From a403831c3814f67385ce1fad219da4964c265329 Mon Sep 17 00:00:00 2001 From: gimenes Date: Sat, 3 Oct 2026 11:58:24 -0300 Subject: [PATCH 01/71] chore(shared): vendor the Blocks content protocol Copies @decocms/blocks/protocol (unpublished v8) into packages/shared/src/vendor/blocks-protocol with a sync script and a SYNC.md note, and exposes it as @decocms/shared/blocks-protocol*. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WNwbSEePYNcY5YCgqZURig --- packages/shared/package.json | 4 + .../shared/src/vendor/blocks-protocol/SYNC.md | 22 + .../src/vendor/blocks-protocol/assets.ts | 127 +++++ .../src/vendor/blocks-protocol/canonical.ts | 140 ++++++ .../src/vendor/blocks-protocol/ciphertext.ts | 175 +++++++ .../src/vendor/blocks-protocol/client.ts | 199 ++++++++ .../conformance/cases/assets.ts | 101 ++++ .../conformance/cases/guards.ts | 408 ++++++++++++++++ .../conformance/cases/reads.ts | 238 +++++++++ .../conformance/cases/types.ts | 9 + .../blocks-protocol/conformance/cases/wire.ts | 207 ++++++++ .../conformance/cases/writes.ts | 450 ++++++++++++++++++ .../conformance/content-hash-fixtures.ts | 101 ++++ .../blocks-protocol/conformance/context.ts | 257 ++++++++++ .../blocks-protocol/conformance/index.ts | 167 +++++++ .../src/vendor/blocks-protocol/errors.ts | 154 ++++++ .../src/vendor/blocks-protocol/index.ts | 61 +++ .../shared/src/vendor/blocks-protocol/keys.ts | 307 ++++++++++++ .../src/vendor/blocks-protocol/params.ts | 76 +++ .../src/vendor/blocks-protocol/secrets.ts | 275 +++++++++++ .../vendor/blocks-protocol/server/assets.ts | 147 ++++++ .../src/vendor/blocks-protocol/server/auth.ts | 65 +++ .../blocks-protocol/server/body-cache.ts | 89 ++++ .../vendor/blocks-protocol/server/content.ts | 199 ++++++++ .../src/vendor/blocks-protocol/server/core.ts | 126 +++++ .../vendor/blocks-protocol/server/handler.ts | 118 +++++ .../src/vendor/blocks-protocol/server/http.ts | 164 +++++++ .../vendor/blocks-protocol/server/index.ts | 26 + .../blocks-protocol/server/methods/apply.ts | 431 +++++++++++++++++ .../blocks-protocol/server/methods/read.ts | 183 +++++++ .../src/vendor/blocks-protocol/server/rpc.ts | 171 +++++++ .../src/vendor/blocks-protocol/storage.ts | 185 +++++++ .../vendor/blocks-protocol/storage/memory.ts | 253 ++++++++++ .../src/vendor/blocks-protocol/types.ts | 197 ++++++++ scripts/sync-blocks-protocol.ts | 98 ++++ 35 files changed, 5930 insertions(+) create mode 100644 packages/shared/src/vendor/blocks-protocol/SYNC.md create mode 100644 packages/shared/src/vendor/blocks-protocol/assets.ts create mode 100644 packages/shared/src/vendor/blocks-protocol/canonical.ts create mode 100644 packages/shared/src/vendor/blocks-protocol/ciphertext.ts create mode 100644 packages/shared/src/vendor/blocks-protocol/client.ts create mode 100644 packages/shared/src/vendor/blocks-protocol/conformance/cases/assets.ts create mode 100644 packages/shared/src/vendor/blocks-protocol/conformance/cases/guards.ts create mode 100644 packages/shared/src/vendor/blocks-protocol/conformance/cases/reads.ts create mode 100644 packages/shared/src/vendor/blocks-protocol/conformance/cases/types.ts create mode 100644 packages/shared/src/vendor/blocks-protocol/conformance/cases/wire.ts create mode 100644 packages/shared/src/vendor/blocks-protocol/conformance/cases/writes.ts create mode 100644 packages/shared/src/vendor/blocks-protocol/conformance/content-hash-fixtures.ts create mode 100644 packages/shared/src/vendor/blocks-protocol/conformance/context.ts create mode 100644 packages/shared/src/vendor/blocks-protocol/conformance/index.ts create mode 100644 packages/shared/src/vendor/blocks-protocol/errors.ts create mode 100644 packages/shared/src/vendor/blocks-protocol/index.ts create mode 100644 packages/shared/src/vendor/blocks-protocol/keys.ts create mode 100644 packages/shared/src/vendor/blocks-protocol/params.ts create mode 100644 packages/shared/src/vendor/blocks-protocol/secrets.ts create mode 100644 packages/shared/src/vendor/blocks-protocol/server/assets.ts create mode 100644 packages/shared/src/vendor/blocks-protocol/server/auth.ts create mode 100644 packages/shared/src/vendor/blocks-protocol/server/body-cache.ts create mode 100644 packages/shared/src/vendor/blocks-protocol/server/content.ts create mode 100644 packages/shared/src/vendor/blocks-protocol/server/core.ts create mode 100644 packages/shared/src/vendor/blocks-protocol/server/handler.ts create mode 100644 packages/shared/src/vendor/blocks-protocol/server/http.ts create mode 100644 packages/shared/src/vendor/blocks-protocol/server/index.ts create mode 100644 packages/shared/src/vendor/blocks-protocol/server/methods/apply.ts create mode 100644 packages/shared/src/vendor/blocks-protocol/server/methods/read.ts create mode 100644 packages/shared/src/vendor/blocks-protocol/server/rpc.ts create mode 100644 packages/shared/src/vendor/blocks-protocol/storage.ts create mode 100644 packages/shared/src/vendor/blocks-protocol/storage/memory.ts create mode 100644 packages/shared/src/vendor/blocks-protocol/types.ts create mode 100644 scripts/sync-blocks-protocol.ts diff --git a/packages/shared/package.json b/packages/shared/package.json index e5f00a120c..b8f5c0bace 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -18,6 +18,10 @@ "./sdk/lib/*": "./src/sdk/lib/*.ts", "./decofile": "./src/decofile/index.ts", "./git-providers": "./src/git-providers/index.ts", + "./blocks-protocol": "./src/vendor/blocks-protocol/index.ts", + "./blocks-protocol/server": "./src/vendor/blocks-protocol/server/index.ts", + "./blocks-protocol/storage/memory": "./src/vendor/blocks-protocol/storage/memory.ts", + "./blocks-protocol/conformance": "./src/vendor/blocks-protocol/conformance/index.ts", "./*": "./src/*.ts" }, "dependencies": { diff --git a/packages/shared/src/vendor/blocks-protocol/SYNC.md b/packages/shared/src/vendor/blocks-protocol/SYNC.md new file mode 100644 index 0000000000..98dd30bf4a --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/SYNC.md @@ -0,0 +1,22 @@ +# Vendored: Blocks content protocol + +A copy of `packages/blocks/src/protocol` from [decocms/blocks](https://github.com/decocms/blocks) +at commit `ccdf6c259a1bf8a0341ea453659ca4bfc9c7db83`, which is `@decocms/blocks/protocol` in the +unpublished `@decocms/blocks@8`. + +**Never edit these files by hand.** Fix the protocol upstream, then resync: + +```sh +bun run scripts/sync-blocks-protocol.ts +``` + +The sync copies the tree at that commit, leaves out `storage/fs/` (Node-only) +and the tests (`__tests__/`, `*.test.ts`), renames files to kebab-case +with their relative imports, formats the result, and updates the commit above. + +Import it through `@decocms/shared/blocks-protocol`, +`@decocms/shared/blocks-protocol/server`, +`@decocms/shared/blocks-protocol/storage/memory` and +`@decocms/shared/blocks-protocol/conformance`. Once `@decocms/blocks@8` is +published, depend on it, point those imports at `@decocms/blocks/protocol*` +and delete this folder: the layout is the same. diff --git a/packages/shared/src/vendor/blocks-protocol/assets.ts b/packages/shared/src/vendor/blocks-protocol/assets.ts new file mode 100644 index 0000000000..f3a74c8eef --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/assets.ts @@ -0,0 +1,127 @@ +/** + * Asset-upload naming rules shared by the asset handler and the storages. + * Browser-safe. + */ + +/** + * Every content type the asset endpoint accepts, with the file extensions a + * file of that type may have (the first is the one appended to a name + * without an extension). The stored file's extension decides the type it's + * served with, so an upload can never pick one the browser would run, such + * as `.html` or `.js`. + */ +export const ASSET_TYPES: Readonly> = { + "image/png": [".png"], + "image/jpeg": [".jpg", ".jpeg"], + "image/webp": [".webp"], + "image/avif": [".avif"], + "image/gif": [".gif"], + "image/x-icon": [".ico"], + "image/vnd.microsoft.icon": [".ico"], + "video/mp4": [".mp4"], + "video/webm": [".webm"], + "font/woff2": [".woff2"], + "font/woff": [".woff"], + "font/ttf": [".ttf"], + "font/otf": [".otf"], + "application/font-woff": [".woff"], + "application/x-font-ttf": [".ttf"], + "application/vnd.ms-fontobject": [".eot"], + "application/pdf": [".pdf"], +}; + +/** + * SVG can carry scripts, so it runs on the site's origin like HTML would. + * Accepted only with `allowSvg`. + */ +export const SVG_ASSET_TYPE = { "image/svg+xml": [".svg"] } as const; + +export interface AssetTypeOptions { + /** Accept `image/svg+xml` uploads (default false: SVG can carry scripts). */ + allowSvg?: boolean; +} + +const mediaType = (contentType: string | null) => + contentType ? contentType.split(";")[0]!.trim().toLowerCase() : ""; + +/** The extensions an upload of `contentType` may have; `null` when the type isn't accepted. */ +export function assetExtensions( + contentType: string | null, + options: AssetTypeOptions = {}, +): readonly string[] | null { + const type = mediaType(contentType); + if (Object.hasOwn(ASSET_TYPES, type)) return ASSET_TYPES[type] ?? null; + if (options.allowSvg && type === "image/svg+xml") + return SVG_ASSET_TYPE[type] ?? null; + return null; +} + +/** True when the asset endpoint accepts `contentType`. */ +export function isAcceptedAssetType( + contentType: string | null, + options: AssetTypeOptions = {}, +): boolean { + return assetExtensions(contentType, options) !== null; +} + +/** + * Fits a sanitized file name to its content type: a name without an + * extension gets the type's, and the extension is lowercased. Returns `null` + * when the name's extension doesn't match the type (`evil.html` sent as + * `image/png`), or the type isn't accepted. + */ +export function assetNameForType( + name: string, + contentType: string | null, + options: AssetTypeOptions = {}, +): string | null { + const extensions = assetExtensions(contentType, options); + if (!extensions) return null; + const dot = name.lastIndexOf("."); + if (dot <= 0) return `${name}${extensions[0]}`; + const extension = name.slice(dot).toLowerCase(); + return extensions.includes(extension) ? name.slice(0, dot) + extension : null; +} + +/** + * Normalizes an upload's file name: its last path segment, lowercased + * extension, with anything but letters, digits, `.`, `_` and `-` replaced. + * Returns `null` when nothing usable is left. + */ +export function sanitizeAssetName(raw: string): string | null { + let name: string; + try { + name = decodeURIComponent(raw); + } catch { + name = raw; + } + name = name.split(/[\\/]/).pop() ?? ""; + name = name + .normalize("NFKD") + .replace(/[̀-ͯ]/g, "") + .replace(/[^A-Za-z0-9._-]+/g, "-") + .replace(/-{2,}/g, "-") + .replace(/^[.-]+/, "") + .replace(/-+(\.|$)/g, "$1"); + if (!name || name === "." || name === "..") return null; + if (name.length > 200) { + const dot = name.lastIndexOf("."); + const ext = dot > 0 ? name.slice(dot).slice(0, 16) : ""; + name = name.slice(0, 200 - ext.length) + ext; + } + return name; +} + +/** + * The name to try when `name` is taken: a short random suffix before the + * extension, such as `banner-3f9a2c.jpg`. + */ +export function suffixedAssetName(name: string): string { + const suffix = Array.from(crypto.getRandomValues(new Uint8Array(3)), (b) => + b.toString(16).padStart(2, "0"), + ).join(""); + const dot = name.lastIndexOf("."); + return dot > 0 + ? `${name.slice(0, dot)}-${suffix}${name.slice(dot)}` + : `${name}-${suffix}`; +} diff --git a/packages/shared/src/vendor/blocks-protocol/canonical.ts b/packages/shared/src/vendor/blocks-protocol/canonical.ts new file mode 100644 index 0000000000..93dbbbfbf7 --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/canonical.ts @@ -0,0 +1,140 @@ +/** + * Canonical JSON and hashing shared by the protocol's request digests. + * + * Canonical form: object keys recursively sorted in JavaScript code-unit + * order, array order preserved, compact UTF-8 JSON with no trailing newline, + * JSON string escaping and number encoding (negative zero becomes zero). + * Values JSON can't represent (undefined, functions, symbols, bigints, + * non-finite numbers, cycles) are rejected rather than silently dropped. + * + * Uses Web Crypto, so it runs in browsers, on Workers and on Node. + */ + +export class CanonicalJsonError extends Error { + constructor(message: string) { + super(message); + this.name = "CanonicalJsonError"; + } +} + +function write( + value: unknown, + path: string, + seen: Set, + out: string[], +): void { + if (value === null) { + out.push("null"); + return; + } + switch (typeof value) { + case "string": + out.push(JSON.stringify(value)); + return; + case "boolean": + out.push(value ? "true" : "false"); + return; + case "number": + if (!Number.isFinite(value)) { + throw new CanonicalJsonError( + `${path}: ${value} can't be represented in JSON`, + ); + } + out.push(Object.is(value, -0) ? "0" : JSON.stringify(value)); + return; + case "object": + break; + default: + throw new CanonicalJsonError( + `${path}: a ${typeof value} can't be represented in JSON`, + ); + } + const object = value as object; + if (seen.has(object)) + throw new CanonicalJsonError( + `${path}: a cycle can't be represented in JSON`, + ); + seen.add(object); + if (Array.isArray(object)) { + out.push("["); + object.forEach((item, index) => { + if (index > 0) out.push(","); + write(item, `${path}/${index}`, seen, out); + }); + out.push("]"); + } else { + const prototype = Object.getPrototypeOf(object); + if (prototype !== Object.prototype && prototype !== null) { + throw new CanonicalJsonError( + `${path}: only plain objects can be represented in JSON`, + ); + } + // Code-unit order, which is what the default sort compares. + const keys = Object.keys(object).sort(); + out.push("{"); + keys.forEach((key, index) => { + if (index > 0) out.push(","); + out.push(JSON.stringify(key), ":"); + write( + (object as Record)[key], + `${path}/${key}`, + seen, + out, + ); + }); + out.push("}"); + } + seen.delete(object); +} + +/** Serializes `value` to canonical JSON. Throws CanonicalJsonError for values JSON can't represent. */ +export function canonicalJson(value: unknown): string { + const out: string[] = []; + write(value, "", new Set(), out); + return out.join(""); +} + +const hex = (bytes: ArrayBuffer) => + Array.from(new Uint8Array(bytes), (b) => + b.toString(16).padStart(2, "0"), + ).join(""); + +/** SHA-256 of `data` (strings are UTF-8 encoded), as lowercase hex. */ +export async function sha256Hex(data: string | Uint8Array): Promise { + const bytes = + typeof data === "string" ? new TextEncoder().encode(data) : data; + return hex(await crypto.subtle.digest("SHA-256", bytes as BufferSource)); +} + +/** + * The version of the canonical content-hashing format below. A snapshot + * envelope records it beside the revision; the revision itself is excluded + * from its own hash. + */ +export const CONTENT_HASH_FORMAT = 1; + +/** + * The content revision of a `blocks` map: SHA-256 over its canonical JSON, + * lowercase hex. The one definition the CLI, the release materializer and + * the site editor's backend share; `contentHashFixtures` (in + * `@decocms/blocks/protocol/conformance`) pins its output. + * + * This identifies content, not storage: a storage's `blocks.list` revision + * is opaque and may be computed differently (from file versions). + */ +export async function computeContentRevision( + blocks: Record, +): Promise { + return sha256Hex(canonicalJson(blocks)); +} + +/** The domain prefix of `blocks.apply` request digests, so they never equal another hash. */ +export const APPLY_DIGEST_DOMAIN = "deco-content/blocks.apply@1\n"; + +/** + * The digest a request key is bound to: SHA-256 over the domain prefix and + * the canonical JSON of every supplied parameter. + */ +export function applyRequestDigest(params: object): Promise { + return sha256Hex(APPLY_DIGEST_DOMAIN + canonicalJson(params)); +} diff --git a/packages/shared/src/vendor/blocks-protocol/ciphertext.ts b/packages/shared/src/vendor/blocks-protocol/ciphertext.ts new file mode 100644 index 0000000000..a0139ebdcb --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/ciphertext.ts @@ -0,0 +1,175 @@ +/** + * The `secret` block's ciphertext format, in one place: the secret guard + * checks it, and `encryptSecret` (`@decocms/blocks/secrets`) produces it. + * + * ``` + * v1... + * ``` + * + * Each segment is unpadded base64url. A fresh AES-256-GCM key encrypts the + * UTF-8 value; the key is wrapped with the site's RSA public key using + * RSA-OAEP with SHA-256. + * + * - `wrappedKey` — the RSA-OAEP output: as long as the modulus, so 256, 384 + * or 512 bytes (2048-, 3072- or 4096-bit keys; `deco` creates 3072-bit ones). + * - `iv` — the AES-GCM nonce: exactly 12 bytes. + * - `ciphertext` — the AES-GCM output, its 16-byte tag included: at least 16 bytes. + * + * Checking the structure needs no key, so the content protocol and + * `deco check` refuse text that merely looks like a ciphertext (`v1.hunter2`). + * Web Crypto only: runs in browsers, on Workers and on Node. + */ + +/** The format version every ciphertext starts with. */ +export const CIPHERTEXT_VERSION = "v1"; + +/** The wrapped-key lengths a ciphertext may carry: 2048-, 3072- and 4096-bit RSA. */ +export const WRAPPED_KEY_BYTES: readonly number[] = [256, 384, 512]; + +/** The AES-GCM nonce length. */ +export const IV_BYTES = 12; + +/** The AES-GCM authentication tag length, the smallest ciphertext segment. */ +export const GCM_TAG_BYTES = 16; + +/** + * The shape of a ciphertext: `v1.` and three base64url segments. A match is + * necessary but not sufficient; `parseCiphertext` also checks segment lengths. + */ +export const CIPHERTEXT_PATTERN = + /^v1\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/; + +/** The decoded segments of a ciphertext. */ +export interface CiphertextParts { + wrappedKey: Uint8Array; + iv: Uint8Array; + ciphertext: Uint8Array; +} + +const BASE64URL = /^[A-Za-z0-9_-]*$/; + +/** Unpadded base64url. */ +export function encodeBase64Url(bytes: Uint8Array): string { + let binary = ""; + for (let i = 0; i < bytes.length; i += 0x8000) { + binary += String.fromCharCode(...bytes.subarray(i, i + 0x8000)); + } + return btoa(binary) + .replace(/\+/g, "-") + .replace(/\//g, "_") + .replace(/=+$/, ""); +} + +/** Decodes unpadded base64url; `null` when `text` isn't canonical base64url. */ +export function decodeBase64Url(text: string): Uint8Array | null { + if (!BASE64URL.test(text) || text.length % 4 === 1) return null; + const padded = + text.replace(/-/g, "+").replace(/_/g, "/") + + "=".repeat((4 - (text.length % 4)) % 4); + let binary: string; + try { + binary = atob(padded); + } catch { + return null; + } + const bytes = Uint8Array.from(binary, (c) => c.charCodeAt(0)); + // Reject non-canonical encodings (stray low bits), so one value has one spelling. + return encodeBase64Url(bytes) === text ? bytes : null; +} + +const validParts = ({ wrappedKey, iv, ciphertext }: CiphertextParts) => + WRAPPED_KEY_BYTES.includes(wrappedKey.byteLength) && + iv.byteLength === IV_BYTES && + ciphertext.byteLength >= GCM_TAG_BYTES; + +/** Splits and decodes a ciphertext; `null` when it isn't well formed. */ +export function parseCiphertext(text: unknown): CiphertextParts | null { + if (typeof text !== "string" || !CIPHERTEXT_PATTERN.test(text)) return null; + const [, rawKey, rawIv, rawCiphertext] = text.split("."); + const wrappedKey = decodeBase64Url(rawKey!); + const iv = decodeBase64Url(rawIv!); + const ciphertext = decodeBase64Url(rawCiphertext!); + if (!wrappedKey || !iv || !ciphertext) return null; + const parts = { wrappedKey, iv, ciphertext }; + return validParts(parts) ? parts : null; +} + +/** Joins decoded segments into a ciphertext. Throws when a segment has the wrong length. */ +export function formatCiphertext(parts: CiphertextParts): string { + if (!validParts(parts)) { + throw new RangeError( + `a ciphertext needs a ${WRAPPED_KEY_BYTES.join("/")}-byte wrapped key, a ${IV_BYTES}-byte iv and at least ${GCM_TAG_BYTES} bytes of ciphertext`, + ); + } + return [ + CIPHERTEXT_VERSION, + encodeBase64Url(parts.wrappedKey), + encodeBase64Url(parts.iv), + encodeBase64Url(parts.ciphertext), + ].join("."); +} + +const PEM_BLOCK = /-----BEGIN ([A-Z ]+)-----([\s\S]*?)-----END \1-----/g; + +/** The DER bytes of a single `PUBLIC KEY` PEM block; `null` for anything else. */ +export function publicKeyDerFromPem(pem: string): Uint8Array | null { + const blocks = [...pem.matchAll(PEM_BLOCK)]; + const block = blocks[0]; + if (blocks.length !== 1 || !block || block[1] !== "PUBLIC KEY") return null; + if (pem.replace(block[0], "").trim() !== "") return null; + const body = (block[2] ?? "").replace(/\s+/g, ""); + if (!/^[A-Za-z0-9+/]+={0,2}$/.test(body)) return null; + try { + return Uint8Array.from(atob(body), (c) => c.charCodeAt(0)); + } catch { + return null; + } +} + +/** A `PUBLIC KEY` PEM block for SPKI `der` bytes, with 64-character lines. */ +export function publicKeyPemFromDer(der: Uint8Array): string { + let binary = ""; + for (const byte of der) binary += String.fromCharCode(byte); + const lines = + btoa(binary) + .match(/.{1,64}/g) + ?.join("\n") ?? ""; + return `-----BEGIN PUBLIC KEY-----\n${lines}\n-----END PUBLIC KEY-----\n`; +} + +/** + * Encrypts `value` for the holder of the private key matching `publicKeyPem` + * (an SPKI `PUBLIC KEY` PEM, such as `.deco/secrets.pub`). + */ +export async function encryptToCiphertext( + publicKeyPem: string, + value: string, +): Promise { + const der = publicKeyDerFromPem(publicKeyPem); + if (!der) + throw new TypeError("the public key must be a single PUBLIC KEY PEM block"); + const rsa = await crypto.subtle.importKey( + "spki", + der as BufferSource, + { name: "RSA-OAEP", hash: "SHA-256" }, + false, + ["encrypt"], + ); + const aesKey = crypto.getRandomValues(new Uint8Array(32)); + const aes = await crypto.subtle.importKey("raw", aesKey, "AES-GCM", false, [ + "encrypt", + ]); + const iv = crypto.getRandomValues(new Uint8Array(IV_BYTES)); + const ciphertext = new Uint8Array( + await crypto.subtle.encrypt( + { name: "AES-GCM", iv }, + aes, + new TextEncoder().encode(value), + ), + ); + const wrappedKey = new Uint8Array( + await crypto.subtle.encrypt({ name: "RSA-OAEP" }, rsa, aesKey), + ); + aesKey.fill(0); + return formatCiphertext({ wrappedKey, iv, ciphertext }); +} diff --git a/packages/shared/src/vendor/blocks-protocol/client.ts b/packages/shared/src/vendor/blocks-protocol/client.ts new file mode 100644 index 0000000000..eef699cd3f --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/client.ts @@ -0,0 +1,199 @@ +/** + * A small content-protocol client: one method per protocol method, plus + * batches. Uses `fetch`, so it runs in browsers, on Workers and on Node. + */ +import { ContentProtocolError, ErrorCode, type RpcErrorObject } from "./errors"; +import { + type DescribeResult, + type MethodName, + type Methods, + PROTOCOL_NAME, + PROTOCOL_VERSION, + type RpcId, +} from "./types"; + +export interface ContentClientOptions { + /** The endpoint URL, such as `http://127.0.0.1:4545/rpc`. */ + endpoint: string | URL; + /** Sent as `Authorization: Bearer `. */ + token?: string; + /** Extra headers on every request. */ + headers?: Record; + /** A fetch implementation (defaults to the global one). */ + fetch?: (input: Request) => Promise; +} + +/** One call of a batch. */ +export type BatchCall = { + [M in MethodName]: { method: M; params?: Methods[M]["params"] }; +}[MethodName]; + +/** The outcome of one batch call: its result or its error. */ +export type BatchOutcome = + | { ok: true; result: unknown } + | { ok: false; error: ContentProtocolError }; + +export interface ContentClient { + /** Throws Unsupported when the endpoint speaks another protocol or major version. */ + describe(): Promise; + schemaGet( + params?: Methods["schema.get"]["params"], + ): Promise; + blocksList( + params?: Methods["blocks.list"]["params"], + ): Promise; + blocksApply( + params: Methods["blocks.apply"]["params"], + ): Promise; + /** Calls one method by name. Throws `ContentProtocolError` on an error response. */ + call( + method: M, + params?: Methods[M]["params"], + ): Promise; + /** Runs several calls in one request; outcomes come back in order. */ + batch(calls: BatchCall[]): Promise; +} + +type WireResponse = { + id: RpcId | null; + result?: unknown; + error?: RpcErrorObject; +}; + +/** + * Decodes a JSON response body, gunzipping it when a custom fetch (such as an + * in-process handler) handed back the raw gzip bytes. + */ +export async function readResponseJson(response: Response): Promise { + let bytes = new Uint8Array(await response.arrayBuffer()); + if (bytes[0] === 0x1f && bytes[1] === 0x8b) { + const stream = new Blob([bytes as BlobPart]) + .stream() + .pipeThrough( + new DecompressionStream("gzip") as unknown as ReadableWritablePair< + Uint8Array, + Uint8Array + >, + ); + bytes = new Uint8Array(await new Response(stream).arrayBuffer()); + } + return JSON.parse(new TextDecoder().decode(bytes)); +} + +/** + * Refuses an endpoint this client can't speak to: another protocol, or + * another major version (minors only add, so any minor is fine). + */ +export function assertSupportedEndpoint( + description: DescribeResult, +): DescribeResult { + const { protocol, version } = (description ?? {}) as Partial; + if (protocol !== PROTOCOL_NAME) { + throw new ContentProtocolError( + ErrorCode.Unsupported, + `the endpoint speaks ${JSON.stringify(protocol)}, not ${PROTOCOL_NAME}`, + ); + } + if (version?.major !== PROTOCOL_VERSION.major) { + throw new ContentProtocolError( + ErrorCode.Unsupported, + `the endpoint speaks ${PROTOCOL_NAME} ${String(version?.major)}.x; this client speaks ${PROTOCOL_VERSION.major}.x`, + ); + } + return description; +} + +export function createContentClient( + options: ContentClientOptions, +): ContentClient { + const doFetch = options.fetch ?? ((request: Request) => fetch(request)); + let nextId = 1; + + async function post(body: unknown): Promise { + const headers: Record = { + "content-type": "application/json", + accept: "application/json", + ...options.headers, + }; + if (options.token !== undefined) + headers.authorization = `Bearer ${options.token}`; + const response = await doFetch( + new Request(options.endpoint, { + method: "POST", + headers, + body: JSON.stringify(body), + }), + ); + let payload: unknown; + try { + payload = await readResponseJson(response); + } catch { + throw new ContentProtocolError( + ErrorCode.Unavailable, + `the endpoint answered HTTP ${response.status} without a JSON-RPC body`, + ); + } + if ( + !response.ok || + (!Array.isArray(payload) && (payload as WireResponse)?.id === null) + ) { + const error = (payload as WireResponse)?.error; + if (error) throw ContentProtocolError.from(error); + throw new ContentProtocolError( + ErrorCode.Unavailable, + `the endpoint answered HTTP ${response.status}`, + ); + } + return payload; + } + + const call = async ( + method: M, + params?: Methods[M]["params"], + ) => { + const id = nextId++; + const response = (await post({ + jsonrpc: "2.0", + id, + method, + params: params ?? {}, + })) as WireResponse; + if (response.error) throw ContentProtocolError.from(response.error); + return response.result as Methods[M]["result"]; + }; + + return { + call, + describe: async () => assertSupportedEndpoint(await call("describe")), + schemaGet: (params) => call("schema.get", params), + blocksList: (params) => call("blocks.list", params), + blocksApply: (params) => call("blocks.apply", params), + async batch(calls) { + const ids = calls.map(() => nextId++); + const responses = (await post( + calls.map((c, i) => ({ + jsonrpc: "2.0", + id: ids[i], + method: c.method, + params: c.params ?? {}, + })), + )) as WireResponse[]; + const byId = new Map(responses.map((r) => [r.id, r])); + return ids.map((id): BatchOutcome => { + const response = byId.get(id); + if (!response) { + return { + ok: false, + error: new ContentProtocolError( + ErrorCode.InternalError, + "no response for this call", + ), + }; + } + return response.error + ? { ok: false, error: ContentProtocolError.from(response.error) } + : { ok: true, result: response.result }; + }); + }, + }; +} diff --git a/packages/shared/src/vendor/blocks-protocol/conformance/cases/assets.ts b/packages/shared/src/vendor/blocks-protocol/conformance/cases/assets.ts new file mode 100644 index 0000000000..527d858bd5 --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/conformance/cases/assets.ts @@ -0,0 +1,101 @@ +/** + * Uploads beside the protocol: `PUT /assets/`. These cases run only + * when the harness passes `assetsEndpoint`. + */ +import { ErrorCode } from "../../errors"; +import { + assert, + assertEqual, + type ConformanceContext, + rawErrorCode, +} from "../context"; +import type { ConformanceCase } from "./types"; + +/** The smallest PNG header: enough bytes for an upload, never executed. */ +const PNG = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + +async function uploadsAccepted(ctx: ConformanceContext) { + if (ctx.options.assetsEndpoint === undefined) + return ctx.skip("no assetsEndpoint"); + const d = await ctx.describe(); + if (d.readOnly || d.assets === null) + return ctx.skip("the endpoint doesn't accept uploads"); + return d; +} + +const pathOf = (body: unknown) => (body as { path?: unknown } | null)?.path; + +export const assetCases: ConformanceCase[] = [ + { + id: "assets/upload", + title: + "an upload answers 201 with /assets/; a taken name gets a suffix, never an overwrite", + async run(ctx) { + await uploadsAccepted(ctx); + const name = `${ctx.prefix}-banner.png`; + const first = await ctx.upload(name, PNG, "image/png"); + assertEqual(first.status, 201, "HTTP status"); + assertEqual(pathOf(first.body), `/assets/${name}`, "the stored path"); + const second = await ctx.upload(name, PNG, "image/png"); + assertEqual(second.status, 201, "HTTP status of the second upload"); + const path = pathOf(second.body); + assert( + typeof path === "string" && + path.startsWith("/assets/") && + path !== `/assets/${name}`, + `a taken name gets another name, got ${String(path)}`, + ); + assert(path.endsWith(".png"), "the suffix goes before the extension"); + }, + }, + { + id: "assets/content-types", + title: + "uploads refuse non-media types, SVG and an extension that doesn't match the type (415)", + async run(ctx) { + await uploadsAccepted(ctx); + const script = ""; + const refused: Array<[name: string, type: string]> = [ + [`${ctx.prefix}-page.html`, "text/html"], + [`${ctx.prefix}-evil.html`, "image/png"], + [`${ctx.prefix}-evil.js`, "image/jpeg"], + [`${ctx.prefix}-logo.svg`, "image/svg+xml"], + [`${ctx.prefix}-data.json`, "application/json"], + ]; + for (const [name, type] of refused) { + const response = await ctx.upload(name, script, type); + assertEqual(response.status, 415, `HTTP status of ${name} as ${type}`); + assertEqual( + rawErrorCode(response.body), + ErrorCode.InvalidRequest, + "the error code", + ); + } + }, + }, + { + id: "assets/read-only", + title: "an endpoint without uploads refuses them", + async run(ctx) { + if (ctx.options.assetsEndpoint === undefined) + return ctx.skip("no assetsEndpoint"); + const d = await ctx.describe(); + if (!d.readOnly && d.assets !== null) + return ctx.skip("the endpoint accepts uploads"); + const response = await ctx.upload( + `${ctx.prefix}-x.png`, + PNG, + "image/png", + ); + assert( + response.status === 403 || response.status === 404, + `HTTP ${response.status} (expected 403 or 404)`, + ); + const code = rawErrorCode(response.body); + assert( + code === ErrorCode.ReadOnly || code === ErrorCode.Unsupported, + `error ${String(code)} (expected ReadOnly or Unsupported)`, + ); + }, + }, +]; diff --git a/packages/shared/src/vendor/blocks-protocol/conformance/cases/guards.ts b/packages/shared/src/vendor/blocks-protocol/conformance/cases/guards.ts new file mode 100644 index 0000000000..f61180ee2d --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/conformance/cases/guards.ts @@ -0,0 +1,408 @@ +/** + * Limits, the secret guard and request-key idempotency. + */ +import { encryptToCiphertext, publicKeyPemFromDer } from "../../ciphertext"; +import { ErrorCode, type InvalidBlockData } from "../../errors"; +import { + assert, + assertEqual, + type ConformanceContext, + expectError, + rawErrorCode, +} from "../context"; +import type { ConformanceCase } from "./types"; + +async function writable(ctx: ConformanceContext) { + const d = await ctx.describe(); + if (d.readOnly) return ctx.skip("read-only endpoint"); + return d; +} + +async function withIdempotency(ctx: ConformanceContext) { + const d = await writable(ctx); + if (!d.writes.idempotency) return ctx.skip("request keys aren't advertised"); + return d; +} + +/** A throwaway public key, for endpoints that don't report one. */ +async function ephemeralPublicKey(): Promise { + const pair = (await crypto.subtle.generateKey( + { + name: "RSA-OAEP", + modulusLength: 2048, + publicExponent: new Uint8Array([1, 0, 1]), + hash: "SHA-256", + }, + true, + ["encrypt", "decrypt"], + )) as CryptoKeyPair; + return publicKeyPemFromDer( + new Uint8Array(await crypto.subtle.exportKey("spki", pair.publicKey)), + ); +} + +/** A real ciphertext: encrypted with the endpoint's public key when it reports one. */ +async function realCiphertext( + ctx: ConformanceContext, + value: string, +): Promise { + const publicKey = + (await ctx.describe()).secrets?.publicKey ?? (await ephemeralPublicKey()); + return encryptToCiphertext(publicKey, value); +} + +const key = (ctx: ConformanceContext, label: string) => + `${ctx.prefix}-${label}-${Math.random().toString(36).slice(2)}`; + +export const guardCases: ConformanceCase[] = [ + { + id: "limits/ops-per-apply", + title: "more names than maxOpsPerApply is LimitExceeded", + async run(ctx) { + const d = await writable(ctx); + const names = Array.from( + { length: d.limits.maxOpsPerApply + 1 }, + (_, i) => `${ctx.prefix}-op-${i}`, + ); + await expectError( + ctx.client.blocksApply({ delete: names }), + ErrorCode.LimitExceeded, + "too many names", + ); + }, + }, + { + id: "limits/block-bytes", + title: "an entry over maxBlockBytes is refused and nothing is written", + async run(ctx) { + const d = await writable(ctx); + if (d.limits.maxBlockBytes + 1024 > d.limits.maxRequestBytes) + return ctx.skip("the request limit is lower"); + const big = ctx.name("big"); + const small = ctx.name("small"); + const error = await expectError( + ctx.client.blocksApply({ + set: { + [big]: { text: "x".repeat(d.limits.maxBlockBytes) }, + [small]: { ok: true }, + }, + }), + ErrorCode.InvalidBlock, + "an oversized entry", + ); + const violations = (error.data as InvalidBlockData).violations; + assert( + violations.some((v) => v.name === big), + "the violation names the entry", + ); + const list = await ctx.client.blocksList(); + assert( + !list.notModified && !(small in list.blocks), + "nothing was written", + ); + }, + }, + { + id: "limits/request-bytes", + title: "a body over maxRequestBytes is HTTP 413 with LimitExceeded", + async run(ctx) { + const d = await ctx.describe(); + if (d.limits.maxRequestBytes > 64 * 1024 * 1024) + return ctx.skip("the request limit is too large to probe"); + const body = JSON.stringify({ + jsonrpc: "2.0", + id: 1, + method: "blocks.apply", + params: { + set: { + [ctx.name("huge")]: { text: "x".repeat(d.limits.maxRequestBytes) }, + }, + }, + }); + const response = await ctx.raw(body); + assertEqual(response.status, 413, "HTTP status"); + assertEqual( + rawErrorCode(response.body), + ErrorCode.LimitExceeded, + "the error code", + ); + }, + }, + { + id: "limits/list-bytes", + title: + "a block list over maxListBytes is LimitExceeded, never a partial map", + async run(ctx) { + const d = await writable(ctx); + const { maxListBytes, maxBlockBytes, maxOpsPerApply } = d.limits; + if (maxListBytes > 512 * 1024) + return ctx.skip("maxListBytes is too large to probe"); + const size = Math.min(maxBlockBytes, 64 * 1024) - 64; + const count = Math.ceil(maxListBytes / size) + 1; + if (count > maxOpsPerApply * 4) + return ctx.skip("would need too many writes"); + for (let i = 0; i < count; i++) { + await ctx.client.blocksApply({ + set: { [ctx.name("fill")]: { text: "x".repeat(size) } }, + }); + } + await expectError( + ctx.client.blocksList(), + ErrorCode.LimitExceeded, + "an oversized list", + ); + }, + }, + { + id: "limits/batch-response-bytes", + title: + "a batch response over maxBatchResponseBytes answers later reads with LimitExceeded, never a write", + async run(ctx) { + const d = await writable(ctx); + const { maxBatchResponseBytes, maxBlockBytes, maxListBytes } = d.limits; + if (maxBatchResponseBytes > 1024 * 1024) + return ctx.skip("maxBatchResponseBytes is too large to probe"); + // Enough content that one list fits the budget but four don't. + const target = Math.floor(maxBatchResponseBytes / 3); + if (target > maxListBytes / 2) + return ctx.skip("maxListBytes is lower than the probe needs"); + const size = Math.min(maxBlockBytes, 64 * 1024) - 64; + for (let filled = 0; filled < target; filled += size) { + await ctx.client.blocksApply({ + set: { [ctx.name("fill")]: { text: "x".repeat(size) } }, + }); + } + const write = ctx.name("batch-write"); + const req = (id: number, method: string, params: unknown) => ({ + jsonrpc: "2.0", + id, + method, + params, + }); + const response = await ctx.rpc([ + req(1, "blocks.list", {}), + req(2, "blocks.list", {}), + req(3, "blocks.list", {}), + req(4, "blocks.list", {}), + req(5, "blocks.apply", { set: { [write]: { kept: true } } }), + req(6, "blocks.list", {}), + ]); + const items = response.body as Array<{ + id: number; + result?: unknown; + error?: { code: number; data?: { limit?: string } }; + }>; + assert( + Array.isArray(items) && items.length === 6, + "one response per call", + ); + assert(!items[0].error, "the first read fits"); + const over = items.filter( + (i) => i.error?.code === ErrorCode.LimitExceeded, + ); + assert(over.length > 0, "a read over the budget is LimitExceeded"); + assert( + over.every((i) => i.error?.data?.limit === "maxBatchResponseBytes"), + "the error names maxBatchResponseBytes", + ); + assert( + items[4].result !== undefined, + "the write still answers with its result", + ); + const list = await ctx.client.blocksList(); + assert(!list.notModified && write in list.blocks, "the write landed"); + }, + }, + { + id: "limits/schema-bytes", + title: + "a schema over maxSchemaBytes is LimitExceeded for schema.get and for writes", + async run(ctx) { + if (!ctx.options.schemaOverLimit) + return ctx.skip("the harness serves a schema within limits"); + await expectError( + ctx.client.schemaGet(), + ErrorCode.LimitExceeded, + "schema.get", + ); + const d = await ctx.describe(); + if (d.readOnly) return; + const name = ctx.name("schema-limited"); + await expectError( + ctx.client.blocksApply({ set: { [name]: { v: 1 } } }), + ErrorCode.LimitExceeded, + "a write checked against the schema", + ); + const list = await ctx.client.blocksList(); + assert( + !list.notModified && !(name in list.blocks), + "nothing was written", + ); + }, + }, + { + id: "secrets/guard", + title: + "a Secret field accepts only a secret block with a well-formed ciphertext", + async run(ctx) { + await writable(ctx); + const fixture = ctx.options.secretField; + if (!fixture) return ctx.skip("no secretField fixture"); + const block = (value: unknown) => ({ + __resolveType: fixture.blockType, + [fixture.field]: value, + }); + const plain = ctx.name("plain-secret"); + const error = await expectError( + ctx.client.blocksApply({ set: { [plain]: block("hunter2") } }), + ErrorCode.InvalidBlock, + "plain text in a Secret field", + ); + const violation = (error.data as InvalidBlockData).violations[0]; + assertEqual(violation?.name, plain, "the violation names the entry"); + // Text that only looks like a ciphertext is plain text all the same. + for (const ciphertext of [ + "hunter2", + "v1.hunter2", + "v1.my-api-key_123", + "v1.QUJD.ZGVm", + ]) { + await expectError( + ctx.client.blocksApply({ + set: { + [ctx.name("bad-cipher")]: block({ + __resolveType: "secret", + ciphertext, + }), + }, + }), + ErrorCode.InvalidBlock, + `the malformed ciphertext ${JSON.stringify(ciphertext)}`, + ); + } + const ok = ctx.name("secret"); + const ciphertext = await realCiphertext(ctx, "conformance-value"); + await ctx.client.blocksApply({ + set: { [ok]: block({ __resolveType: "secret", ciphertext }) }, + }); + const list = await ctx.client.blocksList(); + assert( + !list.notModified && ok in list.blocks, + "an encrypted value is saved", + ); + }, + }, + { + id: "idempotency/unadvertised", + title: + "a request key is refused when the endpoint doesn't advertise receipts", + async run(ctx) { + const d = await writable(ctx); + if (d.writes.idempotency) return ctx.skip("request keys are advertised"); + await expectError( + ctx.client.blocksApply({ + requestKey: key(ctx, "k"), + delete: [ctx.name()], + }), + ErrorCode.Unsupported, + "a request key", + ); + }, + }, + { + id: "idempotency/retry", + title: + "retrying with the same key returns the original result, even after the content moved", + async run(ctx) { + await withIdempotency(ctx); + const name = ctx.name("retry"); + const params = { + requestKey: key(ctx, "retry"), + set: { [name]: { v: 1 } }, + }; + const first = await ctx.client.blocksApply(params); + await ctx.client.blocksApply({ + set: { [ctx.name("advance")]: { v: 1 } }, + }); + const second = await ctx.client.blocksApply(params); + assertEqual(second, first, "the replayed result"); + const list = await ctx.client.blocksList(); + assert( + !list.notModified && list.versions[name] === first.versions[name], + "written once", + ); + }, + }, + { + id: "idempotency/different-params", + title: "reusing a key for a different request is Invalid params", + async run(ctx) { + await withIdempotency(ctx); + const requestKey = key(ctx, "reuse"); + const name = ctx.name("reuse"); + await ctx.client.blocksApply({ requestKey, set: { [name]: { v: 1 } } }); + await expectError( + ctx.client.blocksApply({ requestKey, set: { [name]: { v: 2 } } }), + ErrorCode.InvalidParams, + "a different body", + ); + }, + }, + { + id: "idempotency/simultaneous", + title: "simultaneous duplicates commit once and share one result", + async run(ctx) { + await withIdempotency(ctx); + const params = { + requestKey: key(ctx, "dup"), + set: { [ctx.name("dup")]: { v: 1 } }, + }; + const results = await Promise.all( + [1, 2, 3].map(() => ctx.client.blocksApply(params)), + ); + assertEqual(results[1], results[0], "the second result"); + assertEqual(results[2], results[0], "the third result"); + }, + }, + { + id: "idempotency/restart", + title: "receipts survive a server restart", + async run(ctx) { + await withIdempotency(ctx); + if (!ctx.options.restart) return ctx.skip("no restart hook"); + const params = { + requestKey: key(ctx, "restart"), + set: { [ctx.name("restart")]: { v: 1 } }, + }; + const first = await ctx.client.blocksApply(params); + await ctx.options.restart(); + assertEqual( + await ctx.client.blocksApply(params), + first, + "the replayed result", + ); + }, + }, + { + id: "idempotency/tenant-isolation", + title: "one tenant's receipts never answer another tenant's requests", + async run(ctx) { + await withIdempotency(ctx); + if (!ctx.options.otherTenant) + return ctx.skip("no otherTenant credentials"); + const requestKey = key(ctx, "tenant"); + const a = ctx.name("tenant-a"); + const b = ctx.name("tenant-b"); + await ctx.client.blocksApply({ requestKey, set: { [a]: { v: 1 } } }); + const other = ctx.clientFor(ctx.options.otherTenant); + const result = await other.blocksApply({ + requestKey, + set: { [b]: { v: 1 } }, + }); + assert( + typeof result.versions[b] === "string", + "the other tenant's write ran", + ); + }, + }, +]; diff --git a/packages/shared/src/vendor/blocks-protocol/conformance/cases/reads.ts b/packages/shared/src/vendor/blocks-protocol/conformance/cases/reads.ts new file mode 100644 index 0000000000..5a51f9e4d2 --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/conformance/cases/reads.ts @@ -0,0 +1,238 @@ +/** + * `describe`, `schema.get` and `blocks.list`, including conditional reads. + */ +import { ErrorCode } from "../../errors"; +import { PROTOCOL_NAME, SCHEMA_FORMAT } from "../../types"; +import { assert, assertEqual, expectError } from "../context"; +import type { ConformanceCase } from "./types"; + +const LIMIT_KEYS = [ + "maxOpsPerApply", + "maxBlockBytes", + "maxRequestBytes", + "maxListBytes", + "maxSchemaBytes", + "maxBatchResponseBytes", +] as const; + +export const readCases: ConformanceCase[] = [ + { + id: "describe/shape", + title: + "describe reports the protocol, its version, kind, root, limits and features", + async run(ctx) { + const d = await ctx.client.describe(); + assertEqual(d.protocol, PROTOCOL_NAME, "protocol"); + assertEqual(d.version.major, 1, "version.major"); + assert( + Number.isInteger(d.version.minor) && d.version.minor >= 0, + "version.minor", + ); + assert( + typeof d.server?.name === "string" && + typeof d.server?.version === "string", + "server", + ); + assert( + d.kind === "working-tree" || d.kind === "git", + `kind ${String(d.kind)}`, + ); + assert(typeof d.readOnly === "boolean", "readOnly"); + assert( + typeof d.root === "string" && !d.root.startsWith("/"), + "root is relative", + ); + assertEqual(d.schemaFormat, SCHEMA_FORMAT, "schemaFormat"); + assert(d.refs === null || typeof d.refs.default === "string", "refs"); + assert( + d.writes.idempotency === null || d.writes.idempotency.retentionMs > 0, + "writes.idempotency", + ); + assert( + typeof d.writes.schemaPreconditions === "boolean", + "writes.schemaPreconditions", + ); + assert(d.pollIntervalMs > 0, "pollIntervalMs"); + for (const key of LIMIT_KEYS) assert(d.limits[key] > 0, `limits.${key}`); + assert( + d.preview === null || typeof d.preview.origin === "string", + "preview", + ); + if (d.readOnly) + assertEqual(d.assets, null, "assets is null when read-only"); + if (d.assets) { + assertEqual(d.assets.urlPrefix, "/assets/", "assets.urlPrefix"); + assert( + d.assets.maxBytes > 0 && typeof d.assets.dir === "string", + "assets", + ); + } + assert( + d.secrets === null || typeof d.secrets.publicKey === "string", + "secrets", + ); + }, + }, + { + id: "describe/secrets", + title: "describe returns the committed public key for secrets", + async run(ctx) { + const expected = ctx.options.secretsPublicKey; + if (expected === undefined) + return ctx.skip("no secretsPublicKey fixture"); + const d = await ctx.client.describe(); + assertEqual(d.secrets, { publicKey: expected }, "describe.secrets"); + }, + }, + { + id: "schema/read", + title: + "schema.get returns the schema and a version, and 'not modified' for that version", + async run(ctx) { + if (ctx.options.hasSchema === false) { + await expectError( + ctx.client.schemaGet(), + ErrorCode.NotFound, + "no schema", + ); + return; + } + const first = await ctx.client.schemaGet(); + assert(!first.notModified, "the first read returns the schema"); + assert( + typeof first.version === "string" && first.version.length > 0, + "a version", + ); + assert( + typeof first.schema === "object" && first.schema !== null, + "a schema object", + ); + const again = await ctx.client.schemaGet({ ifNoneMatch: first.version }); + assertEqual( + again, + { notModified: true, version: first.version }, + "the conditional read", + ); + const stale = await ctx.client.schemaGet({ + ifNoneMatch: `${first.version}-stale`, + }); + assert(!stale.notModified, "another version returns the schema"); + }, + }, + { + id: "list/read", + title: + "blocks.list returns every entry with versions, and 'not modified' for its revision", + async run(ctx) { + const first = await ctx.client.blocksList(); + assert(!first.notModified, "the first read returns the map"); + assert( + typeof first.revision === "string" && first.revision.length > 0, + "a revision", + ); + assert(Array.isArray(first.diagnostics), "diagnostics"); + assertEqual( + Object.keys(first.versions).sort(), + Object.keys(first.blocks).sort(), + "one version per entry", + ); + const d = await ctx.describe(); + if (d.refs === null) + assertEqual(first.resolvedRef, null, "resolvedRef without refs"); + const again = await ctx.client.blocksList({ + ifNoneMatch: first.revision, + }); + assertEqual( + again, + { + notModified: true, + revision: first.revision, + resolvedRef: first.resolvedRef, + }, + "the conditional read", + ); + }, + }, + { + id: "list/poll-batch", + title: "one batched poll reads the schema and the blocks conditionally", + async run(ctx) { + const list = await ctx.client.blocksList(); + const calls = [ + { + method: "blocks.list" as const, + params: { ifNoneMatch: list.revision }, + }, + ]; + const outcomes = await ctx.client.batch( + ctx.options.hasSchema === false + ? calls + : [ + { + method: "schema.get" as const, + params: { ifNoneMatch: (await ctx.client.schemaGet()).version }, + }, + ...calls, + ], + ); + for (const outcome of outcomes) { + assert(outcome.ok, "every poll call succeeds"); + assert( + (outcome.result as { notModified: boolean }).notModified, + "nothing changed", + ); + } + }, + }, + { + id: "refs/unsupported", + title: "a ref on an endpoint without branches is Unsupported", + async run(ctx) { + const d = await ctx.describe(); + if (d.refs !== null) return ctx.skip("the endpoint has branches"); + await expectError( + ctx.client.blocksList({ ref: "main" }), + ErrorCode.Unsupported, + "blocks.list ref", + ); + await expectError( + ctx.client.schemaGet({ ref: "main" }), + ErrorCode.Unsupported, + "schema.get ref", + ); + if (!d.readOnly) { + await expectError( + ctx.client.blocksApply({ ref: "main", delete: [ctx.name()] }), + ErrorCode.Unsupported, + "blocks.apply ref", + ); + } + }, + }, + { + id: "auth/token", + title: "a missing or wrong bearer token is HTTP 401 with Unauthorized", + async run(ctx) { + if (ctx.options.token === undefined) + return ctx.skip("the endpoint has no token"); + const body = JSON.stringify({ + jsonrpc: "2.0", + id: 1, + method: "describe", + }); + for (const authorization of ["", "Bearer wrong-token"]) { + const response = await ctx.raw(body, { headers: { authorization } }); + assertEqual( + response.status, + 401, + `HTTP status with "${authorization}"`, + ); + assertEqual( + (response.body as { error?: { code: number } }).error?.code, + ErrorCode.Unauthorized, + "the error code", + ); + } + }, + }, +]; diff --git a/packages/shared/src/vendor/blocks-protocol/conformance/cases/types.ts b/packages/shared/src/vendor/blocks-protocol/conformance/cases/types.ts new file mode 100644 index 0000000000..d85083b9d3 --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/conformance/cases/types.ts @@ -0,0 +1,9 @@ +import type { ConformanceContext } from "../context"; + +/** One black-box check of a content-protocol endpoint. */ +export interface ConformanceCase { + /** A stable id, such as `apply/atomic`. */ + id: string; + title: string; + run(ctx: ConformanceContext): Promise; +} diff --git a/packages/shared/src/vendor/blocks-protocol/conformance/cases/wire.ts b/packages/shared/src/vendor/blocks-protocol/conformance/cases/wire.ts new file mode 100644 index 0000000000..d37876abdc --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/conformance/cases/wire.ts @@ -0,0 +1,207 @@ +/** + * The wire format: JSON-RPC 2.0 envelopes, batches, HTTP status codes, gzip. + */ +import { ErrorCode } from "../../errors"; +import { MAX_BATCH_CALLS } from "../../types"; +import { assert, assertEqual, rawErrorCode } from "../context"; +import type { ConformanceCase } from "./types"; + +const req = (id: unknown, method: string, params?: unknown) => + params === undefined + ? { jsonrpc: "2.0", id, method } + : { jsonrpc: "2.0", id, method, params }; + +export const wireCases: ConformanceCase[] = [ + { + id: "wire/ids-echoed", + title: "string and number ids are echoed back", + async run(ctx) { + const a = await ctx.rpc(req("abc", "describe", {})); + assert(a.status === 200, `HTTP ${a.status}`); + assertEqual((a.body as { id: unknown }).id, "abc", "the string id"); + const b = await ctx.rpc(req(42, "describe")); + assertEqual((b.body as { id: unknown }).id, 42, "the number id"); + assert( + "result" in (b.body as object), + "describe without params succeeds", + ); + }, + }, + { + id: "wire/id-required", + title: "a request without an id is rejected, not run", + async run(ctx) { + const response = await ctx.rpc({ jsonrpc: "2.0", method: "describe" }); + assert(response.status === 200, `HTTP ${response.status}`); + assertEqual( + rawErrorCode(response.body), + ErrorCode.InvalidRequest, + "the error code", + ); + assertEqual((response.body as { id: unknown }).id, null, "the error id"); + }, + }, + { + id: "wire/envelope", + title: + "a bad envelope is Invalid Request; an unknown method is Method not found", + async run(ctx) { + const wrongVersion = await ctx.rpc({ + jsonrpc: "1.0", + id: 1, + method: "describe", + }); + assertEqual( + rawErrorCode(wrongVersion.body), + ErrorCode.InvalidRequest, + "jsonrpc 1.0", + ); + const unknown = await ctx.rpc(req(1, "blocks.rename", {})); + assertEqual( + rawErrorCode(unknown.body), + ErrorCode.MethodNotFound, + "an unknown method", + ); + const arrayParams = await ctx.rpc(req(1, "describe", [])); + assertEqual( + rawErrorCode(arrayParams.body), + ErrorCode.InvalidParams, + "array params", + ); + const unknownParam = await ctx.rpc(req(1, "blocks.list", { since: "x" })); + assertEqual( + rawErrorCode(unknownParam.body), + ErrorCode.InvalidParams, + "an unknown parameter", + ); + }, + }, + { + id: "wire/parse-error", + title: "malformed JSON is a Parse error", + async run(ctx) { + const response = await ctx.raw("{not json"); + assert(response.status === 200, `HTTP ${response.status}`); + assertEqual( + rawErrorCode(response.body), + ErrorCode.ParseError, + "the error code", + ); + }, + }, + { + id: "wire/content-type", + title: "only JSON bodies and POST are accepted", + async run(ctx) { + const form = await ctx.raw("a=1", { + headers: { "content-type": "application/x-www-form-urlencoded" }, + }); + assert(form.status >= 400, `a form post answered HTTP ${form.status}`); + const get = await ctx.raw("", { method: "GET" }); + assert(get.status >= 400, `a GET answered HTTP ${get.status}`); + }, + }, + { + id: "wire/batch-order", + title: "a batch runs in order and returns results in order", + async run(ctx) { + const response = await ctx.rpc([ + req(1, "describe"), + req(2, "nope"), + req("three", "blocks.list", {}), + ]); + assert(Array.isArray(response.body), "a batch answers with an array"); + const items = response.body as Array<{ + id: unknown; + error?: { code: number }; + }>; + assertEqual( + items.map((i) => i.id), + [1, 2, "three"], + "the response ids", + ); + assert(!items[0].error, "the first call succeeds"); + assertEqual( + items[1].error?.code, + ErrorCode.MethodNotFound, + "the second call fails alone", + ); + assert(!items[2].error, "the third call still runs"); + }, + }, + { + id: "wire/batch-limits", + title: `an empty batch is invalid; more than ${MAX_BATCH_CALLS} calls is LimitExceeded`, + async run(ctx) { + const empty = await ctx.rpc([]); + assertEqual( + rawErrorCode(empty.body), + ErrorCode.InvalidRequest, + "an empty batch", + ); + const calls = Array.from({ length: MAX_BATCH_CALLS + 1 }, (_, i) => + req(i + 1, "describe"), + ); + const tooMany = await ctx.rpc(calls); + assertEqual( + rawErrorCode(tooMany.body), + ErrorCode.LimitExceeded, + "an oversized batch", + ); + const ok = await ctx.rpc(calls.slice(0, MAX_BATCH_CALLS)); + assert( + Array.isArray(ok.body) && ok.body.length === MAX_BATCH_CALLS, + `a batch of ${MAX_BATCH_CALLS} runs`, + ); + }, + }, + { + id: "wire/batch-not-atomic", + title: "a batch isn't atomic: a failed write doesn't undo an earlier one", + async run(ctx) { + const description = await ctx.describe(); + if (description.readOnly) return ctx.skip("read-only endpoint"); + const good = ctx.name("batch"); + const response = await ctx.rpc([ + req(1, "blocks.apply", { set: { [good]: { title: "kept" } } }), + req(2, "blocks.apply", { set: { [ctx.name("bad")]: [] } }), + ]); + const items = response.body as Array<{ error?: { code: number } }>; + assert(!items[0].error, "the first write succeeds"); + assertEqual( + items[1].error?.code, + ErrorCode.InvalidBlock, + "the second write fails", + ); + const list = await ctx.client.blocksList(); + assert( + !list.notModified && good in list.blocks, + "the first write landed", + ); + }, + }, + { + id: "wire/gzip", + title: "responses are gzip-compressed when the request accepts it", + async run(ctx) { + const description = await ctx.describe(); + if (description.readOnly) return ctx.skip("read-only endpoint"); + const name = ctx.name("gzip"); + await ctx.client.blocksApply({ + set: { [name]: { text: "x".repeat(8 * 1024) } }, + }); + const response = await ctx.rpc(req(1, "blocks.list", {}), { + "accept-encoding": "gzip", + }); + assertEqual( + response.headers.get("content-encoding"), + "gzip", + "Content-Encoding", + ); + assert( + rawErrorCode(response.body) === undefined, + "the gzip body decodes to a result", + ); + }, + }, +]; diff --git a/packages/shared/src/vendor/blocks-protocol/conformance/cases/writes.ts b/packages/shared/src/vendor/blocks-protocol/conformance/cases/writes.ts new file mode 100644 index 0000000000..22d9a23168 --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/conformance/cases/writes.ts @@ -0,0 +1,450 @@ +/** + * `blocks.apply`: atomic set and delete, set precedence, validation, the + * file-name rule and preconditions. + */ +import { ErrorCode, type InvalidBlockData } from "../../errors"; +import type { BlocksListResult } from "../../types"; +import { + assert, + assertEqual, + type ConformanceContext, + expectError, +} from "../context"; +import type { ConformanceCase } from "./types"; + +type FullList = Extract; + +async function list(ctx: ConformanceContext): Promise { + const result = await ctx.client.blocksList(); + assert(!result.notModified, "an unconditional list returns the map"); + return result; +} + +async function writable(ctx: ConformanceContext) { + const d = await ctx.describe(); + if (d.readOnly) return ctx.skip("read-only endpoint"); + return d; +} + +function violations(error: { data: unknown }): InvalidBlockData["violations"] { + const data = error.data as InvalidBlockData | undefined; + assert( + Array.isArray(data?.violations), + "InvalidBlock carries data.violations", + ); + return data!.violations; +} + +export const writeCases: ConformanceCase[] = [ + { + id: "apply/create-and-read", + title: + "a write returns versions and a revision that blocks.list then reports", + async run(ctx) { + await writable(ctx); + const name = ctx.name("create"); + const value = { + __resolveType: "hero", + title: "Olá — 世界", + items: [1, { a: null }], + }; + const result = await ctx.client.blocksApply({ set: { [name]: value } }); + assert(typeof result.revision === "string", "a revision"); + assert( + typeof result.versions[name] === "string", + "a version for the entry", + ); + const after = await list(ctx); + assertEqual(after.blocks[name], value, "the stored entry"); + assertEqual( + after.versions[name], + result.versions[name], + "the listed version", + ); + assertEqual(after.revision, result.revision, "the listed revision"); + const poll = await ctx.client.blocksList({ + ifNoneMatch: result.revision, + }); + assert( + poll.notModified, + "polling with the returned revision is 'not modified'", + ); + }, + }, + { + id: "apply/atomic-set-delete", + title: "set and delete land together", + async run(ctx) { + await writable(ctx); + const [a, b, c] = [ctx.name("a"), ctx.name("b"), ctx.name("c")]; + await ctx.client.blocksApply({ set: { [a]: { v: 1 }, [b]: { v: 2 } } }); + const result = await ctx.client.blocksApply({ + set: { [c]: { v: 3 } }, + delete: [a], + }); + assertEqual( + result.versions[a], + null, + "a deleted entry's version is null", + ); + const after = await list(ctx); + assert(!(a in after.blocks), "the deleted entry is gone"); + assertEqual(after.blocks[b], { v: 2 }, "the untouched entry stays"); + assertEqual(after.blocks[c], { v: 3 }, "the new entry exists"); + }, + }, + { + id: "apply/set-wins", + title: "set wins when a name is in both set and delete", + async run(ctx) { + await writable(ctx); + const name = ctx.name("both"); + const result = await ctx.client.blocksApply({ + set: { [name]: { kept: true } }, + delete: [name], + }); + assert( + typeof result.versions[name] === "string", + "the entry has a version", + ); + assertEqual( + (await list(ctx)).blocks[name], + { kept: true }, + "the entry exists", + ); + }, + }, + { + id: "apply/delete-missing", + title: "deleting a missing name counts as deleted", + async run(ctx) { + await writable(ctx); + const name = ctx.name("missing"); + const result = await ctx.client.blocksApply({ delete: [name] }); + assertEqual(result.versions, { [name]: null }, "versions"); + }, + }, + { + id: "apply/whole-entry-replace", + title: "set replaces the whole entry (no merge)", + async run(ctx) { + await writable(ctx); + const name = ctx.name("replace"); + await ctx.client.blocksApply({ set: { [name]: { a: 1, b: 2 } } }); + await ctx.client.blocksApply({ set: { [name]: { b: 3 } } }); + assertEqual((await list(ctx)).blocks[name], { b: 3 }, "the entry"); + }, + }, + { + id: "apply/all-or-nothing-validation", + title: "every violation is reported at once and nothing is written", + async run(ctx) { + await writable(ctx); + const good = ctx.name("good"); + const error = await expectError( + ctx.client.blocksApply({ + set: { + [good]: { ok: true }, + [ctx.name("array")]: [] as never, + [ctx.name("null")]: null as never, + }, + }), + ErrorCode.InvalidBlock, + "invalid values", + ); + assertEqual( + violations(error).length, + 2, + "one violation per invalid entry", + ); + assert( + !(good in (await list(ctx)).blocks), + "the valid entry wasn't written", + ); + }, + }, + { + id: "apply/name-rules", + title: "names the site editor can't save are refused", + async run(ctx) { + await writable(ctx); + const p = ctx.prefix; + const bad = [ + "", + `${p}\\x`, + `${p}..x`, + "CON", + "con.backup", + "__proto__", + `${p}-mod.ts`, + `${p}-mod.tsx`, + `.${p}-hidden`, + ".", + ]; + bad.push(`${p}-${"é".repeat(60)}`); // 360 encoded bytes + for (const name of bad) { + const error = await expectError( + ctx.client.blocksApply({ set: { [name]: { x: 1 } } }), + ErrorCode.InvalidBlock, + `the name ${JSON.stringify(name)}`, + ); + assert( + violations(error).some((v) => v.name === name), + `a violation names ${JSON.stringify(name)}`, + ); + } + // A name ending in a source extension can still be deleted. + const result = await ctx.client.blocksApply({ delete: [`${p}-mod.ts`] }); + assertEqual( + result.versions, + { [`${p}-mod.ts`]: null }, + "deleting a .ts name", + ); + }, + }, + { + id: "apply/case-collision", + title: + "a new name that differs from another entry's only in letter case is refused", + async run(ctx) { + await writable(ctx); + const name = ctx.name("CaseName"); + await ctx.client.blocksApply({ set: { [name]: { v: 1 } } }); + await expectError( + ctx.client.blocksApply({ set: { [name.toLowerCase()]: { v: 2 } } }), + ErrorCode.InvalidBlock, + "a case variant", + ); + await ctx.client.blocksApply({ set: { [name]: { v: 3 } } }); + assertEqual( + (await list(ctx)).blocks[name], + { v: 3 }, + "updating the existing name", + ); + }, + }, + { + id: "apply/spellings", + title: + "two spellings of one name are one entry: guards see either, and a write keeps only the written spelling", + async run(ctx) { + await writable(ctx); + const encoded = `${ctx.prefix}-Home%20Page`; + const literal = `${ctx.prefix}-Home Page`; + ctx.track(encoded); + ctx.track(literal); + await expectError( + ctx.client.blocksApply({ + set: { [encoded]: { v: 1 }, [literal]: { v: 2 } }, + }), + ErrorCode.InvalidBlock, + "both spellings in one write", + ); + const created = await ctx.client.blocksApply({ + set: { [encoded]: { v: 1 } }, + }); + // Create-only under another spelling: the entry exists, so the guard fails. + const error = await expectError( + ctx.client.blocksApply({ + set: { [literal]: { v: 2 } }, + ifMatch: { [literal]: null }, + }), + ErrorCode.Conflict, + "create-only on another spelling of an existing entry", + ); + assertEqual( + (error.data as { entries?: unknown }).entries, + { [literal]: { expected: null, actual: created.versions[encoded] } }, + "the conflict reports the existing spelling's version", + ); + assertEqual( + (await list(ctx)).blocks[encoded], + { v: 1 }, + "nothing was deleted", + ); + // A guard on one spelling holds for the other. + const written = await ctx.client.blocksApply({ + set: { [literal]: { v: 2 } }, + ifMatch: { [literal]: created.versions[encoded]! }, + }); + assertEqual( + written.versions[encoded], + null, + "the result reports the deleted spelling as gone, so a client's map drops it", + ); + assert( + typeof written.versions[literal] === "string", + "the written spelling has a version", + ); + const after = await list(ctx); + assertEqual(after.blocks[literal], { v: 2 }, "the written spelling"); + assert( + !(encoded in after.blocks), + "the other spelling was deleted in the same commit", + ); + assert( + !after.diagnostics.some( + (d) => d.name === literal || d.name === encoded, + ), + "no shadowed spelling is left", + ); + }, + }, + { + id: "apply/if-match", + title: + "ifMatch guards a write; a failed guard writes nothing and reports versions", + async run(ctx) { + await writable(ctx); + const name = ctx.name("guarded"); + const other = ctx.name("other"); + const created = await ctx.client.blocksApply({ + set: { [name]: { v: 1 } }, + ifMatch: { [name]: null }, + }); + const v1 = created.versions[name]!; + const error = await expectError( + ctx.client.blocksApply({ + set: { [name]: { v: 2 }, [other]: { v: 2 } }, + ifMatch: { [name]: `${v1}x` }, + }), + ErrorCode.Conflict, + "a stale version", + ); + assertEqual( + (error.data as { entries?: unknown }).entries, + { [name]: { expected: `${v1}x`, actual: v1 } }, + "the conflict's versions", + ); + const after = await list(ctx); + assertEqual( + after.blocks[name], + { v: 1 }, + "the guarded entry is unchanged", + ); + assert(!(other in after.blocks), "nothing else was written"); + await expectError( + ctx.client.blocksApply({ + set: { [name]: { v: 3 } }, + ifMatch: { [name]: null }, + }), + ErrorCode.Conflict, + "create-only on an existing entry", + ); + const updated = await ctx.client.blocksApply({ + set: { [name]: { v: 4 } }, + ifMatch: { [name]: v1 }, + }); + assert(updated.versions[name] !== v1, "a matching version writes"); + }, + }, + { + id: "apply/rename-recipe", + title: "renaming is a set plus a delete, create-only on the new name", + async run(ctx) { + await writable(ctx); + const from = ctx.name("from"); + const to = ctx.name("to"); + const created = await ctx.client.blocksApply({ + set: { [from]: { v: 1 } }, + }); + await ctx.client.blocksApply({ + set: { [to]: { v: 1 } }, + delete: [from], + ifMatch: { [to]: null, [from]: created.versions[from]! }, + }); + const after = await list(ctx); + assert(!(from in after.blocks) && to in after.blocks, "the entry moved"); + }, + }, + { + id: "apply/schema-precondition", + title: "ifSchemaMatch rejects a write when the schema changed", + async run(ctx) { + const d = await writable(ctx); + const name = ctx.name("schema-guarded"); + if (!d.writes.schemaPreconditions) { + await expectError( + ctx.client.blocksApply({ set: { [name]: {} }, ifSchemaMatch: "x" }), + ErrorCode.Unsupported, + "an unadvertised guard", + ); + return; + } + if (ctx.options.hasSchema === false) return ctx.skip("no schema"); + const { version } = await ctx.client.schemaGet(); + const error = await expectError( + ctx.client.blocksApply({ + set: { [name]: {} }, + ifSchemaMatch: `${version}-old`, + }), + ErrorCode.Conflict, + "a stale schema version", + ); + assertEqual( + (error.data as { schema?: unknown }).schema, + { expected: `${version}-old`, actual: version }, + "the conflict's schema versions", + ); + assert(!(name in (await list(ctx)).blocks), "nothing was written"); + await ctx.client.blocksApply({ + set: { [name]: {} }, + ifSchemaMatch: version, + }); + }, + }, + { + id: "apply/unknown-params", + title: + "an unknown parameter is refused, so an unknown guard never becomes an unguarded write", + async run(ctx) { + await writable(ctx); + const name = ctx.name("unknown-param"); + await expectError( + ctx.client.call("blocks.apply", { + set: { [name]: {} }, + ifUnmodifiedSince: "x", + } as never), + ErrorCode.InvalidParams, + "an unknown guard", + ); + assert(!(name in (await list(ctx)).blocks), "nothing was written"); + }, + }, + { + id: "apply/read-only", + title: "a write to a read-only endpoint is ReadOnly", + async run(ctx) { + const d = await ctx.describe(); + if (!d.readOnly) return ctx.skip("the endpoint accepts writes"); + await expectError( + ctx.client.blocksApply({ delete: ["x"] }), + ErrorCode.ReadOnly, + "a write", + ); + }, + }, + { + id: "apply/last-writer-wins", + title: "without guards, the later of two writes wins", + async run(ctx) { + await writable(ctx); + const name = ctx.name("lww"); + await Promise.all([ + ctx.client.blocksApply({ set: { [name]: { from: "a" } } }), + ctx.client.blocksApply({ set: { [name]: { from: "b" } } }), + ]); + const value = (await list(ctx)).blocks[name] as { from: string }; + assert( + value.from === "a" || value.from === "b", + "one of the writes is stored whole", + ); + await ctx.client.blocksApply({ set: { [name]: { from: "c" } } }); + assertEqual( + (await list(ctx)).blocks[name], + { from: "c" }, + "the latest write", + ); + }, + }, +]; diff --git a/packages/shared/src/vendor/blocks-protocol/conformance/content-hash-fixtures.ts b/packages/shared/src/vendor/blocks-protocol/conformance/content-hash-fixtures.ts new file mode 100644 index 0000000000..737c3996f2 --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/conformance/content-hash-fixtures.ts @@ -0,0 +1,101 @@ +/** + * Golden fixtures for canonical content hashing: every implementation that + * computes a content revision (the CLI, the release materializer, the site + * editor's backend) checks it produces these exact bytes and hashes. + * + * ```ts + * import { contentHashFixtures } from "@decocms/blocks/protocol/conformance"; + * for (const f of contentHashFixtures) expect(await myRevision(f.blocks)).toBe(f.revision); + * ``` + */ + +export interface ContentHashFixture { + id: string; + title: string; + /** The `blocks` map: entry name to entry. */ + blocks: Record; + /** Its canonical JSON. */ + canonical: string; + /** SHA-256 of the canonical JSON's UTF-8 bytes, lowercase hex. */ + revision: string; +} + +export const contentHashFixtures: readonly ContentHashFixture[] = [ + { + id: "empty", + title: "an empty map", + blocks: {}, + canonical: "{}", + revision: + "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + }, + { + id: "key-order", + title: + "keys sorted recursively in code-unit order, numeric-looking keys included; arrays keep their order", + blocks: { + b: { z: 1, a: 2 }, + a: [3, 1, 2], + B: true, + "10": "ten", + "2": "two", + "": null, + }, + canonical: + '{"":null,"10":"ten","2":"two","B":true,"a":[3,1,2],"b":{"a":2,"z":1}}', + revision: + "e10ffc5fe1399986d69ff6a5c326d3eca5a4c6ff4134504d2e169676e1155e52", + }, + { + id: "strings", + title: + "JSON string escaping: control characters escaped, non-ASCII and line separators as is", + blocks: { + s: { + text: "Ol\u00e1 \u2014 \u4e16\u754c \ud83d\ude00", + quote: '"\\/', + control: "\u0000\u001f\n\t", + separators: "\u2028\u2029", + }, + }, + canonical: + '{"s":{"control":"\\u0000\\u001f\\n\\t","quote":"\\"\\\\/","separators":"\u2028\u2029","text":"Ol\u00e1 \u2014 \u4e16\u754c \ud83d\ude00"}}', + revision: + "3dbb4dd183b625c383b703df94c2d12ae6271cad0ee7e63c51cf1bfda18c799c", + }, + { + id: "numbers", + title: "JSON number encoding, negative zero as zero", + blocks: { + n: { + zero: -0, + int: 42, + neg: -7, + frac: 0.1, + big: 1e21, + small: 1e-7, + max: 9007199254740991, + }, + }, + canonical: + '{"n":{"big":1e+21,"frac":0.1,"int":42,"max":9007199254740991,"neg":-7,"small":1e-7,"zero":0}}', + revision: + "783d90091668991414259155fb6b37ae5f09afc80a3aea62405fbe515bac535c", + }, + { + id: "page", + title: "a small site", + blocks: { + "pages-Home": { + __resolveType: "page", + path: "/", + sections: [{ __resolveType: "hero", title: "Hi" }], + }, + Header: { __resolveType: "header", links: [] }, + }, + canonical: + '{"Header":{"__resolveType":"header","links":[]},"pages-Home":{"__resolveType":"page","path":"/","sections":[{"__resolveType":"hero","title":"Hi"}]}}', + revision: + "6913c5e6aa28f441d3b0102177b372e4fb7c08bc09bdd8698df85e597282ac96", + }, +]; diff --git a/packages/shared/src/vendor/blocks-protocol/conformance/context.ts b/packages/shared/src/vendor/blocks-protocol/conformance/context.ts new file mode 100644 index 0000000000..0ebaee4d81 --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/conformance/context.ts @@ -0,0 +1,257 @@ +/** + * What every conformance case gets: a client, raw HTTP access, the endpoint's + * `describe`, unique names and assertion helpers. + */ +import { canonicalJson } from "../canonical"; +import { + type ContentClient, + createContentClient, + readResponseJson, +} from "../client"; +import { ContentProtocolError } from "../errors"; +import type { DescribeResult } from "../types"; + +export interface ConformanceOptions { + /** The endpoint URL, such as `http://127.0.0.1:4545/rpc`. */ + endpoint: string | URL; + /** The bearer token the endpoint expects, if any. */ + token?: string; + /** Extra headers on every request (an auth cookie, a tenant header). */ + headers?: Record; + /** A fetch implementation (defaults to the global one); use it to call a handler in process. */ + fetch?: (request: Request) => Promise; + /** + * A block type in the endpoint's schema with a `Secret` field, so the + * secret guard can be checked. Leave out to skip those cases. + */ + secretField?: { blockType: string; field: string }; + /** The `.deco/secrets.pub` content the endpoint should report, if the harness wrote one. */ + secretsPublicKey?: string; + /** Whether the endpoint has a schema (default true). `false` checks NotFound instead. */ + hasSchema?: boolean; + /** + * The endpoint serves a schema larger than its `limits.maxSchemaBytes`, so + * reading it and writes that depend on it must be LimitExceeded. Run such + * an endpoint with only the `limits/schema-bytes` case (pass a filter to + * `runConformance` or `defineConformanceSuite`): every other case expects + * a schema it can read. + */ + schemaOverLimit?: boolean; + /** + * The URL prefix uploads are served under (`PUT `), + * such as `http://127.0.0.1:4545/assets/`. Leave out to skip the upload cases. + */ + assetsEndpoint?: string | URL; + /** + * Restarts the server, keeping its durable state, to check that request-key + * receipts survive a restart. Leave out to skip that case. + */ + restart?: () => Promise; + /** Credentials of another tenant on the same storage, to check receipt isolation. */ + otherTenant?: { token?: string; headers?: Record }; + /** Prefix of every name the suite creates (default `conformance-`). */ + namePrefix?: string; +} + +/** Thrown by `ctx.skip()`: the case doesn't apply to this endpoint. */ +export class SkipCase extends Error { + constructor(reason: string) { + super(reason); + this.name = "SkipCase"; + } +} + +export class ConformanceFailure extends Error { + constructor(message: string) { + super(message); + this.name = "ConformanceFailure"; + } +} + +export interface RawResponse { + status: number; + headers: Headers; + body: unknown; +} + +export class ConformanceContext { + readonly client: ContentClient; + private counter = 0; + private readonly created = new Set(); + private describeResult: DescribeResult | undefined; + + constructor( + readonly options: ConformanceOptions, + readonly prefix: string, + ) { + this.client = createContentClient(options); + } + + /** A client with other credentials (for tenant isolation). */ + clientFor(credentials: { + token?: string; + headers?: Record; + }): ContentClient { + return createContentClient({ ...this.options, ...credentials }); + } + + async describe(): Promise { + this.describeResult ??= await this.client.describe(); + return this.describeResult; + } + + /** A fresh name the suite owns; it's deleted when the case ends. */ + name(label = "entry"): string { + const name = `${this.prefix}-${label}-${++this.counter}`; + this.created.add(name); + return name; + } + + /** Marks a name the case created by other means for cleanup. */ + track(name: string): void { + this.created.add(name); + } + + /** Deletes every name the case created. */ + async cleanup(): Promise { + if (this.created.size === 0) return; + const names = [...this.created]; + this.created.clear(); + const description = await this.describe(); + if (description.readOnly) return; + for (let i = 0; i < names.length; i += description.limits.maxOpsPerApply) { + await this.client + .blocksApply({ + delete: names.slice(i, i + description.limits.maxOpsPerApply), + }) + .catch(() => {}); + } + } + + skip(reason: string): never { + throw new SkipCase(reason); + } + + /** Posts a raw body to the endpoint and decodes the JSON answer. */ + async raw( + body: string | Uint8Array, + init: { method?: string; headers?: Record } = {}, + ): Promise { + const headers: Record = { + "content-type": "application/json", + ...this.options.headers, + ...init.headers, + }; + if (this.options.token !== undefined && !("authorization" in headers)) { + headers.authorization = `Bearer ${this.options.token}`; + } + for (const [key, value] of Object.entries(headers)) + if (value === "") delete headers[key]; + const method = init.method ?? "POST"; + const doFetch = + this.options.fetch ?? ((request: Request) => fetch(request)); + const response = await doFetch( + new Request(this.options.endpoint, { + method, + headers, + body: + method === "GET" || method === "HEAD" + ? undefined + : (body as BodyInit), + }), + ); + let parsed: unknown = null; + try { + parsed = await readResponseJson(response); + } catch { + parsed = null; + } + return { status: response.status, headers: response.headers, body: parsed }; + } + + /** Uploads `body` as `PUT `, with the endpoint's credentials. */ + async upload( + name: string, + body: Uint8Array | string, + contentType: string, + ): Promise { + const base = this.options.assetsEndpoint; + if (base === undefined) return this.skip("no assetsEndpoint"); + const headers: Record = { + ...this.options.headers, + "content-type": contentType, + }; + if (this.options.token !== undefined) + headers.authorization = `Bearer ${this.options.token}`; + const doFetch = + this.options.fetch ?? ((request: Request) => fetch(request)); + const url = new URL( + encodeURIComponent(name), + new URL(String(base)).href.replace(/\/?$/, "/"), + ); + const response = await doFetch( + new Request(url, { method: "PUT", headers, body: body as BodyInit }), + ); + let parsed: unknown = null; + try { + parsed = await readResponseJson(response); + } catch { + parsed = null; + } + return { status: response.status, headers: response.headers, body: parsed }; + } + + /** Posts one JSON-RPC payload (an object or a batch) as JSON. */ + rpc( + payload: unknown, + headers?: Record, + ): Promise { + return this.raw(JSON.stringify(payload), { headers }); + } +} + +export function assert(condition: unknown, message: string): asserts condition { + if (!condition) throw new ConformanceFailure(message); +} + +export function assertEqual( + actual: unknown, + expected: unknown, + message: string, +): void { + const a = canonicalJson(actual ?? null); + const e = canonicalJson(expected ?? null); + if (a !== e) + throw new ConformanceFailure( + `${message}\n expected: ${e}\n actual: ${a}`, + ); +} + +/** Awaits `promise` and checks it fails with protocol error `code`. */ +export async function expectError( + promise: Promise, + code: number, + message: string, +): Promise { + try { + await promise; + } catch (error) { + if (error instanceof ContentProtocolError) { + assert( + error.code === code, + `${message}: expected error ${code}, got ${error.code} (${error.message})`, + ); + return error; + } + throw error; + } + throw new ConformanceFailure( + `${message}: expected error ${code}, but the call succeeded`, + ); +} + +/** The JSON-RPC error code of a raw response body, if it carries one. */ +export function rawErrorCode(body: unknown): number | undefined { + const error = (body as { error?: { code?: unknown } } | null)?.error; + return typeof error?.code === "number" ? error.code : undefined; +} diff --git a/packages/shared/src/vendor/blocks-protocol/conformance/index.ts b/packages/shared/src/vendor/blocks-protocol/conformance/index.ts new file mode 100644 index 0000000000..bc443bbd05 --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/conformance/index.ts @@ -0,0 +1,167 @@ +/** + * `@decocms/blocks/protocol/conformance`: a black-box test suite any + * content-protocol endpoint runs over HTTP, which is how the filesystem + * storage, the site editor's GitHub backend and any other backend are kept to + * the same contract. + * + * Run it with any test runner: + * + * ```ts + * import { describe, it } from "vitest"; + * import { defineConformanceSuite } from "@decocms/blocks/protocol/conformance"; + * + * defineConformanceSuite({ describe, it }, { endpoint: "http://127.0.0.1:4545/rpc", token }); + * ``` + * + * or collect a report with `runConformance(options)`. The suite writes only + * names under its own prefix and deletes them after each case (uploads, which + * the protocol can't delete, stay in the asset folder). Cases that + * don't apply to an endpoint (a read-only one, one without request keys) + * skip themselves. Browser-safe: it only needs `fetch`. + */ +import { assetCases } from "./cases/assets"; +import { guardCases } from "./cases/guards"; +import { readCases } from "./cases/reads"; +import type { ConformanceCase } from "./cases/types"; +import { wireCases } from "./cases/wire"; +import { writeCases } from "./cases/writes"; +import { + ConformanceContext, + type ConformanceOptions, + SkipCase, +} from "./context"; + +export type { ConformanceCase } from "./cases/types"; +export { + type ContentHashFixture, + contentHashFixtures, +} from "./content-hash-fixtures"; +export { + assert, + assertEqual, + ConformanceContext, + ConformanceFailure, + type ConformanceOptions, + expectError, + type RawResponse, + rawErrorCode, + SkipCase, +} from "./context"; + +/** Every conformance case, in the order they run. */ +export const conformanceCases: readonly ConformanceCase[] = [ + ...wireCases, + ...readCases, + ...writeCases, + ...guardCases, + ...assetCases, +]; + +function defaultPrefix(): string { + return `conformance-${Math.random().toString(36).slice(2, 8)}`; +} + +export interface CaseOutcome { + id: string; + title: string; + status: "passed" | "failed" | "skipped"; + /** Why it failed or was skipped. */ + message?: string; +} + +/** Runs one case with its own names, cleaning them up afterwards. */ +export async function runCase( + testCase: ConformanceCase, + options: ConformanceOptions, + prefix: string, +): Promise { + const ctx = new ConformanceContext(options, prefix); + try { + await testCase.run(ctx); + return { id: testCase.id, title: testCase.title, status: "passed" }; + } catch (error) { + if (error instanceof SkipCase) { + return { + id: testCase.id, + title: testCase.title, + status: "skipped", + message: error.message, + }; + } + return { + id: testCase.id, + title: testCase.title, + status: "failed", + message: error instanceof Error ? error.message : String(error), + }; + } finally { + await ctx.cleanup(); + } +} + +export interface ConformanceReport { + passed: number; + failed: number; + skipped: number; + outcomes: CaseOutcome[]; +} + +/** Runs every case (or the ones `filter` keeps) in order and reports the outcomes. */ +export async function runConformance( + options: ConformanceOptions, + filter: (testCase: ConformanceCase) => boolean = () => true, +): Promise { + const prefix = options.namePrefix ?? defaultPrefix(); + const outcomes: CaseOutcome[] = []; + let index = 0; + for (const testCase of conformanceCases.filter(filter)) { + outcomes.push(await runCase(testCase, options, `${prefix}-${++index}`)); + } + const count = (status: CaseOutcome["status"]) => + outcomes.filter((o) => o.status === status).length; + return { + passed: count("passed"), + failed: count("failed"), + skipped: count("skipped"), + outcomes, + }; +} + +/** The parts of a test runner the suite registers itself with (vitest, jest, node:test, bun:test). */ +export interface TestRegistrar { + describe(name: string, body: () => void): void; + it( + name: string, + body: (context?: { skip?: (note?: string) => void }) => Promise, + timeout?: number, + ): void; +} + +/** Registers one test per conformance case with a test runner. */ +export function defineConformanceSuite( + runner: TestRegistrar, + options: ConformanceOptions | (() => ConformanceOptions), + suiteName = "content protocol conformance", + filter: (testCase: ConformanceCase) => boolean = () => true, +): void { + const resolve = typeof options === "function" ? options : () => options; + const prefix = defaultPrefix(); + runner.describe(suiteName, () => { + conformanceCases.filter(filter).forEach((testCase, index) => { + runner.it( + `${testCase.id}: ${testCase.title}`, + async (context) => { + const resolved = resolve(); + const outcome = await runCase( + testCase, + resolved, + `${resolved.namePrefix ?? prefix}-${index + 1}`, + ); + if (outcome.status === "failed") throw new Error(outcome.message); + if (outcome.status === "skipped") context?.skip?.(outcome.message); + }, + 60_000, + ); + }); + }); +} diff --git a/packages/shared/src/vendor/blocks-protocol/errors.ts b/packages/shared/src/vendor/blocks-protocol/errors.ts new file mode 100644 index 0000000000..4735581f02 --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/errors.ts @@ -0,0 +1,154 @@ +/** + * Content-protocol errors. Every failure travels as a JSON-RPC 2.0 `error` + * object (HTTP 200), except a missing or invalid bearer token (401) and a body + * over the size limit (413), which apply to the whole batch. + */ + +export const ErrorCode = { + // Standard JSON-RPC 2.0 codes. + ParseError: -32700, + InvalidRequest: -32600, + MethodNotFound: -32601, + InvalidParams: -32602, + InternalError: -32603, + // Content-protocol codes. + NotFound: -32001, + Conflict: -32002, + InvalidBlock: -32003, + ReadOnly: -32005, + Unsupported: -32006, + LimitExceeded: -32007, + Unavailable: -32008, + Unauthorized: -32010, + Forbidden: -32011, +} as const; + +export type ErrorCodeName = keyof typeof ErrorCode; +export type ErrorCodeValue = (typeof ErrorCode)[ErrorCodeName]; + +/** The JSON-RPC `error` object. */ +export interface RpcErrorObject { + code: number; + message: string; + data?: unknown; +} + +/** One entry's failed `ifMatch` guard. */ +export interface VersionMismatch { + expected: string | null; + actual: string | null; +} + +/** `error.data` of a Conflict (-32002). */ +export interface ConflictData { + entries?: Record; + schema?: { expected: string; actual: string | null }; +} + +/** One rule a `blocks.apply` breaks. */ +export interface BlockViolation { + /** The entry name the violation is about. */ + name: string; + /** A JSON Pointer inside the entry, when the violation is about one value. */ + pointer?: string; + rule: string; + message: string; +} + +/** `error.data` of an InvalidBlock (-32003). */ +export interface InvalidBlockData { + violations: BlockViolation[]; +} + +/** `error.data` of an Unavailable (-32008). */ +export interface UnavailableData { + retryAfterMs?: number; +} + +/** + * An error a method fails with. The server turns it into the JSON-RPC + * `error` object; the client throws it back. + */ +export class ContentProtocolError extends Error { + readonly code: number; + readonly data: unknown; + + constructor(code: number, message: string, data?: unknown) { + super(message); + this.name = "ContentProtocolError"; + this.code = code; + this.data = data; + } + + toJSON(): RpcErrorObject { + return this.data === undefined + ? { code: this.code, message: this.message } + : { code: this.code, message: this.message, data: this.data }; + } + + static from(error: RpcErrorObject): ContentProtocolError { + return new ContentProtocolError(error.code, error.message, error.data); + } +} + +export const notFound = (message: string) => + new ContentProtocolError(ErrorCode.NotFound, message); + +export const conflict = (data: ConflictData) => + new ContentProtocolError(ErrorCode.Conflict, "a precondition failed", data); + +export const invalidBlock = (violations: BlockViolation[]) => + new ContentProtocolError( + ErrorCode.InvalidBlock, + violations.length === 1 + ? `invalid block "${violations[0]!.name}": ${violations[0]!.message}` + : `${violations.length} invalid blocks`, + { violations } satisfies InvalidBlockData, + ); + +export const readOnly = () => + new ContentProtocolError(ErrorCode.ReadOnly, "this endpoint is read-only"); + +export const unsupported = (message: string) => + new ContentProtocolError(ErrorCode.Unsupported, message); + +export const limitExceeded = ( + message: string, + data?: Record, +) => new ContentProtocolError(ErrorCode.LimitExceeded, message, data); + +export const unavailable = (message: string, retryAfterMs?: number) => + new ContentProtocolError( + ErrorCode.Unavailable, + message, + retryAfterMs === undefined + ? undefined + : ({ retryAfterMs } satisfies UnavailableData), + ); + +export const invalidParams = (message: string, data?: unknown) => + new ContentProtocolError(ErrorCode.InvalidParams, message, data); + +export const invalidRequest = (message: string) => + new ContentProtocolError(ErrorCode.InvalidRequest, message); + +export const methodNotFound = (method: string) => + new ContentProtocolError( + ErrorCode.MethodNotFound, + `unknown method "${method}"`, + ); + +export const parseError = () => + new ContentProtocolError(ErrorCode.ParseError, "invalid JSON"); + +export const internalError = () => + new ContentProtocolError(ErrorCode.InternalError, "internal error"); + +export const unauthorized = () => + new ContentProtocolError( + ErrorCode.Unauthorized, + "missing or invalid bearer token", + ); + +export const forbidden = () => + new ContentProtocolError(ErrorCode.Forbidden, "not allowed for this project"); diff --git a/packages/shared/src/vendor/blocks-protocol/index.ts b/packages/shared/src/vendor/blocks-protocol/index.ts new file mode 100644 index 0000000000..098f1cbc28 --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/index.ts @@ -0,0 +1,61 @@ +/** + * `@decocms/blocks/protocol`: the content protocol's method types, errors, + * client and file-name rule. Browser-safe. + * + * The protocol is how the site editor reads and writes saved blocks without + * running a site's code: JSON-RPC 2.0 over one HTTP endpoint with four + * methods — `describe`, `schema.get`, `blocks.list` and `blocks.apply`. + * + * Subpaths: + * - `@decocms/blocks/protocol/keys` — the file-name rule (also re-exported here) + * - `@decocms/blocks/protocol/server` — `createContentHandler(storage)` + * - `@decocms/blocks/protocol/storage/fs` — the filesystem storage (Node only) + * - `@decocms/blocks/protocol/storage/memory` — an in-memory storage (tests, reference) + * - `@decocms/blocks/protocol/conformance` — a black-box test suite over HTTP + * + * The SDK's runtime never imports the protocol, so it never reaches an app bundle. + */ +export { + APPLY_DIGEST_DOMAIN, + applyRequestDigest, + CanonicalJsonError, + CONTENT_HASH_FORMAT, + canonicalJson, + computeContentRevision, + sha256Hex, +} from "./canonical"; +export { + CIPHERTEXT_PATTERN, + CIPHERTEXT_VERSION, + type CiphertextParts, + decodeBase64Url, + encodeBase64Url, + encryptToCiphertext, + formatCiphertext, + GCM_TAG_BYTES, + IV_BYTES, + parseCiphertext, + publicKeyDerFromPem, + publicKeyPemFromDer, + WRAPPED_KEY_BYTES, +} from "./ciphertext"; +export { + assertSupportedEndpoint, + type BatchCall, + type BatchOutcome, + type ContentClient, + type ContentClientOptions, + createContentClient, +} from "./client"; +export * from "./errors"; +export * from "./keys"; +export { + checkSecrets, + isSecretBlock, + isSecretFieldSchema, + isWellFormedCiphertext, + SECRET_BLOCK_TYPE, + SECRET_FORMAT, +} from "./secrets"; +export * from "./storage"; +export * from "./types"; diff --git a/packages/shared/src/vendor/blocks-protocol/keys.ts b/packages/shared/src/vendor/blocks-protocol/keys.ts new file mode 100644 index 0000000000..5dc6e2ef33 --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/keys.ts @@ -0,0 +1,307 @@ +/** + * The one file-name rule for saved blocks (`@decocms/blocks/protocol/keys`). + * + * The site editor, `deco serve` and `deco content` all import this module, so + * an entry the site editor edits is the entry the app renders: + * + * - **Name to file:** `encodeURIComponent(name) + ".json"`, directly in + * `.deco/blocks` (no subfolders). + * - **File to name:** decode the file name, without `.json`, exactly once. If + * decoding fails, the raw name is the entry name. + * - **Two spellings of one name** (files that decode to the same name after + * repeated decoding): one wins — the file whose entry has a `path`, then the + * one that took more decoding, then the lowest file name. The others are + * reported as diagnostics; a write overwrites the winning spelling and + * deletes the others in the same commit. + * - **File content:** `JSON.stringify(entry, null, 2)` plus a newline, UTF-8. + * + * Ported from the v7 `blocks-cli/scripts/lib/blocks-dedupe.ts` and + * `blocks/src/cms/loadDecofileDirectory.ts` (`parseBlockId`), with the mtime + * tie-break dropped: a fresh clone gives every file the same mtime, and the + * rule must give the same answer on GitHub, where there is no mtime. + * + * Browser-safe: no Node APIs. + */ + +/** The extension every saved-block file carries. */ +export const BLOCK_FILE_EXTENSION = ".json"; + +/** + * The longest encoded name (without `.json`) the site editor can save, so the + * file fits a 255-byte file-name limit. + */ +export const MAX_ENCODED_NAME_BYTES = 250; + +/** Windows device names: a file whose stem is one of these can't be created on Windows. */ +const WINDOWS_DEVICE_NAMES = new Set([ + "CON", + "PRN", + "AUX", + "NUL", + ...Array.from({ length: 9 }, (_, i) => `COM${i + 1}`), + ...Array.from({ length: 9 }, (_, i) => `LPT${i + 1}`), +]); + +/** Extensions of source modules: a saved block ending in one would shadow a module. */ +const SOURCE_EXTENSIONS = [ + ".ts", + ".tsx", + ".mts", + ".cts", + ".js", + ".jsx", + ".mjs", + ".cjs", +]; + +const utf8 = new TextEncoder(); + +/** `encodeURIComponent(name) + ".json"`: the file a saved block named `name` is stored in. */ +export function blockFileName(name: string): string { + return `${encodeURIComponent(name)}${BLOCK_FILE_EXTENSION}`; +} + +/** + * True when `file` is a saved-block file name: it ends in `.json`, has a stem, + * isn't in a folder and isn't a dotfile (hidden files are never entries). + */ +export function isBlockFileName(file: string): boolean { + return ( + file.endsWith(BLOCK_FILE_EXTENSION) && + !file.startsWith(".") && + file.length > BLOCK_FILE_EXTENSION.length && + !file.includes("/") && + !file.includes("\\") + ); +} + +function stem(file: string): string { + return file.endsWith(BLOCK_FILE_EXTENSION) + ? file.slice(0, -BLOCK_FILE_EXTENSION.length) + : file; +} + +/** + * The entry name stored in `file`: the stem decoded exactly once. A stem that + * isn't valid percent-encoding (a page literally named `50% off`) is the name + * as it stands. + */ +export function blockNameFromFile(file: string): string { + const raw = stem(file); + try { + return decodeURIComponent(raw); + } catch { + return raw; + } +} + +/** + * Decodes `file`'s stem repeatedly until it stops changing, returning the + * fully decoded name (the spelling-group key) and how many decodes it took. + */ +export function fullyDecodeFileName(file: string): { + name: string; + passes: number; +} { + let name = stem(file); + let passes = 0; + while (name.includes("%")) { + let next: string; + try { + next = decodeURIComponent(name); + } catch { + break; + } + if (next === name) break; + name = next; + passes++; + } + return { name, passes }; +} + +/** The spelling-group key of a saved-block name: every file of the group decodes to it. */ +export function spellingKey(name: string): string { + return fullyDecodeFileName(blockFileName(name)).name; +} + +/** `JSON.stringify(entry, null, 2)` plus a newline: the bytes a saved block is stored as. */ +export function serializeBlock(entry: unknown): string { + return `${JSON.stringify(entry, null, 2)}\n`; +} + +/** True when a parsed entry is a page-like entry: it has a non-empty string `path`. */ +export function entryHasPath(entry: unknown): boolean { + return ( + typeof entry === "object" && + entry !== null && + !Array.isArray(entry) && + typeof (entry as { path?: unknown }).path === "string" && + (entry as { path: string }).path.length > 0 + ); +} + +/** One file that holds (a spelling of) a saved block. */ +export interface SpellingCandidate { + file: string; + /** Whether the file's entry has a non-empty `path`. */ + hasPath: boolean; +} + +/** Orders two spellings of one name: negative when `a` wins. */ +export function compareSpellings( + a: SpellingCandidate, + b: SpellingCandidate, +): number { + if (a.hasPath !== b.hasPath) return a.hasPath ? -1 : 1; + const passesA = fullyDecodeFileName(a.file).passes; + const passesB = fullyDecodeFileName(b.file).passes; + if (passesA !== passesB) return passesB - passesA; + if (a.file === b.file) return 0; + return a.file < b.file ? -1 : 1; +} + +/** The winner of one spelling group and the files it shadows. */ +export interface ResolvedSpelling< + T extends SpellingCandidate = SpellingCandidate, +> { + /** The entry name: the winning file's stem decoded exactly once. */ + name: string; + winner: T; + /** The other spellings, best first. */ + shadowed: T[]; +} + +/** + * Groups candidate files by spelling key and picks one winner per group. + * Returns the groups keyed by entry name, in file-name order of the winners. + */ +export function resolveSpellings( + candidates: readonly T[], +): Map> { + const groups = new Map(); + for (const candidate of candidates) { + const key = fullyDecodeFileName(candidate.file).name; + const group = groups.get(key); + if (group) group.push(candidate); + else groups.set(key, [candidate]); + } + const winners: ResolvedSpelling[] = []; + for (const group of groups.values()) { + const [winner, ...shadowed] = [...group].sort(compareSpellings); + if (!winner) continue; + winners.push({ name: blockNameFromFile(winner.file), winner, shadowed }); + } + winners.sort((a, b) => + a.winner.file < b.winner.file ? -1 : a.winner.file > b.winner.file ? 1 : 0, + ); + return new Map(winners.map((w) => [w.name, w])); +} + +/** Why a name can't be saved. */ +export type NameViolationReason = + | "empty" + | "invalid-character" + | "dot-dot" + | "leading-dot" + | "too-long" + | "case-collision" + | "device-name" + | "reserved" + | "source-extension"; + +export interface NameViolation { + reason: NameViolationReason; + message: string; +} + +export interface NameCheckOptions { + /** + * The names of the entries that already exist. A name that isn't one of + * them is new, and a new name that differs from an existing one only in + * letter case is refused. + */ + existingNames?: Iterable; +} + +/** + * Checks whether the site editor may save an entry under `name`. Returns + * every rule the name breaks (empty when it can be saved). + */ +export function checkBlockName( + name: string, + options: NameCheckOptions = {}, +): NameViolation[] { + const violations: NameViolation[] = []; + if (name.length === 0) { + return [{ reason: "empty", message: "the name is empty" }]; + } + if (name.includes("\\") || name.includes("\0")) { + violations.push({ + reason: "invalid-character", + message: "the name contains a backslash or a NUL character", + }); + } + if (name.includes("..")) { + violations.push({ reason: "dot-dot", message: 'the name contains ".."' }); + } + if (name.startsWith(".")) { + violations.push({ + reason: "leading-dot", + message: 'the name starts with ".", which would make its file hidden', + }); + } + if (name === "__proto__") { + violations.push({ + reason: "reserved", + message: 'the name "__proto__" is reserved', + }); + } + const encoded = encodeURIComponent(name); + if (utf8.encode(encoded).byteLength > MAX_ENCODED_NAME_BYTES) { + violations.push({ + reason: "too-long", + message: `the encoded name is over ${MAX_ENCODED_NAME_BYTES} bytes`, + }); + } + const deviceStem = encoded.split(".")[0]!.toUpperCase(); + if (WINDOWS_DEVICE_NAMES.has(deviceStem)) { + violations.push({ + reason: "device-name", + message: `"${deviceStem}" is a Windows device name`, + }); + } + const lower = name.toLowerCase(); + const extension = SOURCE_EXTENSIONS.find((ext) => lower.endsWith(ext)); + if (extension) { + violations.push({ + reason: "source-extension", + message: `the name ends in "${extension}", which would shadow a source module`, + }); + } + if (options.existingNames) { + let isNew = true; + let collidesWith: string | undefined; + for (const existing of options.existingNames) { + if (existing === name) { + isNew = false; + break; + } + if (collidesWith === undefined && existing.toLowerCase() === lower) + collidesWith = existing; + } + if (isNew && collidesWith !== undefined) { + violations.push({ + reason: "case-collision", + message: `the name differs from the existing entry "${collidesWith}" only in letter case`, + }); + } + } + return violations; +} + +/** Checks a name a write deletes: anything non-empty can be deleted. */ +export function checkDeletedName(name: string): NameViolation[] { + return name.length === 0 + ? [{ reason: "empty", message: "the name is empty" }] + : []; +} diff --git a/packages/shared/src/vendor/blocks-protocol/params.ts b/packages/shared/src/vendor/blocks-protocol/params.ts new file mode 100644 index 0000000000..c22d581331 --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/params.ts @@ -0,0 +1,76 @@ +/** + * Parameter validation for the four methods. `params` is always an object; + * unknown parameters are rejected, so a guard the server doesn't understand + * never turns into an unguarded write. + * + * zod only checks the shape: callers keep using the raw object, because a + * parsed copy of a record would turn an own `__proto__` key into a prototype. + */ +import { z } from "zod"; +import { invalidParams } from "./errors"; +import type { BlocksApplyParams, MethodName, ReadParams } from "./types"; + +const opaque = z.string().min(1).max(1024); + +const readParams = z.strictObject({ + ref: z.string().min(1).max(255).optional(), + ifNoneMatch: opaque.optional(), +}); + +const applyParams = z.strictObject({ + ref: z.string().min(1).max(255).optional(), + requestKey: z.string().min(1).max(256).optional(), + ifSchemaMatch: opaque.optional(), + // Entry shapes are checked later and reported as InvalidBlock, all at once. + set: z.record(z.string(), z.unknown()).optional(), + delete: z.array(z.string()).optional(), + ifMatch: z.record(z.string(), opaque.nullable()).optional(), +}); + +const schemas: Record = { + describe: z.strictObject({}), + "schema.get": readParams, + "blocks.list": readParams, + "blocks.apply": applyParams, +}; + +function describeIssues(error: z.ZodError): string { + return error.issues + .map((issue) => { + const path = issue.path.length ? issue.path.join(".") : "params"; + if (issue.code === "unrecognized_keys") { + return `unknown parameter${issue.keys.length > 1 ? "s" : ""} ${issue.keys + .map((k) => `"${k}"`) + .join(", ")}`; + } + return `${path}: ${issue.message}`; + }) + .join("; "); +} + +/** + * Checks `params` for `method` and returns it as given (never a copy). + * Throws Invalid params (-32602) when it doesn't fit. + */ +export function validateParams( + method: "describe", + params: unknown, +): Record; +export function validateParams( + method: "schema.get" | "blocks.list", + params: unknown, +): ReadParams; +export function validateParams( + method: "blocks.apply", + params: unknown, +): BlocksApplyParams; +export function validateParams(method: MethodName, params: unknown): unknown; +export function validateParams(method: MethodName, params: unknown): unknown { + const value = params === undefined ? {} : params; + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw invalidParams("params must be an object"); + } + const result = schemas[method].safeParse(value); + if (!result.success) throw invalidParams(describeIssues(result.error)); + return value; +} diff --git a/packages/shared/src/vendor/blocks-protocol/secrets.ts b/packages/shared/src/vendor/blocks-protocol/secrets.ts new file mode 100644 index 0000000000..cf5a848cbb --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/secrets.ts @@ -0,0 +1,275 @@ +/** + * The secret guard: the content protocol refuses plain text in a `Secret` + * field, so the site editor and agents can't save a credential by mistake. + * + * A `Secret` field must hold a `secret` block with a well-formed + * `ciphertext` (`{ "__resolveType": "secret", "ciphertext": "v1.…" }`, in the + * format `./ciphertext` pins: three base64url segments of checked lengths), or a + * variant block whose every value does. Every `secret` block anywhere in an + * entry must carry a well-formed `ciphertext`, whatever its field. + * + * The schema marks a `Secret` field with `"format": "secret"` — the contract + * `deco schema` emits for the `Secret` type. A legacy secret loader block + * (`website/loaders/secret.ts`) is left alone. Browser-safe. + */ +import { parseCiphertext } from "./ciphertext"; +import type { BlockViolation } from "./errors"; +import type { DecoMeta } from "./types"; + +/** The built-in block type that holds an encrypted value. */ +export const SECRET_BLOCK_TYPE = "secret"; + +/** The JSON Schema `format` that marks a `Secret` field. */ +export const SECRET_FORMAT = "secret"; + +const MULTIVARIATE_TYPES = new Set([ + "multivariate", + "website/flags/multivariate.ts", +]); +const LAZY_TYPE = "lazy"; +/** + * The legacy (v7) secret loader. Its `encrypted` prop is a string marked + * `"format": "secret"` that already holds the site's own ciphertext, so it + * isn't a `Secret` field; its blocks are migrated, not guarded. + */ +const LEGACY_SECRET_LOADER = /(^|\/)loaders\/secret\.ts$/; +const MAX_DEPTH = 512; + +type Json = Record; + +const isObject = (value: unknown): value is Json => + typeof value === "object" && value !== null && !Array.isArray(value); + +const resolveTypeOf = (value: unknown): string | undefined => + isObject(value) && typeof value.__resolveType === "string" + ? value.__resolveType + : undefined; + +const escapePointer = (key: string) => + key.replace(/~/g, "~0").replace(/\//g, "~1"); + +/** + * True when `ciphertext` is a well-formed secret ciphertext: + * `v1...` with segment lengths an RSA-OAEP and + * AES-256-GCM encryption produces. Checking it needs no key. + */ +export function isWellFormedCiphertext( + ciphertext: unknown, +): ciphertext is string { + return parseCiphertext(ciphertext) !== null; +} + +/** True when a JSON Schema node describes a `Secret` field. */ +export function isSecretFieldSchema(schema: unknown): boolean { + return isObject(schema) && schema.format === SECRET_FORMAT; +} + +/** True when `value` is a `secret` block with a well-formed `ciphertext`. */ +export function isSecretBlock(value: unknown): boolean { + return ( + resolveTypeOf(value) === SECRET_BLOCK_TYPE && + isWellFormedCiphertext((value as Json).ciphertext) + ); +} + +const blockIndexCache = new WeakMap>(); + +/** Every block type in the schema's manifest, by key, to the schema of its arguments. */ +function blockIndex(meta: DecoMeta): Map { + const cached = blockIndexCache.get(meta); + if (cached) return cached; + const index = new Map(); + for (const group of Object.values(meta.manifest?.blocks ?? {})) { + if (!isObject(group)) continue; + for (const [key, schema] of Object.entries(group)) { + if (!index.has(key)) index.set(key, schema); + } + } + blockIndexCache.set(meta, index); + return index; +} + +class SecretWalker { + readonly violations: BlockViolation[] = []; + + constructor( + private readonly name: string, + private readonly meta: DecoMeta | null, + ) {} + + private report(pointer: string, rule: string, message: string) { + this.violations.push({ name: this.name, pointer, rule, message }); + } + + /** Follows local `$ref`s into `schema.definitions`. */ + private deref(schema: unknown): unknown { + let current = schema; + for ( + let hops = 0; + hops < 32 && isObject(current) && typeof current.$ref === "string"; + hops++ + ) { + const match = /^#\/definitions\/(.+)$/.exec(current.$ref); + if (!match) return current; + const key = decodeURIComponent(match[1]!) + .replace(/~1/g, "/") + .replace(/~0/g, "~"); + current = this.meta?.schema?.definitions?.[key]; + } + return current; + } + + /** The schemas of a node, flattened through `allOf`/`anyOf`/`oneOf`. */ + private branches(schema: unknown, depth = 0): Json[] { + const node = this.deref(schema); + if (!isObject(node) || depth > 16) return []; + const out: Json[] = [node]; + for (const combinator of ["allOf", "anyOf", "oneOf"] as const) { + const list = node[combinator]; + if (Array.isArray(list)) + for (const item of list) out.push(...this.branches(item, depth + 1)); + } + return out; + } + + private isSecretField(schema: unknown): boolean { + return this.branches(schema).some(isSecretFieldSchema); + } + + private propertySchema(schema: unknown, key: string): unknown { + for (const branch of this.branches(schema)) { + const properties = branch.properties; + if (isObject(properties) && key in properties) return properties[key]; + } + for (const branch of this.branches(schema)) { + if (isObject(branch.additionalProperties)) + return branch.additionalProperties; + } + return undefined; + } + + private itemSchema(schema: unknown): unknown { + for (const branch of this.branches(schema)) { + if (isObject(branch.items)) return branch.items; + } + return undefined; + } + + /** A value stored in a `Secret` field. */ + checkSecretValue(value: unknown, pointer: string, depth: number) { + if (depth > MAX_DEPTH) + return this.report(pointer, "too-deep", "the value is nested too deeply"); + const type = resolveTypeOf(value); + if (type === SECRET_BLOCK_TYPE) + return this.walkBlock(value as Json, pointer, depth); + if (type !== undefined && MULTIVARIATE_TYPES.has(type)) { + const variants = (value as Json).variants; + if (!Array.isArray(variants)) { + return this.report( + pointer, + "secret-field", + "a Secret field holds a variant block without variants", + ); + } + for (const [index, variant] of variants.entries()) { + const at = `${pointer}/variants/${index}`; + if (!isObject(variant)) { + this.report(at, "secret-field", "a variant must be an object"); + continue; + } + if (variant.rule !== undefined) + this.walkAny(variant.rule, `${at}/rule`, depth + 1); + let inner = variant.value; + let innerAt = `${at}/value`; + if (resolveTypeOf(inner) === LAZY_TYPE) { + inner = (inner as Json).value; + innerAt = `${innerAt}/value`; + } + this.checkSecretValue(inner, innerAt, depth + 1); + } + return; + } + this.report( + pointer, + "secret-field", + 'a Secret field must hold a "secret" block with a well-formed ciphertext, never plain text', + ); + } + + /** A block value: its own definition (from the manifest) describes its arguments. */ + walkBlock(block: Json, pointer: string, depth: number) { + const type = block.__resolveType as string; + if ( + type === SECRET_BLOCK_TYPE && + !isWellFormedCiphertext(block.ciphertext) + ) { + this.report( + pointer, + "secret-ciphertext", + 'a "secret" block must carry a well-formed "ciphertext" (v1...)', + ); + } + const definition = + this.meta && !LEGACY_SECRET_LOADER.test(type) + ? blockIndex(this.meta).get(type) + : undefined; + this.walkObject(block, definition, pointer, depth); + } + + private walkObject( + object: Json, + schema: unknown, + pointer: string, + depth: number, + ) { + for (const [key, child] of Object.entries(object)) { + if (key === "__resolveType") continue; + const childSchema = + schema === undefined ? undefined : this.propertySchema(schema, key); + this.walk( + child, + childSchema, + `${pointer}/${escapePointer(key)}`, + depth + 1, + ); + } + } + + walk(value: unknown, schema: unknown, pointer: string, depth: number) { + if (depth > MAX_DEPTH) + return this.report(pointer, "too-deep", "the value is nested too deeply"); + if (schema !== undefined && this.isSecretField(schema)) { + return this.checkSecretValue(value, pointer, depth); + } + if (Array.isArray(value)) { + const items = schema === undefined ? undefined : this.itemSchema(schema); + for (const [index, item] of value.entries()) { + this.walk(item, items, `${pointer}/${index}`, depth + 1); + } + return; + } + if (!isObject(value)) return; + // A nested block is described by its own definition, not the field's union. + if (resolveTypeOf(value) !== undefined) + return this.walkBlock(value, pointer, depth); + this.walkObject(value, schema, pointer, depth); + } + + walkAny(value: unknown, pointer: string, depth: number) { + this.walk(value, undefined, pointer, depth); + } +} + +/** + * Checks one entry against the secret guard. `meta` is the parsed schema; + * without one, only the `secret` blocks' ciphertexts can be checked. + */ +export function checkSecrets( + name: string, + entry: unknown, + meta: DecoMeta | null, +): BlockViolation[] { + const walker = new SecretWalker(name, meta); + walker.walk(entry, undefined, "", 0); + return walker.violations; +} diff --git a/packages/shared/src/vendor/blocks-protocol/server/assets.ts b/packages/shared/src/vendor/blocks-protocol/server/assets.ts new file mode 100644 index 0000000000..b78566dd9f --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/server/assets.ts @@ -0,0 +1,147 @@ +/** + * `createAssetHandler(storage)`: uploads beside the content protocol. + * + * Uploads aren't one of the four methods. The site editor sends each file to + * `PUT /assets/` with the same token as the protocol; on that path the + * file's own image, video, font or PDF content type is accepted instead of + * JSON, and the name's extension must match it (`banner.html` sent as + * `image/png` is refused; a name without an extension gets the type's). SVG + * is refused unless `allowSvg` is set, since it can carry scripts that would + * run on the site's origin. The storage writes it to its asset folder, never overwriting an + * existing file (a taken name gets a short suffix), and the response carries + * the path the site editor stores in the field: always `/assets/`. + */ + +import { + type AssetTypeOptions, + assetNameForType, + isAcceptedAssetType, + sanitizeAssetName, +} from "../assets"; +import { + type ContentProtocolError, + ErrorCode, + forbidden, + invalidRequest, + limitExceeded, + readOnly, + unauthorized, + unsupported, +} from "../errors"; +import type { ContentStorage } from "../storage"; +import { ASSETS_URL_PREFIX } from "../types"; +import { type AuthOptions, assertAuthOptions, authenticate } from "./auth"; +import { + BodyEncodingError, + BodyTooLargeError, + jsonResponse, + readBody, +} from "./http"; + +const STATUS: Record = { + [ErrorCode.Unauthorized]: 401, + [ErrorCode.Forbidden]: 403, + [ErrorCode.ReadOnly]: 403, + [ErrorCode.Unsupported]: 404, + [ErrorCode.LimitExceeded]: 413, + [ErrorCode.InvalidRequest]: 400, +}; + +export type AssetHandler = (request: Request) => Promise; + +export type AssetHandlerOptions = AuthOptions & AssetTypeOptions; + +export function createAssetHandler( + storage: ContentStorage, + options: AssetHandlerOptions = {}, +): AssetHandler { + assertAuthOptions(options); + const fail = ( + request: Request, + error: ContentProtocolError, + status?: number, + ) => + jsonResponse( + request, + JSON.stringify({ error: error.toJSON() }), + status ?? STATUS[error.code] ?? 400, + ); + + return async (request) => { + if (request.method !== "PUT") { + return jsonResponse( + request, + JSON.stringify({ error: invalidRequest("use PUT").toJSON() }), + 405, + { + allow: "PUT", + }, + ); + } + const auth = await authenticate(request, options); + if (!auth.ok) + return fail( + request, + auth.reason === "unauthorized" ? unauthorized() : forbidden(), + ); + + const description = await storage.describe(); + if (description.readOnly) return fail(request, readOnly()); + if (description.assets === null || !storage.putAsset) { + return fail(request, unsupported("this endpoint doesn't accept uploads")); + } + const contentType = request.headers.get("content-type"); + if (!isAcceptedAssetType(contentType, options)) { + return fail( + request, + invalidRequest("uploads must be an image, video, font or PDF"), + 415, + ); + } + const path = new URL(request.url).pathname; + const marker = path.lastIndexOf(ASSETS_URL_PREFIX); + const sanitized = + marker === -1 + ? null + : sanitizeAssetName(path.slice(marker + ASSETS_URL_PREFIX.length)); + if (!sanitized) + return fail( + request, + invalidRequest("PUT /assets/ needs a file name"), + ); + const name = assetNameForType(sanitized, contentType, options); + if (!name) { + return fail( + request, + invalidRequest( + `the file name's extension doesn't match its content type (${contentType})`, + ), + 415, + ); + } + + const maxBytes = description.assets.maxBytes; + let body: Uint8Array; + try { + body = await readBody(request, maxBytes); + } catch (error) { + if (error instanceof BodyTooLargeError) { + return fail( + request, + limitExceeded(`uploads are limited to ${maxBytes} bytes`), + ); + } + if (error instanceof BodyEncodingError) + return fail(request, invalidRequest(error.message), 415); + throw error; + } + if (body.byteLength === 0) + return fail(request, invalidRequest("the upload is empty")); + const stored = await storage.putAsset(name, body); + return jsonResponse( + request, + JSON.stringify({ path: `${ASSETS_URL_PREFIX}${stored.name}` }), + 201, + ); + }; +} diff --git a/packages/shared/src/vendor/blocks-protocol/server/auth.ts b/packages/shared/src/vendor/blocks-protocol/server/auth.ts new file mode 100644 index 0000000000..fe94c036df --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/server/auth.ts @@ -0,0 +1,65 @@ +/** + * Request authorization shared by the content handler and the asset handler. + */ +import { bearerToken, timingSafeEqualStrings } from "./http"; + +/** + * What an `authorize` hook decides: + * - `true` or `{ scope }` — allowed. `scope` names the tenant and principal, + * and scopes request-key receipts, so one tenant never replays another's. + * - `false` or `"unauthorized"` — a missing or invalid bearer token (HTTP 401). + * - `"forbidden"` — authenticated, but not allowed for this project. + */ +export type AuthorizeResult = + | boolean + | "unauthorized" + | "forbidden" + | { scope?: string }; + +export interface AuthOptions { + /** When set, every request must carry `Authorization: Bearer `. */ + token?: string; + /** Decides whether a request may proceed (runs after the token check). */ + authorize?: (request: Request) => AuthorizeResult | Promise; +} + +/** + * Refuses options that would lock every client out by accident: an empty + * token can never be presented, so every request would be 401. + */ +export function assertAuthOptions(options: AuthOptions): void { + if ( + options.token !== undefined && + (typeof options.token !== "string" || options.token === "") + ) { + throw new TypeError( + "token must be a non-empty string; omit it to accept requests without one", + ); + } +} + +type AuthOutcome = + | { ok: true; scope: string } + | { ok: false; reason: "unauthorized" | "forbidden" }; + +export async function authenticate( + request: Request, + options: AuthOptions, +): Promise { + if (options.token !== undefined) { + const presented = bearerToken(request); + if ( + presented === null || + !(await timingSafeEqualStrings(presented, options.token)) + ) { + return { ok: false, reason: "unauthorized" }; + } + } + if (!options.authorize) return { ok: true, scope: "" }; + const result = await options.authorize(request); + if (result === true) return { ok: true, scope: "" }; + if (result === false || result === "unauthorized") + return { ok: false, reason: "unauthorized" }; + if (result === "forbidden") return { ok: false, reason: "forbidden" }; + return { ok: true, scope: result.scope ?? "" }; +} diff --git a/packages/shared/src/vendor/blocks-protocol/server/body-cache.ts b/packages/shared/src/vendor/blocks-protocol/server/body-cache.ts new file mode 100644 index 0000000000..5cd8047131 --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/server/body-cache.ts @@ -0,0 +1,89 @@ +/** + * A bounded cache of parsed entry bodies, keyed by file and version. + * + * Versions identify content, so a polling client that sees a new revision + * only costs a read and a parse for the files that actually changed. + */ + +export type ParsedBody = + | { ok: true; value: Record; bytes: number } + | { + ok: false; + kind: "invalid-json" | "not-an-object" | "too-large"; + message: string; + bytes: number; + }; + +export class BodyCache { + private readonly entries = new Map(); + private total = 0; + + constructor(private readonly maxBytes: number) {} + + private static key(file: string, version: string) { + return `${file}\0${version}`; + } + + get(file: string, version: string): ParsedBody | undefined { + const key = BodyCache.key(file, version); + const hit = this.entries.get(key); + if (hit) { + // Refresh recency. + this.entries.delete(key); + this.entries.set(key, hit); + } + return hit; + } + + set(file: string, version: string, body: ParsedBody): void { + if (body.bytes > this.maxBytes) return; + const key = BodyCache.key(file, version); + const previous = this.entries.get(key); + if (previous) { + this.total -= previous.bytes; + this.entries.delete(key); + } + this.entries.set(key, body); + this.total += body.bytes; + for (const [oldest, value] of this.entries) { + if (this.total <= this.maxBytes) break; + this.entries.delete(oldest); + this.total -= value.bytes; + } + } +} + +const utf8 = new TextEncoder(); + +/** Parses one stored entry, enforcing the per-entry byte limit. */ +export function parseBody(text: string, maxBlockBytes: number): ParsedBody { + const bytes = utf8.encode(text).byteLength; + if (bytes > maxBlockBytes) { + return { + ok: false, + kind: "too-large", + bytes, + message: `the file is over ${maxBlockBytes} bytes`, + }; + } + let value: unknown; + try { + value = JSON.parse(text); + } catch (error) { + return { + ok: false, + kind: "invalid-json", + bytes, + message: (error as Error).message, + }; + } + if (typeof value !== "object" || value === null || Array.isArray(value)) { + return { + ok: false, + kind: "not-an-object", + bytes, + message: "the file doesn't hold a JSON object", + }; + } + return { ok: true, value: value as Record, bytes }; +} diff --git a/packages/shared/src/vendor/blocks-protocol/server/content.ts b/packages/shared/src/vendor/blocks-protocol/server/content.ts new file mode 100644 index 0000000000..1cc948b3c5 --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/server/content.ts @@ -0,0 +1,199 @@ +/** + * Turns a storage snapshot (files and versions) into the entry map, applying + * the one file-name rule: decode once, resolve spellings, report the rest. + * Shared by `blocks.list` (which needs every body) and `blocks.apply` (which + * needs bodies only to break ties between spellings). + */ +import { limitExceeded, unavailable } from "../errors"; +import { + entryHasPath, + fullyDecodeFileName, + isBlockFileName, + resolveSpellings, +} from "../keys"; +import type { ContentStorage, StorageFile, StorageSnapshot } from "../storage"; +import type { Diagnostic, Limits } from "../types"; +import { type BodyCache, type ParsedBody, parseBody } from "./body-cache"; + +export interface LoadedEntry { + file: string; + version: string; + /** The parsed entry; absent when the body wasn't needed. */ + value?: Record; +} + +export interface LoadedContent { + snapshot: StorageSnapshot; + /** Every entry, by name, in file-name order of the winning files. */ + entries: Map; + /** Every saved-block file, valid or not, by spelling key. */ + groups: Map; + /** Files skipped or shadowed. */ + diagnostics: Diagnostic[]; + /** Uncompressed bytes of the bodies read. */ + bytes: number; + /** + * True when a body read didn't match the snapshot (the file changed or + * vanished in between): the content isn't one consistent snapshot, so the + * caller takes a new one. + */ + moved: boolean; +} + +interface Options { + /** Read every body (`blocks.list`), not only the ones that break ties. */ + readAll: boolean; + limits: Limits; + cache: BodyCache; +} + +async function loadContent( + storage: ContentStorage, + snapshot: StorageSnapshot, + { readAll, limits, cache }: Options, +): Promise { + const files = snapshot.files.filter((f) => isBlockFileName(f.file)); + const groups = new Map(); + for (const file of files) { + const key = fullyDecodeFileName(file.file).name; + const group = groups.get(key); + if (group) group.push(file); + else groups.set(key, [file]); + } + + const diagnostics: Diagnostic[] = []; + let moved = false; + const parsed = new Map(); + const toRead: StorageFile[] = []; + let knownBytes = 0; + for (const group of groups.values()) { + if (!readAll && group.length < 2) continue; + for (const file of group) { + // An existing file over maxBlockBytes is unreadable, like invalid JSON: it's + // left out of the map and reported as a `too-large` diagnostic, so one + // oversized hand edit can't take the whole editor down. The map plus its + // diagnostics still accounts for every file in the snapshot; only the + // aggregate (maxListBytes) answers LimitExceeded. + if (file.size !== undefined && file.size > limits.maxBlockBytes) { + parsed.set(file.file, { + ok: false, + kind: "too-large", + bytes: file.size, + message: `the file is over ${limits.maxBlockBytes} bytes`, + }); + continue; + } + knownBytes += file.size ?? 0; + const hit = cache.get(file.file, file.version); + if (hit) parsed.set(file.file, hit); + else toRead.push(file); + } + } + if (readAll && knownBytes > limits.maxListBytes) { + throw limitExceeded(`the block list is over ${limits.maxListBytes} bytes`, { + limit: "maxListBytes", + }); + } + if (toRead.length > 0) { + const bodies = await storage.readFiles( + snapshot, + toRead.map((f) => f.file), + ); + for (const file of toRead) { + const read = bodies[file.file]; + if (read === undefined) { + moved = true; // vanished since the snapshot + continue; + } + const body = parseBody(read.text, limits.maxBlockBytes); + // Cache under the version of the bytes actually read, never the snapshot's. + cache.set(file.file, read.version, body); + if (read.version !== file.version) { + moved = true; + continue; + } + parsed.set(file.file, body); + } + } + + let bytes = 0; + const candidates: Array< + StorageFile & { hasPath: boolean; value?: Record } + > = []; + for (const file of files) { + const body = parsed.get(file.file); + if (body === undefined) { + // Not read: a single spelling whose body wasn't needed, or a file that vanished. + if (!readAll) candidates.push({ ...file, hasPath: false }); + continue; + } + if (!body.ok) { + diagnostics.push({ + file: file.file, + kind: body.kind, + message: body.message, + }); + continue; + } + bytes += body.bytes; + candidates.push({ + ...file, + hasPath: entryHasPath(body.value), + value: body.value, + }); + } + if (readAll && bytes > limits.maxListBytes) { + throw limitExceeded(`the block list is over ${limits.maxListBytes} bytes`, { + limit: "maxListBytes", + }); + } + + const entries = new Map(); + for (const { name, winner, shadowed } of resolveSpellings( + candidates, + ).values()) { + entries.set(name, { + file: winner.file, + version: winner.version, + value: winner.value, + }); + for (const loser of shadowed) { + diagnostics.push({ + file: loser.file, + kind: "shadowed", + name, + winner: winner.file, + message: `another spelling of "${name}" wins: ${winner.file}`, + }); + } + } + diagnostics.sort((a, b) => (a.file < b.file ? -1 : a.file > b.file ? 1 : 0)); + return { snapshot, entries, groups, diagnostics, bytes, moved }; +} + +/** How many snapshots a read takes before giving up on files that keep changing. */ +const MAX_SNAPSHOT_READS = 3; + +/** + * Takes a snapshot and loads its content, taking a new snapshot when a file + * changed while its body was being read, so the bodies always match the + * versions and revision they're reported with. `shortCircuit` can end the + * read right after the snapshot (a conditional read that's "not modified"). + */ +export async function loadCurrentContent( + storage: ContentStorage, + ref: string | undefined, + options: Options, + shortCircuit?: (snapshot: StorageSnapshot) => boolean, +): Promise<{ snapshot: StorageSnapshot; content: LoadedContent | null }> { + for (let read = 1; read <= MAX_SNAPSHOT_READS; read++) { + const snapshot = await storage.snapshot({ ref }); + if (shortCircuit?.(snapshot)) return { snapshot, content: null }; + const content = await loadContent(storage, snapshot, options); + if (!content.moved) return { snapshot, content }; + } + throw unavailable( + "saved blocks kept changing while being read; retry shortly", + 250, + ); +} diff --git a/packages/shared/src/vendor/blocks-protocol/server/core.ts b/packages/shared/src/vendor/blocks-protocol/server/core.ts new file mode 100644 index 0000000000..867f138eaf --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/server/core.ts @@ -0,0 +1,126 @@ +/** + * The shared state of one content handler: its storage, options and caches. + */ +import { + ContentProtocolError, + invalidBlock, + notFound, + unavailable, + unsupported, +} from "../errors"; +import { blockNameFromFile } from "../keys"; +import { + type ContentStorage, + type StorageDescription, + StorageInvalidFileError, + StorageNotFoundError, + StorageUnavailableError, +} from "../storage"; +import { DEFAULT_LIMITS, type DecoMeta, type Limits } from "../types"; +import type { AuthOptions } from "./auth"; +import { BodyCache } from "./body-cache"; + +export interface ContentHandlerOptions extends AuthOptions { + /** Reported by `describe`, such as `{ name: "deco-cli", version: "8.0.0" }`. */ + server?: { name: string; version: string }; + /** Lowers the protocol's default limits. Higher values are ignored. */ + limits?: Partial; + /** Overrides the poll interval the storage suggests. */ + pollIntervalMs?: number; + /** Where "open the real page" points. */ + preview?: { origin: string } | null; + /** How many times `blocks.apply` attempts a commit when storage moved meanwhile (default 3). */ + maxCommitAttempts?: number; + /** + * The jittered wait before retrying a commit that found storage moved: a + * random delay in `[minMs, maxMs]`, times the attempt number (default + * 50–200 ms), so concurrent writers don't retry in lockstep. + */ + commitRetryDelayMs?: { minMs: number; maxMs: number }; + /** Bytes of parsed entries kept between requests (default 32 MiB). */ + bodyCacheBytes?: number; + /** Called with errors that become an Internal error (-32603), for logging. */ + onError?: (error: unknown) => void; +} + +const DEFAULT_POLL_MS = { "working-tree": 2000, git: 30000 } as const; +const DEFAULT_RETRY_DELAY = { minMs: 50, maxMs: 200 }; + +export class Core { + readonly cache: BodyCache; + /** Requests in flight per scoped request key, so simultaneous duplicates commit once. */ + readonly inflight = new Map>(); + /** The last parsed schema, by version: the secret guard reads it on every write. */ + parsedSchema: { version: string; meta: DecoMeta } | null = null; + + constructor( + readonly storage: ContentStorage, + readonly options: ContentHandlerOptions, + ) { + this.cache = new BodyCache(options.bodyCacheBytes ?? 32 * 1024 * 1024); + } + + get maxCommitAttempts(): number { + return Math.max(1, this.options.maxCommitAttempts ?? 3); + } + + /** Waits before commit attempt `attempt + 1`: jittered, growing with the attempt. */ + async backoff(attempt: number): Promise { + const { minMs, maxMs } = + this.options.commitRetryDelayMs ?? DEFAULT_RETRY_DELAY; + const ms = (minMs + Math.random() * Math.max(0, maxMs - minMs)) * attempt; + if (ms > 0) await new Promise((resolve) => setTimeout(resolve, ms)); + } + + async description(): Promise { + return this.storage.describe(); + } + + /** The effective limits: the lowest of the defaults, the storage's and the options'. */ + limits(description: StorageDescription): Limits { + const out = { ...DEFAULT_LIMITS }; + for (const source of [description.limits, this.options.limits]) { + if (!source) continue; + for (const key of Object.keys(out) as (keyof Limits)[]) { + const value = source[key]; + if (typeof value === "number" && value > 0 && value < out[key]) + out[key] = value; + } + } + return out; + } + + pollIntervalMs(description: StorageDescription): number { + return ( + this.options.pollIntervalMs ?? + description.pollIntervalMs ?? + DEFAULT_POLL_MS[description.kind] + ); + } + + /** Refuses a `ref` on a storage without branches. */ + checkRef(description: StorageDescription, ref: string | undefined): void { + if (ref !== undefined && description.refs === null) { + throw unsupported("this endpoint has no branches; omit ref"); + } + } +} + +/** Maps storage errors to protocol errors; anything unknown is rethrown. */ +export function toProtocolError(error: unknown): ContentProtocolError | null { + if (error instanceof ContentProtocolError) return error; + if (error instanceof StorageNotFoundError) return notFound(error.message); + if (error instanceof StorageInvalidFileError) { + const name = blockNameFromFile(error.file); + return invalidBlock([ + { + name, + rule: "unsupported-name", + message: `this storage can't hold the entry "${name}"`, + }, + ]); + } + if (error instanceof StorageUnavailableError) + return unavailable(error.message, error.retryAfterMs); + return null; +} diff --git a/packages/shared/src/vendor/blocks-protocol/server/handler.ts b/packages/shared/src/vendor/blocks-protocol/server/handler.ts new file mode 100644 index 0000000000..4ca7c91b4d --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/server/handler.ts @@ -0,0 +1,118 @@ +/** + * `createContentHandler(storage)`: the content protocol as a fetch handler, + * `(Request) => Promise`. + * + * `POST ` with `Content-Type: application/json` and a body that's + * one request object or a batch array. Errors come back in the JSON-RPC + * `error` object with HTTP 200, except a missing or invalid bearer token (401) + * and a body over the size limit (413), which apply to the whole batch, and + * requests that aren't the protocol at all: a method other than POST (405), + * or a body that isn't JSON or uses an unsupported encoding (415). Those carry + * a JSON-RPC error body too, with a `null` id. + * Responses are gzip-compressed when the request accepts it. + * + * The handler is path-agnostic: mount it where the endpoint lives (`/rpc` on + * the local server). Transport security that depends on where it runs — CORS, + * `Host` checks, Chrome's local-network preflight — belongs to the server + * that mounts it (`deco serve`). + */ +import { + type ContentProtocolError, + forbidden, + invalidRequest, + limitExceeded, + parseError, + unauthorized, +} from "../errors"; +import type { ContentStorage } from "../storage"; +import { assertAuthOptions, authenticate } from "./auth"; +import { type ContentHandlerOptions, Core } from "./core"; +import { + BodyEncodingError, + BodyTooLargeError, + isJsonContentType, + jsonResponse, + readBody, +} from "./http"; +import { dispatch } from "./rpc"; + +export type ContentHandler = (request: Request) => Promise; + +const errorBody = (error: ContentProtocolError) => + JSON.stringify({ jsonrpc: "2.0", id: null, error: error.toJSON() }); + +export function createContentHandler( + storage: ContentStorage, + options: ContentHandlerOptions = {}, +): ContentHandler { + assertAuthOptions(options); + const core = new Core(storage, options); + + return async (request) => { + if (request.method !== "POST") { + return jsonResponse(request, errorBody(invalidRequest("use POST")), 405, { + allow: "POST", + }); + } + const auth = await authenticate(request, options); + if (!auth.ok) { + return auth.reason === "unauthorized" + ? jsonResponse(request, errorBody(unauthorized()), 401, { + "www-authenticate": 'Bearer realm="deco-content"', + }) + : jsonResponse(request, errorBody(forbidden())); + } + if (!isJsonContentType(request)) { + return jsonResponse( + request, + errorBody(invalidRequest("Content-Type must be application/json")), + 415, + ); + } + + const limits = core.limits(await core.description()); + let bytes: Uint8Array; + try { + bytes = await readBody(request, limits.maxRequestBytes); + } catch (error) { + if (error instanceof BodyTooLargeError) { + return jsonResponse( + request, + errorBody( + limitExceeded( + `the request body is over ${limits.maxRequestBytes} bytes`, + { + limit: "maxRequestBytes", + }, + ), + ), + 413, + ); + } + if (error instanceof BodyEncodingError) { + return jsonResponse( + request, + errorBody(invalidRequest(error.message)), + 415, + ); + } + throw error; + } + + let body: unknown; + try { + body = JSON.parse( + new TextDecoder("utf-8", { fatal: true }).decode(bytes), + ); + } catch { + return jsonResponse(request, errorBody(parseError())); + } + const response = await dispatch( + core, + body, + auth.scope, + limits.maxBatchResponseBytes, + ); + return jsonResponse(request, response); + }; +} diff --git a/packages/shared/src/vendor/blocks-protocol/server/http.ts b/packages/shared/src/vendor/blocks-protocol/server/http.ts new file mode 100644 index 0000000000..81773f1709 --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/server/http.ts @@ -0,0 +1,164 @@ +/** + * HTTP plumbing for the content handler: bounded body reads (gzip request + * bodies included), gzip responses and bearer-token checks. Web-standard APIs + * only (Request, Response, CompressionStream), so it runs on Node, Bun, + * Workers and Deno. + */ +import { sha256Hex } from "../canonical"; + +/** The body was larger than allowed (after decompression). */ +export class BodyTooLargeError extends Error { + constructor(readonly limit: number) { + super(`the request body is over ${limit} bytes`); + this.name = "BodyTooLargeError"; + } +} + +/** The body claimed an encoding it doesn't have, or one we don't decode. */ +export class BodyEncodingError extends Error { + constructor(message: string) { + super(message); + this.name = "BodyEncodingError"; + } +} + +async function readLimited( + stream: ReadableStream, + limit: number, +): Promise { + const reader = stream.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + total += value.byteLength; + if (total > limit) { + await reader.cancel().catch(() => {}); + throw new BodyTooLargeError(limit); + } + chunks.push(value); + } + } finally { + reader.releaseLock(); + } + const out = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + out.set(chunk, offset); + offset += chunk.byteLength; + } + return out; +} + +/** + * Reads the request body as bytes, refusing more than `limit` bytes. A + * `Content-Encoding: gzip` body is decompressed, and the limit applies to the + * decompressed bytes too, so gzip can't bypass it. + */ +export async function readBody( + request: Request, + limit: number, +): Promise { + const declared = Number(request.headers.get("content-length")); + const encoding = (request.headers.get("content-encoding") ?? "identity") + .trim() + .toLowerCase(); + if ( + encoding === "identity" && + Number.isFinite(declared) && + declared > limit + ) { + throw new BodyTooLargeError(limit); + } + if (!request.body) return new Uint8Array(0); + if (encoding === "identity") return readLimited(request.body, limit); + if (encoding !== "gzip") + throw new BodyEncodingError(`unsupported Content-Encoding "${encoding}"`); + const decompressed = request.body.pipeThrough( + new DecompressionStream("gzip") as unknown as ReadableWritablePair< + Uint8Array, + Uint8Array + >, + ); + try { + return await readLimited(decompressed, limit); + } catch (error) { + if (error instanceof BodyTooLargeError) throw error; + throw new BodyEncodingError("the gzip body is corrupt"); + } +} + +/** True when the request's `Accept-Encoding` accepts gzip. */ +function acceptsGzip(request: Request): boolean { + const header = request.headers.get("accept-encoding"); + if (!header) return false; + return header.split(",").some((part) => { + const [coding = "", ...params] = part.trim().toLowerCase().split(";"); + if (coding.trim() !== "gzip" && coding.trim() !== "*") return false; + const q = params.map((p) => p.trim()).find((p) => p.startsWith("q=")); + return q === undefined || Number(q.slice(2)) > 0; + }); +} + +/** Bodies smaller than this aren't worth compressing. */ +const GZIP_THRESHOLD_BYTES = 1024; + +/** A JSON response, gzip-compressed when the request accepts it. */ +export function jsonResponse( + request: Request, + body: string, + status = 200, + headers: Record = {}, +): Response { + const responseHeaders = new Headers({ + "content-type": "application/json; charset=utf-8", + "cache-control": "no-store", + vary: "Accept-Encoding", + ...headers, + }); + const bytes = new TextEncoder().encode(body); + if (bytes.byteLength < GZIP_THRESHOLD_BYTES || !acceptsGzip(request)) { + return new Response(bytes, { status, headers: responseHeaders }); + } + responseHeaders.set("content-encoding", "gzip"); + const stream = new Blob([bytes as BlobPart]) + .stream() + .pipeThrough( + new CompressionStream("gzip") as unknown as ReadableWritablePair< + Uint8Array, + Uint8Array + >, + ); + return new Response(stream, { status, headers: responseHeaders }); +} + +/** True when the request's `Content-Type` is JSON (`application/json`, any parameters). */ +export function isJsonContentType(request: Request): boolean { + const type = request.headers.get("content-type"); + return ( + type !== null && + type.split(";")[0]!.trim().toLowerCase() === "application/json" + ); +} + +/** The bearer token of the request, or `null` without one. */ +export function bearerToken(request: Request): string | null { + const header = request.headers.get("authorization"); + if (!header) return null; + const match = /^Bearer\s+(\S+)\s*$/i.exec(header); + return match?.[1] ?? null; +} + +/** Compares two strings in time independent of where they differ. */ +export async function timingSafeEqualStrings( + a: string, + b: string, +): Promise { + const [ha, hb] = await Promise.all([sha256Hex(a), sha256Hex(b)]); + let diff = 0; + for (let i = 0; i < ha.length; i++) + diff |= ha.charCodeAt(i) ^ hb.charCodeAt(i); + return diff === 0; +} diff --git a/packages/shared/src/vendor/blocks-protocol/server/index.ts b/packages/shared/src/vendor/blocks-protocol/server/index.ts new file mode 100644 index 0000000000..e85b9454f5 --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/server/index.ts @@ -0,0 +1,26 @@ +/** + * `@decocms/blocks/protocol/server`: the content protocol's server core. + * + * `createContentHandler(storage)` serves the four methods over any + * `ContentStorage`; `createAssetHandler(storage)` serves `PUT /assets/` + * uploads. Web-standard APIs only, so both run on Node, Bun, Workers and Deno. + */ + +export { + ASSET_TYPES, + type AssetTypeOptions, + assetExtensions, + assetNameForType, + isAcceptedAssetType, + SVG_ASSET_TYPE, + sanitizeAssetName, + suffixedAssetName, +} from "../assets"; +export { + type AssetHandler, + type AssetHandlerOptions, + createAssetHandler, +} from "./assets"; +export type { AuthOptions, AuthorizeResult } from "./auth"; +export type { ContentHandlerOptions } from "./core"; +export { type ContentHandler, createContentHandler } from "./handler"; diff --git a/packages/shared/src/vendor/blocks-protocol/server/methods/apply.ts b/packages/shared/src/vendor/blocks-protocol/server/methods/apply.ts new file mode 100644 index 0000000000..4c5aa2a940 --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/server/methods/apply.ts @@ -0,0 +1,431 @@ +/** + * `blocks.apply`, the protocol's only write: + * + * 1. All or nothing: every name in `set` and `delete` lands in one storage + * commit, or none does. + * 2. Durable when it returns. + * 3. `set` wins when a name is in both `set` and `delete`. + * 4. Validated first: names, value shapes, sizes and the secret guard are + * checked before anything is written, and every violation is reported. + * 5. Preconditions are optional: a failed `ifMatch` (or `ifSchemaMatch`) + * writes nothing and returns a Conflict; without them the last writer wins. + * + * 6. Two spellings of one name are one entry: a guard compares against the + * entry under any spelling, and the result reports `null` for every other + * spelling the commit deleted. + * + * A commit attempt that finds storage moved (files, the schema, or a request + * key reserved meanwhile) is retried against a new snapshot after a jittered + * backoff, rechecking every guard, up to `maxCommitAttempts` times. + */ +import { applyRequestDigest, canonicalJson } from "../../canonical"; +import { + type BlockViolation, + conflict, + invalidBlock, + invalidParams, + limitExceeded, + readOnly, + unavailable, + unsupported, + type VersionMismatch, +} from "../../errors"; +import { + blockFileName, + blockNameFromFile, + checkBlockName, + checkDeletedName, + fullyDecodeFileName, + isBlockFileName, + serializeBlock, + spellingKey, +} from "../../keys"; +import { checkSecrets } from "../../secrets"; +import type { StorageDescription, StoredReceipt } from "../../storage"; +import type { + BlocksApplyParams, + BlocksApplyResult, + DecoMeta, + Limits, +} from "../../types"; +import { + type LoadedContent, + type LoadedEntry, + loadCurrentContent, +} from "../content"; +import type { Core } from "../core"; +import { parseSchema, readSchema } from "./read"; + +const utf8 = new TextEncoder(); + +interface NormalizedApply { + set: Array<[name: string, value: unknown]>; + /** Names to delete that aren't also set. */ + delete: string[]; + ifMatch: Array<[name: string, version: string | null]>; +} + +function normalize(params: BlocksApplyParams, limits: Limits): NormalizedApply { + const set = Object.entries(params.set ?? {}); + const setNames = new Set(set.map(([name]) => name)); + const deleteNames = [...new Set(params.delete ?? [])].filter( + (name) => !setNames.has(name), + ); + const ops = set.length + deleteNames.length; + if (ops > limits.maxOpsPerApply) { + throw limitExceeded( + `${ops} names in one blocks.apply; the limit is ${limits.maxOpsPerApply}`, + { + limit: "maxOpsPerApply", + }, + ); + } + return { + set, + delete: deleteNames, + ifMatch: Object.entries(params.ifMatch ?? {}), + }; +} + +/** Checks everything that doesn't depend on stored content. */ +function validateStatic( + apply: NormalizedApply, + limits: Limits, + meta: DecoMeta | null, +) { + const violations: BlockViolation[] = []; + const bodies = new Map(); + const bySpelling = new Map(); + for (const [name, value] of apply.set) { + for (const v of checkBlockName(name)) + violations.push({ name, rule: v.reason, message: v.message }); + const key = spellingKey(name); + const twin = bySpelling.get(key); + if (twin !== undefined) { + violations.push({ + name, + rule: "spelling-collision", + message: `"${name}" and "${twin}" are spellings of the same entry`, + }); + } else bySpelling.set(key, name); + if (typeof value !== "object" || value === null || Array.isArray(value)) { + violations.push({ + name, + rule: "not-an-object", + message: "an entry must be a JSON object", + }); + continue; + } + const body = serializeBlock(value); + if (utf8.encode(body).byteLength > limits.maxBlockBytes) { + violations.push({ + name, + rule: "too-large", + message: `the entry is over ${limits.maxBlockBytes} bytes`, + }); + continue; + } + bodies.set(name, body); + violations.push(...checkSecrets(name, value, meta)); + } + for (const name of apply.delete) { + for (const v of checkDeletedName(name)) + violations.push({ name, rule: v.reason, message: v.message }); + } + return { violations, bodies }; +} + +/** The existing entries, by spelling key: two spellings of one name are one entry. */ +type EntriesBySpelling = Map; + +function entriesBySpelling(content: LoadedContent): EntriesBySpelling { + const out: EntriesBySpelling = new Map(); + for (const [name, entry] of content.entries) { + out.set(fullyDecodeFileName(entry.file).name, { name, entry }); + } + return out; +} + +/** Checks the rules that depend on the entries that already exist. */ +function validateAgainst( + content: LoadedContent, + bySpelling: EntriesBySpelling, + apply: NormalizedApply, +): BlockViolation[] { + const violations: BlockViolation[] = []; + const existing = [...content.entries.keys()]; + for (const [name] of apply.set) { + // Another spelling of an existing entry updates that entry; it isn't a new name. + if (bySpelling.has(spellingKey(name))) continue; + const others = apply.set + .filter(([other]) => other !== name) + .map(([other]) => other); + for (const v of checkBlockName(name, { + existingNames: [...existing, ...others], + })) { + if (v.reason === "case-collision") + violations.push({ name, rule: v.reason, message: v.message }); + } + } + return violations; +} + +interface Plan { + put: Record; + delete: string[]; + expected: Record; +} + +/** + * Turns names into file operations: write `encode(name)`, delete every other + * spelling. Deleting any spelling of an entry deletes every spelling. Only + * guarded entries become commit expectations: without `ifMatch`, the last + * writer wins, so a concurrent save of the same entry must not turn into a + * retry storm. + */ +function plan( + content: LoadedContent, + bySpelling: EntriesBySpelling, + apply: NormalizedApply, + bodies: Map, +): Plan { + const versionOf = new Map( + content.snapshot.files + .filter((f) => isBlockFileName(f.file)) + .map((f) => [f.file, f.version]), + ); + const groupFiles = (name: string) => + (content.groups.get(spellingKey(name)) ?? []).map((f) => f.file); + const put: Record = Object.create(null); + const deletes = new Set(); + const expected: Record = Object.create(null); + + for (const [name] of apply.set) { + const file = blockFileName(name); + put[file] = bodies.get(name)!; + for (const other of groupFiles(name)) + if (other !== file) deletes.add(other); + } + for (const name of apply.delete) { + const file = blockFileName(name); + if (versionOf.has(file)) deletes.add(file); + if (bySpelling.has(spellingKey(name))) + for (const other of groupFiles(name)) deletes.add(other); + } + for (const file of Object.keys(put)) deletes.delete(file); + // A guarded entry must still be exactly what the guard saw when the commit lands: + // every spelling of it, so a concurrent save under another spelling is caught too. + for (const [name] of apply.ifMatch) { + for (const file of [blockFileName(name), ...groupFiles(name)]) { + if (isBlockFileName(file)) expected[file] = versionOf.get(file) ?? null; + } + } + return { put, delete: [...deletes], expected }; +} + +/** A guard compares against the entry under any spelling: `null` means no spelling exists. */ +function checkIfMatch(bySpelling: EntriesBySpelling, apply: NormalizedApply) { + const mismatches: Record = {}; + let failed = false; + for (const [name, expected] of apply.ifMatch) { + const actual = bySpelling.get(spellingKey(name))?.entry.version ?? null; + if (actual !== expected) { + mismatches[name] = { expected, actual }; + failed = true; + } + } + return failed ? mismatches : null; +} + +/** + * The result's versions: every name written or deleted, plus `null` for each + * other spelling the commit deleted, so a client's map never keeps an entry + * under a name that's gone. + */ +function resultFromVersions( + apply: NormalizedApply, + revision: string, + fileVersions: Record, + deletedFiles: readonly string[], +): BlocksApplyResult { + const versions: Record = Object.create(null); + const touched = new Set(); + for (const [name] of apply.set) { + versions[name] = fileVersions[blockFileName(name)] ?? null; + touched.add(spellingKey(name)); + } + for (const name of apply.delete) { + versions[name] = null; + touched.add(spellingKey(name)); + } + for (const file of deletedFiles) { + const name = blockNameFromFile(file); + if (!(name in versions) && touched.has(fullyDecodeFileName(file).name)) + versions[name] = null; + } + return { revision, versions }; +} + +function receiptResult( + apply: NormalizedApply, + receipt: StoredReceipt, +): BlocksApplyResult { + return resultFromVersions( + apply, + receipt.revision, + receipt.versions, + receipt.deleted ?? [], + ); +} + +async function loadSchema( + core: Core, + params: BlocksApplyParams, + limits: Limits, +) { + const stored = await readSchema(core, params.ref, limits.maxSchemaBytes); + if (stored === null) return { version: null, meta: null }; + if (core.parsedSchema?.version !== stored.version) { + core.parsedSchema = { + version: stored.version, + meta: parseSchema(stored.text), + }; + } + return { version: stored.version, meta: core.parsedSchema.meta }; +} + +function checkFeatures( + description: StorageDescription, + core: Core, + params: BlocksApplyParams, +) { + if (description.readOnly) throw readOnly(); + core.checkRef(description, params.ref); + if ( + params.requestKey !== undefined && + (!description.idempotency || !core.storage.getReceipt) + ) { + throw unsupported( + "this endpoint doesn't support request keys; omit requestKey", + ); + } +} + +export async function blocksApply( + core: Core, + params: BlocksApplyParams, + scope: string, +): Promise { + const description = await core.description(); + checkFeatures(description, core, params); + if (params.requestKey === undefined) + return applyOnce(core, params, description, null); + + const digest = await applyRequestDigest(params); + const key = canonicalJson([ + scope, + description.root, + params.ref ?? null, + params.requestKey, + ]); + // Simultaneous duplicates: wait for the first, then find its receipt. + for ( + let pending = core.inflight.get(key); + pending; + pending = core.inflight.get(key) + ) { + await pending.catch(() => {}); + } + const run = applyOnce(core, params, description, { key, digest }); + core.inflight.set(key, run); + try { + return await run; + } finally { + if (core.inflight.get(key) === run) core.inflight.delete(key); + } +} + +async function applyOnce( + core: Core, + params: BlocksApplyParams, + description: StorageDescription, + receipt: { key: string; digest: string } | null, +): Promise { + const limits = core.limits(description); + const apply = normalize(params, limits); + + for (let attempt = 1; attempt <= core.maxCommitAttempts; attempt++) { + // Receipts for completed identical requests resolve before any guard is rechecked. + if (receipt) { + const stored = await core.storage.getReceipt!(receipt.key); + if (stored) { + if (stored.digest !== receipt.digest) { + throw invalidParams( + "requestKey was already used for a different request", + ); + } + return receiptResult(apply, stored); + } + } + + const schema = await loadSchema(core, params, limits); + if ( + params.ifSchemaMatch !== undefined && + params.ifSchemaMatch !== schema.version + ) { + throw conflict({ + schema: { expected: params.ifSchemaMatch, actual: schema.version }, + }); + } + const { violations, bodies } = validateStatic(apply, limits, schema.meta); + const { snapshot, content } = await loadCurrentContent( + core.storage, + params.ref, + { + readAll: false, + limits, + cache: core.cache, + }, + ); + const bySpelling = entriesBySpelling(content!); + violations.push(...validateAgainst(content!, bySpelling, apply)); + if (violations.length > 0) throw invalidBlock(violations); + + const mismatches = checkIfMatch(bySpelling, apply); + if (mismatches) throw conflict({ entries: mismatches }); + + const { + put, + delete: deletes, + expected, + } = plan(content!, bySpelling, apply, bodies); + if (Object.keys(put).length === 0 && deletes.length === 0 && !receipt) { + return resultFromVersions(apply, snapshot.revision, {}, []); + } + // The schema the guards and the secret check used must still be current at commit time. + const dependsOnSchema = + params.ifSchemaMatch !== undefined || apply.set.length > 0; + const result = await core.storage.commit({ + ref: params.ref, + base: snapshot, + put, + delete: deletes, + expected, + ...(dependsOnSchema ? { expectedSchemaVersion: schema.version } : {}), + receipt: receipt ?? undefined, + }); + if (result.status === "committed") { + return resultFromVersions( + apply, + result.revision, + result.versions, + deletes, + ); + } + if (attempt < core.maxCommitAttempts) await core.backoff(attempt); + } + throw unavailable( + `storage kept changing; gave up after ${core.maxCommitAttempts} commit attempts`, + 250, + ); +} diff --git a/packages/shared/src/vendor/blocks-protocol/server/methods/read.ts b/packages/shared/src/vendor/blocks-protocol/server/methods/read.ts new file mode 100644 index 0000000000..19090e287f --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/server/methods/read.ts @@ -0,0 +1,183 @@ +/** + * The three read methods: `describe`, `schema.get` and `blocks.list`. + */ +import { publicKeyDerFromPem } from "../../ciphertext"; +import { limitExceeded, notFound, unavailable } from "../../errors"; +import { + ASSETS_URL_PREFIX, + type BlocksListResult, + type DecoMeta, + type DescribeResult, + PROTOCOL_NAME, + PROTOCOL_VERSION, + type ReadParams, + SCHEMA_FORMAT, + type SchemaGetResult, +} from "../../types"; +import { loadCurrentContent } from "../content"; +import type { Core } from "../core"; + +const DEFAULT_SERVER = { name: "deco-blocks", version: "unknown" }; + +/** The largest `.deco/secrets.pub` served (a 16384-bit key's PEM is under 3 KiB). */ +const MAX_PUBLIC_KEY_BYTES = 16 * 1024; + +/** + * The public key `describe` may serve: a single `PUBLIC KEY` PEM block of at + * most 16 KiB. Anything else (a private key committed by mistake, two + * blocks, stray text) is never broadcast to the site editor. + */ +function servablePublicKey(text: string | null): string | null { + if (text === null) return null; + if (new TextEncoder().encode(text).byteLength > MAX_PUBLIC_KEY_BYTES) + return null; + if (text.includes("PRIVATE KEY")) return null; + return publicKeyDerFromPem(text) === null ? null : text; +} + +export async function describe(core: Core): Promise { + const description = await core.description(); + const stored = await core.storage.readSecretsPublicKey(); + const publicKey = servablePublicKey(stored); + if (stored !== null && publicKey === null) { + core.options.onError?.( + new Error( + ".deco/secrets.pub isn't a single PUBLIC KEY PEM block; describe reports no key", + ), + ); + } + const readOnly = description.readOnly; + return { + protocol: PROTOCOL_NAME, + version: { major: PROTOCOL_VERSION.major, minor: PROTOCOL_VERSION.minor }, + server: core.options.server ?? DEFAULT_SERVER, + kind: description.kind, + readOnly, + root: description.root, + schemaFormat: SCHEMA_FORMAT, + refs: description.refs, + writes: { + idempotency: core.storage.getReceipt ? description.idempotency : null, + schemaPreconditions: true, + }, + pollIntervalMs: core.pollIntervalMs(description), + limits: core.limits(description), + preview: core.options.preview ?? null, + assets: + readOnly || description.assets === null + ? null + : { + dir: description.assets.dir, + urlPrefix: ASSETS_URL_PREFIX, + maxBytes: description.assets.maxBytes, + }, + secrets: publicKey === null ? null : { publicKey }, + }; +} + +/** Reads and parses the schema, enforcing the schema byte limit. */ +export async function readSchema( + core: Core, + ref: string | undefined, + maxSchemaBytes: number, +): Promise<{ + version: string; + resolvedRef: string | null; + text: string; +} | null> { + const stored = await core.storage.readSchema({ ref }); + if (stored === null) return null; + if (new TextEncoder().encode(stored.text).byteLength > maxSchemaBytes) { + throw limitExceeded(`the schema is over ${maxSchemaBytes} bytes`, { + limit: "maxSchemaBytes", + }); + } + return stored; +} + +/** Parses schema text; a file caught mid-write is reported as Unavailable, never served torn. */ +export function parseSchema(text: string): DecoMeta { + let schema: unknown; + try { + schema = JSON.parse(text); + } catch { + throw unavailable("the schema file is being written; retry shortly", 500); + } + if (typeof schema !== "object" || schema === null || Array.isArray(schema)) { + throw unavailable("the schema file doesn't hold a JSON object", 500); + } + return schema as DecoMeta; +} + +export async function schemaGet( + core: Core, + params: ReadParams, +): Promise { + const description = await core.description(); + core.checkRef(description, params.ref); + const limits = core.limits(description); + const stored = await core.storage.readSchema({ ref: params.ref }); + if (stored === null) { + throw notFound( + "no schema: neither .deco/schema.gen.json nor .deco/meta.gen.json exists", + ); + } + if ( + params.ifNoneMatch !== undefined && + params.ifNoneMatch === stored.version + ) { + return { notModified: true, version: stored.version }; + } + if ( + new TextEncoder().encode(stored.text).byteLength > limits.maxSchemaBytes + ) { + throw limitExceeded(`the schema is over ${limits.maxSchemaBytes} bytes`, { + limit: "maxSchemaBytes", + }); + } + return { + notModified: false, + version: stored.version, + resolvedRef: stored.resolvedRef, + schema: parseSchema(stored.text), + }; +} + +export async function blocksList( + core: Core, + params: ReadParams, +): Promise { + const description = await core.description(); + core.checkRef(description, params.ref); + const limits = core.limits(description); + const { snapshot, content } = await loadCurrentContent( + core.storage, + params.ref, + { readAll: true, limits, cache: core.cache }, + (s) => + params.ifNoneMatch !== undefined && params.ifNoneMatch === s.revision, + ); + if (content === null) { + return { + notModified: true, + revision: snapshot.revision, + resolvedRef: snapshot.resolvedRef, + }; + } + // Null-prototype maps, so an entry named like an Object.prototype key stays an entry. + const blocks: Record> = Object.create(null); + const versions: Record = Object.create(null); + for (const [name, entry] of content.entries) { + if (entry.value === undefined) continue; + blocks[name] = entry.value; + versions[name] = entry.version; + } + return { + notModified: false, + revision: snapshot.revision, + resolvedRef: snapshot.resolvedRef, + blocks, + versions, + diagnostics: content.diagnostics, + }; +} diff --git a/packages/shared/src/vendor/blocks-protocol/server/rpc.ts b/packages/shared/src/vendor/blocks-protocol/server/rpc.ts new file mode 100644 index 0000000000..d746e43563 --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/server/rpc.ts @@ -0,0 +1,171 @@ +/** + * The JSON-RPC 2.0 layer: envelope validation, batches and dispatch. + * + * - Every request has an `id`; one without is rejected rather than run. + * - A batch runs in order, returns results in order, holds at most 10 calls, + * and isn't atomic. + * - The aggregate response is bounded: once it would pass + * `maxBatchResponseBytes`, later reads answer LimitExceeded instead. + * - A write's response is never replaced, alone or in a batch: the write has + * landed, and answering LimitExceeded would tell the client it hadn't. + */ +import { + type ContentProtocolError, + internalError, + invalidRequest, + limitExceeded, + methodNotFound, +} from "../errors"; +import { validateParams } from "../params"; +import { + MAX_BATCH_CALLS, + METHOD_NAMES, + type MethodName, + type RpcId, +} from "../types"; +import { type Core, toProtocolError } from "./core"; +import { blocksApply } from "./methods/apply"; +import { blocksList, describe, schemaGet } from "./methods/read"; + +const utf8 = new TextEncoder(); + +type Envelope = { id: RpcId; method: MethodName; params: unknown }; + +const isMethod = (method: string): method is MethodName => + (METHOD_NAMES as readonly string[]).includes(method); + +/** Validates one request object; returns the error response's id on failure. */ +function parseEnvelope( + value: unknown, +): Envelope | { error: ContentProtocolError; id: RpcId | null } { + const isObject = + typeof value === "object" && value !== null && !Array.isArray(value); + const raw = (isObject ? value : {}) as Record; + const id = + typeof raw.id === "string" || typeof raw.id === "number" ? raw.id : null; + if (!isObject) + return { error: invalidRequest("a request must be an object"), id: null }; + if (raw.jsonrpc !== "2.0") + return { error: invalidRequest('jsonrpc must be "2.0"'), id }; + if (id === null) { + return { + error: invalidRequest("every request needs a string or number id"), + id: null, + }; + } + if (typeof raw.method !== "string") + return { error: invalidRequest("method must be a string"), id }; + for (const key of Object.keys(raw)) { + if (!["jsonrpc", "id", "method", "params"].includes(key)) { + return { error: invalidRequest(`unknown request member "${key}"`), id }; + } + } + if (!isMethod(raw.method)) return { error: methodNotFound(raw.method), id }; + return { id, method: raw.method, params: raw.params }; +} + +async function run( + core: Core, + envelope: Envelope, + scope: string, +): Promise { + switch (envelope.method) { + case "describe": + validateParams("describe", envelope.params); + return describe(core); + case "schema.get": + return schemaGet(core, validateParams("schema.get", envelope.params)); + case "blocks.list": + return blocksList(core, validateParams("blocks.list", envelope.params)); + case "blocks.apply": + return blocksApply( + core, + validateParams("blocks.apply", envelope.params), + scope, + ); + } +} + +/** True for a well-formed `blocks.apply` request. */ +function isWrite(value: unknown): boolean { + const envelope = parseEnvelope(value); + return !("error" in envelope) && envelope.method === "blocks.apply"; +} + +const errorResponse = (id: RpcId | null, error: ContentProtocolError) => + JSON.stringify({ jsonrpc: "2.0", id, error: error.toJSON() }); + +async function call( + core: Core, + value: unknown, + scope: string, +): Promise { + const envelope = parseEnvelope(value); + if ("error" in envelope) return errorResponse(envelope.id, envelope.error); + try { + const result = await run(core, envelope, scope); + return JSON.stringify({ jsonrpc: "2.0", id: envelope.id, result }); + } catch (error) { + const known = toProtocolError(error); + if (known) return errorResponse(envelope.id, known); + core.options.onError?.(error); + return errorResponse(envelope.id, internalError()); + } +} + +/** + * Runs a parsed request body (one request object or a batch array) and + * returns the serialized response body. + */ +export async function dispatch( + core: Core, + body: unknown, + scope: string, + maxResponseBytes: number, +) { + if (!Array.isArray(body)) { + const response = await call(core, body, scope); + if (isWrite(body) || utf8.encode(response).byteLength <= maxResponseBytes) + return response; + return errorResponse( + parseEnvelope(body).id, + limitExceeded(`the response is over ${maxResponseBytes} bytes`, { + limit: "maxBatchResponseBytes", + }), + ); + } + if (body.length === 0) + return errorResponse(null, invalidRequest("an empty batch")); + if (body.length > MAX_BATCH_CALLS) { + return errorResponse( + null, + limitExceeded(`a batch holds at most ${MAX_BATCH_CALLS} calls`, { + limit: "maxBatchCalls", + }), + ); + } + const responses: string[] = []; + let total = 2; // the brackets + for (const item of body) { + const envelope = parseEnvelope(item); + const id = envelope.id; + const overBudget = () => + errorResponse( + id, + limitExceeded(`the batch response is over ${maxResponseBytes} bytes`, { + limit: "maxBatchResponseBytes", + }), + ); + // Writes always run, so a batch never drops a save; reads stop once over budget. + const write = !("error" in envelope) && envelope.method === "blocks.apply"; + let response = + !write && total > maxResponseBytes + ? overBudget() + : await call(core, item, scope); + const bytes = utf8.encode(response).byteLength + (responses.length ? 1 : 0); + if (total + bytes > maxResponseBytes && !write) response = overBudget(); + total += utf8.encode(response).byteLength + (responses.length ? 1 : 0); + responses.push(response); + } + return `[${responses.join(",")}]`; +} diff --git a/packages/shared/src/vendor/blocks-protocol/storage.ts b/packages/shared/src/vendor/blocks-protocol/storage.ts new file mode 100644 index 0000000000..32abba4e69 --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/storage.ts @@ -0,0 +1,185 @@ +/** + * The storage interface a content-protocol backend implements. + * + * A storage is deliberately small: a snapshot of file names and versions, + * reading file bodies, reading the schema, and one atomic commit attempt. + * The protocol core (`createContentHandler`) owns everything else: the + * JSON-RPC layer, validation, the file-name rule, the secret guard, limits and + * retries. Storages deal in *files* inside `/.deco/blocks`; entry names + * are the core's business. + * + * Browser-safe: types and small error classes only. + */ +import type { Limits, StorageKind } from "./types"; + +/** What a storage tells `describe`. */ +export interface StorageDescription { + kind: StorageKind; + /** The app root, relative to the repository root (`"."` when they're the same). */ + root: string; + readOnly: boolean; + /** Branch support; `null` when the storage has no refs (`ref` params are then Unsupported). */ + refs: null | { default: string; autoCreate: boolean }; + /** Where uploads go; `null` when read-only or uploads go elsewhere. */ + assets: null | { dir: string; maxBytes: number }; + /** + * Durable request-key receipts. Advertise only when `getReceipt` is + * implemented and `commit` persists `attempt.receipt` with the mutation. + */ + idempotency: null | { retentionMs: number }; + /** Overrides the poll interval (default: 2000 for a working tree, 30000 for git). */ + pollIntervalMs?: number; + /** Lowers the protocol's default limits. Higher values are ignored. */ + limits?: Partial; +} + +/** One saved-block file in a snapshot. */ +export interface StorageFile { + /** The file name inside `.deco/blocks`, such as `pages-Home.json`. */ + file: string; + /** An opaque version of the file's content, compared only for equality. */ + version: string; + /** The file's size in bytes, when the storage knows it without reading the body. */ + size?: number; +} + +/** Every saved-block file at one point in time. */ +export interface StorageSnapshot { + /** An opaque revision of the whole set, compared only for equality. */ + revision: string; + /** The ref actually read (it can differ from the requested one); `null` without refs. */ + resolvedRef: string | null; + files: StorageFile[]; +} + +/** A file body as read, with the version of the bytes actually read. */ +export interface StoredFileBody { + /** The file's UTF-8 content. */ + text: string; + /** + * The version of exactly these bytes, computed the way snapshots compute + * versions. It differs from the snapshot's when the file changed in between. + */ + version: string; +} + +/** The schema file, as stored. */ +export interface StoredSchema { + /** An opaque version of the schema bytes. */ + version: string; + /** The raw file content (`schema.gen.json`, else `meta.gen.json`). */ + text: string; + resolvedRef: string | null; +} + +/** A durable receipt for one `blocks.apply` request key. */ +export interface StoredReceipt { + /** The canonical digest of the request the key was first used with. */ + digest: string; + /** The revision that commit produced. */ + revision: string; + /** The versions of the files that commit wrote. */ + versions: Record; + /** The files that commit deleted (`attempt.delete`), so a replay reports the same names. */ + deleted?: string[]; +} + +/** One atomic commit attempt. */ +export interface CommitAttempt { + ref?: string; + /** The snapshot the core validated against. */ + base: StorageSnapshot; + /** Files to write, by file name, with their full content. */ + put: Record; + /** Files to delete. A file that's already gone is fine. */ + delete: string[]; + /** + * Versions the files must still have for the commit to land (`null` = must + * not exist). A storage MUST reject the attempt as stale when any of them + * changed, and MAY reject it when anything else changed since `base`. + */ + expected: Record; + /** + * The schema version the core validated against (`null` = no schema), when + * the write depends on it (`ifSchemaMatch`, or the secret guard). When set, + * a storage MUST reject the attempt as stale if the schema's version at + * commit time differs, checked atomically with `expected`. + */ + expectedSchemaVersion?: string | null; + /** + * Persist this receipt atomically with the mutation (only when idempotency + * is advertised). The key also reserves the request: a storage MUST reject + * the attempt as stale when a receipt for `receipt.key` already exists, + * including one written concurrently by another replica, so two requests + * with the same key never both commit. + */ + receipt?: { key: string; digest: string }; +} + +export type CommitResult = + | { + status: "committed"; + revision: string; + /** The new version of every file in `put`. */ + versions: Record; + } + | { status: "stale" }; + +export interface ContentStorage { + describe(): StorageDescription | Promise; + /** + * Lists the saved-block files. Throws `StorageNotFoundError` when there's no + * `.deco` folder at all, or the ref can't be read. + */ + snapshot(options: { ref?: string }): Promise; + /** + * Reads file bodies (UTF-8 text) from a snapshot, each with the version of + * the bytes read. A file that vanished is left out. A storage that can't + * read at the snapshot (a working tree) returns current bytes; the core + * compares versions to notice. + */ + readFiles( + snapshot: StorageSnapshot, + files: string[], + ): Promise>; + /** Reads `schema.gen.json`, falling back to `meta.gen.json`; `null` when neither exists. */ + readSchema(options: { ref?: string }): Promise; + /** Reads `/.deco/secrets.pub`; `null` without one. */ + readSecretsPublicKey(): Promise; + /** One all-or-nothing commit attempt; durable when it resolves. */ + commit(attempt: CommitAttempt): Promise; + /** Looks up a receipt by its scoped key; `null` when unknown or expired. */ + getReceipt?(key: string): Promise; + /** Stores an uploaded file, never overwriting one; returns the stored file name. */ + putAsset?(name: string, body: Uint8Array): Promise<{ name: string }>; +} + +/** Thrown by a storage when there's no `.deco` folder, or a ref can't be read or created. */ +export class StorageNotFoundError extends Error { + constructor(message: string) { + super(message); + this.name = "StorageNotFoundError"; + } +} + +/** + * Thrown by a storage asked to touch a file that can't be a saved block (a + * dotfile, a path). The core refuses those names first; this is the backstop. + */ +export class StorageInvalidFileError extends Error { + constructor(readonly file: string) { + super(`not a saved-block file name: ${JSON.stringify(file)}`); + this.name = "StorageInvalidFileError"; + } +} + +/** Thrown by a storage when it failed or is rate-limited. */ +export class StorageUnavailableError extends Error { + readonly retryAfterMs: number | undefined; + + constructor(message: string, retryAfterMs?: number) { + super(message); + this.name = "StorageUnavailableError"; + this.retryAfterMs = retryAfterMs; + } +} diff --git a/packages/shared/src/vendor/blocks-protocol/storage/memory.ts b/packages/shared/src/vendor/blocks-protocol/storage/memory.ts new file mode 100644 index 0000000000..7d8a263bba --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/storage/memory.ts @@ -0,0 +1,253 @@ +/** + * `@decocms/blocks/protocol/storage/memory`: an in-memory `ContentStorage`. + * + * A reference implementation of the storage contract, including durable + * request-key receipts persisted atomically with each commit, and a fixture + * for tests: its state can be dumped and restored to simulate a restart, and + * hooks let a test change files between a snapshot and a commit. Browser-safe. + */ +import { suffixedAssetName } from "../assets"; +import { sha256Hex } from "../canonical"; +import { isBlockFileName } from "../keys"; +import { + type CommitAttempt, + type CommitResult, + type ContentStorage, + type StorageDescription, + StorageInvalidFileError, + StorageNotFoundError, + type StorageSnapshot, + type StoredFileBody, + type StoredReceipt, + type StoredSchema, +} from "../storage"; + +const DAY_MS = 24 * 60 * 60 * 1000; + +/** Everything a memory storage holds; JSON-serializable, so it survives a "restart". */ +export interface MemoryStorageState { + /** Saved-block files, by file name, with their content. */ + files: Record; + /** The content of `schema.gen.json`, or `null` without one. */ + schema: string | null; + /** The content of `meta.gen.json`, the fallback schema, or `null`. */ + legacySchema?: string | null; + secretsPublicKey: string | null; + receipts: Record; + assets: Record; + /** `false` simulates an app root with no `.deco` folder at all. */ + hasDecoFolder: boolean; +} + +export interface MemoryStorageOptions { + state?: Partial; + /** Overrides parts of what `describe` reports. */ + description?: Partial; + /** Clock for receipt retention (default `Date.now`). */ + now?: () => number; + /** Runs after each file body is read; a test can change files here, mid-read. */ + afterRead?: (file: string, storage: MemoryStorage) => void | Promise; + /** Runs before each commit attempt's checks; a test can change files here. */ + beforeCommit?: ( + attempt: CommitAttempt, + storage: MemoryStorage, + ) => void | Promise; +} + +export interface MemoryStorage extends ContentStorage { + /** A deep copy of the current state. */ + dump(): MemoryStorageState; + /** Writes or deletes (`null`) a file directly, as an outside edit would. */ + setFile(file: string, content: string | null): void; + /** Replaces `schema.gen.json` (`null` removes it). */ + setSchema(schema: string | object | null): void; + /** How many commits landed. */ + readonly commits: number; +} + +const version = (content: string) => sha256Hex(content); + +export function createMemoryStorage( + options: MemoryStorageOptions = {}, +): MemoryStorage { + const now = options.now ?? Date.now; + const state: MemoryStorageState = structuredClone({ + files: {}, + schema: null, + legacySchema: null, + secretsPublicKey: null, + receipts: {}, + assets: {}, + hasDecoFolder: true, + ...options.state, + }); + const description: StorageDescription = { + kind: "git", + root: ".", + readOnly: false, + refs: null, + assets: { dir: "public/assets", maxBytes: 25 * 1024 * 1024 }, + idempotency: { retentionMs: DAY_MS }, + ...options.description, + }; + let commits = 0; + + async function currentVersions(): Promise> { + const entries = await Promise.all( + Object.entries(state.files).map( + async ([file, content]) => [file, await version(content)] as const, + ), + ); + return new Map(entries); + } + + async function schemaVersion(): Promise { + const text = state.schema ?? state.legacySchema ?? null; + return text === null ? null : version(text); + } + + async function revisionOf(versions: Map): Promise { + const listing = [...versions].sort(([a], [b]) => + a < b ? -1 : a > b ? 1 : 0, + ); + return sha256Hex(listing.map(([file, v]) => `${file}\0${v}`).join("\n")); + } + + let commitQueue: Promise = Promise.resolve(); + + async function commitNow(attempt: CommitAttempt): Promise { + await options.beforeCommit?.(attempt, storage); + for (const file of [ + ...Object.keys(attempt.put), + ...attempt.delete, + ...Object.keys(attempt.expected), + ]) { + if (!isBlockFileName(file)) throw new StorageInvalidFileError(file); + } + if (!state.hasDecoFolder) throw new StorageNotFoundError("no .deco folder"); + if (attempt.receipt && state.receipts[attempt.receipt.key]) + return { status: "stale" }; + if ( + attempt.expectedSchemaVersion !== undefined && + (await schemaVersion()) !== attempt.expectedSchemaVersion + ) { + return { status: "stale" }; + } + const versions = await currentVersions(); + for (const [file, expected] of Object.entries(attempt.expected)) { + if ((versions.get(file) ?? null) !== expected) return { status: "stale" }; + } + // Validate everything before mutating, so the commit is all or nothing. + const written: Record = {}; + for (const [file, content] of Object.entries(attempt.put)) + written[file] = await version(content); + for (const file of attempt.delete) delete state.files[file]; + for (const [file, content] of Object.entries(attempt.put)) + state.files[file] = content; + for (const [file, v] of Object.entries(written)) versions.set(file, v); + for (const file of attempt.delete) + if (!(file in attempt.put)) versions.delete(file); + const revision = await revisionOf(versions); + if (attempt.receipt) { + state.receipts[attempt.receipt.key] = { + digest: attempt.receipt.digest, + revision, + versions: written, + deleted: [...attempt.delete], + createdAt: now(), + }; + } + commits++; + return { status: "committed", revision, versions: written }; + } + + const storage: MemoryStorage = { + get commits() { + return commits; + }, + + describe: () => description, + + async snapshot(): Promise { + if (!state.hasDecoFolder) + throw new StorageNotFoundError("no .deco folder"); + const versions = await currentVersions(); + return { + revision: await revisionOf(versions), + resolvedRef: null, + files: [...versions].map(([file, v]) => ({ + file, + version: v, + size: new TextEncoder().encode(state.files[file]).byteLength, + })), + }; + }, + + async readFiles(_snapshot, files) { + const out: Record = Object.create(null); + for (const file of files) { + if (!(file in state.files)) continue; + const text = state.files[file]; + out[file] = { text, version: await version(text) }; + await options.afterRead?.(file, storage); + } + return out; + }, + + async readSchema(): Promise { + if (!state.hasDecoFolder) return null; + const text = state.schema ?? state.legacySchema ?? null; + if (text === null) return null; + return { version: await version(text), text, resolvedRef: null }; + }, + + async readSecretsPublicKey() { + return state.secretsPublicKey; + }, + + commit(attempt): Promise { + // Serialized, so a commit's checks and its mutation are one atomic step. + const run = commitQueue.then(() => commitNow(attempt)); + commitQueue = run.catch(() => {}); + return run; + }, + + async getReceipt(key) { + const receipt = state.receipts[key]; + if (!receipt) return null; + if ( + description.idempotency && + now() - receipt.createdAt > description.idempotency.retentionMs + ) { + delete state.receipts[key]; + return null; + } + const { createdAt: _createdAt, ...stored } = receipt; + return stored; + }, + + async putAsset(name, body) { + let candidate = name; + while (candidate in state.assets) candidate = suffixedAssetName(name); + state.assets[candidate] = Array.from(body, (b) => + String.fromCharCode(b), + ).join(""); + return { name: candidate }; + }, + + dump: () => structuredClone(state), + + setFile(file, content) { + if (content === null) delete state.files[file]; + else state.files[file] = content; + }, + + setSchema(schema) { + state.schema = + schema === null || typeof schema === "string" + ? schema + : JSON.stringify(schema); + }, + }; + return storage; +} diff --git a/packages/shared/src/vendor/blocks-protocol/types.ts b/packages/shared/src/vendor/blocks-protocol/types.ts new file mode 100644 index 0000000000..9ff31c374e --- /dev/null +++ b/packages/shared/src/vendor/blocks-protocol/types.ts @@ -0,0 +1,197 @@ +/** + * The content protocol's method types: JSON-RPC 2.0 over one HTTP endpoint, + * four methods (`describe`, `schema.get`, `blocks.list`, `blocks.apply`). + */ + +export const PROTOCOL_NAME = "deco-content"; + +/** Minors only add; a client refuses an unknown major. */ +export const PROTOCOL_VERSION = { major: 1, minor: 0 } as const; + +export const SCHEMA_FORMAT = "deco-meta@1"; + +/** The path the local server (`deco serve`) mounts the endpoint at. */ +export const RPC_PATH = "/rpc"; + +/** The longest batch a request may carry. */ +export const MAX_BATCH_CALLS = 10; + +/** The URL prefix every uploaded asset is stored under in a field. */ +export const ASSETS_URL_PREFIX = "/assets/"; + +export interface Limits { + /** Names (set plus delete) in one `blocks.apply`. */ + maxOpsPerApply: number; + /** Bytes of one stored entry. */ + maxBlockBytes: number; + /** Bytes of one HTTP request body, uncompressed. */ + maxRequestBytes: number; + /** Uncompressed bytes of the whole block list. */ + maxListBytes: number; + /** Uncompressed bytes of the schema. */ + maxSchemaBytes: number; + /** Uncompressed bytes of a whole batch response. */ + maxBatchResponseBytes: number; +} + +const MiB = 1024 * 1024; + +/** The protocol's default limits. A storage or a server can lower them. */ +export const DEFAULT_LIMITS: Readonly = Object.freeze({ + maxOpsPerApply: 500, + maxBlockBytes: 1 * MiB, + maxRequestBytes: 8 * MiB, + maxListBytes: 16 * MiB, + maxSchemaBytes: 16 * MiB, + maxBatchResponseBytes: 32 * MiB, +}); + +export type StorageKind = "working-tree" | "git"; + +export interface DescribeResult { + protocol: typeof PROTOCOL_NAME; + version: { major: 1; minor: number }; + server: { name: string; version: string }; + /** The site editor hides publish and draft UI for a working tree. */ + kind: StorageKind; + readOnly: boolean; + /** The app root: the folder that contains `.deco/`, relative to the repository root. */ + root: string; + schemaFormat: typeof SCHEMA_FORMAT; + /** Branches; `null` on the local server. */ + refs: null | { default: string; autoCreate: boolean }; + writes: { + idempotency: null | { retentionMs: number }; + schemaPreconditions: boolean; + }; + /** Local: 2000; git: 30000, plus on window focus. */ + pollIntervalMs: number; + limits: Limits; + /** Where "open the real page" points. */ + preview: null | { origin: string }; + /** Dir relative to the repository root; `null` when read-only or uploads go to hosted storage. */ + assets: null | { + dir: string; + urlPrefix: typeof ASSETS_URL_PREFIX; + maxBytes: number; + }; + /** The contents of `/.deco/secrets.pub`; `null` without one. */ + secrets: null | { publicKey: string }; +} + +/** The schema file, in the site editor's `deco-meta@1` format. */ +export interface DecoMeta { + manifest?: { blocks?: Record> }; + schema?: { + definitions?: Record; + root?: Record; + }; + [key: string]: unknown; +} + +export interface ReadParams { + ref?: string; + ifNoneMatch?: string; +} + +export type SchemaGetParams = ReadParams; + +export type SchemaGetResult = + | { notModified: true; version: string } + | { + notModified: false; + version: string; + resolvedRef: string | null; + schema: DecoMeta; + }; + +export type BlocksListParams = ReadParams; + +/** A file `blocks.list` skipped or shadowed. */ +export interface Diagnostic { + /** The file name inside `.deco/blocks`. */ + file: string; + kind: "invalid-json" | "not-an-object" | "too-large" | "shadowed"; + message: string; + /** For `shadowed`: the entry the file is a spelling of. */ + name?: string; + /** For `shadowed`: the file that won. */ + winner?: string; +} + +export type BlocksListResult = + | { notModified: true; revision: string; resolvedRef: string | null } + | { + notModified: false; + revision: string; + resolvedRef: string | null; + /** Every saved block, by name. */ + blocks: Record>; + /** One opaque version per entry. */ + versions: Record; + /** Files skipped or shadowed. */ + diagnostics: Diagnostic[]; + }; + +export interface BlocksApplyParams { + ref?: string; + /** Retry the same logical write; only when advertised. */ + requestKey?: string; + /** Reject if the schema changed; only when advertised. */ + ifSchemaMatch?: string; + /** Whole-entry replace (create or update). */ + set?: Record>; + /** A missing name counts as deleted. */ + delete?: string[]; + /** A version the entry must have; `null` = must not exist. */ + ifMatch?: Record; +} + +export interface BlocksApplyResult { + revision: string; + versions: Record; +} + +/** The four methods, by wire name. */ +export interface Methods { + describe: { + params: Record | undefined; + result: DescribeResult; + }; + "schema.get": { + params: SchemaGetParams | undefined; + result: SchemaGetResult; + }; + "blocks.list": { + params: BlocksListParams | undefined; + result: BlocksListResult; + }; + "blocks.apply": { params: BlocksApplyParams; result: BlocksApplyResult }; +} + +export type MethodName = keyof Methods; + +export const METHOD_NAMES: readonly MethodName[] = [ + "describe", + "schema.get", + "blocks.list", + "blocks.apply", +]; + +/** A JSON-RPC request id. The protocol requires one on every request. */ +export type RpcId = string | number; + +export interface RpcRequest { + jsonrpc: "2.0"; + id: RpcId; + method: M; + params?: Methods[M]["params"]; +} + +export type RpcResponse = + | { jsonrpc: "2.0"; id: RpcId | null; result: R } + | { + jsonrpc: "2.0"; + id: RpcId | null; + error: { code: number; message: string; data?: unknown }; + }; diff --git a/scripts/sync-blocks-protocol.ts b/scripts/sync-blocks-protocol.ts new file mode 100644 index 0000000000..4e6ee3639c --- /dev/null +++ b/scripts/sync-blocks-protocol.ts @@ -0,0 +1,98 @@ +#!/usr/bin/env bun +/** + * Copies the Blocks content protocol (`@decocms/blocks/protocol`) into + * `packages/shared/src/vendor/blocks-protocol/` until `@decocms/blocks@8` is + * published. See that folder's SYNC.md. + * + * bun run scripts/sync-blocks-protocol.ts + * + * Copies the tree at `` (not the working tree), leaves out the + * Node-only filesystem storage and the tests, renames files to kebab-case + * (this repo's file-name rule) with their relative imports, formats, and + * points SYNC.md at the commit. + */ + +import { mkdir, rm, writeFile } from "node:fs/promises"; +import { dirname, join, posix } from "node:path"; + +const SOURCE_DIR = "packages/blocks/src/protocol"; +const repoRoot = join(import.meta.dir, ".."); +const targetDir = join(repoRoot, "packages/shared/src/vendor/blocks-protocol"); +const syncNote = join(targetDir, "SYNC.md"); + +function excluded(path: string): boolean { + return ( + path.startsWith("storage/fs/") || + path.startsWith("__tests__/") || + /\.test\.ts$/.test(path) + ); +} + +const kebab = (segment: string) => + segment.replace(/([a-z0-9])([A-Z])/g, "$1-$2").toLowerCase(); + +const kebabPath = (path: string) => path.split("/").map(kebab).join("/"); + +function git(cwd: string, args: string[]): string { + const result = Bun.spawnSync(["git", ...args], { cwd }); + if (result.exitCode !== 0) { + throw new Error(`git ${args.join(" ")}: ${result.stderr.toString()}`); + } + return result.stdout.toString(); +} + +/** Rewrites relative import specifiers to the kebab-case file names. */ +function rewriteImports(source: string): string { + return source.replace( + /(from\s+|import\s*\(\s*)"(\.{1,2}\/[^"]+)"/g, + (_match, prefix: string, specifier: string) => + `${prefix}"${posix.normalize(kebabPath(specifier)).replace(/^(?!\.)/, "./")}"`, + ); +} + +async function main() { + const [checkout, commit] = process.argv.slice(2); + if (!checkout || !commit) { + console.error( + "usage: bun run scripts/sync-blocks-protocol.ts ", + ); + process.exit(1); + } + const sha = git(checkout, ["rev-parse", commit]).trim(); + const files = git(checkout, [ + "ls-tree", + "-r", + "--name-only", + sha, + `${SOURCE_DIR}/`, + ]) + .split("\n") + .filter(Boolean) + .map((path) => path.slice(SOURCE_DIR.length + 1)) + .filter((path) => path.endsWith(".ts") && !excluded(path)); + + const note = await Bun.file(syncNote).text(); + await rm(targetDir, { recursive: true, force: true }); + for (const path of files) { + const source = git(checkout, ["show", `${sha}:${SOURCE_DIR}/${path}`]); + const target = join(targetDir, kebabPath(path)); + await mkdir(dirname(target), { recursive: true }); + await writeFile(target, rewriteImports(source)); + } + await writeFile( + syncNote, + note.replace(/at commit `[0-9a-f]+`/, `at commit \`${sha}\``), + ); + const format = Bun.spawnSync( + ["bunx", "biome", "format", "--write", targetDir], + { + cwd: repoRoot, + stdout: "inherit", + stderr: "inherit", + }, + ); + if (format.exitCode !== 0) process.exit(format.exitCode ?? 1); + console.log(`synced ${files.length} files from ${sha}`); +} + +await main(); From 6398a60525b56bff9c93f582a2877645677c9ac1 Mon Sep 17 00:00:00 2001 From: gimenes Date: Sat, 3 Oct 2026 11:58:53 -0300 Subject: [PATCH 02/71] feat(shared): add the site_editor_content_protocol org flag Default off. Gates the content-protocol editing path for next-major Blocks sites. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WNwbSEePYNcY5YCgqZURig --- packages/shared/src/organization/schema.ts | 6 ++++++ packages/shared/src/tools/tool-io.ts | 3 +++ 2 files changed, 9 insertions(+) diff --git a/packages/shared/src/organization/schema.ts b/packages/shared/src/organization/schema.ts index 9c6ffa0b7a..a022c7c834 100644 --- a/packages/shared/src/organization/schema.ts +++ b/packages/shared/src/organization/schema.ts @@ -277,6 +277,12 @@ export const OrgFlagsSchema = z.object({ .describe( "Use the redesigned blocks editor for every member of the organization. Off by default — the classic editor stays until an admin opts the org in.", ), + site_editor_content_protocol: z + .boolean() + .optional() + .describe( + "Edit next-major Blocks sites through the content protocol: a project with a committed schema (`.deco/schema.gen.json` or `meta.gen.json`) is edited without running its code, on GitHub or through a connected `deco serve`. Off by default — with it off, every project stays on the legacy editing path.", + ), }); export type OrgFlags = z.infer; diff --git a/packages/shared/src/tools/tool-io.ts b/packages/shared/src/tools/tool-io.ts index 03b4d5ab1a..2e76e25d67 100644 --- a/packages/shared/src/tools/tool-io.ts +++ b/packages/shared/src/tools/tool-io.ts @@ -133,6 +133,7 @@ export interface StudioToolIO { site_create_enabled?: boolean | undefined; delivery_lanes_enabled?: boolean | undefined; new_blocks_editor?: boolean | undefined; + site_editor_content_protocol?: boolean | undefined; } | null | undefined; @@ -211,6 +212,7 @@ export interface StudioToolIO { site_create_enabled?: boolean | undefined; delivery_lanes_enabled?: boolean | undefined; new_blocks_editor?: boolean | undefined; + site_editor_content_protocol?: boolean | undefined; } | undefined; submodule_credentials?: { host: string; secretId: string }[] | undefined; @@ -285,6 +287,7 @@ export interface StudioToolIO { site_create_enabled?: boolean | undefined; delivery_lanes_enabled?: boolean | undefined; new_blocks_editor?: boolean | undefined; + site_editor_content_protocol?: boolean | undefined; } | null | undefined; From fe930395de9b94d323fd3af8c3f9c8744e434751 Mon Sep 17 00:00:00 2001 From: gimenes Date: Sat, 3 Oct 2026 12:25:14 -0300 Subject: [PATCH 03/71] feat(api): serve the content protocol over a project's repository Adds a ContentStorage over RepoContentClient (one repo, app root and branch; blob shas as versions, the branch head as the revision) behind the vendored createContentHandler, at POST /api/:org/decofile/:virtualMcpId/:branch/rpc GET /api/:org/decofile/:virtualMcpId/:branch/draft-grant Both sit under the existing scope middleware (session, cms plan, repository binding) and answer 404 unless the org has the site_editor_content_protocol flag. They don't need a preview server. A missing branch reads as the default branch until the first write creates it. The tracked blocks.gen.json regeneration moves to gen-artifact.ts, shared with the commit coalescer. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WNwbSEePYNcY5YCgqZURig --- apps/api/src/api/routes/decofile.ts | 86 ++++- apps/api/src/decofile/commit-coalescer.ts | 36 +-- apps/api/src/decofile/gen-artifact.ts | 48 +++ apps/api/src/decofile/repo-content-storage.ts | 299 ++++++++++++++++++ 4 files changed, 442 insertions(+), 27 deletions(-) create mode 100644 apps/api/src/decofile/gen-artifact.ts create mode 100644 apps/api/src/decofile/repo-content-storage.ts diff --git a/apps/api/src/api/routes/decofile.ts b/apps/api/src/api/routes/decofile.ts index 59bb521296..3bd0a0d716 100644 --- a/apps/api/src/api/routes/decofile.ts +++ b/apps/api/src/api/routes/decofile.ts @@ -8,6 +8,8 @@ * PATCH /api/:org/decofile/:virtualMcpId/:branch write blocks (session) * POST /api/:org/decofile/:virtualMcpId/:branch/publish merge into default (session) * GET /api/:org/decofile/:virtualMcpId/:branch/status drift vs default (session) + * POST /api/:org/decofile/:virtualMcpId/:branch/rpc content protocol (session, flag) + * GET /api/:org/decofile/:virtualMcpId/:branch/draft-grant draft token (session, flag) * * The surface is inert unless the virtual MCP has both a preview server URL * (`previewServerUrl`, legacy `productionUrl`) and a GitHub repo — what a CMS @@ -15,6 +17,11 @@ * does NOT gate it; that switch only picks the runtime a NEW thread is stamped * with, and gating this on it would strand an already-stamped session. * + * The two content-protocol routes serve next-major Blocks sites (see + * `decofile/repo-content-storage.ts`) and exist only behind the + * `site_editor_content_protocol` org flag. They don't need a preview server: + * the protocol never renders, and a project without one just has no preview. + * * Anonymous access: `resolveOrgFromPath` lets unauthenticated requests through * (membership is only enforced for signed-in principals), so the GET handler * self-enforces the signed draft token, mirroring automation-webhooks.ts. @@ -47,6 +54,9 @@ import { import { signDraftToken, verifyDraftToken } from "@/decofile/draft-token"; import { repoGitRebase } from "@/decofile/git-compat"; import { readDecofileSnapshot } from "@/decofile/read-decofile"; +import { createRepoContentStorage } from "@/decofile/repo-content-storage"; +import { createContentHandler } from "@decocms/shared/blocks-protocol/server"; +import { orgFlagEnabled } from "@decocms/shared/organization/schema"; import { projectPlanningPostsForPreview } from "@/decofile/blog-draft-projection"; import { orgHasFeature } from "@/core/plan-feature-gate"; import type { Env } from "../hono-env"; @@ -59,6 +69,7 @@ interface DecofileScope { repository: RepositoryBinding; /** Present only for session-authenticated (member) requests. */ userId: string | null; + previewServerUrl: string | null; } type DecofileEnv = Env & { @@ -152,7 +163,8 @@ const resolveDecofileScope = createMiddleware(async (c, next) => { // Anonymous: only the plain GET is reachable, and only with a valid token. const token = c.req.query("token"); const isPlainGet = - c.req.method === "GET" && !c.req.path.match(/\/(publish|status)$/); + c.req.method === "GET" && + !c.req.path.match(/\/(publish|status|draft-grant)$/); if (!isPlainGet) return c.json({ error: "Unauthorized" }, 401); if ( !token || @@ -192,7 +204,7 @@ const resolveDecofileScope = createMiddleware(async (c, next) => { * threads, and gating the data plane on it would strand a session already * stamped `cms` the moment someone flips the switch off. */ const previewServerUrl = resolvePreviewServerUrl(metadata); - if (!previewServerUrl) { + if (!previewServerUrl && !isContentProtocolPath(c.req.path)) { return c.json({ error: "Project has no preview server configured" }, 404); } @@ -211,10 +223,34 @@ const resolveDecofileScope = createMiddleware(async (c, next) => { packagePath: runtime?.path?.replace(/^\/+|\/+$/g, "") || null, repository, userId, + previewServerUrl, }); return next(); }); +/** The content-protocol routes, which work without a preview server. */ +function isContentProtocolPath(path: string): boolean { + return /\/(rpc|draft-grant)$/.test(path); +} + +async function contentProtocolEnabled( + c: Context, +): Promise { + const ctx = c.var.studioContext; + const settings = await ctx.storage.organizationSettings.get( + c.get("decofileScope").organizationId, + ); + return orgFlagEnabled(settings?.flags, "site_editor_content_protocol"); +} + +function signScopeDraftToken(scope: DecofileScope): string { + return signDraftToken({ + organizationId: scope.organizationId, + virtualMcpId: scope.virtualMcpId, + branch: scope.branch, + }); +} + /** * The authority (host[:port]) the editor should bake into the `?__draft=` * pointer. `window.location.host` is wrong in the native app — the webview's @@ -496,5 +532,51 @@ export function createDecofileRoutes() { } }); + app.post("/:virtualMcpId/:branch/rpc", async (c) => { + if (!(await contentProtocolEnabled(c))) { + return c.json({ error: "Not found" }, 404); + } + const scope = c.get("decofileScope"); + const client = await contentClientForScope(c); + // Per request: the storage carries this caller's repository credential. + // The body cache is off for the same reason, and the blob cache under + // the storage already makes repeated reads cheap. + const handler = createContentHandler( + createRepoContentStorage({ + client, + packagePath: scope.packagePath, + branch: scope.branch, + coAuthor: coAuthorFromStudioContext(c.var.studioContext), + }), + { + server: { name: "studio-github", version: "1" }, + preview: scope.previewServerUrl + ? { origin: new URL(scope.previewServerUrl).origin } + : null, + bodyCacheBytes: 0, + onError: (error) => + console.error("content protocol: internal error", { + organizationId: scope.organizationId, + virtualMcpId: scope.virtualMcpId, + error: error instanceof Error ? error.message : String(error), + }), + }, + ); + return handler(c.req.raw); + }); + + /** The grant a protocol client puts in its `?__draft=` pointer. */ + app.get("/:virtualMcpId/:branch/draft-grant", async (c) => { + if (!(await contentProtocolEnabled(c))) { + return c.json({ error: "Not found" }, 404); + } + const scope = c.get("decofileScope"); + return c.json( + { token: signScopeDraftToken(scope), apiHost: requestApiHost(c) }, + 200, + { "Cache-Control": "no-store" }, + ); + }); + return app; } diff --git a/apps/api/src/decofile/commit-coalescer.ts b/apps/api/src/decofile/commit-coalescer.ts index e5b3df6a33..6c4840cf12 100644 --- a/apps/api/src/decofile/commit-coalescer.ts +++ b/apps/api/src/decofile/commit-coalescer.ts @@ -2,8 +2,7 @@ import { appendCoAuthorTrailer, type CoAuthorIdentity, } from "@decocms/sandbox/shared"; -import { blockKeyToFileStem, mergeBlocks } from "@decocms/shared/decofile"; -import { repoIdentityKey } from "@decocms/shared/git-providers"; +import { blockKeyToFileStem } from "@decocms/shared/decofile"; import { exponentialBackoffWithJitter, sleep } from "@decocms/shared/std"; import { type FileChange, @@ -15,9 +14,9 @@ import { blockEntriesInTree, blocksDirPath, primeBlobCache, - resolveBlockContents, resolveOrCreateHead, } from "./read-decofile"; +import { regenerateGenArtifact } from "./gen-artifact"; /** * Per-(virtualMcpId, branch) commit coalescer. Autosaves arrive every ~700ms @@ -174,28 +173,15 @@ async function commitBatch(batch: Batch): Promise { if (writes.length === 0) return headSha; - // Repos that track the merged artifact get it regenerated in-commit; - // gitignored repos (the common case) never have the tree entry. - const genPath = packagePath - ? `${packagePath}/.deco/blocks.gen.json` - : ".deco/blocks.gen.json"; - if (tree.some((e) => e.type === "blob" && e.path === genPath)) { - const files = await resolveBlockContents( - client, - nextBlocks.values(), - blobMemo, - ); - const { decofile: genContent, skipped } = mergeBlocks(files); - if (skipped.length > 0) { - console.warn("decofile gen: dropped blocks that were not valid JSON", { - repo: repoIdentityKey(client.repo), - branch, - packagePath, - blocks: skipped.map((s) => s.key), - }); - } - writes.push({ path: genPath, content: genContent }); - } + const gen = await regenerateGenArtifact({ + client, + tree, + packagePath, + branch, + nextBlocks: nextBlocks.values(), + memo: blobMemo, + }); + if (gen) writes.push(gen); try { const { sha } = await client.commitFiles({ diff --git a/apps/api/src/decofile/gen-artifact.ts b/apps/api/src/decofile/gen-artifact.ts new file mode 100644 index 0000000000..81df90c113 --- /dev/null +++ b/apps/api/src/decofile/gen-artifact.ts @@ -0,0 +1,48 @@ +import { mergeBlocks } from "@decocms/shared/decofile"; +import { repoIdentityKey } from "@decocms/shared/git-providers"; +import type { FileChange, RepoContentClient, TreeEntry } from "@/git-providers"; +import { type BlockSource, resolveBlockContents } from "./read-decofile"; + +/** Repo-relative path of the merged `blocks.gen.json` artifact. */ +function genArtifactPath(packagePath: string | null): string { + return packagePath + ? `${packagePath}/.deco/blocks.gen.json` + : ".deco/blocks.gen.json"; +} + +/** + * The regenerated `blocks.gen.json`, for a commit that changes `.deco/blocks`. + * + * Repos that track the merged artifact get it rewritten in the same commit, so + * it never disagrees with the block files; gitignored repos (the common case) + * have no tree entry and get `null`. `nextBlocks` is the post-commit view of + * the blocks dir; `memo` is the blob memo a compare-and-swap retry loop + * threads through, so a retry never re-reads a blob it already resolved. + */ +export async function regenerateGenArtifact(params: { + client: RepoContentClient; + tree: TreeEntry[]; + packagePath: string | null; + branch: string; + nextBlocks: Iterable; + memo: Map; +}): Promise { + const { client, tree, packagePath, branch } = params; + const genPath = genArtifactPath(packagePath); + if (!tree.some((e) => e.type === "blob" && e.path === genPath)) return null; + const files = await resolveBlockContents( + client, + params.nextBlocks, + params.memo, + ); + const { decofile, skipped } = mergeBlocks(files); + if (skipped.length > 0) { + console.warn("decofile gen: dropped blocks that were not valid JSON", { + repo: repoIdentityKey(client.repo), + branch, + packagePath, + blocks: skipped.map((s) => s.key), + }); + } + return { path: genPath, content: decofile }; +} diff --git a/apps/api/src/decofile/repo-content-storage.ts b/apps/api/src/decofile/repo-content-storage.ts new file mode 100644 index 0000000000..c6d31fef15 --- /dev/null +++ b/apps/api/src/decofile/repo-content-storage.ts @@ -0,0 +1,299 @@ +/** + * The content protocol's storage over a project's Git repository: the site + * editor's GitHub backend (the hosted side of `deco serve`). + * + * One storage is bound to one `(repository, app root, branch)`. The vendored + * protocol core (`createContentHandler`) owns names, validation, the secret + * guard, limits and retries; this file only lists, reads and commits files + * under `/.deco/`, through the same `RepoContentClient`, blob cache and + * compare-and-swap commit the legacy decofile routes use. + * + * Versions are git blob shas and the revision is the branch head sha. Before + * the first write a missing branch reads as the default branch (reported in + * `resolvedRef`); the first commit creates it at the head it was read from. + */ + +import { + appendCoAuthorTrailer, + type CoAuthorIdentity, +} from "@decocms/sandbox/shared"; +import { + blockNameFromFile, + type CommitAttempt, + type CommitResult, + type ContentStorage, + type StorageDescription, + type StorageSnapshot, + type StoredFileBody, + type StoredSchema, + StorageNotFoundError, + StorageUnavailableError, + unsupported, +} from "@decocms/shared/blocks-protocol"; +import { + type FileChange, + type RepoContentClient, + RepoWriteConflict, + repoRateLimitRetryAfterMs, + requireBranchHead, +} from "@/git-providers"; +import { regenerateGenArtifact } from "./gen-artifact"; +import { + type BlockSource, + blockEntriesInTree, + blocksDirPath, + gitBlobSha, + primeBlobCache, + resolveBlockContents, +} from "./read-decofile"; + +export interface RepoContentStorageOptions { + client: RepoContentClient; + /** The app root inside the repository; `null` at the repository root. */ + packagePath: string | null; + /** The one branch this storage reads and writes. */ + branch: string; + /** Acting user, appended as a `Co-authored-by:` trailer when present. */ + coAuthor?: CoAuthorIdentity | null; +} + +function decoPath(packagePath: string | null, file: string): string { + return packagePath ? `${packagePath}/.deco/${file}` : `.deco/${file}`; +} + +/** Turns provider failures into the storage errors the core understands. */ +async function guarded(work: () => Promise): Promise { + try { + return await work(); + } catch (error) { + const retryAfterMs = repoRateLimitRetryAfterMs(error); + if (retryAfterMs !== undefined) { + throw new StorageUnavailableError( + "the Git provider is rate-limiting this repository", + retryAfterMs ?? undefined, + ); + } + throw error; + } +} + +function commitMessage(attempt: CommitAttempt): string { + const summarize = (files: string[]): string => { + const names = files.map(blockNameFromFile); + const shown = names.slice(0, 3).join(", "); + return names.length > 3 ? `${shown} (+${names.length - 3} more)` : shown; + }; + const parts: string[] = []; + const put = Object.keys(attempt.put); + if (put.length > 0) parts.push(`update ${summarize(put)}`); + if (attempt.delete.length > 0) + parts.push(`delete ${summarize(attempt.delete)}`); + return `chore(decofile): ${parts.join("; ")}`; +} + +export function createRepoContentStorage( + options: RepoContentStorageOptions, +): ContentStorage { + const { client, packagePath, branch } = options; + const blocksPrefix = `${blocksDirPath(packagePath)}/`; + + const checkRef = (ref: string | undefined) => { + if (ref !== undefined && ref !== branch) { + throw unsupported( + `this endpoint is bound to the branch "${branch}"; omit ref`, + ); + } + }; + + /** The bound branch's head, or the default branch's while it doesn't exist. */ + const readHead = async (): Promise<{ sha: string; ref: string }> => { + const head = await client.getBranch(branch); + if (head) return { sha: head.sha, ref: branch }; + const defaultBranch = await client.getDefaultBranch(); + return { + sha: await requireBranchHead(client, defaultBranch), + ref: defaultBranch, + }; + }; + + /** `schema.gen.json`, else `meta.gen.json`, at one commit. */ + const schemaEntryAt = async (sha: string) => { + const paths = [ + decoPath(packagePath, "schema.gen.json"), + decoPath(packagePath, "meta.gen.json"), + ]; + const entries = await client.getEntriesAtPaths(sha, paths); + for (const path of paths) { + const entry = entries.get(path); + if (entry?.type === "blob") return entry; + } + return null; + }; + + /** The schema the bound branch sees, falling back to the default branch's. */ + const resolveSchema = async (head: { sha: string; ref: string }) => { + const entry = await schemaEntryAt(head.sha); + if (entry) return { entry, ref: head.ref }; + const defaultBranch = await client.getDefaultBranch(); + if (defaultBranch === head.ref) return null; + const fallback = await schemaEntryAt( + await requireBranchHead(client, defaultBranch), + ); + return fallback ? { entry: fallback, ref: defaultBranch } : null; + }; + + const readBlobText = async (stem: string, sha: string): Promise => { + const [file] = await resolveBlockContents(client, [{ stem, sha }]); + return file!.content; + }; + + return { + describe(): StorageDescription { + return { + kind: "git", + root: packagePath ?? ".", + readOnly: false, + refs: { default: branch, autoCreate: true }, + // Uploads go to Studio's own file storage, never into the repository. + assets: null, + idempotency: null, + }; + }, + + snapshot: ({ ref }) => + guarded(async (): Promise => { + checkRef(ref); + const head = await readHead(); + const tree = await client.listDecofileEntries(head.sha, packagePath); + if (tree.length === 0 && (await schemaEntryAt(head.sha)) === null) { + throw new StorageNotFoundError( + `no .deco folder in ${packagePath ?? "the repository root"}`, + ); + } + return { + revision: head.sha, + resolvedRef: head.ref, + files: blockEntriesInTree(tree, packagePath).map((entry) => ({ + file: entry.path.slice(blocksPrefix.length), + version: entry.sha, + size: entry.size, + })), + }; + }), + + readFiles: (snapshot, files) => + guarded(async () => { + const wanted = new Set(files); + const sources = snapshot.files.filter((f) => wanted.has(f.file)); + // A snapshot is a commit, so the bytes always match its versions. + const contents = await resolveBlockContents( + client, + sources.map((f) => ({ stem: f.file, sha: f.version })), + ); + const out: Record = {}; + sources.forEach((f, i) => { + out[f.file] = { text: contents[i]!.content, version: f.version }; + }); + return out; + }), + + readSchema: ({ ref }) => + guarded(async (): Promise => { + checkRef(ref); + const schema = await resolveSchema(await readHead()); + if (!schema) return null; + return { + version: schema.entry.sha, + text: await readBlobText(schema.entry.path, schema.entry.sha), + resolvedRef: schema.ref, + }; + }), + + readSecretsPublicKey: () => + guarded(async () => { + const head = await readHead(); + const path = decoPath(packagePath, "secrets.pub"); + const entry = (await client.getEntriesAtPaths(head.sha, [path])).get( + path, + ); + return entry?.type === "blob" + ? readBlobText(entry.path, entry.sha) + : null; + }), + + commit: (attempt) => + guarded(async (): Promise => { + checkRef(attempt.ref); + const base = attempt.base.revision; + try { + // The whole head is the guard: any commit since `base` is stale, + // which covers every per-file expectation the core asks for. + const current = await client.getBranch(branch); + if (current === null) { + await client.createBranch(branch, base); + } else if (current.sha !== base) { + return { status: "stale" }; + } + if (attempt.expectedSchemaVersion !== undefined) { + const schema = await resolveSchema({ sha: base, ref: branch }); + if ((schema?.entry.sha ?? null) !== attempt.expectedSchemaVersion) { + return { status: "stale" }; + } + } + + const tree = await client.listDecofileEntries(base, packagePath); + const present = new Set(attempt.base.files.map((f) => f.file)); + const nextBlocks = new Map( + blockEntriesInTree(tree, packagePath).map((e) => [ + e.path, + { stem: e.stem, sha: e.sha }, + ]), + ); + const changes: FileChange[] = []; + const versions: Record = {}; + for (const [file, content] of Object.entries(attempt.put)) { + const path = `${blocksPrefix}${file}`; + changes.push({ path, content }); + nextBlocks.set(path, { + stem: file.slice(0, -".json".length), + content, + }); + versions[file] = gitBlobSha(content); + await primeBlobCache(client.repo, content); + } + for (const file of attempt.delete) { + if (!present.has(file)) continue; + const path = `${blocksPrefix}${file}`; + changes.push({ path, deleted: true }); + nextBlocks.delete(path); + } + if (changes.length === 0) { + return { status: "committed", revision: base, versions }; + } + const gen = await regenerateGenArtifact({ + client, + tree, + packagePath, + branch, + nextBlocks: nextBlocks.values(), + memo: new Map(), + }); + if (gen) changes.push(gen); + + const { sha } = await client.commitFiles({ + branch, + message: appendCoAuthorTrailer( + commitMessage(attempt), + options.coAuthor, + ), + expectedHead: base, + changes, + }); + return { status: "committed", revision: sha, versions }; + } catch (error) { + if (error instanceof RepoWriteConflict) return { status: "stale" }; + throw error; + } + }), + }; +} From 2938627f1d9898f99c76e74cde2bb0470d104d38 Mon Sep 17 00:00:00 2001 From: gimenes Date: Sat, 3 Oct 2026 12:25:14 -0300 Subject: [PATCH 04/71] test(e2e): content-protocol conformance against the GitHub backend Runs the protocol's black-box conformance suite against the new endpoint over the GitHub stub, plus the flag gate, session auth, the bound branch, branch creation on first write and the tracked blocks.gen.json. The stub now hashes blobs the way git does, so versions computed from content agree with its tree. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WNwbSEePYNcY5YCgqZURig --- packages/e2e/fixtures/github-stub.ts | 7 +- .../e2e/tests/content-protocol-github.spec.ts | 269 ++++++++++++++++++ 2 files changed, 275 insertions(+), 1 deletion(-) create mode 100644 packages/e2e/tests/content-protocol-github.spec.ts diff --git a/packages/e2e/fixtures/github-stub.ts b/packages/e2e/fixtures/github-stub.ts index 5989b10e6c..053ab3572c 100644 --- a/packages/e2e/fixtures/github-stub.ts +++ b/packages/e2e/fixtures/github-stub.ts @@ -158,8 +158,13 @@ function repoKey(owner: string, name: string): string { return `${owner}/${name}`; } +/** Git's own blob id, so callers that hash content locally agree with the stub. */ function putBlob(repo: RepoState, content: string): string { - const sha = sha1(`blob:${content}`); + const bytes = Buffer.from(content, "utf-8"); + const sha = createHash("sha1") + .update(`blob ${bytes.length}\0`) + .update(bytes) + .digest("hex"); repo.blobs.set(sha, content); return sha; } diff --git a/packages/e2e/tests/content-protocol-github.spec.ts b/packages/e2e/tests/content-protocol-github.spec.ts new file mode 100644 index 0000000000..0b0b12227e --- /dev/null +++ b/packages/e2e/tests/content-protocol-github.spec.ts @@ -0,0 +1,269 @@ +/** + * The site editor's GitHub backend for next-major Blocks sites: the content + * protocol served over a project's repository. + * + * POST /api/:org/decofile/:virtualMcpId/:branch/rpc (session, org flag) + * GET /api/:org/decofile/:virtualMcpId/:branch/draft-grant (session, org flag) + * + * The protocol's own black-box conformance suite runs against the endpoint, + * with the repository on the local GitHub stub (see fixtures/fast-preview.ts). + * The cases below it cover what's specific to this backend: the flag, auth, + * the bound branch, branch creation on first write and the tracked + * `blocks.gen.json`. + */ + +import type { APIRequestContext } from "@playwright/test"; +import { + publicKeyPemFromDer, + type BlocksApplyResult, + type BlocksListResult, + type DescribeResult, +} from "@decocms/shared/blocks-protocol"; +import { runConformance } from "@decocms/shared/blocks-protocol/conformance"; +import { signUpViaApi } from "../fixtures/auth-api"; +import { + createFastPreviewProject, + type FastPreviewProject, + inspectStubRepo, + seedStubRepo, + uniqueOwner, +} from "../fixtures/fast-preview"; +import { callSelfMcpTool, findOrgId } from "../fixtures/mcp-tools"; +import { expect, getE2EAppOrigin, newApiContext, test } from "../fixtures/test"; + +const SECRET_BLOCK = "newsletter"; +const SECRET_FIELD = "apiKey"; + +/** A small deco-meta@1 schema with a block type that has a Secret field. */ +const schema = { + manifest: { + blocks: { + sections: { + hero: { $ref: "#/definitions/aGVybw==" }, + [SECRET_BLOCK]: { $ref: "#/definitions/bmV3c2xldHRlcg==" }, + }, + }, + }, + schema: { + definitions: { + "aGVybw==": { + type: "object", + properties: { title: { type: "string" } }, + }, + "bmV3c2xldHRlcg==": { + type: "object", + properties: { + listId: { type: "string" }, + [SECRET_FIELD]: { type: "string", format: "secret" }, + }, + }, + }, + }, +}; + +async function generatePublicKeyPem(): Promise { + const pair = (await crypto.subtle.generateKey( + { + name: "RSA-OAEP", + modulusLength: 2048, + publicExponent: new Uint8Array([1, 0, 1]), + hash: "SHA-256", + }, + true, + ["encrypt", "decrypt"], + )) as CryptoKeyPair; + return publicKeyPemFromDer( + new Uint8Array(await crypto.subtle.exportKey("spki", pair.publicKey)), + ); +} + +const rpcPath = (p: FastPreviewProject, branch: string) => + `/api/${p.org}/decofile/${p.vmcpId}/${branch}/rpc`; + +/** A fetch that goes through the Playwright context, so it carries the session. */ +function contextFetch(ctx: APIRequestContext) { + return async (request: Request): Promise => { + const body = + request.method === "GET" || request.method === "HEAD" + ? undefined + : Buffer.from(await request.arrayBuffer()); + const res = await ctx.fetch(request.url, { + method: request.method, + headers: Object.fromEntries(request.headers), + data: body, + maxRedirects: 0, + }); + // The context hands back the decoded body; Content-Encoding stays as sent. + const headers = new Headers(res.headers()); + headers.delete("content-length"); + return new Response(new Uint8Array(await res.body()), { + status: res.status(), + headers, + }); + }; +} + +async function rpc( + ctx: APIRequestContext, + path: string, + method: string, + params: unknown = {}, +): Promise<{ status: number; result?: T; error?: { code: number } }> { + const res = await ctx.post(path, { + data: { jsonrpc: "2.0", id: 1, method, params }, + }); + if (res.status() !== 200) return { status: res.status() }; + const body = (await res.json()) as { result?: T; error?: { code: number } }; + return { status: 200, ...body }; +} + +async function enableContentProtocol( + ctx: APIRequestContext, + orgSlug: string, +): Promise { + await callSelfMcpTool(ctx, orgSlug, "ORGANIZATION_SETTINGS_UPDATE", { + organizationId: await findOrgId(ctx, orgSlug), + flags: { site_editor_content_protocol: true }, + }); +} + +async function setUp( + ctx: APIRequestContext, + files: Record, +): Promise { + const user = await signUpViaApi(ctx); + const owner = uniqueOwner(); + const project = await createFastPreviewProject(ctx, user.orgSlug, { + owner, + repo: "site", + }); + await seedStubRepo(ctx, { + owner, + repo: "site", + defaultBranch: "main", + branches: { main: { files } }, + }); + return project; +} + +test.describe("content protocol on GitHub", () => { + test("passes the protocol conformance suite", async ({ playwright }) => { + test.setTimeout(10 * 60_000); + const ctx = await newApiContext(playwright); + try { + const publicKey = await generatePublicKeyPem(); + const project = await setUp(ctx, { + ".deco/schema.gen.json": JSON.stringify(schema), + ".deco/secrets.pub": publicKey, + ".deco/blocks/hero-home.json": '{"__resolveType":"hero"}\n', + }); + await enableContentProtocol(ctx, project.org); + + const report = await runConformance({ + endpoint: `${getE2EAppOrigin()}${rpcPath(project, "main")}`, + fetch: contextFetch(ctx), + secretField: { blockType: SECRET_BLOCK, field: SECRET_FIELD }, + secretsPublicKey: publicKey, + }); + const failures = report.outcomes.filter((o) => o.status === "failed"); + expect(failures, JSON.stringify(failures, null, 2)).toEqual([]); + expect(report.passed).toBeGreaterThan(20); + } finally { + await ctx.dispose(); + } + }); + + test("is off without the org flag, and needs a session", async ({ + playwright, + }) => { + const ctx = await newApiContext(playwright); + const anon = await newApiContext(playwright); + try { + const project = await setUp(ctx, { + ".deco/schema.gen.json": JSON.stringify(schema), + }); + const path = rpcPath(project, "main"); + const grantPath = path.replace(/\/rpc$/, "/draft-grant"); + + expect((await rpc(ctx, path, "describe")).status).toBe(404); + expect((await ctx.get(grantPath)).status()).toBe(404); + + await enableContentProtocol(ctx, project.org); + const described = await rpc(ctx, path, "describe"); + expect(described.result).toMatchObject({ + protocol: "deco-content", + server: { name: "studio-github" }, + kind: "git", + root: ".", + refs: { default: "main", autoCreate: true }, + assets: null, + preview: { origin: "https://site.example.com" }, + }); + const grant = await ctx.get(grantPath); + expect(grant.status()).toBe(200); + expect(await grant.json()).toMatchObject({ + token: expect.any(String), + apiHost: expect.any(String), + }); + + expect((await rpc(anon, path, "describe")).status).toBe(401); + expect((await anon.get(grantPath)).status()).toBe(401); + } finally { + await ctx.dispose(); + await anon.dispose(); + } + }); + + test("reads the default branch until the first write creates the branch", async ({ + playwright, + }) => { + const ctx = await newApiContext(playwright); + try { + const project = await setUp(ctx, { + ".deco/meta.gen.json": JSON.stringify(schema), + ".deco/blocks.gen.json": '{"hero-home":{"__resolveType":"hero"}}', + ".deco/blocks/hero-home.json": '{"__resolveType":"hero"}\n', + }); + await enableContentProtocol(ctx, project.org); + const path = rpcPath(project, "draft-1"); + + const before = await rpc(ctx, path, "blocks.list"); + expect(before.result).toMatchObject({ + resolvedRef: "main", + blocks: { "hero-home": { __resolveType: "hero" } }, + }); + expect( + (await rpc(ctx, path, "blocks.list", { ref: "main" })).error?.code, + ).toBe(-32006); + expect((await inspectStubRepo(ctx, project.owner, "site")).refs).toEqual({ + main: expect.any(String), + }); + + const applied = await rpc(ctx, path, "blocks.apply", { + set: { "pages-Home": { __resolveType: "hero", title: "Hi" } }, + }); + expect(applied.result?.revision).toEqual(expect.any(String)); + + const repo = await inspectStubRepo(ctx, project.owner, "site"); + expect(repo.refs["draft-1"]).toBe(applied.result?.revision); + expect(repo.refs["main"]).not.toBe(repo.refs["draft-1"]); + const files = repo.branches["draft-1"]!.files; + expect(files[".deco/blocks/pages-Home.json"]).toBe( + `${JSON.stringify({ __resolveType: "hero", title: "Hi" }, null, 2)}\n`, + ); + // The tracked merged artifact is regenerated in the same commit. + expect(JSON.parse(files[".deco/blocks.gen.json"]!)).toEqual({ + "hero-home": { __resolveType: "hero" }, + "pages-Home": { __resolveType: "hero", title: "Hi" }, + }); + + const after = await rpc(ctx, path, "blocks.list"); + expect(after.result).toMatchObject({ + resolvedRef: "draft-1", + revision: applied.result?.revision, + }); + } finally { + await ctx.dispose(); + } + }); +}); From a47113daed7a047327171c1de289b3dfbb72956c Mon Sep 17 00:00:00 2001 From: gimenes Date: Sat, 3 Oct 2026 12:26:39 -0300 Subject: [PATCH 05/71] feat(web): content backend seam for the content protocol useContentBackend decides, per project and branch, whether the editor reads and writes the legacy way or through the Blocks content protocol: a connected deco serve, or Studio's GitHub backend when a Fast Preview session's branch has a committed schema. Behind the org flag; pending while the flag or the probe loads, so a project never switches backends mid-read. On the protocol, useDecofile and useLiveMeta poll blocks.list and schema.get with ifNoneMatch at describe.pollIntervalMs and on focus, the write hooks send one blocks.apply each and adopt its revision, and a poll keeps entries still being saved. On GitHub the draft pointer comes from the new draft-grant route plus the list revision. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WNwbSEePYNcY5YCgqZURig --- .../sandbox/blocks/blocks-panel.tsx | 7 +- .../sections-editor/content-backend.test.ts | 126 +++++++++++++ .../sections-editor/content-backend.ts | 142 ++++++++++++++ .../sections-editor/content-protocol-api.ts | 140 ++++++++++++++ .../deco-serve-connection.test.ts | 43 +++++ .../sections-editor/deco-serve-connection.ts | 75 ++++++++ .../sections-editor/deco-serve-status.ts | 14 ++ .../sections-editor/use-content-backend.ts | 174 ++++++++++++++++++ .../sections-editor/use-decofile.ts | 41 ++++- .../sections-editor/use-delete-block.ts | 33 +++- .../sections-editor/use-live-meta.ts | 45 ++++- .../sections-editor/use-move-blocks.ts | 40 +++- .../sections-editor/use-save-block.ts | 35 +++- .../src/hooks/use-deco-serve-connection.ts | 48 +++++ apps/web/src/hooks/use-local-storage.ts | 7 +- .../src/hooks/use-organization-settings.ts | 13 ++ apps/web/src/lib/localstorage-keys.ts | 4 + apps/web/src/lib/query-keys.ts | 12 ++ 18 files changed, 971 insertions(+), 28 deletions(-) create mode 100644 apps/web/src/components/sections-editor/content-backend.test.ts create mode 100644 apps/web/src/components/sections-editor/content-backend.ts create mode 100644 apps/web/src/components/sections-editor/content-protocol-api.ts create mode 100644 apps/web/src/components/sections-editor/deco-serve-connection.test.ts create mode 100644 apps/web/src/components/sections-editor/deco-serve-connection.ts create mode 100644 apps/web/src/components/sections-editor/deco-serve-status.ts create mode 100644 apps/web/src/components/sections-editor/use-content-backend.ts create mode 100644 apps/web/src/hooks/use-deco-serve-connection.ts diff --git a/apps/web/src/components/sandbox/blocks/blocks-panel.tsx b/apps/web/src/components/sandbox/blocks/blocks-panel.tsx index a09336a597..b607867c9e 100644 --- a/apps/web/src/components/sandbox/blocks/blocks-panel.tsx +++ b/apps/web/src/components/sandbox/blocks/blocks-panel.tsx @@ -22,6 +22,7 @@ import { BlocksEmptyState, BlocksErrorState, } from "@/layouts/main-panel-tabs/blocks-tab-states"; +import { useContentBackend } from "@/components/sections-editor/use-content-backend"; import { PanelLoading } from "@/layouts/main-panel-boundary"; const SectionsEditor = lazy(() => @@ -67,6 +68,7 @@ export function BlocksPanel({ previewUrl, } : null; + const contentBackend = useContentBackend(virtualMcpId, currentBranch); const decofile = useDecofile(fetchParams, { fetchEnabled: devServerReady }); const meta = useLiveMeta(fetchParams, { fetchEnabled: devServerReady }); const state = resolveBlocksTabState({ @@ -74,7 +76,10 @@ export function BlocksPanel({ decofile: toBlocksQueryState(decofile), meta: toBlocksQueryState(meta), hasEditableContent: hasEditableDecoContent(decofile.data, meta.data), - fastPreviewActive: useSessionRuntime(virtualMcpId).runtime === "cms", + fastPreviewActive: + useSessionRuntime(virtualMcpId).runtime === "cms" || + contentBackend.kind === "protocol" || + contentBackend.kind === "unavailable", }); const panel = (children: ReactNode) => ( diff --git a/apps/web/src/components/sections-editor/content-backend.test.ts b/apps/web/src/components/sections-editor/content-backend.test.ts new file mode 100644 index 0000000000..72b2786ce3 --- /dev/null +++ b/apps/web/src/components/sections-editor/content-backend.test.ts @@ -0,0 +1,126 @@ +import { describe, expect, test } from "bun:test"; +import { + blockKeysOfWrite, + contentCapabilities, + mergePolledBlocks, + selectContentBackend, +} from "./content-backend"; + +describe("selectContentBackend", () => { + const base = { + flagEnabled: true, + hasServeConnection: false, + hasLocalTunnel: false, + runtime: "cms" as const, + githubSchema: "present" as const, + }; + + test("waits for the org flag", () => { + expect(selectContentBackend({ ...base, flagEnabled: undefined })).toBe( + "pending", + ); + }); + + test("the flag off is always legacy", () => { + expect( + selectContentBackend({ + ...base, + flagEnabled: false, + hasServeConnection: true, + }), + ).toBe("legacy"); + }); + + test("a connected deco serve wins, over the tunnel and any runtime", () => { + expect( + selectContentBackend({ + ...base, + hasServeConnection: true, + hasLocalTunnel: true, + runtime: "sandbox", + githubSchema: "absent", + }), + ).toBe("protocol-local"); + }); + + test("the legacy tunnel and sandbox sessions stay legacy", () => { + expect(selectContentBackend({ ...base, hasLocalTunnel: true })).toBe( + "legacy", + ); + expect(selectContentBackend({ ...base, runtime: "sandbox" })).toBe( + "legacy", + ); + }); + + test("a cms session follows the committed schema", () => { + expect(selectContentBackend(base)).toBe("protocol-github"); + expect(selectContentBackend({ ...base, githubSchema: "absent" })).toBe( + "legacy", + ); + expect(selectContentBackend({ ...base, githubSchema: "loading" })).toBe( + "pending", + ); + }); +}); + +describe("contentCapabilities", () => { + test("the protocol never runs site code", () => { + expect( + Object.values( + contentCapabilities({ + kind: "unavailable", + source: "local", + reason: "unreachable", + }), + ), + ).toEqual([false, false, false, false, false, false]); + expect(Object.values(contentCapabilities({ kind: "legacy" }))).toEqual([ + true, + true, + true, + true, + true, + true, + ]); + }); +}); + +describe("mergePolledBlocks", () => { + test("the remote map wins", () => { + expect( + mergePolledBlocks({ a: 1, b: 2 }, { a: 0, c: 3 }, new Set()), + ).toEqual({ a: 1, b: 2 }); + }); + + test("entries still being saved keep their local value", () => { + expect( + mergePolledBlocks( + { a: 1, b: 2 }, + { a: "editing", b: 2, d: "new" }, + new Set(["a", "d"]), + ), + ).toEqual({ a: "editing", b: 2, d: "new" }); + }); + + test("an entry being deleted stays deleted", () => { + expect( + mergePolledBlocks({ a: 1, gone: 2 }, { a: 1 }, new Set(["gone"])), + ).toEqual({ a: 1 }); + }); + + test("with no local copy, the remote map as is", () => { + const remote = { a: 1 }; + expect(mergePolledBlocks(remote, undefined, new Set(["a"]))).toBe(remote); + }); +}); + +describe("blockKeysOfWrite", () => { + test("reads the save, delete and move mutation shapes", () => { + expect(blockKeysOfWrite({ blockKey: "home", data: {} })).toEqual(["home"]); + expect( + blockKeysOfWrite({ writes: { b: {}, c: {} }, deletes: ["a", 1] }), + ).toEqual(["b", "c", "a"]); + expect(blockKeysOfWrite(undefined)).toEqual([]); + expect(blockKeysOfWrite("nope")).toEqual([]); + }); +}); diff --git a/apps/web/src/components/sections-editor/content-backend.ts b/apps/web/src/components/sections-editor/content-backend.ts new file mode 100644 index 0000000000..18334e22a4 --- /dev/null +++ b/apps/web/src/components/sections-editor/content-backend.ts @@ -0,0 +1,142 @@ +/** + * Where the site editor reads and writes a project's content. + * + * - `legacy`: the running site (`/live/_meta`, `/.decofile`), the sandbox + * working tree, or the Fast Preview decofile API — everything that existed + * before next-major Blocks. + * - `protocol`: the Blocks content protocol, which needs only the committed + * schema and `.deco/blocks` and never runs the site's code. Served either by + * a `deco serve` on the editor's machine (`local`) or by Studio's GitHub + * backend (`github`). + * + * Pure: the selection and the poll merge are unit-tested without mocks. + */ + +import type { + ContentClient, + DescribeResult, + SchemaGetResult, +} from "@decocms/shared/blocks-protocol"; + +export type ContentSource = "github" | "local"; + +export interface ProtocolBackend { + kind: "protocol"; + source: ContentSource; + client: ContentClient; + describe: DescribeResult; + /** The first `schema.get`, read while probing, so the editor needn't read it twice. */ + schema: Extract | null; + /** Distinguishes cache entries of different endpoints for one project. */ + cacheKeySuffix: string; +} + +export type ContentBackend = + /** Not decided yet: the flag or the probe is still loading. */ + | { kind: "pending" } + | { kind: "legacy" } + | ProtocolBackend + /** A connected `deco serve` that can't be used right now. */ + | { + kind: "unavailable"; + source: "local"; + reason: "unauthorized" | "unreachable"; + }; + +/** What the editor may do on a backend; the protocol never runs site code. */ +export interface ContentCapabilities { + /** Re-render the preview in place through `/live/previews`. */ + inPlaceRender: boolean; + /** Gallery thumbnails and block previews rendered by the site. */ + livePreviews: boolean; + /** Pickers whose options come from running a loader. */ + invoke: boolean; + /** The Run button and runnable (saved loader/action) blocks. */ + runBlocks: boolean; + /** Installing commerce apps from the editor. */ + installApps: boolean; + /** Encrypting v7 secrets through the site's encrypt action. */ + v7Secrets: boolean; +} + +export function contentCapabilities( + backend: ContentBackend, +): ContentCapabilities { + // A connected `deco serve` that's down is still a protocol project. + const legacy = backend.kind === "legacy" || backend.kind === "pending"; + return { + inPlaceRender: legacy, + livePreviews: legacy, + invoke: legacy, + runBlocks: legacy, + installApps: legacy, + v7Secrets: legacy, + }; +} + +export type BackendDecision = + | "pending" + | "legacy" + | "protocol-local" + | "protocol-github"; + +/** + * Which backend a project's editor uses. A connected `deco serve` wins, then + * the legacy Local tunnel; a Fast Preview (`cms`) session uses the GitHub + * backend when the branch has a committed schema. Sandbox sessions stay + * legacy: their pod's working tree shares the branch, and commits from here + * would make the two diverge. + */ +export function selectContentBackend(input: { + /** The org flag; `undefined` while the org settings load. */ + flagEnabled: boolean | undefined; + hasServeConnection: boolean; + hasLocalTunnel: boolean; + runtime: "cms" | "sandbox"; + /** The GitHub probe: does the branch have a committed schema? */ + githubSchema: "present" | "absent" | "loading"; +}): BackendDecision { + if (input.flagEnabled === undefined) return "pending"; + if (!input.flagEnabled) return "legacy"; + if (input.hasServeConnection) return "protocol-local"; + if (input.hasLocalTunnel || input.runtime !== "cms") return "legacy"; + if (input.githubSchema === "loading") return "pending"; + return input.githubSchema === "present" ? "protocol-github" : "legacy"; +} + +/** + * Applies a polled block map over the local copy: the remote map wins, except + * for entries the editor is still saving, which keep their local value (or + * stay deleted) until their own write lands. + */ +export function mergePolledBlocks( + remote: Record, + local: Record | undefined, + saving: ReadonlySet, +): Record { + if (!local || saving.size === 0) return remote; + const next = { ...remote }; + for (const key of saving) { + if (Object.hasOwn(local, key)) next[key] = local[key]; + else delete next[key]; + } + return next; +} + +/** + * The block names an in-flight decofile write touches, from the variables of + * the write mutations (`{ blockKey }` or `{ writes, deletes }`). + */ +export function blockKeysOfWrite(variables: unknown): string[] { + if (!variables || typeof variables !== "object") return []; + const v = variables as Record; + const keys: string[] = []; + if (typeof v.blockKey === "string") keys.push(v.blockKey); + if (v.writes && typeof v.writes === "object") { + keys.push(...Object.keys(v.writes)); + } + if (Array.isArray(v.deletes)) { + keys.push(...v.deletes.filter((k): k is string => typeof k === "string")); + } + return keys; +} diff --git a/apps/web/src/components/sections-editor/content-protocol-api.ts b/apps/web/src/components/sections-editor/content-protocol-api.ts new file mode 100644 index 0000000000..9628bc157d --- /dev/null +++ b/apps/web/src/components/sections-editor/content-protocol-api.ts @@ -0,0 +1,140 @@ +/** + * The editor's reads and writes over the content protocol. The data hooks + * (`useDecofile`, `useLiveMeta`, the write hooks) call these when the project + * has a protocol backend, and keep their legacy paths otherwise. + * + * Polling follows the protocol: each read carries the revision (or schema + * version) it last saw and gets a short "not modified" when nothing changed; + * after a write the client adopts the returned revision, so its next poll is + * "not modified" unless someone else wrote. + */ + +import type { QueryClient } from "@tanstack/react-query"; +import { KEYS } from "@/lib/query-keys"; +import { + blockKeysOfWrite, + mergePolledBlocks, + type ProtocolBackend, +} from "./content-backend"; +import { + type DecofilePatchBody, + type DecofileScopeParams, + decofileWriteMutationKey, + setDecofileDraft, + throwResponseError, +} from "./decofile-api"; +import type { LiveMeta } from "./resolve-schema"; + +/** The block names the project's in-flight writes touch. */ +function savingBlockKeys( + queryClient: QueryClient, + params: DecofileScopeParams, +): Set { + const pending = queryClient.getMutationCache().findAll({ + mutationKey: decofileWriteMutationKey( + params.orgSlug, + params.virtualMcpId, + params.branch, + ), + status: "pending", + }); + return new Set(pending.flatMap((m) => blockKeysOfWrite(m.state.variables))); +} + +/** + * Refreshes the `?__draft=` pointer for a GitHub backend: the grant comes + * from Studio, the version is the branch head the protocol reported. + */ +async function refreshDraftPointer( + queryClient: QueryClient, + backend: ProtocolBackend, + params: DecofileScopeParams, + revision: string, +): Promise { + if (backend.source !== "github") return; + const res = await fetch( + `/api/${params.orgSlug}/decofile/${encodeURIComponent(params.virtualMcpId)}/${encodeURIComponent(params.branch)}/draft-grant`, + { cache: "no-store" }, + ); + if (!res.ok) return throwResponseError(res, "Draft grant"); + const grant = (await res.json()) as { token: string; apiHost?: string }; + setDecofileDraft(queryClient, params, { + version: revision, + token: grant.token, + apiHost: grant.apiHost ?? window.location.host, + }); +} + +/** `blocks.list`, conditional on the revision this cache entry last saw. */ +export async function readProtocolBlocks( + queryClient: QueryClient, + backend: ProtocolBackend, + params: DecofileScopeParams, + cacheKey: string, +): Promise> { + const current = queryClient.getQueryData>( + KEYS.decofile(cacheKey), + ); + const revisionKey = KEYS.contentRevision(cacheKey); + const seen = queryClient.getQueryData(revisionKey); + const result = await backend.client.blocksList( + current && seen ? { ifNoneMatch: seen } : {}, + ); + if (result.notModified && current) return current; + if (result.notModified) { + // The revision outlived the map it described: read it whole. + queryClient.removeQueries({ queryKey: revisionKey }); + return readProtocolBlocks(queryClient, backend, params, cacheKey); + } + queryClient.setQueryData(revisionKey, result.revision); + await refreshDraftPointer(queryClient, backend, params, result.revision); + return mergePolledBlocks( + result.blocks, + current, + savingBlockKeys(queryClient, params), + ); +} + +/** One `blocks.apply`; adopts the returned revision. */ +export async function applyProtocolPatch( + queryClient: QueryClient, + backend: ProtocolBackend, + params: DecofileScopeParams, + cacheKey: string, + patch: DecofilePatchBody, +): Promise<{ revision: string }> { + const result = await backend.client.blocksApply({ + set: patch.set as Record> | undefined, + delete: patch.delete, + }); + queryClient.setQueryData(KEYS.contentRevision(cacheKey), result.revision); + await refreshDraftPointer(queryClient, backend, params, result.revision); + return { revision: result.revision }; +} + +/** `schema.get`, conditional on the version this cache entry last saw. */ +export async function readProtocolMeta( + queryClient: QueryClient, + backend: ProtocolBackend, + queryKey: readonly unknown[], + versionKey: string, +): Promise { + const current = queryClient.getQueryData(queryKey); + const revisionKey = KEYS.contentRevision(versionKey); + const seen = queryClient.getQueryData(revisionKey); + // The probe already read the schema once. + if (!current && !seen && backend.schema) { + queryClient.setQueryData(revisionKey, backend.schema.version); + return backend.schema.schema as LiveMeta; + } + const result = await backend.client.schemaGet( + current && seen ? { ifNoneMatch: seen } : {}, + ); + if (result.notModified && current) return current; + if (result.notModified) { + queryClient.removeQueries({ queryKey: revisionKey }); + return readProtocolMeta(queryClient, backend, queryKey, versionKey); + } + queryClient.setQueryData(revisionKey, result.version); + return result.schema as LiveMeta; +} diff --git a/apps/web/src/components/sections-editor/deco-serve-connection.test.ts b/apps/web/src/components/sections-editor/deco-serve-connection.test.ts new file mode 100644 index 0000000000..bc89334828 --- /dev/null +++ b/apps/web/src/components/sections-editor/deco-serve-connection.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, test } from "bun:test"; +import { + parseConnectFragment, + parseStoredConnection, +} from "./deco-serve-connection"; + +describe("parseConnectFragment", () => { + test("reads the endpoint and token deco serve prints", () => { + expect( + parseConnectFragment( + `#endpoint=${encodeURIComponent("http://127.0.0.1:4545/rpc")}&token=abc123`, + ), + ).toEqual({ endpoint: "http://127.0.0.1:4545/rpc", token: "abc123" }); + }); + + test("refuses incomplete or unsafe links", () => { + expect(parseConnectFragment("")).toBeNull(); + expect(parseConnectFragment("#token=abc")).toBeNull(); + expect( + parseConnectFragment("#endpoint=http://127.0.0.1:4545/rpc"), + ).toBeNull(); + expect( + parseConnectFragment("#endpoint=javascript:alert(1)&token=abc"), + ).toBeNull(); + expect(parseConnectFragment("#endpoint=not a url&token=abc")).toBeNull(); + expect( + parseConnectFragment( + `#endpoint=http://127.0.0.1/rpc&token=${"x".repeat(2000)}`, + ), + ).toBeNull(); + }); +}); + +describe("parseStoredConnection", () => { + test("accepts only a well-formed connection", () => { + expect( + parseStoredConnection({ endpoint: "http://127.0.0.1:1/rpc", token: "t" }), + ).toEqual({ endpoint: "http://127.0.0.1:1/rpc", token: "t" }); + expect(parseStoredConnection(null)).toBeNull(); + expect(parseStoredConnection("http://127.0.0.1:1/rpc")).toBeNull(); + expect(parseStoredConnection({ endpoint: "x", token: "" })).toBeNull(); + }); +}); diff --git a/apps/web/src/components/sections-editor/deco-serve-connection.ts b/apps/web/src/components/sections-editor/deco-serve-connection.ts new file mode 100644 index 0000000000..709b0c4f1e --- /dev/null +++ b/apps/web/src/components/sections-editor/deco-serve-connection.ts @@ -0,0 +1,75 @@ +/** + * A connection to `deco serve`, the content-protocol server the Blocks CLI + * runs on the editor's machine. The CLI prints a connect link, + * `/connect#endpoint=&token=`; the token travels in the + * fragment so it never reaches logs or `Referer` headers. + */ + +import { z } from "zod"; + +export const DecoServeConnectionSchema = z.object({ + /** The protocol endpoint, such as `http://127.0.0.1:4545/rpc`. */ + endpoint: z + .string() + .max(2048) + .refine((value) => { + try { + const url = new URL(value); + return url.protocol === "http:" || url.protocol === "https:"; + } catch { + return false; + } + }), + token: z.string().min(1).max(1024), +}); + +export type DecoServeConnection = z.infer; + +/** Reads a connect link's fragment; `null` when it isn't a valid one. */ +export function parseConnectFragment(hash: string): DecoServeConnection | null { + const params = new URLSearchParams(hash.replace(/^#/, "")); + const parsed = DecoServeConnectionSchema.safeParse({ + endpoint: params.get("endpoint") ?? "", + token: params.get("token") ?? "", + }); + return parsed.success ? parsed.data : null; +} + +/** A stored connection, or `null` for anything else in storage. */ +export function parseStoredConnection( + value: unknown, +): DecoServeConnection | null { + const parsed = DecoServeConnectionSchema.safeParse(value); + return parsed.success ? parsed.data : null; +} + +const PENDING_KEY = "studio:deco-serve-pending"; + +/** + * Holds a connect link's connection for this tab until a project is picked, + * so it survives the login redirect (which keeps no URL fragment). + */ +export function stashPendingConnection(connection: DecoServeConnection): void { + try { + sessionStorage.setItem(PENDING_KEY, JSON.stringify(connection)); + } catch { + // Storage blocked: the flow then asks for the link again. + } +} + +export function readPendingConnection(): DecoServeConnection | null { + try { + const raw = sessionStorage.getItem(PENDING_KEY); + return raw ? parseStoredConnection(JSON.parse(raw)) : null; + } catch { + return null; + } +} + +export function clearPendingConnection(): void { + try { + sessionStorage.removeItem(PENDING_KEY); + } catch { + // Nothing to clear. + } +} diff --git a/apps/web/src/components/sections-editor/deco-serve-status.ts b/apps/web/src/components/sections-editor/deco-serve-status.ts new file mode 100644 index 0000000000..7b0d26f78a --- /dev/null +++ b/apps/web/src/components/sections-editor/deco-serve-status.ts @@ -0,0 +1,14 @@ +import { + ContentProtocolError, + ErrorCode, +} from "@decocms/shared/blocks-protocol"; + +/** Why a `deco serve` request failed: a bad token (the server restarted) or no server. */ +export function decoServeErrorReason( + error: unknown, +): "unauthorized" | "unreachable" { + return error instanceof ContentProtocolError && + error.code === ErrorCode.Unauthorized + ? "unauthorized" + : "unreachable"; +} diff --git a/apps/web/src/components/sections-editor/use-content-backend.ts b/apps/web/src/components/sections-editor/use-content-backend.ts new file mode 100644 index 0000000000..6118c335fa --- /dev/null +++ b/apps/web/src/components/sections-editor/use-content-backend.ts @@ -0,0 +1,174 @@ +import { useQuery } from "@tanstack/react-query"; +import { + assertSupportedEndpoint, + type ContentClient, + createContentClient, + type DescribeResult, + ErrorCode, + type SchemaGetResult, +} from "@decocms/shared/blocks-protocol"; +import { useProjectContext } from "@/sdk"; +import { useDecoServeConnection } from "@/hooks/use-deco-serve-connection"; +import { useLocalPreviewUrl } from "@/hooks/use-local-preview-url"; +import { useOrgFlagState } from "@/hooks/use-organization-settings"; +import { useSessionRuntime } from "@/hooks/use-session-runtime"; +import { KEYS } from "@/lib/query-keys"; +import { + type ContentBackend, + type ProtocolBackend, + selectContentBackend, +} from "./content-backend"; +import { decoServeErrorReason } from "./deco-serve-status"; + +type FullSchema = Extract; + +interface Probe { + client: ContentClient; + describe: DescribeResult; + schema: FullSchema | null; +} + +/** The Studio GitHub backend's endpoint for one project and branch. */ +export function githubContentEndpoint(params: { + orgSlug: string; + virtualMcpId: string; + branch: string; +}): string { + return new URL( + `/api/${params.orgSlug}/decofile/${encodeURIComponent(params.virtualMcpId)}/${encodeURIComponent(params.branch)}/rpc`, + window.location.origin, + ).href; +} + +/** `describe` plus the schema, in one request. */ +async function probe(client: ContentClient): Promise { + const [described, schema] = await client.batch([ + { method: "describe" }, + { method: "schema.get" }, + ]); + if (!described?.ok) throw described?.error ?? new Error("no describe"); + const describe = assertSupportedEndpoint(described.result as DescribeResult); + if (schema?.ok) { + return { client, describe, schema: schema.result as FullSchema }; + } + if (schema?.error.code === ErrorCode.NotFound) { + return { client, describe, schema: null }; + } + throw schema?.error ?? new Error("no schema.get"); +} + +/** + * The content backend of a project's editor (see `selectContentBackend`). + * `pending` while the org flag or the backend probe loads: callers hold their + * reads until then, so a project never reads from one backend and then + * switches to another. + */ +export function useContentBackend( + virtualMcpId: string | null | undefined, + branch: string | null | undefined, +): ContentBackend { + const { org } = useProjectContext(); + const flagEnabled = useOrgFlagState("site_editor_content_protocol"); + const { connection } = useDecoServeConnection(virtualMcpId); + const { url: tunnel } = useLocalPreviewUrl(virtualMcpId); + const { runtime } = useSessionRuntime(virtualMcpId); + + const githubEnabled = + !!flagEnabled && + !connection && + !tunnel && + runtime === "cms" && + !!virtualMcpId && + !!branch; + const github = useQuery({ + queryKey: KEYS.contentBackend( + org.slug, + virtualMcpId ?? "", + branch ?? "", + "github", + ), + queryFn: async (): Promise => { + const client = createContentClient({ + endpoint: githubContentEndpoint({ + orgSlug: org.slug, + virtualMcpId: virtualMcpId!, + branch: branch!, + }), + }); + const result = await probe(client); + // No committed schema: a legacy site. + return result.schema ? result : null; + }, + enabled: githubEnabled, + staleTime: Number.POSITIVE_INFINITY, + retry: 2, + }); + + const localEnabled = !!flagEnabled && !!connection; + const local = useQuery({ + queryKey: KEYS.contentBackend( + org.slug, + virtualMcpId ?? "", + "local", + connection?.endpoint ?? "", + connection?.token ?? "", + ), + queryFn: () => + probe( + createContentClient({ + endpoint: connection!.endpoint, + token: connection!.token, + }), + ), + enabled: localEnabled, + staleTime: Number.POSITIVE_INFINITY, + retry: false, + // Reconnects on its own once `deco serve` is (re)started. + refetchInterval: (query) => + query.state.status === "error" ? 5_000 : false, + }); + + const decision = selectContentBackend({ + flagEnabled, + hasServeConnection: !!connection, + hasLocalTunnel: !!tunnel, + runtime, + githubSchema: github.data + ? "present" + : github.data === null || github.isError + ? "absent" + : "loading", + }); + + if (decision === "pending") return { kind: "pending" }; + if (decision === "legacy") return { kind: "legacy" }; + if (decision === "protocol-github") { + return toBackend("github", github.data!, ""); + } + if (local.data) { + return toBackend("local", local.data, `:serve:${connection!.endpoint}`); + } + if (local.isError) { + return { + kind: "unavailable", + source: "local", + reason: decoServeErrorReason(local.error), + }; + } + return { kind: "pending" }; +} + +function toBackend( + source: ProtocolBackend["source"], + probed: Probe, + cacheKeySuffix: string, +): ProtocolBackend { + return { + kind: "protocol", + source, + client: probed.client, + describe: probed.describe, + schema: probed.schema, + cacheKeySuffix, + }; +} diff --git a/apps/web/src/components/sections-editor/use-decofile.ts b/apps/web/src/components/sections-editor/use-decofile.ts index cba854011d..ce4582d492 100644 --- a/apps/web/src/components/sections-editor/use-decofile.ts +++ b/apps/web/src/components/sections-editor/use-decofile.ts @@ -10,6 +10,8 @@ import { buildDecofileFetchUrl } from "./preview-fetch-url"; import { readCommittedJson } from "./read-committed-file"; import { decofileErrorStatus } from "./decofile-read-status"; import { usePackagePath } from "./use-package-path"; +import { useContentBackend } from "./use-content-backend"; +import { readProtocolBlocks } from "./content-protocol-api"; interface UseDecofileParams { orgSlug: string; @@ -32,13 +34,28 @@ export function decofileCacheKey(input: { virtualMcpId: string; branch: string; localPreviewUrl?: string | null; + /** A content-protocol backend's suffix (a `deco serve` endpoint), if any. */ + protocolSuffix?: string | null; }): string { const base = `${input.orgSlug}/${input.virtualMcpId}/${input.branch}`; + if (input.protocolSuffix) return `${base}${input.protocolSuffix}`; return input.localPreviewUrl ? `${base}:local:${input.localPreviewUrl}` : base; } +/** The decofile cache key for a project, given its content backend. */ +export function useDecofileCacheKey( + params: { orgSlug: string; virtualMcpId: string; branch: string } | null, +): string { + const { url: localPreviewUrl } = useLocalPreviewUrl(params?.virtualMcpId); + const backend = useContentBackend(params?.virtualMcpId, params?.branch); + if (!params) return ""; + return backend.kind === "protocol" + ? decofileCacheKey({ ...params, protocolSuffix: backend.cacheKeySuffix }) + : decofileCacheKey({ ...params, localPreviewUrl }); +} + export function useDecofile( params: UseDecofileParams | null, options?: { fetchEnabled?: boolean }, @@ -51,7 +68,10 @@ export function useDecofile( */ const { url: localPreviewUrl } = useLocalPreviewUrl(params?.virtualMcpId); const localOverride = !!localPreviewUrl; - const key = params ? decofileCacheKey({ ...params, localPreviewUrl }) : ""; + // Next-major Blocks sites: the content protocol, polled (see content-backend). + const backend = useContentBackend(params?.virtualMcpId, params?.branch); + const protocol = backend.kind === "protocol" ? backend : null; + const key = useDecofileCacheKey(params); // `fetchEnabled` means the dev server is up, so the live `/.decofile` route is // worth hitting. When it's down we read `.deco/blocks.gen.json` straight from // the working tree — and if that artifact is absent (it's commonly gitignored) @@ -82,6 +102,14 @@ export function useDecofile( return useQuery({ queryKey: KEYS.decofile(key), queryFn: async () => { + if (protocol) { + return readProtocolBlocks(queryClient, protocol, params!, key); + } + if (backend.kind === "unavailable") { + const err = new Error("decofile unavailable (deco serve unreachable)"); + (err as { status?: number }).status = 502; + throw err; + } if (localOverride) { const res = await fetch( new URL("/.decofile", localPreviewUrl).toString(), @@ -142,8 +170,11 @@ export function useDecofile( }); throw err; }, - enabled: !!params, - staleTime: 30_000, + enabled: !!params && backend.kind !== "pending", + staleTime: protocol ? 0 : 30_000, + refetchInterval: protocol?.describe.pollIntervalMs, + refetchIntervalInBackground: false, + refetchOnWindowFocus: protocol ? "always" : undefined, // 502 = preview unreachable / nothing available yet. Retrying just hammers // a known-down endpoint; sandbox-events-context re-invalidates this query // on first daemon contact (and again when the dev server reports @@ -158,7 +189,9 @@ export function useDecofile( // 404 = no decofile route on this repo (not a deco site) — as terminal as // 502 for retry purposes. retry: (failureCount, error) => { - if (fastPreviewActive || localOverride) return failureCount < 3; + if (protocol || fastPreviewActive || localOverride) { + return failureCount < 3; + } const status = (error as { status?: number }).status; return status !== 502 && status !== 404 && failureCount < 2; }, diff --git a/apps/web/src/components/sections-editor/use-delete-block.ts b/apps/web/src/components/sections-editor/use-delete-block.ts index 6451b2292a..37b764a99a 100644 --- a/apps/web/src/components/sections-editor/use-delete-block.ts +++ b/apps/web/src/components/sections-editor/use-delete-block.ts @@ -1,7 +1,6 @@ import { useMutation, useQueryClient } from "@tanstack/react-query"; import { useSessionRuntime } from "@/hooks/use-session-runtime"; import { useLocalPreviewUrl } from "@/hooks/use-local-preview-url"; -import { decofileCacheKey } from "./use-decofile"; import { usePackagePath } from "./use-package-path"; import { KEYS } from "@/lib/query-keys"; import { decoBlockFilePath } from "./deco-block-key"; @@ -15,6 +14,9 @@ import { } from "./decofile-api"; import { sandboxGitStatusQueryKey } from "../thread/repository/sandbox-git-api"; import { useOptionalChatTask } from "@/components/chat/chat-context"; +import { useContentBackend } from "./use-content-backend"; +import { applyProtocolPatch } from "./content-protocol-api"; +import { decofileCacheKey, useDecofileCacheKey } from "./use-decofile"; import { buildSandboxUrl } from "@/sdk/sandbox-url"; interface UseDeleteBlockParams { @@ -47,17 +49,42 @@ export function useDeleteBlock({ * reading it without the tunnel patched a decofile nobody is rendering. */ const { url: localPreviewUrl } = useLocalPreviewUrl(virtualMcpId); - const cacheKey = decofileCacheKey({ + const backend = useContentBackend(virtualMcpId, branch); + const protocol = backend.kind === "protocol" ? backend : null; + const protocolCacheKey = useDecofileCacheKey({ orgSlug, virtualMcpId, branch, - localPreviewUrl, }); + const cacheKey = protocol + ? protocolCacheKey + : decofileCacheKey({ orgSlug, virtualMcpId, branch, localPreviewUrl }); return useMutation({ mutationKey: decofileWriteMutationKey(orgSlug, virtualMcpId, branch), scope: decofileWriteScope(orgSlug, virtualMcpId, branch), mutationFn: async ({ blockKey }: { blockKey: string }) => { + if (protocol) { + await applyProtocolPatch( + queryClient, + protocol, + { orgSlug, virtualMcpId, branch }, + cacheKey, + { delete: [blockKey] }, + ); + if (protocol.source === "github") { + // The commit moved the branch head; refresh the header's branch meta. + await queryClient.invalidateQueries({ + queryKey: sandboxGitStatusQueryKey({ + orgSlug, + virtualMcpId, + branch, + threadId, + }), + }); + } + return { ok: true as const, existed: true }; + } // Local: no persistence — the optimistic cache removal is the delete. if (localPreviewUrl) return { ok: true as const, existed: true }; if (fastPreviewActive) { diff --git a/apps/web/src/components/sections-editor/use-live-meta.ts b/apps/web/src/components/sections-editor/use-live-meta.ts index 27ca22f723..3b9409da44 100644 --- a/apps/web/src/components/sections-editor/use-live-meta.ts +++ b/apps/web/src/components/sections-editor/use-live-meta.ts @@ -1,4 +1,4 @@ -import { type Query, useQuery } from "@tanstack/react-query"; +import { type Query, useQuery, useQueryClient } from "@tanstack/react-query"; import { exponentialBackoffWithJitter } from "@decocms/shared/std"; import { KEYS } from "@/lib/query-keys"; import { decoRepoPath } from "./deco-repo-path"; @@ -10,6 +10,8 @@ import { useSessionRuntime } from "@/hooks/use-session-runtime"; import { useLocalPreviewUrl } from "@/hooks/use-local-preview-url"; import { usePackagePath } from "./use-package-path"; import type { LiveMeta } from "./resolve-schema"; +import { useContentBackend } from "./use-content-backend"; +import { readProtocolMeta } from "./content-protocol-api"; interface UseLiveMetaParams { orgSlug: string; @@ -114,11 +116,36 @@ export function useLiveMeta( params?.virtualMcpId, ); const fastPreviewActive = !localOverride && runtime === "cms"; + // Next-major Blocks sites read the committed schema over the content + // protocol: never from the running site. + const backend = useContentBackend(params?.virtualMcpId, params?.branch); + const protocol = backend.kind === "protocol" ? backend : null; + const queryClient = useQueryClient(); + const queryKey = params + ? protocol + ? liveMetaQueryKey({ + ...params, + previewUrl: `protocol${protocol.cacheKeySuffix}`, + productionUrl: null, + }) + : liveMetaQueryKey({ ...params, previewUrl, productionUrl }) + : KEYS.liveMeta(""); return useQuery({ - queryKey: params - ? liveMetaQueryKey({ ...params, previewUrl, productionUrl }) - : KEYS.liveMeta(""), + queryKey, queryFn: async () => { + if (protocol) { + return readProtocolMeta( + queryClient, + protocol, + queryKey, + `schema:${queryKey.join("/")}`, + ); + } + if (backend.kind === "unavailable") { + const err = new Error("live meta unavailable (deco serve unreachable)"); + (err as { status?: number }).status = 502; + throw err; + } const fetchMeta = async (baseUrl: string): Promise => { const url = new URL("/live/_meta", baseUrl).href; const res = await fetch(url, { cache: "no-store" }).catch(() => null); @@ -172,10 +199,12 @@ export function useLiveMeta( (err as { status?: number }).status = 502; throw err; }, - enabled: !!params, - refetchInterval: options?.refetchInterval, + enabled: !!params && backend.kind !== "pending", + refetchInterval: protocol + ? protocol.describe.pollIntervalMs + : options?.refetchInterval, refetchIntervalInBackground: false, - staleTime: 300_000, + staleTime: protocol ? 0 : 300_000, // 502 = nothing reachable yet. The sandbox lifecycle re-invalidates this // query when the dev server comes up (see sandbox-events-context), so // retrying just hammers a known-down endpoint. Sandbox-less Fast Preview @@ -184,7 +213,7 @@ export function useLiveMeta( // retries ARE the recovery there. // Local mode, like sandbox-less Fast Preview, has no lifecycle event to re-invalidate a 502, so bounded retries are its recovery. retry: (failureCount, error) => - fastPreviewActive || localOverride + protocol || fastPreviewActive || localOverride ? failureCount < 3 : (error as { status?: number }).status !== 502 && failureCount < 3, retryDelay: (attempt) => diff --git a/apps/web/src/components/sections-editor/use-move-blocks.ts b/apps/web/src/components/sections-editor/use-move-blocks.ts index 56aead1c8e..01fd5af213 100644 --- a/apps/web/src/components/sections-editor/use-move-blocks.ts +++ b/apps/web/src/components/sections-editor/use-move-blocks.ts @@ -1,7 +1,6 @@ import { useMutation, useQueryClient } from "@tanstack/react-query"; import { useSessionRuntime } from "@/hooks/use-session-runtime"; import { useLocalPreviewUrl } from "@/hooks/use-local-preview-url"; -import { decofileCacheKey } from "./use-decofile"; import { usePackagePath } from "./use-package-path"; import { KEYS } from "@/lib/query-keys"; import { sanitizeSecretsForPersistence } from "@decocms/shared/decofile"; @@ -16,6 +15,9 @@ import { } from "./decofile-api"; import { sandboxGitStatusQueryKey } from "../thread/repository/sandbox-git-api"; import { useOptionalChatTask } from "@/components/chat/chat-context"; +import { useContentBackend } from "./use-content-backend"; +import { applyProtocolPatch } from "./content-protocol-api"; +import { decofileCacheKey, useDecofileCacheKey } from "./use-decofile"; import { buildSandboxUrl } from "@/sdk/sandbox-url"; interface UseMoveBlocksParams { @@ -63,15 +65,45 @@ export function useMoveBlocks({ * nothing reads AND tried a real write against the branch. */ const { url: localPreviewUrl } = useLocalPreviewUrl(virtualMcpId); - const queryKey = KEYS.decofile( - decofileCacheKey({ orgSlug, virtualMcpId, branch, localPreviewUrl }), - ); + const backend = useContentBackend(virtualMcpId, branch); + const protocol = backend.kind === "protocol" ? backend : null; + const protocolCacheKey = useDecofileCacheKey({ + orgSlug, + virtualMcpId, + branch, + }); + const cacheKey = protocol + ? protocolCacheKey + : decofileCacheKey({ orgSlug, virtualMcpId, branch, localPreviewUrl }); + const queryKey = KEYS.decofile(cacheKey); const mutation = useMutation({ mutationKey: decofileWriteMutationKey(orgSlug, virtualMcpId, branch), scope: decofileWriteScope(orgSlug, virtualMcpId, branch), mutationFn: async ({ writes: rawWrites, deletes }: BlockMove) => { const writes = sanitizeSecretsForPersistence(rawWrites); + if (protocol) { + // One `blocks.apply`: the write and the delete land together. + const applied = await applyProtocolPatch( + queryClient, + protocol, + { orgSlug, virtualMcpId, branch }, + cacheKey, + { set: writes, delete: deletes.filter((key) => !(key in writes)) }, + ); + if (protocol.source === "github") { + // The commit moved the branch head; refresh the header's branch meta. + await queryClient.invalidateQueries({ + queryKey: sandboxGitStatusQueryKey({ + orgSlug, + virtualMcpId, + branch, + threadId, + }), + }); + } + return applied; + } // Local: no persistence — the optimistic cache write is the save. if (localPreviewUrl) return { ok: true as const }; if (fastPreviewActive) { diff --git a/apps/web/src/components/sections-editor/use-save-block.ts b/apps/web/src/components/sections-editor/use-save-block.ts index 79d3b45708..e923d66560 100644 --- a/apps/web/src/components/sections-editor/use-save-block.ts +++ b/apps/web/src/components/sections-editor/use-save-block.ts @@ -2,7 +2,9 @@ import { useEffect, useRef } from "react"; import { useMutation, useQueryClient } from "@tanstack/react-query"; import { useSessionRuntime } from "@/hooks/use-session-runtime"; import { useLocalPreviewUrl } from "@/hooks/use-local-preview-url"; -import { decofileCacheKey } from "./use-decofile"; +import { useDecofileCacheKey } from "./use-decofile"; +import { useContentBackend } from "./use-content-backend"; +import { applyProtocolPatch } from "./content-protocol-api"; import { usePackagePath } from "./use-package-path"; import { toast } from "sonner"; import { sanitizeSecretsForPersistence } from "@decocms/shared/decofile"; @@ -49,12 +51,10 @@ export function useSaveBlock({ * in-place render picks the merged decofile up and repaints the tunnel frame. */ const { url: localPreviewUrl } = useLocalPreviewUrl(virtualMcpId); - const cacheKey = decofileCacheKey({ - orgSlug, - virtualMcpId, - branch, - localPreviewUrl, - }); + // Next-major Blocks sites: one `blocks.apply` per save. + const backend = useContentBackend(virtualMcpId, branch); + const protocol = backend.kind === "protocol" ? backend : null; + const cacheKey = useDecofileCacheKey({ orgSlug, virtualMcpId, branch }); return useMutation({ mutationKey: decofileWriteMutationKey(orgSlug, virtualMcpId, branch), @@ -69,6 +69,27 @@ export function useSaveBlock({ }) => { // Fail closed before any write path: a raw secret must never be persisted. data = sanitizeSecretsForPersistence(data); + if (protocol) { + const applied = await applyProtocolPatch( + queryClient, + protocol, + { orgSlug, virtualMcpId, branch }, + cacheKey, + { set: { [blockKey]: data } }, + ); + if (protocol.source === "github") { + // The commit moved the branch head; refresh the header's branch meta. + await queryClient.invalidateQueries({ + queryKey: sandboxGitStatusQueryKey({ + orgSlug, + virtualMcpId, + branch, + threadId, + }), + }); + } + return applied; + } // Local: no persistence — the optimistic cache write is the save. if (localPreviewUrl) return { blockKey, data }; if (fastPreviewActive) { diff --git a/apps/web/src/hooks/use-deco-serve-connection.ts b/apps/web/src/hooks/use-deco-serve-connection.ts new file mode 100644 index 0000000000..ccfc8eabcc --- /dev/null +++ b/apps/web/src/hooks/use-deco-serve-connection.ts @@ -0,0 +1,48 @@ +import { + type DecoServeConnection, + parseStoredConnection, +} from "@/components/sections-editor/deco-serve-connection"; +import { useQueryClient } from "@tanstack/react-query"; +import { + localStorageQueryKey, + useLocalStorage, +} from "@/hooks/use-local-storage"; +import { LOCALSTORAGE_KEYS } from "@/lib/localstorage-keys"; + +export interface DecoServeConnectionState { + /** The project's connected `deco serve`, or `null`. */ + connection: DecoServeConnection | null; + set: (next: DecoServeConnection) => void; + clear: () => void; +} + +/** + * The `deco serve` this browser connected a project to. Per-browser and + * per-project, like the Local tunnel URL: it points at a server only this + * machine can reach, so it never belongs on shared project metadata. + */ +export function useDecoServeConnection( + virtualMcpId: string | null | undefined, +): DecoServeConnectionState { + const key = LOCALSTORAGE_KEYS.decoServeConnection(virtualMcpId ?? ""); + const [stored, setStored] = useLocalStorage(key, null); + return { + connection: virtualMcpId ? parseStoredConnection(stored) : null, + set: (next) => setStored(next), + clear: () => setStored(null), + }; +} + +/** Connects a project picked by its id (the connect flow's project list). */ +export function useSaveDecoServeConnection() { + const queryClient = useQueryClient(); + return (virtualMcpId: string, connection: DecoServeConnection) => { + const key = LOCALSTORAGE_KEYS.decoServeConnection(virtualMcpId); + try { + localStorage.setItem(key, JSON.stringify(connection)); + } catch { + // Storage blocked: the connection lasts for this session only. + } + queryClient.setQueryData(localStorageQueryKey(key), connection); + }; +} diff --git a/apps/web/src/hooks/use-local-storage.ts b/apps/web/src/hooks/use-local-storage.ts index dc774295e7..a7c57f5027 100644 --- a/apps/web/src/hooks/use-local-storage.ts +++ b/apps/web/src/hooks/use-local-storage.ts @@ -44,12 +44,17 @@ function initializeFromStorage( return next; } +/** The cache entry {@link useLocalStorage} mirrors a key into. */ +export function localStorageQueryKey(key: string) { + return ["localStorage", key] as const; +} + export function useLocalStorage( key: string, initializer: T | ((existing: T | undefined) => T), ): [T, (value: T | ((prev: T) => T)) => void] { const queryClientInstance = useQueryClient(); - const queryKey = ["localStorage", key] as const; + const queryKey = localStorageQueryKey(key); // Use TanStack Query to read from localStorage const { data: value } = useQuery({ diff --git a/apps/web/src/hooks/use-organization-settings.ts b/apps/web/src/hooks/use-organization-settings.ts index b1b783362c..ac87d9cca2 100644 --- a/apps/web/src/hooks/use-organization-settings.ts +++ b/apps/web/src/hooks/use-organization-settings.ts @@ -233,6 +233,19 @@ export function useOrgFlag(flag: keyof OrgFlags): boolean { return data ?? DEFAULT_ON_FLAGS.has(flag); } +/** + * {@link useOrgFlag}, but `undefined` while the org settings load — for + * gates that must not act on the pre-load default (routing reads to one + * backend, then switching them to another). + */ +export function useOrgFlagState(flag: keyof OrgFlags): boolean | undefined { + const { data, isError } = useOrganizationSettings((s) => + orgFlagEnabled(s.flags, flag), + ); + // A failed read falls back to the default rather than waiting forever. + return data ?? (isError ? DEFAULT_ON_FLAGS.has(flag) : undefined); +} + /** * Writer for the org's blocks editor. Not `useSetOrgFlag`: that goes through * ORGANIZATION_SETTINGS_UPDATE (org:manage), while any member may switch the diff --git a/apps/web/src/lib/localstorage-keys.ts b/apps/web/src/lib/localstorage-keys.ts index 2a5f9cac85..cf52c81003 100644 --- a/apps/web/src/lib/localstorage-keys.ts +++ b/apps/web/src/lib/localstorage-keys.ts @@ -57,4 +57,8 @@ export const LOCALSTORAGE_KEYS = { * project metadata. */ localPreviewUrl: (virtualMcpId: string) => `studio:local-preview-url:${virtualMcpId}`, + /** Per-project `deco serve` connection (endpoint + token), per-browser for + * the same reason as the tunnel URL. */ + decoServeConnection: (virtualMcpId: string) => + `studio:deco-serve-connection:${virtualMcpId}`, } as const; diff --git a/apps/web/src/lib/query-keys.ts b/apps/web/src/lib/query-keys.ts index cd8c8714f0..9ebb6de9a6 100644 --- a/apps/web/src/lib/query-keys.ts +++ b/apps/web/src/lib/query-keys.ts @@ -767,6 +767,18 @@ export const KEYS = { // Variadic so an invalidation call can pass just the org/vmid/branch prefix // and still partial-match the full org/vmid/branch/previewUrl query key. liveMeta: (...parts: string[]) => ["live-meta", ...parts] as const, + // Which content backend (legacy / content protocol) a project's editor uses. + contentBackend: (...parts: string[]) => + ["content-backend", ...parts] as const, + // Last content-protocol revision/version seen for a decofile or schema read. + contentRevision: (cacheKey: string) => + ["content-revision", cacheKey] as const, + // The repo's committed `deno.json`, read for its pinned deco-apps version. + denoJson: (orgSlug: string, virtualMcpId: string, branch: string) => + ["deno-json", orgSlug, virtualMcpId, branch] as const, + // The repo's committed `package.json`, read for its pinned blog-app version. + packageJson: (orgSlug: string, virtualMcpId: string, branch: string) => + ["package-json", orgSlug, virtualMcpId, branch] as const, sandboxInvoke: (sandboxKey: string, loaderKey: string) => ["sandbox-invoke", sandboxKey, loaderKey] as const, // `threadId` is part of the key because it is part of the URL: two sessions From 45929de47477e45a961d5bdfa7f62bb4202e54a8 Mon Sep 17 00:00:00 2001 From: gimenes Date: Sat, 3 Oct 2026 12:26:46 -0300 Subject: [PATCH 06/71] feat(web): connect the site editor to deco serve MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /connect#endpoint=…&token=… (the link deco serve prints) takes the connection out of the fragment, keeps it for the tab across the login redirect and hands over to /$org/connect, which calls describe (Chrome's Local Network Access prompt), lets the editor pick the project and opens its site editor. The connection is stored per project in this browser. The site editor shows a Local server chip with Disconnect, says when the server restarted with a new token or isn't running, and hides the publish and terminal surfaces: the developer commits. Image and file uploads go to PUT /assets/ when describe.assets allows. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WNwbSEePYNcY5YCgqZURig --- .../sections-editor/deco-serve-chip.tsx | 55 ++++++ .../sections-editor/fields/file-field.tsx | 16 ++ .../fields/use-serve-asset-upload.ts | 49 +++++ apps/web/src/i18n/en/deco-serve.ts | 23 +++ apps/web/src/i18n/en/index.ts | 2 + apps/web/src/i18n/pt-br/deco-serve.ts | 25 +++ apps/web/src/i18n/pt-br/index.ts | 2 + apps/web/src/router.tsx | 17 ++ apps/web/src/routes/connect.tsx | 101 ++++++++++ apps/web/src/routes/orgs/connect.tsx | 172 ++++++++++++++++++ .../routes/workspace/agent-site-editor.tsx | 46 +++-- 11 files changed, 493 insertions(+), 15 deletions(-) create mode 100644 apps/web/src/components/sections-editor/deco-serve-chip.tsx create mode 100644 apps/web/src/components/sections-editor/fields/use-serve-asset-upload.ts create mode 100644 apps/web/src/i18n/en/deco-serve.ts create mode 100644 apps/web/src/i18n/pt-br/deco-serve.ts create mode 100644 apps/web/src/routes/connect.tsx create mode 100644 apps/web/src/routes/orgs/connect.tsx diff --git a/apps/web/src/components/sections-editor/deco-serve-chip.tsx b/apps/web/src/components/sections-editor/deco-serve-chip.tsx new file mode 100644 index 0000000000..5b4a2162bf --- /dev/null +++ b/apps/web/src/components/sections-editor/deco-serve-chip.tsx @@ -0,0 +1,55 @@ +import { Button } from "@decocms/ui/components/button.tsx"; +import { cn } from "@decocms/ui/lib/utils.ts"; +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from "@decocms/ui/components/tooltip.tsx"; +import { useDecoServeConnection } from "@/hooks/use-deco-serve-connection"; +import { useT } from "@/i18n/use-t.ts"; +import { useContentBackend } from "./use-content-backend"; + +/** The site editor's "Local server" chip: what it edits, its state, Disconnect. */ +export function DecoServeChip({ + virtualMcpId, + branch, +}: { + virtualMcpId: string; + branch: string | null; +}) { + const t = useT(); + const { connection, clear } = useDecoServeConnection(virtualMcpId); + const backend = useContentBackend(virtualMcpId, branch); + if (!connection) return null; + const problem = + backend.kind === "unavailable" + ? backend.reason === "unauthorized" + ? t("decoServe.status.unauthorized") + : t("decoServe.status.unreachable") + : null; + return ( +
+ + + + {t("decoServe.chip.label")} + + + + {problem ?? t("decoServe.chip.tooltip")} + + + +
+ ); +} diff --git a/apps/web/src/components/sections-editor/fields/file-field.tsx b/apps/web/src/components/sections-editor/fields/file-field.tsx index b8d408c80b..0bfe61d9e4 100644 --- a/apps/web/src/components/sections-editor/fields/file-field.tsx +++ b/apps/web/src/components/sections-editor/fields/file-field.tsx @@ -12,6 +12,7 @@ import { import { matchSiteSlugConfig } from "@/components/file-picker/match-site-slug-config"; import { useFileConfigsQuery } from "@/hooks/use-file-configs"; import { useFilePickerUpload } from "@/hooks/use-file-picker"; +import { useServeAssetUpload } from "./use-serve-asset-upload"; import { ClickToReplaceOverlay } from "./click-to-replace-overlay"; import { extractUrl } from "./extract-url"; import { FieldLabel } from "./field-label"; @@ -47,6 +48,8 @@ export function FileField({ const configsQuery = useFileConfigsQuery(); const upload = useFilePickerUpload(); + // A connected `deco serve` writes uploads into the repository instead. + const serveUpload = useServeAssetUpload(sandbox); const lockedConfig = matchSiteSlugConfig( configsQuery.data?.configs ?? [], sandbox?.siteSlug, @@ -64,6 +67,19 @@ export function FileField({ } } + if (serveUpload) { + try { + onChange(await serveUpload(list[0]!)); + } catch (err) { + toast.error( + err instanceof Error + ? err.message + : t("sectionsEditor.fileField.uploadFailed"), + ); + } + return; + } + const targetConfigId = resolveTargetConfigId( configsQuery.data?.configs ?? [], lockedConfig?.id, diff --git a/apps/web/src/components/sections-editor/fields/use-serve-asset-upload.ts b/apps/web/src/components/sections-editor/fields/use-serve-asset-upload.ts new file mode 100644 index 0000000000..381e48ca5d --- /dev/null +++ b/apps/web/src/components/sections-editor/fields/use-serve-asset-upload.ts @@ -0,0 +1,49 @@ +import { useDecoServeConnection } from "@/hooks/use-deco-serve-connection"; +import { useContentBackend } from "../use-content-backend"; +import type { SandboxConfig } from "./field-props"; + +/** + * Uploads to a connected `deco serve`, when it takes them (`describe.assets`): + * `PUT /assets/` with its token, answered with the path the + * field stores (`/assets/`). `null` otherwise — on GitHub uploads keep + * going to Studio's file storage. + */ +export function useServeAssetUpload( + sandbox: SandboxConfig | null | undefined, +): ((file: File) => Promise) | null { + const backend = useContentBackend(sandbox?.virtualMcpId, sandbox?.branch); + const { connection } = useDecoServeConnection(sandbox?.virtualMcpId); + if ( + backend.kind !== "protocol" || + backend.source !== "local" || + !backend.describe.assets || + !connection + ) { + return null; + } + const origin = new URL(connection.endpoint).origin; + return async (file) => { + const res = await fetch( + `${origin}/assets/${encodeURIComponent(file.name)}`, + { + method: "PUT", + headers: { + authorization: `Bearer ${connection.token}`, + "content-type": file.type || "application/octet-stream", + }, + body: file, + }, + ); + const body = (await res.json().catch(() => null)) as { + path?: unknown; + error?: { message?: unknown }; + } | null; + if (!res.ok || typeof body?.path !== "string") { + const message = body?.error?.message; + throw new Error( + typeof message === "string" ? message : `Upload failed (${res.status})`, + ); + } + return body.path; + }; +} diff --git a/apps/web/src/i18n/en/deco-serve.ts b/apps/web/src/i18n/en/deco-serve.ts new file mode 100644 index 0000000000..63cdfc3d9c --- /dev/null +++ b/apps/web/src/i18n/en/deco-serve.ts @@ -0,0 +1,23 @@ +export const decoServe = { + "decoServe.chip.disconnect": "Disconnect", + "decoServe.chip.label": "Local server", + "decoServe.chip.tooltip": + "Editing the files on your machine through deco serve. Nothing is committed: review the changes and commit them yourself.", + "decoServe.connect.invalidLinkDescription": + "Run deco serve and open the site editor link it prints.", + "decoServe.connect.invalidLinkTitle": "This connect link is incomplete", + "decoServe.connect.noProjects": + "This organization has no projects yet. Import the site's repository first.", + "decoServe.connect.notEnabledDescription": + "Editing a local server isn't enabled for this organization.", + "decoServe.connect.notEnabledTitle": "Not available", + "decoServe.connect.pickDescription": + "Pick the project for {root}, served at {endpoint}.", + "decoServe.connect.pickTitle": "Connect your local server", + "decoServe.connect.reaching": "Reaching {endpoint}…", + "decoServe.connect.retry": "Try again", + "decoServe.status.unauthorized": + "The local server restarted with a new token. Open the link deco serve printed to reconnect.", + "decoServe.status.unreachable": + "The local server isn't answering. Start it with npx @decocms/blocks serve, and allow this site to reach your machine if Chrome asks.", +} as const; diff --git a/apps/web/src/i18n/en/index.ts b/apps/web/src/i18n/en/index.ts index 0cdef207ed..0b07309de2 100644 --- a/apps/web/src/i18n/en/index.ts +++ b/apps/web/src/i18n/en/index.ts @@ -34,6 +34,7 @@ import { common } from "./common.ts"; import { reportsOnboarding } from "./reports-onboarding.ts"; import { collections } from "./collections.ts"; import { chooseEditor } from "./choose-editor.ts"; +import { decoServe } from "./deco-serve.ts"; import { chat } from "./chat.ts"; import { credits } from "./credits.ts"; import { automations } from "./automations.ts"; @@ -83,6 +84,7 @@ export const en = { ...reportsOnboarding, ...collections, ...chooseEditor, + ...decoServe, ...chat, ...credits, ...automations, diff --git a/apps/web/src/i18n/pt-br/deco-serve.ts b/apps/web/src/i18n/pt-br/deco-serve.ts new file mode 100644 index 0000000000..7b2b985a74 --- /dev/null +++ b/apps/web/src/i18n/pt-br/deco-serve.ts @@ -0,0 +1,25 @@ +import type { decoServe as decoServeEn } from "../en/deco-serve.ts"; + +export const decoServe = { + "decoServe.chip.disconnect": "Desconectar", + "decoServe.chip.label": "Servidor local", + "decoServe.chip.tooltip": + "Editando os arquivos da sua máquina pelo deco serve. Nada é commitado: revise as mudanças e faça o commit você mesmo.", + "decoServe.connect.invalidLinkDescription": + "Rode o deco serve e abra o link do editor do site que ele mostra.", + "decoServe.connect.invalidLinkTitle": "Este link de conexão está incompleto", + "decoServe.connect.noProjects": + "Esta organização ainda não tem projetos. Importe primeiro o repositório do site.", + "decoServe.connect.notEnabledDescription": + "A edição por servidor local não está habilitada para esta organização.", + "decoServe.connect.notEnabledTitle": "Indisponível", + "decoServe.connect.pickDescription": + "Escolha o projeto de {root}, servido em {endpoint}.", + "decoServe.connect.pickTitle": "Conecte seu servidor local", + "decoServe.connect.reaching": "Conectando a {endpoint}…", + "decoServe.connect.retry": "Tentar novamente", + "decoServe.status.unauthorized": + "O servidor local reiniciou com um novo token. Abra o link que o deco serve mostrou para reconectar.", + "decoServe.status.unreachable": + "O servidor local não está respondendo. Inicie-o com npx @decocms/blocks serve e permita que este site acesse sua máquina se o Chrome pedir.", +} satisfies Record; diff --git a/apps/web/src/i18n/pt-br/index.ts b/apps/web/src/i18n/pt-br/index.ts index d2c0e0e214..52609d792f 100644 --- a/apps/web/src/i18n/pt-br/index.ts +++ b/apps/web/src/i18n/pt-br/index.ts @@ -34,6 +34,7 @@ import { common } from "./common.ts"; import { reportsOnboarding } from "./reports-onboarding.ts"; import { collections } from "./collections.ts"; import { chooseEditor } from "./choose-editor.ts"; +import { decoServe } from "./deco-serve.ts"; import { chat } from "./chat.ts"; import { credits } from "./credits.ts"; import { automations } from "./automations.ts"; @@ -81,6 +82,7 @@ export const ptBR = { ...reportsOnboarding, ...collections, ...chooseEditor, + ...decoServe, ...chat, ...credits, ...automations, diff --git a/apps/web/src/router.tsx b/apps/web/src/router.tsx index 9f480f2364..07dccb556d 100644 --- a/apps/web/src/router.tsx +++ b/apps/web/src/router.tsx @@ -311,6 +311,14 @@ const chooseEditorRoute = createRoute({ ), }); +// `deco serve` connect link: `/connect#endpoint=…&token=…`. Root route so the +// fragment is read before the login redirect, which would drop it. +const connectRoute = createRoute({ + getParentRoute: () => rootRoute, + path: "/connect", + component: lazyRouteComponent(() => import("./routes/connect.tsx")), +}); + // Public report, readable without a session — hence outside the org shell. const reportRoute = createRoute({ getParentRoute: () => rootRoute, @@ -1597,7 +1605,15 @@ const threadRouteWithChildren = threadRoute.addChildren([ threadSessionWithChildren, ]); +// Binds a `deco serve` connection to one of the org's projects. +const orgConnectRoute = createRoute({ + getParentRoute: () => orgRoute, + path: "/connect", + component: lazyRouteComponent(() => import("./routes/orgs/connect.tsx")), +}); + const orgRouteWithChildren = orgRoute.addChildren([ + orgConnectRoute, threadRouteWithChildren, orgMembersRedirectRoute, settingsWithChildren, @@ -1615,6 +1631,7 @@ const routeTree = rootRoute.addChildren([ reportsOnboardingRoute, legacyCommerceOnboardingRoute, chooseEditorRoute, + connectRoute, reportRoute, loginRoute, cliAuthSuccessRoute, diff --git a/apps/web/src/routes/connect.tsx b/apps/web/src/routes/connect.tsx new file mode 100644 index 0000000000..e953fe91db --- /dev/null +++ b/apps/web/src/routes/connect.tsx @@ -0,0 +1,101 @@ +/** + * `/connect#endpoint=…&token=…` — the link `deco serve` prints. + * + * Takes the connection out of the fragment (so the token never stays in the + * address bar or history), holds it for this tab and hands over to the org's + * connect screen, which picks the project. The fragment doesn't survive the + * login redirect, which is why it's stashed before the auth gate. + */ + +import { useState } from "react"; +import { Navigate } from "@tanstack/react-router"; +import { useQuery } from "@tanstack/react-query"; +import { Spinner } from "@decocms/ui/components/spinner.tsx"; +import { + parseConnectFragment, + readPendingConnection, + stashPendingConnection, +} from "@/components/sections-editor/deco-serve-connection"; +import RequiredAuthLayout from "@/layouts/required-auth-layout"; +import { listOrganizationsCached } from "@/lib/auth-client"; +import { readLastLocation } from "@/lib/last-location"; +import { LOCALSTORAGE_KEYS } from "@/lib/localstorage-keys"; +import { KEYS } from "@/lib/query-keys"; +import { useT } from "@/i18n/use-t.ts"; + +function takeConnectionFromUrl(): boolean { + const connection = parseConnectFragment(window.location.hash); + if (connection) { + stashPendingConnection(connection); + window.history.replaceState( + null, + "", + window.location.pathname + window.location.search, + ); + } + return readPendingConnection() !== null; +} + +function lastOrgSlug(): string | null { + try { + return ( + readLastLocation()?.org ?? + localStorage.getItem(LOCALSTORAGE_KEYS.lastOrgSlug()) + ); + } catch { + return null; + } +} + +function Centered({ children }: { children: React.ReactNode }) { + return ( +
+
+ {children} +
+
+ ); +} + +function ToOrg() { + const remembered = lastOrgSlug(); + const orgs = useQuery({ + queryKey: KEYS.contentBackend("connect", "organizations"), + queryFn: listOrganizationsCached, + enabled: !remembered, + }); + const org = remembered ?? orgs.data?.data?.[0]?.slug ?? null; + if (org) { + return ; + } + if (orgs.isPending) { + return ( + + + + ); + } + return ; +} + +export default function ConnectRoute() { + const t = useT(); + const [hasConnection] = useState(takeConnectionFromUrl); + if (!hasConnection) { + return ( + +

+ {t("decoServe.connect.invalidLinkTitle")} +

+

+ {t("decoServe.connect.invalidLinkDescription")} +

+ + ); + } + return ( + + + + ); +} diff --git a/apps/web/src/routes/orgs/connect.tsx b/apps/web/src/routes/orgs/connect.tsx new file mode 100644 index 0000000000..99e98e0eb2 --- /dev/null +++ b/apps/web/src/routes/orgs/connect.tsx @@ -0,0 +1,172 @@ +/** + * `/$org/connect` — the second half of a `deco serve` connect link (see + * `routes/connect.tsx`): checks the server answers, then binds it to the + * project the editor picks and opens that project's site editor. + * + * The first request to the server is what makes Chrome ask whether this site + * may reach a server on the editor's machine (Local Network Access). + */ + +import { Suspense } from "react"; +import { useNavigate } from "@tanstack/react-router"; +import { useQuery } from "@tanstack/react-query"; +import { createContentClient } from "@decocms/shared/blocks-protocol"; +import { Button } from "@decocms/ui/components/button.tsx"; +import { Spinner } from "@decocms/ui/components/spinner.tsx"; +import { AgentAvatar } from "@/components/agent-icon"; +import { + clearPendingConnection, + readPendingConnection, +} from "@/components/sections-editor/deco-serve-connection"; +import { useSaveDecoServeConnection } from "@/hooks/use-deco-serve-connection"; +import { PROJECT_ROUTE } from "@/hooks/use-destination-route"; +import { useOrgFlag } from "@/hooks/use-organization-settings"; +import { scopableProjects } from "@/hooks/use-project-scope"; +import { useT } from "@/i18n/use-t.ts"; +import { KEYS } from "@/lib/query-keys"; +import { useProjectContext, useVirtualMCPs } from "@/sdk"; +import { decoServeErrorReason } from "@/components/sections-editor/deco-serve-status"; + +function Centered({ children }: { children: React.ReactNode }) { + return ( +
+
+ {children} +
+
+ ); +} + +function ProjectPicker({ + onPick, +}: { + onPick: (project: { id: string }) => void; +}) { + const t = useT(); + const { org } = useProjectContext(); + const projects = scopableProjects(useVirtualMCPs({ pageSize: 1000 })) + .filter((p) => p.id !== org.id) + .sort((a, b) => a.title.localeCompare(b.title)); + if (projects.length === 0) { + return ( +

+ {t("decoServe.connect.noProjects")} +

+ ); + } + return ( +
+ {projects.map((project) => ( + + ))} +
+ ); +} + +export default function OrgConnectRoute() { + const t = useT(); + const navigate = useNavigate(); + const { org } = useProjectContext(); + const enabled = useOrgFlag("site_editor_content_protocol"); + const saveConnection = useSaveDecoServeConnection(); + const pending = readPendingConnection(); + + const described = useQuery({ + queryKey: KEYS.contentBackend( + "connect", + pending?.endpoint ?? "", + pending?.token ?? "", + ), + queryFn: () => createContentClient(pending!).describe(), + enabled: enabled && !!pending, + retry: false, + staleTime: Number.POSITIVE_INFINITY, + }); + + if (!enabled) { + return ( + +

+ {t("decoServe.connect.notEnabledTitle")} +

+

+ {t("decoServe.connect.notEnabledDescription")} +

+ + ); + } + if (!pending) { + return ( + +

+ {t("decoServe.connect.invalidLinkTitle")} +

+

+ {t("decoServe.connect.invalidLinkDescription")} +

+ + ); + } + if (described.isPending) { + return ( + + +

+ {t("decoServe.connect.reaching", { endpoint: pending.endpoint })} +

+ + ); + } + if (described.isError) { + return ( + +

+ {decoServeErrorReason(described.error) === "unauthorized" + ? t("decoServe.status.unauthorized") + : t("decoServe.status.unreachable")} +

+ + + ); + } + + return ( + +
+

+ {t("decoServe.connect.pickTitle")} +

+

+ {t("decoServe.connect.pickDescription", { + root: described.data.root, + endpoint: pending.endpoint, + })} +

+
+ }> + { + saveConnection(project.id, pending); + clearPendingConnection(); + navigate({ + to: PROJECT_ROUTE.siteEditorContent, + params: { org: org.slug, agentId: project.id }, + }); + }} + /> + + + ); +} diff --git a/apps/web/src/routes/workspace/agent-site-editor.tsx b/apps/web/src/routes/workspace/agent-site-editor.tsx index 4716c21a2e..c625d2d14d 100644 --- a/apps/web/src/routes/workspace/agent-site-editor.tsx +++ b/apps/web/src/routes/workspace/agent-site-editor.tsx @@ -15,6 +15,9 @@ import { shouldShowTerminalDrawer } from "@/layouts/main-panel-tabs/terminal-dra import { PreviewDrawerHost } from "@/layouts/main-panel-tabs/preview-drawer-host"; import { ChatLayout } from "@/components/chat-layout"; import { useVirtualMCP } from "@/sdk"; +import { DecoServeChip } from "@/components/sections-editor/deco-serve-chip"; +import { useDecoServeConnection } from "@/hooks/use-deco-serve-connection"; +import { useOrgFlag } from "@/hooks/use-organization-settings"; function SiteEditorActions() { const session = useOptionalChatTask(); @@ -23,27 +26,36 @@ function SiteEditorActions() { const runtime = useSessionRuntime(entity?.id).runtime; // Local mode edits are ephemeral (nothing to promote) → withhold publish. const { url: localPreviewUrl } = useLocalPreviewUrl(entity?.id); + // A connected `deco serve` edits the working tree: the developer commits. + const { connection: serveConnection } = useDecoServeConnection(entity?.id); + const servingLocally = + useOrgFlag("site_editor_content_protocol") && !!serveConnection; if (!entity) return null; return ( <> + {servingLocally && ( + + )}
- {!localPreviewUrl && agentShowsRepositoryHeaderActions(entity) && ( - <> - -
- {runtime === "cms" ? ( - - ) : ( - - )} -
- - )} + {!localPreviewUrl && + !servingLocally && + agentShowsRepositoryHeaderActions(entity) && ( + <> + +
+ {runtime === "cms" ? ( + + ) : ( + + )} +
+ + )} ); } @@ -55,8 +67,12 @@ function SiteEditorDrawer() { const activeTabId = useActivePanelTabId(); const sessionRuntime = useSessionRuntime(entity?.id).runtime; const { url: localPreviewUrl } = useLocalPreviewUrl(entity?.id); + const { connection: serveConnection } = useDecoServeConnection(entity?.id); + const servingLocally = + useOrgFlag("site_editor_content_protocol") && !!serveConnection; const showDrawer = !localPreviewUrl && + !servingLocally && shouldShowTerminalDrawer({ hasClonableSource: agentHasClonableSource(entity?.metadata) || From 09cf0da50e9fe8ac8be461f9584dcd8656435108 Mon Sep 17 00:00:00 2001 From: gimenes Date: Sat, 3 Oct 2026 12:26:56 -0300 Subject: [PATCH 07/71] feat(web): next-major content shapes and protocol degradations - A key in the manifest is a block type, so short names like `hero` aren't read as saved blocks, are offered in block pickers, and can't be taken by a new saved block. Without the manifest, the module-path heuristic is unchanged. - Lazy fields render the form of T and store the lazy block around the value; `lazy` is never offered as a pick. The built-in `multivariate` keeps each variant's value in a lazy block; the legacy names keep plain values. - Redirects list and edit the flat `redirect` shape next to the nested one, keep the shape they were read in, and edit its optional status (301/302/307/308). New redirects stay nested. - On the protocol, Secret fields are write-only: what an editor types is encrypted in the browser with describe.secrets.publicKey and only the secret block with its ciphertext is saved. v7 secrets say they can't be edited there. The content protocol never runs site code, so on it: - the preview is the real app (the draft on GitHub, the dev app for a connected deco serve) with no in-place /live/previews renders; - the add-section gallery shows each block's name, description and schema @image instead of rendered thumbnails; - loader-backed pickers fall back to the schema's options or free text; - Run, runnable loaders and actions, and app install are hidden. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WNwbSEePYNcY5YCgqZURig --- .../components/sandbox/content/app-editor.tsx | 1 + .../sandbox/content/content-browser.tsx | 34 +++++-- .../sandbox/content/redirect-data.test.ts | 55 ++++++++++++ .../sandbox/content/redirect-data.ts | 75 +++++++++++++--- .../sandbox/content/redirect-editor.tsx | 53 ++++++++++- .../sandbox/content/redirect-type-badge.tsx | 7 +- .../components/sandbox/preview/preview.tsx | 42 ++++++--- .../sections-editor/add-section-modal.tsx | 67 ++++++++++---- .../sections-editor/block-type-utils.test.ts | 22 +++++ .../sections-editor/block-type-utils.ts | 14 ++- .../fields/media-variants.test.ts | 16 ++++ .../sections-editor/fields/media-variants.ts | 23 +++-- .../fields/protocol-secret-field.tsx | 90 +++++++++++++++++++ .../fields/protocol-secret.test.ts | 66 ++++++++++++++ .../sections-editor/fields/protocol-secret.ts | 19 ++++ .../fields/secret-field.test.ts | 10 ++- .../sections-editor/fields/secret-field.tsx | 6 +- .../sections-editor/lazy-value.test.ts | 73 +++++++++++++++ .../components/sections-editor/lazy-value.ts | 42 +++++++++ .../sections-editor/matcher-rules.ts | 6 +- .../sections-editor/page-sections.test.ts | 9 ++ .../sections-editor/page-sections.ts | 7 ++ .../resolve-schema-short-names.test.ts | 67 ++++++++++++++ .../sections-editor/resolve-schema.ts | 32 +++++-- .../sections-editor/schema-form.tsx | 29 +++++- .../sections-editor/section-catalog.ts | 4 +- .../sections-editor/section-preview-image.ts | 16 ++++ .../sections-editor/sections-editor.tsx | 21 +++-- .../components/sections-editor/seo-schema.ts | 13 ++- .../use-section-preview-base.ts | 6 ++ apps/web/src/i18n/en/sandbox.ts | 4 + apps/web/src/i18n/en/sections-editor.ts | 6 ++ apps/web/src/i18n/pt-br/sandbox.ts | 4 + apps/web/src/i18n/pt-br/sections-editor.ts | 6 ++ 34 files changed, 858 insertions(+), 87 deletions(-) create mode 100644 apps/web/src/components/sections-editor/fields/protocol-secret-field.tsx create mode 100644 apps/web/src/components/sections-editor/fields/protocol-secret.test.ts create mode 100644 apps/web/src/components/sections-editor/fields/protocol-secret.ts create mode 100644 apps/web/src/components/sections-editor/lazy-value.test.ts create mode 100644 apps/web/src/components/sections-editor/lazy-value.ts create mode 100644 apps/web/src/components/sections-editor/resolve-schema-short-names.test.ts diff --git a/apps/web/src/components/sandbox/content/app-editor.tsx b/apps/web/src/components/sandbox/content/app-editor.tsx index 13f4e3be63..0ae4968f7b 100644 --- a/apps/web/src/components/sandbox/content/app-editor.tsx +++ b/apps/web/src/components/sandbox/content/app-editor.tsx @@ -58,6 +58,7 @@ export function AppEditor({ // back to the Fast Preview production deployment while the sandbox boots. const sectionPreviewBase = useSectionPreviewBase({ virtualMcpId, + branch, sandboxUrl: previewBaseUrl, }); const resolveType = diff --git a/apps/web/src/components/sandbox/content/content-browser.tsx b/apps/web/src/components/sandbox/content/content-browser.tsx index d806262a9e..f28056c348 100644 --- a/apps/web/src/components/sandbox/content/content-browser.tsx +++ b/apps/web/src/components/sandbox/content/content-browser.tsx @@ -123,6 +123,8 @@ import { EmptyMessage } from "./empty-message"; import { SectionsRightPane } from "./sections-right-pane"; import { ItemActions } from "./item-actions"; import { ItemRow } from "./item-row"; +import { useContentBackend } from "@/components/sections-editor/use-content-backend"; +import { contentCapabilities } from "@/components/sections-editor/content-backend"; import { categoryAncestors, type CategoryTreeRow, @@ -260,8 +262,12 @@ export function ContentBrowser({ deepLinkPage }: ContentBrowserProps) { const lifecycle = useSandboxLifecycle(); const { runtime, previewServerUrl } = useSessionRuntime(virtualMcpId); const fastPreviewActive = runtime === "cms"; + // A content-protocol backend has nothing to boot either. + const contentBackend = useContentBackend(virtualMcpId, branch); + const protocolActive = + contentBackend.kind === "protocol" || contentBackend.kind === "unavailable"; const gate = resolveContentSandboxGate({ - fastPreviewActive, + fastPreviewActive: fastPreviewActive || protocolActive, previewState: lifecycle.previewState, lifecyclePhase: vmEvents.lifecycle.phase, }); @@ -287,9 +293,14 @@ export function ContentBrowser({ deepLinkPage }: ContentBrowserProps) { orgSlug={org.slug} virtualMcpId={virtualMcpId} branch={branch} - previewUrl={lifecycle.previewUrl} + // The protocol never runs site code: nothing executes against a pod. + previewUrl={protocolActive ? null : lifecycle.previewUrl} sitePreviewUrl={ - fastPreviewActive ? previewServerUrl : lifecycle.previewUrl + contentBackend.kind === "protocol" && contentBackend.source === "local" + ? (contentBackend.describe.preview?.origin ?? null) + : fastPreviewActive + ? previewServerUrl + : lifecycle.previewUrl } deepLinkPage={deepLinkPage} devServerReady={gate.devServerReady} @@ -349,6 +360,9 @@ function ContentBrowserReady({ const t = useT(); const threadId = useOptionalChatTask()?.taskId ?? null; const fetchParams = { orgSlug, virtualMcpId, branch, threadId, previewUrl }; + const contentCaps = contentCapabilities( + useContentBackend(virtualMcpId, branch), + ); const { data: decofile, isLoading: decofileLoading } = useDecofile( fetchParams, { fetchEnabled: devServerReady }, @@ -460,7 +474,7 @@ function ContentBrowserReady({ const { catalog: appCatalog, isLoading: appCatalogLoading } = useDecoAppsCatalog(meta ?? undefined, decofile ?? undefined, { - enabled: activeCollection === "apps", + enabled: contentCaps.installApps && activeCollection === "apps", }); const saveBlock = useSaveBlock(fetchParams); @@ -528,8 +542,12 @@ function ContentBrowserReady({ ? allBlogEntries[activeCollection] : []; - const loadersCount = countAvailableRunnables(meta, "loaders"); - const actionsCount = countAvailableRunnables(meta, "actions"); + const loadersCount = contentCaps.runBlocks + ? countAvailableRunnables(meta, "loaders") + : 0; + const actionsCount = contentCaps.runBlocks + ? countAvailableRunnables(meta, "actions") + : 0; // Loader/action-only sites are still editable — don't gate them out. if ( @@ -550,7 +568,7 @@ function ContentBrowserReady({ const counts: CollectionCounts = { pages: pages.length, sections: globalSections.length, - apps: appCatalog.length, + apps: contentCaps.installApps ? appCatalog.length : 0, loaders: loadersCount, actions: actionsCount, redirects: redirects.length, @@ -1758,7 +1776,7 @@ function ItemList({ selection?.collection === "redirects" && selection.key === entry.key } - trailing={} + trailing={} onClick={() => onSelect({ collection: "redirects", key: entry.key }) } diff --git a/apps/web/src/components/sandbox/content/redirect-data.test.ts b/apps/web/src/components/sandbox/content/redirect-data.test.ts index 5d808ba81e..4cf8523ff8 100644 --- a/apps/web/src/components/sandbox/content/redirect-data.test.ts +++ b/apps/web/src/components/sandbox/content/redirect-data.test.ts @@ -6,6 +6,7 @@ import { extractRedirects, generateRedirectBlockKey, getRedirectPayload, + redirectStatus, } from "./redirect-data"; test("REDIRECT_STATUS locks the HTTP status contract", () => { @@ -120,6 +121,60 @@ describe("buildRedirectBlock / getRedirectPayload round-trip", () => { }); }); +describe("flat redirects (next-major `redirect`)", () => { + const flat = { + __resolveType: "redirect", + from: "/summer", + to: "/sale", + permanent: false, + status: 308, + discardQueryParameters: true, + }; + + test("are listed next to nested ones", () => { + const decofile = { + "redirects-flat": flat, + "redirects-nested": redirectBlock({ from: "/a", to: "/b" }), + }; + const byKey = Object.fromEntries( + extractRedirects(decofile).map((r) => [r.key, r]), + ); + expect(byKey["redirects-flat"]).toEqual({ + key: "redirects-flat", + from: "/summer", + to: "/sale", + type: "temporary", + discardQueryParameters: true, + flat: true, + status: 308, + }); + expect(byKey["redirects-nested"]?.flat).toBeUndefined(); + }); + + test("write back in the shape they were read", () => { + expect(buildRedirectBlock(getRedirectPayload(flat))).toEqual(flat); + const minimal = { + __resolveType: "redirect", + from: "/x", + to: "/y", + permanent: true, + }; + expect(buildRedirectBlock(getRedirectPayload(minimal))).toEqual(minimal); + }); + + test("answer with the status that wins", () => { + const read = (block: Record) => + redirectStatus(getRedirectPayload(block)); + expect(read(flat)).toBe(308); + expect(read({ ...flat, status: undefined })).toBe(302); + expect(read({ ...flat, status: 418, permanent: true })).toBe(301); + expect(read(redirectBlock({ from: "/a", to: "/b" }))).toBe(307); + expect( + read(redirectBlock({ from: "/a", to: "/b", type: "permanent" })), + ).toBe(301); + }); +}); + describe("generateRedirectBlockKey", () => { test("derives a slug from the `from` path, stripping the query string", () => { const key = generateRedirectBlockKey({}, "/Bazar-Farm/Short?map=c"); diff --git a/apps/web/src/components/sandbox/content/redirect-data.ts b/apps/web/src/components/sandbox/content/redirect-data.ts index 6e4d1953ab..737beb3f85 100644 --- a/apps/web/src/components/sandbox/content/redirect-data.ts +++ b/apps/web/src/components/sandbox/content/redirect-data.ts @@ -9,6 +9,14 @@ * The site's routes include an inline `website/loaders/redirects.ts` (plural) * that auto-discovers ALL such blocks via `resolveTypeSelector`, so CRUD here * is just create/update/delete of these blocks — no routes/site wiring needed. + * + * Next-major Blocks also reads a flat shape, the built-in `redirect`: + * + * { "__resolveType": "redirect", from, to, permanent, status?, discardQueryParameters? } + * + * where `status` (301/302/307/308) wins over `permanent` (301, else 302). An + * entry keeps the shape it was read in; new redirects use the nested shape, + * which every site understands. */ export const REDIRECT_RESOLVE_TYPE = "website/loaders/redirect.ts"; @@ -23,6 +31,12 @@ export const REDIRECT_LOADER_RESOLVE_TYPES: ReadonlySet = new Set([ "website/loaders/redirects.ts", ]); +/** The next-major built-in, stored flat. */ +const FLAT_REDIRECT_RESOLVE_TYPE = "redirect"; + +export const REDIRECT_STATUS_CODES = [301, 302, 307, 308] as const; +export type RedirectStatusCode = (typeof REDIRECT_STATUS_CODES)[number]; + export type RedirectType = "temporary" | "permanent"; /** HTTP status the deco redirect handler emits for each type. */ @@ -31,29 +45,54 @@ export const REDIRECT_STATUS: Record = { permanent: 301, }; -export interface RedirectEntry { - key: string; +export interface RedirectPayload { from: string; to: string; type: RedirectType; discardQueryParameters: boolean; + /** Stored in the flat (`redirect`) shape; absent for the nested one. */ + flat?: true; + /** Flat shape only: an explicit status, which wins over `type`. */ + status?: RedirectStatusCode; } -export interface RedirectPayload { - from: string; - to: string; - type: RedirectType; - discardQueryParameters: boolean; +export interface RedirectEntry extends RedirectPayload { + key: string; +} + +/** The status code a redirect answers with. A flat temporary one is a 302. */ +export function redirectStatus(payload: RedirectPayload): number { + if (!payload.flat) return REDIRECT_STATUS[payload.type]; + return payload.status ?? (payload.type === "permanent" ? 301 : 302); } const asStr = (v: unknown): string => (typeof v === "string" ? v : ""); const asType = (v: unknown): RedirectType => v === "permanent" ? "permanent" : "temporary"; +const asStatus = (v: unknown): RedirectStatusCode | undefined => + REDIRECT_STATUS_CODES.find((code) => code === v); + +/** A flat `redirect` block, defensively narrowed. */ +function readFlatRedirect(block: Record): RedirectPayload { + const status = asStatus(block.status); + return { + from: asStr(block.from), + to: asStr(block.to), + type: block.permanent === true ? "permanent" : "temporary", + discardQueryParameters: block.discardQueryParameters === true, + flat: true, + ...(status ? { status } : {}), + }; +} + /** The `redirect` sub-object of a redirect block, defensively narrowed. */ function readRedirect( block: Record | undefined, ): RedirectPayload { + if (block?.__resolveType === FLAT_REDIRECT_RESOLVE_TYPE) { + return readFlatRedirect(block); + } const raw = block && typeof block.redirect === "object" && @@ -77,9 +116,13 @@ export function extractRedirects( for (const [key, val] of Object.entries(decofile)) { if (!val || typeof val !== "object" || Array.isArray(val)) continue; const obj = val as Record; - if (obj.__resolveType !== REDIRECT_RESOLVE_TYPE) continue; - const { from, to, type, discardQueryParameters } = readRedirect(obj); - out.push({ key, from, to, type, discardQueryParameters }); + if ( + obj.__resolveType !== REDIRECT_RESOLVE_TYPE && + obj.__resolveType !== FLAT_REDIRECT_RESOLVE_TYPE + ) { + continue; + } + out.push({ key, ...readRedirect(obj) }); } return out; } @@ -95,6 +138,18 @@ export function getRedirectPayload( export function buildRedirectBlock( payload: RedirectPayload, ): Record { + if (payload.flat) { + return { + __resolveType: FLAT_REDIRECT_RESOLVE_TYPE, + from: payload.from, + to: payload.to, + permanent: payload.type === "permanent", + ...(payload.status ? { status: payload.status } : {}), + ...(payload.discardQueryParameters + ? { discardQueryParameters: true } + : {}), + }; + } const redirect: Record = { from: payload.from, to: payload.to, diff --git a/apps/web/src/components/sandbox/content/redirect-editor.tsx b/apps/web/src/components/sandbox/content/redirect-editor.tsx index e2bba539bc..e04b03ef22 100644 --- a/apps/web/src/components/sandbox/content/redirect-editor.tsx +++ b/apps/web/src/components/sandbox/content/redirect-editor.tsx @@ -16,11 +16,16 @@ import type { TranslationKey } from "@/i18n/use-t.ts"; import { buildRedirectBlock, getRedirectPayload, - REDIRECT_STATUS, + REDIRECT_STATUS_CODES, type RedirectPayload, + type RedirectStatusCode, type RedirectType, + redirectStatus, } from "./redirect-data"; +/** The status select's value for "no explicit status". */ +const STATUS_FROM_TYPE = "type"; + const TYPE_OPTIONS: Array<{ value: RedirectType; labelKey: TranslationKey }> = [ { value: "temporary", labelKey: "sandbox.redirectEditor.typeTemporary" }, { value: "permanent", labelKey: "sandbox.redirectEditor.typePermanent" }, @@ -118,7 +123,11 @@ export function RedirectEditor({ {TYPE_OPTIONS.map((option) => ( {t(option.labelKey, { - status: REDIRECT_STATUS[option.value], + status: redirectStatus({ + ...payload, + type: option.value, + status: undefined, + }), })} ))} @@ -131,6 +140,46 @@ export function RedirectEditor({

+ {payload.flat && ( +
+ + +

+ {t("sandbox.redirectEditor.statusDescription")} +

+
+ )} +