diff --git a/.changeset/storage-do-sqlite.md b/.changeset/storage-do-sqlite.md new file mode 100644 index 00000000..cd0d08d3 --- /dev/null +++ b/.changeset/storage-do-sqlite.md @@ -0,0 +1,5 @@ +--- +"@cloudhal/storage": minor +--- + +New `@cloudhal/storage/do-sqlite`: `DoSqliteDriver`, a `SqlDriver` over a SQLite-backed Durable Object's storage, with an atomic `batch` (`transactionSync`) and D1's limits (RFC-0036). New blob stores at the root: `SqlBlobStore` keeps blobs in a table of the same database with per-object and total caps, `LayeredBlobStore` reads a writable store over a read-only one, and `FetcherBlobStore` reads static files through a Worker `ASSETS` binding. `@cloudhal/storage/testing` gains `createMiniflareDoSqlite()`, which runs any `SqlDriver` suite on Durable Object SQLite. diff --git a/AGENTS.md b/AGENTS.md index 2d1cb0fc..323ec26d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,7 +26,7 @@ pnpm i18n:export / pnpm i18n:import [--locale xx] # translations as nested ICU - `packages/core-contracts/schemas/*.schema.json` — **the only hand-edited contract source**. `common.schema.json` holds shared `$defs`. - `packages/core-contracts/src/generated/`, `crates/cloudhal-schema/src/generated/` — generated; never edit. - `packages/core-contracts/fixtures/{valid,invalid}//` — golden fixtures used by both languages. -- `packages/storage` — `SqlDriver`/`BlobStore` + adapters (`sqlite`, `d1`, `fs`, `r2`) + migrations (`migrations/*.ts`, plain SQL). Conformance suites in `src/testing.ts`. +- `packages/storage` — `SqlDriver`/`BlobStore` + adapters (`sqlite`, `d1`, `do-sqlite`, `fs`, `r2`; `SqlBlobStore`, `LayeredBlobStore`, `FetcherBlobStore` at the root) + migrations (`migrations/*.ts`, plain SQL). Conformance suites in `src/testing.ts`. - `packages/core-ts` — `SiteGraphStore` (CAS on `rev`, snapshots, diff, publish, rollback). Behavioural suite in `src/testing.ts` runs on sqlite and Miniflare D1. `./local` opens the offline store; `seedStarterSite` seeds a demo. `./surfaces` is RFC-0020's generative UI: the catalog (primitives + block types), eval-free surface checks, bindings, repairs, JSON Patch (`patchPage`), `catalogPrompt`; `./interop` converts A2UI v0.9 and json-render to and from surfaces; `./design` is RFC-0021's DESIGN.md lint and guidance. `search.ts` is RFC-0019's FTS5 site search (index written with the publish batch; articles, pages, terms and authors, RFC-0029); `search-provider.ts` is RFC-0029's `SearchProvider` (`fts5`, `memory`, `withFallback`), set with `SiteGraphStoreOptions.search`; `search-ai.ts` is the experimental Cloudflare AI Search provider (`hybridProvider`, `searchProviderFromEnv`, `cloudhal.json` `search`, docs/guides/search.md). - `packages/schedules` — RFC-0008: time model (only `temporal-polyfill/implementation`, `now` injected, never `Temporal.Now`), RRULE subset, release model (`rl1:`), automation/run state, taint and budgets, `runDue`; `./sql` is `SqlScheduleStore`, `./engine` the Temporal-free part core-ts and the site Worker use. Worker-safe; conformance suites in `src/testing.ts` on sqlite and Miniflare D1. - `packages/astro-renderer` — `GraphSource` (live/snapshot), `cloudhal()` integration (virtual config + dev route-cache watcher), `.astro` section components (`Surface.astro` for RFC-0020 surfaces; `./surfaces` renders them, `./kit` has the table of contents and reading time). Routes live in starters, not here. diff --git a/packages/core-ts/test/store.test.ts b/packages/core-ts/test/store.test.ts index b38bbff7..88ef14fa 100644 --- a/packages/core-ts/test/store.test.ts +++ b/packages/core-ts/test/store.test.ts @@ -1,7 +1,7 @@ -import { MemoryBlobStore } from "@cloudhal/storage"; +import { MemoryBlobStore, SqlBlobStore } from "@cloudhal/storage"; import { D1Driver } from "@cloudhal/storage/d1"; import { BetterSqlite3Driver } from "@cloudhal/storage/sqlite"; -import { createMiniflareD1 } from "@cloudhal/storage/testing"; +import { createMiniflareD1, createMiniflareDoSqlite } from "@cloudhal/storage/testing"; import { describeSiteGraphStore } from "../src/testing.ts"; import { correctionsSuite } from "../src/testing-corrections.ts"; import { gatesSuite } from "../src/testing-gates.ts"; @@ -44,3 +44,19 @@ releasesSuite("Cloudflare D1 (Miniflare) + memory blobs", miniflare); correctionsSuite("Cloudflare D1 (Miniflare) + memory blobs", miniflare); searchSuite("Cloudflare D1 (Miniflare) + memory blobs", miniflare); searchKindsSuite("Cloudflare D1 (Miniflare) + memory blobs", miniflare); + +// RFC-0036: the playground's backend. Blobs live in the same Durable Object database. +let doSqlite: Awaited> | undefined; +const durableObject = { + async create() { + doSqlite ??= await createMiniflareDoSqlite(); + const sql = doSqlite.driver("store"); + return { sql, blobs: new SqlBlobStore(sql) }; + }, + async dispose() { + await doSqlite?.dispose(); + doSqlite = undefined; + }, +}; +describeSiteGraphStore("Durable Object SQLite (Miniflare) + sql blobs", durableObject); +searchSuite("Durable Object SQLite (Miniflare) + sql blobs", durableObject); diff --git a/packages/storage/README.md b/packages/storage/README.md index dfcf43ca..bf9f1b0d 100644 --- a/packages/storage/README.md +++ b/packages/storage/README.md @@ -4,12 +4,13 @@ Storage interfaces and adapters for Cloudhal. The interface is shaped by the mos | Export | What | |---|---| -| `@cloudhal/storage` | `SqlDriver`, `BlobStore`, limits, `migrate()`, `migrations`, `MemoryBlobStore` | +| `@cloudhal/storage` | `SqlDriver`, `BlobStore`, limits, `migrate()`, `migrations`, `MemoryBlobStore`, `SqlBlobStore` (blobs in a SQL table, capped), `LayeredBlobStore` (read-only base under a writable upper), `FetcherBlobStore` (read-only, over a Worker `ASSETS` binding) | | `@cloudhal/storage/sqlite` | `BetterSqlite3Driver` — local files or `:memory:` | | `@cloudhal/storage/d1` | `D1Driver` — Cloudflare D1 binding | +| `@cloudhal/storage/do-sqlite` | `DoSqliteDriver` — a SQLite-backed Durable Object's `ctx.storage` (RFC-0036); `batch` runs in `transactionSync` | | `@cloudhal/storage/fs` | `FsBlobStore` — directory on disk | | `@cloudhal/storage/r2` | `R2BlobStore` — Cloudflare R2 binding | -| `@cloudhal/storage/testing` | conformance suites + Miniflare helpers (`createMiniflareD1`, `createMiniflareR2`) | +| `@cloudhal/storage/testing` | conformance suites + Miniflare helpers (`createMiniflareD1`, `createMiniflareR2`, `createMiniflareDoSqlite`) | ```ts import { migrate, migrations } from "@cloudhal/storage"; @@ -24,4 +25,18 @@ await sql.batch([ Migrations are TypeScript modules under `migrations/` (plain SQL strings) so they bundle for Workers without filesystem access. Each migration is applied as one atomic batch. -Every adapter must pass `describeSqlDriverConformance` / `describeBlobStoreConformance`; the D1 and R2 suites run against Miniflare (workerd in-process), so no Cloudflare account is needed. +Every adapter must pass `describeSqlDriverConformance` / `describeBlobStoreConformance`; the D1, R2 and Durable Object suites run against Miniflare (workerd in-process), so no Cloudflare account is needed. `createMiniflareDoSqlite()` runs `DoSqliteDriver` inside a test-only Durable Object and hands Node a driver whose every call is one request to it, so any `SqlDriver` suite (core-ts's included) runs on Durable Object SQLite. + +A whole site in one Durable Object (the playground's shape): + +```ts +import { FetcherBlobStore, LayeredBlobStore, migrate, migrations, SqlBlobStore } from "@cloudhal/storage"; +import { DoSqliteDriver } from "@cloudhal/storage/do-sqlite"; + +const sql = new DoSqliteDriver(ctx.storage); +await migrate(sql, migrations); +const blobs = new LayeredBlobStore( + new FetcherBlobStore(env.ASSETS, "demo/"), // read-only demo media + new SqlBlobStore(sql, { maxBytes: 8 * 1024 * 1024 }), // the session's own blobs +); +``` diff --git a/packages/storage/package.json b/packages/storage/package.json index 7ce61d01..c7020387 100644 --- a/packages/storage/package.json +++ b/packages/storage/package.json @@ -1,7 +1,7 @@ { "name": "@cloudhal/storage", "version": "0.2.0", - "description": "Cloudhal storage interfaces (SQL driver, blob store) with Cloudflare D1/R2 and local SQLite/filesystem adapters", + "description": "Cloudhal storage interfaces (SQL driver, blob store) with Cloudflare D1/R2/Durable Object SQLite and local SQLite/filesystem adapters", "license": "Apache-2.0", "homepage": "https://cloudhal.dev", "repository": { @@ -17,6 +17,7 @@ ".": "./src/index.ts", "./sqlite": "./src/sql/better-sqlite3.ts", "./d1": "./src/sql/d1.ts", + "./do-sqlite": "./src/sql/do-sqlite.ts", "./r2": "./src/blob/r2.ts", "./fs": "./src/blob/fs.ts", "./testing": "./src/testing.ts", @@ -70,6 +71,10 @@ "types": "./dist/src/sql/d1.d.ts", "default": "./dist/src/sql/d1.js" }, + "./do-sqlite": { + "types": "./dist/src/sql/do-sqlite.d.ts", + "default": "./dist/src/sql/do-sqlite.js" + }, "./r2": { "types": "./dist/src/blob/r2.d.ts", "default": "./dist/src/blob/r2.js" diff --git a/packages/storage/src/blob/fetcher.ts b/packages/storage/src/blob/fetcher.ts new file mode 100644 index 00000000..fb2fa130 --- /dev/null +++ b/packages/storage/src/blob/fetcher.ts @@ -0,0 +1,73 @@ +import { + assertBlobKey, + type BlobMeta, + type BlobStore, + StorageError, + sha256OfBytes, +} from "../types.ts"; + +/** The part of a Worker `Fetcher` binding (e.g. `ASSETS`) this store uses. */ +export interface BlobFetcher { + fetch(input: string): Promise; +} + +export interface FetcherBlobStoreOptions { + /** Keys `list` reports (static assets cannot be listed). Default: none. */ + readonly keys?: readonly string[]; +} + +/** + * A read-only `BlobStore` over static files (RFC-0036 §2): `key` is fetched as + * `/` from the Worker's `ASSETS` binding. Writes throw `StorageError`. Used as the + * base layer of a `LayeredBlobStore` for demo media. + */ +export class FetcherBlobStore implements BlobStore { + readonly name = "fetcher"; + readonly #fetcher: BlobFetcher; + readonly #prefix: string; + readonly #keys: readonly string[]; + + constructor(fetcher: BlobFetcher, prefix: string, options: FetcherBlobStoreOptions = {}) { + this.#fetcher = fetcher; + this.#prefix = prefix.replace(/^\/+/, ""); + this.#keys = [...(options.keys ?? [])].sort(); + } + + async #fetch(key: string): Promise { + assertBlobKey(key); + const res = await this.#fetcher.fetch(`https://assets.local/${this.#prefix}${key}`); + if (res.status === 404) return null; + if (!res.ok) throw new StorageError(`fetching ${key} failed with ${res.status}`); + return res; + } + + async get(key: string): Promise { + const res = await this.#fetch(key); + return res ? new Uint8Array(await res.arrayBuffer()) : null; + } + + async head(key: string): Promise { + // A full read: the size and SHA-256 must be exact, and demo media are small. + const res = await this.#fetch(key); + if (!res) return null; + const data = new Uint8Array(await res.arrayBuffer()); + const ct = res.headers.get("content-type"); + return { + size: data.byteLength, + sha256: await sha256OfBytes(data), + ...(ct ? { contentType: ct } : {}), + }; + } + + async put(key: string): Promise { + throw new StorageError(`blob store "${this.name}" is read-only (put ${key})`); + } + + async delete(key: string): Promise { + throw new StorageError(`blob store "${this.name}" is read-only (delete ${key})`); + } + + async list(prefix = ""): Promise { + return this.#keys.filter((k) => k.startsWith(prefix)); + } +} diff --git a/packages/storage/src/blob/layered.ts b/packages/storage/src/blob/layered.ts new file mode 100644 index 00000000..56668257 --- /dev/null +++ b/packages/storage/src/blob/layered.ts @@ -0,0 +1,85 @@ +import { + assertBlobKey, + type BlobMeta, + type BlobPutOptions, + type BlobRange, + type BlobStore, + type BlobStream, + bytesStream, + resolveBlobRange, + StorageError, +} from "../types.ts"; + +/** + * Reads `upper` first, then `base`; every write goes to `upper`; `list` merges both (RFC-0036 §2). + * The playground puts read-only demo media (`base`) under a writable per-session store (`upper`). + * `base` is never written: deleting a key only removes `upper`'s copy, so a base object stays + * readable after a delete. + */ +export class LayeredBlobStore implements BlobStore { + readonly name: string; + readonly #base: BlobStore; + readonly #upper: BlobStore; + + constructor(base: BlobStore, upper: BlobStore) { + this.#base = base; + this.#upper = upper; + this.name = `layered(${upper.name}/${base.name})`; + } + + async get(key: string): Promise { + return (await this.#upper.get(key)) ?? this.#base.get(key); + } + + async head(key: string): Promise { + return (await this.#upper.head(key)) ?? this.#base.head(key); + } + + put(key: string, data: Uint8Array, options?: BlobPutOptions): Promise { + return this.#upper.put(key, data, options); + } + + async putStream( + key: string, + data: ReadableStream, + options: BlobPutOptions & { size: number; sha256: string }, + ): Promise { + if (this.#upper.putStream) return this.#upper.putStream(key, data, options); + const bytes = new Uint8Array(await new Response(data).arrayBuffer()); + return this.#upper.put(key, bytes, options); + } + + async getStream(key: string, options: { range?: BlobRange } = {}): Promise { + assertBlobKey(key); + for (const store of [this.#upper, this.#base]) { + if (store.getStream) { + const s = await store.getStream(key, options); + if (s) return s; + continue; + } + const data = await store.get(key); + if (data) return streamOf(data, options.range); + } + return null; + } + + delete(key: string): Promise { + return this.#upper.delete(key); + } + + async list(prefix = ""): Promise { + const [a, b] = await Promise.all([this.#upper.list(prefix), this.#base.list(prefix)]); + return [...new Set([...a, ...b])].sort(); + } +} + +function streamOf(data: Uint8Array, range: BlobRange | undefined): BlobStream { + if (!range) return { body: bytesStream(data), size: data.byteLength }; + const r = resolveBlobRange(range, data.byteLength); + if (!r) throw new StorageError("range not satisfiable"); + return { + body: bytesStream(data.slice(r.offset, r.offset + r.length)), + size: data.byteLength, + range: r, + }; +} diff --git a/packages/storage/src/blob/sql.ts b/packages/storage/src/blob/sql.ts new file mode 100644 index 00000000..bb1c6b87 --- /dev/null +++ b/packages/storage/src/blob/sql.ts @@ -0,0 +1,167 @@ +import { + assertBlobKey, + BlobIntegrityError, + type BlobMeta, + type BlobPutOptions, + type BlobRange, + type BlobStore, + type BlobStream, + bytesStream, + readAllBytes, + resolveBlobRange, + type SqlDriver, + StorageError, + StorageLimitError, + sha256OfBytes, + verifyBlob, +} from "../types.ts"; + +export interface SqlBlobStoreOptions { + /** Cap on the sum of all object sizes; a put past it throws `StorageLimitError`. Default: none. */ + readonly maxBytes?: number; + /** Cap on one object. Default 1 MiB, under DO SQLite's and D1's 2 MB value limit. */ + readonly maxObjectBytes?: number; +} + +const DDL = + "CREATE TABLE IF NOT EXISTS _blobs (key TEXT PRIMARY KEY, data BLOB NOT NULL, content_type TEXT, size INTEGER NOT NULL, sha256 TEXT NOT NULL)"; + +/** + * A `BlobStore` kept in a table of the same SQL database (RFC-0036 §2): one database holds a + * whole site, as in a playground session's Durable Object. Works over any `SqlDriver`. Every put + * records the SHA-256 and checks a declared one; the total-size cap is checked in the same + * statement that writes, so concurrent puts cannot overshoot it. + */ +export class SqlBlobStore implements BlobStore { + readonly name = "sql"; + readonly #sql: SqlDriver; + readonly #maxBytes: number; + readonly #maxObjectBytes: number; + #ready: Promise | undefined; + + constructor(sql: SqlDriver, options: SqlBlobStoreOptions = {}) { + this.#sql = sql; + this.#maxBytes = options.maxBytes ?? Number.MAX_SAFE_INTEGER; + this.#maxObjectBytes = options.maxObjectBytes ?? 1024 * 1024; + } + + #table(): Promise { + this.#ready ??= this.#sql.exec(DDL).catch((e: unknown) => { + this.#ready = undefined; + throw e; + }); + return this.#ready; + } + + async get(key: string): Promise { + assertBlobKey(key); + await this.#table(); + const row = await this.#sql.get<{ data: Uint8Array | ArrayBuffer }>( + "SELECT data FROM _blobs WHERE key = ?", + [key], + ); + return row ? new Uint8Array(row.data) : null; + } + + async head(key: string): Promise { + assertBlobKey(key); + await this.#table(); + const row = await this.#sql.get<{ size: number; content_type: string | null; sha256: string }>( + "SELECT size, content_type, sha256 FROM _blobs WHERE key = ?", + [key], + ); + if (!row) return null; + return { + size: row.size, + sha256: row.sha256, + ...(row.content_type === null ? {} : { contentType: row.content_type }), + }; + } + + async put(key: string, data: Uint8Array, options: BlobPutOptions = {}): Promise { + assertBlobKey(key); + if (options.sha256 !== undefined) await verifyBlob(data, options.sha256); + if (data.byteLength > this.#maxObjectBytes) + throw new StorageLimitError( + `blob is ${data.byteLength} bytes; the limit is ${this.#maxObjectBytes}`, + ); + await this.#table(); + const sha256 = options.sha256 ?? (await sha256OfBytes(data)); + const res = await this.#sql.run( + `INSERT INTO _blobs (key, data, content_type, size, sha256) + SELECT ?, ?, ?, ?, ? + WHERE (SELECT coalesce(sum(size), 0) FROM _blobs WHERE key <> ?) + ? <= ? + ON CONFLICT (key) DO UPDATE SET data = excluded.data, content_type = excluded.content_type, + size = excluded.size, sha256 = excluded.sha256`, + [ + key, + data, + options.contentType ?? null, + data.byteLength, + sha256, + key, + data.byteLength, + this.#maxBytes, + ], + ); + if (res.changes === 0) + throw new StorageLimitError(`blob store is full (limit ${this.#maxBytes} bytes)`); + } + + async putStream( + key: string, + data: ReadableStream, + options: BlobPutOptions & { size: number; sha256: string }, + ): Promise { + assertBlobKey(key); + if (options.size > this.#maxObjectBytes) + throw new StorageLimitError( + `blob is ${options.size} bytes; the limit is ${this.#maxObjectBytes}`, + ); + const bytes = await readAllBytes(data, options.size); + if (bytes.byteLength !== options.size) + throw new BlobIntegrityError(`expected ${options.size} bytes, got ${bytes.byteLength}`); + await this.put(key, bytes, options); + } + + async getStream(key: string, options: { range?: BlobRange } = {}): Promise { + const data = await this.get(key); + if (!data) return null; + const meta = await this.head(key); + const ct = meta?.contentType === undefined ? {} : { contentType: meta.contentType }; + const size = data.byteLength; + if (!options.range) return { body: bytesStream(data), size, ...ct }; + const range = resolveBlobRange(options.range, size); + if (!range) throw new StorageError("range not satisfiable"); + return { + body: bytesStream(data.slice(range.offset, range.offset + range.length)), + size, + range, + ...ct, + }; + } + + async delete(key: string): Promise { + assertBlobKey(key); + await this.#table(); + await this.#sql.run("DELETE FROM _blobs WHERE key = ?", [key]); + } + + async list(prefix = ""): Promise { + await this.#table(); + const rows = await this.#sql.all<{ key: string }>( + "SELECT key FROM _blobs WHERE substr(key, 1, ?) = ? ORDER BY key", + [prefix.length, prefix], + ); + return rows.map((r) => r.key); + } + + /** Sum of all object sizes, for usage displays and caps. */ + async totalBytes(): Promise { + await this.#table(); + const row = await this.#sql.get<{ n: number }>( + "SELECT coalesce(sum(size), 0) AS n FROM _blobs", + ); + return row?.n ?? 0; + } +} diff --git a/packages/storage/src/index.ts b/packages/storage/src/index.ts index cd61fff7..ea63ac27 100644 --- a/packages/storage/src/index.ts +++ b/packages/storage/src/index.ts @@ -3,10 +3,18 @@ * * Interfaces: `SqlDriver` (get/all/run/atomic batch/exec) and `BlobStore` (get/put/delete/list). * Adapters: `@cloudhal/storage/sqlite` (better-sqlite3), `@cloudhal/storage/d1`, `@cloudhal/storage/fs`, - * `@cloudhal/storage/r2`, plus `MemoryBlobStore` here. Conformance suites: `@cloudhal/storage/testing`. + * `@cloudhal/storage/r2`, `@cloudhal/storage/do-sqlite` (Durable Object SQLite), plus `MemoryBlobStore`, + * `SqlBlobStore`, `LayeredBlobStore` and `FetcherBlobStore` here. Conformance suites: `@cloudhal/storage/testing`. */ export { migrations } from "../migrations/index.ts"; +export { + type BlobFetcher, + FetcherBlobStore, + type FetcherBlobStoreOptions, +} from "./blob/fetcher.ts"; +export { LayeredBlobStore } from "./blob/layered.ts"; export { MemoryBlobStore } from "./blob/memory.ts"; +export { SqlBlobStore, type SqlBlobStoreOptions } from "./blob/sql.ts"; export * from "./migrate.ts"; export * from "./types.ts"; diff --git a/packages/storage/src/sql/do-sqlite.ts b/packages/storage/src/sql/do-sqlite.ts new file mode 100644 index 00000000..b4398827 --- /dev/null +++ b/packages/storage/src/sql/do-sqlite.ts @@ -0,0 +1,94 @@ +import { + assertBatchLimits, + assertStatementLimits, + type RunResult, + type SqlDriver, + type SqlStatement, + type SqlValue, + StorageError, +} from "../types.ts"; + +type Binding = string | number | null | ArrayBuffer; + +/** + * SQLite-backed Durable Object adapter (RFC-0036 §2) over `ctx.storage` of the DO that owns the + * database. Every call runs synchronously on the DO's own SQLite; the async signature is the + * `SqlDriver` contract. `batch` runs inside `transactionSync`, so a throw rolls every statement back; + * the transaction is never handed to callers, and `BEGIN` is refused by the runtime, so the + * batch-only rule holds here as on D1 (ADR-0002). D1's limits are enforced like on every adapter. + */ +export class DoSqliteDriver implements SqlDriver { + readonly name = "do-sqlite"; + readonly #storage: DurableObjectStorage; + + constructor(storage: DurableObjectStorage) { + this.#storage = storage; + } + + /** Bytes the database takes on disk (`ctx.storage.sql.databaseSize`), for size caps. */ + get databaseSize(): number { + return this.#storage.sql.databaseSize; + } + + #rows(sql: string, params: readonly SqlValue[] = []): T[] { + assertStatementLimits(sql, params); + const cursor = this.#storage.sql.exec(sql, ...params.map(toBinding)); + return cursor.toArray().map((row) => fromRow(row) as T); + } + + #run(sql: string, params: readonly SqlValue[] = []): RunResult { + this.#rows(sql, params); + const meta = this.#storage.sql.exec("SELECT changes() AS c, last_insert_rowid() AS id").one(); + const changes = Number(meta.c ?? 0); + return { changes, lastRowId: changes > 0 ? Number(meta.id) : null }; + } + + async get>( + sql: string, + params?: readonly SqlValue[], + ): Promise { + return this.#rows(sql, params)[0]; + } + + async all>( + sql: string, + params?: readonly SqlValue[], + ): Promise { + return this.#rows(sql, params); + } + + async run(sql: string, params?: readonly SqlValue[]): Promise { + return this.#run(sql, params); + } + + async batch(statements: readonly SqlStatement[]): Promise { + assertBatchLimits(statements); + try { + return this.#storage.transactionSync(() => statements.map((s) => this.#run(s.sql, s.params))); + } catch (cause) { + throw new StorageError(`batch failed: ${(cause as Error).message}`, cause); + } + } + + async exec(sql: string): Promise { + this.#storage.sql.exec(sql); + } + + async close(): Promise { + // The database belongs to the Durable Object; nothing to release. + } +} + +function toBinding(value: SqlValue): Binding { + if (value instanceof Uint8Array) + return value.buffer.slice(value.byteOffset, value.byteOffset + value.byteLength) as ArrayBuffer; + return value; +} + +/** BLOB columns come back as ArrayBuffer; hand them out as Uint8Array like the other adapters. */ +function fromRow(row: Record): Record { + const out: Record = {}; + for (const [k, v] of Object.entries(row)) + out[k] = v instanceof ArrayBuffer ? new Uint8Array(v) : v; + return out; +} diff --git a/packages/storage/src/testing-do.ts b/packages/storage/src/testing-do.ts new file mode 100644 index 00000000..fab082a5 --- /dev/null +++ b/packages/storage/src/testing-do.ts @@ -0,0 +1,146 @@ +/** + * Test harness for `DoSqliteDriver` (RFC-0036 §2 invariant 6). A Miniflare Worker holds a test-only, + * SQLite-backed Durable Object that runs the real driver on its own `ctx.storage`; from Node, every + * `SqlDriver` call is one request to that object. Suites written for any `SqlDriver` (the storage + * conformance suite, core-ts's behavioural suite) thereby run against Durable Object SQLite. + */ +import { + BlobIntegrityError, + type RunResult, + type SqlDriver, + type SqlStatement, + type SqlValue, + StorageError, + StorageLimitError, +} from "./types.ts"; + +const WORKER = ` +import { DurableObject } from "cloudflare:workers"; +import { DoSqliteDriver } from "./sql/do-sqlite.EXT"; +const b64 = (u) => { let s = ""; for (const c of u) s += String.fromCharCode(c); return btoa(s); }; +const unb64 = (s) => Uint8Array.from(atob(s), (c) => c.charCodeAt(0)); +const revive = (_k, v) => (v && typeof v === "object" && typeof v.$b === "string" ? unb64(v.$b) : v); +const encode = (v) => JSON.stringify(v ?? null, (_k, x) => (x instanceof Uint8Array ? { $b: b64(x) } : x)); +export class TestDb extends DurableObject { + constructor(ctx, env) { super(ctx, env); this.driver = new DoSqliteDriver(ctx.storage); } + async fetch(req) { + const { op, args } = JSON.parse(await req.text(), revive); + try { + return new Response(encode({ ok: await this.driver[op](...args) })); + } catch (e) { + return new Response(encode({ error: { name: e.name, message: e.message } })); + } + } +} +export default { + fetch(req, env) { + const db = new URL(req.url).searchParams.get("db") ?? "default"; + return env.DB.get(env.DB.idFromName(db)).fetch(req); + }, +}; +`; + +const ERRORS: Record Error> = { + StorageError, + StorageLimitError, + BlobIntegrityError, +}; + +const toB64 = (u: Uint8Array) => Buffer.from(u).toString("base64"); +const encode = (v: unknown) => + JSON.stringify(v, (_k, x) => (x instanceof Uint8Array ? { $b: toB64(x) } : x)); +const revive = (_k: string, v: unknown) => + v && typeof v === "object" && typeof (v as { $b?: unknown }).$b === "string" + ? new Uint8Array(Buffer.from((v as { $b: string }).$b, "base64")) + : v; + +type Dispatch = ( + url: string, + init: { method: string; body: string }, +) => Promise<{ text(): Promise }>; + +/** Node-side `SqlDriver` whose every call runs `DoSqliteDriver` inside the test Durable Object. */ +class RemoteDoSqliteDriver implements SqlDriver { + readonly name = "do-sqlite"; + readonly #dispatch: Dispatch; + readonly #db: string; + + constructor(dispatch: Dispatch, db: string) { + this.#dispatch = dispatch; + this.#db = db; + } + + async #call(op: string, args: unknown[]): Promise { + const res = await this.#dispatch(`http://do.test/?db=${encodeURIComponent(this.#db)}`, { + method: "POST", + body: encode({ op, args }), + }); + const out = JSON.parse(await res.text(), revive) as { + ok?: T; + error?: { name: string; message: string }; + }; + if (out.error) { + const E = ERRORS[out.error.name] ?? Error; + throw new E(out.error.message); + } + return out.ok as T; + } + + async get>(sql: string, params?: readonly SqlValue[]) { + return (await this.#call("get", [sql, params ?? []])) ?? undefined; + } + all>(sql: string, params?: readonly SqlValue[]) { + return this.#call("all", [sql, params ?? []]); + } + run(sql: string, params?: readonly SqlValue[]) { + return this.#call("run", [sql, params ?? []]); + } + batch(statements: readonly SqlStatement[]) { + return this.#call("batch", [statements]); + } + async exec(sql: string) { + await this.#call("exec", [sql]); + } + async close() {} +} + +/** + * Spins up a Miniflare Worker with a SQLite-backed Durable Object running `DoSqliteDriver`. + * `driver(db)` returns a driver for the object named `db` (a fresh, empty database per name). + */ +export async function createMiniflareDoSqlite(): Promise<{ + driver(db?: string): SqlDriver; + dispose(): Promise; +}> { + const { Miniflare } = await import("miniflare"); + const { readFile } = await import("node:fs/promises"); + const { stripTypeScriptTypes } = await import("node:module"); + const { fileURLToPath } = await import("node:url"); + // Sources in the repo are TypeScript; the published package carries compiled JavaScript. + const ext = import.meta.url.endsWith(".ts") ? "ts" : "js"; + const load = async (rel: string) => { + const text = await readFile(fileURLToPath(new URL(rel, import.meta.url).href), "utf8"); + return ext === "ts" ? stripTypeScriptTypes(text) : text; + }; + const root = fileURLToPath(new URL(".", import.meta.url).href); + const mf = new Miniflare({ + compatibilityDate: "2026-07-01", + modulesRoot: root, + modules: [ + { type: "ESModule", path: `${root}worker.js`, contents: WORKER.replace("EXT", ext) }, + { + type: "ESModule", + path: `${root}sql/do-sqlite.${ext}`, + contents: await load(`./sql/do-sqlite.${ext}`), + }, + { type: "ESModule", path: `${root}types.${ext}`, contents: await load(`./types.${ext}`) }, + ], + durableObjects: { DB: { className: "TestDb", useSQLite: true } }, + }); + await mf.ready; + const dispatch: Dispatch = (url, init) => mf.dispatchFetch(url, init); + return { + driver: (db = "default") => new RemoteDoSqliteDriver(dispatch, db), + dispose: () => mf.dispose(), + }; +} diff --git a/packages/storage/src/testing.ts b/packages/storage/src/testing.ts index cac930c8..2ef846d5 100644 --- a/packages/storage/src/testing.ts +++ b/packages/storage/src/testing.ts @@ -357,3 +357,4 @@ export async function createMiniflareR2(): Promise<{ bucket: R2Bucket; dispose() const bucket = (await mf.getR2Bucket("BUCKET")) as unknown as R2Bucket; return { bucket, dispose: () => mf.dispose() }; } +export { createMiniflareDoSqlite } from "./testing-do.ts"; diff --git a/packages/storage/test/blob-layers.test.ts b/packages/storage/test/blob-layers.test.ts new file mode 100644 index 00000000..3171d09d --- /dev/null +++ b/packages/storage/test/blob-layers.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, it } from "vitest"; +import { FetcherBlobStore } from "../src/blob/fetcher.ts"; +import { LayeredBlobStore } from "../src/blob/layered.ts"; +import { MemoryBlobStore } from "../src/blob/memory.ts"; +import { SqlBlobStore } from "../src/blob/sql.ts"; +import { BetterSqlite3Driver } from "../src/sql/better-sqlite3.ts"; +import { describeBlobStoreConformance } from "../src/testing.ts"; +import { StorageError, StorageLimitError, sha256OfBytes } from "../src/types.ts"; + +const enc = new TextEncoder(); +const dec = new TextDecoder(); + +describeBlobStoreConformance("sql over better-sqlite3", { + create: async () => new SqlBlobStore(await BetterSqlite3Driver.open(":memory:")), +}); + +describeBlobStoreConformance("layered (memory over empty memory)", { + create: async () => new LayeredBlobStore(new MemoryBlobStore(), new MemoryBlobStore()), +}); + +/** A stand-in for the Worker's ASSETS binding. */ +function assets(files: Record) { + const calls: string[] = []; + return { + calls, + async fetch(input: string) { + const path = new URL(input).pathname.slice(1); + calls.push(path); + const f = files[path]; + if (!f) return new Response("not found", { status: 404 }); + return new Response(f.body, { headers: f.type ? { "content-type": f.type } : {} }); + }, + }; +} + +describe("SqlBlobStore caps", () => { + it("refuses an object over maxObjectBytes and a put past maxBytes; overwrites count once", async () => { + const store = new SqlBlobStore(await BetterSqlite3Driver.open(":memory:"), { + maxBytes: 10, + maxObjectBytes: 6, + }); + await expect(store.put("a", new Uint8Array(7))).rejects.toThrow(StorageLimitError); + await store.put("a", new Uint8Array(6)); + await store.put("b", new Uint8Array(4)); + await expect(store.put("c", new Uint8Array(1))).rejects.toThrow(StorageLimitError); + expect(await store.head("c")).toBeNull(); + await store.put("a", new Uint8Array(5)); // replaces 6 bytes: 9 in all + expect(await store.totalBytes()).toBe(9); + await store.delete("b"); + await store.put("c", new Uint8Array(4)); + expect(await store.list()).toEqual(["a", "c"]); + }); + + it("records the SHA-256 of every object", async () => { + const store = new SqlBlobStore(await BetterSqlite3Driver.open(":memory:")); + const data = enc.encode("hello"); + await store.put("h", data, { contentType: "text/plain" }); + expect(await store.head("h")).toEqual({ + size: 5, + contentType: "text/plain", + sha256: await sha256OfBytes(data), + }); + }); + + it("lists by prefix without LIKE wildcards leaking", async () => { + const store = new SqlBlobStore(await BetterSqlite3Driver.open(":memory:")); + for (const k of ["a_b/1", "axb/1", "a_b2"]) await store.put(k, enc.encode(k)); + expect(await store.list("a_b/")).toEqual(["a_b/1"]); + }); +}); + +describe("FetcherBlobStore", () => { + const files = { + "demo/media/a.png": { body: "PNG", type: "image/png" }, + "demo/media/b.txt": { body: "text" }, + }; + + it("reads static files under its prefix; missing is null", async () => { + const fetcher = assets(files); + const store = new FetcherBlobStore(fetcher, "demo/", { keys: ["media/b.txt", "media/a.png"] }); + expect(dec.decode((await store.get("media/a.png")) as Uint8Array)).toBe("PNG"); + expect(await store.head("media/a.png")).toEqual({ + size: 3, + contentType: "image/png", + sha256: await sha256OfBytes(enc.encode("PNG")), + }); + expect(await store.get("media/none")).toBeNull(); + expect(await store.head("media/none")).toBeNull(); + expect(await store.list("media/")).toEqual(["media/a.png", "media/b.txt"]); + expect(fetcher.calls[0]).toBe("demo/media/a.png"); + }); + + it("is read-only and checks keys before fetching", async () => { + const fetcher = assets(files); + const store = new FetcherBlobStore(fetcher, "demo/"); + await expect(store.put("x")).rejects.toThrow(StorageError); + await expect(store.delete("media/a.png")).rejects.toThrow(StorageError); + await expect(store.get("../secret")).rejects.toThrow(StorageError); + expect(fetcher.calls).toEqual([]); + }); + + it("reports other failures as StorageError", async () => { + const store = new FetcherBlobStore( + { fetch: async () => new Response("", { status: 500 }) }, + "", + ); + await expect(store.get("a")).rejects.toThrow(StorageError); + }); +}); + +describe("LayeredBlobStore", () => { + it("reads upper over base, writes to upper, merges lists and keeps base objects on delete", async () => { + const base = new FetcherBlobStore( + assets({ "demo/media/a": { body: "base-a" }, "demo/media/b": { body: "base-b" } }), + "demo/", + { + keys: ["media/a", "media/b"], + }, + ); + const upper = new MemoryBlobStore(); + const store = new LayeredBlobStore(base, upper); + expect(dec.decode((await store.get("media/a")) as Uint8Array)).toBe("base-a"); + await store.put("media/a", enc.encode("mine")); + await store.put("nodes/n.json", enc.encode("{}")); + expect(dec.decode((await store.get("media/a")) as Uint8Array)).toBe("mine"); + expect(await upper.list()).toEqual(["media/a", "nodes/n.json"]); + expect(await store.list()).toEqual(["media/a", "media/b", "nodes/n.json"]); + expect(await store.list("media/")).toEqual(["media/a", "media/b"]); + await store.delete("media/a"); + expect(dec.decode((await store.get("media/a")) as Uint8Array)).toBe("base-a"); + const stream = await store.getStream("media/b", { range: { suffix: 1 } }); + expect(dec.decode(await new Response(stream?.body).arrayBuffer())).toBe("b"); + expect(await store.getStream("media/none")).toBeNull(); + }); +}); diff --git a/packages/storage/test/do-sqlite.test.ts b/packages/storage/test/do-sqlite.test.ts new file mode 100644 index 00000000..195d40f7 --- /dev/null +++ b/packages/storage/test/do-sqlite.test.ts @@ -0,0 +1,53 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { migrations } from "../migrations/index.ts"; +import { SqlBlobStore } from "../src/blob/sql.ts"; +import { migrate } from "../src/migrate.ts"; +import { + createMiniflareDoSqlite, + describeBlobStoreConformance, + describeSqlDriverConformance, +} from "../src/testing.ts"; + +// One Miniflare for the whole file: suites run in order, never in parallel (test-load rule). +let handle: Awaited> | undefined; +const open = async (db: string) => { + handle ??= await createMiniflareDoSqlite(); + return handle.driver(db); +}; + +describeSqlDriverConformance("Durable Object SQLite (Miniflare)", { + create: () => open("conformance"), +}); + +describeBlobStoreConformance("sql over Durable Object SQLite (Miniflare)", { + create: async () => new SqlBlobStore(await open("blobs")), +}); + +describe("DoSqliteDriver", () => { + beforeAll(async () => { + await open("init"); + }); + afterAll(async () => { + await handle?.dispose(); + handle = undefined; + }); + + it("applies every Cloudhal migration, FTS5 included", async () => { + const driver = await open("migrations"); + const { applied } = await migrate(driver, migrations); + expect(applied).toEqual(migrations.map((m) => m.name)); + expect(applied.length).toBeGreaterThan(20); + const fts = await driver.all<{ name: string }>( + "SELECT name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'", + ); + expect(fts.length).toBeGreaterThan(0); + expect((await migrate(driver, migrations)).applied).toEqual([]); + }); + + it("keeps one database per object name", async () => { + const a = await open("iso-a"); + const b = await open("iso-b"); + await a.exec("CREATE TABLE only_a (x INTEGER)"); + await expect(b.run("INSERT INTO only_a (x) VALUES (1)")).rejects.toThrow(); + }); +});