feat(storage): Durable Object SQLite driver and SQL, layered and fetcher blob stores (RFC-0036 phase 1)

.changeset/storage-do-sqlite.md

@@ -0,0 +1,5 @@
+---
+"@cloudhal/storage": minor
+---
+
+New `@cloudhal/storage/do-sqlite`: `DoSqliteDriver`, a `SqlDriver` over a SQLite-backed Durable Object's storage, with an atomic `batch` (`transactionSync`) and D1's limits (RFC-0036). New blob stores at the root: `SqlBlobStore` keeps blobs in a table of the same database with per-object and total caps, `LayeredBlobStore` reads a writable store over a read-only one, and `FetcherBlobStore` reads static files through a Worker `ASSETS` binding. `@cloudhal/storage/testing` gains `createMiniflareDoSqlite()`, which runs any `SqlDriver` suite on Durable Object SQLite.

AGENTS.md

@@ -26,7 +26,7 @@ pnpm i18n:export / pnpm i18n:import [--locale xx]  # translations as nested ICU
 - `packages/core-contracts/schemas/*.schema.json` — **the only hand-edited contract source**. `common.schema.json` holds shared `$defs`.
 - `packages/core-contracts/src/generated/`, `crates/cloudhal-schema/src/generated/` — generated; never edit.
 - `packages/core-contracts/fixtures/{valid,invalid}/<schema>/` — golden fixtures used by both languages.
-- `packages/storage` — `SqlDriver`/`BlobStore` + adapters (`sqlite`, `d1`, `fs`, `r2`) + migrations (`migrations/*.ts`, plain SQL). Conformance suites in `src/testing.ts`.
+- `packages/storage` — `SqlDriver`/`BlobStore` + adapters (`sqlite`, `d1`, `do-sqlite`, `fs`, `r2`; `SqlBlobStore`, `LayeredBlobStore`, `FetcherBlobStore` at the root) + migrations (`migrations/*.ts`, plain SQL). Conformance suites in `src/testing.ts`.
 - `packages/core-ts` — `SiteGraphStore` (CAS on `rev`, snapshots, diff, publish, rollback). Behavioural suite in `src/testing.ts` runs on sqlite and Miniflare D1. `./local` opens the offline store; `seedStarterSite` seeds a demo. `./surfaces` is RFC-0020's generative UI: the catalog (primitives + block types), eval-free surface checks, bindings, repairs, JSON Patch (`patchPage`), `catalogPrompt`; `./interop` converts A2UI v0.9 and json-render to and from surfaces; `./design` is RFC-0021's DESIGN.md lint and guidance. `search.ts` is RFC-0019's FTS5 site search (index written with the publish batch; articles, pages, terms and authors, RFC-0029); `search-provider.ts` is RFC-0029's `SearchProvider` (`fts5`, `memory`, `withFallback`), set with `SiteGraphStoreOptions.search`; `search-ai.ts` is the experimental Cloudflare AI Search provider (`hybridProvider`, `searchProviderFromEnv`, `cloudhal.json` `search`, docs/guides/search.md).
 - `packages/schedules` — RFC-0008: time model (only `temporal-polyfill/implementation`, `now` injected, never `Temporal.Now`), RRULE subset, release model (`rl1:`), automation/run state, taint and budgets, `runDue`; `./sql` is `SqlScheduleStore`, `./engine` the Temporal-free part core-ts and the site Worker use. Worker-safe; conformance suites in `src/testing.ts` on sqlite and Miniflare D1.
 - `packages/astro-renderer` — `GraphSource` (live/snapshot), `cloudhal()` integration (virtual config + dev route-cache watcher), `.astro` section components (`Surface.astro` for RFC-0020 surfaces; `./surfaces` renders them, `./kit` has the table of contents and reading time). Routes live in starters, not here.

packages/core-ts/test/store.test.ts

@@ -1,7 +1,7 @@
-import { MemoryBlobStore } from "@cloudhal/storage";
+import { MemoryBlobStore, SqlBlobStore } from "@cloudhal/storage";
 import { D1Driver } from "@cloudhal/storage/d1";
 import { BetterSqlite3Driver } from "@cloudhal/storage/sqlite";
-import { createMiniflareD1 } from "@cloudhal/storage/testing";
+import { createMiniflareD1, createMiniflareDoSqlite } from "@cloudhal/storage/testing";
 import { describeSiteGraphStore } from "../src/testing.ts";
 import { correctionsSuite } from "../src/testing-corrections.ts";
 import { gatesSuite } from "../src/testing-gates.ts";
@@ -44,3 +44,19 @@ releasesSuite("Cloudflare D1 (Miniflare) + memory blobs", miniflare);
 correctionsSuite("Cloudflare D1 (Miniflare) + memory blobs", miniflare);
 searchSuite("Cloudflare D1 (Miniflare) + memory blobs", miniflare);
 searchKindsSuite("Cloudflare D1 (Miniflare) + memory blobs", miniflare);
+
+// RFC-0036: the playground's backend. Blobs live in the same Durable Object database.
+let doSqlite: Awaited<ReturnType<typeof createMiniflareDoSqlite>> | undefined;
+const durableObject = {
+  async create() {
+    doSqlite ??= await createMiniflareDoSqlite();
+    const sql = doSqlite.driver("store");
+    return { sql, blobs: new SqlBlobStore(sql) };
+  },
+  async dispose() {
+    await doSqlite?.dispose();
+    doSqlite = undefined;
+  },
+};
+describeSiteGraphStore("Durable Object SQLite (Miniflare) + sql blobs", durableObject);
+searchSuite("Durable Object SQLite (Miniflare) + sql blobs", durableObject);

packages/storage/README.md

@@ -4,12 +4,13 @@ Storage interfaces and adapters for Cloudhal. The interface is shaped by the mos
 
 | Export | What |
 |---|---|
-| `@cloudhal/storage` | `SqlDriver`, `BlobStore`, limits, `migrate()`, `migrations`, `MemoryBlobStore` |
+| `@cloudhal/storage` | `SqlDriver`, `BlobStore`, limits, `migrate()`, `migrations`, `MemoryBlobStore`, `SqlBlobStore` (blobs in a SQL table, capped), `LayeredBlobStore` (read-only base under a writable upper), `FetcherBlobStore` (read-only, over a Worker `ASSETS` binding) |
 | `@cloudhal/storage/sqlite` | `BetterSqlite3Driver` — local files or `:memory:` |
 | `@cloudhal/storage/d1` | `D1Driver` — Cloudflare D1 binding |
+| `@cloudhal/storage/do-sqlite` | `DoSqliteDriver` — a SQLite-backed Durable Object's `ctx.storage` (RFC-0036); `batch` runs in `transactionSync` |
 | `@cloudhal/storage/fs` | `FsBlobStore` — directory on disk |
 | `@cloudhal/storage/r2` | `R2BlobStore` — Cloudflare R2 binding |
-| `@cloudhal/storage/testing` | conformance suites + Miniflare helpers (`createMiniflareD1`, `createMiniflareR2`) |
+| `@cloudhal/storage/testing` | conformance suites + Miniflare helpers (`createMiniflareD1`, `createMiniflareR2`, `createMiniflareDoSqlite`) |
 
 ```ts
 import { migrate, migrations } from "@cloudhal/storage";
@@ -24,4 +25,18 @@ await sql.batch([
 
 Migrations are TypeScript modules under `migrations/` (plain SQL strings) so they bundle for Workers without filesystem access. Each migration is applied as one atomic batch.
 
-Every adapter must pass `describeSqlDriverConformance` / `describeBlobStoreConformance`; the D1 and R2 suites run against Miniflare (workerd in-process), so no Cloudflare account is needed.
+Every adapter must pass `describeSqlDriverConformance` / `describeBlobStoreConformance`; the D1, R2 and Durable Object suites run against Miniflare (workerd in-process), so no Cloudflare account is needed. `createMiniflareDoSqlite()` runs `DoSqliteDriver` inside a test-only Durable Object and hands Node a driver whose every call is one request to it, so any `SqlDriver` suite (core-ts's included) runs on Durable Object SQLite.
+
+A whole site in one Durable Object (the playground's shape):
+
+```ts
+import { FetcherBlobStore, LayeredBlobStore, migrate, migrations, SqlBlobStore } from "@cloudhal/storage";
+import { DoSqliteDriver } from "@cloudhal/storage/do-sqlite";
+
+const sql = new DoSqliteDriver(ctx.storage);
+await migrate(sql, migrations);
+const blobs = new LayeredBlobStore(
+  new FetcherBlobStore(env.ASSETS, "demo/"),             // read-only demo media
+  new SqlBlobStore(sql, { maxBytes: 8 * 1024 * 1024 }), // the session's own blobs
+);
+```

packages/storage/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@cloudhal/storage",
   "version": "0.2.0",
-  "description": "Cloudhal storage interfaces (SQL driver, blob store) with Cloudflare D1/R2 and local SQLite/filesystem adapters",
+  "description": "Cloudhal storage interfaces (SQL driver, blob store) with Cloudflare D1/R2/Durable Object SQLite and local SQLite/filesystem adapters",
   "license": "Apache-2.0",
   "homepage": "https://cloudhal.dev",
   "repository": {
@@ -17,6 +17,7 @@
     ".": "./src/index.ts",
     "./sqlite": "./src/sql/better-sqlite3.ts",
     "./d1": "./src/sql/d1.ts",
+    "./do-sqlite": "./src/sql/do-sqlite.ts",
     "./r2": "./src/blob/r2.ts",
     "./fs": "./src/blob/fs.ts",
     "./testing": "./src/testing.ts",
@@ -70,6 +71,10 @@
         "types": "./dist/src/sql/d1.d.ts",
         "default": "./dist/src/sql/d1.js"
       },
+      "./do-sqlite": {
+        "types": "./dist/src/sql/do-sqlite.d.ts",
+        "default": "./dist/src/sql/do-sqlite.js"
+      },
       "./r2": {
         "types": "./dist/src/blob/r2.d.ts",
         "default": "./dist/src/blob/r2.js"

packages/storage/src/blob/fetcher.ts

@@ -0,0 +1,73 @@
+import {
+  assertBlobKey,
+  type BlobMeta,
+  type BlobStore,
+  StorageError,
+  sha256OfBytes,
+} from "../types.ts";
+
+/** The part of a Worker `Fetcher` binding (e.g. `ASSETS`) this store uses. */
+export interface BlobFetcher {
+  fetch(input: string): Promise<Response>;
+}
+
+export interface FetcherBlobStoreOptions {
+  /** Keys `list` reports (static assets cannot be listed). Default: none. */
+  readonly keys?: readonly string[];
+}
+
+/**
+ * A read-only `BlobStore` over static files (RFC-0036 §2): `key` is fetched as
+ * `/<prefix><key>` from the Worker's `ASSETS` binding. Writes throw `StorageError`. Used as the
+ * base layer of a `LayeredBlobStore` for demo media.
+ */
+export class FetcherBlobStore implements BlobStore {
+  readonly name = "fetcher";
+  readonly #fetcher: BlobFetcher;
+  readonly #prefix: string;
+  readonly #keys: readonly string[];
+
+  constructor(fetcher: BlobFetcher, prefix: string, options: FetcherBlobStoreOptions = {}) {
+    this.#fetcher = fetcher;
+    this.#prefix = prefix.replace(/^\/+/, "");
+    this.#keys = [...(options.keys ?? [])].sort();
+  }
+
+  async #fetch(key: string): Promise<Response | null> {
+    assertBlobKey(key);
+    const res = await this.#fetcher.fetch(`https://assets.local/${this.#prefix}${key}`);
+    if (res.status === 404) return null;
+    if (!res.ok) throw new StorageError(`fetching ${key} failed with ${res.status}`);
+    return res;
+  }
+
+  async get(key: string): Promise<Uint8Array | null> {
+    const res = await this.#fetch(key);
+    return res ? new Uint8Array(await res.arrayBuffer()) : null;
+  }
+
+  async head(key: string): Promise<BlobMeta | null> {
+    // A full read: the size and SHA-256 must be exact, and demo media are small.
+    const res = await this.#fetch(key);
+    if (!res) return null;
+    const data = new Uint8Array(await res.arrayBuffer());
+    const ct = res.headers.get("content-type");
+    return {
+      size: data.byteLength,
+      sha256: await sha256OfBytes(data),
+      ...(ct ? { contentType: ct } : {}),
+    };
+  }
+
+  async put(key: string): Promise<void> {
+    throw new StorageError(`blob store "${this.name}" is read-only (put ${key})`);
+  }
+
+  async delete(key: string): Promise<void> {
+    throw new StorageError(`blob store "${this.name}" is read-only (delete ${key})`);
+  }
+
+  async list(prefix = ""): Promise<string[]> {
+    return this.#keys.filter((k) => k.startsWith(prefix));
+  }
+}

packages/storage/src/blob/layered.ts

@@ -0,0 +1,85 @@
+import {
+  assertBlobKey,
+  type BlobMeta,
+  type BlobPutOptions,
+  type BlobRange,
+  type BlobStore,
+  type BlobStream,
+  bytesStream,
+  resolveBlobRange,
+  StorageError,
+} from "../types.ts";
+
+/**
+ * Reads `upper` first, then `base`; every write goes to `upper`; `list` merges both (RFC-0036 §2).
+ * The playground puts read-only demo media (`base`) under a writable per-session store (`upper`).
+ * `base` is never written: deleting a key only removes `upper`'s copy, so a base object stays
+ * readable after a delete.
+ */
+export class LayeredBlobStore implements BlobStore {
+  readonly name: string;
+  readonly #base: BlobStore;
+  readonly #upper: BlobStore;
+
+  constructor(base: BlobStore, upper: BlobStore) {
+    this.#base = base;
+    this.#upper = upper;
+    this.name = `layered(${upper.name}/${base.name})`;
+  }
+
+  async get(key: string): Promise<Uint8Array | null> {
+    return (await this.#upper.get(key)) ?? this.#base.get(key);
+  }
+
+  async head(key: string): Promise<BlobMeta | null> {
+    return (await this.#upper.head(key)) ?? this.#base.head(key);
+  }
+
+  put(key: string, data: Uint8Array, options?: BlobPutOptions): Promise<void> {
+    return this.#upper.put(key, data, options);
+  }
+
+  async putStream(
+    key: string,
+    data: ReadableStream<Uint8Array>,
+    options: BlobPutOptions & { size: number; sha256: string },
+  ): Promise<void> {
+    if (this.#upper.putStream) return this.#upper.putStream(key, data, options);
+    const bytes = new Uint8Array(await new Response(data).arrayBuffer());
+    return this.#upper.put(key, bytes, options);
+  }
+
+  async getStream(key: string, options: { range?: BlobRange } = {}): Promise<BlobStream | null> {
+    assertBlobKey(key);
+    for (const store of [this.#upper, this.#base]) {
+      if (store.getStream) {
+        const s = await store.getStream(key, options);
+        if (s) return s;
+        continue;
+      }
+      const data = await store.get(key);
+      if (data) return streamOf(data, options.range);
+    }
+    return null;
+  }
+
+  delete(key: string): Promise<void> {
+    return this.#upper.delete(key);
+  }
+
+  async list(prefix = ""): Promise<string[]> {
+    const [a, b] = await Promise.all([this.#upper.list(prefix), this.#base.list(prefix)]);
+    return [...new Set([...a, ...b])].sort();
+  }
+}
+
+function streamOf(data: Uint8Array, range: BlobRange | undefined): BlobStream {
+  if (!range) return { body: bytesStream(data), size: data.byteLength };
+  const r = resolveBlobRange(range, data.byteLength);
+  if (!r) throw new StorageError("range not satisfiable");
+  return {
+    body: bytesStream(data.slice(r.offset, r.offset + r.length)),
+    size: data.byteLength,
+    range: r,
+  };
+}