diff --git a/.changeset/rfc45-step4-field-operations.md b/.changeset/rfc45-step4-field-operations.md new file mode 100644 index 00000000..edfde757 --- /dev/null +++ b/.changeset/rfc45-step4-field-operations.md @@ -0,0 +1,18 @@ +--- +"@cloudhal/core-ts": patch +"@cloudhal/api": patch +"@cloudhal/cli": patch +--- + +RFC-0045 rollout step 4: the four field operations work on reference and repeater fields. The types themselves stay refused on every production write path until step 7 (`fields_unsupported`, now from the dry run's check of the definitions the operation ends with); the shared suites enable them with `SiteGraphStoreOptions.fieldTypes`. Nothing changes for operations on the nine existing types. + +- **`sub`** on all four operations (`FieldOperation.sub`, `FieldMigration.sub`): with it, `rename_field`, `change_field_type` and `remove_field` rename, convert or remove one sub-field of a repeater in every item. An item left empty is dropped, and a list left empty is no value. `set_field_localized` refuses `sub` (sub-fields share their repeater's `localized`). Readers, writers and the rows read items through the marker; a top-level reader ignores a sub-field rename's `fromKey` (`renamedFrom`). +- **New `change_field_type` inputs:** `collections`, `maxItems`, `subFields` and `itemLabel`, each refused where the target type does not take it. A reference changes to a reference only to switch `multiple`. A conversion that carries ids over may only widen the target collections. A repeater converts to text or to a reference only with exactly one matching sub-field. +- **New conversion rows** (`fields/conversions-references.ts`): reference single ↔ multiple, text ↔ reference (through the prefetched `ConvertContext.entries` maps, never the entry's own group), multiline text → repeater, repeater → text, and multiple reference ↔ repeater. The maps are read once per operation pass, and by `EntryEngine.write` while such a conversion runs. +- **Text ↔ reference values** are decided by one rule (`fields/text-reference.ts`), per value, from the `entry_refs` rows that phase C's pending check reads too: each operation pass loads the path's rows once, and a person's write reads its node's. A value with its row is kept; a legacy id-shaped text is written as a reference when its group is usable, else reported; a value that does not convert keeps its rows as they are. Readers use a display rule. Rebuilds (pull, push, import, restore) read a converting path through its earlier type. The renderer's conversion maps read archived entries too, so an archived target lends its title. +- **Phase B's target guard** covers only the ids a rewrite newly names; ids a reshape or a rename carries over stay as stored, rows kept. **`set_field_localized`** updates the rows' `shared` in the batch of its site write (phase A, and the revert's). +- **Revert** drops restored references whose group has no member left in the site, and reports them in `FieldRevertResult.dropped`. +- **Values that stop converting while an operation runs** (a target archived or deleted, a second entry with the same title) are reported in `unconvertible` and keep the operation running. `continueFieldOperation` and `revertFieldOperation` take `onInvalid: "clear"` to drop them, as do the admin API's continue and revert routes (in the body) and `cloudhal fields continue|revert --clear`. +- **Releases (#213):** when the read of the published site node fails, the run retries it once, then purges everything; it no longer falls back to the working site row. +- **`multiple`** on a type change to a type that has none is ignored, as before. +- **CLI:** `cloudhal fields retype` takes `--collections a,b`, `--max-items n` and `--sub-fields `, and every action takes `--sub `. The admin API's operation route parses the same inputs. MCP's operation tools take them with rollout step 5. diff --git a/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md b/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md index e62e0b3e..c7c55344 100644 --- a/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md +++ b/docs/rfcs/0045-collections-stage-4-references-and-repeaters.md @@ -1046,9 +1046,10 @@ The riskiest SQL and interplay. Every phase PR's review checks the items its pha - Shards are streamed, and the byte and node budgets are checked before each parse, so an out-of-memory isolate can never skip a purge silently. - The short-circuit for sites without reference definitions. - `RELEASE_EXECUTOR_COST` is updated for the re-plan. + - The release gate reads the published site node, retries once, then purges everything; it never falls back to the working row (#213). 3. **Marker interplay** (phases 2 to 4). - `rename_field`'s `root_key` and `fromKey` lag in the fragment (over-blocks only) and in the rows `INSERT` (`fromKey` match). - - Step 4: the live `referencedBy` reads candidates from `entry_refs`, and a text value has no row until phase B rewrites its entry. So during a text → reference conversion it lists a converting referrer only after phase B, or step 4 makes it read candidates by collection while such a marker runs. The snapshot's index already resolves through the conversion maps (3a). + - Step 4: the live `referencedBy` reads candidates from `entry_refs`, and a text value has no row until phase B rewrites its entry. So during a text → reference conversion it lists a converting referrer only after phase B, or step 4 makes it read candidates by collection while such a marker runs. The snapshot's index already resolves through the conversion maps (3a). Step 4 chose "only after phase B" (Amended in 4). - Text → reference conversion: readers convert on the fly, `field_operation_running` refuses target deletes, moves and group changes, and any marker on `from` or `to` purges everything. - `set_field_localized`'s phase-A `UPDATE` of `shared`. - `entryRefsRebuild` reading through `fromKey`. @@ -1225,7 +1226,7 @@ Rollout step 2 shipped in two PRs: 2a, store and integrity, and 2b, the consumer - **Release runs.** - **The gate is the published site node,** not the working row. A release never carries the site node, so its snapshot publishes the live one's; `publishRelease` returns that ref. Only a node that defines references, or still converts from them, re-plans. Every other site sends the plan stored at pin time, as before 3b (nothing when it purges nothing). - That gate costs one blob read per release run that has a purger, except on a first publish: a snapshot's site node is captured before the publish flips the pointer, so the working row equals it only on a site's first publish. - - The read is counted but never refused, so a site without references never depends on what other runs left. If it fails, the working row decides: no references, the stored plan; otherwise purge everything. + - The read is counted but never refused, so a site without references never depends on what other runs left. If it fails, it is retried once, then the run purges everything ([#213](https://github.com/cloudhal-dev/cloudhal/issues/213), fixed in step 4). The working row never decides: a draft that dropped its reference fields would send the closure-less stored plan and leave referrer pages stale. - **The re-plan** goes from the pointer the batch moved (`from`) to the new snapshot. That is `baseSnapshotId` whenever something was live; a first publish purges everything. - It runs over the tick's counting driver, never refused, and spends no shard reads on `to`. - If it throws, the run purges everything, never the stored plan. @@ -1246,3 +1247,126 @@ Rollout step 2 shipped in two PRs: 2a, store and integrity, and 2b, the consumer The rare site-moved retry stays outside it, as before. - **The pin-time plan skips the closure** (`references: false`). It would read every body one by one, and for sites with references it is the batch's *expected* outcome only: the run-time re-plan, or purge-everything, covers referrers. + +### Amended in 4 (2026-11-12, rollout step 4) + +- **`sub`** (§7). The marker sits on the repeater: `{ id, sub, fromKey | fromType }`, or `{ id, sub }` while a sub-field is removed. Phase A keeps a removed sub-field's definition and phase C drops it, as for a whole field. Every item goes through the sub-field's own per-value rule (`fields/migration-sub.ts`), so readers, writers and phase B agree. + - A sub-field rename's `fromKey` names a sub-key. Top-level readers ignore it (`renamedFrom`), and the rows `INSERT`'s `fromKey` match requires `migration.sub IS NULL`. + - `referenceUses`, `definitionOfPath` and `fieldAssetRefs` read an item's old sub-key while the rename runs. + - `set_field_localized` refuses `sub`, because sub-fields share their repeater's `localized`. Removing a repeater's only sub-field is refused: remove the repeater instead. +- **`change_field_type`'s request checks** (`fields-operations/retype.ts`): + - an input the target type does not take is refused, and with `sub` only `collections` (to a reference) and `options` apply; + - a reference changes to a reference only to switch `multiple`; + - a conversion that carries ids over (reference ↔ reference, reference ↔ repeater) may only widen the target collections, so phase B never writes a carried id where the field no longer allows it; + - a repeater converts to text or to a reference only when it has exactly one sub-field, of type text or rich text, or reference. Readers know only the new definition and the marker, so they read each item's one value from its shape. + - Repeater → text sets `multiline: true`. A rich-text sub-field's items join as their Markdown source, as stored, because the marker does not record the sub-field's old type. + - The dry run checks the site the operation leaves against the contract and `checkFieldDefinitions`, with the store's capability. Until step 7, that is where reference and repeater targets are refused (`fields_unsupported`); `field_operation_unsupported` is no longer raised. +- **Text → reference never picks the entry's own group** (`ConvertContext.group`), the same rule a person's write follows. +- **Phase B's target guard covers only the ids a rewrite newly names** (`rewriteTargetGuard`, mode `fresh`), meaning ids that appear nowhere in the stored value. Ids a reshape or a rename carries over are not checked. A value naming a deleted target therefore finishes its conversion as stored, with its row, as rows are kept for a missing target. Guarding carried ids would skip such an entry on every pass, and phase C would never run. +- **Writers during an operation** (`EntryEngine.write`) load the conversion maps before `normalizeEntryFields`, only while a text ↔ reference conversion runs. They then compare the draft with the stored entry normalized through the same markers, for the target guard and for canonical ids. A person's edit that leaves a reshaped or renamed value alone is therefore not re-checked. Text → reference values converted on a person's write are not guarded as newly chosen: deletes in the target collections are refused meanwhile (`field_operation_running`), and only archiving can race them. +- **`hasReferenceFields` includes a field still converting from references.** Until phase B rewrites them, its entries keep rows, which every write, rewrite and guard must still see. +- **`shared` follows `localized` in every operation site write** (`references/localize.ts`, phase A and the revert's first step), as a diff of the two site bodies anchored on the new site hash. In a site with reference fields, every entry write carries the definitions anchor. A holder write planned before phase A therefore fails its batch, and its retry reads the new definitions: R16 resolves by retry. +- **Revert `dropped`.** Each revert pass reads the members of the ids its decisions name, once. It drops each id with no member in a collection its field allows, whatever the member's status, and guards the rest with a member requirement (mode `present`). A target deleted between the read and the batch skips the entry, and the next pass drops it. +- **Watch item 3, live `referencedBy`:** it reads candidates from `entry_refs`, so during a text → reference conversion it lists a converting referrer only after phase B rewrites that entry. That is the documented behaviour, and nothing reads candidates by collection while a marker runs. The live-versus-snapshot comparison runs with no marker active, so the difference is never compared. +- **The CLI and the admin API** take `collections`, `maxItems`, `subFields` and `sub`, and the API also takes `itemLabel`. MCP's operation tools keep their step-1 inputs until step 5. +- **[#213](https://github.com/cloudhal-dev/cloudhal/issues/213)** (from 3b): the release gate's read of the published site node is retried once, then the run purges everything. + +#### Review round 1 (2026-11-12, PM review of PR #214) + +- **Values that stop converting while an operation runs** (root cause 1). The dry run sees only the start state; meanwhile a target can be archived or deleted, or a second entry can take the same title. One rule: + - **Phase C's pending check reads the value where the JSON type cannot tell.** For text ↔ single reference, top-level and in `sub` items, a string that is no group id is pending to a reference, and a string that still is one is pending from a reference. The test is a `GLOB` plus length bounds, the SQL form of `GROUP_ID_PATTERN`. A list of ids and a list of items are told apart by their first element. + - **Each pass reports its unconvertible entries** (`FieldOperationResult.unconvertible`, `FieldRevertResult.unconvertible`: `{ entryId, reason }`, next to `skipped`). They keep the marker, and with it `field_operation_running`. + - **The way out.** `continueFieldOperation` and `revertFieldOperation` take `onInvalid: "clear"`, which drops what no longer converts, so the marker can always be lifted. The admin API's continue and revert routes take it in the body, and the CLI's `continue` and `revert` take `--clear`. The CLI stops looping while entries are unconvertible and says how to go on. + - Id-shaped legacy text (`entry_…` naming no group) converting to a reference is the one value this cannot tell apart. The dry run counts it unconvertible, and readers show nothing for it. +- **The "fits" shortcut** (root cause 2). A group id is valid text, and id-shaped text is a valid single reference. For text ↔ reference, top-level and in items, the shortcut is therefore never taken, in `retype`, `normalizeThroughMarkers`, `resolveFields` (`convertsEveryValue`), phase B and revert. Every value goes through the table, and the table keeps a value already in the new type: + - text → reference keeps an id of a usable target group that is not the entry's own (`ConversionEntries.usable`); + - reference → text keeps a string that is no group id. + + This is the PM's baseline rule in value form. A value a person wrote under the new type is the value the table keeps, and a person's newly chosen id still differs from the normalized stored one, so the writer guards it with the own-group check. Readers need no baseline. Phase B's guard treats every id in a value converting from text as newly chosen (`carriedPart`), so a converted id is guarded. +- **Revert of a sub-field removal** counts the definition as restored only once the repeater has the sub-field again, so the definition, its values, their rows and the guard come back. +- **`hasReferenceFields`** counts any marker whose source type can hold references (`fromType` reference or repeater). A repeater converting away keeps its rows visible until each entry is rewritten. +- **`shared` after a rename's revert** matches the definition's key and its `fromKey` (`root_key IN (key, fromKey)`), so a revert that also restores an older `localized` updates the rows of entries under either key. +- **Release dependencies through the markers.** For a field carrying a marker, every group-id string in its value, and in its old key's value, is dropped from the dependency walk. Field values name no other entry. This covers a sub-field rename's old key and a conversion from references. +- **The edition copy** reads a reference sub-value through a running sub-field rename, as `referenceUses` does, so an unusable id is dropped and reported. +- **The 100-references cap** is counted by the dry run and by every pass's decider for a conversion. Over the cap, the value is unconvertible, or cleared under `onInvalid: "clear"`. +- **Collections widen only where ids stay.** The check runs only when the field stays a reference, or becomes one with ids carried over. A sub-field reference → text is no longer held to another sub-field's collections. +- **Titles for reference → text** come from every member, any status, so an archived target converts to its title. The title chosen is the default-locale member's, else the group's original's, else the member with the lowest locale, then id, whatever the read order. Only a group with no member in the site is unconvertible. Archived members still match no text. +- **Repeater → multiple reference** deduplicates ids, first seen first. +- **`multiple`** is ignored where it does not apply, as before step 4. The CLI sends `multiple: false` for a reference target without `--multiple`, so multiple → single can be asked for. +- **Malformed `subFields`** (entries that are not objects) are a 400 in the admin API and a CLI error, never a 500. + +#### Review round 2 (2026-11-12, PM verification of c8c00f9e) + +Round 1 dropped the "fits" shortcut for text ↔ reference, but then the table re-decided values that were already converted. This round replaces that with one rule. + +- **One rule for "already in the new type"** (`fields/text-reference.ts`). Every caller uses it: the dry run, phase B, the revert's decider, a person's write (`normalizeThroughMarkers`), `resolveFields` with and without maps, and so the renderer sources. + - **A person's write that changes the value** (it differs from the stored value) is kept when it fits, and the writer's guard checks it as newly chosen. `normalizeEntryFields` passes the stored entry, and items are compared one by one. + - **No maps** (`checkRequiredToPublish`, prechecks): a value is kept as stored, never read as missing for want of maps. + - The pristine rule this round introduced, and the gap it left, are replaced by round 3's rule below. +- **Phase C's pending check follows the rows, not the shape** (`textReferencePending`, on top-level values and `sub` items, correlated on the pending query's `nodes.id`, run on SQLite, D1 and DO). + - To a reference: a value is pending when it is text that is not a group id, or a group id with no `entry_refs` row for (node, field path, value). + - From a reference: a value is pending while any row is left for (node, field path). + - A multiple reference keeps its shape test: arrays are converted, strings are pending. +- **The cap** (`fields-operations/cap.ts`) is applied by the dry run, phase B and the revert's decider. Under `clear`, a `sub` conversion over the cap loses only the converted sub-values (an emptied item goes, an emptied list is no value), never the repeater. +- **A sub-field operation's revert** starts from the repeater as it is now and puts back only the named sub-field (`revert-sub.ts`): + - a removed sub-field goes back in at its old index; + - a renamed or retyped one is replaced by its earlier definition under the inverse marker; + - a different sub-field of the same key is refused (`field_key_in_use`). + + A removal counts as reverted only when the repeater holds the earlier sub-field unchanged (`sameJson`). Sub-fields added since stay. +- **The renderer's maps read every entry**, archived ones too, as the store does. An archived target or an archived default-locale member lends its title to reference → text on both sources. +- **The edition copy** carries text that a running text → reference conversion has not converted yet as it is, rather than dropping it as missing. The edition's own write converts it. +- **Repeater → reference is always multiple**, as its row is reference (multiple) ↔ repeater. A `multiple: false` (the CLI's default) is ignored there, and `maxItems` is kept. + +#### Review round 3 (2026-11-12, PM verification of 23bffe00) + +- **The decision reads the facts phase C reads.** Round 2 decided legacy values by whole-entry pristineness (the node's hash against the `before` snapshot), while phase C reads rows. Many writes move a hash without touching the field: a title edit, a status change, source-forward, a term merge or retire, the tags migration. Each one left the two sides disagreeing. `pristine` is gone from these rows' decision (`fields/text-reference.ts`); the other conversion rows keep RFC-0041's rule. + - Each pass of the dry run, phase B and the revert loads the collection's `entry_refs` rows for the converting field path once, beside the maps (`references/conversion-rows.ts`, filtered by `site_id`, `source_collection` and `field`). A person's write reads its node's rows. + - **Text → reference**, for an id-shaped value (a string, or every element of a multiple's list): + - the entry's own group fails first, in every caller; + - in the target shape with a row for every id, it is kept; + - written under the new type by a person's write (it differs from the stored value) and fitting, it is kept, and the writer's guard checks it; + - otherwise it is legacy, and `check` writes it as a reference (a bare string wrapped for a multiple) under the fresh guard, with its rows, when every id names a usable group. Else it fails, is reported, and `clear` drops it; + - under a multiple definition a bare string is never kept; + - any other text goes through the table (`byText`). + - **Reference → text.** A row left for the node and path means the value is still a reference: a group in `titles` converts to its title, any other fails (reported; `clear` removes it and its rows). No row left means it is text already, an id-shaped title included. + - **A value the decision does not accept keeps its rows as they are** in a person's write (`held`): a rejected text → reference value gets no row, and a reference → text value that does not convert keeps its rows until phase B reports or clears it. + - **Readers** (`resolveFields`, the renderer) have no rows. They keep a display rule: an id-shaped value naming a member group shows as a reference, except the entry's own group; a group's title shows for reference → text, any other string as it is. + - Round 2's accepted gap is closed: legacy text is decided by its rows, whatever wrote the entry. +- **The edition copy** carries text a running text → reference conversion has not converted only when the edition site's maps convert it to exactly one group, so the edition's own write converts it. Any other text is dropped and reported, as in round 1. + +#### Review round 4 (2026-11-13, PM verification of 0d170eab) + +- **Reference → text is decided per value.** + - A string is still a reference only with its own row (`rows.has`); with no row of its own it is text already, even next to a held reference item in the same repeater. + - A non-string (a list, `[]` included) is never text yet and goes through the table: joined titles, no value, or a failure. + - `rows.any` is phase C's pending predicate only. +- **`held` is per value.** A person's write keeps stored rows only for the (field path, group id) pairs the decision failed (`HeldRef`), writes rows for every accepted value, and deletes rows for ids no longer in the draft. `rewriteItems` returns item-level results: in a person's write (`partial`) a failing item stays as stored while the others convert, and the operation's passes still treat any failing item as the entry's failure. The writer's guard skips held values, which are stored as they were and reported by phase B. +- **The writer's "already held" comes from the rows.** While a text → reference conversion runs on a definition or `sub`, an id counts as already held only with its row for that path (`ConvertingRows`, which `normalizeWrite` already reads); every other id is newly chosen, as `rewriteTargetGuard`'s rule treats them. The edition copy follows the same rule. +- **Rebuilds read a converting path through its earlier type** (`rebuildRows`; pull, push, `import --replace` and `--revert`, snapshot restore). + - From text: no rows for the path, so phase B checks every value again. + - From references: a stored value that is still a group id of the site keeps its row. An id-shaped title naming no group stays text. + + Sites without such a marker are unchanged, and imports are not refused. +- **The edition's `textConverts`** uses the converter's own rule: `byText` without the entry's own group, exactly one group left. + +#### Review round 5 (2026-11-13, PM verification of 973494e9) + +- **Rebuilds.** + - Converting from text, only a bare string loses its row; a multiple's list (stored only by phase B or a guarded write) keeps its rows. + - Converting from references, every group-id-shaped stored string keeps its row, whether or not its group still has a member, as the live writer keeps it, so phase B reports a deleted target. +- **Text → multiple reference's pending check follows the rows:** a list is pending while any element is not a group id with its own row for (node, path, element) (`listRowsPending`). +- **Held only for stored values.** A person's write holds a failing value only when it is the stored one, compared per item with `rewriteItems`' canonical-item match (`FailedValue.stored`); one the person newly wrote is guarded like any choice. + - The own-group check runs over every use of a changed key, held or not. + - Held pairs are counted per occurrence: an id the draft also holds in an accepted or converted item is not held. +- **A repeater converts item by item in a person's write only when every failing sub-value still fits the new sub-field** (an id-shaped value). Otherwise the repeater stays as stored, so the write checks; phase B reports it. +- **`loadNodeRows` filters `source_collection`** by the draft's collection, here and in the edition copy, so a move into a converting collection holds nothing from the old one. +- **Phase B's guard on a converting path** treats an id that already has its row for the path as carried, read from the pass's rows, as `guardedUses` does. An accepted item whose target was archived later no longer skips the entry forever. +- **The edition's held check is per path:** on the converting sub-field only `rows.has`, and other sub-fields' stored ids only for those sub-fields. + +#### Final gate (2026-11-13, PM verification of cff168d6) + +- **Per-occurrence held subtraction applies only to paths converting to references** (`fromType: "text"`), where another occurrence gets its own guard and row. Converting from references, every stored failing id stays held, so a copy of a failing item cannot drop its rows and turn a dangling id into text unreported. +- **Rebuilds read a running reference (multiple) ↔ repeater conversion through its old type** (`reshapedRows`, top level). A stored value still in the old shape keeps its rows: from a reference, each id-shaped string at the field path; from a repeater, each item's id-shaped sub-value at `.`. A mid-operation import therefore keeps the target locked. +- **The edition's per-sub-field held check applies only while a text → reference marker runs on the definition.** Otherwise every stored id of the field is held, as before step 4. + diff --git a/packages/api/src/field-operations.ts b/packages/api/src/field-operations.ts index c6427bae..aea263a7 100644 --- a/packages/api/src/field-operations.ts +++ b/packages/api/src/field-operations.ts @@ -5,17 +5,18 @@ * handlers through `/api/admin` with the deployment token (`cloudhal fields … --remote`). * * - `POST …/collections/:key/fields/operations` ← `{ op, …, dryRun? }` → the result (`done: false` - * when the budget ran out); `?continue=` resumes one instead + * when the budget ran out); `?continue=` resumes one instead, with + * `{ onInvalid: "clear" }` dropping values that stopped converting (RFC-0045 §7) * - `GET …/collections/:key/fields/operations/:operationId` → its status - * - `POST …/collections/:key/fields/operations/:operationId/revert` → the revert result (call again - * while `done: false`) + * - `POST …/collections/:key/fields/operations/:operationId/revert` ← `{ onInvalid? }` → the revert + * result (call again while `done: false`) * * Refusals are the store's codes: `operation_in_progress` (409), `field_values_unconvertible` (422, * with the dry run's report in `details.report`), `field_operation_incomplete` (409), and the * definition codes (`unknown_field`, `field_key_duplicate`, `field_key_reserved`, …). */ import { holds } from "@cloudhal/agent-runtime"; -import type { Actor, FieldOption } from "@cloudhal/core-contracts"; +import type { Actor, FieldOption, SubFieldDefinition } from "@cloudhal/core-contracts"; import { continueFieldOperation, FIELD_OPERATIONS, @@ -25,7 +26,7 @@ import { runFieldOperation, type SiteGraphStore, } from "@cloudhal/core-ts"; -import { type EnabledFieldType, FIELD_TYPES, type FieldType } from "@cloudhal/core-ts/fields"; +import { ALL_FIELD_TYPES, type FieldType } from "@cloudhal/core-ts/fields"; import type { Context } from "hono"; import { Hono } from "hono"; import type { Env } from "./context.ts"; @@ -41,20 +42,52 @@ const str = (b: Body, name: string): string => { return v; }; -/** The request body as an operation on `collection`; 400 for a malformed one. */ +/** RFC-0045 §7: a repeater's sub-field, when the body names one. */ +function subOf(b: Body): { sub?: string } { + if (b.sub === undefined) return {}; + if (typeof b.sub !== "string" || !b.sub) throw badRequest("sub must be a sub-field key"); + return { sub: b.sub }; +} + +/** RFC-0045 §7: a type change's inputs for references and repeaters, each only when given. */ +function stage4Of(b: Body) { + const strings = (v: unknown) => Array.isArray(v) && v.every((x) => typeof x === "string"); + if (b.collections !== undefined && !strings(b.collections)) + throw badRequest("collections must be a list of collection keys"); + if (b.maxItems !== undefined && !Number.isInteger(b.maxItems)) + throw badRequest("maxItems must be a whole number"); + const record = (v: unknown) => typeof v === "object" && v !== null && !Array.isArray(v); + if (b.subFields !== undefined && (!Array.isArray(b.subFields) || !b.subFields.every(record))) + throw badRequest("subFields must be a list of sub-field definitions"); + if (b.itemLabel !== undefined && typeof b.itemLabel !== "string") + throw badRequest("itemLabel must be a sub-field key"); + return { + ...(b.collections !== undefined ? { collections: b.collections as string[] } : {}), + ...(b.maxItems !== undefined ? { maxItems: b.maxItems as number } : {}), + ...(b.subFields !== undefined ? { subFields: b.subFields as SubFieldDefinition[] } : {}), + ...(b.itemLabel !== undefined ? { itemLabel: b.itemLabel as string } : {}), + }; +} + +/** + * The request body as an operation on `collection`; 400 for a malformed one. Reference and + * repeater targets parse; the store refuses them (`fields_unsupported`) until RFC-0045 step 7. + */ export function operationOf(collection: string, b: Body): FieldOperation { const op = b.op; if (typeof op !== "string" || !FIELD_OPERATIONS.includes(op as FieldOperation["op"])) throw badRequest(`op must be one of ${FIELD_OPERATIONS.join(", ")}`); - if (op === "rename_field") return { op, collection, from: str(b, "from"), to: str(b, "to") }; - if (op === "remove_field") return { op, collection, key: str(b, "key") }; + const sub = subOf(b); + if (op === "rename_field") + return { op, collection, from: str(b, "from"), to: str(b, "to"), ...sub }; + if (op === "remove_field") return { op, collection, key: str(b, "key"), ...sub }; if (op === "set_field_localized") { if (typeof b.localized !== "boolean") throw badRequest("localized must be a boolean"); - return { op, collection, key: str(b, "key"), localized: b.localized }; + return { op, collection, key: str(b, "key"), localized: b.localized, ...sub }; } const type = str(b, "type"); - if (!FIELD_TYPES.includes(type as EnabledFieldType)) - throw badRequest(`type must be one of ${FIELD_TYPES.join(", ")}`); + if (!ALL_FIELD_TYPES.includes(type as FieldType)) + throw badRequest(`type must be one of ${ALL_FIELD_TYPES.join(", ")}`); if (b.onInvalid !== undefined && b.onInvalid !== "clear") throw badRequest('onInvalid is "clear" or absent'); if (b.multiple !== undefined && typeof b.multiple !== "boolean") @@ -66,12 +99,24 @@ export function operationOf(collection: string, b: Body): FieldOperation { collection, key: str(b, "key"), type: type as FieldType, + ...sub, + ...stage4Of(b), ...(b.options ? { options: b.options as FieldOption[] } : {}), ...(b.multiple !== undefined ? { multiple: b.multiple as boolean } : {}), ...(b.onInvalid === "clear" ? { onInvalid: "clear" as const } : {}), }; } +/** + * RFC-0045 §7: `{ onInvalid: "clear" }` on a continue or a revert drops the values that stopped + * converting while the operation ran, so it can always finish. + */ +function clearOf(b: Body): { onInvalid?: "clear" } { + if (b.onInvalid === undefined) return {}; + if (b.onInvalid !== "clear") throw badRequest('onInvalid is "clear" or absent'); + return { onInvalid: "clear" }; +} + /** The operation must belong to the collection in the path. */ async function ownStatus(store: SiteGraphStore, siteId: string, key: string, id: string) { const status = await fieldOperationStatus(store, siteId, id); @@ -87,11 +132,14 @@ export function fieldOperationHandlers(actor: (c: Context) => Actor) { const siteId = param(c, "siteId"); const key = param(c, "key"); const resume = c.req.query("continue"); + const b = ((await c.req.json().catch(() => null)) ?? {}) as Body; if (resume) { + const clear = clearOf(b); await ownStatus(store, siteId, key, resume); - return c.json(await continueFieldOperation(store, siteId, resume, { actor: actor(c) })); + return c.json( + await continueFieldOperation(store, siteId, resume, { actor: actor(c), ...clear }), + ); } - const b = ((await c.req.json().catch(() => null)) ?? {}) as Body; if (b.dryRun !== undefined && typeof b.dryRun !== "boolean") throw badRequest("dryRun must be a boolean"); const op = operationOf(key, b); @@ -114,7 +162,10 @@ export function fieldOperationHandlers(actor: (c: Context) => Actor) { const siteId = param(c, "siteId"); const id = param(c, "operationId"); await ownStatus(store, siteId, param(c, "key"), id); - return c.json(await revertFieldOperation(store, siteId, id, { actor: actor(c) })); + const b = ((await c.req.json().catch(() => null)) ?? {}) as Body; + return c.json( + await revertFieldOperation(store, siteId, id, { actor: actor(c), ...clearOf(b) }), + ); }); } diff --git a/packages/api/src/testing-fields.ts b/packages/api/src/testing-fields.ts index 93d0c437..658cb298 100644 --- a/packages/api/src/testing-fields.ts +++ b/packages/api/src/testing-fields.ts @@ -183,14 +183,54 @@ export function describeFieldsApi(label: string, factory: ApiStoreFactory): void expect(refused.body.error).toBe("fields_unsupported"); } expect((await site()).collections?.events).toBeUndefined(); - // and no field operation turns a field into one (the API takes the nine types only) + // and no field operation turns a field into one: the route parses RFC-0045's inputs, and + // the store's dry run refuses the end state (rollout step 4) expect((await put(EVENTS)).status).toBe(200); const ops = `/api/sites/${siteId}/collections/events/fields/operations`; - const retype = await call(ops, { + for (const dryRun of [true, false]) { + const retype = await call(ops, { + method: "POST", + body: JSON.stringify({ + op: "change_field_type", + key: "venue", + type: "reference", + collections: ["events"], + dryRun, + }), + }); + expect(retype.status).toBe(409); + expect(retype.body.error).toBe("fields_unsupported"); + } + const malformed = await call(ops, { method: "POST", - body: JSON.stringify({ op: "change_field_type", key: "venue", type: "reference" }), + body: JSON.stringify({ + op: "change_field_type", + key: "venue", + type: "reference", + maxItems: "x", + }), }); - expect(retype.status).toBe(400); + expect(malformed.status).toBe(400); + for (const subFields of [[1], "x", [["a"]]]) { + const bad = await call(ops, { + method: "POST", + body: JSON.stringify({ + op: "change_field_type", + key: "venue", + type: "repeater", + subFields, + }), + }); + expect(bad.status).toBe(400); + } + const unknownClear = await call(`${ops}?continue=snap_nothing`, { + method: "POST", + body: JSON.stringify({ onInvalid: "drop" }), + }); + expect(unknownClear.status).toBe(400); + expect((await site()).collections?.events?.fields?.find((d) => d.key === "venue")?.type).toBe( + "text", + ); }); it("field operations: dry run, run, status, continue and revert (RFC-0041 §6)", async () => { diff --git a/packages/astro-renderer/src/live-source.ts b/packages/astro-renderer/src/live-source.ts index 055a5c46..322a02ef 100644 --- a/packages/astro-renderer/src/live-source.ts +++ b/packages/astro-renderer/src/live-source.ts @@ -281,7 +281,9 @@ export class LiveSource implements GraphSource { [...needs].map((c) => this.#store.listContentEntries(this.#siteId, c)), ) ).flat(); - return visible(read, true); + // Every entry, archived ones too: an archived target still lends its title (reference → + // text), and `conversionEntries` keeps archived ones out of what text matches. + return read; }); this.#conversions.set(collection, { at, maps }); maps.catch(() => this.#conversions.delete(collection)); diff --git a/packages/astro-renderer/src/reference-conversions.ts b/packages/astro-renderer/src/reference-conversions.ts index 687c75b7..67e30daf 100644 --- a/packages/astro-renderer/src/reference-conversions.ts +++ b/packages/astro-renderer/src/reference-conversions.ts @@ -3,11 +3,14 @@ * (`ConvertContext.entries`), so `resolveFields` converts an entry phase B has not rewritten yet * exactly as the writers and the operation will. * - * - The snapshot source builds them from the snapshot's own entries (every one not archived, - * drafts included: the graph it renders), lazily, once per snapshot, isolate and collection, - * and only while the collection carries such a marker. - * - The live source builds them from the working copy (the graph phase B reads) before resolving - * an entry of a collection that carries one. + * - The snapshot source builds them from the snapshot's own entries (every one, drafts and + * archived ones included: the graph it renders), lazily, once per snapshot, isolate and + * collection, and only while the collection carries such a marker. + * - The live source builds them from the working copy (the graph phase B reads), every entry too, + * before resolving an entry of a collection that carries one. + * + * Both pass every entry, as the store does: `conversionEntries` keeps archived ones out of what + * text matches (`byText`, `usable`), and lets every member lend a title (`titles`). * * Without such a marker neither reads anything. Worker-safe. */ diff --git a/packages/astro-renderer/src/source.ts b/packages/astro-renderer/src/source.ts index 0ded19fa..90daccdd 100644 --- a/packages/astro-renderer/src/source.ts +++ b/packages/astro-renderer/src/source.ts @@ -174,7 +174,10 @@ interface Loaded { published: ContentEntry[]; /** Published entries with a public URL (unlisted ones too); `entries` holds the listed. */ reachable: ContentEntry[]; - /** RFC-0045 §7: every entry not archived, drafts included; kept only while a conversion runs. */ + /** + * RFC-0045 §7: every entry, drafts and archived ones included (`conversionEntries` filters what + * matches text itself); kept only while a conversion runs. + */ current: ContentEntry[]; /** Every published entry's derived gate (RFC-0042 §3), computed once per snapshot. */ derived: ReadonlyMap; @@ -385,7 +388,7 @@ export class SnapshotSource implements GraphSource { entries: shownEntries, published: shownEntries, reachable: shownEntries, - current: siteConverts(site as Site) ? visible(entries, true) : [], + current: siteConverts(site as Site) ? entries : [], derived, resolved: new Map(), taxonomies, diff --git a/packages/astro-renderer/test/references.test.ts b/packages/astro-renderer/test/references.test.ts index bafd528c..ea01223e 100644 --- a/packages/astro-renderer/test/references.test.ts +++ b/packages/astro-renderer/test/references.test.ts @@ -5,6 +5,7 @@ * active); gate instants; depth, cycles and the per-request budget. */ import type { ContentEntry, Site } from "@cloudhal/core-contracts"; +import { canonicalJson, nodeHash } from "@cloudhal/core-ts"; import { afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { entryFields, type FieldsSource, type RenderedFields } from "../src/fields.ts"; import { linkLocalizer, localizeLinks } from "../src/locales.ts"; @@ -372,3 +373,48 @@ describe("reference links go through the locale middleware once (RFC-0045 §4)", expect(h, h).not.toMatch(/^\/(en|tr)\/(en|tr)\//); }); }); + +describe("reference → text titles on both sources (RFC-0045 §7, PR #214 round 2)", () => { + it("an archived target and an archived default-locale member lend their titles, as in the store", async () => { + const w = await refsWorld(); + const { e, store } = w; + // the hall's English member is archived; its Turkish member stays + await store.updateContentEntry({ + ...((await store.getContentEntry(e.hall?.id as string)) as ContentEntry), + status: "archived", + }); + // `change_field_type` reference → text, phase A: venue is text with the marker + const site = (await store.getSite(w.site.id)) as Site; + const events = site.collections?.events; + const fields = (events?.fields ?? []).map((d) => + d.key === "venue" + ? { + key: "venue", + type: "text", + labels: d.labels, + localized: false, + migration: { id: "fop_01HZX8K3Q9CD", fromType: "reference" }, + } + : d, + ); + const next = { + ...site, + rev: site.rev + 1, + collections: { ...site.collections, events: { ...events, fields } }, + } as unknown as Site; + await store.sql.run("UPDATE nodes SET rev = ?, hash = ?, body = ? WHERE id = ?", [ + next.rev, + await nodeHash(next), + canonicalJson(next), + site.id, + ]); + const snap = await snapshotOf(w, next); + const live = new LiveSource(store, site.id, { includeDrafts: false, now: () => T }); + for (const source of [snap, live]) { + const night = await source.entryFields(e.night as ContentEntry); + expect(night.venue?.text).toBe("Title vault"); + const launch = await source.entryFields(e.launch as ContentEntry); + expect(launch.venue?.text).toBe("Title hall"); + } + }); +}); diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 6b82eecb..34af3c91 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -75,6 +75,10 @@ export const OPTIONS = { options: { type: "string" }, multiple: { type: "boolean" }, clear: { type: "boolean" }, + collections: { type: "string" }, + "max-items": { type: "string" }, + "sub-fields": { type: "string" }, + sub: { type: "string" }, } as const; /** diff --git a/packages/cli/src/cli/help.ts b/packages/cli/src/cli/help.ts index 7ebc96bc..a611e6d7 100644 --- a/packages/cli/src/cli/help.ts +++ b/packages/cli/src/cli/help.ts @@ -97,7 +97,7 @@ const CONTENT = ` cloudhal export [--out ] [--site ] [--no-media] [-- cloudhal migrate tags-to-terms [--dry-run] [--revert ] move entry tags into tag terms (RFC-0002); resumable; --revert undoes a run - cloudhal fields rename | retype [--options a,b] [--multiple] [--clear] | localize on|off | remove | continue|revert [--dry-run] [--remote] + cloudhal fields rename | retype [--options a,b] [--multiple] [--collections a,b] [--max-items n] [--sub-fields file.json] [--clear] | localize on|off | remove | continue|revert [--clear] [--sub ] [--dry-run] [--remote] change a collection's typed fields with their values (RFC-0041): snapshotted, resumable; --dry-run reports and greps the theme; revert undoes a run diff --git a/packages/cli/src/commands/fields.ts b/packages/cli/src/commands/fields.ts index 74e8574f..e017c2aa 100644 --- a/packages/cli/src/commands/fields.ts +++ b/packages/cli/src/commands/fields.ts @@ -8,7 +8,7 @@ */ import { readdir, readFile } from "node:fs/promises"; import path from "node:path"; -import type { FieldOption } from "@cloudhal/core-contracts"; +import type { FieldOption, SubFieldDefinition } from "@cloudhal/core-contracts"; import { continueFieldOperation, type FieldOperation, @@ -17,7 +17,7 @@ import { revertFieldOperation, runFieldOperation, } from "@cloudhal/core-ts"; -import { type EnabledFieldType, FIELD_TYPES, type FieldType } from "@cloudhal/core-ts/fields"; +import { ALL_FIELD_TYPES, type FieldType } from "@cloudhal/core-ts/fields"; import { openLocalStore } from "@cloudhal/core-ts/local"; import { humanizeSlug } from "@cloudhal/core-ts/names"; import type { CliCommand } from "../cli/types.ts"; @@ -28,7 +28,41 @@ import { brief, deployment } from "./sync.ts"; const CLI_ACTOR = { kind: "human", id: "user_localcli0001", name: "cloudhal CLI" } as const; export const FIELDS_USAGE = - "usage: cloudhal fields rename | retype [--options a,b] [--multiple] [--clear] | localize on|off | remove | continue|revert [--dry-run] [--remote]"; + "usage: cloudhal fields rename | retype [--options a,b] [--multiple] [--collections a,b] [--max-items n] [--sub-fields file.json] [--clear] | localize on|off | remove | continue|revert [--clear] [--sub ] [--dry-run] [--remote]"; + +/** What `parseFieldsArgs` takes besides the positionals (`subFields` already read from its file). */ +export interface FieldsFlags { + dryRun?: boolean; + options?: string; + multiple?: boolean; + clear?: boolean; + /** RFC-0045 §7: a reference's target collections, `a,b`. */ + collections?: string; + maxItems?: string; + subFields?: SubFieldDefinition[]; + /** A repeater's sub-field: the operation renames, converts or removes it in every item. */ + sub?: string; +} + +/** `--max-items 20` as a positive integer. */ +function maxItemsOf(v: string): number { + const n = Number(v); + if (!Number.isInteger(n) || n < 1) throw new Error("--max-items is a whole number from 1"); + return n; +} + +/** The type change's RFC-0045 inputs, each only when given. */ +function stage4Inputs(flags: FieldsFlags) { + const list = flags.collections + ?.split(",") + .map((c) => c.trim()) + .filter(Boolean); + return { + ...(list?.length ? { collections: list } : {}), + ...(flags.maxItems !== undefined ? { maxItems: maxItemsOf(flags.maxItems) } : {}), + ...(flags.subFields ? { subFields: flags.subFields } : {}), + }; +} export interface FieldsCliOptions { readonly projectDir: string; @@ -39,7 +73,7 @@ export interface FieldsCliOptions { type Request = | { kind: "run"; op: FieldOperation; dryRun: boolean } - | { kind: "continue" | "revert"; collection: string; id: string }; + | { kind: "continue" | "revert"; collection: string; id: string; clear?: boolean }; export type FieldsCliResult = | { kind: "run"; result: FieldOperationResult; theme: string[] } @@ -60,7 +94,7 @@ function optionsOf(list: string, locale: string): FieldOption[] { /** The positionals and flags as a request (throws the usage when they do not fit). */ export function parseFieldsArgs( rest: readonly string[], - flags: { dryRun?: boolean; options?: string; multiple?: boolean; clear?: boolean }, + flags: FieldsFlags, locale = "en", ): Request { const [action, collection, a, b] = rest; @@ -69,36 +103,51 @@ export function parseFieldsArgs( }; const dryRun = flags.dryRun === true; const c = collection as string; + const sub = flags.sub ? { sub: flags.sub } : {}; switch (action) { case "rename": need(a, b); return { kind: "run", dryRun, - op: { op: "rename_field", collection: c, from: a as string, to: b as string }, + op: { op: "rename_field", collection: c, from: a as string, to: b as string, ...sub }, }; case "remove": need(a); - return { kind: "run", dryRun, op: { op: "remove_field", collection: c, key: a as string } }; + return { + kind: "run", + dryRun, + op: { op: "remove_field", collection: c, key: a as string, ...sub }, + }; case "localize": need(a); if (b !== "on" && b !== "off") throw new Error(FIELDS_USAGE); return { kind: "run", dryRun, - op: { op: "set_field_localized", collection: c, key: a as string, localized: b === "on" }, + op: { + op: "set_field_localized", + collection: c, + key: a as string, + localized: b === "on", + ...sub, + }, }; case "retype": { need(a, b); - if (!FIELD_TYPES.includes(b as EnabledFieldType)) - throw new Error(`the type is one of ${FIELD_TYPES.join(", ")}`); + // The store refuses reference and repeater targets until RFC-0045 rollout step 7. + if (!ALL_FIELD_TYPES.includes(b as FieldType)) + throw new Error(`the type is one of ${ALL_FIELD_TYPES.join(", ")}`); const op: FieldOperation = { op: "change_field_type", collection: c, key: a as string, type: b as FieldType, + ...sub, ...(flags.options ? { options: optionsOf(flags.options, locale) } : {}), - ...(b === "select" ? { multiple: flags.multiple === true } : {}), + ...(b === "select" && !flags.sub ? { multiple: flags.multiple === true } : {}), + ...(b === "reference" && !flags.sub ? { multiple: flags.multiple === true } : {}), + ...stage4Inputs(flags), ...(flags.clear ? { onInvalid: "clear" as const } : {}), }; return { kind: "run", dryRun, op }; @@ -106,7 +155,13 @@ export function parseFieldsArgs( case "continue": case "revert": need(a); - return { kind: action, collection: c, id: a as string }; + // RFC-0045 §7: `--clear` drops values that stopped converting while the operation ran. + return { + kind: action, + collection: c, + id: a as string, + ...(flags.clear ? { clear: true } : {}), + }; default: throw new Error(FIELDS_USAGE); } @@ -137,14 +192,19 @@ export async function themeReferences(projectDir: string, key: string): Promise< const keyOf = (op: FieldOperation) => (op.op === "rename_field" ? op.from : op.key); +/** Another call can make progress: nothing changed meanwhile, nothing stopped converting. */ +const again = (r: { done: boolean; skipped: unknown[]; unconvertible?: unknown[] | undefined }) => + !r.done && r.skipped.length === 0 && (r.unconvertible ?? []).length === 0; + async function runLocal(options: FieldsCliOptions, req: Request): Promise { const cfg = await readProjectConfig(options.projectDir); const local = await openLocalStore(path.join(options.projectDir, cfg.dataDir)); - const actor = { actor: CLI_ACTOR }; + const clear = req.kind !== "run" && req.clear ? { onInvalid: "clear" as const } : {}; + const actor = { actor: CLI_ACTOR, ...clear }; try { if (req.kind === "revert") { let result = await revertFieldOperation(local.store, cfg.siteId, req.id, actor); - while (!result.done && result.skipped.length === 0) + while (again(result)) result = await revertFieldOperation(local.store, cfg.siteId, req.id, actor); return { kind: "revert", result }; } @@ -152,7 +212,7 @@ async function runLocal(options: FieldsCliOptions, req: Request): Promise `dropped ${d.entryId} ${d.field} → ${d.targetId} (no entry of that group is left)`, + ), + ...(x.unconvertible ?? []).map((u) => ` ${u.entryId} does not convert back: ${u.reason}`), + ...(x.done + ? [`snapshot ${x.snapshotId}`] + : (x.unconvertible ?? []).length > 0 + ? ["not finished: fix those values, or run the revert again with --clear"] + : ["not finished: run the revert again"]), ]; } const { result: x, theme } = r; @@ -249,8 +317,12 @@ export function describeFields(r: FieldsCliResult): string[] { lines.push(`theme code reads fields.${p.key} directly and needs a change: ${theme.join(", ")}`); if (x.dryRun) return lines; if (!x.done) { + const stuck = x.unconvertible ?? []; + for (const u of stuck) lines.push(` ${u.entryId} no longer converts: ${u.reason}`); lines.push( - `not finished (${x.skipped.length} entries changed meanwhile): cloudhal fields continue ${p.collection} ${x.operationId}`, + stuck.length > 0 + ? `not finished (${stuck.length} values stopped converting): fix them, or cloudhal fields continue ${p.collection} ${x.operationId} --clear` + : `not finished (${x.skipped.length} entries changed meanwhile): cloudhal fields continue ${p.collection} ${x.operationId}`, ); return lines; } @@ -261,6 +333,16 @@ export function describeFields(r: FieldsCliResult): string[] { return lines; } +/** `--sub-fields `: a JSON list of sub-field definitions (checked by the store). */ +async function readSubFields(projectDir: string, file: string): Promise { + const text = await readFile(path.resolve(projectDir, file), "utf8"); + const list = JSON.parse(text) as unknown; + const record = (v: unknown) => typeof v === "object" && v !== null && !Array.isArray(v); + if (!Array.isArray(list) || !list.every(record)) + throw new Error(`${file} holds a JSON list of sub-field definitions (objects)`); + return list as SubFieldDefinition[]; +} + /** `cloudhal fields …`. */ export const fieldsCommand: CliCommand = async (rest, values, { projectDir }) => { const locale = values.options ? await localDefaultLocale(projectDir) : "en"; @@ -271,6 +353,12 @@ export const fieldsCommand: CliCommand = async (rest, values, { projectDir }) => ...(values.options ? { options: values.options } : {}), ...(values.multiple ? { multiple: true } : {}), ...(values.clear ? { clear: true } : {}), + ...(values.collections ? { collections: values.collections } : {}), + ...(values["max-items"] ? { maxItems: values["max-items"] } : {}), + ...(values["sub-fields"] + ? { subFields: await readSubFields(projectDir, values["sub-fields"]) } + : {}), + ...(values.sub ? { sub: values.sub } : {}), }, locale, ); diff --git a/packages/cli/test/fields-cli.test.ts b/packages/cli/test/fields-cli.test.ts index 7cafbd1d..f5520c95 100644 --- a/packages/cli/test/fields-cli.test.ts +++ b/packages/cli/test/fields-cli.test.ts @@ -95,6 +95,74 @@ describe("cloudhal fields (RFC-0041 §6)", () => { expect(() => parseFieldsArgs(["nope"], {})).toThrow(/usage: cloudhal fields/); }); + it("parses RFC-0045's inputs: --collections, --max-items, --sub-fields and --sub", () => { + expect( + parseFieldsArgs(["retype", "events", "venue", "reference"], { + collections: "places, halls", + maxItems: "5", + multiple: true, + }), + ).toMatchObject({ + op: { type: "reference", collections: ["places", "halls"], maxItems: 5, multiple: true }, + }); + const subFields = [{ key: "line", type: "text" as const, labels: { en: { label: "Line" } } }]; + expect( + parseFieldsArgs(["retype", "events", "steps", "repeater"], { subFields }).kind === "run" && + parseFieldsArgs(["retype", "events", "steps", "repeater"], { subFields }), + ).toMatchObject({ op: { type: "repeater", subFields } }); + expect( + parseFieldsArgs(["rename", "events", "speakers", "fullName"], { sub: "name" }), + ).toMatchObject({ op: { op: "rename_field", from: "speakers", sub: "name", to: "fullName" } }); + expect(parseFieldsArgs(["remove", "events", "speakers"], { sub: "seats" })).toMatchObject({ + op: { op: "remove_field", key: "speakers", sub: "seats" }, + }); + expect(() => + parseFieldsArgs(["retype", "events", "venue", "reference"], { maxItems: "0" }), + ).toThrow(/max-items/); + // without --multiple a reference target is single, so multiple → single can be asked for + expect( + parseFieldsArgs(["retype", "events", "venue", "reference"], { collections: "places" }), + ).toMatchObject({ op: { type: "reference", multiple: false } }); + // continue and revert take --clear (RFC-0045 §7: values that stopped converting) + expect(parseFieldsArgs(["continue", "events", "snap_1"], { clear: true })).toEqual({ + kind: "continue", + collection: "events", + id: "snap_1", + clear: true, + }); + expect(parseFieldsArgs(["revert", "events", "snap_1"], {})).toEqual({ + kind: "revert", + collection: "events", + id: "snap_1", + }); + }); + + it("reads --sub-fields from a file; the local store still refuses the types (step 7)", async () => { + await writeFile( + path.join(projectDir, "subs.json"), + JSON.stringify([{ key: "line", type: "text", labels: { en: { label: "Line" } } }]), + ); + const cwd = vi.spyOn(process, "cwd").mockReturnValue(projectDir); + await expect( + main([ + "fields", + "retype", + "events", + "venue", + "repeater", + "--sub-fields", + "subs.json", + "--dry-run", + ]), + ).rejects.toThrow(/not enabled on this server/); + await writeFile(path.join(projectDir, "bad.json"), JSON.stringify([1, "x"])); + await expect( + main(["fields", "retype", "events", "venue", "repeater", "--sub-fields", "bad.json"]), + ).rejects.toThrow(/JSON list of sub-field definitions/); + cwd.mockRestore(); + expect(await fields()).toEqual({ venue: "Hall" }); + }); + it("--dry-run reports, greps the theme and writes nothing; a run and its revert", async () => { expect(await themeReferences(projectDir, "venue")).toEqual([ path.join("src", "layouts", "Event.astro"), diff --git a/packages/core-ts/src/fields-operations/cap.ts b/packages/core-ts/src/fields-operations/cap.ts new file mode 100644 index 00000000..4681a93a --- /dev/null +++ b/packages/core-ts/src/fields-operations/cap.ts @@ -0,0 +1,49 @@ +/** + * RFC-0045 §1 and §7: a conversion into references keeps the entry within 100 reference values. + * Over the cap the value is unconvertible, or cleared under `onInvalid: "clear"`: a field's whole + * value, or for a `sub` marker only the converted sub-values (an item left empty goes, and an + * emptied list is no value), never the repeater's other sub-fields. Phase B, the dry run and the + * revert's decider all apply it. Pure. + */ +import type { FieldDefinition } from "@cloudhal/core-contracts"; +import type { ValueChange } from "../fields/migration.ts"; +import { isRecord } from "../fields/types.ts"; +import { referenceCountIssues } from "../references/values.ts"; +import type { Decision } from "./rewrite.ts"; + +/** `values` without the converted value of `def` (its sub-values only, for a `sub` marker). */ +function withoutConverted(def: FieldDefinition, values: Record) { + const { [def.key]: value, ...rest } = values; + const sub = def.migration?.sub; + if (sub === undefined || !Array.isArray(value)) return rest; + const items = value + .map((item) => { + if (!isRecord(item)) return item; + const { [sub]: _converted, ...kept } = item; + return kept; + }) + .filter((item) => !isRecord(item) || Object.keys(item).length > 0); + return items.length > 0 ? { ...rest, [def.key]: items } : rest; +} + +/** The decision for a rewritten entry once the cap is counted. */ +export function withinReferenceCap( + defs: readonly FieldDefinition[], + def: FieldDefinition, + r: { + values: Record; + change: ValueChange | undefined; + onInvalid?: "clear" | undefined; + }, +): Decision { + const over = + def.migration?.fromType !== undefined ? referenceCountIssues(defs, r.values)[0] : undefined; + if (!over) return { fields: r.values, change: r.change }; + const before = r.change?.before; + if (r.onInvalid !== "clear") + return { change: { kind: "unconvertible", before, after: undefined, reason: over.message } }; + return { + fields: withoutConverted(def, r.values), + change: { kind: "cleared", before, after: undefined, reason: over.message }, + }; +} diff --git a/packages/core-ts/src/fields-operations/plan-sub.ts b/packages/core-ts/src/fields-operations/plan-sub.ts new file mode 100644 index 00000000..b219e372 --- /dev/null +++ b/packages/core-ts/src/fields-operations/plan-sub.ts @@ -0,0 +1,200 @@ +/** + * RFC-0045 §7 `sub`: what an operation on one sub-field of a repeater asks of the site node. The + * marker sits on the repeater: `{ id, sub: , fromKey | fromType }`, or `{ id, sub }` + * while the sub-field is being removed (phase A keeps its definition, phase C drops it, as for a + * top-level field). Pure but for the predicates' SQL text. + */ +import type { FieldDefinition, FieldMigration } from "@cloudhal/core-contracts"; +import { InvariantError } from "../errors.ts"; +import { RESERVED_FIELD_KEYS, type RepeaterFieldDefinition } from "../fields/types.ts"; +import type { Cond } from "../node-sql.ts"; +import { retyped } from "./retype.ts"; +import type { FieldOperation } from "./types.ts"; + +const KEY = /^[a-z][A-Za-z0-9]{0,39}$/; + +function invalid( + code: "invalid_fields" | "unknown_field" | "field_key_duplicate" | "field_key_reserved", + message: string, +): never { + throw new InvariantError(code, message, { issues: [] }); +} + +/** Checks a `sub` request against the repeater; throws when it can never run as written. */ +export function checkSub(def: FieldDefinition, op: FieldOperation & { sub: string }): void { + if (def.type !== "repeater") + invalid( + "invalid_fields", + `sub names a repeater's sub-field; ${def.key} is a ${def.type} field`, + ); + if (op.op === "set_field_localized") + invalid("invalid_fields", `sub-fields share ${def.key}'s localized; change it without sub`); + const keys = def.fields.map((s) => s.key); + if (!keys.includes(op.sub)) + invalid( + "unknown_field", + `${op.sub} is not a sub-field of ${def.key}; its sub-fields are ${keys.join(", ")}`, + ); + if (op.op === "rename_field") { + if (!KEY.test(op.to)) + invalid("field_key_reserved", `${op.to} is not a field key (lower camelCase)`); + if (RESERVED_FIELD_KEYS.has(op.to)) invalid("field_key_reserved", `${op.to} is reserved`); + if (keys.includes(op.to)) + invalid("field_key_duplicate", `${op.to} is already a sub-field of ${def.key}`); + } + if (op.op === "remove_field" && keys.length === 1) + invalid( + "invalid_fields", + `${op.sub} is ${def.key}'s only sub-field; remove ${def.key} instead`, + ); +} + +/** The repeater with `from` renamed to `to` (its `itemLabel` follows). */ +function renamedSub(def: RepeaterFieldDefinition, from: string, to: string): FieldDefinition { + const fields = def.fields.map((s) => (s.key === from ? { ...s, key: to } : s)); + const label = def.itemLabel === from ? { itemLabel: to } : {}; + return { ...def, fields, ...label } as FieldDefinition; +} + +/** The repeater without the sub-field `key` (an `itemLabel` naming it goes too). */ +export function withoutSub(def: RepeaterFieldDefinition, key: string): FieldDefinition { + const { itemLabel, ...rest } = def; + const fields = def.fields.filter((s) => s.key !== key); + const label = itemLabel !== undefined && itemLabel !== key ? { itemLabel } : {}; + return { ...rest, ...label, fields } as FieldDefinition; +} + +/** The repeater the operation ends with (`op.sub` set). */ +export function subTarget( + def: FieldDefinition, + op: FieldOperation & { sub: string }, +): FieldDefinition { + const repeater = def as RepeaterFieldDefinition; + switch (op.op) { + case "rename_field": + return renamedSub(repeater, op.sub, op.to); + case "remove_field": + return withoutSub(repeater, op.sub); + case "change_field_type": + return retyped(def, op); + case "set_field_localized": + return def; + } +} + +/** Phase A's marker of a sub-field operation. */ +export function subMarkerOf( + id: string, + def: FieldDefinition, + op: FieldOperation & { sub: string }, +): FieldMigration { + const was = (def as RepeaterFieldDefinition).fields.find((s) => s.key === op.sub); + switch (op.op) { + case "rename_field": + return { id, sub: op.to, fromKey: op.sub }; + case "change_field_type": + return { id, sub: op.sub, fromType: was?.type ?? "text" }; + default: + return { id, sub: op.sub }; + } +} + +/** True when the repeater already shows the sub-field operation's end state. */ +export function subApplied( + now: FieldDefinition, + op: FieldOperation & { sub: string }, + target: FieldDefinition, +): boolean { + if (now.type !== "repeater" || target.type !== "repeater") return false; + const has = (k: string) => now.fields.some((s) => s.key === k); + if (op.op === "rename_field") return has(op.to) && !has(op.sub); + if (op.op === "remove_field") return !has(op.sub); + const want = target.fields.find((s) => s.key === op.sub)?.type; + return now.fields.find((s) => s.key === op.sub)?.type === want; +} + +/** + * Entries whose items the sub-field marker still has work for: an item holding the old key (a + * rename), the sub-value being removed, or a sub-value not stored as the new type's JSON. + */ +export function subPending( + base: Cond, + def: FieldDefinition, + jsonTypes: (d: FieldDefinition) => string[], +): Cond | undefined { + const m = def.migration; + if (def.type !== "repeater" || m?.sub === undefined) return undefined; + const items = `$.fields.${def.key}`; + const holds = (key: string, extra = "", params: Cond["params"] = []): Cond => ({ + sql: `${base.sql} AND EXISTS (SELECT 1 FROM json_each(body, ?) i + WHERE json_type(i.value, ?) IS NOT NULL${extra})`, + params: [...base.params, items, `$.${key}`, ...params], + }); + if (m.fromKey !== undefined) return holds(m.fromKey); + if (m.fromType === undefined) return holds(m.sub); + const sub = def.fields.find((s) => s.key === m.sub); + if (!sub) return undefined; + const types = jsonTypes({ ...sub, localized: def.localized } as unknown as FieldDefinition); + const at = `$.${m.sub}`; + const field = `${def.key}.${m.sub}`; + const rows = textReferencePending(sub.type, m.fromType, { doc: "i.value", path: at, field }); + // To a reference, per item; from one, the rows say what is left, whatever the items look like. + const perItem = m.fromType === "text" ? rows : undefined; + const mismatch: Cond = { + sql: `EXISTS (SELECT 1 FROM json_each(body, ?) i WHERE json_type(i.value, ?) IS NOT NULL + AND (json_type(i.value, ?) NOT IN (${types.map(() => "?").join(", ")})${perItem ? ` OR ${perItem.sql}` : ""}))`, + params: [items, at, at, ...types, ...(perItem?.params ?? [])], + }; + const left = perItem ? undefined : rows; + return { + sql: `${base.sql} AND (${mismatch.sql}${left ? ` OR ${left.sql}` : ""})`, + params: [...base.params, ...mismatch.params, ...(left?.params ?? [])], + }; +} + +/** `expr` is a translation group id (`GROUP_ID_PATTERN`: `entry_` and 6 to 32 letters or digits). */ +const isGroupId = (expr: string) => + `(${expr} GLOB 'entry_*' AND length(${expr}) BETWEEN 12 AND 38 AND substr(${expr}, 7) NOT GLOB '*[^A-Za-z0-9]*')`; + +/** + * RFC-0045 §7: text → multiple reference: a list is pending while any element is not a group id + * with its own `entry_refs` row for (node, field path, element), as `text-reference.ts` decides. + */ +export function listRowsPending(path: string, field: string): Cond { + return { + sql: `(json_type(body, ?) = 'array' AND EXISTS (SELECT 1 FROM json_each(body, ?) e + WHERE NOT (${isGroupId("e.value")} AND EXISTS (SELECT 1 FROM entry_refs r + WHERE r.source_id = nodes.id AND r.field = ? AND r.target_id = e.value))))`, + params: [path, path, field], + }; +} + +/** + * RFC-0045 §7: phase C's pending test for text ↔ single reference follows the rows, not the shape + * (both store one string; `text-reference.ts` decides values the same way): + * - to a reference, a string is converted only when it is a group id with its `entry_refs` row + * for (node, field path, value): what phase B or a person's write left; + * - from a reference, a value is converted once no row is left for (node, field path). + * + * `doc` is the JSON the value sits in at `path`; `field` is the rows' field path. The node is the + * pending query's `nodes` row. + */ +export function textReferencePending( + type: string, + from: string | undefined, + at: { doc: string; path: string; field: string }, +): Cond | undefined { + const { doc, path, field } = at; + if (type === "text" && from === "reference") + return { + sql: "EXISTS (SELECT 1 FROM entry_refs r WHERE r.source_id = nodes.id AND r.field = ?)", + params: [field], + }; + if (type !== "reference" || from !== "text") return undefined; + const value = `json_extract(${doc}, ?)`; + return { + sql: `(json_type(${doc}, ?) = 'text' AND (NOT ${isGroupId(value)} OR NOT EXISTS ( + SELECT 1 FROM entry_refs r WHERE r.source_id = nodes.id AND r.field = ? AND r.target_id = ${value})))`, + params: [path, path, path, path, field, path], + }; +} diff --git a/packages/core-ts/src/fields-operations/plan.ts b/packages/core-ts/src/fields-operations/plan.ts index eb52065d..41bf9f83 100644 --- a/packages/core-ts/src/fields-operations/plan.ts +++ b/packages/core-ts/src/fields-operations/plan.ts @@ -6,22 +6,26 @@ */ import type { FieldDefinition, FieldMigration, Site } from "@cloudhal/core-contracts"; import { InvariantError } from "../errors.ts"; -import { isReferenceFieldType, RESERVED_FIELD_KEYS } from "../fields/types.ts"; +import { RESERVED_FIELD_KEYS } from "../fields/types.ts"; import type { Cond } from "../node-sql.ts"; import { renameSettingsReferences } from "../store/collection-settings.ts"; +import { + checkSub, + listRowsPending, + subApplied, + subMarkerOf, + subPending, + subTarget, + textReferencePending, + withoutSub, +} from "./plan-sub.ts"; +import { checkRetype, retyped } from "./retype.ts"; import { FIELD_OPERATIONS, type FieldOperation, finalKey, operationKey } from "./types.ts"; const KEY = /^[a-z][A-Za-z0-9]{0,39}$/; const COLLECTION = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; -function refuse( - code: - | "unknown_field" - | "field_key_duplicate" - | "field_key_reserved" - | "field_operation_unsupported", - msg: string, -) { +function refuse(code: "unknown_field" | "field_key_duplicate" | "field_key_reserved", msg: string) { return new InvariantError(code, msg, { issues: [] }); } @@ -55,33 +59,9 @@ export function runningMarker(defs: readonly FieldDefinition[]): FieldMigration return defs.find((d) => d.migration)?.migration; } -const COMMON = ["key", "type", "labels", "localized", "requiredToPublish"] as const; - -/** The definition `change_field_type` ends with: the shared properties plus the new type's. */ -function retyped(def: FieldDefinition, op: Extract) { - const base: Record = {}; - const was = def as unknown as Record; - for (const k of COMMON) if (was[k] !== undefined) base[k] = was[k]; - base.type = op.type; - if (op.type === "boolean") delete base.requiredToPublish; - const searchable = "searchable" in def ? def.searchable : undefined; - if ((op.type === "text" || op.type === "richText") && searchable !== undefined) - base.searchable = searchable; - if (op.type === "select") { - const options = op.options ?? (def.type === "select" ? def.options : undefined); - if (!options?.length) - throw new InvariantError("invalid_fields", "a select needs its options", { - issues: [{ path: "options", code: "invalid_value", message: "at least one option" }], - }); - base.options = options; - const multiple = op.multiple ?? (def.type === "select" ? def.multiple : undefined); - if (multiple) base.multiple = true; - } - return base as unknown as FieldDefinition; -} - /** The definition the operation ends with; `null` for `remove_field`. */ export function targetDefinition(def: FieldDefinition, op: FieldOperation): FieldDefinition | null { + if (op.sub !== undefined) return subTarget(def, { ...op, sub: op.sub }); switch (op.op) { case "rename_field": return { ...def, key: op.to }; @@ -97,17 +77,10 @@ export function targetDefinition(def: FieldDefinition, op: FieldOperation): Fiel /** Checks the request against the collection; throws when it can never run as written. */ export function checkOperation(site: Site, op: FieldOperation): void { const { defs, def } = currentDefinition(site, op); - // RFC-0045 §2.6: no operation turns a field into, or rewrites, a reference or repeater until - // rollout step 4. - if ( - isReferenceFieldType(def.type) || - (op.op === "change_field_type" && isReferenceFieldType(op.type)) - ) - throw refuse( - "field_operation_unsupported", - `field operations on reference and repeater fields are not available yet (${def.key})`, - ); - if (op.op === "rename_field") { + // RFC-0045 §7: reference and repeater fields take every operation; the types themselves stay + // refused by `checkFieldDefinitions`' gate until rollout step 7. + if (op.sub !== undefined) checkSub(def, { ...op, sub: op.sub }); + else if (op.op === "rename_field") { if (!KEY.test(op.to)) throw refuse("field_key_reserved", `${op.to} is not a field key (lower camelCase)`); if (RESERVED_FIELD_KEYS.has(op.to)) @@ -115,12 +88,12 @@ export function checkOperation(site: Site, op: FieldOperation): void { if (defs.some((d) => d.key === op.to)) throw refuse("field_key_duplicate", `${op.to} is already a field of ${op.collection}`); } - if (op.op === "change_field_type" && def.type === op.type && op.type !== "select") - throw new InvariantError("invalid_fields", `${def.key} already is a ${op.type} field`); + if (op.op === "change_field_type") checkRetype(def, op); } /** Phase A's marker: the operation id and what readers fall back to. */ export function markerOf(id: string, def: FieldDefinition, op: FieldOperation): FieldMigration { + if (op.sub !== undefined) return subMarkerOf(id, def, { ...op, sub: op.sub }); switch (op.op) { case "rename_field": return { id, fromKey: def.key }; @@ -147,10 +120,12 @@ export function withFields(site: Site, collection: string, fields: FieldDefiniti */ export function phaseASite(site: Site, op: FieldOperation, id: string): Site { const { defs, def, index } = currentDefinition(site, op); - const marked = { ...(targetDefinition(def, op) ?? def), migration: markerOf(id, def, op) }; + // A removal keeps the definition (a sub-field's too) until phase C. + const target = op.op === "remove_field" ? def : (targetDefinition(def, op) ?? def); + const marked = { ...target, migration: markerOf(id, def, op) }; const fields = defs.map((d, i) => (i === index ? (marked as FieldDefinition) : d)); const next = withFields(site, op.collection, fields); - return op.op === "rename_field" + return op.op === "rename_field" && op.sub === undefined ? renameSettingsReferences(next, op.collection, op.from, op.to) : next; } @@ -168,10 +143,19 @@ export function phaseCSite(site: Site, collection: string, id: string): Site | u if (!found) return undefined; const { defs, def, index } = found; const { migration, ...plain } = def; - const removing = migration && Object.keys(migration).length === 1; - const fields = removing - ? defs.filter((_, i) => i !== index) - : defs.map((d, i) => (i === index ? (plain as FieldDefinition) : d)); + const removing = + migration?.fromKey === undefined && + migration?.fromType === undefined && + migration?.fromLocalized === undefined; + const sub = migration?.sub; + const ended = + removing && sub !== undefined && plain.type === "repeater" + ? withoutSub(plain, sub) + : (plain as FieldDefinition); + const fields = + removing && sub === undefined + ? defs.filter((_, i) => i !== index) + : defs.map((d, i) => (i === index ? ended : d)); return withFields(site, collection, fields); } @@ -180,15 +164,14 @@ export function isApplied(site: Site, op: FieldOperation, target: FieldDefinitio const defs = site.collections?.[op.collection]?.fields ?? []; if (defs.some((d) => d.migration)) return false; const now = defs.find((d) => d.key === finalKey(op)); + if (op.sub !== undefined) + return !!now && !!target && subApplied(now, { ...op, sub: op.sub }, target); if (op.op === "remove_field") return now === undefined; if (!now || !target) return false; if (op.op === "rename_field") return !defs.some((d) => d.key === op.from); if (op.op === "set_field_localized") return now.localized === op.localized; - return ( - now.type === target.type && - (now.type !== "select" || - (target.type === "select" && (now.multiple === true) === (target.multiple === true))) - ); + const list = (d: FieldDefinition) => "multiple" in d && d.multiple === true; + return now.type === target.type && list(now) === list(target); } const IN_COLLECTION = @@ -199,10 +182,18 @@ function jsonTypes(def: FieldDefinition): string[] { if (def.type === "number") return ["integer", "real"]; if (def.type === "boolean") return ["true", "false"]; if (def.type === "datetime") return ["object"]; - if (def.type === "select" && def.multiple) return ["array"]; + if ((def.type === "select" || def.type === "reference") && def.multiple) return ["array"]; + if (def.type === "repeater") return ["array"]; return ["text"]; } +/** The JSON type of an unconverted list's first element, where both types store arrays. */ +function listItemType(def: FieldDefinition, from: string): string | undefined { + if (def.type === "repeater" && from === "reference") return "text"; + if (def.type === "reference" && def.multiple && from === "repeater") return "object"; + return undefined; +} + /** * Entries of the collection the marker still has work for, as a SQL condition (one predicate * per operation; keys match the FieldKey pattern, so they are safe in a JSON path). @@ -216,6 +207,7 @@ export function pendingCondition( if (!m) return undefined; const path = `$.fields.${def.key}`; const base = { sql: IN_COLLECTION, params: [siteId, collection] }; + if (m.sub !== undefined) return subPending(base, def, jsonTypes); if (m.fromKey !== undefined) return { sql: `${base.sql} AND json_type(body, ?) IS NOT NULL`, @@ -223,9 +215,24 @@ export function pendingCondition( }; if (m.fromType !== undefined) { const types = jsonTypes(def); + // RFC-0045 §7: a list of ids and a list of items are both arrays; their first element differs. + const item = listItemType(def, m.fromType); + const multiple = "multiple" in def && def.multiple === true; + const id = !multiple + ? textReferencePending(def.type, m.fromType, { doc: "body", path, field: def.key }) + : def.type === "reference" && m.fromType === "text" + ? listRowsPending(path, def.key) + : undefined; return { - sql: `${base.sql} AND json_type(body, ?) IS NOT NULL AND json_type(body, ?) NOT IN (${types.map(() => "?").join(", ")})`, - params: [...base.params, path, path, ...types], + sql: `${base.sql} AND json_type(body, ?) IS NOT NULL AND (json_type(body, ?) NOT IN (${types.map(() => "?").join(", ")})${item ? " OR json_type(body, ?) = ?" : ""}${id ? ` OR ${id.sql}` : ""})`, + params: [ + ...base.params, + path, + path, + ...types, + ...(item ? [`${path}[0]`, item] : []), + ...(id?.params ?? []), + ], }; } if (m.fromLocalized !== undefined) { diff --git a/packages/core-ts/src/fields-operations/references.ts b/packages/core-ts/src/fields-operations/references.ts index 8c24b235..e8eb7707 100644 --- a/packages/core-ts/src/fields-operations/references.ts +++ b/packages/core-ts/src/fields-operations/references.ts @@ -34,7 +34,8 @@ function inUse(message: string, details: Record): never { /** Settings that name the field: a removal refuses, a type change re-runs the settings checks. */ export function checkSettingsReferences(site: Site, op: FieldOperation): void { - if (op.op === "remove_field") { + // Settings name fields, never a repeater's sub-fields (RFC-0045 `sub`). + if (op.op === "remove_field" && op.sub === undefined) { const paths = settingsUses(site, op.collection, op.key); if (paths.length === 0) return; inUse(`${op.collection}.${op.key} is named by ${paths.join(", ")}; remove it there first`, { @@ -77,7 +78,7 @@ export async function checkSectionReferences( site: Site, op: FieldOperation, ): Promise { - if (op.op === "set_field_localized") return; + if (op.op === "set_field_localized" || op.sub !== undefined) return; const key = operationKey(op); const pages = await pagesUsing(ctx.store.sql, sectionUse(site.id, op.collection, key)); if (pages.length === 0) return; diff --git a/packages/core-ts/src/fields-operations/retype.ts b/packages/core-ts/src/fields-operations/retype.ts new file mode 100644 index 00000000..f52a9bcc --- /dev/null +++ b/packages/core-ts/src/fields-operations/retype.ts @@ -0,0 +1,188 @@ +/** + * RFC-0041 §6 and RFC-0045 §7: the definition `change_field_type` ends with, and the request + * checks the new types need. A field keeps its shared properties (key, labels, `localized`, + * `requiredToPublish`) and takes the new type's from the request: `options` and `multiple` for a + * select, `collections`, `multiple` and `maxItems` for a reference, `subFields`, `maxItems` and + * `itemLabel` for a repeater. With `sub`, one sub-field of a repeater changes type. Pure; the dry + * run then runs `checkFieldDefinitions` on the site the operation leaves. + */ +import type { FieldDefinition, SubFieldDefinition } from "@cloudhal/core-contracts"; +import { InvariantError } from "../errors.ts"; +import { isRecord, type RepeaterFieldDefinition } from "../fields/types.ts"; +import type { FieldOperation } from "./types.ts"; + +type Retype = Extract; + +const COMMON = ["key", "type", "labels", "localized", "requiredToPublish"] as const; + +function invalid(path: string, message: string): never { + throw new InvariantError("invalid_fields", message, { + issues: [{ path, code: "invalid_value", message }], + }); +} + +/** The reference collections a definition (or its one reference sub-field) holds, if any. */ +function referenceCollections(def: FieldDefinition): readonly string[] | undefined { + if (def.type === "reference") return def.collections; + if (def.type !== "repeater" || !Array.isArray(def.fields)) return undefined; + const subs = def.fields.filter((s) => isRecord(s) && s.type === "reference"); + return subs.length > 0 + ? subs.flatMap((s) => (s.type === "reference" ? s.collections : [])) + : undefined; +} + +/** A select's options: the request's, else the field's own. */ +function optionsOf(op: Retype, was: { type: string; options?: unknown }) { + const options = + op.options ?? (was.type === "select" ? (was.options as Retype["options"]) : undefined); + if (!options?.length) invalid("options", "a select needs its options"); + return options; +} + +/** The new type's own properties of a top-level field. */ +function ownProperties(def: FieldDefinition, op: Retype): Record { + const out: Record = {}; + if (op.type === "select") { + out.options = optionsOf(op, def); + if (op.multiple ?? (def.type === "select" ? def.multiple : undefined)) out.multiple = true; + } + if (op.type === "reference") { + const collections = op.collections ?? referenceCollections(def); + if (!collections?.length) invalid("collections", "a reference needs its target collections"); + out.collections = [...new Set(collections)]; + // From a repeater it stays a list; a reference keeps its own unless the request says. + // From a repeater it is always a list (the row is reference (multiple) ↔ repeater), so a + // `multiple: false` (the CLI's default) is ignored there, and `maxItems` kept. + const multiple = + def.type === "repeater" || (op.multiple ?? (def.type === "reference" && def.multiple)); + if (multiple) out.multiple = true; + const maxItems = op.maxItems ?? ("maxItems" in def ? def.maxItems : undefined); + if (multiple && maxItems !== undefined) out.maxItems = maxItems; + if (def.type === "reference" && def.where) out.where = def.where; + } + if (op.type === "repeater") { + if (!op.subFields?.length) invalid("subFields", "a repeater needs its sub-fields"); + out.fields = op.subFields; + const maxItems = op.maxItems ?? ("maxItems" in def ? def.maxItems : undefined); + if (maxItems !== undefined) out.maxItems = maxItems; + if (op.itemLabel !== undefined) out.itemLabel = op.itemLabel; + } + // RFC-0045 §7: a repeater's items join by line breaks. + if (op.type === "text" && def.type === "repeater") out.multiline = true; + return out; +} + +/** The sub-field `op.sub` with its new type (sub-fields have no `multiple`, `maxItems` or items). */ +function retypedSub(sub: SubFieldDefinition, op: Retype): SubFieldDefinition { + const out: Record = { key: sub.key, type: op.type, labels: sub.labels }; + if ("requiredToPublish" in sub && sub.requiredToPublish !== undefined && op.type !== "boolean") + out.requiredToPublish = sub.requiredToPublish; + if (op.type === "select") out.options = optionsOf(op, sub); + if (op.type === "reference") { + const collections = op.collections ?? (sub.type === "reference" ? sub.collections : undefined); + if (!collections?.length) invalid("collections", "a reference needs its target collections"); + out.collections = [...new Set(collections)]; + } + return out as unknown as SubFieldDefinition; +} + +/** The definition `change_field_type` ends with: the shared properties plus the new type's. */ +export function retyped(def: FieldDefinition, op: Retype): FieldDefinition { + if (op.sub !== undefined) { + const repeater = def as RepeaterFieldDefinition; + const fields = repeater.fields.map((s) => (s.key === op.sub ? retypedSub(s, op) : s)); + return { ...repeater, fields } as FieldDefinition; + } + const base: Record = {}; + const was = def as unknown as Record; + for (const k of COMMON) if (was[k] !== undefined) base[k] = was[k]; + base.type = op.type; + if (op.type === "boolean") delete base.requiredToPublish; + const searchable = "searchable" in def ? def.searchable : undefined; + if ((op.type === "text" || op.type === "richText") && searchable !== undefined) + base.searchable = searchable; + return { ...base, ...ownProperties(def, op) } as unknown as FieldDefinition; +} + +/** + * The inputs only some targets take: refused elsewhere, so a request means what it says. + * `multiple` is not among them: it is ignored where it does not apply, as before RFC-0045 + * (agents often fill every optional boolean). + */ +const TAKES: Record<"collections" | "maxItems" | "subFields" | "itemLabel", readonly string[]> = { + collections: ["reference"], + maxItems: ["reference", "repeater"], + subFields: ["repeater"], + itemLabel: ["repeater"], +}; + +/** The value type a definition (or a sub-field) holds, for the same-type rule. */ +const isList = (def: { type: string; multiple?: boolean | undefined }) => def.multiple === true; + +/** + * RFC-0045 §7: the request checks of a type change (beyond `checkFieldDefinitions` on the result): + * inputs the target takes, a same-type change that reshapes (select, reference single ↔ + * multiple), a repeater source with one sub-field, and id-carrying conversions only widening the + * target collections, so phase B never writes a carried id where the field no longer allows it. + */ +export function checkRetype(def: FieldDefinition, op: Retype): void { + const sub = + op.sub === undefined + ? undefined + : (def as RepeaterFieldDefinition).fields.find((s) => s.key === op.sub); + const from: { type: string; multiple?: boolean | undefined } = sub ?? def; + for (const [input, types] of Object.entries(TAKES)) + if ( + op[input as keyof typeof TAKES] !== undefined && + (!types.includes(op.type) || (sub !== undefined && input !== "collections")) + ) + invalid( + input, + `${input} does not apply to ${sub ? "a sub-field's" : `a ${op.type}`} type change`, + ); + if (from.type === op.type) { + const reshapes = + (op.type === "select" && sub === undefined) || + (op.type === "reference" && + sub === undefined && + (op.multiple ?? isList(from)) !== isList(from)); + if (!reshapes) + throw new InvariantError( + "invalid_fields", + `${op.sub ?? def.key} already is a ${op.type} field`, + ); + } + if ( + sub === undefined && + def.type === "repeater" && + (op.type === "text" || op.type === "reference") + ) { + const [only, ...more] = def.fields; + const fits = + op.type === "text" + ? only?.type === "text" || only?.type === "richText" + : only?.type === "reference"; + if (!only || more.length > 0 || !fits) + invalid( + "type", + `only a repeater with one ${op.type === "text" ? "text or rich-text" : "reference"} sub-field converts to ${op.type}`, + ); + } + const carried = sub + ? sub.type === "reference" + ? sub.collections + : undefined + : referenceCollections(def); + // Only a field (or sub-field) that stays a reference keeps ids, so only it must widen. + const target = retyped(def, op); + const after = sub + ? undefined + : op.type === "reference" || op.type === "repeater" + ? referenceCollections(target) + : undefined; + if (carried && after && from.type !== "text" && !carried.every((c) => after.includes(c))) + invalid( + "collections", + `a type change keeps every collection ${op.sub ?? def.key} allows (${carried.join(", ")})`, + ); +} diff --git a/packages/core-ts/src/fields-operations/revert-sub.ts b/packages/core-ts/src/fields-operations/revert-sub.ts new file mode 100644 index 00000000..85c580e0 --- /dev/null +++ b/packages/core-ts/src/fields-operations/revert-sub.ts @@ -0,0 +1,65 @@ +/** + * RFC-0045 §7 `sub`: the revert's first step for a sub-field operation. It starts from the + * repeater as it is now and puts back only the named sub-field, so sub-fields added since stay: + * a removed sub-field is inserted at its old index; a renamed or retyped one is replaced by its + * earlier definition, under the inverse marker. A sub-field of the same key that differs from the + * earlier one is refused (`field_key_in_use`). Pure. + */ +import type { FieldDefinition, FieldMigration } from "@cloudhal/core-contracts"; +import { InvariantError } from "../errors.ts"; +import { type RepeaterFieldDefinition, sameJson } from "../fields/types.ts"; +import type { FieldOperation } from "./types.ts"; + +type SubOp = FieldOperation & { sub: string }; + +/** The earlier sub-field and its index in the earlier repeater. */ +function earlierSub(earlier: FieldDefinition, key: string) { + const fields = earlier.type === "repeater" ? earlier.fields : []; + const index = fields.findIndex((s) => s.key === key); + return { sub: fields[index], index }; +} + +/** True when a sub-field removal is undone: the repeater holds the earlier sub-field again. */ +export function subRestored(now: FieldDefinition, earlier: FieldDefinition, key: string): boolean { + const was = earlierSub(earlier, key).sub; + return now.type === "repeater" && now.fields.some((s) => s.key === key && sameJson(s, was)); +} + +/** The current repeater with the operation's sub-field put back as it was. */ +export function restoredSubDefinition( + now: FieldDefinition, + earlier: FieldDefinition, + op: SubOp, + marker: FieldMigration | undefined, +): FieldDefinition { + const { sub: was, index } = earlierSub(earlier, op.sub); + if (now.type !== "repeater" || !was) return earlier; + const repeater = now as RepeaterFieldDefinition; + const fields = [...repeater.fields]; + let itemLabel = repeater.itemLabel; + if (op.op === "remove_field") { + const held = fields.find((s) => s.key === op.sub); + if (held && !sameJson(held, was)) + throw new InvariantError( + "field_key_in_use", + `${now.key} has another sub-field ${op.sub} now; rename or remove it before the revert`, + { issues: [] }, + ); + if (!held) fields.splice(Math.min(index, fields.length), 0, was); + const label = earlier.type === "repeater" ? earlier.itemLabel : undefined; + if (itemLabel === undefined && label === op.sub) itemLabel = op.sub; + } else { + const key = op.op === "rename_field" ? op.to : op.sub; + const at = fields.findIndex((s) => s.key === key); + if (at >= 0) fields[at] = was; + else fields.splice(Math.min(index, fields.length), 0, was); + if (itemLabel === key) itemLabel = op.sub; + } + const { itemLabel: _l, migration: _m, ...rest } = repeater; + return { + ...rest, + fields, + ...(itemLabel === undefined ? {} : { itemLabel }), + ...(marker ? { migration: marker } : {}), + } as FieldDefinition; +} diff --git a/packages/core-ts/src/fields-operations/revert.ts b/packages/core-ts/src/fields-operations/revert.ts index 19c2dabc..0d36e540 100644 --- a/packages/core-ts/src/fields-operations/revert.ts +++ b/packages/core-ts/src/fields-operations/revert.ts @@ -14,11 +14,29 @@ import type { ContentEntry, FieldDefinition, FieldMigration, Site } from "@cloudhal/core-contracts"; import { InvariantError } from "../errors.ts"; import { convertContext, rewriteFields } from "../fields/migration.ts"; -import { sameJson } from "../fields/types.ts"; +import { fieldDefinitions, sameJson } from "../fields/types.ts"; +import { loadConversionEntries } from "../references/conversion-load.ts"; +import { loadFieldRows } from "../references/conversion-rows.ts"; +import { hasReferenceFields } from "../references/issues.ts"; +import { + changedReferenceIds, + type DroppedReference, + dropMissingReferences, +} from "../references/revert-drops.ts"; +import { readMembers, targetProblem } from "../references/targets.ts"; import { renameSettingsReferences } from "../store/collection-settings.ts"; +import { withinReferenceCap } from "./cap.ts"; import { markedField, markerGuard, pendingCondition, phaseCSite, withFields } from "./plan.ts"; import { checkSectionReferences, checkSettingsReferences, renamedPages } from "./references.ts"; -import { type Decision, loadEntries, type Row, rewritePass } from "./rewrite.ts"; +import { restoredSubDefinition, subRestored } from "./revert-sub.ts"; +import { + type Decide, + type Decision, + isSameSiteTranslation, + loadEntries, + type Row, + rewritePass, +} from "./rewrite.ts"; import { beforeSiteOf } from "./run.ts"; import { labelOf, @@ -48,6 +66,13 @@ function inverseMarker( op: FieldOperation, now: FieldDefinition | undefined, ): FieldMigration | undefined { + // RFC-0045 `sub`: the marker reads the sub-field's end state as the earlier one's source. + if (op.sub !== undefined) { + if (op.op === "rename_field") return { id, sub: op.sub, fromKey: op.to }; + if (op.op !== "change_field_type" || now?.type !== "repeater") return undefined; + const sub = now.fields.find((f) => f.key === op.sub); + return sub ? { id, sub: op.sub, fromType: sub.type } : undefined; + } switch (op.op) { case "rename_field": return { id, fromKey: op.to }; @@ -68,18 +93,25 @@ function restoredSite(site: Site, op: FieldOperation, beforeSite: Site, id: stri const defs = [...(site.collections?.[op.collection]?.fields ?? [])]; const at = defs.findIndex((d) => d.key === finalKey(op)); const marker = inverseMarker(id, op, defs[at]); - const def = (marker ? { ...earlier, migration: marker } : earlier) as FieldDefinition; + const now = defs[at]; + // RFC-0045 `sub`: only the named sub-field goes back; the repeater keeps what was added since. + const def = + op.sub !== undefined && now + ? restoredSubDefinition(now, earlier, { ...op, sub: op.sub }, marker) + : ((marker ? { ...earlier, migration: marker } : earlier) as FieldDefinition); if (at >= 0) defs[at] = def; else defs.splice(Math.min(index, defs.length), 0, def); const restored = withFields(site, op.collection, defs); // RFC-0042 §4: a rename's references in the stage 3 settings follow the key back. - return op.op === "rename_field" + return op.op === "rename_field" && op.sub === undefined ? renameSettingsReferences(restored, op.collection, op.to, op.from) : restored; } /** The operation that undoes `op` as far as references are concerned (RFC-0042 §4). */ function inverseOf(op: FieldOperation, beforeSite: Site): FieldOperation | undefined { + // Settings and sections never name a repeater's sub-fields. + if (op.sub !== undefined) return undefined; if (op.op === "rename_field") return { ...op, from: op.to, to: op.from }; if (op.op !== "change_field_type") return undefined; const was = beforeSite.collections?.[op.collection]?.fields?.find((d) => d.key === op.key); @@ -90,6 +122,12 @@ function inverseOf(op: FieldOperation, beforeSite: Site): FieldOperation | undef key: op.key, type: was.type, ...(was.type === "select" ? { options: was.options, multiple: was.multiple === true } : {}), + ...(was.type === "reference" + ? { collections: was.collections, multiple: was.multiple === true, maxItems: was.maxItems } + : {}), + ...(was.type === "repeater" + ? { subFields: was.fields, maxItems: was.maxItems, itemLabel: was.itemLabel } + : {}), }; } @@ -154,6 +192,44 @@ async function restorable( return out; } +/** + * RFC-0045 §7: the pass's decisions with restored references whose group has no member left in + * the site (in an allowed collection, any status) dropped; one read of their members per pass. + * The pass then guards the rest (`targets: "present"`), so a target deleted meanwhile skips the + * entry, and the next pass drops it. + */ +async function droppingMissing( + ctx: OperationContext, + site: Site, + input: { rows: readonly Row[]; decide: Decide }, +): Promise<{ decide: Decide; drops: Map }> { + const drops = new Map(); + if (!hasReferenceFields(site)) return { decide: input.decide, drops }; + const defsOf = (row: Row) => fieldDefinitions(site, (row.entry as ContentEntry).collection); + const valuesOf = (d: Decision) => d.replace?.fields ?? d.fields ?? {}; + const ids = new Set(); + for (const row of input.rows) { + const d = input.decide(row, isSameSiteTranslation(row.entry)); + if (d) + for (const id of changedReferenceIds(defsOf(row), valuesOf(d), row.entry.fields ?? {})) + ids.add(id); + } + const members = await readMembers(ctx.store.sql, site.id, [...ids]); + const keep = (groupId: string, collections: readonly string[]) => + targetProblem(site, { groupId, collections, fresh: false }, members) === undefined; + const decide: Decide = (row, translation) => { + const d = input.decide(row, translation); + if (!d) return d; + const r = dropMissingReferences(defsOf(row), valuesOf(d), row.entry.fields ?? {}, keep); + if (r.dropped.length === 0) return d; + drops.set(row.entry.id, r.dropped); + return d.replace + ? { ...d, replace: { ...d.replace, fields: r.fields } } + : { ...d, fields: r.fields }; + }; + return { decide, drops }; +} + export async function revertFieldOperation( store: OperationContext["store"], siteId: string, @@ -178,8 +254,15 @@ export async function revertFieldOperation( const beforeSite = await beforeSiteOf(ctx, before); let site = await requireSite(ctx, siteId); const removal = op.op === "remove_field"; + // A sub-field removal is undone once the repeater holds the earlier sub-field (RFC-0045 `sub`). + const earlierDef = beforeSite.collections?.[op.collection]?.fields?.find( + (d) => d.key === operationKey(op), + ); const restoredDef = (site.collections?.[op.collection]?.fields ?? []).some((d) => - removal ? d.key === op.key : d.migration?.id === operationId, + !removal + ? d.migration?.id === operationId + : d.key === op.key && + (op.sub === undefined || (!!earlierDef && subRestored(d, earlierDef, op.sub))), ); if (!restoredDef) { const running = (site.collections?.[op.collection]?.fields ?? []).find((d) => d.migration); @@ -190,26 +273,48 @@ export async function revertFieldOperation( before: new Map(before.nodes.map((n) => [n.id, n.hash])), after: new Map(after.nodes.map((n) => [n.id, n.hash])), }; - const out: FieldRevertResult = { operationId, done: false, restored: 0, edited: [], skipped: [] }; + const out: FieldRevertResult = { + operationId, + done: false, + restored: 0, + edited: [], + dropped: [], + unconvertible: [], + skipped: [], + }; for (let pass = 0; pass < 3; pass++) { site = await requireSite(ctx, siteId); const marked = markedField(site, op.collection, operationId); const rows = await loadEntries(ctx, siteId, op.collection); const earlier = await restorable(ctx, rows, snaps); - const convert = convertContext(site); - const decide = (row: Row, translation: boolean): Decision | undefined => { + // RFC-0045 §7: the inverse conversion's maps, read once before the pass. + const entries = marked ? await loadConversionEntries(ctx.store, site, [marked.def]) : undefined; + const rowsOf = marked + ? await loadFieldRows(ctx.store.sql, { siteId, collection: op.collection, def: marked.def }) + : undefined; + const convert = { ...convertContext(site), entries }; + const raw = (row: Row, translation: boolean): Decision | undefined => { const body = earlier.get(row.entry.id); if (body) return { replace: body }; if (!marked) return undefined; const r = rewriteFields({ def: marked.def, values: row.entry.fields, - convert, + convert: { ...convert, group: row.entry.translation?.group ?? row.entry.id }, translation, pristine: false, + onInvalid: options.onInvalid, + rows: rowsOf?.(row.entry.id), + }); + if (!r.values) return r.change ? { change: r.change } : undefined; + // RFC-0045 §1: converting back into references counts the cap too. + return withinReferenceCap(fieldDefinitions(site, op.collection), marked.def, { + values: r.values, + change: r.change, + onInvalid: options.onInvalid, }); - return r.values ? { fields: r.values, change: r.change } : undefined; }; + const { decide, drops } = await droppingMissing(ctx, site, { rows, decide: raw }); const r = await rewritePass({ ctx, site, @@ -220,11 +325,15 @@ export async function revertFieldOperation( freshAlso: (sourceId) => (earlier.has(sourceId) ? snaps.before.get(sourceId) : undefined), done: out.restored + out.edited.length, lists: { collection: op.collection, keys: operationKeys(op) }, + targets: "present", }); - for (const w of r.written) + for (const w of r.written) { if (w.decision?.replace) out.restored++; else if (w.decision?.fields) out.edited.push(w.id); + for (const d of drops.get(w.id) ?? []) out.dropped.push({ entryId: w.id, ...d }); + } out.skipped = r.skipped; + out.unconvertible = r.unconvertible; if (r.exhausted) return out; if (r.skipped.length === 0) break; } @@ -243,5 +352,5 @@ export async function revertFieldOperation( metadata: { [META.kind]: FIELD_MIGRATION, [META.id]: operationId, [META.phase]: "reverted" }, }); await settleLists(ctx, siteId, out.restored + out.edited.length); - return { ...out, done: true, skipped: [], snapshotId: snapshot.id }; + return { ...out, done: true, skipped: [], unconvertible: [], snapshotId: snapshot.id }; } diff --git a/packages/core-ts/src/fields-operations/rewrite.ts b/packages/core-ts/src/fields-operations/rewrite.ts index e196eed8..f0f14361 100644 --- a/packages/core-ts/src/fields-operations/rewrite.ts +++ b/packages/core-ts/src/fields-operations/rewrite.ts @@ -16,6 +16,7 @@ import { import type { SqlStatement } from "@cloudhal/storage"; import { ConflictError } from "../errors.ts"; import type { ValueChange } from "../fields/migration.ts"; +import type { FieldRows } from "../fields/text-reference.ts"; import { sameJson } from "../fields/types.ts"; import { nodeHash } from "../hash.ts"; import { passResetStatement } from "../list-values/state-sql.ts"; @@ -83,6 +84,15 @@ export interface PassInput { readonly freshAlso?: ((sourceId: string) => string | undefined) | undefined; /** Entries this call already rewrote (the budget counts across passes). */ readonly done: number; + /** + * RFC-0045 §7: what the target guard requires of changed reference values: `fresh` (phase B) + * only newly named ids, `present` (a revert) a member for every one (`rewriteTargetGuard`). + */ + readonly targets?: "fresh" | "present" | undefined; + /** RFC-0045 §7: a running text → reference path and its rows per node (phase B's guard). */ + readonly converting?: + | { readonly path: string; readonly rowsOf: (nodeId: string) => FieldRows } + | undefined; /** * RFC-0042 §4: the collection and the field keys the operation rewrites. Each batch resets * (`passResetStatement`) the list values of collections listing one of them, or any other key a @@ -99,6 +109,8 @@ export interface PassResult { written: { id: string; decision: Decision | undefined }[]; /** The budget ran out before every planned entry was written. */ exhausted: boolean; + /** RFC-0045 §7: entries left as they are because a value does not convert, with the reason. */ + unconvertible: { entryId: string; reason: string }[]; } /** Sources before the translations made from them (chains are short; capped at 8). */ @@ -161,7 +173,13 @@ async function statementsOf( ): Promise { // RFC-0045 §2.6: a rewrite that changes reference values writes them only under the target guard. const targets = p.fieldsChanged - ? rewriteTargetGuard(input.site, p.next, p.stored.entry) + ? rewriteTargetGuard(input.site, p.next, p.stored.entry, { + mode: input.targets, + converting: input.converting && { + path: input.converting.path, + rows: input.converting.rowsOf(p.next.id), + }, + }) : undefined; const cas = await casUpdateStatement( "contentEntry", @@ -193,7 +211,17 @@ function resetKeys(keys: readonly string[], planned: readonly Planned[]): string export async function rewritePass(input: PassInput): Promise { const { ctx } = input; const { planned, decisions } = await planPass(input, ctx.stamp()); - const out: PassResult = { rewritten: 0, skipped: [], written: [], exhausted: false }; + const out: PassResult = { + rewritten: 0, + skipped: [], + written: [], + exhausted: false, + unconvertible: [...decisions].flatMap(([entryId, d]) => + d?.change?.kind === "unconvertible" && !d.fields && !d.replace + ? [{ entryId, reason: d.change.reason ?? "does not convert" }] + : [], + ), + }; // RFC-0045 §2.1: rows and target pairs come from the pass's definitions, so anchor them. let anchor: Cond | undefined; if (hasReferenceFields(input.site)) diff --git a/packages/core-ts/src/fields-operations/run.ts b/packages/core-ts/src/fields-operations/run.ts index c98f6da8..bb38368d 100644 --- a/packages/core-ts/src/fields-operations/run.ts +++ b/packages/core-ts/src/fields-operations/run.ts @@ -5,9 +5,21 @@ * over what is still pending) and the `after` snapshot. Only the draft graph changes: the published * snapshot keeps the old definitions with the old values until the next publish. */ -import type { FieldDefinition, Site, Snapshot } from "@cloudhal/core-contracts"; +import { + assertValid, + type FieldDefinition, + type Site, + type Snapshot, +} from "@cloudhal/core-contracts"; import { InvariantError } from "../errors.ts"; -import { convertContext, rewriteFields } from "../fields/migration.ts"; +import { checkFieldDefinitions, refuseFieldIssues } from "../fields/index.ts"; +import { convertContext, rewriteFields, type ValueChange } from "../fields/migration.ts"; +import type { FieldRows } from "../fields/text-reference.ts"; +import { fieldDefinitions } from "../fields/types.ts"; +import { referenceDefinitionOptions } from "../references/capability.ts"; +import { loadConversionEntries } from "../references/conversion-load.ts"; +import { convertingPath, loadFieldRows } from "../references/conversion-rows.ts"; +import { withinReferenceCap } from "./cap.ts"; import { checkOperation, currentDefinition, @@ -50,25 +62,44 @@ import { const PASSES = 3; const DRY_RUN_ID = "snap_fielddryrun"; -/** How entries change under `def`'s marker; `pristine` per entry from the `before` snapshot. */ -function decider( +/** + * How entries change under `def`'s marker; `pristine` per entry from the `before` snapshot. A + * text ↔ reference conversion's maps and its `entry_refs` rows are read once here, before the + * pass (RFC-0045 §7): its values are decided by the rows phase C reads. + */ +async function decider( + ctx: OperationContext, site: Site, def: FieldDefinition, - op: FieldOperation, - pristine: (id: string, hash: string) => boolean, -): Decide { - const convert = convertContext(site); - const onInvalid = op.op === "change_field_type" ? op.onInvalid : undefined; + input: { + op: FieldOperation; + pristine: (id: string, hash: string) => boolean; + /** A continuing call's `onInvalid` (RFC-0045 §7), else the request's. */ + onInvalid?: "clear" | undefined; + /** The pass's rows, when the caller read them for its guard too. */ + rowsOf?: ((nodeId: string) => FieldRows) | undefined; + }, +): Promise { + const { op, pristine } = input; + const entries = await loadConversionEntries(ctx.store, site, [def]); + const rowsOf = + input.rowsOf ?? + (await loadFieldRows(ctx.store.sql, { siteId: site.id, collection: op.collection, def })); + const convert = { ...convertContext(site), entries }; + const onInvalid = input.onInvalid ?? (op.op === "change_field_type" ? op.onInvalid : undefined); + const defs = fieldDefinitions(site, op.collection); return (row, translation) => { const r = rewriteFields({ def, values: row.entry.fields, - convert, + convert: { ...convert, group: row.entry.translation?.group ?? row.entry.id }, translation, pristine: pristine(row.entry.id, row.hash), onInvalid, + rows: rowsOf?.(row.entry.id), }); - return r.values || r.change ? { fields: r.values, change: r.change } : undefined; + if (!r.values) return r.change ? { fields: undefined, change: r.change } : undefined; + return withinReferenceCap(defs, def, { values: r.values, change: r.change, onInvalid }); }; } @@ -79,17 +110,33 @@ async function dryRun( op: FieldOperation, ): Promise { const { def } = currentDefinition(site, op); - const marked = markedField(phaseASite(site, op, DRY_RUN_ID), op.collection, DRY_RUN_ID); + const phaseA = phaseASite(site, op, DRY_RUN_ID); + const marked = markedField(phaseA, op.collection, DRY_RUN_ID); const rows = await loadEntries(ctx, site.id, op.collection); return buildReport({ op, before: def, after: targetDefinition(def, op), rows, - decide: decider(site, marked?.def as FieldDefinition, op, () => true), + decide: await decider(ctx, phaseA, marked?.def as FieldDefinition, { + op, + pristine: () => true, + }), }); } +/** + * RFC-0045 §7: the definitions the operation ends with pass the contract and + * `checkFieldDefinitions` (with the store's capability: the types stay refused until rollout step + * 7), so the dry run refuses what phase A or C could not write. + */ +async function checkEndState(ctx: OperationContext, site: Site, op: FieldOperation) { + const end = phaseCSite(phaseASite(site, op, DRY_RUN_ID), op.collection, DRY_RUN_ID) ?? site; + assertValid("site", end); + const options = await referenceDefinitionOptions(ctx.store, end, true); + refuseFieldIssues(checkFieldDefinitions(end, undefined, options)); +} + function result( record: Pick & { after?: Snapshot | undefined }, report: FieldOperationReport, @@ -103,6 +150,7 @@ function result( report, rewritten: 0, skipped: [], + unconvertible: [], beforeSnapshotId: record.before.id, ...(record.after ? { afterSnapshotId: record.after.id } : {}), ...extra, @@ -129,6 +177,7 @@ export async function runFieldOperation( // RFC-0042 §4: what list fields, expiry, structured data and sections make of the change. await checkSectionReferences(ctx, site, op); checkSettingsReferences(site, op); + await checkEndState(ctx, site, op); const report = await dryRun(ctx, site, op); if (options.dryRun) return { @@ -139,6 +188,7 @@ export async function runFieldOperation( report, rewritten: 0, skipped: [], + unconvertible: [], }; if (report.refusal) throw new InvariantError("field_values_unconvertible", report.refusal.message, { report }); @@ -190,7 +240,10 @@ async function remaining( after: targetDefinition(def, op), rows: marked ? await loadEntries(ctx, site.id, op.collection) : [], decide: marked - ? decider(site, marked.def, op, (id, hash) => hashes.get(id) === hash) + ? await decider(ctx, site, marked.def, { + op, + pristine: (id, hash) => hashes.get(id) === hash, + }) : () => undefined, }); } @@ -219,7 +272,7 @@ async function proceed( const running = runningMarker(site.collections?.[op.collection]?.fields ?? []); if (running) refuseRunning(op.collection, running.id); const sections = - op.op === "rename_field" + op.op === "rename_field" && op.sub === undefined ? (s: Site) => renamedPages(ctx, s, { collection: op.collection, from: op.from, to: op.to }) : undefined; @@ -230,23 +283,39 @@ async function proceed( const hashes = new Map(before.nodes.map((n) => [n.id, n.hash])); let rewritten = 0; let skipped: FieldOperationResult["skipped"] = []; + let unconvertible: FieldOperationResult["unconvertible"] = []; for (let pass = 0; pass < PASSES; pass++) { site = await requireSite(ctx, siteId); const marked = markedField(site, op.collection, id); if (!marked) break; + // RFC-0045 §7: one read of the converting path's rows, for the decisions and the guard. + const path = convertingPath(marked.def); + const rowsOf = await loadFieldRows(ctx.store.sql, { + siteId, + collection: op.collection, + def: marked.def, + }); const r = await rewritePass({ ctx, site, rows: await loadEntries(ctx, siteId, op.collection), defs: marked.defs, - decide: decider(site, marked.def, op, (eid, hash) => hashes.get(eid) === hash), + decide: await decider(ctx, site, marked.def, { + op, + pristine: (eid, hash) => hashes.get(eid) === hash, + onInvalid: options.onInvalid, + rowsOf, + }), + converting: path !== undefined && rowsOf ? { path, rowsOf } : undefined, guard: markerGuard(siteId, op.collection, id), done: rewritten, lists: { collection: op.collection, keys: operationKeys(op) }, }); rewritten += r.rewritten; skipped = r.skipped; - if (r.exhausted) return result(record, report, { rewritten, skipped, done: false }); + unconvertible = r.unconvertible; + if (r.exhausted) + return result(record, report, { rewritten, skipped, unconvertible, done: false }); if (skipped.length === 0) break; } const marked = markedField(site, op.collection, id); @@ -255,7 +324,8 @@ async function proceed( const wrote = await writeSite(ctx, siteId, (s) => phaseCSite(s, op.collection, id), { guard: nothingPending(pending), }); - if (wrote === "guarded") return result(record, report, { rewritten, skipped, done: false }); + if (wrote === "guarded") + return result(record, report, { rewritten, skipped, unconvertible, done: false }); } const after = await ctx.store.createSnapshot(siteId, { actor: options.actor, diff --git a/packages/core-ts/src/fields-operations/state.ts b/packages/core-ts/src/fields-operations/state.ts index c1264ca6..32299945 100644 --- a/packages/core-ts/src/fields-operations/state.ts +++ b/packages/core-ts/src/fields-operations/state.ts @@ -16,6 +16,7 @@ import { settleListValues } from "../list-values/backfill.ts"; import { planListState } from "../list-values/site-plan.ts"; import { type Cond, casUpdateStatement, OpBatch, runOpBatch } from "../node-sql.ts"; import { referenceDefinitionOptions } from "../references/capability.ts"; +import { sharedUpdates } from "../references/localize.ts"; import type { PageRewrite } from "./references.ts"; import { FIELD_MIGRATION, @@ -101,11 +102,13 @@ export async function writeSite( ), ); const rewrites = (await pages?.(site)) ?? []; + const hash = await nodeHash(next); // RFC-0042 §4: a marker on a listed field changes its fingerprint (phases A and C both). - const lists = (await planListState(site, next)).statements( - await nodeHash(next), - next.updatedAt, - ); + // RFC-0045 §7: a field's new `localized` reaches its `entry_refs` rows in the same batch. + const lists = [ + ...(await planListState(site, next)).statements(hash, next.updatedAt), + ...sharedUpdates(site, next, hash), + ]; if (rewrites.length > 0) { if (await writeWithPages(ctx, { site, next, guard, rewrites, lists })) return "written"; } else { diff --git a/packages/core-ts/src/fields-operations/types.ts b/packages/core-ts/src/fields-operations/types.ts index eab5aef9..dd6da632 100644 --- a/packages/core-ts/src/fields-operations/types.ts +++ b/packages/core-ts/src/fields-operations/types.ts @@ -2,7 +2,12 @@ * RFC-0041 §6: the four field operations, their reports and results. Shared by the runner, the * API routes and the CLI. */ -import type { Actor, FieldDefinition, FieldOption } from "@cloudhal/core-contracts"; +import type { + Actor, + FieldDefinition, + FieldOption, + SubFieldDefinition, +} from "@cloudhal/core-contracts"; import type { FieldType } from "../fields/types.ts"; import type { SiteGraphStore } from "../store.ts"; @@ -32,22 +37,41 @@ export const FIELD_OPERATIONS: readonly FieldOperationName[] = [ "remove_field", ]; -/** One operation as requested: what an agent tool, the API and the CLI send. */ +/** + * One operation as requested: what an agent tool, the API and the CLI send. RFC-0045 §7: `sub` + * names a repeater's sub-field (`from`/`key` is then the repeater), which the operation renames, + * converts or removes in every item. + */ export type FieldOperation = - | { op: "rename_field"; collection: string; from: string; to: string } + | { op: "rename_field"; collection: string; from: string; to: string; sub?: string | undefined } | { op: "change_field_type"; collection: string; key: string; type: FieldType; + sub?: string | undefined; /** The options of a new select (default: the field's own when it already is one). */ options?: FieldOption[] | undefined; multiple?: boolean | undefined; + /** RFC-0045: a reference's target collections (required to a reference). */ + collections?: string[] | undefined; + /** RFC-0045: a multiple reference's or a repeater's item cap. */ + maxItems?: number | undefined; + /** RFC-0045: a repeater's sub-fields (required to a repeater) and the one naming items. */ + subFields?: SubFieldDefinition[] | undefined; + itemLabel?: string | undefined; /** Drop values the table cannot convert; without it such values refuse the operation. */ onInvalid?: "clear" | undefined; } - | { op: "set_field_localized"; collection: string; key: string; localized: boolean } - | { op: "remove_field"; collection: string; key: string }; + | { + op: "set_field_localized"; + collection: string; + key: string; + localized: boolean; + /** Refused: sub-fields share their repeater's `localized`. */ + sub?: string | undefined; + } + | { op: "remove_field"; collection: string; key: string; sub?: string | undefined }; /** Counts of what happens to values (per locale too). */ export interface FieldValueCounts { @@ -106,6 +130,12 @@ export interface FieldOperationResult { rewritten: number; /** Entries changed meanwhile, retried by the next call. */ skipped: { entryId: string; reason: string }[]; + /** + * RFC-0045 §7: entries whose value stopped converting while the operation ran (a target + * archived, a second entry with the same title, a target deleted). They keep the operation + * running; continuing it with `onInvalid: "clear"` drops them. + */ + unconvertible: { entryId: string; reason: string }[]; beforeSnapshotId?: string; afterSnapshotId?: string; } @@ -130,6 +160,13 @@ export interface FieldRevertResult { restored: number; /** Entries edited since the operation: converted back through the marker instead. */ edited: string[]; + /** + * RFC-0045 §7: restored references whose group has no member left in the site (in a collection + * the field allows), dropped from the restored value. + */ + dropped: { entryId: string; field: string; targetId: string }[]; + /** Edited entries whose value does not convert back (see `FieldOperationResult`). */ + unconvertible: { entryId: string; reason: string }[]; skipped: { entryId: string; reason: string }[]; snapshotId?: string; } @@ -141,6 +178,11 @@ export interface FieldOperationOptions { now?: (() => Date) | undefined; /** Stop after this much and answer `done: false` (default `FIELD_OPERATION_BUDGET`). */ budget?: { entries?: number; ms?: number } | undefined; + /** + * RFC-0045 §7: continuing or reverting, drop the values that no longer convert (the way out of + * an operation whose values became unconvertible while it ran). + */ + onInvalid?: "clear" | undefined; } /** The store plus the clock. */ @@ -167,14 +209,22 @@ export function operationContext( }; } -/** The field key the operation names (a rename's current key). */ +/** The field key the operation names (a rename's current key; with `sub`, the repeater). */ export const operationKey = (op: FieldOperation): string => op.op === "rename_field" ? op.from : op.key; -/** The field key once the operation is done. */ -export const finalKey = (op: FieldOperation): string => (op.op === "rename_field" ? op.to : op.key); +/** The field key once the operation is done (a sub-field rename keeps the repeater's). */ +export const finalKey = (op: FieldOperation): string => + op.op === "rename_field" && op.sub === undefined ? op.to : operationKey(op); /** Every field key the operation rewrites in entries (a rename's both). */ export const operationKeys = (op: FieldOperation): string[] => [ ...new Set([operationKey(op), finalKey(op)]), ]; + +/** The sub-field the operation names (RFC-0045 `sub`), if any. */ +export const operationSub = (op: FieldOperation): string | undefined => op.sub; + +/** `speakers.name` for a sub-field operation, else the field key. */ +export const operationPath = (op: FieldOperation): string => + op.sub === undefined ? operationKey(op) : `${operationKey(op)}.${op.sub}`; diff --git a/packages/core-ts/src/fields/conversions-references.ts b/packages/core-ts/src/fields/conversions-references.ts new file mode 100644 index 00000000..48bec338 --- /dev/null +++ b/packages/core-ts/src/fields/conversions-references.ts @@ -0,0 +1,129 @@ +/** + * RFC-0045 §7: the rows of `change_field_type`'s closed table for references and repeaters. + * + * | from → to | conversion | unconvertible | + * |---|---|---| + * | reference single ↔ multiple | one id ↔ a list of one; an empty list is no value | more than one id to single | + * | text → reference | the one group among the target collections with a usable member whose slug, or default-locale title, equals the text (case-folded); never the entry's own group | no match, or more than one | + * | reference → text | a member's title, any status (`ConversionEntries.titles`; a list: comma separated) | a group with no member in the site | + * | text → repeater | one item per non-empty line into the repeater's one `text` sub-field | a line over its `maxLength`, more lines than `maxItems` | + * | repeater → text | the items' one value each, joined by line breaks | items with other shapes | + * | reference ↔ repeater | one item per id into the repeater's one `reference` sub-field, and back | items with other shapes | + * + * The text ↔ reference rows read the prefetched maps (`ConvertContext.entries`), so the table stays + * pure and synchronous. Whether a value is already in the new type is decided before them, by one + * rule every caller shares (`text-reference.ts`). Repeater → reference drops repeated ids. Readers know only the new definition and the marker, so the repeater rows + * read an item's shape from the value: one own key per item. The operation refuses, before phase + * A, a repeater source that has more than one sub-field (`checkOperation`), so a reader never sees + * an item that phase B would treat differently. A rich-text sub-field's items join as their + * Markdown, as stored: the marker does not say which type the sub-field had. Pure and Worker-safe. + */ +import type { FieldDefinition } from "@cloudhal/core-contracts"; +import { foldText } from "../references/conversion-maps.ts"; +import type { Conversion, ConvertContext } from "./conversions.ts"; +import { isRecord } from "./types.ts"; + +type Step = (value: unknown, to: FieldDefinition, ctx: ConvertContext) => Conversion; + +const fail = (reason: string): Conversion => ({ ok: false, reason }); +const ok = (value: unknown): Conversion => ({ ok: true, value }); +const quote = (text: string) => `"${text.slice(0, 40)}"`; + +/** A reference value as a list of ids (a single id is a list of one), or undefined. */ +function idsOf(value: unknown): string[] | undefined { + if (typeof value === "string") return [value]; + if (Array.isArray(value) && value.every((v) => typeof v === "string")) return value; + return undefined; +} + +/** A list of ids into `to`: a multiple reference takes them all, a single one at most one. */ +function asReference(ids: readonly string[], to: FieldDefinition): Conversion { + if (to.type !== "reference") return fail("not a reference"); + if (ids.length === 0) return ok(undefined); + if (to.multiple) return ok([...ids]); + if (ids.length === 1) return ok(ids[0]); + return fail(`${ids.length} entries chosen; a single reference keeps one`); +} + +/** The key of a repeater's only sub-field of `type`, when it has exactly one sub-field. */ +function onlySub(to: FieldDefinition, type: "text" | "reference"): string | undefined { + if (to.type !== "repeater" || to.fields.length !== 1) return undefined; + const [sub] = to.fields; + return sub?.type === type ? sub.key : undefined; +} + +/** The items' one value each, or undefined when an item has any other shape. */ +function soleValues(value: unknown): string[] | undefined { + if (!Array.isArray(value)) return undefined; + const out: string[] = []; + for (const item of value) { + if (!isRecord(item)) return undefined; + const keys = Object.keys(item); + const v = keys.length === 1 ? item[keys[0] as string] : undefined; + if (typeof v !== "string") return undefined; + out.push(v); + } + return out; +} + +function textToReference(value: string, to: FieldDefinition, ctx: ConvertContext): Conversion { + if (to.type !== "reference") return fail("not a reference"); + const text = foldText(value); + if (text === "") return ok(undefined); + if (!ctx.entries) return fail("the entries to match could not be read"); + const groups = (ctx.entries.byText.get(text) ?? []).filter((g) => g !== ctx.group); + if (groups.length === 0) + return fail(`no entry of ${to.collections.join(", ")} has the title or slug ${quote(value)}`); + if (groups.length > 1) return fail(`${quote(value)} matches ${groups.length} entries`); + return asReference(groups, to); +} + +function textToRepeater(value: string, to: FieldDefinition): Conversion { + const key = onlySub(to, "text"); + if (key === undefined) return fail("text converts to a repeater with one text sub-field only"); + const lines = value + .split("\n") + .map((l) => l.trim()) + .filter((l) => l !== ""); + return ok(lines.length > 0 ? lines.map((line) => ({ [key]: line })) : undefined); +} + +/** The steps from `text` to the new types (`conversions.ts` routes them here). */ +export function textToStage4(value: unknown, to: FieldDefinition, ctx: ConvertContext): Conversion { + if (typeof value !== "string") return fail("not text"); + if (to.type === "reference") return textToReference(value, to, ctx); + if (to.type === "repeater") return textToRepeater(value, to); + return fail(`text does not convert to ${to.type}`); +} + +/** One step per new source type; the target decides inside it. */ +export const STAGE4_FROM: { readonly reference: Step; readonly repeater: Step } = { + reference(value, to, ctx) { + const ids = idsOf(value); + if (!ids) return fail("not a reference value"); + if (to.type === "reference") return asReference(ids, to); + if (to.type === "repeater") { + const key = onlySub(to, "reference"); + if (key === undefined) + return fail("references convert to a repeater with one reference sub-field only"); + return ok(ids.length > 0 ? ids.map((id) => ({ [key]: id })) : undefined); + } + if (to.type !== "text") return fail(`references do not convert to ${to.type}`); + if (ids.length === 0) return ok(undefined); + if (!ctx.entries) return fail("the entries' titles could not be read"); + const titles: string[] = []; + for (const id of ids) { + const title = ctx.entries.titles.get(id); + if (title === undefined) return fail(`${id} has no entry in this site`); + titles.push(title); + } + return ok(titles.join(", ")); + }, + repeater(value, to) { + const values = soleValues(value); + if (!values) return fail("only a repeater whose items hold one value each converts"); + if (to.type === "reference") return asReference([...new Set(values)], to); + if (to.type !== "text") return fail(`repeaters do not convert to ${to.type}`); + return ok(values.length > 0 ? values.join("\n") : undefined); + }, +}; diff --git a/packages/core-ts/src/fields/conversions.ts b/packages/core-ts/src/fields/conversions.ts index 56aa33ab..92e3234d 100644 --- a/packages/core-ts/src/fields/conversions.ts +++ b/packages/core-ts/src/fields/conversions.ts @@ -16,10 +16,13 @@ * | datetime → date | the day in the value's own zone | * | select ↔ select | single ↔ multiple (one value ↔ a list of one; an empty list is no value) | * + * RFC-0045 §7 adds the reference and repeater rows (`conversions-references.ts`). + * * Every result must also fit the new definition (`valueIssues`); anything else is unconvertible. */ import type { FieldDefinition } from "@cloudhal/core-contracts"; import type { ConversionEntries } from "../references/conversion-maps.ts"; +import { STAGE4_FROM, textToStage4 } from "./conversions-references.ts"; import { escapeAtLineStarts, unlink } from "./markdown-scan.ts"; import type { FieldType } from "./types.ts"; import { valueIssues } from "./value-checks.ts"; @@ -35,6 +38,8 @@ export interface ConvertContext { * a conversion runs (`conversion-maps.ts`). */ readonly entries?: ConversionEntries | undefined; + /** RFC-0045 §7: the converted entry's own translation group, which text → reference skips. */ + readonly group?: string | undefined; } /** `value: undefined` means "no value" (the key is removed). */ @@ -42,15 +47,20 @@ export type Conversion = { ok: true; value: unknown } | { ok: false; reason: str /** The table, from → the types it converts to (besides itself). */ export const FIELD_CONVERSIONS: Readonly>> = { - text: ["richText", "number", "select", "url"], + text: ["richText", "number", "select", "url", "reference", "repeater"], richText: ["text"], number: ["text"], select: ["text", "select"], url: ["text"], date: ["datetime"], datetime: ["date"], + reference: ["text", "reference", "repeater"], + repeater: ["text", "reference"], }; +/** Types whose same-type change converts values (single ↔ multiple). */ +const RESHAPED: ReadonlySet = new Set(["select", "reference"]); + /** True when the table converts `from` values to `to` (a type always converts to itself). */ export function canConvert(from: FieldType, to: FieldType): boolean { return from === to || FIELD_CONVERSIONS[from]?.includes(to) === true; @@ -192,7 +202,7 @@ const FROM: Partial> = { return fail(`"${value.slice(0, 40)}" is not a number`); return ok(Number(value)); } - return fail(`text does not convert to ${to.type}`); + return textToStage4(value, to, ctx); }, richText(value, to) { if (!isString(value) || to.type !== "text") return fail(`rich text converts to text only`); @@ -223,6 +233,7 @@ const FROM: Partial> = { return fail("a date and time converts to a date"); return ok(dayIn(v.at, v.timeZone)); }, + ...STAGE4_FROM, }; /** @@ -236,7 +247,7 @@ export function convertFieldValue( ctx: ConvertContext, ): Conversion { let out: Conversion; - if (from === to.type && to.type !== "select") out = ok(value); + if (from === to.type && !RESHAPED.has(to.type)) out = ok(value); else if (!canConvert(from, to.type)) return fail(`${from} values do not convert to ${to.type}`); else { const step = FROM[from]; diff --git a/packages/core-ts/src/fields/index.ts b/packages/core-ts/src/fields/index.ts index f43d14b9..410cd809 100644 --- a/packages/core-ts/src/fields/index.ts +++ b/packages/core-ts/src/fields/index.ts @@ -160,6 +160,7 @@ export { type ReferenceFieldDefinition, type ReferenceFieldType, type RepeaterFieldDefinition, + renamedFrom, type SiteFields, type SubFieldType, } from "./types.ts"; diff --git a/packages/core-ts/src/fields/migration-sub.ts b/packages/core-ts/src/fields/migration-sub.ts new file mode 100644 index 00000000..668f8e56 --- /dev/null +++ b/packages/core-ts/src/fields/migration-sub.ts @@ -0,0 +1,139 @@ +/** + * RFC-0045 §7 `sub`: a field operation on one sub-field of a repeater. The marker sits on the + * repeater (`migration: { id, sub, fromKey | fromType }`; `sub` alone: the sub-field is being + * removed), and every item goes through the same per-value rule a top-level field does: the + * sub-field is read as a field (`subAsField`) carrying the marker without `sub`, and RFC-0041 §6's + * `rewriteFields` runs on the item. An item left empty is dropped, and a list left empty is no + * value. Pure and Worker-safe. + */ +import type { FieldDefinition, FieldMigration } from "@cloudhal/core-contracts"; +import { canonicalJson } from "../hash.ts"; +import type { ConvertContext } from "./conversions.ts"; +import type { FieldRows } from "./text-reference.ts"; +import { isRecord, type RepeaterFieldDefinition } from "./types.ts"; + +/** One item's rewrite: what `rewriteFields` answers for the sub-field read as a field. */ +export interface ItemRewrite { + readonly values: Record | undefined; + readonly change: { readonly kind: string; readonly reason?: string | undefined } | undefined; +} + +export type RewriteItem = (input: { + def: FieldDefinition; + values: Readonly>; + convert: ConvertContext; + translation: boolean; + pristine: boolean; + onInvalid?: "clear" | undefined; + baseline?: { readonly value: unknown } | undefined; + rows?: FieldRows | undefined; +}) => ItemRewrite; + +/** A sub-field as a field definition: the repeater's `localized`, and `migration` when given. */ +export function subAsField( + repeater: RepeaterFieldDefinition, + key: string, + migration?: FieldMigration, +): FieldDefinition | undefined { + const sub = repeater.fields.find((s) => s.key === key); + if (!sub) return undefined; + return { + ...sub, + localized: repeater.localized, + ...(migration ? { migration } : {}), + } as unknown as FieldDefinition; +} + +/** The sub-field marker as a top-level one (`sub` dropped), or undefined without `sub`. */ +export function subMarker(def: FieldDefinition): FieldMigration | undefined { + const m = def.migration; + if (m?.sub === undefined) return undefined; + const { sub: _sub, ...rest } = m; + return rest; +} + +/** The stored items, as canonical JSON. */ +function storedItems(value: unknown): Set { + return new Set(Array.isArray(value) ? value.filter(isRecord).map((i) => canonicalJson(i)) : []); +} + +export interface ItemsResult { + /** The new list (`undefined`: no value), or `unchanged`. */ + readonly next: unknown[] | undefined | "unchanged"; + /** The first unconvertible item's reason (with `onInvalid` not `clear`). */ + readonly unconvertible?: string | undefined; + /** Every unconvertible sub-value, as stored (its item is kept as it is in `next`). */ + readonly failed?: readonly { readonly value: unknown; readonly stored: boolean }[] | undefined; + /** Some sub-value was dropped by `onInvalid: "clear"`. */ + readonly cleared?: string | undefined; +} + +/** + * The repeater's items under its `sub` marker. `rewrite` is RFC-0041's `rewriteFields`, passed + * in so this module and `migration.ts` do not import each other. + */ +export function rewriteItems( + def: RepeaterFieldDefinition, + value: unknown, + input: { + convert: ConvertContext; + pristine: boolean; + onInvalid?: "clear" | undefined; + /** A person's write: the repeater's stored value (an item found there unchanged is not written). */ + baseline?: { readonly value: unknown } | undefined; + /** The node's rows for `.` (every item's sub-value shares the path). */ + rows?: FieldRows | undefined; + rewrite: RewriteItem; + }, +): ItemsResult { + const m = def.migration; + const marker = subMarker(def); + if (m?.sub === undefined || !marker || !Array.isArray(value)) return { next: "unchanged" }; + // A removal's sub-field is still defined (phase C drops it); a rename's carries the new key. + const sub = subAsField(def, m.sub, marker); + if (!sub) return { next: "unchanged" }; + let changed = false; + let cleared: string | undefined; + let unconvertible: string | undefined; + const failed: { value: unknown; stored: boolean }[] = []; + const out: unknown[] = []; + const stored = input.baseline === undefined ? undefined : storedItems(input.baseline.value); + for (const item of value) { + if (!isRecord(item)) { + out.push(item); + continue; + } + // An item the write left as stored keeps its sub-value's baseline; any other is written. + const baseline = stored && { + value: stored.has(canonicalJson(item)) ? item[sub.key] : undefined, + }; + const r = input.rewrite({ + def: sub, + values: item, + convert: input.convert, + translation: false, + pristine: input.pristine, + onInvalid: input.onInvalid, + baseline, + rows: input.rows, + }); + // Item by item: a failing sub-value stays as stored, the others convert. + if (r.change?.kind === "unconvertible") { + unconvertible ??= r.change.reason ?? "unconvertible"; + // Stored: the write left this item as it was (no baseline: the operation's own pass). + failed.push({ value: item[sub.key], stored: !stored || stored.has(canonicalJson(item)) }); + out.push(item); + continue; + } + if (r.change?.kind === "cleared") cleared ??= r.change.reason; + if (!r.values) { + out.push(item); + continue; + } + changed = true; + if (Object.keys(r.values).length > 0) out.push(r.values); + } + const failures = unconvertible === undefined ? {} : { unconvertible, failed }; + if (!changed) return { next: "unchanged", ...failures }; + return { next: out.length > 0 ? out : undefined, cleared, ...failures }; +} diff --git a/packages/core-ts/src/fields/migration.ts b/packages/core-ts/src/fields/migration.ts index 3a85b27d..34b9c968 100644 --- a/packages/core-ts/src/fields/migration.ts +++ b/packages/core-ts/src/fields/migration.ts @@ -7,8 +7,10 @@ * reader agrees however many entries remain. Pure and Worker-safe. */ import type { FieldDefinition } from "@cloudhal/core-contracts"; -import { type ConvertContext, convertFieldValue } from "./conversions.ts"; -import { isRecord, type SiteFields, sameJson } from "./types.ts"; +import { type Conversion, type ConvertContext, convertFieldValue } from "./conversions.ts"; +import { rewriteItems, subAsField } from "./migration-sub.ts"; +import { type FieldRows, textReferenceOutcome } from "./text-reference.ts"; +import { isRecord, renamedFrom, type SiteFields, sameJson } from "./types.ts"; import { valueIssues } from "./value-checks.ts"; export type FieldMigrationKind = "rename" | "retype" | "localize" | "remove"; @@ -23,8 +25,12 @@ export function migrationKind(def: FieldDefinition): FieldMigrationKind | undefi return "remove"; } -/** True while an operation removes the field: readers treat it as gone. */ -export const isBeingRemoved = (def: FieldDefinition): boolean => migrationKind(def) === "remove"; +/** + * True while an operation removes the field: readers treat it as gone. A sub-field removal + * (RFC-0045 `sub`) removes only that sub-value from items; the field stays. + */ +export const isBeingRemoved = (def: FieldDefinition): boolean => + migrationKind(def) === "remove" && def.migration?.sub === undefined; /** The conversion context of a site (`UTC` and `en` when it names none). */ export function convertContext(site: SiteFields) { @@ -49,6 +55,21 @@ export interface RewriteInput { readonly pristine: boolean; /** `"clear"` drops values that cannot be converted; otherwise they stay as stored. */ readonly onInvalid?: "clear" | undefined; + /** + * RFC-0045 §7: a person's write only: the field's stored value, so a value the write changed + * counts as written under the new type (`text-reference.ts`). + */ + readonly baseline?: { readonly value: unknown } | undefined; + /** + * RFC-0045 §7: the node's `entry_refs` rows for the converting field path, which decide text ↔ + * reference values for the writers (`text-reference.ts`); absent for readers. + */ + readonly rows?: FieldRows | undefined; + /** + * A person's write: a repeater converts its items one by one, keeping a failing item as stored + * (phase B, the dry run and the revert treat any failing item as the entry's failure). + */ + readonly partial?: boolean | undefined; } /** How one entry's value changes under a marker. */ @@ -63,6 +84,17 @@ export interface RewriteResult { /** The new `fields`, or undefined when nothing changes. */ readonly values: Record | undefined; readonly change: ValueChange | undefined; + /** + * RFC-0045 §7: the values that did not convert, as stored (a field's value, or a repeater's + * failing sub-values), so a person's write can keep their rows as they are. + */ + readonly failed?: readonly FailedValue[] | undefined; +} + +/** A value that did not convert, and whether it is the stored one (a person did not write it). */ +export interface FailedValue { + readonly value: unknown; + readonly stored: boolean; } const NOTHING: RewriteResult = { values: undefined, change: undefined }; @@ -78,9 +110,31 @@ function retype(input: RewriteInput, values: Readonly>): const from = def.migration?.fromType; if (!from || !Object.hasOwn(values, def.key)) return NOTHING; const before = values[def.key]; - const fits = valueIssues(def, before, "v").length === 0; + // RFC-0045 §7: for text ↔ reference, "fits" says nothing; one rule decides (`text-reference.ts`). + const written = input.baseline !== undefined && !sameJson(before, input.baseline.value); + const decided = textReferenceOutcome({ + from, + def, + value: before, + convert, + rows: input.rows, + written, + }); + if (decided?.kind === "keep") return NOTHING; + if (decided?.kind === "check") { + // Kept, but written again (a list for a multiple reference) so the guard and rows run. + const value = "multiple" in def && def.multiple && !Array.isArray(before) ? [before] : before; + return { + values: { ...values, [def.key]: value }, + change: { kind: "converted", before, after: value }, + }; + } + const fits = decided === undefined && valueIssues(def, before, "v").length === 0; if (fits && (!pristine || from === def.type)) return NOTHING; - const out = convertFieldValue(from, def, before, convert); + const out: Conversion = + decided?.kind === "fail" + ? { ok: false, reason: decided.reason } + : convertFieldValue(from, def, before, convert); if (out.ok) { if (sameJson(out.value, before)) return NOTHING; const next = @@ -95,13 +149,61 @@ function retype(input: RewriteInput, values: Readonly>): return { values: undefined, change: { kind: "unconvertible", before, after: undefined, reason: out.reason }, + failed: [{ value: before, stored: !written }], }; } +/** RFC-0045 §7 `sub`: the repeater's items rewritten, each through the sub-field's own rule. */ +function rewriteSub(input: RewriteInput, values: Readonly>): RewriteResult { + const { def } = input; + if (def.type !== "repeater" || !Object.hasOwn(values, def.key)) return NOTHING; + const before = values[def.key]; + const r = rewriteItems(def, before, { + convert: input.convert, + pristine: input.pristine, + onInvalid: input.onInvalid, + baseline: input.baseline, + rows: input.rows, + rewrite: rewriteFields, + }); + if (r.unconvertible !== undefined) { + const change = { + kind: "unconvertible" as const, + before, + after: undefined, + reason: r.unconvertible, + }; + // A person's write keeps the items that converted, when every failing one still fits the new + // sub-field (an id-shaped value: else the write would not check); the passes fail the entry. + const sub = def.migration?.sub === undefined ? undefined : subAsField(def, def.migration.sub); + const fit = (r.failed ?? []).every((f) => !sub || valueIssues(sub, f.value, "v").length === 0); + const kept = + input.partial && fit && r.next !== "unchanged" + ? r.next === undefined + ? without(values, def.key) + : { ...values, [def.key]: r.next } + : undefined; + return { values: kept, change, failed: r.failed }; + } + if (r.next === "unchanged") return NOTHING; + const next = r.next === undefined ? without(values, def.key) : { ...values, [def.key]: r.next }; + const kind = r.cleared !== undefined ? "cleared" : SUB_KIND[migrationKind(def) ?? "remove"]; + const reason = r.cleared === undefined ? {} : { reason: r.cleared }; + return { values: next, change: { kind, before, after: r.next, ...reason } }; +} + +const SUB_KIND = { + rename: "moved", + retype: "converted", + localize: "converted", + remove: "removed", +} as const satisfies Record; + /** What an entry's `fields` become under `input.def`'s marker. */ export function rewriteFields(input: RewriteInput): RewriteResult { const { def } = input; const values = isRecord(input.values) ? input.values : {}; + if (def.migration?.sub !== undefined) return rewriteSub(input, values); switch (migrationKind(def)) { case "rename": { const from = def.migration?.fromKey as string; @@ -132,6 +234,13 @@ export function rewriteFields(input: RewriteInput): RewriteResult { } } +/** The value `def` reads from stored `fields`: its key, else a running rename's old one. */ +function storedValue(def: FieldDefinition, fields: Readonly>): unknown { + if (Object.hasOwn(fields, def.key)) return fields[def.key]; + const from = renamedFrom(def); + return from !== undefined && Object.hasOwn(fields, from) ? fields[from] : undefined; +} + /** * An entry's `fields` normalized through every marker of its collection (the store runs it on a * person's write during an operation, so concurrent edits converge). Unconvertible values stay. @@ -139,13 +248,26 @@ export function rewriteFields(input: RewriteInput): RewriteResult { export function normalizeThroughMarkers( defs: readonly FieldDefinition[], values: Readonly> | undefined, - context: { convert: ConvertContext; translation: boolean }, + context: { + convert: ConvertContext; + translation: boolean; + /** The stored entry's `fields`, for a person's write (RFC-0045 §7: what the write changed). */ + stored?: Readonly> | undefined; + /** RFC-0045 §7: the node's rows for a running text ↔ reference conversion's path. */ + rows?: FieldRows | undefined; + /** Called for a definition with values that do not convert (kept as stored), with them. */ + onUnconvertible?: ((def: FieldDefinition, failed: readonly FailedValue[]) => void) | undefined; + }, ): Record | undefined { let current = values; let changed = false; + const { stored, onUnconvertible, ...rest } = context; for (const def of defs) { if (!def.migration) continue; - const r = rewriteFields({ def, values: current, ...context, pristine: false }); + const baseline = stored === undefined ? undefined : { value: storedValue(def, stored) }; + const partial = stored !== undefined; + const r = rewriteFields({ def, values: current, ...rest, pristine: false, baseline, partial }); + if (r.change?.kind === "unconvertible") onUnconvertible?.(def, r.failed ?? []); if (r.values) { current = r.values; changed = true; diff --git a/packages/core-ts/src/fields/resolve.ts b/packages/core-ts/src/fields/resolve.ts index 8ea33ee1..9e3ced84 100644 --- a/packages/core-ts/src/fields/resolve.ts +++ b/packages/core-ts/src/fields/resolve.ts @@ -15,12 +15,15 @@ */ import type { ContentEntry, FieldDefinition } from "@cloudhal/core-contracts"; import { type ConvertContext, convertFieldValue } from "./conversions.ts"; -import { convertContext, isBeingRemoved } from "./migration.ts"; +import { convertContext, isBeingRemoved, rewriteFields } from "./migration.ts"; +import { rewriteItems } from "./migration-sub.ts"; +import { textReferenceOutcome } from "./text-reference.ts"; import { type FieldIssue, fieldDefinitions, isKnownFieldType, isRecord, + renamedFrom, type SiteFields, } from "./types.ts"; import { valueIssues } from "./value-checks.ts"; @@ -60,7 +63,7 @@ function readsOriginal(entry: FieldsEntry, original: FieldsOriginal | undefined) function read(values: unknown, def: FieldDefinition): unknown { if (!isRecord(values)) return undefined; if (Object.hasOwn(values, def.key)) return values[def.key]; - const from = def.migration?.fromKey; + const from = renamedFrom(def); return from && Object.hasOwn(values, from) ? values[from] : undefined; } @@ -92,23 +95,43 @@ export function resolveFields( convert?: Pick, ): ResolvedFields { const out: ResolvedFields = {}; + // RFC-0045 §7: text → reference never picks the entry's own group. + const context = { + ...convertContext(site), + group: entry.translation?.group ?? entry.id, + ...convert, + }; for (const def of fieldDefinitions(site, entry.collection)) { if (isBeingRemoved(def) || !isKnownFieldType(def.type)) continue; - const { value, fromOriginal } = pick(def, entry, original); + const picked = pick(def, entry, original); + const { fromOriginal } = picked; + const value = throughSub(def, picked.value, context); if (value === undefined) { out[def.key] = { def, value: undefined, fromOriginal: false }; continue; } + // A sub-field marker converted the items already (`throughSub`). + const converting = def.migration?.sub === undefined ? def.migration?.fromType : undefined; + // RFC-0045 §7: text ↔ reference values are decided by the one shared rule. + const decided = converting + ? textReferenceOutcome({ + from: converting, + def, + value, + convert: context, + written: false, + }) + : undefined; const issues = valueIssues(def, value, `fields/${def.key}`); const first = issues[0]; - if (!first) { + if (decided?.kind === "keep" || decided?.kind === "check" || (!decided && !first)) { out[def.key] = { def, value, fromOriginal }; continue; } - const converting = def.migration?.fromType; - const converted = converting - ? convertFieldValue(converting, def, value, { ...convertContext(site), ...convert }) - : undefined; + const converted = + converting && decided?.kind !== "fail" + ? convertFieldValue(converting, def, value, context) + : undefined; if (converted?.ok) { out[def.key] = { def, @@ -122,17 +145,33 @@ export function resolveFields( value: undefined, fromOriginal, issue: { - path: first.path, + path: first?.path ?? `fields/${def.key}`, code: "field_value_unreadable", message: converting ? `a ${converting} value that does not convert to ${def.type}` - : first.message, + : (first?.message ?? "unreadable"), }, }; } return out; } +/** + * RFC-0045 §7 `sub`: a repeater's items read through its sub-field marker. A sub-value the table + * cannot convert reads as none (an item left empty is dropped), as a top-level value does. + */ +function throughSub(def: FieldDefinition, value: unknown, convert: ConvertContext): unknown { + if (def.type !== "repeater" || def.migration?.sub === undefined || value === undefined) + return value; + const r = rewriteItems(def, value, { + convert, + pristine: false, + onInvalid: "clear", + rewrite: rewriteFields, + }); + return r.next === "unchanged" ? value : r.next; +} + /** "Has a value" for `requiredToPublish`: not absent, not an empty string, not an empty list. */ export function hasFieldValue(value: unknown): boolean { if (value === undefined || value === null) return false; diff --git a/packages/core-ts/src/fields/text-reference.ts b/packages/core-ts/src/fields/text-reference.ts new file mode 100644 index 00000000..21e92c04 --- /dev/null +++ b/packages/core-ts/src/fields/text-reference.ts @@ -0,0 +1,119 @@ +/** + * RFC-0045 §7: the one rule for "is this value already in the new type?" while `change_field_type` + * runs text ↔ reference (single or multiple, top-level or in a `sub` item). A group id is valid + * text and id-shaped text a valid single reference, so neither "fits" nor the entry's hash can + * say. The writers decide by the facts phase C reads, the node's `entry_refs` rows for the field + * path (`FieldRows`): the dry run, phase B and the revert load them once per pass, a person's write + * reads its node's. Readers without rows (`resolveFields`, the renderer) use a display rule. + * + * - **Text → reference**, for an id-shaped value (a string, or every element of a multiple's + * list). The entry's own group fails first, in every caller. In the target shape with a row for + * every id: kept. Written under the new type by a person's write (it differs from the stored + * value) and fitting: kept, and the writer's guard checks it. Otherwise it is legacy text that + * happens to be an id: `check` writes it as a reference (a bare string wrapped for a multiple) + * under the fresh guard, with its rows, when every id names a usable group; else it fails, is + * reported, and `clear` drops it. Any other text goes through the table (`byText`). Display: an + * id-shaped value naming a member group shows as a reference. + * - **Reference → text**, per value. A string with its own row (`rows.has`) is still a reference: + * a group in `titles` converts to its title, any other fails (reported; `clear` removes it and + * its rows). A string with no row of its own is text already, an id-shaped title included. A + * non-string (a list, `[]` too) is never text yet and goes through the table (joined titles, no + * value, or a failure). Display: a group in `titles` shows its title, any other string as it is. + * - **No maps** (`checkRequiredToPublish`, prechecks): kept as stored. + * + * Other conversion rows keep RFC-0041's pristine rule. Pure. + */ +import type { FieldDefinition } from "@cloudhal/core-contracts"; +import type { ConvertContext } from "./conversions.ts"; +import { GROUP_ID_PATTERN } from "./props.ts"; +import type { FieldType } from "./types.ts"; +import { valueIssues } from "./value-checks.ts"; + +const GROUP_ID = new RegExp(GROUP_ID_PATTERN); + +/** What a value under a text ↔ reference marker is. */ +export type TextReferenceOutcome = + /** Already in the new type: left as stored. */ + | { readonly kind: "keep" } + /** Legacy id-shaped text naming a usable group: written as a reference, guard and rows run. */ + | { readonly kind: "check" } + /** Through the conversion table. */ + | { readonly kind: "table" } + | { readonly kind: "fail"; readonly reason: string }; + +/** The node's `entry_refs` rows for the converting field path (`venue`, `speakers.host`). */ +export interface FieldRows { + /** A row names this group. */ + has(target: string): boolean; + /** Any row is left. */ + readonly any: boolean; +} + +export interface TextReferenceInput { + readonly from: FieldType; + readonly def: FieldDefinition; + readonly value: unknown; + readonly convert: Pick; + /** + * The rows phase C reads, for the writers that decide (the dry run, phase B, the revert, a + * person's write). Absent: a reader's display rule. + */ + readonly rows?: FieldRows | undefined; + /** A person's write changed the value (its stored value differs). */ + readonly written: boolean; +} + +/** True for the rows this module decides: text → reference and reference → text. */ +export const isTextReference = (from: string | undefined, to: string): boolean => + (from === "text" && to === "reference") || (from === "reference" && to === "text"); + +const OWN: TextReferenceOutcome = { + kind: "fail", + reason: "an entry cannot reference its own translation group", +}; + +/** Text → reference over the ids of a value (a string, or a multiple's list). */ +function toReference(input: TextReferenceInput, ids: readonly string[]): TextReferenceOutcome { + const { def, convert, rows } = input; + if (ids.some((id) => id === convert.group)) return OWN; + const maps = convert.entries; + if (def.type !== "reference" || !maps) return { kind: "keep" }; + // In the target shape and every id with its row: what phase B or an accepted write left. + const shaped = Array.isArray(input.value) === (def.multiple === true); + if (!rows) + return ids.every((id) => maps.titles.has(id)) && shaped ? { kind: "keep" } : { kind: "table" }; + if (shaped && ids.every((id) => rows.has(id))) return { kind: "keep" }; + if (input.written && shaped && valueIssues(def, input.value, "v").length === 0) + return { kind: "keep" }; + const gone = ids.find((id) => !maps.usable.has(id)); + if (gone === undefined) return { kind: "check" }; + return { kind: "fail", reason: `${gone} names no usable entry of ${def.collections.join(", ")}` }; +} + +/** + * The decision for one value, or undefined when the row is not text ↔ reference. Phase C's + * pending check reads the same rows (`plan-sub.ts`), so the two cannot disagree. + */ +export function textReferenceOutcome(input: TextReferenceInput): TextReferenceOutcome | undefined { + const { from, def, value, convert, rows } = input; + if (!isTextReference(from, def.type)) return undefined; + const maps = convert.entries; + if (def.type === "reference") { + const ids = Array.isArray(value) ? value : [value]; + const idShaped = ids.length > 0 && ids.every((v) => typeof v === "string" && GROUP_ID.test(v)); + if (idShaped) return toReference(input, ids as string[]); + if (!maps) return { kind: "keep" }; + if (input.written && valueIssues(def, value, "v").length === 0) return { kind: "keep" }; + return { kind: "table" }; + } + // Reference → text, per value: a string is still a reference only with its own row; a + // non-string (a list, `[]` included) is never text yet, so it goes through the table. + if (!maps) return { kind: "keep" }; + if (input.written && valueIssues(def, value, "v").length === 0) return { kind: "keep" }; + if (typeof value !== "string") return { kind: "table" }; + const still = rows ? rows.has(value) : maps.titles.has(value); + if (!still) return { kind: "keep" }; + return maps.titles.has(value) + ? { kind: "table" } + : { kind: "fail", reason: `${value} has no entry in this site` }; +} diff --git a/packages/core-ts/src/fields/types.ts b/packages/core-ts/src/fields/types.ts index 0fbabb29..8f9dca8b 100644 --- a/packages/core-ts/src/fields/types.ts +++ b/packages/core-ts/src/fields/types.ts @@ -172,6 +172,13 @@ export function fieldDefinitions(site: SiteFields, collection: string): readonly return site.collections?.[collection]?.fields ?? []; } +/** + * The top-level key a running `rename_field` still reads values from. A sub-field rename (RFC-0045 + * `migration.sub`) keeps the repeater's key, so its `fromKey` names a sub-field: none here. + */ +export const renamedFrom = (def: Pick): string | undefined => + def.migration?.sub === undefined ? def.migration?.fromKey : undefined; + export const isRecord = (v: unknown): v is Record => typeof v === "object" && v !== null && !Array.isArray(v); diff --git a/packages/core-ts/src/list-values/rows.ts b/packages/core-ts/src/list-values/rows.ts index d23d680f..4920b2fc 100644 --- a/packages/core-ts/src/list-values/rows.ts +++ b/packages/core-ts/src/list-values/rows.ts @@ -9,7 +9,7 @@ import type { ContentEntry, FieldDefinition } from "@cloudhal/core-contracts"; import { FIELD_QUERY_LIMITS, sortKey } from "../fields/query.ts"; import { type FieldsOriginal, hasFieldValue, resolveFields } from "../fields/resolve.ts"; -import type { SiteFields } from "../fields/types.ts"; +import { renamedFrom, type SiteFields } from "../fields/types.ts"; /** One row: `has` is 0/1 on a field row and 2 on an option row of a multiple select. */ export interface ListValueRow { @@ -152,7 +152,7 @@ export function originalIdOf(entry: ListEntry): string | undefined { function ownKey(fields: ListEntry["fields"], def: FieldDefinition): boolean { if (!fields) return false; - const from = def.migration?.fromKey; + const from = renamedFrom(def); return Object.hasOwn(fields, def.key) || (from !== undefined && Object.hasOwn(fields, from)); } diff --git a/packages/core-ts/src/media/field-refs.ts b/packages/core-ts/src/media/field-refs.ts index fed713cc..43175f4b 100644 --- a/packages/core-ts/src/media/field-refs.ts +++ b/packages/core-ts/src/media/field-refs.ts @@ -8,7 +8,7 @@ */ import type { Asset, FieldDefinition } from "@cloudhal/core-contracts"; import { ASSET_ID_PATTERN } from "../fields/props.ts"; -import { type FieldIssue, isRecord } from "../fields/types.ts"; +import { type FieldIssue, isRecord, renamedFrom } from "../fields/types.ts"; import { type AssetRefUse, markdownAssetIds } from "./refs.ts"; const ASSET_ID = new RegExp(ASSET_ID_PATTERN); @@ -54,10 +54,17 @@ export function fieldAssetRefs( const value = Object.hasOwn(values, def.key) ? values[def.key] : undefined; for (const id of valueAssets(def, value)) add(id, `fields.${def.key}`, `fields/${def.key}`); if (def.type !== "repeater" || !Array.isArray(value)) continue; + // RFC-0045 §7: a running sub-field rename's items still hold the old key. + const m = def.migration; + const held = (item: Record, key: string) => { + if (Object.hasOwn(item, key)) return item[key]; + const was = m?.sub === key ? m.fromKey : undefined; + return was !== undefined && Object.hasOwn(item, was) ? item[was] : undefined; + }; value.forEach((item, i) => { if (!isRecord(item)) return; for (const sub of def.fields) - for (const id of valueAssets(sub, Object.hasOwn(item, sub.key) ? item[sub.key] : undefined)) + for (const id of valueAssets(sub, held(item, sub.key))) add(id, `fields.${def.key}.${sub.key}`, `fields/${def.key}/${i}/${sub.key}`); }); } @@ -76,10 +83,10 @@ export function migratedFieldValues( defs: readonly FieldDefinition[], values: Readonly> | undefined, ): Readonly> | undefined { - if (!values || !defs.some((d) => d.migration?.fromKey)) return values; + if (!values || !defs.some((d) => renamedFrom(d))) return values; const out: Record = { ...values }; for (const def of defs) { - const from = def.migration?.fromKey; + const from = renamedFrom(def); if (from && !Object.hasOwn(values, def.key) && Object.hasOwn(values, from)) out[def.key] = values[from]; } diff --git a/packages/core-ts/src/references/conversion-load.ts b/packages/core-ts/src/references/conversion-load.ts new file mode 100644 index 00000000..fdffaf6f --- /dev/null +++ b/packages/core-ts/src/references/conversion-load.ts @@ -0,0 +1,33 @@ +/** + * RFC-0045 §7 "Where the graph reads happen", on the store's side: the prefetched maps of a + * running text ↔ reference conversion, built from one read of the working copy's target + * collections. Each field-operation pass (the dry run, each phase-B pass, each revert pass) builds + * them once before it plans, and `EntryEngine.write` builds them before it normalizes a write, + * only while such a marker is active. The live source reads the same entries + * (`astro-renderer/src/reference-conversions.ts`), so every converter agrees. + */ +import type { ContentEntry, FieldDefinition, Site } from "@cloudhal/core-contracts"; +import { + type ConversionEntries, + conversionCollections, + conversionEntries, +} from "./conversion-maps.ts"; + +/** What the maps read of the store. */ +export interface ConversionSource { + listContentEntries(siteId: string, collection?: string): Promise; +} + +/** The maps `defs`' running conversions read, or `undefined` when none needs any (no read). */ +export async function loadConversionEntries( + store: ConversionSource, + site: Pick, + defs: readonly FieldDefinition[], +): Promise { + const needs = conversionCollections(defs); + if (!needs) return undefined; + const entries = needs.has("*") + ? await store.listContentEntries(site.id) + : (await Promise.all([...needs].map((c) => store.listContentEntries(site.id, c)))).flat(); + return conversionEntries(entries, site.defaultLocale, needs); +} diff --git a/packages/core-ts/src/references/conversion-maps.ts b/packages/core-ts/src/references/conversion-maps.ts index 196ab8ce..de775982 100644 --- a/packages/core-ts/src/references/conversion-maps.ts +++ b/packages/core-ts/src/references/conversion-maps.ts @@ -6,11 +6,12 @@ * * - `byText`: case-folded slug or default-locale title → the groups with such a member among the * target collections (text → reference picks the one group; none or several is unconvertible); - * - `titles`: group → its default-locale member's title (reference → text). + * - `titles`: group → a member's title, chosen without regard to order (reference → text); + * - `usable`: groups with a member that is not archived. * - * Archived entries count nothing: a converted value is a newly chosen target, which the store's - * target guard requires to be usable (§2.1). Pure and Worker-safe; the conversion rows that read - * the maps land with rollout step 4. + * Archived entries match no text: a converted value is a newly chosen target, which the store's + * target guard requires to be usable (§2.1). They still lend their title: reference → text finds + * only a group with no member in the site unconvertible. Pure and Worker-safe. */ import type { ContentEntry, FieldDefinition } from "@cloudhal/core-contracts"; import { memberGroup } from "./members.ts"; @@ -19,8 +20,13 @@ import { memberGroup } from "./members.ts"; export interface ConversionEntries { /** Case-folded slug or default-locale title → group ids, in first-seen order. */ readonly byText: ReadonlyMap; - /** Group id → its default-locale member's title. */ + /** + * Group id → a member's title, any status: the default-locale member's, else the group's + * original's, else the member with the lowest locale (then id). Every group with a member. + */ readonly titles: ReadonlyMap; + /** The groups with a member that is not archived (a value text → reference may keep). */ + readonly usable: ReadonlySet; } /** The one case fold every converter compares with. */ @@ -65,7 +71,8 @@ export function conversionEntries( collections?: ReadonlySet, ): ConversionEntries { const byText = new Map(); - const titles = new Map(); + const titled = new Map(); + const usable = new Set(); const all = collections === undefined || collections.has("*"); const add = (text: string, group: string) => { const key = foldText(text); @@ -75,13 +82,18 @@ export function conversionEntries( else if (!list.includes(group)) list.push(group); }; for (const e of entries) { - if (e.status === "archived") continue; if (!all && !collections.has(e.collection)) continue; const group = memberGroup(e); const isDefault = e.locale === defaultLocale; - if (isDefault && !titles.has(group)) titles.set(group, e.title); + // Order-independent, so every source picks the same title for a group. + const rank = `${isDefault ? 0 : e.id === group ? 1 : 2}\u0000${e.locale}\u0000${e.id}`; + const held = titled.get(group); + if (!held || rank < held.rank) titled.set(group, { rank, title: e.title }); + if (e.status === "archived") continue; + usable.add(group); add(e.slug, group); if (isDefault) add(e.title, group); } - return { byText, titles }; + const titles = new Map([...titled].map(([g, t]) => [g, t.title] as const)); + return { byText, titles, usable }; } diff --git a/packages/core-ts/src/references/conversion-rows.ts b/packages/core-ts/src/references/conversion-rows.ts new file mode 100644 index 00000000..98765927 --- /dev/null +++ b/packages/core-ts/src/references/conversion-rows.ts @@ -0,0 +1,63 @@ +/** + * RFC-0045 §7: the `entry_refs` rows a text ↔ reference decision reads (`text-reference.ts`), the + * same facts phase C's pending check reads. A pass of the dry run, phase B or the revert loads the + * collection's rows for the converting field path once; a person's write reads its own node's. + * Every read filters `site_id`. + */ +import type { FieldDefinition } from "@cloudhal/core-contracts"; +import type { SqlDriver } from "@cloudhal/storage"; +import { type FieldRows, isTextReference } from "../fields/text-reference.ts"; + +/** The rows' field path of a running text ↔ reference conversion (`venue`, `speakers.host`). */ +export function convertingPath(def: FieldDefinition): string | undefined { + const m = def.migration; + if (m?.fromType === undefined) return undefined; + if (m.sub === undefined) return isTextReference(m.fromType, def.type) ? def.key : undefined; + const sub = def.type === "repeater" ? def.fields.find((s) => s.key === m.sub) : undefined; + return sub && isTextReference(m.fromType, sub.type) ? `${def.key}.${m.sub}` : undefined; +} + +const NONE: FieldRows = { has: () => false, any: false }; + +function rowsOf(targets: readonly string[] | undefined): FieldRows { + if (!targets?.length) return NONE; + const set = new Set(targets); + return { has: (t) => set.has(t), any: true }; +} + +/** Per node, its rows for `def`'s converting path; undefined when `def` converts no such row. */ +export async function loadFieldRows( + sql: SqlDriver, + at: { siteId: string; collection: string; def: FieldDefinition }, +): Promise<((nodeId: string) => FieldRows) | undefined> { + const path = convertingPath(at.def); + if (path === undefined) return undefined; + const rows = await sql.all<{ s: string; t: string }>( + "SELECT source_id AS s, target_id AS t FROM entry_refs WHERE site_id = ? AND source_collection = ? AND field = ?", + [at.siteId, at.collection, path], + ); + const by = new Map(); + for (const r of rows) by.set(r.s, [...(by.get(r.s) ?? []), r.t]); + return (nodeId) => rowsOf(by.get(nodeId)); +} + +/** One node's rows for the converting path of `defs` (a person's write), or undefined. */ +export async function loadNodeRows( + sql: SqlDriver, + at: { + siteId: string; + nodeId: string | undefined; + /** The draft's collection: rows the node held in another one are not held here. */ + collection: string; + defs: readonly FieldDefinition[]; + }, +): Promise { + const path = at.defs.map(convertingPath).find((p) => p !== undefined); + if (path === undefined) return undefined; + if (at.nodeId === undefined) return NONE; + const rows = await sql.all<{ t: string }>( + "SELECT target_id AS t FROM entry_refs WHERE site_id = ? AND source_id = ? AND source_collection = ? AND field = ?", + [at.siteId, at.nodeId, at.collection, path], + ); + return rowsOf(rows.map((r) => r.t)); +} diff --git a/packages/core-ts/src/references/editions.ts b/packages/core-ts/src/references/editions.ts index 5df9027a..b1d37c39 100644 --- a/packages/core-ts/src/references/editions.ts +++ b/packages/core-ts/src/references/editions.ts @@ -17,6 +17,8 @@ */ import type { ContentEntry, FieldCondition, FieldDefinition } from "@cloudhal/core-contracts"; import type { SqlDriver } from "@cloudhal/storage"; +import { GROUP_ID_PATTERN } from "../fields/props.ts"; +import type { FieldRows } from "../fields/text-reference.ts"; import { FIELD_LIMITS, fieldDefinitions, @@ -24,6 +26,8 @@ import { type SiteFields, sameJson, } from "../fields/types.ts"; +import { conversionCollections, conversionEntries, foldText } from "./conversion-maps.ts"; +import { convertingPath, loadNodeRows } from "./conversion-rows.ts"; import { entryRefRows, holdsReferences, storedKey } from "./rows.ts"; import { readMembers, targetProblem } from "./targets.ts"; import { referenceUses } from "./values.ts"; @@ -42,6 +46,8 @@ export interface EditionDrop { interface Target { collections: readonly string[]; where?: readonly FieldCondition[] | undefined; + /** A repeater's sub-field the value sits in (its held ids are read per path). */ + sub?: string | undefined; } /** Whether the edition takes `groupId` for a target, and why not. */ @@ -52,6 +58,24 @@ interface Carry { dropped: EditionDrop[]; /** References still allowed in this entry (the per-entry cap). */ budget: { left: number }; + /** + * RFC-0045 §7: under a running text → reference conversion, text the edition site's maps + * convert to exactly one group (its own write converts it); any other text is dropped. + */ + converts: (text: string) => boolean; +} + +const GROUP_ID = new RegExp(GROUP_ID_PATTERN); + +/** A string that is no group id under a running text → reference marker (top-level or `sub`). */ +function unconvertedText( + m: FieldDefinition["migration"], + key: string, + value: unknown, + sub = false, +): boolean { + const converting = m?.fromType === "text" && (sub ? m.sub === key : m.sub === undefined); + return converting && typeof value === "string" && !GROUP_ID.test(value); } /** One reference id: kept, or dropped with its reason. */ @@ -75,6 +99,10 @@ function carryReference( at: { path: string; carry: Carry }, ): unknown { const { path, carry } = at; + // Text a running text → reference conversion has not converted yet: the edition's own write + // converts it through its markers, so it is carried as it is. + if (unconvertedText(def.migration, def.key, value) && at.carry.converts(value as string)) + return value; const ids = Array.isArray(value) ? value : [value]; const pathOf = (i: number) => (Array.isArray(value) ? `${path}/${i}` : path); const target = { collections: def.collections, where: def.where }; @@ -115,11 +143,19 @@ function carryRepeater( return; } const next: Record = { ...item }; + // A running sub-field rename's items still hold the old key (as `referenceUses` reads them). + const m = def.migration; for (const sub of def.fields) { - if (sub.type !== "reference" || next[sub.key] === undefined) continue; - const target = { collections: sub.collections }; - if (!keep(next[sub.key], { path: `${path}/${i}/${sub.key}`, target, carry })) - delete next[sub.key]; + if (sub.type !== "reference") continue; + const key = + next[sub.key] === undefined && m?.sub === sub.key && m.fromKey !== undefined + ? m.fromKey + : sub.key; + if (next[key] === undefined) continue; + if (unconvertedText(m, sub.key, next[key], true) && carry.converts(next[key] as string)) + continue; + const target = { collections: sub.collections, sub: sub.key }; + if (!keep(next[key], { path: `${path}/${i}/${key}`, target, carry })) delete next[key]; } if (Object.keys(next).length > 0) items.push(next); else carry.dropped.push({ path: `${path}/${i}`, reason: "empty" }); @@ -140,9 +176,59 @@ async function usableOn( }; } -/** The group ids a stored value holds for `def`: the freshness test only (not a count). */ -const heldBy = (def: FieldDefinition, value: unknown): Set => - new Set(referenceUses(def, value, "").map((u) => u.groupId)); +/** + * Whether text converts to exactly one group on the edition site, under `defs`' running text → + * reference conversion: the site's maps over its target collections (one read, only then). + */ +async function textConverts( + sql: SqlDriver, + toSite: SiteFields & { id: string }, + at: { defs: readonly FieldDefinition[]; group: string | undefined }, +): Promise<(text: string) => boolean> { + const { defs, group } = at; + const needs = conversionCollections(defs); + if (!needs) return () => false; + const rows = await sql.all<{ body: string }>( + `SELECT body FROM nodes WHERE site_id = ? AND kind = 'contentEntry' + AND json_extract(body, '$.collection') IN (SELECT value FROM json_each(?))`, + [toSite.id, JSON.stringify([...needs])], + ); + const maps = conversionEntries( + rows.map((r) => JSON.parse(r.body) as ContentEntry), + toSite.defaultLocale ?? "en", + needs, + ); + // The converter's own rule: one group left once the entry's own is set aside. + return (text) => (maps.byText.get(foldText(text)) ?? []).filter((g) => g !== group).length === 1; +} + +/** + * Whether a stored value already holds a group for `def` (the freshness test only, not a count). + * While a text → reference conversion runs on `def`, only an id with its row is held on the + * converting path, and a repeater's other sub-fields hold their own stored ids only (RFC-0045 §7). + * Otherwise every stored id of the field is held, as before step 4. + */ +function heldByRows( + def: FieldDefinition, + value: unknown, + rows: FieldRows | undefined, +): (group: string, sub?: string) => boolean { + const all = new Set(referenceUses(def, value, "").map((u) => u.groupId)); + const fromText = convertingPath(def) !== undefined && def.migration?.fromType === "text"; + // Without such a marker: every stored id of the field, as before RFC-0045 step 4. + if (!fromText) return (g) => all.has(g); + const converting = def.migration?.sub; + // Per path: a sub-field's stored ids hold only for that sub-field. + const ids = new Map>(); + for (const u of referenceUses(def, value, "")) { + const at = def.type === "repeater" ? (u.path.split("/")[2] ?? "") : ""; + ids.set(at, (ids.get(at) ?? new Set()).add(u.groupId)); + } + return (g, sub = "") => + converting === undefined || converting === sub + ? rows?.has(g) === true + : ids.get(sub)?.has(g) === true; +} /** * `entry`'s fields as written on site `toSite` (its own definitions), with what was dropped. @@ -152,7 +238,8 @@ export async function editionReferences( sql: SqlDriver, toSite: SiteFields & { id: string }, input: { - entry: Pick; + entry: Pick & + Partial>; /** The edition's stored version of the entry (a re-save): its values are not fresh. */ stored?: Pick | undefined; }, @@ -162,6 +249,15 @@ export async function editionReferences( const ids = [...new Set(entryRefRows(defs, entry.fields).map((r) => r.t))]; if (ids.length === 0) return { fields: entry.fields, dropped: [] }; const usable = await usableOn(sql, toSite, ids); + const group = entry.translation?.group ?? entry.id; + const converts = await textConverts(sql, toSite, { defs, group }); + // RFC-0045 §7: on a path converting from text, only an id with its row is already held. + const rows = await loadNodeRows(sql, { + siteId: toSite.id, + nodeId: input.stored ? entry.id : undefined, + collection: entry.collection, + defs, + }); const before: Record = input.stored?.collection === entry.collection ? { ...(input.stored.fields ?? {}) } : {}; const budget = { left: FIELD_LIMITS.referencesPerEntry }; @@ -179,11 +275,12 @@ export async function editionReferences( } // Pass 2: the changed keys, carried under what the cap leaves. for (const { def, key } of changed) { - const held = Object.hasOwn(before, key) ? heldBy(def, before[key]) : new Set(); + const held = Object.hasOwn(before, key) ? heldByRows(def, before[key], rows) : () => false; const carry: Carry = { - usable: (g, target) => usable(g, target, !held.has(g)), + usable: (g, target) => usable(g, target, !held(g, target.sub)), dropped, budget, + converts, }; const at = { path: `fields/${key}`, carry }; const value = diff --git a/packages/core-ts/src/references/issues.ts b/packages/core-ts/src/references/issues.ts index fbb75d21..f18e1216 100644 --- a/packages/core-ts/src/references/issues.ts +++ b/packages/core-ts/src/references/issues.ts @@ -37,10 +37,18 @@ export interface ReferenceSnapshot { entries: readonly ContentEntry[]; } -/** True when some collection of the site has a reference or repeater field. */ +/** + * True when some collection of the site has a reference or repeater field, or a field still + * converting from one (RFC-0045 §7: until phase B rewrites them, its entries keep rows, which + * every write and guard must still see). + */ export function hasReferenceFields(site: Pick): boolean { return Object.values(site.collections ?? {}).some((c) => - (c?.fields ?? []).some((d) => isReferenceFieldType(d.type)), + (c?.fields ?? []).some( + (d) => + isReferenceFieldType(d.type) || + (d.migration?.fromType !== undefined && isReferenceFieldType(d.migration.fromType)), + ), ); } diff --git a/packages/core-ts/src/references/localize.ts b/packages/core-ts/src/references/localize.ts new file mode 100644 index 00000000..d8664481 --- /dev/null +++ b/packages/core-ts/src/references/localize.ts @@ -0,0 +1,31 @@ +/** + * RFC-0045 §7, `set_field_localized` phase A: the rows' `shared` follows a reference or repeater + * field's new `localized` in the same batch as the site write, anchored on the site's new hash, + * so guards and purges read `resolveFields`' rule from that moment. Rows a racing holder write + * inserts afterwards read `shared` from the site row at batch time (R16). The revert's site + * write goes through the same diff, back. Pure. + */ +import type { Site } from "@cloudhal/core-contracts"; +import type { SqlStatement } from "@cloudhal/storage"; +import { renamedFrom } from "../fields/types.ts"; +import { holdsReferences, sharedOf } from "./rows.ts"; + +/** One `UPDATE entry_refs SET shared` per field whose `localized` the write changes. */ +export function sharedUpdates(site: Site, next: Site, hash: string): SqlStatement[] { + const out: SqlStatement[] = []; + for (const [collection, settings] of Object.entries(next.collections ?? {})) + for (const def of settings?.fields ?? []) { + // A rename's revert restores the old key and may restore an older `localized` with it. + const from = renamedFrom(def); + const keys = [def.key, ...(from === undefined ? [] : [from])]; + const was = site.collections?.[collection]?.fields?.find((d) => keys.includes(d.key)); + if (!was || was.localized === def.localized || !holdsReferences(def)) continue; + out.push({ + sql: `UPDATE entry_refs SET shared = ? + WHERE site_id = ? AND source_collection = ? AND root_key IN (?, ?) + AND EXISTS (SELECT 1 FROM nodes WHERE id = ? AND hash = ?)`, + params: [sharedOf(def), site.id, collection, def.key, from ?? def.key, site.id, hash], + }); + } + return out; +} diff --git a/packages/core-ts/src/references/rebuild.ts b/packages/core-ts/src/references/rebuild.ts index 764acdcc..2d23f4c4 100644 --- a/packages/core-ts/src/references/rebuild.ts +++ b/packages/core-ts/src/references/rebuild.ts @@ -5,15 +5,95 @@ * replace a concurrent writer's fresher ones (R10, the `asset_refs` lesson of #196). * * `shared` is computed in JS from the same site row, so no `NULL` can arise, and body keys are - * read through any `migration.fromKey` the site carries (`entryRefRows`). There is no membership + * read through any `migration.fromKey` the site carries (`entryRefRows`); a running text ↔ + * reference conversion's path is read through its earlier type (`rebuildRows`). There is no membership * join: rows are kept for every stored value, and the site filters keep them harmless. The * `INSERT` is chunked below 90 KB a statement; the chunks count against the import batch's cap. * Worker-safe. */ -import type { ContentEntry, Site } from "@cloudhal/core-contracts"; +import type { ContentEntry, FieldDefinition, Site } from "@cloudhal/core-contracts"; import type { SqlStatement } from "@cloudhal/storage"; +import { GROUP_ID_PATTERN } from "../fields/props.ts"; +import { fieldDefinitions, isRecord } from "../fields/types.ts"; import { chunks, REFS_STATEMENT_BYTES } from "../media/refs-rebuild.ts"; -import { entryRefRowsOf } from "./rows.ts"; +import { convertingPath } from "./conversion-rows.ts"; +import { type EntryRefRow, entryRefRows, sharedOf, storedKey } from "./rows.ts"; + +const GROUP_ID = new RegExp(GROUP_ID_PATTERN); + +/** + * RFC-0045 §7: an entry's rows for a rebuild, read through a running text ↔ reference + * conversion's earlier type, so they mean only what phase B or an accepted write left. Converting + * from text, a bare string gets no row, so phase B checks it again; a multiple's list keeps its + * rows (only phase B or a guarded write stores one). Converting from references, every stored + * group-id-shaped string keeps its row, whether or not its group still has a member, as the live + * writer keeps it; phase B then reports a deleted target. Without such a marker, `entryRefRows`. + */ +export function rebuildRows(site: Pick, entry: ContentEntry): EntryRefRow[] { + const defs = fieldDefinitions(site, entry.collection); + const rows = entryRefRows(defs, entry.fields); + const values = isRecord(entry.fields) ? entry.fields : {}; + const out = [...rows]; + for (const def of defs) { + const f = convertingPath(def); + if (f === undefined) { + out.push(...reshapedRows(def, values, out)); + continue; + } + const k = storedKey(def, values) ?? def.key; + const sub = def.migration?.sub; + const stored = values[k]; + if (def.migration?.fromType === "text") { + // A bare string may be legacy text; a multiple's list came from phase B or a guarded write. + if (sub !== undefined || !Array.isArray(stored)) + out.splice(0, out.length, ...out.filter((row) => row.f !== f)); + continue; + } + const found = + sub === undefined + ? Array.isArray(stored) + ? stored + : [stored] + : Array.isArray(stored) + ? stored.map((i) => (isRecord(i) ? i[sub] : undefined)) + : []; + for (const t of new Set(found)) + if (typeof t === "string" && GROUP_ID.test(t)) out.push({ t, f, k, shared: sharedOf(def) }); + } + return out; +} + +/** + * A running reference (multiple) ↔ repeater conversion (top level): a stored value still in the + * old shape is read through the old type, so its rows stand until phase B rewrites it. From a + * reference, each id-shaped string of the list gets a row at the field path; from a repeater, + * each item's id-shaped sub-value at `.`. + */ +function reshapedRows( + def: FieldDefinition, + values: Readonly>, + have: readonly EntryRefRow[], +): EntryRefRow[] { + const from = def.migration?.fromType; + if (def.migration?.sub !== undefined || (from !== "reference" && from !== "repeater")) return []; + const k = storedKey(def, values) ?? def.key; + const stored = values[k]; + const found: { f: string; t: string }[] = []; + const id = (v: unknown): v is string => typeof v === "string" && GROUP_ID.test(v); + for (const v of Array.isArray(stored) ? stored : [stored]) { + if (from === "reference" && id(v)) found.push({ f: k, t: v }); + if (from === "repeater" && isRecord(v)) + for (const [sub, t] of Object.entries(v)) if (id(t)) found.push({ f: `${k}.${sub}`, t }); + } + const seen = new Set(have.map((r) => `${r.f} ${r.t}`)); + const out: EntryRefRow[] = []; + for (const { f, t } of found) { + if (seen.has(`${f} ${t}`)) continue; + seen.add(`${f} ${t}`); + out.push({ t, f, k, shared: sharedOf(def) }); + } + return out; +} const INSERT_SQL = `INSERT INTO entry_refs (site_id, target_id, source_id, source_collection, field, root_key, shared, source_status) SELECT ?, json_extract(r.value, '$.t'), json_extract(r.value, '$.n'), json_extract(r.value, '$.c'), @@ -31,14 +111,15 @@ export function entryRefItems( if (!siteRow) return []; const site = JSON.parse(siteRow.body) as Pick; const out: string[] = []; - for (const r of rows) { - if (r.kind !== "contentEntry") continue; - const entry = JSON.parse(r.body) as ContentEntry; - for (const row of entryRefRowsOf(site, entry)) + const entries = rows.flatMap((r) => + r.kind === "contentEntry" ? [{ id: r.id, entry: JSON.parse(r.body) as ContentEntry }] : [], + ); + for (const { id, entry } of entries) { + for (const row of rebuildRows(site, entry)) out.push( JSON.stringify({ t: row.t, - n: r.id, + n: id, c: entry.collection, f: row.f, k: row.k, diff --git a/packages/core-ts/src/references/release-purge.ts b/packages/core-ts/src/references/release-purge.ts index 7001d9ab..38a2bd89 100644 --- a/packages/core-ts/src/references/release-purge.ts +++ b/packages/core-ts/src/references/release-purge.ts @@ -7,8 +7,9 @@ * that read happens except on a first publish: a snapshot's site node is captured before the * publish flips the pointer, so the row differs from it at least in `publishedSnapshotId` and * `rev`. The read is counted but never refused (one call per release, from the sizing's margin), - * so a site without references never depends on what other runs left; if it fails, the working - * row decides (no references: the stored plan; else everything). Only a node that defines + * so a site without references never depends on what other runs left. If it fails, it is retried + * once, then the run purges everything; never the working row (#213), since a draft that + * dropped its reference fields would send the closure-less stored plan. Only a node that defines * references, or still converts from them, re-plans; every other site sends the plan stored at * pin time, as before 3b (nothing when it purges nothing). * - **Re-planned** `from → snapshotId` (`from`: the pointer the batch moved), like every other @@ -32,7 +33,7 @@ * release (the re-plan's 2 `getSnapshot`, bookkeeping 4, `recordPurge` 1); the built-in jobs' 40 * blob reads. Per release, counted from the allowance itself: `refreshSearch`'s worst case * (`MAX_RELEASE_ITEMS` × 2 + the site + ~10 taxonomies ≈ 211), `labelOf` and the publish's own - * blobs (2), the gate read (1); and the purge's sends (≤ 4). + * blobs (2), the gate read (1, 2 on a retry); and the purge's sends (≤ 4). * - **Free** (40 statements, 1 release per tick): * 1,000 − 2 × (40 + 7) − 2 × 40 − 2 × (211 + 2 + 4) = 392, so 350. * - **Paid** (900 statements; about 27 releases fit a tick's D1, at 33 statements each with claim @@ -134,14 +135,13 @@ async function planFor(input: ReleasePurgeInput): Promise { // site without references never depends on what other runs left. let node: Site = site; const published = input.publishedSite; - if (published && !published.current) - try { - node = await planner.readSnapshotNode(published.ref); - } catch (e) { - // the working row decides: no references there, the stored plan; else everything - if (!definesReferences(site)) return input.stored; - return everything(`the published site node could not be read: ${message(e)}`); - } + if (published && !published.current) { + const read = await readTwice(() => planner.readSnapshotNode(published.ref)); + // Never the working row (#213): a draft that dropped its reference fields would send the + // closure-less stored plan and leave referrer pages stale. + if (!read.ok) return everything(`the published site node could not be read: ${read.error}`); + node = read.value; + } if (!definesReferences(node)) return input.stored; if (!input.from) return { mode: "everything", reason: "nothing was live before the release" }; try { @@ -171,6 +171,21 @@ async function planFor(input: ReleasePurgeInput): Promise { const message = (e: unknown) => (e instanceof Error ? e.message : String(e)); +/** The gate read, retried once: a transient blob error should not cost a whole-site purge. */ +async function readTwice( + read: () => Promise, +): Promise<{ ok: true; value: T } | { ok: false; error: string }> { + try { + return { ok: true, value: await read() }; + } catch { + try { + return { ok: true, value: await read() }; + } catch (e) { + return { ok: false, error: message(e) }; + } + } +} + /** Sends the run's purge (see the module comment); a throw only from the purger itself. */ export async function sendReleasePurge(input: ReleasePurgeInput): Promise { const plan = await planFor(input); diff --git a/packages/core-ts/src/references/revert-drops.ts b/packages/core-ts/src/references/revert-drops.ts new file mode 100644 index 00000000..23ffb41f --- /dev/null +++ b/packages/core-ts/src/references/revert-drops.ts @@ -0,0 +1,89 @@ +/** + * RFC-0045 §7, `revertFieldOperation`: a restored reference whose group has no member left in the + * site (in a collection its field allows) is dropped from the value it would restore, and + * reported (`FieldRevertResult.dropped`). Only values the revert changes are looked at: a value it + * leaves as stored keeps its rows, like any write that does not touch it. A repeater item left + * empty is dropped, and a list left empty is no value. Pure. + */ +import type { FieldDefinition } from "@cloudhal/core-contracts"; +import { isRecord, sameJson } from "../fields/types.ts"; +import { holdsReferences, storedKey } from "./rows.ts"; +import { referenceUses } from "./values.ts"; + +type Values = Readonly>; + +/** A dropped value: the field path (`venue`, `speakers.person`) and the group it named. */ +export interface DroppedReference { + field: string; + targetId: string; +} + +/** The reference-holding definitions whose value `next` changes against `stored`. */ +function changed(defs: readonly FieldDefinition[], next: Values, stored: Values) { + return defs.filter((def) => { + if (!holdsReferences(def) || !Object.hasOwn(next, def.key)) return false; + const key = storedKey(def, stored); + return !sameJson(next[def.key], key === undefined ? undefined : stored[key]); + }); +} + +/** The group ids the changed values of `next` name (one read of their members per pass). */ +export function changedReferenceIds( + defs: readonly FieldDefinition[], + next: Values, + stored: Values, +): string[] { + return changed(defs, next, stored).flatMap((def) => + referenceUses(def, next[def.key], "").map((u) => u.groupId), + ); +} + +type Keep = (groupId: string, collections: readonly string[]) => boolean; + +function dropIn(def: FieldDefinition, value: unknown, keep: Keep, gone: DroppedReference[]) { + if (def.type === "reference") { + const ok = (id: unknown) => typeof id !== "string" || keep(id, def.collections); + const note = (id: unknown) => gone.push({ field: def.key, targetId: String(id) }); + if (!Array.isArray(value)) { + if (ok(value)) return value; + note(value); + return undefined; + } + const kept = value.filter((id) => ok(id) || (note(id), false)); + return kept.length > 0 ? kept : undefined; + } + if (def.type !== "repeater" || !Array.isArray(value)) return value; + const items: unknown[] = []; + for (const item of value) { + if (!isRecord(item)) { + items.push(item); + continue; + } + const next = { ...item }; + for (const sub of def.fields) { + const id = next[sub.key]; + if (sub.type !== "reference" || typeof id !== "string" || keep(id, sub.collections)) continue; + gone.push({ field: `${def.key}.${sub.key}`, targetId: id }); + delete next[sub.key]; + } + if (Object.keys(next).length > 0) items.push(next); + } + return items.length > 0 ? items : undefined; +} + +/** `next` without the changed reference values `keep` refuses, and what went. */ +export function dropMissingReferences( + defs: readonly FieldDefinition[], + next: Values, + stored: Values, + keep: Keep, +): { fields: Record; dropped: DroppedReference[] } { + const fields: Record = { ...next }; + const dropped: DroppedReference[] = []; + for (const def of changed(defs, next, stored)) { + const value = dropIn(def, fields[def.key], keep, dropped); + if (value === undefined) delete fields[def.key]; + else fields[def.key] = value; + } + return { fields, dropped }; +} diff --git a/packages/core-ts/src/references/rows.ts b/packages/core-ts/src/references/rows.ts index 11a1f47a..b313af49 100644 --- a/packages/core-ts/src/references/rows.ts +++ b/packages/core-ts/src/references/rows.ts @@ -6,7 +6,7 @@ * Pairs are deduplicated after the writer canonicalized member ids to group ids. Pure. */ import type { FieldDefinition, Site } from "@cloudhal/core-contracts"; -import { fieldDefinitions, isRecord } from "../fields/types.ts"; +import { fieldDefinitions, isRecord, renamedFrom } from "../fields/types.ts"; import { referenceUses } from "./values.ts"; /** One row's varying part: target group, field path, top-level key. */ @@ -37,7 +37,7 @@ export function storedKey( values: Record, ): string | undefined { if (Object.hasOwn(values, def.key)) return def.key; - const from = def.migration?.fromKey; + const from = renamedFrom(def); return from !== undefined && Object.hasOwn(values, from) ? from : undefined; } @@ -85,10 +85,13 @@ export function definitionOfPath( field: string, ): { def: FieldDefinition; collections: readonly string[] } | undefined { const [root, sub] = field.split("."); - const def = defs.find((d) => d.key === root) ?? defs.find((d) => d.migration?.fromKey === root); + const def = defs.find((d) => d.key === root) ?? defs.find((d) => renamedFrom(d) === root); if (!def) return undefined; if (def.type === "reference" && sub === undefined) return { def, collections: def.collections }; if (def.type !== "repeater" || sub === undefined) return undefined; - const s = def.fields.find((x) => x.key === sub); + // A row written before a sub-field rename names the old sub-key (RFC-0045 §7 `sub`). + const m = def.migration; + const key = m?.sub !== undefined && m.fromKey === sub ? m.sub : sub; + const s = def.fields.find((x) => x.key === key); return s?.type === "reference" ? { def, collections: s.collections } : undefined; } diff --git a/packages/core-ts/src/references/values.ts b/packages/core-ts/src/references/values.ts index f87a1af1..3012aef1 100644 --- a/packages/core-ts/src/references/values.ts +++ b/packages/core-ts/src/references/values.ts @@ -30,11 +30,15 @@ export function referenceUses(def: FieldDefinition, value: unknown, path: string } if (def.type !== "repeater" || !Array.isArray(value)) return []; const subs = def.fields.filter((s) => s.type === "reference"); + // RFC-0045 §7: a running sub-field rename's items still hold the old key. + const m = def.migration; + const old = (key: string) => (m?.sub === key ? m.fromKey : undefined); const out: ReferenceUse[] = []; value.forEach((item, i) => { if (!isRecord(item)) return; for (const sub of subs) { - const v = ownValue(item, sub.key); + const was = old(sub.key); + const v = ownValue(item, sub.key) ?? (was === undefined ? undefined : ownValue(item, was)); if (typeof v === "string" && sub.type === "reference") out.push({ path: `${path}/${i}/${sub.key}`, groupId: v, collections: sub.collections }); } diff --git a/packages/core-ts/src/references/writer.ts b/packages/core-ts/src/references/writer.ts index b613f850..8cb4a911 100644 --- a/packages/core-ts/src/references/writer.ts +++ b/packages/core-ts/src/references/writer.ts @@ -11,19 +11,21 @@ * with reference fields, and on writes that change `fields` or `collection` elsewhere. * - **The rows**, anchored on the entry's new hash: an anchored `DELETE`, then one plain * `INSERT … SELECT` that reads `shared` from the site row as it is when the batch runs, by key - * or `migration.fromKey` (the exact key wins; an aggregate, not a correlated `ORDER BY`, which - * workerd's SQLite does not resolve). A key with no reference or repeater definition (absent, or + * or a top-level rename's `migration.fromKey` (a sub-field rename's names a sub-key, RFC-0045 + * §7 `sub`; the exact key wins; an aggregate, not a correlated `ORDER BY`, which workerd's + * SQLite does not resolve). A key with no reference or repeater definition (absent, or * retyped by a concurrent import) yields `NULL` and fails the batch (`isSharedNullViolation` → * `ConflictError`); it is never `OR IGNORE`. */ import type { ContentEntry, FieldDefinition, Site } from "@cloudhal/core-contracts"; import type { SqlDriver, SqlStatement } from "@cloudhal/storage"; import { refuseFieldIssues } from "../fields/refusal.ts"; +import type { FieldRows } from "../fields/text-reference.ts"; import { type FieldIssue, fieldDefinitions, isRecord, sameJson } from "../fields/types.ts"; import { and, type Cond } from "../node-sql.ts"; import { hasReferenceFields } from "./issues.ts"; import { memberGroup } from "./members.ts"; -import { definitionsPath, entryRefRows, holdsReferences } from "./rows.ts"; +import { definitionsPath, entryRefRows, holdsReferences, storedKey } from "./rows.ts"; import { definitionsAnchor, type TargetPair, targetGuard } from "./sql.ts"; import { type MemberRow, readMembers, targetProblem } from "./targets.ts"; import { type ReferenceUse, referenceUses } from "./values.ts"; @@ -98,6 +100,8 @@ export interface EntryRefsPlan { interface Use extends ReferenceUse { fresh: boolean; def: FieldDefinition; + /** A stored value a running conversion did not accept: only the own-group check applies. */ + held?: boolean; } /** The uses the target guard covers: changed keys, or every own value when the entry goes live. */ @@ -105,7 +109,12 @@ function guardedUses( defs: readonly FieldDefinition[], draft: ContentEntry, stored: ContentEntry | undefined, + at: { converting?: ConvertingRows | undefined; held?: readonly HeldRef[] | undefined } = {}, ): Use[] { + const { converting } = at; + // A value the conversion held is stored as it was: phase B reports it, no target guard decides + // it, but the own-group rule still applies (`useIssues`). + const kept = new Set((at.held ?? []).map((h) => `${h.f} ${h.t}`)); const values = draft.fields ?? {}; const same = stored?.collection === draft.collection; const before = same ? (stored?.fields ?? {}) : {}; @@ -116,8 +125,17 @@ function guardedUses( const changed = !sameJson(values[def.key], before[def.key]); if (!changed && !goingLive) continue; const had = new Set(referenceUses(def, before[def.key], "").map((u) => u.groupId)); - for (const use of referenceUses(def, values[def.key], `fields/${def.key}`)) - out.push({ ...use, def, fresh: !had.has(use.groupId) }); + for (const use of referenceUses(def, values[def.key], `fields/${def.key}`)) { + // RFC-0045 §7: on a path converting from text, only an id with its row is already held. + const path = def.type === "repeater" ? `${def.key}.${use.path.split("/")[3]}` : def.key; + if (kept.has(`${path} ${use.groupId}`)) { + out.push({ ...use, def, fresh: false, held: true }); + continue; + } + const held = + converting?.path === path ? converting.rows.has(use.groupId) : had.has(use.groupId); + out.push({ ...use, def, fresh: !held }); + } } return out; } @@ -166,19 +184,49 @@ function useIssues(site: Site, draft: ContentEntry, uses: readonly Use[], member }); continue; } + // A held value is stored as it was: phase B reports it; only the own-group rule applies. + if (use.held) continue; const issue = useIssue(site, use, members); if (issue) issues.push(issue); } return issues; } +/** A stored row a write leaves as it is: a value a running conversion did not accept (§7). */ +export interface HeldRef { + /** The field path (`venue`, `speakers.host`). */ + readonly f: string; + readonly t: string; +} + +/** The node's rows for a running text ↔ reference conversion's path, as the writer read them. */ +export interface ConvertingRows { + readonly path: string; + readonly rows: FieldRows; +} + /** The rows statements for `draft` (an anchored `DELETE`, then the plain `INSERT … SELECT`). */ -export function entryRefStatements(site: Site, draft: ContentEntry, hash: string): SqlStatement[] { - const rows = entryRefRows(fieldDefinitions(site, draft.collection), draft.fields); +export function entryRefStatements( + site: Site, + draft: ContentEntry, + hash: string, + /** + * RFC-0045 §7: the values a running text ↔ reference conversion did not accept, per value + * (field path and group id). Their stored rows stay as they are, and they get no new one, until + * phase B reports or clears them; every other row follows the draft. + */ + held: readonly HeldRef[] = [], +): SqlStatement[] { + const kept = new Set(held.map((h) => `${h.f} ${h.t}`)); + const rows = entryRefRows(fieldDefinitions(site, draft.collection), draft.fields).filter( + (r) => !kept.has(`${r.f} ${r.t}`), + ); + const keep = + kept.size > 0 ? " AND field || ' ' || target_id NOT IN (SELECT value FROM json_each(?))" : ""; const out: SqlStatement[] = [ { - sql: `DELETE FROM entry_refs WHERE source_id = ? AND ${ANCHOR}`, - params: [draft.id, draft.id, hash], + sql: `DELETE FROM entry_refs WHERE source_id = ?${keep} AND ${ANCHOR}`, + params: [draft.id, ...(kept.size > 0 ? [JSON.stringify([...kept])] : []), draft.id, hash], }, ]; if (rows.length === 0) return out; @@ -189,12 +237,14 @@ export function entryRefStatements(site: Site, draft: ContentEntry, hash: string max(CASE WHEN json_extract(d.value, '$.key') = json_extract(p.value, '$.k') THEN CASE json_extract(d.value, '$.localized') WHEN 1 THEN 2 ELSE 1 END END), max(CASE WHEN json_extract(d.value, '$.migration.fromKey') = json_extract(p.value, '$.k') + AND json_extract(d.value, '$.migration.sub') IS NULL THEN CASE json_extract(d.value, '$.localized') WHEN 1 THEN 2 ELSE 1 END END)) FROM nodes s, json_each(json_extract(s.body, ?)) d WHERE s.id = ? AND s.kind = 'site' AND json_extract(d.value, '$.type') IN ('reference', 'repeater') AND (json_extract(d.value, '$.key') = json_extract(p.value, '$.k') - OR json_extract(d.value, '$.migration.fromKey') = json_extract(p.value, '$.k'))), + OR (json_extract(d.value, '$.migration.fromKey') = json_extract(p.value, '$.k') + AND json_extract(d.value, '$.migration.sub') IS NULL))), ? FROM json_each(?) p WHERE ${ANCHOR}`, params: [ @@ -216,25 +266,84 @@ export function entryRefStatements(site: Site, draft: ContentEntry, hash: string function pairsOf(uses: readonly Use[]): TargetPair[] { const pairs = new Map(); for (const u of uses) { + if (u.held) continue; const pair: TargetPair = { t: u.groupId, c: u.collections, f: u.fresh ? 1 : 0 }; pairs.set(JSON.stringify(pair), pair); } return [...pairs.values()]; } +/** Every string a stored value holds, at any depth (the ids a rewrite carries over). */ +function strings(value: unknown, out = new Set()): Set { + if (typeof value === "string") out.add(value); + else if (Array.isArray(value)) for (const v of value) strings(v, out); + else if (isRecord(value)) for (const v of Object.values(value)) strings(v, out); + return out; +} + /** - * §2.6: the target guard of a field-operation rewrite, over the reference values it changes - * (undefined when none change). A miss skips the entry for the next pass, like a concurrent edit. - * The rewrite's own statements are guarded on the site still carrying the operation's marker, so - * the definitions the pairs come from cannot change under it. + * The part of a stored value whose strings are ids carried over: none of a value converting from + * text (top-level, or the sub-field of a `sub` marker), whose strings name nothing yet. + */ +function carriedPart(def: FieldDefinition, was: unknown): unknown { + const m = def.migration; + if (m?.fromType !== "text") return was; + if (m.sub === undefined || !Array.isArray(was)) return undefined; + const sub = m.sub; + return was.map((item) => { + if (!isRecord(item)) return item; + const { [sub]: _converting, ...rest } = item; + return rest; + }); +} + +/** + * §2.6 and §7: the target guard of a field-operation rewrite, over the reference values it + * changes (undefined when none change). A miss skips the entry for the next pass, like a + * concurrent edit. The rewrite's own statements are guarded on the site still carrying the + * operation's marker, so the definitions the pairs come from cannot change under it. + * + * - `fresh` (phase B): only ids the stored value does not already hold, newly chosen, so a + * conversion never writes a deleted or archived target. Ids it carries over (a reshape, a + * rename) stay as stored, as rows are kept for a missing target. + * - `present` (a revert, which first drops restored ids with no member): every changed id needs + * a member in an allowed collection, whatever its status. */ export function rewriteTargetGuard( site: Site, next: ContentEntry, stored: ContentEntry, + options: { + mode?: "fresh" | "present" | undefined; + /** The node's rows on a running text → reference path: an id with its row is carried. */ + converting?: ConvertingRows | undefined; + } = {}, ): Cond | undefined { - const defs = fieldDefinitions(site, next.collection); - return targetGuard(site.id, pairsOf(guardedUses(defs, next, stored))); + const mode = options.mode ?? "fresh"; + const { converting } = options; + const values = next.fields ?? {}; + const before = stored.collection === next.collection ? (stored.fields ?? {}) : {}; + const pairs = new Map(); + for (const def of fieldDefinitions(site, next.collection)) { + if (!holdsReferences(def) || !Object.hasOwn(values, def.key)) continue; + const key = storedKey(def, before); + const was = key === undefined ? undefined : before[key]; + // A value converting from text is newly chosen even when phase B writes it unchanged (a + // pristine legacy id, `text-reference.ts`): its ids are checked anyway. + const fromText = mode === "fresh" && def.migration?.fromType === "text"; + if (!fromText && sameJson(values[def.key], was)) continue; + const carried = strings(carriedPart(def, was)); + for (const use of referenceUses(def, values[def.key], "")) { + if (mode === "fresh" && carried.has(use.groupId)) continue; + // An accepted value already has its row for the path: carried, like `guardedUses` reads it. + const path = def.type === "repeater" ? `${def.key}.${use.path.split("/")[2]}` : def.key; + if (mode === "fresh" && converting?.path === path && converting.rows.has(use.groupId)) + continue; + const pair: TargetPair = { t: use.groupId, c: use.collections, f: mode === "fresh" ? 1 : 0 }; + pairs.set(JSON.stringify(pair), pair); + } + } + return targetGuard(site.id, [...pairs.values()]); } /** True when the write changes the entry's values or collection (the definitions anchor applies). */ @@ -252,9 +361,21 @@ const changesFields = (draft: ContentEntry, stored: ContentEntry | undefined) => export async function planEntryRefs( sql: SqlDriver, site: Site, - draft: ContentEntry, - stored: ContentEntry | undefined, + write: { + draft: ContentEntry; + stored: ContentEntry | undefined; + /** + * `stored` through the field-operation markers (RFC-0045 §7): which values the write changes, + * so a value an operation reshapes or renames, and the writer left alone, is not re-checked. + */ + baseline: ContentEntry | undefined; + /** Values a running conversion did not accept: their rows stay as they are. */ + held?: readonly HeldRef[] | undefined; + /** The node's rows for a running text → reference path: which ids it already holds. */ + converting?: ConvertingRows | undefined; + }, ): Promise { + const { draft, stored } = write; const references = hasReferenceFields(site); const anchor = references || changesFields(draft, stored) @@ -262,14 +383,17 @@ export async function planEntryRefs( : undefined; if (!references) return { guard: anchor, statements: () => [], diagnose: async () => {} }; const defs = fieldDefinitions(site, draft.collection); - const uses = guardedUses(defs, draft, stored); + const uses = guardedUses(defs, draft, write.baseline, { + converting: write.converting, + held: write.held, + }); const groups = [...new Set(uses.map((u) => u.groupId))]; const check = async () => refuseFieldIssues(useIssues(site, draft, uses, await readMembers(sql, site.id, groups))); await check(); return { guard: and(anchor, targetGuard(site.id, pairsOf(uses))), - statements: (hash) => entryRefStatements(site, draft, hash), + statements: (hash) => entryRefStatements(site, draft, hash, write.held), diagnose: check, }; } diff --git a/packages/core-ts/src/release-deps.ts b/packages/core-ts/src/release-deps.ts index e4170d7e..2c0b4a4f 100644 --- a/packages/core-ts/src/release-deps.ts +++ b/packages/core-ts/src/release-deps.ts @@ -9,7 +9,8 @@ */ import type { FieldDefinition, Site } from "@cloudhal/core-contracts"; import { type NodeVersion, referencedNodeIds } from "@cloudhal/schedules/engine"; -import { fieldDefinitions, isRecord } from "./fields/types.ts"; +import { GROUP_ID_PATTERN } from "./fields/props.ts"; +import { fieldDefinitions, isRecord, renamedFrom } from "./fields/types.ts"; /** One value with its reference ids removed (a repeater keeps its other sub-values). */ function withoutReferences(def: FieldDefinition, value: unknown): unknown { @@ -24,16 +25,33 @@ function withoutReferences(def: FieldDefinition, value: unknown): unknown { ); } -/** An entry body without its reference values, under the site's definitions. */ +const GROUP_ID = new RegExp(GROUP_ID_PATTERN); + +/** + * A value of a field an operation is rewriting, without the group ids it may still hold in its + * old shape (RFC-0045 §7: a running conversion from references, a sub-field rename's old key). + * Field values name no other entry, so every group-id string is a reference. + */ +function withoutIds(value: unknown): unknown { + if (typeof value === "string") return GROUP_ID.test(value) ? undefined : value; + if (Array.isArray(value)) return value.map(withoutIds).filter((v) => v !== undefined); + if (!isRecord(value)) return value; + return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, withoutIds(v)])); +} + +/** An entry body without its reference values, under the site's definitions (and markers). */ export function stripReferences(site: Pick, body: unknown): unknown { if (!isRecord(body) || typeof body.collection !== "string" || !isRecord(body.fields)) return body; const defs = fieldDefinitions(site, body.collection); - if (!defs.some((d) => d.type === "reference" || d.type === "repeater")) return body; + if (!defs.some((d) => d.type === "reference" || d.type === "repeater" || d.migration)) + return body; const fields: Record = { ...body.fields }; for (const def of defs) - for (const key of [def.key, def.migration?.fromKey]) - if (key !== undefined && Object.hasOwn(fields, key)) - fields[key] = withoutReferences(def, fields[key]); + for (const key of [def.key, renamedFrom(def)]) + if (key !== undefined && Object.hasOwn(fields, key)) { + const value = withoutReferences(def, fields[key]); + fields[key] = def.migration ? withoutIds(value) : value; + } return { ...body, fields }; } diff --git a/packages/core-ts/src/store/entries.ts b/packages/core-ts/src/store/entries.ts index 7d7f2b0c..3580c855 100644 --- a/packages/core-ts/src/store/entries.ts +++ b/packages/core-ts/src/store/entries.ts @@ -3,6 +3,7 @@ import { assertValid, type ContentEntry } from "@cloudhal/core-contracts"; import { indexMarkdown, withoutIndex } from "../body-index.ts"; import { prepareCorrections } from "../corrections.ts"; import { ConflictError, NotFoundError } from "../errors.ts"; +import { fieldDefinitions } from "../fields/types.ts"; import { nodeHash } from "../hash.ts"; import { assertUnlocked } from "../holds.ts"; import { ID_PREFIX } from "../ids.ts"; @@ -11,13 +12,15 @@ import { fillMissingSrc } from "../media/refs.ts"; import { and, casUpdateStatement, insertNodeStatement, OpBatch, runOpBatch } from "../node-sql.ts"; import { uniquenessKey } from "../nodes.ts"; import { entryPath } from "../permalinks.ts"; +import { loadConversionEntries } from "../references/conversion-load.ts"; +import { convertingPath, loadNodeRows } from "../references/conversion-rows.ts"; import { planLeave } from "../references/leave.ts"; import { dropEntryRefsStatement } from "../references/rebuild.ts"; import { canonicalReferences } from "../references/writer.ts"; import { canonicalTerms } from "../taxonomy.ts"; import { entersCollection, planNewCollection } from "./collection-paths.ts"; import type { StoreContext } from "./context.ts"; -import { normalizeEntryFields, planEntryFields } from "./entry-fields.ts"; +import { normalizeWrite, planEntryFields } from "./entry-fields.ts"; import { planEntryIntegrity } from "./entry-integrity.ts"; import { planSourceForward } from "./source-forward.ts"; import { checkLocalizedNode } from "./translations.ts"; @@ -63,15 +66,24 @@ export class EntryEngine { }); draft = corrections ? { ...rest, corrections } : (rest as ContentEntry); } - draft = normalizeEntryFields(site, draft); + // RFC-0045 §7: through the field-operation markers, stored entry too (`normalizeWrite`). + const normalized = await normalizeWrite(this.#ctx, site, { draft, stored }); + const { baseline, held, rows: converting } = normalized; + draft = normalized.draft; // RFC-0045 §1: a member's id is stored as its group's. - draft = await canonicalReferences(this.#ctx.sql, site, { draft, stored }); + draft = await canonicalReferences(this.#ctx.sql, site, { draft, stored: baseline }); assertValid("content-entry", canonicalTerms(draft)); checkLocalizedNode("contentEntry", site, draft); // RFC-0041 §3: typed field values, requiredToPublish and their asset references. const fields = await planEntryFields(this.#ctx, site, draft, stored); // RFC-0033 links; RFC-0045 §2 targets, anchors, `entry_refs`, a move or group change. - const integrity = await planEntryIntegrity(this.#ctx, site, { draft, stored }); + const integrity = await planEntryIntegrity(this.#ctx, site, { + draft, + stored, + baseline, + held, + converting, + }); if (stored) await assertUnlocked(this.#ctx.sql, stored.siteId, [stored.id], options.releaseEdit); if (draft.body.format === "markdown") { diff --git a/packages/core-ts/src/store/entry-fields.ts b/packages/core-ts/src/store/entry-fields.ts index 0b7e6ef0..1f68d9c0 100644 --- a/packages/core-ts/src/store/entry-fields.ts +++ b/packages/core-ts/src/store/entry-fields.ts @@ -24,10 +24,16 @@ import { normalizeThroughMarkers, refuseFieldIssues, } from "../fields/index.ts"; -import { sameJson } from "../fields/types.ts"; +import type { FailedValue } from "../fields/migration.ts"; +import type { FieldRows } from "../fields/text-reference.ts"; +import { isRecord, sameJson } from "../fields/types.ts"; import { fieldAssetIssues, fieldAssetRefs } from "../media/field-refs.ts"; import type { AssetRefUse } from "../media/refs.ts"; import type { Cond } from "../node-sql.ts"; +import { loadConversionEntries } from "../references/conversion-load.ts"; +import type { ConversionEntries } from "../references/conversion-maps.ts"; +import { convertingPath, loadNodeRows } from "../references/conversion-rows.ts"; +import type { HeldRef } from "../references/writer.ts"; import type { StoreContext } from "./context.ts"; /** What the field checks add to an entry write. */ @@ -68,17 +74,123 @@ async function originalOf( * a value still in the old type converts, a removed field or a translation's copy of a field that * stops being localized goes), so a person's edit and the operation converge. */ -export function normalizeEntryFields(site: Site, draft: ContentEntry): ContentEntry { +export function normalizeEntryFields( + site: Site, + draft: ContentEntry, + options: { + /** RFC-0045 §7: the text ↔ reference rows' maps, and the node's rows for the path. */ + entries?: ConversionEntries | undefined; + rows?: FieldRows | undefined; + /** + * A person's write, with the stored entry (none: a new entry): a value the write changed is + * written under the new type. Absent: the stored entry itself, normalized as the baseline. + */ + write?: { stored: ContentEntry | undefined } | undefined; + /** The definitions whose value does not convert (kept as stored; the writer leaves its rows). */ + onUnconvertible?: ((def: FieldDefinition, failed: readonly FailedValue[]) => void) | undefined; + } = {}, +): ContentEntry { const defs = fieldDefinitions(site, draft.collection); if (!defs.some((d) => d.migration)) return draft; const t = draft.translation; + const stored = options.write?.stored; const fields = normalizeThroughMarkers(defs, draft.fields, { - convert: convertContext(site), + // RFC-0045 §7: the text ↔ reference rows' maps, and the group text → reference skips. + convert: { ...convertContext(site), entries: options.entries, group: t?.group ?? draft.id }, + stored: options.write + ? stored?.collection === draft.collection + ? (stored.fields ?? {}) + : {} + : undefined, + rows: options.rows, + onUnconvertible: options.onUnconvertible, translation: !!t && t.group !== draft.id && t.source?.siteId === undefined, }); return fields ? { ...draft, fields } : draft; } +/** The strings of a value (a string, or a list's). */ +const idsOf = (value: unknown): string[] => + (Array.isArray(value) ? value : [value]).filter((v): v is string => typeof v === "string"); + +/** The ids the draft holds at a converting path: the field's value, or the sub-field's items'. */ +function idsAtPath(def: FieldDefinition, fields: ContentEntry["fields"]): string[] { + const value = (fields ?? {})[def.key]; + const sub = def.migration?.sub; + if (sub === undefined) return idsOf(value); + return Array.isArray(value) ? value.flatMap((i) => (isRecord(i) ? idsOf(i[sub]) : [])) : []; +} + +/** + * The held pairs. Converting to references, per occurrence: an id the draft also holds in an + * accepted or converted item (it occurs more often than in the failing stored ones) is not held, + * so it gets its guard and row. Converting from references, every stored failing id is held. + */ +function heldRefs( + failures: readonly { def: FieldDefinition; f: string; ids: readonly string[] }[], + fields: ContentEntry["fields"], +): HeldRef[] { + const out: HeldRef[] = []; + for (const { def, f, ids } of failures) { + const count = (list: readonly string[], id: string) => list.filter((x) => x === id).length; + const all = idsAtPath(def, fields); + // Only converting to references does another occurrence get its own guard and row; from + // references, every stored failing id keeps its rows until phase B reports it. + const toReference = def.migration?.fromType === "text"; + for (const t of new Set(ids)) + if (!toReference || count(all, t) <= count(ids, t)) out.push({ f, t }); + } + return out; +} + +/** + * RFC-0045 §7: a write's draft and its stored entry through the markers. A running text ↔ + * reference conversion's maps, and the node's rows for its path, are read only while one runs: + * the values are decided by the rows phase C reads. The stored entry, normalized the same way, is + * the baseline of what the write changed; `held` lists the paths whose value did not convert, + * whose rows the write leaves as they are. + */ +export async function normalizeWrite( + ctx: StoreContext, + site: Site, + write: { draft: ContentEntry; stored: ContentEntry | undefined }, +): Promise<{ + draft: ContentEntry; + baseline: ContentEntry | undefined; + held: HeldRef[]; + rows: { path: string; rows: FieldRows } | undefined; +}> { + const { stored } = write; + const defs = fieldDefinitions(site, write.draft.collection); + const entries = await loadConversionEntries(ctx.graph, site, defs); + const rows = entries + ? await loadNodeRows(ctx.sql, { + siteId: site.id, + nodeId: stored?.id, + collection: write.draft.collection, + defs, + }) + : undefined; + const failures: { def: FieldDefinition; f: string; ids: string[] }[] = []; + const draft = normalizeEntryFields(site, write.draft, { + entries, + rows, + write: { stored }, + // Per value: only a stored value that did not convert keeps its rows as they are; one the + // person newly wrote is guarded like any choice. + onUnconvertible: (def, failed) => { + const f = convertingPath(def); + if (f === undefined) return; + const ids = failed.filter((x) => x.stored).flatMap((x) => idsOf(x.value)); + failures.push({ def, f, ids }); + }, + }); + const held = heldRefs(failures, draft.fields); + const baseline = stored && normalizeEntryFields(site, stored, { entries, rows }); + const path = defs.map(convertingPath).find((p) => p !== undefined); + return { draft, baseline, held, rows: rows && path !== undefined ? { path, rows } : undefined }; +} + /** Checks the field part of an entry write and plans its asset references; throws on issues. */ export async function planEntryFields( ctx: StoreContext, diff --git a/packages/core-ts/src/store/entry-integrity.ts b/packages/core-ts/src/store/entry-integrity.ts index 0c227758..f371f420 100644 --- a/packages/core-ts/src/store/entry-integrity.ts +++ b/packages/core-ts/src/store/entry-integrity.ts @@ -8,7 +8,7 @@ import type { ContentEntry, Site } from "@cloudhal/core-contracts"; import type { SqlStatement } from "@cloudhal/storage"; import { and, type Cond } from "../node-sql.ts"; import { planLeave } from "../references/leave.ts"; -import { planEntryRefs } from "../references/writer.ts"; +import { type ConvertingRows, type HeldRef, planEntryRefs } from "../references/writer.ts"; import type { StoreContext } from "./context.ts"; import { checkTranslationLinks } from "./translations.ts"; @@ -23,7 +23,16 @@ export interface EntryIntegrityPlan { export async function planEntryIntegrity( ctx: StoreContext, site: Site, - write: { draft: ContentEntry; stored: ContentEntry | undefined }, + write: { + draft: ContentEntry; + stored: ContentEntry | undefined; + /** `stored` through the field-operation markers, as the draft was (RFC-0045 §7). */ + baseline?: ContentEntry | undefined; + /** RFC-0045 §7: values that did not convert; their rows stay as they are. */ + held?: readonly HeldRef[] | undefined; + /** The node's rows for a running text → reference path (what it already holds). */ + converting?: ConvertingRows | undefined; + }, ): Promise { const { draft, stored } = write; const linked = await checkTranslationLinks(ctx, { @@ -32,7 +41,13 @@ export async function planEntryIntegrity( node: draft, stored, }); - const references = await planEntryRefs(ctx.sql, site, draft, stored); + const references = await planEntryRefs(ctx.sql, site, { + draft, + stored, + baseline: write.baseline ?? stored, + held: write.held, + converting: write.converting, + }); const leave = stored ? await planLeave(ctx.sql, site, { stored, next: draft }) : undefined; return { guard: and(references.guard, linked, leave?.guard), diff --git a/packages/core-ts/src/testing-field-operations-references.ts b/packages/core-ts/src/testing-field-operations-references.ts new file mode 100644 index 00000000..76753611 --- /dev/null +++ b/packages/core-ts/src/testing-field-operations-references.ts @@ -0,0 +1,1477 @@ +/** + * RFC-0045 rollout step 4 behavioural suite: the four field operations on reference and repeater + * fields, on every backend (`describeSiteGraphStore`). The types stay refused in production; this + * suite enables them with `SiteGraphStoreOptions.fieldTypes` on a store of its own. + * + * Covered: every new conversion row end to end, with `entry_refs` rows; `sub` (rename, convert, + * remove in every item, asset rows following); phase B's target guard (a target archived or + * deleted while it runs); resume; revert with a deleted target (`dropped`); `set_field_localized` + * phase A's `shared` in both directions with a racing holder write (R16); a racing holder write + * during a rename (`fromKey`); `field_operation_running` during a text → reference conversion, + * including a delete planned before phase A commits and batched after it (R12). + */ +import type { Actor, ContentEntry, FieldDefinition, Site } from "@cloudhal/core-contracts"; +import { describe, expect, it } from "vitest"; +import { GATES_ENFORCED } from "./availability.ts"; +import { ConflictError, InvariantError } from "./errors.ts"; +import { + continueFieldOperation, + type FieldOperation, + fieldOperationStatus, + revertFieldOperation, + runFieldOperation, +} from "./fields-operations/index.ts"; +import { nodeHash } from "./hash.ts"; +import { PNG_1X1 } from "./media/fixtures.ts"; +import { SiteGraphStore } from "./store.ts"; +import { exportSite, importSite } from "./sync.ts"; +import { racingDriver, type SuiteContext } from "./testing-taxonomy.ts"; + +const person: Actor = { kind: "human", id: "user_01HZX8K3Q9CD", name: "Halil" }; +const actor = { actor: person }; +const L = (label: string) => ({ en: { label } }); +const ENABLED = { supported: () => true }; + +const text = (key: string, over: Record = {}): FieldDefinition => + ({ key, type: "text", labels: L(key), localized: false, ...over }) as FieldDefinition; +const reference = (key: string, over: Record = {}): FieldDefinition => + ({ + key, + type: "reference", + labels: L(key), + localized: false, + collections: ["places"], + ...over, + }) as FieldDefinition; +const SPEAKERS: FieldDefinition = { + key: "speakers", + type: "repeater", + labels: L("Speakers"), + localized: false, + fields: [ + { key: "person", type: "reference", labels: L("Person"), collections: ["places"] }, + { key: "name", type: "text", labels: L("Name") }, + { key: "seats", type: "text", labels: L("Seats") }, + { key: "photo", type: "image", labels: L("Photo") }, + ], +}; + +async function code(p: Promise): Promise { + try { + await p; + } catch (e) { + if (e instanceof InvariantError) return e.code; + if (e instanceof ConflictError) return "conflict"; + throw e; + } + return "ok"; +} + +/** After a delete or write refused at batch time, the caller's retry. */ +const isAsset = (q: string) => + /^DELETE FROM asset_refs WHERE node_id = \? AND field LIKE 'fields/.test(q); + +export function fieldOperationsReferencesSuite(ctx: SuiteContext): void { + describe("RFC-0045 §7 field operations on references and repeaters", () => { + let n = 0; + let tick = 0; + const now = () => new Date(Date.UTC(2026, 10, 12, 9, 0, tick++)); + const make = (sql = ctx.sql()) => + new SiteGraphStore({ + sql, + blobs: ctx.blobs(), + now, + gates: GATES_ENFORCED, + fieldTypes: ENABLED, + }); + const store = () => make(); + const racing = (hook: () => Promise, when?: (q: string) => boolean) => + make(racingDriver(ctx.sql(), hook, when)); + const run = ( + site: Site, + op: FieldOperation, + extra: Record = {}, + on = store(), + ) => runFieldOperation(on, site.id, op, { ...actor, ...extra }); + + async function world( + events: FieldDefinition[], + extra: NonNullable = {}, + ): Promise { + n++; + const project = await store().createProject({ + name: "Ops", + slug: `refops-${n}`, + ownerId: person.id, + siteIds: [], + }); + return store().createSite({ + projectId: project.id, + name: "Ops", + slug: `refops-site-${n}`, + defaultLocale: "en", + locales: ["en", "tr"], + domains: [], + pageIds: [], + theme: { name: "blog" }, + collections: { events: { fields: events as [FieldDefinition] }, places: {}, ...extra }, + }); + } + const entry = (site: Site, collection: string, over: Partial = {}) => + store().createContentEntry({ + siteId: site.id, + collection, + slug: `${collection}-${Math.random().toString(36).slice(2, 8)}`, + locale: "en", + status: "draft", + title: `A ${collection}`, + body: { format: "markdown", value: "Body" }, + fields: {}, + tags: [], + author: person, + ...over, + } as never); + const place = (site: Site, title: string, over: Partial = {}) => + entry(site, "places", { title, slug: title.toLowerCase().replace(/ /g, "-"), ...over }); + const fresh = async (e: ContentEntry) => (await store().getContentEntry(e.id)) as ContentEntry; + const freshSite = async (site: Site) => (await store().getSite(site.id)) as Site; + const def = async (site: Site, key: string) => + ((await store().getSite(site.id)) as Site).collections?.events?.fields?.find( + (d) => d.key === key, + ); + const rows = async (sourceId: string) => + ( + await ctx + .sql() + .all<{ t: string; f: string; k: string; s: number }>( + "SELECT target_id AS t, field AS f, root_key AS k, shared AS s FROM entry_refs WHERE source_id = ? ORDER BY field, target_id", + [sourceId], + ) + ).map((r) => ({ ...r, s: Number(r.s) })); + const sharedOf = async (sourceId: string) => (await rows(sourceId)).map((r) => r.s); + + describe("conversion rows", () => { + it("text → reference and back: rows appear with phase B, and the revert restores the text", async () => { + const site = await world([text("venue")]); + const hall = await place(site, "Main Hall"); + const a = await entry(site, "events", { fields: { venue: "main hall" } }); + const b = await entry(site, "events", { fields: { venue: "Nowhere" } }); + const op: FieldOperation = { + op: "change_field_type", + collection: "events", + key: "venue", + type: "reference", + collections: ["places"], + }; + const dry = await run(site, op, { dryRun: true }); + expect(dry.report).toMatchObject({ converted: 1, unconvertible: 1 }); + expect(dry.report.refusal?.code).toBe("field_values_unconvertible"); + const done = await run(site, { ...op, onInvalid: "clear" }); + expect(done.done).toBe(true); + expect((await fresh(a)).fields).toEqual({ venue: hall.id }); + expect((await fresh(b)).fields).toEqual({}); + expect(await rows(a.id)).toEqual([{ t: hall.id, f: "venue", k: "venue", s: 1 }]); + const back = await revertFieldOperation(store(), site.id, done.operationId, actor); + expect(back).toMatchObject({ done: true, restored: 2, dropped: [] }); + expect((await fresh(a)).fields).toEqual({ venue: "main hall" }); + expect((await fresh(b)).fields).toEqual({ venue: "Nowhere" }); + expect(await rows(a.id)).toEqual([]); + }); + + it("reference → text: the default-locale title; rows go", async () => { + const site = await world([reference("venue", { multiple: true })]); + const hall = await place(site, "Main Hall"); + const room = await place(site, "Room"); + const e = await entry(site, "events", { fields: { venue: [hall.id, room.id] } }); + const done = await run(site, { + op: "change_field_type", + collection: "events", + key: "venue", + type: "text", + }); + expect(done.done).toBe(true); + expect((await fresh(e)).fields).toEqual({ venue: "Main Hall, Room" }); + expect(await rows(e.id)).toEqual([]); + }); + + it("reference single ↔ multiple, keeping the rows", async () => { + const site = await world([reference("venue")]); + const hall = await place(site, "Main Hall"); + const e = await entry(site, "events", { fields: { venue: hall.id } }); + const op: FieldOperation = { + op: "change_field_type", + collection: "events", + key: "venue", + type: "reference", + multiple: true, + }; + expect((await run(site, op)).done).toBe(true); + expect((await fresh(e)).fields).toEqual({ venue: [hall.id] }); + expect(await def(site, "venue")).toMatchObject({ multiple: true, collections: ["places"] }); + expect(await rows(e.id)).toEqual([{ t: hall.id, f: "venue", k: "venue", s: 1 }]); + expect((await run(site, { ...op, multiple: false })).done).toBe(true); + expect((await fresh(e)).fields).toEqual({ venue: hall.id }); + // same shape twice is no type change; narrowing the collections is refused + expect(await code(run(site, { ...op, multiple: false }))).toBe("invalid_fields"); + expect(await code(run(site, { ...op, collections: ["events"] }))).toBe("invalid_fields"); + }); + + it("multiline text ↔ repeater, one item per line", async () => { + const site = await world([text("steps", { multiline: true })]); + const e = await entry(site, "events", { fields: { steps: "Mix\n\nBake" } }); + const op: FieldOperation = { + op: "change_field_type", + collection: "events", + key: "steps", + type: "repeater", + subFields: [{ key: "line", type: "text", labels: L("Line") }], + maxItems: 10, + }; + // a target definition that would not pass checkFieldDefinitions is refused by the dry run + expect(await code(run(site, { ...op, itemLabel: "nope" }, { dryRun: true }))).toBe( + "unknown_field", + ); + const done = await run(site, op); + expect(done.done).toBe(true); + expect((await fresh(e)).fields).toEqual({ steps: [{ line: "Mix" }, { line: "Bake" }] }); + const back = await run(site, { + op: "change_field_type", + collection: "events", + key: "steps", + type: "text", + }); + expect(back.done).toBe(true); + expect((await fresh(e)).fields).toEqual({ steps: "Mix\nBake" }); + expect(await def(site, "steps")).toMatchObject({ type: "text", multiline: true }); + }); + + it("reference (multiple) ↔ repeater with one reference sub-field; rows follow the path", async () => { + const site = await world([reference("hosts", { multiple: true })]); + const hall = await place(site, "Main Hall"); + const e = await entry(site, "events", { fields: { hosts: [hall.id] } }); + const part = await run( + site, + { + op: "change_field_type", + collection: "events", + key: "hosts", + type: "repeater", + subFields: [ + { key: "person", type: "reference", labels: L("Person"), collections: ["places"] }, + ], + }, + { budget: { entries: 0 } }, + ); + // a list of ids and a list of items are both arrays: still pending until rewritten + expect((await fieldOperationStatus(store(), site.id, part.operationId)).pending).toBe(1); + const done = await continueFieldOperation(store(), site.id, part.operationId, actor); + expect(done.done).toBe(true); + expect((await fresh(e)).fields).toEqual({ hosts: [{ person: hall.id }] }); + expect(await rows(e.id)).toEqual([{ t: hall.id, f: "hosts.person", k: "hosts", s: 1 }]); + const back = await revertFieldOperation(store(), site.id, done.operationId, actor); + expect(back.done).toBe(true); + expect((await fresh(e)).fields).toEqual({ hosts: [hall.id] }); + expect(await rows(e.id)).toEqual([{ t: hall.id, f: "hosts", k: "hosts", s: 1 }]); + }); + }); + + describe("sub", () => { + it("renames a sub-field in every item; rows and asset rows follow", async () => { + const site = await world([SPEAKERS]); + const hall = await place(site, "Main Hall"); + const { asset } = await store().ingestAsset({ + siteId: site.id, + bytes: PNG_1X1, + filename: "p.png", + uploadedBy: person, + source: { kind: "upload" }, + }); + const e = await entry(site, "events", { + fields: { speakers: [{ person: hall.id, photo: asset.id }, { name: "Ada" }] }, + }); + const assets = async () => + ( + await ctx + .sql() + .all<{ field: string }>("SELECT field FROM asset_refs WHERE node_id = ?", [e.id]) + ).map((r) => r.field); + const before = await assets(); + expect(before).toHaveLength(1); + for (const [from, to] of [ + ["person", "host"], + ["photo", "portrait"], + ] as const) { + const done = await run(site, { + op: "rename_field", + collection: "events", + from: "speakers", + sub: from, + to, + }); + expect(done.done).toBe(true); + } + expect((await fresh(e)).fields).toEqual({ + speakers: [{ host: hall.id, portrait: asset.id }, { name: "Ada" }], + }); + expect(await rows(e.id)).toEqual([{ t: hall.id, f: "speakers.host", k: "speakers", s: 1 }]); + expect(await assets()).toEqual([before[0]?.replace("photo", "portrait")]); + // the asset stays in use + expect(await code(store().deleteAsset(asset))).toBe("in_use"); + // a key the repeater already has, or a sub-field of a non-repeater, is refused + const dup = { + op: "rename_field", + collection: "events", + from: "speakers", + sub: "name", + to: "seats", + } as const; + expect(await code(run(site, dup))).toBe("field_key_duplicate"); + }); + + it("converts a sub-field in every item, and removes one (an item left empty goes)", async () => { + const site = await world([SPEAKERS]); + const hall = await place(site, "Main Hall"); + const a = await entry(site, "events", { + fields: { speakers: [{ name: "Main Hall", seats: "12" }, { seats: "3" }] }, + }); + const host = await run(site, { + op: "change_field_type", + collection: "events", + key: "speakers", + sub: "name", + type: "reference", + collections: ["places"], + }); + expect(host.done).toBe(true); + expect((await fresh(a)).fields).toEqual({ + speakers: [{ name: hall.id, seats: "12" }, { seats: "3" }], + }); + expect(await rows(a.id)).toContainEqual({ + t: hall.id, + f: "speakers.name", + k: "speakers", + s: 1, + }); + const removed = await run(site, { + op: "remove_field", + collection: "events", + key: "speakers", + sub: "seats", + }); + expect(removed.done).toBe(true); + expect((await fresh(a)).fields).toEqual({ speakers: [{ name: hall.id }] }); + const speakers = await def(site, "speakers"); + expect(speakers?.type === "repeater" && speakers.fields.map((s) => s.key)).toEqual([ + "person", + "name", + "photo", + ]); + // sub-fields share the repeater's localized + expect( + await code( + run(site, { + op: "set_field_localized", + collection: "events", + key: "speakers", + sub: "name", + localized: true, + }), + ), + ).toBe("invalid_fields"); + }); + }); + + describe("request checks", () => { + it("a sub-field reference → text is not held to another sub-field's collections", async () => { + const site = await world([ + { + ...SPEAKERS, + fields: [ + ...(SPEAKERS.type === "repeater" ? SPEAKERS.fields : []), + { key: "room", type: "reference", labels: L("Room"), collections: ["events"] }, + ], + } as FieldDefinition, + ]); + const hall = await place(site, "Main Hall"); + const e = await entry(site, "events", { fields: { speakers: [{ person: hall.id }] } }); + const done = await run(site, { + op: "change_field_type", + collection: "events", + key: "speakers", + sub: "person", + type: "text", + }); + expect(done.done).toBe(true); + expect((await fresh(e)).fields).toEqual({ speakers: [{ person: "Main Hall" }] }); + }); + + it("multiple is ignored where it does not apply, as before", async () => { + const site = await world([text("seats")]); + const e = await entry(site, "events", { fields: { seats: "12" } }); + const done = await run(site, { + op: "change_field_type", + collection: "events", + key: "seats", + type: "number", + multiple: false, + }); + expect(done.done).toBe(true); + expect((await fresh(e)).fields).toEqual({ seats: 12 }); + }); + }); + + describe("phase B's target guard (§2.6)", () => { + it("a target archived during phase B is never written: reported, then continued with clear", async () => { + const site = await world([text("venue")]); + const hall = await place(site, "Main Hall"); + const e = await entry(site, "events", { fields: { venue: "Main Hall" } }); + const op: FieldOperation = { + op: "change_field_type", + collection: "events", + key: "venue", + type: "reference", + collections: ["places"], + }; + let deleted = ""; + const raced = racing(async () => { + deleted = await code(store().deleteContentEntry(await fresh(hall))); + await store().updateContentEntry({ ...(await fresh(hall)), status: "archived" }); + }, isAsset); + const stuck = await run(site, op, {}, raced); + expect(deleted).toBe("field_operation_running"); + // the guard skipped it; the next pass read the maps again, and nothing matches now + expect(stuck).toMatchObject({ done: false, skipped: [] }); + expect(stuck.unconvertible).toEqual([{ entryId: e.id, reason: expect.any(String) }]); + expect((await fresh(e)).fields).toEqual({ venue: "Main Hall" }); + expect(await code(store().deleteContentEntry(await fresh(hall)))).toBe( + "field_operation_running", + ); + const cleared = await continueFieldOperation(store(), site.id, stuck.operationId, { + ...actor, + onInvalid: "clear", + }); + expect(cleared.done).toBe(true); + expect((await fresh(e)).fields).toEqual({}); + expect(await rows(e.id)).toEqual([]); + expect(await code(store().deleteContentEntry(await fresh(hall)))).toBe("ok"); + }); + + it("a second entry with the same title during the run: single and multiple, then cleared", async () => { + for (const multiple of [false, true]) { + const site = await world([text("venue")]); + await place(site, "Main Hall"); + const e = await entry(site, "events", { fields: { venue: "Main Hall" } }); + const op: FieldOperation = { + op: "change_field_type", + collection: "events", + key: "venue", + type: "reference", + collections: ["places"], + multiple, + }; + const started = await run(site, op, { budget: { entries: 0 } }); + await place(site, "Main Hall", { slug: "main-hall-2" }); + const stuck = await continueFieldOperation(store(), site.id, started.operationId, actor); + expect(stuck.done).toBe(false); + expect(stuck.unconvertible.map((u) => u.entryId)).toEqual([e.id]); + expect(stuck.unconvertible[0]?.reason).toMatch(/matches 2 entries/); + const done = await continueFieldOperation(store(), site.id, started.operationId, { + ...actor, + onInvalid: "clear", + }); + expect(done.done).toBe(true); + expect((await fresh(e)).fields).toEqual({}); + } + }); + + it("reference → text with a target deleted mid-run: the id is pending, then cleared", async () => { + const site = await world([reference("venue")]); + const hall = await place(site, "Main Hall"); + const e = await entry(site, "events", { fields: { venue: hall.id } }); + const op: FieldOperation = { + op: "change_field_type", + collection: "events", + key: "venue", + type: "text", + }; + const started = await run(site, op, { budget: { entries: 0 } }); + expect(await code(store().deleteContentEntry(await fresh(hall)))).toBe("ok"); + const stuck = await continueFieldOperation(store(), site.id, started.operationId, actor); + expect(stuck.done).toBe(false); + expect(stuck.unconvertible.map((u) => u.entryId)).toEqual([e.id]); + expect((await fresh(e)).fields).toEqual({ venue: hall.id }); + const done = await continueFieldOperation(store(), site.id, started.operationId, { + ...actor, + onInvalid: "clear", + }); + expect(done.done).toBe(true); + expect((await fresh(e)).fields).toEqual({}); + expect(await rows(e.id)).toEqual([]); + }); + + it("a person's edit of another field mid-run converts the value through the table", async () => { + const site = await world([reference("venue"), text("note")]); + const hall = await place(site, "Main Hall"); + const e = await entry(site, "events", { fields: { venue: hall.id } }); + const toText: FieldOperation = { + op: "change_field_type", + collection: "events", + key: "venue", + type: "text", + }; + const a = await run(site, toText, { budget: { entries: 0 } }); + // the stored id is valid text, yet it is not left as it is + const edited = await store().updateContentEntry({ + ...(await fresh(e)), + fields: { venue: hall.id, note: "n" }, + }); + expect(edited.fields).toEqual({ venue: "Main Hall", note: "n" }); + expect(await rows(e.id)).toEqual([]); + expect((await continueFieldOperation(store(), site.id, a.operationId, actor)).done).toBe( + true, + ); + // and back: id-shaped text is no reference until the table says so + const toRef: FieldOperation = { + ...toText, + type: "reference", + collections: ["places"], + } as FieldOperation; + const b = await run(site, toRef, { budget: { entries: 0 } }); + const again = await store().updateContentEntry({ ...(await fresh(e)), title: "Edited" }); + expect(again.fields).toEqual({ venue: hall.id, note: "n" }); + expect(await rows(e.id)).toEqual([{ t: hall.id, f: "venue", k: "venue", s: 1 }]); + expect((await continueFieldOperation(store(), site.id, b.operationId, actor)).done).toBe( + true, + ); + }); + + it("a target deleted while ids are carried over stays as stored, rows kept", async () => { + const site = await world([reference("hosts", { multiple: true })]); + const hall = await place(site, "Main Hall"); + const gone = await place(site, "Gone"); + const e = await entry(site, "events", { fields: { hosts: [hall.id, gone.id] } }); + const raced = racing(async () => { + // a draft referrer locks nothing, so this delete goes through + expect(await code(store().deleteContentEntry(await fresh(gone)))).toBe("ok"); + }, isAsset); + const done = await run( + site, + { + op: "change_field_type", + collection: "events", + key: "hosts", + type: "repeater", + subFields: [ + { key: "person", type: "reference", labels: L("Person"), collections: ["places"] }, + ], + }, + {}, + raced, + ); + expect(done.done).toBe(true); + expect((await fresh(e)).fields).toEqual({ + hosts: [{ person: hall.id }, { person: gone.id }], + }); + expect((await rows(e.id)).map((r) => r.t).sort()).toEqual([hall.id, gone.id].sort()); + }); + + it("resumes after its budget; a person's write meanwhile converts the same way", async () => { + const site = await world([text("venue")]); + const hall = await place(site, "Main Hall"); + const made: ContentEntry[] = []; + for (let i = 0; i < 3; i++) + made.push(await entry(site, "events", { fields: { venue: "Main Hall" } })); + const op: FieldOperation = { + op: "change_field_type", + collection: "events", + key: "venue", + type: "reference", + collections: ["places"], + }; + const part = await run(site, op, { budget: { entries: 1 } }); + expect(part).toMatchObject({ done: false, rewritten: 1 }); + const last = made[2] as ContentEntry; + expect((await fresh(last)).fields).toEqual({ venue: "Main Hall" }); + const edited = await store().updateContentEntry({ + ...(await fresh(last)), + title: "Edited", + }); + expect(edited.fields).toEqual({ venue: hall.id }); + expect(await rows(last.id)).toEqual([{ t: hall.id, f: "venue", k: "venue", s: 1 }]); + const rest = await continueFieldOperation(store(), site.id, part.operationId, actor); + expect(rest).toMatchObject({ done: true, resumed: true }); + for (const m of made) expect((await fresh(m)).fields).toEqual({ venue: hall.id }); + }); + }); + + describe("revert", () => { + it("drops a restored reference whose group has no member left, and reports it", async () => { + const site = await world([reference("venue", { multiple: true })]); + const hall = await place(site, "Main Hall"); + const room = await place(site, "Room"); + const e = await entry(site, "events", { fields: { venue: [hall.id, room.id] } }); + const done = await run(site, { + op: "change_field_type", + collection: "events", + key: "venue", + type: "text", + }); + expect((await fresh(e)).fields).toEqual({ venue: "Main Hall, Room" }); + await store().deleteContentEntry(await fresh(room)); + const back = await revertFieldOperation(store(), site.id, done.operationId, actor); + expect(back).toMatchObject({ done: true, restored: 1 }); + expect(back.dropped).toEqual([{ entryId: e.id, field: "venue", targetId: room.id }]); + expect((await fresh(e)).fields).toEqual({ venue: [hall.id] }); + expect(await rows(e.id)).toEqual([{ t: hall.id, f: "venue", k: "venue", s: 1 }]); + }); + }); + + describe("already converted (round 2: one rule, rows decide what is pending)", () => { + const toRef = (over: Partial> = {}) => + ({ + op: "change_field_type", + collection: "events", + key: "venue", + type: "reference", + collections: ["places"], + ...over, + }) as FieldOperation; + + it("text → multiple reference over several calls with budget 1, with and without clear", async () => { + for (const onInvalid of [undefined, "clear" as const]) { + const site = await world([text("venue")]); + const hall = await place(site, "Main Hall"); + const made: ContentEntry[] = []; + for (let i = 0; i < 3; i++) + made.push(await entry(site, "events", { fields: { venue: "Main Hall" } })); + let r = await run(site, toRef({ multiple: true }), { budget: { entries: 1 } }); + for (let i = 0; i < 5 && !r.done; i++) { + expect(r.unconvertible).toEqual([]); + r = await continueFieldOperation(store(), site.id, r.operationId, { + ...actor, + budget: { entries: 1 }, + onInvalid, + }); + } + expect(r.done).toBe(true); + for (const m of made) { + expect((await fresh(m)).fields).toEqual({ venue: [hall.id] }); + expect(await rows(m.id)).toEqual([{ t: hall.id, f: "venue", k: "venue", s: 1 }]); + } + } + }); + + it("convert, archive the target, continue with clear: the reference and its row stay", async () => { + const site = await world([text("venue")]); + const hall = await place(site, "Main Hall"); + const a = await entry(site, "events", { fields: { venue: "Main Hall" } }); + const b = await entry(site, "events", { fields: { venue: "Main Hall" } }); + const part = await run(site, toRef(), { budget: { entries: 1 } }); + expect((await fresh(a)).fields).toEqual({ venue: hall.id }); + await store().updateContentEntry({ ...(await fresh(hall)), status: "archived" }); + const done = await continueFieldOperation(store(), site.id, part.operationId, { + ...actor, + onInvalid: "clear", + }); + expect(done.done).toBe(true); + expect((await fresh(a)).fields).toEqual({ venue: hall.id }); + expect(await rows(a.id)).toEqual([{ t: hall.id, f: "venue", k: "venue", s: 1 }]); + // the one still unconverted matches no usable entry now: cleared + expect((await fresh(b)).fields).toEqual({}); + }); + + it("a pristine legacy id is checked, gets its row and locks; the revert gives the text back", async () => { + const site = await world([text("venue")]); + const hall = await place(site, "Main Hall"); + const e = await entry(site, "events", { status: "published", fields: { venue: hall.id } }); + const done = await run(site, toRef()); + expect(done).toMatchObject({ done: true, rewritten: 1 }); + expect(await rows(e.id)).toEqual([{ t: hall.id, f: "venue", k: "venue", s: 1 }]); + expect(await code(store().deleteContentEntry(await fresh(hall)))).toBe("entry_referenced"); + const back = await revertFieldOperation(store(), site.id, done.operationId, actor); + expect(back.done).toBe(true); + expect((await def(site, "venue"))?.type).toBe("text"); + expect((await fresh(e)).fields).toEqual({ venue: hall.id }); + expect(await rows(e.id)).toEqual([]); + }); + + it("reference → text where the title is id-shaped finishes", async () => { + const site = await world([reference("venue")]); + const odd = await place(site, "entry_2025report", { slug: "report-2025" }); + const e = await entry(site, "events", { fields: { venue: odd.id } }); + const done = await run(site, { + op: "change_field_type", + collection: "events", + key: "venue", + type: "text", + }); + expect(done.done).toBe(true); + expect((await fresh(e)).fields).toEqual({ venue: "entry_2025report" }); + expect(await rows(e.id)).toEqual([]); + }); + + it("a requiredToPublish field mid-operation: editing a published entry succeeds", async () => { + const site = await world([text("venue", { requiredToPublish: true })]); + const hall = await place(site, "Main Hall"); + const a = await entry(site, "events", { + status: "published", + fields: { venue: "Main Hall" }, + }); + await entry(site, "events", { status: "published", fields: { venue: "Main Hall" } }); + const part = await run(site, toRef(), { budget: { entries: 1 } }); + expect(part.done).toBe(false); + const edited = await store().updateContentEntry({ ...(await fresh(a)), title: "Edited" }); + expect(edited.fields).toEqual({ venue: hall.id }); + expect((await continueFieldOperation(store(), site.id, part.operationId, actor)).done).toBe( + true, + ); + }); + + it("a sub over-cap clear drops only the converted sub-values; the other sub-fields stay", async () => { + const site = await world([ + reference("hosts", { multiple: true, maxItems: 50 }), + reference("lead"), + { ...SPEAKERS, maxItems: 50 } as FieldDefinition, + ]); + const places: ContentEntry[] = []; + for (let i = 0; i < 50; i++) + places.push(await place(site, i === 0 ? "Main Hall" : `Place ${i}`)); + const hall = places[0] as ContentEntry; + const items = Array.from({ length: 50 }, (_, i) => ({ name: "Main Hall", seats: `s${i}` })); + const e = await entry(site, "events", { + fields: { hosts: places.map((p) => p.id), lead: hall.id, speakers: items }, + }); + const op: FieldOperation = { + op: "change_field_type", + collection: "events", + key: "speakers", + sub: "name", + type: "reference", + collections: ["places"], + }; + const dry = await run(site, op, { dryRun: true }); + expect(dry.report.unconvertible).toBe(1); + const done = await run(site, { ...op, onInvalid: "clear" }); + expect(done.done).toBe(true); + const after = (await fresh(e)).fields; + expect(after?.hosts).toHaveLength(50); + expect(after?.speakers).toEqual(items.map(({ seats }) => ({ seats }))); + }); + + it("a sub revert keeps a sub-field added later", async () => { + const site = await world([SPEAKERS]); + const e = await entry(site, "events", { fields: { speakers: [{ name: "Ada" }] } }); + const done = await run(site, { + op: "rename_field", + collection: "events", + from: "speakers", + sub: "name", + to: "fullName", + }); + const now = (await store().getSite(site.id)) as Site; + const speakers = now.collections?.events?.fields?.find((d) => d.key === "speakers"); + const bio = { key: "bio", type: "text", labels: L("Bio") } as const; + await store().updateSite({ + ...now, + collections: { + ...now.collections, + events: { + fields: [ + { + ...speakers, + fields: [...(speakers?.type === "repeater" ? speakers.fields : []), bio], + }, + ], + }, + }, + } as unknown as Site); + const back = await revertFieldOperation(store(), site.id, done.operationId, actor); + expect(back.done).toBe(true); + const restored = await def(site, "speakers"); + expect(restored?.type === "repeater" && restored.fields.map((f) => f.key)).toEqual([ + "person", + "name", + "seats", + "photo", + "bio", + ]); + expect((await fresh(e)).fields).toEqual({ speakers: [{ name: "Ada" }] }); + }); + }); + + describe("decided by the rows (round 3: other writes do not change what is converted)", () => { + const toRef = (over: Partial> = {}) => + ({ + op: "change_field_type", + collection: "events", + key: "venue", + type: "reference", + collections: ["places"], + ...over, + }) as FieldOperation; + const title = async (e: ContentEntry, t = "Edited") => + store().updateContentEntry({ ...(await fresh(e)), title: t }); + + it("multiple: a legacy id with a title edit before phase B is converted, with a row", async () => { + const site = await world([text("venue")]); + const hall = await place(site, "Main Hall"); + const e = await entry(site, "events", { fields: { venue: hall.id } }); + const part = await run(site, toRef({ multiple: true }), { budget: { entries: 0 } }); + await title(e); + const done = await continueFieldOperation(store(), site.id, part.operationId, actor); + expect(done).toMatchObject({ done: true, unconvertible: [] }); + expect((await fresh(e)).fields).toEqual({ venue: [hall.id] }); + expect(await rows(e.id)).toEqual([{ t: hall.id, f: "venue", k: "venue", s: 1 }]); + }); + + it("single: a translation's legacy id stays decided by rows when its source is forwarded", async () => { + const site = await world([text("venue", { localized: true }), text("note")]); + const hall = await place(site, "Main Hall"); + const original = await entry(site, "events", { fields: { venue: hall.id, note: "a" } }); + const member = (await store().translationGroup(site, original.id)).members.find( + (m) => m.id === original.id, + ); + const tr = await entry(site, "events", { + locale: "tr", + slug: original.slug, + fields: { venue: hall.id }, + translation: { + group: original.id, + method: "human", + source: { + id: original.id, + locale: "en", + rev: original.rev, + hash: member?.hash as string, + }, + }, + }); + const part = await run(site, toRef(), { budget: { entries: 0 } }); + // a non-localized change forwards the translation's source: its hash moves, its venue not + const before = (await fresh(tr)).translation?.source?.hash; + await store().updateContentEntry({ + ...(await fresh(original)), + fields: { venue: hall.id, note: "b" }, + }); + expect((await fresh(tr)).translation?.source?.hash).not.toBe(before); + const done = await continueFieldOperation(store(), site.id, part.operationId, actor); + expect(done.done).toBe(true); + expect((await fresh(tr)).fields).toEqual({ venue: hall.id }); + expect(await rows(tr.id)).toEqual([{ t: hall.id, f: "venue", k: "venue", s: 2 }]); + }); + + it("single: a legacy id whose group was archived, then a title edit: reported with no row, then cleared", async () => { + const site = await world([text("venue")]); + const gone = await place(site, "Gone"); + const e = await entry(site, "events", { fields: { venue: gone.id } }); + const part = await run(site, toRef(), { budget: { entries: 0 } }); + await store().updateContentEntry({ ...(await fresh(gone)), status: "archived" }); + const edited = await title(e); + expect(edited.fields).toEqual({ venue: gone.id }); + expect(await rows(e.id)).toEqual([]); + const stuck = await continueFieldOperation(store(), site.id, part.operationId, actor); + expect(stuck.done).toBe(false); + expect(stuck.unconvertible.map((u) => u.entryId)).toEqual([e.id]); + expect(await rows(e.id)).toEqual([]); + const done = await continueFieldOperation(store(), site.id, part.operationId, { + ...actor, + onInvalid: "clear", + }); + expect(done.done).toBe(true); + expect((await fresh(e)).fields).toEqual({}); + }); + + it("a legacy id of the entry's own group, then a title edit: it fails and gets no row", async () => { + const site = await world([text("venue")]); + const e = await entry(site, "events", { fields: {} }); + await store().updateContentEntry({ ...(await fresh(e)), fields: { venue: e.id } }); + const op = toRef({ collections: ["places", "events"], onInvalid: "clear" }); + const part = await run(site, op, { budget: { entries: 0 } }); + const edited = await title(e); + expect(edited.fields).toEqual({ venue: e.id }); + expect(await rows(e.id)).toEqual([]); + const done = await continueFieldOperation(store(), site.id, part.operationId, actor); + expect(done.done).toBe(true); + expect((await fresh(e)).fields).toEqual({}); + expect(await rows(e.id)).toEqual([]); + }); + + it("the revert of text → reference with an edited entry whose target is gone: reported, then cleared", async () => { + const site = await world([text("venue")]); + const hall = await place(site, "Main Hall"); + const e = await entry(site, "events", { fields: { venue: "Main Hall" } }); + const done = await run(site, toRef()); + await title(e, "Edited after"); + expect(await code(store().deleteContentEntry(await fresh(hall)))).toBe("ok"); + const stuck = await revertFieldOperation(store(), site.id, done.operationId, actor); + expect(stuck.done).toBe(false); + expect(stuck.unconvertible.map((u) => u.entryId)).toEqual([e.id]); + const back = await revertFieldOperation(store(), site.id, done.operationId, { + ...actor, + onInvalid: "clear", + }); + expect(back.done).toBe(true); + expect((await fresh(e)).fields).toEqual({}); + expect(await rows(e.id)).toEqual([]); + }); + + it("a term merge touching an entry not converted yet: it is converted", async () => { + const site = await world([text("venue")]); + const hall = await place(site, "Main Hall"); + const [, tag] = await store().createDefaultTaxonomies(await freshSite(site)); + const term = (slug: string) => + store().createTerm({ + siteId: site.id, + taxonomyId: (tag as { id: string }).id, + slug, + status: "active", + visibility: "public", + labels: { en: { name: slug } }, + }); + const [a, b] = [await term("old"), await term("new")]; + const e = await entry(site, "events", { + fields: { venue: hall.id }, + terms: { tag: { ids: [a.id] } }, + } as never); + const part = await run(site, toRef(), { budget: { entries: 0 } }); + const hash = (await fresh(e)).rev; + await store().mergeTerms([a], b); + expect((await fresh(e)).rev).not.toBe(hash); + const doneOp = await continueFieldOperation(store(), site.id, part.operationId, actor); + expect(doneOp.done).toBe(true); + expect((await fresh(e)).fields).toEqual({ venue: hall.id }); + expect(await rows(e.id)).toEqual([{ t: hall.id, f: "venue", k: "venue", s: 1 }]); + }); + }); + + describe("per value (round 4)", () => { + const toRef = (over: Partial> = {}) => + ({ + op: "change_field_type", + collection: "events", + key: "venue", + type: "reference", + collections: ["places"], + ...over, + }) as FieldOperation; + + it("a sub item written as text next to a held reference item stays text", async () => { + const site = await world([SPEAKERS]); + const hall = await place(site, "Main Hall"); + const e = await entry(site, "events", { fields: { speakers: [{ person: hall.id }] } }); + const op: FieldOperation = { + op: "change_field_type", + collection: "events", + key: "speakers", + sub: "person", + type: "text", + }; + const part = await run(site, op, { budget: { entries: 0 } }); + expect(await code(store().deleteContentEntry(await fresh(hall)))).toBe("ok"); + const written = await store().updateContentEntry({ + ...(await fresh(e)), + fields: { speakers: [{ person: hall.id }, { person: "Keynote" }] }, + }); + expect(written.fields).toEqual({ speakers: [{ person: hall.id }, { person: "Keynote" }] }); + // the held reference keeps its row; the text item has none + expect(await rows(e.id)).toEqual([ + { t: hall.id, f: "speakers.person", k: "speakers", s: 1 }, + ]); + const stuck = await continueFieldOperation(store(), site.id, part.operationId, actor); + expect(stuck.unconvertible.map((u) => u.entryId)).toEqual([e.id]); + const done = await continueFieldOperation(store(), site.id, part.operationId, { + ...actor, + onInvalid: "clear", + }); + expect(done.done).toBe(true); + expect((await fresh(e)).fields).toEqual({ speakers: [{ person: "Keynote" }] }); + expect(await rows(e.id)).toEqual([]); + }); + + it("held per value: one archived legacy item is reported, a new usable item gets its row", async () => { + const site = await world([SPEAKERS]); + const hall = await place(site, "Main Hall"); + const gone = await place(site, "Gone"); + const e = await entry(site, "events", { fields: { speakers: [{ name: gone.id }] } }); + const op: FieldOperation = { + op: "change_field_type", + collection: "events", + key: "speakers", + sub: "name", + type: "reference", + collections: ["places"], + }; + const part = await run(site, op, { budget: { entries: 0 } }); + await store().updateContentEntry({ ...(await fresh(gone)), status: "archived" }); + await store().updateContentEntry({ + ...(await fresh(e)), + fields: { speakers: [{ name: gone.id }, { name: hall.id }] }, + }); + expect(await rows(e.id)).toEqual([{ t: hall.id, f: "speakers.name", k: "speakers", s: 1 }]); + const stuck = await continueFieldOperation(store(), site.id, part.operationId, actor); + expect(stuck.done).toBe(false); + expect(stuck.unconvertible.map((u) => u.entryId)).toEqual([e.id]); + expect(await rows(e.id)).toEqual([{ t: hall.id, f: "speakers.name", k: "speakers", s: 1 }]); + }); + + it("a person writing [Y, Z] over an archived legacy Z is refused for Z", async () => { + const site = await world([text("venue")]); + const y = await place(site, "Y place"); + const z = await place(site, "Z place"); + const e = await entry(site, "events", { fields: { venue: z.id } }); + await run(site, toRef({ multiple: true }), { budget: { entries: 0 } }); + await store().updateContentEntry({ ...(await fresh(z)), status: "archived" }); + expect( + await code( + store().updateContentEntry({ ...(await fresh(e)), fields: { venue: [y.id, z.id] } }), + ), + ).toBe("unknown_entry"); + expect(await rows(e.id)).toEqual([]); + }); + + it("a mid-operation import rebuilds rows through the marker, then the operation continues", async () => { + // text → reference: an archived legacy id gets no row from the rebuild, and is reported + const a = await world([text("venue")]); + const gone = await place(a, "Gone"); + const legacy = await entry(a, "events", { fields: { venue: gone.id } }); + const toRefOp = await run(a, toRef(), { budget: { entries: 0 } }); + await store().updateContentEntry({ ...(await fresh(gone)), status: "archived" }); + const storage = { sql: ctx.sql(), blobs: ctx.blobs() }; + await importSite(storage, await exportSite(storage, a.id), { + mode: "replace", + fieldTypes: ENABLED, + }); + expect(await rows(legacy.id)).toEqual([]); + const stuck = await continueFieldOperation(store(), a.id, toRefOp.operationId, actor); + expect(stuck.unconvertible.map((u) => u.entryId)).toEqual([legacy.id]); + // reference → text: the rebuild keeps rows for ids, which still convert to titles + const b = await world([reference("venue")]); + const hall = await place(b, "Main Hall"); + const e = await entry(b, "events", { fields: { venue: hall.id } }); + const toText = await run( + b, + { op: "change_field_type", collection: "events", key: "venue", type: "text" }, + { budget: { entries: 0 } }, + ); + await importSite(storage, await exportSite(storage, b.id), { + mode: "replace", + fieldTypes: ENABLED, + }); + expect(await rows(e.id)).toEqual([{ t: hall.id, f: "venue", k: "venue", s: 1 }]); + const done = await continueFieldOperation(store(), b.id, toText.operationId, actor); + expect(done.done).toBe(true); + expect((await fresh(e)).fields).toEqual({ venue: "Main Hall" }); + expect(await rows(e.id)).toEqual([]); + }); + }); + + describe("round 5 edges", () => { + const toRef = (over: Partial> = {}) => + ({ + op: "change_field_type", + collection: "events", + key: "venue", + type: "reference", + collections: ["places"], + ...over, + }) as FieldOperation; + const storage = () => ({ sql: ctx.sql(), blobs: ctx.blobs() }); + const reimport = async (site: Site) => + importSite(storage(), await exportSite(storage(), site.id), { + mode: "replace", + fieldTypes: ENABLED, + }); + + it("a rebuild keeps an accepted list's rows; only a bare string is checked again", async () => { + const site = await world([text("venue")]); + const hall = await place(site, "Main Hall"); + const a = await entry(site, "events", { fields: { venue: "Main Hall" } }); + const b = await entry(site, "events", { fields: { venue: "Main Hall" } }); + const part = await run(site, toRef({ multiple: true }), { budget: { entries: 1 } }); + expect((await fresh(a)).fields).toEqual({ venue: [hall.id] }); + await store().updateContentEntry({ ...(await fresh(hall)), status: "archived" }); + await reimport(site); + expect(await rows(a.id)).toEqual([{ t: hall.id, f: "venue", k: "venue", s: 1 }]); + const stuck = await continueFieldOperation(store(), site.id, part.operationId, actor); + expect(stuck.unconvertible.map((u) => u.entryId)).toEqual([b.id]); + expect((await fresh(a)).fields).toEqual({ venue: [hall.id] }); + expect(await rows(a.id)).toEqual([{ t: hall.id, f: "venue", k: "venue", s: 1 }]); + }); + + it("a rebuild keeps rows for a deleted target converting to text; phase B reports it", async () => { + const site = await world([reference("venue")]); + const gone = await place(site, "Gone"); + const e = await entry(site, "events", { fields: { venue: gone.id } }); + const part = await run( + site, + { op: "change_field_type", collection: "events", key: "venue", type: "text" }, + { budget: { entries: 0 } }, + ); + expect(await code(store().deleteContentEntry(await fresh(gone)))).toBe("ok"); + await reimport(site); + expect(await rows(e.id)).toEqual([{ t: gone.id, f: "venue", k: "venue", s: 1 }]); + const stuck = await continueFieldOperation(store(), site.id, part.operationId, actor); + expect(stuck.unconvertible.map((u) => u.entryId)).toEqual([e.id]); + expect((await fresh(e)).fields).toEqual({ venue: gone.id }); + }); + + it("a person writing their own group id during text → reference is refused", async () => { + const site = await world([text("venue")]); + const e = await entry(site, "events", { fields: { venue: "x" } }); + await run(site, toRef({ collections: ["places", "events"], onInvalid: "clear" }), { + budget: { entries: 0 }, + }); + expect( + await code(store().updateContentEntry({ ...(await fresh(e)), fields: { venue: e.id } })), + ).toBe("invalid_fields"); + expect(await rows(e.id)).toEqual([]); + }); + + it("an id a new item adds is guarded, even when an untouched failing item holds it", async () => { + const site = await world([SPEAKERS]); + const z = await place(site, "Z place"); + const e = await entry(site, "events", { fields: { speakers: [{ name: z.id }] } }); + const op: FieldOperation = { + op: "change_field_type", + collection: "events", + key: "speakers", + sub: "name", + type: "reference", + collections: ["places"], + }; + await run(site, op, { budget: { entries: 0 } }); + await store().updateContentEntry({ ...(await fresh(z)), status: "archived" }); + expect( + await code( + store().updateContentEntry({ + ...(await fresh(e)), + fields: { speakers: [{ name: z.id }, { name: z.id, seats: "2" }] }, + }), + ), + ).toBe("unknown_entry"); + // the untouched failing item alone still writes, kept as it is + const edited = await store().updateContentEntry({ ...(await fresh(e)), title: "Edited" }); + expect(edited.fields).toEqual({ speakers: [{ name: z.id }] }); + expect(await rows(e.id)).toEqual([]); + }); + + it("a move into a converting collection does not hold the old collection's rows", async () => { + const site = await world([text("venue")], { talks: { fields: [reference("venue")] } }); + const gone = await place(site, "Gone"); + const t = await entry(site, "talks", { fields: { venue: gone.id } }); + expect(await rows(t.id)).toEqual([{ t: gone.id, f: "venue", k: "venue", s: 1 }]); + await run(site, toRef(), { budget: { entries: 0 } }); + await store().updateContentEntry({ ...(await fresh(gone)), status: "archived" }); + expect( + await code(store().updateContentEntry({ ...(await fresh(t)), collection: "events" })), + ).toBe("unknown_entry"); + }); + + it("phase B does not re-check a sub-value it accepted when another item converts later", async () => { + const site = await world([SPEAKERS]); + const hall = await place(site, "Main Hall"); + const room = await place(site, "Room"); + const e = await entry(site, "events", { + fields: { speakers: [{ name: "Main Hall" }, { name: room.id }] }, + }); + const op: FieldOperation = { + op: "change_field_type", + collection: "events", + key: "speakers", + sub: "name", + type: "reference", + collections: ["places"], + }; + const part = await run(site, op, { budget: { entries: 0 } }); + // the room is archived: a title edit converts item 1 alone and holds item 2 + await store().updateContentEntry({ ...(await fresh(room)), status: "archived" }); + const written = await store().updateContentEntry({ ...(await fresh(e)), title: "Edited" }); + expect(written.fields).toEqual({ speakers: [{ name: hall.id }, { name: room.id }] }); + expect(await rows(e.id)).toEqual([{ t: hall.id, f: "speakers.name", k: "speakers", s: 1 }]); + // item 2 becomes usable again, and item 1's target is archived meanwhile + await store().updateContentEntry({ ...(await fresh(room)), status: "published" }); + await store().updateContentEntry({ ...(await fresh(hall)), status: "archived" }); + const done = await continueFieldOperation(store(), site.id, part.operationId, actor); + expect(done).toMatchObject({ done: true, skipped: [] }); + expect((await rows(e.id)).map((r) => r.t).sort()).toEqual([hall.id, room.id].sort()); + }); + + it("a person's write over a failing text item keeps the repeater as stored", async () => { + const site = await world([SPEAKERS]); + await place(site, "Main Hall"); + const room = await place(site, "Room"); + const e = await entry(site, "events", { + fields: { speakers: [{ name: "Main Hall" }, { name: "Room" }] }, + }); + const op: FieldOperation = { + op: "change_field_type", + collection: "events", + key: "speakers", + sub: "name", + type: "reference", + collections: ["places"], + }; + await run(site, op, { budget: { entries: 0 } }); + await store().updateContentEntry({ + ...(await fresh(room)), + title: "Hall B", + slug: "hall-b", + }); + const written = await store().updateContentEntry({ ...(await fresh(e)), title: "Edited" }); + expect(written.fields).toEqual({ speakers: [{ name: "Main Hall" }, { name: "Room" }] }); + }); + }); + + describe("final gate", () => { + it("reference → text sub: a failing stored item and a written copy of its id keep the row", async () => { + const site = await world([SPEAKERS]); + const r = await place(site, "R place"); + const e = await entry(site, "events", { fields: { speakers: [{ person: r.id }] } }); + const part = await run( + site, + { + op: "change_field_type", + collection: "events", + key: "speakers", + sub: "person", + type: "text", + }, + { budget: { entries: 0 } }, + ); + expect(await code(store().deleteContentEntry(await fresh(r)))).toBe("ok"); + await store().updateContentEntry({ + ...(await fresh(e)), + fields: { speakers: [{ person: r.id }, { person: r.id, name: "copy" }] }, + }); + expect(await rows(e.id)).toEqual([{ t: r.id, f: "speakers.person", k: "speakers", s: 1 }]); + const stuck = await continueFieldOperation(store(), site.id, part.operationId, actor); + expect(stuck.done).toBe(false); + expect(stuck.unconvertible.map((u) => u.entryId)).toEqual([e.id]); + }); + + it("a mid-operation reimport keeps reference → repeater rows, so the target stays locked", async () => { + const site = await world([reference("hosts", { multiple: true })]); + const p = await place(site, "P place"); + const e = await entry(site, "events", { status: "published", fields: { hosts: [p.id] } }); + await run( + site, + { + op: "change_field_type", + collection: "events", + key: "hosts", + type: "repeater", + subFields: [ + { key: "person", type: "reference", labels: L("Person"), collections: ["places"] }, + ], + }, + { budget: { entries: 0 } }, + ); + const storage = { sql: ctx.sql(), blobs: ctx.blobs() }; + await importSite(storage, await exportSite(storage, site.id), { + mode: "replace", + fieldTypes: ENABLED, + }); + expect(await rows(e.id)).toEqual([{ t: p.id, f: "hosts", k: "hosts", s: 1 }]); + expect(await code(store().deleteContentEntry(await fresh(p)))).toBe("entry_referenced"); + }); + }); + + describe("revert, after edits", () => { + it("text → reference: an entry edited after the operation converts back to its title", async () => { + const site = await world([text("venue")]); + const hall = await place(site, "Main Hall"); + const e = await entry(site, "events", { fields: { venue: "main hall" } }); + const done = await run(site, { + op: "change_field_type", + collection: "events", + key: "venue", + type: "reference", + collections: ["places"], + }); + await store().updateContentEntry({ ...(await fresh(e)), title: "Edited after" }); + const back = await revertFieldOperation(store(), site.id, done.operationId, actor); + expect(back).toMatchObject({ done: true, restored: 0, edited: [e.id] }); + expect((await fresh(e)).fields).toEqual({ venue: "Main Hall" }); + expect(await rows(e.id)).toEqual([]); + }); + + it("a sub-field removal's revert restores the sub-field, its values and their rows", async () => { + const site = await world([SPEAKERS]); + const hall = await place(site, "Main Hall"); + const e = await entry(site, "events", { + fields: { speakers: [{ person: hall.id, name: "Ada" }, { person: hall.id }] }, + }); + const done = await run(site, { + op: "remove_field", + collection: "events", + key: "speakers", + sub: "person", + }); + expect((await fresh(e)).fields).toEqual({ speakers: [{ name: "Ada" }] }); + expect(await rows(e.id)).toEqual([]); + const back = await revertFieldOperation(store(), site.id, done.operationId, actor); + expect(back.done).toBe(true); + const speakers = await def(site, "speakers"); + expect(speakers?.type === "repeater" && speakers.fields.map((f) => f.key)).toContain( + "person", + ); + expect((await fresh(e)).fields).toEqual({ + speakers: [{ person: hall.id, name: "Ada" }, { person: hall.id }], + }); + expect(await rows(e.id)).toEqual([ + { t: hall.id, f: "speakers.person", k: "speakers", s: 1 }, + ]); + }); + + it("a repeater converting away holds its rows until rewritten, then drops them", async () => { + const site = await world([SPEAKERS]); + const hall = await place(site, "Main Hall"); + const e = await entry(site, "events", { + status: "published", + fields: { speakers: [{ person: hall.id }] }, + }); + const op: FieldOperation = { + op: "change_field_type", + collection: "events", + key: "speakers", + type: "number", + onInvalid: "clear", + }; + const started = await run(site, op, { budget: { entries: 0 } }); + // the stale row still locks while the marker runs + expect(await code(store().deleteContentEntry(await fresh(hall)))).toBe("entry_referenced"); + expect( + (await continueFieldOperation(store(), site.id, started.operationId, actor)).done, + ).toBe(true); + expect((await fresh(e)).fields).toEqual({}); + expect(await rows(e.id)).toEqual([]); + expect(await code(store().deleteContentEntry(await fresh(hall)))).toBe("ok"); + }); + + it("a rename's revert that also restores an older localized updates shared", async () => { + const site = await world([reference("venue")]); + const hall = await place(site, "Main Hall"); + const e = await entry(site, "events", { fields: { venue: hall.id } }); + const renamed = await run(site, { + op: "rename_field", + collection: "events", + from: "venue", + to: "place", + }); + await run(site, { + op: "set_field_localized", + collection: "events", + key: "place", + localized: true, + }); + expect(await sharedOf(e.id)).toEqual([2]); + const back = await revertFieldOperation(store(), site.id, renamed.operationId, actor); + expect(back.done).toBe(true); + expect((await def(site, "venue"))?.localized).toBe(false); + expect(await sharedOf(e.id)).toEqual([1]); + }); + }); + + describe("marker interplay (watch item 3)", () => { + it("set_field_localized: phase A sets shared in its batch, both ways; a racing holder write retries (R16)", async () => { + const site = await world([reference("venue")]); + const hall = await place(site, "Main Hall"); + const a = await entry(site, "events", { fields: { venue: hall.id } }); + const b = await entry(site, "events", { fields: { venue: hall.id } }); + expect(await sharedOf(a.id)).toEqual([1]); + for (const localized of [true, false]) { + const op: FieldOperation = { + op: "set_field_localized", + collection: "events", + key: "venue", + localized, + }; + // b's status-only write is planned before phase A and batched after it: the definitions + // anchor fails it, and its retry reads `shared` from the new definition + const writer = racing(() => run(site, op).then(() => {})); + expect( + await code(writer.updateContentEntry({ ...(await fresh(b)), title: `T ${localized}` })), + ).toBe("conflict"); + const want = localized ? 2 : 1; + // a's rows were written before phase A, which corrected them in its own batch + expect(await sharedOf(a.id)).toEqual([want]); + await store().updateContentEntry({ ...(await fresh(b)), title: `T ${localized}` }); + expect(await sharedOf(b.id)).toEqual([want]); + expect((await def(site, "venue"))?.localized).toBe(localized); + } + }); + + it("rename_field: rows keep the old key until each entry is rewritten, and still lock (fromKey)", async () => { + const site = await world([reference("venue")]); + const hall = await place(site, "Main Hall"); + const a = await entry(site, "events", { status: "published", fields: { venue: hall.id } }); + const b = await entry(site, "events", { fields: { venue: hall.id } }); + const op: FieldOperation = { + op: "rename_field", + collection: "events", + from: "venue", + to: "place", + }; + let id = ""; + const writer = racing(async () => { + id = (await run(site, op, { budget: { entries: 0 } })).operationId; + }); + expect(await code(writer.updateContentEntry({ ...(await fresh(b)), title: "Raced" }))).toBe( + "conflict", + ); + // the retry goes through the marker: the new key, and its rows with it + const retried = await store().updateContentEntry({ ...(await fresh(b)), title: "Raced" }); + expect(retried.fields).toEqual({ place: hall.id }); + expect(await rows(b.id)).toEqual([{ t: hall.id, f: "place", k: "place", s: 1 }]); + // a is not rewritten yet: its row names the old key, matched through migration.fromKey + expect(await rows(a.id)).toEqual([{ t: hall.id, f: "venue", k: "venue", s: 1 }]); + expect(await code(store().deleteContentEntry(await fresh(hall)))).toBe("entry_referenced"); + expect((await continueFieldOperation(store(), site.id, id, actor)).done).toBe(true); + expect(await rows(a.id)).toEqual([{ t: hall.id, f: "place", k: "place", s: 1 }]); + }); + + it("R12: target deletes, moves and group changes wait while text → reference runs", async () => { + const site = await world([text("venue")]); + const hall = await place(site, "Main Hall", { status: "published" }); + const other = await place(site, "Other"); + await entry(site, "events", { status: "published", fields: { venue: "Main Hall" } }); + const op: FieldOperation = { + op: "change_field_type", + collection: "events", + key: "venue", + type: "reference", + collections: ["places"], + }; + let id = ""; + // planned before phase A commits, batched after it: the collections anchor fails it… + const del = racing(async () => { + id = (await run(site, op, { budget: { entries: 0 } })).operationId; + }); + expect(await code(del.deleteContentEntry(hall))).toBe("conflict"); + // …and the retry is refused while the marker is there, as are moves and group changes + expect(await code(store().deleteContentEntry(await fresh(hall)))).toBe( + "field_operation_running", + ); + expect( + await code(store().updateContentEntry({ ...(await fresh(hall)), collection: "events" })), + ).toBe("field_operation_running"); + // a group change: `other` joins the hall's group as a valid Turkish translation + const source = await fresh(hall); + const translation = { + group: hall.id, + method: "human" as const, + source: { id: hall.id, locale: "en", rev: source.rev, hash: await nodeHash(source) }, + }; + expect( + await code( + store().updateContentEntry({ ...(await fresh(other)), locale: "tr", translation }), + ), + ).toBe("field_operation_running"); + expect((await continueFieldOperation(store(), site.id, id, actor)).done).toBe(true); + // phase C dropped the marker: the published referrer's row now locks the hall + expect(await code(store().deleteContentEntry(await fresh(hall)))).toBe("entry_referenced"); + expect(await code(store().deleteContentEntry(await fresh(other)))).toBe("ok"); + }); + }); + }); +} diff --git a/packages/core-ts/src/testing-reference-gate.ts b/packages/core-ts/src/testing-reference-gate.ts index 97f1a449..f976ba0c 100644 --- a/packages/core-ts/src/testing-reference-gate.ts +++ b/packages/core-ts/src/testing-reference-gate.ts @@ -126,19 +126,24 @@ export function referenceGateSuite(ctx: SuiteContext): void { expect((await store().getSite(site.id))?.collections).toEqual(site.collections); }); - it("refuses a field operation that turns a field into one (step 4 adds them)", async () => { + it("refuses a field operation that turns a field into one, dry run included", async () => { const site = await setup(); - for (const type of ["reference", "repeater"] as const) - expect( - await code( - runFieldOperation( - store(), - site.id, - { op: "change_field_type", collection: "events", key: "room", type }, - { actor: person }, + const into = [ + { type: "reference", collections: ["events"] }, + { type: "repeater", subFields: [{ key: "line", type: "text", labels: L("Line") }] }, + ] as const; + for (const target of into) + for (const dryRun of [true, false]) + expect( + await code( + runFieldOperation( + store(), + site.id, + { op: "change_field_type", collection: "events", key: "room", ...target } as never, + { actor: person, dryRun }, + ), ), - ), - ).toBe("field_operation_unsupported"); + ).toBe("fields_unsupported"); expect((await store().getSite(site.id))?.collections).toEqual(site.collections); }); diff --git a/packages/core-ts/src/testing-references.ts b/packages/core-ts/src/testing-references.ts index ede7853e..d28e67af 100644 --- a/packages/core-ts/src/testing-references.ts +++ b/packages/core-ts/src/testing-references.ts @@ -571,15 +571,26 @@ export function referencesSuite(ctx: SuiteContext): void { expect((await rows(event.id)).map((x) => x.target_id)).toEqual([kept.id]); }); - it("refuses operations that name a reference or repeater field (step 4)", async () => { + it("operations name reference and repeater fields since step 4 (testing-field-operations)", async () => { const site = await world(); - for (const op of [ - { op: "rename_field", collection: "events", from: "venue", to: "place" }, - { op: "remove_field", collection: "events", key: "speakers" }, - ] as const) - expect(await code(runFieldOperation(store(), site.id, op, { actor: person }))).toBe( - "field_operation_unsupported", - ); + const hall = await entry(site, "places"); + const event = await entry(site, "events", { fields: { venue: hall.id } }); + const op = { + op: "rename_field", + collection: "events", + from: "venue", + to: "place", + } as const; + expect((await runFieldOperation(store(), site.id, op, { actor: person })).done).toBe(true); + expect(await rows(event.id)).toEqual([ + { + target_id: hall.id, + field: "place", + root_key: "place", + shared: 1, + source_status: "draft", + }, + ]); }); }); diff --git a/packages/core-ts/src/testing.ts b/packages/core-ts/src/testing.ts index 42c88016..d62b9b5e 100644 --- a/packages/core-ts/src/testing.ts +++ b/packages/core-ts/src/testing.ts @@ -13,6 +13,7 @@ import { assetsSuite } from "./testing-assets.ts"; import { collectionsSuite } from "./testing-collections.ts"; import { entryQuerySuite } from "./testing-entry-query.ts"; import { fieldOperationsSuite } from "./testing-field-operations.ts"; +import { fieldOperationsReferencesSuite } from "./testing-field-operations-references.ts"; import { fieldsSuite } from "./testing-fields.ts"; import { listingsSuite } from "./testing-listings.ts"; import { menuSuite } from "./testing-menu.ts"; @@ -278,6 +279,7 @@ export function describeSiteGraphStore(label: string, factory: StoreFactory): vo collectionsSuite(suite); fieldsSuite(suite); fieldOperationsSuite(suite); + fieldOperationsReferencesSuite(suite); referenceGateSuite(suite); referencesSuite(suite); referenceReadsSuite(suite); diff --git a/packages/core-ts/test/fields-conversions-references.test.ts b/packages/core-ts/test/fields-conversions-references.test.ts new file mode 100644 index 00000000..600c8a8e --- /dev/null +++ b/packages/core-ts/test/fields-conversions-references.test.ts @@ -0,0 +1,422 @@ +/** RFC-0045 §7: the conversion rows for references and repeaters, and `sub` markers. Pure. */ +import type { ContentEntry, FieldDefinition } from "@cloudhal/core-contracts"; +import { describe, expect, it } from "vitest"; +import { + canConvert, + conversionEntries, + convertFieldValue, + normalizeThroughMarkers, + resolveFields, + rewriteFields, +} from "../src/fields/index.ts"; +import { textReferenceOutcome } from "../src/fields/text-reference.ts"; + +const L = { en: { label: "x" } }; +const def = (over: Record) => + ({ key: "v", labels: L, localized: false, ...over }) as unknown as FieldDefinition; +const ref = (multiple = false, maxItems?: number) => + def({ type: "reference", collections: ["places"], multiple, ...(maxItems ? { maxItems } : {}) }); +const lines = (over: Record = {}) => + def({ type: "repeater", fields: [{ key: "line", type: "text", labels: L, ...over }] }); +const people = def({ + type: "repeater", + fields: [{ key: "person", type: "reference", labels: L, collections: ["places"] }], +}); + +const HALL = "entry_HALL000001"; +const ROOM = "entry_ROOM000001"; +const member = (id: string, over: Partial = {}) => + ({ + id, + collection: "places", + locale: "en", + slug: id.slice(6).toLowerCase(), + title: id === HALL ? "Main Hall" : "Room", + status: "published", + ...over, + }) as ContentEntry; +const entries = conversionEntries( + [ + member(HALL), + member("entry_HALLTR0001", { + locale: "tr", + title: "Ana Salon", + translation: { group: HALL } as never, + }), + member(ROOM), + member("entry_ROOMTWIN01", { title: "Room", slug: "room-twin" }), + member("entry_GONE000001", { title: "Gone", status: "archived" }), + ], + "en", +); +const ctx = { timeZone: "UTC", defaultLocale: "en", entries }; + +describe("RFC-0045 §7 conversion rows", () => { + it("lists the new rows", () => { + for (const [from, to] of [ + ["text", "reference"], + ["reference", "text"], + ["reference", "reference"], + ["text", "repeater"], + ["repeater", "text"], + ["reference", "repeater"], + ["repeater", "reference"], + ] as const) + expect(canConvert(from, to)).toBe(true); + expect(canConvert("number", "reference")).toBe(false); + expect(canConvert("repeater", "number")).toBe(false); + }); + + it("reference single ↔ multiple: one id ↔ a list of one; an empty list is no value", () => { + expect(convertFieldValue("reference", ref(true), HALL, ctx)).toEqual({ + ok: true, + value: [HALL], + }); + expect(convertFieldValue("reference", ref(false), [HALL], ctx)).toEqual({ + ok: true, + value: HALL, + }); + expect(convertFieldValue("reference", ref(false), [], ctx)).toEqual({ + ok: true, + value: undefined, + }); + expect(convertFieldValue("reference", ref(false), [HALL, ROOM], ctx)).toMatchObject({ + ok: false, + reason: expect.stringMatching(/2 entries/), + }); + // the new definition's own rules still apply + expect(convertFieldValue("reference", ref(true, 1), [HALL, ROOM], ctx).ok).toBe(false); + }); + + it("text → reference: the one group with that slug or default-locale title, case-folded", () => { + expect(convertFieldValue("text", ref(), " main HALL ", ctx)).toEqual({ + ok: true, + value: HALL, + }); + expect(convertFieldValue("text", ref(), "hall000001", ctx)).toEqual({ ok: true, value: HALL }); + expect(convertFieldValue("text", ref(true), "Main Hall", ctx)).toEqual({ + ok: true, + value: [HALL], + }); + expect(convertFieldValue("text", ref(), "", ctx)).toEqual({ ok: true, value: undefined }); + // a translation's title is not the default locale's; archived entries match nothing + expect(convertFieldValue("text", ref(), "Ana Salon", ctx).ok).toBe(false); + expect(convertFieldValue("text", ref(), "Gone", ctx).ok).toBe(false); + expect(convertFieldValue("text", ref(), "Nowhere", ctx)).toMatchObject({ ok: false }); + expect(convertFieldValue("text", ref(), "Room", ctx)).toMatchObject({ + ok: false, + reason: expect.stringMatching(/matches 2 entries/), + }); + // never the entry's own group; without the maps nothing converts + expect(convertFieldValue("text", ref(), "Main Hall", { ...ctx, group: HALL }).ok).toBe(false); + expect(convertFieldValue("text", ref(), "Main Hall", { ...ctx, entries: undefined }).ok).toBe( + false, + ); + }); + + it("reference → text: the default-locale title, a list comma separated", () => { + const text = def({ type: "text" }); + expect(convertFieldValue("reference", text, HALL, ctx)).toEqual({ + ok: true, + value: "Main Hall", + }); + expect(convertFieldValue("reference", text, [HALL, ROOM], ctx)).toEqual({ + ok: true, + value: "Main Hall, Room", + }); + expect(convertFieldValue("reference", text, "entry_NOBODY0001", ctx).ok).toBe(false); + }); + + it("text → repeater: one item per non-empty line into the one text sub-field", () => { + expect(convertFieldValue("text", lines(), "2 eggs\n\n flour \r\nsalt", ctx)).toEqual({ + ok: true, + value: [{ line: "2 eggs" }, { line: "flour" }, { line: "salt" }], + }); + expect(convertFieldValue("text", lines({ maxLength: 3 }), "salt\nflour", ctx).ok).toBe(false); + const many = Array.from({ length: 21 }, (_, i) => `l${i}`).join("\n"); + expect(convertFieldValue("text", lines(), many, ctx).ok).toBe(false); + const two = def({ + type: "repeater", + fields: [ + { key: "line", type: "text", labels: L }, + { key: "n", type: "number", labels: L }, + ], + }); + expect(convertFieldValue("text", two, "a", ctx).ok).toBe(false); + }); + + it("repeater → text: items joined by line breaks; other shapes do not convert", () => { + const text = def({ type: "text", multiline: true }); + expect(convertFieldValue("repeater", text, [{ line: "a" }, { line: "b" }], ctx)).toEqual({ + ok: true, + value: "a\nb", + }); + expect(convertFieldValue("repeater", text, [{ line: "a", n: 2 }], ctx).ok).toBe(false); + expect(convertFieldValue("repeater", text, [{ n: 2 }], ctx).ok).toBe(false); + }); + + it("reference (multiple) ↔ repeater with one reference sub-field", () => { + expect(convertFieldValue("reference", people, [HALL, ROOM], ctx)).toEqual({ + ok: true, + value: [{ person: HALL }, { person: ROOM }], + }); + expect(convertFieldValue("repeater", ref(true), [{ person: HALL }], ctx)).toEqual({ + ok: true, + value: [HALL], + }); + expect(convertFieldValue("reference", lines(), [HALL], ctx).ok).toBe(false); + expect(convertFieldValue("repeater", ref(true), [{ person: HALL, talk: "x" }], ctx).ok).toBe( + false, + ); + }); +}); + +describe("RFC-0045 §7 sub-field markers", () => { + const speakers = (migration: Record, extra: Record[] = []) => + def({ + key: "speakers", + type: "repeater", + fields: [ + { key: "fullName", type: "text", labels: L }, + { key: "seats", type: "number", labels: L }, + ...extra, + ], + migration: { id: "snap_01HZX8K3Q9OP", ...migration }, + }); + const rewrite = (d: FieldDefinition, values: Record, onInvalid?: "clear") => + rewriteFields({ def: d, values, convert: ctx, translation: false, pristine: true, onInvalid }); + + it("rename: the sub-key moves in every item; the field keeps its key", () => { + const d = speakers({ sub: "fullName", fromKey: "name" }); + const r = rewrite(d, { speakers: [{ name: "Ada", seats: 2 }, { seats: 3 }], other: 1 }); + expect(r.values).toEqual({ speakers: [{ fullName: "Ada", seats: 2 }, { seats: 3 }], other: 1 }); + expect(r.change?.kind).toBe("moved"); + // readers see the end state already + const site = { collections: { events: { fields: [d] } } } as never; + const resolved = resolveFields(site, { + id: "entry_EVENT00001", + collection: "events", + fields: { speakers: [{ name: "Ada" }] }, + }); + expect(resolved.speakers?.value).toEqual([{ fullName: "Ada" }]); + }); + + it("retype: each item's sub-value converts; unconvertible refuses unless cleared", () => { + const d = speakers({ sub: "seats", fromType: "text" }); + const ok = rewrite(d, { speakers: [{ fullName: "A", seats: "12" }] }); + expect(ok.values).toEqual({ speakers: [{ fullName: "A", seats: 12 }] }); + const bad = rewrite(d, { speakers: [{ fullName: "A", seats: "lots" }] }); + expect(bad).toMatchObject({ values: undefined, change: { kind: "unconvertible" } }); + // clear drops the sub-value; an item left empty goes, and an emptied list is no value + const cleared = rewrite( + d, + { speakers: [{ fullName: "A", seats: "x" }, { seats: "y" }] }, + "clear", + ); + expect(cleared.values).toEqual({ speakers: [{ fullName: "A" }] }); + expect(cleared.change?.kind).toBe("cleared"); + expect(rewrite(d, { speakers: [{ seats: "y" }] }, "clear").values).toEqual({}); + }); + + it("text → reference inside items reads the maps; a person's write converges", () => { + const d = speakers({ sub: "host", fromType: "text" }, [ + { key: "host", type: "reference", labels: L, collections: ["places"] }, + ]); + const values = { speakers: [{ fullName: "A", host: "main hall" }] }; + expect(rewrite(d, values).values).toEqual({ speakers: [{ fullName: "A", host: HALL }] }); + expect(normalizeThroughMarkers([d], values, { convert: ctx, translation: false })).toEqual({ + speakers: [{ fullName: "A", host: HALL }], + }); + }); + + it("remove: the sub-value goes from every item, empty items are dropped, the field stays", () => { + const d = speakers({ sub: "seats" }); + const r = rewrite(d, { speakers: [{ fullName: "A", seats: 1 }, { seats: 2 }] }); + expect(r.values).toEqual({ speakers: [{ fullName: "A" }] }); + expect(r.change?.kind).toBe("removed"); + expect(rewrite(d, { speakers: [{ seats: 2 }] }).values).toEqual({}); + const site = { collections: { events: { fields: [d] } } } as never; + const resolved = resolveFields(site, { + id: "entry_EVENT00001", + collection: "events", + fields: { speakers: [{ fullName: "A", seats: 1 }] }, + }); + expect(resolved.speakers?.value).toEqual([{ fullName: "A" }]); + }); +}); + +describe("RFC-0045 §7 review round (PR #214)", () => { + it("reference → text: an archived target lends its title; without a default-locale member, a fixed fallback", () => { + const maps = conversionEntries( + [ + member("entry_GONE000001", { title: "Gone", status: "archived" }), + // no English member: the original's title, whatever the order + member("entry_TRONLY0001", { + locale: "tr", + title: "Ana", + translation: { group: "entry_TRONLY0001" } as never, + }), + member("entry_TRONLY0002", { + locale: "de", + title: "Haupt", + translation: { group: "entry_TRONLY0001" } as never, + }), + // no original either: the lowest locale + member("entry_ORPHAN0002", { + locale: "tr", + title: "Yetim", + translation: { group: "entry_ORPHAN0001" } as never, + }), + member("entry_ORPHAN0003", { + locale: "de", + title: "Waise", + translation: { group: "entry_ORPHAN0001" } as never, + }), + ], + "en", + ); + const reversed = conversionEntries( + [ + ...[ + member("entry_ORPHAN0003", { + locale: "de", + title: "Waise", + translation: { group: "entry_ORPHAN0001" } as never, + }), + member("entry_ORPHAN0002", { + locale: "tr", + title: "Yetim", + translation: { group: "entry_ORPHAN0001" } as never, + }), + ], + ], + "en", + ); + const text = def({ type: "text" }); + const c = { timeZone: "UTC", defaultLocale: "en", entries: maps }; + expect(convertFieldValue("reference", text, "entry_GONE000001", c)).toEqual({ + ok: true, + value: "Gone", + }); + expect(convertFieldValue("reference", text, "entry_TRONLY0001", c)).toEqual({ + ok: true, + value: "Ana", + }); + expect(convertFieldValue("reference", text, "entry_ORPHAN0001", c)).toEqual({ + ok: true, + value: "Waise", + }); + expect(reversed.titles.get("entry_ORPHAN0001")).toBe("Waise"); + // archived members still match no text + expect(convertFieldValue("text", ref(), "Gone", c).ok).toBe(false); + }); + + it("one rule decides by the rows phase C reads (text-reference.ts)", () => { + const rows = (...targets: string[]) => ({ + has: (t: string) => targets.includes(t), + any: targets.length > 0, + }); + const at = (over: Partial[0]>) => + textReferenceOutcome({ + from: "text", + def: ref(), + value: HALL, + convert: { entries }, + written: false, + ...over, + })?.kind; + const GONE = "entry_GONE000001"; + // text → reference with rows: in the target shape and with its row, kept + expect(at({ value: HALL, rows: rows(HALL) })).toBe("keep"); + expect(at({ def: ref(true), value: [HALL], rows: rows(HALL) })).toBe("keep"); + expect(at({ value: GONE, rows: rows(GONE) })).toBe("keep"); + // no row: legacy text that happens to be an id; checked when usable, else it fails + expect(at({ value: HALL, rows: rows() })).toBe("check"); + expect(at({ def: ref(true), value: HALL, rows: rows(HALL) })).toBe("check"); + expect(at({ value: GONE, rows: rows() })).toBe("fail"); + expect(at({ value: "entry_NOBODY0001", rows: rows() })).toBe("fail"); + // the own group fails first, rows or not, in every caller + expect(at({ value: HALL, rows: rows(HALL), convert: { entries, group: HALL } })).toBe("fail"); + expect(at({ value: HALL, convert: { entries, group: HALL } })).toBe("fail"); + // a person's write under the new type: kept when it fits and has the shape + expect(at({ value: "entry_NOBODY0001", rows: rows(), written: true })).toBe("keep"); + expect(at({ def: ref(true), value: HALL, rows: rows(), written: true })).toBe("check"); + // other text goes through the table; no maps: kept as stored + expect(at({ value: "Main Hall", rows: rows() })).toBe("table"); + expect(at({ value: "Main Hall", convert: {} })).toBe("keep"); + // display (no rows): an id naming a member group shows as a reference + expect(at({ value: GONE })).toBe("keep"); + expect(at({ value: "entry_NOBODY0001" })).toBe("table"); + // reference → text: a row left means still a reference; no row means text already + const text = def({ type: "text" }); + expect(at({ from: "reference", def: text, value: HALL, rows: rows(HALL) })).toBe("table"); + expect( + at({ + from: "reference", + def: text, + value: "entry_NOBODY0001", + rows: rows("entry_NOBODY0001"), + }), + ).toBe("fail"); + expect(at({ from: "reference", def: text, value: HALL, rows: rows() })).toBe("keep"); + expect(at({ from: "reference", def: text, value: "entry_2025report", rows: rows() })).toBe( + "keep", + ); + expect(at({ from: "reference", def: text, value: "entry_2025report" })).toBe("keep"); + expect(at({ from: "reference", def: text, value: HALL })).toBe("table"); + // per value: a list is never text yet; a string with no row of its own is text already + expect(at({ from: "reference", def: text, value: [], rows: rows() })).toBe("table"); + expect(at({ from: "reference", def: text, value: [HALL], rows: rows() })).toBe("table"); + expect(at({ from: "reference", def: text, value: "Keynote", rows: rows(HALL) })).toBe("keep"); + expect(convertFieldValue("reference", text, [], ctx)).toEqual({ ok: true, value: undefined }); + expect(convertFieldValue("reference", text, [HALL], ctx)).toEqual({ + ok: true, + value: "Main Hall", + }); + // other rows are not decided here + expect(at({ from: "text", def: def({ type: "number" }) })).toBeUndefined(); + }); + + it("repeater → multiple reference: repeated ids collapse, first seen first", () => { + expect( + convertFieldValue( + "repeater", + ref(true), + [{ person: HALL }, { person: ROOM }, { person: HALL }], + ctx, + ), + ).toEqual({ ok: true, value: [HALL, ROOM] }); + }); + + it("readers never keep an unconverted string that happens to fit (text ↔ reference)", () => { + const venue = def({ + key: "venue", + type: "reference", + collections: ["places"], + migration: { id: "snap_01HZX8K3Q9OP", fromType: "text" }, + }); + const site = { collections: { events: { fields: [venue] } } } as never; + const read = (v: string) => + resolveFields( + site, + { id: "entry_EVENT00001", collection: "events", fields: { venue: v } }, + undefined, + { + entries, + }, + ).venue; + expect(read("Main Hall")?.value).toBe(HALL); + expect(read("entry_Zz9xQ1AB")).toMatchObject({ + value: undefined, + issue: { code: "field_value_unreadable" }, + }); + const rewrite = rewriteFields({ + def: venue, + values: { venue: "entry_Zz9xQ1AB" }, + convert: ctx, + translation: false, + pristine: false, + }); + expect(rewrite.change?.kind).toBe("unconvertible"); + }); +}); diff --git a/packages/core-ts/test/references-purge.test.ts b/packages/core-ts/test/references-purge.test.ts index 4167c52d..d172d662 100644 --- a/packages/core-ts/test/references-purge.test.ts +++ b/packages/core-ts/test/references-purge.test.ts @@ -486,31 +486,95 @@ describe("planCachePurge: the referrer closure (RFC-0045 §2)", () => { return { sent, allowance, outcome }; }; - it("a site without references sends the stored plan with the allowance spent, or the gate read failing", async () => { + it("a site without references sends the stored plan with the allowance spent", async () => { const plain = await seedStarterSite(store, { name: "Plain", slug: "plain-release" }); const a = await store.createSnapshot(plain.id, { actor: human, reason: "publish" }); await store.publish(plain, a.id); const b = await store.createSnapshot(plain.id, { actor: human, reason: "manual" }); const spent = await run({ siteId: plain.id, left: 0, from: a.id, to: b.id }); expect(spent.sent).toEqual([stored]); - const failing = new SiteGraphStore({ + }); + + // #213: the working row never decides. A draft that dropped its reference fields would send + // the closure-less stored plan while the live node still shows cards. + const failingGets = (fails: number) => { + let calls = 0; + const graph = new SiteGraphStore({ sql: store.sql, blobs: new Proxy(store.blobs, { get(target, prop, receiver) { const v = Reflect.get(target, prop, receiver); - if (prop === "get") return async () => Promise.reject(new Error("R2 went away")); + if (prop === "get") + return async (...args: unknown[]) => { + calls++; + if (calls <= fails) throw new Error("R2 went away"); + return (v as (...a: unknown[]) => unknown).apply(target, args); + }; return typeof v === "function" ? v.bind(target) : v; }, }), }); - const broken = await run({ - graph: failing, + return { graph, calls: () => calls }; + }; + + it("a gate read that fails twice purges everything, even where the working row has no references", async () => { + const { hall } = await world(); + const from = (await fresh()).publishedSnapshotId as string; + await edit(hall, { title: "Main hall" }); + const to = await snap("manual"); + // the working copy drops every reference definition (as a finished remove_field would) + const s = await fresh(); + const events = s.collections?.events; + const next = { + ...s, + rev: s.rev + 1, + collections: { + ...s.collections, + events: { + ...events, + fields: (events?.fields ?? []).filter((d) => d.type !== "reference"), + }, + }, + } as unknown as Site; + await store.sql.run("UPDATE nodes SET rev = ?, hash = ?, body = ? WHERE id = ?", [ + next.rev, + await nodeHash(next), + canonicalJson(next), + s.id, + ]); + const broken = failingGets(2); + const out = await run({ graph: broken.graph, left: 100, from, to: to.id }); + expect(broken.calls()).toBe(2); + expect(out.sent).toHaveLength(1); + expect(out.sent[0]).toMatchObject({ mode: "everything" }); + expect((out.sent[0] as { reason: string }).reason).toMatch(/published site node/); + }); + + it("a gate read that fails once is retried, and a site without references sends the stored plan", async () => { + const plain = await seedStarterSite(store, { name: "Plain", slug: "plain-retry" }); + const a = await store.createSnapshot(plain.id, { actor: human, reason: "publish" }); + await store.publish(plain, a.id); + const b = await store.createSnapshot(plain.id, { actor: human, reason: "manual" }); + const flaky = failingGets(1); + const out = await run({ + graph: flaky.graph, + siteId: plain.id, + left: 100, + from: a.id, + to: b.id, + }); + expect(flaky.calls()).toBe(2); + expect(out.sent).toEqual([stored]); + // and failing for good: everything, not the stored plan + const down = failingGets(99); + const gone = await run({ + graph: down.graph, siteId: plain.id, left: 100, from: a.id, to: b.id, }); - expect(broken.sent).toEqual([stored]); + expect(gone.sent).toEqual([expect.objectContaining({ mode: "everything" })]); }); it("a re-plan with more terms than reads left purges everything before reading anything", async () => { diff --git a/packages/core-ts/test/references-store.test.ts b/packages/core-ts/test/references-store.test.ts index 27136dd6..10f3256b 100644 --- a/packages/core-ts/test/references-store.test.ts +++ b/packages/core-ts/test/references-store.test.ts @@ -294,6 +294,48 @@ describe("release dependencies (§2.7)", () => { it("leaves bodies without reference definitions as they are", () => { expect(stripReferences({ collections: {} }, body)).toBe(body); + // RFC-0045 §7: through the markers, a sub-field rename's old key and a conversion from references + const marked = { + collections: { + events: { + fields: [ + { + key: "speakers", + type: "repeater", + labels: { en: { label: "S" } }, + localized: false, + fields: [ + { + key: "host", + type: "reference", + labels: { en: { label: "H" } }, + collections: ["places"], + }, + ], + migration: { id: "snap_01HZX8K3Q9OP", sub: "host", fromKey: "person" }, + }, + { + key: "venue", + type: "text", + labels: { en: { label: "V" } }, + localized: false, + migration: { id: "snap_01HZX8K3Q9OP", fromType: "reference" }, + }, + ], + }, + }, + } as never; + const stale = { + collection: "events", + fields: { + speakers: [{ person: "entry_HALL000001", name: "Ada" }], + venue: "entry_ROOM000001", + }, + }; + expect(stripReferences(marked, stale)).toEqual({ + collection: "events", + fields: { speakers: [{ name: "Ada" }], venue: undefined }, + }); }); }); diff --git a/packages/ui/src/blocks/entry-fields-model.ts b/packages/ui/src/blocks/entry-fields-model.ts index c4e7b2aa..87ec8cc4 100644 --- a/packages/ui/src/blocks/entry-fields-model.ts +++ b/packages/ui/src/blocks/entry-fields-model.ts @@ -4,7 +4,7 @@ * for its original's value. No React, no English: words come from the caller. */ import type { FieldDefinition } from "@cloudhal/core-contracts"; -import { optionLabel } from "@cloudhal/core-ts/fields"; +import { optionLabel, renamedFrom } from "@cloudhal/core-ts/fields"; import { dateTimeFormat, formatList, formatNumber } from "@cloudhal/messages/format"; import { formatDay } from "./schedule-model.ts"; @@ -17,7 +17,7 @@ const isRecord = (v: unknown): v is Record => export function readField(values: FieldValues | undefined, def: FieldDefinition): unknown { if (!values) return undefined; if (Object.hasOwn(values, def.key)) return values[def.key]; - const from = def.migration?.fromKey; + const from = renamedFrom(def); return from && Object.hasOwn(values, from) ? values[from] : undefined; } @@ -37,7 +37,7 @@ export function writeField( v: unknown, ): Record { const next: Record = { ...(values ?? {}) }; - const from = def.migration?.fromKey; + const from = renamedFrom(def); if (from) delete next[from]; if (hasValue(v)) next[def.key] = v; else delete next[def.key];