Private books in the teaching corpus; collectEntities compares by content

CHANGELOG.md

@@ -183,12 +183,45 @@ npm trusts this repository's `release.yml`.
   legacy view (which drafts nothing) resolves a requested `semantic` to
   `locator` itself. `projectionContentHash` moves to
   `src/ingest/projections.ts`, beside `projectionProblem`.
+- Private books in the teaching corpus: `privateBooksDir` in `archive.config.ts`
+  (off by default; the example content does not change), one markdown file per
+  book. `buildPrivateBooks` (`src/corpus.ts`) reads the frontmatter (`title`,
+  `about`, and optionally `authors`, `identifiers`, `date`, `version`, `themes`,
+  `exposure`, `publicTitle`, `requireReview`, `fragmentBy`, `maxFragmentChars`,
+  `firstPage`) and cuts the body with `fragmentByHeadings` or
+  `fragmentByPageMarkers` into pieces whose locators are structural; a leading
+  `# Title` that repeats the frontmatter title is dropped as the title page.
+  `fromPrivateBook` (`src/adapters/teaching.ts`) yields one private entity with
+  one fragment per piece (`book:x#ch12`, `book:x#ch12.s3`, `book:x#p184`), the
+  heading inside the fragment's text and nowhere else, `publicTitle` and
+  `requireReview` on the entity's policy, and the metadata lint run over the
+  whole book before `npm run index` drafts or embeds anything; two pieces that
+  share a locator are refused by position and scheme. `npm run index` reads the books
+  beside the records and the notes; a book's `semantic` request is drafted per
+  fragment against the whole book's text, as CONTRACT §12 describes.
 
 ### Removed
 
 - `scripts/build-blind-artifact.mjs`. Review is open, not blind; the script's
   anonymized tarball has no remaining use.
 
+### Fixed
+
+- `collectEntities` compared two descriptions of one entity by
+  `JSON.stringify`, so the same entity emitted with its properties in another
+  insertion order (two adapters, or one adapter run twice) was refused as
+  "described two ways". It now compares by content, with object keys sorted at
+  every level; the order inside a list (creators, identifiers) is still
+  content. A regression test pins both.
+- A private book's page marker (`<<<page N>>>`) is bounded by the reader to a
+  short label of letters, digits, `.` and `-` before it becomes a locator
+  value and part of a fragment id, so the refusals that name a fragment have
+  a safe error surface; the value stays linted metadata, checked against the
+  book by the adapter's metadata lint. A marker outside the bound is refused
+  by piece position, never echoed.
+- A bare year in frontmatter (`date: 1900`, which YAML reads as a number) was
+  dropped silently by every reader; it is now the date at year precision.
+
 ### Breaking in the 3.0.0 line (summary)
 
 Recorded here in one place; `docs/CONTRACT.md` is the normative text.

README.md

@@ -65,8 +65,11 @@ notebook in `example-content/notebook/` is the private layer. Each note
 declares the public page it routes to (`about`), where the moment lives
 (`locator`), a linted display name (`label`), and, optionally, the `exposure`
 it wants; its `title` and body stay private, embedded so retrieval can find the
-moment. The teaching adapters (`src/adapters/teaching.ts`) turn those files
-into entities and fragments; a consumer writes its own.
+moment. A private **book** (`privateBooksDir`, off by default) is one entity
+with many fragments: the file is cut on its chapter headings or page markers,
+and each piece travels as where it is (`ch. 12`, `p. 184`) or as its gist,
+never as text. The teaching adapters (`src/adapters/teaching.ts`) turn those
+files into entities and fragments; a consumer writes its own.
 
 > In production ([Ask the Archive](https://lukefwalton.com/ask/)), published
 > podcast passages are public fragments while unpublished transcript text is
@@ -269,8 +272,29 @@ get `temperature: 0`).
    index it without ever serving it — see `docs/CONTRACT.md` §3 and §5. No
    private layer? Remove `privateNotesDir` from the config and the engine runs
    public-only.
-4. Replace `example-content/` with your corpus and rerun `npm run index`.
-5. Rewrite `eval/gold.yaml` for your corpus — keep the refusals.
+4. A private book is one markdown file in `privateBooksDir` (set it in the
+   config; it is off by default). The frontmatter is the book's public
+   surface: `title` (required; checked against the text unless
+   `publicTitle: true` says it is public by construction), `about` (the public
+   URL a hit routes to, required), and optionally `authors` (names, or
+   `{ name, role }` entries), `identifiers` (`{ scheme, value }`: isbn, doi),
+   `date`, `version` ("manuscript"), `themes`, `exposure` (`locator` by
+   default; `semantic` has `npm run index` draft one gist per fragment for you
+   to authorize in `artifacts/projections.json`; `none`), and
+   `requireReview: true` to serve no gist you have not marked reviewed. The
+   body is the private text. It is cut into fragments on its `#`/`##` headings
+   (`fragmentBy: headings`, the default: "Chapter 12" is `ch. 12`, any other
+   heading is counted among them; a leading `# Title` that repeats the
+   frontmatter title is the title page, not a chapter) or on page markers
+   (`fragmentBy: pages`: a `<<<page 184>>>` line, or a form feed with
+   `firstPage`); pieces longer than `maxFragmentChars` (6000) are sub-split on
+   paragraphs (`ch. 12, §3`). The locator is the only authored-looking thing a
+   hit carries about a chapter, and it is structural; the heading goes into the
+   embedding and nowhere else. The same build-time lint that checks a note's
+   label checks the book's title, version, and creators against the whole book
+   before anything is drafted or embedded.
+5. Replace `example-content/` with your corpus and rerun `npm run index`.
+6. Rewrite `eval/gold.yaml` for your corpus — keep the refusals.
 
 ## Commands
 

archive.config.ts

@@ -23,6 +23,10 @@ export const config: ArchiveConfig = {
   // production this is chunked podcast transcripts; here, notebook entries.
   // Remove the line to run public-only.
   privateNotesDir: './example-content/notebook',
+  // Private books: one markdown file per book, cut into fragments on its
+  // chapter headings (or page markers), served as where a moment is or as a
+  // gist the author authorizes, never as text. See README "Make it yours".
+  // privateBooksDir: './example-content/books',
   embeddingModel: 'text-embedding-3-large',
   // Default is a widely available chat model. Swap for any Responses-API model
   // you have access to (e.g. a reasoning model — see src/answer.ts).

docs/CONTRACT.md

@@ -847,9 +847,17 @@ yields an entity with one `text` fragment whose locator is `[{ scheme: 'whole',
 value: '' }]`; `fromPrivateNote` yields a private entity with one `locator`
 fragment whose linted label becomes the entity title and whose locator string
 becomes `[{ scheme: 'note', value }]`, and whose frontmatter may ask for an
-`exposure`. The example content does not change. The demo gains one entity that
-exercises `semantic` in a following release: a public-domain novel with its
-chapter gists committed beside the vectors, which needs a keyed build.
+`exposure`. `fromPrivateBook` yields a private entity with one fragment per
+piece the corpus reader cut (`buildPrivateBooks`: `fragmentByHeadings` or
+`fragmentByPageMarkers`, then `splitLong`), ids `${entityId}#${locatorKey}`,
+each fragment's text its heading and its piece joined as a note's title and
+body are, the frontmatter's `publicTitle` and `requireReview` on the entity's
+`policy`, and the section 6 metadata lint run over the whole book before the
+build drafts or embeds anything; `collectEntities` then groups the pairs, comparing
+two descriptions of one entity by content with keys sorted at every level, not
+by property order. The example content does not change. The demo gains one
+entity that exercises `semantic` in a following release: a public-domain novel
+with its chapter gists committed beside the vectors, which needs a keyed build.
 
 ## 13. What remains owned rather than guaranteed
 

scripts/smoke-dist.mjs

@@ -35,6 +35,8 @@ for (const name of [
   'answerQuestion',
   'fromArchiveRecord',
   'fromPrivateNote',
+  'fromPrivateBook',
+  'buildPrivateBooks',
 ]) {
   assert.equal(typeof root[name], 'function', `missing export: ${name}`);
 }

src/adapters/teaching.ts

@@ -11,10 +11,10 @@
 // the same two shapes reconstructed losslessly, so today's retrieval, prompt,
 // eval and demo code keep running on a schema-4 index without changes.
 
-import type { Entity, Fragment } from '../contract.js';
+import type { Entity, EntityPolicy, Fragment } from '../contract.js';
 import { locatorKey, renderLocatorLabel } from '../locator.js';
-import { assertPublicSafeField } from '../public-safe.js';
-import type { ArchiveRecord, PrivateNote } from '../types.js';
+import { assertPublicSafeField, assertPublicSafeMetadata } from '../public-safe.js';
+import type { ArchiveRecord, PrivateBook, PrivateNote } from '../types.js';
 
 export const WHOLE_LOCATOR = { scheme: 'whole', value: '' } as const;
 
@@ -96,6 +96,83 @@ export function fromPrivateNote(note: PrivateNote): { entity: Entity; fragment:
   return { entity, fragment };
 }
 
+/**
+ * A private book is one entity with many fragments (CONTRACT.md §8, §12): one
+ * per piece the corpus reader cut, ids `${entityId}#${locatorKey}` (book:x#ch12,
+ * book:x#ch12.s3, book:x#p184), each fragment's text its heading and its piece
+ * joined as a note's title and body are, so a chapter title is in the
+ * embedding and never leaves `text`. The entity default is the exposure the
+ * frontmatter asked for, carried on every fragment as a request until the
+ * build resolves it against a projection. The frontmatter's `publicTitle` and
+ * `requireReview` become the entity's policy.
+ *
+ * The strings a hit on the book would carry (title, version, creators,
+ * themes, locator labels) go through the metadata lint here, against the whole
+ * book, before anything is drafted or embedded: a quoting title fails in the
+ * reader's output, not after a paid build. The store runs the same lint again
+ * at every load (CONTRACT.md §12).
+ */
+export function fromPrivateBook(book: PrivateBook): { entity: Entity; fragment: Fragment }[] {
+  const disclosure = { raw: 'private', exposure: book.exposure ?? 'locator' } as const;
+  const policy: EntityPolicy = {
+    ...(book.publicTitle ? { publicTitle: true } : {}),
+    ...(book.requireReview ? { requireReview: true } : {}),
+  };
+  const entity: Entity = {
+    id: book.id,
+    type: book.type,
+    title: book.title,
+    attribution: book.attribution,
+    ...(book.date ? { date: book.date } : {}),
+    ...(book.version ? { version: book.version } : {}),
+    url: book.url,
+    identifiers: book.identifiers,
+    ...(book.themes.length > 0 ? { themes: book.themes } : {}),
+    disclosure,
+    ...(Object.keys(policy).length > 0 ? { policy } : {}),
+  };
+  if (book.pieces.length === 0) {
+    throw new Error(`private book '${book.id}' has no pieces; a book is at least one fragment.`);
+  }
+  const fragments: Fragment[] = [];
+  const keys = new Map<string, number>();
+  for (const [index, piece] of book.pieces.entries()) {
+    const key = locatorKey(piece.locator);
+    const first = keys.get(key);
+    if (first !== undefined) {
+      // Two chapters numbered the same, or two pages marked the same: the
+      // locator is the id, so the author fixes the source. Named by position
+      // and scheme, never by value: a chapter value is a parsed number or an
+      // ordinal, but a page value is the author's own marker token, and this
+      // message is what `npm run index` prints (STANDARDS §4).
+      const schemes = piece.locator.map((l) => l.scheme).join('.');
+      throw new Error(
+        `private book '${book.id}': pieces ${first + 1} and ${index + 1} of ${book.pieces.length} share a ` +
+          `'${schemes}' locator; each chapter heading or page marker must be distinct.`,
+      );
+    }
+    keys.set(key, index);
+    // The key carries the locator values (ch12, p184, p184.s2). A chapter or
+    // section value is a parsed number or an ordinal; a page value is the
+    // author's marker token, an authored string in the linted bucket
+    // (STANDARDS §1, §4). The metadata lint below checks every locator value
+    // and label against the whole book before this id goes anywhere, so an id
+    // is safe to print for the same reason the hit's locatorLabel is: linted,
+    // not trusted. The reader (src/corpus.ts) additionally bounds a page token
+    // to a short label, so the lint's own refusal, which names the id, has a
+    // safe error surface.
+    fragments.push({
+      id: `${book.id}#${key}`,
+      entityId: book.id,
+      locator: piece.locator,
+      text: joinNoteText(piece.heading ?? '', piece.text),
+      disclosure,
+    });
+  }
+  assertPublicSafeMetadata(entity, fragments, { path: `private book '${book.id}'` });
+  return fragments.map((fragment) => ({ entity, fragment }));
+}
+
 /**
  * Reverse of fromPrivateNote. The private title is the text before the first
  * blank line and the body is what follows: a note's title is one frontmatter

src/cli/build-index.ts

@@ -1,4 +1,5 @@
-// npm run index — read both layers, draft the gists the author asked for,
+// npm run index — read the public collections, the private notes, and the
+// private books; draft the gists the author asked for,
 // resolve each fragment's exposure, embed what changed, and write the private
 // index (artifacts/index.json) and the author's projections file
 // (artifacts/projections.json). Both are private material and gitignored.
@@ -25,9 +26,9 @@ import { createHash } from 'node:crypto';
 import OpenAI from 'openai';
 
 import { config } from '../../archive.config.js';
-import { fromArchiveRecord, fromPrivateNote } from '../adapters/teaching.js';
+import { fromArchiveRecord, fromPrivateBook, fromPrivateNote } from '../adapters/teaching.js';
 import type { Entity, EntityPolicy, Fragment, SemanticProjection } from '../contract.js';
-import { buildCorpus, buildPrivateNotes } from '../corpus.js';
+import { buildCorpus, buildPrivateBooks, buildPrivateNotes } from '../corpus.js';
 import { embedStringFor } from '../embed-string.js';
 import { batchInputs, embedBatch, truncateForEmbedding } from '../embedding.js';
 import { collectEntities } from '../ingest/collect.js';
@@ -107,9 +108,15 @@ async function main(): Promise<void> {
     );
   }
   const notes = buildPrivateNotes(config);
-  console.log(`Corpus: ${records.length} records, ${notes.length} private notes`);
-
-  const sources: Source[] = [...records.map(fromArchiveRecord), ...notes.map(fromPrivateNote)];
+  const books = buildPrivateBooks(config);
+  console.log(`Corpus: ${records.length} records, ${notes.length} private notes, ${books.length} private books`);
+
+  // A record or a note is one fragment; a book is one entity with many.
+  const sources: Source[] = [
+    ...records.map(fromArchiveRecord),
+    ...notes.map(fromPrivateNote),
+    ...books.flatMap(fromPrivateBook),
+  ];
   const lint = lintPolicy(config.gist);
   if (lint) for (const { entity } of sources) entity.policy = { ...entity.policy, lint };
   // Entities once, every fragment under its entity (CONTRACT.md §12). The