From 8a46f0af2e773d784fcc915d2fc47cdb685cbf8f Mon Sep 17 00:00:00 2001 From: "mads.thines" Date: Wed, 14 Oct 2026 22:13:41 +0000 Subject: [PATCH 01/16] fix(byod): generate bootstrap.sql from the migrations The hand-written BYOD schema had drifted: it defined 6 of the 48 RPCs the edge calls, and its memory_write still took p_ttl_days while both write paths send p_ttl_seconds, p_kind, p_host and p_origin_*, so PostgREST cannot resolve the call (PGRST202). Generate the file from the migrations instead: one transaction, a guard against re-running, and the migration history recorded so the database upgrades with supabase db push. Co-authored-by: dash0-dev[bot] <257284812+dash0-dev[bot]@users.noreply.github.com> --- scripts/codegen/gen-byod-bootstrap.mjs | 184 + scripts/codegen/gen-byod-bootstrap.test.mjs | 62 + supabase/byod/bootstrap.sql | 22491 +++++++++++++++++- 3 files changed, 22227 insertions(+), 510 deletions(-) create mode 100644 scripts/codegen/gen-byod-bootstrap.mjs create mode 100644 scripts/codegen/gen-byod-bootstrap.test.mjs diff --git a/scripts/codegen/gen-byod-bootstrap.mjs b/scripts/codegen/gen-byod-bootstrap.mjs new file mode 100644 index 000000000..e6ea43d6d --- /dev/null +++ b/scripts/codegen/gen-byod-bootstrap.mjs @@ -0,0 +1,184 @@ +#!/usr/bin/env node +// Generate the BYOD ("bring your own database") bootstrap from the migrations. +// Source of truth: supabase/migrations/*.sql (applied in filename order) +// Generated file: supabase/byod/bootstrap.sql +// +// node scripts/codegen/gen-byod-bootstrap.mjs regenerate bootstrap.sql +// node scripts/codegen/gen-byod-bootstrap.mjs --check exit 1 if it is stale (CI) +// +// Why generated: bootstrap.sql used to be a hand-written SUBSET of the schema. +// The edge function kept moving with the migrations and the subset did not — +// by 00113 it defined 6 of the 48 RPCs the edge calls, and its `memory_write` +// still took `p_ttl_days` while both write paths send `p_ttl_seconds`, `p_kind`, +// `p_host` and `p_origin_*`. PostgREST resolves an RPC by argument NAME, so a +// BYOD database built from it misses the function entirely (PGRST202). A second +// hand-written copy of the schema drifts; a projection of the migrations cannot. +// +// The output is the migrations, verbatim and in order, inside ONE transaction: +// * a guard refuses to run against a database that already holds the schema, +// so a second paste into the SQL Editor fails closed instead of half-applying; +// * every migration's version is recorded in Supabase's migration history +// (`supabase_migrations.schema_migrations`), so a database bootstrapped from +// this file upgrades with `supabase db push` like any linked project. +// +// Zero-dependency (node built-ins only), deterministic (no timestamps), so the +// --check comparison is a plain string equality. +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const ROOT = fileURLToPath(new URL('../../', import.meta.url)); +export const MIGRATIONS_DIR = path.join(ROOT, 'supabase/migrations'); +export const OUTPUT_PATH = path.join(ROOT, 'supabase/byod/bootstrap.sql'); + +// The Supabase CLI's own migration filename grammar: `_.sql`. +const MIGRATION_FILE = /^(\d+)_(.+)\.sql$/; + +/** + * Parse a migration filename into the (version, name) pair the Supabase CLI + * records. Returns null for a file the CLI would not treat as a migration. + */ +export function parseMigrationName(file) { + const m = MIGRATION_FILE.exec(file); + return m ? { version: m[1], name: m[2] } : null; +} + +/** Quote a value as a SQL string literal. */ +function sqlLiteral(value) { + return `'${String(value).replaceAll("'", "''")}'`; +} + +function banner(text) { + const rule = '-- ' + '─'.repeat(77); + return `${rule}\n-- ${text}\n${rule}`; +} + +/** + * Render bootstrap.sql from an ordered list of migrations. + * @param {{ file: string, sql: string }[]} migrations already sorted by filename + * @returns {string} + */ +export function renderBootstrap(migrations) { + if (migrations.length === 0) throw new Error('no migrations to render'); + const parsed = migrations.map(({ file, sql }) => { + const id = parseMigrationName(file); + if (!id) throw new Error(`not a migration filename: ${file}`); + return { file, sql: sql.endsWith('\n') ? sql : `${sql}\n`, ...id }; + }); + const newest = parsed[parsed.length - 1].file; + + const header = `-- LoreKit BYOD bootstrap schema +-- +-- ⚠ GENERATED FILE — DO NOT EDIT BY HAND. +-- Generated by \`node scripts/codegen/gen-byod-bootstrap.mjs\` from the +-- ${parsed.length} migrations in supabase/migrations/ (newest: ${newest}). +-- CI fails when this file and the migrations disagree; edit a migration (or add +-- one) and regenerate instead. +-- +-- What it does: applies the FULL LoreKit schema — the same migrations the hosted +-- project runs — to an empty Supabase project, in one transaction, and records +-- each migration in Supabase's migration history. +-- +-- Fresh database only. A guard below aborts (and rolls everything back) when the +-- LoreKit schema is already present. Apply with ON_ERROR_STOP so psql stops at +-- the first error instead of carrying on past it: +-- +-- psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -f supabase/byod/bootstrap.sql +-- +-- or paste the whole file into the Supabase dashboard → SQL Editor and run it. +-- $DATABASE_URL is the connection string from the Supabase dashboard → Connect; +-- its password is the DATABASE password, not the service-role API key. +-- +-- Upgrades: because the migration history is recorded, a later LoreKit release +-- upgrades the database with the Supabase CLI from a checkout of the repo: +-- +-- supabase link --project-ref +-- supabase db push +-- +-- Billing: memories stored in your own database are NOT counted against +-- LoreKit's hosted memory-count billing — LoreKit has no visibility into it. +-- The memory cap and rate limit are read from \`lorekit_default_limit\` / +-- \`user_limits\` in YOUR database; adjust them there. +-- +-- See docs/byod.md for the full setup. + +begin; + +-- Fail closed on a database that already holds the schema: re-running the +-- migrations from the top would collide with the existing objects partway +-- through. Raising here rolls back the transaction before anything changes. +do $guard$ +begin + if to_regclass('public.memories') is not null then + raise exception using + message = 'LoreKit schema already present: bootstrap.sql only initialises an empty database.', + hint = 'Upgrade an existing BYOD database with \`supabase db push\` (see docs/byod.md).'; + end if; +end +$guard$; + +-- The Supabase CLI's migration-history table. \`if not exists\` keeps this +-- compatible with a project the CLI has already touched; the CLI adds any +-- further columns it needs on its next run. +create schema if not exists supabase_migrations; +create table if not exists supabase_migrations.schema_migrations ( + version text not null primary key, + statements text[], + name text +); +`; + + const body = parsed + .map( + ({ file, sql, version, name }) => ` +${banner(file)} + +${sql} +insert into supabase_migrations.schema_migrations (version, name) +values (${sqlLiteral(version)}, ${sqlLiteral(name)}) +on conflict (version) do nothing; +`, + ) + .join(''); + + return `${header}${body} +commit; +`; +} + +/** Read every migration in filename order. */ +export function readMigrations(dir = MIGRATIONS_DIR) { + return fs + .readdirSync(dir) + .filter((f) => parseMigrationName(f)) + .sort() + .map((file) => ({ file, sql: fs.readFileSync(path.join(dir, file), 'utf8') })); +} + +function main() { + const expected = renderBootstrap(readMigrations()); + const rel = path.relative(ROOT, OUTPUT_PATH); + + if (process.argv.includes('--check')) { + let actual = null; + try { + actual = fs.readFileSync(OUTPUT_PATH, 'utf8'); + } catch { + // Missing file: reported as stale below. + } + if (actual !== expected) { + console.error(`${rel} is out of date with supabase/migrations/.`); + console.error('Run: node scripts/codegen/gen-byod-bootstrap.mjs'); + process.exit(1); + } + console.log(`${rel} matches supabase/migrations/.`); + return; + } + + fs.writeFileSync(OUTPUT_PATH, expected); + console.log(`Wrote ${rel}.`); +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main(); +} diff --git a/scripts/codegen/gen-byod-bootstrap.test.mjs b/scripts/codegen/gen-byod-bootstrap.test.mjs new file mode 100644 index 000000000..074d33fcc --- /dev/null +++ b/scripts/codegen/gen-byod-bootstrap.test.mjs @@ -0,0 +1,62 @@ +// Unit tests for the BYOD bootstrap generator's pure logic. +// node --test scripts/codegen/gen-byod-bootstrap.test.mjs +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { + parseMigrationName, + renderBootstrap, + readMigrations, + OUTPUT_PATH, +} from './gen-byod-bootstrap.mjs'; + +test('parseMigrationName follows the Supabase CLI grammar', () => { + assert.deepEqual(parseMigrationName('00001_memories.sql'), { version: '00001', name: 'memories' }); + assert.deepEqual(parseMigrationName('00113_a_b_c.sql'), { version: '00113', name: 'a_b_c' }); + assert.equal(parseMigrationName('README.md'), null); + assert.equal(parseMigrationName('memories.sql'), null); +}); + +test('renderBootstrap keeps migrations verbatim, in order, inside one transaction', () => { + const out = renderBootstrap([ + { file: '00001_first.sql', sql: 'create table a (id int);' }, + { file: '00002_second.sql', sql: 'create table b (id int);\n' }, + ]); + const a = out.indexOf('create table a (id int);'); + const b = out.indexOf('create table b (id int);'); + assert.ok(a > 0 && b > a, 'migrations appear in the given order'); + assert.ok(out.indexOf('\nbegin;\n') < a, 'transaction opens before the first migration'); + assert.ok(out.trimEnd().endsWith('commit;'), 'transaction commits at the end'); + assert.equal(out.match(/^begin;$/gm).length, 1, 'exactly one transaction'); +}); + +test('renderBootstrap records every version in the Supabase migration history', () => { + const out = renderBootstrap([ + { file: '00001_first.sql', sql: 'select 1;' }, + { file: '00002_it\'s.sql', sql: 'select 2;' }, + ]); + assert.match(out, /values \('00001', 'first'\)/); + assert.match(out, /values \('00002', 'it''s'\)/, 'quotes are escaped'); + assert.match(out, /create table if not exists supabase_migrations\.schema_migrations/); +}); + +test('renderBootstrap guards against re-running on an initialised database', () => { + const out = renderBootstrap([{ file: '00001_first.sql', sql: 'select 1;' }]); + const guard = out.indexOf("to_regclass('public.memories') is not null"); + assert.ok(guard > 0 && guard < out.indexOf('select 1;'), 'guard runs before any migration'); +}); + +test('renderBootstrap is deterministic and rejects bad input', () => { + const input = [{ file: '00001_first.sql', sql: 'select 1;' }]; + assert.equal(renderBootstrap(input), renderBootstrap(input)); + assert.throws(() => renderBootstrap([]), /no migrations/); + assert.throws(() => renderBootstrap([{ file: 'nope.sql', sql: '' }]), /not a migration filename/); +}); + +test('the committed bootstrap.sql is generated from the current migrations', () => { + assert.equal( + fs.readFileSync(OUTPUT_PATH, 'utf8'), + renderBootstrap(readMigrations()), + 'run: node scripts/codegen/gen-byod-bootstrap.mjs', + ); +}); diff --git a/supabase/byod/bootstrap.sql b/supabase/byod/bootstrap.sql index 598cfa402..e70071838 100644 --- a/supabase/byod/bootstrap.sql +++ b/supabase/byod/bootstrap.sql @@ -1,53 +1,73 @@ --- LoreKit BYOD Bootstrap Schema --- Apply this file to your own Supabase project to enable LoreKit memory.* tools. +-- LoreKit BYOD bootstrap schema -- --- Usage: --- psql "$DATABASE_URL" -f supabase/byod/bootstrap.sql --- # or via lorekit CLI: --- lorekit bootstrap +-- ⚠ GENERATED FILE — DO NOT EDIT BY HAND. +-- Generated by `node scripts/codegen/gen-byod-bootstrap.mjs` from the +-- 108 migrations in supabase/migrations/ (newest: 00113_explorer_reverse_activity_filters_and_opened_sort.sql). +-- CI fails when this file and the migrations disagree; edit a migration (or add +-- one) and regenerate instead. -- --- This is a subset of the full LoreKit schema. Org sharing, audit logs, --- webhook secrets, and billing tables are NOT included — those require --- the hosted LoreKit product. +-- What it does: applies the FULL LoreKit schema — the same migrations the hosted +-- project runs — to an empty Supabase project, in one transaction, and records +-- each migration in Supabase's migration history. -- --- Billing note: --- Memories stored in your own database are NOT counted against LoreKit's --- hosted memory-count billing. LoreKit has no visibility into your private --- database and cannot meter memories stored there. BYOD users are on a --- flat-rate or open-source tier. You are responsible for configuring memory --- limits and rate limiting in your own project. +-- Fresh database only. A guard below aborts (and rolls everything back) when the +-- LoreKit schema is already present. Apply with ON_ERROR_STOP so psql stops at +-- the first error instead of carrying on past it: -- --- Schema upgrades: --- When LoreKit ships schema changes that affect the memory.* tools, you will --- need to apply updates manually. Run `lorekit doctor` to check compatibility. - --- ───────────────────────────────────────────────────────────────────────────── --- 0. Extensions --- ───────────────────────────────────────────────────────────────────────────── - -create extension if not exists "uuid-ossp"; +-- psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -f supabase/byod/bootstrap.sql +-- +-- or paste the whole file into the Supabase dashboard → SQL Editor and run it. +-- $DATABASE_URL is the connection string from the Supabase dashboard → Connect; +-- its password is the DATABASE password, not the service-role API key. +-- +-- Upgrades: because the migration history is recorded, a later LoreKit release +-- upgrades the database with the Supabase CLI from a checkout of the repo: +-- +-- supabase link --project-ref +-- supabase db push +-- +-- Billing: memories stored in your own database are NOT counted against +-- LoreKit's hosted memory-count billing — LoreKit has no visibility into it. +-- The memory cap and rate limit are read from `lorekit_default_limit` / +-- `user_limits` in YOUR database; adjust them there. +-- +-- See docs/byod.md for the full setup. --- ───────────────────────────────────────────────────────────────────────────── --- 1. Shared trigger function --- ───────────────────────────────────────────────────────────────────────────── +begin; -create or replace function set_updated_at() -returns trigger language plpgsql as $$ +-- Fail closed on a database that already holds the schema: re-running the +-- migrations from the top would collide with the existing objects partway +-- through. Raising here rolls back the transaction before anything changes. +do $guard$ begin - new.updated_at = now(); - return new; -end; -$$; + if to_regclass('public.memories') is not null then + raise exception using + message = 'LoreKit schema already present: bootstrap.sql only initialises an empty database.', + hint = 'Upgrade an existing BYOD database with `supabase db push` (see docs/byod.md).'; + end if; +end +$guard$; + +-- The Supabase CLI's migration-history table. `if not exists` keeps this +-- compatible with a project the CLI has already touched; the CLI adds any +-- further columns it needs on its next run. +create schema if not exists supabase_migrations; +create table if not exists supabase_migrations.schema_migrations ( + version text not null primary key, + statements text[], + name text +); -- ───────────────────────────────────────────────────────────────────────────── --- 2. memories table +-- 00001_memories.sql -- ───────────────────────────────────────────────────────────────────────────── +-- LoreKit memories table +-- Stores agent memory/lessons with canonical scope strings, full-text search, and RLS. + create table if not exists memories ( id uuid primary key default gen_random_uuid(), user_id uuid references auth.users on delete cascade, - -- org_id is accepted in the schema for compatibility with the hosted write RPC - -- signature, but BYOD does not enforce org membership (no orgs table). org_id text, scope text not null, key text not null, @@ -55,66 +75,37 @@ create table if not exists memories ( tags text[] not null default '{}', source_agent text, trigger text, - -- Author attribution columns (additive, nullable). - created_by uuid references auth.users on delete set null, - updated_by uuid references auth.users on delete set null, - -- Generated FTS column for full-text search. + -- Generated FTS column for full-text search via to_tsvector fts tsvector generated always as ( to_tsvector('english', coalesce(key, '') || ' ' || coalesce(value, '')) ) stored, created_at timestamptz not null default now(), updated_at timestamptz not null default now(), - -- Soft-delete: archived rows are hidden from normal reads. - archived_at timestamptz, - -- Optional TTL: when set, the row is treated as invisible after this timestamp. - expires_at timestamptz + -- user_id can be null for service-role writes (CI); uniqueness still holds + constraint memories_user_scope_key_unique unique nulls not distinct (user_id, scope, key) ); -- Indexes -create index if not exists memories_fts_idx on memories using gin(fts); -create index if not exists memories_scope_idx on memories (scope); -create index if not exists memories_user_idx on memories (user_id); -create index if not exists memories_scope_key on memories (scope, key); -create index if not exists memories_archived_at_idx on memories (archived_at) - where archived_at is not null; -create index if not exists memories_expires_at_idx on memories (expires_at) - where expires_at is not null; - --- Partial unique indexes (matching the hosted schema's ON CONFLICT predicates). --- Drop the old plain unique constraint if it exists from a previous install. -alter table memories drop constraint if exists memories_user_scope_key_unique; - -create unique index if not exists memories_user_scope_key_active_unique - on memories (user_id, scope, key) - where org_id is null and user_id is not null and archived_at is null; - -create unique index if not exists memories_null_user_scope_key_active_unique - on memories (scope, key) - where org_id is null and user_id is null and archived_at is null; +create index if not exists memories_fts_idx on memories using gin(fts); +create index if not exists memories_scope_idx on memories (scope); +create index if not exists memories_user_idx on memories (user_id); +create index if not exists memories_scope_key on memories (scope, key); --- Enable RLS +-- Enable Row Level Security alter table memories enable row level security; --- RLS: users can read their own active (non-archived) rows. -drop policy if exists "rls_read" on memories; +-- Policy: users can read their own rows, or rows where org_id matches their JWT claim create policy "rls_read" on memories for select using ( - archived_at is null - and user_id = auth.uid() - ); - --- RLS: users can read their own archived rows (dashboard archive view). -drop policy if exists "rls_read_archived" on memories; -create policy "rls_read_archived" - on memories for select - using ( - archived_at is not null - and user_id = auth.uid() + user_id = auth.uid() + or ( + org_id is not null + and org_id = (auth.jwt() ->> 'org_id') + ) ); --- RLS: users can insert their own rows; service_role bypasses via superuser. -drop policy if exists "rls_insert" on memories; +-- Policy: users can insert their own rows; service_role bypasses via superuser create policy "rls_insert" on memories for insert with check ( @@ -122,8 +113,7 @@ create policy "rls_insert" or auth.role() = 'service_role' ); --- RLS: users can update their own rows. -drop policy if exists "rls_update" on memories; +-- Policy: users can update their own rows create policy "rls_update" on memories for update using ( @@ -131,8 +121,7 @@ create policy "rls_update" or auth.role() = 'service_role' ); --- RLS: users can delete their own rows. -drop policy if exists "rls_delete" on memories; +-- Policy: users can delete their own rows create policy "rls_delete" on memories for delete using ( @@ -140,205 +129,265 @@ create policy "rls_delete" or auth.role() = 'service_role' ); --- Trigger to keep updated_at current. -drop trigger if exists memories_updated_at on memories; -create trigger memories_updated_at +-- Trigger to keep updated_at current +create or replace function set_updated_at() +returns trigger language plpgsql as $$ +begin + new.updated_at = now(); + return new; +end; +$$; + +create or replace trigger memories_updated_at before update on memories for each row execute function set_updated_at(); +insert into supabase_migrations.schema_migrations (version, name) +values ('00001', 'memories') +on conflict (version) do nothing; + -- ───────────────────────────────────────────────────────────────────────────── --- 3. api_tokens table +-- 00002_api_tokens.sql -- ───────────────────────────────────────────────────────────────────────────── +-- LoreKit API tokens +-- Allows users to generate durable tokens for agent/CI authentication. +-- Tokens are NEVER stored in plain text — only as SHA-256 hashes. +-- The token prefix (first 12 chars) is stored for display/identification. +-- +-- Token format: lk_{perm}_{32 random alphanumeric chars} +-- lk_rw_... read+write (all 5 MCP tools) +-- lk_ro_... read-only (memory.read, memory.list, memory.search only) + create table if not exists api_tokens ( id uuid primary key default gen_random_uuid(), user_id uuid references auth.users on delete cascade not null, name text not null check (length(name) >= 1 and length(name) <= 100), - -- First 12 chars + "..." for display — safe to store, max 16 chars. + -- First 12 chars + "..." for display ("lk_rw_aBcD1...") — safe to store, max 16 chars token_prefix text not null check (length(token_prefix) <= 16), - -- SHA-256 hex of the full token — used for lookup on every request. + -- SHA-256 hex of the full token — used for lookup on every request token_hash text not null unique, - -- Array of granted permissions: 'read' | 'write'. + -- Array of granted permissions: 'read' | 'write' permissions text[] not null default '{"read","write"}', - -- Scoping (migration 00068). EMPTY scopes = unrestricted; org_access is a - -- tri-state and org_ids is non-empty iff org_access = 'selected'. - scopes text[] not null default '{}', - org_access text not null default 'all', - org_ids uuid[] not null default '{}', last_used_at timestamptz, created_at timestamptz not null default now() ); --- `create table if not exists` above is a no-op on an install that predates --- 00068, so the columns are added separately for that case. -alter table api_tokens - add column if not exists scopes text[] not null default '{}', - add column if not exists org_access text not null default 'all', - add column if not exists org_ids uuid[] not null default '{}'; +create index if not exists api_tokens_user_idx on api_tokens(user_id); +create index if not exists api_tokens_hash_idx on api_tokens(token_hash); -create index if not exists api_tokens_user_idx on api_tokens(user_id); -create index if not exists api_tokens_hash_idx on api_tokens(token_hash); +alter table api_tokens enable row level security; --- Shape gate for the scope allowlist. A CHECK cannot hold a subquery, and --- "every element satisfies P" needs one — hence the immutable helper. -create or replace function lorekit_api_token_scopes_valid(p_patterns text[]) -returns boolean -language sql -immutable -as $$ - select p_patterns is null or not exists ( - select 1 - from unnest(p_patterns) as t(pattern) - where (t.pattern ~ '^[a-z0-9._:/-]+(/|::)\*$' or t.pattern ~ '^[a-z0-9._:/-]+$') is not true - or (length(t.pattern) <= 200) is not true - ); -$$; +-- Users can only see and manage their own tokens +create policy "rls_api_tokens_select" + on api_tokens for select + using (user_id = auth.uid()); -alter table api_tokens drop constraint if exists api_tokens_scopes_len; -alter table api_tokens add constraint api_tokens_scopes_len - check (cardinality(scopes) <= 50); +create policy "rls_api_tokens_insert" + on api_tokens for insert + with check (user_id = auth.uid()); -alter table api_tokens drop constraint if exists api_tokens_scopes_shape; -alter table api_tokens add constraint api_tokens_scopes_shape - check (lorekit_api_token_scopes_valid(scopes)); +create policy "rls_api_tokens_delete" + on api_tokens for delete + using (user_id = auth.uid()); -alter table api_tokens drop constraint if exists api_tokens_org_access_valid; -alter table api_tokens add constraint api_tokens_org_access_valid - check (org_access in ('all', 'personal', 'selected')); +-- Service role can look up tokens for auth validation (bypasses RLS) +-- No update policy needed — last_used_at is updated via service role -alter table api_tokens drop constraint if exists api_tokens_org_ids_match_access; -alter table api_tokens add constraint api_tokens_org_ids_match_access - check ((org_access = 'selected') = (cardinality(org_ids) > 0)); +insert into supabase_migrations.schema_migrations (version, name) +values ('00002', 'api_tokens') +on conflict (version) do nothing; -alter table api_tokens drop constraint if exists api_tokens_org_ids_len; -alter table api_tokens add constraint api_tokens_org_ids_len - check (cardinality(org_ids) <= 50); +-- ───────────────────────────────────────────────────────────────────────────── +-- 00003_archive.sql +-- ───────────────────────────────────────────────────────────────────────────── --- `{null}` has cardinality 1, so neither CHECK above catches a NULL element, --- and a NULL org id on an authorization column is "unknown", not "none". --- Mirrored from 00068. -alter table api_tokens drop constraint if exists api_tokens_org_ids_not_null; -alter table api_tokens add constraint api_tokens_org_ids_not_null - check (array_position(org_ids, null) is null); +-- LoreKit archive / soft-delete support +-- +-- Adds archived_at to memories so agents and the dashboard can soft-delete a +-- lesson (archive) instead of hard-deleting it. Archived rows are hidden from +-- all normal reads (RLS select policy, tool queries) but can be listed via the +-- memory.list_archived MCP tool or the dashboard Archive tab. +-- +-- A pg_cron job (or the memory.purge RPC called by the dashboard) hard-deletes +-- rows where archived_at < now() - (retention_days * interval '1 day'). --- The two request-time predicates. Kept byte-identical to the CURRENT hosted --- definitions — `lorekit_api_token_scope_allowed` as re-issued by 00069 §8, --- `lorekit_api_token_org_allowed` as first issued by 00068 — a BYOD install --- that answers these differently is a BYOD install with a different --- authorization boundary. Mirror the LATEST definition, never the one the --- column was introduced with: 00068's looser scope predicate treated any --- trailing `*` as a prefix wildcard, so a stored mid-token `*` WIDENED the key. --- --- `lorekit_api_token_set_scoping` is deliberately NOT mirrored: it validates --- org ids against `lorekit_member_org_ids`, and this file has no orgs by design --- (see the header). The org columns are still created so the row shape the --- transports select is identical on both deployments — with no orgs, every row --- is personal and `lorekit_api_token_org_allowed` returns true regardless of --- the tenancy, so the tri-state is inert rather than wrong. -create or replace function lorekit_api_token_scope_allowed( - p_patterns text[], - p_scope text -) -returns boolean -language sql -immutable -as $$ - select case - when p_patterns is null or cardinality(p_patterns) = 0 then true - when p_scope is null then false - else exists ( - select 1 - from unnest(p_patterns) as pattern - -- SCOPE_PATTERN's shape, verbatim (00069 §8). A `*` is a wildcard only - -- directly after `/` or `::`; a pattern that fails this test is DROPPED - -- rather than matched, so a stored mid-token wildcard can only ever - -- narrow the key. The guard is needed even though `api_tokens_scopes_shape` - -- exists, because the column can hold a value the CHECK never saw — a - -- BYOD install bootstrapped before this file grew the constraint, or a - -- constraint dropped by hand. - where pattern ~ '^[a-z0-9._:/-]+((/|::)\*)?$' - and case - when right(pattern, 1) = '*' - -- Escape LIKE's single-character wildcard in the literal prefix so - -- `repo::my_org/*` stays owner-exact instead of also matching - -- `repo::myXorg/...`. `%` and `\` cannot occur — the shape test's - -- charset excludes them. - then p_scope like replace(left(pattern, -1), '_', '\_') || '%' - else p_scope = pattern - end +-- 1. Add the soft-delete timestamp column. +alter table memories + add column if not exists archived_at timestamptz; + +-- 2. Index for efficient purge sweeps (only touches archived rows). +create index if not exists memories_archived_at_idx + on memories (archived_at) + where archived_at is not null; + +-- 3. Drop the old unique constraint and recreate it excluding archived rows. +-- This allows the same (user_id, scope, key) to be re-created after an archive. +alter table memories + drop constraint if exists memories_user_scope_key_unique; + +create unique index if not exists memories_user_scope_key_active_unique + on memories (user_id, scope, key) + where archived_at is null; + +-- Nulls-not-distinct variant for rows where user_id IS NULL (service-role writes). +create unique index if not exists memories_null_user_scope_key_active_unique + on memories (scope, key) + where user_id is null and archived_at is null; + +-- 4. Update the RLS select policy to hide archived rows from normal reads. +drop policy if exists "rls_read" on memories; + +create policy "rls_read" + on memories for select + using ( + archived_at is null + and ( + user_id = auth.uid() + or ( + org_id is not null + and org_id = (auth.jwt() ->> 'org_id') + ) ) - end; -$$; + ); -create or replace function lorekit_api_token_org_allowed( - p_org_access text, - p_org_ids uuid[], - p_org_id uuid +-- 5. RLS policy for reading archived rows (dashboard archive view). +create policy "rls_read_archived" + on memories for select + using ( + archived_at is not null + and ( + user_id = auth.uid() + or ( + org_id is not null + and org_id = (auth.jwt() ->> 'org_id') + ) + ) + ); + +-- 6. Soft-archive RPC — sets archived_at, returns the row id. +-- Called by the MCP memory.archive tool and the dashboard server action. +-- Uses SECURITY DEFINER so the RLS update policy (user_id = auth.uid()) +-- still applies — the function runs as the table owner but only touches +-- the row if the caller passes their own user_id. +create or replace function archive_memory( + p_user_id uuid, + p_scope text, + p_key text ) -returns boolean -language sql -immutable +returns uuid +language plpgsql +security definer +set search_path = public as $$ - select case - when p_org_id is null then true - when p_org_access = 'all' then true - when p_org_access = 'personal' then false - -- `coalesce(…, false)`: a NULL element in p_org_ids makes `= any(…)` NULL, - -- and an authorization predicate must never return NULL. Kept identical to - -- 00068 — see the rationale there. - when p_org_access = 'selected' - then coalesce(p_org_id = any(coalesce(p_org_ids, '{}'::uuid[])), false) - else false - end; -$$; +declare + v_id uuid; +begin + update memories + set archived_at = now() + where user_id = p_user_id + and scope = p_scope + and key = p_key + and archived_at is null + returning id into v_id; --- The GRANTS are part of the mirror, not decoration: a `create function` is --- PUBLIC-executable by default, so omitting these left a BYOD install with three --- authorization predicates any role could call. Byte-for-byte bodies with --- different grants is not a mirror. -revoke execute on function lorekit_api_token_scopes_valid(text[]) from public, anon; -revoke execute on function lorekit_api_token_scope_allowed(text[], text) from public, anon; -revoke execute on function lorekit_api_token_org_allowed(text, uuid[], uuid) from public, anon; -grant execute on function lorekit_api_token_scopes_valid(text[]) to authenticated, service_role; -grant execute on function lorekit_api_token_scope_allowed(text[], text) to authenticated, service_role; -grant execute on function lorekit_api_token_org_allowed(text, uuid[], uuid) to authenticated, service_role; + return v_id; -- null if row not found or already archived +end; +$$; -alter table api_tokens enable row level security; +-- 7. Restore-from-archive RPC — clears archived_at. +create or replace function restore_memory( + p_user_id uuid, + p_scope text, + p_key text +) +returns uuid +language plpgsql +security definer +set search_path = public +as $$ +declare + v_id uuid; +begin + update memories + set archived_at = null + where user_id = p_user_id + and scope = p_scope + and key = p_key + and archived_at is not null + returning id into v_id; -drop policy if exists "rls_api_tokens_select" on api_tokens; -create policy "rls_api_tokens_select" - on api_tokens for select - using (user_id = auth.uid()); + return v_id; +end; +$$; -drop policy if exists "rls_api_tokens_insert" on api_tokens; -create policy "rls_api_tokens_insert" - on api_tokens for insert - with check (user_id = auth.uid()); +-- 8. Purge RPC — hard-deletes archived rows older than retention_days. +-- Returns the count of deleted rows. +-- Intended to be called by a scheduled job or the dashboard "Run purge" button. +create or replace function purge_archived_memories( + p_user_id uuid, + p_retention_days integer default 30 +) +returns integer +language plpgsql +security definer +set search_path = public +as $$ +declare + v_count integer; +begin + delete from memories + where user_id = p_user_id + and archived_at is not null + and archived_at < now() - (p_retention_days * interval '1 day') + ; + get diagnostics v_count = row_count; + return v_count; +end; +$$; -drop policy if exists "rls_api_tokens_delete" on api_tokens; -create policy "rls_api_tokens_delete" - on api_tokens for delete - using (user_id = auth.uid()); +insert into supabase_migrations.schema_migrations (version, name) +values ('00003', 'archive') +on conflict (version) do nothing; -- ───────────────────────────────────────────────────────────────────────────── --- 4. Limits: default config, per-user overrides, cap trigger +-- 00004_limits.sql -- ───────────────────────────────────────────────────────────────────────────── --- 4a. Single source of truth for default limits. --- BYOD users are responsible for adjusting these in their own project. +-- LoreKit abuse guardrails: per-user memory cap + request rate limiting. +-- +-- Two enforcement planes sharing one config source: +-- * Cap plane (authoritative): enforce_memory_cap() BEFORE INSERT trigger on +-- memories, keyed off NEW.user_id. Counts active (non-archived) rows and +-- rejects the insert once the user's max_memories limit is reached. Upsert +-- updates (existing key) never trip this — only NEW rows go through +-- BEFORE INSERT. +-- * Rate plane (transport): lorekit_check_rate_limit() RPC, a Postgres-backed +-- fixed-window counter. Called by the app layer (Deno edge fn / Node +-- server) right after auth resolves, per request, per user. +-- +-- Config source: lorekit_default_limit(key) is the single free-tier default; +-- user_limits holds nullable per-user overrides; lorekit_get_limit resolves +-- COALESCE(override, default). No numeric limit is hardcoded in app code. +-- +-- Service-role (NEW.user_id / p_user_id IS NULL — CI/internal) is exempt from +-- both guardrails. + +-- 1. Single source of truth for free-tier default limits. create or replace function lorekit_default_limit(p_key text) returns integer language sql immutable as $$ select case p_key - when 'max_memories' then 5000 + when 'max_memories' then 1000 when 'requests_per_minute' then 120 else null end; $$; --- 4b. Per-user override table. +-- 2. Per-user override table. Absence of a row (or a null column) means the +-- user is on the free-tier default — lorekit_get_limit() never returns null. create table if not exists user_limits ( user_id uuid primary key references auth.users on delete cascade, max_memories integer, @@ -349,17 +398,20 @@ create table if not exists user_limits ( alter table user_limits enable row level security; -drop policy if exists "rls_user_limits_select" on user_limits; +-- Users can see their own override row (so the dashboard/CLI can show "your +-- limit"); raising a limit is a service-role/admin upsert for now — no +-- insert/update policy for regular users. create policy "rls_user_limits_select" on user_limits for select using (user_id = auth.uid()); -drop trigger if exists user_limits_updated_at on user_limits; -create trigger user_limits_updated_at +create or replace trigger user_limits_updated_at before update on user_limits for each row execute function set_updated_at(); --- 4c. Resolve effective limit for a user. +-- 3. Resolve the effective limit for a user: override if set, else the +-- free-tier default. Security definer so it can read user_limits +-- regardless of the caller's RLS visibility (needed inside the trigger). create or replace function lorekit_get_limit(p_user_id uuid, p_key text) returns integer language plpgsql @@ -379,8 +431,12 @@ begin end; $$; --- 4d. Memory cap trigger — counts active (non-archived) rows and rejects the --- insert at/over the limit. Service-role (user_id IS NULL) is exempt. +-- 4. Cap trigger — the authoritative guardrail. Counts NEW.user_id's active +-- (archived_at IS NULL) rows and rejects the insert at/over the limit. +-- Service-role / CI writes (NEW.user_id IS NULL) are exempt. +-- Raises a custom SQLSTATE ('LK001') so the app layer can distinguish this +-- from any other DB error and translate it into an actionable LimitError +-- (see packages/mcp-core/src/limits.ts and supabase/functions/mcp/limits.ts). create or replace function enforce_memory_cap() returns trigger language plpgsql @@ -412,16 +468,12 @@ begin end; $$; -drop trigger if exists memories_enforce_cap on memories; -create trigger memories_enforce_cap +create or replace trigger memories_enforce_cap before insert on memories for each row execute function enforce_memory_cap(); --- ───────────────────────────────────────────────────────────────────────────── --- 5. Rate limiting --- ───────────────────────────────────────────────────────────────────────────── - --- 5a. Rate-limit counter table. +-- 5. Rate-limit counter table — one tiny row per (user, window). Postgres- +-- backed (not in-memory) because edge isolates are stateless/short-lived. create table if not exists rate_limit_counters ( user_id uuid not null, window_start timestamptz not null, @@ -433,9 +485,12 @@ create index if not exists rate_limit_counters_window_idx on rate_limit_counters (window_start); alter table rate_limit_counters enable row level security; --- No policies: only touched via the SECURITY DEFINER RPC below. +-- No policies: this table is a transport-layer implementation detail, only +-- ever touched via the security-definer RPC below (service-role / RPC path). --- 5b. Atomic fixed-window rate-limit check. +-- 6. Atomic fixed-window rate-limit check. Increments the current window's +-- counter and returns whether the request is allowed, plus how long until +-- the next window opens (Retry-After). create or replace function lorekit_check_rate_limit( p_user_id uuid, p_window_seconds integer default 60 @@ -474,7 +529,12 @@ begin end; $$; --- 5c. Reaper for stale rate-limit windows. +-- 7. Reaper for stale rate-limit windows. rate_limit_counters gains one row per +-- (user, window) on every request, but only the current window is ever read +-- — anything older is dead weight that bloats the table and its index and +-- slowly degrades the RPC above (which runs on every authenticated request). +-- Hard-delete windows older than p_older_than. Mirrors purge_archived_memories +-- in 00003_archive.sql. Returns the count of deleted rows. create or replace function lorekit_purge_rate_limit_counters( p_older_than interval default interval '1 hour' ) @@ -493,185 +553,20301 @@ begin end; $$; +-- Schedule the reaper every 15 minutes when pg_cron is available. Guarded so +-- the migration still applies on instances without the extension — the function +-- above can then be driven by an external scheduler or the dashboard instead. +-- Idempotent: cron.schedule() upserts by job name. +do $$ +begin + if exists (select 1 from pg_extension where extname = 'pg_cron') then + perform cron.schedule( + 'lorekit-purge-rate-limit-counters', + '*/15 * * * *', + $cron$select lorekit_purge_rate_limit_counters()$cron$ + ); + end if; +end; +$$; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00004', 'limits') +on conflict (version) do nothing; + -- ───────────────────────────────────────────────────────────────────────────── --- 6. Archive / restore / purge RPCs +-- 00005_webhook_secrets.sql -- ───────────────────────────────────────────────────────────────────────────── -create or replace function archive_memory( - p_user_id uuid, - p_scope text, - p_key text -) -returns uuid -language plpgsql -security definer -set search_path = public -as $$ +-- LoreKit webhook secrets +-- Stores per-user HMAC secrets used to verify GitHub webhook payloads. +-- Secrets are stored in plaintext because: +-- 1. They are Supabase-internal (the DB itself is the trust boundary). +-- 2. The edge function needs the raw value to compute HMAC-SHA256 per request. +-- 3. The dashboard needs to display the value once on first setup, matching +-- the api_tokens "show once" UX pattern (except this value IS retrievable +-- because the edge function needs to read it on every webhook call). +-- +-- One active secret per user. Regenerating creates a new row and soft-deletes +-- the old one (active = false) so webhook deliveries in-flight don't fail +-- immediately — GitHub retries with the old secret for a short window. + +create table if not exists webhook_secrets ( + id uuid primary key default gen_random_uuid(), + user_id uuid references auth.users on delete cascade not null, + secret text not null check (length(secret) = 64), -- 32 raw bytes → 64 hex chars + active boolean not null default true, + created_at timestamptz not null default now() +); + +create index if not exists webhook_secrets_user_active_idx + on webhook_secrets (user_id, active) + where active = true; + +alter table webhook_secrets enable row level security; + +-- Users can read their own secrets (needed to display in the dashboard) +create policy "rls_webhook_secrets_select" + on webhook_secrets for select + using (user_id = auth.uid()); + +create policy "rls_webhook_secrets_insert" + on webhook_secrets for insert + with check (user_id = auth.uid()); + +create policy "rls_webhook_secrets_update" + on webhook_secrets for update + using (user_id = auth.uid()); + +-- Service role reads the active secret for HMAC verification on each webhook +-- call (bypasses RLS — no additional policy needed for service role). + +insert into supabase_migrations.schema_migrations (version, name) +values ('00005', 'webhook_secrets') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00006_grants.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- Explicit table privileges for the Supabase API roles. +-- +-- Supabase cloud projects grant DML to anon/authenticated/service_role via +-- ambient "default privileges", so this is effectively a no-op in staging and +-- production. But a fresh local `supabase start` (newer CLI) only grants +-- TRUNCATE/REFERENCES/TRIGGER to these roles, leaving them without +-- SELECT/INSERT/UPDATE/DELETE — so any service-role or api_key request fails +-- with "permission denied for table ...", and PostgREST won't expose an +-- ON CONFLICT arbiter for upserts. Granting explicitly makes the schema +-- self-contained and identical across every environment. +-- +-- This grants no new *effective* access: anon/authenticated remain governed by +-- the RLS policies defined in earlier migrations; service_role bypasses RLS as +-- it already does. It only makes the underlying table privileges explicit. + +grant select, insert, update, delete on all tables in schema public + to anon, authenticated, service_role; + +-- Tables created by future migrations inherit the same grants automatically. +alter default privileges in schema public + grant select, insert, update, delete on tables + to anon, authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00006', 'grants') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00007_memory_write_rpc.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- Fix memory.write against the partial unique indexes introduced in 00003. +-- +-- 00003 dropped the plain `memories_user_scope_key_unique` constraint and +-- replaced it with two PARTIAL unique indexes (WHERE archived_at IS NULL): +-- • memories_user_scope_key_active_unique (user_id, scope, key) +-- • memories_null_user_scope_key_active_unique (scope, key) WHERE user_id IS NULL +-- +-- A plain `.upsert(..., { onConflict: 'user_id,scope,key' })` cannot target a +-- partial index — Postgres raises "there is no unique or exclusion constraint +-- matching the ON CONFLICT specification", so every write failed. `supabase-js` +-- can't express a partial-index arbiter, so writes go through this RPC, which +-- branches on whether user_id is null and names the matching partial index +-- predicate in ON CONFLICT. +-- +-- Behaviour preserved: the BEFORE INSERT memory-cap trigger still fires on the +-- insert path (not on conflict-update); archived rows are excluded, so a +-- (user_id, scope, key) can be re-created after it is archived. + +create or replace function memory_write( + p_user_id uuid, + p_scope text, + p_key text, + p_value text, + p_tags text[] default '{}', + p_source_agent text default null, + p_trigger text default null +) +returns table (id uuid, created_at timestamptz) +language plpgsql +set search_path = public +as $$ +begin + if p_user_id is null then + -- service-role / CI writes: (scope, key) partial index for null user_id + return query + insert into memories (user_id, scope, key, value, tags, source_agent, trigger, updated_at) + values (null, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, now()) + on conflict (scope, key) where user_id is null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now() + returning memories.id, memories.created_at; + else + -- user-scoped writes (api_key / user): (user_id, scope, key) partial index + return query + insert into memories (user_id, scope, key, value, tags, source_agent, trigger, updated_at) + values (p_user_id, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, now()) + on conflict (user_id, scope, key) where archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now() + returning memories.id, memories.created_at; + end if; +end; +$$; + +grant execute on function memory_write(uuid, text, text, text, text[], text, text) + to anon, authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00007', 'memory_write_rpc') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00008_webhook_secrets_repo.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- Repo-scoped webhook secrets. +-- +-- Problem: the original webhook_secrets model (00005) is one active secret +-- per user, matched at lookup time by the GitHub *owner login*. That breaks +-- for org-owned repos (repository.owner.login is the org, not the LoreKit +-- user's personal GitHub login) and requires an O(all-users) listUsers scan. +-- +-- Fix: add a nullable `repo` column (canonical lowercased `owner/name`) so a +-- user can register a secret per repository, matched deterministically by +-- the delivery's `repository.full_name` — no auth.users join needed. +-- +-- Existing null-`repo` rows are kept as-is (a legacy/global fallback secret) +-- for back-compat; this migration does not backfill or touch 00005. +-- Forward-only per CLAUDE.md — never edit an applied migration. + +alter table webhook_secrets add column if not exists repo text; + +-- Canonical owner/name, lowercased. Null = legacy/global fallback row. +alter table webhook_secrets + add constraint webhook_secrets_repo_format + check (repo is null or repo ~ '^[a-z0-9._-]+/[a-z0-9._-]+$'); + +-- One active secret per (user, repo). coalesce(repo, '') folds the legacy +-- null-repo row into a single active row per user, portably — no +-- NULLS NOT DISTINCT dependency (PG15+-only, and the repo's local Supabase +-- PostgREST predates it per the CI split note in CLAUDE.md). +drop index if exists webhook_secrets_user_active_idx; +create unique index webhook_secrets_user_repo_active_uidx + on webhook_secrets (user_id, coalesce(repo, '')) + where active = true; + +-- Edge lookup path: match active secrets by repo directly. +create index webhook_secrets_repo_active_idx + on webhook_secrets (repo) + where active = true and repo is not null; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00008', 'webhook_secrets_repo') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00009_memory_write_created_at.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- Allow memory.write to backdate a memory's creation date (migration support). +-- +-- When importing pre-existing memories into LoreKit, the caller can pass the +-- memory's ORIGINAL creation date so the dashboard dates it correctly instead +-- of showing the migration wall-clock time. This adds an optional +-- `p_created_at timestamptz` parameter to the memory_write RPC (00007). +-- +-- Semantics: +-- • p_created_at IS NULL (default / not supplied): behaviour unchanged — the +-- insert uses now() for both created_at and updated_at. +-- • p_created_at supplied: the INSERT path sets BOTH created_at AND updated_at +-- to that value, so a freshly-migrated memory presents its original date in +-- every dashboard view (some order by created_at, some by updated_at) until +-- it is next edited. The app layer validates it and rejects future dates. +-- • The ON CONFLICT (update) path is unchanged: created_at is preserved (a +-- creation date never moves on a subsequent write) and updated_at = now(). +-- +-- Adding a parameter changes the function signature, so CREATE OR REPLACE alone +-- would leave a stale 7-arg overload behind and make PostgREST calls ambiguous. +-- Drop the old signature first, then create the widened one. Forward-only. + +drop function if exists memory_write(uuid, text, text, text, text[], text, text); + +create or replace function memory_write( + p_user_id uuid, + p_scope text, + p_key text, + p_value text, + p_tags text[] default '{}', + p_source_agent text default null, + p_trigger text default null, + p_created_at timestamptz default null +) +returns table (id uuid, created_at timestamptz) +language plpgsql +set search_path = public +as $$ +begin + if p_user_id is null then + -- service-role / CI writes: (scope, key) partial index for null user_id + return query + insert into memories (user_id, scope, key, value, tags, source_agent, trigger, created_at, updated_at) + values (null, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now())) + on conflict (scope, key) where user_id is null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now() + returning memories.id, memories.created_at; + else + -- user-scoped writes (api_key / user): (user_id, scope, key) partial index + return query + insert into memories (user_id, scope, key, value, tags, source_agent, trigger, created_at, updated_at) + values (p_user_id, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now())) + on conflict (user_id, scope, key) where archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now() + returning memories.id, memories.created_at; + end if; +end; +$$; + +grant execute on function memory_write(uuid, text, text, text, text[], text, text, timestamptz) + to anon, authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00009', 'memory_write_created_at') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00010_audit_log.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- LoreKit audit log — append-only trail for security/data-affecting actions. +-- +-- CAPTURE MODEL: app-layer, explicit (Decision D1). Every audit row is +-- written by application code right after the primary operation succeeds +-- (packages/mcp-core/src/audit.ts + supabase/functions/mcp/audit.ts on the +-- MCP side, packages/web/src/lib/audit-log.ts on the dashboard side) — NOT by +-- a DB trigger on the data tables (memories / api_tokens / webhook_secrets). +-- App-layer capture can see the resolved actor (auth.uid() for dashboard +-- actions, the resolved token/user for MCP, null for service-role/CI) and can +-- shape a human-readable `target` + `metadata`, which a trigger on those +-- tables cannot do cleanly. +-- +-- THE ONE DELIBERATE EXCEPTION: `user_limits` changes (Decision D2). No +-- app-layer path writes that table today (raising a user's limit is a raw +-- SQL upsert — see docs/limits.md), so there is no call site to instrument. +-- An AFTER INSERT OR UPDATE OR DELETE trigger (audit_user_limits(), below) +-- is the only way to capture it, mirroring the enforce_memory_cap() +-- DB-trigger precedent from 00004_limits.sql (the write-path actor is not +-- app-visible there either). If an admin server action is ever added for +-- `user_limits`, instrument it there too — the trigger is a safety net, not +-- a reason to skip app-layer capture when a call site exists. +-- +-- RLS (Decision D5): append-only. Users may SELECT only their own rows. +-- A scoped INSERT policy permits the authenticated server-action client +-- (mirrors api_tokens' insert policy). NO update/delete policy — once +-- written, a row cannot be changed or removed via the API surface (only a +-- service-role/admin operating directly on the DB could, which is outside +-- RLS's scope by design). + +create table if not exists audit_log ( + id uuid primary key default gen_random_uuid(), + -- Nullable: service-role/CI writes (memory.* via a service-role token) and + -- JWT-authenticated MCP calls (see supabase/functions/mcp/audit.ts doc + -- comment) don't attribute to a resolvable app user_id; RLS SELECT then + -- never surfaces those rows to any user, mirroring the memory-cap exemption. + user_id uuid references auth.users on delete cascade, + action text not null check (action in ( + 'api_key.create', + 'api_key.revoke', + 'webhook_secret.create', + 'webhook_secret.rotate', + 'webhook_secret.deactivate', + 'memory.create', + 'memory.update', + 'memory.archive', + 'memory.restore', + 'memory.delete', + 'limit.override' + )), + resource_type text, + resource_id text, + target text, + metadata jsonb, + created_at timestamptz not null default now() +); + +-- Primary read pattern: "my recent activity", newest first. +create index if not exists audit_log_user_created_idx on audit_log (user_id, created_at desc); + +-- Secondary pattern: filter by action type (the Settings → Audit Logs filter pills). +create index if not exists audit_log_action_idx on audit_log (action); + +alter table audit_log enable row level security; + +-- Users can see only their own audit trail. +create policy "rls_audit_log_select" + on audit_log for select + using (user_id = auth.uid()); + +-- Scoped insert for the authenticated server-action client (mirrors +-- api_tokens' insert policy exactly). Service-role writes (MCP memory +-- mutations) bypass RLS entirely, as usual. +create policy "rls_audit_log_insert" + on audit_log for insert + with check (user_id = auth.uid()); + +-- Deliberately NO update or delete policy — the log is append-only / +-- immutable via the API surface (Decision D5). + +-- ── user_limits audit trigger (Decision D2 — the one exception) ──────────── +-- +-- Fires on every insert/update/delete against user_limits (i.e. every limit +-- override) and records a `limit.override` row. SECURITY DEFINER so it can +-- write audit_log regardless of the caller's RLS visibility, matching +-- enforce_memory_cap()'s shape in 00004_limits.sql. The acting user_id is not +-- known to this trigger (overrides are applied via raw SQL / a future admin +-- path, not a session-scoped request) — recorded as NULL, same as any other +-- service-role-attributed write. +create or replace function audit_user_limits() +returns trigger +language plpgsql +security definer +set search_path = public +as $$ +declare + v_row_user_id uuid; + v_metadata jsonb; +begin + v_row_user_id := coalesce(new.user_id, old.user_id); + + v_metadata := jsonb_build_object( + 'op', tg_op, + 'max_memories', case when tg_op = 'DELETE' then old.max_memories else new.max_memories end, + 'requests_per_minute', case when tg_op = 'DELETE' then old.requests_per_minute else new.requests_per_minute end + ); + + -- user_limits is keyed by user_id (it IS the row's primary key), so the row + -- identity (resource_id) and the affected subject (target) are intentionally + -- the same value here — not a copy-paste. The actual changed limit values + -- live in metadata, not target. + insert into audit_log (user_id, action, resource_type, resource_id, target, metadata) + values (null, 'limit.override', 'user_limits', v_row_user_id::text, v_row_user_id::text, v_metadata); + + if tg_op = 'DELETE' then + return old; + end if; + return new; +end; +$$; + +create or replace trigger user_limits_audit + after insert or update or delete on user_limits + for each row execute function audit_user_limits(); + +insert into supabase_migrations.schema_migrations (version, name) +values ('00010', 'audit_log') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00011_memory_write_inserted.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- Distinguish memory.create from memory.update for audit logging (Decision D4). +-- +-- memory_write() (00007, widened by 00009) performs an upsert — the caller +-- cannot otherwise tell whether the row was newly inserted or an existing +-- (scope, key) was updated. Postgres exposes this cheaply and atomically via +-- `xmax = 0` on the RETURNING clause: a freshly-inserted row's xmax is 0 +-- (never yet updated/deleted by a transaction), while an ON CONFLICT DO +-- UPDATE path always leaves xmax set to the current transaction's id. +-- +-- This is purely additive: a new `inserted boolean` column appended to the +-- RETURNS TABLE. Existing callers that read named fields (`id`, `created_at`) +-- are unaffected; packages/mcp-core/src/tools/write.ts and the edge mirror +-- (supabase/functions/mcp/tools.ts toolWrite) read the new field to record +-- `memory.create` vs `memory.update` in the audit log. +-- +-- Widening the RETURNS TABLE changes the function's return signature, so +-- CREATE OR REPLACE alone is insufficient — drop the old signature first, +-- then create the widened one (same drop+recreate shape as 00009). Forward-only. + +drop function if exists memory_write(uuid, text, text, text, text[], text, text, timestamptz); + +create or replace function memory_write( + p_user_id uuid, + p_scope text, + p_key text, + p_value text, + p_tags text[] default '{}', + p_source_agent text default null, + p_trigger text default null, + p_created_at timestamptz default null +) +returns table (id uuid, created_at timestamptz, inserted boolean) +language plpgsql +set search_path = public +as $$ +begin + if p_user_id is null then + -- service-role / CI writes: (scope, key) partial index for null user_id + return query + insert into memories (user_id, scope, key, value, tags, source_agent, trigger, created_at, updated_at) + values (null, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now())) + on conflict (scope, key) where user_id is null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now() + -- xmax is an xid; compare its text form to '0' so the insert-vs-update + -- discriminator reads explicitly rather than relying on an implicit + -- xid-to-integer cast (see the header comment for the idiom). + returning memories.id, memories.created_at, (xmax::text = '0') as inserted; + else + -- user-scoped writes (api_key / user): (user_id, scope, key) partial index + return query + insert into memories (user_id, scope, key, value, tags, source_agent, trigger, created_at, updated_at) + values (p_user_id, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now())) + on conflict (user_id, scope, key) where archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now() + -- xmax is an xid; compare its text form to '0' so the insert-vs-update + -- discriminator reads explicitly rather than relying on an implicit + -- xid-to-integer cast (see the header comment for the idiom). + returning memories.id, memories.created_at, (xmax::text = '0') as inserted; + end if; +end; +$$; + +grant execute on function memory_write(uuid, text, text, text, text[], text, text, timestamptz) + to anon, authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00011', 'memory_write_inserted') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00012_audit_log_search.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- Keyset pagination + name search support for the Settings → Audit Logs feed +-- (see packages/web/src/lib/pagination/ and the extended listAuditLog in +-- packages/web/src/lib/audit-log.ts). Forward-only, additive: no existing +-- column, constraint, or RLS policy is touched. +-- +-- 1. `pg_trgm` + a GIN trigram index on `audit_log.target` so the +-- case-insensitive substring `name` search (`.ilike('target', '%…%')`) +-- stays index-backed instead of a sequential scan as the table grows. +-- A plain btree index can't serve a leading-wildcard ILIKE; trigram GIN +-- is the standard Postgres answer (mirrors the `memories_fts_idx` GIN +-- index pattern in 00001_memories.sql for full-text search). +-- +-- 2. A `(user_id, created_at desc, id)` index covering the keyset seek. +-- 00010's `audit_log_user_created_idx (user_id, created_at desc)` serves +-- the ORDER BY but doesn't include `id`, so the keyset predicate's +-- tiebreaker (`created_at.eq.,and(id.lt.)`) can't be resolved from +-- that index alone. Adding `id` makes the whole keyset seek index-only. + +create extension if not exists pg_trgm; + +create index if not exists audit_log_target_trgm_idx + on audit_log using gin (target gin_trgm_ops); + +create index if not exists audit_log_user_created_id_idx + on audit_log (user_id, created_at desc, id); + +insert into supabase_migrations.schema_migrations (version, name) +values ('00012', 'audit_log_search') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00013_memory_keyset_index.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- Keyset-pagination covering index for the Lore Explorer's server-side +-- paginated list of memories (ordered newest-first, tiebroken by id). +-- +-- Mirrors the pattern established by 00012_audit_log_search.sql for +-- audit_log: the `(user_id, created_at desc, id)` index covers the keyset +-- seek used by `listMemories` (the `(created_at desc, id desc)` order + +-- the `.or()` "after cursor" predicate). The `id` tiebreaker is required so +-- the seek is deterministic when multiple rows share the same `created_at`; +-- 00001's `memories_user_idx (user_id)` lacks both `created_at` and `id`. +-- +-- Also enables pg_trgm trigram search on `memories.key` and `memories.value` +-- for the key/value substring search (`ilike '%…%'`), mirroring the +-- 00012_audit_log_search.sql precedent for `audit_log.target`. +create extension if not exists pg_trgm; + +create index if not exists memories_user_created_at_id_idx + on memories (user_id, created_at desc, id desc); + +create index if not exists memories_key_trgm_idx + on memories using gin (key gin_trgm_ops); + +create index if not exists memories_value_trgm_idx + on memories using gin (value gin_trgm_ops); + +insert into supabase_migrations.schema_migrations (version, name) +values ('00013', 'memory_keyset_index') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00014_orgs.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- LoreKit organizations: server-side org entity + membership table. +-- +-- Foundational schema for shared/team lore (org-owned memories land in +-- 00013/00014; the write path itself is Phase 2 — see plan.md). `orgs` is +-- the org identity; `org_members` is the SOLE membership-truth table. +-- `.lorekit/config.json` (Phase 4, advisory-only) never substitutes for it — +-- see CLAUDE.md. +-- +-- lorekit_member_org_ids() is the SINGLE tenant-visibility predicate +-- source: both the memories RLS policies (00013) and the mirrored TS helper +-- (packages/mcp-core/src/tenant-scope.ts, applyTenantScope) consume its +-- result via RPC — never a hand-copied membership check. SECURITY DEFINER + +-- STABLE, mirroring the enforce_memory_cap()/lorekit_get_limit() pattern +-- (00004_limits.sql): keyed off the authenticated identity, never a +-- client-supplied claim. + +create table if not exists orgs ( + id uuid primary key default gen_random_uuid(), + slug text not null unique check (slug = lower(slug)), + name text not null, + created_by uuid references auth.users on delete set null, + created_at timestamptz not null default now(), + updated_at timestamptz not null default now() +); + +alter table orgs enable row level security; + +create table if not exists org_members ( + org_id uuid not null references orgs(id) on delete cascade, + user_id uuid not null references auth.users on delete cascade, + role text not null default 'member' check (role in ('owner', 'admin', 'member')), + created_at timestamptz not null default now(), + primary key (org_id, user_id) +); + +alter table org_members enable row level security; + +create index if not exists org_members_user_idx on org_members (user_id); + +-- Members can read the orgs they belong to, and only their own membership +-- row. No insert/update/delete policy yet: Phase 3 (invite/accept server +-- actions, Supabase-JWT scoped) owns the membership write path — until then +-- only a service-role client can create orgs/memberships. +create policy "rls_orgs_select" + on orgs for select + using ( + id in (select org_id from org_members where user_id = auth.uid()) + ); + +create policy "rls_org_members_select" + on org_members for select + using (user_id = auth.uid()); + +create or replace trigger orgs_updated_at + before update on orgs + for each row execute function set_updated_at(); + +-- The single membership-truth function. STABLE (read-only, same result +-- within a statement) + SECURITY DEFINER so the memories RLS policy (00013) +-- can call it without granting SELECT on org_members directly, and so the +-- edge api_key path (service-role, RLS-bypass) gets identical semantics via +-- an explicit RPC call (see tenant-scope.ts). Reads ONLY org_members — never +-- memories — so it cannot recurse into the memories RLS policy that calls it. +create or replace function lorekit_member_org_ids(p_user_id uuid) +returns setof uuid +language sql +stable +security definer +set search_path = public +as $$ + select org_id from org_members where user_id = p_user_id; +$$; + +-- NOT granted to `anon`: the function takes a bare p_user_id, so an +-- unauthenticated caller with EXECUTE could enumerate any user's org +-- membership via PostgREST RPC. RLS policies that call it run as the +-- querying role, and only authenticated/service_role ever read `memories`. +grant execute on function lorekit_member_org_ids(uuid) to authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00014', 'orgs') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00015_memories_org_fk.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- Replace the dormant, unsafe org_id scaffolding on memories with a real FK +-- and a membership-verified RLS read policy. +-- +-- 00001_memories.sql introduced `org_id text` plus an RLS read policy that +-- trusted a client-asserted, single-valued JWT claim +-- (`org_id = auth.jwt() ->> 'org_id'`). That is unsafe: any caller can mint +-- a JWT (or, for the api_key path, is not even gated by RLS) asserting any +-- org_id and read that org's lore. This migration: +-- 1. Drops the two policies that reference the unsafe claim (00003 widened +-- them to also gate on archived_at, so both rls_read and +-- rls_read_archived must be dropped and recreated). +-- 2. Converts org_id from free-text to a nullable uuid FK -> orgs(id). The +-- column has never been written by any code path (grep-verified), so +-- `org_id::uuid` casts every existing (null) value cleanly. Pre-flight +-- safety note for whoever applies this migration to a non-fresh +-- database (dev/staging/production): run +-- select count(*) from memories where org_id is not null; +-- first. A non-zero count means the cast will fail loudly (forward-only +-- migrations do not paper over that) — investigate before proceeding. +-- 3. Recreates both read policies using lorekit_member_org_ids(auth.uid()) +-- (00014_orgs.sql) — the single SECURITY DEFINER membership source — +-- instead of the client-asserted claim. +-- +-- Writes are untouched here: memory_write (00007/00009/00011) never sets +-- org_id, so every row stays personal-only until Phase 2. The org-aware +-- unique arbiter lands in 00014. + +drop policy if exists "rls_read" on memories; +drop policy if exists "rls_read_archived" on memories; + +alter table memories + alter column org_id type uuid using org_id::uuid; + +alter table memories + add constraint memories_org_id_fkey + foreign key (org_id) references orgs(id) on delete cascade; + +create policy "rls_read" + on memories for select + using ( + archived_at is null + and ( + user_id = auth.uid() + or org_id in (select lorekit_member_org_ids(auth.uid())) + ) + ); + +create policy "rls_read_archived" + on memories for select + using ( + archived_at is not null + and ( + user_id = auth.uid() + or org_id in (select lorekit_member_org_ids(auth.uid())) + ) + ); + +insert into supabase_migrations.schema_migrations (version, name) +values ('00015', 'memories_org_fk') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00016_memories_org_unique_arbiter.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- Make org ownership a first-class partition of the (scope, key) uniqueness +-- arbiter, and re-pin memory_write's ON CONFLICT predicates to match. +-- +-- 00003 introduced two partial unique indexes: +-- memories_user_scope_key_active_unique (user_id, scope, key) +-- WHERE archived_at IS NULL +-- memories_null_user_scope_key_active_unique (scope, key) +-- WHERE user_id IS NULL AND archived_at IS NULL +-- +-- Now that org_id is a real FK (00013), those two predicates are ambiguous +-- for an org-owned row (org_id IS NOT NULL, user_id may be NULL or set to the +-- creator) — it could collide with either partition. This migration replaces +-- them with THREE mutually-exclusive partial unique indexes: +-- • org-owned (org_id, scope, key) WHERE org_id IS NOT NULL AND archived_at IS NULL +-- • personal (user_id, scope, key) WHERE org_id IS NULL AND user_id IS NOT NULL AND archived_at IS NULL +-- • service (scope, key) WHERE org_id IS NULL AND user_id IS NULL AND archived_at IS NULL +-- An org row can never collide with (or impersonate) the cap-exempt +-- user_id IS NULL service partition, because the org index requires +-- org_id IS NOT NULL while the service index requires org_id IS NULL. +-- +-- Postgres' ON CONFLICT arbiter inference requires the inference clause's +-- predicate to match the target partial index's predicate, so memory_write's +-- two ON CONFLICT ... WHERE clauses must be re-pinned in the same migration +-- or every write breaks. No signature change (still no p_org_id parameter — +-- writes stay personal-only, Requirement R7), so CREATE OR REPLACE suffices +-- (matches the "same drop+recreate shape only when the signature/return +-- changes" note in 00009/00011 — here only the body changes). + +drop index if exists memories_user_scope_key_active_unique; +drop index if exists memories_null_user_scope_key_active_unique; + +create unique index if not exists memories_user_scope_key_active_unique + on memories (user_id, scope, key) + where org_id is null and user_id is not null and archived_at is null; + +create unique index if not exists memories_null_user_scope_key_active_unique + on memories (scope, key) + where org_id is null and user_id is null and archived_at is null; + +create unique index if not exists memories_org_scope_key_active_unique + on memories (org_id, scope, key) + where org_id is not null and archived_at is null; + +create or replace function memory_write( + p_user_id uuid, + p_scope text, + p_key text, + p_value text, + p_tags text[] default '{}', + p_source_agent text default null, + p_trigger text default null, + p_created_at timestamptz default null +) +returns table (id uuid, created_at timestamptz, inserted boolean) +language plpgsql +set search_path = public +as $$ +begin + if p_user_id is null then + -- service-role / CI writes: (scope, key) partial index for + -- org_id IS NULL AND user_id IS NULL. + return query + insert into memories (user_id, scope, key, value, tags, source_agent, trigger, created_at, updated_at) + values (null, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now())) + on conflict (scope, key) where org_id is null and user_id is null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now() + returning memories.id, memories.created_at, (xmax::text = '0') as inserted; + else + -- user-scoped writes (api_key / user): (user_id, scope, key) partial + -- index for org_id IS NULL AND user_id IS NOT NULL. Phase 1 never sets + -- org_id on a write, so every insert lands in this personal partition. + return query + insert into memories (user_id, scope, key, value, tags, source_agent, trigger, created_at, updated_at) + values (p_user_id, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now())) + on conflict (user_id, scope, key) where org_id is null and user_id is not null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now() + returning memories.id, memories.created_at, (xmax::text = '0') as inserted; + end if; +end; +$$; + +grant execute on function memory_write(uuid, text, text, text, text[], text, text, timestamptz) + to anon, authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00016', 'memories_org_unique_arbiter') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00017_org_roles_and_author.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- Org write foundations, part 1: a `viewer` role and author attribution. +-- +-- Phase 2 (org-owned writes) opens the write path Phase 1 deliberately left +-- closed. This migration lays the two pieces the write/delete RPCs (00017, +-- 00018) depend on: +-- +-- 1. `viewer` joins the `org_members.role` CHECK. Phase 1 only ever needed +-- owner/admin/member (nobody could write yet); Phase 2 needs a +-- read-only role so "a viewer cannot write/delete" is expressible. +-- 2. `created_by` / `updated_by` land on `memories`, additive and nullable, +-- neighbour-symmetric with the existing `created_at`/`updated_at` pair +-- (created_by preserved on clobber like created_at; updated_by tracks +-- the last writer like updated_at tracks the last write time). Org rows +-- keep `user_id NULL` (see 00013/00014) so a dedicated author pair +-- records who wrote an org row without overloading `user_id` — reusing +-- `user_id` would pool org rows into the writer's personal cap count +-- and personal RLS partition. +-- +-- `lorekit_org_role` / `lorekit_org_can` are the single SQL capability +-- source (mirrors the `lorekit_member_org_ids` SECURITY DEFINER shape, +-- 00012_orgs.sql): every role -> capability decision for org writes/deletes +-- routes through `lorekit_org_can`, never a hand-copied TS matrix. Reads are +-- untouched — `lorekit_member_org_ids` stays the sole read-visibility +-- predicate (R8). + +alter table org_members drop constraint org_members_role_check; +alter table org_members add constraint org_members_role_check + check (role in ('owner', 'admin', 'member', 'viewer')); + +alter table memories + add column if not exists created_by uuid references auth.users on delete set null, + add column if not exists updated_by uuid references auth.users on delete set null; + +-- Resolve a user's role within an org, or NULL if they are not a member. +-- STABLE (read-only within a statement) + SECURITY DEFINER so it can be +-- called from the memory_write/memory_delete RPCs (also SECURITY DEFINER) +-- regardless of the caller's RLS visibility into org_members. +create or replace function lorekit_org_role(p_user_id uuid, p_org_id uuid) +returns text +language sql +stable +security definer +set search_path = public +as $$ + select role from org_members where user_id = p_user_id and org_id = p_org_id; +$$; + +grant execute on function lorekit_org_role(uuid, uuid) to anon, authenticated, service_role; + +-- The single capability source for org write/delete authorization. +-- Capabilities: 'write' | 'archive' | 'restore' | 'hard_delete'. +-- Matrix: viewer -> none; member -> write/archive/restore; admin & owner -> +-- additionally hard_delete. A non-member (NULL role) is denied everything. +-- This is the ONLY place the role -> capability matrix is encoded — the +-- app-layer org-permissions.ts module translates the resulting LK002 denial, +-- it does not re-derive the matrix (see plan.md Decisions). +create or replace function lorekit_org_can(p_user_id uuid, p_org_id uuid, p_capability text) +returns boolean +language plpgsql +stable +security definer +set search_path = public +as $$ +declare + v_role text := lorekit_org_role(p_user_id, p_org_id); +begin + if v_role is null then + return false; + end if; + + return case p_capability + when 'write' then v_role in ('member', 'admin', 'owner') + when 'archive' then v_role in ('member', 'admin', 'owner') + when 'restore' then v_role in ('member', 'admin', 'owner') + when 'hard_delete' then v_role in ('admin', 'owner') + else false + end; +end; +$$; + +grant execute on function lorekit_org_can(uuid, uuid, text) to anon, authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00017', 'org_roles_and_author') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00018_org_limits.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- Org write foundations, part 2: a tenant-keyed memory cap. +-- +-- `org_limits` + `lorekit_get_org_limit` mirror `user_limits` + +-- `lorekit_get_limit` (00004_limits.sql) exactly — a nullable per-org +-- override table, COALESCEd against the same free-tier default via +-- `lorekit_default_limit`, so raising an org's cap is a single row upsert +-- (paid-tier-ready, no billing built yet). Only `max_memories` is carried — +-- rate limiting stays caller-keyed this phase (see plan.md Decisions), so an +-- org `requests_per_minute` column would be dead configuration. +-- +-- `enforce_memory_cap()` gains an org branch that MUST run BEFORE the +-- `user_id IS NULL` service exemption: an org-owned row always has +-- `user_id IS NULL` (00013/00014), so without this ordering an org write +-- would silently inherit the cap-exempt service partition. Branch order is +-- now org -> service-exempt -> personal. + +create table if not exists org_limits ( + org_id uuid primary key references orgs(id) on delete cascade, + max_memories integer, + created_at timestamptz not null default now(), + updated_at timestamptz not null default now() +); + +alter table org_limits enable row level security; + +-- Members can see their org's limit row (dashboard/CLI "your org's limit"); +-- raising a limit is a service-role/admin upsert for now — no insert/update +-- policy for regular members, mirroring user_limits' shape. +create policy "rls_org_limits_select" + on org_limits for select + using ( + org_id in (select lorekit_member_org_ids(auth.uid())) + ); + +create or replace trigger org_limits_updated_at + before update on org_limits + for each row execute function set_updated_at(); + +-- Resolve the effective limit for an org: override if set, else the same +-- free-tier default lorekit_get_limit uses. Security definer so it can read +-- org_limits regardless of the caller's RLS visibility (needed inside the +-- cap trigger). +create or replace function lorekit_get_org_limit(p_org_id uuid, p_key text) +returns integer +language plpgsql +security definer +set search_path = public +as $$ +declare + v_override integer; +begin + if p_key = 'max_memories' then + select max_memories into v_override from org_limits where org_id = p_org_id; + end if; + + return coalesce(v_override, lorekit_default_limit(p_key)); +end; +$$; + +grant execute on function lorekit_get_org_limit(uuid, text) to anon, authenticated, service_role; + +-- Index backing the org branch's active-row count below +-- (`where org_id = new.org_id and archived_at is null`), which runs on every +-- org-owned INSERT. Without it large orgs seq-scan `memories` per write. +-- Partial (archived_at is null) + org_id-keyed to match the count predicate. +create index if not exists memories_org_id_active_idx + on memories (org_id) + where archived_at is null; + +-- Tenant-keyed cap trigger. Branch order: org -> service-exempt -> personal. +-- The org branch must precede the service exemption because org rows have +-- user_id IS NULL — checking `new.user_id is null` first would silently +-- exempt every org write from any cap at all. +create or replace function enforce_memory_cap() +returns trigger +language plpgsql +security definer +set search_path = public +as $$ +declare + v_limit integer; + v_count integer; +begin + if new.org_id is not null then + v_limit := lorekit_get_org_limit(new.org_id, 'max_memories'); + + select count(*) into v_count + from memories + where org_id = new.org_id + and archived_at is null; + + if v_count >= v_limit then + raise exception using + errcode = 'LK001', + message = format('memory_cap_exceeded: limit=%s', v_limit); + end if; + + return new; + end if; + + if new.user_id is null then + return new; -- true service-role / CI exemption (both org_id and user_id null) + end if; + + v_limit := lorekit_get_limit(new.user_id, 'max_memories'); + + select count(*) into v_count + from memories + where user_id = new.user_id + and archived_at is null; + + if v_count >= v_limit then + raise exception using + errcode = 'LK001', + message = format('memory_cap_exceeded: limit=%s', v_limit); + end if; + + return new; +end; +$$; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00018', 'org_limits') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00019_memory_write_org.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- memory_write gains org-owned writes: a trailing p_org_slug selector, author +-- attribution, and an authorization-gated org branch. +-- +-- org_id is authorization-derived INSIDE the RPC, never trusted from the +-- caller (R1/R7): when p_org_slug is supplied, this resolves the slug to an +-- org_id and requires lorekit_org_can(p_user_id, org_id, 'write') — a +-- non-member or a viewer raises SQLSTATE 'LK002' (a distinct signal from the +-- memory-cap LK001) and writes no row. A caller passing any slug it is not a +-- write-capable member of is rejected regardless of what it claims. +-- +-- An unresolvable slug raises a distinct 'unknown_org' error (not LK002) so +-- the caller can tell "no such org" from "not permitted" (see plan.md Edge +-- Cases). +-- +-- Author attribution: every branch now sets created_by/updated_by = the +-- writer (null for the service branch, since it has no writer identity). On +-- an org upsert-clobber, created_by/created_at are preserved (matching the +-- existing personal-clobber created_at semantics) while updated_by/updated_at +-- advance to the clobbering writer. +-- +-- Adding a trailing parameter changes the function signature, so +-- CREATE OR REPLACE alone would leave the old 8-arg overload behind and make +-- PostgREST calls ambiguous — drop the old signature first (same shape as +-- 00009/00011). Forward-only. + +drop function if exists memory_write(uuid, text, text, text, text[], text, text, timestamptz); + +create or replace function memory_write( + p_user_id uuid, + p_scope text, + p_key text, + p_value text, + p_tags text[] default '{}', + p_source_agent text default null, + p_trigger text default null, + p_created_at timestamptz default null, + p_org_slug text default null +) +returns table (id uuid, created_at timestamptz, inserted boolean) +language plpgsql +set search_path = public +as $$ +declare + v_org_id uuid; +begin + if p_org_slug is not null then + -- Qualify orgs.id explicitly: the RETURNS TABLE's `id` output column + -- would otherwise make a bare `select id` ambiguous inside this function. + select o.id into v_org_id from orgs o where o.slug = p_org_slug; + if v_org_id is null then + raise exception using + errcode = 'P0001', + message = format('unknown_org: %s', p_org_slug); + end if; + + if not lorekit_org_can(p_user_id, v_org_id, 'write') then + raise exception using + errcode = 'LK002', + message = format('org_permission_denied: org=%s', p_org_slug); + end if; + + -- Org-owned write: (org_id, scope, key) partial index for + -- org_id IS NOT NULL. user_id stays NULL so the row is never counted + -- against the writer's personal cap or personal RLS partition (R3/R9); + -- created_by/updated_by record the writer instead. + return query + insert into memories ( + user_id, org_id, scope, key, value, tags, source_agent, trigger, + created_at, updated_at, created_by, updated_by + ) + values ( + null, v_org_id, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), p_user_id, p_user_id + ) + on conflict (org_id, scope, key) where org_id is not null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + updated_by = p_user_id + -- created_by/created_at are NOT in this SET list: they are preserved + -- from the original row on clobber, matching the existing + -- created_at-preservation semantics (00009). + returning memories.id, memories.created_at, (xmax::text = '0') as inserted; + + elsif p_user_id is null then + -- service-role / CI writes: (scope, key) partial index for + -- org_id IS NULL AND user_id IS NULL. No writer identity to attribute. + return query + insert into memories (user_id, scope, key, value, tags, source_agent, trigger, created_at, updated_at, created_by, updated_by) + values (null, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), null, null) + on conflict (scope, key) where org_id is null and user_id is null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now() + returning memories.id, memories.created_at, (xmax::text = '0') as inserted; + else + -- user-scoped writes (api_key / user): (user_id, scope, key) partial + -- index for org_id IS NULL AND user_id IS NOT NULL. + return query + insert into memories (user_id, scope, key, value, tags, source_agent, trigger, created_at, updated_at, created_by, updated_by) + values (p_user_id, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), p_user_id, p_user_id) + on conflict (user_id, scope, key) where org_id is null and user_id is not null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + updated_by = p_user_id + returning memories.id, memories.created_at, (xmax::text = '0') as inserted; + end if; +end; +$$; + +grant execute on function memory_write(uuid, text, text, text, text[], text, text, timestamptz, text) + to anon, authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00019', 'memory_write_org') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00020_memory_delete_org.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- Role-gated memory_delete RPC — the org side of destructive ops. +-- +-- The edge api_key path is a service-role client that bypasses RLS, so a raw +-- org-targeted `.delete()`/`.update()` would have NO gate at all. Role +-- enforcement for org rows must therefore live in a SECURITY DEFINER RPC that +-- calls lorekit_org_can (00015), mirroring memory_write's org branch. +-- +-- Personal delete (p_org_slug NULL) mirrors today's raw .eq('user_id', ...) +-- behavior exactly — the well-tested personal path is intentionally left +-- untouched elsewhere (tools.ts, mcp-core delete.ts) to minimize blast radius +-- on a security-critical change; this RPC only absorbs the org-targeted case +-- plus a personal fallback so callers have one entry point. +-- +-- Capability required: 'archive' for a soft-delete (p_force = false), +-- 'hard_delete' for p_force = true. A viewer is denied both; a member may +-- soft-archive but not hard-delete; admin/owner may do both. Any denial +-- raises SQLSTATE 'LK002' (same signal as memory_write's org-permission +-- denial) so the app layer translates it identically. +-- +-- An unresolvable slug raises 'unknown_org', matching memory_write. + +create or replace function memory_delete( + p_user_id uuid, + p_org_slug text default null, + p_scope text default null, + p_key text default null, + p_force boolean default false +) +returns table (deleted boolean, archived boolean) +language plpgsql +security definer +set search_path = public +as $$ +declare + v_org_id uuid; + v_count integer; +begin + if p_org_slug is not null then + select o.id into v_org_id from orgs o where o.slug = p_org_slug; + if v_org_id is null then + raise exception using + errcode = 'P0001', + message = format('unknown_org: %s', p_org_slug); + end if; + + if p_force then + if not lorekit_org_can(p_user_id, v_org_id, 'hard_delete') then + raise exception using + errcode = 'LK002', + message = format('org_permission_denied: org=%s capability=hard_delete', p_org_slug); + end if; + + delete from memories + where org_id = v_org_id and scope = p_scope and key = p_key; + get diagnostics v_count = row_count; + + return query select (v_count > 0), false; + else + if not lorekit_org_can(p_user_id, v_org_id, 'archive') then + raise exception using + errcode = 'LK002', + message = format('org_permission_denied: org=%s capability=archive', p_org_slug); + end if; + + update memories + set archived_at = now() + where org_id = v_org_id and scope = p_scope and key = p_key and archived_at is null; + get diagnostics v_count = row_count; + + return query select false, (v_count > 0); + end if; + else + -- Personal delete: mirrors the existing .eq('user_id', p_user_id) path. + if p_force then + delete from memories + where user_id = p_user_id and scope = p_scope and key = p_key; + get diagnostics v_count = row_count; + + return query select (v_count > 0), false; + else + update memories + set archived_at = now() + where user_id = p_user_id and scope = p_scope and key = p_key and archived_at is null; + get diagnostics v_count = row_count; + + return query select false, (v_count > 0); + end if; + end if; +end; +$$; + +grant execute on function memory_delete(uuid, text, text, text, boolean) to anon, authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00020', 'memory_delete_org') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00021_org_invites.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- Org sharing Phase 3, migration 1 of 3: the `org_invites` table + its RLS +-- read policies, and widening `org_members` visibility to co-members. +-- +-- `org_invites` is the identity-bound invitation entity the 00012 migration +-- comment deferred to "Phase 3": a pending invite addressed to an email OR a +-- GitHub handle (never a forwardable share-link — see plan.md Decisions). +-- `role` deliberately excludes `owner` (ownership is non-transferable in v1 — +-- see plan.md Out of Scope #2) and `status` tracks the invite's lifecycle. +-- Only ONE pending invite may exist per (org, identity) — enforced by two +-- partial unique indexes, mirroring the `webhook_secrets` partial-unique +-- pattern (00008). +-- +-- NO insert/update/delete RLS policy exists on `org_invites` — every mutation +-- (invite/revoke/accept/decline) is a SECURITY DEFINER RPC (00020) that +-- authorizes via `lorekit_org_can`, exactly like `org_members`/`orgs` before +-- it. RLS here governs reads only: an org's owners/admins see all its +-- invites; an invited user sees pending invites addressed to their VERIFIED +-- JWT identity claims (`email`, `user_metadata.user_name`) — never a +-- client-supplied identity string. +-- +-- Also widens `rls_org_members_select` from own-row-only (00012) to all +-- co-members of a shared org, via `lorekit_org_role` (SECURITY DEFINER, reads +-- org_members bypassing RLS — the same anti-recursion shape used throughout +-- 00012/00015). The dashboard member list needs to show co-members; this +-- supersedes the Phase 1 own-row-only assertion in migrations.test.sql §7, +-- which is updated forward in the same PR (a test file, not a shipped +-- migration — see plan.md Decisions). + +create table if not exists org_invites ( + id uuid primary key default gen_random_uuid(), + org_id uuid not null references orgs(id) on delete cascade, + invitee_email text, -- lowercased + invitee_handle text, -- lowercased GitHub handle + role text not null default 'member' check (role in ('admin', 'member', 'viewer')), + status text not null default 'pending' check (status in ('pending', 'accepted', 'declined', 'revoked')), + invited_by uuid references auth.users on delete set null, + created_at timestamptz not null default now(), + responded_at timestamptz, + expires_at timestamptz, + check (invitee_email is not null or invitee_handle is not null) -- identity-bound +); + +-- One pending invite per (org, identity) — a second pending invite for the +-- same identity would be redundant/confusing; revoke or let it resolve first. +create unique index if not exists org_invites_pending_email_idx + on org_invites (org_id, invitee_email) where status = 'pending' and invitee_email is not null; +create unique index if not exists org_invites_pending_handle_idx + on org_invites (org_id, invitee_handle) where status = 'pending' and invitee_handle is not null; +create index if not exists org_invites_org_idx on org_invites (org_id); + +alter table org_invites enable row level security; + +-- Owners/admins of the org see all its invites (single-source capability +-- check — 'invite' is a management capability an owner/admin holds). +create policy "rls_org_invites_select_manage" on org_invites for select + using (lorekit_org_can(auth.uid(), org_id, 'invite')); + +-- The invited user sees pending invites addressed to their VERIFIED identity +-- claims — never a client-supplied string. auth.jwt() reads the request's +-- verified JWT (same GUC-backed function used throughout 00012/00015). +create policy "rls_org_invites_select_invitee" on org_invites for select + using ( + status = 'pending' and ( + invitee_email = lower(auth.jwt() ->> 'email') or + invitee_handle = lower(auth.jwt() -> 'user_metadata' ->> 'user_name') + ) + ); + +-- Widen member visibility from own-row-only (00012) to all co-members of a +-- shared org. lorekit_org_role is SECURITY DEFINER + STABLE and reads +-- org_members bypassing RLS, so this policy cannot recurse into itself. +drop policy "rls_org_members_select" on org_members; +create policy "rls_org_members_select" on org_members for select + using (lorekit_org_role(auth.uid(), org_id) is not null); + +insert into supabase_migrations.schema_migrations (version, name) +values ('00021', 'org_invites') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00022_org_management_rpcs.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- Org sharing Phase 3, migration 2 of 3: the membership-truth WRITE path. +-- +-- Every state transition for orgs/org_members/org_invites goes through one of +-- the 10 SECURITY DEFINER RPCs below — never a direct RLS-gated table write +-- (org_invites/org_members/orgs carry no insert/update/delete RLS policy). +-- Two problems make direct writes unsafe here (see plan.md Decisions): +-- +-- 1. Owner-bootstrap: the FIRST membership row can't be gated by existing +-- membership, so a self-insert-as-owner RLS policy would let any +-- authenticated user seize any org by inserting an owner row for a slug +-- they don't own. lorekit_org_create sidesteps this by being the only +-- path that can ever insert the first owner, atomically with the org +-- row, bound to auth.uid(). +-- 2. Non-atomic accept: inserting the membership row and flipping the +-- invite's status are two statements; RLS-gated direct writes would +-- make that two round-trips (a TOCTOU window). The RPC does both in one +-- transaction. +-- +-- Every RPC below resolves the actor as `auth.uid()` — NEVER a caller-passed +-- user-id parameter — because these are dashboard actions invoked under a +-- real Supabase user JWT session (unlike memory_write/memory_delete, which +-- take an explicit p_user_id because the edge api_key path calls them from a +-- service-role client with no session JWT of its own). This is the same +-- anti-forgery property Phase 2 relies on, applied at the call boundary +-- instead of via a parameter: no argument the caller supplies can name a +-- different identity than the one the JWT authenticated. +-- +-- lorekit_org_can (00015) is extended with six management capabilities — +-- never a second matrix. It stays the SOLE role -> capability source; these +-- RPCs only add capability-independent INVARIANTS (last-owner protection, +-- admin-cannot-touch-owner-or-admin, invite-addressed-to-caller) that a +-- static role matrix cannot express, since they depend on the mutation's +-- specific target/state, not just the actor's role. +create or replace function lorekit_org_can(p_user_id uuid, p_org_id uuid, p_capability text) +returns boolean +language plpgsql +stable +security definer +set search_path = public +as $$ +declare + v_role text := lorekit_org_role(p_user_id, p_org_id); +begin + if v_role is null then + return false; + end if; + + return case p_capability + when 'write' then v_role in ('member', 'admin', 'owner') + when 'archive' then v_role in ('member', 'admin', 'owner') + when 'restore' then v_role in ('member', 'admin', 'owner') + when 'hard_delete' then v_role in ('admin', 'owner') + when 'invite' then v_role in ('admin', 'owner') + when 'revoke_invite' then v_role in ('admin', 'owner') + when 'remove_member' then v_role in ('admin', 'owner') + when 'change_role' then v_role in ('admin', 'owner') + when 'rename_org' then v_role in ('admin', 'owner') + when 'delete_org' then v_role = 'owner' + else false + end; +end; +$$; + +-- ── 1. Org lifecycle ───────────────────────────────────────────────────────── + +-- Creates the org AND the creator's owner membership atomically. No +-- capability check: this is the owner-bootstrap path — anyone authenticated +-- may create an org and becomes its sole owner. A duplicate slug surfaces +-- the orgs.slug unique_violation naturally. +create or replace function lorekit_org_create(p_slug text, p_name text) +returns uuid +language plpgsql +security definer +set search_path = public +as $$ +declare + v_org_id uuid := gen_random_uuid(); +begin + insert into orgs (id, slug, name, created_by) + values (v_org_id, p_slug, p_name, auth.uid()); + + insert into org_members (org_id, user_id, role) + values (v_org_id, auth.uid(), 'owner'); + + return v_org_id; +end; +$$; + +grant execute on function lorekit_org_create(text, text) to authenticated, service_role; + +create or replace function lorekit_org_rename(p_org_id uuid, p_name text) +returns void +language plpgsql +security definer +set search_path = public +as $$ +begin + if not lorekit_org_can(auth.uid(), p_org_id, 'rename_org') then + raise exception using errcode = 'LK002', message = format('org_permission_denied: org=%s capability=rename_org', p_org_id); + end if; + + update orgs set name = p_name where id = p_org_id; +end; +$$; + +grant execute on function lorekit_org_rename(uuid, text) to authenticated, service_role; + +-- Owner-only. Cascades org_members/org_invites via their `on delete cascade` +-- FKs to orgs(id) — no manual cleanup needed. +create or replace function lorekit_org_delete(p_org_id uuid) +returns void +language plpgsql +security definer +set search_path = public +as $$ +begin + if not lorekit_org_can(auth.uid(), p_org_id, 'delete_org') then + raise exception using errcode = 'LK002', message = format('org_permission_denied: org=%s capability=delete_org', p_org_id); + end if; + + delete from orgs where id = p_org_id; +end; +$$; + +grant execute on function lorekit_org_delete(uuid) to authenticated, service_role; + +-- ── 2. Invite lifecycle ────────────────────────────────────────────────────── + +-- Shared identity-match check for accept/decline: true iff the invite is +-- addressed to the CALLER's verified JWT identity (email or GitHub handle). +-- Each comparison is explicitly `coalesce`d to false — `NULL = x` evaluates +-- to SQL NULL, not false, so a one-sided invite (email set, handle NULL, or +-- vice versa) would otherwise poison the `or` into NULL. `if not (NULL)` is +-- treated as `if not false` by plpgsql (the branch is skipped, no exception +-- raised) — a real privilege-escalation bug this coalesce closes, caught by +-- migrations.test.sql's AC-6 "wrong user accepts" negative assertion. +create or replace function lorekit_invite_addressed_to_caller(p_invite org_invites) +returns boolean +language sql +stable +set search_path = public +as $$ + select + coalesce(p_invite.invitee_email = lower(auth.jwt() ->> 'email'), false) + or coalesce(p_invite.invitee_handle = lower(auth.jwt() -> 'user_metadata' ->> 'user_name'), false); +$$; + +-- Owner/admin only. The org_invites.role CHECK (00019) independently rejects +-- an attempt to invite an 'owner' — this RPC does not re-derive that guard. +create or replace function lorekit_org_invite( + p_org_id uuid, + p_invitee_email text default null, + p_invitee_handle text default null, + p_role text default 'member' +) +returns uuid +language plpgsql +security definer +set search_path = public +as $$ +declare + v_invite_id uuid; +begin + if not lorekit_org_can(auth.uid(), p_org_id, 'invite') then + raise exception using errcode = 'LK002', message = format('org_permission_denied: org=%s capability=invite', p_org_id); + end if; + + insert into org_invites (org_id, invitee_email, invitee_handle, role, invited_by) + values ( + p_org_id, + case when p_invitee_email is not null then lower(p_invitee_email) end, + case when p_invitee_handle is not null then lower(p_invitee_handle) end, + p_role, + auth.uid() + ) + returning id into v_invite_id; + + return v_invite_id; +end; +$$; + +grant execute on function lorekit_org_invite(uuid, text, text, text) to authenticated, service_role; + +create or replace function lorekit_org_invite_revoke(p_invite_id uuid) +returns void +language plpgsql +security definer +set search_path = public +as $$ +declare + inv org_invites; +begin + select * into inv from org_invites where id = p_invite_id; + if inv.id is null then + raise exception using errcode = 'LK002', message = 'invite not found'; + end if; + + if not lorekit_org_can(auth.uid(), inv.org_id, 'revoke_invite') then + raise exception using errcode = 'LK002', message = format('org_permission_denied: org=%s capability=revoke_invite', inv.org_id); + end if; + + -- Only a still-pending invite can be revoked — guards the silent + -- re-revoke and the revoke-of-already-accepted cases (accepted invites are + -- undone via member removal, not revoke). + if inv.status <> 'pending' then + raise exception using errcode = 'LK002', message = 'invite is not pending'; + end if; + + update org_invites set status = 'revoked', responded_at = now() where id = p_invite_id; +end; +$$; + +grant execute on function lorekit_org_invite_revoke(uuid) to authenticated, service_role; + +-- Binds the new membership row to the CALLER (auth.uid()), NEVER the invited +-- string. Takes no user-id parameter by design (the anti-TOCTOU fix — see +-- plan.md). The invited email/handle is only ever a MATCH TARGET for "is this +-- invite addressed to me?", never the identity that gets inserted. +create or replace function lorekit_org_invite_accept(p_invite_id uuid) +returns void +language plpgsql +security definer +set search_path = public +as $$ +declare + inv org_invites; +begin + select * into inv from org_invites where id = p_invite_id; + if inv.id is null or inv.status <> 'pending' + or (inv.expires_at is not null and inv.expires_at < now()) then + raise exception using errcode = 'LK002', message = 'invite is not open'; + end if; + + if not lorekit_invite_addressed_to_caller(inv) then + raise exception using errcode = 'LK002', message = 'invite is not addressed to you'; + end if; + + insert into org_members (org_id, user_id, role) + values (inv.org_id, auth.uid(), inv.role) + on conflict (org_id, user_id) do nothing; + + update org_invites set status = 'accepted', responded_at = now() where id = inv.id; +end; +$$; + +grant execute on function lorekit_org_invite_accept(uuid) to authenticated, service_role; + +-- Invitee-only. Mirrors the accept RPC's identity check but flips status to +-- 'declined' and creates no membership. +create or replace function lorekit_org_invite_decline(p_invite_id uuid) +returns void +language plpgsql +security definer +set search_path = public +as $$ +declare + inv org_invites; +begin + select * into inv from org_invites where id = p_invite_id; + if inv.id is null or inv.status <> 'pending' then + raise exception using errcode = 'LK002', message = 'invite is not open'; + end if; + + if not lorekit_invite_addressed_to_caller(inv) then + raise exception using errcode = 'LK002', message = 'invite is not addressed to you'; + end if; + + update org_invites set status = 'declined', responded_at = now() where id = inv.id; +end; +$$; + +grant execute on function lorekit_org_invite_decline(uuid) to authenticated, service_role; + +-- ── 3. Member management ───────────────────────────────────────────────────── + +-- Owner/admin only, plus two invariants a static role matrix cannot express: +-- - an admin actor may only act on member/viewer targets (never owner/admin) +-- - the last remaining owner can never be removed +create or replace function lorekit_org_member_remove(p_org_id uuid, p_target_user_id uuid) +returns void +language plpgsql +security definer +set search_path = public +as $$ +declare + v_actor_role text; + v_target_role text; + v_owner_count int; +begin + if not lorekit_org_can(auth.uid(), p_org_id, 'remove_member') then + raise exception using errcode = 'LK002', message = format('org_permission_denied: org=%s capability=remove_member', p_org_id); + end if; + + v_actor_role := lorekit_org_role(auth.uid(), p_org_id); + v_target_role := lorekit_org_role(p_target_user_id, p_org_id); + + if v_target_role is null then + raise exception using errcode = 'LK002', message = 'target is not a member of this org'; + end if; + + if v_actor_role = 'admin' and v_target_role in ('owner', 'admin') then + raise exception using errcode = 'LK002', message = 'an admin cannot remove an owner or another admin'; + end if; + + if v_target_role = 'owner' then + select count(*) into v_owner_count from org_members where org_id = p_org_id and role = 'owner'; + if v_owner_count <= 1 then + raise exception using errcode = 'LK002', message = 'the last owner cannot be removed — transfer or delete the org first'; + end if; + end if; + + delete from org_members where org_id = p_org_id and user_id = p_target_user_id; +end; +$$; + +grant execute on function lorekit_org_member_remove(uuid, uuid) to authenticated, service_role; + +-- Owner/admin only. Cannot assign 'owner' (ownership is non-transferable in +-- v1); cannot demote the last owner; an admin actor may only act on +-- member/viewer targets, mirroring lorekit_org_member_remove's invariant. +create or replace function lorekit_org_member_role(p_org_id uuid, p_target_user_id uuid, p_role text) +returns void +language plpgsql +security definer +set search_path = public +as $$ +declare + v_actor_role text; + v_target_role text; + v_owner_count int; +begin + if p_role = 'owner' then + raise exception using errcode = 'LK002', message = 'cannot assign owner via changeMemberRole — ownership is not transferable in v1'; + end if; + + if not lorekit_org_can(auth.uid(), p_org_id, 'change_role') then + raise exception using errcode = 'LK002', message = format('org_permission_denied: org=%s capability=change_role', p_org_id); + end if; + + v_actor_role := lorekit_org_role(auth.uid(), p_org_id); + v_target_role := lorekit_org_role(p_target_user_id, p_org_id); + + if v_target_role is null then + raise exception using errcode = 'LK002', message = 'target is not a member of this org'; + end if; + + if v_actor_role = 'admin' and v_target_role in ('owner', 'admin') then + raise exception using errcode = 'LK002', message = 'an admin cannot change the role of an owner or another admin'; + end if; + + if v_target_role = 'owner' then + select count(*) into v_owner_count from org_members where org_id = p_org_id and role = 'owner'; + if v_owner_count <= 1 then + raise exception using errcode = 'LK002', message = 'the last owner cannot be demoted — transfer or delete the org first'; + end if; + end if; + + update org_members set role = p_role where org_id = p_org_id and user_id = p_target_user_id; +end; +$$; + +grant execute on function lorekit_org_member_role(uuid, uuid, text) to authenticated, service_role; + +-- Self-service: any member may remove their OWN membership (no capability +-- check — leaving is not a management action), except the last owner, who +-- must transfer ownership or delete the org first. +create or replace function lorekit_org_leave(p_org_id uuid) +returns void +language plpgsql +security definer +set search_path = public +as $$ +declare + v_role text := lorekit_org_role(auth.uid(), p_org_id); + v_owner_count int; +begin + if v_role is null then + raise exception using errcode = 'LK002', message = 'you are not a member of this org'; + end if; + + if v_role = 'owner' then + select count(*) into v_owner_count from org_members where org_id = p_org_id and role = 'owner'; + if v_owner_count <= 1 then + raise exception using errcode = 'LK002', message = 'the last owner cannot leave — transfer or delete the org first'; + end if; + end if; + + delete from org_members where org_id = p_org_id and user_id = auth.uid(); +end; +$$; + +grant execute on function lorekit_org_leave(uuid) to authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00022', 'org_management_rpcs') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00023_audit_log_org_actions.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- Org sharing Phase 3, migration 3 of 3: extend the audit_log `action` CHECK +-- with the 10 org-management actions so orgs.ts/org-invites.ts's +-- recordAuditEvent calls can insert them. Forward-only: drop + re-add the +-- CHECK (a constraint can't be widened in place) rather than editing the +-- shipped 00010 migration. + +alter table audit_log drop constraint audit_log_action_check; + +alter table audit_log add constraint audit_log_action_check check (action in ( + 'api_key.create', + 'api_key.revoke', + 'webhook_secret.create', + 'webhook_secret.rotate', + 'webhook_secret.deactivate', + 'memory.create', + 'memory.update', + 'memory.archive', + 'memory.restore', + 'memory.delete', + 'limit.override', + 'org.create', + 'org.rename', + 'org.delete', + 'member.invite', + 'member.accept', + 'member.decline', + 'member.revoke', + 'member.remove', + 'member.role_change', + 'member.leave' +)); + +insert into supabase_migrations.schema_migrations (version, name) +values ('00023', 'audit_log_org_actions') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00024_org_member_identities.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- Org sharing Phase 4 addition: real member identities for the dashboard +-- member list. This REVERSES plan.md Decision D1's "web-only, no handle +-- resolution for other members" deferral — an explicit orchestrator override +-- of the Phase 4 dashboard-UX plan (see +-- .agent/claude/org-sharing-phase-4-dashboard/plan.md Decisions, D1). +-- +-- `org_members` exposes only `user_id`. `auth.users.raw_user_meta_data` +-- (GitHub OAuth: `user_name` + `avatar_url` — the SAME claims 00019/00020 +-- already read off the caller's own verified JWT) is otherwise only readable +-- by a service-role client, never a bare `select * from auth.users` from +-- `anon`/`authenticated`. This migration adds ONE narrow, membership-gated +-- read: a caller may resolve the handle/avatar of any user who shares an org +-- WITH them, and ONLY for that org — never a global user directory. +-- +-- SECURITY DEFINER + STABLE (read-only), gated via the existing +-- `lorekit_org_role` membership-truth function (00015) — the exact same +-- "NULL role denies everything" shape used throughout 00012/00015/00020. +-- No insert/update path, no direct grant on `auth.users` to anon/authenticated. +-- +-- Takes `p_org_id` (not a slug) to match every sibling Phase 3 management RPC +-- (`lorekit_org_rename`/`_delete`/`_member_remove`/`_member_role`/`_leave` all +-- take `p_org_id`) — the dashboard always already has the org's uuid in scope +-- (from `listMyOrgs()`/`listMembers()`), so a slug->id lookup would be an +-- extra, inconsistent hop for this one RPC alone. +create or replace function lorekit_org_members_list(p_org_id uuid) +returns table ( + user_id uuid, + handle text, + avatar_url text, + role text, + joined_at timestamptz +) +language plpgsql +stable +security definer +set search_path = public +as $$ +begin + -- Non-member (including a non-existent org — lorekit_org_role resolves to + -- NULL either way): return an EMPTY set, never raise and never leak + -- whether the org exists. Mirrors the "NULL role denies everything" gate + -- shape used throughout 00012/00015/00020. + if lorekit_org_role(auth.uid(), p_org_id) is null then + return; + end if; + + return query + select + m.user_id, + coalesce( + u.raw_user_meta_data ->> 'user_name', + u.raw_user_meta_data ->> 'preferred_username' + ) as handle, + u.raw_user_meta_data ->> 'avatar_url' as avatar_url, + m.role, + m.created_at as joined_at + from org_members m + join auth.users u on u.id = m.user_id + where m.org_id = p_org_id + order by m.created_at asc; +end; +$$; + +-- No `anon` grant (unlike lorekit_member_org_ids/lorekit_org_role, which +-- return only booleans/ids): this function returns PII (handle, avatar_url) +-- for other users, so it is authenticated-only, defense in depth beyond the +-- membership gate above. +grant execute on function lorekit_org_members_list(uuid) to authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00024', 'org_member_identities') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00025_safe_org_deletion.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- Safe org deletion — soft-delete + retention + explicit purge. +-- +-- The #73 ON DELETE CASCADE discussion resolved to KEEP the cascade (an org's +-- memberships/invites/limits and its `memories.org_id` FK all cascade on a real +-- delete) and add RECOVERY one layer above it: `lorekit_org_delete` now +-- SOFT-deletes (stamps `orgs.deleted_at`), and a separate owner-only +-- `lorekit_org_purge` performs the real cascading delete as an explicit second +-- step (the permanent-delete / future grace-period-job path). +-- +-- A soft-deleted org's lore disappears from every member's reads through ONE +-- change: `lorekit_member_org_ids()` (the single tenant-visibility predicate, +-- consumed by both the `memories` RLS read policies AND the mirrored TS +-- `applyTenantScope`, which reads this function's OUTPUT list) now excludes +-- soft-deleted orgs. No read-path code outside this function changes; the +-- `tenant-scope.ts` mirror and its drift guards (edge-parity / tenant-scope-usage) +-- stay green because the function's contract (a list of visible org ids) is +-- unchanged — only its result shrinks. +-- ═════════════════════════════════════════════════════════════════════════ + +-- 1. The soft-delete marker. Nullable; NULL = live (the overwhelming majority), +-- a timestamp = soft-deleted at that instant. Added BEFORE the function +-- recreate below so the join target exists. +alter table orgs add column if not exists deleted_at timestamptz; + +-- 2. Re-create the single membership-truth function to hide soft-deleted orgs. +-- Signature / STABLE / SECURITY DEFINER / search_path all preserved verbatim +-- from 00014 — only the body gains the `orgs` join + `deleted_at is null` +-- filter. Because the memories RLS policies and the edge `applyTenantScope` +-- helper both route through this function, a soft-deleted org's memories +-- vanish from ALL reads with no other change. +create or replace function lorekit_member_org_ids(p_user_id uuid) +returns setof uuid +language sql +stable +security definer +set search_path = public +as $$ + select om.org_id + from org_members om + join orgs o on o.id = om.org_id + where om.user_id = p_user_id + and o.deleted_at is null; +$$; + +grant execute on function lorekit_member_org_ids(uuid) to authenticated, service_role; + +-- 3. The `orgs` SELECT policy (00014) uses a DIRECT org_members subquery, not +-- `lorekit_member_org_ids`, so it must be changed explicitly — otherwise a +-- soft-deleted org would still be directly readable via a `from orgs` select +-- (e.g. the org switcher). `rls_org_limits_select` (00018) DOES route through +-- the function above, so it is hidden transitively with no change here. +drop policy if exists "rls_orgs_select" on orgs; +create policy "rls_orgs_select" + on orgs for select + using ( + deleted_at is null + and id in (select org_id from org_members where user_id = auth.uid()) + ); + +-- 4. `lorekit_org_delete` becomes a SOFT delete: stamp `deleted_at` instead of +-- removing the row. Same owner-only authorization (`delete_org` capability) +-- and `auth.uid()` actor pattern as 00022. Idempotent: a second call on an +-- already-soft-deleted org is a no-op (the `deleted_at is null` guard) and +-- still returns void. +create or replace function lorekit_org_delete(p_org_id uuid) +returns void +language plpgsql +security definer +set search_path = public +as $$ +begin + if not lorekit_org_can(auth.uid(), p_org_id, 'delete_org') then + raise exception using errcode = 'LK002', message = format('org_permission_denied: org=%s capability=delete_org', p_org_id); + end if; + + update orgs set deleted_at = now() where id = p_org_id and deleted_at is null; +end; +$$; + +grant execute on function lorekit_org_delete(uuid) to authenticated, service_role; + +-- 5. NEW: `lorekit_org_purge` — the real, irreversible cascading delete. Same +-- owner-only gate as delete (there is no separate "purge" capability; +-- permanent removal is the owner's prerogative, identical trust level to +-- soft-delete). The `delete from orgs` cascades to org_members / org_invites +-- / org_limits and nulls/cascades `memories.org_id` per the FK definitions. +-- Not authorization-gated on `deleted_at` — an owner may purge a live org +-- directly (skip the retention window) or purge one already soft-deleted. +-- No app-layer/server-action caller ships this PR: the audit_log action +-- CHECK (00023) has no `org.purge` value, so wiring a dashboard purge button +-- would require a second CHECK migration. Shipped SQL-first as the explicit +-- permanent-delete / grace-period-job entry point; the dashboard delete flow +-- uses the soft-delete above. +create or replace function lorekit_org_purge(p_org_id uuid) +returns void +language plpgsql +security definer +set search_path = public +as $$ +begin + if not lorekit_org_can(auth.uid(), p_org_id, 'delete_org') then + raise exception using errcode = 'LK002', message = format('org_permission_denied: org=%s capability=purge', p_org_id); + end if; + + delete from orgs where id = p_org_id; +end; +$$; + +grant execute on function lorekit_org_purge(uuid) to authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00025', 'safe_org_deletion') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00026_scope_org_bindings.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- Scope → org binding: bind a scope (e.g. a repo) to an org so writes under it +-- are routed to the org automatically, without each write naming the org. +-- +-- Authorization stays server-derived, never caller-trusted (the Phase 2 rule): +-- • Creating a binding requires the actor be an admin/owner of the org +-- (new `manage_scopes` capability on lorekit_org_can). +-- • A write under a bound scope is routed to the org ONLY IF the writer is a +-- write-capable member of that org. A non-member's (or viewer's) write +-- falls back to PERSONAL — never rejected, never silently lost: memory_write +-- reports the bound org's slug back so the caller can surface an actionable +-- notice ("saved personal; this scope is shared with , ask an admin"). +-- +-- Precedence: an explicit p_org_slug on memory_write always wins; the binding +-- only applies when no org was named. +-- ═════════════════════════════════════════════════════════════════════════ + +-- 1. The binding table. One org per scope GLOBALLY (unique on scope) so +-- auto-routing is unambiguous — a scope can't be bound to two orgs. A +-- soft-deleted org's binding is harmless: memory_write joins orgs and skips +-- deleted ones (and the FK cascade removes the binding on a real purge). +create table org_scope_bindings ( + id uuid primary key default gen_random_uuid(), + org_id uuid not null references orgs(id) on delete cascade, + scope text not null, + created_by uuid, + created_at timestamptz not null default now() +); + +create unique index org_scope_bindings_scope_uniq on org_scope_bindings (scope); +create index org_scope_bindings_org_idx on org_scope_bindings (org_id); + +alter table org_scope_bindings enable row level security; + +-- Members of the org can see its bindings (the dashboard's "Shared scopes" +-- list). No insert/update/delete policy — all writes go through the RPCs below, +-- mirroring orgs/org_members/org_invites (Phase 3). +create policy "rls_scope_bindings_select" + on org_scope_bindings for select + using (org_id in (select lorekit_member_org_ids(auth.uid()))); + +grant select on org_scope_bindings to authenticated, service_role; + +-- 2. Extend lorekit_org_can with `manage_scopes` (admin/owner) — the single +-- role→capability source, extended, never re-derived. Body is 00022's +-- verbatim plus the one new capability line. +create or replace function lorekit_org_can(p_user_id uuid, p_org_id uuid, p_capability text) +returns boolean +language plpgsql +stable +security definer +set search_path = public +as $$ +declare + v_role text := lorekit_org_role(p_user_id, p_org_id); +begin + if v_role is null then + return false; + end if; + + return case p_capability + when 'write' then v_role in ('member', 'admin', 'owner') + when 'archive' then v_role in ('member', 'admin', 'owner') + when 'restore' then v_role in ('member', 'admin', 'owner') + when 'hard_delete' then v_role in ('admin', 'owner') + when 'invite' then v_role in ('admin', 'owner') + when 'revoke_invite' then v_role in ('admin', 'owner') + when 'remove_member' then v_role in ('admin', 'owner') + when 'change_role' then v_role in ('admin', 'owner') + when 'rename_org' then v_role in ('admin', 'owner') + when 'manage_scopes' then v_role in ('admin', 'owner') + when 'delete_org' then v_role = 'owner' + else false + end; +end; +$$; + +-- 3. Bind / unbind RPCs. SECURITY DEFINER, actor = auth.uid() (dashboard JWT +-- session — the Phase 3 pattern; NO caller-supplied user-id). Gated on +-- `manage_scopes`. Bind is idempotent for the same (org, scope); a scope +-- already bound to a DIFFERENT org raises a distinct 'scope_bound_elsewhere' +-- so the caller can report the conflict. +create or replace function lorekit_scope_bind(p_org_id uuid, p_scope text) +returns uuid +language plpgsql +security definer +set search_path = public +as $$ +declare + v_actor uuid := auth.uid(); + v_existing_org uuid; + v_id uuid; +begin + if p_scope is null or length(trim(p_scope)) = 0 then + raise exception using errcode = 'P0001', message = 'scope is required'; + end if; + + if not lorekit_org_can(v_actor, p_org_id, 'manage_scopes') then + raise exception using errcode = 'LK002', + message = format('org_permission_denied: org=%s capability=manage_scopes', p_org_id); + end if; + + select org_id, id into v_existing_org, v_id from org_scope_bindings where scope = p_scope; + if v_existing_org = p_org_id then + -- Already bound to this org — idempotent no-op; return the existing id. + return v_id; + elsif v_existing_org is not null then + raise exception using errcode = 'P0001', + message = format('scope_bound_elsewhere: %s', p_scope); + end if; + + insert into org_scope_bindings (org_id, scope, created_by) + values (p_org_id, p_scope, v_actor) + returning id into v_id; + return v_id; +end; +$$; + +create or replace function lorekit_scope_unbind(p_org_id uuid, p_scope text) +returns void +language plpgsql +security definer +set search_path = public +as $$ +declare + v_actor uuid := auth.uid(); +begin + if p_scope is null or length(trim(p_scope)) = 0 then + raise exception using errcode = 'P0001', message = 'scope is required'; + end if; + + if not lorekit_org_can(v_actor, p_org_id, 'manage_scopes') then + raise exception using errcode = 'LK002', + message = format('org_permission_denied: org=%s capability=manage_scopes', p_org_id); + end if; + + delete from org_scope_bindings where org_id = p_org_id and scope = p_scope; +end; +$$; + +grant execute on function lorekit_scope_bind(uuid, text) to authenticated, service_role; +grant execute on function lorekit_scope_unbind(uuid, text) to authenticated, service_role; + +-- 4. Recreate memory_write with scope-binding auto-routing. The return type +-- grows two additive columns — `org_routed` (did the row land in an org?) +-- and `binding_org_slug` (the bound org's slug when a binding exists for the +-- scope, regardless of routing) — so the caller can build the graceful +-- fallback notice. Changing the RETURNS TABLE shape requires a drop first. +drop function if exists memory_write(uuid, text, text, text, text[], text, text, timestamptz, text); + +create or replace function memory_write( + p_user_id uuid, + p_scope text, + p_key text, + p_value text, + p_tags text[] default '{}', + p_source_agent text default null, + p_trigger text default null, + p_created_at timestamptz default null, + p_org_slug text default null +) +returns table (id uuid, created_at timestamptz, inserted boolean, org_routed boolean, binding_org_slug text) +language plpgsql +set search_path = public +as $$ +declare + v_org_id uuid; + v_binding_org uuid; + v_binding_slug text; +begin + if p_org_slug is not null then + -- Explicit org selector — takes precedence over any binding. + select o.id into v_org_id from orgs o where o.slug = p_org_slug and o.deleted_at is null; + if v_org_id is null then + raise exception using errcode = 'P0001', message = format('unknown_org: %s', p_org_slug); + end if; + if not lorekit_org_can(p_user_id, v_org_id, 'write') then + raise exception using errcode = 'LK002', message = format('org_permission_denied: org=%s', p_org_slug); + end if; + else + -- No explicit org: consult the scope binding. Resolve the bound org (if any, + -- excluding soft-deleted orgs). Route to it only when the writer is a + -- write-capable member; otherwise fall through to a personal/service write + -- but still report the bound slug so the caller can surface a notice. + select b.org_id, o.slug into v_binding_org, v_binding_slug + from org_scope_bindings b + join orgs o on o.id = b.org_id + where b.scope = p_scope and o.deleted_at is null; + + if v_binding_org is not null + and p_user_id is not null + and lorekit_org_can(p_user_id, v_binding_org, 'write') then + v_org_id := v_binding_org; + end if; + end if; + + if v_org_id is not null then + -- Org-owned write (explicit slug OR binding-routed). user_id stays NULL; + -- created_by/updated_by record the writer. org_routed = true. + return query + insert into memories ( + user_id, org_id, scope, key, value, tags, source_agent, trigger, + created_at, updated_at, created_by, updated_by + ) + values ( + null, v_org_id, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), p_user_id, p_user_id + ) + on conflict (org_id, scope, key) where org_id is not null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + updated_by = p_user_id + returning memories.id, memories.created_at, (xmax::text = '0') as inserted, + true as org_routed, v_binding_slug as binding_org_slug; + + elsif p_user_id is null then + -- service-role / CI write. + return query + insert into memories (user_id, scope, key, value, tags, source_agent, trigger, created_at, updated_at, created_by, updated_by) + values (null, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), null, null) + on conflict (scope, key) where org_id is null and user_id is null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now() + returning memories.id, memories.created_at, (xmax::text = '0') as inserted, + false as org_routed, v_binding_slug as binding_org_slug; + else + -- user-scoped (api_key / user) personal write — includes the binding + -- fallback case (a binding exists but the writer can't write to it). + return query + insert into memories (user_id, scope, key, value, tags, source_agent, trigger, created_at, updated_at, created_by, updated_by) + values (p_user_id, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), p_user_id, p_user_id) + on conflict (user_id, scope, key) where org_id is null and user_id is not null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + updated_by = p_user_id + returning memories.id, memories.created_at, (xmax::text = '0') as inserted, + false as org_routed, v_binding_slug as binding_org_slug; + end if; +end; +$$; + +grant execute on function memory_write(uuid, text, text, text, text[], text, text, timestamptz, text) + to anon, authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00026', 'scope_org_bindings') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00027_audit_log_scope_actions.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- Scope-binding UI (follow-up to 00026): extend the audit_log `action` CHECK +-- with `scope.bind` and `scope.unbind` so the dashboard's scope-bindings +-- server actions can call recordAuditEvent for bind/unbind mutations. +-- +-- Forward-only: drop + re-add the CHECK (a constraint can't be widened in +-- place) — the same pattern as 00023_audit_log_org_actions.sql. + +alter table audit_log drop constraint audit_log_action_check; + +alter table audit_log add constraint audit_log_action_check check (action in ( + 'api_key.create', + 'api_key.revoke', + 'webhook_secret.create', + 'webhook_secret.rotate', + 'webhook_secret.deactivate', + 'memory.create', + 'memory.update', + 'memory.archive', + 'memory.restore', + 'memory.delete', + 'limit.override', + 'org.create', + 'org.rename', + 'org.delete', + 'member.invite', + 'member.accept', + 'member.decline', + 'member.revoke', + 'member.remove', + 'member.role_change', + 'member.leave', + 'scope.bind', + 'scope.unbind' +)); + +insert into supabase_migrations.schema_migrations (version, name) +values ('00027', 'audit_log_scope_actions') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00028_invite_org_details.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- Invite-details modal: lets a PENDING (non-member) invitee see which org +-- invited them before accepting. `listPendingInvitesForMe`'s `orgs(name, +-- slug)` embed resolves to null for a non-member (the org read policy, +-- 00014/00025, is member-only), so today the banner falls back to the +-- literal "Someone invited you to an organization" with no name — the UX +-- problem this migration's read path fixes. +-- +-- SECURITY DEFINER, gated on the EXISTING `lorekit_invite_addressed_to_caller` +-- (00022) — never re-derived, never re-widened. Returns Tier-A fields ONLY +-- (org name, slug, created_at, an aggregate member COUNT, the inviter's +-- handle/avatar) — explicitly NO per-member identity list (Tier B, declined +-- by the user in Phase 0). This migration adds NO policy and touches NO +-- existing function or policy — it is a pure, additive read path, mirroring +-- `lorekit_org_members_list`'s (00024) "gate, then join auth.users for one +-- identity" shape, applied to the ADDRESSED-INVITEE gate instead of the +-- membership gate. +create or replace function lorekit_invite_org_details(p_invite_id uuid) +returns table ( + org_name text, + org_slug text, + org_created_at timestamptz, + member_count int, + inviter_handle text, + inviter_avatar_url text +) +language plpgsql +stable +security definer +set search_path = public +as $$ +declare + inv org_invites; +begin + select * into inv from org_invites where id = p_invite_id; + + -- No such invite, not pending, or not addressed to the caller: return an + -- EMPTY set, never raise and never leak whether the invite exists — the + -- same "gate denies silently, by returning nothing" shape used throughout + -- 00022/00024. + if inv.id is null or inv.status <> 'pending' or not lorekit_invite_addressed_to_caller(inv) then + return; + end if; + + return query + select + o.name, + o.slug, + o.created_at, + (select count(*) from org_members where org_id = inv.org_id)::int, + coalesce(u.raw_user_meta_data ->> 'user_name', u.raw_user_meta_data ->> 'preferred_username'), + u.raw_user_meta_data ->> 'avatar_url' + from orgs o + left join auth.users u on u.id = inv.invited_by + where o.id = inv.org_id and o.deleted_at is null; +end; +$$; + +-- No `anon` grant: this function returns PII (inviter handle/avatar, member +-- count) about an org the caller isn't yet a member of. Authenticated-only, +-- the same defense-in-depth posture as `lorekit_org_members_list` (00024) +-- and every other PII-bearing definer function in this schema. +grant execute on function lorekit_invite_org_details(uuid) to authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00028', 'invite_org_details') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00030_memory_ttl.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- Memory TTL (time-to-live) — per-row auto-expiry for transient memories. +-- +-- Adds an optional `expires_at` column to the memories table. When set, the +-- row is treated as invisible by all active-memory read paths once the +-- timestamp passes, without requiring the caller to manually archive or +-- delete it. Expired rows are physically removed by the new +-- `purge_expired_memories` RPC (called by the MCP `memory.purge` tool or a +-- scheduled job) — the existing `purge_archived_memories` RPC is unchanged. +-- +-- Design choices: +-- • `expires_at` is nullable — omitting it means the memory lives forever +-- (current default behaviour, fully backward-compatible). +-- • The active-read filter is extended to: +-- archived_at IS NULL AND (expires_at IS NULL OR expires_at > now()) +-- NOTE: The RLS policies on the memories table perform the archived_at +-- check. The expires_at check is applied at the query layer (toolRead, +-- toolList, toolSearch, etc.) via an additional .or() filter, keeping +-- the RLS policies lean (they cannot call now() cheaply in a policy). +-- • Expired rows remain physically present until purged so the sweep is a +-- cheap index scan, not a table scan, and no data is silently lost. +-- • On UPDATE (upsert conflict): p_ttl_days IS NOT NULL → refreshes +-- expires_at; p_ttl_days IS NULL → leaves existing expires_at unchanged +-- so a value-only update never accidentally clears an existing TTL. +-- ═════════════════════════════════════════════════════════════════════════ + +-- 1. Add the expiry column (nullable, no default → backward-compatible). +alter table memories + add column if not exists expires_at timestamptz; + +-- 2. Partial index for fast expiry sweeps (only non-null expires_at rows). +create index if not exists memories_expires_at_idx + on memories (expires_at) + where expires_at is not null; + +-- 3. Recreate memory_write with an optional p_ttl_days parameter. +-- +-- p_ttl_days: when provided (>= 1), sets expires_at = now() + p_ttl_days * INTERVAL '1 day'. +-- On INSERT: expires_at is computed from p_ttl_days (or left NULL if omitted). +-- On UPDATE: expires_at is refreshed ONLY when p_ttl_days is not null — +-- omitting it on an update leaves the existing expiry unchanged. +-- The return type grows an additive `expires_at` column so callers can +-- surface the computed expiry. Changing RETURNS TABLE shape requires DROP first. +drop function if exists memory_write(uuid, text, text, text, text[], text, text, timestamptz, text); + +create or replace function memory_write( + p_user_id uuid, + p_scope text, + p_key text, + p_value text, + p_tags text[] default '{}', + p_source_agent text default null, + p_trigger text default null, + p_created_at timestamptz default null, + p_org_slug text default null, + p_ttl_days integer default null +) +returns table ( + id uuid, + created_at timestamptz, + inserted boolean, + org_routed boolean, + binding_org_slug text, + expires_at timestamptz +) +language plpgsql +set search_path = public +as $$ +declare + v_org_id uuid; + v_binding_org uuid; + v_binding_slug text; + v_expires_at timestamptz; +begin + -- Validate TTL: must be a positive integer when supplied. + if p_ttl_days is not null then + if p_ttl_days < 1 or p_ttl_days > 365 then + raise exception using errcode = 'P0001', + message = format('ttl_days must be between 1 and 365, got %s', p_ttl_days); + end if; + v_expires_at := now() + (p_ttl_days * interval '1 day'); + end if; + + if p_org_slug is not null then + select o.id into v_org_id from orgs o where o.slug = p_org_slug and o.deleted_at is null; + if v_org_id is null then + raise exception using errcode = 'P0001', message = format('unknown_org: %s', p_org_slug); + end if; + if not lorekit_org_can(p_user_id, v_org_id, 'write') then + raise exception using errcode = 'LK002', message = format('org_permission_denied: org=%s', p_org_slug); + end if; + else + select b.org_id, o.slug into v_binding_org, v_binding_slug + from org_scope_bindings b + join orgs o on o.id = b.org_id + where b.scope = p_scope and o.deleted_at is null; + + if v_binding_org is not null + and p_user_id is not null + and lorekit_org_can(p_user_id, v_binding_org, 'write') then + v_org_id := v_binding_org; + end if; + end if; + + if v_org_id is not null then + return query + insert into memories ( + user_id, org_id, scope, key, value, tags, source_agent, trigger, + created_at, updated_at, created_by, updated_by, expires_at + ) + values ( + null, v_org_id, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), + p_user_id, p_user_id, v_expires_at + ) + on conflict (org_id, scope, key) where org_id is not null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + updated_by = p_user_id, + expires_at = case when p_ttl_days is not null then v_expires_at else memories.expires_at end + returning + memories.id, memories.created_at, (xmax::text = '0') as inserted, + true as org_routed, v_binding_slug as binding_org_slug, memories.expires_at; + + elsif p_user_id is null then + return query + insert into memories ( + user_id, scope, key, value, tags, source_agent, trigger, + created_at, updated_at, created_by, updated_by, expires_at + ) + values ( + null, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), + null, null, v_expires_at + ) + on conflict (scope, key) where org_id is null and user_id is null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + expires_at = case when p_ttl_days is not null then v_expires_at else memories.expires_at end + returning + memories.id, memories.created_at, (xmax::text = '0') as inserted, + false as org_routed, v_binding_slug as binding_org_slug, memories.expires_at; + + else + return query + insert into memories ( + user_id, scope, key, value, tags, source_agent, trigger, + created_at, updated_at, created_by, updated_by, expires_at + ) + values ( + p_user_id, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), + p_user_id, p_user_id, v_expires_at + ) + on conflict (user_id, scope, key) where org_id is null and user_id is not null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + updated_by = p_user_id, + expires_at = case when p_ttl_days is not null then v_expires_at else memories.expires_at end + returning + memories.id, memories.created_at, (xmax::text = '0') as inserted, + false as org_routed, v_binding_slug as binding_org_slug, memories.expires_at; + end if; +end; +$$; + +grant execute on function memory_write(uuid, text, text, text, text[], text, text, timestamptz, text, integer) + to anon, authenticated, service_role; + +-- 4. New purge_expired_memories RPC — hard-deletes active rows whose +-- expires_at is in the past. Separate from purge_archived_memories so +-- each sweep is independently callable and narrowly indexed. +create or replace function purge_expired_memories( + p_user_id uuid +) +returns integer +language plpgsql +security definer +set search_path = public +as $$ +declare + v_count integer; +begin + delete from memories + where user_id = p_user_id + and expires_at is not null + and expires_at < now() + and archived_at is null -- archived rows stay with purge_archived_memories + ; + get diagnostics v_count = row_count; + return v_count; +end; +$$; + +-- Intentionally excludes anon: an unauthenticated caller could supply any +-- p_user_id and hard-delete another user's expired rows (no auth.uid() check +-- inside the function body). Authenticated + service_role are sufficient. +grant execute on function purge_expired_memories(uuid) to authenticated, service_role; + +comment on column memories.expires_at is + 'Optional timestamp after which the row is treated as invisible by active-read + paths. NULL means the row never expires. Physically deleted by + purge_expired_memories() or the MCP memory.purge tool.'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00030', 'memory_ttl') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00031_memory_ttl_clear.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- Add p_clear_ttl to memory_write so callers can explicitly remove an +-- existing expires_at (set it back to NULL → the row never expires again). +-- +-- Without this, the only way to clear a TTL is a raw UPDATE, which bypasses +-- the write RPC's authorization logic (org cap, org permission checks, audit). +-- +-- Design: +-- p_clear_ttl boolean default false +-- When true: sets expires_at = NULL (clears the TTL). +-- When false: existing behaviour — p_ttl_days controls expires_at. +-- p_clear_ttl takes precedence: if both p_clear_ttl = true and +-- p_ttl_days IS NOT NULL, the clear wins. +-- ═════════════════════════════════════════════════════════════════════════ + +drop function if exists memory_write(uuid, text, text, text, text[], text, text, timestamptz, text, integer); + +create or replace function memory_write( + p_user_id uuid, + p_scope text, + p_key text, + p_value text, + p_tags text[] default '{}', + p_source_agent text default null, + p_trigger text default null, + p_created_at timestamptz default null, + p_org_slug text default null, + p_ttl_days integer default null, + p_clear_ttl boolean default false +) +returns table ( + id uuid, + created_at timestamptz, + inserted boolean, + org_routed boolean, + binding_org_slug text, + expires_at timestamptz +) +language plpgsql +set search_path = public +as $$ +declare + v_org_id uuid; + v_binding_org uuid; + v_binding_slug text; + v_expires_at timestamptz; + -- Tri-state for the expires_at UPDATE branch: + -- 'clear' → set to NULL + -- 'set' → set to v_expires_at + -- 'keep' → leave unchanged (CASE WHEN ... ELSE memories.expires_at END) + v_ttl_action text := 'keep'; +begin + if p_clear_ttl then + -- Explicit clear wins over any p_ttl_days value. + v_ttl_action := 'clear'; + elsif p_ttl_days is not null then + if p_ttl_days < 1 or p_ttl_days > 365 then + raise exception using errcode = 'P0001', + message = format('ttl_days must be between 1 and 365, got %s', p_ttl_days); + end if; + v_expires_at := now() + (p_ttl_days * interval '1 day'); + v_ttl_action := 'set'; + end if; + + if p_org_slug is not null then + select o.id into v_org_id from orgs o where o.slug = p_org_slug and o.deleted_at is null; + if v_org_id is null then + raise exception using errcode = 'P0001', message = format('unknown_org: %s', p_org_slug); + end if; + if not lorekit_org_can(p_user_id, v_org_id, 'write') then + raise exception using errcode = 'LK002', message = format('org_permission_denied: org=%s', p_org_slug); + end if; + else + select b.org_id, o.slug into v_binding_org, v_binding_slug + from org_scope_bindings b + join orgs o on o.id = b.org_id + where b.scope = p_scope and o.deleted_at is null; + + if v_binding_org is not null + and p_user_id is not null + and lorekit_org_can(p_user_id, v_binding_org, 'write') then + v_org_id := v_binding_org; + end if; + end if; + + if v_org_id is not null then + return query + insert into memories ( + user_id, org_id, scope, key, value, tags, source_agent, trigger, + created_at, updated_at, created_by, updated_by, expires_at + ) + values ( + null, v_org_id, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), + p_user_id, p_user_id, v_expires_at + ) + on conflict (org_id, scope, key) where org_id is not null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + updated_by = p_user_id, + expires_at = case v_ttl_action + when 'clear' then null + when 'set' then v_expires_at + else memories.expires_at + end + returning + memories.id, memories.created_at, (xmax::text = '0') as inserted, + true as org_routed, v_binding_slug as binding_org_slug, memories.expires_at; + + elsif p_user_id is null then + return query + insert into memories ( + user_id, scope, key, value, tags, source_agent, trigger, + created_at, updated_at, created_by, updated_by, expires_at + ) + values ( + null, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), + null, null, v_expires_at + ) + on conflict (scope, key) where org_id is null and user_id is null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + expires_at = case v_ttl_action + when 'clear' then null + when 'set' then v_expires_at + else memories.expires_at + end + returning + memories.id, memories.created_at, (xmax::text = '0') as inserted, + false as org_routed, v_binding_slug as binding_org_slug, memories.expires_at; + + else + return query + insert into memories ( + user_id, scope, key, value, tags, source_agent, trigger, + created_at, updated_at, created_by, updated_by, expires_at + ) + values ( + p_user_id, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), + p_user_id, p_user_id, v_expires_at + ) + on conflict (user_id, scope, key) where org_id is null and user_id is not null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + updated_by = p_user_id, + expires_at = case v_ttl_action + when 'clear' then null + when 'set' then v_expires_at + else memories.expires_at + end + returning + memories.id, memories.created_at, (xmax::text = '0') as inserted, + false as org_routed, v_binding_slug as binding_org_slug, memories.expires_at; + end if; +end; +$$; + +grant execute on function memory_write(uuid, text, text, text, text[], text, text, timestamptz, text, integer, boolean) + to anon, authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00031', 'memory_ttl_clear') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00032_plans.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- LoreKit billing plans — Phase 1 (free plan only). +-- +-- Introduces a `plan` concept without any billing integration: +-- * `user_plans` table tracks which plan a user is on (default: 'free'). +-- * `lorekit_default_limit` is updated so plan-aware defaults can be read +-- from the table — a future paid tier only needs to insert a row here. +-- * The free-plan `max_memories` default rises from 1000 → 5000. +-- * A `plan_name` column is added to `user_limits` so any manual override +-- row can be annotated with its originating plan (audit trail). +-- +-- Security posture: +-- * Users can read their own plan row (dashboard "you are on the free plan"). +-- * Only service-role / admin can insert or update plan rows (no self-upgrade). +-- * RLS on `user_plans` mirrors `user_limits` exactly. +-- +-- Existing `user_limits` per-user overrides continue to take precedence over +-- the plan default via the existing `COALESCE(override, default)` chain in +-- `lorekit_get_limit` — no change needed there. Raising a user's cap beyond +-- their plan ceiling remains a service-role upsert into `user_limits`. + +-- 1. Plan definitions table — the authoritative per-plan limit defaults. +-- Adding a new plan in future is a single INSERT here. +create table if not exists plans ( + name text primary key, -- e.g. 'free', 'pro', 'team' + max_memories integer not null, + requests_per_minute integer not null, + description text, + created_at timestamptz not null default now(), + updated_at timestamptz not null default now() +); + +-- Trigger to keep updated_at current on plan changes. +create or replace trigger plans_updated_at + before update on plans + for each row execute function set_updated_at(); + +-- Anyone can read plan definitions (they are public limits, not PII). +alter table plans enable row level security; +create policy "rls_plans_select" + on plans for select + using (true); + +-- Seed the free plan. ON CONFLICT so the migration is idempotent on re-runs. +insert into plans (name, max_memories, requests_per_minute, description) +values ( + 'free', + 5000, + 120, + 'Free plan: up to 5000 stored memories, 120 requests per minute.' +) +on conflict (name) do update + set max_memories = excluded.max_memories, + requests_per_minute = excluded.requests_per_minute, + description = excluded.description, + updated_at = now(); + +-- 2. Per-user plan assignment. Absence of a row means the user is on 'free'. +create table if not exists user_plans ( + user_id uuid primary key references auth.users on delete cascade, + plan_name text not null default 'free' references plans(name), + created_at timestamptz not null default now(), + updated_at timestamptz not null default now() +); + +alter table user_plans enable row level security; + +-- Users can read their own plan row. +create policy "rls_user_plans_select" + on user_plans for select + using (user_id = auth.uid()); + +create or replace trigger user_plans_updated_at + before update on user_plans + for each row execute function set_updated_at(); + +-- 3. Annotate user_limits with the plan that originated the override (optional, +-- for audit/analytics). Nullable so existing rows are unaffected. +alter table user_limits + add column if not exists plan_name text references plans(name); + +-- 4. Update lorekit_default_limit to read from the `plans` table when a +-- matching row exists, falling back to hard-coded values for safety. +-- The user-plan lookup goes through lorekit_get_limit (below) so this +-- function stays plan-agnostic — it is only the universal fallback. +create or replace function lorekit_default_limit(p_key text) +returns integer +language sql +stable -- changed from immutable: now reads the plans table +security definer +set search_path = public +as $$ + select case p_key + when 'max_memories' then (select max_memories from plans where name = 'free') + when 'requests_per_minute' then (select requests_per_minute from plans where name = 'free') + else null + end; +$$; + +-- 5. Plan-aware limit resolver. Replaces lorekit_get_limit with a version that +-- reads the user's plan first, then falls back to the per-user override, +-- then to the plan default. The precedence chain is: +-- user_limits override > plan default > lorekit_default_limit (free). +-- +-- This ensures a pro-plan user gets their plan ceiling even without an +-- explicit user_limits row, while a user_limits override still wins +-- (needed when an admin manually caps a misbehaving user below their plan). +create or replace function lorekit_get_limit(p_user_id uuid, p_key text) +returns integer +language plpgsql +security definer +set search_path = public +as $$ +declare + v_override integer; + v_plan_name text; + v_plan_limit integer; +begin + -- Step 1: per-user manual override (highest precedence). + if p_key = 'max_memories' then + select max_memories into v_override from user_limits where user_id = p_user_id; + elsif p_key = 'requests_per_minute' then + select requests_per_minute into v_override from user_limits where user_id = p_user_id; + end if; + if v_override is not null then + return v_override; + end if; + + -- Step 2: plan-based limit (falls back to 'free' when no row exists). + select coalesce(up.plan_name, 'free') + into v_plan_name + from user_plans up + where up.user_id = p_user_id; + + if v_plan_name is null then + v_plan_name := 'free'; + end if; + + if p_key = 'max_memories' then + select max_memories into v_plan_limit from plans where name = v_plan_name; + elsif p_key = 'requests_per_minute' then + select requests_per_minute into v_plan_limit from plans where name = v_plan_name; + end if; + + -- Step 3: ultimate fallback (handles unknown keys gracefully). + return coalesce(v_plan_limit, lorekit_default_limit(p_key)); +end; +$$; + +-- Grant read on new tables so authenticated + anon selects work under RLS. +grant select on table plans to anon, authenticated, service_role; +grant select on table user_plans to anon, authenticated, service_role; +grant insert, update on table user_plans to service_role; +grant insert, update on table user_limits to service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00032', 'plans') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00033_performance_indexes.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- Performance optimisations for the hot read paths in tools.ts. +-- +-- Analysis of the three most-called query patterns: +-- +-- 1. toolRead — SELECT … FROM memories WHERE scope=? AND key=? +-- AND archived_at IS NULL AND (expires_at IS NULL OR expires_at > now()) +-- Existing : memories_scope_key (scope, key) — hits the filter, but forces +-- a heap fetch + recheck for archived_at / expires_at on every row. +-- Fix : Replace with a PARTIAL index that pre-filters active, non-expired +-- rows and adds user_id so api_key queries can satisfy the full +-- predicate from the index alone (index-only scan). +-- +-- 2. toolList — SELECT … FROM memories WHERE scope=? AND archived_at IS NULL +-- AND (expires_at IS NULL OR expires_at > now()) +-- ORDER BY updated_at DESC LIMIT ? +-- Existing : memories_scope_idx (scope) — covers the WHERE but not the ORDER. +-- Postgres sorts in-memory; large scopes cause a filesort. +-- Fix : Add (scope, updated_at DESC) so the planner can satisfy the +-- order from the index and stop early after LIMIT rows. +-- +-- 3. toolListArchived — WHERE scope=? AND archived_at IS NOT NULL +-- ORDER BY archived_at DESC LIMIT ? +-- Existing : No index covering (scope, archived_at). +-- Fix : Add a PARTIAL index (archived_at IS NOT NULL) on (scope, archived_at DESC). +-- +-- 4. memberOrgIds RPC — called on every api_key read (toolRead / toolList / +-- toolSearch / toolListArchived). The RPC itself is already fast, +-- but adding a covering index on org_members (user_id, org_id) +-- means the SECURITY DEFINER function never hits the heap. +-- +-- 5. rate_limit_counters — no additional index added. The existing +-- rate_limit_counters_window_idx (window_start) from 00004_limits.sql +-- already covers the reaper's DELETE. A partial index would need +-- a volatile predicate (now()), which PostgreSQL disallows. +-- +-- All indexes are created with IF NOT EXISTS so they can be applied to a +-- running production database (CONCURRENTLY is not used inside migrations +-- because it cannot run inside a transaction). + +-- 1. Active-memories point-read covering index. +-- Covers: toolRead, and the cap trigger's active-row count. +create index if not exists memories_scope_key_active_idx + on memories (scope, key, user_id) + where archived_at is null; + +-- 2. List-by-scope ordered index — eliminates the filesort for toolList. +create index if not exists memories_scope_updated_at_idx + on memories (scope, updated_at desc) + where archived_at is null; + +-- 3. Archived list ordered index — eliminates the filesort for toolListArchived. +create index if not exists memories_scope_archived_at_idx + on memories (scope, archived_at desc) + where archived_at is not null; + +-- 4. org_members covering index — eliminates heap fetch for memberOrgIds RPC. +create index if not exists org_members_user_org_idx + on org_members (user_id, org_id); + +-- 5. rate_limit_counters — no additional index needed. +-- The existing `rate_limit_counters_window_idx (window_start)` from +-- 00004_limits.sql already covers the reaper's DELETE predicate +-- (window_start < now() - interval). A partial index on this table +-- would require a volatile predicate (now()), which PostgreSQL rejects +-- for index predicates — so the existing index is the right choice. + +-- 6. Schedule automatic expired-memory purge via pg_cron (when available). +-- Expired memories (expires_at < now()) are invisible to all read paths but +-- still occupy cap headroom and bloat the table. Purge them nightly. +-- lorekit_purge_all_expired_memories() is defined below. +create or replace function lorekit_purge_all_expired_memories() +returns integer +language plpgsql +security definer +set search_path = public +as $$ +declare + v_count integer; +begin + delete from memories + where expires_at is not null + and expires_at < now(); + get diagnostics v_count = row_count; + return v_count; +end; +$$; + +grant execute on function lorekit_purge_all_expired_memories() to service_role; + +-- Schedule nightly at 03:00 UTC when pg_cron is available. +do $$ +begin + if exists (select 1 from pg_extension where extname = 'pg_cron') then + perform cron.schedule( + 'lorekit-purge-expired-memories', + '0 3 * * *', + $cron$select lorekit_purge_all_expired_memories()$cron$ + ); + end if; +end; +$$; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00033', 'performance_indexes') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00034_usage_events.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- Usage events — structured telemetry for post-processing and plan analysis. +-- +-- Captures every significant MCP tool call outcome as a lightweight structured +-- row so you can answer questions like: +-- * How many writes/reads/searches per user per day? (→ plan sizing) +-- * What fraction of writes hit the cap? (→ conversion signal) +-- * What is the p50/p95 memory count at cap-hit time? (→ plan threshold) +-- * How many active memories does a median user hold? (→ free-plan ceiling) +-- * Which scope types are most used? (→ product prioritisation) +-- * What is the rate-limit hit rate per plan? (→ plan RPM calibration) +-- +-- Design decisions: +-- * Append-only (no UPDATE/DELETE RLS policies). Rows age out via a pg_cron +-- job (lorekit_purge_old_usage_events) or a retention policy. +-- * user_id is stored pseudonymously (the Supabase UUID, never email/handle). +-- * No memory key/value/scope string is stored — just bounded categorical +-- attributes (tool_name, scope_type, outcome, plan_name) that are safe for +-- aggregation dashboards without privacy review. +-- * auth_type discriminates api_key vs JWT vs service writes so you can filter +-- out internal/CI traffic in analytics. +-- * duration_ms tracks wall-clock handler time for latency-percentile analysis. +-- +-- RLS: users can read their own rows (self-service "my usage" views); service- +-- role writes; no user insert/update/delete. + +create table if not exists usage_events ( + id uuid primary key default gen_random_uuid(), + user_id uuid references auth.users on delete set null, -- null for service-role + org_id uuid references orgs(id) on delete set null, -- null for personal writes + plan_name text, -- plan at time of event + tool_name text not null, -- e.g. 'memory.write' + scope_type text, -- 'global'|'project'|'repo'|'branch'|null + auth_type text not null, -- 'api_key'|'jwt'|'service' + outcome text not null, -- 'ok'|'cap_exceeded'|'rate_limited'|'permission_denied'|'error' + duration_ms integer, -- wall-clock handler time + memory_count integer, -- active memories at time of event (write path only) + created_at timestamptz not null default now() +); + +alter table usage_events enable row level security; + +-- Users can read their own usage rows (self-service dashboard). +create policy "rls_usage_events_select" + on usage_events for select + using (user_id = auth.uid()); + +-- Service-role can insert (the only write path — no user insert policy). +-- (service_role bypasses RLS by default in Supabase; this is documentation.) + +-- Covering indexes for the most common analytics queries: +-- * Per-user time-series: (user_id, created_at) for daily/weekly rollups. +-- * Funnel by tool + outcome: (tool_name, outcome, created_at). +-- * Plan analysis: (plan_name, outcome, created_at). +create index if not exists usage_events_user_created_at_idx + on usage_events (user_id, created_at desc); + +create index if not exists usage_events_tool_outcome_idx + on usage_events (tool_name, outcome, created_at desc); + +create index if not exists usage_events_plan_outcome_idx + on usage_events (plan_name, outcome, created_at desc); + +-- Reaper: hard-delete usage events older than p_older_than (default 90 days). +-- Called by pg_cron. Returns the count of deleted rows. +create or replace function lorekit_purge_old_usage_events( + p_older_than interval default interval '90 days' +) +returns integer +language plpgsql +security definer +set search_path = public +as $$ +declare + v_count integer; +begin + delete from usage_events + where created_at < now() - p_older_than; + get diagnostics v_count = row_count; + return v_count; +end; +$$; + +grant execute on function lorekit_purge_old_usage_events(interval) to service_role; + +-- Schedule quarterly cleanup (retain 90 days) — runs weekly on Sundays at 02:00 UTC. +do $$ +begin + if exists (select 1 from pg_extension where extname = 'pg_cron') then + perform cron.schedule( + 'lorekit-purge-old-usage-events', + '0 2 * * 0', + $cron$select lorekit_purge_old_usage_events()$cron$ + ); + end if; +end; +$$; + +-- Helper RPC: record one usage event. Accepts nullable fields gracefully. +-- Called from the edge function and Node server after each tool call completes. +-- Returns the new row's id for correlation. +create or replace function lorekit_record_usage_event( + p_user_id uuid default null, + p_org_id uuid default null, + p_plan_name text default null, + p_tool_name text default null, + p_scope_type text default null, + p_auth_type text default null, + p_outcome text default null, + p_duration_ms integer default null, + p_memory_count integer default null +) +returns uuid +language plpgsql +security definer +set search_path = public +as $$ +declare + v_id uuid; +begin + insert into usage_events ( + user_id, org_id, plan_name, + tool_name, scope_type, auth_type, + outcome, duration_ms, memory_count + ) values ( + p_user_id, p_org_id, p_plan_name, + p_tool_name, p_scope_type, p_auth_type, + p_outcome, p_duration_ms, p_memory_count + ) + returning id into v_id; + return v_id; +exception + when others then + -- Never let telemetry writes break the primary operation. + return null; +end; +$$; + +grant execute on function lorekit_record_usage_event(uuid, uuid, text, text, text, text, text, integer, integer) + to anon, authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00034', 'usage_events') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00035_memory_count.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- lorekit_memory_count(p_user_id) — efficient active memory count for the +-- /settings/plan page usage bar. +-- +-- The dashboard can't do a raw SELECT COUNT(*) per page load without either +-- bypassing RLS (service-role) or doing a full table scan under the user's +-- RLS context. This SECURITY DEFINER function does the scoped count with the +-- index-backed predicate that enforce_memory_cap() uses, so no extra full-scan +-- occurs. It reuses memories_user_idx + the archived_at IS NULL partial index +-- (memories_org_id_active_idx for the org branch). +-- +-- Returns: JSONB { "count": , "limit": , "plan": } +-- so the caller gets everything it needs in one round-trip. + +create or replace function lorekit_memory_count(p_user_id uuid) +returns jsonb +language plpgsql +security definer +stable +set search_path = public +as $$ +declare + v_count integer; + v_limit integer; + v_plan_name text; +begin + -- Count active personal memories for this user. + select count(*) + into v_count + from memories + where user_id = p_user_id + and archived_at is null; + + -- Resolve effective limit and plan name in the same function call. + v_limit := lorekit_get_limit(p_user_id, 'max_memories'); + + select coalesce(up.plan_name, 'free') + into v_plan_name + from user_plans up + where up.user_id = p_user_id; + + if v_plan_name is null then + v_plan_name := 'free'; + end if; + + return jsonb_build_object( + 'count', v_count, + 'limit', v_limit, + 'plan', v_plan_name + ); +end; +$$; + +-- Grant to authenticated so the Next.js server action can call it under the +-- user JWT (supabase-js createServerClient uses the user's JWT session). +grant execute on function lorekit_memory_count(uuid) + to anon, authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00035', 'memory_count') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00036_fix_memory_count.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- Fix memory count consistency (bug fix). +-- +-- Root cause: lorekit_memory_count (00035) only counted personal memories +-- (WHERE user_id = p_user_id AND archived_at IS NULL), while the dashboard +-- query fetched all memories visible under RLS — including org-owned rows +-- (user_id IS NULL, org_id set) and archived rows. The two numbers diverged +-- for any user who (a) belongs to an org or (b) has archived memories. +-- +-- Correct definition: active (archived_at IS NULL), non-expired memories that +-- the user owns personally OR through org membership. This matches what the +-- dashboard should show and what the enforcement cap cares about (the cap is +-- per-personal-user or per-org, not a combined total, but the displayed count +-- should reflect what the user sees). +-- +-- The dashboard-side fix (filter archived rows, add org memberships to the +-- count) lives in the Next.js query layer — no migration needed there. +-- +-- This migration drops and recreates lorekit_memory_count with: +-- * personal count: user_id = p_user_id AND archived_at IS NULL +-- * org count: org_id IN (lorekit_member_org_ids(p_user_id)) AND archived_at IS NULL +-- * total: personal + org (what the user sees in the dashboard) +-- Both personal and org counts are returned so the UI can decide what to show. + +create or replace function lorekit_memory_count(p_user_id uuid) +returns jsonb +language plpgsql +security definer +stable +set search_path = public +as $$ +declare + v_personal_count integer; + v_org_count integer; + v_limit integer; + v_plan_name text; + v_org_ids uuid[]; +begin + -- Resolve org memberships once. + select array_agg(org_id) + into v_org_ids + from org_members om + join orgs o on o.id = om.org_id + where om.user_id = p_user_id + and o.deleted_at is null; + + -- Personal active memories. + select count(*) + into v_personal_count + from memories + where user_id = p_user_id + and archived_at is null; + + -- Org-owned active memories across all orgs the user belongs to. + if v_org_ids is not null and cardinality(v_org_ids) > 0 then + select count(*) + into v_org_count + from memories + where org_id = any(v_org_ids) + and archived_at is null; + else + v_org_count := 0; + end if; + + -- Resolve effective personal limit and plan name. + v_limit := lorekit_get_limit(p_user_id, 'max_memories'); + + select coalesce(up.plan_name, 'free') + into v_plan_name + from user_plans up + where up.user_id = p_user_id; + + if v_plan_name is null then + v_plan_name := 'free'; + end if; + + return jsonb_build_object( + 'count', v_personal_count + v_org_count, + 'personal_count', v_personal_count, + 'org_count', v_org_count, + 'limit', v_limit, + 'plan', v_plan_name + ); +end; +$$; + +grant execute on function lorekit_memory_count(uuid) + to anon, authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00036', 'fix_memory_count') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00037_github_installations.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- GitHub App installation tracking. +-- +-- A GitHub App installation covers one or more repositories and is identified +-- by a numeric installation_id. When a user installs the LoreKit GitHub App, +-- GitHub delivers an `installation` event; the edge webhook handler (behind the +-- GITHUB_APP_ENABLED flag) reconciles that event here. +-- +-- Authorization posture (the Phase 2 rule): +-- • The reconcile/upsert RPC is SECURITY DEFINER and is called only from the +-- edge function with the service-role key — never by an authenticated user +-- directly. No caller-supplied user_id is trusted: the edge handler looks +-- up the GitHub account id in auth.users and supplies the resolved uuid, or +-- NULL for a pending installation. +-- • RLS on github_installations: a user reads only the rows that are linked +-- to their own auth.uid() (status='linked'). Pending rows (user_id IS NULL) +-- are invisible to authenticated users until reconciled. +-- • installation_repositories is readable only through the parent +-- github_installations row: the service role reads it; the dashboard reads +-- via a joined server action. +-- +-- Fail-safe pending-identity: +-- An installation can arrive before any matching LoreKit user exists (e.g., +-- installed from GitHub Marketplace before signing in). Those installations +-- are stored with user_id = NULL + status = 'pending'. A later login or +-- membership event that matches the github_account_id transitions them to +-- 'linked'. They are NEVER dropped. +-- +-- Idempotency: +-- UNIQUE on installation_id + ON CONFLICT in the upsert RPC ensures that +-- redelivered `installation` events produce exactly one row. +-- ═════════════════════════════════════════════════════════════════════════ + +-- 1. github_installations — one row per GitHub App installation. +create table github_installations ( + id uuid primary key default gen_random_uuid(), + installation_id bigint not null, + github_account_id bigint not null, + github_account_login text not null, + account_type text not null check (account_type in ('User', 'Organization')), + user_id uuid references auth.users(id) on delete set null, + status text not null default 'pending' + check (status in ('pending', 'linked', 'removed')), + created_at timestamptz not null default now(), + updated_at timestamptz not null default now() +); + +create unique index github_installations_installation_id_uniq + on github_installations (installation_id); + +create index github_installations_github_account_id_idx + on github_installations (github_account_id); + +create index github_installations_user_id_idx + on github_installations (user_id) + where user_id is not null; + +alter table github_installations enable row level security; + +-- Users can read only their own linked installations (pending rows with +-- user_id NULL are invisible to authenticated users by this policy). +create policy "rls_github_installations_select" + on github_installations for select + using (user_id = auth.uid() and status = 'linked'); + +grant select on github_installations to authenticated, service_role; + +-- 2. installation_repositories — repos covered by an installation. +create table installation_repositories ( + id uuid primary key default gen_random_uuid(), + installation_id bigint not null references github_installations(installation_id) on delete cascade, + full_name text not null, + active boolean not null default true, + added_at timestamptz not null default now(), + removed_at timestamptz +); + +create index installation_repositories_installation_id_idx + on installation_repositories (installation_id); + +create index installation_repositories_full_name_idx + on installation_repositories (full_name) + where active = true; + +-- Unique: one active row per (installation_id, full_name). +create unique index installation_repositories_active_uniq + on installation_repositories (installation_id, full_name) + where active = true; + +alter table installation_repositories enable row level security; + +-- Readable only by service_role (the edge function) and by authenticated users +-- whose matching github_installations row is linked to their uid. +create policy "rls_installation_repositories_select" + on installation_repositories for select + using ( + installation_id in ( + select i.installation_id + from github_installations i + where i.user_id = auth.uid() and i.status = 'linked' + ) + ); + +grant select on installation_repositories to authenticated, service_role; + +-- 3. Reconcile/upsert RPC — called by the edge function (service-role key). +-- SECURITY DEFINER: operates on github_installations as the definer (bypasses +-- RLS), so the edge function cannot accidentally trigger the authenticated +-- select policy. Caller supplies a pre-resolved p_user_id (or NULL). +-- Idempotent: ON CONFLICT (installation_id) DO UPDATE. +create or replace function lorekit_installation_upsert( + p_installation_id bigint, + p_github_account_id bigint, + p_github_account_login text, + p_account_type text, + p_user_id uuid, -- NULL → pending + p_status text, -- 'pending' | 'linked' | 'removed' + p_repos text[] -- full_names to mark active; empty = no repo changes +) +returns uuid +language plpgsql +security definer +set search_path = public +as $$ +declare + v_id uuid; +begin + -- Upsert the installation row. + insert into github_installations ( + installation_id, github_account_id, github_account_login, account_type, + user_id, status, updated_at + ) + values ( + p_installation_id, p_github_account_id, p_github_account_login, p_account_type, + p_user_id, p_status, now() + ) + on conflict (installation_id) do update set + github_account_id = excluded.github_account_id, + github_account_login = excluded.github_account_login, + account_type = excluded.account_type, + user_id = case + -- Only overwrite user_id when transitioning to linked + -- or when updating an already-linked row. A pending + -- install that re-delivers must not lose a previously + -- linked user_id. + when excluded.status = 'linked' then excluded.user_id + when github_installations.status = 'linked' then github_installations.user_id + else excluded.user_id + end, + status = case + -- Never regress from 'linked' to 'pending'. + when github_installations.status = 'linked' + and excluded.status = 'pending' + then 'linked' + else excluded.status + end, + updated_at = now() + returning id into v_id; + + -- Upsert covered repos when provided. + if array_length(p_repos, 1) is not null then + -- Insert new active repo rows; skip if already active. + insert into installation_repositories (installation_id, full_name, active, added_at) + select p_installation_id, unnest(p_repos), true, now() + on conflict (installation_id, full_name) where active = true + do nothing; + end if; + + return v_id; +end; +$$; + +grant execute on function lorekit_installation_upsert(bigint, bigint, text, text, uuid, text, text[]) + to service_role; + +-- 4. Remove repos covered by an installation (installation_repositories action='removed'). +create or replace function lorekit_installation_remove_repos( + p_installation_id bigint, + p_repos text[] +) +returns void +language plpgsql +security definer +set search_path = public +as $$ +begin + update installation_repositories + set active = false, removed_at = now() + where installation_id = p_installation_id + and full_name = any(p_repos) + and active = true; +end; +$$; + +grant execute on function lorekit_installation_remove_repos(bigint, text[]) + to service_role; + +-- 5. Mark an installation as removed (soft-delete its repo rows too). +create or replace function lorekit_installation_remove( + p_installation_id bigint +) +returns void +language plpgsql +security definer +set search_path = public +as $$ +begin + update github_installations + set status = 'removed', updated_at = now() + where installation_id = p_installation_id; + + update installation_repositories + set active = false, removed_at = now() + where installation_id = p_installation_id + and active = true; +end; +$$; + +grant execute on function lorekit_installation_remove(bigint) + to service_role; + +-- 6. Identity lookup helper — given a GitHub numeric account id (stored as +-- text in auth.identities.provider_id), return the matching auth.users.id. +-- Used by the edge webhook handler to reconcile an installation to a LoreKit +-- user without exposing the auth schema through PostgREST. +-- Returns NULL when no matching user exists (pending path). +create or replace function lorekit_find_user_by_github_id(p_github_account_id text) +returns uuid +language sql +stable +security definer +set search_path = public +as $$ + select i.user_id + from auth.identities i + where i.provider = 'github' + and i.provider_id = p_github_account_id + limit 1; +$$; + +grant execute on function lorekit_find_user_by_github_id(text) + to service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00037', 'github_installations') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00038_memory_ttl_seconds.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- Extend memory_write TTL to support seconds and minutes in addition to days. +-- +-- Previously the RPC accepted p_ttl_days (integer days) and computed +-- expires_at = now() + p_ttl_days * INTERVAL '1 day'. +-- +-- This migration replaces p_ttl_days with p_ttl_seconds (integer seconds) +-- so callers can set fine-grained expiry (e.g. 30 s, 5 min) without +-- being forced to round up to a whole day. +-- +-- The application layer (mcp-core and the Deno edge function) normalises +-- all three user-visible units (ttl_days, ttl_minutes, ttl_seconds) to +-- seconds via parseTtl() before calling the RPC, so the DB only ever +-- receives an integer number of seconds. +-- +-- Design: +-- p_ttl_seconds: when provided (>= 1), sets +-- expires_at = now() + p_ttl_seconds * INTERVAL '1 second' +-- On INSERT: expires_at is computed from p_ttl_seconds (or NULL if omitted). +-- On UPDATE: expires_at is refreshed ONLY when p_ttl_seconds is not null — +-- omitting it on an update leaves the existing expiry unchanged. +-- p_clear_ttl still wins over p_ttl_seconds when both are supplied. +-- Validation: p_ttl_seconds must be between 1 and 31 536 000 (365 days). +-- ═════════════════════════════════════════════════════════════════════════ + +drop function if exists memory_write(uuid, text, text, text, text[], text, text, timestamptz, text, integer, boolean); + +create or replace function memory_write( + p_user_id uuid, + p_scope text, + p_key text, + p_value text, + p_tags text[] default '{}', + p_source_agent text default null, + p_trigger text default null, + p_created_at timestamptz default null, + p_org_slug text default null, + p_ttl_seconds integer default null, + p_clear_ttl boolean default false +) +returns table ( + id uuid, + created_at timestamptz, + inserted boolean, + org_routed boolean, + binding_org_slug text, + expires_at timestamptz +) +language plpgsql +set search_path = public +as $$ +declare + v_org_id uuid; + v_binding_org uuid; + v_binding_slug text; + v_expires_at timestamptz; + -- Tri-state for the expires_at UPDATE branch: + -- 'clear' → set to NULL + -- 'set' → set to v_expires_at + -- 'keep' → leave unchanged (CASE WHEN ... ELSE memories.expires_at END) + v_ttl_action text := 'keep'; +begin + if p_clear_ttl then + -- Explicit clear wins over any p_ttl_seconds value. + v_ttl_action := 'clear'; + elsif p_ttl_seconds is not null then + if p_ttl_seconds < 1 or p_ttl_seconds > 31536000 then + raise exception using errcode = 'P0001', + message = format('ttl_seconds must be between 1 and 31536000, got %s', p_ttl_seconds); + end if; + v_expires_at := now() + (p_ttl_seconds * interval '1 second'); + v_ttl_action := 'set'; + end if; + + if p_org_slug is not null then + select o.id into v_org_id from orgs o where o.slug = p_org_slug and o.deleted_at is null; + if v_org_id is null then + raise exception using errcode = 'P0001', message = format('unknown_org: %s', p_org_slug); + end if; + if not lorekit_org_can(p_user_id, v_org_id, 'write') then + raise exception using errcode = 'LK002', message = format('org_permission_denied: org=%s', p_org_slug); + end if; + else + select b.org_id, o.slug into v_binding_org, v_binding_slug + from org_scope_bindings b + join orgs o on o.id = b.org_id + where b.scope = p_scope and o.deleted_at is null; + + if v_binding_org is not null + and p_user_id is not null + and lorekit_org_can(p_user_id, v_binding_org, 'write') then + v_org_id := v_binding_org; + end if; + end if; + + if v_org_id is not null then + return query + insert into memories ( + user_id, org_id, scope, key, value, tags, source_agent, trigger, + created_at, updated_at, created_by, updated_by, expires_at + ) + values ( + null, v_org_id, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), + p_user_id, p_user_id, v_expires_at + ) + on conflict (org_id, scope, key) where org_id is not null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + updated_by = p_user_id, + expires_at = case v_ttl_action + when 'clear' then null + when 'set' then v_expires_at + else memories.expires_at + end + returning + memories.id, memories.created_at, (xmax::text = '0') as inserted, + true as org_routed, v_binding_slug as binding_org_slug, memories.expires_at; + + elsif p_user_id is null then + return query + insert into memories ( + user_id, scope, key, value, tags, source_agent, trigger, + created_at, updated_at, created_by, updated_by, expires_at + ) + values ( + null, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), + null, null, v_expires_at + ) + on conflict (scope, key) where org_id is null and user_id is null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + expires_at = case v_ttl_action + when 'clear' then null + when 'set' then v_expires_at + else memories.expires_at + end + returning + memories.id, memories.created_at, (xmax::text = '0') as inserted, + false as org_routed, v_binding_slug as binding_org_slug, memories.expires_at; + + else + return query + insert into memories ( + user_id, scope, key, value, tags, source_agent, trigger, + created_at, updated_at, created_by, updated_by, expires_at + ) + values ( + p_user_id, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), + p_user_id, p_user_id, v_expires_at + ) + on conflict (user_id, scope, key) where org_id is null and user_id is not null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + updated_by = p_user_id, + expires_at = case v_ttl_action + when 'clear' then null + when 'set' then v_expires_at + else memories.expires_at + end + returning + memories.id, memories.created_at, (xmax::text = '0') as inserted, + false as org_routed, v_binding_slug as binding_org_slug, memories.expires_at; + end if; +end; +$$; + +grant execute on function memory_write(uuid, text, text, text, text[], text, text, timestamptz, text, integer, boolean) + to anon, authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00038', 'memory_ttl_seconds') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00039_memory_scopes.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- lorekit_memory_scopes(p_user_id) — distinct visible scopes with their +-- active memory counts, for GET /memories/scopes (the REST equivalent of the +-- CLI's `listScopes()`, which returns an "unsupported" sentinel today). +-- +-- Why an RPC and not a client-side dedupe: the obvious implementation is +-- `select scope from memories` followed by a Set() in TypeScript — which is +-- what packages/web/src/lib/queries/lore.ts does. That is silently wrong past +-- PostgREST's default row cap (db-max-rows): the query returns the first N +-- rows and the caller dedupes those, so a user with more memories than the cap +-- loses whole scopes with no error and no truncation signal. Aggregating in +-- Postgres is exact regardless of row count and returns one row per scope. +-- +-- Visibility reuses lorekit_member_org_ids(p_user_id) — the SINGLE +-- tenant-visibility predicate source (00014_orgs.sql) — composed exactly as +-- the memories RLS read policies do (00015_memories_org_fk.sql): +-- user_id = OR org_id IN (select lorekit_member_org_ids()) +-- No hand-rolled membership check here; a soft-deleted org drops out of +-- lorekit_member_org_ids (00025) and therefore out of this result too. +-- +-- The count matches what GET /memories returns by default +-- (supabase/functions/memories/handlers/list.ts): non-archived AND +-- non-expired. Archived rows are reachable via `?archived=true` and are +-- deliberately excluded from this count rather than split into a second +-- column — the endpoint answers "what can I list right now". +-- +-- SECURITY DEFINER + STABLE, mirroring lorekit_memory_count (00035/00036): +-- read-only, keyed off an explicit p_user_id rather than a client-asserted +-- claim, so the api_key edge path (service-role, RLS-bypass) and the JWT path +-- get identical semantics. +-- +-- The `p_user_id is null and auth.role() = 'service_role'` branch is the +-- service-role (CI) escape hatch, matching how every other read path treats +-- that credential — GET /memories applies no tenant filter for it. It is safe +-- because `auth.role()` reads the VERIFIED JWT role claim PostgREST sets, not +-- request input: an `authenticated` caller passing p_user_id => null gets +-- nothing back (user_id = null is NULL, and lorekit_member_org_ids(null) is +-- empty), so the function fails closed. Same predicate `rls_insert` (00001) +-- uses to recognise service-role. + +-- plpgsql rather than a plain SQL body (unlike lorekit_member_org_ids) for one +-- specific reason: the caller contract includes the ORDER BY. A single-statement +-- SQL function is a candidate for inlining, and an inlined set-returning +-- function's internal ordering is not guaranteed to survive into the outer plan. +-- A plpgsql RETURN QUERY is never inlined, so `order by scope asc` is part of +-- the result rather than an accident of the planner. Same language choice as +-- lorekit_memory_count (00035/00036). +create or replace function lorekit_memory_scopes(p_user_id uuid) +returns table (scope text, count bigint) +language plpgsql +stable +security definer +set search_path = public +as $$ +-- The RETURNS TABLE columns (scope, count) are also plpgsql OUT variables, so an +-- unqualified reference to either inside the query would be ambiguous. Every +-- reference below is table-qualified, and this directive makes the column win +-- regardless, so the body can never be mis-resolved against the OUT variables. +#variable_conflict use_column +begin + return query + select m.scope, count(*) as count + from memories m + where ( + (p_user_id is null and auth.role() = 'service_role') + or m.user_id = p_user_id + or m.org_id in (select lorekit_member_org_ids(p_user_id)) + ) + and m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + group by m.scope + order by m.scope asc; +end; +$$; + +-- NOT granted to `anon`, for the same reason lorekit_member_org_ids is not +-- (00014): the function takes a bare p_user_id, so an unauthenticated caller +-- with EXECUTE could enumerate any user's scope names — and scope names are +-- themselves sensitive (they embed repo and project names) — via a PostgREST +-- RPC call. `authenticated` is safe because the edge functions resolve +-- p_user_id from the verified credential, never from request input. +grant execute on function lorekit_memory_scopes(uuid) to authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00039', 'memory_scopes') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00041_org_actor_override.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- Org actor override — let the REST edge functions serve `lk_*` API tokens. +-- +-- ── The problem ────────────────────────────────────────────────────────── +-- Every org management RPC (00022, plus the 00024/00025 re-definitions) +-- resolves the acting user as `auth.uid()`. That is exactly right for the +-- dashboard, which calls them under a real Supabase user JWT session. +-- +-- It is unusable for `supabase/functions/orgs/`'s `api_key` auth tier. A +-- `lk_rw_*` / `lk_ro_*` token is verified by the edge function itself +-- (SHA-256 lookup in `api_tokens`), after which the function talks to +-- Postgres with the SERVICE-ROLE key. There is no end-user JWT on that +-- connection, so `auth.uid()` is NULL and every `lorekit_org_can(...)` +-- check denies. The practical consequence today is that `GET/POST/PATCH/ +-- DELETE /orgs*` are registered `requires: 'jwt'` and API tokens get a 403 — +-- which is why the CLI still has to keep an MCP transport alive purely for +-- `org.create` / `org.list` / `org.rename` / `org.delete`. +-- +-- ── The fix, and why it is the precedent already in this repo ──────────── +-- `memory_write` (00019) and `memory_delete` (00020) hit the same wall in +-- Phase 2 and solved it the same way: take the actor as an explicit +-- parameter, because the edge api_key path calls from a service-role client +-- with no session JWT of its own. 00022's header comment names that +-- distinction explicitly. This migration extends the same treatment to the +-- org-management RPCs the REST surface actually exposes. +-- +-- ── THE SECURITY INVARIANT ─────────────────────────────────────────────── +-- `p_actor_user_id` is NOT a "who am I" parameter. It is a privileged +-- override that only a service-role connection can exercise: +-- +-- lorekit_org_actor(p_actor_user_id) = +-- auth.role() = 'service_role' -> coalesce(p_actor_user_id, auth.uid()) +-- otherwise -> auth.uid() +-- +-- 1. AN `authenticated` CALLER'S `p_actor_user_id` IS IGNORED, ENTIRELY. +-- If you are `authenticated`, the parameter never reaches the result: the +-- `else` branch returns `auth.uid()` and nothing else. A dashboard user +-- (or anyone who obtains an anon-key + user JWT and calls PostgREST +-- directly) can pass any UUID they like — their own, an owner's, a +-- stranger's — and still acts as themselves. There is no combination of +-- request input that changes this, because the discriminator is not +-- request input. +-- +-- 2. THE DISCRIMINATOR IS A VERIFIED JWT CLAIM, NOT A REQUEST FIELD. +-- `auth.role()` reads the `role` claim out of `request.jwt.claims`, which +-- PostgREST sets from the JWT it has already cryptographically VERIFIED +-- against the project's JWT secret. A client cannot set it: it is not a +-- header, a query parameter, or a body field, and forging it requires the +-- signing secret. This is the SAME predicate `rls_insert` (00001) has +-- relied on since the first migration to grant service-role a bypass, and +-- the same one `lorekit_memory_scopes` (00039) uses for its CI escape +-- hatch. It is not a new trust assumption — it is the existing one. +-- +-- 3. WHO ACTUALLY HOLDS SERVICE-ROLE. Only `SUPABASE_SERVICE_ROLE_KEY` +-- produces that claim, and it lives exclusively in server-side secrets +-- (the edge functions, CI). The edge function does not take the actor +-- from the request either: it resolves it from the `api_tokens` row it +-- just authenticated by token hash (`resolveRestAuth` -> +-- `auth.userId` -> `actorUserId(auth)`). So the chain is +-- "presented token -> hashed -> matched row -> that row's owner", never +-- "caller said so". A client with a `lk_*` token can only ever act as the +-- user that token belongs to. +-- +-- 4. IT FAILS CLOSED. When the actor resolves to NULL (service-role with no +-- override and no `sub` claim), every capability check becomes +-- `lorekit_org_can(null, ...)`, whose `lorekit_org_role(null, ...)` is +-- NULL, whose `if v_role is null then return false` denies everything. +-- The one RPC with no capability check — `lorekit_org_create`, the +-- owner-bootstrap path — therefore gains an EXPLICIT null-actor guard +-- below; without it a service-role call with no actor would insert an org +-- with a NULL `created_by` (and then fail confusingly on the NOT NULL +-- `org_members.user_id`). +-- +-- 5. IT IS BACKWARD COMPATIBLE. The new parameter is TRAILING and DEFAULTS +-- to NULL, and PostgREST resolves overloads by named arguments, so every +-- existing caller that omits it — all of `packages/web/src/lib/orgs.ts`, +-- `org-invites.ts`, `org-members.ts`, and the MCP `org.*` tools in +-- `supabase/functions/mcp/tools.ts` — keeps working unchanged and keeps +-- resolving its actor from `auth.uid()`. +-- +-- ── Deliberately NOT changed: accept / decline / leave ─────────────────── +-- `lorekit_org_invite_accept`, `lorekit_org_invite_decline` and +-- `lorekit_org_leave` keep their pure `auth.uid()` actor and gain no +-- override. They are not REST routes, so nothing needs it yet, and for +-- accept/decline it would be actively unsafe to bolt on: they do not merely +-- resolve an actor, they MATCH the invite's `invitee_email` / +-- `invitee_handle` against the caller's VERIFIED JWT identity claims +-- (`email`, `user_metadata.user_name`) via `lorekit_invite_addressed_to_caller` +-- (00022). A service-role connection carries no such claims, so an override +-- would have to also invent an identity to match against — i.e. re-derive +-- the anti-TOCTOU check that is the whole point of that RPC. Giving +-- service-role an identity source it can trust is a separate change with its +-- own threat model. `lorekit_org_leave` is left alone for consistency with +-- them and because `DELETE /orgs/:slug/members/:userId`'s self-removal branch +-- is the only caller; under an api_key token that branch now fails closed +-- with LK002 ("you are not a member of this org") rather than silently +-- removing the wrong row. Documented as a known gap, not an accident. +-- ═════════════════════════════════════════════════════════════════════════ + +-- ── 0. The actor resolver ──────────────────────────────────────────────── +-- SECURITY DEFINER + STABLE, mirroring the shape of every other resolver in +-- this schema (`lorekit_org_role`, `lorekit_member_org_ids`). `stable` is +-- correct: it reads only session settings, which do not change within a +-- statement. +create or replace function lorekit_org_actor(p_actor_user_id uuid) +returns uuid +language sql +stable +security definer +set search_path = public +as $$ + select case + when auth.role() = 'service_role' then coalesce(p_actor_user_id, auth.uid()) + else auth.uid() + end; +$$; + +-- No `anon` grant. The function returns only the caller's own resolved +-- identity (or NULL), so it leaks nothing, but there is no reason for an +-- unauthenticated role to call it — same posture as `lorekit_member_org_ids`. +-- +-- THE `revoke ... from public` IS LOAD-BEARING, not belt-and-braces. Postgres +-- grants EXECUTE on a newly created function to PUBLIC by default, and `anon` +-- inherits it. Naming only `authenticated, service_role` in a GRANT therefore +-- does NOT withhold it from `anon` — it just adds two roles that already had +-- access. Every `revoke` below pairs with its `grant` for that reason. +-- +-- This is not hypothetical: the assertion in `supabase/tests/migrations.test.sql` +-- ("lorekit_org_members_list must not be granted to anon") FAILED the first +-- time it ran against this migration, because 00024 created that function with +-- a grant and no revoke. It returns other members' GitHub handles and avatars. +-- +-- Known remaining gap, deliberately out of scope here: the same pattern applies +-- to functions this migration does not define — `lorekit_member_org_ids` +-- (00014), `lorekit_memory_count` (00035/00036), `lorekit_memory_scopes` +-- (00039) and `lorekit_invite_org_details` (00028) are all still reachable by +-- `anon` through PUBLIC. Closing those wants one sweep across the whole schema, +-- not a half-fix smuggled into an org-actor migration. +revoke execute on function lorekit_org_actor(uuid) from public; +grant execute on function lorekit_org_actor(uuid) to authenticated, service_role; + +comment on function lorekit_org_actor(uuid) is + 'Resolves the acting user for an org RPC. Returns auth.uid() for every caller ' + 'except a verified service_role JWT, which may name an actor explicitly. An ' + 'authenticated caller''s p_actor_user_id is always ignored.'; + +-- ── 1. Org lifecycle ───────────────────────────────────────────────────── + +-- Bodies below are copied VERBATIM from the migration that last defined each +-- function — 00022 for create/rename/invite/invite_revoke/member_remove/ +-- member_role, 00025 for delete (the soft-delete rewrite), 00024 for +-- members_list — with exactly one class of change: `auth.uid()` becomes +-- `v_actor`, resolved once at the top from `lorekit_org_actor(...)`. Starting +-- from 00022 for `delete`/`members_list` would silently revert the +-- soft-delete and the identity-resolving member list. +-- +-- Each function is DROPPED first rather than `create or replace`d: adding a +-- parameter creates a new OVERLOAD instead of replacing, and two overloads +-- that differ only by a defaulted trailing argument make a named-argument +-- PostgREST call ambiguous. Dropping also drops the grants, so every one is +-- re-issued below, identical to the original. + +drop function if exists lorekit_org_create(text, text); + +-- Creates the org AND the creator's owner membership atomically. No +-- capability check: this is the owner-bootstrap path — anyone authenticated +-- may create an org and becomes its sole owner. A duplicate slug surfaces +-- the orgs.slug unique_violation naturally. +-- +-- Unlike every sibling below, this RPC has no `lorekit_org_can` gate to fail +-- closed on a NULL actor, so it needs its OWN explicit guard (invariant 4 in +-- the header). LK002 is reused deliberately: `translateDbError` already maps +-- it to a 403, which is the honest answer to "service-role called this with +-- nobody to own the result". +create or replace function lorekit_org_create( + p_slug text, + p_name text, + p_actor_user_id uuid default null +) +returns uuid +language plpgsql +security definer +set search_path = public +as $$ +declare + v_actor uuid := lorekit_org_actor(p_actor_user_id); + v_org_id uuid := gen_random_uuid(); +begin + if v_actor is null then + raise exception using errcode = 'LK002', message = 'org_actor_unresolved: lorekit_org_create requires an actor (an authenticated session, or an explicit p_actor_user_id from service_role)'; + end if; + + insert into orgs (id, slug, name, created_by) + values (v_org_id, p_slug, p_name, v_actor); + + insert into org_members (org_id, user_id, role) + values (v_org_id, v_actor, 'owner'); + + return v_org_id; +end; +$$; + +revoke execute on function lorekit_org_create(text, text, uuid) from public; +grant execute on function lorekit_org_create(text, text, uuid) to authenticated, service_role; + +drop function if exists lorekit_org_rename(uuid, text); + +create or replace function lorekit_org_rename( + p_org_id uuid, + p_name text, + p_actor_user_id uuid default null +) +returns void +language plpgsql +security definer +set search_path = public +as $$ +declare + v_actor uuid := lorekit_org_actor(p_actor_user_id); +begin + if not lorekit_org_can(v_actor, p_org_id, 'rename_org') then + raise exception using errcode = 'LK002', message = format('org_permission_denied: org=%s capability=rename_org', p_org_id); + end if; + + update orgs set name = p_name where id = p_org_id; +end; +$$; + +revoke execute on function lorekit_org_rename(uuid, text, uuid) from public; +grant execute on function lorekit_org_rename(uuid, text, uuid) to authenticated, service_role; + +drop function if exists lorekit_org_delete(uuid); + +-- Body from 00025 (the SOFT delete), not 00022 (which really deleted the +-- row). Owner-only. Idempotent: a second call on an already-soft-deleted org +-- is a no-op via the `deleted_at is null` guard. +create or replace function lorekit_org_delete( + p_org_id uuid, + p_actor_user_id uuid default null +) +returns void +language plpgsql +security definer +set search_path = public +as $$ +declare + v_actor uuid := lorekit_org_actor(p_actor_user_id); +begin + if not lorekit_org_can(v_actor, p_org_id, 'delete_org') then + raise exception using errcode = 'LK002', message = format('org_permission_denied: org=%s capability=delete_org', p_org_id); + end if; + + update orgs set deleted_at = now() where id = p_org_id and deleted_at is null; +end; +$$; + +revoke execute on function lorekit_org_delete(uuid, uuid) from public; +grant execute on function lorekit_org_delete(uuid, uuid) to authenticated, service_role; + +-- ── 2. Invite lifecycle ────────────────────────────────────────────────── +-- `lorekit_invite_addressed_to_caller` is untouched, as are +-- `lorekit_org_invite_accept` / `_decline` (see the header). + +drop function if exists lorekit_org_invite(uuid, text, text, text); + +-- Owner/admin only. The org_invites.role CHECK (00021) independently rejects +-- an attempt to invite an 'owner' — this RPC does not re-derive that guard. +create or replace function lorekit_org_invite( + p_org_id uuid, + p_invitee_email text default null, + p_invitee_handle text default null, + p_role text default 'member', + p_actor_user_id uuid default null +) +returns uuid +language plpgsql +security definer +set search_path = public +as $$ +declare + v_actor uuid := lorekit_org_actor(p_actor_user_id); + v_invite_id uuid; +begin + if not lorekit_org_can(v_actor, p_org_id, 'invite') then + raise exception using errcode = 'LK002', message = format('org_permission_denied: org=%s capability=invite', p_org_id); + end if; + + insert into org_invites (org_id, invitee_email, invitee_handle, role, invited_by) + values ( + p_org_id, + case when p_invitee_email is not null then lower(p_invitee_email) end, + case when p_invitee_handle is not null then lower(p_invitee_handle) end, + p_role, + v_actor + ) + returning id into v_invite_id; + + return v_invite_id; +end; +$$; + +revoke execute on function lorekit_org_invite(uuid, text, text, text, uuid) from public; +grant execute on function lorekit_org_invite(uuid, text, text, text, uuid) to authenticated, service_role; + +drop function if exists lorekit_org_invite_revoke(uuid); + +create or replace function lorekit_org_invite_revoke( + p_invite_id uuid, + p_actor_user_id uuid default null +) +returns void +language plpgsql +security definer +set search_path = public +as $$ +declare + v_actor uuid := lorekit_org_actor(p_actor_user_id); + inv org_invites; +begin + select * into inv from org_invites where id = p_invite_id; + if inv.id is null then + raise exception using errcode = 'LK002', message = 'invite not found'; + end if; + + if not lorekit_org_can(v_actor, inv.org_id, 'revoke_invite') then + raise exception using errcode = 'LK002', message = format('org_permission_denied: org=%s capability=revoke_invite', inv.org_id); + end if; + + -- Only a still-pending invite can be revoked — guards the silent + -- re-revoke and the revoke-of-already-accepted cases (accepted invites are + -- undone via member removal, not revoke). + if inv.status <> 'pending' then + raise exception using errcode = 'LK002', message = 'invite is not pending'; + end if; + + update org_invites set status = 'revoked', responded_at = now() where id = p_invite_id; +end; +$$; + +revoke execute on function lorekit_org_invite_revoke(uuid, uuid) from public; +grant execute on function lorekit_org_invite_revoke(uuid, uuid) to authenticated, service_role; + +-- ── 3. Member management ───────────────────────────────────────────────── + +drop function if exists lorekit_org_member_remove(uuid, uuid); + +-- Owner/admin only, plus two invariants a static role matrix cannot express: +-- - an admin actor may only act on member/viewer targets (never owner/admin) +-- - the last remaining owner can never be removed +create or replace function lorekit_org_member_remove( + p_org_id uuid, + p_target_user_id uuid, + p_actor_user_id uuid default null +) +returns void +language plpgsql +security definer +set search_path = public +as $$ +declare + v_actor uuid := lorekit_org_actor(p_actor_user_id); + v_actor_role text; + v_target_role text; + v_owner_count int; +begin + if not lorekit_org_can(v_actor, p_org_id, 'remove_member') then + raise exception using errcode = 'LK002', message = format('org_permission_denied: org=%s capability=remove_member', p_org_id); + end if; + + v_actor_role := lorekit_org_role(v_actor, p_org_id); + v_target_role := lorekit_org_role(p_target_user_id, p_org_id); + + if v_target_role is null then + raise exception using errcode = 'LK002', message = 'target is not a member of this org'; + end if; + + if v_actor_role = 'admin' and v_target_role in ('owner', 'admin') then + raise exception using errcode = 'LK002', message = 'an admin cannot remove an owner or another admin'; + end if; + + if v_target_role = 'owner' then + select count(*) into v_owner_count from org_members where org_id = p_org_id and role = 'owner'; + if v_owner_count <= 1 then + raise exception using errcode = 'LK002', message = 'the last owner cannot be removed — transfer or delete the org first'; + end if; + end if; + + delete from org_members where org_id = p_org_id and user_id = p_target_user_id; +end; +$$; + +revoke execute on function lorekit_org_member_remove(uuid, uuid, uuid) from public; +grant execute on function lorekit_org_member_remove(uuid, uuid, uuid) to authenticated, service_role; + +drop function if exists lorekit_org_member_role(uuid, uuid, text); + +-- Owner/admin only. Cannot assign 'owner' (ownership is non-transferable in +-- v1); cannot demote the last owner; an admin actor may only act on +-- member/viewer targets, mirroring lorekit_org_member_remove's invariant. +create or replace function lorekit_org_member_role( + p_org_id uuid, + p_target_user_id uuid, + p_role text, + p_actor_user_id uuid default null +) +returns void +language plpgsql +security definer +set search_path = public +as $$ +declare + v_actor uuid := lorekit_org_actor(p_actor_user_id); + v_actor_role text; + v_target_role text; + v_owner_count int; +begin + if p_role = 'owner' then + raise exception using errcode = 'LK002', message = 'cannot assign owner via changeMemberRole — ownership is not transferable in v1'; + end if; + + if not lorekit_org_can(v_actor, p_org_id, 'change_role') then + raise exception using errcode = 'LK002', message = format('org_permission_denied: org=%s capability=change_role', p_org_id); + end if; + + v_actor_role := lorekit_org_role(v_actor, p_org_id); + v_target_role := lorekit_org_role(p_target_user_id, p_org_id); + + if v_target_role is null then + raise exception using errcode = 'LK002', message = 'target is not a member of this org'; + end if; + + if v_actor_role = 'admin' and v_target_role in ('owner', 'admin') then + raise exception using errcode = 'LK002', message = 'an admin cannot change the role of an owner or another admin'; + end if; + + if v_target_role = 'owner' then + select count(*) into v_owner_count from org_members where org_id = p_org_id and role = 'owner'; + if v_owner_count <= 1 then + raise exception using errcode = 'LK002', message = 'the last owner cannot be demoted — transfer or delete the org first'; + end if; + end if; + + update org_members set role = p_role where org_id = p_org_id and user_id = p_target_user_id; +end; +$$; + +revoke execute on function lorekit_org_member_role(uuid, uuid, text, uuid) from public; +grant execute on function lorekit_org_member_role(uuid, uuid, text, uuid) to authenticated, service_role; + +-- ── 4. Member identity listing (read) ──────────────────────────────────── + +drop function if exists lorekit_org_members_list(uuid); + +-- Body from 00024. Membership-gated PII read: a caller may resolve the +-- handle/avatar of anyone who shares an org WITH them, and only for that org. +-- A non-member (or a non-existent org — `lorekit_org_role` is NULL either +-- way) gets an EMPTY set, never an error, so existence is never leaked. A +-- service-role call with no actor therefore also gets an empty set: this +-- read fails closed exactly like the write RPCs, without a special case. +create or replace function lorekit_org_members_list( + p_org_id uuid, + p_actor_user_id uuid default null +) +returns table ( + user_id uuid, + handle text, + avatar_url text, + role text, + joined_at timestamptz +) +language plpgsql +stable +security definer +set search_path = public +as $$ +declare + v_actor uuid := lorekit_org_actor(p_actor_user_id); +begin + if lorekit_org_role(v_actor, p_org_id) is null then + return; + end if; + + return query + select + m.user_id, + coalesce( + u.raw_user_meta_data ->> 'user_name', + u.raw_user_meta_data ->> 'preferred_username' + ) as handle, + u.raw_user_meta_data ->> 'avatar_url' as avatar_url, + m.role, + m.created_at as joined_at + from org_members m + join auth.users u on u.id = m.user_id + where m.org_id = p_org_id + order by m.created_at asc; +end; +$$; + +-- No `anon` grant (unlike lorekit_member_org_ids/lorekit_org_role, which +-- return only booleans/ids): this function returns PII (handle, avatar_url) +-- for other users, so it is authenticated-only, defense in depth beyond the +-- membership gate above. +revoke execute on function lorekit_org_members_list(uuid, uuid) from public; +grant execute on function lorekit_org_members_list(uuid, uuid) to authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00041', 'org_actor_override') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00042_audit_log_github_app_action.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- GitHub App installation linking: extend the audit_log `action` CHECK with +-- `github_app.installation_linked` so `handleSetupReturn` +-- (packages/web/src/lib/github-installations.ts) can record the link. +-- +-- WHY: that call site has been auditing `github_app.installation_linked` since +-- the GitHub App shipped, but the CHECK — last set by +-- 00027_audit_log_scope_actions.sql over 23 actions — never admitted it. The +-- insert therefore failed the constraint on EVERY App link, and because +-- `recordAuditEvent` is deliberately non-throwing (a failed audit must never +-- break the operation it audits) the failure was swallowed and logged to the +-- server console only. The result was a silent, permanent hole in the audit +-- trail: no user ever saw an error, and no `github_app.installation_linked` +-- row was ever written. Widening the CHECK is the whole fix. +-- +-- The action list below is now derived from ONE source — +-- packages/schemas/src/audit.ts's `AUDIT_ACTIONS` — and +-- packages/mcp-core/src/audit-vocabulary.spec.ts parses the newest +-- `audit_log` action-CHECK migration (this file) and fails if the two sets +-- differ. A future action must be added in both places, in one commit. +-- +-- Forward-only: drop + re-add the CHECK (a constraint can't be widened in +-- place) — the same pattern as 00023_audit_log_org_actions.sql and +-- 00027_audit_log_scope_actions.sql. + +alter table audit_log drop constraint audit_log_action_check; + +alter table audit_log add constraint audit_log_action_check check (action in ( + 'api_key.create', + 'api_key.revoke', + 'webhook_secret.create', + 'webhook_secret.rotate', + 'webhook_secret.deactivate', + 'memory.create', + 'memory.update', + 'memory.archive', + 'memory.restore', + 'memory.delete', + 'limit.override', + 'org.create', + 'org.rename', + 'org.delete', + 'member.invite', + 'member.accept', + 'member.decline', + 'member.revoke', + 'member.remove', + 'member.role_change', + 'member.leave', + 'scope.bind', + 'scope.unbind', + 'github_app.installation_linked' +)); + +insert into supabase_migrations.schema_migrations (version, name) +values ('00042', 'audit_log_github_app_action') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00043_usage_stats.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- lorekit_usage_stats(p_user_id, p_since, p_until) — aggregate usage counts a +-- caller can read back about THEIR OWN activity, for GET /memories/usage. +-- +-- usage_events (00034) already captures one row per MCP tool call and one per +-- REST route (tool_name, outcome, scope_type, auth_type, duration_ms, …). What +-- was missing was any way to READ those rows back as aggregates: the table had +-- an RLS select policy but no endpoint, RPC, tool or CLI consumed it. This is +-- that read side — "how many reads/writes today, per scope, per outcome" so a +-- human or agent can judge whether LoreKit is helping. +-- +-- Aggregate in Postgres, not with a select + client-side reduce, for the same +-- reason as lorekit_memory_scopes (00039): a client-side rollup is silently +-- truncated past PostgREST's row cap, dropping whole buckets with no error. One +-- grouped row per (tool_name, outcome, scope_type) is exact at any volume, and +-- the group cardinality is bounded (tool_name × outcome × scope_type are all +-- bounded categoricals). +-- +-- Visibility is self-only, matching the usage_events RLS select policy +-- (user_id = auth.uid()): a caller sees only their own events. The +-- `p_user_id is null and auth.role() = 'service_role'` branch is the CI / +-- service-role escape hatch, exactly as lorekit_memory_scopes uses it and safe +-- for the same reason — auth.role() reads the VERIFIED JWT role claim PostgREST +-- sets, never request input, so an `authenticated` caller passing p_user_id => +-- null gets nothing back (user_id = null is NULL) and the function fails closed. +-- +-- The [p_since, p_until) window is half-open and both bounds are optional +-- (NULL = unbounded), so the caller controls the period: p_since = start-of-day +-- answers "today", p_since = now()-7d answers "this week", both NULL is +-- all-time. total_duration_ms is summed for latency-per-bucket analysis. +-- +-- SECURITY DEFINER + STABLE (read-only, keyed on an explicit p_user_id rather +-- than a client-asserted claim) and plpgsql rather than SQL for the same +-- ordering reason as lorekit_memory_scopes: a plpgsql RETURN QUERY is never +-- inlined, so the ORDER BY is part of the contract, not an accident of the +-- planner. +create or replace function lorekit_usage_stats( + p_user_id uuid, + p_since timestamptz default null, + p_until timestamptz default null +) +returns table ( + tool_name text, + outcome text, + scope_type text, + event_count bigint, + total_duration_ms bigint +) +language plpgsql +stable +security definer +set search_path = public +as $$ +-- The RETURNS TABLE columns double as OUT variables; qualify every reference and +-- make the column win on any residual ambiguity. +#variable_conflict use_column +begin + return query + select + e.tool_name, + e.outcome, + e.scope_type, + count(*) as event_count, + coalesce(sum(e.duration_ms), 0) as total_duration_ms + from usage_events e + where ( + (p_user_id is null and auth.role() = 'service_role') + or e.user_id = p_user_id + ) + and (p_since is null or e.created_at >= p_since) + and (p_until is null or e.created_at < p_until) + group by e.tool_name, e.outcome, e.scope_type + order by event_count desc, e.tool_name asc; +end; +$$; + +-- NOT granted to `anon`, matching lorekit_memory_scopes / lorekit_member_org_ids: +-- the function takes a bare p_user_id, so an unauthenticated caller with EXECUTE +-- could read any user's usage aggregates. `authenticated` is safe because the +-- edge functions resolve p_user_id from the verified credential, never request +-- input; `service_role` is the CI escape hatch. +grant execute on function lorekit_usage_stats(uuid, timestamptz, timestamptz) + to authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00043', 'usage_stats') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00044_usage_event_dimensions.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- Usage-event dimensions for G1 (real record counts) + G2 (per-PR/session +-- correlation), and the matching read-side extension of lorekit_usage_stats. +-- +-- G1 — "read 600 MEMORIES today", not "600 read CALLS": usage_events counted one +-- row per tool call but never how many records that call returned. Add +-- result_count (nullable — only reads/expiry populate it) and sum it in the +-- stats RPC as record_count, distinct from the call count. +-- +-- G2 — "5 memories during THIS PR": add a nullable, bounded correlation_id the +-- write paths fill from a client-supplied X-LoreKit-Correlation-Id header, and +-- let lorekit_usage_stats filter by it. +-- +-- Forward-only and additive: both columns are nullable so every existing row and +-- every caller that omits them is unaffected; the two RPCs are DROPped and +-- recreated with trailing DEFAULT params so PostgREST's named-argument +-- resolution keeps every current caller working. + +-- ── G1/G2 columns ─────────────────────────────────────────────────────────── +alter table usage_events add column if not exists result_count integer; +alter table usage_events add column if not exists correlation_id text; + +-- Bound the correlation id (cardinality + storage). char_length, not octet, is +-- fine for the printable-ASCII set the app validator (parseCorrelationId) +-- enforces; the DB check is the backstop, not the primary gate. +alter table usage_events drop constraint if exists usage_events_correlation_id_len; +alter table usage_events add constraint usage_events_correlation_id_len + check (correlation_id is null or char_length(correlation_id) <= 200); + +-- Index the grouped/filtered read: "my events for correlation X, newest first". +create index if not exists usage_events_user_correlation_idx + on usage_events (user_id, correlation_id, created_at desc) + where correlation_id is not null; + +-- ── writer: add p_result_count + p_correlation_id ─────────────────────────── +-- DROP first (not CREATE OR REPLACE): adding params changes the signature, and a +-- bare CREATE would leave the old overload behind and make calls ambiguous. +drop function if exists lorekit_record_usage_event(uuid, uuid, text, text, text, text, text, integer, integer); + +create or replace function lorekit_record_usage_event( + p_user_id uuid default null, + p_org_id uuid default null, + p_plan_name text default null, + p_tool_name text default null, + p_scope_type text default null, + p_auth_type text default null, + p_outcome text default null, + p_duration_ms integer default null, + p_memory_count integer default null, + p_result_count integer default null, + p_correlation_id text default null +) +returns uuid +language plpgsql +security definer +set search_path = public +as $$ +declare + v_id uuid; +begin + insert into usage_events ( + user_id, org_id, plan_name, + tool_name, scope_type, auth_type, + outcome, duration_ms, memory_count, + result_count, correlation_id + ) values ( + p_user_id, p_org_id, p_plan_name, + p_tool_name, p_scope_type, p_auth_type, + p_outcome, p_duration_ms, p_memory_count, + p_result_count, p_correlation_id + ) + returning id into v_id; + return v_id; +exception + when others then + -- Never let telemetry writes break the primary operation. + return null; +end; +$$; + +grant execute on function lorekit_record_usage_event(uuid, uuid, text, text, text, text, text, integer, integer, integer, text) + to anon, authenticated, service_role; + +-- ── reader: record_count column + correlation filter ──────────────────────── +-- DROP the 00043 3-arg version and recreate with a trailing p_correlation_id and +-- an added record_count in the returned rows. Everything else — self-only +-- visibility, the service-role escape hatch, the plpgsql-not-SQL ordering +-- rationale, the grant surface — is unchanged from 00043. +drop function if exists lorekit_usage_stats(uuid, timestamptz, timestamptz); + +create or replace function lorekit_usage_stats( + p_user_id uuid, + p_since timestamptz default null, + p_until timestamptz default null, + p_correlation_id text default null +) +returns table ( + tool_name text, + outcome text, + scope_type text, + event_count bigint, + record_count bigint, + total_duration_ms bigint +) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +begin + return query + select + e.tool_name, + e.outcome, + e.scope_type, + count(*) as event_count, + coalesce(sum(e.result_count), 0) as record_count, + coalesce(sum(e.duration_ms), 0) as total_duration_ms + from usage_events e + where ( + (p_user_id is null and auth.role() = 'service_role') + or e.user_id = p_user_id + ) + and (p_since is null or e.created_at >= p_since) + and (p_until is null or e.created_at < p_until) + and (p_correlation_id is null or e.correlation_id = p_correlation_id) + group by e.tool_name, e.outcome, e.scope_type + order by event_count desc, e.tool_name asc; +end; +$$; + +-- The `revoke ... from public` is LOAD-BEARING, not belt-and-braces — the same +-- reason 00041 pairs one with every grant. Postgres grants EXECUTE on a newly +-- created function to PUBLIC by default and `anon` inherits it, so naming only +-- `authenticated, service_role` in the GRANT does NOT withhold it from `anon`. +-- This function takes a bare p_user_id, so an `anon` caller with EXECUTE could +-- read ANY user's usage aggregates. +revoke execute on function lorekit_usage_stats(uuid, timestamptz, timestamptz, text) from public; +grant execute on function lorekit_usage_stats(uuid, timestamptz, timestamptz, text) + to authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00044', 'usage_event_dimensions') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00045_memory_expiry_events.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- G3 — make expiry OBSERVABLE ("6 lessons got expired"). +-- +-- Expiry was invisible to analytics: TTL'd rows are filtered out of reads +-- (expires_at in the past) but no event marks the moment, and +-- purge_expired_memories deleted them while returning only a count the caller +-- usually discarded. There is no other discrete "expired" moment to hook — a +-- lazy read never deletes, it just excludes — so the purge is the right and +-- only pattern-fitting seam: capture the count it already computes as a usage +-- event instead of throwing it away. +-- +-- purge_expired_memories now records ONE usage_events row per run that deleted +-- anything, tool_name 'memory.expired', result_count = rows expired. GET +-- /memories/usage then surfaces `expired` = sum(record_count) over that bucket. +-- The signature and return value are unchanged, so every existing caller (the +-- memory.purge_expired MCP tool, POST /memories/purge-expired) is unaffected — +-- and both of those still record their own 'memory.purge_expired' CALL event, so +-- the call and its effect stay distinct. +-- +-- Forward-only: CREATE OR REPLACE, same signature as 00030. + +create or replace function purge_expired_memories( + p_user_id uuid +) +returns integer +language plpgsql +security definer +set search_path = public +as $$ +declare + v_count integer; +begin + delete from memories + where user_id = p_user_id + and expires_at is not null + and expires_at < now() + and archived_at is null -- archived rows stay with purge_archived_memories + ; + get diagnostics v_count = row_count; + + -- Emit an expiry event so the count is tallyable per period/scope. Only when + -- something actually expired — an empty purge is not an expiry event. + -- lorekit_record_usage_event swallows its own errors, so this can never break + -- the purge; auth_type 'service' marks it as a system-generated effect (it is + -- not a caller read/write). Depends on 00044's added p_result_count param. + if v_count > 0 then + perform lorekit_record_usage_event( + p_user_id => p_user_id, + p_tool_name => 'memory.expired', + p_scope_type => null, + p_auth_type => 'service', + p_outcome => 'ok', + p_result_count => v_count + ); + end if; + + return v_count; +end; +$$; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00045', 'memory_expiry_events') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00046_memory_rpc_actor_guard.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- Memory RPC actor guard — close a caller-supplied-user-id access-control gap. +-- +-- ── The problem ────────────────────────────────────────────────────────── +-- Four SECURITY DEFINER RPCs act on a caller-supplied `p_user_id` with no +-- comparison against the authenticated caller: +-- +-- archive_memory(p_user_id, scope, key) — 00003 +-- restore_memory(p_user_id, scope, key) — 00003 +-- purge_archived_memories(p_user_id, retention) — 00003 (hard DELETE) +-- purge_expired_memories(p_user_id) — 00030 (hard DELETE) +-- +-- SECURITY DEFINER runs the body as the table owner, and no migration sets +-- FORCE ROW LEVEL SECURITY on `memories`, so the owner BYPASSES the `rls_read` +-- update/select policies entirely. The row that is touched is chosen solely by +-- the `where user_id = p_user_id` predicate — a value the caller supplies. +-- The 00003 comment ("Uses SECURITY DEFINER so the RLS update policy +-- (user_id = auth.uid()) still applies") has the mechanism backwards: DEFINER +-- is precisely the construct that stops that policy applying. +-- +-- Worse, three of the four keep PostgreSQL's default `EXECUTE` grant to +-- PUBLIC (00003 never revoked it; only the org RPCs got that treatment in +-- 00041). purge_expired_memories added a grant to `authenticated` but never +-- revoked PUBLIC either, so its "intentionally excludes anon" comment did not +-- hold. Supabase exposes public-schema functions over PostgREST, so as +-- deployed any caller could: +-- +-- POST /rest/v1/rpc/purge_archived_memories +-- { "p_user_id": "", "p_retention_days": 0 } +-- +-- and hard-delete another user's archived rows — a textbook broken-access- +-- control (OWASP A01) IDOR. The application layer is careful everywhere +-- (the dashboard resolves user.id from the session; the MCP purge tool +-- refuses without a resolved user id) so the product path is sound; the gap +-- is at the grant boundary, which the app layer cannot close. +-- +-- ── The fix, and why it is the precedent already in this repo ──────────── +-- 00041_org_actor_override.sql solved the identical problem for the org +-- RPCs: resolve the actor so that a caller-supplied id is honoured ONLY on a +-- verified service-role connection and is otherwise ignored in favour of +-- `auth.uid()`. This migration applies the same rule to the four memory RPCs. +-- +-- effective actor = +-- auth.role() = 'service_role' -> coalesce(p_user_id, auth.uid()) +-- otherwise -> auth.uid() +-- +-- ── THE SECURITY INVARIANT (identical to lorekit_org_actor) ────────────── +-- 1. AN `authenticated` CALLER'S `p_user_id` IS IGNORED, ENTIRELY. The `else` +-- branch returns `auth.uid()` and nothing else, so a dashboard user (or +-- anyone with an anon-key + user JWT hitting PostgREST directly) can name +-- any UUID and still only ever acts on their OWN rows. No request input +-- changes this, because the discriminator is not request input. +-- 2. THE DISCRIMINATOR IS A VERIFIED JWT CLAIM, NOT A REQUEST FIELD. +-- `auth.role()` reads the `role` claim PostgREST sets from the JWT it has +-- already cryptographically verified. Forging it requires the signing +-- secret. This is the same predicate `rls_insert` (00001) and +-- `lorekit_org_actor` (00041) already rely on. +-- 3. WHO HOLDS SERVICE-ROLE: only `SUPABASE_SERVICE_ROLE_KEY`, held in +-- server-side secrets (edge functions, CI). The edge/MCP path does not take +-- the actor from the request body — it resolves it from the `api_tokens` +-- row it authenticated by hash — so the chain is "presented token -> +-- hashed -> matched row -> that row's owner", never "caller said so". +-- 4. IT FAILS CLOSED. When the actor resolves to NULL (service-role, no +-- override, no `sub`), `where user_id = NULL` matches no row, so the +-- UPDATE/DELETE touches nothing. +-- 5. IT IS BEHAVIOUR-PRESERVING FOR LEGITIMATE CALLERS. The signatures are +-- unchanged (plain `create or replace`, no new overload). The dashboard +-- already passes user.id == auth.uid(), so resolving to auth.uid() is a +-- no-op for it; the edge/MCP service-role path already passes the resolved +-- token owner, which the service-role branch honours via coalesce. +-- ═════════════════════════════════════════════════════════════════════════ + +-- 1. archive_memory — soft-archive the CALLER's own row. +create or replace function archive_memory( + p_user_id uuid, + p_scope text, + p_key text +) +returns uuid +language plpgsql +security definer +set search_path = public +as $$ +declare + v_id uuid; + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; +begin + update memories + set archived_at = now() + where user_id = v_actor + and scope = p_scope + and key = p_key + and archived_at is null + returning id into v_id; + + return v_id; -- null if row not found or already archived +end; +$$; + +-- 2. restore_memory — clear archived_at on the CALLER's own row. +create or replace function restore_memory( + p_user_id uuid, + p_scope text, + p_key text +) +returns uuid +language plpgsql +security definer +set search_path = public +as $$ +declare + v_id uuid; + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; +begin + update memories + set archived_at = null + where user_id = v_actor + and scope = p_scope + and key = p_key + and archived_at is not null + returning id into v_id; + + return v_id; +end; +$$; + +-- 3. purge_archived_memories — hard-delete the CALLER's own archived rows. +create or replace function purge_archived_memories( + p_user_id uuid, + p_retention_days integer default 30 +) +returns integer +language plpgsql +security definer +set search_path = public +as $$ +declare + v_count integer; + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; +begin + delete from memories + where user_id = v_actor + and archived_at is not null + and archived_at < now() - (p_retention_days * interval '1 day') + ; + get diagnostics v_count = row_count; + return v_count; +end; +$$; + +-- 4. purge_expired_memories — hard-delete the CALLER's own expired rows. +create or replace function purge_expired_memories( + p_user_id uuid +) +returns integer +language plpgsql +security definer +set search_path = public +as $$ +declare + v_count integer; + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; +begin + delete from memories + where user_id = v_actor + and expires_at is not null + and expires_at < now() + and archived_at is null -- archived rows stay with purge_archived_memories + ; + get diagnostics v_count = row_count; + + -- Preserve 00045's expiry-event recording (this migration is applied AFTER + -- 00045, so its create-or-replace would otherwise silently drop it). Record + -- for the RESOLVED actor, and only when something actually expired. + -- lorekit_record_usage_event swallows its own errors, so it can never break + -- the purge. + if v_count > 0 then + perform lorekit_record_usage_event( + p_user_id => v_actor, + p_tool_name => 'memory.expired', + p_scope_type => null, + p_auth_type => 'service', + p_outcome => 'ok', + p_result_count => v_count + ); + end if; + + return v_count; +end; +$$; + +-- ── Grants ─────────────────────────────────────────────────────────────── +-- `create or replace` preserves the pre-existing grants, so revoke the +-- default PUBLIC EXECUTE explicitly (00003 never did) and re-issue least- +-- privilege grants, matching the posture 00041 established for the org RPCs. +-- Now that the actor guard makes a caller-supplied id inert for anyone but a +-- service-role connection, these are safe to keep on `authenticated`. +revoke execute on function archive_memory(uuid, text, text) from public; +revoke execute on function restore_memory(uuid, text, text) from public; +revoke execute on function purge_archived_memories(uuid, integer) from public; +revoke execute on function purge_expired_memories(uuid) from public; + +grant execute on function archive_memory(uuid, text, text) to authenticated, service_role; +grant execute on function restore_memory(uuid, text, text) to authenticated, service_role; +grant execute on function purge_archived_memories(uuid, integer) to authenticated, service_role; +grant execute on function purge_expired_memories(uuid) to authenticated, service_role; + +-- Correct the record on all four functions. 00003's file comment above +-- archive_memory claimed "Uses SECURITY DEFINER so the RLS update policy +-- (user_id = auth.uid()) still applies" — backwards: DEFINER runs as the table +-- owner and, absent FORCE ROW LEVEL SECURITY, BYPASSES that policy. What +-- actually scopes each of these is the resolved actor below, not RLS. A +-- `comment on function` is the durable place to say so (a comment on a +-- superseded body in 00003 can't be edited forward-only). +comment on function archive_memory(uuid, text, text) is + 'Soft-archives the effective caller''s own row. SECURITY DEFINER BYPASSES the + memories RLS policies (no FORCE ROW LEVEL SECURITY), so the row is scoped by + the resolved actor, NOT by RLS — correcting 00003''s backwards note. The + actor is resolved by the same service-role-gated rule as lorekit_org_actor + (00041): a caller-supplied p_user_id is honoured only on a verified + service-role connection, otherwise auth.uid() wins. Fails closed on NULL.'; +comment on function restore_memory(uuid, text, text) is + 'Restores the effective caller''s own archived row. Same actor rule and same + RLS-bypass caveat as archive_memory (see 00046 header; corrects 00003).'; +comment on function purge_archived_memories(uuid, integer) is + 'Hard-deletes the effective caller''s archived rows past the retention window. + The effective user id is resolved by the same service-role-gated rule as + lorekit_org_actor (00041): a caller-supplied p_user_id is honoured only on a + verified service-role connection and is otherwise ignored in favour of + auth.uid(). SECURITY DEFINER bypasses RLS; the actor scopes the delete. + Fails closed on a NULL actor.'; +comment on function purge_expired_memories(uuid) is + 'Hard-deletes the effective caller''s expired (non-archived) rows. Same + service-role-gated actor rule and RLS-bypass caveat as + purge_archived_memories (see 00046 header).'; + +-- ═════════════════════════════════════════════════════════════════════════ +-- 5. memory_delete — the omitted fifth RPC of this family. +-- +-- 00020's memory_delete has the SAME caller-supplied-p_user_id flaw as the +-- four above, and it is the most dangerous: SECURITY DEFINER, granted to +-- `anon`, and it hard-DELETEs (p_force=true) or archives on a bare +-- `where user_id = p_user_id` with no auth.uid()/service-role gate. Over +-- PostgREST that is an anon-reachable, cross-tenant DESTRUCTIVE IDOR: +-- POST /rest/v1/rpc/memory_delete +-- { "p_user_id":"", "p_scope":"global", "p_key":"k", "p_force":true } +-- The org branch is exploitable too — it gates on +-- lorekit_org_can(p_user_id, ...) with the CALLER's p_user_id, so naming any +-- privileged member deletes that org's lore as a non-member. It was simply +-- omitted from this migration's original list; recreate it with the identical +-- actor guard, applied to BOTH the personal user_id filter AND the org +-- capability check, so the org path authorizes the REAL caller, never a named +-- member. Behaviour-preserving: the edge api_key path (service-role) still +-- names the token owner via coalesce; a JWT caller resolves to its own +-- auth.uid() (which the edge already passes as p_user_id). +create or replace function memory_delete( + p_user_id uuid, + p_org_slug text default null, + p_scope text default null, + p_key text default null, + p_force boolean default false +) +returns table (deleted boolean, archived boolean) +language plpgsql +security definer +set search_path = public +as $$ +declare + v_org_id uuid; + v_count integer; + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; +begin + if p_org_slug is not null then + select o.id into v_org_id from orgs o where o.slug = p_org_slug; + if v_org_id is null then + raise exception using + errcode = 'P0001', + message = format('unknown_org: %s', p_org_slug); + end if; + + if p_force then + if not lorekit_org_can(v_actor, v_org_id, 'hard_delete') then + raise exception using + errcode = 'LK002', + message = format('org_permission_denied: org=%s capability=hard_delete', p_org_slug); + end if; + + delete from memories + where org_id = v_org_id and scope = p_scope and key = p_key; + get diagnostics v_count = row_count; + + return query select (v_count > 0), false; + else + if not lorekit_org_can(v_actor, v_org_id, 'archive') then + raise exception using + errcode = 'LK002', + message = format('org_permission_denied: org=%s capability=archive', p_org_slug); + end if; + + update memories + set archived_at = now() + where org_id = v_org_id and scope = p_scope and key = p_key and archived_at is null; + get diagnostics v_count = row_count; + + return query select false, (v_count > 0); + end if; + else + -- Personal delete: scoped to the RESOLVED actor, never a named p_user_id. + if p_force then + delete from memories + where user_id = v_actor and scope = p_scope and key = p_key; + get diagnostics v_count = row_count; + + return query select (v_count > 0), false; + else + update memories + set archived_at = now() + where user_id = v_actor and scope = p_scope and key = p_key and archived_at is null; + get diagnostics v_count = row_count; + + return query select false, (v_count > 0); + end if; + end if; +end; +$$; + +-- 00020:92 granted memory_delete to `anon` EXPLICITLY (unlike the 00003/00030 +-- RPCs, which only had the default PUBLIC grant). `revoke ... from public` does +-- NOT remove a per-role grant, so anon must be revoked by name too — otherwise +-- anon keeps EXECUTE (the actor guard makes the call inert, but least-privilege +-- means removing the grant, not just relying on the guard). +revoke execute on function memory_delete(uuid, text, text, text, boolean) from public, anon; +grant execute on function memory_delete(uuid, text, text, text, boolean) to authenticated, service_role; + +comment on function memory_delete(uuid, text, text, text, boolean) is + 'Archives (p_force=false) or hard-deletes (p_force=true) the effective + caller''s own memory, or an org''s memory when p_org_slug is given and the + caller has the archive/hard_delete capability. The effective actor is + resolved by the same service-role-gated rule as the rest of this family + (00046) and lorekit_org_actor (00041): a caller-supplied p_user_id is + honoured only on a verified service-role connection, otherwise auth.uid() + wins — scoping BOTH the personal user_id filter and the org capability + check to the real caller. SECURITY DEFINER bypasses RLS; the actor is the + gate. Fails closed on a NULL actor.'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00046', 'memory_rpc_actor_guard') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00047_readfn_grant_hardening.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- Read-function grant hardening — the "clearly-safe" half of the deferred +-- security-review sweep (docs/tasks/security-review-deferred-findings.md). +-- +-- Same root cause as 00046: PostgreSQL grants EXECUTE to PUBLIC by default, and +-- only 00041/00046 ever revoked it — so a set of SECURITY DEFINER functions +-- that take a bare `p_user_id` (or are service-role-only) stayed anon-reachable +-- over PostgREST, leaking other users' data or letting anon call service-only +-- procedures. This migration closes the CLEARLY-SAFE subset: +-- +-- * lorekit_memory_scopes / lorekit_memory_count — edge-called reads that +-- trust p_user_id → get the 00046/00041 actor guard (honour p_user_id only +-- under a verified service-role connection, else auth.uid()) AND lose the +-- PUBLIC/anon grant, closing both the anon and the authenticated cross-user +-- read. +-- * lorekit_find_user_by_github_id + the three installation RPCs + +-- lorekit_purge_old_usage_events + lorekit_purge_all_expired_memories — +-- service-role / webhook / cron only → simply lose PUBLIC EXECUTE. +-- +-- DELIBERATELY OUT OF SCOPE (left for a separate, carefully-tested pass): +-- functions embedded in RLS policies or the insert/rate-limit hot path +-- (lorekit_member_org_ids, lorekit_org_role/_can, lorekit_check_rate_limit, +-- lorekit_purge_rate_limit_counters, lorekit_get_limit/_org_limit/default_limit, +-- lorekit_record_usage_event) — a wrong grant/actor change there is a +-- production-availability incident. See the findings doc, "[rls/hot-path]". +-- ═════════════════════════════════════════════════════════════════════════ + +-- ── 1. lorekit_memory_scopes — actor guard (preserves the CI escape hatch) ── +create or replace function lorekit_memory_scopes(p_user_id uuid) +returns table (scope text, count bigint) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + -- Honour a caller-supplied p_user_id only on a verified service-role + -- connection (the edge resolves the token owner and passes it); an + -- authenticated caller is pinned to its own auth.uid(), closing the + -- cross-user scope-name enumeration. A NULL actor under service-role is the + -- CI escape hatch, preserved by the first visibility branch below. + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; +begin + return query + select m.scope, count(*) as count + from memories m + where ( + (v_actor is null and auth.role() = 'service_role') + or m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) + ) + and m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + group by m.scope + order by m.scope asc; +end; +$$; + +revoke execute on function lorekit_memory_scopes(uuid) from public, anon; +grant execute on function lorekit_memory_scopes(uuid) to authenticated, service_role; + +comment on function lorekit_memory_scopes(uuid) is + 'Per-scope active counts visible to the EFFECTIVE caller. Actor resolved by + the 00046/00041 service-role-gated rule: a caller-supplied p_user_id is + honoured only on a verified service-role connection, otherwise auth.uid() + wins — so an authenticated caller can never enumerate another user''s scope + names. p_user_id IS NULL under service-role is the CI escape hatch.'; + +-- ── 2. lorekit_memory_count — actor guard ─────────────────────────────────── +create or replace function lorekit_memory_count(p_user_id uuid) +returns jsonb +language plpgsql +security definer +stable +set search_path = public +as $$ +declare + v_personal_count integer; + v_org_count integer; + v_limit integer; + v_plan_name text; + v_org_ids uuid[]; + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; +begin + -- Resolve org memberships once (for the resolved actor). + select array_agg(org_id) + into v_org_ids + from org_members om + join orgs o on o.id = om.org_id + where om.user_id = v_actor + and o.deleted_at is null; + + -- Personal active memories. + select count(*) + into v_personal_count + from memories + where user_id = v_actor + and archived_at is null; + + -- Org-owned active memories across all orgs the actor belongs to. + if v_org_ids is not null and cardinality(v_org_ids) > 0 then + select count(*) + into v_org_count + from memories + where org_id = any(v_org_ids) + and archived_at is null; + else + v_org_count := 0; + end if; + + -- Resolve effective personal limit and plan name. + v_limit := lorekit_get_limit(v_actor, 'max_memories'); + + select coalesce(up.plan_name, 'free') + into v_plan_name + from user_plans up + where up.user_id = v_actor; + + if v_plan_name is null then + v_plan_name := 'free'; + end if; + + return jsonb_build_object( + 'count', coalesce(v_personal_count, 0) + coalesce(v_org_count, 0), + 'personal_count', coalesce(v_personal_count, 0), + 'org_count', v_org_count, + 'limit', v_limit, + 'plan', v_plan_name + ); +end; +$$; + +revoke execute on function lorekit_memory_count(uuid) from public, anon; +grant execute on function lorekit_memory_count(uuid) to authenticated, service_role; + +comment on function lorekit_memory_count(uuid) is + 'Personal + org memory counts, limit and plan for the EFFECTIVE caller. Same + service-role-gated actor rule as lorekit_memory_scopes (00047) — an + authenticated caller cannot read another user''s counts or plan tier.'; + +-- ── 3. Service-role / webhook / cron-only functions — drop PUBLIC EXECUTE ──── +-- These take no p_user_id (or are procedures the app never calls as a normal +-- user), so no actor guard applies; they should simply not be anon-reachable. +-- They were granted `to service_role` only, but the default PUBLIC grant was +-- never revoked. `find_user_by_github_id` also loses authenticated (it maps a +-- GitHub id to an internal user UUID — a deanonymisation oracle). +revoke execute on function lorekit_find_user_by_github_id(text) from public, anon, authenticated; +revoke execute on function lorekit_installation_upsert(bigint, bigint, text, text, uuid, text, text[]) from public, anon, authenticated; +revoke execute on function lorekit_installation_remove_repos(bigint, text[]) from public, anon, authenticated; +revoke execute on function lorekit_installation_remove(bigint) from public, anon, authenticated; +revoke execute on function lorekit_purge_old_usage_events(interval) from public, anon, authenticated; +revoke execute on function lorekit_purge_all_expired_memories() from public, anon, authenticated; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00047', 'readfn_grant_hardening') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00048_memory_origin.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- Memory origin (provenance): where a lesson was RECORDED FROM. +-- +-- A memory's `scope` says WHERE the lesson applies. It does not say where the +-- lesson came from: a `global` lesson can be learned while reviewing a pull +-- request, and a `repo::`-scoped lesson says nothing about which branch, +-- commit, or PR taught it. The dashboard could therefore only ever render a +-- "Repo" row by string-splitting the scope — and never a branch, commit, or +-- pull-request link. +-- +-- Four nullable, independently-optional columns close that gap: +-- origin_repo owner/name of the repository the write happened in +-- origin_branch the git branch the write happened on +-- origin_commit the commit SHA that was checked out +-- origin_pr the pull request number the work belonged to +-- +-- Design notes: +-- * Separate typed columns, not a JSONB blob — each one is independently +-- queryable and CHECK-constrainable, and the shapes are known and stable. +-- * `origin_branch` is NOT lowercased (unlike a `branch::` scope, which is +-- canonically lowercased and therefore produces `/tree/` links that 404 on +-- a mixed-case branch). Stored verbatim so its GitHub link resolves. +-- * On UPSERT each field uses `coalesce(excluded.x, memories.x)`: the newest +-- write wins when it knows the value, and a write that does NOT know it +-- (an agent with no git context) never erases what a previous write did +-- know. "Last known origin", never a regression to unknown. +-- * The application layer validates and normalises all four before they get +-- here (packages/mcp-core/src/origin.ts, mirrored self-contained into +-- supabase/functions/_shared/provenance/origin.ts). The CHECK constraints below are +-- the backstop, deliberately looser than the app-layer regexes so a future +-- provider with a slightly different ref shape does not require a +-- migration to accept. +-- * No index: no read path filters on these columns today (they are rendered +-- from a row already fetched by `(scope, key)` or by the keyset list). Add +-- one with the feature that first filters on origin, not speculatively. +-- +-- The RPC widening follows the established pattern (00009, 00030, 00031, +-- 00038): DROP the old overload FIRST — `create or replace` alone would leave +-- a stale N-arg overload and PostgREST, which resolves RPCs by ARGUMENT NAME, +-- would see an ambiguous call — then recreate with the new params appended +-- LAST and DEFAULTed, then re-issue the GRANTs against the new signature. +-- Forward-only. +-- ═════════════════════════════════════════════════════════════════════════ + +alter table memories add column if not exists origin_repo text; +alter table memories add column if not exists origin_branch text; +alter table memories add column if not exists origin_commit text; +alter table memories add column if not exists origin_pr integer; + +alter table memories drop constraint if exists memories_origin_repo_check; +-- Shape-aware, not length-only. `../evil` is a well-formed "owner/name" to a +-- naive check, and the dashboard turns it into https://github.com/../evil, +-- which the browser resolves to https://github.com/evil — a link to a +-- repository the memory never came from. The regex admits exactly one +-- `owner/name` pair of [A-Za-z0-9._-] segments, and the second clause rejects +-- a segment made only of dots, so no relative path can be stored even by a +-- direct INSERT that bypasses the app-layer validator. +alter table memories add constraint memories_origin_repo_check + check ( + origin_repo is null + or ( + length(origin_repo) between 3 and 140 + and origin_repo ~ '^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$' + and origin_repo !~ '(^|/)\.+(/|$)' + ) + ); + +alter table memories drop constraint if exists memories_origin_branch_check; +-- Branch names are near-freeform (git-check-ref-format is a deny list, and the +-- app layer applies it in full), so this backstop only guards the two +-- properties the render path depends on: a bounded length, and no relative +-- path segment that would retarget the /tree/ link. +alter table memories add constraint memories_origin_branch_check + check ( + origin_branch is null + or (length(origin_branch) between 1 and 255 and origin_branch !~ '(^|/)\.\.(/|$)') + ); + +alter table memories drop constraint if exists memories_origin_commit_check; +alter table memories add constraint memories_origin_commit_check + check (origin_commit is null or origin_commit ~ '^[0-9a-f]{7,40}$'); + +alter table memories drop constraint if exists memories_origin_pr_check; +alter table memories add constraint memories_origin_pr_check + check (origin_pr is null or origin_pr >= 1); + +comment on column memories.origin_repo is 'Provenance: owner/name of the repository this memory was last recorded from.'; +comment on column memories.origin_branch is 'Provenance: git branch this memory was last recorded from (verbatim case).'; +comment on column memories.origin_commit is 'Provenance: commit SHA checked out when this memory was last recorded.'; +comment on column memories.origin_pr is 'Provenance: pull request number this memory was last recorded from.'; + +drop function if exists memory_write(uuid, text, text, text, text[], text, text, timestamptz, text, integer, boolean); + +create or replace function memory_write( + p_user_id uuid, + p_scope text, + p_key text, + p_value text, + p_tags text[] default '{}', + p_source_agent text default null, + p_trigger text default null, + p_created_at timestamptz default null, + p_org_slug text default null, + p_ttl_seconds integer default null, + p_clear_ttl boolean default false, + p_origin_repo text default null, + p_origin_branch text default null, + p_origin_commit text default null, + p_origin_pr integer default null +) +returns table ( + id uuid, + created_at timestamptz, + inserted boolean, + org_routed boolean, + binding_org_slug text, + expires_at timestamptz +) +language plpgsql +set search_path = public +as $$ +declare + v_org_id uuid; + v_binding_org uuid; + v_binding_slug text; + v_expires_at timestamptz; + -- Tri-state for the expires_at UPDATE branch: + -- 'clear' → set to NULL + -- 'set' → set to v_expires_at + -- 'keep' → leave unchanged (CASE WHEN ... ELSE memories.expires_at END) + v_ttl_action text := 'keep'; +begin + if p_clear_ttl then + -- Explicit clear wins over any p_ttl_seconds value. + v_ttl_action := 'clear'; + elsif p_ttl_seconds is not null then + if p_ttl_seconds < 1 or p_ttl_seconds > 31536000 then + raise exception using errcode = 'P0001', + message = format('ttl_seconds must be between 1 and 31536000, got %s', p_ttl_seconds); + end if; + v_expires_at := now() + (p_ttl_seconds * interval '1 second'); + v_ttl_action := 'set'; + end if; + + if p_org_slug is not null then + select o.id into v_org_id from orgs o where o.slug = p_org_slug and o.deleted_at is null; + if v_org_id is null then + raise exception using errcode = 'P0001', message = format('unknown_org: %s', p_org_slug); + end if; + if not lorekit_org_can(p_user_id, v_org_id, 'write') then + raise exception using errcode = 'LK002', message = format('org_permission_denied: org=%s', p_org_slug); + end if; + else + select b.org_id, o.slug into v_binding_org, v_binding_slug + from org_scope_bindings b + join orgs o on o.id = b.org_id + where b.scope = p_scope and o.deleted_at is null; + + if v_binding_org is not null + and p_user_id is not null + and lorekit_org_can(p_user_id, v_binding_org, 'write') then + v_org_id := v_binding_org; + end if; + end if; + + if v_org_id is not null then + return query + insert into memories ( + user_id, org_id, scope, key, value, tags, source_agent, trigger, + created_at, updated_at, created_by, updated_by, expires_at, + origin_repo, origin_branch, origin_commit, origin_pr + ) + values ( + null, v_org_id, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), + p_user_id, p_user_id, v_expires_at, + p_origin_repo, p_origin_branch, p_origin_commit, p_origin_pr + ) + on conflict (org_id, scope, key) where org_id is not null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + updated_by = p_user_id, + -- Last KNOWN origin wins: a write that does not know a field leaves the + -- previously recorded one intact rather than erasing it. + origin_repo = coalesce(excluded.origin_repo, memories.origin_repo), + origin_branch = coalesce(excluded.origin_branch, memories.origin_branch), + origin_commit = coalesce(excluded.origin_commit, memories.origin_commit), + origin_pr = coalesce(excluded.origin_pr, memories.origin_pr), + expires_at = case v_ttl_action + when 'clear' then null + when 'set' then v_expires_at + else memories.expires_at + end + returning + memories.id, memories.created_at, (xmax::text = '0') as inserted, + true as org_routed, v_binding_slug as binding_org_slug, memories.expires_at; + + elsif p_user_id is null then + return query + insert into memories ( + user_id, scope, key, value, tags, source_agent, trigger, + created_at, updated_at, created_by, updated_by, expires_at, + origin_repo, origin_branch, origin_commit, origin_pr + ) + values ( + null, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), + null, null, v_expires_at, + p_origin_repo, p_origin_branch, p_origin_commit, p_origin_pr + ) + on conflict (scope, key) where org_id is null and user_id is null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + origin_repo = coalesce(excluded.origin_repo, memories.origin_repo), + origin_branch = coalesce(excluded.origin_branch, memories.origin_branch), + origin_commit = coalesce(excluded.origin_commit, memories.origin_commit), + origin_pr = coalesce(excluded.origin_pr, memories.origin_pr), + expires_at = case v_ttl_action + when 'clear' then null + when 'set' then v_expires_at + else memories.expires_at + end + returning + memories.id, memories.created_at, (xmax::text = '0') as inserted, + false as org_routed, v_binding_slug as binding_org_slug, memories.expires_at; + + else + return query + insert into memories ( + user_id, scope, key, value, tags, source_agent, trigger, + created_at, updated_at, created_by, updated_by, expires_at, + origin_repo, origin_branch, origin_commit, origin_pr + ) + values ( + p_user_id, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), + p_user_id, p_user_id, v_expires_at, + p_origin_repo, p_origin_branch, p_origin_commit, p_origin_pr + ) + on conflict (user_id, scope, key) where org_id is null and user_id is not null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + updated_by = p_user_id, + origin_repo = coalesce(excluded.origin_repo, memories.origin_repo), + origin_branch = coalesce(excluded.origin_branch, memories.origin_branch), + origin_commit = coalesce(excluded.origin_commit, memories.origin_commit), + origin_pr = coalesce(excluded.origin_pr, memories.origin_pr), + expires_at = case v_ttl_action + when 'clear' then null + when 'set' then v_expires_at + else memories.expires_at + end + returning + memories.id, memories.created_at, (xmax::text = '0') as inserted, + false as org_routed, v_binding_slug as binding_org_slug, memories.expires_at; + end if; +end; +$$; + +grant execute on function memory_write(uuid, text, text, text, text[], text, text, timestamptz, text, integer, boolean, text, text, text, integer) + to anon, authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00048', 'memory_origin') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00049_memory_scopes_last_activity.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- lorekit_memory_scopes — add `last_activity` to the per-scope rollup. +-- +-- WHY: the dashboard's Scope Health cards show, per scope, a total and "last +-- activity". Both were derived client-side from a `select scope, created_at +-- ... limit 1000` in packages/web/src/lib/queries/dashboard.ts — the exact +-- shape 00039 exists to replace. That query is silently wrong twice over: past +-- the row cap it drops whole scopes (00039's rationale) and, because the cap +-- is applied to `created_at desc` rows, it also understates the total for +-- every scope that survives. +-- +-- Returning max(created_at) alongside the count makes the endpoint answer the +-- whole question exactly, at any volume, in one round trip. It is the same +-- grouped scan — `max()` over a column already being read — so there is no new +-- cost. +-- +-- Forward-only: the return type changes, so the function must be DROPped +-- first (`create or replace` cannot add a column to a RETURNS TABLE). Callers +-- that select `scope`/`count` are unaffected; the new column is additive and +-- the CLI's RemoteStore.listScopes() ignores unknown fields. +-- +-- Everything else is carried over verbatim from 00047: the service-role-gated +-- actor guard, the visibility predicate composed from lorekit_member_org_ids, +-- the active-rows-only definition (non-archived AND non-expired), the +-- `order by scope asc` contract, plpgsql-not-SQL so that ordering survives, and +-- the revoked PUBLIC/anon grant. +-- ═════════════════════════════════════════════════════════════════════════ + +drop function if exists lorekit_memory_scopes(uuid); + +create function lorekit_memory_scopes(p_user_id uuid) +returns table (scope text, count bigint, last_activity timestamptz) +language plpgsql +stable +security definer +set search_path = public +as $$ +-- The RETURNS TABLE columns are also plpgsql OUT variables, so an unqualified +-- reference to any of them inside the query would be ambiguous. Every reference +-- below is table-qualified, and this directive makes the column win regardless. +#variable_conflict use_column +declare + -- Honour a caller-supplied p_user_id only on a verified service-role + -- connection (the edge resolves the token owner and passes it); an + -- authenticated caller is pinned to its own auth.uid(), closing the + -- cross-user scope-name enumeration. A NULL actor under service-role is the + -- CI escape hatch, preserved by the first visibility branch below. + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; +begin + return query + select m.scope, count(*) as count, max(m.created_at) as last_activity + from memories m + where ( + (v_actor is null and auth.role() = 'service_role') + or m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) + ) + and m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + group by m.scope + order by m.scope asc; +end; +$$; + +revoke execute on function lorekit_memory_scopes(uuid) from public, anon; +grant execute on function lorekit_memory_scopes(uuid) to authenticated, service_role; + +comment on function lorekit_memory_scopes(uuid) is + 'Per-scope active counts and last activity visible to the EFFECTIVE caller. + Actor resolved by the 00046/00041 service-role-gated rule: a caller-supplied + p_user_id is honoured only on a verified service-role connection, otherwise + auth.uid() wins — so an authenticated caller can never enumerate another + user''s scope names. p_user_id IS NULL under service-role is the CI escape + hatch. last_activity is max(created_at) over the same counted rows.'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00049', 'memory_scopes_last_activity') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00050_memory_tags.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- lorekit_memory_tags(p_user_id, p_archived) — the label (tags) catalog, for +-- GET /memories/tags. +-- +-- WHY an RPC rather than a client-side tally: this is lorekit_memory_scopes +-- (00039) applied to the second unbounded dimension. The dashboard's label +-- filter built its catalog from `select tags from memories` plus a +-- client-side reduce (packages/web/src/lib/queries/lore.ts), which is +-- truncated without warning past PostgREST's row cap — so a label used only by +-- older memories vanishes from its own filter, and every count is understated. +-- One grouped row per distinct label is exact at any volume. +-- +-- WHY it is archived-aware: the catalog must describe the population it will +-- filter. The Explorer renders the bar in both the active and the archived +-- view and `GET /memories` partitions on archived_at, so a catalog pinned to +-- active rows describes the wrong population in archived mode — wrong counts, +-- and archive-only labels missing entirely. p_archived selects the partition, +-- exactly as `?archived=` does on the list route. +-- +-- Expiry: the active partition additionally excludes expired rows, matching +-- lorekit_memory_scopes and the default GET /memories predicate. The archived +-- partition does NOT — an archived row is out of the active set already, and +-- `?archived=true` on the list route applies no expiry filter either, so +-- applying one here would make the catalog disagree with the list it labels. +-- +-- Ordering is count desc, then tag asc: the caller renders a picker, and a bar +-- that reshuffles under the cursor for equal counts is a usability bug. As with +-- every other read RPC here, plpgsql (not SQL) so the ORDER BY cannot be lost +-- to inlining. +-- +-- Actor resolution, visibility and grants are the 00046/00047 rule verbatim — +-- a caller-supplied p_user_id is honoured only on a verified service-role +-- connection, and PUBLIC/anon get no EXECUTE (label names are as sensitive as +-- scope names: they are free text written by agents). +-- ═════════════════════════════════════════════════════════════════════════ + +create or replace function lorekit_memory_tags( + p_user_id uuid, + p_archived boolean default false +) +returns table (tag text, count bigint) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; +begin + return query + select t.tag, count(*) as count + from memories m + -- unnest, not a GIN-assisted filter: the catalog is a full rollup of a + -- text[] column, so every visible row contributes each of its labels. + cross join lateral unnest(m.tags) as t(tag) + where ( + (v_actor is null and auth.role() = 'service_role') + or m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) + ) + and ( + case + when p_archived then m.archived_at is not null + else m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + end + ) + and t.tag is not null + and btrim(t.tag) <> '' + group by t.tag + order by count(*) desc, t.tag asc; +end; +$$; + +revoke execute on function lorekit_memory_tags(uuid, boolean) from public, anon; +grant execute on function lorekit_memory_tags(uuid, boolean) to authenticated, service_role; + +comment on function lorekit_memory_tags(uuid, boolean) is + 'Label (memories.tags) catalog with per-label counts, for the partition + selected by p_archived, visible to the EFFECTIVE caller. Same + service-role-gated actor rule and tenant predicate as + lorekit_memory_scopes. Ordered count desc, tag asc.'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00050', 'memory_tags') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00051_memory_activity.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- lorekit_memory_activity(p_user_id, p_bucket, p_since, p_until) — memories +-- created per UTC hour/day per scope, for GET /memories/activity. +-- +-- WHY: the dashboard's stat cards (24h / 7d / 30d sparkbars + period-over- +-- period change) and the contribution heatmap both need "how many memories, +-- when, in which scope". Both were computed client-side from a raw +-- `select scope, created_at ... limit 1000` — truncated without warning past +-- the cap, and shipping up to 1000 rows to a browser to produce ~60 numbers. +-- +-- Bucketing in Postgres is exact at any volume and the payload is bounded by +-- (buckets × active scopes) rather than by memory count. `date_trunc` anchors +-- each bucket at the START of the UTC hour/day, which is precisely where the +-- client's own bucket boundaries fall (`Math.floor(t / HOUR_MS) * HOUR_MS` and +-- `Date.UTC(y, m, d)` in packages/web/src/lib/aggregations.ts), so a client +-- tallying these rows gets the same numbers it got from raw rows. +-- +-- The [p_since, p_until) window is half-open, matching lorekit_usage_stats +-- (00043). Both bounds are optional; the CALLER is responsible for choosing a +-- window, and the edge handler defaults it rather than leaving it unbounded. +-- +-- p_bucket is a bounded categorical ('hour' | 'day'), validated here as well as +-- at the edge. It is passed to date_trunc as a plpgsql function ARGUMENT, not +-- interpolated into SQL text, so there is no injection surface — this function +-- builds no dynamic SQL at all (no EXECUTE, no format()). The check earns its +-- place anyway: date_trunc on an unknown field raises an opaque 22023 from +-- inside the query, so validating up front turns that into a named, testable +-- error (AC-5 in supabase/tests/migrations.test.sql). Anything else raises. +-- +-- Actor resolution, visibility, grants, and plpgsql-for-ordering are the +-- 00046/00047 rule verbatim. +-- ═════════════════════════════════════════════════════════════════════════ + +create or replace function lorekit_memory_activity( + p_user_id uuid, + p_bucket text default 'day', + p_since timestamptz default null, + p_until timestamptz default null +) +returns table (bucket timestamptz, scope text, count bigint) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; +begin + if p_bucket is null or p_bucket not in ('hour', 'day') then + raise exception 'invalid bucket %, expected hour or day', p_bucket + using errcode = '22023'; + end if; + + return query + select date_trunc(p_bucket, m.created_at at time zone 'UTC') at time zone 'UTC' as bucket, + m.scope, + count(*) as count + from memories m + where ( + (v_actor is null and auth.role() = 'service_role') + or m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) + ) + and m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + and (p_since is null or m.created_at >= p_since) + and (p_until is null or m.created_at < p_until) + group by 1, m.scope + order by 1 asc, m.scope asc; +end; +$$; + +revoke execute on function lorekit_memory_activity(uuid, text, timestamptz, timestamptz) from public, anon; +grant execute on function lorekit_memory_activity(uuid, text, timestamptz, timestamptz) to authenticated, service_role; + +comment on function lorekit_memory_activity(uuid, text, timestamptz, timestamptz) is + 'Memories created per UTC hour/day per scope over the half-open + [p_since, p_until) window, visible to the EFFECTIVE caller. Same + service-role-gated actor rule and tenant predicate as + lorekit_memory_scopes. p_bucket is validated against (hour, day).'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00051', 'memory_activity') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00052_memory_facets.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- lorekit_memory_facets(p_user_id, p_archived) — the value catalog for EVERY +-- filterable dimension, for GET /memories/facets. +-- +-- WHY it exists: the Explorer grew from a labels-only picker into a +-- multi-dimension filter menu (labels, agent, trigger, repo, branch, pull +-- request). Each new dimension is another unbounded free-text column, so each +-- would otherwise repeat the exact bug 00039 and 00050 were written to fix — +-- `select from memories` plus a browser-side tally is truncated +-- without warning past PostgREST's row cap, so a value used only by older +-- memories disappears from its own filter and every count is understated. +-- +-- WHY one RPC and not six: the menu's headline affordance is cross-dimension +-- type-ahead ("type `main`, get `Branch → main`"), which needs every +-- dimension's values before the user has chosen a dimension. Six round trips +-- to render one list is six chances to render it half-populated. +-- +-- WHY it overlaps lorekit_memory_tags (00050): that function is the +-- single-dimension label catalog `GET /memories/tags` serves to the CLI and to +-- older clients, and deleting it would be a breaking API change. Both read the +-- same rows under the same predicate, so their `tag` rows agree; the union +-- branch below is deliberately written in the same shape as 00050's body so +-- the agreement is visible rather than asserted. +-- +-- Expiry, ordering, actor resolution, visibility and grants are 00050's rules +-- verbatim: the active partition additionally excludes expired rows and the +-- archived partition does not; ordering is count desc then value asc so a +-- picker never reshuffles under the cursor for equal counts; a caller-supplied +-- p_user_id is honoured only on a verified service-role connection; and +-- PUBLIC/anon get no EXECUTE (branch names, repo names and agent names are at +-- least as sensitive as the scope names 00039 withholds). +-- ═════════════════════════════════════════════════════════════════════════ + +create or replace function lorekit_memory_facets( + p_user_id uuid, + p_archived boolean default false +) +returns table (facet text, value text, count bigint) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; +begin + return query + with visible as ( + select m.tags, m.source_agent, m.trigger, + m.origin_repo, m.origin_branch, m.origin_pr + from memories m + where ( + (v_actor is null and auth.role() = 'service_role') + or m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) + ) + and ( + case + when p_archived then m.archived_at is not null + else m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + end + ) + ), cells as ( + -- Labels: unnest, not a GIN-assisted filter — a rollup of a text[] column + -- means every visible row contributes each of its labels (00050). + select 'tag'::text as facet, t.tag as value + from visible v + cross join lateral unnest(v.tags) as t(tag) + union all + select 'source_agent', v.source_agent from visible v + union all + select 'trigger', v.trigger from visible v + union all + select 'origin_repo', v.origin_repo from visible v + union all + select 'origin_branch', v.origin_branch from visible v + -- A pull request number is rendered and filtered as text on the wire, so + -- it is cast once here rather than in every consumer. + union all + select 'origin_pr', v.origin_pr::text from visible v + ) + select c.facet, c.value, count(*) as count + from cells c + -- A null column is "this memory has no branch", not a value to offer: an + -- option that matches by absence needs an `is not set` operator, which the + -- list route does not have yet. Blank strings are dropped for 00050's + -- reason — they are unrenderable and unselectable. + where c.value is not null + and btrim(c.value) <> '' + group by c.facet, c.value + order by c.facet asc, count(*) desc, c.value asc; +end; +$$; + +revoke execute on function lorekit_memory_facets(uuid, boolean) from public, anon; +grant execute on function lorekit_memory_facets(uuid, boolean) to authenticated, service_role; + +comment on function lorekit_memory_facets(uuid, boolean) is + 'Value catalog with counts for every filterable memory dimension (tag, + source_agent, trigger, origin_repo, origin_branch, origin_pr) over the + partition selected by p_archived, visible to the EFFECTIVE caller. Same + service-role-gated actor rule, tenant predicate and ordering as + lorekit_memory_tags.'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00052', 'memory_facets') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00053_read_activity.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- lorekit_read_activity(p_user_id, p_bucket, p_since, p_until) — memory +-- RECORDS read per UTC hour/day, for GET /memories/read-activity. +-- +-- WHY: the Overview's third stat card claims to measure reads, but no +-- reads-over-time series existed, so it charted memories WRITTEN under a +-- "scopes active" headline — a chart whose bars measured one thing and whose +-- number measured another. Reads are already recorded: every MCP tool call and +-- every REST route writes one `usage_events` row (00034), and 00044 added +-- `result_count`, the RECORDS a call touched. Summing that over the read tools +-- is an additive metric, so the sparkbar's bars add up to the headline — the +-- property every card on that page is now held to. +-- +-- Modelled on lorekit_memory_activity (00051): same bucket validation, same +-- half-open [p_since, p_until) window, same date_trunc anchoring so a JS client +-- bucketing by UTC hour/day tallies identically. +-- +-- Visibility is SELF-ONLY, unlike 00051. Usage is a per-user ledger: there is +-- no org sharing of read events (a co-member's reads are not the caller's +-- activity, and usage_events has no org_id at all), so this uses the same +-- self-only predicate + service-role escape hatch as lorekit_usage_stats +-- (00043) and deliberately does NOT join lorekit_member_org_ids — that function +-- is the tenant predicate for `memories` visibility, not for usage. +-- +-- The read-tool list is an inline literal, mirroring how 00045 filters +-- `memory.expired` inline. `packages/mcp-core/src/permissions.ts`'s READ_TOOLS +-- carries the same four strings, but it is the MCP permission gate, not a SQL +-- artefact, and there is no mechanism to share a TS constant with a migration. +-- Keep the two in step by hand if a read tool is ever added. +-- +-- It mirrors READ_TOOLS — the memory read-FAMILY — and deliberately NOT +-- `usage-stats.ts`'s broader READ_TOOL_NAMES, which additionally classifies +-- `memory.scopes`, `memory.usage`, `org.list`, `org.get`, `member.list` and +-- `member.invite_list` as reads. Those are analytics and org listings, not +-- memory records, so `GET /usage`'s `records_read` is legitimately LARGER than +-- the sum here; a card headlined "Memories read" must not count `org.list`. +-- +-- `having sum(...) > 0` keeps the payload sparse: only buckets that actually +-- read something come back, exactly as 00051's aggregate is sparse. +-- ═════════════════════════════════════════════════════════════════════════ + +create or replace function lorekit_read_activity( + p_user_id uuid, + p_bucket text default 'day', + p_since timestamptz default null, + p_until timestamptz default null +) +returns table (bucket timestamptz, count bigint) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; +begin + if p_bucket is null or p_bucket not in ('hour', 'day') then + raise exception 'invalid bucket %, expected hour or day', p_bucket + using errcode = '22023'; + end if; + + return query + select date_trunc(p_bucket, ue.created_at at time zone 'UTC') at time zone 'UTC' as bucket, + sum(coalesce(ue.result_count, 0))::bigint as count + from usage_events ue + where ( + (v_actor is null and auth.role() = 'service_role') + or ue.user_id = v_actor + ) + and ue.tool_name in ('memory.read', 'memory.list', 'memory.search', + 'memory.list_archived') + and (p_since is null or ue.created_at >= p_since) + and (p_until is null or ue.created_at < p_until) + group by 1 + having sum(coalesce(ue.result_count, 0)) > 0 + order by 1 asc; +end; +$$; + +revoke execute on function lorekit_read_activity(uuid, text, timestamptz, timestamptz) from public, anon; +grant execute on function lorekit_read_activity(uuid, text, timestamptz, timestamptz) to authenticated, service_role; + +comment on function lorekit_read_activity(uuid, text, timestamptz, timestamptz) is + 'Memory RECORDS read (sum of usage_events.result_count over memory.read / + memory.list / memory.search / memory.list_archived — permissions.ts''s + READ_TOOLS) per UTC hour/day over the half-open + [p_since, p_until) window. Visibility is SELF-ONLY with the same + service-role + NULL escape hatch as lorekit_usage_stats — usage is a + per-user ledger and is never org-shared. p_bucket is validated against + (hour, day). Buckets with no records read are omitted.'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00053', 'read_activity') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00054_usage_event_client.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- usage_events.client — WHICH SURFACE made the call — and the read-activity +-- metric that stops counting the dashboard's own reads. +-- +-- THE BUG: reloading the dashboard made the Overview's "Memories read" card go +-- up, every time. The dashboard is a CLIENT of LoreKit's own REST API (a +-- deliberate decision — see CLAUDE.md), so rendering the Lore Explorer issues a +-- real `GET /memories`, the router records a real `usage_events` row with a +-- real `result_count`, and `lorekit_read_activity` (00053) sums it. The card +-- was therefore measuring "records this account read INCLUDING the ones the +-- page fetched in order to draw this very card" — a metric that responds to +-- looking at it. Visualising your lore is not consuming it; the card is meant +-- to answer "how much have my agents read". +-- +-- THE FIX, in two halves: +-- +-- 1. Record WHO called. `usage_events` already carried `auth_type` (HOW the +-- caller authenticated) and `tool_name` (WHAT they asked for) but nothing +-- for WHICH SURFACE — and neither existing column can stand in. A +-- dashboard read and an agent read over a Supabase JWT are both +-- `auth_type = 'jwt'` calling `memory.list`; distinguishing them by +-- auth tier would be a guess that silently mis-attributes the moment an +-- agent authenticates with a JWT. So: a new nullable `client` column, +-- filled from the client-supplied `X-LoreKit-Client` header at the ONE +-- recording site each surface already has (`_shared/api/router.ts`, +-- `mcp/mcp-handler.ts`) and validated against a closed vocabulary by the +-- pure `parseUsageClient` (`_shared/telemetry/usage-stats.ts`). Unattributed stays +-- NULL — the 00044 posture: a telemetry dimension can never fail the +-- call it is measuring. +-- +-- 2. Exclude the dashboard from the read METRIC, not from the LEDGER. The +-- events are still written in full, so `GET /memories/usage` still totals +-- every read and the exclusion is reversible with one more migration. +-- Only `lorekit_read_activity` — the series behind the "Memories read" +-- card — filters `client is distinct from 'dashboard'`. +-- +-- The exclusion is inlined in the RPC rather than parameterised, mirroring how +-- 00053 inlines its read-tool list and 00045 inlines `memory.expired`: it is +-- the definition of the metric, not a caller's choice. Keep the literal in step +-- with `DASHBOARD_CLIENT` in `packages/mcp-core/src/usage-stats.ts` by hand — +-- there is no mechanism to share a TS constant with a migration. +-- +-- Forward-only and additive: the column is nullable and the writer's new +-- parameter is trailing + defaulted, so every existing row and every caller +-- that omits it is unaffected. +-- ═════════════════════════════════════════════════════════════════════════ + +-- ── the column ────────────────────────────────────────────────────────────── +alter table usage_events add column if not exists client text; + +-- Bound the value as a BACKSTOP, not as the primary gate — exactly the reason +-- 00044 gives for `usage_events_correlation_id_len`. The authoritative closed +-- vocabulary is `parseUsageClient`; a CHECK enumerating the members would turn +-- adding a surface into a migration, while an unbounded column would let a +-- malformed header inflate analytics cardinality. A short length cap gives the +-- storage/cardinality guarantee without pinning the vocabulary in SQL. +alter table usage_events drop constraint if exists usage_events_client_len; +alter table usage_events add constraint usage_events_client_len + check (client is null or (char_length(client) between 1 and 32)); + +-- Index the read this exists for: "my read events, excluding one client, +-- newest first". Partial on the non-null rows because a NULL client carries no +-- information to filter on. +create index if not exists usage_events_user_client_idx + on usage_events (user_id, client, created_at desc) + where client is not null; + +-- ── writer: add a trailing p_client ───────────────────────────────────────── +-- DROP first (not CREATE OR REPLACE): adding a parameter changes the signature, +-- and a bare CREATE would leave the 00044 overload behind and make every call +-- ambiguous. Body is 00044's verbatim plus the one column. +drop function if exists lorekit_record_usage_event(uuid, uuid, text, text, text, text, text, integer, integer, integer, text); + +create or replace function lorekit_record_usage_event( + p_user_id uuid default null, + p_org_id uuid default null, + p_plan_name text default null, + p_tool_name text default null, + p_scope_type text default null, + p_auth_type text default null, + p_outcome text default null, + p_duration_ms integer default null, + p_memory_count integer default null, + p_result_count integer default null, + p_correlation_id text default null, + p_client text default null +) +returns uuid +language plpgsql +security definer +set search_path = public +as $$ +declare + v_id uuid; +begin + insert into usage_events ( + user_id, org_id, plan_name, + tool_name, scope_type, auth_type, + outcome, duration_ms, memory_count, + result_count, correlation_id, client + ) values ( + p_user_id, p_org_id, p_plan_name, + p_tool_name, p_scope_type, p_auth_type, + p_outcome, p_duration_ms, p_memory_count, + p_result_count, p_correlation_id, p_client + ) + returning id into v_id; + return v_id; +exception + when others then + -- Never let telemetry writes break the primary operation. + return null; +end; +$$; + +grant execute on function lorekit_record_usage_event(uuid, uuid, text, text, text, text, text, integer, integer, integer, text, text) + to anon, authenticated, service_role; + +-- ── reader: the "Memories read" series stops counting the dashboard ───────── +-- CREATE OR REPLACE, not DROP: the signature is unchanged from 00053, so every +-- caller (`GET /memories/read-activity`) is untouched. Everything else — the +-- bucket validation, the half-open window, the self-only visibility with the +-- service-role escape hatch, the read-tool list, the sparse `having` — is +-- 00053's verbatim. +create or replace function lorekit_read_activity( + p_user_id uuid, + p_bucket text default 'day', + p_since timestamptz default null, + p_until timestamptz default null +) +returns table (bucket timestamptz, count bigint) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; +begin + if p_bucket is null or p_bucket not in ('hour', 'day') then + raise exception 'invalid bucket %, expected hour or day', p_bucket + using errcode = '22023'; + end if; + + return query + select date_trunc(p_bucket, ue.created_at at time zone 'UTC') at time zone 'UTC' as bucket, + sum(coalesce(ue.result_count, 0))::bigint as count + from usage_events ue + where ( + (v_actor is null and auth.role() = 'service_role') + or ue.user_id = v_actor + ) + and ue.tool_name in ('memory.read', 'memory.list', 'memory.search', + 'memory.list_archived') + -- The dashboard reading lore in order to DRAW this chart is not a read + -- the chart should report. `is distinct from` (not `<>`) because the + -- column is nullable and `null <> 'dashboard'` is null, which would + -- silently drop every unattributed event — including every row written + -- before this migration. + and ue.client is distinct from 'dashboard' + and (p_since is null or ue.created_at >= p_since) + and (p_until is null or ue.created_at < p_until) + group by 1 + having sum(coalesce(ue.result_count, 0)) > 0 + order by 1 asc; +end; +$$; + +revoke execute on function lorekit_read_activity(uuid, text, timestamptz, timestamptz) from public, anon; +grant execute on function lorekit_read_activity(uuid, text, timestamptz, timestamptz) to authenticated, service_role; + +comment on function lorekit_read_activity(uuid, text, timestamptz, timestamptz) is + 'Memory RECORDS read (sum of usage_events.result_count over memory.read / + memory.list / memory.search / memory.list_archived — permissions.ts''s + READ_TOOLS) per UTC hour/day over the half-open [p_since, p_until) window, + EXCLUDING events attributed to the dashboard client (00054): browsing your + own lore in the web UI is visualisation, not consumption, so it must not + inflate the "Memories read" card it is drawing. Unattributed (NULL client) + events are counted. Visibility is SELF-ONLY with the same service-role + + NULL escape hatch as lorekit_usage_stats — usage is a per-user ledger and is + never org-shared. p_bucket is validated against (hour, day). Buckets with no + records read are omitted.'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00054', 'usage_event_client') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00055_blog_post_likes.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- blog_post_likes — a public, anonymous, cumulative "like" counter for /blog. +-- +-- WHY this table looks unlike every other one in this schema: the blog is a +-- PUBLIC, statically-generated surface with no auth gate, and a like +-- accumulates across ALL visitors — signed-in or not. So, uniquely here, the +-- read policy and the increment RPC are granted to `anon`. There is no +-- user_id, no tenant, no owner-scoped RLS: the counter is one global total per +-- post slug, exactly like a page-view counter. +-- +-- The per-user/session cap (max 100 likes) is enforced CLIENT-side, in +-- localStorage, because an anonymous visitor has no server identity to key a +-- per-session tally on. The server's job is only to (a) accumulate the global +-- total atomically and (b) refuse an obviously abusive single call: the RPC +-- clamps p_delta to [1, 100] and validates the slug shape, so no single request +-- can inflate a counter by more than one session's worth, and no caller can +-- create junk rows under arbitrary keys. A determined client can still script +-- the endpoint — that is inherent to an anonymous public counter and accepted +-- for a blog vanity metric; the clamp bounds the per-call blast radius. +-- +-- Writes go ONLY through the SECURITY DEFINER RPC (there is no insert/update +-- RLS policy), so the slug validation and delta clamp cannot be bypassed by a +-- direct PostgREST write. Reads are a plain SELECT under the public policy. +-- ═════════════════════════════════════════════════════════════════════════ + +create table if not exists blog_post_likes ( + slug text primary key + check (slug ~ '^[a-z0-9]+(?:-[a-z0-9]+)*$' and length(slug) <= 128), + likes bigint not null default 0 check (likes >= 0), + updated_at timestamptz not null default now() +); + +alter table blog_post_likes enable row level security; + +-- Public read: anyone (signed-in or not) can see the totals. This is the whole +-- point — the count is shown to every blog visitor. +drop policy if exists blog_post_likes_public_read on blog_post_likes; +create policy blog_post_likes_public_read + on blog_post_likes for select + to anon, authenticated + using (true); + +-- No insert / update / delete policy is defined on purpose: every write must go +-- through lorekit_blog_like below, so the slug CHECK and the [1,100] delta clamp +-- are always applied. + +-- ── lorekit_blog_like(slug, delta) → new global total ────────────────────── +-- Atomic upsert-increment. SECURITY DEFINER so `anon` can accumulate the total +-- without any table-level write grant. Validates the slug and clamps the delta. +create or replace function lorekit_blog_like(p_slug text, p_delta int default 1) +returns bigint +language plpgsql +security definer +set search_path = public +as $$ +declare + -- One click is +1; a batched flush of a rapid-tap burst sends its accumulated + -- delta. Clamp to a single session's worth so no call can inflate the counter + -- past the client-enforced cap, and floor at 1 so a stray 0/negative is a + -- no-op increment rather than a silent decrement. + v_delta int := least(greatest(coalesce(p_delta, 1), 1), 100); + v_total bigint; +begin + if p_slug is null + or p_slug !~ '^[a-z0-9]+(?:-[a-z0-9]+)*$' + or length(p_slug) > 128 then + raise exception 'invalid blog slug %', p_slug using errcode = '22023'; + end if; + + insert into blog_post_likes as l (slug, likes, updated_at) + values (p_slug, v_delta, now()) + on conflict (slug) + do update set likes = l.likes + v_delta, updated_at = now() + returning l.likes into v_total; + + return v_total; +end; +$$; + +revoke execute on function lorekit_blog_like(text, int) from public; +grant execute on function lorekit_blog_like(text, int) to anon, authenticated, service_role; + +comment on table blog_post_likes is + 'Public, anonymous, cumulative like counter for /blog posts — one global total + per post slug. No user_id/tenant: reads are public and writes go only through + lorekit_blog_like. The per-session 100-like cap is enforced client-side.'; + +comment on function lorekit_blog_like(text, int) is + 'Atomically add p_delta (clamped to [1,100]) likes to a blog post and return + the new global total. SECURITY DEFINER + granted to anon so an unauthenticated + blog visitor can like a post. Validates the slug shape (lowercase kebab, <=128) + so no junk rows can be created.'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00055', 'blog_post_likes') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00056_memory_kind_host.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- memories.kind + memories.host — the bucket TAXONOMY as first-class columns, +-- and the same two dimensions on usage_events so operations are attributable +-- to a family and an owner. +-- +-- WHY: a loop's memory bucket has always encoded two facts in its tag string — +-- WHAT KIND of memory it is (a procedural `lesson`, a transient outcome `bus`, +-- or a durable per-repo `signal`) and WHICH HOST owns it (`reviewer`, `aw`, …). +-- Reading either meant parsing `loop::-lessons` on the client. Promoting +-- them to columns lets a caller filter (`?kind=lesson`), lets usage analytics +-- group by kind/host, and lets the dashboard show the three families instead of +-- implying them. The authoritative vocabulary lives in agent-skills' +-- `agents/shared/rules/memory-buckets.md`. +-- +-- kind is a closed 3-value vocabulary; host is open (a new host is a new agent, +-- not a migration). Following 00054's `client` precedent, neither is pinned by a +-- CHECK enumerating members — the Zod `MemoryWriteSchema` is the authoritative +-- gate for kind, and a length backstop bounds cardinality/storage without +-- turning "add a host" into a migration. +-- +-- Forward-only and additive: both columns are nullable, and each RPC is DROPped +-- and recreated with its new params appended-and-defaulted, so every existing +-- row and every caller that omits them is unaffected. A memory written before +-- this migration has NULL kind/host; the app derives them from the legacy tag +-- (see tags.ts `inferKindHost`) so no data backfill is required. +-- ═════════════════════════════════════════════════════════════════════════ + +-- ── columns on memories ────────────────────────────────────────────────────── +alter table memories add column if not exists kind text; +alter table memories add column if not exists host text; + +-- Length backstops (not the primary gate — see 00054). kind's closed vocabulary +-- is enforced by MemoryWriteSchema; host is open free-text like source_agent. +alter table memories drop constraint if exists memories_kind_len; +alter table memories add constraint memories_kind_len + check (kind is null or (char_length(kind) between 1 and 32)); +alter table memories drop constraint if exists memories_host_len; +alter table memories add constraint memories_host_len + check (host is null or (char_length(host) between 1 and 64)); + +-- Index the family/owner read: "lessons for host reviewer", newest first. +create index if not exists memories_kind_host_idx + on memories (kind, host) + where kind is not null; + +-- ── writer: memory_write gains p_kind + p_host ────────────────────────────── +-- DROP first (not CREATE OR REPLACE): adding params changes the signature, and a +-- bare CREATE would leave the 00048 15-arg overload behind and make calls +-- ambiguous. Body is 00048's verbatim plus the two columns in every branch. +-- kind/host use the origin_* "last KNOWN value wins" merge on update so a write +-- that does not know them (an older client) keeps the previously recorded value +-- rather than erasing it. +drop function if exists memory_write(uuid, text, text, text, text[], text, text, timestamptz, text, integer, boolean, text, text, text, integer); + +create or replace function memory_write( + p_user_id uuid, + p_scope text, + p_key text, + p_value text, + p_tags text[] default '{}', + p_source_agent text default null, + p_trigger text default null, + p_created_at timestamptz default null, + p_org_slug text default null, + p_ttl_seconds integer default null, + p_clear_ttl boolean default false, + p_origin_repo text default null, + p_origin_branch text default null, + p_origin_commit text default null, + p_origin_pr integer default null, + p_kind text default null, + p_host text default null +) +returns table ( + id uuid, + created_at timestamptz, + inserted boolean, + org_routed boolean, + binding_org_slug text, + expires_at timestamptz +) +language plpgsql +set search_path = public +as $$ +declare + v_org_id uuid; + v_binding_org uuid; + v_binding_slug text; + v_expires_at timestamptz; + v_ttl_action text := 'keep'; +begin + if p_clear_ttl then + v_ttl_action := 'clear'; + elsif p_ttl_seconds is not null then + if p_ttl_seconds < 1 or p_ttl_seconds > 31536000 then + raise exception using errcode = 'P0001', + message = format('ttl_seconds must be between 1 and 31536000, got %s', p_ttl_seconds); + end if; + v_expires_at := now() + (p_ttl_seconds * interval '1 second'); + v_ttl_action := 'set'; + end if; + + if p_org_slug is not null then + select o.id into v_org_id from orgs o where o.slug = p_org_slug and o.deleted_at is null; + if v_org_id is null then + raise exception using errcode = 'P0001', message = format('unknown_org: %s', p_org_slug); + end if; + if not lorekit_org_can(p_user_id, v_org_id, 'write') then + raise exception using errcode = 'LK002', message = format('org_permission_denied: org=%s', p_org_slug); + end if; + else + select b.org_id, o.slug into v_binding_org, v_binding_slug + from org_scope_bindings b + join orgs o on o.id = b.org_id + where b.scope = p_scope and o.deleted_at is null; + + if v_binding_org is not null + and p_user_id is not null + and lorekit_org_can(p_user_id, v_binding_org, 'write') then + v_org_id := v_binding_org; + end if; + end if; + + if v_org_id is not null then + return query + insert into memories ( + user_id, org_id, scope, key, value, tags, source_agent, trigger, + created_at, updated_at, created_by, updated_by, expires_at, + origin_repo, origin_branch, origin_commit, origin_pr, kind, host + ) + values ( + null, v_org_id, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), + p_user_id, p_user_id, v_expires_at, + p_origin_repo, p_origin_branch, p_origin_commit, p_origin_pr, p_kind, p_host + ) + on conflict (org_id, scope, key) where org_id is not null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + updated_by = p_user_id, + origin_repo = coalesce(excluded.origin_repo, memories.origin_repo), + origin_branch = coalesce(excluded.origin_branch, memories.origin_branch), + origin_commit = coalesce(excluded.origin_commit, memories.origin_commit), + origin_pr = coalesce(excluded.origin_pr, memories.origin_pr), + kind = coalesce(excluded.kind, memories.kind), + host = coalesce(excluded.host, memories.host), + expires_at = case v_ttl_action + when 'clear' then null + when 'set' then v_expires_at + else memories.expires_at + end + returning + memories.id, memories.created_at, (xmax::text = '0') as inserted, + true as org_routed, v_binding_slug as binding_org_slug, memories.expires_at; + + elsif p_user_id is null then + return query + insert into memories ( + user_id, scope, key, value, tags, source_agent, trigger, + created_at, updated_at, created_by, updated_by, expires_at, + origin_repo, origin_branch, origin_commit, origin_pr, kind, host + ) + values ( + null, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), + null, null, v_expires_at, + p_origin_repo, p_origin_branch, p_origin_commit, p_origin_pr, p_kind, p_host + ) + on conflict (scope, key) where org_id is null and user_id is null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + origin_repo = coalesce(excluded.origin_repo, memories.origin_repo), + origin_branch = coalesce(excluded.origin_branch, memories.origin_branch), + origin_commit = coalesce(excluded.origin_commit, memories.origin_commit), + origin_pr = coalesce(excluded.origin_pr, memories.origin_pr), + kind = coalesce(excluded.kind, memories.kind), + host = coalesce(excluded.host, memories.host), + expires_at = case v_ttl_action + when 'clear' then null + when 'set' then v_expires_at + else memories.expires_at + end + returning + memories.id, memories.created_at, (xmax::text = '0') as inserted, + false as org_routed, v_binding_slug as binding_org_slug, memories.expires_at; + + else + return query + insert into memories ( + user_id, scope, key, value, tags, source_agent, trigger, + created_at, updated_at, created_by, updated_by, expires_at, + origin_repo, origin_branch, origin_commit, origin_pr, kind, host + ) + values ( + p_user_id, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), + p_user_id, p_user_id, v_expires_at, + p_origin_repo, p_origin_branch, p_origin_commit, p_origin_pr, p_kind, p_host + ) + on conflict (user_id, scope, key) where org_id is null and user_id is not null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + updated_by = p_user_id, + origin_repo = coalesce(excluded.origin_repo, memories.origin_repo), + origin_branch = coalesce(excluded.origin_branch, memories.origin_branch), + origin_commit = coalesce(excluded.origin_commit, memories.origin_commit), + origin_pr = coalesce(excluded.origin_pr, memories.origin_pr), + kind = coalesce(excluded.kind, memories.kind), + host = coalesce(excluded.host, memories.host), + expires_at = case v_ttl_action + when 'clear' then null + when 'set' then v_expires_at + else memories.expires_at + end + returning + memories.id, memories.created_at, (xmax::text = '0') as inserted, + false as org_routed, v_binding_slug as binding_org_slug, memories.expires_at; + end if; +end; +$$; + +grant execute on function memory_write(uuid, text, text, text, text[], text, text, timestamptz, text, integer, boolean, text, text, text, integer, text, text) + to anon, authenticated, service_role; + +-- ── usage_events: kind + host as tracked dimensions ───────────────────────── +alter table usage_events add column if not exists kind text; +alter table usage_events add column if not exists host text; + +alter table usage_events drop constraint if exists usage_events_kind_len; +alter table usage_events add constraint usage_events_kind_len + check (kind is null or (char_length(kind) between 1 and 32)); +alter table usage_events drop constraint if exists usage_events_host_len; +alter table usage_events add constraint usage_events_host_len + check (host is null or (char_length(host) between 1 and 64)); + +-- Index the family/owner analytics roll-up: "my events for kind X", newest first. +create index if not exists usage_events_user_kind_idx + on usage_events (user_id, kind, created_at desc) + where kind is not null; + +-- ── writer: record_usage_event gains p_kind + p_host ──────────────────────── +-- DROP first — adding params changes the signature (00054's 12-arg overload +-- would otherwise remain and make calls ambiguous). Body is 00054's verbatim +-- plus the two columns. +drop function if exists lorekit_record_usage_event(uuid, uuid, text, text, text, text, text, integer, integer, integer, text, text); + +create or replace function lorekit_record_usage_event( + p_user_id uuid default null, + p_org_id uuid default null, + p_plan_name text default null, + p_tool_name text default null, + p_scope_type text default null, + p_auth_type text default null, + p_outcome text default null, + p_duration_ms integer default null, + p_memory_count integer default null, + p_result_count integer default null, + p_correlation_id text default null, + p_client text default null, + p_kind text default null, + p_host text default null +) +returns uuid +language plpgsql +security definer +set search_path = public +as $$ +declare + v_id uuid; +begin + insert into usage_events ( + user_id, org_id, plan_name, + tool_name, scope_type, auth_type, + outcome, duration_ms, memory_count, + result_count, correlation_id, client, kind, host + ) values ( + p_user_id, p_org_id, p_plan_name, + p_tool_name, p_scope_type, p_auth_type, + p_outcome, p_duration_ms, p_memory_count, + p_result_count, p_correlation_id, p_client, p_kind, p_host + ) + returning id into v_id; + return v_id; +exception + when others then + -- Never let telemetry writes break the primary operation. + return null; +end; +$$; + +grant execute on function lorekit_record_usage_event(uuid, uuid, text, text, text, text, text, integer, integer, integer, text, text, text, text) + to anon, authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00056', 'memory_kind_host') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00057_memory_facets_drilldown.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- lorekit_memory_facets — drill-down (filter-aware) counts + kind/host. +-- +-- WHAT CHANGES: 00052's facets were GLOBAL — every dimension's counts were the +-- total over all visible memories, ignoring whatever the user had already +-- selected. A drill-down filter menu wants the opposite: once you pick +-- `kind=lesson`, the counts shown for `host`, `source_agent`, … should narrow +-- to "how many LESSON memories also carry this host", so the numbers you see +-- are what you'd actually get by adding that filter. +-- +-- THE SELF-EXCLUSION RULE (the easy thing to get wrong): when counting a +-- dimension D's values, apply every OTHER active filter but NOT D's own. If D's +-- own selection were applied to D's own counts, D would collapse to just the +-- value(s) already picked and every other value would read 0 — you could never +-- see what else you could switch to. This is "OR within a dimension, AND across +-- dimensions", the exact model ListMemoriesQuerySchema's `*_mode` filters use. +-- +-- HOW: compute each dimension's match flag ONCE per row in `base` (does this row +-- satisfy that dimension's filter?), then a dimension's cells require the AND of +-- all the OTHER flags. A dimension the user has not filtered on has a flag that +-- is trivially true, so its counts fall back to the fully-filtered set — which +-- is exactly right. +-- +-- ALSO: adds `kind` and `host` (migration 00056) as two new dimensions. +-- +-- Forward-only. The function is DROPped and recreated with a wider signature; +-- every new parameter is optional and defaulted, and the handler is updated in +-- lockstep. With no filters supplied every flag is true and the result is +-- byte-for-byte 00052's global catalog, so callers that pass only +-- (p_user_id, p_archived) are unaffected. +-- ═════════════════════════════════════════════════════════════════════════ + +drop function if exists lorekit_memory_facets(uuid, boolean); + +create or replace function lorekit_memory_facets( + p_user_id uuid, + p_archived boolean default false, + -- Active filters, mirroring GET /memories' query params so the menu can pass + -- its current filter state verbatim. All optional: null/absent = not filtered. + p_scope text default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in' +) +returns table (facet text, value text, count bigint) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + -- `origin_pr` is an INTEGER column, so it must be compared numerically, not as + -- text: `GET /memories` sends the digits-only list bare to the integer column + -- (`applyScalarFilter(..., { quote: false })`), where Postgres reads `007` as + -- 7 and matches PR 7. A `m.origin_pr::text = any(...)` comparison here would + -- match nothing for the same input, so the two routes would disagree on the + -- same query string. Non-numeric entries are DROPPED rather than raising, and + -- a list that reduces to empty applies no filter at all (`array_agg` over no + -- rows is NULL) — the list route's documented behaviour, verbatim. + v_origin_pr integer[] := ( + select array_agg(x::integer) + from unnest(coalesce(p_origin_pr, '{}'::text[])) as x + where x ~ '^[0-9]+$' + ); +begin + return query + with base as ( + select + m.tags, m.source_agent, m.trigger, m.kind, m.host, + m.origin_repo, m.origin_branch, m.origin_pr, + -- Per-dimension match flag: is this row kept by THIS dimension's filter? + -- A null filter is "not filtered" → trivially true, so an untouched + -- dimension never narrows anything. + (p_tags is null or case coalesce(p_tags_mode, 'any') + when 'all' then m.tags @> p_tags + when 'none' then not (m.tags && p_tags) + else m.tags && p_tags + end) as ok_tag, + (p_source_agent is null or case coalesce(p_source_agent_mode, 'in') + when 'nin' then (m.source_agent is not null and m.source_agent <> all(p_source_agent)) + else m.source_agent = any(p_source_agent) + end) as ok_source_agent, + (p_trigger is null or case coalesce(p_trigger_mode, 'in') + when 'nin' then (m.trigger is not null and m.trigger <> all(p_trigger)) + else m.trigger = any(p_trigger) + end) as ok_trigger, + (p_kind is null or case coalesce(p_kind_mode, 'in') + when 'nin' then (m.kind is not null and m.kind <> all(p_kind)) + else m.kind = any(p_kind) + end) as ok_kind, + (p_host is null or case coalesce(p_host_mode, 'in') + when 'nin' then (m.host is not null and m.host <> all(p_host)) + else m.host = any(p_host) + end) as ok_host, + (p_origin_repo is null or case coalesce(p_origin_repo_mode, 'in') + when 'nin' then (m.origin_repo is not null and m.origin_repo <> all(p_origin_repo)) + else m.origin_repo = any(p_origin_repo) + end) as ok_origin_repo, + (p_origin_branch is null or case coalesce(p_origin_branch_mode, 'in') + when 'nin' then (m.origin_branch is not null and m.origin_branch <> all(p_origin_branch)) + else m.origin_branch = any(p_origin_branch) + end) as ok_origin_branch, + (v_origin_pr is null or case coalesce(p_origin_pr_mode, 'in') + when 'nin' then (m.origin_pr is not null and m.origin_pr <> all(v_origin_pr)) + else m.origin_pr = any(v_origin_pr) + end) as ok_origin_pr + from memories m + where ( + (v_actor is null and auth.role() = 'service_role') + or m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) + ) + and ( + case + when p_archived then m.archived_at is not null + else m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + end + ) + -- Scope is a hard filter, not a facet dimension, so it is always applied. + and (p_scope is null or m.scope = p_scope) + ), cells as ( + -- Each dimension's cells are counted over rows kept by every OTHER + -- dimension's filter (self-exclusion), so a dimension's own selection never + -- suppresses its own alternative values. + select 'tag'::text as facet, t.tag as value + from base b + cross join lateral unnest(b.tags) as t(tag) + where b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr + union all + select 'source_agent', b.source_agent from base b + where b.ok_tag and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr + union all + select 'trigger', b.trigger from base b + where b.ok_tag and b.ok_source_agent and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr + union all + select 'kind', b.kind from base b + where b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr + union all + select 'host', b.host from base b + where b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr + union all + select 'origin_repo', b.origin_repo from base b + where b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_branch and b.ok_origin_pr + union all + select 'origin_branch', b.origin_branch from base b + where b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_pr + union all + select 'origin_pr', b.origin_pr::text from base b + where b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch + ) + select c.facet, c.value, count(*) as count + from cells c + where c.value is not null + and btrim(c.value) <> '' + group by c.facet, c.value + order by c.facet asc, count(*) desc, c.value asc; +end; +$$; + +revoke execute on function lorekit_memory_facets( + uuid, boolean, text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text +) from public, anon; +grant execute on function lorekit_memory_facets( + uuid, boolean, text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text +) to authenticated, service_role; + +comment on function lorekit_memory_facets( + uuid, boolean, text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text +) is + 'Value catalog with counts for every filterable memory dimension (tag, + source_agent, trigger, kind, host, origin_repo, origin_branch, origin_pr) + over the partition selected by p_archived, visible to the EFFECTIVE caller. + Counts are DRILL-DOWN (00057): each dimension is counted with every OTHER + active filter applied but not its own (self-exclusion), so the numbers match + what adding that value would yield. With no filters supplied the result is + 00052''s global catalog. Same service-role-gated actor rule and ordering as + lorekit_memory_tags.'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00057', 'memory_facets_drilldown') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00058_usage_event_scope.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- usage_events.scope — WHICH SCOPE a call touched — and the read-activity +-- series grouped by it. +-- +-- THE GAP: reads already carry `scope_type` (00034: `global` / `project` / +-- `repo` / `branch`), which answers "how much do I read at repo level" but not +-- "how much do I read from `repo::mthines/lorekit`". `scope_type` is a +-- deliberately low-cardinality telemetry dimension and cannot be widened into +-- the exact string without changing what it means, so per-scope read totals — +-- the number the Explorer's stats header needs beside its per-scope WRITE +-- totals (`lorekit_memory_activity`, 00051) — were simply unanswerable. Writes +-- have had the exact scope since 00051; reads have not. +-- +-- THE FIX, mirroring 00054's structure exactly (that migration added `client` +-- the same way, and for the same class of reason): +-- +-- 1. A new nullable `scope` column on `usage_events`, filled at the ONE +-- recording site each surface already has (`_shared/api/router.ts`, +-- `mcp/mcp-handler.ts`) from the caller-supplied scope, normalised through +-- the canonical validator by the pure `safeValidateScope`. Unresolvable or +-- absent stays NULL — the 00044/00054 posture: a telemetry dimension can +-- never fail the call it is measuring. +-- +-- 2. `lorekit_read_activity` returns one row per `(bucket, scope)` instead of +-- one per `bucket`, exactly the shape `lorekit_memory_activity` (00051) +-- already returns for writes, plus an optional `p_scope` FILTER. +-- +-- WHY A FILTER PARAMETER AND NOT A COMPANION "per-scope total" RPC: the buckets +-- are additive (a sum of `result_count`), so the filtered buckets' SUM *is* the +-- per-scope headline. A second function would be a second definition of the +-- same number, free to drift from the bars it is drawn above — the property +-- 00053 exists to protect. +-- +-- BACKWARD COMPATIBILITY of the grouped shape: the sole consumer today is +-- `packages/web/src/lib/aggregations.ts`'s `computeCountTrend`, which sums +-- `row.count` into UTC time-slots and reads no other field. Splitting one row +-- per bucket into N rows per bucket therefore produces the identical trend — +-- the same argument 00051 made for the write series. +-- +-- DEFERRED (R10): `purge_expired_memories` (00045) is deliberately UNTOUCHED. +-- It records one `memory.expired` event per purge run, but the run is per-USER +-- and spans every scope that user owns, so there is no single scope to +-- attribute it to — splitting that event per scope is a separate change with +-- its own shape decision, not the cheap case this migration handles. +-- +-- Forward-only and additive throughout: the column is nullable, the writer's +-- new parameter is trailing + defaulted, and the reader's is trailing + +-- defaulted, so every existing row and every existing caller is unaffected. +-- ═════════════════════════════════════════════════════════════════════════ + +-- ── the column ────────────────────────────────────────────────────────────── +alter table usage_events add column if not exists scope text; + +-- Bound the value as a BACKSTOP, not as the primary gate — the same reasoning +-- 00044 gives for `usage_events_correlation_id_len` and 00054 for +-- `usage_events_client_len`. The authoritative validator is `safeValidateScope` +-- (the canonical `validateScope` wrapped to return null instead of throwing); a +-- CHECK re-encoding the scope grammar in SQL would be a second, drift-prone +-- copy of it, while an unbounded column would let a malformed value inflate +-- analytics cardinality. 200 chars is 00044's `usage_events_correlation_id_len` +-- ceiling reused verbatim — deliberately NOT "the same ceiling `memories.scope` +-- uses", which has none: that column is plain `text not null` +-- (`00001_memories.sql:8`) with no CHECK, so a memory may legitimately be +-- written under a scope this telemetry column would refuse. That asymmetry is +-- the point of a backstop on a DIMENSION rather than on the record itself. +-- +-- BACKSTOP means the recording side clamps FIRST: `safeValidateScope` returns +-- null above `USAGE_SCOPE_MAX` (= 200), so a grammatical but over-long scope is +-- recorded as unattributed instead of reaching this CHECK. It has to — the +-- violation would be raised inside `lorekit_record_usage_event`, whose `when +-- others` handler swallows it and returns null, dropping the WHOLE event rather +-- than just the scope dimension. Same clamp-then-check pairing 00044 has between +-- `parseCorrelationId` and `usage_events_correlation_id_len`. +alter table usage_events drop constraint if exists usage_events_scope_len; +alter table usage_events add constraint usage_events_scope_len + check (scope is null or (char_length(scope) between 1 and 200)); + +-- Index the read this exists for: "my read events under THIS scope, newest +-- first". Partial on the non-null rows because a NULL scope carries no +-- information to filter on — the same shape as 00054's client index. +create index if not exists usage_events_user_scope_created_idx + on usage_events (user_id, scope, created_at desc) + where scope is not null; + +-- ── writer: add a trailing p_scope ────────────────────────────────────────── +-- DROP first (not CREATE OR REPLACE): adding a parameter changes the signature, +-- and a bare CREATE would leave the previous overload behind and make every +-- call ambiguous. The signature dropped here is the FOURTEEN-argument one from +-- 00056 (which added `p_kind`/`p_host`), not 00054's twelve — a stale drop +-- target is silent, leaving both overloads live and every named-argument call +-- ambiguous at runtime. Body is 00056's verbatim plus the one column. +drop function if exists lorekit_record_usage_event(uuid, uuid, text, text, text, text, text, integer, integer, integer, text, text, text, text); + +create or replace function lorekit_record_usage_event( + p_user_id uuid default null, + p_org_id uuid default null, + p_plan_name text default null, + p_tool_name text default null, + p_scope_type text default null, + p_auth_type text default null, + p_outcome text default null, + p_duration_ms integer default null, + p_memory_count integer default null, + p_result_count integer default null, + p_correlation_id text default null, + p_client text default null, + p_kind text default null, + p_host text default null, + p_scope text default null +) +returns uuid +language plpgsql +security definer +set search_path = public +as $$ +declare + v_id uuid; +begin + insert into usage_events ( + user_id, org_id, plan_name, + tool_name, scope_type, auth_type, + outcome, duration_ms, memory_count, + result_count, correlation_id, client, kind, host, + scope + ) values ( + p_user_id, p_org_id, p_plan_name, + p_tool_name, p_scope_type, p_auth_type, + p_outcome, p_duration_ms, p_memory_count, + p_result_count, p_correlation_id, p_client, p_kind, p_host, + p_scope + ) + returning id into v_id; + return v_id; +exception + when others then + -- Never let telemetry writes break the primary operation. + return null; +end; +$$; + +grant execute on function lorekit_record_usage_event(uuid, uuid, text, text, text, text, text, integer, integer, integer, text, text, text, text, text) + to anon, authenticated, service_role; + +-- ── reader: one row per (bucket, scope), plus an optional scope filter ────── +-- DROP first: the RETURN TYPE gains a column, which CREATE OR REPLACE cannot +-- do ("cannot change return type of existing function"), and the parameter list +-- gains a trailing entry. Everything else — the bucket validation, the +-- half-open window, the self-only visibility with the service-role escape +-- hatch, the read-tool list, the dashboard exclusion, the sparse `having` — is +-- 00053/00054's verbatim. +drop function if exists lorekit_read_activity(uuid, text, timestamptz, timestamptz); + +create or replace function lorekit_read_activity( + p_user_id uuid, + p_bucket text default 'day', + p_since timestamptz default null, + p_until timestamptz default null, + p_scope text default null +) +returns table (bucket timestamptz, scope text, count bigint) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; +begin + if p_bucket is null or p_bucket not in ('hour', 'day') then + raise exception 'invalid bucket %, expected hour or day', p_bucket + using errcode = '22023'; + end if; + + return query + select date_trunc(p_bucket, ue.created_at at time zone 'UTC') at time zone 'UTC' as bucket, + ue.scope as scope, + sum(coalesce(ue.result_count, 0))::bigint as count + from usage_events ue + where ( + (v_actor is null and auth.role() = 'service_role') + or ue.user_id = v_actor + ) + and ue.tool_name in ('memory.read', 'memory.list', 'memory.search', + 'memory.list_archived') + -- The dashboard reading lore in order to DRAW this chart is not a read + -- the chart should report. `is distinct from` (not `<>`) because the + -- column is nullable and `null <> 'dashboard'` is null, which would + -- silently drop every unattributed event — including every row written + -- before this migration. + and ue.client is distinct from 'dashboard' + -- The optional per-scope filter. `=`, not `is not distinct from`: a + -- caller asking for a named scope wants events attributed to it, never + -- the unattributable NULL-scope remainder. Omitting the parameter + -- returns every scope INCLUDING those NULL rows, so the unfiltered + -- account total stays complete. + and (p_scope is null or ue.scope = p_scope) + and (p_since is null or ue.created_at >= p_since) + and (p_until is null or ue.created_at < p_until) + group by 1, ue.scope + having sum(coalesce(ue.result_count, 0)) > 0 + -- The scope tiebreak is 00051's verbatim (`order by 1 asc, m.scope asc`). + -- Bucket alone stopped being a total order the moment a bucket could hold + -- several scopes, and this function's header claims it mirrors that shape, + -- so intra-bucket row order must be deterministic here too. + order by 1 asc, ue.scope asc; +end; +$$; + +revoke execute on function lorekit_read_activity(uuid, text, timestamptz, timestamptz, text) from public, anon; +grant execute on function lorekit_read_activity(uuid, text, timestamptz, timestamptz, text) to authenticated, service_role; + +comment on function lorekit_read_activity(uuid, text, timestamptz, timestamptz, text) is + 'Memory RECORDS read (sum of usage_events.result_count over memory.read / + memory.list / memory.search / memory.list_archived — permissions.ts''s + READ_TOOLS) per UTC hour/day AND per scope over the half-open + [p_since, p_until) window, EXCLUDING events attributed to the dashboard + client (00054). One row per (bucket, scope), mirroring + lorekit_memory_activity (00051); scope is NULL for events whose scope could + not be resolved, and those rows are counted in the unfiltered result. + p_scope restricts the result to one exact scope, whose buckets SUM to the + per-scope headline — the metric is additive, so no companion total function + exists to drift from it. Visibility is SELF-ONLY with the same service-role + + NULL escape hatch as lorekit_usage_stats — usage is a per-user ledger and is + never org-shared. p_bucket is validated against (hour, day). Buckets with no + records read are omitted.'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00058', 'usage_event_scope') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00059_memory_seen_count.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- memories.seen_count — RECURRENCE as a real column, counted by the writer. +-- +-- WHY: recurrence is already the load-bearing signal in LoreKit's own guidance. +-- `packages/cli/skill/lorekit-setup/rules/self-improvement-loops.md` gates +-- promotion on `seen_count >= 3`, `lorekit-groom`'s volatile-key rule exists +-- precisely because a per-sighting identifier in the key "never dedups and +-- leaves seen_count frozen at 1", and `lessons-view.mjs`'s LINT_RULES cites the +-- same field. None of it was ever true: the counter lived only as hand-written +-- text inside a lesson body's `` comment, so +-- nothing incremented it, nothing could read it, and every consumer above was +-- describing a column that did not exist. +-- +-- This makes it real, and puts the increment where the recurrence actually +-- happens: the upsert. A write whose (tenant, scope, key) already resolves to a +-- live row IS the second sighting of that lesson — that is the definition the +-- skill documents ("a recurrence resolves to an UPDATE that increments +-- seen_count by 1"), so the RPC counts it rather than trusting a caller to pass +-- a number it would have to read back first. +-- +-- Forward-only and additive. The column is NOT NULL DEFAULT 1 — one sighting is +-- what an existing row has demonstrably had, so the backfill is the default and +-- no data migration is required. `memory_write` keeps its 00056 signature +-- unchanged (no new parameter — the count is derived, never supplied), so every +-- existing caller is untouched and this is a CREATE OR REPLACE rather than the +-- DROP-first dance a signature change forces. +-- +-- No index: no read path filters or orders on seen_count yet. Same call as +-- 00048 made for the origin_* columns — add one with the query that needs it. +-- ═════════════════════════════════════════════════════════════════════════ + +-- ── column on memories ────────────────────────────────────────────────────── +alter table memories add column if not exists seen_count integer not null default 1; + +-- A sighting count is >= 1 by construction: the row exists because it was +-- written at least once. The backstop makes a corrupt/hand-edited 0 or negative +-- value fail loudly here rather than silently sinking a lesson's salience in +-- whatever ranks on this column later. +alter table memories drop constraint if exists memories_seen_count_positive; +alter table memories add constraint memories_seen_count_positive + check (seen_count >= 1); + +-- ── writer: memory_write counts the recurrence ────────────────────────────── +-- Body is 00056's verbatim plus `seen_count` in every branch. The signature is +-- IDENTICAL to 00056's, so CREATE OR REPLACE is safe and leaves no ambiguous +-- overload behind — unlike 00048/00056, this migration adds no parameter. +-- +-- Insert branches set 1 explicitly rather than leaning on the column default, +-- so the first-sighting value is visible in the same statement that the +-- increment is, and the two cannot drift if the default is ever changed. +-- +-- The increment is `memories.seen_count + 1`, NOT `excluded.seen_count + 1`: +-- `excluded` is the row the INSERT proposed, which always carries the literal 1 +-- and would pin every recurrence at 2. +-- +-- Reviving an ARCHIVED key is deliberately NOT a recurrence. Every conflict +-- predicate below is partial on `archived_at is null` (00016), so a write +-- against an archived key inserts a fresh row and starts at 1 — the lesson was +-- retired and is being learned again, which is the honest reading and matches +-- the existing created_at semantics on that path. +create or replace function memory_write( + p_user_id uuid, + p_scope text, + p_key text, + p_value text, + p_tags text[] default '{}', + p_source_agent text default null, + p_trigger text default null, + p_created_at timestamptz default null, + p_org_slug text default null, + p_ttl_seconds integer default null, + p_clear_ttl boolean default false, + p_origin_repo text default null, + p_origin_branch text default null, + p_origin_commit text default null, + p_origin_pr integer default null, + p_kind text default null, + p_host text default null +) +returns table ( + id uuid, + created_at timestamptz, + inserted boolean, + org_routed boolean, + binding_org_slug text, + expires_at timestamptz +) +language plpgsql +set search_path = public +as $$ +declare + v_org_id uuid; + v_binding_org uuid; + v_binding_slug text; + v_expires_at timestamptz; + v_ttl_action text := 'keep'; +begin + if p_clear_ttl then + v_ttl_action := 'clear'; + elsif p_ttl_seconds is not null then + if p_ttl_seconds < 1 or p_ttl_seconds > 31536000 then + raise exception using errcode = 'P0001', + message = format('ttl_seconds must be between 1 and 31536000, got %s', p_ttl_seconds); + end if; + v_expires_at := now() + (p_ttl_seconds * interval '1 second'); + v_ttl_action := 'set'; + end if; + + if p_org_slug is not null then + select o.id into v_org_id from orgs o where o.slug = p_org_slug and o.deleted_at is null; + if v_org_id is null then + raise exception using errcode = 'P0001', message = format('unknown_org: %s', p_org_slug); + end if; + if not lorekit_org_can(p_user_id, v_org_id, 'write') then + raise exception using errcode = 'LK002', message = format('org_permission_denied: org=%s', p_org_slug); + end if; + else + select b.org_id, o.slug into v_binding_org, v_binding_slug + from org_scope_bindings b + join orgs o on o.id = b.org_id + where b.scope = p_scope and o.deleted_at is null; + + if v_binding_org is not null + and p_user_id is not null + and lorekit_org_can(p_user_id, v_binding_org, 'write') then + v_org_id := v_binding_org; + end if; + end if; + + if v_org_id is not null then + return query + insert into memories ( + user_id, org_id, scope, key, value, tags, source_agent, trigger, + created_at, updated_at, created_by, updated_by, expires_at, + origin_repo, origin_branch, origin_commit, origin_pr, kind, host, seen_count + ) + values ( + null, v_org_id, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), + p_user_id, p_user_id, v_expires_at, + p_origin_repo, p_origin_branch, p_origin_commit, p_origin_pr, p_kind, p_host, 1 + ) + on conflict (org_id, scope, key) where org_id is not null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + updated_by = p_user_id, + seen_count = memories.seen_count + 1, + origin_repo = coalesce(excluded.origin_repo, memories.origin_repo), + origin_branch = coalesce(excluded.origin_branch, memories.origin_branch), + origin_commit = coalesce(excluded.origin_commit, memories.origin_commit), + origin_pr = coalesce(excluded.origin_pr, memories.origin_pr), + kind = coalesce(excluded.kind, memories.kind), + host = coalesce(excluded.host, memories.host), + expires_at = case v_ttl_action + when 'clear' then null + when 'set' then v_expires_at + else memories.expires_at + end + returning + memories.id, memories.created_at, (xmax::text = '0') as inserted, + true as org_routed, v_binding_slug as binding_org_slug, memories.expires_at; + + elsif p_user_id is null then + return query + insert into memories ( + user_id, scope, key, value, tags, source_agent, trigger, + created_at, updated_at, created_by, updated_by, expires_at, + origin_repo, origin_branch, origin_commit, origin_pr, kind, host, seen_count + ) + values ( + null, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), + null, null, v_expires_at, + p_origin_repo, p_origin_branch, p_origin_commit, p_origin_pr, p_kind, p_host, 1 + ) + on conflict (scope, key) where org_id is null and user_id is null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + seen_count = memories.seen_count + 1, + origin_repo = coalesce(excluded.origin_repo, memories.origin_repo), + origin_branch = coalesce(excluded.origin_branch, memories.origin_branch), + origin_commit = coalesce(excluded.origin_commit, memories.origin_commit), + origin_pr = coalesce(excluded.origin_pr, memories.origin_pr), + kind = coalesce(excluded.kind, memories.kind), + host = coalesce(excluded.host, memories.host), + expires_at = case v_ttl_action + when 'clear' then null + when 'set' then v_expires_at + else memories.expires_at + end + returning + memories.id, memories.created_at, (xmax::text = '0') as inserted, + false as org_routed, v_binding_slug as binding_org_slug, memories.expires_at; + + else + return query + insert into memories ( + user_id, scope, key, value, tags, source_agent, trigger, + created_at, updated_at, created_by, updated_by, expires_at, + origin_repo, origin_branch, origin_commit, origin_pr, kind, host, seen_count + ) + values ( + p_user_id, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), + p_user_id, p_user_id, v_expires_at, + p_origin_repo, p_origin_branch, p_origin_commit, p_origin_pr, p_kind, p_host, 1 + ) + on conflict (user_id, scope, key) where org_id is null and user_id is not null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + updated_by = p_user_id, + seen_count = memories.seen_count + 1, + origin_repo = coalesce(excluded.origin_repo, memories.origin_repo), + origin_branch = coalesce(excluded.origin_branch, memories.origin_branch), + origin_commit = coalesce(excluded.origin_commit, memories.origin_commit), + origin_pr = coalesce(excluded.origin_pr, memories.origin_pr), + kind = coalesce(excluded.kind, memories.kind), + host = coalesce(excluded.host, memories.host), + expires_at = case v_ttl_action + when 'clear' then null + when 'set' then v_expires_at + else memories.expires_at + end + returning + memories.id, memories.created_at, (xmax::text = '0') as inserted, + false as org_routed, v_binding_slug as binding_org_slug, memories.expires_at; + end if; +end; +$$; + +grant execute on function memory_write(uuid, text, text, text, text[], text, text, timestamptz, text, integer, boolean, text, text, text, integer, text, text) + to anon, authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00059', 'memory_seen_count') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00060_memory_embeddings.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- memories.embedding — the semantic-search column, landed DORMANT. +-- +-- WHY SEPARATELY, AND WHY EMPTY: nothing reads or writes this column in this +-- migration, and that is the entire point. The embedding arc has three +-- independent risks — a schema change, a provider dependency with a real cost +-- and latency, and a ranking-fusion tuning problem — and bundling them makes +-- the first impossible to review and the last impossible to abandon. Landing +-- the schema alone means the write pipeline (which needs an API key, a backfill +-- and a budget) and the search modes (which need recall tuning) can each be +-- decided on their own evidence, and either can be dropped without unpicking a +-- migration. +-- +-- After this migration the database is byte-for-byte equivalent in behaviour: +-- every existing row has `embedding is null`, no handler selects the column, no +-- RPC writes it, and `MEMORY_SELECT` does not name it. The only observable +-- change is that `pgvector` is enabled and two nullable columns exist. +-- +-- ── The two choices a reviewer should actually check ────────────────────── +-- +-- DIMENSION 1536, and it is a constraint rather than a preference. pgvector's +-- HNSW index refuses more than 2000 dimensions on the `vector` type, so a +-- 3072-dimension model (OpenAI `text-embedding-3-large`, Gemini's large +-- embedding) cannot be ANN-indexed here at all without switching the column to +-- `halfvec` and halving precision. 1536 is `text-embedding-3-small`'s native +-- width, sits comfortably under the ceiling, and is the width every mainstream +-- provider can produce — several support Matryoshka truncation to it, so +-- choosing 1536 does NOT lock the pipeline to one vendor. Changing this later +-- is a table rewrite, which is exactly why it is being decided now, in +-- isolation, rather than discovered during the pipeline PR. +-- +-- HNSW, NOT IVFFLAT, and the emptiness is the reason. An IVFFlat index +-- partitions the vector space by clustering the rows that exist AT BUILD TIME; +-- built on an empty table it produces meaningless centroids and has to be +-- dropped and rebuilt once data lands — which is a second migration and an +-- outage-shaped operation on a large table. HNSW builds a graph incrementally +-- as rows arrive, so an index created on zero rows is correct and stays correct. +-- Given this column is deliberately empty today, IVFFlat would be the wrong +-- structure by construction. +-- +-- COSINE (`vector_cosine_ops`) because text embeddings from every provider +-- worth using are L2-normalised, where cosine distance and inner product rank +-- identically and cosine is the one that stays correct if a future provider is +-- not normalised. +-- +-- PARTIAL (`where embedding is not null`) so the index describes only rows that +-- have one. Today that is none, so the index costs nothing; during the backfill +-- it grows with coverage instead of being sized for the whole table up front. +-- ═════════════════════════════════════════════════════════════════════════ + +-- Supabase provisions `vector` in the `extensions` schema on hosted projects +-- and `if not exists` makes this a no-op there; locally it is what installs it. +-- Same shape as `pg_trgm` in 00012 — no schema qualification, so it resolves +-- wherever the project already puts extensions. +create extension if not exists vector; + +-- ── the column ────────────────────────────────────────────────────────────── +-- Nullable with no default: null means "not embedded yet", which is the state +-- every existing row is in and the state the backfill queries for. A default +-- would make "never embedded" indistinguishable from "embedded as zeroes". +alter table memories add column if not exists embedding vector(1536); + +-- WHICH MODEL produced the vector. Not decoration: embeddings from two models +-- are not comparable, so a silent provider swap would leave the table holding +-- two incompatible vector spaces and a similarity search would return confident +-- nonsense across the boundary. Recording it is what makes a re-embed +-- targetable (`where embedding_model <> $current`) instead of a full rebuild, +-- and it is far cheaper to add now, on an empty column, than after a backfill. +alter table memories add column if not exists embedding_model text; + +-- Length backstop only — the vocabulary is the pipeline's to police, exactly as +-- 00056 argued for `kind`/`host`. This bounds storage and cardinality without +-- turning "support another provider" into a migration. +alter table memories drop constraint if exists memories_embedding_model_len; +alter table memories add constraint memories_embedding_model_len + check (embedding_model is null or (char_length(embedding_model) between 1 and 128)); + +-- A vector without a model is unattributable, and a model without a vector is a +-- lie about what the row holds. Both-or-neither, enforced here rather than +-- trusted to the writer, because the failure it prevents is silent. +alter table memories drop constraint if exists memories_embedding_model_pairing; +alter table memories add constraint memories_embedding_model_pairing + check ((embedding is null) = (embedding_model is null)); + +-- ── the ANN index ─────────────────────────────────────────────────────────── +-- `m` and `ef_construction` are pgvector's defaults, stated explicitly so the +-- build parameters are visible in the schema rather than inherited from +-- whatever version happens to be installed. Recall tuning belongs with the +-- search modes, where it can be measured. +create index if not exists memories_embedding_hnsw_idx + on memories using hnsw (embedding vector_cosine_ops) + with (m = 16, ef_construction = 64) + where embedding is not null; + +-- Coverage lookup for the backfill: "which rows still need embedding", newest +-- first. Without it a resumable backfill sequentially scans the whole table on +-- every batch, which is the shape that makes a backfill quadratic. +create index if not exists memories_embedding_pending_idx + on memories (created_at desc) + where embedding is null; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00060', 'memory_embeddings') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00061_memories_keyset_index.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- Give the memories list seek its `id` tiebreaker. +-- +-- `memories_scope_updated_at_idx` (00033) was designed for the query toolList +-- ran AT THE TIME: +-- +-- … WHERE scope = ? AND archived_at IS NULL +-- ORDER BY updated_at DESC LIMIT ? +-- +-- so `(scope, updated_at desc)` satisfied the order exactly. Keyset pagination +-- landed afterwards and changed both halves of that query. It now orders by a +-- COMPOSITE key and seeks on it: +-- +-- … WHERE scope = ? AND archived_at IS NULL +-- AND (updated_at < ?ts OR (updated_at = ?ts AND id < ?id)) +-- ORDER BY updated_at DESC, id DESC LIMIT ? +-- +-- (`supabase/functions/mcp/tools.ts` toolList / `memories/handlers/list.ts`, +-- predicate built by `_shared/api/paginate.ts` + `mcp/cursor.ts`.) +-- +-- The existing index stops at `updated_at`, so `id` is not available to the +-- scan: the tiebreaker becomes a heap recheck on every candidate row, and the +-- planner cannot stop early at LIMIT when a timestamp is not unique. This is +-- the same omission 00012 fixed for `audit_log` — 00010's +-- `(user_id, created_at desc)` lacked the same `id` column, for the same +-- keyset predicate. +-- +-- The fix here is 00012's fix with ONE deliberate difference: 00012 built +-- `(user_id, created_at desc, id)`, with `id` ASC. The seek it serves orders by +-- `created_at DESC, id DESC`, and a (DESC, ASC) index yields that pair in +-- neither scan direction, so 00012 recovered the index-only tiebreaker lookup +-- but not the ordered scan. `id desc` below matches the ORDER BY exactly. The +-- corresponding correction on `audit_log_user_created_id_idx` is a separate, +-- separately-measured migration on a different table and is not made here. +-- +-- Cost is concentrated where it is felt most: deep pagination over a large +-- scope, which is the shape the busiest caller actually has (37% of list calls +-- in a recent 24h window came back with `has_more`, and the deepest walk paged +-- a single scope ~a dozen times at 543–765 ms per page). +-- +-- The partial predicate is carried over verbatim from 00033 so the two indexes +-- describe the same row population. The expiry filter +-- (`expires_at IS NULL OR expires_at > now()`) is deliberately NOT in the +-- predicate: `now()` is volatile and PostgreSQL rejects it in an index +-- predicate — the same reason 00033 gave for adding no `rate_limit_counters` +-- index. +-- +-- Forward-only and additive. `IF NOT EXISTS` so it can be applied to a running +-- production database (`CONCURRENTLY` is not used inside migrations because it +-- cannot run in a transaction). +-- +-- LOCK WINDOW: because the build is not `CONCURRENTLY`, it takes a SHARE lock +-- on `memories` for the duration — reads continue, writes BLOCK until the +-- index is built. `IF NOT EXISTS` makes re-application free but does not +-- shorten the first build. Sizing that window is a deploy-time call on the +-- live table, not something this file can assert; if it is ever too long for +-- the write path, the escape hatch is to build the index out-of-band with +-- `CREATE INDEX CONCURRENTLY` before deploying, after which this migration +-- is a no-op. +-- +-- SCOPE: `sort=updated_at` only. `GET /memories` also accepts +-- `sort=created_at`, whose keyset seeks `(created_at, id)` under the same +-- `scope` filter, and no index covers that pair either. That page is +-- deliberately out of scope here: the measured cost is on the default sort +-- (`memory.list` is the most-called MCP tool and always sorts by `updated_at`), +-- and every index on `memories` is write amplification on the hottest table in +-- the product, so the second one should be justified by its own measurement +-- rather than added on symmetry. Tracked as a follow-up, not fixed here. +-- +-- The superseded `memories_scope_updated_at_idx` is deliberately LEFT IN PLACE. +-- Dropping an index in the same migration that adds its replacement gives the +-- planner no fallback if the new one is not yet warm, and this is the hottest +-- read path in the product; retiring it is a separate, observable decision. + +create index if not exists memories_scope_updated_at_id_idx + on memories (scope, updated_at desc, id desc) + where archived_at is null; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00061', 'memories_keyset_index') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00062_memory_embedding_write.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- lorekit_memory_set_embedding — the ONE authorised path for writing a +-- memory's vector. +-- +-- ── WHY THIS EXISTS ────────────────────────────────────────────────────── +-- The write-path embedder (`_shared/embedding/embed-on-write.ts`) updated `memories` +-- directly, with whatever client the request happened to arrive on. That is +-- correct on two of the three auth tiers and SILENTLY wrong on the third: +-- +-- * `service` and `api_key` both hold a service-role client, and `rls_update` +-- (00001) admits `auth.role() = 'service_role'`, so the update lands. +-- +-- * a Supabase JWT holds an RLS-scoped client, and `rls_update` admits only +-- `user_id = auth.uid()`. The READ policies were widened for orgs in 00015 +-- (`org_id in (select lorekit_member_org_ids(auth.uid()))`); `rls_update` +-- never was. An ORG-OWNED memory carries `user_id is null` (00019), so the +-- UPDATE matched ZERO ROWS — and a zero-row update is not an error in +-- PostgREST. The embedding was dropped with no signal anywhere, and the +-- row waited for a manual backfill that an operator had no reason to run. +-- +-- The asymmetry is the whole bug: SELECT knows about orgs, UPDATE does not. +-- +-- ── WHY NOT SIMPLY WIDEN `rls_update` ──────────────────────────────────── +-- Making UPDATE symmetric with SELECT would fix the embed and open a hole: +-- every org MEMBER could then rewrite any column of any org memory straight +-- through PostgREST, with no role check at all. Org writes are gated on +-- `lorekit_org_can(…, 'write')` — a VIEWER must not write — and that gate lives +-- in RPCs precisely because RLS cannot express it. Widening the policy would +-- silently demote a role check to a membership check, on every column, for +-- every caller. The embedding is not worth that. +-- +-- ── WHY NOT MIRROR `memory_write`'s BRANCHING IN THE EDGE ──────────────── +-- Two copies of an ownership rule drift, and the copy in TypeScript is the one +-- no migration test ever runs. 00019's branching would have to be restated in +-- Deno and kept in step by review alone. +-- +-- ── WHAT THIS DOES INSTEAD ─────────────────────────────────────────────── +-- One SECURITY DEFINER function — the shape 00022/00041 established for every +-- org state transition — with authorisation derived INSIDE it from the same +-- predicates the rest of the schema already uses: +-- +-- * `lorekit_org_actor` (00041) resolves WHO is acting: a service-role +-- caller may name the user it acts for, a JWT caller is only ever itself. +-- * `lorekit_org_can` (00017) is the ROLE gate for an org-owned row — +-- the same 'write' capability `memory_write` demanded to create it. +-- * personal rows compare against the resolved actor directly. +-- +-- Nothing here re-implements an ownership rule; it composes the existing ones. +-- If the ownership model changes, it changes in those functions and this +-- inherits it. +-- +-- ── IT REPORTS WHETHER IT WROTE, AND THAT IS THE OTHER HALF OF THE FIX ─── +-- The original defect was not merely that the update failed — it is that it +-- failed INVISIBLY. This reports whether a row was actually written so the +-- caller can record a miss on its span. If some future ownership model escapes +-- these predicates, it surfaces as a signal instead of an empty column nobody +-- notices until a semantic search quietly comes back thin. +-- +-- `returns table (written boolean)` rather than a bare `returns boolean`: it is +-- the shape `memory_delete` (00020) already uses for a did-it-land answer, and +-- the one the edge's traced client types honestly — `TracedQuery` resolves to +-- `PostgrestResponse`, so a scalar-returning function would have to be read +-- through an `unknown` cast at every call site. +-- +-- ── search_path NAMES `extensions` ON PURPOSE ──────────────────────────── +-- Every other function in this schema pins `set search_path = public`. This one +-- cannot: it casts to `vector(1536)`, and Supabase provisions `vector` in the +-- `extensions` schema on hosted projects (see 00060's header). Pinning `public` +-- alone would resolve locally, where the extension lands in the search path +-- already, and fail in production with `type "vector" does not exist` — inside a +-- backgrounded task whose errors are swallowed by design. Naming both schemas +-- is what makes the local run and the hosted run the same run. +-- ═════════════════════════════════════════════════════════════════════════ + +-- ═════════════════════════════════════════════════════════════════════════ +-- PART 1 — an embedding write must not disturb `updated_at`. +-- +-- `memories_updated_at` (00001) is a BEFORE UPDATE trigger that sets +-- `updated_at = now()` unconditionally. That is right for an edit and WRONG for +-- an embedding: the vector is a DERIVED artefact, not a change to what the +-- lesson says. `updated_at` is the recency signal `POST /memories/search` and +-- `GET /memories/relevant` order by, that `memory.list` keysets on +-- (`memories_scope_updated_at_id_idx`), and that `lesson-rank` scores. +-- +-- On the write path the damage is nil — the row was created moments earlier. +-- A WHOLE-STORE BACKFILL is the problem: it walks `created_at desc` and would +-- restamp every row in that order, collapsing the store's real recency ordering +-- into the order the backfill happened to run in. The old values are not +-- recoverable afterwards, so this is a one-way corruption of the ranking signal +-- that nothing would report. +-- +-- FIXED AT THE STORAGE LAYER, not at the two call sites. Both the RPC below and +-- the backfill's PostgREST PATCH trip the same trigger, and so would any future +-- writer — a rule each caller has to remember is a rule that gets forgotten +-- exactly once. +-- +-- `set_updated_at` itself is LEFT ALONE: five other tables share it +-- (`user_limits`, `orgs`, `org_limits`, `plans`, `user_plans`), and none of them +-- has an embedding column or this problem. Only the `memories` trigger is +-- retargeted. +-- +-- The predicate is "did anything OTHER than the embedding columns change?", +-- asked by masking those columns plus `updated_at` and comparing the whole rows. +-- Column-complete by construction: a column added to `memories` later is +-- included automatically, where an explicit list would silently stop covering it. +-- ═════════════════════════════════════════════════════════════════════════ + +create or replace function lorekit_memories_set_updated_at() +returns trigger +language plpgsql +set search_path = public, extensions +as $$ +declare + v_new memories := new; + v_old memories := old; + v_embedding_changed boolean; +begin + -- The embedding columns must ACTUALLY have changed for this to be an embedding + -- write. Without this precondition the rule reads "nothing meaningful changed + -- → preserve", which also catches a plain NO-OP re-write — and `memory_write` + -- upserts, so an agent re-saving an identical lesson would silently stop + -- refreshing `updated_at`. That is a recency-contract change reaching well + -- beyond embeddings, in a migration whose whole point is that a DERIVED column + -- must not disturb the row. Preserve only when a vector actually moved; + -- everything else, including a no-op, keeps the behaviour `set_updated_at` + -- always had. + v_embedding_changed := + new.embedding is distinct from old.embedding + or new.embedding_model is distinct from old.embedding_model; + + -- Mask the derived columns and the one being decided. + -- + -- `fts` MUST be masked too, and for a reason that is easy to get wrong: + -- it is a GENERATED column, and Postgres computes generated columns AFTER + -- BEFORE-row triggers run. So `new.fts` is NULL here while `old.fts` holds + -- the stored value, and an unmasked comparison finds every update "changed" + -- — which silently turns this whole function back into `set_updated_at`. + -- It fails in the safe direction (a bump that should not happen, never a + -- missing bump), which is exactly why it needs a test rather than a reading. + -- `fts` is derived from `key || ' ' || value`, so masking it loses nothing: + -- a real edit is already visible in those columns. + v_new.embedding := null; v_new.embedding_model := null; v_new.updated_at := null; v_new.fts := null; + v_old.embedding := null; v_old.embedding_model := null; v_old.updated_at := null; v_old.fts := null; + + if v_embedding_changed and v_new is not distinct from v_old then + -- A vector moved and nothing else did: an embedding write. Preserve the + -- row's real recency rather than restamping it. + new.updated_at := old.updated_at; + else + new.updated_at := now(); + end if; + + return new; +end; +$$; + +create or replace trigger memories_updated_at + before update on memories + for each row execute function lorekit_memories_set_updated_at(); + +comment on function lorekit_memories_set_updated_at() is + 'BEFORE UPDATE on memories: bumps updated_at like set_updated_at, EXCEPT when the only change is ' + 'the embedding columns — a derived artefact must not rewrite the recency signal that search, ' + 'relevant, the keyset index and lesson-rank all read. A whole-store backfill would otherwise ' + 'restamp every row in created_at desc order, irrecoverably.'; + +-- ═════════════════════════════════════════════════════════════════════════ +-- PART 2 — the authorised write path itself. +-- ═════════════════════════════════════════════════════════════════════════ + +-- Dropped first for the reason 00019/00009 give: `create or replace` cannot +-- change a function's RETURN TYPE, so amending this later (a third output +-- column, say) would fail against a database that already has it. Forward-only. +drop function if exists lorekit_memory_set_embedding(uuid, uuid, text, text); + +create function lorekit_memory_set_embedding( + p_memory_id uuid, + p_actor_user_id uuid, + p_embedding text, + p_model text +) +returns table (written boolean) +language plpgsql +volatile +security definer +set search_path = public, extensions +as $$ +declare + v_actor uuid; + v_is_svc boolean; + v_user_id uuid; + v_org_id uuid; + v_rows integer; +begin + if p_memory_id is null then + return query select false; + return; + end if; + + -- Both-or-neither, per the 00060 CHECK. Refused here so the caller gets a + -- named reason rather than an opaque constraint violation surfacing from a + -- background task. `embedOnWrite` always sends both; this is the guard for + -- the next caller. + if (p_embedding is null) <> (p_model is null) then + raise exception using + errcode = 'P0001', + message = 'embedding and embedding_model must be set together (00060 pairing)'; + end if; + + v_actor := lorekit_org_actor(p_actor_user_id); + v_is_svc := auth.role() = 'service_role'; + + select m.user_id, m.org_id into v_user_id, v_org_id + from memories m + where m.id = p_memory_id; + + -- No such row (deleted between the write and this call). Not an error: the + -- caller's contract is "never fail a memory write because of an embedding". + if not found then + return query select false; + return; + end if; + + if v_org_id is not null then + -- Org-owned. Membership alone is NOT enough — the same 'write' capability + -- 00019 required to create the row is required to embed it. A null actor + -- fails closed here (`lorekit_org_role(null, …)` is null), so the service + -- tier is admitted by its own branch rather than by an accident of NULL + -- comparison. + if not (v_is_svc and v_actor is null) + and not lorekit_org_can(v_actor, v_org_id, 'write') then + return query select false; + return; + end if; + + elsif v_user_id is not null then + -- Personal. `is distinct from` rather than `<>` so a null actor is a + -- mismatch instead of an unknown that falls through. + if v_actor is distinct from v_user_id then + return query select false; + return; + end if; + + else + -- Neither user- nor org-owned: written by the service tier with + -- `p_user_id => null`. Only the service tier may touch it. + if not v_is_svc then + return query select false; + return; + end if; + end if; + + update memories + set embedding = p_embedding::vector(1536), + embedding_model = p_model + where id = p_memory_id; + + get diagnostics v_rows = row_count; + return query select v_rows > 0; +end; +$$; + +-- THE REVOKE IS LOAD-BEARING, not belt-and-braces: Postgres grants EXECUTE on a +-- newly created function to PUBLIC by default, and `anon` inherits it. Naming +-- `authenticated, service_role` in the GRANT alone would NOT withhold it — see +-- the same note in 00041, where `migrations.test.sql`'s anon assertion caught +-- exactly this. +revoke execute on function lorekit_memory_set_embedding(uuid, uuid, text, text) from public; +grant execute on function lorekit_memory_set_embedding(uuid, uuid, text, text) + to authenticated, service_role; + +comment on function lorekit_memory_set_embedding(uuid, uuid, text, text) is + 'Writes memories.embedding + embedding_model for one row, authorised inside the function ' + '(lorekit_org_actor + lorekit_org_can for org-owned rows, actor identity for personal rows). ' + 'Returns true when a row was written, false when none matched — the write path records that ' + 'miss on its span rather than dropping the embedding silently, which is what rls_update did ' + 'for org-owned memories before 00062.'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00062', 'memory_embedding_write') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00063_activity_filters.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- lorekit_memory_activity — filter-aware (scope + dimension filters). +-- +-- WHAT CHANGES: 00051's activity counted every visible memory in the window, +-- ignoring the Explorer's filter bar. The Explorer's stat header follows a +-- scope selection AND the dimension filters (label / agent / trigger / kind / +-- host / repo / branch / PR), and its numbers have to agree with the LIST under +-- them — the list already applies exactly these predicates (GET /memories). So +-- the written/scopes cards must count the same set the list shows. +-- +-- HOW: the SAME per-dimension predicate as GET /memories and lorekit_memory_facets +-- (00057) — "OR within a dimension, AND across dimensions", each `*_mode` +-- choosing `in`/`nin` (and `tags` also `all`/`none`). Unlike the facets RPC there +-- is NO self-exclusion here: a straight count applies every filter directly, so +-- the CTE-of-flags collapses to a flat WHERE. `scope` is a hard filter (00057's +-- treatment). Every parameter is optional and defaulted, so a caller passing +-- only (p_user_id, p_bucket, p_since, p_until) gets byte-for-byte 00051's result. +-- +-- Read-activity is deliberately NOT widened here: usage_events carries no +-- per-memory tag/agent/repo (only `scope`, 00058), so a dimension filter is +-- unanswerable for reads — the Read card stays scope-level by design. +-- +-- Forward-only. The function is DROPped and recreated with a wider signature; +-- the handler is updated in lockstep. Bucket validation, the half-open +-- [p_since, p_until) window, the active-only + tenant predicate, and the +-- date_trunc anchoring are 00051 verbatim. +-- ═════════════════════════════════════════════════════════════════════════ + +drop function if exists lorekit_memory_activity(uuid, text, timestamptz, timestamptz); + +create or replace function lorekit_memory_activity( + p_user_id uuid, + p_bucket text default 'day', + p_since timestamptz default null, + p_until timestamptz default null, + -- Active filters, mirroring GET /memories' query params so the stat header can + -- pass its filter bar verbatim. All optional: null/absent = not filtered. + p_scope text default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in' +) +returns table (bucket timestamptz, scope text, count bigint) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + -- `origin_pr` is an INTEGER column: coerce the digits-only list numerically so + -- this route matches GET /memories on the same input (00057's rationale). A + -- non-numeric entry is dropped; a list reducing to empty applies no filter. + v_origin_pr integer[] := ( + select array_agg(x::integer) + from unnest(coalesce(p_origin_pr, '{}'::text[])) as x + where x ~ '^[0-9]+$' + ); +begin + if p_bucket is null or p_bucket not in ('hour', 'day') then + raise exception 'invalid bucket %, expected hour or day', p_bucket + using errcode = '22023'; + end if; + + return query + select date_trunc(p_bucket, m.created_at at time zone 'UTC') at time zone 'UTC' as bucket, + m.scope, + count(*) as count + from memories m + where ( + (v_actor is null and auth.role() = 'service_role') + or m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) + ) + and m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + and (p_since is null or m.created_at >= p_since) + and (p_until is null or m.created_at < p_until) + -- Scope is a hard filter, always applied (00057's treatment). + and (p_scope is null or m.scope = p_scope) + -- Dimension filters — every one applied (no self-exclusion for a count). + and (p_tags is null or case coalesce(p_tags_mode, 'any') + when 'all' then m.tags @> p_tags + when 'none' then not (m.tags && p_tags) + else m.tags && p_tags + end) + and (p_source_agent is null or case coalesce(p_source_agent_mode, 'in') + when 'nin' then (m.source_agent is not null and m.source_agent <> all(p_source_agent)) + else m.source_agent = any(p_source_agent) + end) + and (p_trigger is null or case coalesce(p_trigger_mode, 'in') + when 'nin' then (m.trigger is not null and m.trigger <> all(p_trigger)) + else m.trigger = any(p_trigger) + end) + and (p_kind is null or case coalesce(p_kind_mode, 'in') + when 'nin' then (m.kind is not null and m.kind <> all(p_kind)) + else m.kind = any(p_kind) + end) + and (p_host is null or case coalesce(p_host_mode, 'in') + when 'nin' then (m.host is not null and m.host <> all(p_host)) + else m.host = any(p_host) + end) + and (p_origin_repo is null or case coalesce(p_origin_repo_mode, 'in') + when 'nin' then (m.origin_repo is not null and m.origin_repo <> all(p_origin_repo)) + else m.origin_repo = any(p_origin_repo) + end) + and (p_origin_branch is null or case coalesce(p_origin_branch_mode, 'in') + when 'nin' then (m.origin_branch is not null and m.origin_branch <> all(p_origin_branch)) + else m.origin_branch = any(p_origin_branch) + end) + and (v_origin_pr is null or case coalesce(p_origin_pr_mode, 'in') + when 'nin' then (m.origin_pr is not null and m.origin_pr <> all(v_origin_pr)) + else m.origin_pr = any(v_origin_pr) + end) + group by 1, m.scope + order by 1 asc, m.scope asc; +end; +$$; + +revoke execute on function lorekit_memory_activity( + uuid, text, timestamptz, timestamptz, text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text +) from public, anon; +grant execute on function lorekit_memory_activity( + uuid, text, timestamptz, timestamptz, text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text +) to authenticated, service_role; + +comment on function lorekit_memory_activity( + uuid, text, timestamptz, timestamptz, text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text +) is + 'Memories created per UTC hour/day per scope over the half-open + [p_since, p_until) window, visible to the EFFECTIVE caller, narrowed by the + optional scope + dimension filters (00063) — the SAME predicate as + GET /memories and lorekit_memory_facets, so the stat header agrees with the + list. With no filters supplied the result is byte-for-byte 00051''s.'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00063', 'activity_filters') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00064_owner_filter.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- Owner as a first-class filter DIMENSION (00064). +-- +-- WHAT CHANGES: ownership (personal vs a shared org) was the ONE Explorer +-- filter narrowed CLIENT-side — a separate `OwnershipFilterBar` over the loaded +-- pages, so it could not agree with the server-filtered list, the drill-down +-- facet counts, or the stat header, and it silently missed rows past the page +-- the browser happened to hold. This folds it into the SAME "OR within a +-- dimension, AND across dimensions" machinery every other dimension already +-- uses (00057 facets, 00063 activity, GET /memories), as a new `owner` facet. +-- +-- THE OWNER IDENTITY of a row is `personal` when `org_id is null`, else the +-- owning org's SLUG (stable, unlike its uuid or its display name). It is +-- resolved by a LEFT JOIN to `orgs`; a personal row has no org and takes the +-- literal `personal`. Slugs resolve against the caller's member orgs ONLY — +-- which the visibility predicate already guarantees, since an org-owned row is +-- visible solely through the `lorekit_member_org_ids` branch, so its joined +-- slug is always one of the caller's own orgs. +-- +-- THE OWNER PREDICATE (mirrored byte-for-byte in both functions and in the +-- GET /memories handler): +-- in → ('personal' = any(p_owner) AND org_id is null) +-- OR (org_id is not null AND o.slug = any(p_owner)) +-- nin → identity is not null AND identity <> all(p_owner) -- identity above +-- The `nin` guard mirrors every other scalar dimension's `(col is not null AND +-- col <> all(...))` shape; the computed identity is never null in practice +-- (personal → 'personal', visible org row → its slug), but the guard keeps the +-- three-valued-logic behaviour identical to its neighbours. +-- +-- SELF-EXCLUSION, as for every drill-down dimension in 00057: the `owner` facet +-- cell applies every OTHER dimension's flag but NOT `ok_owner`, so standing on +-- `owner=personal` still lists the org you could switch to; and every OTHER +-- dimension's cell now additionally requires `ok_owner`, so picking an owner +-- narrows their counts to the list's set. +-- +-- Forward-only. Both functions are DROPped at their 00057 / 00063 signatures +-- and recreated with `p_owner text[]` / `p_owner_mode text` APPENDED at the end +-- (defaulted, so a positional caller passing only the earlier args is +-- unaffected, and a bare call is byte-for-byte the prior behaviour). The DROP of +-- the exact prior signature is deliberate — leaving it in place would make the +-- named-arg handler calls ambiguous between two overloads. +-- ═════════════════════════════════════════════════════════════════════════ + +-- ── lorekit_memory_facets ──────────────────────────────────────────────────── + +drop function if exists lorekit_memory_facets( + uuid, boolean, text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text +); + +create or replace function lorekit_memory_facets( + p_user_id uuid, + p_archived boolean default false, + p_scope text default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in', + -- Owner dimension (00064). `personal` plus one slug per member org with + -- visible rows. All optional: null/absent = not filtered. + p_owner text[] default null, + p_owner_mode text default 'in' +) +returns table (facet text, value text, count bigint) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + v_origin_pr integer[] := ( + select array_agg(x::integer) + from unnest(coalesce(p_origin_pr, '{}'::text[])) as x + where x ~ '^[0-9]+$' + ); +begin + return query + with base as ( + select + m.tags, m.source_agent, m.trigger, m.kind, m.host, + m.origin_repo, m.origin_branch, m.origin_pr, + m.org_id, o.slug as org_slug, + (p_tags is null or case coalesce(p_tags_mode, 'any') + when 'all' then m.tags @> p_tags + when 'none' then not (m.tags && p_tags) + else m.tags && p_tags + end) as ok_tag, + (p_source_agent is null or case coalesce(p_source_agent_mode, 'in') + when 'nin' then (m.source_agent is not null and m.source_agent <> all(p_source_agent)) + else m.source_agent = any(p_source_agent) + end) as ok_source_agent, + (p_trigger is null or case coalesce(p_trigger_mode, 'in') + when 'nin' then (m.trigger is not null and m.trigger <> all(p_trigger)) + else m.trigger = any(p_trigger) + end) as ok_trigger, + (p_kind is null or case coalesce(p_kind_mode, 'in') + when 'nin' then (m.kind is not null and m.kind <> all(p_kind)) + else m.kind = any(p_kind) + end) as ok_kind, + (p_host is null or case coalesce(p_host_mode, 'in') + when 'nin' then (m.host is not null and m.host <> all(p_host)) + else m.host = any(p_host) + end) as ok_host, + (p_origin_repo is null or case coalesce(p_origin_repo_mode, 'in') + when 'nin' then (m.origin_repo is not null and m.origin_repo <> all(p_origin_repo)) + else m.origin_repo = any(p_origin_repo) + end) as ok_origin_repo, + (p_origin_branch is null or case coalesce(p_origin_branch_mode, 'in') + when 'nin' then (m.origin_branch is not null and m.origin_branch <> all(p_origin_branch)) + else m.origin_branch = any(p_origin_branch) + end) as ok_origin_branch, + (v_origin_pr is null or case coalesce(p_origin_pr_mode, 'in') + when 'nin' then (m.origin_pr is not null and m.origin_pr <> all(v_origin_pr)) + else m.origin_pr = any(v_origin_pr) + end) as ok_origin_pr, + -- Owner: the computed identity is `personal` (org_id null) or the org slug. + (p_owner is null or case coalesce(p_owner_mode, 'in') + when 'nin' then ( + (case when m.org_id is null then 'personal' else o.slug end) is not null + and (case when m.org_id is null then 'personal' else o.slug end) <> all(p_owner) + ) + else ( + ('personal' = any(p_owner) and m.org_id is null) + or (m.org_id is not null and o.slug = any(p_owner)) + ) + end) as ok_owner + from memories m + -- A personal row has no org, so this is a LEFT join; org rows resolve to + -- their slug. Visible org rows are always the caller's own orgs (the + -- visibility predicate below admits them only via lorekit_member_org_ids), + -- so `o.slug` is never a slug the caller cannot see. + left join orgs o on o.id = m.org_id + where ( + (v_actor is null and auth.role() = 'service_role') + or m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) + ) + and ( + case + when p_archived then m.archived_at is not null + else m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + end + ) + and (p_scope is null or m.scope = p_scope) + ), cells as ( + select 'tag'::text as facet, t.tag as value + from base b + cross join lateral unnest(b.tags) as t(tag) + where b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner + union all + select 'source_agent', b.source_agent from base b + where b.ok_tag and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner + union all + select 'trigger', b.trigger from base b + where b.ok_tag and b.ok_source_agent and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner + union all + select 'kind', b.kind from base b + where b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner + union all + select 'host', b.host from base b + where b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner + union all + select 'origin_repo', b.origin_repo from base b + where b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner + union all + select 'origin_branch', b.origin_branch from base b + where b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_pr and b.ok_owner + union all + select 'origin_pr', b.origin_pr::text from base b + where b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_owner + union all + -- Owner is the ONE dimension self-excluded here (every other flag, NOT + -- ok_owner), so a drilled-in owner still lists the alternative owner. + select 'owner', case when b.org_id is null then 'personal' else b.org_slug end from base b + where b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr + ) + select c.facet, c.value, count(*) as count + from cells c + where c.value is not null + and btrim(c.value) <> '' + group by c.facet, c.value + order by c.facet asc, count(*) desc, c.value asc; +end; +$$; + +revoke execute on function lorekit_memory_facets( + uuid, boolean, text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text +) from public, anon; +grant execute on function lorekit_memory_facets( + uuid, boolean, text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text +) to authenticated, service_role; + +comment on function lorekit_memory_facets( + uuid, boolean, text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text +) is + 'Value catalog with counts for every filterable memory dimension (tag, + source_agent, trigger, kind, host, origin_repo, origin_branch, origin_pr, + owner) over the partition selected by p_archived, visible to the EFFECTIVE + caller. `owner` (00064) is `personal` for org_id-null rows, else the org + slug. Counts are DRILL-DOWN (00057): each dimension is counted with every + OTHER active filter applied but not its own (self-exclusion), so the numbers + match what adding that value would yield. With no filters supplied the result + is 00052''s global catalog plus the owner dimension. Same service-role-gated + actor rule and ordering as lorekit_memory_tags.'; + +-- ── lorekit_memory_activity ────────────────────────────────────────────────── + +drop function if exists lorekit_memory_activity( + uuid, text, timestamptz, timestamptz, text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text +); + +create or replace function lorekit_memory_activity( + p_user_id uuid, + p_bucket text default 'day', + p_since timestamptz default null, + p_until timestamptz default null, + p_scope text default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in', + -- Owner dimension (00064) — the SAME predicate as lorekit_memory_facets and + -- GET /memories, so the stat header agrees with the list under an owner pill. + p_owner text[] default null, + p_owner_mode text default 'in' +) +returns table (bucket timestamptz, scope text, count bigint) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + v_origin_pr integer[] := ( + select array_agg(x::integer) + from unnest(coalesce(p_origin_pr, '{}'::text[])) as x + where x ~ '^[0-9]+$' + ); +begin + if p_bucket is null or p_bucket not in ('hour', 'day') then + raise exception 'invalid bucket %, expected hour or day', p_bucket + using errcode = '22023'; + end if; + + return query + select date_trunc(p_bucket, m.created_at at time zone 'UTC') at time zone 'UTC' as bucket, + m.scope, + count(*) as count + from memories m + -- LEFT join for the owner predicate — a personal row has no org (00064). + left join orgs o on o.id = m.org_id + where ( + (v_actor is null and auth.role() = 'service_role') + or m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) + ) + and m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + and (p_since is null or m.created_at >= p_since) + and (p_until is null or m.created_at < p_until) + and (p_scope is null or m.scope = p_scope) + and (p_tags is null or case coalesce(p_tags_mode, 'any') + when 'all' then m.tags @> p_tags + when 'none' then not (m.tags && p_tags) + else m.tags && p_tags + end) + and (p_source_agent is null or case coalesce(p_source_agent_mode, 'in') + when 'nin' then (m.source_agent is not null and m.source_agent <> all(p_source_agent)) + else m.source_agent = any(p_source_agent) + end) + and (p_trigger is null or case coalesce(p_trigger_mode, 'in') + when 'nin' then (m.trigger is not null and m.trigger <> all(p_trigger)) + else m.trigger = any(p_trigger) + end) + and (p_kind is null or case coalesce(p_kind_mode, 'in') + when 'nin' then (m.kind is not null and m.kind <> all(p_kind)) + else m.kind = any(p_kind) + end) + and (p_host is null or case coalesce(p_host_mode, 'in') + when 'nin' then (m.host is not null and m.host <> all(p_host)) + else m.host = any(p_host) + end) + and (p_origin_repo is null or case coalesce(p_origin_repo_mode, 'in') + when 'nin' then (m.origin_repo is not null and m.origin_repo <> all(p_origin_repo)) + else m.origin_repo = any(p_origin_repo) + end) + and (p_origin_branch is null or case coalesce(p_origin_branch_mode, 'in') + when 'nin' then (m.origin_branch is not null and m.origin_branch <> all(p_origin_branch)) + else m.origin_branch = any(p_origin_branch) + end) + and (v_origin_pr is null or case coalesce(p_origin_pr_mode, 'in') + when 'nin' then (m.origin_pr is not null and m.origin_pr <> all(v_origin_pr)) + else m.origin_pr = any(v_origin_pr) + end) + -- Owner: `personal` for org_id-null rows, else the org slug (00064). + and (p_owner is null or case coalesce(p_owner_mode, 'in') + when 'nin' then ( + (case when m.org_id is null then 'personal' else o.slug end) is not null + and (case when m.org_id is null then 'personal' else o.slug end) <> all(p_owner) + ) + else ( + ('personal' = any(p_owner) and m.org_id is null) + or (m.org_id is not null and o.slug = any(p_owner)) + ) + end) + group by 1, m.scope + order by 1 asc, m.scope asc; +end; +$$; + +revoke execute on function lorekit_memory_activity( + uuid, text, timestamptz, timestamptz, text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, text[], text +) from public, anon; +grant execute on function lorekit_memory_activity( + uuid, text, timestamptz, timestamptz, text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, text[], text +) to authenticated, service_role; + +comment on function lorekit_memory_activity( + uuid, text, timestamptz, timestamptz, text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, text[], text +) is + 'Memories created per UTC hour/day per scope over the half-open + [p_since, p_until) window, visible to the EFFECTIVE caller, narrowed by the + optional scope + dimension filters (00063) plus the owner dimension (00064, + `personal` / org slug) — the SAME predicate as GET /memories and + lorekit_memory_facets, so the stat header agrees with the list. With no + filters supplied the result is byte-for-byte 00051''s.'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00064', 'owner_filter') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00065_memory_scopes_order_by_count.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- lorekit_memory_scopes — order the per-scope rollup by COUNT, not scope name. +-- +-- WHY: the Lore Explorer's scope chip strip renders `GET /memories/scopes` in +-- the order the endpoint returns (`fetchScopes` maps but never re-sorts), so the +-- endpoint's ORDER BY *is* the chip order. Alphabetical put a one-memory scope +-- ahead of the account's busiest one; sorting by count surfaces the scopes a +-- reader actually works in first. +-- +-- This also makes the two catalog endpoints consistent: `lorekit_memory_tags` +-- (00050) already orders `count desc, label asc`. `/scopes` was the odd one out +-- at `scope asc`; both now lead with frequency and break ties by name. +-- +-- Signature and return type are unchanged, so a plain `create or replace` (no +-- DROP) suffices — unlike 00049, which had to DROP to add a column. Everything +-- else is carried over verbatim from 00049: the service-role-gated actor guard, +-- the `lorekit_member_org_ids` visibility predicate, the active-rows-only +-- definition (non-archived AND non-expired), `last_activity`, plpgsql-not-SQL so +-- the ordering survives, and the revoked PUBLIC/anon grant. +-- +-- `count desc` is emitted as `count(*) desc` rather than the output alias `count` +-- to stay unambiguous under `#variable_conflict use_column`; `scope asc` is the +-- deterministic tiebreaker so equal-count scopes keep a stable order. +-- ═════════════════════════════════════════════════════════════════════════ + +create or replace function lorekit_memory_scopes(p_user_id uuid) +returns table (scope text, count bigint, last_activity timestamptz) +language plpgsql +stable +security definer +set search_path = public +as $$ +-- The RETURNS TABLE columns are also plpgsql OUT variables, so an unqualified +-- reference to any of them inside the query would be ambiguous. Every reference +-- below is table-qualified, and this directive makes the column win regardless. +#variable_conflict use_column +declare + -- Honour a caller-supplied p_user_id only on a verified service-role + -- connection (the edge resolves the token owner and passes it); an + -- authenticated caller is pinned to its own auth.uid(), closing the + -- cross-user scope-name enumeration. A NULL actor under service-role is the + -- CI escape hatch, preserved by the first visibility branch below. + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; +begin + return query + select m.scope, count(*) as count, max(m.created_at) as last_activity + from memories m + where ( + (v_actor is null and auth.role() = 'service_role') + or m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) + ) + and m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + group by m.scope + order by count(*) desc, m.scope asc; +end; +$$; + +revoke execute on function lorekit_memory_scopes(uuid) from public, anon; +grant execute on function lorekit_memory_scopes(uuid) to authenticated, service_role; + +comment on function lorekit_memory_scopes(uuid) is + 'Per-scope active counts and last activity visible to the EFFECTIVE caller, + ordered by count desc then scope asc (matching lorekit_memory_tags). Actor + resolved by the 00046/00041 service-role-gated rule: a caller-supplied + p_user_id is honoured only on a verified service-role connection, otherwise + auth.uid() wins — so an authenticated caller can never enumerate another + user''s scope names. p_user_id IS NULL under service-role is the CI escape + hatch. last_activity is max(created_at) over the same counted rows.'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00065', 'memory_scopes_order_by_count') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00066_dimension_filter_helpers.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- Reusable dimension-filter predicates — one definition of what each mode +-- means, composed by every aggregate that narrows memories by dimension. +-- +-- WHY: two aggregate functions apply the SAME per-dimension predicate — the +-- facet catalog `lorekit_memory_facets` (00057) and the write series +-- `lorekit_memory_activity` (00063), both later extended with the owner +-- dimension in 00064 — and each spelled it out INLINE, once per dimension. +-- `lorekit_memory_facets` carried eight `case` flags in its base CTE; +-- `lorekit_memory_activity` carried the same eight again in a flat WHERE. +-- Sixteen near-identical copies of a rule that is easy to get subtly wrong: +-- +-- `nin` means `value is not null and value <> all(filter)` +-- +-- The null test is load-bearing and non-obvious. `m.source_agent <> all(...)` +-- alone is NULL for a row with no agent, which reads as false, so "agent is not +-- aw" would silently drop every memory nobody attributed — the rows most likely +-- to be the ones you are hunting for. Written out sixteen times, one copy +-- eventually loses it, and the two callers disagree about a filter neither +-- author touched. +-- +-- So the predicate becomes three tiny functions and every caller composes them. +-- They are `language sql` + `immutable` + a single expression, which is what +-- makes PostgreSQL INLINE them into the calling query: the planner still sees a +-- plain boolean expression over the column and can still use an index. A +-- plpgsql helper would have been a per-row function call and a planner barrier. +-- +-- ── What this migration DELIBERATELY does NOT change ──────────────────────── +-- +-- This is a pure refactor. `lorekit_memory_facets` and `lorekit_memory_activity` +-- keep their 00064 SIGNATURES verbatim (including `p_owner` / `p_owner_mode`) +-- and their 00064 BEHAVIOUR to the row — only the eight text/tags/int dimension +-- predicates change, from inline `case` blocks to helper calls that are +-- identical to them. The owner dimension stays INLINE: it is not a scalar +-- text/tags/int column but a LEFT JOIN-computed identity (`personal` / org +-- slug), so it is not one of the three helpers. `migrations.test.sql` §68 / +-- §69 (incl. the owner §69c) is the executable proof that behaviour did not +-- move; §80 (added here) pins the helpers directly. +-- +-- `create or replace` (not drop): the signatures are unchanged, so the body is +-- swapped in place and the 00064 grants persist. The revoke/grant is re-stated +-- for the same self-documenting reason every prior migration re-states it. +-- ═════════════════════════════════════════════════════════════════════════ + +-- ── 1. The three predicates ───────────────────────────────────────────────── + +-- A scalar text column against a value list. Covers source_agent, trigger, +-- kind, host, origin_repo and origin_branch — every single-valued text +-- dimension the filter bar offers. +-- +-- `immutable`, not `stable`: the result depends only on the arguments. That is +-- also what lets the planner fold it into an index-usable expression. +create or replace function lorekit_match_text( + p_value text, + p_filter text[], + p_mode text +) +returns boolean +language sql +immutable +parallel safe +as $$ + select coalesce( + p_filter is null + or case coalesce(p_mode, 'in') + -- The null test that is the whole reason this function exists: a row + -- with no value must NOT satisfy "is not one of these". Without it the + -- comparison is NULL, which filters the row out — silently hiding + -- every unattributed row from a negated filter. + when 'nin' then (p_value is not null and p_value <> all(p_filter)) + else p_value = any(p_filter) + end, + false); +$$; + +comment on function lorekit_match_text(text, text[], text) is + 'Does a scalar text column satisfy a value-list filter? `nin` negates and + requires the value to be NON-NULL, so an unattributed row is excluded from a + negated filter rather than silently dropped by NULL logic. A null filter is + "not filtered" and matches everything. Inlinable (sql/immutable), so callers + keep index access.'; + +-- A `text[]` column (memories.tags) against a label list. Three modes rather +-- than two, because a column holding MANY values admits containment: `all` is +-- @>, `any` is &&, and `none` is the negation of `any`. +create or replace function lorekit_match_tags( + p_value text[], + p_filter text[], + p_mode text +) +returns boolean +language sql +immutable +parallel safe +as $$ + select coalesce( + p_filter is null + or case coalesce(p_mode, 'any') + when 'all' then p_value @> p_filter + -- NOT(carries any), never NOT(carries all): the latter would also + -- admit a row carrying all but one of the named labels. + when 'none' then not (p_value && p_filter) + else p_value && p_filter + end, + false); +$$; + +comment on function lorekit_match_tags(text[], text[], text) is + 'Does a label array satisfy a label filter? `all` is containment (@>), `any` + is overlap (&&), `none` is the negation of `any` — never the negation of + `all`, which would admit a row carrying all but one named label. A null + filter matches everything. Inlinable.'; + +-- An integer column (memories.origin_pr) against a value list. +-- +-- Separate from the text helper because the column is an INTEGER and the +-- comparison must be numeric: `GET /memories` sends the digits bare to the +-- integer column, so Postgres reads `007` as 7 and matches PR 7. Comparing +-- `origin_pr::text` would match nothing for that same input and the two routes +-- would disagree about one query string. The digits-only coercion (dropping a +-- non-numeric entry, applying no filter when the list reduces to empty) happens +-- at the CALLER, so this helper only ever receives an already-numeric array. +create or replace function lorekit_match_int( + p_value integer, + p_filter integer[], + p_mode text +) +returns boolean +language sql +immutable +parallel safe +as $$ + select coalesce( + p_filter is null + or case coalesce(p_mode, 'in') + when 'nin' then (p_value is not null and p_value <> all(p_filter)) + else p_value = any(p_filter) + end, + false); +$$; + +comment on function lorekit_match_int(integer, integer[], text) is + 'Integer counterpart of lorekit_match_text, for memories.origin_pr. Numeric + comparison so `007` matches PR 7, exactly as GET /memories does. Inlinable.'; + +-- These are pure predicates that touch no data, so the default PUBLIC EXECUTE is +-- harmless — but this repo revokes PUBLIC/anon on every function it ships rather +-- than relying on the default, so match that pattern here too. It does not +-- affect inlining (which depends on the sql/immutable/single-expression shape, +-- not on the grant), and the SECURITY DEFINER callers reach them as their owner. +revoke execute on function lorekit_match_text(text, text[], text) from public, anon; +revoke execute on function lorekit_match_tags(text[], text[], text) from public, anon; +revoke execute on function lorekit_match_int(integer, integer[], text) from public, anon; +grant execute on function lorekit_match_text(text, text[], text) to authenticated, service_role; +grant execute on function lorekit_match_tags(text[], text[], text) to authenticated, service_role; +grant execute on function lorekit_match_int(integer, integer[], text) to authenticated, service_role; + +-- ── 2. lorekit_memory_facets — composes the helpers ───────────────────────── +-- +-- Signature UNCHANGED from 00064 (so no caller moves) and behaviour unchanged: +-- this replaces the eight inline `case` flags in the base CTE with the helper +-- calls they are now identical to. The point is that there is one definition of +-- what `nin` means rather than two — the whole reason the helpers exist, and +-- undermined by leaving this function as the second copy. Owner stays inline: +-- it is a LEFT JOIN-computed identity, not one of the three column helpers. +-- +-- Everything else — the actor rule, the tenant predicate (lorekit_member_org_ids), +-- the archived partition, the LEFT JOIN orgs, the owner predicate + owner-dimension +-- self-exclusion, the null/blank drop and the ordering — is 00064's verbatim. +create or replace function lorekit_memory_facets( + p_user_id uuid, + p_archived boolean default false, + p_scope text default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in', + -- Owner dimension (00064). `personal` plus one slug per member org with + -- visible rows. All optional: null/absent = not filtered. + p_owner text[] default null, + p_owner_mode text default 'in' +) +returns table (facet text, value text, count bigint) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + v_origin_pr integer[] := ( + select array_agg(x::integer) + from unnest(coalesce(p_origin_pr, '{}'::text[])) as x + where x ~ '^[0-9]+$' + ); +begin + return query + with base as ( + select + m.tags, m.source_agent, m.trigger, m.kind, m.host, + m.origin_repo, m.origin_branch, m.origin_pr, + m.org_id, o.slug as org_slug, + -- Per-dimension match flag, now from the shared predicates so it cannot + -- drift from lorekit_memory_activity's. A null filter is "not filtered" → + -- the helper returns true, so an untouched dimension never narrows. + lorekit_match_tags(m.tags, p_tags, p_tags_mode) as ok_tag, + lorekit_match_text(m.source_agent, p_source_agent, p_source_agent_mode) as ok_source_agent, + lorekit_match_text(m.trigger, p_trigger, p_trigger_mode) as ok_trigger, + lorekit_match_text(m.kind, p_kind, p_kind_mode) as ok_kind, + lorekit_match_text(m.host, p_host, p_host_mode) as ok_host, + lorekit_match_text(m.origin_repo, p_origin_repo, p_origin_repo_mode) as ok_origin_repo, + lorekit_match_text(m.origin_branch, p_origin_branch, p_origin_branch_mode) as ok_origin_branch, + lorekit_match_int (m.origin_pr, v_origin_pr, p_origin_pr_mode) as ok_origin_pr, + -- Owner: the computed identity is `personal` (org_id null) or the org slug. + -- Stays inline — it is not one of the three column helpers (00064). + (p_owner is null or case coalesce(p_owner_mode, 'in') + when 'nin' then ( + (case when m.org_id is null then 'personal' else o.slug end) is not null + and (case when m.org_id is null then 'personal' else o.slug end) <> all(p_owner) + ) + else ( + ('personal' = any(p_owner) and m.org_id is null) + or (m.org_id is not null and o.slug = any(p_owner)) + ) + end) as ok_owner + from memories m + -- A personal row has no org, so this is a LEFT join; org rows resolve to + -- their slug. Visible org rows are always the caller's own orgs (the + -- visibility predicate below admits them only via lorekit_member_org_ids), + -- so `o.slug` is never a slug the caller cannot see. + left join orgs o on o.id = m.org_id + where ( + (v_actor is null and auth.role() = 'service_role') + or m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) + ) + and ( + case + when p_archived then m.archived_at is not null + else m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + end + ) + and (p_scope is null or m.scope = p_scope) + ), cells as ( + select 'tag'::text as facet, t.tag as value + from base b + cross join lateral unnest(b.tags) as t(tag) + where b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner + union all + select 'source_agent', b.source_agent from base b + where b.ok_tag and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner + union all + select 'trigger', b.trigger from base b + where b.ok_tag and b.ok_source_agent and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner + union all + select 'kind', b.kind from base b + where b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner + union all + select 'host', b.host from base b + where b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner + union all + select 'origin_repo', b.origin_repo from base b + where b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner + union all + select 'origin_branch', b.origin_branch from base b + where b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_pr and b.ok_owner + union all + select 'origin_pr', b.origin_pr::text from base b + where b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_owner + union all + -- Owner is the ONE dimension self-excluded here (every other flag, NOT + -- ok_owner), so a drilled-in owner still lists the alternative owner. + select 'owner', case when b.org_id is null then 'personal' else b.org_slug end from base b + where b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr + ) + select c.facet, c.value, count(*) as count + from cells c + where c.value is not null + and btrim(c.value) <> '' + group by c.facet, c.value + order by c.facet asc, count(*) desc, c.value asc; +end; +$$; + +revoke execute on function lorekit_memory_facets( + uuid, boolean, text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text +) from public, anon; +grant execute on function lorekit_memory_facets( + uuid, boolean, text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text +) to authenticated, service_role; + +comment on function lorekit_memory_facets( + uuid, boolean, text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text +) is + 'Value catalog with counts for every filterable memory dimension (tag, + source_agent, trigger, kind, host, origin_repo, origin_branch, origin_pr, + owner) over the partition selected by p_archived, visible to the EFFECTIVE + caller. `owner` (00064) is `personal` for org_id-null rows, else the org + slug. Counts are DRILL-DOWN (00057): each dimension is counted with every + OTHER active filter applied but not its own (self-exclusion), so the numbers + match what adding that value would yield. The eight text/tags/int dimension + predicates come from lorekit_match_text / _tags / _int (00066) so their + semantics cannot drift from lorekit_memory_activity; owner stays inline. With + no filters supplied the result is 00052''s global catalog plus the owner + dimension. Same service-role-gated actor rule and ordering as + lorekit_memory_tags.'; + +-- ── 3. lorekit_memory_activity — composes the same helpers ────────────────── +-- +-- Signature UNCHANGED from 00064. Unlike the facet catalog there is NO +-- self-exclusion here: a straight count applies every filter directly, so the +-- eight dimension predicates collapse to a flat WHERE. Replacing the inline +-- `case` blocks with the helper calls keeps the series byte-identical to 00064 +-- and provably in step with the facet catalog. Owner stays inline. Bucket +-- validation, the half-open [p_since, p_until) window, the active-only + tenant +-- predicate, the LEFT JOIN orgs and the date_trunc anchoring are 00064 verbatim. +create or replace function lorekit_memory_activity( + p_user_id uuid, + p_bucket text default 'day', + p_since timestamptz default null, + p_until timestamptz default null, + p_scope text default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in', + -- Owner dimension (00064) — the SAME predicate as lorekit_memory_facets and + -- GET /memories, so the stat header agrees with the list under an owner pill. + p_owner text[] default null, + p_owner_mode text default 'in' +) +returns table (bucket timestamptz, scope text, count bigint) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + -- `origin_pr` is an INTEGER column: coerce the digits-only list numerically so + -- this route matches GET /memories on the same input (00057's rationale). A + -- non-numeric entry is dropped; a list reducing to empty applies no filter. + v_origin_pr integer[] := ( + select array_agg(x::integer) + from unnest(coalesce(p_origin_pr, '{}'::text[])) as x + where x ~ '^[0-9]+$' + ); +begin + if p_bucket is null or p_bucket not in ('hour', 'day') then + raise exception 'invalid bucket %, expected hour or day', p_bucket + using errcode = '22023'; + end if; + + return query + select date_trunc(p_bucket, m.created_at at time zone 'UTC') at time zone 'UTC' as bucket, + m.scope, + count(*) as count + from memories m + -- LEFT join for the owner predicate — a personal row has no org (00064). + left join orgs o on o.id = m.org_id + where ( + (v_actor is null and auth.role() = 'service_role') + or m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) + ) + and m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + and (p_since is null or m.created_at >= p_since) + and (p_until is null or m.created_at < p_until) + -- Scope is a hard filter, always applied (00057's treatment). + and (p_scope is null or m.scope = p_scope) + -- Dimension filters — AND across, OR within, from the shared predicates + -- (00066) so the semantics cannot drift from lorekit_memory_facets. No + -- self-exclusion here: a straight count applies every one directly. + and lorekit_match_tags(m.tags, p_tags, p_tags_mode) + and lorekit_match_text(m.source_agent, p_source_agent, p_source_agent_mode) + and lorekit_match_text(m.trigger, p_trigger, p_trigger_mode) + and lorekit_match_text(m.kind, p_kind, p_kind_mode) + and lorekit_match_text(m.host, p_host, p_host_mode) + and lorekit_match_text(m.origin_repo, p_origin_repo, p_origin_repo_mode) + and lorekit_match_text(m.origin_branch, p_origin_branch, p_origin_branch_mode) + and lorekit_match_int (m.origin_pr, v_origin_pr, p_origin_pr_mode) + -- Owner: `personal` for org_id-null rows, else the org slug (00064). + and (p_owner is null or case coalesce(p_owner_mode, 'in') + when 'nin' then ( + (case when m.org_id is null then 'personal' else o.slug end) is not null + and (case when m.org_id is null then 'personal' else o.slug end) <> all(p_owner) + ) + else ( + ('personal' = any(p_owner) and m.org_id is null) + or (m.org_id is not null and o.slug = any(p_owner)) + ) + end) + group by 1, m.scope + order by 1 asc, m.scope asc; +end; +$$; + +revoke execute on function lorekit_memory_activity( + uuid, text, timestamptz, timestamptz, text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, text[], text +) from public, anon; +grant execute on function lorekit_memory_activity( + uuid, text, timestamptz, timestamptz, text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, text[], text +) to authenticated, service_role; + +comment on function lorekit_memory_activity( + uuid, text, timestamptz, timestamptz, text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, text[], text +) is + 'Memories created per UTC hour/day per scope over the half-open + [p_since, p_until) window, visible to the EFFECTIVE caller, narrowed by the + optional scope + dimension filters (00063) plus the owner dimension (00064, + `personal` / org slug). The eight text/tags/int dimension predicates come + from lorekit_match_text / _tags / _int (00066) — the SAME predicates + lorekit_memory_facets composes and the SAME behaviour GET /memories has, so + the stat header agrees with the list. With no filters supplied the result is + byte-for-byte 00051''s.'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00066', 'dimension_filter_helpers') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00067_memory_list_rpc.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00067 — lorekit_memory_list: the keyset list read as a SQL function +-- +-- WHY THIS EXISTS +-- +-- This PR's edge handlers moved the Explorer's filters off the query string and +-- into a JSON body (`POST /memories/list` / `/facets` / `/activity`), because a +-- comma-joined dimension is capped at 2048 characters and several dimensions +-- compose a URL the gateway refuses outright (`414 URI too long`, with no +-- LoreKit error envelope at all). That transport change needed no migration — +-- it is a decoder (`dimensionsFromBody`) beside the one the query form already +-- used. 00066 is a different thing: the shared `lorekit_match_text` / +-- `lorekit_match_tags` / `lorekit_match_int` dimension predicates this function +-- composes below. +-- +-- That fixed the CLIENT → EDGE hop and left the EDGE → POSTGREST hop exactly as +-- it was. `handleList` built its predicates with postgrest-js, whose `.or()` is +-- a QUERY PARAM and whose `.select()` is a GET — so a dimension carrying 200 +-- values still composed a ~7 KB `or=` operand on the internal request and still +-- died at the same gateway, now as a `500` instead of a `414`. The wall had +-- moved one hop downstream, which is worse than not having moved: the error is +-- no longer attributable to the thing the caller did. +-- +-- A SQL function takes its arguments as real `text[]` parameters over a POST +-- body (`/rest/v1/rpc/…`), so the value set never appears in a URL on either +-- hop. `lorekit_memory_facets` and `lorekit_memory_activity` were already built +-- this way, which is precisely why THEY had no such wall — this brings the list +-- read onto the same footing rather than inventing a third shape for it. +-- +-- ONE PREDICATE, STILL +-- +-- Both `GET /memories` and `POST /memories/list` call this function. The +-- transports differ in how a request is spelled and in nothing else, which was +-- the safety argument for adding the body form and stays true here — the +-- difference is that the single shared predicate now lives in SQL beside the +-- other two dimension readers instead of in TypeScript beside neither. +-- +-- WHAT STAYS IN TYPESCRIPT, AND WHY +-- +-- Three values are computed by the caller and passed in already-resolved, so +-- that each is encoded exactly once in this repo (the rule 00063 and 00066 both +-- cite for NOT mirroring `q` and `expiring_within_days` into plpgsql): +-- +-- * `p_q` — the LIKE needle, already escaped by `likeNeedle`. +-- Postgres LIKE's default escape character is the +-- backslash, which is what `likeNeedle` emits, so the +-- pattern applies here unchanged. +-- * `p_key_prefix` — likewise escaped; this function appends the one +-- active `%`. +-- * `p_expires_after` / `p_expires_on_or_before` +-- — the asymmetric "expiring soon" window, already +-- computed by `expiringWindow`. A second +-- `now()`-relative boundary in SQL is exactly the +-- drift those comments warn about. +-- +-- DYNAMIC SQL, DELIBERATELY +-- +-- The sort column is interpolated as an IDENTIFIER rather than compared with a +-- `case` expression, because `order by case when … then created_at else +-- updated_at end` is not an indexable expression and would give up +-- `memories_scope_updated_at_id_idx` (00061) — the index the keyset page exists +-- to ride. It is the ONLY interpolated fragment, it is chosen from a two-value +-- whitelist below, and every caller-supplied value is bound with `using`. A +-- `p_sort` that is neither known value falls back to `updated_at` rather than +-- raising: the column is validated by `MemorySortSchema` before it ever gets +-- here, so an unknown value means a bug, and a list route should degrade to its +-- default order rather than 500. +-- ───────────────────────────────────────────────────────────────────────────── + +create or replace function lorekit_memory_list( + p_user_id uuid, + p_archived boolean default false, + p_scope text default null, + p_key text default null, + -- Already LIKE-escaped by `likeNeedle`; the trailing `%` is added here and is + -- the only active wildcard. + p_key_prefix text default null, + -- Already LIKE-escaped by `likeNeedle`; wrapped in `%…%` here. + p_q text default null, + p_created_since timestamptz default null, + p_created_until timestamptz default null, + -- The "expiring soon" window, precomputed by `expiringWindow`: the half-open + -- `(after, on_or_before]` pair. Both null = the filter is not applied. + p_expires_after timestamptz default null, + p_expires_on_or_before timestamptz default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in', + p_owner text[] default null, + p_owner_mode text default 'in', + p_sort text default 'updated_at', + -- Keyset cursor: the sort value and id of the last row of the previous page. + -- Both null = first page. The edge decodes and shape-validates the opaque + -- cursor (`_shared/api/paginate.ts`) before splitting it into these two. + p_cursor_ts timestamptz default null, + p_cursor_id uuid default null, + -- The caller passes limit + 1 and derives `hasMore` from the overflow row, + -- exactly as the PostgREST path did — the pagination contract is unchanged. + p_limit integer default 51 +) +returns table ( + id uuid, + scope text, + key text, + value text, + tags text[], + source_agent text, + trigger text, + created_at timestamptz, + updated_at timestamptz, + expires_at timestamptz, + archived_at timestamptz, + origin_repo text, + origin_branch text, + origin_commit text, + origin_pr integer, + kind text, + host text, + seen_count integer, + org_id uuid, + created_by uuid, + updated_by uuid, + -- The `orgs(id,name,slug)` embed, flattened. `shapeMemoryRow` folds these + -- three back into the nested `org` object the API returns, so the response + -- body is byte-identical to the PostgREST path's. + org_name text, + org_slug text +) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + -- Same service-role-gated actor rule as lorekit_memory_facets / + -- lorekit_memory_activity: a service_role caller may act as a named user, and + -- anyone else is themselves regardless of what they passed. + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + -- `origin_pr` is an integer column and the wire form is text. A non-numeric + -- entry is DROPPED, never an error: the filter can be built from a + -- hand-editable URL, and one bad entry should narrow the filter rather than + -- break the page. + -- + -- The digit count is BOUNDED, which a bare `^[0-9]+$` is not: `99999999999` + -- is all digits and still overflows int4, so `x::integer` would raise 22003 + -- and turn a hand-editable filter value into a 500 — the exact opposite of + -- the drop this comment promises. Nine significant digits is the widest run + -- that cannot overflow, and the leading `0*` keeps the zero-padded form + -- working (`007` → 7, as GET /memories has always resolved it). The guard is + -- a regex rather than a `x::numeric <= 2147483647` conjunct because WHERE + -- conjuncts have no guaranteed evaluation order, so a cast there could still + -- see a non-numeric value and raise 22P02. + v_origin_pr integer[] := ( + select array_agg(x::integer) + from unnest(coalesce(p_origin_pr, '{}'::text[])) as x + where x ~ '^0*[0-9]{1,9}$' + ); + -- Whitelisted identifier — see the header. Anything unrecognised degrades to + -- the route's default order instead of raising. + v_sort text := case when p_sort = 'created_at' then 'created_at' else 'updated_at' end; +begin + return query execute format($q$ + select + m.id, m.scope, m.key, m.value, m.tags, m.source_agent, m.trigger, + m.created_at, m.updated_at, m.expires_at, m.archived_at, + m.origin_repo, m.origin_branch, m.origin_commit, m.origin_pr, + m.kind, m.host, m.seen_count, + m.org_id, m.created_by, m.updated_by, + o.name as org_name, o.slug as org_slug + from memories m + -- LEFT: a personal row has no org. Visible org rows are only ever the + -- caller's own (the visibility predicate admits them via + -- lorekit_member_org_ids), so o.slug is never a slug they cannot see. + left join orgs o on o.id = m.org_id + where ( + ($1 is null and auth.role() = 'service_role') + or m.user_id = $1 + or m.org_id in (select lorekit_member_org_ids($1)) + ) + -- The archived partition. The live branch additionally hides an expired + -- row; the archived branch has no liveness guard, which is why the + -- expiring-soon window below re-states its own lower bound. + and ( + case + when $2 then m.archived_at is not null + else m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + end + ) + and ($3 is null or m.scope = $3) + and ($4 is null or m.key = $4) + -- Prefix match on key. The needle arrives LIKE-escaped, so the `%%` + -- appended here is the only active wildcard. Doubled because this + -- comment is inside the format() template: an undoubled percent there + -- is read as a type specifier and raises, exactly like the predicates + -- below. + and ($5 is null or m.key ilike $5 || '%%') + -- Substring over key OR value, the `q` filter. Same escaped-needle + -- contract; both wildcards are added here. + and ($6 is null or m.key ilike '%%' || $6 || '%%' or m.value ilike '%%' || $6 || '%%') + -- Half-open [since, until) creation window. + and ($7 is null or m.created_at >= $7) + and ($8 is null or m.created_at < $8) + -- "Expiring soon": `expires_at` in (after, on_or_before]. TWO predicates, + -- not three — a row with no TTL fails both comparisons on its own, + -- because `null > x` is null, so no `is not null` guard is needed. Both + -- are plain conjuncts, so this narrows the partition above rather than + -- widening it. Range-scans memories_expires_at_idx (00030). + and ($9 is null or m.expires_at > $9) + and ($10 is null or m.expires_at <= $10) + -- The dimension predicates, from the shared helpers (00066), so this + -- reader cannot drift from lorekit_memory_facets or _activity. AND + -- across dimensions, OR within one. A null filter is "not filtered". + and lorekit_match_tags(m.tags, $11, $12) + and lorekit_match_text(m.source_agent, $13, $14) + and lorekit_match_text(m.trigger, $15, $16) + and lorekit_match_text(m.kind, $17, $18) + and lorekit_match_text(m.host, $19, $20) + and lorekit_match_text(m.origin_repo, $21, $22) + and lorekit_match_text(m.origin_branch, $23, $24) + and lorekit_match_int (m.origin_pr, $25, $26) + -- Owner (00064): the computed identity `personal` (org_id null) or the + -- org slug. Inline rather than a helper because it is not a plain + -- column — identical expression to the other two readers'. + and ($27 is null or case coalesce($28, 'in') + when 'nin' then ( + (case when m.org_id is null then 'personal' else o.slug end) is not null + and (case when m.org_id is null then 'personal' else o.slug end) <> all($27) + ) + else ( + ('personal' = any($27) and m.org_id is null) + or (m.org_id is not null and o.slug = any($27)) + ) + end) + -- Keyset. Strictly after the previous page's last row in the composite + -- (sort, id) order below, which is why the tie-break on id is part of + -- the predicate and not just the ordering. + and ( + $29 is null + or m.%1$I < $29 + or (m.%1$I = $29 and m.id < $30) + ) + order by m.%1$I desc, m.id desc + limit $31 + $q$, v_sort) + using + v_actor, coalesce(p_archived, false), p_scope, p_key, p_key_prefix, p_q, + p_created_since, p_created_until, p_expires_after, p_expires_on_or_before, + p_tags, p_tags_mode, + p_source_agent, p_source_agent_mode, + p_trigger, p_trigger_mode, + p_kind, p_kind_mode, + p_host, p_host_mode, + p_origin_repo, p_origin_repo_mode, + p_origin_branch, p_origin_branch_mode, + v_origin_pr, p_origin_pr_mode, + p_owner, p_owner_mode, + p_cursor_ts, p_cursor_id, + greatest(coalesce(p_limit, 51), 1); +end; +$$; + +revoke execute on function lorekit_memory_list( + uuid, boolean, text, text, text, text, timestamptz, timestamptz, timestamptz, + timestamptz, text[], text, text[], text, text[], text, text[], text, text[], + text, text[], text, text[], text, text[], text, text[], text, text, + timestamptz, uuid, integer +) from public, anon; +grant execute on function lorekit_memory_list( + uuid, boolean, text, text, text, text, timestamptz, timestamptz, timestamptz, + timestamptz, text[], text, text[], text, text[], text, text[], text, text[], + text, text[], text, text[], text, text[], text, text[], text, text, + timestamptz, uuid, integer +) to authenticated, service_role; + +comment on function lorekit_memory_list( + uuid, boolean, text, text, text, text, timestamptz, timestamptz, timestamptz, + timestamptz, text[], text, text[], text, text[], text, text[], text, text[], + text, text[], text, text[], text, text[], text, text[], text, text, + timestamptz, uuid, integer +) is + 'One keyset page of the memories visible to the EFFECTIVE caller, ordered by + p_sort desc then id desc, narrowed by every filter GET /memories and + POST /memories/list accept. Backs BOTH of those routes, so the two transports + share one predicate. Exists as a FUNCTION rather than a PostgREST query + because postgrest-js emits filters as URL query params: a dimension carrying + a few hundred values composed an internal request the gateway refused + (414 → surfaced as 500), which is the same wall the body transport removed + one hop upstream. The eight text/tags/int dimension predicates come from + lorekit_match_text / _tags / _int (00066), so this reader cannot drift from + lorekit_memory_facets or lorekit_memory_activity; owner stays inline. p_q and + p_key_prefix arrive already LIKE-escaped and the expiring-soon window arrives + already computed, so neither escaping nor the now()-relative boundary is + implemented twice. Callers pass limit + 1 and derive hasMore from the + overflow row.'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00067', 'memory_list_rpc') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00068_api_token_scoping.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- API token scoping: restrict a key to specific scopes and/or specific orgs. +-- +-- Until now a key inherited the ENTIRE visibility of its owning user — every +-- personal memory plus every org they belong to (`applyTenantScope`). The only +-- capability axis was read/write (`permissions text[]`, 00002). That makes a +-- token handed to one repo's CI a credential for the whole account, which is +-- the thing this migration fixes: a key can now be narrowed to a set of scope +-- patterns and to a tenancy (personal only / all orgs / named orgs), on top of +-- the read/write axis it already had. +-- +-- ── Decisions, and why ──────────────────────────────────────────────────── +-- +-- 1. EMPTY MEANS UNRESTRICTED. `scopes = '{}'` is "every scope the owner can +-- see", not "no scope at all". The alternative (empty = deny) reads safer +-- but silently bricks every existing key on deploy AND the key the dashboard +-- auto-generates on first login (`onboarding-steps.tsx`), which has no UI to +-- pick scopes from. Restriction is therefore opt-in and additive, and the +-- column's default reproduces today's behaviour exactly. `permissions` (a +-- text[] with NO check and an unrepresentable empty set) is the cautionary +-- tale next door — hence the explicit CHECKs below. +-- +-- 2. ORG ACCESS IS A TRI-STATE, NOT A LIST. A bare `org_ids uuid[]` cannot say +-- "personal only": an empty list would have to mean both "unrestricted" and +-- "no orgs", and those are the two most different answers in the space. So +-- tenancy is an enum (`all` | `personal` | `selected`) with the list only +-- meaningful — and only permitted — under `selected`. +-- +-- 3. SCOPE PATTERNS REUSE THE EXISTING WILDCARD GRAMMAR. `repo::mthines/*` is +-- already a thing users type (`expandScopeForSearch`), so a key allowlist +-- accepts the same shape rather than inventing a second matcher. The CHECK's +-- charset is deliberately byte-identical to that function's PostgREST +-- injection guard (`[a-z0-9._:/-]`) — the values end up in the same kind of +-- predicate, so they get the same gate — and so is its WILDCARD POSITION: a +-- trailing `*` counts only after `/` or `::`, exactly as that function +-- requires. An "any trailing star" rule would have let `repo::mthines/lore*` +-- allowlist `repo::mthines/lorekit-private` while being REFUSED as a search +-- filter — two different grammars wearing one syntax. +-- +-- 4. THE KEY WINS OVER A SCOPE→ORG BINDING. `org_scope_bindings` (00026) +-- auto-routes a write under a bound scope into that org. A key restricted to +-- `personal` writing under a bound scope must NOT land in an org the key was +-- never granted — the binding is a convenience, the key restriction is an +-- authorization boundary, and a boundary that a convenience can widen is not +-- one. `lorekit_api_token_org_allowed` is the predicate that says so; wiring +-- it into `memory_write`'s routing is the next migration's job, together +-- with the transport-side enforcement. +-- +-- 5. WRITES GO THROUGH AN RPC, NOT AN UPDATE POLICY. 00002 deliberately has no +-- UPDATE policy on `api_tokens`. Adding one now would put an authorization +-- decision (may this actor point this key at this org?) in a WITH CHECK +-- expression, which cannot consult `org_members` without re-deriving the +-- tenancy predicate a second time. `lorekit_api_token_set_scoping` is the +-- Phase-3 shape instead: SECURITY DEFINER, actor from `auth.uid()`, never a +-- caller-supplied user id. +-- +-- Nothing here ENFORCES anything yet — this migration is the data model plus +-- the two predicates. The transports (`mcp/auth.ts`, `_shared/api/auth.ts`, +-- `mcp-handler.ts`, `router.ts`, `applyTenantScope`) start consulting them in +-- the following change, so this one is a no-op for every existing key. +-- ═════════════════════════════════════════════════════════════════════════ + +-- ───────────────────────────────────────────────────────────────────────────── +-- 1. The columns +-- ───────────────────────────────────────────────────────────────────────────── + +alter table api_tokens + add column if not exists scopes text[] not null default '{}', + add column if not exists org_access text not null default 'all', + add column if not exists org_ids uuid[] not null default '{}'; + +comment on column api_tokens.scopes is + 'Allowlist of scope patterns this key may touch. EMPTY = unrestricted. A ' + 'pattern is a canonical scope, or an owner wildcard whose trailing "*" comes ' + 'directly after a "/" or a "::" (repo::mthines/*), matched by ' + 'lorekit_api_token_scope_allowed().'; + +comment on column api_tokens.org_access is + 'Tenancy this key may reach: all (every org the owner belongs to, plus ' + 'personal), personal (personal rows only), selected (personal plus the orgs ' + 'in org_ids).'; + +comment on column api_tokens.org_ids is + 'The orgs a "selected" key may reach. Non-empty if and only if org_access = ' + '''selected''.'; + +-- Bounded, so a key cannot become a thousand-element predicate evaluated on +-- every request. 50 is far past any real key and far short of a problem. +alter table api_tokens + drop constraint if exists api_tokens_scopes_len; +alter table api_tokens + add constraint api_tokens_scopes_len + check (cardinality(scopes) <= 50); + +-- Every element is a canonical scope or an owner wildcard. The charset mirrors +-- `expandScopeForSearch`'s injection guard exactly — see decision 3. The 200 +-- ceiling is USAGE_SCOPE_MAX, reused so a scope that can be recorded is a scope +-- that can be allowlisted. +-- +-- A helper function rather than an inline predicate because a CHECK cannot +-- contain a subquery, and "every element of an array satisfies P" needs one. +-- IMMUTABLE and total, so the constraint is index-safe and a NULL never slips +-- past it as "unknown, therefore fine". +create or replace function lorekit_api_token_scopes_valid(p_patterns text[]) +returns boolean +language sql +immutable +as $$ + select p_patterns is null or not exists ( + select 1 + from unnest(p_patterns) as t(pattern) + -- `is not true` rather than `!~`: a NULL ELEMENT makes both the regex and + -- the length test NULL, the row is filtered out, and `not exists` reports + -- the array valid — the unknown-therefore-fine case this gate exists to + -- refuse. + where (t.pattern ~ '^[a-z0-9._:/-]+(/|::)\*$' or t.pattern ~ '^[a-z0-9._:/-]+$') is not true + or (length(t.pattern) <= 200) is not true + ); +$$; + +comment on function lorekit_api_token_scopes_valid(text[]) is + 'Shape gate for api_tokens.scopes: each element is a canonical scope or an ' + 'owner wildcard (a trailing ''*'' only after ''/'' or ''::''), over ' + 'expandScopeForSearch''s charset, at most 200 chars. Total: a NULL element is ' + 'INVALID, never unknown-therefore-fine.'; + +revoke execute on function lorekit_api_token_scopes_valid(text[]) from public, anon; +grant execute on function lorekit_api_token_scopes_valid(text[]) to authenticated, service_role; + +alter table api_tokens + drop constraint if exists api_tokens_scopes_shape; +alter table api_tokens + add constraint api_tokens_scopes_shape + check (lorekit_api_token_scopes_valid(scopes)); + +alter table api_tokens + drop constraint if exists api_tokens_org_access_valid; +alter table api_tokens + add constraint api_tokens_org_access_valid + check (org_access in ('all', 'personal', 'selected')); + +-- The list and the mode cannot disagree. Written as an equality of two booleans +-- rather than two implications so neither direction can be forgotten: a +-- 'selected' key with no orgs would be an unreachable key, and a 'personal' key +-- carrying org ids would be a lie the next reader believes. +alter table api_tokens + drop constraint if exists api_tokens_org_ids_match_access; +alter table api_tokens + add constraint api_tokens_org_ids_match_access + check ((org_access = 'selected') = (cardinality(org_ids) > 0)); + +alter table api_tokens + drop constraint if exists api_tokens_org_ids_len; +alter table api_tokens + add constraint api_tokens_org_ids_len + check (cardinality(org_ids) <= 50); + +-- The org-list twin of the NULL-element rule `lorekit_api_token_scopes_valid` +-- enforces for `scopes`. Nothing above catches it: `{null}` has cardinality 1, +-- so both the length cap and the tenancy-match CHECK are satisfied, and a NULL +-- element is not a "no orgs" list — it is an unknown one, which on an +-- authorization column is the worst of the three answers. +-- +-- `array_position` rather than a subquery because a CHECK cannot contain one, +-- and it is the subquery-free form: it compares with IS NOT DISTINCT FROM, so +-- it finds NULLs, and returns NULL (not 0) when there is no match. +alter table api_tokens + drop constraint if exists api_tokens_org_ids_not_null; +alter table api_tokens + add constraint api_tokens_org_ids_not_null + check (array_position(org_ids, null) is null); + +-- ───────────────────────────────────────────────────────────────────────────── +-- 2. The predicates +-- +-- Both are IMMUTABLE and take the key's stored columns as arguments rather than +-- a token id: the transports already hold the row after the hash lookup, and a +-- predicate that re-reads the table would turn one query per request into two. +-- They live in SQL — not only in the edge — because the write path's last +-- unbypassable gate is inside `memory_write`, which cannot call TypeScript. +-- ───────────────────────────────────────────────────────────────────────────── + +create or replace function lorekit_api_token_scope_allowed( + p_patterns text[], + p_scope text +) +returns boolean +language sql +immutable +as $$ + select case + -- Decision 1: an unrestricted key is the default and the common case, so it + -- is the first branch and costs no array walk. + when p_patterns is null or cardinality(p_patterns) = 0 then true + -- A scopeless operation (memory.purge_expired, the account-wide reads) + -- cannot be matched against an allowlist. Refusing it is the fail-closed + -- answer: a key narrowed to one repo has no business sweeping the account. + when p_scope is null then false + else exists ( + select 1 + from unnest(p_patterns) as pattern + where case + when right(pattern, 1) = '*' + -- Escape LIKE's single-character wildcard in the literal prefix so + -- `repo::my_org/*` stays owner-exact instead of also matching + -- `repo::myXorg/...`. `%` and `\` cannot occur — the CHECK's charset + -- excludes them. Same reasoning, same escape, as expandScopeForSearch. + then p_scope like replace(left(pattern, -1), '_', '\_') || '%' + else p_scope = pattern + end + ) + end; +$$; + +comment on function lorekit_api_token_scope_allowed(text[], text) is + 'May a key whose api_tokens.scopes is p_patterns touch p_scope? Empty ' + 'allowlist = yes (unrestricted). NULL scope = no (fail closed).'; + +create or replace function lorekit_api_token_org_allowed( + p_org_access text, + p_org_ids uuid[], + p_org_id uuid +) +returns boolean +language sql +immutable +as $$ + select case + -- A personal row (org_id IS NULL) is reachable under every tenancy: the key + -- belongs to that user, and 'personal' is a narrowing of which ORGS are + -- reachable, never a revocation of the owner's own memories. + when p_org_id is null then true + when p_org_access = 'all' then true + when p_org_access = 'personal' then false + -- `coalesce(…, false)` is load-bearing: a NULL ELEMENT in p_org_ids makes + -- `= any(…)` evaluate to NULL, and this function would then hand its caller + -- NULL from an authorization predicate — neither true nor false. The + -- api_tokens_org_ids_not_null CHECK stops such a row being written here, but + -- this is precisely the function the comment below says is also fed rows + -- from a BYOD install that may predate the CHECK, so it cannot rely on it. + when p_org_access = 'selected' + then coalesce(p_org_id = any(coalesce(p_org_ids, '{}'::uuid[])), false) + -- Unreachable while api_tokens_org_access_valid holds; fail closed anyway, + -- because this function is also called with values read back from a BYOD + -- install whose bootstrap may predate the CHECK. + else false + end; +$$; + +comment on function lorekit_api_token_org_allowed(text, uuid[], uuid) is + 'May a key with this tenancy reach a row owned by p_org_id (NULL = personal)? ' + 'Authoritative over org_scope_bindings auto-routing — see 00068 decision 4.'; + +revoke execute on function lorekit_api_token_scope_allowed(text[], text) from public, anon; +revoke execute on function lorekit_api_token_org_allowed(text, uuid[], uuid) from public, anon; +grant execute on function lorekit_api_token_scope_allowed(text[], text) to authenticated, service_role; +grant execute on function lorekit_api_token_org_allowed(text, uuid[], uuid) to authenticated, service_role; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 3. Setting the scoping — decision 5 +-- +-- The dashboard calls this with the user's own JWT session, so the actor is +-- `auth.uid()` and there is no caller-supplied user id to spoof (00046's rule, +-- applied to a table that had no UPDATE path at all). +-- ───────────────────────────────────────────────────────────────────────────── + +-- No DEFAULTS on the three scoping arguments, deliberately. With them, a caller +-- that sent only `p_org_access` would silently reset `scopes` to `{}` — WIDENING +-- the key to every scope — because "omitted" and "clear this" would be the same +-- request. Requiring all three makes the full intended state explicit on every +-- call, so a partial update cannot fail open. +create or replace function lorekit_api_token_set_scoping( + p_token_id uuid, + p_scopes text[], + p_org_access text, + p_org_ids uuid[] +) +returns table ( + id uuid, + scopes text[], + org_access text, + org_ids uuid[] +) +language plpgsql +volatile +security definer +set search_path = public +as $$ +declare + v_actor uuid := auth.uid(); + v_scopes text[]; + v_org_ids uuid[]; + v_owner uuid; + v_stray uuid; +begin + if v_actor is null then + raise exception 'LK001: authentication required' using errcode = '28000'; + end if; + + -- These used to be `coalesce(p_scopes, '{}')` / `coalesce(p_org_ids, '{}')` in + -- the declare block, which reopened the hole giving the arguments no DEFAULT + -- was meant to close. No DEFAULT stops an OMITTED argument; the coalesce let + -- an EXPLICIT null through and turned it into '{}' — and for `scopes`, '{}' + -- means UNRESTRICTED (decision 1). So `set_scoping(id, null, 'all', '{}')` + -- silently widened the key to every scope, which is exactly the fail-open + -- partial update decision 5 refuses. + -- + -- Refused rather than treated as "leave unchanged": that would make this a + -- partial-update API that has to read and merge the current row, and decision + -- 5 chose "state the full intended state on every call". Same reasoning, and + -- now the same treatment, as the NULL `p_org_access` guard below. + if p_scopes is null then + raise exception 'LK004: scopes must not be null; pass an empty array to un-scope the key' + using errcode = '22023'; + end if; + + if p_org_ids is null then + raise exception 'LK004: org_ids must not be null; pass an empty array for a key with no orgs' + using errcode = '22023'; + end if; + + v_scopes := p_scopes; + v_org_ids := p_org_ids; + + -- Ownership first, and by SELECT rather than by letting the UPDATE match zero + -- rows: "you do not own this key" and "no such key" must be the same answer to + -- the caller (no existence oracle), but they must be a real error rather than + -- a silent no-op that the dashboard would render as success. + select user_id into v_owner from api_tokens where api_tokens.id = p_token_id; + if v_owner is null or v_owner <> v_actor then + raise exception 'LK003: api token not found' using errcode = 'P0002'; + end if; + + -- Re-state the table's CHECKs here so a bad argument comes back as a legible + -- LK004 the dashboard can render, not as a raw constraint-violation string + -- naming an internal constraint. The CHECKs stay the authority — this is the + -- error message, not the gate. + if cardinality(v_scopes) > 50 then + raise exception 'LK004: at most 50 scope patterns per key' using errcode = '22023'; + end if; + + if not lorekit_api_token_scopes_valid(v_scopes) then + raise exception 'LK004: a scope pattern may contain only [a-z0-9._:/-], with an optional trailing "*" directly after a "/" or a "::"' + using errcode = '22023'; + end if; + + if cardinality(v_org_ids) > 50 then + raise exception 'LK004: at most 50 orgs per key' using errcode = '22023'; + end if; + + -- `is null or` is load-bearing, not defensive noise. `null not in (…)` is + -- NULL, `if NULL` takes the false branch, and the `<>` guard below is NULL for + -- the same reason — so a NULL tenancy would fall through BOTH re-statements + -- and reach the UPDATE, where the column's NOT NULL rejects it as a raw 23502 + -- naming an internal constraint. That is the exact unreadable failure these + -- re-statements exist to prevent. All three scoping arguments are now read + -- raw and each carries its own explicit NULL refusal — `p_scopes` and + -- `p_org_ids` above, this one here. + -- + -- Refusing rather than coalescing to 'all' is deliberate: 'all' is the WIDEST + -- tenancy, so defaulting a missing one would let an under-specified call widen + -- the key — the same fail-open shape decision 5 refuses by giving the + -- arguments no DEFAULT in the first place. That is also why the two arguments + -- above are no longer coalesced: their default, '{}', means UNRESTRICTED. + if p_org_access is null or p_org_access not in ('all', 'personal', 'selected') then + raise exception 'LK004: org_access must be all, personal or selected' + using errcode = '22023'; + end if; + + if (p_org_access = 'selected') <> (cardinality(v_org_ids) > 0) then + raise exception 'LK004: org_ids must be non-empty for org_access=selected and empty otherwise' + using errcode = '22023'; + end if; + + -- A key cannot be pointed at an org its OWNER cannot reach. Without this the + -- column would be a wish rather than a restriction: the request-time check + -- intersects with the owner's real membership anyway, so an unreachable org id + -- would sit in the row looking like access that was granted. Rejecting it here + -- keeps the row honest and the failure legible. + -- A NULL ELEMENT has to go first, because the membership guard below cannot + -- see it: a NULL org id IS selected into v_stray (`m.org_id = null` is NULL, + -- so `not exists` is true), but then `v_stray is not null` is false and the + -- raise never fires. Two layers of three-valued logic cancelling out into + -- "looks like a member". The CHECK is the authority; this is the legible + -- LK004, same division of labour as the other re-statements above. + if array_position(v_org_ids, null) is not null then + raise exception 'LK004: org_ids must not contain a null element' + using errcode = '22023'; + end if; + + select t.org_id into v_stray + from unnest(v_org_ids) as t(org_id) + where not exists ( + select 1 from lorekit_member_org_ids(v_actor) as m(org_id) + where m.org_id = t.org_id + ) + limit 1; + if v_stray is not null then + raise exception 'LK002: not a member of org %', v_stray using errcode = '42501'; + end if; + + -- Qualified in RETURNING, bare on the left of SET — the shape `memory_write` + -- has used since 00007, which is what keeps the OUT parameter names from + -- colliding with the identically-named columns. + return query + update api_tokens + set scopes = v_scopes, + org_access = p_org_access, + org_ids = v_org_ids + where api_tokens.id = p_token_id + and api_tokens.user_id = v_actor + returning api_tokens.id, api_tokens.scopes, api_tokens.org_access, api_tokens.org_ids; +end; +$$; + +comment on function lorekit_api_token_set_scoping(uuid, text[], text, uuid[]) is + 'Owner-only update of a key''s scope allowlist and tenancy. Actor is ' + 'auth.uid(); org ids are validated against the actor''s own membership.'; + +revoke execute on function lorekit_api_token_set_scoping(uuid, text[], text, uuid[]) + from public, anon; +grant execute on function lorekit_api_token_set_scoping(uuid, text[], text, uuid[]) + to authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00068', 'api_token_scoping') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00069_api_token_scoping_enforcement.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- API token scoping, part 2 of 2: the SQL-side enforcement. +-- +-- 00068 added the data model (`api_tokens.scopes` / `org_access` / `org_ids`) +-- and the two predicates, and enforced nothing. This migration teaches the SEVEN +-- functions that the transports CANNOT stand in front of — two mutation gates +-- and five per-scope aggregates — and re-issues one of 00068's own predicates: +-- +-- 1. `memory_write` — the last unbypassable gate on the write path, on BOTH +-- axes. The edge holds the service-role key, so every check above this +-- function is advisory by construction; and the scope→org BINDING lives in +-- here, where no transport can see it. 00068 decision 4 says the key +-- restriction beats the binding, and this is the only place that sentence +-- can be true. +-- +-- 2. `lorekit_memory_scopes` — the scope catalog. Filtering reads without +-- filtering this one would leave scoping leaking exactly what it hides: a +-- scope string IS a repo or project name, so a key narrowed to one repo +-- could still enumerate every repo on the account. +-- +-- 3. `lorekit_memory_activity`, 4. `lorekit_read_activity`, 5. `lorekit_memory_tags` +-- and 6. `lorekit_memory_facets` — the other four per-scope aggregates. Each +-- returns one row per scope, label or repo NAME, and `_facets`'s +-- `origin_repo` is a repository name outright — so narrowing only +-- `lorekit_memory_scopes` would move the leak to the charts and the filter +-- pills rather than close it. `lorekit_usage_stats` is deliberately NOT in +-- this list: it rolls up by the bounded `scope_type`, never a name. +-- +-- 7. `memory_delete` — the delete twin of `memory_write`. Its org branch picks +-- its own rows, so no transport-side filter can reach them. +-- +-- 8. `lorekit_memory_list` — the list read itself. 00067 (merged to main +-- while this branch was open) moved it out of PostgREST and into a SQL +-- function, which took the whole family out of `applyRestTenantScope`'s +-- reach. Ungated it would be the one read a scoped key still sees whole. +-- +-- And `lorekit_api_token_scope_allowed` itself is re-issued (body section 8) so +-- a stored pattern outside `SCOPE_PATTERN`'s shape is dropped rather than +-- widening the key. +-- +-- All of them take the calling key's restriction as DEFAULTED trailing parameters, so +-- every existing caller — a dashboard JWT, the Node `mcp-server`, CI on the +-- service role — keeps the pre-00069 behaviour with no call-site change, and an +-- unscoped key is byte-for-byte unaffected. +-- +-- ── Why the parameters are trustworthy ──────────────────────────────────── +-- +-- "Caller says which key it is" would be no boundary at all. It is safe here +-- for the same reason `p_user_id` is safe in 00046: the ONLY caller that can +-- reach these functions with a restriction attached is the edge, on a verified +-- service-role connection, and it reads the restriction out of `api_tokens` by +-- token hash microseconds earlier. An `authenticated` caller cannot forge a +-- WIDER restriction than it already has, because the widest possible value is +-- the default — passing `('all', '{}')` is what every non-key caller already +-- does. The failure mode of a forged value is therefore self-denial. +-- +-- ── Why the changed signatures are DROPped first ────────────────────────── +-- +-- Adding a defaulted parameter does not replace the old signature, it creates +-- an OVERLOAD — and then every existing call becomes ambiguous. Forward-only, +-- drop-then-create, grants re-issued (the 00026 / 00059 pattern). +-- ═════════════════════════════════════════════════════════════════════════ + +-- ───────────────────────────────────────────────────────────────────────────── +-- 1. memory_write +-- +-- Carried over verbatim from 00059 apart from the two new parameters and the +-- two `lorekit_api_token_org_allowed` guards they feed — the org branch (an +-- explicitly named org) and the binding branch (an auto-routed one). +-- ───────────────────────────────────────────────────────────────────────────── + +drop function if exists memory_write( + uuid, text, text, text, text[], text, text, timestamptz, text, + integer, boolean, text, text, text, integer, text, text +); + +create or replace function memory_write( + p_user_id uuid, + p_scope text, + p_key text, + p_value text, + p_tags text[] default '{}', + p_source_agent text default null, + p_trigger text default null, + p_created_at timestamptz default null, + p_org_slug text default null, + p_ttl_seconds integer default null, + p_clear_ttl boolean default false, + p_origin_repo text default null, + p_origin_branch text default null, + p_origin_commit text default null, + p_origin_pr integer default null, + p_kind text default null, + p_host text default null, + -- The CALLING KEYs restriction — BOTH axes, defaulted so every existing + -- caller (JWT, the Node path, CI service-role) keeps the pre-00069 behaviour + -- untouched. The scope allowlist is here for the same reason the tenancy is: + -- the edge holds the service-role key, so the dispatchers refusal is + -- advisory, and this is the last gate a write cannot go around. + p_key_scopes text[] default '{}', + p_key_org_access text default 'all', + p_key_org_ids uuid[] default '{}' +) +returns table ( + id uuid, + created_at timestamptz, + inserted boolean, + org_routed boolean, + binding_org_slug text, + expires_at timestamptz +) +language plpgsql +set search_path = public +as $$ +declare + v_org_id uuid; + v_binding_org uuid; + v_binding_slug text; + v_expires_at timestamptz; + v_ttl_action text := 'keep'; +begin + -- The scope allowlist, checked FIRST and for every branch. Both transports + -- already refuse a named scope outside the allowlist, but both run on the + -- service-role client, so those refusals are advisory by construction — this + -- is the one place on the write path that a caller cannot route around. + -- LK002, the same code the org denial raises, so `translateDbError` answers + -- the REST caller a 403 and the MCP caller a forbidden error without a second + -- mapping. + if not lorekit_api_token_scope_allowed(p_key_scopes, p_scope) then + raise exception using errcode = 'LK002', + message = format('key_scope_denied: scope=%s', p_scope); + end if; + + if p_clear_ttl then + v_ttl_action := 'clear'; + elsif p_ttl_seconds is not null then + if p_ttl_seconds < 1 or p_ttl_seconds > 31536000 then + raise exception using errcode = 'P0001', + message = format('ttl_seconds must be between 1 and 31536000, got %s', p_ttl_seconds); + end if; + v_expires_at := now() + (p_ttl_seconds * interval '1 second'); + v_ttl_action := 'set'; + end if; + + if p_org_slug is not null then + select o.id into v_org_id from orgs o where o.slug = p_org_slug and o.deleted_at is null; + if v_org_id is null then + raise exception using errcode = 'P0001', message = format('unknown_org: %s', p_org_slug); + end if; + if not lorekit_org_can(p_user_id, v_org_id, 'write') then + raise exception using errcode = 'LK002', message = format('org_permission_denied: org=%s', p_org_slug); + end if; + -- Defence in depth. The transport already refuses a key writing into an + -- org outside its tenancy, but this RPC is the LAST gate that cannot be + -- bypassed: the edge holds the service-role key, so every check above it + -- is advisory by construction. + if not lorekit_api_token_org_allowed(p_key_org_access, p_key_org_ids, v_org_id) then + raise exception using errcode = 'LK002', + message = format('key_org_denied: org=%s', p_org_slug); + end if; + else + select b.org_id, o.slug into v_binding_org, v_binding_slug + from org_scope_bindings b + join orgs o on o.id = b.org_id + where b.scope = p_scope and o.deleted_at is null; + + -- 00068 decision 4: THE KEY WINS OVER THE BINDING. Auto-routing is a + -- convenience; the keys tenancy is an authorization boundary, and a + -- boundary a convenience can widen is not one. A key that may not reach + -- the bound org falls back to a PERSONAL write — the same graceful + -- outcome a non-member already gets, and `binding_org_slug` still comes + -- back so the caller can surface the same actionable notice. + if v_binding_org is not null + and p_user_id is not null + and lorekit_org_can(p_user_id, v_binding_org, 'write') + and lorekit_api_token_org_allowed(p_key_org_access, p_key_org_ids, v_binding_org) then + v_org_id := v_binding_org; + end if; + end if; + + if v_org_id is not null then + return query + insert into memories ( + user_id, org_id, scope, key, value, tags, source_agent, trigger, + created_at, updated_at, created_by, updated_by, expires_at, + origin_repo, origin_branch, origin_commit, origin_pr, kind, host, seen_count + ) + values ( + null, v_org_id, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), + p_user_id, p_user_id, v_expires_at, + p_origin_repo, p_origin_branch, p_origin_commit, p_origin_pr, p_kind, p_host, 1 + ) + on conflict (org_id, scope, key) where org_id is not null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + updated_by = p_user_id, + seen_count = memories.seen_count + 1, + origin_repo = coalesce(excluded.origin_repo, memories.origin_repo), + origin_branch = coalesce(excluded.origin_branch, memories.origin_branch), + origin_commit = coalesce(excluded.origin_commit, memories.origin_commit), + origin_pr = coalesce(excluded.origin_pr, memories.origin_pr), + kind = coalesce(excluded.kind, memories.kind), + host = coalesce(excluded.host, memories.host), + expires_at = case v_ttl_action + when 'clear' then null + when 'set' then v_expires_at + else memories.expires_at + end + returning + memories.id, memories.created_at, (xmax::text = '0') as inserted, + true as org_routed, v_binding_slug as binding_org_slug, memories.expires_at; + + elsif p_user_id is null then + return query + insert into memories ( + user_id, scope, key, value, tags, source_agent, trigger, + created_at, updated_at, created_by, updated_by, expires_at, + origin_repo, origin_branch, origin_commit, origin_pr, kind, host, seen_count + ) + values ( + null, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), + null, null, v_expires_at, + p_origin_repo, p_origin_branch, p_origin_commit, p_origin_pr, p_kind, p_host, 1 + ) + on conflict (scope, key) where org_id is null and user_id is null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + seen_count = memories.seen_count + 1, + origin_repo = coalesce(excluded.origin_repo, memories.origin_repo), + origin_branch = coalesce(excluded.origin_branch, memories.origin_branch), + origin_commit = coalesce(excluded.origin_commit, memories.origin_commit), + origin_pr = coalesce(excluded.origin_pr, memories.origin_pr), + kind = coalesce(excluded.kind, memories.kind), + host = coalesce(excluded.host, memories.host), + expires_at = case v_ttl_action + when 'clear' then null + when 'set' then v_expires_at + else memories.expires_at + end + returning + memories.id, memories.created_at, (xmax::text = '0') as inserted, + false as org_routed, v_binding_slug as binding_org_slug, memories.expires_at; + + else + return query + insert into memories ( + user_id, scope, key, value, tags, source_agent, trigger, + created_at, updated_at, created_by, updated_by, expires_at, + origin_repo, origin_branch, origin_commit, origin_pr, kind, host, seen_count + ) + values ( + p_user_id, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), + p_user_id, p_user_id, v_expires_at, + p_origin_repo, p_origin_branch, p_origin_commit, p_origin_pr, p_kind, p_host, 1 + ) + on conflict (user_id, scope, key) where org_id is null and user_id is not null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + updated_by = p_user_id, + seen_count = memories.seen_count + 1, + origin_repo = coalesce(excluded.origin_repo, memories.origin_repo), + origin_branch = coalesce(excluded.origin_branch, memories.origin_branch), + origin_commit = coalesce(excluded.origin_commit, memories.origin_commit), + origin_pr = coalesce(excluded.origin_pr, memories.origin_pr), + kind = coalesce(excluded.kind, memories.kind), + host = coalesce(excluded.host, memories.host), + expires_at = case v_ttl_action + when 'clear' then null + when 'set' then v_expires_at + else memories.expires_at + end + returning + memories.id, memories.created_at, (xmax::text = '0') as inserted, + false as org_routed, v_binding_slug as binding_org_slug, memories.expires_at; + end if; +end; +$$; + +-- Grants re-issued after the DROP. The role list is carried over UNCHANGED from +-- 00059, `anon` included. That grant looks anomalous next to 00046, which +-- revoked `anon` from `memory_delete` and the four archive/purge RPCs and did +-- not touch `memory_write` — but tightening it here would be an unrelated +-- behaviour change smuggled into an authorization migration, and this function +-- is reachable on the BYOD/anon path. Left exactly as found, flagged for its +-- own change. +grant execute on function memory_write( + uuid, text, text, text, text[], text, text, timestamptz, text, + integer, boolean, text, text, text, integer, text, text, text[], text, uuid[] +) to anon, authenticated, service_role; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 2. lorekit_memory_scopes +-- +-- Carried over verbatim from 00065 apart from the three new parameters and the +-- two predicate calls in the WHERE clause. +-- +-- The org predicate is applied to `m.org_id`, which is NULL for a personal row — +-- and `lorekit_api_token_org_allowed` answers true for NULL under every tenancy, +-- so a `personal` key still sees its own scopes and only loses the org ones. +-- ───────────────────────────────────────────────────────────────────────────── + +drop function if exists lorekit_memory_scopes(uuid); + +create or replace function lorekit_memory_scopes( + p_user_id uuid, + p_key_scopes text[] default '{}', + p_key_org_access text default 'all', + p_key_org_ids uuid[] default '{}' +) +returns table (scope text, count bigint, last_activity timestamptz) +language plpgsql +stable +security definer +set search_path = public +as $$ +-- The RETURNS TABLE columns are also plpgsql OUT variables, so an unqualified +-- reference to any of them inside the query would be ambiguous. Every reference +-- below is table-qualified, and this directive makes the column win regardless. +#variable_conflict use_column +declare + -- Honour a caller-supplied p_user_id only on a verified service-role + -- connection (the edge resolves the token owner and passes it); an + -- authenticated caller is pinned to its own auth.uid(), closing the + -- cross-user scope-name enumeration. A NULL actor under service-role is the + -- CI escape hatch, preserved by the first visibility branch below. + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; +begin + return query + select m.scope, count(*) as count, max(m.created_at) as last_activity + from memories m + where ( + (v_actor is null and auth.role() = 'service_role') + or m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) + ) + -- The calling key's own restriction, on top of the caller's visibility. + -- Never instead of it: a key can only ever NARROW what its owner can + -- already see, so these are ANDed with the predicate above and the + -- account boundary stays exactly where 00046 put it. + and lorekit_api_token_scope_allowed(p_key_scopes, m.scope) + and lorekit_api_token_org_allowed(p_key_org_access, p_key_org_ids, m.org_id) + and m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + group by m.scope + order by count(*) desc, m.scope asc; +end; +$$; + +revoke execute on function lorekit_memory_scopes(uuid, text[], text, uuid[]) from public, anon; +grant execute on function lorekit_memory_scopes(uuid, text[], text, uuid[]) + to authenticated, service_role; + +comment on function lorekit_memory_scopes(uuid, text[], text, uuid[]) is + 'Per-scope active counts and last activity visible to the EFFECTIVE caller, + further narrowed by the CALLING KEY''s scope allowlist and tenancy (00068). + Ordered by count desc then scope asc (matching lorekit_memory_tags). Actor + resolved by the 00046/00041 service-role-gated rule: a caller-supplied + p_user_id is honoured only on a verified service-role connection, otherwise + auth.uid() wins — so an authenticated caller can never enumerate another + user''s scope names, and a scoped key can never enumerate scope names outside + its allowlist (a scope string IS a repo or project name). p_user_id IS NULL + under service-role is the CI escape hatch. The key parameters default to + unrestricted, so a non-key caller is unaffected. last_activity is + max(created_at) over the same counted rows.'; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 3. lorekit_memory_activity +-- 4. lorekit_read_activity +-- +-- The two remaining per-scope catalogs. `lorekit_memory_scopes` above is not +-- the only endpoint that returns one row per scope name: `GET /memories/activity` +-- and `GET /memories/read-activity` do too, and a scope string IS a repo or +-- project name — so narrowing the catalog while leaving the two activity series +-- unnarrowed would let a key restricted to one repo enumerate every repo on the +-- account through the charts instead of through the catalog. +-- +-- Same shape as `lorekit_memory_scopes`: defaulted trailing parameters, the two +-- 00068 predicates ANDed on top of the caller's own visibility (never instead +-- of it), and an unscoped key byte-for-byte unaffected. Narrowed HERE rather +-- than post-filtered in the edge for the reason the catalog is: the rows are +-- aggregates, so dropping rows afterwards would report counts that do not add +-- up, and a truncated response would under-report silently. +-- +-- `lorekit_read_activity` reads `usage_events`, which has no `org_id`, so it +-- takes the scope allowlist only — its visibility is already self-only (usage +-- is a per-user ledger, never org-shared) and there is no org axis to narrow. +-- Its `scope` column is NULLABLE (an unattributable read); a NULL scope names +-- nothing and so leaks nothing, and `lorekit_api_token_scope_allowed` is given +-- the explicit NULL pass below so the account total stays complete rather than +-- silently dropping every pre-00058 row for a scoped key. +-- ───────────────────────────────────────────────────────────────────────────── + +drop function if exists lorekit_memory_activity( + uuid, text, timestamptz, timestamptz, text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, text[], text +); + +create or replace function lorekit_memory_activity( + p_user_id uuid, + p_bucket text default 'day', + p_since timestamptz default null, + p_until timestamptz default null, + p_scope text default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in', + p_owner text[] default null, + p_owner_mode text default 'in', + -- The CALLING KEY's restriction (00068), defaulted to unrestricted. + p_key_scopes text[] default '{}', + p_key_org_access text default 'all', + p_key_org_ids uuid[] default '{}' +) +returns table (bucket timestamptz, scope text, count bigint) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + -- `origin_pr` is an INTEGER column: coerce the digits-only list numerically so + -- this route matches GET /memories on the same input (00057's rationale). A + -- non-numeric entry is dropped; a list reducing to empty applies no filter. + v_origin_pr integer[] := ( + select array_agg(x::integer) + from unnest(coalesce(p_origin_pr, '{}'::text[])) as x + where x ~ '^[0-9]+$' + ); +begin + if p_bucket is null or p_bucket not in ('hour', 'day') then + raise exception 'invalid bucket %, expected hour or day', p_bucket + using errcode = '22023'; + end if; + + return query + select date_trunc(p_bucket, m.created_at at time zone 'UTC') at time zone 'UTC' as bucket, + m.scope, + count(*) as count + from memories m + -- LEFT join for the owner predicate — a personal row has no org (00064). + left join orgs o on o.id = m.org_id + where ( + (v_actor is null and auth.role() = 'service_role') + or m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) + ) + -- The calling key's own restriction, ANDed on top of the caller's + -- visibility and never instead of it (the lorekit_memory_scopes rule). + and lorekit_api_token_scope_allowed(p_key_scopes, m.scope) + and lorekit_api_token_org_allowed(p_key_org_access, p_key_org_ids, m.org_id) + and m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + and (p_since is null or m.created_at >= p_since) + and (p_until is null or m.created_at < p_until) + -- Scope is a hard filter, always applied (00057's treatment). + and (p_scope is null or m.scope = p_scope) + -- Dimension filters — AND across, OR within, from the shared predicates + -- (00066) so the semantics cannot drift from lorekit_memory_facets. No + -- self-exclusion here: a straight count applies every one directly. + and lorekit_match_tags(m.tags, p_tags, p_tags_mode) + and lorekit_match_text(m.source_agent, p_source_agent, p_source_agent_mode) + and lorekit_match_text(m.trigger, p_trigger, p_trigger_mode) + and lorekit_match_text(m.kind, p_kind, p_kind_mode) + and lorekit_match_text(m.host, p_host, p_host_mode) + and lorekit_match_text(m.origin_repo, p_origin_repo, p_origin_repo_mode) + and lorekit_match_text(m.origin_branch, p_origin_branch, p_origin_branch_mode) + and lorekit_match_int (m.origin_pr, v_origin_pr, p_origin_pr_mode) + -- Owner: `personal` for org_id-null rows, else the org slug (00064). + and (p_owner is null or case coalesce(p_owner_mode, 'in') + when 'nin' then ( + (case when m.org_id is null then 'personal' else o.slug end) is not null + and (case when m.org_id is null then 'personal' else o.slug end) <> all(p_owner) + ) + else ( + ('personal' = any(p_owner) and m.org_id is null) + or (m.org_id is not null and o.slug = any(p_owner)) + ) + end) + group by 1, m.scope + order by 1 asc, m.scope asc; +end; +$$; + +revoke execute on function lorekit_memory_activity( + uuid, text, timestamptz, timestamptz, text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[] +) from public, anon; +grant execute on function lorekit_memory_activity( + uuid, text, timestamptz, timestamptz, text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[] +) to authenticated, service_role; + +comment on function lorekit_memory_activity( + uuid, text, timestamptz, timestamptz, text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[] +) is + 'Memories created per UTC hour/day per scope over the half-open + [p_since, p_until) window, visible to the EFFECTIVE caller, narrowed by the + optional scope + dimension filters (00063) plus the owner dimension (00064, + `personal` / org slug), and further narrowed by the CALLING KEY''s scope + allowlist and tenancy (00068) — this series returns one row per scope NAME, + so leaving it unnarrowed would leak exactly what lorekit_memory_scopes hides. + The eight text/tags/int dimension predicates come from lorekit_match_text / + _tags / _int (00066). The key parameters default to unrestricted, so a + non-key caller is unaffected and the result stays byte-for-byte 00066''s.'; + +drop function if exists lorekit_read_activity(uuid, text, timestamptz, timestamptz, text); + +create or replace function lorekit_read_activity( + p_user_id uuid, + p_bucket text default 'day', + p_since timestamptz default null, + p_until timestamptz default null, + p_scope text default null, + -- The CALLING KEY's scope allowlist (00068). No org parameters: `usage_events` + -- is a per-user ledger with no org_id, so there is no tenancy axis to narrow. + p_key_scopes text[] default '{}' +) +returns table (bucket timestamptz, scope text, count bigint) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; +begin + if p_bucket is null or p_bucket not in ('hour', 'day') then + raise exception 'invalid bucket %, expected hour or day', p_bucket + using errcode = '22023'; + end if; + + return query + select date_trunc(p_bucket, ue.created_at at time zone 'UTC') at time zone 'UTC' as bucket, + ue.scope as scope, + sum(coalesce(ue.result_count, 0))::bigint as count + from usage_events ue + where ( + (v_actor is null and auth.role() = 'service_role') + or ue.user_id = v_actor + ) + and ue.tool_name in ('memory.read', 'memory.list', 'memory.search', + 'memory.list_archived') + -- The dashboard reading lore in order to DRAW this chart is not a read + -- the chart should report. `is distinct from` (not `<>`) because the + -- column is nullable and `null <> 'dashboard'` is null, which would + -- silently drop every unattributed event — including every row written + -- before this migration. + and ue.client is distinct from 'dashboard' + -- The calling key's scope allowlist (00068). A NULL scope is an + -- unattributable read: it names nothing, so it leaks nothing, and it is + -- passed through rather than dropped so the account total a scoped key + -- sees stays the sum of the rows it is allowed to attribute plus the + -- ones nobody could. Dropping them would silently erase every event + -- written before 00058 from a scoped key's chart. + and (ue.scope is null or lorekit_api_token_scope_allowed(p_key_scopes, ue.scope)) + -- The optional per-scope filter. `=`, not `is not distinct from`: a + -- caller asking for a named scope wants events attributed to it, never + -- the unattributable NULL-scope remainder. Omitting the parameter + -- returns every scope INCLUDING those NULL rows, so the unfiltered + -- account total stays complete. + and (p_scope is null or ue.scope = p_scope) + and (p_since is null or ue.created_at >= p_since) + and (p_until is null or ue.created_at < p_until) + group by 1, ue.scope + having sum(coalesce(ue.result_count, 0)) > 0 + -- The scope tiebreak is 00051's verbatim (`order by 1 asc, m.scope asc`). + -- Bucket alone stopped being a total order the moment a bucket could hold + -- several scopes, and this function's header claims it mirrors that shape, + -- so intra-bucket row order must be deterministic here too. + order by 1 asc, ue.scope asc; +end; +$$; + +revoke execute on function lorekit_read_activity(uuid, text, timestamptz, timestamptz, text, text[]) + from public, anon; +grant execute on function lorekit_read_activity(uuid, text, timestamptz, timestamptz, text, text[]) + to authenticated, service_role; + +comment on function lorekit_read_activity(uuid, text, timestamptz, timestamptz, text, text[]) is + 'Memory RECORDS read per UTC hour/day per scope over the half-open + [p_since, p_until) window, summed from usage_events.result_count over the + read tools, excluding dashboard-originated reads. Further narrowed by the + CALLING KEY''s scope allowlist (00068): this series returns one row per scope + NAME, so leaving it unnarrowed would leak what lorekit_memory_scopes hides. + A NULL scope is unattributable, names nothing and is passed through, so the + account total stays complete. No org parameters — usage_events is a per-user + ledger with no org axis. The key parameter defaults to unrestricted, so a + non-key caller sees byte-for-byte 00058''s result.'; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 5. memory_delete +-- +-- The delete twin of `memory_write`, and the last gate on the other mutation +-- path the transports cannot stand in front of. `DELETE /memories?…&org=` and +-- the MCP `memory.delete` tool both route their org form through this function, +-- which chooses the rows itself — so there is no query left for the edge to +-- filter, and a transport-side check is advisory anyway (service-role client). +-- +-- Without the key parameters the org branch enforced `lorekit_org_can` and +-- nothing else: a key whose tenancy is `personal`, or `selected` over a +-- different org, could still hard-delete an org-owned memory as long as its +-- OWNER held the capability. The scope axis is added for the same +-- defence-in-depth reason it was added to `memory_write` — both dispatchers +-- refuse a named scope outside the allowlist, and both are advisory. +-- +-- Actor resolution, the capability gates and the personal branch are 00046 +-- verbatim; only the two guards and the three defaulted parameters are new. +-- ───────────────────────────────────────────────────────────────────────────── + +drop function if exists memory_delete(uuid, text, text, text, boolean); + +create or replace function memory_delete( + p_user_id uuid, + p_org_slug text default null, + p_scope text default null, + p_key text default null, + p_force boolean default false, + -- The CALLING KEY's restriction (00068), defaulted to unrestricted so every + -- existing caller — dashboard JWT, the Node path, CI on the service role — + -- keeps the pre-00069 behaviour with no call-site change. + p_key_scopes text[] default '{}', + p_key_org_access text default 'all', + p_key_org_ids uuid[] default '{}' +) +returns table (deleted boolean, archived boolean) +language plpgsql +security definer +set search_path = public +as $$ +declare + v_org_id uuid; + v_count integer; + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; +begin + -- The scope allowlist, before either branch. LK002 for the same reason + -- `memory_write` uses it: `translateDbError` already maps it to a 403, so no + -- second mapping is needed on either surface. + if not lorekit_api_token_scope_allowed(p_key_scopes, p_scope) then + raise exception using errcode = 'LK002', + message = format('key_scope_denied: scope=%s', p_scope); + end if; + + if p_org_slug is not null then + select o.id into v_org_id from orgs o where o.slug = p_org_slug; + if v_org_id is null then + raise exception using + errcode = 'P0001', + message = format('unknown_org: %s', p_org_slug); + end if; + + -- The tenancy half. Checked BEFORE the capability gates so a key that may + -- not reach this org is refused on the key, not on its owner's role — the + -- two denials are different facts and `key_org_denied` is the true one. + if not lorekit_api_token_org_allowed(p_key_org_access, p_key_org_ids, v_org_id) then + raise exception using errcode = 'LK002', + message = format('key_org_denied: org=%s', p_org_slug); + end if; + + if p_force then + if not lorekit_org_can(v_actor, v_org_id, 'hard_delete') then + raise exception using + errcode = 'LK002', + message = format('org_permission_denied: org=%s capability=hard_delete', p_org_slug); + end if; + + delete from memories + where org_id = v_org_id and scope = p_scope and key = p_key; + get diagnostics v_count = row_count; + + return query select (v_count > 0), false; + else + if not lorekit_org_can(v_actor, v_org_id, 'archive') then + raise exception using + errcode = 'LK002', + message = format('org_permission_denied: org=%s capability=archive', p_org_slug); + end if; + + update memories + set archived_at = now() + where org_id = v_org_id and scope = p_scope and key = p_key and archived_at is null; + get diagnostics v_count = row_count; + + return query select false, (v_count > 0); + end if; + else + -- Personal delete: scoped to the RESOLVED actor, never a named p_user_id. + if p_force then + delete from memories + where user_id = v_actor and scope = p_scope and key = p_key; + get diagnostics v_count = row_count; + + return query select (v_count > 0), false; + else + update memories + set archived_at = now() + where user_id = v_actor and scope = p_scope and key = p_key and archived_at is null; + get diagnostics v_count = row_count; + + return query select false, (v_count > 0); + end if; + end if; +end; +$$; + +-- 00046's revoke is carried over verbatim against the NEW signature: 00020 +-- granted `anon` EXPLICITLY, and `revoke … from public` does not remove a +-- per-role grant, so anon must be named again here or a re-created function +-- silently comes back reachable. +revoke execute on function memory_delete(uuid, text, text, text, boolean, text[], text, uuid[]) + from public, anon; +grant execute on function memory_delete(uuid, text, text, text, boolean, text[], text, uuid[]) + to authenticated, service_role; + +comment on function memory_delete(uuid, text, text, text, boolean, text[], text, uuid[]) is + 'Archives (p_force=false) or hard-deletes (p_force=true) the effective + caller''s own memory, or an org''s memory when p_org_slug is given and the + actor holds the archive / hard_delete capability. Further gated by the + CALLING KEY''s scope allowlist and tenancy (00068): the org branch chooses + its rows inside this function, so no transport-side filter can cover it, and + both dispatchers run on the service-role client and are advisory. The key + parameters default to unrestricted, so a non-key caller is unaffected.'; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 6. lorekit_memory_tags +-- 7. lorekit_memory_facets +-- +-- The last two aggregate catalogs on the memories surface. `lorekit_memory_tags` +-- rolls up labels and `lorekit_memory_facets` rolls up every filterable +-- dimension — including `origin_repo`, which is a repository name by +-- construction, so a key restricted to one repo could enumerate every repo on +-- the account through the facet list even with the scope catalog and both +-- activity series narrowed. Narrowing four of six catalogs moves a leak; it does +-- not close one. +-- +-- `GET /memories/usage` is deliberately NOT in this list: `lorekit_usage_stats` +-- rolls up by `scope_type`, a bounded low-cardinality value (`repo`, `project`, +-- `global`), never a scope NAME, so there is nothing there to enumerate. +-- +-- Both take the two predicates in the row-visibility WHERE, where every value +-- these functions emit is derived from — one place per function, rather than +-- one per emitted facet. Bodies are 00050 / 00066 verbatim otherwise. +-- ───────────────────────────────────────────────────────────────────────────── + +drop function if exists lorekit_memory_tags(uuid, boolean); + +create or replace function lorekit_memory_tags( + p_user_id uuid, + p_archived boolean default false, + p_key_scopes text[] default '{}', + p_key_org_access text default 'all', + p_key_org_ids uuid[] default '{}' +) +returns table (tag text, count bigint) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; +begin + return query + select t.tag, count(*) as count + from memories m + -- unnest, not a GIN-assisted filter: the catalog is a full rollup of a + -- text[] column, so every visible row contributes each of its labels. + cross join lateral unnest(m.tags) as t(tag) + where ( + (v_actor is null and auth.role() = 'service_role') + or m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) + ) + -- The calling key's restriction, ANDed on top of the caller's own + -- visibility and never instead of it (the lorekit_memory_scopes rule). + and lorekit_api_token_scope_allowed(p_key_scopes, m.scope) + and lorekit_api_token_org_allowed(p_key_org_access, p_key_org_ids, m.org_id) + and ( + case + when p_archived then m.archived_at is not null + else m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + end + ) + and t.tag is not null + and btrim(t.tag) <> '' + group by t.tag + order by count(*) desc, t.tag asc; +end; +$$; + +revoke execute on function lorekit_memory_tags(uuid, boolean, text[], text, uuid[]) + from public, anon; +grant execute on function lorekit_memory_tags(uuid, boolean, text[], text, uuid[]) + to authenticated, service_role; + +comment on function lorekit_memory_tags(uuid, boolean, text[], text, uuid[]) is + 'Label (memories.tags) catalog with per-label counts, for the partition + selected by p_archived, visible to the EFFECTIVE caller and further narrowed + by the CALLING KEY''s scope allowlist and tenancy (00068). Same + service-role-gated actor rule and tenant predicate as lorekit_memory_scopes. + Ordered count desc, tag asc. The key parameters default to unrestricted, so a + non-key caller sees byte-for-byte 00050''s result.'; + +drop function if exists lorekit_memory_facets( + uuid, boolean, text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text +); + +create or replace function lorekit_memory_facets( + p_user_id uuid, + p_archived boolean default false, + p_scope text default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in', + -- Owner dimension (00064). `personal` plus one slug per member org with + -- visible rows. All optional: null/absent = not filtered. + p_owner text[] default null, + p_owner_mode text default 'in', + -- The CALLING KEY's restriction (00068), defaulted to unrestricted. + p_key_scopes text[] default '{}', + p_key_org_access text default 'all', + p_key_org_ids uuid[] default '{}' +) +returns table (facet text, value text, count bigint) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + v_origin_pr integer[] := ( + select array_agg(x::integer) + from unnest(coalesce(p_origin_pr, '{}'::text[])) as x + where x ~ '^[0-9]+$' + ); +begin + return query + with base as ( + select + m.tags, m.source_agent, m.trigger, m.kind, m.host, + m.origin_repo, m.origin_branch, m.origin_pr, + m.org_id, o.slug as org_slug, + -- Per-dimension match flag, now from the shared predicates so it cannot + -- drift from lorekit_memory_activity's. A null filter is "not filtered" → + -- the helper returns true, so an untouched dimension never narrows. + lorekit_match_tags(m.tags, p_tags, p_tags_mode) as ok_tag, + lorekit_match_text(m.source_agent, p_source_agent, p_source_agent_mode) as ok_source_agent, + lorekit_match_text(m.trigger, p_trigger, p_trigger_mode) as ok_trigger, + lorekit_match_text(m.kind, p_kind, p_kind_mode) as ok_kind, + lorekit_match_text(m.host, p_host, p_host_mode) as ok_host, + lorekit_match_text(m.origin_repo, p_origin_repo, p_origin_repo_mode) as ok_origin_repo, + lorekit_match_text(m.origin_branch, p_origin_branch, p_origin_branch_mode) as ok_origin_branch, + lorekit_match_int (m.origin_pr, v_origin_pr, p_origin_pr_mode) as ok_origin_pr, + -- Owner: the computed identity is `personal` (org_id null) or the org slug. + -- Stays inline — it is not one of the three column helpers (00064). + (p_owner is null or case coalesce(p_owner_mode, 'in') + when 'nin' then ( + (case when m.org_id is null then 'personal' else o.slug end) is not null + and (case when m.org_id is null then 'personal' else o.slug end) <> all(p_owner) + ) + else ( + ('personal' = any(p_owner) and m.org_id is null) + or (m.org_id is not null and o.slug = any(p_owner)) + ) + end) as ok_owner + from memories m + -- A personal row has no org, so this is a LEFT join; org rows resolve to + -- their slug. Visible org rows are always the caller's own orgs (the + -- visibility predicate below admits them only via lorekit_member_org_ids), + -- so `o.slug` is never a slug the caller cannot see. + left join orgs o on o.id = m.org_id + where ( + (v_actor is null and auth.role() = 'service_role') + or m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) + ) + -- The calling key's restriction, applied ONCE here in the row-visibility + -- predicate every emitted facet value is derived from — `origin_repo` is + -- a repository name by construction, so an unnarrowed facet list leaks + -- exactly what the scope catalog hides. + and lorekit_api_token_scope_allowed(p_key_scopes, m.scope) + and lorekit_api_token_org_allowed(p_key_org_access, p_key_org_ids, m.org_id) + and ( + case + when p_archived then m.archived_at is not null + else m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + end + ) + and (p_scope is null or m.scope = p_scope) + ), cells as ( + select 'tag'::text as facet, t.tag as value + from base b + cross join lateral unnest(b.tags) as t(tag) + where b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner + union all + select 'source_agent', b.source_agent from base b + where b.ok_tag and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner + union all + select 'trigger', b.trigger from base b + where b.ok_tag and b.ok_source_agent and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner + union all + select 'kind', b.kind from base b + where b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner + union all + select 'host', b.host from base b + where b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner + union all + select 'origin_repo', b.origin_repo from base b + where b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner + union all + select 'origin_branch', b.origin_branch from base b + where b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_pr and b.ok_owner + union all + select 'origin_pr', b.origin_pr::text from base b + where b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_owner + union all + -- Owner is the ONE dimension self-excluded here (every other flag, NOT + -- ok_owner), so a drilled-in owner still lists the alternative owner. + select 'owner', case when b.org_id is null then 'personal' else b.org_slug end from base b + where b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr + ) + select c.facet, c.value, count(*) as count + from cells c + where c.value is not null + and btrim(c.value) <> '' + group by c.facet, c.value + order by c.facet asc, count(*) desc, c.value asc; +end; +$$; + +revoke execute on function lorekit_memory_facets( + uuid, boolean, text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[] +) from public, anon; +grant execute on function lorekit_memory_facets( + uuid, boolean, text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[] +) to authenticated, service_role; + +comment on function lorekit_memory_facets( + uuid, boolean, text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[] +) is + 'Value catalog with counts for every filterable memory dimension (tag, + source_agent, trigger, kind, host, origin_repo, origin_branch, origin_pr, + owner) over the partition selected by p_archived, visible to the EFFECTIVE + caller and further narrowed by the CALLING KEY''s scope allowlist and tenancy + (00068) — `origin_repo` is a repository name, so an unnarrowed facet list + would leak what lorekit_memory_scopes hides. `owner` (00064) is `personal` + for org_id-null rows, else the org slug. Counts are DRILL-DOWN (00057). The + key parameters default to unrestricted, so a non-key caller sees + byte-for-byte 00066''s result.'; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 8. lorekit_api_token_scope_allowed — the same shape test as the edge +-- +-- 00068 treated any pattern ending in `*` as a prefix wildcard. The authority +-- on what a pattern may look like is `SCOPE_PATTERN` in `schemas/api-key.ts` +-- (and `api_tokens_scopes_shape`, which mirrors it): a `*` is a wildcard only +-- directly after `/` or `::`. Under the looser rule a stored `repo::mthines/lore*` +-- became the LIKE prefix `repo::mthines/lore%` and reached every repo starting +-- with those letters — a pattern that WIDENS the key, which is the one direction +-- these predicates must never move. +-- +-- Both guards exist because the column can hold a value the CHECK never saw: a +-- BYOD install bootstrapped before 00068, or a constraint dropped by hand. A +-- non-conforming pattern now contributes nothing, exactly as the edge's +-- `keyScopeFilter` drops it — and a key whose patterns ALL fail the shape test +-- matches no row, which is the fail-closed answer for "restricted, with no +-- usable patterns". +-- +-- `create or replace` with an unchanged signature, so the 00068 grants stand. +-- ───────────────────────────────────────────────────────────────────────────── + +create or replace function lorekit_api_token_scope_allowed( + p_patterns text[], + p_scope text +) +returns boolean +language sql +immutable +as $$ + select case + -- Decision 1: an unrestricted key is the default and the common case, so it + -- is the first branch and costs no array walk. + when p_patterns is null or cardinality(p_patterns) = 0 then true + -- A scopeless operation (memory.purge_expired, the account-wide reads) + -- cannot be matched against an allowlist. Refusing it is the fail-closed + -- answer: a key narrowed to one repo has no business sweeping the account. + when p_scope is null then false + else exists ( + select 1 + from unnest(p_patterns) as pattern + -- SCOPE_PATTERN's shape, verbatim. A pattern that fails it is dropped + -- rather than matched literally, so it can only ever narrow. + where pattern ~ '^[a-z0-9._:/-]+((/|::)\*)?$' + and case + when right(pattern, 1) = '*' + -- Escape LIKE's single-character wildcard in the literal prefix so + -- `repo::my_org/*` stays owner-exact instead of also matching + -- `repo::myXorg/...`. `%` and `\` cannot occur — the CHECK's charset + -- excludes them. Same reasoning, same escape, as expandScopeForSearch. + then p_scope like replace(left(pattern, -1), '_', '\_') || '%' + else p_scope = pattern + end + ) + end; +$$; + +comment on function lorekit_api_token_scope_allowed(text[], text) is + 'May a key whose api_tokens.scopes is p_patterns touch p_scope? Empty ' + 'allowlist = yes (unrestricted). NULL scope = no (fail closed). A pattern ' + 'outside SCOPE_PATTERN''s shape is DROPPED, not matched, so a stored ' + 'mid-token wildcard cannot widen the key (00069).'; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 9. lorekit_memory_list +-- +-- 00067 (merged to main while this branch was open) moved the list read out of +-- PostgREST and into a SQL function, so a wide filter never becomes a URL. That +-- relocated the ENTIRE list read behind a predicate this migration's transports +-- cannot reach: `applyRestTenantScope` no longer runs on that path, and the +-- function's own visibility predicate knows nothing about a key restriction. +-- Left alone, `GET /memories` and `POST /memories/list` would be the one +-- family a scoped key still reads unnarrowed — the widest possible hole, in the +-- most-used endpoint. +-- +-- Carried over verbatim from 00067 apart from the three parameters and the two +-- predicate calls they feed. DROP-then-create for the usual reason: a defaulted +-- parameter is an overload, not a replacement. +-- ───────────────────────────────────────────────────────────────────────────── + +drop function if exists lorekit_memory_list( + uuid, boolean, text, text, text, text, timestamptz, timestamptz, timestamptz, + timestamptz, text[], text, text[], text, text[], text, text[], text, text[], + text, text[], text, text[], text, text[], text, text[], text, text, + timestamptz, uuid, integer +); + +create or replace function lorekit_memory_list( + p_user_id uuid, + p_archived boolean default false, + p_scope text default null, + p_key text default null, + -- Already LIKE-escaped by `likeNeedle`; the trailing `%` is added here and is + -- the only active wildcard. + p_key_prefix text default null, + -- Already LIKE-escaped by `likeNeedle`; wrapped in `%…%` here. + p_q text default null, + p_created_since timestamptz default null, + p_created_until timestamptz default null, + -- The "expiring soon" window, precomputed by `expiringWindow`: the half-open + -- `(after, on_or_before]` pair. Both null = the filter is not applied. + p_expires_after timestamptz default null, + p_expires_on_or_before timestamptz default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in', + p_owner text[] default null, + p_owner_mode text default 'in', + p_sort text default 'updated_at', + -- Keyset cursor: the sort value and id of the last row of the previous page. + -- Both null = first page. The edge decodes and shape-validates the opaque + -- cursor (`_shared/api/paginate.ts`) before splitting it into these two. + p_cursor_ts timestamptz default null, + p_cursor_id uuid default null, + -- The caller passes limit + 1 and derives `hasMore` from the overflow row, + -- exactly as the PostgREST path did — the pagination contract is unchanged. + p_limit integer default 51, + -- The CALLING KEY's restriction (00069). Defaulted, so every existing caller + -- — a dashboard JWT, CI on the service role — keeps the pre-00070 behaviour. + p_key_scopes text[] default '{}', + p_key_org_access text default 'all', + p_key_org_ids uuid[] default '{}' +) +returns table ( + id uuid, + scope text, + key text, + value text, + tags text[], + source_agent text, + trigger text, + created_at timestamptz, + updated_at timestamptz, + expires_at timestamptz, + archived_at timestamptz, + origin_repo text, + origin_branch text, + origin_commit text, + origin_pr integer, + kind text, + host text, + seen_count integer, + org_id uuid, + created_by uuid, + updated_by uuid, + -- The `orgs(id,name,slug)` embed, flattened. `shapeMemoryRow` folds these + -- three back into the nested `org` object the API returns, so the response + -- body is byte-identical to the PostgREST path's. + org_name text, + org_slug text +) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + -- Same service-role-gated actor rule as lorekit_memory_facets / + -- lorekit_memory_activity: a service_role caller may act as a named user, and + -- anyone else is themselves regardless of what they passed. + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + -- `origin_pr` is an integer column and the wire form is text. A non-numeric + -- entry is DROPPED, never an error: the filter can be built from a + -- hand-editable URL, and one bad entry should narrow the filter rather than + -- break the page. + -- + -- The digit count is BOUNDED, which a bare `^[0-9]+$` is not: `99999999999` + -- is all digits and still overflows int4, so `x::integer` would raise 22003 + -- and turn a hand-editable filter value into a 500 — the exact opposite of + -- the drop this comment promises. Nine significant digits is the widest run + -- that cannot overflow, and the leading `0*` keeps the zero-padded form + -- working (`007` → 7, as GET /memories has always resolved it). The guard is + -- a regex rather than a `x::numeric <= 2147483647` conjunct because WHERE + -- conjuncts have no guaranteed evaluation order, so a cast there could still + -- see a non-numeric value and raise 22P02. + v_origin_pr integer[] := ( + select array_agg(x::integer) + from unnest(coalesce(p_origin_pr, '{}'::text[])) as x + where x ~ '^0*[0-9]{1,9}$' + ); + -- Whitelisted identifier — see the header. Anything unrecognised degrades to + -- the route's default order instead of raising. + v_sort text := case when p_sort = 'created_at' then 'created_at' else 'updated_at' end; +begin + return query execute format($q$ + select + m.id, m.scope, m.key, m.value, m.tags, m.source_agent, m.trigger, + m.created_at, m.updated_at, m.expires_at, m.archived_at, + m.origin_repo, m.origin_branch, m.origin_commit, m.origin_pr, + m.kind, m.host, m.seen_count, + m.org_id, m.created_by, m.updated_by, + o.name as org_name, o.slug as org_slug + from memories m + -- LEFT: a personal row has no org. Visible org rows are only ever the + -- caller's own (the visibility predicate admits them via + -- lorekit_member_org_ids), so o.slug is never a slug they cannot see. + left join orgs o on o.id = m.org_id + where ( + ($1 is null and auth.role() = 'service_role') + or m.user_id = $1 + or m.org_id in (select lorekit_member_org_ids($1)) + ) + -- The calling key's own restriction, ANDed onto the caller's visibility + -- and never instead of it: a key can only NARROW what its owner already + -- sees, so the account boundary above is untouched. This is the same + -- pair every other key-aware reader applies; it is here because 00067 + -- moved the list read into this function, and a reader the enforcement + -- does not reach is a reader that leaks. + and lorekit_api_token_scope_allowed($32, m.scope) + and lorekit_api_token_org_allowed($33, $34, m.org_id) + -- The archived partition. The live branch additionally hides an expired + -- row; the archived branch has no liveness guard, which is why the + -- expiring-soon window below re-states its own lower bound. + and ( + case + when $2 then m.archived_at is not null + else m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + end + ) + and ($3 is null or m.scope = $3) + and ($4 is null or m.key = $4) + -- Prefix match on key. The needle arrives LIKE-escaped, so the `%%` + -- appended here is the only active wildcard. Doubled because this + -- comment is inside the format() template: an undoubled percent there + -- is read as a type specifier and raises, exactly like the predicates + -- below. + and ($5 is null or m.key ilike $5 || '%%') + -- Substring over key OR value, the `q` filter. Same escaped-needle + -- contract; both wildcards are added here. + and ($6 is null or m.key ilike '%%' || $6 || '%%' or m.value ilike '%%' || $6 || '%%') + -- Half-open [since, until) creation window. + and ($7 is null or m.created_at >= $7) + and ($8 is null or m.created_at < $8) + -- "Expiring soon": `expires_at` in (after, on_or_before]. TWO predicates, + -- not three — a row with no TTL fails both comparisons on its own, + -- because `null > x` is null, so no `is not null` guard is needed. Both + -- are plain conjuncts, so this narrows the partition above rather than + -- widening it. Range-scans memories_expires_at_idx (00030). + and ($9 is null or m.expires_at > $9) + and ($10 is null or m.expires_at <= $10) + -- The dimension predicates, from the shared helpers (00066), so this + -- reader cannot drift from lorekit_memory_facets or _activity. AND + -- across dimensions, OR within one. A null filter is "not filtered". + and lorekit_match_tags(m.tags, $11, $12) + and lorekit_match_text(m.source_agent, $13, $14) + and lorekit_match_text(m.trigger, $15, $16) + and lorekit_match_text(m.kind, $17, $18) + and lorekit_match_text(m.host, $19, $20) + and lorekit_match_text(m.origin_repo, $21, $22) + and lorekit_match_text(m.origin_branch, $23, $24) + and lorekit_match_int (m.origin_pr, $25, $26) + -- Owner (00064): the computed identity `personal` (org_id null) or the + -- org slug. Inline rather than a helper because it is not a plain + -- column — identical expression to the other two readers'. + and ($27 is null or case coalesce($28, 'in') + when 'nin' then ( + (case when m.org_id is null then 'personal' else o.slug end) is not null + and (case when m.org_id is null then 'personal' else o.slug end) <> all($27) + ) + else ( + ('personal' = any($27) and m.org_id is null) + or (m.org_id is not null and o.slug = any($27)) + ) + end) + -- Keyset. Strictly after the previous page's last row in the composite + -- (sort, id) order below, which is why the tie-break on id is part of + -- the predicate and not just the ordering. + and ( + $29 is null + or m.%1$I < $29 + or (m.%1$I = $29 and m.id < $30) + ) + order by m.%1$I desc, m.id desc + limit $31 + $q$, v_sort) + using + v_actor, coalesce(p_archived, false), p_scope, p_key, p_key_prefix, p_q, + p_created_since, p_created_until, p_expires_after, p_expires_on_or_before, + p_tags, p_tags_mode, + p_source_agent, p_source_agent_mode, + p_trigger, p_trigger_mode, + p_kind, p_kind_mode, + p_host, p_host_mode, + p_origin_repo, p_origin_repo_mode, + p_origin_branch, p_origin_branch_mode, + v_origin_pr, p_origin_pr_mode, + p_owner, p_owner_mode, + p_cursor_ts, p_cursor_id, + greatest(coalesce(p_limit, 51), 1), + coalesce(p_key_scopes, '{}'::text[]), + coalesce(p_key_org_access, 'all'), + coalesce(p_key_org_ids, '{}'::uuid[]); +end; +$$; + +revoke execute on function lorekit_memory_list( + uuid, boolean, text, text, text, text, timestamptz, timestamptz, timestamptz, + timestamptz, text[], text, text[], text, text[], text, text[], text, text[], + text, text[], text, text[], text, text[], text, text[], text, text, + timestamptz, uuid, integer, text[], text, uuid[] +) from public, anon; +grant execute on function lorekit_memory_list( + uuid, boolean, text, text, text, text, timestamptz, timestamptz, timestamptz, + timestamptz, text[], text, text[], text, text[], text, text[], text, text[], + text, text[], text, text[], text, text[], text, text[], text, text, + timestamptz, uuid, integer, text[], text, uuid[] +) to authenticated, service_role; + +comment on function lorekit_memory_list( + uuid, boolean, text, text, text, text, timestamptz, timestamptz, timestamptz, + timestamptz, text[], text, text[], text, text[], text, text[], text, text[], + text, text[], text, text[], text, text[], text, text[], text, text, + timestamptz, uuid, integer, text[], text, uuid[] +) is + 'One keyset page of the memories visible to the EFFECTIVE caller, further + narrowed by the CALLING KEY''s scope allowlist and tenancy (00069). Otherwise + identical to the function 00067 introduced; the key parameters default to + unrestricted, so a non-key caller is unaffected.'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00069', 'api_token_scoping_enforcement') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00070_audit_log_api_key_scope_change.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- API key scoping, part 3: the audit vocabulary. +-- +-- 00068 gave a key a scope allowlist and a tenancy; 00069 enforced them. Both +-- are AUTHORIZATION state, so changing them has to leave a trail — and today it +-- cannot: `audit_log_action_check` has admitted `api_key.create` and +-- `api_key.revoke` since 00010 and nothing else on that resource. +-- +-- The failure mode this avoids is the one 00042 documents at length: +-- `recordAuditEvent` is deliberately non-throwing (a failed audit must never +-- break the operation it audits), so a call with an action the CHECK rejects is +-- swallowed and logged to the server console only. The result is a silent, +-- permanent hole in the trail — no error anyone sees, and no row. Widening the +-- CHECK BEFORE the call site ships is the whole fix. +-- +-- One action, not two (`api_key.scope_change` rather than a set/clear pair): +-- clearing a key's scoping is the same operation with an empty argument, and +-- the metadata carries the before/after. A vocabulary that splits on the VALUE +-- of a field rather than the operation grows a term every time the field grows +-- a state. +-- +-- Forward-only: drop + re-add (a CHECK cannot be widened in place) — the +-- pattern of 00023, 00027 and 00042. The list below is derived from the ONE +-- source, `packages/schemas/src/audit.ts`'s `AUDIT_ACTIONS`, and +-- `packages/mcp-core/src/audit-vocabulary.spec.ts` parses the newest +-- action-CHECK migration (this file) and fails if the two sets differ. + +alter table audit_log drop constraint audit_log_action_check; + +alter table audit_log add constraint audit_log_action_check check (action in ( + 'api_key.create', + 'api_key.revoke', + 'api_key.scope_change', + 'webhook_secret.create', + 'webhook_secret.rotate', + 'webhook_secret.deactivate', + 'memory.create', + 'memory.update', + 'memory.archive', + 'memory.restore', + 'memory.delete', + 'limit.override', + 'org.create', + 'org.rename', + 'org.delete', + 'member.invite', + 'member.accept', + 'member.decline', + 'member.revoke', + 'member.remove', + 'member.role_change', + 'member.leave', + 'scope.bind', + 'scope.unbind', + 'github_app.installation_linked' +)); + +insert into supabase_migrations.schema_migrations (version, name) +values ('00070', 'audit_log_api_key_scope_change') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00071_api_token_scope_authorized_removal.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- API-token scope-authorized removal, and a not-found / not-permitted signal. +-- +-- ── The problem ────────────────────────────────────────────────────────── +-- Since 00046 every memory-removal path is pinned to the resolved actor's OWN +-- rows: `where user_id = v_actor` (personal) or an org capability (org). An +-- `lk_*` API token therefore reaches the store on the service-role connection +-- (getDb: "api_key ... use service-role") yet can only remove rows it wrote. +-- The two MCP tools that back `memory.archive` and personal `memory.delete` +-- (`toolArchive`, `toolDelete`) run a raw `.update()/.delete()` filtered by +-- `.eq('user_id', userId)` and ignore the key's scope allowlist entirely, so a +-- key deliberately SCOPED to `repo::x` still cannot manage rows in `repo::x` +-- that another principal wrote. Worse, a 0-row result returns +-- `{archived:false}` — indistinguishable from "no such memory", so the caller +-- cannot tell "already gone" from "not yours". +-- +-- ── The change (two additive parts) ────────────────────────────────────── +-- 1. `existed` return column. `memory_delete` now returns whether a row for +-- (scope,key) that the caller may SEE exists, so the edge can map a 0-row +-- removal to `not_found` (nothing there) vs `forbidden` (present, but this +-- call removed nothing — another principal's row outside the key's reach, +-- or already archived). For a NON-service-role caller `existed` never +-- reflects another user's row, so the RPC cannot be used to enumerate the +-- store. +-- +-- 2. Scope-authorized personal removal. A key that carries a NON-EMPTY scope +-- allowlist AND reaches this function on the trusted service-role +-- connection may archive / hard-delete EVERY writer's row for (scope,key) +-- within the scopes and orgs it is scoped to — the authority the owner +-- granted by scoping the key. Both conjuncts are load-bearing: +-- • `auth.role() = 'service_role'` — the edge resolves `p_key_scopes` +-- from the AUTHENTICATED token, never from request input. Without this +-- gate a raw `authenticated` PostgREST caller could pass an allowlist +-- and delete another user's rows — the exact IDOR 00046 closed. For +-- such a caller `auth.uid()` still pins the actor and the widening is +-- never reached. +-- • `array_length(p_key_scopes, 1) is not null` — an UNSCOPED key ('{}') +-- gets no widening and stays own-rows-only, so scoping a key UP is what +-- grants management authority, and the default key is unaffected. +-- The scope itself is already asserted allowed at the top (LK002 otherwise), +-- and org tenancy is re-checked per row via `lorekit_api_token_org_allowed`. +-- +-- Actor resolution, the org capability gates and the grants are 00068 verbatim; +-- only the `existed` column, the `v_scope_managed` widening and the personal +-- branch's WHERE are new. +-- ───────────────────────────────────────────────────────────────────────────── + +drop function if exists memory_delete(uuid, text, text, text, boolean, text[], text, uuid[]); + +create or replace function memory_delete( + p_user_id uuid, + p_org_slug text default null, + p_scope text default null, + p_key text default null, + p_force boolean default false, + p_key_scopes text[] default '{}', + p_key_org_access text default 'all', + p_key_org_ids uuid[] default '{}' +) +returns table (deleted boolean, archived boolean, existed boolean) +language plpgsql +security definer +set search_path = public +as $$ +declare + v_org_id uuid; + v_count integer; + v_existed boolean; + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + -- A scoped key on the trusted service-role connection manages the whole + -- allowed scope, not just its owner's rows. See the header for why BOTH + -- conjuncts are load-bearing (service_role gate closes the p_key_scopes IDOR; + -- the length check keeps an unscoped key own-rows-only). + v_scope_managed boolean := ( + auth.role() = 'service_role' + and array_length(p_key_scopes, 1) is not null + ); + -- The SERVICE tier — the bare `SUPABASE_SERVICE_ROLE_KEY` (CI, the smoke + -- suite, an operator script). It authenticates as no user and no key, so both + -- surfaces resolve `p_user_id => null` for it (REST `auth.userId` is unset for + -- `type:'service'`; MCP's `toolUserId` returns null for anything that is not + -- an api_key). `auth.uid()` is null on that connection too, so v_actor is NULL + -- and every `m.user_id = v_actor` disjunct below is NULL — before this the + -- personal branch matched ZERO rows for the tier that is meant to reach all of + -- them, and the edge reported the row it could see but not touch as a 403. + -- + -- Own-rows-only is the API-KEY rule (00046), and null actor is precisely "no + -- key, no user": the tier that holds the service-role secret can write the + -- table directly, so pinning it here protected nothing. This restores what + -- the raw-query paths have always done — `if (userId) …eq('user_id', userId)`, + -- i.e. no owner filter when there is no user (mcp-core/src/tools/delete.ts). + -- + -- It cannot widen anyone else: an api_key ALWAYS carries `p_user_id` (the + -- `api_tokens.user_id` it was matched by), and a non-service-role caller fails + -- the role conjunct, so `auth.uid()` keeps pinning it — the 00046 IDOR guard + -- is untouched on both. + v_service_tier boolean := ( + auth.role() = 'service_role' + and p_user_id is null + ); +begin + -- The scope allowlist, before either branch. LK002 maps to a 403 on both + -- surfaces, so no second mapping is needed here. + if not lorekit_api_token_scope_allowed(p_key_scopes, p_scope) then + raise exception using errcode = 'LK002', + message = format('key_scope_denied: scope=%s', p_scope); + end if; + + if p_org_slug is not null then + select o.id into v_org_id from orgs o where o.slug = p_org_slug; + if v_org_id is null then + raise exception using + errcode = 'P0001', + message = format('unknown_org: %s', p_org_slug); + end if; + + -- The tenancy half, before the capability gates so the denial names the + -- key, not its owner's role. + if not lorekit_api_token_org_allowed(p_key_org_access, p_key_org_ids, v_org_id) then + raise exception using errcode = 'LK002', + message = format('key_org_denied: org=%s', p_org_slug); + end if; + + v_existed := exists( + select 1 from memories m + where m.org_id = v_org_id and m.scope = p_scope and m.key = p_key + ); + + if p_force then + if not lorekit_org_can(v_actor, v_org_id, 'hard_delete') then + raise exception using + errcode = 'LK002', + message = format('org_permission_denied: org=%s capability=hard_delete', p_org_slug); + end if; + + delete from memories + where org_id = v_org_id and scope = p_scope and key = p_key; + get diagnostics v_count = row_count; + + return query select (v_count > 0), false, v_existed; + else + if not lorekit_org_can(v_actor, v_org_id, 'archive') then + raise exception using + errcode = 'LK002', + message = format('org_permission_denied: org=%s capability=archive', p_org_slug); + end if; + + update memories + set archived_at = now() + where org_id = v_org_id and scope = p_scope and key = p_key and archived_at is null; + get diagnostics v_count = row_count; + + return query select false, (v_count > 0), v_existed; + end if; + else + -- Personal / scope-authorized branch. + -- + -- `existed` is the caller's VISIBILITY of (scope,key), deliberately BROADER + -- than the removal WHERE below so that `existed=true, removed=false` is a + -- meaningful "present but not removed by this call" (forbidden / already + -- archived) rather than collapsing into not_found. A service-role caller + -- (the edge, which can already read the whole scope via memory.list) sees + -- any row; a non-service-role caller sees only its own / member-org rows, + -- so it can never enumerate another user's memories. + v_existed := case + when auth.role() = 'service_role' then exists( + select 1 from memories m where m.scope = p_scope and m.key = p_key + ) + else exists( + select 1 from memories m + where m.scope = p_scope and m.key = p_key + and ( m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) ) + ) + end; + + if p_force then + delete from memories m + where m.scope = p_scope and m.key = p_key + and ( v_service_tier + or m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) + or ( v_scope_managed + and lorekit_api_token_org_allowed(p_key_org_access, p_key_org_ids, m.org_id) ) ); + get diagnostics v_count = row_count; + + return query select (v_count > 0), false, (v_existed or v_count > 0); + else + update memories m + set archived_at = now() + where m.scope = p_scope and m.key = p_key and m.archived_at is null + and ( v_service_tier + or m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) + or ( v_scope_managed + and lorekit_api_token_org_allowed(p_key_org_access, p_key_org_ids, m.org_id) ) ); + get diagnostics v_count = row_count; + + return query select false, (v_count > 0), (v_existed or v_count > 0); + end if; + end if; +end; +$$; + +-- 00046/00068's revoke is carried over verbatim against the (unchanged) +-- signature: 00020 granted `anon` EXPLICITLY, and `revoke … from public` does +-- not remove a per-role grant, so anon must be named again or the re-created +-- function silently comes back reachable. +revoke execute on function memory_delete(uuid, text, text, text, boolean, text[], text, uuid[]) + from public, anon; +grant execute on function memory_delete(uuid, text, text, text, boolean, text[], text, uuid[]) + to authenticated, service_role; + +comment on function memory_delete(uuid, text, text, text, boolean, text[], text, uuid[]) is + 'Archives (p_force=false) or hard-deletes (p_force=true) a memory and reports + whether a row for (scope,key) the caller may see existed. A key with a + non-empty scope allowlist reaching this function on the service-role + connection manages every writer''s row within the scopes/orgs it is scoped + to; an unscoped key, and any non-service-role caller, stay pinned to their + own (and member-org) rows, preserving the 00046 actor guard. The service tier + (service-role connection, no p_user_id — no user and no key) is unpinned: it + has no actor to be pinned to. Org removals remain gated on the archive / + hard_delete capability.'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00071', 'api_token_scope_authorized_removal') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00072_api_token_scope_authorized_restore.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- Scope-authorized restore, symmetric with 00071's archive/delete. +-- +-- 00071 let a scoped key archive/delete any writer's row within its allowlist, +-- but restore stayed own-rows-only (`restore_memory` was `where user_id = +-- v_actor`, and both the MCP `toolRestore` and the REST restore handler ran a +-- raw `.update({archived_at:null}).eq('user_id', userId)`). So a key could +-- archive a teammate's row in its scope and then be unable to un-archive it. +-- +-- This rewrites `restore_memory` to mirror `memory_delete` (00071): it takes the +-- calling key's scoping, applies the same service-role + non-empty-allowlist +-- widening, and returns `existed` so a 0-row restore is reported as not_found vs +-- forbidden. The signature changes (adds the three key params) and the return +-- becomes a row, so the function is DROPped and recreated; no edge/Node/web +-- caller referenced the old `returns uuid` form (both surfaces did a direct +-- update), so this revives the RPC as the single authorization site rather than +-- changing a live contract. +-- +-- `existed` here is scoped to ARCHIVED rows: restore acts only on an archived +-- row, so an already-active row is not "restorable" and reads as not_found, +-- which is the truthful signal for the caller. +-- ───────────────────────────────────────────────────────────────────────────── + +drop function if exists restore_memory(uuid, text, text); + +create or replace function restore_memory( + p_user_id uuid, + p_scope text default null, + p_key text default null, + p_key_scopes text[] default '{}', + p_key_org_access text default 'all', + p_key_org_ids uuid[] default '{}' +) +returns table (restored boolean, existed boolean) +language plpgsql +security definer +set search_path = public +as $$ +declare + v_count integer; + v_existed boolean; + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + -- Same gate as memory_delete (00071): a scoped key on the trusted + -- service-role connection manages the whole allowlist; an unscoped key and + -- any non-service-role caller stay own-rows-only (auth.uid() pins them), + -- preserving the 00046 actor guard against a request-supplied p_key_scopes. + v_scope_managed boolean := ( + auth.role() = 'service_role' + and array_length(p_key_scopes, 1) is not null + ); + -- The SERVICE tier, exactly as in 00071's `memory_delete` — see that header + -- for the full argument. Short form: the bare service-role key authenticates + -- as no user and no key, so `p_user_id` is null AND `auth.uid()` is null, + -- leaving v_actor NULL and every ownership disjunct below NULL. Restore has to + -- answer this the same way removal does, or a row the service tier can archive + -- is a row it cannot un-archive. + v_service_tier boolean := ( + auth.role() = 'service_role' + and p_user_id is null + ); +begin + if not lorekit_api_token_scope_allowed(p_key_scopes, p_scope) then + raise exception using errcode = 'LK002', + message = format('key_scope_denied: scope=%s', p_scope); + end if; + + -- Visibility of an ARCHIVED (restorable) row, broader than the restore WHERE + -- for a service-role caller so `existed=true, restored=false` is a meaningful + -- forbidden signal; a non-service-role caller never sees another user's row. + v_existed := case + when auth.role() = 'service_role' then exists( + select 1 from memories m + where m.scope = p_scope and m.key = p_key and m.archived_at is not null + ) + else exists( + select 1 from memories m + where m.scope = p_scope and m.key = p_key and m.archived_at is not null + and ( m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) ) + ) + end; + + update memories m + set archived_at = null + where m.scope = p_scope and m.key = p_key and m.archived_at is not null + and ( v_service_tier + or m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) + or ( v_scope_managed + and lorekit_api_token_org_allowed(p_key_org_access, p_key_org_ids, m.org_id) ) ); + get diagnostics v_count = row_count; + + return query select (v_count > 0), (v_existed or v_count > 0); +end; +$$; + +revoke execute on function restore_memory(uuid, text, text, text[], text, uuid[]) + from public, anon; +grant execute on function restore_memory(uuid, text, text, text[], text, uuid[]) + to authenticated, service_role; + +comment on function restore_memory(uuid, text, text, text[], text, uuid[]) is + 'Un-archives a memory and reports whether a restorable (archived) row the + caller may see existed. A key with a non-empty scope allowlist reaching this + on the service-role connection restores any writer''s row within its + scopes/orgs; an unscoped key and any non-service-role caller stay pinned to + their own (and member-org) rows — the 00046/00071 actor guard, applied to + restore. The service tier (service-role connection, no p_user_id — no user and + no key) is unpinned: it has no actor to be pinned to.'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00072', 'api_token_scope_authorized_restore') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00073_memory_write_security_definer.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- Fix: memory_write is missing SECURITY DEFINER, so an org-scoped write from +-- a JWT-authenticated caller (tier 3 auth — a real user session, not the +-- service-role client used by API tokens) fails with: +-- +-- new row violates row-level security policy for table "memories" +-- +-- Root cause: the org-write branch (introduced in 00019, carried unchanged +-- through every later revision including 00069) inserts the row with +-- `user_id = null` for an org-owned memory: +-- +-- insert into memories (user_id, org_id, ...) values (null, v_org_id, ...) +-- +-- `memory_write` runs `language plpgsql` WITHOUT `security definer`, so it +-- executes with the CALLER's privileges. The table's `rls_insert` policy +-- (00001_memories.sql) is: +-- +-- with check (user_id = auth.uid() or auth.role() = 'service_role') +-- +-- For an `authenticated` role (a real user, auth.role() != 'service_role') +-- inserting a row where user_id IS NULL, neither disjunct holds, so Postgres +-- rejects the insert at the RLS layer — even when the function's OWN +-- authorization check (`lorekit_org_can(p_user_id, v_org_id, 'write')`, +-- already evaluated above the insert) approved the write. A caller using a +-- service-role-backed API token never hits this, because `auth.role() = +-- 'service_role'` satisfies the policy directly — which is why this was only +-- ever observed from JWT-authenticated (dashboard/user-session) callers. +-- +-- Every sibling write/delete RPC added alongside or after memory_write +-- (memory_delete, restore_memory, lorekit_memory_scopes, lorekit_memory_list, +-- ...) is `security definer` for exactly this reason: each does its own +-- explicit authorization inside the function body (org role checks, API-key +-- scope/org allowlists) and is not meant to additionally rely on — or be +-- blocked by — table-level RLS. memory_write was the one RPC never updated to +-- match when the org-write branch was added, and this migration corrects that +-- oversight. No other change: authorization is unchanged, since it was +-- already fully derived inside the function (LK002 checks) rather than from +-- RLS. +-- +-- Signature carried over verbatim from 00069 (drop-then-create, per this +-- repo's forward-only migration convention — grants re-issued identically). +-- ═════════════════════════════════════════════════════════════════════════ + +drop function if exists memory_write( + uuid, text, text, text, text[], text, text, timestamptz, text, + integer, boolean, text, text, text, integer, text, text, text[], text, uuid[] +); + +create or replace function memory_write( + p_user_id uuid, + p_scope text, + p_key text, + p_value text, + p_tags text[] default '{}', + p_source_agent text default null, + p_trigger text default null, + p_created_at timestamptz default null, + p_org_slug text default null, + p_ttl_seconds integer default null, + p_clear_ttl boolean default false, + p_origin_repo text default null, + p_origin_branch text default null, + p_origin_commit text default null, + p_origin_pr integer default null, + p_kind text default null, + p_host text default null, + -- The CALLING KEYs restriction — BOTH axes, defaulted so every existing + -- caller (JWT, the Node path, CI service-role) keeps the pre-00069 behaviour + -- untouched. The scope allowlist is here for the same reason the tenancy is: + -- the edge holds the service-role key, so the dispatchers refusal is + -- advisory, and this is the last gate a write cannot go around. + p_key_scopes text[] default '{}', + p_key_org_access text default 'all', + p_key_org_ids uuid[] default '{}' +) +returns table ( + id uuid, + created_at timestamptz, + inserted boolean, + org_routed boolean, + binding_org_slug text, + expires_at timestamptz +) +language plpgsql +security definer +set search_path = public +as $$ +declare + v_org_id uuid; + v_binding_org uuid; + v_binding_slug text; + v_expires_at timestamptz; + v_ttl_action text := 'keep'; +begin + -- The scope allowlist, checked FIRST and for every branch. Both transports + -- already refuse a named scope outside the allowlist, but both run on the + -- service-role client, so those refusals are advisory by construction — this + -- is the one place on the write path that a caller cannot route around. + -- LK002, the same code the org denial raises, so `translateDbError` answers + -- the REST caller a 403 and the MCP caller a forbidden error without a second + -- mapping. + if not lorekit_api_token_scope_allowed(p_key_scopes, p_scope) then + raise exception using errcode = 'LK002', + message = format('key_scope_denied: scope=%s', p_scope); + end if; + + if p_clear_ttl then + v_ttl_action := 'clear'; + elsif p_ttl_seconds is not null then + if p_ttl_seconds < 1 or p_ttl_seconds > 31536000 then + raise exception using errcode = 'P0001', + message = format('ttl_seconds must be between 1 and 31536000, got %s', p_ttl_seconds); + end if; + v_expires_at := now() + (p_ttl_seconds * interval '1 second'); + v_ttl_action := 'set'; + end if; + + if p_org_slug is not null then + select o.id into v_org_id from orgs o where o.slug = p_org_slug and o.deleted_at is null; + if v_org_id is null then + raise exception using errcode = 'P0001', message = format('unknown_org: %s', p_org_slug); + end if; + if not lorekit_org_can(p_user_id, v_org_id, 'write') then + raise exception using errcode = 'LK002', message = format('org_permission_denied: org=%s', p_org_slug); + end if; + -- Defence in depth. The transport already refuses a key writing into an + -- org outside its tenancy, but this RPC is the LAST gate that cannot be + -- bypassed: the edge holds the service-role key, so every check above it + -- is advisory by construction. + if not lorekit_api_token_org_allowed(p_key_org_access, p_key_org_ids, v_org_id) then + raise exception using errcode = 'LK002', + message = format('key_org_denied: org=%s', p_org_slug); + end if; + else + select b.org_id, o.slug into v_binding_org, v_binding_slug + from org_scope_bindings b + join orgs o on o.id = b.org_id + where b.scope = p_scope and o.deleted_at is null; + + -- 00068 decision 4: THE KEY WINS OVER THE BINDING. Auto-routing is a + -- convenience; the keys tenancy is an authorization boundary, and a + -- boundary a convenience can widen is not one. A key that may not reach + -- the bound org falls back to a PERSONAL write — the same graceful + -- outcome a non-member already gets, and `binding_org_slug` still comes + -- back so the caller can surface the same actionable notice. + if v_binding_org is not null + and p_user_id is not null + and lorekit_org_can(p_user_id, v_binding_org, 'write') + and lorekit_api_token_org_allowed(p_key_org_access, p_key_org_ids, v_binding_org) then + v_org_id := v_binding_org; + end if; + end if; + + if v_org_id is not null then + return query + insert into memories ( + user_id, org_id, scope, key, value, tags, source_agent, trigger, + created_at, updated_at, created_by, updated_by, expires_at, + origin_repo, origin_branch, origin_commit, origin_pr, kind, host, seen_count + ) + values ( + null, v_org_id, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), + p_user_id, p_user_id, v_expires_at, + p_origin_repo, p_origin_branch, p_origin_commit, p_origin_pr, p_kind, p_host, 1 + ) + on conflict (org_id, scope, key) where org_id is not null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + updated_by = p_user_id, + seen_count = memories.seen_count + 1, + origin_repo = coalesce(excluded.origin_repo, memories.origin_repo), + origin_branch = coalesce(excluded.origin_branch, memories.origin_branch), + origin_commit = coalesce(excluded.origin_commit, memories.origin_commit), + origin_pr = coalesce(excluded.origin_pr, memories.origin_pr), + kind = coalesce(excluded.kind, memories.kind), + host = coalesce(excluded.host, memories.host), + expires_at = case v_ttl_action + when 'clear' then null + when 'set' then v_expires_at + else memories.expires_at + end + returning + memories.id, memories.created_at, (xmax::text = '0') as inserted, + true as org_routed, v_binding_slug as binding_org_slug, memories.expires_at; + + elsif p_user_id is null then + return query + insert into memories ( + user_id, scope, key, value, tags, source_agent, trigger, + created_at, updated_at, created_by, updated_by, expires_at, + origin_repo, origin_branch, origin_commit, origin_pr, kind, host, seen_count + ) + values ( + null, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), + null, null, v_expires_at, + p_origin_repo, p_origin_branch, p_origin_commit, p_origin_pr, p_kind, p_host, 1 + ) + on conflict (scope, key) where org_id is null and user_id is null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + seen_count = memories.seen_count + 1, + origin_repo = coalesce(excluded.origin_repo, memories.origin_repo), + origin_branch = coalesce(excluded.origin_branch, memories.origin_branch), + origin_commit = coalesce(excluded.origin_commit, memories.origin_commit), + origin_pr = coalesce(excluded.origin_pr, memories.origin_pr), + kind = coalesce(excluded.kind, memories.kind), + host = coalesce(excluded.host, memories.host), + expires_at = case v_ttl_action + when 'clear' then null + when 'set' then v_expires_at + else memories.expires_at + end + returning + memories.id, memories.created_at, (xmax::text = '0') as inserted, + false as org_routed, v_binding_slug as binding_org_slug, memories.expires_at; + + else + return query + insert into memories ( + user_id, scope, key, value, tags, source_agent, trigger, + created_at, updated_at, created_by, updated_by, expires_at, + origin_repo, origin_branch, origin_commit, origin_pr, kind, host, seen_count + ) + values ( + p_user_id, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), + p_user_id, p_user_id, v_expires_at, + p_origin_repo, p_origin_branch, p_origin_commit, p_origin_pr, p_kind, p_host, 1 + ) + on conflict (user_id, scope, key) where org_id is null and user_id is not null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + updated_by = p_user_id, + seen_count = memories.seen_count + 1, + origin_repo = coalesce(excluded.origin_repo, memories.origin_repo), + origin_branch = coalesce(excluded.origin_branch, memories.origin_branch), + origin_commit = coalesce(excluded.origin_commit, memories.origin_commit), + origin_pr = coalesce(excluded.origin_pr, memories.origin_pr), + kind = coalesce(excluded.kind, memories.kind), + host = coalesce(excluded.host, memories.host), + expires_at = case v_ttl_action + when 'clear' then null + when 'set' then v_expires_at + else memories.expires_at + end + returning + memories.id, memories.created_at, (xmax::text = '0') as inserted, + false as org_routed, v_binding_slug as binding_org_slug, memories.expires_at; + end if; +end; +$$; + +grant execute on function memory_write( + uuid, text, text, text, text[], text, text, timestamptz, text, + integer, boolean, text, text, text, integer, text, text, text[], text, uuid[] +) to anon, authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00073', 'memory_write_security_definer') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00074_db_query_stats.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- Query-level profiling: read the top statements out of pg_stat_statements so +-- the `profiling` Edge Function can ship them to Dash0 as OTel metrics. +-- +-- WHY THIS EXISTS, and why it is not a CPU profiler. +-- +-- Dash0 collects profiles with a host-level eBPF agent. There is no host here: +-- every LoreKit runtime is managed serverless (Supabase Edge Functions on +-- managed Deno isolates, the dashboard on Vercel), so no agent can be attached +-- and no userland V8 profiler is exposed to sample from. A CPU profile would +-- also say little — these handlers are I/O-bound, so a sampled stack is mostly +-- "awaiting fetch". +-- +-- The time actually goes into SQL, and Postgres already profiles itself. +-- `createTracedClient` gives us a CLIENT span per round-trip, which measures +-- each call from the CALLER's side (round-trip, including network); what it +-- cannot see is server-side cost per query SHAPE, aggregated across every +-- caller — which statement is expensive, how often it runs, whether a plan +-- regressed. That is what `pg_stat_statements` holds, and it is the closest +-- thing to a profile this architecture can produce. See docs/otel.md → +-- "Query-level profiling". +-- +-- WHAT SHIPS HERE +-- 1. `lorekit_db_query_stats()` — the reader. Bounded, service-role only. +-- 2. `lorekit_export_db_query_stats()` — the scheduled trigger that pokes the +-- Edge Function via pg_net. INERT until an operator provisions two Vault +-- secrets, so this migration turns nothing on by itself. +-- +-- Requires the `pg_stat_statements` extension. Absent (or readable by nobody) +-- is handled as "no rows", never an error: profiling is an observability +-- surface and must not be able to fail a migration or a request. +-- ───────────────────────────────────────────────────────────────────────────── + +-- ── 1. The reader ─────────────────────────────────────────────────────────── +-- +-- Returns the top `p_limit` statements of the CURRENT database by cumulative +-- execution time, newest stats-reset timestamp attached. +-- +-- The counters are cumulative since `stats_reset` and are exported as such — +-- OTel cumulative monotonic sums, with `stats_since` becoming each datapoint's +-- `startTimeUnixNano`. Dash0 then does the differencing itself and reads a +-- reset as a new series rather than as negative traffic. Computing deltas here +-- would mean persisting the previous snapshot and re-implementing reset +-- detection; see supabase/functions/_shared/telemetry/otlp-metrics.ts. +-- +-- Dynamic SQL on purpose: a STATIC reference to +-- `extensions.pg_stat_statements` would make the whole function body +-- unresolvable on an instance where the extension is not installed, turning an +-- optional feature into a hard failure. The schema is resolved from +-- `pg_extension` rather than hardcoded to `extensions` (where Supabase puts +-- it) so a self-hosted instance that installed it into `public` also works. +create or replace function lorekit_db_query_stats(p_limit integer default 20) +returns table ( + queryid text, + query text, + toplevel boolean, + calls bigint, + total_exec_ms double precision, + rows_returned bigint, + stats_since timestamptz +) +language plpgsql +security definer +set search_path = public +as $$ +declare + v_schema text; + v_limit integer; +begin + -- Hard cap. Every returned row becomes a metric SERIES in Dash0, so an + -- unbounded p_limit is an unbounded cardinality bill. 200 is far above any + -- real top-N and low enough to stay a rounding error. + v_limit := least(greatest(coalesce(p_limit, 20), 1), 200); + + select n.nspname + into v_schema + from pg_extension e + join pg_namespace n on n.oid = e.extnamespace + where e.extname = 'pg_stat_statements'; + + -- Not installed → no rows. The caller reports "0 datapoints", which is + -- honest and actionable, instead of a 500 nobody can distinguish from a + -- broken exporter. + if v_schema is null then + return; + end if; + + return query execute format($fmt$ + select + s.queryid::text, + -- Collapse whitespace and truncate: this string becomes a metric + -- ATTRIBUTE, and pg_stat_statements keeps the full normalised statement, + -- which for a wide upsert runs to kilobytes. The queryid is the stable + -- identity; the text is the human label. + left(regexp_replace(btrim(s.query), '\s+', ' ', 'g'), 512), + s.toplevel, + s.calls, + s.total_exec_time, + s.rows, + i.stats_reset + from %1$I.pg_stat_statements s + left join %1$I.pg_stat_statements_info i on true + -- This database only. A shared cluster's other databases are somebody + -- else's workload and would both mislead and inflate cardinality. + where s.dbid = (select oid from pg_database where datname = current_database()) + -- Never profile the profiler — BOTH halves of it. Excluding only the + -- view's own name is not enough: the exporter reaches this function + -- through PostgREST, so `select * from lorekit_db_query_stats($1)` is + -- itself a statement pg_stat_statements records, and at one scrape a + -- minute it climbs its own top-N and then reports on itself forever. The + -- pg_net poke is recorded too, for the same reason. + and s.query not ilike '%%pg_stat_statements%%' + and s.query not ilike '%%lorekit_db_query_stats%%' + and s.query not ilike '%%lorekit_export_db_query_stats%%' + order by s.total_exec_time desc + limit %2$s + $fmt$, v_schema, v_limit); +exception + -- The closed set of "this instance cannot serve the read" failures, each + -- swallowed to "no rows" for the reason above: an observability read must + -- never be the thing that breaks. A genuine mistake in the statement itself + -- (a typo, a wrong cast) is NOT in this set and still raises loudly — which + -- is why this is an enumeration and not `when others`. + -- + -- object_not_in_prerequisite_state — the extension is INSTALLED but not in + -- `shared_preload_libraries`, so the view exists and raises on read. + -- Supabase preloads it; a self-hosted instance that ran + -- `create extension` without editing postgresql.conf lands exactly here, + -- and it is the one failure mode the presence check above cannot see. + -- insufficient_privilege — reading other roles' statements needs + -- `pg_read_all_stats`. Without it Postgres normally just narrows the + -- result to our own statements rather than erroring, but a locked-down + -- instance can refuse outright. + -- undefined_column / undefined_table — an extension version predating + -- `toplevel` / `total_exec_time` / `pg_stat_statements_info` (pre-PG14), + -- or the view dropped from under us. + when object_not_in_prerequisite_state + or insufficient_privilege + or undefined_column + or undefined_table then + return; +end; +$$; + +comment on function lorekit_db_query_stats(integer) is + 'Top-N statements of this database by cumulative exec time, for the profiling ' + 'Edge Function. Cumulative counters + stats_reset; empty when ' + 'pg_stat_statements is absent or unreadable.'; + +-- Service-role only. These are cross-tenant query shapes — the whole cluster's +-- workload aggregated — so this is an operator surface, never a tenant one. +-- `authenticated` and `anon` must NOT reach it, and PostgREST exposes any +-- function executable by them. +revoke all on function lorekit_db_query_stats(integer) from public; +revoke all on function lorekit_db_query_stats(integer) from anon, authenticated; +grant execute on function lorekit_db_query_stats(integer) to service_role; + +-- ── 2. The scheduled trigger ──────────────────────────────────────────────── +-- +-- pg_cron cannot POST, so the DB pokes the `profiling` Edge Function through +-- pg_net and the function does the OTLP export. The function is where the +-- Dash0 credentials ALREADY live (OTEL_EXPORTER_OTLP_* Supabase secrets) and +-- where the one OTLP payload builder lives, so routing through it adds no new +-- secret surface for Dash0 and no second copy of the resource attributes. +-- +-- OFF BY DEFAULT, deliberately — the same posture as the embedding pipeline +-- (docs/embeddings.md). Two Vault secrets are required and neither is created +-- here: +-- +-- lorekit_profiling_url — https://.supabase.co/functions/v1/profiling +-- lorekit_profiling_key — the project's service-role key +-- +-- select vault.create_secret('https://pqokxlhvnosogizsjztg.supabase.co/functions/v1/profiling', 'lorekit_profiling_url'); +-- select vault.create_secret('', 'lorekit_profiling_key'); +-- +-- Until both exist this function returns 'disabled' and posts nothing. That +-- keeps the gate in ONE place: the cron job below is scheduled +-- unconditionally, so an operator turns profiling on by adding the secrets — +-- no migration re-run, no schedule edit. +create or replace function lorekit_export_db_query_stats() +returns text +language plpgsql +security definer +set search_path = public +as $$ +declare + v_url text; + v_key text; +begin + -- pg_net absent → nothing can be posted. Checked before Vault so the reason + -- reported is the real blocker. + if not exists (select 1 from pg_extension where extname = 'pg_net') then + return 'skipped: pg_net not installed'; + end if; + + -- Vault absent (a self-hosted instance without it) is the same "not + -- configured" state as a missing secret, not an error. + if to_regclass('vault.decrypted_secrets') is null then + return 'disabled: vault unavailable'; + end if; + + select decrypted_secret into v_url + from vault.decrypted_secrets where name = 'lorekit_profiling_url'; + select decrypted_secret into v_key + from vault.decrypted_secrets where name = 'lorekit_profiling_key'; + + if v_url is null or v_key is null then + return 'disabled: set the lorekit_profiling_url and lorekit_profiling_key vault secrets to enable'; + end if; + + -- Fire-and-forget by design: pg_net queues the request and returns an id. + -- A cron tick must not block on Dash0, and the NEXT tick re-reads cumulative + -- counters anyway — a dropped tick loses resolution, never data, which is + -- the whole reason for exporting cumulative sums instead of deltas. + perform net.http_post( + url := v_url, + body := '{}'::jsonb, + headers := jsonb_build_object( + 'Content-Type', 'application/json', + 'Authorization', 'Bearer ' || v_key + ), + timeout_milliseconds := 10000 + ); + + return 'posted'; +end; +$$; + +comment on function lorekit_export_db_query_stats() is + 'Pokes the profiling Edge Function to export query stats to Dash0. Inert ' + 'until the lorekit_profiling_url + lorekit_profiling_key vault secrets exist.'; + +revoke all on function lorekit_export_db_query_stats() from public; +revoke all on function lorekit_export_db_query_stats() from anon, authenticated; +grant execute on function lorekit_export_db_query_stats() to service_role; + +-- Every minute. This is a metric SCRAPE, so the interval is the resolution of +-- every rate() over the result — the 15-minute and daily cadences used by the +-- reapers elsewhere in this schema would make the data useless for finding a +-- slow query. Cheap: one bounded read of an in-memory view plus one queued +-- HTTP request, and a no-op entirely while profiling is off. +-- +-- Guarded so the migration still applies without pg_cron (the function can +-- then be driven by an external scheduler). Idempotent: cron.schedule() upserts +-- by job name. +do $$ +begin + if exists (select 1 from pg_extension where extname = 'pg_cron') then + perform cron.schedule( + 'lorekit-export-db-query-stats', + '* * * * *', + $cron$select lorekit_export_db_query_stats()$cron$ + ); + end if; +end; +$$; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00074', 'db_query_stats') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00075_memory_write_returns_full_row.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- Perf: memory_write returns the full display row, so POST /memories no +-- longer needs a trailing SELECT to shape the response body. +-- +-- Telemetry (docs/otel.md → server-side load-test attribution): every +-- POST /memories paid a SECOND edge→PostgREST round trip — a +-- `.from('memories').select(...).eq('id', row.id).single()` right after the +-- RPC returned — for the sole purpose of re-reading the row `memory_write` +-- had just inserted or updated, so the response body could match +-- `MemoryEntrySchema`. That hop measured ~206ms server-side wait despite the +-- underlying SQL taking single-digit milliseconds (`pg_stat_statements`) — +-- the edge→PostgREST hop cost, not the query, per CLAUDE.md's "central fact". +-- +-- Every column `handlers/create.ts` re-fetched is already available inside +-- `memory_write`'s own INSERT/UPDATE — it is the same row, in the same +-- statement, before the round trip even starts. Returning them directly +-- removes the extra hop instead of re-deriving anything. +-- +-- `org` (the joined org name/slug) is deliberately NOT added here: the +-- existing follow-up SELECT never requested it either (its literal column +-- list omits `org_id`/`orgs(...)`), so this migration is a strict return-set +-- superset of what the handler already read from the second query, not a new +-- widening of the write response's public contract. +-- ═════════════════════════════════════════════════════════════════════════ + +drop function if exists memory_write( + uuid, text, text, text, text[], text, text, timestamptz, text, + integer, boolean, text, text, text, integer, text, text, text[], text, uuid[] +); + +create or replace function memory_write( + p_user_id uuid, + p_scope text, + p_key text, + p_value text, + p_tags text[] default '{}', + p_source_agent text default null, + p_trigger text default null, + p_created_at timestamptz default null, + p_org_slug text default null, + p_ttl_seconds integer default null, + p_clear_ttl boolean default false, + p_origin_repo text default null, + p_origin_branch text default null, + p_origin_commit text default null, + p_origin_pr integer default null, + p_kind text default null, + p_host text default null, + p_key_scopes text[] default '{}', + p_key_org_access text default 'all', + p_key_org_ids uuid[] default '{}' +) +returns table ( + id uuid, + created_at timestamptz, + inserted boolean, + org_routed boolean, + binding_org_slug text, + expires_at timestamptz, + -- Everything else `handlers/create.ts`'s trailing SELECT used to fetch, + -- straight off the row the INSERT/UPDATE already touched. + scope text, + key text, + value text, + tags text[], + source_agent text, + trigger text, + updated_at timestamptz, + archived_at timestamptz, + origin_repo text, + origin_branch text, + origin_commit text, + origin_pr integer, + kind text, + host text, + seen_count integer +) +language plpgsql +security definer +set search_path = public +as $$ +-- The new RETURNS TABLE columns (scope, key, value, tags, source_agent, +-- trigger, updated_at, archived_at, origin_*, kind, host, seen_count) are also +-- plpgsql OUT variables, and `scope`/`key` collide with the bare column names +-- used in the `on conflict (…, scope, key)` arbiter lists below — Postgres +-- parses an arbiter list as a general expression context, unlike an INSERT +-- column list or an UPDATE SET target, so it raises "column reference ... +-- is ambiguous" there without this. Every other reference in the body is +-- table-qualified already; this directive (the same one 00039/00043/00050/… +-- use for exactly this OUT-column shape) makes the column win regardless, so +-- the body can never be mis-resolved against the OUT variables. +#variable_conflict use_column +declare + v_org_id uuid; + v_binding_org uuid; + v_binding_slug text; + v_expires_at timestamptz; + v_ttl_action text := 'keep'; +begin + if not lorekit_api_token_scope_allowed(p_key_scopes, p_scope) then + raise exception using errcode = 'LK002', + message = format('key_scope_denied: scope=%s', p_scope); + end if; + + if p_clear_ttl then + v_ttl_action := 'clear'; + elsif p_ttl_seconds is not null then + if p_ttl_seconds < 1 or p_ttl_seconds > 31536000 then + raise exception using errcode = 'P0001', + message = format('ttl_seconds must be between 1 and 31536000, got %s', p_ttl_seconds); + end if; + v_expires_at := now() + (p_ttl_seconds * interval '1 second'); + v_ttl_action := 'set'; + end if; + + if p_org_slug is not null then + select o.id into v_org_id from orgs o where o.slug = p_org_slug and o.deleted_at is null; + if v_org_id is null then + raise exception using errcode = 'P0001', message = format('unknown_org: %s', p_org_slug); + end if; + if not lorekit_org_can(p_user_id, v_org_id, 'write') then + raise exception using errcode = 'LK002', message = format('org_permission_denied: org=%s', p_org_slug); + end if; + if not lorekit_api_token_org_allowed(p_key_org_access, p_key_org_ids, v_org_id) then + raise exception using errcode = 'LK002', + message = format('key_org_denied: org=%s', p_org_slug); + end if; + else + select b.org_id, o.slug into v_binding_org, v_binding_slug + from org_scope_bindings b + join orgs o on o.id = b.org_id + where b.scope = p_scope and o.deleted_at is null; + + if v_binding_org is not null + and p_user_id is not null + and lorekit_org_can(p_user_id, v_binding_org, 'write') + and lorekit_api_token_org_allowed(p_key_org_access, p_key_org_ids, v_binding_org) then + v_org_id := v_binding_org; + end if; + end if; + + if v_org_id is not null then + return query + insert into memories ( + user_id, org_id, scope, key, value, tags, source_agent, trigger, + created_at, updated_at, created_by, updated_by, expires_at, + origin_repo, origin_branch, origin_commit, origin_pr, kind, host, seen_count + ) + values ( + null, v_org_id, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), + p_user_id, p_user_id, v_expires_at, + p_origin_repo, p_origin_branch, p_origin_commit, p_origin_pr, p_kind, p_host, 1 + ) + on conflict (org_id, scope, key) where org_id is not null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + updated_by = p_user_id, + seen_count = memories.seen_count + 1, + origin_repo = coalesce(excluded.origin_repo, memories.origin_repo), + origin_branch = coalesce(excluded.origin_branch, memories.origin_branch), + origin_commit = coalesce(excluded.origin_commit, memories.origin_commit), + origin_pr = coalesce(excluded.origin_pr, memories.origin_pr), + kind = coalesce(excluded.kind, memories.kind), + host = coalesce(excluded.host, memories.host), + expires_at = case v_ttl_action + when 'clear' then null + when 'set' then v_expires_at + else memories.expires_at + end + returning + memories.id, memories.created_at, (xmax::text = '0') as inserted, + true as org_routed, v_binding_slug as binding_org_slug, memories.expires_at, + memories.scope, memories.key, memories.value, memories.tags, + memories.source_agent, memories.trigger, memories.updated_at, memories.archived_at, + memories.origin_repo, memories.origin_branch, memories.origin_commit, memories.origin_pr, + memories.kind, memories.host, memories.seen_count; + + elsif p_user_id is null then + return query + insert into memories ( + user_id, scope, key, value, tags, source_agent, trigger, + created_at, updated_at, created_by, updated_by, expires_at, + origin_repo, origin_branch, origin_commit, origin_pr, kind, host, seen_count + ) + values ( + null, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), + null, null, v_expires_at, + p_origin_repo, p_origin_branch, p_origin_commit, p_origin_pr, p_kind, p_host, 1 + ) + on conflict (scope, key) where org_id is null and user_id is null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + seen_count = memories.seen_count + 1, + origin_repo = coalesce(excluded.origin_repo, memories.origin_repo), + origin_branch = coalesce(excluded.origin_branch, memories.origin_branch), + origin_commit = coalesce(excluded.origin_commit, memories.origin_commit), + origin_pr = coalesce(excluded.origin_pr, memories.origin_pr), + kind = coalesce(excluded.kind, memories.kind), + host = coalesce(excluded.host, memories.host), + expires_at = case v_ttl_action + when 'clear' then null + when 'set' then v_expires_at + else memories.expires_at + end + returning + memories.id, memories.created_at, (xmax::text = '0') as inserted, + false as org_routed, v_binding_slug as binding_org_slug, memories.expires_at, + memories.scope, memories.key, memories.value, memories.tags, + memories.source_agent, memories.trigger, memories.updated_at, memories.archived_at, + memories.origin_repo, memories.origin_branch, memories.origin_commit, memories.origin_pr, + memories.kind, memories.host, memories.seen_count; + + else + return query + insert into memories ( + user_id, scope, key, value, tags, source_agent, trigger, + created_at, updated_at, created_by, updated_by, expires_at, + origin_repo, origin_branch, origin_commit, origin_pr, kind, host, seen_count + ) + values ( + p_user_id, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), + p_user_id, p_user_id, v_expires_at, + p_origin_repo, p_origin_branch, p_origin_commit, p_origin_pr, p_kind, p_host, 1 + ) + on conflict (user_id, scope, key) where org_id is null and user_id is not null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + updated_by = p_user_id, + seen_count = memories.seen_count + 1, + origin_repo = coalesce(excluded.origin_repo, memories.origin_repo), + origin_branch = coalesce(excluded.origin_branch, memories.origin_branch), + origin_commit = coalesce(excluded.origin_commit, memories.origin_commit), + origin_pr = coalesce(excluded.origin_pr, memories.origin_pr), + kind = coalesce(excluded.kind, memories.kind), + host = coalesce(excluded.host, memories.host), + expires_at = case v_ttl_action + when 'clear' then null + when 'set' then v_expires_at + else memories.expires_at + end + returning + memories.id, memories.created_at, (xmax::text = '0') as inserted, + false as org_routed, v_binding_slug as binding_org_slug, memories.expires_at, + memories.scope, memories.key, memories.value, memories.tags, + memories.source_agent, memories.trigger, memories.updated_at, memories.archived_at, + memories.origin_repo, memories.origin_branch, memories.origin_commit, memories.origin_pr, + memories.kind, memories.host, memories.seen_count; + end if; +end; +$$; + +grant execute on function memory_write( + uuid, text, text, text, text[], text, text, timestamptz, text, + integer, boolean, text, text, text, integer, text, text, text[], text, uuid[] +) to anon, authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00075', 'memory_write_returns_full_row') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00076_memories_tags_gin_index.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- GIN index on memories.tags — fixes the occasional multi-second stall behind +-- the "web — elevated backend p95 latency" check (Dash0 issue 9019610890880570001). +-- +-- WHAT WAS HAPPENING +-- +-- `getOnboardingState()` (packages/web/src/lib/onboarding-server.ts), awaited +-- by every `/overview` render, issues two count-only reads against `memories`: +-- +-- select id, { count: 'exact', head: true } +-- select id, { count: 'exact', head: true }.contains('tags', ['source::pr-webhook']) +-- +-- The second predicate compiles to `tags @> ARRAY['source::pr-webhook']`. The +-- `tags` column (00001_memories.sql) has never had a supporting index — the +-- table has a GIN index on `fts`, a btree on `scope`, `user_id`, and +-- `(scope, key)`, but nothing on `tags` — so this containment check runs a +-- sequential scan over every row RLS leaves visible. Traced instances of +-- `GET /overview` (2026-08-24, service.name=web, service.namespace=lorekit) +-- show this exact fetch at 0.55–0.9s normally and spiking to 7.559s in the +-- trace that crossed the alert's 4s p95 threshold — a seq-scan signature, not +-- network variance, since the sibling unfiltered count on the same table +-- stayed under 0.8s throughout. +-- +-- THE FIX +-- +-- A standard GIN index on a `text[]` column supports `@>`/`<@`/`&&` directly +-- (default `array_ops` opclass) with no query-side change — the existing +-- `.contains(...)` call starts using it as soon as this migration applies. +-- `CONCURRENTLY` is deliberately NOT used: Supabase migrations run inside a +-- transaction and `CREATE INDEX CONCURRENTLY` cannot run inside one, and the +-- `memories` table is small enough today that a regular (briefly +-- lock-holding) build is the right trade-off, matching every other index in +-- 00001_memories.sql. +-- +-- Scope note: this migration only fixes the missing index. The separate +-- architectural question — `onboarding-server.ts` reads `memories` directly +-- via supabase-js instead of through the `/memories` REST endpoint, per the +-- "Data access goes through a REST endpoint" rule in the root CLAUDE.md — is +-- out of scope for this fix; it is a pre-existing surface predating that rule +-- and migrating it is a larger, separate change. +-- ───────────────────────────────────────────────────────────────────────────── + +create index if not exists memories_tags_idx on memories using gin (tags); + +comment on index memories_tags_idx is + 'GIN index supporting tags @> / <@ / && containment queries (e.g. ' + 'onboarding-server.ts''s source::pr-webhook check), avoiding a sequential ' + 'scan on every /overview render.'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00076', 'memories_tags_gin_index') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00077_org_actor_functions_anon_revoke.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- Close residual `anon` EXECUTE grants left over on seven functions whose +-- migrations only ever revoked PUBLIC, not `anon` by name. +-- +-- ── The problem ────────────────────────────────────────────────────────── +-- Every migration below follows the schema's standard pattern of running +-- `revoke execute on function ... from public;` before granting to +-- `authenticated, service_role` — intended to strip PostgreSQL's default +-- PUBLIC EXECUTE grant on a newly CREATEd function. +-- +-- That pattern assumes `anon`'s only path to EXECUTE is inheriting the +-- PUBLIC grant. On this project's Supabase Postgres image, a newly CREATEd +-- function in `public` also picks up a *direct* `ALTER DEFAULT PRIVILEGES` +-- grant to `anon` (separate from, and not touched by, `revoke ... from +-- public`) — the same class of gap 00046 already had to close for +-- `memory_delete` with an explicit `revoke ... from public, anon`. Wherever +-- a later migration also named `anon` explicitly, the gap was already +-- closed (e.g. `restore_memory` in 00072, `memory_delete` itself in 00069/ +-- 00071, the 00047 read-function hardening, `lorekit_db_query_stats` in +-- 00074). The seven below never got that explicit `anon` revoke, so a fresh +-- CREATE (00041) or a CREATE OR REPLACE that only re-asserted the PUBLIC +-- revoke (00046/00062) left the default-privilege grant to `anon` intact: +-- +-- * `lorekit_org_actor(uuid)` (00041) — resolves the acting user for +-- every org RPC. +-- * `lorekit_org_members_list(uuid, uuid)` (00041) — returns other org +-- members' GitHub handles and avatars (PII), gated on membership in the +-- function body — a gate that only matters if the function can be +-- reached in the first place. +-- * `archive_memory(uuid, text, text)`, `purge_archived_memories(uuid, +-- integer)`, `purge_expired_memories(uuid)` (00003/00030, re-asserted by +-- 00046) — the same caller-supplied-`p_user_id` RPCs 00046's own header +-- names as needing this treatment; only `memory_delete` was named +-- against `anon` there, the other three were not. +-- * `lorekit_usage_stats(uuid, timestamptz, timestamptz, text)` (00043, +-- re-created by 00044) — a bare `p_user_id` reader; anon EXECUTE would +-- expose any user's usage aggregates. +-- * `lorekit_memory_set_embedding(uuid, uuid, text, text)` (00062). +-- +-- `supabase/tests/migrations.test.sql` already asserts every one of these +-- must not be executable by `anon`; this migration is the fix those +-- assertions caught. +-- +-- ── The fix ────────────────────────────────────────────────────────────── +-- Revoke EXECUTE from `anon` by name, matching 00046's precedent for +-- `memory_delete`. `public` is included too so this is idempotent +-- regardless of which grant path is present in a given environment; +-- `authenticated, service_role` are unaffected. + +revoke execute on function lorekit_org_actor(uuid) from public, anon; +revoke execute on function lorekit_org_members_list(uuid, uuid) from public, anon; +revoke execute on function archive_memory(uuid, text, text) from public, anon; +revoke execute on function purge_archived_memories(uuid, integer) from public, anon; +revoke execute on function purge_expired_memories(uuid) from public, anon; +revoke execute on function lorekit_usage_stats(uuid, timestamptz, timestamptz, text) from public, anon; +revoke execute on function lorekit_memory_set_embedding(uuid, uuid, text, text) from public, anon; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00077', 'org_actor_functions_anon_revoke') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00078_usage_event_scope_count.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- usage_events.scope_count — HOW MANY scopes an array-bearing call named. +-- +-- THE GAP: `memory.search` takes a `scopes` ARRAY; `usage_events.scope` (00058) +-- is ONE text column. So every search recorded `scope = null` — not because the +-- call was unscoped, but because the recording site (`safeValidateScope`) had +-- nothing single-valued to write. In a live 30-day sample this null bucket was +-- the SINGLE LARGEST slice of the per-scope read series (145,260 of 358,782 +-- records, ~40%) — almost entirely `memory.search`. +-- +-- THE FIX — option (a) from the two considered (see the PR description for the +-- full comparison): a bounded `scope_count` column, `scope` kept exactly as +-- accurate as it already is. +-- +-- 1. `scope_count`: how many scopes the call named, for ANY scope-bearing +-- tool. Null for a singular-`scope` tool (the RPC's own `default null`). +-- Recorded at the SAME two sites `scope`/`scope_type` already are +-- (`_shared/api/router.ts`, `mcp/mcp-handler.ts`). +-- 2. `scope` stays populated for the SINGLE-scope case, whether that single +-- scope arrived as `?scope=`/`toolArgs.scope` OR as a one-element +-- `scopes` array — a search over exactly one repo is exactly as +-- attributable as a `memory.list` over the same repo. TWO OR MORE scopes +-- still record `scope = null`; `scope_count` is the honest answer for +-- that case rather than a guessed single value. +-- 3. Why NOT `scopes text[]` (option b, alongside `scope`): the per-scope +-- read series (`lorekit_read_activity`) is a LEADERBOARD whose bars must +-- sum to the account total — a search over 4 scopes attributed to all 4 +-- would double-, triple-, quadruple-count its records against that +-- additive invariant. `scope_count` keeps ONE authoritative `scope` +-- column and turns the unattributed bucket into an EXPLAINABLE number +-- ("N searches spanning multiple scopes") instead of a bare null. +-- +-- MCP-SIDE BONUS FIX (same functions this migration's application code +-- touches, not a schema change): `mcp-handler.ts` computed `scopeType` via the +-- array-aware `scopeTypeAttribute(rawScope, toolArgs.scopes)` — correctly +-- resolving `mixed` for a multi-repo search — but then gated it behind +-- `rawScope ? scopeType : null` before recording it, discarding that value for +-- every `memory.search` call regardless of what it actually computed. +-- `scopeType` is already total (null exactly when there is nothing to report), +-- so the gate could only ever discard information; it never added safety. Fixed +-- alongside this migration since it is the same code path and the same finding +-- ("scope_type resolves the array to mixed" was the SPAN attribute's behaviour, +-- not the DATABASE COLUMN's, until this fix). +-- +-- FAIL-SAFE POSTURE, matching 00054/00058 exactly: `scope_count` is a +-- MEASUREMENT, recorded at the same site and with the same "must never fail +-- the call it measures" contract as `scope`/`client`/`correlation_id`. Filtering +-- BY a scope stays fail-LOUD (`handleReadActivity`'s `validateScope`, unchanged +-- by this migration) — recording and filtering are deliberately different +-- postures, not a single one applied inconsistently. +-- +-- Forward-only and additive: the column is nullable, the writer's new +-- parameter is trailing + defaulted, and existing rows are unaffected. +-- ═════════════════════════════════════════════════════════════════════════ + +-- ── the column ────────────────────────────────────────────────────────────── +alter table usage_events add column if not exists scope_count integer; + +-- Bound as a BACKSTOP, not the primary gate — the same reasoning 00044 gives +-- for `usage_events_correlation_id_len` and 00054/00058 for their own columns. +-- The recording site (`toolArgs.scopes.length` / `body.scopes.length`) is +-- already a small, real count; this exists only to stop a malformed or +-- adversarial value from inflating a GROUP BY. 256 is generous headroom over +-- any real `memory.search` call while still being finite. +alter table usage_events drop constraint if exists usage_events_scope_count_range; +alter table usage_events add constraint usage_events_scope_count_range + check (scope_count is null or (scope_count >= 0 and scope_count <= 256)); + +-- ── writer: add a trailing p_scope_count ───────────────────────────────────── +-- DROP first (not CREATE OR REPLACE): a new parameter changes the signature, +-- and CREATE OR REPLACE would leave 00058's fifteen-argument overload live +-- alongside this one, making every named-argument call ambiguous. Body is +-- 00058's verbatim plus the one column. +drop function if exists lorekit_record_usage_event(uuid, uuid, text, text, text, text, text, integer, integer, integer, text, text, text, text, text); + +create or replace function lorekit_record_usage_event( + p_user_id uuid default null, + p_org_id uuid default null, + p_plan_name text default null, + p_tool_name text default null, + p_scope_type text default null, + p_auth_type text default null, + p_outcome text default null, + p_duration_ms integer default null, + p_memory_count integer default null, + p_result_count integer default null, + p_correlation_id text default null, + p_client text default null, + p_kind text default null, + p_host text default null, + p_scope text default null, + p_scope_count integer default null +) +returns uuid +language plpgsql +security definer +set search_path = public +as $$ +declare + v_id uuid; +begin + insert into usage_events ( + user_id, org_id, plan_name, + tool_name, scope_type, auth_type, + outcome, duration_ms, memory_count, + result_count, correlation_id, client, kind, host, + scope, scope_count + ) values ( + p_user_id, p_org_id, p_plan_name, + p_tool_name, p_scope_type, p_auth_type, + p_outcome, p_duration_ms, p_memory_count, + p_result_count, p_correlation_id, p_client, p_kind, p_host, + p_scope, p_scope_count + ) + returning id into v_id; + return v_id; +exception + when others then + -- Never let telemetry writes break the primary operation. + return null; +end; +$$; + +grant execute on function lorekit_record_usage_event(uuid, uuid, text, text, text, text, text, integer, integer, integer, text, text, text, text, text, integer) + to anon, authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00078', 'usage_event_scope_count') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00079_usage_stats_client_kind_host.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- lorekit_usage_stats — group by client, kind, host too. +-- +-- THE GAP: `usage_events` has stored `client` (00054), `kind`/`host` (00056) +-- for months, and `GET /memories/usage` exposes none of them. "Who is +-- reading" (client) and "which agent family" (kind × host) are both +-- unanswerable from the API despite the columns already existing — the exact +-- shape of dead data 00056's own history warns about (kind/host sat NULL for +-- a release because nothing wired them into the write payload; this is the +-- READ half of the same mistake, just without the bug — the columns are +-- populated, only unexposed). +-- +-- SIGNATURE CHANGE: the RETURNS TABLE gains three columns, so this is a DROP +-- + recreate (matching 00054/00058/00076's own precedent on this exact +-- function), not a bare CREATE OR REPLACE, which cannot change a return type +-- and would leave the four-column overload live for an ambiguous call. +-- +-- CORRECTNESS: widening the GROUP BY only ever REFINES existing buckets — +-- summing `event_count` over the new, finer-grained rows reproduces the +-- exact total the old query gave, because no join or row duplication is +-- introduced. `summarizeUsageRows`/`rollupByScopeType` (usage-stats.ts) need +-- no code change for this reason; their own unit tests gain a regression +-- case proving a row set split across client/kind/host still summarises +-- identically to the un-split version. +-- +-- SCOPE_TYPE BUCKETING: unchanged from 00044 — legacy free-text values pass +-- through verbatim on this RPC (bucketing happens at READ time in the web +-- client, per the global constraint on this work; this RPC's job is to +-- return the true grouped rows, not to pre-judge which values are "known"). +-- +-- HOST CARDINALITY: `host` is OPEN free-text (a new host is a new agent, not +-- a migration) — the one dimension of the three that can blow up a busy +-- account's row count. Bounded here to the top 20 hosts by event count +-- within the query's own window; anything outside that top 20 collapses to +-- the literal `'other'` (NULL host stays NULL — a scopeless/hostless call is +-- not "some other host", it has none). This is a per-QUERY top-20, computed +-- fresh from the window's own data, not a global config — a different window +-- can legitimately show a different top 20. +-- ═════════════════════════════════════════════════════════════════════════ + +drop function if exists lorekit_usage_stats(uuid, timestamptz, timestamptz, text); + +create or replace function lorekit_usage_stats( + p_user_id uuid, + p_since timestamptz default null, + p_until timestamptz default null, + p_correlation_id text default null +) +returns table ( + tool_name text, + outcome text, + scope_type text, + client text, + kind text, + host text, + event_count bigint, + record_count bigint, + total_duration_ms bigint +) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +begin + return query + with visible as ( + select e.* + from usage_events e + where ( + (p_user_id is null and auth.role() = 'service_role') + or e.user_id = p_user_id + ) + and (p_since is null or e.created_at >= p_since) + and (p_until is null or e.created_at < p_until) + and (p_correlation_id is null or e.correlation_id = p_correlation_id) + ), + top_hosts as ( + select v.host + from visible v + where v.host is not null + group by v.host + order by count(*) desc, v.host asc + limit 20 + ) + select + v.tool_name, + v.outcome, + v.scope_type, + v.client, + v.kind, + case + when v.host is null then null + when v.host in (select th.host from top_hosts th) then v.host + else 'other' + end as host, + count(*) as event_count, + coalesce(sum(v.result_count), 0) as record_count, + coalesce(sum(v.duration_ms), 0) as total_duration_ms + from visible v + group by v.tool_name, v.outcome, v.scope_type, v.client, v.kind, + case + when v.host is null then null + when v.host in (select th.host from top_hosts th) then v.host + else 'other' + end + order by event_count desc, v.tool_name asc; +end; +$$; + +revoke execute on function lorekit_usage_stats(uuid, timestamptz, timestamptz, text) from public; +grant execute on function lorekit_usage_stats(uuid, timestamptz, timestamptz, text) + to authenticated, service_role; + +comment on function lorekit_usage_stats(uuid, timestamptz, timestamptz, text) is + 'Usage events grouped by (tool_name, outcome, scope_type, client, kind, host) + over the half-open [p_since, p_until) window, self-only visibility with the + service-role + NULL actor escape hatch. host is bounded to the window''s own + top 20 by event count; anything else collapses to ''other'' (NULL host stays + NULL). Summing event_count over every row reproduces the pre-00079 total — + the extra dimensions only refine existing buckets, never duplicate rows.'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00079', 'usage_stats_client_kind_host') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00080_read_activity_read_kind.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- lorekit_read_activity gains read_kind — the honesty fix for "Memories read". +-- +-- THE PROBLEM: "Memories read: 494,839" in a live 30-day sample is 99.7% bulk +-- list/search output — the SessionStart hook injecting ~31 lessons per call, +-- ~16,000 times. Targeted `memory.read` accounts for 1,487 records. One +-- number carries two completely different behaviours: a hook priming context +-- and an agent deliberately opening a specific lesson. +-- +-- THE FIX: split the series by `read_kind` — `'targeted'` for `memory.read` +-- (one exact scope+key), `'bulk'` for `memory.list` / `memory.search` / +-- `memory.list_archived` (every row a listing call returned). +-- +-- WHICH "READ": this is `lorekit_read_activity`'s OWN narrow inline 4-tool +-- list (00053's `tool_name in ('memory.read', 'memory.list', 'memory.search', +-- 'memory.list_archived')`), extended in place with a `case` on the SAME +-- list — NOT `usage-stats.ts`'s broader `READ_TOOL_NAMES`, which also counts +-- `memory.scopes` / `memory.usage` / `org.list`. A card headlined "Memories +-- read" must not count `org.list`. The two definitions are deliberately +-- different elsewhere in this schema (see migration 00077's memory_read_daily, +-- which makes the identical choice for the identical reason) and this +-- migration does not blur them. +-- +-- SIGNATURE CHANGE: the RETURNS TABLE gains a column, so DROP + recreate. +-- Body is 00069's verbatim (the `p_key_scopes` allowlist narrowing, the +-- dashboard-client exclusion, the half-open window, the UTC date_trunc +-- anchoring) plus the one `read_kind` case expression, folded into both the +-- SELECT list and the GROUP BY. +-- ═════════════════════════════════════════════════════════════════════════ + +drop function if exists lorekit_read_activity(uuid, text, timestamptz, timestamptz, text, text[]); + +create or replace function lorekit_read_activity( + p_user_id uuid, + p_bucket text default 'day', + p_since timestamptz default null, + p_until timestamptz default null, + p_scope text default null, + -- The CALLING KEY's scope allowlist (00068). No org parameters: `usage_events` + -- is a per-user ledger with no org_id, so there is no tenancy axis to narrow. + p_key_scopes text[] default '{}' +) +returns table (bucket timestamptz, scope text, read_kind text, count bigint) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; +begin + if p_bucket is null or p_bucket not in ('hour', 'day') then + raise exception 'invalid bucket %, expected hour or day', p_bucket + using errcode = '22023'; + end if; + + return query + select date_trunc(p_bucket, ue.created_at at time zone 'UTC') at time zone 'UTC' as bucket, + ue.scope as scope, + case when ue.tool_name = 'memory.read' then 'targeted' else 'bulk' end as read_kind, + sum(coalesce(ue.result_count, 0))::bigint as count + from usage_events ue + where ( + (v_actor is null and auth.role() = 'service_role') + or ue.user_id = v_actor + ) + and ue.tool_name in ('memory.read', 'memory.list', 'memory.search', + 'memory.list_archived') + -- The dashboard reading lore in order to DRAW this chart is not a read + -- the chart should report. `is distinct from` (not `<>`) because the + -- column is nullable and `null <> 'dashboard'` is null, which would + -- silently drop every unattributed event — including every row written + -- before this migration. + and ue.client is distinct from 'dashboard' + -- The calling key's scope allowlist (00068). A NULL scope is an + -- unattributable read: it names nothing, so it leaks nothing, and it is + -- passed through rather than dropped. + and (ue.scope is null or lorekit_api_token_scope_allowed(p_key_scopes, ue.scope)) + -- The optional per-scope filter. `=`, not `is not distinct from`: a + -- caller asking for a named scope wants events attributed to it, never + -- the unattributable NULL-scope remainder. + and (p_scope is null or ue.scope = p_scope) + and (p_since is null or ue.created_at >= p_since) + and (p_until is null or ue.created_at < p_until) + group by 1, ue.scope, + case when ue.tool_name = 'memory.read' then 'targeted' else 'bulk' end + having sum(coalesce(ue.result_count, 0)) > 0 + -- Bucket, scope, THEN read_kind: extends 00051/00069's tiebreak with the + -- new dimension rather than replacing it, so intra-bucket row order + -- stays a total order now that a (bucket, scope) pair can hold two rows. + order by 1 asc, ue.scope asc, read_kind asc; +end; +$$; + +revoke execute on function lorekit_read_activity(uuid, text, timestamptz, timestamptz, text, text[]) + from public, anon; +grant execute on function lorekit_read_activity(uuid, text, timestamptz, timestamptz, text, text[]) + to authenticated, service_role; + +comment on function lorekit_read_activity(uuid, text, timestamptz, timestamptz, text, text[]) is + 'Memory RECORDS read per UTC hour/day, scope, AND read_kind + (targeted = memory.read, bulk = memory.list/search/list_archived — this + function''s own narrow 4-tool definition, NOT usage-stats.ts''s broader + READ_TOOL_NAMES) over the half-open [p_since, p_until) window, excluding + the dashboard client (00054) and narrowed to the calling key''s scope + allowlist (00068/00069). One row per (bucket, scope, read_kind); retrieved + + opened sum to the same total the pre-00080 function gave for the same + window. Visibility is SELF-ONLY with the service-role + NULL escape hatch.'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00080', 'read_activity_read_kind') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00081_usage_memory_count_peak.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- lorekit_usage_memory_count_peak — usage_events.memory_count surfaced. +-- +-- THE GAP: `usage_events.memory_count` (migration 00034) is stamped on every +-- WRITE event with the account's active memory count at that moment, and is +-- exposed by no endpoint and no view. It is a free time series of "how full +-- is this account" that nothing reads. +-- +-- NOT A REPLACEMENT for the existing live count: `/settings/plan` already +-- shows an ACCURATE, LIVE count via `lorekit_memory_count()` (00035, a real +-- `count(*)` against `memories`) in `PlanUsageBar`. That answers "how full +-- am I RIGHT NOW". `usage_events.memory_count` answers a different question +-- — "how full WAS I, over a window" — since it is a snapshot taken at each +-- write, not a live query. This migration adds the peak-over-a-window +-- reading alongside the existing live one, not instead of it. +-- +-- WHY A SEPARATE SCALAR RPC, NOT A `lorekit_usage_stats` COLUMN: +-- `lorekit_usage_stats` groups by (tool_name, outcome, scope_type, client, +-- kind, host) — `memory_count` is a per-EVENT snapshot, not something that +-- sums or means sensibly across a group the way `event_count`/`record_count` +-- do. A `max(memory_count)` bolted onto that grouped return would repeat the +-- same scalar on every row for no reason. A dedicated single-value function +-- is the honest shape for a single-value answer. +-- +-- NO LIMIT IS RETURNED OR HARDCODED HERE — this function answers "how full +-- WAS the account", not "how close to the cap". The caller already has the +-- limit from `lorekit_get_limit`/`lorekit_memory_count` and pairs the two +-- client-side, per the "no numeric limit hardcoded in app code" rule. +-- +-- SERVICE-ROLE WITH NO TARGET USER returns NULL — there is no single +-- account's headroom to report for an unscoped service-role call, matching +-- `lorekit_usage_stats`'s own null-user semantics but returning null instead +-- of an account-wide aggregate (there is no "everyone's peak" that means +-- anything). +-- ═════════════════════════════════════════════════════════════════════════ + +create or replace function lorekit_usage_memory_count_peak( + p_user_id uuid, + p_since timestamptz default null, + p_until timestamptz default null +) +returns integer +language plpgsql +stable +security definer +set search_path = public +as $$ +declare + v_actor uuid := case + when auth.role() = 'service_role' then p_user_id + else auth.uid() + end; + v_peak integer; +begin + -- No specific account to report on — service-role with no target user. + if v_actor is null then + return null; + end if; + + select max(e.memory_count) into v_peak + from usage_events e + where e.user_id = v_actor + and e.memory_count is not null + and (p_since is null or e.created_at >= p_since) + and (p_until is null or e.created_at < p_until); + + return v_peak; +end; +$$; + +revoke execute on function lorekit_usage_memory_count_peak(uuid, timestamptz, timestamptz) from public, anon; +grant execute on function lorekit_usage_memory_count_peak(uuid, timestamptz, timestamptz) + to authenticated, service_role; + +comment on function lorekit_usage_memory_count_peak(uuid, timestamptz, timestamptz) is + 'The highest memories.count snapshot recorded on a WRITE event + (usage_events.memory_count, migration 00034) in the half-open + [p_since, p_until) window -- "how full WAS this account", distinct from + lorekit_memory_count()''s live "how full is it now". Returns null for a + service-role caller with no target user, or when the window has no write + events at all. Self-only: v_actor is auth.uid() for a JWT caller and the + verified service-role''s own p_user_id otherwise -- an authenticated + caller can never pass another user''s id through p_user_id.'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00081', 'usage_memory_count_peak') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00082_usage_event_session_kind.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- usage_events.session_kind — WAS this read in a local session, CI, or a PR +-- automation? +-- +-- THE GAP: `correlation_id` (00044) is bounded, indexed, filterable, and its +-- own code comment names the intended values ("a PR ref, a branch, a session +-- id") -- but it was populated ONLY when a human hand-exported +-- LOREKIT_CORRELATION_ID. Nothing derived it: there was not one reference to +-- GITHUB_RUN_ID or GITHUB_ACTIONS anywhere in packages/cli, packages/mcp-core, +-- supabase, or plugins outside a Vitest reporter config. So "was this lore +-- read in a local session, a PR automation, or a CI job" was designed for +-- and unanswerable. +-- +-- THE FIX has two halves, and this migration is the DB half: +-- +-- 1. `session_kind` -- bounded, closed to local|ci|pr|unknown, the SAME +-- client/scope_type CHECK pattern (migration 00054): bound in app code, +-- a length CHECK on the column as a backstop, never a CHECK enumerating +-- members. This is the column every chart groups on; correlation_id +-- stays the unbounded drill-down key. +-- 2. Recording sites (`_shared/api/router.ts`, `mcp/mcp-handler.ts`) read a +-- NEW `X-LoreKit-Session-Kind` header the CLI now derives and sends +-- (see `packages/cli/src/shared/mcp.mjs`'s `deriveSessionContext` and +-- the shared hook engine) -- the edge never derives, it only validates, +-- the same posture `X-LoreKit-Client` already has. +-- +-- Forward-only and additive: the column is nullable, the writer's new +-- parameter is trailing + defaulted, so every existing row and caller is +-- unaffected. Retroactive for NEW traffic only. +-- ═════════════════════════════════════════════════════════════════════════ + +alter table usage_events add column if not exists session_kind text; + +-- Bound as a BACKSTOP, matching 00054's `usage_events_client_len` exactly: +-- the primary gate is the app-side `parseSessionKind`, this only stops a +-- direct insert from putting an unbounded value into a grouped column. +alter table usage_events drop constraint if exists usage_events_session_kind_len; +alter table usage_events add constraint usage_events_session_kind_len + check (session_kind is null or (char_length(session_kind) between 1 and 16)); + +-- ── writer: add a trailing p_session_kind ──────────────────────────────── +-- DROP first: a new parameter changes the signature. The dropped signature is +-- 00076's fifteen-argument one (…, p_scope, p_scope_count) -- a stale drop +-- target is silent, leaving two overloads live and every named-argument call +-- ambiguous. +drop function if exists lorekit_record_usage_event(uuid, uuid, text, text, text, text, text, integer, integer, integer, text, text, text, text, text, integer); + +create or replace function lorekit_record_usage_event( + p_user_id uuid default null, + p_org_id uuid default null, + p_plan_name text default null, + p_tool_name text default null, + p_scope_type text default null, + p_auth_type text default null, + p_outcome text default null, + p_duration_ms integer default null, + p_memory_count integer default null, + p_result_count integer default null, + p_correlation_id text default null, + p_client text default null, + p_kind text default null, + p_host text default null, + p_scope text default null, + p_scope_count integer default null, + p_session_kind text default null +) +returns uuid +language plpgsql +security definer +set search_path = public +as $$ +declare + v_id uuid; +begin + insert into usage_events ( + user_id, org_id, plan_name, + tool_name, scope_type, auth_type, + outcome, duration_ms, memory_count, + result_count, correlation_id, client, kind, host, + scope, scope_count, session_kind + ) values ( + p_user_id, p_org_id, p_plan_name, + p_tool_name, p_scope_type, p_auth_type, + p_outcome, p_duration_ms, p_memory_count, + p_result_count, p_correlation_id, p_client, p_kind, p_host, + p_scope, p_scope_count, p_session_kind + ) + returning id into v_id; + return v_id; +exception + when others then + -- Never let telemetry writes break the primary operation. + return null; +end; +$$; + +grant execute on function lorekit_record_usage_event(uuid, uuid, text, text, text, text, text, integer, integer, integer, text, text, text, text, text, integer, text) + to anon, authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00082', 'usage_event_session_kind') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00083_usage_runs.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- lorekit_usage_runs — enumerate the runs `?correlation_id=` can only filter. +-- +-- THE GAP: `GET /memories/usage?correlation_id=` filters TO one run, but +-- nothing ENUMERATES which runs exist — a user has no way to discover which +-- correlation ids to even ask about. This is the payoff view: a browsable +-- list of runs (local sessions, CI jobs, PR automations — migration 00082's +-- session_kind), each showing what it read, from which scopes, how long it +-- took, and whether it wrote anything back. +-- +-- WHICH "READ"/"WRITE": the BROADER usage-stats.ts READ_TOOL_NAMES / +-- WRITE_TOOL_NAMES vocabulary (memory.scopes/usage/org.list included on the +-- read side) — a run summary answers "what did this run do overall", not the +-- narrower "memories read" card question lorekit_read_activity/ +-- memory_read_daily answer. Spelled out as a literal list here (this is SQL, +-- not TypeScript, so the two cannot share a runtime import) and named +-- explicitly in the function comment so a future reader does not assume it +-- matches the narrow one. +-- +-- NO ORG AXIS: usage_events is a per-user ledger with no org_id column to +-- narrow by — same posture as lorekit_usage_stats/lorekit_read_activity. +-- +-- KEYSET PAGINATION, NOT OFFSET: follows the Audit Logs precedent +-- (`lib/pagination/keyset.ts`/`cursor.ts` on the web side) — an opaque +-- (last_seen, correlation_id) cursor, and the RPC applies its OWN user_id +-- filter so a forged cursor cannot widen visibility, only mis-page the +-- caller's own rows. Grouping happens BEFORE the keyset predicate (a CTE), +-- since the sort key (last_seen = max(created_at)) only exists post-aggregate. +-- +-- WINDOW: bounded by the CALLER (the REST handler defaults to a bounded +-- window when none is given, the UNBOUNDED_STATS_RANGE posture +-- `lib/queries/explorer-stats.ts` already established) — an all-time +-- enumeration over the highest-volume table in the schema is the performance +-- cliff that posture exists to avoid. +-- +-- INDEX: `usage_events_user_correlation_idx (user_id, correlation_id, +-- created_at desc) where correlation_id is not null` (00044) already orders +-- exactly what this function groups and filters on, so the GROUP BY can walk +-- it as a sorted scan rather than a hash aggregate over the whole table. No +-- new index added — the existing one is the right shape for this query, +-- verification of the actual plan is deferred to CI's Integration smoke job +-- (this sandbox has no live Postgres to EXPLAIN against). +-- ═════════════════════════════════════════════════════════════════════════ + +create or replace function lorekit_usage_runs( + p_user_id uuid, + p_since timestamptz default null, + p_until timestamptz default null, + p_cursor_last_seen timestamptz default null, + p_cursor_correlation_id text default null, + p_limit integer default 20 +) +returns table ( + correlation_id text, + session_kind text, + first_seen timestamptz, + last_seen timestamptz, + read_events bigint, + records_read bigint, + write_events bigint, + distinct_scopes bigint, + total_duration_ms bigint +) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + v_limit integer := least(greatest(coalesce(p_limit, 20), 1), 100); + -- usage-stats.ts's READ_TOOL_NAMES / WRITE_TOOL_NAMES, spelled out here + -- since SQL cannot import the TypeScript source of truth. Keep in step by + -- hand if that list ever changes. + v_read_tools text[] := array['memory.read', 'memory.list', 'memory.search', + 'memory.scopes', 'memory.list_archived', + 'memory.usage', 'org.list', 'org.get', + 'member.list', 'member.invite_list']; + v_write_tools text[] := array['memory.write', 'memory.delete', 'memory.archive', + 'memory.restore', 'memory.purge', 'memory.purge_expired', + 'org.create', 'org.rename', 'org.delete', + 'member.role_change', 'member.remove', + 'member.invite', 'member.revoke']; +begin + return query + with runs as ( + select + ue.correlation_id, + -- A run is assumed internally consistent (one CLI invocation's + -- environment does not change mid-run); max() is a simplification + -- for the rare case a correlation id is reused across differing + -- session_kind values, not a claim that every event agreed. + max(ue.session_kind) as session_kind, + min(ue.created_at) as first_seen, + max(ue.created_at) as last_seen, + count(*) filter (where ue.tool_name = any(v_read_tools)) as read_events, + coalesce(sum(ue.result_count) filter (where ue.tool_name = any(v_read_tools)), 0) as records_read, + count(*) filter (where ue.tool_name = any(v_write_tools)) as write_events, + count(distinct ue.scope) filter (where ue.scope is not null) as distinct_scopes, + coalesce(sum(ue.duration_ms), 0) as total_duration_ms + from usage_events ue + where ( + (v_actor is null and auth.role() = 'service_role') + or ue.user_id = v_actor + ) + and ue.correlation_id is not null + and (p_since is null or ue.created_at >= p_since) + and (p_until is null or ue.created_at < p_until) + group by ue.correlation_id + ) + select r.correlation_id, r.session_kind, r.first_seen, r.last_seen, + r.read_events, r.records_read, r.write_events, r.distinct_scopes, + r.total_duration_ms + from runs r + where p_cursor_last_seen is null + or (r.last_seen, r.correlation_id) < (p_cursor_last_seen, p_cursor_correlation_id) + order by r.last_seen desc, r.correlation_id desc + limit v_limit; +end; +$$; + +revoke execute on function lorekit_usage_runs(uuid, timestamptz, timestamptz, timestamptz, text, integer) from public, anon; +grant execute on function lorekit_usage_runs(uuid, timestamptz, timestamptz, timestamptz, text, integer) + to authenticated, service_role; + +comment on function lorekit_usage_runs(uuid, timestamptz, timestamptz, timestamptz, text, integer) is + 'Enumerates runs (distinct correlation_id values) with per-run session_kind, + first/last seen, read/write event and record counts, distinct scopes + touched, and total duration. Uses usage-stats.ts''s BROADER + READ_TOOL_NAMES/WRITE_TOOL_NAMES (not lorekit_read_activity''s narrower + 4-tool "read" definition) -- a run summary answers "what did this run do + overall". Keyset-paginated on (last_seen desc, correlation_id desc); the + caller supplies the previous page''s last row as the cursor. Self-only + visibility, same escape hatch as lorekit_usage_stats. No org axis -- + usage_events is a per-user ledger.'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00083', 'usage_runs') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00084_memory_read_counters.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- Per-memory read counters + daily rollup — closing the structural gap: +-- `usage_events` records HOW MANY records a call touched, never WHICH. There +-- is no `memory_id` on the read ledger and none of the tables in this schema +-- is a per-memory read table, so "is this lesson earning its place" is absent +-- from the data, not merely unsurfaced. +-- +-- THE CHEAP SHAPE, deliberately NOT a per-read event table: at the observed +-- ~495K read records/month for a SINGLE account, an event table would be the +-- highest-volume table in the schema by an order of magnitude on day one and +-- would need retention + rollup from the start. Counters are bounded by +-- memories × days × read_kind instead of by traffic, and answer everything +-- except the finest-grained "show me every individual read event" question. +-- +-- 1. `memories.read_count` / `.last_read_at` — the two numbers a reader +-- wants on the detail sheet without any join: "how many times, and when +-- last". `not null default 0` so every existing memory starts at a real, +-- countable zero rather than a NULL that reads as "unknown". +-- +-- 2. `memory_read_daily(memory_id, day, read_kind, count)` — the +-- per-context breakdown a bare counter cannot answer ("read this week vs +-- last month"). `read_kind` distinguishes `targeted` (`memory.read`, one +-- exact scope+key) from `bulk` (`memory.list` / `memory.search` / +-- `memory.list_archived`, every row a listing call returned) — reusing +-- `lorekit_read_activity`'s OWN narrow 4-tool "read" definition, not the +-- broader `READ_TOOL_NAMES` in `usage-stats.ts` (which also counts +-- `memory.scopes` / `memory.usage` / `org.list` — none of which touch a +-- single memory row, so none of them ever call the writer below at all). +-- This is the DOCUMENTED choice migration 00053 already forces every +-- "split by read definition" decision to make explicit; see +-- `lorekit_record_memory_reads`'s comment. +-- +-- Deliberately WITHOUT a `session_kind` dimension (local/CI/PR +-- automation): that requires deriving `session_kind` at the point of the +-- call, which is separate, not-yet-shipped work. Adding it later is a +-- column addition to this same table, not a redesign — `read_kind` +-- alone already answers "opened vs skimmed", the question this migration +-- exists to unlock. +-- +-- WRITE PATH: `lorekit_record_memory_reads(p_memory_ids uuid[], p_read_kind +-- text)`, called once per request with every memory id that request actually +-- returned — a bulk `list` returning 31 rows is ONE call with a 31-element +-- array, never 31 statements. Same fail-safe contract as +-- `lorekit_record_usage_event`: a counter update must never fail the read it +-- is measuring, so the whole body is wrapped in `when others` and swallows +-- any error rather than propagating it. No-ops on an empty/null array (a +-- call that matched nothing touched no memory) and skips ids under +-- concurrent deletion via `on conflict do nothing`-style tolerance (see the +-- function body). +-- +-- CUTOVER: every existing memory starts at `read_count = 0`, which is +-- indistinguishable from "never read" — it is really "never read SINCE this +-- migration". Consumers (the Wave E "hot/cold lore" work) MUST carry this +-- migration's deployment date through their copy as the cutover, or the very +-- first cold-lore view flags the entire pre-existing library as unused. This +-- migration does not itself claim a cutover date — deployment time is server +-- history, not migration-file content — but the requirement is recorded here +-- so the consuming PR cannot skip it. +-- +-- RETENTION on `memory_read_daily`: follows `usage_events`' +-- `lorekit_purge_old_usage_events` precedent (a scheduled pg_cron sweep) — +-- deferred to the PR that actually schedules it, since this migration adds no +-- pg_cron job of its own and an unbounded promise here would be undeliverable. +-- The table is additive-only and its own row count is bounded by +-- memories × days × 2 (read_kind), which grows far slower than usage_events. +-- +-- CASCADE: `on delete cascade` from `memories` — an archived memory KEEPS its +-- counters (archiving never deletes the row), a PURGED memory's rollup rows +-- are removed with it, so no orphaned `memory_read_daily` row can ever +-- outlive the memory it counted reads for. +-- ═════════════════════════════════════════════════════════════════════════ + +-- ── memories: the two counters ─────────────────────────────────────────── +alter table memories add column if not exists read_count integer not null default 0; +alter table memories add column if not exists last_read_at timestamptz; + +-- A non-negative backstop, matching the `usage_events.scope_count` (00076) +-- reasoning: the app-side increment is the primary gate, this is insurance +-- against a direct insert/update putting a negative count on a row a "most +-- read" ranking will sort by. +alter table memories drop constraint if exists memories_read_count_non_negative; +alter table memories add constraint memories_read_count_non_negative + check (read_count >= 0); + +-- ── memory_read_daily: the per-context rollup ──────────────────────────── +create table if not exists memory_read_daily ( + memory_id uuid not null references memories(id) on delete cascade, + day date not null, + read_kind text not null, + count integer not null default 0, + primary key (memory_id, day, read_kind) +); + +-- Bounded, closed, app-code vocabulary — the `client`/`scope_type` CHECK +-- pattern (migration 00054), not an enumerating gate: `read_kind` is closed to +-- exactly two values today (`targeted`/`bulk`), and this CHECK is the +-- backstop against a third value smuggling itself into a grouped column, not +-- the primary authority for what the values mean. +alter table memory_read_daily drop constraint if exists memory_read_daily_read_kind_check; +alter table memory_read_daily add constraint memory_read_daily_read_kind_check + check (read_kind in ('targeted', 'bulk')); + +alter table memory_read_daily drop constraint if exists memory_read_daily_count_non_negative; +alter table memory_read_daily add constraint memory_read_daily_count_non_negative + check (count >= 0); + +-- The one read this table exists for: "this memory's read history, most +-- recent day first" (the per-memory consumption panel on the detail sheet). +-- The primary key already covers memory_id lookups; this orders them. +create index if not exists memory_read_daily_memory_day_idx + on memory_read_daily (memory_id, day desc); + +alter table memory_read_daily enable row level security; +-- No policies: this table is never read or written through PostgREST/RLS — +-- every access is through the SECURITY DEFINER functions below (the write) and +-- the ranked RPC a later PR adds (the read), exactly like `usage_events`. RLS +-- enabled with zero policies denies every direct PostgREST request by default, +-- which is the point: a service-role or SECURITY DEFINER context bypasses RLS +-- entirely, so this is a backstop against the one path that is not one of those +-- — an anon/authenticated key querying the table directly. + +revoke all on memory_read_daily from anon, authenticated; +grant select, insert, update on memory_read_daily to service_role; + +-- ── the writer ──────────────────────────────────────────────────────────── +create or replace function lorekit_record_memory_reads( + p_memory_ids uuid[], + p_read_kind text +) +returns void +language plpgsql +security definer +set search_path = public +as $$ +begin + if p_memory_ids is null or array_length(p_memory_ids, 1) is null then + return; + end if; + + -- ONE statement for every id, not one per id — the hot-path requirement a + -- bulk `list` returning 31 rows imposes. `unnest` turns the array into a rowset + -- the UPDATE can join against directly. + update memories m + set read_count = m.read_count + 1, + last_read_at = now() + from unnest(p_memory_ids) as ids(id) + where m.id = ids.id; + + -- Upsert today's rollup row per memory — one INSERT, conflict-aggregated, + -- same shape as the update above. `current_date` is UTC-anchored the same + -- way `usage_events`' other date_trunc'd reads are (`at time zone 'UTC'`), + -- so a rollup day boundary agrees with every other UTC-bucketed series in + -- this schema. + insert into memory_read_daily (memory_id, day, read_kind, count) + select ids.id, (now() at time zone 'UTC')::date, p_read_kind, 1 + from unnest(p_memory_ids) as ids(id) + on conflict (memory_id, day, read_kind) + do update set count = memory_read_daily.count + 1; +exception + when others then + -- Never let a counter write break the read it is measuring. + return; +end; +$$; + +revoke execute on function lorekit_record_memory_reads(uuid[], text) from public, anon; +grant execute on function lorekit_record_memory_reads(uuid[], text) to authenticated, service_role; + +comment on function lorekit_record_memory_reads(uuid[], text) is + 'Increments memories.read_count/.last_read_at and today''s memory_read_daily + row (UTC day, keyed by read_kind) for every memory id a read call actually + returned, in ONE statement regardless of array size. read_kind is + targeted (memory.read) or bulk (memory.list / memory.search / + memory.list_archived) -- lorekit_read_activity''s own narrow "read" + definition, not usage-stats.ts''s broader READ_TOOL_NAMES. Fail-safe: never + throws, so a counter-write failure cannot fail the read it measures. No-op + on a null/empty array.'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00084', 'memory_read_counters') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00085_memory_read_ranking.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- lorekit_memory_read_ranking — hot and cold lore, ranked by memories.read_count +-- (migration 00084). +-- +-- THE PAYOFF: with per-memory counters in place, "these 40 lessons have not +-- been read since " becomes a real ranked query instead of a shrug — +-- exactly the prune-list input the `lorekit-groom` skill exists to consume. +-- `hot` (read_count desc) surfaces the most-consumed lore; `cold` (read_count +-- asc, oldest-created first) surfaces the least. +-- +-- COUNTING_SINCE / THE CUTOVER: `read_count = 0` on a memory created before +-- migration 00084 shipped is NOT "never read" — it is "not read since 00084 +-- started counting". A memory written last year could have been read 200 +-- times under the old, uncounted regime. This function returns no such date +-- itself (a SQL function has no honest way to know when its OWN migration was +-- applied to THIS database); the calling handler stamps it from a constant +-- recording that deploy date, and every consumer MUST render it rather than +-- the bare word "never" — see `ReadRankingResponseSchema.counting_since`. +-- +-- SCOPE: active (non-archived, non-expired) memories only, mirroring +-- `lorekit_memory_scopes` (00065) — an archived memory is already pruned, so +-- ranking it for pruning again is noise. Same org-shared + own-rows visibility +-- predicate as every other per-user analytics RPC in this file family +-- (`lorekit_memory_scopes`, `lorekit_read_activity`), and the same +-- service-role + NULL actor escape hatch. +-- +-- INDEX: `(user_id, read_count)` partial on `archived_at is null` — the +-- `memories_keyset_index` (00061) / `memory_scopes_order_by_count` (00065) +-- precedent for indexing an ordered analytics read. `now()` is intentionally +-- NOT in the partial predicate (a volatile function there defeats the +-- planner's ability to use the index for an unrelated snapshot) — the +-- non-expired filter is applied in the query body instead, exactly as 00065 +-- does for its own scope rollup. +-- ═════════════════════════════════════════════════════════════════════════ + +create index if not exists memories_user_read_count_idx + on memories (user_id, read_count) + where archived_at is null; + +create or replace function lorekit_memory_read_ranking( + p_user_id uuid, + p_direction text default 'hot', + p_scope text default null, + p_limit integer default 20, + p_key_scopes text[] default '{}' +) +returns table ( + id uuid, + scope text, + key text, + read_count integer, + last_read_at timestamptz, + seen_count integer, + created_at timestamptz +) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + v_limit integer := least(greatest(coalesce(p_limit, 20), 1), 100); +begin + if p_direction is null or p_direction not in ('hot', 'cold') then + raise exception 'invalid direction %, expected hot or cold', p_direction + using errcode = '22023'; + end if; + + return query + select m.id, m.scope, m.key, m.read_count, m.last_read_at, m.seen_count, m.created_at + from memories m + where ( + (v_actor is null and auth.role() = 'service_role') + or m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) + ) + and m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + and (p_scope is null or m.scope = p_scope) + -- A scoped API key (00068/00069) sees only ITS allowed scopes on an + -- unfiltered call, the same narrowing lorekit_read_activity applies — + -- an unscoped/JWT caller passes an empty p_key_scopes, under which + -- lorekit_api_token_scope_allowed admits everything. + and lorekit_api_token_scope_allowed(p_key_scopes, m.scope) + order by + case when p_direction = 'hot' then m.read_count end desc, + case when p_direction = 'hot' then m.last_read_at end desc nulls last, + case when p_direction = 'cold' then m.read_count end asc, + case when p_direction = 'cold' then m.created_at end asc, + m.id desc + limit v_limit; +end; +$$; + +revoke execute on function lorekit_memory_read_ranking(uuid, text, text, integer, text[]) from public, anon; +grant execute on function lorekit_memory_read_ranking(uuid, text, text, integer, text[]) to authenticated, service_role; + +comment on function lorekit_memory_read_ranking(uuid, text, text, integer, text[]) is + 'Memories ranked by read_count (migration 00084) — hot (most-read first) or + cold (least-read, oldest-created first among ties). Active rows only + (non-archived, non-expired), same org-shared + self visibility as + lorekit_memory_scopes. read_count = 0 means "not read since 00084 started + counting", never "never read" -- the calling handler must stamp and every + consumer must render the counting_since qualifier. p_limit is clamped to + [1, 100].'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00085', 'memory_read_ranking') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00086_reindex_memories_tags_idx.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- Rebuild + verify memories_tags_idx — the GIN index 00076 added is not +-- preventing the seq-scan it was meant to fix. +-- +-- WHAT'S HAPPENING (2026-08-28, service.name=web, service.namespace=lorekit) +-- +-- The "web — elevated backend p95 latency" check fired again +-- (Dash0 issues 8667967338373977430 and 9019610890880570001), and the +-- offending trace reproduces EXACTLY the signature 00076 diagnosed and was +-- meant to close: +-- +-- HEAD /rest/v1/memories?select=id 0.398s +-- HEAD /rest/v1/memories?select=id&tags=cs.%7Bsource%3A%3Apr-webhook%7D 101.697s +-- +-- Same sibling-comparison signature as before — the unfiltered count stays +-- sub-second while the `tags @> ARRAY['source::pr-webhook']` containment +-- read (still `getOnboardingState()`, still awaited by every `/overview` +-- render) balloons to 100+ seconds. `memories_tags_idx` (00076) should be +-- serving this predicate; a query that should be an index scan is behaving +-- like a sequential scan again, four days after the fix merged. +-- +-- WHAT THIS MIGRATION DOES +-- +-- It does not change the query or the index definition — 00076's GIN index +-- on `tags` with the default `array_ops` opclass is still the right shape +-- for `@>`/`<@`/`&&`. This is a repair-and-verify pass for the cases that +-- would silently defeat an otherwise-correct index without any schema +-- change showing up in a diff: +-- +-- 1. The index exists but is marked INVALID (e.g. a prior CONCURRENTLY +-- build — none is used here or in 00076, but a manual out-of-band +-- rebuild could have left one) or has drifted from the heap due to +-- bloat, so the planner (correctly) stops trusting it. +-- 2. Table/index statistics are stale, so the planner's cost estimate for +-- the containment scan no longer favours the index over a seq scan +-- even though the index itself is fine. +-- +-- `REINDEX INDEX` rebuilds the index from scratch against the current heap, +-- resolving (1) unconditionally. `ANALYZE` refreshes the planner's +-- statistics on `memories`, resolving (2). `ANALYZE` is always safe and +-- cheap to re-run even when it wasn't the problem. `REINDEX INDEX` (no +-- `CONCURRENTLY`) is NOT free in that case, though: it takes an ACCESS +-- EXCLUSIVE lock on `memories` for the rebuild, briefly blocking reads and +-- writes to the table regardless of whether the index was already healthy. +-- `memories` is small enough today that this brief lock is an acceptable +-- trade-off — consistent with 00076's own reasoning for skipping +-- `CONCURRENTLY` — but it is a real (if short) cost, not a no-op. +-- +-- `CONCURRENTLY` is deliberately NOT used, matching 00076's own reasoning: +-- Supabase migrations run inside a transaction and neither +-- `REINDEX INDEX CONCURRENTLY` nor `CREATE INDEX CONCURRENTLY` can run +-- inside one. The `memories` table remains small enough that a regular +-- (briefly lock-holding) REINDEX is the right trade-off, consistent with +-- every other index build in this schema (00001_memories.sql, 00076). +-- +-- `create index if not exists` is kept ahead of the REINDEX as a safety net +-- in case the index was dropped outright rather than merely degraded — +-- `REINDEX INDEX` errors on a missing index, so recreating it first keeps +-- this migration safely re-runnable regardless of which failure mode +-- produced today's regression. +-- ───────────────────────────────────────────────────────────────────────────── + +create index if not exists memories_tags_idx on memories using gin (tags); + +reindex index memories_tags_idx; + +analyze memories; + +comment on index memories_tags_idx is + 'GIN index supporting tags @> / <@ / && containment queries (e.g. ' + 'onboarding-server.ts''s source::pr-webhook check), avoiding a sequential ' + 'scan on every /overview render. Rebuilt + ANALYZEd by 00086 after the ' + 'same seq-scan signature reappeared four days after 00076 first added it ' + '(Dash0 issues 8667967338373977430 / 9019610890880570001).'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00086', 'reindex_memories_tags_idx') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00088_retention_policies.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- Retention policies: scoped, saved rules that AUTO-ARCHIVE (never hard-delete) +-- matching lessons. The automated generalization of the manual TTL/expiry +-- grooming that already exists (purge_expired_memories, the lorekit-groom +-- skill). v1 is PERSONAL-owned (user_id-keyed) only — org-owned policies are +-- explicitly deferred to a later phase. +-- +-- ONE candidate function (lorekit_groom_candidates) is the single source of +-- truth for "what matches" — preview, run-now, and the nightly sweep all +-- resolve the SAME candidates, so a previewed count always equals what a run +-- archives. Archiving reuses the existing soft-archive mechanism +-- (archived_at, 00003_archive.sql) — a policy NEVER hard-deletes; hard purge +-- stays exactly where it is (memory.purge / memory.purge_expired, manual, +-- confirm-or---yes). +-- +-- ── "unseen_days" and memories.last_seen_at ───────────────────────────────── +-- LoreKit does not yet track per-memory READ activity (usage_events records +-- reads in aggregate, not per row). Rather than wiring real read-tracking +-- through every read handler — a much larger change than this migration's +-- scope — `last_seen_at` ships here as a nullable column that starts out NULL +-- for every row (nothing sets it yet) and is reserved for that follow-up. +-- lorekit_groom_candidates treats a NULL last_seen_at as negative infinity, +-- so "unseen for N days" is unconditionally TRUE for a never-seen row — this +-- is the literal reading of "unseen_days includes never-seen lessons" and +-- requires no additional wiring to be correct today (every row is +-- "never-seen" until read-tracking lands). Wiring memory.read/list/search to +-- bump it is deliberately left as a follow-up, noted in +-- docs/adding-an-operation.md's worked-example feature. + +-- 1. retention_policies — one row per saved rule. +create table if not exists retention_policies ( + id uuid primary key default gen_random_uuid(), + user_id uuid not null references auth.users on delete cascade, + scope text not null, + name text not null, + -- 'review' — surfaced in the dashboard for a human to run manually. + -- 'auto' — swept nightly by lorekit_groom_sweep, IF enabled=true. + mode text not null default 'review' check (mode in ('review', 'auto')), + -- Auto mode starts DISABLED per policy — a saved rule must be explicitly + -- turned on before it archives anything unattended. + enabled boolean not null default false, + min_age_days integer check (min_age_days is null or min_age_days > 0), + unseen_days integer check (unseen_days is null or unseen_days > 0), + max_seen_count integer check (max_seen_count is null or max_seen_count >= 0), + created_at timestamptz not null default now(), + updated_at timestamptz not null default now() +); + +create index if not exists retention_policies_user_id_idx on retention_policies (user_id); + +alter table retention_policies enable row level security; + +-- Owner-only, full CRUD via RLS — unlike memories, there is no org-shared +-- read path yet (v1 is personal-owned), so a single policy suffices. +create policy "rls_retention_policies_owner" + on retention_policies for all + using (user_id = auth.uid()) + with check (user_id = auth.uid()); + +create or replace trigger retention_policies_updated_at + before update on retention_policies + for each row execute function set_updated_at(); + +-- 2. memories.protected — excluded from every grooming candidate set, +-- regardless of policy. memories.last_seen_at — see header note above. +alter table memories add column if not exists protected boolean not null default false; +alter table memories add column if not exists last_seen_at timestamptz; + +-- 3. lorekit_groom_candidates — THE single source of truth for "what +-- matches" a set of conditions. Called directly by groom.preview, and by +-- lorekit_groom_run below (never a second query). SECURITY DEFINER so it +-- can be called by an unprivileged token's resolved user_id, matching the +-- purge_archived_memories / restore_memory precedent — the explicit +-- p_user_id parameter (never auth.uid()) is what lets an api_key caller's +-- resolved user reach it. +-- +-- Scope matching: p_scope = 'global' matches every memory; an exact scope +-- match always matches; a 'repo::owner/repo' policy additionally matches +-- every 'branch::owner/repo::*' memory (a branch scope's repo portion IS +-- the containing repo scope's value) — the "::"-delimited hierarchy the +-- plan's Decisions describe. 'project::*' and 'branch::*' policies match +-- only their exact scope, since no narrower scope type nests under them. +create or replace function lorekit_groom_candidates( + p_user_id uuid, + p_scope text, + p_min_age_days integer default null, + p_unseen_days integer default null, + p_max_seen_count integer default null +) +returns table (id uuid, scope text, key text) +language sql +stable +security definer +set search_path = public +as $$ + select m.id, m.scope, m.key + from memories m + where m.user_id = p_user_id + and m.archived_at is null + and m.protected = false + and ( + p_scope = 'global' + or m.scope = p_scope + or ( + p_scope like 'repo::%' + -- starts_with (not LIKE) so `_`/`%` in the repo name are literal, not + -- wildcards — an over-match bug this repo hit before (#231, #260). + and starts_with(m.scope, 'branch::' || substring(p_scope from 7) || '::') + ) + ) + and (p_min_age_days is null or m.created_at <= now() - (p_min_age_days * interval '1 day')) + and (p_unseen_days is null or coalesce(m.last_seen_at, '-infinity'::timestamptz) <= now() - (p_unseen_days * interval '1 day')) + and (p_max_seen_count is null or m.seen_count <= p_max_seen_count) + order by m.scope, m.key; +$$; + +grant execute on function lorekit_groom_candidates(uuid, text, integer, integer, integer) + to authenticated, service_role; + +-- 4. lorekit_groom_run — archives every candidate in ONE transaction (this +-- function's body), returns the count + the archived keys, and NEVER +-- deletes a row. Deliberately does NOT write audit_log itself — LoreKit's +-- audit capture model is app-layer, explicit recordAudit calls right after +-- the primary operation succeeds (CLAUDE.md "Audit logging is captured at +-- the app layer"), not a DB-side insert. Both callers (the MCP groom.run +-- tool and the REST POST /groom/run handler) write one 'memory.archive' +-- audit row per key in the returned `keys` array after this RPC returns. +create or replace function lorekit_groom_run( + p_user_id uuid, + p_scope text, + p_min_age_days integer default null, + p_unseen_days integer default null, + p_max_seen_count integer default null +) +returns table (archived integer, keys jsonb) +language plpgsql +security definer +set search_path = public +as $$ +declare + v_ids uuid[]; + v_keys jsonb; + v_count integer := 0; +begin + select coalesce(array_agg(c.id), '{}'), + coalesce(jsonb_agg(jsonb_build_object('scope', c.scope, 'key', c.key)), '[]'::jsonb) + into v_ids, v_keys + from lorekit_groom_candidates(p_user_id, p_scope, p_min_age_days, p_unseen_days, p_max_seen_count) c; + + if array_length(v_ids, 1) is null then + return query select 0, '[]'::jsonb; + return; + end if; + + update memories m + set archived_at = now() + where m.id = any(v_ids) + and m.archived_at is null; -- defensive; candidates are already unarchived + get diagnostics v_count = row_count; + + return query select v_count, v_keys; +end; +$$; + +grant execute on function lorekit_groom_run(uuid, text, integer, integer, integer) + to authenticated, service_role; + +-- 5. lorekit_groom_sweep — the nightly cron target. Archives ONLY for +-- policies with mode='auto' AND enabled=true; 'review' policies and +-- disabled 'auto' policies are left untouched (they surface in the +-- dashboard for a human to run via groom.run instead). Returns the total +-- number of memories archived across every swept policy. +create or replace function lorekit_groom_sweep() +returns integer +language plpgsql +security definer +set search_path = public +as $$ +declare + v_policy retention_policies%rowtype; + v_result record; + v_total integer := 0; +begin + for v_policy in + select * from retention_policies where mode = 'auto' and enabled = true + loop + select * into v_result + from lorekit_groom_run( + v_policy.user_id, v_policy.scope, + v_policy.min_age_days, v_policy.unseen_days, v_policy.max_seen_count + ); + v_total := v_total + coalesce(v_result.archived, 0); + end loop; + + return v_total; +end; +$$; + +grant execute on function lorekit_groom_sweep() to service_role; + +-- 6. lorekit_memory_protect — toggle memories.protected for one lesson. +-- SECURITY DEFINER, mirroring archive_memory/restore_memory (00003): +-- it runs as the table owner but only touches the row addressed by the +-- caller's own resolved user_id. +create or replace function lorekit_memory_protect( + p_user_id uuid, + p_scope text, + p_key text, + p_protected boolean +) +returns boolean +language plpgsql +security definer +set search_path = public +as $$ +declare + v_id uuid; +begin + update memories + set protected = p_protected + where user_id = p_user_id + and scope = p_scope + and key = p_key + and archived_at is null + returning id into v_id; + + return v_id is not null; +end; +$$; + +grant execute on function lorekit_memory_protect(uuid, text, text, boolean) + to authenticated, service_role; + +-- 7. Policy CRUD RPCs — SECURITY DEFINER, owner-scoped by explicit p_user_id +-- (never auth.uid()), matching every mutation RPC above. Routing CRUD +-- through RPCs rather than the generated PostgREST/postgrest-js Database +-- types is deliberate: `retention_policies` is a brand-new table the +-- generated `database.types.ts` mirror does not know about, and every +-- other mutable resource in this schema (memories, orgs, api_tokens) is +-- already reached through a SECURITY DEFINER function rather than a raw +-- `.from(table)` call for the same api_key/service-role-bypasses-RLS +-- reason — this is the existing pattern, not a new one. +create or replace function lorekit_policy_list(p_user_id uuid) +returns setof retention_policies +language sql +stable +security definer +set search_path = public +as $$ + select * from retention_policies where user_id = p_user_id order by created_at desc; +$$; + +grant execute on function lorekit_policy_list(uuid) to authenticated, service_role; + +-- Every policy-CRUD RPC below RETURNS SETOF (zero or one row) rather than a +-- bare composite. A plpgsql function declared `returns retention_policies` +-- that never assigns its %rowtype OUT variable still returns a NON-NULL +-- composite with every field null — not SQL NULL — so a caller checking +-- "found" by nullability would misread a not-found UPDATE/DELETE as a hit. +-- SETOF sidesteps that ambiguity entirely: zero matching rows is a zero-length +-- result set, one match is a one-row set, and the postgrest-js caller reads +-- `data[0]` with no `.single()`/`.maybeSingle()` needed (TracedRpcQuery only +-- exposes `.single()`, which errors on a zero-row result — the wrong shape for +-- an expected "not found"). +create or replace function lorekit_policy_create( + p_user_id uuid, + p_scope text, + p_name text, + p_mode text default 'review', + p_enabled boolean default false, + p_min_age_days integer default null, + p_unseen_days integer default null, + p_max_seen_count integer default null +) +returns setof retention_policies +language sql +security definer +set search_path = public +as $$ + insert into retention_policies (user_id, scope, name, mode, enabled, min_age_days, unseen_days, max_seen_count) + values (p_user_id, p_scope, p_name, p_mode, p_enabled, p_min_age_days, p_unseen_days, p_max_seen_count) + returning *; +$$; + +grant execute on function lorekit_policy_create(uuid, text, text, text, boolean, integer, integer, integer) + to authenticated, service_role; + +-- Partial update via a JSONB patch: a key ABSENT from p_patch leaves the +-- column unchanged; a key present with a JSON null CLEARS it (sets the +-- column to NULL); a key present with a value SETS it. The `?` (has-key) +-- operator is what makes "absent" and "present-but-null" distinguishable — +-- `coalesce(p_patch->>'x', x)` alone could never clear a field. Zero rows +-- back means p_id does not exist or is not owned by p_user_id — "not found". +create or replace function lorekit_policy_update( + p_user_id uuid, + p_id uuid, + p_patch jsonb +) +returns setof retention_policies +language sql +security definer +set search_path = public +as $$ + update retention_policies + set name = case when p_patch ? 'name' then p_patch ->> 'name' else name end, + mode = case when p_patch ? 'mode' then p_patch ->> 'mode' else mode end, + enabled = case when p_patch ? 'enabled' then (p_patch ->> 'enabled')::boolean else enabled end, + min_age_days = case when p_patch ? 'min_age_days' then (p_patch ->> 'min_age_days')::integer else min_age_days end, + unseen_days = case when p_patch ? 'unseen_days' then (p_patch ->> 'unseen_days')::integer else unseen_days end, + max_seen_count = case when p_patch ? 'max_seen_count' then (p_patch ->> 'max_seen_count')::integer else max_seen_count end + where id = p_id and user_id = p_user_id + returning *; +$$; + +grant execute on function lorekit_policy_update(uuid, uuid, jsonb) to authenticated, service_role; + +create or replace function lorekit_policy_delete(p_user_id uuid, p_id uuid) +returns setof retention_policies +language sql +security definer +set search_path = public +as $$ + delete from retention_policies + where id = p_id and user_id = p_user_id + returning *; +$$; + +grant execute on function lorekit_policy_delete(uuid, uuid) to authenticated, service_role; + +-- 8. Nightly sweep schedule — guarded so this migration still applies on an +-- instance without pg_cron (see the identical guard in +-- 00003_archive.sql / 00004_limits.sql's rate-limit reaper). 03:17 UTC to +-- land off the hour, away from other scheduled jobs. +do $$ +begin + if exists (select 1 from pg_extension where extname = 'pg_cron') then + perform cron.schedule( + 'lorekit-groom-sweep', + '17 3 * * *', + $cron$select lorekit_groom_sweep()$cron$ + ); + end if; +end; +$$; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00088', 'retention_policies') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00089_audit_log_groom_actions.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- Retention policies, part 2: the audit vocabulary. +-- +-- 00088 added policy.* (create/update/delete) and memory.protect as +-- security/data-affecting mutations. Every such action needs a trail, and +-- `recordAudit`/`recordAuditEvent` deliberately never throw on a rejected +-- action — a call with an action the CHECK refuses is swallowed and logged to +-- the server console only, leaving a silent, permanent hole (00070's exact +-- rationale). Widening the CHECK before the call sites ship is the whole fix. +-- +-- `groom.run` reuses the EXISTING `memory.archive` action for its per-lesson +-- audit rows (see 00088's lorekit_groom_run) rather than minting a new +-- `memory.groom` — the archive path already has that action, and reusing it +-- keeps grooming's trail in the existing vocabulary instead of growing a +-- parallel one for what is, from audit_log's point of view, an archive. +-- +-- Forward-only: drop + re-add (a CHECK cannot be widened in place) — the +-- 00023/00027/00042/00070 pattern. The list below is derived from the ONE +-- source, `packages/schemas/src/domain/audit.ts`'s `AUDIT_ACTIONS`, and +-- `packages/mcp-core/src/audit/audit-vocabulary.spec.ts` parses the newest +-- action-CHECK migration (this file) and fails if the two sets differ. + +alter table audit_log drop constraint audit_log_action_check; + +alter table audit_log add constraint audit_log_action_check check (action in ( + 'api_key.create', + 'api_key.revoke', + 'api_key.scope_change', + 'webhook_secret.create', + 'webhook_secret.rotate', + 'webhook_secret.deactivate', + 'memory.create', + 'memory.update', + 'memory.archive', + 'memory.restore', + 'memory.delete', + 'limit.override', + 'org.create', + 'org.rename', + 'org.delete', + 'member.invite', + 'member.accept', + 'member.decline', + 'member.revoke', + 'member.remove', + 'member.role_change', + 'member.leave', + 'scope.bind', + 'scope.unbind', + 'github_app.installation_linked', + 'policy.create', + 'policy.update', + 'policy.delete', + 'memory.protect' +)); + +insert into supabase_migrations.schema_migrations (version, name) +values ('00089', 'audit_log_groom_actions') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00090_memory_pivot.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- lorekit_memory_pivot — the TWO-dimensional facet count. +-- +-- ── WHY ────────────────────────────────────────────────────────────────── +-- `lorekit_memory_facets` (00052/00057, helper-ised in 00066) answers "how +-- many memories carry each value, per dimension" — a ONE-dimensional pivot. +-- The Explorer's new matrix instrument asks the next question: "how many +-- carry value X of dimension A *and* value Y of dimension B", which is the +-- same aggregate with a second group-by. +-- +-- The alternative the dashboard could reach for is one facets call per row +-- value with that value pushed into the filters — an N+1 whose N is chosen by +-- the UI, over the largest table in the schema. The rule in +-- `packages/web/CLAUDE.md` is explicit that an aggregate the dashboard needs +-- belongs in Postgres behind an endpoint, so this is that endpoint's function. +-- +-- ── SELF-EXCLUSION ON *BOTH* AXES, AND WHY IT IS THE FEATURE ───────────── +-- 00057 established that a facet is counted with every OTHER active filter +-- applied but not its own, so the dimension you are standing in still lists +-- everything you could switch to. A matrix has TWO dimensions in that +-- position, so both are excluded. +-- +-- This is not a nicety. Count a matrix under its own axes' filters and the +-- instant you click a cell every other cell reads zero — the grid blacks +-- itself out and there is no second click. The instrument is only navigable +-- because the counts answer "what would selecting this cell yield", which is +-- exactly what excluding the two axes computes. +-- +-- ── ONE ROW-VISIBILITY PREDICATE, COPIED FROM NOWHERE ─────────────────── +-- The base CTE is `lorekit_memory_facets`' verbatim: the same tenant test +-- (`lorekit_member_org_ids`, as the memories RLS read policies compose it), +-- the same calling-key restriction (00068/00069 — `origin_repo` is a +-- repository name by construction, so an unnarrowed pivot leaks exactly what +-- the scope catalog hides), the same archived/expired partition, and the same +-- per-dimension flags from the shared `lorekit_match_*` helpers. A pivot that +-- narrowed differently from the facet menu beside it would be a second +-- implementation of the same question. +-- +-- ── THE VALUE PROJECTION IS A FUNCTION, NOT A CASE PER AXIS ───────────── +-- `tag` is the one dimension whose column holds MANY values per row, so it +-- needs `unnest` while the other eight are scalars. With two axes that is +-- four combinations to spell out inline. `lorekit_memory_facet_value` returns +-- a SETOF instead — one row for a scalar, N for tags — so each axis is one +-- `cross join lateral` and the axes cannot be handled asymmetrically. +-- +-- A memory carrying two tags therefore contributes to two cells on a `tag` +-- axis. That is the same double-count `lorekit_memory_facets` already does +-- through its own `unnest`, so a matrix row total agrees with the facet count +-- for that value rather than disagreeing by the multi-tag rows. +-- ═════════════════════════════════════════════════════════════════════════ + +-- ── The per-facet value projection ────────────────────────────────────── +-- Takes the already-selected columns rather than a row id, so it stays +-- `immutable` and inlinable and never re-reads `memories`. +create or replace function lorekit_memory_facet_value( + p_facet text, + p_tags text[], + p_source_agent text, + p_trigger text, + p_kind text, + p_host text, + p_origin_repo text, + p_origin_branch text, + p_origin_pr integer, + p_org_id uuid, + p_org_slug text +) +returns setof text +language sql +immutable +parallel safe +as $$ + -- The eight scalar dimensions. A null column yields a null value, which the + -- caller drops — the same "a null column value yields NO facet row" omission + -- `lorekit_memory_facets` has, so the two agree on absence. + select case p_facet + when 'source_agent' then p_source_agent + when 'trigger' then p_trigger + when 'kind' then p_kind + when 'host' then p_host + when 'origin_repo' then p_origin_repo + when 'origin_branch' then p_origin_branch + when 'origin_pr' then p_origin_pr::text + -- Owner is a computed identity, not a column: `personal` for a row + -- with no org, else the org's slug (00064). + when 'owner' then case when p_org_id is null then 'personal' else p_org_slug end + end + where p_facet <> 'tag' + union all + select t.tag + from unnest(coalesce(p_tags, '{}'::text[])) as t(tag) + where p_facet = 'tag'; +$$; + +revoke execute on function lorekit_memory_facet_value( + text, text[], text, text, text, text, text, text, integer, uuid, text +) from public, anon; +grant execute on function lorekit_memory_facet_value( + text, text[], text, text, text, text, text, text, integer, uuid, text +) to authenticated, service_role; + +comment on function lorekit_memory_facet_value( + text, text[], text, text, text, text, text, text, integer, uuid, text +) is + 'The value(s) a memory row carries for one facet name: exactly one row for + the eight scalar dimensions (null when the column is null), and one row per + label for `tag`. Lets lorekit_memory_pivot treat both axes identically + instead of spelling out the scalar/array combinations.'; + + +-- ── The pivot ─────────────────────────────────────────────────────────── +create or replace function lorekit_memory_pivot( + p_row_facet text, + p_col_facet text, + p_user_id uuid default null, + p_archived boolean default false, + p_scope text default null, + p_limit integer default 400, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in', + p_owner text[] default null, + p_owner_mode text default 'in', + -- The CALLING KEY's restriction (00068), defaulted to unrestricted. + p_key_scopes text[] default '{}', + p_key_org_access text default 'all', + p_key_org_ids uuid[] default '{}' +) +returns table (row_value text, col_value text, count bigint) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + v_origin_pr integer[] := ( + select array_agg(x::integer) + from unnest(coalesce(p_origin_pr, '{}'::text[])) as x + where x ~ '^[0-9]+$' + ); +begin + return query + with base as ( + select + m.tags, m.source_agent, m.trigger, m.kind, m.host, + m.origin_repo, m.origin_branch, m.origin_pr, + m.org_id, o.slug as org_slug, + lorekit_match_tags(m.tags, p_tags, p_tags_mode) as ok_tag, + lorekit_match_text(m.source_agent, p_source_agent, p_source_agent_mode) as ok_source_agent, + lorekit_match_text(m.trigger, p_trigger, p_trigger_mode) as ok_trigger, + lorekit_match_text(m.kind, p_kind, p_kind_mode) as ok_kind, + lorekit_match_text(m.host, p_host, p_host_mode) as ok_host, + lorekit_match_text(m.origin_repo, p_origin_repo, p_origin_repo_mode) as ok_origin_repo, + lorekit_match_text(m.origin_branch, p_origin_branch, p_origin_branch_mode) as ok_origin_branch, + lorekit_match_int (m.origin_pr, v_origin_pr, p_origin_pr_mode) as ok_origin_pr, + -- Owner stays inline for lorekit_memory_facets' reason: it is a + -- LEFT JOIN-computed identity, not one of the three column helpers. + (p_owner is null or case coalesce(p_owner_mode, 'in') + when 'nin' then ( + (case when m.org_id is null then 'personal' else o.slug end) is not null + and (case when m.org_id is null then 'personal' else o.slug end) <> all(p_owner) + ) + else ( + ('personal' = any(p_owner) and m.org_id is null) + or (m.org_id is not null and o.slug = any(p_owner)) + ) + end) as ok_owner + from memories m + left join orgs o on o.id = m.org_id + where ( + (v_actor is null and auth.role() = 'service_role') + or m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) + ) + and lorekit_api_token_scope_allowed(p_key_scopes, m.scope) + and lorekit_api_token_org_allowed(p_key_org_access, p_key_org_ids, m.org_id) + and ( + case + when p_archived then m.archived_at is not null + else m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + end + ) + and (p_scope is null or m.scope = p_scope) + ) + select rv.value, cv.value, count(*)::bigint + from base b + cross join lateral lorekit_memory_facet_value( + p_row_facet, b.tags, b.source_agent, b.trigger, b.kind, b.host, + b.origin_repo, b.origin_branch, b.origin_pr, b.org_id, b.org_slug + ) as rv(value) + cross join lateral lorekit_memory_facet_value( + p_col_facet, b.tags, b.source_agent, b.trigger, b.kind, b.host, + b.origin_repo, b.origin_branch, b.origin_pr, b.org_id, b.org_slug + ) as cv(value) + -- Each dimension's own filter applies UNLESS it is one of the two axes — + -- see the self-exclusion note at the top. One line per dimension so adding + -- a tenth is one line here, not a re-derivation. + where (p_row_facet = 'tag' or p_col_facet = 'tag' or b.ok_tag) + and (p_row_facet = 'source_agent' or p_col_facet = 'source_agent' or b.ok_source_agent) + and (p_row_facet = 'trigger' or p_col_facet = 'trigger' or b.ok_trigger) + and (p_row_facet = 'kind' or p_col_facet = 'kind' or b.ok_kind) + and (p_row_facet = 'host' or p_col_facet = 'host' or b.ok_host) + and (p_row_facet = 'origin_repo' or p_col_facet = 'origin_repo' or b.ok_origin_repo) + and (p_row_facet = 'origin_branch' or p_col_facet = 'origin_branch' or b.ok_origin_branch) + and (p_row_facet = 'origin_pr' or p_col_facet = 'origin_pr' or b.ok_origin_pr) + and (p_row_facet = 'owner' or p_col_facet = 'owner' or b.ok_owner) + -- A null column value yields no cell, matching lorekit_memory_facets. + and rv.value is not null + and cv.value is not null + group by rv.value, cv.value + -- Densest cells first, so a caller that has to truncate keeps the ones + -- worth drawing. Deterministic tie-break so a page boundary is stable. + order by count(*) desc, rv.value asc, cv.value asc + limit greatest(p_limit, 0); +end; +$$; + +revoke execute on function lorekit_memory_pivot( + text, text, uuid, boolean, text, integer, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[] +) from public, anon; +grant execute on function lorekit_memory_pivot( + text, text, uuid, boolean, text, integer, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[] +) to authenticated, service_role; + +comment on function lorekit_memory_pivot( + text, text, uuid, boolean, text, integer, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[] +) is + 'Two-dimensional facet counts: how many visible memories carry each + (row_facet value, col_facet value) pair. Same row visibility, key + restriction, archived/expired partition and dimension predicates as + lorekit_memory_facets, with BOTH axes self-excluded from the filters so a + drilled-in matrix still shows every cell you could move to. Ordered count + desc, row asc, col asc and bounded by p_limit.'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00090', 'memory_pivot') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00092_list_retention_conditions.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- Explorer preview of retention-policy conditions: the SAME three thresholds +-- `lorekit_groom_candidates` (00088) matches a saved policy on, added to +-- `lorekit_memory_list` (00067/00069) so the dashboard's Lore Explorer can +-- filter its list to the lessons a policy WOULD catch, before any policy is +-- ever saved. This is the "verify before you run it" gap: `groom.preview` +-- already answers "how many and which keys", but the Explorer renders full +-- lesson cards, not a bare key list, and a reader wants to see those cards +-- narrowed the same way the filter bar narrows every other dimension. +-- +-- Deliberately NOT folded into `lorekit_groom_candidates` itself, and not a +-- ninth `MemoryDimensions` entry: these three are numeric THRESHOLDS against +-- `created_at`/`last_seen_at`/`seen_count`, not a categorical value-list like +-- `tags`/`kind`/`host`, so they compose with the list route's OWN predicate +-- rather than the dimension-filter helpers (00066). The predicate expression +-- is copied verbatim from `lorekit_groom_candidates` (00088) — same three +-- conditions, same "never-seen coalesces to `-infinity`" reading of +-- `unseen_days` — so a value that would match a saved policy always matches +-- here too; `groom.spec.ts`'s TypeScript mirror is unaffected, since this +-- migration adds a second SQL caller of the same rule rather than changing it. +-- +-- Purely ADDITIVE: three new trailing parameters, all defaulting to `null` +-- ("not filtered"), so behaviour is unchanged for every existing caller — +-- the CLI, the MCP surface, and every caller that does not pass them see the +-- exact same rows as before. Callers use PostgREST's named-argument RPC form +-- (a JSON object, not a positional call), so appending parameters at the END +-- of the signature — rather than inserting them in reading order — needs no +-- renumbering of any `$N` placeholder already in the function body; the +-- three new placeholders are likewise appended to the end of the +-- `using (...)` list. +-- +-- `create or replace` only replaces a function with the SAME argument list; +-- it cannot change a signature. The existing 35-arg overload (00069) has to +-- be dropped first, or it and this new 38-arg one coexist as two overloads — +-- PostgREST's named-argument RPC form then can't pick one and every call +-- through `lorekit_memory_list` becomes ambiguous. +drop function if exists lorekit_memory_list( + uuid, boolean, text, text, text, text, timestamptz, timestamptz, timestamptz, + timestamptz, text[], text, text[], text, text[], text, text[], text, text[], + text, text[], text, text[], text, text[], text, text[], text, text, + timestamptz, uuid, integer, text[], text, uuid[] +); + +create or replace function lorekit_memory_list( + p_user_id uuid, + p_archived boolean default false, + p_scope text default null, + p_key text default null, + p_key_prefix text default null, + p_q text default null, + p_created_since timestamptz default null, + p_created_until timestamptz default null, + p_expires_after timestamptz default null, + p_expires_on_or_before timestamptz default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in', + p_owner text[] default null, + p_owner_mode text default 'in', + p_sort text default 'updated_at', + p_cursor_ts timestamptz default null, + p_cursor_id uuid default null, + p_limit integer default 51, + p_key_scopes text[] default '{}', + p_key_org_access text default 'all', + p_key_org_ids uuid[] default '{}', + -- The retention-policy preview trio (see header). All optional; `null` + -- (the default) applies no filter, exactly like every other parameter here. + p_min_age_days integer default null, + p_unseen_days integer default null, + p_max_seen_count integer default null +) +returns table ( + id uuid, + scope text, + key text, + value text, + tags text[], + source_agent text, + trigger text, + created_at timestamptz, + updated_at timestamptz, + expires_at timestamptz, + archived_at timestamptz, + origin_repo text, + origin_branch text, + origin_commit text, + origin_pr integer, + kind text, + host text, + seen_count integer, + org_id uuid, + created_by uuid, + updated_by uuid, + org_name text, + org_slug text +) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + v_origin_pr integer[] := ( + select array_agg(x::integer) + from unnest(coalesce(p_origin_pr, '{}'::text[])) as x + where x ~ '^0*[0-9]{1,9}$' + ); + v_sort text := case when p_sort = 'created_at' then 'created_at' else 'updated_at' end; +begin + return query execute format($q$ + select + m.id, m.scope, m.key, m.value, m.tags, m.source_agent, m.trigger, + m.created_at, m.updated_at, m.expires_at, m.archived_at, + m.origin_repo, m.origin_branch, m.origin_commit, m.origin_pr, + m.kind, m.host, m.seen_count, + m.org_id, m.created_by, m.updated_by, + o.name as org_name, o.slug as org_slug + from memories m + left join orgs o on o.id = m.org_id + where ( + ($1 is null and auth.role() = 'service_role') + or m.user_id = $1 + or m.org_id in (select lorekit_member_org_ids($1)) + ) + and lorekit_api_token_scope_allowed($32, m.scope) + and lorekit_api_token_org_allowed($33, $34, m.org_id) + and ( + case + when $2 then m.archived_at is not null + else m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + end + ) + and ($3 is null or m.scope = $3) + and ($4 is null or m.key = $4) + and ($5 is null or m.key ilike $5 || '%%') + and ($6 is null or m.key ilike '%%' || $6 || '%%' or m.value ilike '%%' || $6 || '%%') + and ($7 is null or m.created_at >= $7) + and ($8 is null or m.created_at < $8) + and ($9 is null or m.expires_at > $9) + and ($10 is null or m.expires_at <= $10) + and lorekit_match_tags(m.tags, $11, $12) + and lorekit_match_text(m.source_agent, $13, $14) + and lorekit_match_text(m.trigger, $15, $16) + and lorekit_match_text(m.kind, $17, $18) + and lorekit_match_text(m.host, $19, $20) + and lorekit_match_text(m.origin_repo, $21, $22) + and lorekit_match_text(m.origin_branch, $23, $24) + and lorekit_match_int (m.origin_pr, $25, $26) + and ($27 is null or case coalesce($28, 'in') + when 'nin' then ( + (case when m.org_id is null then 'personal' else o.slug end) is not null + and (case when m.org_id is null then 'personal' else o.slug end) <> all($27) + ) + else ( + ('personal' = any($27) and m.org_id is null) + or (m.org_id is not null and o.slug = any($27)) + ) + end) + -- The retention-policy preview trio, copied verbatim from + -- `lorekit_groom_candidates` (00088) — see header. + and ($35 is null or m.created_at <= now() - ($35 * interval '1 day')) + and ($36 is null or coalesce(m.last_seen_at, '-infinity'::timestamptz) <= now() - ($36 * interval '1 day')) + and ($37 is null or m.seen_count <= $37) + and ( + $29 is null + or m.%1$I < $29 + or (m.%1$I = $29 and m.id < $30) + ) + order by m.%1$I desc, m.id desc + limit $31 + $q$, v_sort) + using + v_actor, coalesce(p_archived, false), p_scope, p_key, p_key_prefix, p_q, + p_created_since, p_created_until, p_expires_after, p_expires_on_or_before, + p_tags, p_tags_mode, + p_source_agent, p_source_agent_mode, + p_trigger, p_trigger_mode, + p_kind, p_kind_mode, + p_host, p_host_mode, + p_origin_repo, p_origin_repo_mode, + p_origin_branch, p_origin_branch_mode, + v_origin_pr, p_origin_pr_mode, + p_owner, p_owner_mode, + p_cursor_ts, p_cursor_id, + greatest(coalesce(p_limit, 51), 1), + coalesce(p_key_scopes, '{}'::text[]), + coalesce(p_key_org_access, 'all'), + coalesce(p_key_org_ids, '{}'::uuid[]), + p_min_age_days, p_unseen_days, p_max_seen_count; +end; +$$; + +revoke execute on function lorekit_memory_list( + uuid, boolean, text, text, text, text, timestamptz, timestamptz, timestamptz, + timestamptz, text[], text, text[], text, text[], text, text[], text, text[], + text, text[], text, text[], text, text[], text, text[], text, text, + timestamptz, uuid, integer, text[], text, uuid[], integer, integer, integer +) from public, anon; +grant execute on function lorekit_memory_list( + uuid, boolean, text, text, text, text, timestamptz, timestamptz, timestamptz, + timestamptz, text[], text, text[], text, text[], text, text[], text, text[], + text, text[], text, text[], text, text[], text, text[], text, text, + timestamptz, uuid, integer, text[], text, uuid[], integer, integer, integer +) to authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00092', 'list_retention_conditions') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00093_policy_dimension_filters.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- Retention policies, part 3: the SAME eight dimension filters the Lore +-- Explorer's filter bar offers (label/kind/host/agent/trigger/repo/branch/PR +-- — `lib/filters.ts`'s `FILTER_FIELDS`, minus `owner`: a policy's `scope` +-- already partitions personal-vs-org lore, so a second ownership predicate +-- would either agree with it or silently fight it). +-- +-- Until now a policy could only narrow by scope + the three age/activity +-- thresholds (00088). "Create retention policy" on the Explorer therefore +-- dropped every dimension filter a reader had set — the exact gap this +-- migration closes, so a policy created FROM a filtered Explorer view +-- actually reproduces that view's narrowing, not just its age conditions. +-- +-- Reuses the dimension-filter helpers `lorekit_memory_facets`/ +-- `lorekit_memory_list` already compose (`lorekit_match_text` / +-- `lorekit_match_tags`, migration 00066) — one definition of what `nin` / +-- `all`/`any`/`none` mean, not a third copy of it. `origin_pr` matches as +-- TEXT here (`m.origin_pr::text` against a digit-string filter), not via +-- `lorekit_match_int`: the wire and the Explorer's filter bar both speak +-- digit strings for this dimension (`ListMemoriesBodySchema.origin_pr` is a +-- string list too), and a policy's stored `origin_pr` column is `text[]` to +-- match — avoiding a second representation for the one dimension that would +-- otherwise need it. +-- +-- 1. retention_policies — eight new (filter, mode) column pairs, all +-- nullable and defaulting to null/'in' — additive, so every existing row +-- reads back with every new filter "not narrowed", unchanged behaviour. +alter table retention_policies + add column if not exists tags text[], + add column if not exists tags_mode text default 'any', + add column if not exists source_agent text[], + add column if not exists source_agent_mode text default 'in', + add column if not exists trigger text[], + add column if not exists trigger_mode text default 'in', + add column if not exists kind text[], + add column if not exists kind_mode text default 'in', + add column if not exists host text[], + add column if not exists host_mode text default 'in', + add column if not exists origin_repo text[], + add column if not exists origin_repo_mode text default 'in', + add column if not exists origin_branch text[], + add column if not exists origin_branch_mode text default 'in', + add column if not exists origin_pr text[], + add column if not exists origin_pr_mode text default 'in'; + +-- 2. lorekit_groom_candidates — the single source of truth for "what +-- matches", extended with the eight filters. `language sql`, so the new +-- parameters are referenced by NAME in the body, not by a `$N` position — +-- unlike `lorekit_memory_list` (00092), this function has no dynamic +-- `execute format(...)` to renumber. Appended at the end of the argument +-- list so this is additive for the same reason 00092's append was: every +-- caller uses PostgREST's named-argument RPC form, so an appended, +-- defaulted parameter changes no existing caller's behaviour. +-- +-- `create or replace` only replaces a function whose PARAMETER LIST +-- matches exactly — a longer list (even all-defaulted) creates a SECOND +-- overload instead, and a 5-arg call then becomes ambiguous ("not unique") +-- because both overloads can satisfy it via defaults. Drop the 00088 +-- signature first so there is exactly one `lorekit_groom_candidates`. +drop function if exists lorekit_groom_candidates(uuid, text, integer, integer, integer); + +create or replace function lorekit_groom_candidates( + p_user_id uuid, + p_scope text, + p_min_age_days integer default null, + p_unseen_days integer default null, + p_max_seen_count integer default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in' +) +returns table (id uuid, scope text, key text) +language sql +stable +security definer +set search_path = public +as $$ + select m.id, m.scope, m.key + from memories m + where m.user_id = p_user_id + and m.archived_at is null + and m.protected = false + and ( + p_scope = 'global' + or m.scope = p_scope + or ( + p_scope like 'repo::%' + and starts_with(m.scope, 'branch::' || substring(p_scope from 7) || '::') + ) + ) + and (p_min_age_days is null or m.created_at <= now() - (p_min_age_days * interval '1 day')) + and (p_unseen_days is null or coalesce(m.last_seen_at, '-infinity'::timestamptz) <= now() - (p_unseen_days * interval '1 day')) + and (p_max_seen_count is null or m.seen_count <= p_max_seen_count) + and lorekit_match_tags(m.tags, p_tags, p_tags_mode) + and lorekit_match_text(m.source_agent, p_source_agent, p_source_agent_mode) + and lorekit_match_text(m.trigger, p_trigger, p_trigger_mode) + and lorekit_match_text(m.kind, p_kind, p_kind_mode) + and lorekit_match_text(m.host, p_host, p_host_mode) + and lorekit_match_text(m.origin_repo, p_origin_repo, p_origin_repo_mode) + and lorekit_match_text(m.origin_branch, p_origin_branch, p_origin_branch_mode) + and lorekit_match_text(m.origin_pr::text, p_origin_pr, p_origin_pr_mode) + order by m.scope, m.key; +$$; + +grant execute on function lorekit_groom_candidates( + uuid, text, integer, integer, integer, + text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text +) to authenticated, service_role; + +-- 3. lorekit_groom_run — forwards the same eight filters to +-- lorekit_groom_candidates (never a second candidate query). Same +-- overload hazard as above — drop the 00088 5-arg signature first. +drop function if exists lorekit_groom_run(uuid, text, integer, integer, integer); + +create or replace function lorekit_groom_run( + p_user_id uuid, + p_scope text, + p_min_age_days integer default null, + p_unseen_days integer default null, + p_max_seen_count integer default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in' +) +returns table (archived integer, keys jsonb) +language plpgsql +security definer +set search_path = public +as $$ +declare + v_ids uuid[]; + v_keys jsonb; + v_count integer := 0; +begin + select coalesce(array_agg(c.id), '{}'), + coalesce(jsonb_agg(jsonb_build_object('scope', c.scope, 'key', c.key)), '[]'::jsonb) + into v_ids, v_keys + from lorekit_groom_candidates( + p_user_id, p_scope, p_min_age_days, p_unseen_days, p_max_seen_count, + p_tags, p_tags_mode, p_source_agent, p_source_agent_mode, + p_trigger, p_trigger_mode, p_kind, p_kind_mode, p_host, p_host_mode, + p_origin_repo, p_origin_repo_mode, p_origin_branch, p_origin_branch_mode, + p_origin_pr, p_origin_pr_mode + ) c; + + if array_length(v_ids, 1) is null then + return query select 0, '[]'::jsonb; + return; + end if; + + update memories m + set archived_at = now() + where m.id = any(v_ids) + and m.archived_at is null; + get diagnostics v_count = row_count; + + return query select v_count, v_keys; +end; +$$; + +grant execute on function lorekit_groom_run( + uuid, text, integer, integer, integer, + text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text +) to authenticated, service_role; + +-- 4. lorekit_groom_sweep — passes a swept policy's new filter columns +-- through to lorekit_groom_run, unchanged shape otherwise. +create or replace function lorekit_groom_sweep() +returns integer +language plpgsql +security definer +set search_path = public +as $$ +declare + v_policy retention_policies%rowtype; + v_result record; + v_total integer := 0; +begin + for v_policy in + select * from retention_policies where mode = 'auto' and enabled = true + loop + select * into v_result + from lorekit_groom_run( + v_policy.user_id, v_policy.scope, + v_policy.min_age_days, v_policy.unseen_days, v_policy.max_seen_count, + v_policy.tags, v_policy.tags_mode, + v_policy.source_agent, v_policy.source_agent_mode, + v_policy.trigger, v_policy.trigger_mode, + v_policy.kind, v_policy.kind_mode, + v_policy.host, v_policy.host_mode, + v_policy.origin_repo, v_policy.origin_repo_mode, + v_policy.origin_branch, v_policy.origin_branch_mode, + v_policy.origin_pr, v_policy.origin_pr_mode + ); + v_total := v_total + coalesce(v_result.archived, 0); + end loop; + + return v_total; +end; +$$; + +grant execute on function lorekit_groom_sweep() to service_role; + +-- 5. lorekit_policy_create — the eight filters, appended. Same overload +-- hazard as `lorekit_groom_candidates` above — drop the 00088 8-arg +-- signature first. +drop function if exists lorekit_policy_create(uuid, text, text, text, boolean, integer, integer, integer); + +create or replace function lorekit_policy_create( + p_user_id uuid, + p_scope text, + p_name text, + p_mode text default 'review', + p_enabled boolean default false, + p_min_age_days integer default null, + p_unseen_days integer default null, + p_max_seen_count integer default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in' +) +returns setof retention_policies +language sql +security definer +set search_path = public +as $$ + insert into retention_policies ( + user_id, scope, name, mode, enabled, min_age_days, unseen_days, max_seen_count, + tags, tags_mode, source_agent, source_agent_mode, trigger, trigger_mode, + kind, kind_mode, host, host_mode, origin_repo, origin_repo_mode, + origin_branch, origin_branch_mode, origin_pr, origin_pr_mode + ) + values ( + p_user_id, p_scope, p_name, p_mode, p_enabled, p_min_age_days, p_unseen_days, p_max_seen_count, + p_tags, p_tags_mode, p_source_agent, p_source_agent_mode, p_trigger, p_trigger_mode, + p_kind, p_kind_mode, p_host, p_host_mode, p_origin_repo, p_origin_repo_mode, + p_origin_branch, p_origin_branch_mode, p_origin_pr, p_origin_pr_mode + ) + returning *; +$$; + +grant execute on function lorekit_policy_create( + uuid, text, text, text, boolean, integer, integer, integer, + text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text +) to authenticated, service_role; + +-- 6. lorekit_policy_update — the same has-key / json-null-clears JSONB patch +-- convention as 00088, extended to the eight new array + mode columns. An +-- array column's clear needs its own branch (`p_patch->'tags' = 'null'::jsonb`) +-- because `jsonb_array_elements_text` raises on a JSON null rather than +-- returning zero rows. +create or replace function lorekit_policy_update( + p_user_id uuid, + p_id uuid, + p_patch jsonb +) +returns setof retention_policies +language sql +security definer +set search_path = public +as $$ + update retention_policies + set name = case when p_patch ? 'name' then p_patch ->> 'name' else name end, + mode = case when p_patch ? 'mode' then p_patch ->> 'mode' else mode end, + enabled = case when p_patch ? 'enabled' then (p_patch ->> 'enabled')::boolean else enabled end, + min_age_days = case when p_patch ? 'min_age_days' then (p_patch ->> 'min_age_days')::integer else min_age_days end, + unseen_days = case when p_patch ? 'unseen_days' then (p_patch ->> 'unseen_days')::integer else unseen_days end, + max_seen_count = case when p_patch ? 'max_seen_count' then (p_patch ->> 'max_seen_count')::integer else max_seen_count end, + tags = case when p_patch ? 'tags' then (case when p_patch -> 'tags' = 'null'::jsonb then null else array(select jsonb_array_elements_text(p_patch -> 'tags')) end) else tags end, + tags_mode = case when p_patch ? 'tags_mode' then p_patch ->> 'tags_mode' else tags_mode end, + source_agent = case when p_patch ? 'source_agent' then (case when p_patch -> 'source_agent' = 'null'::jsonb then null else array(select jsonb_array_elements_text(p_patch -> 'source_agent')) end) else source_agent end, + source_agent_mode = case when p_patch ? 'source_agent_mode' then p_patch ->> 'source_agent_mode' else source_agent_mode end, + trigger = case when p_patch ? 'trigger' then (case when p_patch -> 'trigger' = 'null'::jsonb then null else array(select jsonb_array_elements_text(p_patch -> 'trigger')) end) else trigger end, + trigger_mode = case when p_patch ? 'trigger_mode' then p_patch ->> 'trigger_mode' else trigger_mode end, + kind = case when p_patch ? 'kind' then (case when p_patch -> 'kind' = 'null'::jsonb then null else array(select jsonb_array_elements_text(p_patch -> 'kind')) end) else kind end, + kind_mode = case when p_patch ? 'kind_mode' then p_patch ->> 'kind_mode' else kind_mode end, + host = case when p_patch ? 'host' then (case when p_patch -> 'host' = 'null'::jsonb then null else array(select jsonb_array_elements_text(p_patch -> 'host')) end) else host end, + host_mode = case when p_patch ? 'host_mode' then p_patch ->> 'host_mode' else host_mode end, + origin_repo = case when p_patch ? 'origin_repo' then (case when p_patch -> 'origin_repo' = 'null'::jsonb then null else array(select jsonb_array_elements_text(p_patch -> 'origin_repo')) end) else origin_repo end, + origin_repo_mode = case when p_patch ? 'origin_repo_mode' then p_patch ->> 'origin_repo_mode' else origin_repo_mode end, + origin_branch = case when p_patch ? 'origin_branch' then (case when p_patch -> 'origin_branch' = 'null'::jsonb then null else array(select jsonb_array_elements_text(p_patch -> 'origin_branch')) end) else origin_branch end, + origin_branch_mode = case when p_patch ? 'origin_branch_mode' then p_patch ->> 'origin_branch_mode' else origin_branch_mode end, + origin_pr = case when p_patch ? 'origin_pr' then (case when p_patch -> 'origin_pr' = 'null'::jsonb then null else array(select jsonb_array_elements_text(p_patch -> 'origin_pr')) end) else origin_pr end, + origin_pr_mode = case when p_patch ? 'origin_pr_mode' then p_patch ->> 'origin_pr_mode' else origin_pr_mode end + where id = p_id and user_id = p_user_id + returning *; +$$; + +grant execute on function lorekit_policy_update(uuid, uuid, jsonb) to authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00093', 'policy_dimension_filters') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00094_memory_list_total_count.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- The Lore Explorer's header shows how many memories a scope/search/filter/ +-- retention view actually matches ("12 of 128"), but `lorekit_memory_list` +-- (00067/00069/00092) only ever returned a PAGE — `limit + 1` rows, so the +-- REST handler could tell "is there another page" but not "how many rows +-- match in total". The dashboard was filling that gap with how many rows it +-- had LOADED so far, which understates the true match for any view spanning +-- more than one page (a "50+ of 2897" that never resolves past 50 for an +-- account with thousands of matches) — a floor pretending to be a fact. +-- +-- Adds ONE extra output column, `total_count`, computed with +-- `count(*) over ()` — a window function evaluated over every row the WHERE +-- clause matched, BEFORE the `limit` in the same query cuts it down to a page. +-- This is the standard "exact total alongside a LIMITed page" technique and +-- costs no second round trip: every row of the page carries the same +-- `total_count`, so the caller reads it off the first row (or 0 for an empty +-- page) rather than issuing a companion COUNT query. +-- +-- Purely ADDITIVE to behaviour: the WHERE clause, its 38 parameters and their +-- defaults are UNCHANGED, so the CLI, the MCP surface and every existing +-- caller see the exact same rows in the exact same order — only the returned +-- ROW SHAPE gains one column. `create or replace` cannot change a return +-- type, so the existing 38-arg overload has to be dropped first, exactly as +-- 00092 dropped the 35-arg one it replaced. +drop function if exists lorekit_memory_list( + uuid, boolean, text, text, text, text, timestamptz, timestamptz, timestamptz, + timestamptz, text[], text, text[], text, text[], text, text[], text, text[], + text, text[], text, text[], text, text[], text, text[], text, text, + timestamptz, uuid, integer, text[], text, uuid[], integer, integer, integer +); + +create or replace function lorekit_memory_list( + p_user_id uuid, + p_archived boolean default false, + p_scope text default null, + p_key text default null, + p_key_prefix text default null, + p_q text default null, + p_created_since timestamptz default null, + p_created_until timestamptz default null, + p_expires_after timestamptz default null, + p_expires_on_or_before timestamptz default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in', + p_owner text[] default null, + p_owner_mode text default 'in', + p_sort text default 'updated_at', + p_cursor_ts timestamptz default null, + p_cursor_id uuid default null, + p_limit integer default 51, + p_key_scopes text[] default '{}', + p_key_org_access text default 'all', + p_key_org_ids uuid[] default '{}', + p_min_age_days integer default null, + p_unseen_days integer default null, + p_max_seen_count integer default null +) +returns table ( + id uuid, + scope text, + key text, + value text, + tags text[], + source_agent text, + trigger text, + created_at timestamptz, + updated_at timestamptz, + expires_at timestamptz, + archived_at timestamptz, + origin_repo text, + origin_branch text, + origin_commit text, + origin_pr integer, + kind text, + host text, + seen_count integer, + org_id uuid, + created_by uuid, + updated_by uuid, + org_name text, + org_slug text, + -- Exact count of every row the WHERE clause matched, ignoring `limit` — + -- see header. Identical on every row of the result; a caller reads it off + -- the first row (or treats an empty result as 0). + total_count integer +) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + v_origin_pr integer[] := ( + select array_agg(x::integer) + from unnest(coalesce(p_origin_pr, '{}'::text[])) as x + where x ~ '^0*[0-9]{1,9}$' + ); + v_sort text := case when p_sort = 'created_at' then 'created_at' else 'updated_at' end; +begin + return query execute format($q$ + select + m.id, m.scope, m.key, m.value, m.tags, m.source_agent, m.trigger, + m.created_at, m.updated_at, m.expires_at, m.archived_at, + m.origin_repo, m.origin_branch, m.origin_commit, m.origin_pr, + m.kind, m.host, m.seen_count, + m.org_id, m.created_by, m.updated_by, + o.name as org_name, o.slug as org_slug, + (count(*) over ())::integer as total_count + from memories m + left join orgs o on o.id = m.org_id + where ( + ($1 is null and auth.role() = 'service_role') + or m.user_id = $1 + or m.org_id in (select lorekit_member_org_ids($1)) + ) + and lorekit_api_token_scope_allowed($32, m.scope) + and lorekit_api_token_org_allowed($33, $34, m.org_id) + and ( + case + when $2 then m.archived_at is not null + else m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + end + ) + and ($3 is null or m.scope = $3) + and ($4 is null or m.key = $4) + and ($5 is null or m.key ilike $5 || '%%') + and ($6 is null or m.key ilike '%%' || $6 || '%%' or m.value ilike '%%' || $6 || '%%') + and ($7 is null or m.created_at >= $7) + and ($8 is null or m.created_at < $8) + and ($9 is null or m.expires_at > $9) + and ($10 is null or m.expires_at <= $10) + and lorekit_match_tags(m.tags, $11, $12) + and lorekit_match_text(m.source_agent, $13, $14) + and lorekit_match_text(m.trigger, $15, $16) + and lorekit_match_text(m.kind, $17, $18) + and lorekit_match_text(m.host, $19, $20) + and lorekit_match_text(m.origin_repo, $21, $22) + and lorekit_match_text(m.origin_branch, $23, $24) + and lorekit_match_int (m.origin_pr, $25, $26) + and ($27 is null or case coalesce($28, 'in') + when 'nin' then ( + (case when m.org_id is null then 'personal' else o.slug end) is not null + and (case when m.org_id is null then 'personal' else o.slug end) <> all($27) + ) + else ( + ('personal' = any($27) and m.org_id is null) + or (m.org_id is not null and o.slug = any($27)) + ) + end) + and ($35 is null or m.created_at <= now() - ($35 * interval '1 day')) + and ($36 is null or coalesce(m.last_seen_at, '-infinity'::timestamptz) <= now() - ($36 * interval '1 day')) + and ($37 is null or m.seen_count <= $37) + and ( + $29 is null + or m.%1$I < $29 + or (m.%1$I = $29 and m.id < $30) + ) + order by m.%1$I desc, m.id desc + limit $31 + $q$, v_sort) + using + v_actor, coalesce(p_archived, false), p_scope, p_key, p_key_prefix, p_q, + p_created_since, p_created_until, p_expires_after, p_expires_on_or_before, + p_tags, p_tags_mode, + p_source_agent, p_source_agent_mode, + p_trigger, p_trigger_mode, + p_kind, p_kind_mode, + p_host, p_host_mode, + p_origin_repo, p_origin_repo_mode, + p_origin_branch, p_origin_branch_mode, + v_origin_pr, p_origin_pr_mode, + p_owner, p_owner_mode, + p_cursor_ts, p_cursor_id, + greatest(coalesce(p_limit, 51), 1), + coalesce(p_key_scopes, '{}'::text[]), + coalesce(p_key_org_access, 'all'), + coalesce(p_key_org_ids, '{}'::uuid[]), + p_min_age_days, p_unseen_days, p_max_seen_count; +end; +$$; + +revoke execute on function lorekit_memory_list( + uuid, boolean, text, text, text, text, timestamptz, timestamptz, timestamptz, + timestamptz, text[], text, text[], text, text[], text, text[], text, text[], + text, text[], text, text[], text, text[], text, text[], text, text, + timestamptz, uuid, integer, text[], text, uuid[], integer, integer, integer +) from public, anon; +grant execute on function lorekit_memory_list( + uuid, boolean, text, text, text, text, timestamptz, timestamptz, timestamptz, + timestamptz, text[], text, text[], text, text[], text, text[], text, text[], + text, text[], text, text[], text, text[], text, text[], text, text, + timestamptz, uuid, integer, text[], text, uuid[], integer, integer, integer +) to authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00094', 'memory_list_total_count') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00095_groom_sweep_telemetry.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- Retention-policy sweep telemetry. +-- +-- `lorekit_groom_sweep()` (00088, re-created in 00093) is the nightly +-- auto-archive pass — and it has ALWAYS been invisible end-to-end. pg_cron +-- calls it directly as a raw SQL statement, entirely inside Postgres: no +-- span, no metric, nothing distinguishes "ran and archived nothing" from +-- "did not run at all" from "failed silently". An operator staring at +-- unchanged data has no telemetry to tell those apart from here — the exact +-- report this migration exists to fix. +-- +-- Mirrors the ALREADY-SOLVED identical problem one migration family over: +-- `lorekit_export_db_query_stats()` / the `profiling` Edge Function (00074, +-- docs/otel.md "Query-level profiling"). Same shape, applied to the sweep +-- instead of the stats scrape: +-- +-- pg_cron (nightly, 03:17 UTC) +-- └── lorekit_export_groom_sweep() ← inert without vault secrets +-- └── pg_net → POST /functions/v1/groom-sweep +-- └── lorekit_groom_sweep_and_record() ← THIS migration +-- └── lorekit_groom_sweep() ← UNCHANGED (00093) +-- +-- `lorekit_groom_sweep_and_record()` does not touch `lorekit_groom_sweep()`'s +-- tested behaviour at all — it calls it exactly as pg_cron always has, then +-- records the result into a persistent counter row so the Edge Function can +-- export TRUE cumulative sums (matching db-query-metrics.ts's convention) +-- instead of a per-run delta needing its own reset-detection. + +-- 1. groom_sweep_stats — a SINGLETON counter row. `id` is fixed to TRUE by +-- the check constraint so there is exactly one row, ever; every sweep +-- UPDATEs it rather than INSERTing a new one, the same "one row, updated +-- forever" shape pg_stat_statements itself uses for its own counters. +-- `started_at` is set once, at row creation, and becomes every exported +-- datapoint's `startTimeUnixNano` — the series start a cumulative counter +-- needs for a backend to read a reset (this table dropped and recreated) +-- as a new series rather than as negative traffic. +create table if not exists groom_sweep_stats ( + id boolean primary key default true check (id), + runs_total bigint not null default 0, + archived_total bigint not null default 0, + policies_evaluated integer not null default 0, + started_at timestamptz not null default now(), + last_run_at timestamptz +); + +insert into groom_sweep_stats (id) values (true) on conflict (id) do nothing; + +comment on table groom_sweep_stats is + 'Singleton cumulative counters for the nightly retention-policy sweep — ' + 'updated by lorekit_groom_sweep_and_record, read by the groom-sweep Edge ' + 'Function to export lorekit.groom.sweep.{runs,archived} to Dash0.'; + +alter table groom_sweep_stats enable row level security; +-- No policy is created: this is an operator/service-role surface exactly +-- like retention_policies' RPCs are for CRUD, but here there is no owning +-- user at all (the sweep spans every user's auto+enabled policies), so no +-- `authenticated`-reachable policy is correct — only the SECURITY DEFINER +-- function below (and the service role, which bypasses RLS) may touch it. + +-- 2. lorekit_groom_sweep_and_record — runs the existing, unchanged +-- `lorekit_groom_sweep()`, then atomically records this run's result into +-- `groom_sweep_stats` and returns the accumulated totals PLUS this run's +-- own snapshot (archived_this_run, policies_evaluated) for the Edge +-- Function's span attributes. The snapshot fields are deliberately NOT +-- part of the exported metrics (see groom-sweep-metrics.ts) — they are +-- per-run values, not cumulative counters, and mixing the two shapes in +-- one Sum metric would misread a run-over-run drop as a counter reset. +create or replace function lorekit_groom_sweep_and_record() +returns table ( + runs_total bigint, + archived_total bigint, + archived_this_run integer, + policies_evaluated integer, + started_at timestamptz, + last_run_at timestamptz +) +language plpgsql +security definer +set search_path = public +as $$ +declare + v_archived integer; + v_policies integer; +begin + select lorekit_groom_sweep() into v_archived; + + select count(*) into v_policies + from retention_policies where mode = 'auto' and enabled = true; + + return query + update groom_sweep_stats + set runs_total = groom_sweep_stats.runs_total + 1, + archived_total = groom_sweep_stats.archived_total + coalesce(v_archived, 0), + policies_evaluated = v_policies, + last_run_at = now() + where id = true + returning + groom_sweep_stats.runs_total, + groom_sweep_stats.archived_total, + coalesce(v_archived, 0), + v_policies, + groom_sweep_stats.started_at, + groom_sweep_stats.last_run_at; +end; +$$; + +comment on function lorekit_groom_sweep_and_record() is + 'Runs lorekit_groom_sweep() unchanged, then records the result into ' + 'groom_sweep_stats. Called by the groom-sweep Edge Function so every ' + 'nightly sweep produces a span + cumulative Dash0 metrics.'; + +revoke all on function lorekit_groom_sweep_and_record() from public; +revoke all on function lorekit_groom_sweep_and_record() from anon, authenticated; +grant execute on function lorekit_groom_sweep_and_record() to service_role; + +-- 3. lorekit_export_groom_sweep — pokes the `groom-sweep` Edge Function +-- through pg_net, IDENTICAL gating and shape to +-- lorekit_export_db_query_stats (00074): OFF by default until an operator +-- provisions two Vault secrets, so this migration turns nothing on by +-- itself and adds no new secret-handling code path to audit. +-- +-- select vault.create_secret('https://.supabase.co/functions/v1/groom-sweep', 'lorekit_groom_sweep_url'); +-- select vault.create_secret('', 'lorekit_groom_sweep_key'); +create or replace function lorekit_export_groom_sweep() +returns text +language plpgsql +security definer +set search_path = public +as $$ +declare + v_url text; + v_key text; +begin + if not exists (select 1 from pg_extension where extname = 'pg_net') then + return 'skipped: pg_net not installed'; + end if; + + if to_regclass('vault.decrypted_secrets') is null then + return 'disabled: vault unavailable'; + end if; + + select decrypted_secret into v_url + from vault.decrypted_secrets where name = 'lorekit_groom_sweep_url'; + select decrypted_secret into v_key + from vault.decrypted_secrets where name = 'lorekit_groom_sweep_key'; + + if v_url is null or v_key is null then + return 'disabled: set the lorekit_groom_sweep_url and lorekit_groom_sweep_key vault secrets to enable'; + end if; + + -- Fire-and-forget, same reasoning as the query-stats poke: pg_net queues + -- the request and returns immediately, so a cron tick never blocks on + -- Dash0 being slow or unreachable. A dropped tick here loses one night's + -- run count, never data — the cumulative counters keep going from wherever + -- they last landed. + perform net.http_post( + url := v_url, + body := '{}'::jsonb, + headers := jsonb_build_object( + 'Content-Type', 'application/json', + 'Authorization', 'Bearer ' || v_key + ), + timeout_milliseconds := 30000 + ); + + return 'posted'; +end; +$$; + +comment on function lorekit_export_groom_sweep() is + 'Pokes the groom-sweep Edge Function to run the nightly retention sweep ' + 'and export its telemetry to Dash0. Inert until the ' + 'lorekit_groom_sweep_url + lorekit_groom_sweep_key vault secrets exist.'; + +revoke all on function lorekit_export_groom_sweep() from public; +revoke all on function lorekit_export_groom_sweep() from anon, authenticated; +grant execute on function lorekit_export_groom_sweep() to service_role; + +-- 4. Reschedule the nightly job to call the new, observable entry point. +-- `cron.schedule()` upserts by job name (same name, same time as 00088: +-- 'lorekit-groom-sweep', 03:17 UTC) so this REPLACES the command in +-- place — no unschedule step needed, and no gap in the schedule. +-- Guarded identically to 00088 so this migration still applies on an +-- instance without pg_cron. +do $$ +begin + if exists (select 1 from pg_extension where extname = 'pg_cron') then + perform cron.schedule( + 'lorekit-groom-sweep', + '17 3 * * *', + $cron$select lorekit_export_groom_sweep()$cron$ + ); + end if; +end; +$$; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00095', 'groom_sweep_telemetry') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00096_memories_user_updated_at_id_idx.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- Covering index for the one memories read that has NO scope predicate: +-- `GET /memories/clusters` run without `?scope=` (`handlers/clusters.ts`). +-- +-- Every other hot read on this table filters `scope = ?` first and is served +-- by `memories_scope_key_active_idx` / `memories_scope_updated_at_id_idx` +-- (00033, 00061). The unscoped clusters query has no such leading column: +-- +-- SELECT scope,key,value,seen_count,updated_at FROM memories +-- WHERE archived_at IS NULL +-- AND (expires_at IS NULL OR expires_at > now()) +-- ORDER BY updated_at DESC, id DESC +-- LIMIT 150 +-- +-- For a JWT caller this reaches Postgres under RLS's `rls_read` predicate +-- (`user_id = auth.uid() OR org_id = jwt org_id`, 00001); for an api_key +-- caller with no org membership, `applyRestTenantScope` adds an explicit +-- `.eq('user_id', userId)` (`_shared/api/tenant.ts`). Either way the query is +-- effectively scoped to ONE user's rows, but the only index covering +-- `user_id` is the bare `memories_user_idx (user_id)` from 00001 — it has no +-- `archived_at` predicate and no `updated_at`/`id` ordering, so the planner +-- has to choose between an index scan on `user_id` followed by a heap +-- recheck + filesort, or a sequential scan filtered on `archived_at IS +-- NULL`. Neither is stable: which one the planner picks (and how many rows +-- of the OTHER tenants' data it has to skip past to find this user's 150 +-- newest) depends on live statistics, so the SAME query with the SAME +-- 150-row result swings from ~150ms to several seconds run to run — the +-- p95 latency alert on the `api` service (`lorekit` namespace) fired on +-- exactly this variance, not on a change in traffic or row count. +-- +-- The fix mirrors 00061's `(scope, updated_at desc, id desc)` index one +-- column over: replace the leading `scope` with `user_id`, so the ONE +-- unscoped query gets the same shape of covering, order-satisfying index +-- every scoped query already has. `IF NOT EXISTS` so it applies safely to a +-- running production database; not `CONCURRENTLY` because migrations run +-- inside a transaction (matches every prior index migration on this table). +-- +-- The expiry filter (`expires_at IS NULL OR expires_at > now()`) is +-- deliberately NOT in the predicate, for the same reason 00061 left it out +-- of its own index: `now()` is volatile and PostgreSQL rejects a volatile +-- expression in an index predicate. +create index if not exists memories_user_updated_at_id_idx + on memories (user_id, updated_at desc, id desc) + where archived_at is null; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00096', 'memories_user_updated_at_id_idx') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00098_groom_unseen_days_last_read_at.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- Fix: `unseen_days` was checking a column nothing ever wrote. +-- +-- 00088 introduced `memories.last_seen_at` for the `unseen_days` retention +-- condition, but its own header said so explicitly: "nothing sets it yet... +-- wiring memory.read/list/search to bump it is deliberately left as a +-- follow-up." That follow-up never landed — `last_seen_at` has no writer +-- anywhere in this codebase, so it is permanently NULL for every row, +-- including rows read constantly. `coalesce(last_seen_at, '-infinity')` then +-- evaluates to true for EVERY memory regardless of actual read activity, so +-- `unseen_days` has been a silent no-op since it shipped: `min_age_days` and +-- `max_seen_count` narrowed a retention view, `unseen_days` narrowed nothing. +-- +-- 00084 (which predates 00088) already ships the real signal: +-- `memories.last_read_at`, bumped by `lorekit_record_memory_reads` on every +-- `memory.read`/`list`/`search`/`list_archived` call that actually returns +-- the row. This migration repoints `unseen_days` at that column instead of +-- the dead one, in both callers that evaluate it — `lorekit_groom_candidates` +-- (00088/00093) and `lorekit_memory_list` (00092/00094) — so a saved policy's +-- preview, a run, the nightly sweep, and the Explorer's retention-conditions +-- filter all agree, exactly as they did before this fix, just against a +-- column that is actually written. +-- +-- The "never-seen matches any threshold" reading is unchanged: `last_read_at` +-- is NULL until a row is first read (or if it predates 00084's cutover), and +-- `coalesce(last_read_at, '-infinity')` treats that the same way the dead +-- column's NULL always did — this is a data-source fix, not a semantics +-- change, and `grooming.mdx`'s documented contract ("a lesson that has never +-- been read matches at ANY threshold") was already written for this reading. +-- +-- `last_seen_at` has no remaining reader or writer after this migration — +-- dropped rather than left as inert dead weight for a future reader to trip +-- over again. +-- +-- Both functions keep their EXISTING signatures (no parameter added, removed, +-- or reordered), so `create or replace` is safe here without a preceding +-- `drop function` — unlike 00093/00094, which changed the argument or return +-- shape and had to drop the old overload first. + +-- ── lorekit_groom_candidates (00088, extended by 00093) ───────────────────── +create or replace function lorekit_groom_candidates( + p_user_id uuid, + p_scope text, + p_min_age_days integer default null, + p_unseen_days integer default null, + p_max_seen_count integer default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in' +) +returns table (id uuid, scope text, key text) +language sql +stable +security definer +set search_path = public +as $$ + select m.id, m.scope, m.key + from memories m + where m.user_id = p_user_id + and m.archived_at is null + and m.protected = false + and ( + p_scope = 'global' + or m.scope = p_scope + or ( + p_scope like 'repo::%' + and starts_with(m.scope, 'branch::' || substring(p_scope from 7) || '::') + ) + ) + and (p_min_age_days is null or m.created_at <= now() - (p_min_age_days * interval '1 day')) + and (p_unseen_days is null or coalesce(m.last_read_at, '-infinity'::timestamptz) <= now() - (p_unseen_days * interval '1 day')) + and (p_max_seen_count is null or m.seen_count <= p_max_seen_count) + and lorekit_match_tags(m.tags, p_tags, p_tags_mode) + and lorekit_match_text(m.source_agent, p_source_agent, p_source_agent_mode) + and lorekit_match_text(m.trigger, p_trigger, p_trigger_mode) + and lorekit_match_text(m.kind, p_kind, p_kind_mode) + and lorekit_match_text(m.host, p_host, p_host_mode) + and lorekit_match_text(m.origin_repo, p_origin_repo, p_origin_repo_mode) + and lorekit_match_text(m.origin_branch, p_origin_branch, p_origin_branch_mode) + and lorekit_match_text(m.origin_pr::text, p_origin_pr, p_origin_pr_mode) + order by m.scope, m.key; +$$; + +grant execute on function lorekit_groom_candidates( + uuid, text, integer, integer, integer, + text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text +) to authenticated, service_role; + +-- ── lorekit_memory_list (00067/00069/00092/00094) ─────────────────────────── +create or replace function lorekit_memory_list( + p_user_id uuid, + p_archived boolean default false, + p_scope text default null, + p_key text default null, + p_key_prefix text default null, + p_q text default null, + p_created_since timestamptz default null, + p_created_until timestamptz default null, + p_expires_after timestamptz default null, + p_expires_on_or_before timestamptz default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in', + p_owner text[] default null, + p_owner_mode text default 'in', + p_sort text default 'updated_at', + p_cursor_ts timestamptz default null, + p_cursor_id uuid default null, + p_limit integer default 51, + p_key_scopes text[] default '{}', + p_key_org_access text default 'all', + p_key_org_ids uuid[] default '{}', + p_min_age_days integer default null, + p_unseen_days integer default null, + p_max_seen_count integer default null +) +returns table ( + id uuid, + scope text, + key text, + value text, + tags text[], + source_agent text, + trigger text, + created_at timestamptz, + updated_at timestamptz, + expires_at timestamptz, + archived_at timestamptz, + origin_repo text, + origin_branch text, + origin_commit text, + origin_pr integer, + kind text, + host text, + seen_count integer, + org_id uuid, + created_by uuid, + updated_by uuid, + org_name text, + org_slug text, + total_count integer +) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + v_origin_pr integer[] := ( + select array_agg(x::integer) + from unnest(coalesce(p_origin_pr, '{}'::text[])) as x + where x ~ '^0*[0-9]{1,9}$' + ); + v_sort text := case when p_sort = 'created_at' then 'created_at' else 'updated_at' end; +begin + return query execute format($q$ + select + m.id, m.scope, m.key, m.value, m.tags, m.source_agent, m.trigger, + m.created_at, m.updated_at, m.expires_at, m.archived_at, + m.origin_repo, m.origin_branch, m.origin_commit, m.origin_pr, + m.kind, m.host, m.seen_count, + m.org_id, m.created_by, m.updated_by, + o.name as org_name, o.slug as org_slug, + (count(*) over ())::integer as total_count + from memories m + left join orgs o on o.id = m.org_id + where ( + ($1 is null and auth.role() = 'service_role') + or m.user_id = $1 + or m.org_id in (select lorekit_member_org_ids($1)) + ) + and lorekit_api_token_scope_allowed($32, m.scope) + and lorekit_api_token_org_allowed($33, $34, m.org_id) + and ( + case + when $2 then m.archived_at is not null + else m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + end + ) + and ($3 is null or m.scope = $3) + and ($4 is null or m.key = $4) + and ($5 is null or m.key ilike $5 || '%%') + and ($6 is null or m.key ilike '%%' || $6 || '%%' or m.value ilike '%%' || $6 || '%%') + and ($7 is null or m.created_at >= $7) + and ($8 is null or m.created_at < $8) + and ($9 is null or m.expires_at > $9) + and ($10 is null or m.expires_at <= $10) + and lorekit_match_tags(m.tags, $11, $12) + and lorekit_match_text(m.source_agent, $13, $14) + and lorekit_match_text(m.trigger, $15, $16) + and lorekit_match_text(m.kind, $17, $18) + and lorekit_match_text(m.host, $19, $20) + and lorekit_match_text(m.origin_repo, $21, $22) + and lorekit_match_text(m.origin_branch, $23, $24) + and lorekit_match_int (m.origin_pr, $25, $26) + and ($27 is null or case coalesce($28, 'in') + when 'nin' then ( + (case when m.org_id is null then 'personal' else o.slug end) is not null + and (case when m.org_id is null then 'personal' else o.slug end) <> all($27) + ) + else ( + ('personal' = any($27) and m.org_id is null) + or (m.org_id is not null and o.slug = any($27)) + ) + end) + and ($35 is null or m.created_at <= now() - ($35 * interval '1 day')) + and ($36 is null or coalesce(m.last_read_at, '-infinity'::timestamptz) <= now() - ($36 * interval '1 day')) + and ($37 is null or m.seen_count <= $37) + and ( + $29 is null + or m.%1$I < $29 + or (m.%1$I = $29 and m.id < $30) + ) + order by m.%1$I desc, m.id desc + limit $31 + $q$, v_sort) + using + v_actor, coalesce(p_archived, false), p_scope, p_key, p_key_prefix, p_q, + p_created_since, p_created_until, p_expires_after, p_expires_on_or_before, + p_tags, p_tags_mode, + p_source_agent, p_source_agent_mode, + p_trigger, p_trigger_mode, + p_kind, p_kind_mode, + p_host, p_host_mode, + p_origin_repo, p_origin_repo_mode, + p_origin_branch, p_origin_branch_mode, + v_origin_pr, p_origin_pr_mode, + p_owner, p_owner_mode, + p_cursor_ts, p_cursor_id, + greatest(coalesce(p_limit, 51), 1), + coalesce(p_key_scopes, '{}'::text[]), + coalesce(p_key_org_access, 'all'), + coalesce(p_key_org_ids, '{}'::uuid[]), + p_min_age_days, p_unseen_days, p_max_seen_count; +end; +$$; + +grant execute on function lorekit_memory_list( + uuid, boolean, text, text, text, text, timestamptz, timestamptz, timestamptz, + timestamptz, text[], text, text[], text, text[], text, text[], text, text[], + text, text[], text, text[], text, text[], text, text[], text, text, + timestamptz, uuid, integer, text[], text, uuid[], integer, integer, integer +) to authenticated, service_role; + +-- ── drop the dead column ───────────────────────────────────────────────── +-- No function, view, index, or RLS policy reads or writes `last_seen_at` +-- after the two replacements above — it was never wired to anything in the +-- first place. Dropping it rather than leaving it as inert dead weight. +alter table memories drop column if exists last_seen_at; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00098', 'groom_unseen_days_last_read_at') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00099_last_opened_at_targeted_agent_reads.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- `unseen_days` should mean "no agent has deliberately opened this lesson in +-- N days" — not "nothing touched this row", which is what `last_read_at` +-- (repointed here from the dead `last_seen_at` by 00098) actually measures. +-- +-- THE GAP 00098 LEFT: `last_read_at` is bumped by `lorekit_record_memory_reads` +-- for EVERY read that returns a row, `targeted` (`memory.read` / `GET /:id`) +-- OR `bulk` (`memory.list` / `.search` / `.list_archived`) — and `GET /:id` is +-- the SAME route the web dashboard's LessonDetailSheet calls to render a +-- lesson for a human, plus whatever the CLI's remote `read`/`show` command +-- calls. So `last_read_at` moves when: an agent asks for this exact lesson +-- (the signal `unseen_days` wants), a bulk `list`/`search` happens to include +-- it (no one "saw" it — it rode along in a results page), or a human opens the +-- dashboard's detail sheet (browsing your own lore is not consuming it — the +-- same reasoning migration 00054 already applied to `lorekit_read_activity`). +-- A policy set to "unseen 90d" was therefore satisfied by the Explorer being +-- opened, not by an agent reading the lesson back. +-- +-- THE FIX: a new, narrower column — `memories.last_opened_at` — bumped ONLY +-- when a read is BOTH `read_kind = 'targeted'` AND attributed to an agent +-- surface (`client` = `mcp` or `cli`, the `X-LoreKit-Client` / MCP-transport +-- vocabulary from migration 00054). `unseen_days` in both +-- `lorekit_groom_candidates` and `lorekit_memory_list` now reads THIS column +-- instead of `last_read_at`. +-- +-- `last_read_at`/`read_count` are UNTOUCHED and keep their existing, broader +-- meaning (any read, from any surface, targeted or bulk) — that is the +-- dashboard's "Consumption" row, and it stays useful precisely because it +-- answers a different question ("has this been read at all") than +-- `last_opened_at` ("has an agent deliberately reached for this one lesson"). +-- Both are worth keeping: a lesson can ride along in list pages constantly +-- while never once being individually opened, or vice versa. +-- +-- `lorekit_record_memory_reads` gains a trailing `p_client text default null` +-- — additive, so every existing caller that omits it is unaffected and keeps +-- recording `read_count`/`last_read_at`/`memory_read_daily` exactly as before. +-- Adding a parameter changes the signature, so the old overload is DROPped +-- first, matching 00093/00094/00054's precedent. + +-- ── the column ────────────────────────────────────────────────────────────── +alter table memories add column if not exists last_opened_at timestamptz; + +comment on column memories.last_opened_at is + 'Last time this exact memory was individually retrieved by an agent over + MCP or the CLI (read_kind = targeted, client in (mcp, cli)) — migration + 00099. NULL means never targeted-opened by an agent. Distinct from + last_read_at (migration 00084/00098), which also moves on a bulk + list/search appearance or a human viewing the web dashboard.'; + +-- ── writer: add a trailing p_client, gate last_opened_at on it ───────────── +drop function if exists lorekit_record_memory_reads(uuid[], text); + +create or replace function lorekit_record_memory_reads( + p_memory_ids uuid[], + p_read_kind text, + p_client text default null +) +returns void +language plpgsql +security definer +set search_path = public +as $$ +begin + if p_memory_ids is null or array_length(p_memory_ids, 1) is null then + return; + end if; + + -- ONE statement for every id, not one per id — the hot-path requirement a + -- bulk `list` returning 31 rows imposes. `unnest` turns the array into a rowset + -- the UPDATE can join against directly. + update memories m + set read_count = m.read_count + 1, + last_read_at = now(), + -- Narrower than the two columns above: only a TARGETED read + -- attributed to an agent surface counts as "opened". `p_client` is + -- validated app-side (`parseUsageClient`) before it reaches here, so + -- this is a closed two-value check, not a free-text comparison. + last_opened_at = case + when p_read_kind = 'targeted' and p_client in ('mcp', 'cli') + then now() + else m.last_opened_at + end + from unnest(p_memory_ids) as ids(id) + where m.id = ids.id; + + -- Upsert today's rollup row per memory — one INSERT, conflict-aggregated, + -- same shape as the update above. `current_date` is UTC-anchored the same + -- way `usage_events`' other date_trunc'd reads are (`at time zone 'UTC'`), + -- so a rollup day boundary agrees with every other UTC-bucketed series in + -- this schema. + insert into memory_read_daily (memory_id, day, read_kind, count) + select ids.id, (now() at time zone 'UTC')::date, p_read_kind, 1 + from unnest(p_memory_ids) as ids(id) + on conflict (memory_id, day, read_kind) + do update set count = memory_read_daily.count + 1; +exception + when others then + -- Never let a counter write break the read it is measuring. + return; +end; +$$; + +revoke execute on function lorekit_record_memory_reads(uuid[], text, text) from public, anon; +grant execute on function lorekit_record_memory_reads(uuid[], text, text) to authenticated, service_role; + +comment on function lorekit_record_memory_reads(uuid[], text, text) is + 'Increments memories.read_count/.last_read_at and today''s memory_read_daily + row (UTC day, keyed by read_kind) for every memory id a read call actually + returned, in ONE statement regardless of array size. Also sets + memories.last_opened_at (migration 00099) when read_kind = targeted AND + client in (mcp, cli) -- an agent deliberately reaching for this one lesson, + as opposed to a bulk list/search appearance or a dashboard view. read_kind + is targeted (memory.read / GET /:id) or bulk (memory.list / memory.search / + memory.list_archived). Fail-safe: never throws, so a counter-write failure + cannot fail the read it measures. No-op on a null/empty array.'; + +-- ── unseen_days now reads last_opened_at, not last_read_at ───────────────── +-- Both functions keep their EXISTING signatures (no parameter added, removed, +-- or reordered), so `create or replace` is safe without a preceding +-- `drop function`. + +create or replace function lorekit_groom_candidates( + p_user_id uuid, + p_scope text, + p_min_age_days integer default null, + p_unseen_days integer default null, + p_max_seen_count integer default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in' +) +returns table (id uuid, scope text, key text) +language sql +stable +security definer +set search_path = public +as $$ + select m.id, m.scope, m.key + from memories m + where m.user_id = p_user_id + and m.archived_at is null + and m.protected = false + and ( + p_scope = 'global' + or m.scope = p_scope + or ( + p_scope like 'repo::%' + and starts_with(m.scope, 'branch::' || substring(p_scope from 7) || '::') + ) + ) + and (p_min_age_days is null or m.created_at <= now() - (p_min_age_days * interval '1 day')) + and (p_unseen_days is null or coalesce(m.last_opened_at, '-infinity'::timestamptz) <= now() - (p_unseen_days * interval '1 day')) + and (p_max_seen_count is null or m.seen_count <= p_max_seen_count) + and lorekit_match_tags(m.tags, p_tags, p_tags_mode) + and lorekit_match_text(m.source_agent, p_source_agent, p_source_agent_mode) + and lorekit_match_text(m.trigger, p_trigger, p_trigger_mode) + and lorekit_match_text(m.kind, p_kind, p_kind_mode) + and lorekit_match_text(m.host, p_host, p_host_mode) + and lorekit_match_text(m.origin_repo, p_origin_repo, p_origin_repo_mode) + and lorekit_match_text(m.origin_branch, p_origin_branch, p_origin_branch_mode) + and lorekit_match_text(m.origin_pr::text, p_origin_pr, p_origin_pr_mode) + order by m.scope, m.key; +$$; + +grant execute on function lorekit_groom_candidates( + uuid, text, integer, integer, integer, + text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text +) to authenticated, service_role; + +create or replace function lorekit_memory_list( + p_user_id uuid, + p_archived boolean default false, + p_scope text default null, + p_key text default null, + p_key_prefix text default null, + p_q text default null, + p_created_since timestamptz default null, + p_created_until timestamptz default null, + p_expires_after timestamptz default null, + p_expires_on_or_before timestamptz default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in', + p_owner text[] default null, + p_owner_mode text default 'in', + p_sort text default 'updated_at', + p_cursor_ts timestamptz default null, + p_cursor_id uuid default null, + p_limit integer default 51, + p_key_scopes text[] default '{}', + p_key_org_access text default 'all', + p_key_org_ids uuid[] default '{}', + p_min_age_days integer default null, + p_unseen_days integer default null, + p_max_seen_count integer default null +) +returns table ( + id uuid, + scope text, + key text, + value text, + tags text[], + source_agent text, + trigger text, + created_at timestamptz, + updated_at timestamptz, + expires_at timestamptz, + archived_at timestamptz, + origin_repo text, + origin_branch text, + origin_commit text, + origin_pr integer, + kind text, + host text, + seen_count integer, + org_id uuid, + created_by uuid, + updated_by uuid, + org_name text, + org_slug text, + total_count integer +) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + v_origin_pr integer[] := ( + select array_agg(x::integer) + from unnest(coalesce(p_origin_pr, '{}'::text[])) as x + where x ~ '^0*[0-9]{1,9}$' + ); + v_sort text := case when p_sort = 'created_at' then 'created_at' else 'updated_at' end; +begin + return query execute format($q$ + select + m.id, m.scope, m.key, m.value, m.tags, m.source_agent, m.trigger, + m.created_at, m.updated_at, m.expires_at, m.archived_at, + m.origin_repo, m.origin_branch, m.origin_commit, m.origin_pr, + m.kind, m.host, m.seen_count, + m.org_id, m.created_by, m.updated_by, + o.name as org_name, o.slug as org_slug, + (count(*) over ())::integer as total_count + from memories m + left join orgs o on o.id = m.org_id + where ( + ($1 is null and auth.role() = 'service_role') + or m.user_id = $1 + or m.org_id in (select lorekit_member_org_ids($1)) + ) + and lorekit_api_token_scope_allowed($32, m.scope) + and lorekit_api_token_org_allowed($33, $34, m.org_id) + and ( + case + when $2 then m.archived_at is not null + else m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + end + ) + and ($3 is null or m.scope = $3) + and ($4 is null or m.key = $4) + and ($5 is null or m.key ilike $5 || '%%') + and ($6 is null or m.key ilike '%%' || $6 || '%%' or m.value ilike '%%' || $6 || '%%') + and ($7 is null or m.created_at >= $7) + and ($8 is null or m.created_at < $8) + and ($9 is null or m.expires_at > $9) + and ($10 is null or m.expires_at <= $10) + and lorekit_match_tags(m.tags, $11, $12) + and lorekit_match_text(m.source_agent, $13, $14) + and lorekit_match_text(m.trigger, $15, $16) + and lorekit_match_text(m.kind, $17, $18) + and lorekit_match_text(m.host, $19, $20) + and lorekit_match_text(m.origin_repo, $21, $22) + and lorekit_match_text(m.origin_branch, $23, $24) + and lorekit_match_int (m.origin_pr, $25, $26) + and ($27 is null or case coalesce($28, 'in') + when 'nin' then ( + (case when m.org_id is null then 'personal' else o.slug end) is not null + and (case when m.org_id is null then 'personal' else o.slug end) <> all($27) + ) + else ( + ('personal' = any($27) and m.org_id is null) + or (m.org_id is not null and o.slug = any($27)) + ) + end) + and ($35 is null or m.created_at <= now() - ($35 * interval '1 day')) + and ($36 is null or coalesce(m.last_opened_at, '-infinity'::timestamptz) <= now() - ($36 * interval '1 day')) + and ($37 is null or m.seen_count <= $37) + and ( + $29 is null + or m.%1$I < $29 + or (m.%1$I = $29 and m.id < $30) + ) + order by m.%1$I desc, m.id desc + limit $31 + $q$, v_sort) + using + v_actor, coalesce(p_archived, false), p_scope, p_key, p_key_prefix, p_q, + p_created_since, p_created_until, p_expires_after, p_expires_on_or_before, + p_tags, p_tags_mode, + p_source_agent, p_source_agent_mode, + p_trigger, p_trigger_mode, + p_kind, p_kind_mode, + p_host, p_host_mode, + p_origin_repo, p_origin_repo_mode, + p_origin_branch, p_origin_branch_mode, + v_origin_pr, p_origin_pr_mode, + p_owner, p_owner_mode, + p_cursor_ts, p_cursor_id, + greatest(coalesce(p_limit, 51), 1), + coalesce(p_key_scopes, '{}'::text[]), + coalesce(p_key_org_access, 'all'), + coalesce(p_key_org_ids, '{}'::uuid[]), + p_min_age_days, p_unseen_days, p_max_seen_count; +end; +$$; + +grant execute on function lorekit_memory_list( + uuid, boolean, text, text, text, text, timestamptz, timestamptz, timestamptz, + timestamptz, text[], text, text[], text, text[], text, text[], text, text[], + text, text[], text, text[], text, text[], text, text[], text, text, + timestamptz, uuid, integer, text[], text, uuid[], integer, integer, integer +) to authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00099', 'last_opened_at_targeted_agent_reads') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00100_unseen_days_never_opened_uses_created_at.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- Fix: `unseen_days` matched EVERY lesson no agent had ever opened. +-- +-- 00099 gave `unseen_days` an honest column to key on — `last_opened_at`, +-- bumped only by a targeted MCP/CLI read — and kept 00088's reading of a NULL: +-- +-- coalesce(m.last_opened_at, '-infinity') <= now() - (unseen_days * '1 day') +-- +-- `-infinity` means "never opened == infinitely stale", so the predicate is +-- VACUOUSLY TRUE for every row with a NULL `last_opened_at`, whatever the +-- threshold. A lesson written yesterday and not yet read by anyone satisfies +-- "not opened in 3650 days". +-- +-- Two things make that worse than a rounding error: +-- +-- 1. `last_opened_at` is new in 00099 and was NOT backfilled, so it is NULL +-- for the ENTIRE pre-existing store. On the day 00099 lands, `unseen_days` +-- therefore selects everything the other two conditions let through — +-- including lore an agent opened an hour before the migration ran. +-- 2. Retention policies act on this. 00098 fixed `unseen_days` narrowing +-- NOTHING (a silent no-op); left alone, 00099 converts that into +-- `unseen_days` matching EVERYTHING, which under `mode = 'delete'` is a +-- no-op in the direction that loses data. +-- +-- THE FIX: fall back to `created_at`, not `-infinity`. The staleness clock +-- starts when the lesson was written, so "not opened in N days" is literally +-- true of every row returned: +-- +-- * never opened, written 7 days ago, unseen_days 90 → NO match (7 < 90). +-- * never opened, written 200 days ago, unseen_days 90 → match. +-- * opened 100 days ago, unseen_days 90 → match. +-- +-- It also makes the un-backfilled NULL safe by construction: a pre-00099 row +-- falls back to its own creation date, which is the most conservative honest +-- answer available — we do not know when it was last opened, but we do know it +-- has existed, unopened as far as we can tell, since it was written. +-- +-- `min_age_days` stays a separate condition rather than becoming redundant: +-- for a never-opened row the two now compose as "at least X days old AND at +-- least Y days un-opened", where Y is the binding constraint when Y > X. +-- +-- Forward-only, matching 00098's precedent (which corrected 00088 rather than +-- editing it): 00099 has already been applied to preview databases, so an +-- in-place edit to that file would never re-run there. + +-- ── the two functions, re-issued with the created_at fallback ───────────── +-- Both functions keep their EXISTING signatures (no parameter added, removed, +-- or reordered), so `create or replace` is safe without a preceding +-- `drop function`. + +create or replace function lorekit_groom_candidates( + p_user_id uuid, + p_scope text, + p_min_age_days integer default null, + p_unseen_days integer default null, + p_max_seen_count integer default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in' +) +returns table (id uuid, scope text, key text) +language sql +stable +security definer +set search_path = public +as $$ + select m.id, m.scope, m.key + from memories m + where m.user_id = p_user_id + and m.archived_at is null + and m.protected = false + and ( + p_scope = 'global' + or m.scope = p_scope + or ( + p_scope like 'repo::%' + and starts_with(m.scope, 'branch::' || substring(p_scope from 7) || '::') + ) + ) + and (p_min_age_days is null or m.created_at <= now() - (p_min_age_days * interval '1 day')) + and (p_unseen_days is null or coalesce(m.last_opened_at, m.created_at) <= now() - (p_unseen_days * interval '1 day')) + and (p_max_seen_count is null or m.seen_count <= p_max_seen_count) + and lorekit_match_tags(m.tags, p_tags, p_tags_mode) + and lorekit_match_text(m.source_agent, p_source_agent, p_source_agent_mode) + and lorekit_match_text(m.trigger, p_trigger, p_trigger_mode) + and lorekit_match_text(m.kind, p_kind, p_kind_mode) + and lorekit_match_text(m.host, p_host, p_host_mode) + and lorekit_match_text(m.origin_repo, p_origin_repo, p_origin_repo_mode) + and lorekit_match_text(m.origin_branch, p_origin_branch, p_origin_branch_mode) + and lorekit_match_text(m.origin_pr::text, p_origin_pr, p_origin_pr_mode) + order by m.scope, m.key; +$$; + +grant execute on function lorekit_groom_candidates( + uuid, text, integer, integer, integer, + text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text +) to authenticated, service_role; + +create or replace function lorekit_memory_list( + p_user_id uuid, + p_archived boolean default false, + p_scope text default null, + p_key text default null, + p_key_prefix text default null, + p_q text default null, + p_created_since timestamptz default null, + p_created_until timestamptz default null, + p_expires_after timestamptz default null, + p_expires_on_or_before timestamptz default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in', + p_owner text[] default null, + p_owner_mode text default 'in', + p_sort text default 'updated_at', + p_cursor_ts timestamptz default null, + p_cursor_id uuid default null, + p_limit integer default 51, + p_key_scopes text[] default '{}', + p_key_org_access text default 'all', + p_key_org_ids uuid[] default '{}', + p_min_age_days integer default null, + p_unseen_days integer default null, + p_max_seen_count integer default null +) +returns table ( + id uuid, + scope text, + key text, + value text, + tags text[], + source_agent text, + trigger text, + created_at timestamptz, + updated_at timestamptz, + expires_at timestamptz, + archived_at timestamptz, + origin_repo text, + origin_branch text, + origin_commit text, + origin_pr integer, + kind text, + host text, + seen_count integer, + org_id uuid, + created_by uuid, + updated_by uuid, + org_name text, + org_slug text, + total_count integer +) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + v_origin_pr integer[] := ( + select array_agg(x::integer) + from unnest(coalesce(p_origin_pr, '{}'::text[])) as x + where x ~ '^0*[0-9]{1,9}$' + ); + v_sort text := case when p_sort = 'created_at' then 'created_at' else 'updated_at' end; +begin + return query execute format($q$ + select + m.id, m.scope, m.key, m.value, m.tags, m.source_agent, m.trigger, + m.created_at, m.updated_at, m.expires_at, m.archived_at, + m.origin_repo, m.origin_branch, m.origin_commit, m.origin_pr, + m.kind, m.host, m.seen_count, + m.org_id, m.created_by, m.updated_by, + o.name as org_name, o.slug as org_slug, + (count(*) over ())::integer as total_count + from memories m + left join orgs o on o.id = m.org_id + where ( + ($1 is null and auth.role() = 'service_role') + or m.user_id = $1 + or m.org_id in (select lorekit_member_org_ids($1)) + ) + and lorekit_api_token_scope_allowed($32, m.scope) + and lorekit_api_token_org_allowed($33, $34, m.org_id) + and ( + case + when $2 then m.archived_at is not null + else m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + end + ) + and ($3 is null or m.scope = $3) + and ($4 is null or m.key = $4) + and ($5 is null or m.key ilike $5 || '%%') + and ($6 is null or m.key ilike '%%' || $6 || '%%' or m.value ilike '%%' || $6 || '%%') + and ($7 is null or m.created_at >= $7) + and ($8 is null or m.created_at < $8) + and ($9 is null or m.expires_at > $9) + and ($10 is null or m.expires_at <= $10) + and lorekit_match_tags(m.tags, $11, $12) + and lorekit_match_text(m.source_agent, $13, $14) + and lorekit_match_text(m.trigger, $15, $16) + and lorekit_match_text(m.kind, $17, $18) + and lorekit_match_text(m.host, $19, $20) + and lorekit_match_text(m.origin_repo, $21, $22) + and lorekit_match_text(m.origin_branch, $23, $24) + and lorekit_match_int (m.origin_pr, $25, $26) + and ($27 is null or case coalesce($28, 'in') + when 'nin' then ( + (case when m.org_id is null then 'personal' else o.slug end) is not null + and (case when m.org_id is null then 'personal' else o.slug end) <> all($27) + ) + else ( + ('personal' = any($27) and m.org_id is null) + or (m.org_id is not null and o.slug = any($27)) + ) + end) + and ($35 is null or m.created_at <= now() - ($35 * interval '1 day')) + and ($36 is null or coalesce(m.last_opened_at, m.created_at) <= now() - ($36 * interval '1 day')) + and ($37 is null or m.seen_count <= $37) + and ( + $29 is null + or m.%1$I < $29 + or (m.%1$I = $29 and m.id < $30) + ) + order by m.%1$I desc, m.id desc + limit $31 + $q$, v_sort) + using + v_actor, coalesce(p_archived, false), p_scope, p_key, p_key_prefix, p_q, + p_created_since, p_created_until, p_expires_after, p_expires_on_or_before, + p_tags, p_tags_mode, + p_source_agent, p_source_agent_mode, + p_trigger, p_trigger_mode, + p_kind, p_kind_mode, + p_host, p_host_mode, + p_origin_repo, p_origin_repo_mode, + p_origin_branch, p_origin_branch_mode, + v_origin_pr, p_origin_pr_mode, + p_owner, p_owner_mode, + p_cursor_ts, p_cursor_id, + greatest(coalesce(p_limit, 51), 1), + coalesce(p_key_scopes, '{}'::text[]), + coalesce(p_key_org_access, 'all'), + coalesce(p_key_org_ids, '{}'::uuid[]), + p_min_age_days, p_unseen_days, p_max_seen_count; +end; +$$; + +grant execute on function lorekit_memory_list( + uuid, boolean, text, text, text, text, timestamptz, timestamptz, timestamptz, + timestamptz, text[], text, text[], text, text[], text, text[], text, text[], + text, text[], text, text[], text, text[], text, text[], text, text, + timestamptz, uuid, integer, text[], text, uuid[], integer, integer, integer +) to authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00100', 'unseen_days_never_opened_uses_created_at') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00101_max_read_count_condition.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- 00101_max_read_count_condition.sql +-- +-- A fourth retention condition: `max_read_count` — match only lessons READ at +-- most N times. +-- +-- WHY IT IS NOT max_seen_count. `seen_count` (00059) counts WRITES: a +-- `memory.write` against an existing key bumps it, so it measures how often a +-- lesson RECURRED, not whether anyone used it. "Written once and never +-- consumed" is the classic dead-lore shape and no existing condition could +-- express it — `max_seen_count: 1` catches every one-off write, useful and +-- useless lore alike. +-- +-- WHICH COUNTER. `memories.read_count` (00084) — the BROAD one, incremented on +-- every read including a bulk `memory.list`/`memory.search` appearance. It is +-- deliberately NOT the narrow targeted-open signal `unseen_days` uses +-- (`last_opened_at`, 00099): there is no targeted-only counter, and inventing +-- one would restart every lesson at zero. Every surface must therefore SAY +-- that bulk reads count, rather than labelling both conditions "seen" — the +-- mismatch 00100 was written to clean up. +-- +-- CUTOVER, inherited from 00084. `read_count` starts at 0 for a memory created +-- before that migration, so a low count can mean "not read" or "not counted +-- yet". Unlike `unseen_days` (00100) there is no second column to fall back +-- to, so this condition is honestly conservative in the OPPOSITE direction: it +-- over-matches on old rows rather than under-matching. `lorekit_memory_read_ranking` +-- (00085) already carries the same caveat as `counting_since`, and the docs +-- repeat it wherever this condition is offered. +-- +-- SIGNATURES. `p_max_read_count` is APPENDED to every parameter list rather +-- than slotted beside `p_max_seen_count`, so existing positional callers keep +-- working. Adding a parameter changes the signature, so each function is +-- DROPPED first — `create or replace` would leave a second overload behind and +-- make every call ambiguous. `lorekit_policy_update` takes a jsonb patch and +-- so needs no drop. +-- +-- The index that serves it already exists: `memories_user_read_count_idx` +-- on `(user_id, read_count) where archived_at is null` (00085). + +-- ── the policy column ───────────────────────────────────────────────────── +alter table retention_policies add column if not exists max_read_count integer; + +alter table retention_policies drop constraint if exists retention_policies_max_read_count_non_negative; +alter table retention_policies add constraint retention_policies_max_read_count_non_negative + check (max_read_count is null or max_read_count >= 0); + +comment on column retention_policies.max_read_count is + 'Match only lessons read at most this many times (memories.read_count, 00084 -- + counts bulk list/search appearances too, unlike unseen_days). NULL = unconstrained.'; + +-- ── 1. lorekit_groom_candidates ─────────────────────────────────────────── +drop function if exists lorekit_groom_candidates( + uuid, text, integer, integer, integer, + text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text +); + +create or replace function lorekit_groom_candidates( + p_user_id uuid, + p_scope text, + p_min_age_days integer default null, + p_unseen_days integer default null, + p_max_seen_count integer default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in', + p_max_read_count integer default null +) +returns table (id uuid, scope text, key text) +language sql +stable +security definer +set search_path = public +as $$ + select m.id, m.scope, m.key + from memories m + where m.user_id = p_user_id + and m.archived_at is null + and m.protected = false + and ( + p_scope = 'global' + or m.scope = p_scope + or ( + p_scope like 'repo::%' + and starts_with(m.scope, 'branch::' || substring(p_scope from 7) || '::') + ) + ) + and (p_min_age_days is null or m.created_at <= now() - (p_min_age_days * interval '1 day')) + and (p_unseen_days is null or coalesce(m.last_opened_at, m.created_at) <= now() - (p_unseen_days * interval '1 day')) + and (p_max_seen_count is null or m.seen_count <= p_max_seen_count) + and (p_max_read_count is null or m.read_count <= p_max_read_count) + and lorekit_match_tags(m.tags, p_tags, p_tags_mode) + and lorekit_match_text(m.source_agent, p_source_agent, p_source_agent_mode) + and lorekit_match_text(m.trigger, p_trigger, p_trigger_mode) + and lorekit_match_text(m.kind, p_kind, p_kind_mode) + and lorekit_match_text(m.host, p_host, p_host_mode) + and lorekit_match_text(m.origin_repo, p_origin_repo, p_origin_repo_mode) + and lorekit_match_text(m.origin_branch, p_origin_branch, p_origin_branch_mode) + and lorekit_match_text(m.origin_pr::text, p_origin_pr, p_origin_pr_mode) + order by m.scope, m.key; +$$; + +grant execute on function lorekit_groom_candidates( + uuid, text, integer, integer, integer, + text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, integer +) to authenticated, service_role; + +-- ── 2. lorekit_groom_run ────────────────────────────────────────────────── +drop function if exists lorekit_groom_run( + uuid, text, integer, integer, integer, + text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text +); + +create or replace function lorekit_groom_run( + p_user_id uuid, + p_scope text, + p_min_age_days integer default null, + p_unseen_days integer default null, + p_max_seen_count integer default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in', + p_max_read_count integer default null +) +returns table (archived integer, keys jsonb) +language plpgsql +security definer +set search_path = public +as $$ +declare + v_ids uuid[]; + v_keys jsonb; + v_count integer := 0; +begin + select coalesce(array_agg(c.id), '{}'), + coalesce(jsonb_agg(jsonb_build_object('scope', c.scope, 'key', c.key)), '[]'::jsonb) + into v_ids, v_keys + from lorekit_groom_candidates( + p_user_id, p_scope, p_min_age_days, p_unseen_days, p_max_seen_count, + p_tags, p_tags_mode, p_source_agent, p_source_agent_mode, + p_trigger, p_trigger_mode, p_kind, p_kind_mode, p_host, p_host_mode, + p_origin_repo, p_origin_repo_mode, p_origin_branch, p_origin_branch_mode, + p_origin_pr, p_origin_pr_mode, p_max_read_count + ) c; + + if array_length(v_ids, 1) is null then + return query select 0, '[]'::jsonb; + return; + end if; + + update memories m + set archived_at = now() + where m.id = any(v_ids) + and m.archived_at is null; + get diagnostics v_count = row_count; + + return query select v_count, v_keys; +end; +$$; + +grant execute on function lorekit_groom_run( + uuid, text, integer, integer, integer, + text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, integer +) to authenticated, service_role; + +-- ── 3. lorekit_groom_sweep ──────────────────────────────────────── +create or replace function lorekit_groom_sweep() +returns integer +language plpgsql +security definer +set search_path = public +as $$ +declare + v_policy retention_policies%rowtype; + v_result record; + v_total integer := 0; +begin + for v_policy in + select * from retention_policies where mode = 'auto' and enabled = true + loop + select * into v_result + from lorekit_groom_run( + v_policy.user_id, v_policy.scope, + v_policy.min_age_days, v_policy.unseen_days, v_policy.max_seen_count, + v_policy.tags, v_policy.tags_mode, + v_policy.source_agent, v_policy.source_agent_mode, + v_policy.trigger, v_policy.trigger_mode, + v_policy.kind, v_policy.kind_mode, + v_policy.host, v_policy.host_mode, + v_policy.origin_repo, v_policy.origin_repo_mode, + v_policy.origin_branch, v_policy.origin_branch_mode, + v_policy.origin_pr, v_policy.origin_pr_mode, + v_policy.max_read_count + ); + v_total := v_total + coalesce(v_result.archived, 0); + end loop; + + return v_total; +end; +$$; + +grant execute on function lorekit_groom_sweep() to service_role; + +-- ── 4. lorekit_policy_create ────────────────────────────────────────────── +drop function if exists lorekit_policy_create( + uuid, text, text, text, boolean, integer, integer, integer, + text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text +); + +create or replace function lorekit_policy_create( + p_user_id uuid, + p_scope text, + p_name text, + p_mode text default 'review', + p_enabled boolean default false, + p_min_age_days integer default null, + p_unseen_days integer default null, + p_max_seen_count integer default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in', + p_max_read_count integer default null +) +returns setof retention_policies +language sql +security definer +set search_path = public +as $$ + insert into retention_policies ( + user_id, scope, name, mode, enabled, min_age_days, unseen_days, max_seen_count, + max_read_count, + tags, tags_mode, source_agent, source_agent_mode, trigger, trigger_mode, + kind, kind_mode, host, host_mode, origin_repo, origin_repo_mode, + origin_branch, origin_branch_mode, origin_pr, origin_pr_mode + ) + values ( + p_user_id, p_scope, p_name, p_mode, p_enabled, p_min_age_days, p_unseen_days, p_max_seen_count, + p_max_read_count, + p_tags, p_tags_mode, p_source_agent, p_source_agent_mode, p_trigger, p_trigger_mode, + p_kind, p_kind_mode, p_host, p_host_mode, p_origin_repo, p_origin_repo_mode, + p_origin_branch, p_origin_branch_mode, p_origin_pr, p_origin_pr_mode + ) + returning *; +$$; + +grant execute on function lorekit_policy_create( + uuid, text, text, text, boolean, integer, integer, integer, + text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, integer +) to authenticated, service_role; + +-- ── 5. lorekit_policy_update ─────────────────────────────────── +create or replace function lorekit_policy_update( + p_user_id uuid, + p_id uuid, + p_patch jsonb +) +returns setof retention_policies +language sql +security definer +set search_path = public +as $$ + update retention_policies + set name = case when p_patch ? 'name' then p_patch ->> 'name' else name end, + mode = case when p_patch ? 'mode' then p_patch ->> 'mode' else mode end, + enabled = case when p_patch ? 'enabled' then (p_patch ->> 'enabled')::boolean else enabled end, + min_age_days = case when p_patch ? 'min_age_days' then (p_patch ->> 'min_age_days')::integer else min_age_days end, + unseen_days = case when p_patch ? 'unseen_days' then (p_patch ->> 'unseen_days')::integer else unseen_days end, + max_seen_count = case when p_patch ? 'max_seen_count' then (p_patch ->> 'max_seen_count')::integer else max_seen_count end, + max_read_count = case when p_patch ? 'max_read_count' then (p_patch ->> 'max_read_count')::integer else max_read_count end, + tags = case when p_patch ? 'tags' then (case when p_patch -> 'tags' = 'null'::jsonb then null else array(select jsonb_array_elements_text(p_patch -> 'tags')) end) else tags end, + tags_mode = case when p_patch ? 'tags_mode' then p_patch ->> 'tags_mode' else tags_mode end, + source_agent = case when p_patch ? 'source_agent' then (case when p_patch -> 'source_agent' = 'null'::jsonb then null else array(select jsonb_array_elements_text(p_patch -> 'source_agent')) end) else source_agent end, + source_agent_mode = case when p_patch ? 'source_agent_mode' then p_patch ->> 'source_agent_mode' else source_agent_mode end, + trigger = case when p_patch ? 'trigger' then (case when p_patch -> 'trigger' = 'null'::jsonb then null else array(select jsonb_array_elements_text(p_patch -> 'trigger')) end) else trigger end, + trigger_mode = case when p_patch ? 'trigger_mode' then p_patch ->> 'trigger_mode' else trigger_mode end, + kind = case when p_patch ? 'kind' then (case when p_patch -> 'kind' = 'null'::jsonb then null else array(select jsonb_array_elements_text(p_patch -> 'kind')) end) else kind end, + kind_mode = case when p_patch ? 'kind_mode' then p_patch ->> 'kind_mode' else kind_mode end, + host = case when p_patch ? 'host' then (case when p_patch -> 'host' = 'null'::jsonb then null else array(select jsonb_array_elements_text(p_patch -> 'host')) end) else host end, + host_mode = case when p_patch ? 'host_mode' then p_patch ->> 'host_mode' else host_mode end, + origin_repo = case when p_patch ? 'origin_repo' then (case when p_patch -> 'origin_repo' = 'null'::jsonb then null else array(select jsonb_array_elements_text(p_patch -> 'origin_repo')) end) else origin_repo end, + origin_repo_mode = case when p_patch ? 'origin_repo_mode' then p_patch ->> 'origin_repo_mode' else origin_repo_mode end, + origin_branch = case when p_patch ? 'origin_branch' then (case when p_patch -> 'origin_branch' = 'null'::jsonb then null else array(select jsonb_array_elements_text(p_patch -> 'origin_branch')) end) else origin_branch end, + origin_branch_mode = case when p_patch ? 'origin_branch_mode' then p_patch ->> 'origin_branch_mode' else origin_branch_mode end, + origin_pr = case when p_patch ? 'origin_pr' then (case when p_patch -> 'origin_pr' = 'null'::jsonb then null else array(select jsonb_array_elements_text(p_patch -> 'origin_pr')) end) else origin_pr end, + origin_pr_mode = case when p_patch ? 'origin_pr_mode' then p_patch ->> 'origin_pr_mode' else origin_pr_mode end + where id = p_id and user_id = p_user_id + returning *; +$$; + +grant execute on function lorekit_policy_update(uuid, uuid, jsonb) to authenticated, service_role; + +-- ── 6. lorekit_memory_list ──────────────────────────────────────────────── +drop function if exists lorekit_memory_list( + uuid, boolean, text, text, text, text, timestamptz, timestamptz, timestamptz, + timestamptz, text[], text, text[], text, text[], text, text[], text, text[], + text, text[], text, text[], text, text[], text, text[], text, text, + timestamptz, uuid, integer, text[], text, uuid[], integer, integer, integer +); + +create or replace function lorekit_memory_list( + p_user_id uuid, + p_archived boolean default false, + p_scope text default null, + p_key text default null, + p_key_prefix text default null, + p_q text default null, + p_created_since timestamptz default null, + p_created_until timestamptz default null, + p_expires_after timestamptz default null, + p_expires_on_or_before timestamptz default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in', + p_owner text[] default null, + p_owner_mode text default 'in', + p_sort text default 'updated_at', + p_cursor_ts timestamptz default null, + p_cursor_id uuid default null, + p_limit integer default 51, + p_key_scopes text[] default '{}', + p_key_org_access text default 'all', + p_key_org_ids uuid[] default '{}', + p_min_age_days integer default null, + p_unseen_days integer default null, + p_max_seen_count integer default null, + p_max_read_count integer default null +) +returns table ( + id uuid, + scope text, + key text, + value text, + tags text[], + source_agent text, + trigger text, + created_at timestamptz, + updated_at timestamptz, + expires_at timestamptz, + archived_at timestamptz, + origin_repo text, + origin_branch text, + origin_commit text, + origin_pr integer, + kind text, + host text, + seen_count integer, + org_id uuid, + created_by uuid, + updated_by uuid, + org_name text, + org_slug text, + total_count integer +) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + v_origin_pr integer[] := ( + select array_agg(x::integer) + from unnest(coalesce(p_origin_pr, '{}'::text[])) as x + where x ~ '^0*[0-9]{1,9}$' + ); + v_sort text := case when p_sort = 'created_at' then 'created_at' else 'updated_at' end; +begin + return query execute format($q$ + select + m.id, m.scope, m.key, m.value, m.tags, m.source_agent, m.trigger, + m.created_at, m.updated_at, m.expires_at, m.archived_at, + m.origin_repo, m.origin_branch, m.origin_commit, m.origin_pr, + m.kind, m.host, m.seen_count, + m.org_id, m.created_by, m.updated_by, + o.name as org_name, o.slug as org_slug, + (count(*) over ())::integer as total_count + from memories m + left join orgs o on o.id = m.org_id + where ( + ($1 is null and auth.role() = 'service_role') + or m.user_id = $1 + or m.org_id in (select lorekit_member_org_ids($1)) + ) + and lorekit_api_token_scope_allowed($32, m.scope) + and lorekit_api_token_org_allowed($33, $34, m.org_id) + and ( + case + when $2 then m.archived_at is not null + else m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + end + ) + and ($3 is null or m.scope = $3) + and ($4 is null or m.key = $4) + and ($5 is null or m.key ilike $5 || '%%') + and ($6 is null or m.key ilike '%%' || $6 || '%%' or m.value ilike '%%' || $6 || '%%') + and ($7 is null or m.created_at >= $7) + and ($8 is null or m.created_at < $8) + and ($9 is null or m.expires_at > $9) + and ($10 is null or m.expires_at <= $10) + and lorekit_match_tags(m.tags, $11, $12) + and lorekit_match_text(m.source_agent, $13, $14) + and lorekit_match_text(m.trigger, $15, $16) + and lorekit_match_text(m.kind, $17, $18) + and lorekit_match_text(m.host, $19, $20) + and lorekit_match_text(m.origin_repo, $21, $22) + and lorekit_match_text(m.origin_branch, $23, $24) + and lorekit_match_int (m.origin_pr, $25, $26) + and ($27 is null or case coalesce($28, 'in') + when 'nin' then ( + (case when m.org_id is null then 'personal' else o.slug end) is not null + and (case when m.org_id is null then 'personal' else o.slug end) <> all($27) + ) + else ( + ('personal' = any($27) and m.org_id is null) + or (m.org_id is not null and o.slug = any($27)) + ) + end) + and ($35 is null or m.created_at <= now() - ($35 * interval '1 day')) + and ($36 is null or coalesce(m.last_opened_at, m.created_at) <= now() - ($36 * interval '1 day')) + and ($37 is null or m.seen_count <= $37) + and ($38 is null or m.read_count <= $38) + and ( + $29 is null + or m.%1$I < $29 + or (m.%1$I = $29 and m.id < $30) + ) + order by m.%1$I desc, m.id desc + limit $31 + $q$, v_sort) + using + v_actor, coalesce(p_archived, false), p_scope, p_key, p_key_prefix, p_q, + p_created_since, p_created_until, p_expires_after, p_expires_on_or_before, + p_tags, p_tags_mode, + p_source_agent, p_source_agent_mode, + p_trigger, p_trigger_mode, + p_kind, p_kind_mode, + p_host, p_host_mode, + p_origin_repo, p_origin_repo_mode, + p_origin_branch, p_origin_branch_mode, + v_origin_pr, p_origin_pr_mode, + p_owner, p_owner_mode, + p_cursor_ts, p_cursor_id, + greatest(coalesce(p_limit, 51), 1), + coalesce(p_key_scopes, '{}'::text[]), + coalesce(p_key_org_access, 'all'), + coalesce(p_key_org_ids, '{}'::uuid[]), + p_min_age_days, p_unseen_days, p_max_seen_count, p_max_read_count; +end; +$$; + +grant execute on function lorekit_memory_list( + uuid, boolean, text, text, text, text, timestamptz, timestamptz, timestamptz, + timestamptz, text[], text, text[], text, text[], text, text[], text, text[], + text, text[], text, text[], text, text[], text, text[], text, text, + timestamptz, uuid, integer, text[], text, uuid[], integer, integer, integer, + integer +) to authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00101', 'max_read_count_condition') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00102_github_relevance_configs.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- Server-side review-comment relevance classification — per-installation +-- configuration. +-- +-- WHAT THIS ENABLES +-- A review agent posts findings as PR review comments. Later, the thread is +-- resolved, declined, or merged still open. That outcome is the only honest +-- signal about whether the finding was worth posting, and until now capturing +-- it required every consumer repository to install a GitHub Actions workflow +-- plus a LOREKIT_API_KEY secret. The GitHub App already receives the same +-- deliveries, so the classification can run server-side and a consumer gets +-- it by installing the App — no workflow, no secret. +-- +-- WHY IT NEEDS CONFIGURATION AT ALL +-- LoreKit must NOT learn any particular agent's vocabulary. It does not know +-- what a "finder", a "defect class", or a fingerprint is, and it must never +-- have to. So the account that installed the App declares, once, three +-- things about its own review agent: +-- +-- • marker_open / marker_close — the literal delimiters the agent wraps its +-- per-finding identifier in, inside the comment body. +-- • bucket_tag — the LoreKit tag the outcome is filed under. +-- • key_prefix — the key namespace the outcome is filed in. +-- • agent_name — what to record as the finding's author. +-- +-- `agent_name` exists because the consuming reader filters on it: another +-- bot's declined finding and a human's incidental comment have their own +-- false-positive profiles and must never train this agent's suppressor. The +-- account is the only party that can name its agent, and a comment carrying +-- the account's own marker is by definition from it — so the marker is the +-- attribution and this column is what it attributes to. LoreKit does not +-- otherwise use the value. +-- +-- The identifier between the delimiters is copied VERBATIM and treated as an +-- opaque string. LoreKit never parses it, never splits it, never derives +-- meaning from it. That is the whole separation: LoreKit owns the mechanism +-- (receive, read, classify, write); the account owns the vocabulary. +-- +-- WHY LITERAL DELIMITERS AND NOT A REGEX +-- A caller-supplied regex compiled inside an edge function is a +-- catastrophic-backtracking vector, and "reject the dangerous patterns" is +-- not a check anyone can write soundly. Two literal delimiters answer the +-- same question in linear time with nothing to validate beyond length, and a +-- human can read the stored config and know exactly what it matches. +-- +-- SECURITY POSTURE — the config IS the boundary +-- Anyone who can comment on a PR in a covered repository can cause a write. +-- That is already true of the `source::pr-webhook` candidate feed, but a +-- classified record is filed under a DERIVED key, so an unbounded key would +-- let a commenter address — and overwrite — an arbitrary existing record. +-- Three things bound it: +-- a) The account, not the comment, supplies `key_prefix` and `bucket_tag`. +-- A commenter can only ever influence the suffix, inside a namespace the +-- account itself declared. +-- b) The extracted identifier must pass a strict charset + length check +-- (`isSafeMarkerValue`, comment-relevance.ts) or nothing is written. +-- c) The write is scoped to `repo::` of the delivery, which is +-- HMAC-verified before any of this runs. +-- +-- WHY THE ROW CARRIES NO user_id +-- Ownership comes from the installation, resolved at write time through +-- `github_installations.user_id`. Storing it here too would let the two +-- disagree after an installation is transferred, and the installation row is +-- the one the reconcile path already keeps correct. +-- +-- Authorization posture (matches 00037 and the Phase 3 org rule): +-- • RLS grants SELECT on rows whose installation is linked to auth.uid(). +-- • There is NO insert/update/delete policy. Every state transition goes +-- through a SECURITY DEFINER RPC that checks installation ownership. +-- • The edge function reads through `lorekit_relevance_config_for_repo`, +-- service-role only, which resolves repo → installation → owner → config +-- in one round trip. +-- ═════════════════════════════════════════════════════════════════════════ + +-- 1. github_relevance_configs — one row per (installation, bucket). +create table github_relevance_configs ( + id uuid primary key default gen_random_uuid(), + installation_id bigint not null + references github_installations(installation_id) on delete cascade, + + -- Literal delimiters around the agent's opaque per-finding identifier. + -- `marker_open` is at least 4 characters so a config cannot be written that + -- matches near-arbitrary prose; `marker_close` may be short (`-->`). + marker_open text not null check (length(marker_open) between 4 and 120), + marker_close text not null check (length(marker_close) between 1 and 120), + + -- Where the outcome is filed. Both are the account's vocabulary, never a + -- commenter's — see the security posture above. + bucket_tag text not null check (bucket_tag ~ '^[a-z0-9]+(::[a-z0-9._-]+)+$'), + key_prefix text not null check (length(key_prefix) between 1 and 200), + agent_name text not null check (length(agent_name) between 1 and 60), + + -- Taxonomy applied to the written record. LoreKit infers kind/host only from + -- a `loop::` tag, so a `ci::`-tagged bucket must state them explicitly. + record_kind text not null default 'signal' + check (record_kind in ('lesson', 'bus', 'signal')), + record_host text check (record_host is null or length(record_host) between 1 and 60), + + ttl_days int not null default 60 check (ttl_days between 1 and 365), + active boolean not null default true, + created_at timestamptz not null default now(), + updated_at timestamptz not null default now() +); + +-- One active config per (installation, bucket): an account can classify into +-- more than one bucket, but never twice into the same one with two different +-- markers, which would make the write order decide the record. +create unique index github_relevance_configs_active_uniq + on github_relevance_configs (installation_id, bucket_tag) + where active = true; + +create index github_relevance_configs_installation_id_idx + on github_relevance_configs (installation_id) + where active = true; + +alter table github_relevance_configs enable row level security; + +-- Readable by the owner of the installation, and by the service role. +create policy "rls_github_relevance_configs_select" + on github_relevance_configs for select + using ( + installation_id in ( + select i.installation_id + from github_installations i + where i.user_id = auth.uid() and i.status = 'linked' + ) + ); + +grant select on github_relevance_configs to authenticated, service_role; + +-- 2. Edge lookup — repo full_name → the configs to classify with, plus the +-- LoreKit user the resulting records belong to. +-- +-- Returns ZERO rows for every honest no-op, and the caller treats them +-- identically: the repo is not covered by any installation, the installation +-- is still `pending` (nobody to attribute the write to), or the account has +-- declared no marker. None of those is an error. +-- +-- `status = 'linked'` is load-bearing rather than cosmetic: a webhook write +-- with a null user_id lands in the service-role partition, which no user +-- token can read back (see `applyTenantScope`), so classifying for a pending +-- installation would produce records that exist and are unreachable. +create or replace function lorekit_relevance_config_for_repo(p_repo text) +returns table ( + installation_id bigint, + user_id uuid, + marker_open text, + marker_close text, + bucket_tag text, + key_prefix text, + agent_name text, + record_kind text, + record_host text, + ttl_days int +) +language sql +stable +security definer +set search_path = public +as $$ + select c.installation_id, + i.user_id, + c.marker_open, + c.marker_close, + c.bucket_tag, + c.key_prefix, + c.agent_name, + c.record_kind, + c.record_host, + c.ttl_days + from installation_repositories r + join github_installations i + on i.installation_id = r.installation_id + join github_relevance_configs c + on c.installation_id = r.installation_id + where r.full_name = lower(p_repo) + and r.active = true + and i.status = 'linked' + and i.user_id is not null + and c.active = true; +$$; + +-- Service-role only, and the revoke is not optional. This function takes an +-- ARBITRARY `p_repo` and returns another account's installation owner plus its +-- whole declared vocabulary, with no `auth.uid()` gate in the body — the +-- table's select-RLS does not apply inside a `security definer` function. A +-- newly CREATEd function in `public` carries PostgreSQL's default PUBLIC +-- EXECUTE, and on this project's Supabase image ALSO a direct default-privilege +-- grant to `anon` that `revoke ... from public` does not touch (00077's whole +-- subject). Granting to `service_role` without revoking first therefore leaves +-- the function callable by every `anon`/`authenticated` caller through +-- PostgREST. Same three statements as `lorekit_db_query_stats` (00074) and +-- `lorekit_groom_sweep_and_record` (00095). +revoke all on function lorekit_relevance_config_for_repo(text) from public; +revoke all on function lorekit_relevance_config_for_repo(text) from anon, authenticated; +grant execute on function lorekit_relevance_config_for_repo(text) to service_role; + +-- 3. Owner-only upsert. The caller proves ownership by owning the installation; +-- no caller-supplied user_id is trusted and there is no UPDATE policy to +-- reach the table any other way. +create or replace function lorekit_relevance_config_set( + p_installation_id bigint, + p_marker_open text, + p_marker_close text, + p_bucket_tag text, + p_key_prefix text, + p_agent_name text, + p_record_kind text default 'signal', + p_record_host text default null, + p_ttl_days int default 60 +) +returns uuid +language plpgsql +security definer +set search_path = public +as $$ +declare + v_id uuid; +begin + if not exists ( + select 1 + from github_installations i + where i.installation_id = p_installation_id + and i.user_id = auth.uid() + and i.status = 'linked' + ) then + raise exception 'installation not found or not linked to caller'; + end if; + + insert into github_relevance_configs ( + installation_id, marker_open, marker_close, bucket_tag, key_prefix, agent_name, + record_kind, record_host, ttl_days, active, updated_at + ) + values ( + p_installation_id, p_marker_open, p_marker_close, p_bucket_tag, p_key_prefix, p_agent_name, + coalesce(p_record_kind, 'signal'), p_record_host, coalesce(p_ttl_days, 60), true, now() + ) + on conflict (installation_id, bucket_tag) where active = true + do update set + marker_open = excluded.marker_open, + marker_close = excluded.marker_close, + key_prefix = excluded.key_prefix, + agent_name = excluded.agent_name, + record_kind = excluded.record_kind, + record_host = excluded.record_host, + ttl_days = excluded.ttl_days, + updated_at = now() + returning id into v_id; + + return v_id; +end; +$$; + +-- `anon` is named explicitly for the reason 00077 exists: the default-privilege +-- grant to `anon` survives a bare `revoke ... from public`. The body's +-- `auth.uid()` gate already denies an anonymous caller, so this is defence in +-- depth — but leaving the function on PostgREST's anonymous surface is the +-- exposure this schema closes by policy, not case by case. +revoke execute on function lorekit_relevance_config_set(bigint, text, text, text, text, text, text, text, int) + from public, anon; +grant execute on function lorekit_relevance_config_set(bigint, text, text, text, text, text, text, text, int) + to authenticated; + +-- 4. Owner-only deactivate. Soft, so the unique index frees up for a new +-- config while the history of what was configured stays readable. +create or replace function lorekit_relevance_config_deactivate( + p_installation_id bigint, + p_bucket_tag text +) +returns void +language plpgsql +security definer +set search_path = public +as $$ +begin + if not exists ( + select 1 + from github_installations i + where i.installation_id = p_installation_id + and i.user_id = auth.uid() + and i.status = 'linked' + ) then + raise exception 'installation not found or not linked to caller'; + end if; + + update github_relevance_configs + set active = false, updated_at = now() + where installation_id = p_installation_id + and bucket_tag = p_bucket_tag + and active = true; +end; +$$; + +revoke execute on function lorekit_relevance_config_deactivate(bigint, text) + from public, anon; +grant execute on function lorekit_relevance_config_deactivate(bigint, text) + to authenticated; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00102', 'github_relevance_configs') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00103_reads_must_not_restamp_updated_at.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- 00103 — a READ must not rewrite `updated_at`. +-- +-- THE BUG, observed in production on 2026-09-03: +-- +-- BEFORE updated_at = 2026-09-03T17:49:09.236723+00:00 +-- GET /memories?scope=repo::mthines/agent-skills&limit=100 +-- AFTER updated_at = 2026-09-03T17:50:04.978342+00:00 +-- +-- No write of any kind was issued. `lorekit_record_memory_reads` (00099) does +-- +-- update memories m set read_count = m.read_count + 1, last_read_at = now() +-- +-- to record that a read happened, and that UPDATE fires the BEFORE-row trigger +-- `lorekit_memories_set_updated_at`, which stamps `updated_at = now()`. Because +-- a single `memory.list` page carries ~55 rows and every one of them gets the +-- counter bump, ONE bulk read restamps ~55 memories. +-- +-- WHAT IT COST, measured on the same account the day this was written: +-- * 0 of 2,475 sampled memories had an `updated_at` older than 14 days, +-- against a median row age of 22 days and a maximum of 77 — freshness was +-- unreadable because every row looked touched-just-now. +-- * `updated_at` is the DEFAULT SORT for both the dashboard Explorer and +-- `memory.list order=recency`, so the ordering agents receive lore in was +-- being driven by which rows happened to be read most recently — that is, +-- by nothing. +-- * The detail sheet's "Last updated" field was showing a last-READ time +-- under a last-WRITTEN label. +-- +-- THE FIX is the one migration 00062 already established for the embedding +-- columns, applied to the read counters: a DERIVED column must not rewrite the +-- recency signal that search, `relevant`, the keyset index and lesson-rank all +-- read. 00062's function is EXTENDED rather than replaced, so `memories` keeps +-- exactly one BEFORE UPDATE trigger and the two exemptions are stated in one +-- place instead of racing each other. +-- +-- The predicate keeps 00062's shape for the reason its comment gives: mask the +-- derived columns plus `updated_at`, then compare the WHOLE rows, so a column +-- added to `memories` later is covered automatically where an explicit +-- "did any real column change" list would silently stop covering it. +-- +-- The `v_*_changed` preconditions are equally load-bearing, and for 00062's +-- reason: without them the rule degrades to "nothing meaningful changed → +-- preserve", which also catches a plain NO-OP re-write — and `memory_write` +-- upserts, so an agent re-saving an identical lesson would stop refreshing +-- `updated_at`. Preserve only when a derived column ACTUALLY moved and nothing +-- else did. +-- +-- `set_updated_at` itself is still LEFT ALONE — six other tables share it +-- (`user_limits`, `orgs`, `org_limits`, `plans`, `user_plans`, +-- `retention_policies`) and none of them has counters or this problem. +-- ═════════════════════════════════════════════════════════════════════════ + +create or replace function lorekit_memories_set_updated_at() +returns trigger +language plpgsql +set search_path = public, extensions +as $$ +declare + v_new memories := new; + v_old memories := old; + v_embedding_changed boolean; + v_counters_changed boolean; +begin + -- 00062's exemption: an embedding write. + v_embedding_changed := + new.embedding is distinct from old.embedding + or new.embedding_model is distinct from old.embedding_model; + + -- This migration's exemption: a read-counter write from + -- `lorekit_record_memory_reads`. `last_opened_at` is included because a + -- TARGETED read moves it in the same statement as `read_count`; leaving it + -- unmasked would make every targeted read look like a real edit. + v_counters_changed := + new.read_count is distinct from old.read_count + or new.last_read_at is distinct from old.last_read_at + or new.last_opened_at is distinct from old.last_opened_at; + + -- Mask the derived columns and the one being decided. + -- + -- `fts` MUST be masked too, for 00062's reason: it is a GENERATED column and + -- Postgres computes generated columns AFTER BEFORE-row triggers run, so + -- `new.fts` is NULL here while `old.fts` holds the stored value. An unmasked + -- comparison finds every update "changed", which silently turns this whole + -- function back into `set_updated_at`. It fails in the safe direction (a bump + -- that should not happen, never a missing bump), which is exactly why it + -- needs a test rather than a reading — see migrations.test.sql §101. + v_new.embedding := null; v_new.embedding_model := null; + v_new.read_count := null; v_new.last_read_at := null; v_new.last_opened_at := null; + v_new.updated_at := null; v_new.fts := null; + + v_old.embedding := null; v_old.embedding_model := null; + v_old.read_count := null; v_old.last_read_at := null; v_old.last_opened_at := null; + v_old.updated_at := null; v_old.fts := null; + + if (v_embedding_changed or v_counters_changed) + and v_new is not distinct from v_old then + -- A derived column moved and nothing else did. Preserve the row's real + -- recency rather than restamping it. + new.updated_at := old.updated_at; + else + new.updated_at := now(); + end if; + + return new; +end; +$$; + +-- The trigger already points here (00062); recreated so a fresh database and a +-- migrated one land in the same state regardless of which migrations ran. +create or replace trigger memories_updated_at + before update on memories + for each row execute function lorekit_memories_set_updated_at(); + +comment on function lorekit_memories_set_updated_at() is + 'BEFORE UPDATE on memories: bumps updated_at like set_updated_at, EXCEPT when the only change is ' + 'a DERIVED column — the embedding pair (00062) or the read counters read_count/last_read_at/' + 'last_opened_at (00103). Neither may rewrite the recency signal that search, relevant, the keyset ' + 'index, lesson-rank and BOTH default sorts (Explorer, memory.list order=recency) all read. Before ' + '00103 a single bulk memory.list restamped every row on the page, so updated_at reported ' + 'last-READ under a last-WRITTEN name and recency ordering was effectively arbitrary.'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00103', 'reads_must_not_restamp_updated_at') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00104_opened_count_pull_through.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- 00104 — `opened_count`: the numerator that makes lore value comparable. +-- +-- THE PROBLEM. All three counters a memory carries are ABSOLUTE COUNTS ON THE +-- SUPPLY SIDE — how often the system produced this lesson: +-- +-- seen_count (00059) how many times it was WRITTEN. 88.4% of a live +-- 2,475-row sample sit at exactly 1, so it has almost +-- no variance to rank by. +-- read_count (00084) how many times it was READ — but 99.80% of recorded +-- reads are BULK ride-alongs in a list/search page, so +-- this ranks SCOPE BREADTH, not usefulness. A `global` +-- lesson is delivered on every session; a `branch` +-- lesson almost never. Sorting by it puts every global +-- lesson on top whether or not anything used it. +-- last_opened_at (00099) when an agent last deliberately fetched THIS lesson. +-- The honest signal — and a bare timestamp, with no +-- count and no denominator to compare rows by. +-- +-- Value lives on the DEMAND side: given that a lesson was offered, did anything +-- take it up? That is a RATIO, and a ratio cancels the confound — scope breadth +-- appears in both numerator and denominator and divides away. A `global` lesson +-- delivered 1,300 times and opened twice scores 0.15%; a `branch` lesson +-- delivered 3 times and opened twice scores 67%. For the first time those two +-- numbers are comparable, and the second is obviously the more valuable. +-- +-- PULL-THROUGH is defined here as `opened_count / read_count` — targeted opens +-- over TOTAL reads, not over bulk reads. A proper fraction in [0, 1]: bounded, +-- so it sorts; and defined for a lesson that has only ever been fetched +-- deliberately (which `targeted / bulk` would divide by zero on). +-- +-- WHY A COLUMN, when `memory_read_daily(memory_id, day, read_kind, count)` +-- (00084) has stored the split since the counters shipped: +-- +-- 1. It has to be FILTERABLE at scale. `max_opened_count` (00105) is a +-- retention condition, and `lorekit_groom_candidates` runs it over a whole +-- scope. A per-memory aggregate over the rollup cannot be indexed; a +-- column can, and gets `memories_user_opened_count_idx` below, mirroring +-- 00085's index on `read_count`. +-- 2. It cannot DISAGREE with `last_opened_at`. Both move in the SAME +-- statement under the SAME gate inside `lorekit_record_memory_reads`, so +-- "opened 4 times" and "last opened on ..." are always the same event +-- stream. Deriving one from the rollup and reading the other from the row +-- is two sources for one fact. +-- +-- The rollup is still the authority on HISTORY, which is what the backfill +-- below reads: every targeted read ever recorded, so the column ships exact +-- rather than starting every lesson at zero. +-- +-- ORDER IS LOAD-BEARING IN THIS FILE. The trigger is taught to mask +-- `opened_count` BEFORE the backfill runs. Reversed, the backfill's UPDATE +-- would restamp `updated_at` on every memory that has ever been opened — the +-- exact whole-store recency wipe 00103 was written to stop, and unrecoverable +-- once done. +-- ═════════════════════════════════════════════════════════════════════════ + +-- ── 1. the column ───────────────────────────────────────────────────────── +alter table memories add column if not exists opened_count integer not null default 0; + +comment on column memories.opened_count is + 'How many times an agent DELIBERATELY fetched this exact lesson -- the count + whose timestamp is last_opened_at (00099), moved by the same gate in the same + statement: read_kind = targeted AND client in (mcp, cli). Never moved by a + bulk list/search ride-along or by a human browsing the dashboard. Divided by + read_count (00084, which counts every read) it gives PULL-THROUGH: of all the + times this lesson was surfaced, how often was it a deliberate fetch. That + ratio is comparable across scopes where the raw counts are not, because scope + breadth cancels. Not null, defaults to 0; backfilled from memory_read_daily + by 00104.'; + +-- ── 2. teach the trigger to mask it — BEFORE the backfill ───────────────── +-- 00103 exempted read_count/last_read_at/last_opened_at from restamping +-- `updated_at`; opened_count is the fourth column of that same derived set and +-- moves in the same statement, so omitting it here would defeat 00103 for every +-- TARGETED read. Asserted by migrations.test.sql §101 AC-2b, which does exactly +-- that read and checks the stamp did not move. +create or replace function lorekit_memories_set_updated_at() +returns trigger +language plpgsql +set search_path = public, extensions +as $$ +declare + v_new memories := new; + v_old memories := old; + v_embedding_changed boolean; + v_counters_changed boolean; +begin + -- 00062's exemption: an embedding write. + v_embedding_changed := + new.embedding is distinct from old.embedding + or new.embedding_model is distinct from old.embedding_model; + + -- 00103's exemption, extended by 00104 with opened_count: a read-counter + -- write from `lorekit_record_memory_reads`. + v_counters_changed := + new.read_count is distinct from old.read_count + or new.last_read_at is distinct from old.last_read_at + or new.last_opened_at is distinct from old.last_opened_at + or new.opened_count is distinct from old.opened_count; + + -- Mask the derived columns and the one being decided. + -- + -- `fts` MUST be masked too, for 00062's reason: it is a GENERATED column and + -- Postgres computes generated columns AFTER BEFORE-row triggers, so + -- `new.fts` is NULL here while `old.fts` holds the stored value. An unmasked + -- comparison finds every update "changed", which silently turns this whole + -- function back into `set_updated_at`. It fails in the safe direction (a bump + -- that should not happen, never a missing bump), which is exactly why it + -- needs a test rather than a reading — see migrations.test.sql §101. + v_new.embedding := null; v_new.embedding_model := null; + v_new.read_count := null; v_new.last_read_at := null; v_new.last_opened_at := null; + v_new.opened_count := null; + v_new.updated_at := null; v_new.fts := null; + + v_old.embedding := null; v_old.embedding_model := null; + v_old.read_count := null; v_old.last_read_at := null; v_old.last_opened_at := null; + v_old.opened_count := null; + v_old.updated_at := null; v_old.fts := null; + + if (v_embedding_changed or v_counters_changed) + and v_new is not distinct from v_old then + -- A derived column moved and nothing else did. Preserve the row's real + -- recency rather than restamping it. + new.updated_at := old.updated_at; + else + new.updated_at := now(); + end if; + + return new; +end; +$$; + +comment on function lorekit_memories_set_updated_at() is + 'BEFORE UPDATE on memories: bumps updated_at like set_updated_at, EXCEPT when the only change is ' + 'a DERIVED column — the embedding pair (00062) or the read counters read_count/last_read_at/' + 'last_opened_at (00103) and opened_count (00104). None may rewrite the recency signal that ' + 'search, relevant, the keyset index, lesson-rank and BOTH default sorts (Explorer, memory.list ' + 'order=recency) all read. Before 00103 a single bulk memory.list restamped every row on the page, ' + 'so updated_at reported last-READ under a last-WRITTEN name and recency ordering was arbitrary.'; + +-- ── 3. backfill from the rollup ─────────────────────────────────────────── +-- Every targeted read `memory_read_daily` has ever recorded, so the column is +-- exact from day one instead of restarting the whole store at zero — the +-- cutover problem 00101 had to document for `read_count` and this one does not. +-- Runs AFTER the trigger above, so it preserves `updated_at` (see the header). +update memories m + set opened_count = t.n + from ( + select memory_id, sum(count)::integer as n + from memory_read_daily + where read_kind = 'targeted' + group by memory_id + ) t + where m.id = t.memory_id + and m.opened_count is distinct from t.n; + +-- ── 4. the index the retention condition and the ranking need ───────────── +-- Mirrors `memories_user_read_count_idx` (00085) so `max_opened_count` (00105) +-- and any "never chosen" query are served the same way `max_read_count` is. +create index if not exists memories_user_opened_count_idx + on memories (user_id, opened_count) + where archived_at is null; + +-- ── 5. the writer ───────────────────────────────────────────────────────── +-- Same signature, so `create or replace` is enough — no drop needed. The +-- increment is gated on EXACTLY the condition that sets `last_opened_at`, +-- written as one shared `case` so the pair cannot drift apart. +create or replace function lorekit_record_memory_reads( + p_memory_ids uuid[], + p_read_kind text, + p_client text default null +) +returns void +language plpgsql +security definer +set search_path = public +as $$ +declare + -- Resolved ONCE, outside the statement: `p_read_kind`/`p_client` are call + -- arguments, not per-row values, so "is this a targeted agent open" is one + -- boolean for the whole batch rather than a predicate re-evaluated per row — + -- and, more importantly, the count and the timestamp below cannot be gated + -- on two expressions that drift. + v_opened boolean := p_read_kind = 'targeted' and p_client in ('mcp', 'cli'); +begin + if p_memory_ids is null or array_length(p_memory_ids, 1) is null then + return; + end if; + + -- ONE statement for every id, not one per id — the hot-path requirement a + -- bulk `list` returning 31 rows imposes. `unnest` turns the array into a rowset + -- the UPDATE can join against directly. + update memories m + set read_count = m.read_count + 1, + last_read_at = now(), + -- Narrower than the two columns above: only a TARGETED read + -- attributed to an agent surface counts as "opened". `p_client` is + -- validated app-side (`parseUsageClient`) before it reaches here, so + -- this is a closed two-value check, not a free-text comparison. + last_opened_at = case when v_opened then now() else m.last_opened_at end, + opened_count = m.opened_count + case when v_opened then 1 else 0 end + from unnest(p_memory_ids) as ids(id) + where m.id = ids.id; + + -- Upsert today's rollup row per memory — one INSERT, conflict-aggregated, + -- same shape as the update above. `current_date` is UTC-anchored the same + -- way `usage_events`' other date_trunc'd reads are (`at time zone 'UTC'`), + -- so a rollup day boundary agrees with every other UTC-bucketed series in + -- this schema. + insert into memory_read_daily (memory_id, day, read_kind, count) + select ids.id, (now() at time zone 'UTC')::date, p_read_kind, 1 + from unnest(p_memory_ids) as ids(id) + on conflict (memory_id, day, read_kind) + do update set count = memory_read_daily.count + 1; +exception + when others then + -- Never let a counter write break the read it is measuring. + return; +end; +$$; + +comment on function lorekit_record_memory_reads(uuid[], text, text) is + 'Increments memories.read_count/.last_read_at and today''s memory_read_daily + row (UTC day, keyed by read_kind) for every memory id a read call actually + returned, in ONE statement regardless of array size. Also sets + memories.last_opened_at (00099) and increments memories.opened_count (00104) + when read_kind = targeted AND client in (mcp, cli) -- an agent deliberately + reaching for this one lesson, as opposed to it riding along in a page. Those + two move together off one shared gate so a count and its timestamp can never + disagree. Swallows its own errors: a counter must never break the read it is + measuring.'; + +-- ── 6. surface the counters on the list RPC ─────────────────────────────── +-- `lorekit_memory_list` returned NONE of the four read counters, so the lesson +-- CARD could not show any of them and comparing two lessons meant opening two +-- detail sheets. Adding them to `returns table` is a return-TYPE change, so the +-- function must be dropped first; the parameter list is unchanged, so the drop +-- signature is 00101's verbatim. +drop function if exists lorekit_memory_list( + uuid, boolean, text, text, text, text, timestamptz, timestamptz, timestamptz, + timestamptz, text[], text, text[], text, text[], text, text[], text, text[], + text, text[], text, text[], text, text[], text, text[], text, text, + timestamptz, uuid, integer, text[], text, uuid[], integer, integer, integer, + integer +); + +create or replace function lorekit_memory_list( + p_user_id uuid, + p_archived boolean default false, + p_scope text default null, + p_key text default null, + p_key_prefix text default null, + p_q text default null, + p_created_since timestamptz default null, + p_created_until timestamptz default null, + p_expires_after timestamptz default null, + p_expires_on_or_before timestamptz default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in', + p_owner text[] default null, + p_owner_mode text default 'in', + p_sort text default 'updated_at', + p_cursor_ts timestamptz default null, + p_cursor_id uuid default null, + p_limit integer default 51, + p_key_scopes text[] default '{}', + p_key_org_access text default 'all', + p_key_org_ids uuid[] default '{}', + p_min_age_days integer default null, + p_unseen_days integer default null, + p_max_seen_count integer default null, + p_max_read_count integer default null +) +returns table ( + id uuid, + scope text, + key text, + value text, + tags text[], + source_agent text, + trigger text, + created_at timestamptz, + updated_at timestamptz, + expires_at timestamptz, + archived_at timestamptz, + origin_repo text, + origin_branch text, + origin_commit text, + origin_pr integer, + kind text, + host text, + seen_count integer, + read_count integer, + opened_count integer, + last_read_at timestamptz, + last_opened_at timestamptz, + org_id uuid, + created_by uuid, + updated_by uuid, + org_name text, + org_slug text, + total_count integer +) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + v_origin_pr integer[] := ( + select array_agg(x::integer) + from unnest(coalesce(p_origin_pr, '{}'::text[])) as x + where x ~ '^0*[0-9]{1,9}$' + ); + v_sort text := case when p_sort = 'created_at' then 'created_at' else 'updated_at' end; +begin + return query execute format($q$ + select + m.id, m.scope, m.key, m.value, m.tags, m.source_agent, m.trigger, + m.created_at, m.updated_at, m.expires_at, m.archived_at, + m.origin_repo, m.origin_branch, m.origin_commit, m.origin_pr, + m.kind, m.host, m.seen_count, + m.read_count, m.opened_count, m.last_read_at, m.last_opened_at, + m.org_id, m.created_by, m.updated_by, + o.name as org_name, o.slug as org_slug, + (count(*) over ())::integer as total_count + from memories m + left join orgs o on o.id = m.org_id + where ( + ($1 is null and auth.role() = 'service_role') + or m.user_id = $1 + or m.org_id in (select lorekit_member_org_ids($1)) + ) + and lorekit_api_token_scope_allowed($32, m.scope) + and lorekit_api_token_org_allowed($33, $34, m.org_id) + and ( + case + when $2 then m.archived_at is not null + else m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + end + ) + and ($3 is null or m.scope = $3) + and ($4 is null or m.key = $4) + and ($5 is null or m.key ilike $5 || '%%') + and ($6 is null or m.key ilike '%%' || $6 || '%%' or m.value ilike '%%' || $6 || '%%') + and ($7 is null or m.created_at >= $7) + and ($8 is null or m.created_at < $8) + and ($9 is null or m.expires_at > $9) + and ($10 is null or m.expires_at <= $10) + and lorekit_match_tags(m.tags, $11, $12) + and lorekit_match_text(m.source_agent, $13, $14) + and lorekit_match_text(m.trigger, $15, $16) + and lorekit_match_text(m.kind, $17, $18) + and lorekit_match_text(m.host, $19, $20) + and lorekit_match_text(m.origin_repo, $21, $22) + and lorekit_match_text(m.origin_branch, $23, $24) + and lorekit_match_int (m.origin_pr, $25, $26) + and ($27 is null or case coalesce($28, 'in') + when 'nin' then ( + (case when m.org_id is null then 'personal' else o.slug end) is not null + and (case when m.org_id is null then 'personal' else o.slug end) <> all($27) + ) + else ( + ('personal' = any($27) and m.org_id is null) + or (m.org_id is not null and o.slug = any($27)) + ) + end) + and ($35 is null or m.created_at <= now() - ($35 * interval '1 day')) + and ($36 is null or coalesce(m.last_opened_at, m.created_at) <= now() - ($36 * interval '1 day')) + and ($37 is null or m.seen_count <= $37) + and ($38 is null or m.read_count <= $38) + and ( + $29 is null + or m.%1$I < $29 + or (m.%1$I = $29 and m.id < $30) + ) + order by m.%1$I desc, m.id desc + limit $31 + $q$, v_sort) + using + v_actor, coalesce(p_archived, false), p_scope, p_key, p_key_prefix, p_q, + p_created_since, p_created_until, p_expires_after, p_expires_on_or_before, + p_tags, p_tags_mode, + p_source_agent, p_source_agent_mode, + p_trigger, p_trigger_mode, + p_kind, p_kind_mode, + p_host, p_host_mode, + p_origin_repo, p_origin_repo_mode, + p_origin_branch, p_origin_branch_mode, + v_origin_pr, p_origin_pr_mode, + p_owner, p_owner_mode, + p_cursor_ts, p_cursor_id, + greatest(coalesce(p_limit, 51), 1), + coalesce(p_key_scopes, '{}'::text[]), + coalesce(p_key_org_access, 'all'), + coalesce(p_key_org_ids, '{}'::uuid[]), + p_min_age_days, p_unseen_days, p_max_seen_count, p_max_read_count; +end; +$$; + +grant execute on function lorekit_memory_list( + uuid, boolean, text, text, text, text, timestamptz, timestamptz, timestamptz, + timestamptz, text[], text, text[], text, text[], text, text[], text, text[], + text, text[], text, text[], text, text[], text, text[], text, text, + timestamptz, uuid, integer, text[], text, uuid[], integer, integer, integer, + integer +) to authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00104', 'opened_count_pull_through') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00105_max_opened_count_condition.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- 00105 — `max_opened_count`: the retention condition whose 0 means something. +-- +-- WHY max_read_count IS NOT ENOUGH. 00101 added `max_read_count` over +-- `read_count`, which counts EVERY read including bulk list/search +-- ride-alongs. Measured on a live 2,772-memory store the day this was written, +-- that makes the filter almost unusable at the low end: +-- +-- max_read_count <= 5 -> 0 memories +-- max_read_count <= 26 -> the first non-empty result +-- max_read_count <= 50 -> roughly half the store +-- max_read_count <= 400 -> essentially everything +-- +-- Its whole usable range is a narrow band in the middle, and the grooming UI +-- suggests `0` — a value that can never match anything a `memory.list` has ever +-- paged over, which is every lesson in an active scope. Worse, what it ranks on +-- is SCOPE BREADTH: a `global` lesson rides along in every session and a +-- `branch` lesson almost never, so "read at most N times" selects narrow scopes +-- and calls them unused. +-- +-- `opened_count` (00104) counts ONLY a deliberate agent fetch of this exact +-- lesson. `max_opened_count => 0` therefore means exactly what a reader +-- expects — "no agent has ever reached for this" — and it means the same thing +-- for a `global` lesson as for a `branch` one. It is the condition +-- `max_read_count` was reaching for. +-- +-- BOTH ARE KEPT, and `max_read_count` is not deprecated here. It answers a real +-- and different question — "what is this lesson COSTING me in context" — which +-- is the other axis of the delivered x chosen grid. Pairing them is what makes +-- the interesting quadrant expressible for the first time: +-- +-- max_opened_count => 0 "nothing ever chose it" +-- (with no max_read_count) ... at any delivery volume +-- +-- and, for the prune list that actually saves context, a policy that leaves +-- `max_read_count` unset while setting `max_opened_count => 0` catches the +-- heavily-delivered, never-chosen lessons that `max_read_count` structurally +-- cannot reach. +-- +-- NO CUTOVER CAVEAT, unlike 00101. `max_read_count` had to warn that a +-- long-lived lesson can show a low count it never earned, because `read_count` +-- started at 0 when 00084 shipped. `opened_count` was BACKFILLED from +-- `memory_read_daily` in 00104, so it is exact over the whole history the +-- rollup holds and no lesson looks falsely unopened. +-- +-- SIGNATURES follow 00101 exactly: `p_max_opened_count` is APPENDED last so +-- positional callers keep working, and every function whose signature changes +-- is DROPPED first — `create or replace` would leave a second overload behind +-- and make every call ambiguous. `lorekit_policy_update` takes a jsonb patch, +-- so it needs no drop; `lorekit_groom_sweep` takes no arguments. +-- +-- The index it needs already exists: `memories_user_opened_count_idx` (00104). +-- ═════════════════════════════════════════════════════════════════════════ + +-- ── the policy column ───────────────────────────────────────────────────── +alter table retention_policies add column if not exists max_opened_count integer; + +alter table retention_policies drop constraint if exists retention_policies_max_opened_count_non_negative; +alter table retention_policies add constraint retention_policies_max_opened_count_non_negative + check (max_opened_count is null or max_opened_count >= 0); + +comment on column retention_policies.max_opened_count is + 'Match only lessons an agent has DELIBERATELY fetched at most this many times + (memories.opened_count, 00104). Unlike max_read_count (00101) a bulk + list/search ride-along does not count, so 0 means "nothing ever chose this" + rather than "this lesson happens to live in a narrow scope". NULL = + unconstrained.'; + +-- ── 1. lorekit_groom_candidates ─────────────────────────────────────────── +drop function if exists lorekit_groom_candidates( + uuid, text, integer, integer, integer, + text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, integer +); + +create or replace function lorekit_groom_candidates( + p_user_id uuid, + p_scope text, + p_min_age_days integer default null, + p_unseen_days integer default null, + p_max_seen_count integer default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in', + p_max_read_count integer default null, + p_max_opened_count integer default null +) +returns table (id uuid, scope text, key text) +language sql +stable +security definer +set search_path = public +as $$ + select m.id, m.scope, m.key + from memories m + where m.user_id = p_user_id + and m.archived_at is null + and m.protected = false + and ( + p_scope = 'global' + or m.scope = p_scope + or ( + p_scope like 'repo::%' + and starts_with(m.scope, 'branch::' || substring(p_scope from 7) || '::') + ) + ) + and (p_min_age_days is null or m.created_at <= now() - (p_min_age_days * interval '1 day')) + and (p_unseen_days is null or coalesce(m.last_opened_at, m.created_at) <= now() - (p_unseen_days * interval '1 day')) + and (p_max_seen_count is null or m.seen_count <= p_max_seen_count) + and (p_max_read_count is null or m.read_count <= p_max_read_count) + and (p_max_opened_count is null or m.opened_count <= p_max_opened_count) + and lorekit_match_tags(m.tags, p_tags, p_tags_mode) + and lorekit_match_text(m.source_agent, p_source_agent, p_source_agent_mode) + and lorekit_match_text(m.trigger, p_trigger, p_trigger_mode) + and lorekit_match_text(m.kind, p_kind, p_kind_mode) + and lorekit_match_text(m.host, p_host, p_host_mode) + and lorekit_match_text(m.origin_repo, p_origin_repo, p_origin_repo_mode) + and lorekit_match_text(m.origin_branch, p_origin_branch, p_origin_branch_mode) + and lorekit_match_text(m.origin_pr::text, p_origin_pr, p_origin_pr_mode) + order by m.scope, m.key; +$$; + +grant execute on function lorekit_groom_candidates( + uuid, text, integer, integer, integer, + text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, integer, integer +) to authenticated, service_role; + +-- ── 2. lorekit_groom_run ────────────────────────────────────────────────── +drop function if exists lorekit_groom_run( + uuid, text, integer, integer, integer, + text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, integer +); + +create or replace function lorekit_groom_run( + p_user_id uuid, + p_scope text, + p_min_age_days integer default null, + p_unseen_days integer default null, + p_max_seen_count integer default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in', + p_max_read_count integer default null, + p_max_opened_count integer default null +) +returns table (archived integer, keys jsonb) +language plpgsql +security definer +set search_path = public +as $$ +declare + v_ids uuid[]; + v_keys jsonb; + v_count integer := 0; +begin + select coalesce(array_agg(c.id), '{}'), + coalesce(jsonb_agg(jsonb_build_object('scope', c.scope, 'key', c.key)), '[]'::jsonb) + into v_ids, v_keys + from lorekit_groom_candidates( + p_user_id, p_scope, p_min_age_days, p_unseen_days, p_max_seen_count, + p_tags, p_tags_mode, p_source_agent, p_source_agent_mode, + p_trigger, p_trigger_mode, p_kind, p_kind_mode, p_host, p_host_mode, + p_origin_repo, p_origin_repo_mode, p_origin_branch, p_origin_branch_mode, + p_origin_pr, p_origin_pr_mode, p_max_read_count, p_max_opened_count + ) c; + + if array_length(v_ids, 1) is null then + return query select 0, '[]'::jsonb; + return; + end if; + + update memories m + set archived_at = now() + where m.id = any(v_ids) + and m.archived_at is null; + get diagnostics v_count = row_count; + + return query select v_count, v_keys; +end; +$$; + +grant execute on function lorekit_groom_run( + uuid, text, integer, integer, integer, + text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, integer, integer +) to authenticated, service_role; + +-- ── 3. lorekit_groom_sweep ──────────────────────────────────────────────── +-- No signature change; recreated only to pass the new policy column through. +create or replace function lorekit_groom_sweep() +returns integer +language plpgsql +security definer +set search_path = public +as $$ +declare + v_policy retention_policies%rowtype; + v_result record; + v_total integer := 0; +begin + for v_policy in + select * from retention_policies where mode = 'auto' and enabled = true + loop + select * into v_result + from lorekit_groom_run( + v_policy.user_id, v_policy.scope, + v_policy.min_age_days, v_policy.unseen_days, v_policy.max_seen_count, + v_policy.tags, v_policy.tags_mode, + v_policy.source_agent, v_policy.source_agent_mode, + v_policy.trigger, v_policy.trigger_mode, + v_policy.kind, v_policy.kind_mode, + v_policy.host, v_policy.host_mode, + v_policy.origin_repo, v_policy.origin_repo_mode, + v_policy.origin_branch, v_policy.origin_branch_mode, + v_policy.origin_pr, v_policy.origin_pr_mode, + v_policy.max_read_count, v_policy.max_opened_count + ); + v_total := v_total + coalesce(v_result.archived, 0); + end loop; + + return v_total; +end; +$$; + +grant execute on function lorekit_groom_sweep() to service_role; + +-- ── 4. lorekit_policy_create ────────────────────────────────────────────── +drop function if exists lorekit_policy_create( + uuid, text, text, text, boolean, integer, integer, integer, + text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, integer +); + +create or replace function lorekit_policy_create( + p_user_id uuid, + p_scope text, + p_name text, + p_mode text default 'review', + p_enabled boolean default false, + p_min_age_days integer default null, + p_unseen_days integer default null, + p_max_seen_count integer default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in', + p_max_read_count integer default null, + p_max_opened_count integer default null +) +returns setof retention_policies +language sql +security definer +set search_path = public +as $$ + insert into retention_policies ( + user_id, scope, name, mode, enabled, min_age_days, unseen_days, max_seen_count, + max_read_count, max_opened_count, + tags, tags_mode, source_agent, source_agent_mode, trigger, trigger_mode, + kind, kind_mode, host, host_mode, origin_repo, origin_repo_mode, + origin_branch, origin_branch_mode, origin_pr, origin_pr_mode + ) + values ( + p_user_id, p_scope, p_name, p_mode, p_enabled, p_min_age_days, p_unseen_days, p_max_seen_count, + p_max_read_count, p_max_opened_count, + p_tags, p_tags_mode, p_source_agent, p_source_agent_mode, p_trigger, p_trigger_mode, + p_kind, p_kind_mode, p_host, p_host_mode, p_origin_repo, p_origin_repo_mode, + p_origin_branch, p_origin_branch_mode, p_origin_pr, p_origin_pr_mode + ) + returning *; +$$; + +grant execute on function lorekit_policy_create( + uuid, text, text, text, boolean, integer, integer, integer, + text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, integer, integer +) to authenticated, service_role; + +-- ── 5. lorekit_policy_update ────────────────────────────────────────────── +-- jsonb patch, so the signature is unchanged and no drop is needed. One more +-- `case when p_patch ? ...` line, matching every other condition: an absent key +-- leaves the column alone, an explicit null clears it. +create or replace function lorekit_policy_update( + p_user_id uuid, + p_id uuid, + p_patch jsonb +) +returns setof retention_policies +language sql +security definer +set search_path = public +as $$ + update retention_policies + set name = case when p_patch ? 'name' then p_patch ->> 'name' else name end, + mode = case when p_patch ? 'mode' then p_patch ->> 'mode' else mode end, + enabled = case when p_patch ? 'enabled' then (p_patch ->> 'enabled')::boolean else enabled end, + min_age_days = case when p_patch ? 'min_age_days' then (p_patch ->> 'min_age_days')::integer else min_age_days end, + unseen_days = case when p_patch ? 'unseen_days' then (p_patch ->> 'unseen_days')::integer else unseen_days end, + max_seen_count = case when p_patch ? 'max_seen_count' then (p_patch ->> 'max_seen_count')::integer else max_seen_count end, + max_read_count = case when p_patch ? 'max_read_count' then (p_patch ->> 'max_read_count')::integer else max_read_count end, + max_opened_count = case when p_patch ? 'max_opened_count' then (p_patch ->> 'max_opened_count')::integer else max_opened_count end, + tags = case when p_patch ? 'tags' then (case when p_patch -> 'tags' = 'null'::jsonb then null else array(select jsonb_array_elements_text(p_patch -> 'tags')) end) else tags end, + tags_mode = case when p_patch ? 'tags_mode' then p_patch ->> 'tags_mode' else tags_mode end, + source_agent = case when p_patch ? 'source_agent' then (case when p_patch -> 'source_agent' = 'null'::jsonb then null else array(select jsonb_array_elements_text(p_patch -> 'source_agent')) end) else source_agent end, + source_agent_mode = case when p_patch ? 'source_agent_mode' then p_patch ->> 'source_agent_mode' else source_agent_mode end, + trigger = case when p_patch ? 'trigger' then (case when p_patch -> 'trigger' = 'null'::jsonb then null else array(select jsonb_array_elements_text(p_patch -> 'trigger')) end) else trigger end, + trigger_mode = case when p_patch ? 'trigger_mode' then p_patch ->> 'trigger_mode' else trigger_mode end, + kind = case when p_patch ? 'kind' then (case when p_patch -> 'kind' = 'null'::jsonb then null else array(select jsonb_array_elements_text(p_patch -> 'kind')) end) else kind end, + kind_mode = case when p_patch ? 'kind_mode' then p_patch ->> 'kind_mode' else kind_mode end, + host = case when p_patch ? 'host' then (case when p_patch -> 'host' = 'null'::jsonb then null else array(select jsonb_array_elements_text(p_patch -> 'host')) end) else host end, + host_mode = case when p_patch ? 'host_mode' then p_patch ->> 'host_mode' else host_mode end, + origin_repo = case when p_patch ? 'origin_repo' then (case when p_patch -> 'origin_repo' = 'null'::jsonb then null else array(select jsonb_array_elements_text(p_patch -> 'origin_repo')) end) else origin_repo end, + origin_repo_mode = case when p_patch ? 'origin_repo_mode' then p_patch ->> 'origin_repo_mode' else origin_repo_mode end, + origin_branch = case when p_patch ? 'origin_branch' then (case when p_patch -> 'origin_branch' = 'null'::jsonb then null else array(select jsonb_array_elements_text(p_patch -> 'origin_branch')) end) else origin_branch end, + origin_branch_mode = case when p_patch ? 'origin_branch_mode' then p_patch ->> 'origin_branch_mode' else origin_branch_mode end, + origin_pr = case when p_patch ? 'origin_pr' then (case when p_patch -> 'origin_pr' = 'null'::jsonb then null else array(select jsonb_array_elements_text(p_patch -> 'origin_pr')) end) else origin_pr end, + origin_pr_mode = case when p_patch ? 'origin_pr_mode' then p_patch ->> 'origin_pr_mode' else origin_pr_mode end + where id = p_id and user_id = p_user_id + returning *; +$$; + +grant execute on function lorekit_policy_update(uuid, uuid, jsonb) to authenticated, service_role; + +-- ── 6. lorekit_memory_list ──────────────────────────────────────────────── +-- The Explorer's retention filter bar reads through this, so the condition has +-- to exist on both surfaces or the preview and the policy disagree. Parameter +-- change, so the 00104 definition is dropped by its exact argument list first. +drop function if exists lorekit_memory_list( + uuid, boolean, text, text, text, text, timestamptz, timestamptz, timestamptz, + timestamptz, text[], text, text[], text, text[], text, text[], text, text[], + text, text[], text, text[], text, text[], text, text[], text, text, + timestamptz, uuid, integer, text[], text, uuid[], integer, integer, integer, + integer +); + +create or replace function lorekit_memory_list( + p_user_id uuid, + p_archived boolean default false, + p_scope text default null, + p_key text default null, + p_key_prefix text default null, + p_q text default null, + p_created_since timestamptz default null, + p_created_until timestamptz default null, + p_expires_after timestamptz default null, + p_expires_on_or_before timestamptz default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in', + p_owner text[] default null, + p_owner_mode text default 'in', + p_sort text default 'updated_at', + p_cursor_ts timestamptz default null, + p_cursor_id uuid default null, + p_limit integer default 51, + p_key_scopes text[] default '{}', + p_key_org_access text default 'all', + p_key_org_ids uuid[] default '{}', + p_min_age_days integer default null, + p_unseen_days integer default null, + p_max_seen_count integer default null, + p_max_read_count integer default null, + p_max_opened_count integer default null +) +returns table ( + id uuid, + scope text, + key text, + value text, + tags text[], + source_agent text, + trigger text, + created_at timestamptz, + updated_at timestamptz, + expires_at timestamptz, + archived_at timestamptz, + origin_repo text, + origin_branch text, + origin_commit text, + origin_pr integer, + kind text, + host text, + seen_count integer, + read_count integer, + opened_count integer, + last_read_at timestamptz, + last_opened_at timestamptz, + org_id uuid, + created_by uuid, + updated_by uuid, + org_name text, + org_slug text, + total_count integer +) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + v_origin_pr integer[] := ( + select array_agg(x::integer) + from unnest(coalesce(p_origin_pr, '{}'::text[])) as x + where x ~ '^0*[0-9]{1,9}$' + ); + v_sort text := case when p_sort = 'created_at' then 'created_at' else 'updated_at' end; +begin + return query execute format($q$ + select + m.id, m.scope, m.key, m.value, m.tags, m.source_agent, m.trigger, + m.created_at, m.updated_at, m.expires_at, m.archived_at, + m.origin_repo, m.origin_branch, m.origin_commit, m.origin_pr, + m.kind, m.host, m.seen_count, + m.read_count, m.opened_count, m.last_read_at, m.last_opened_at, + m.org_id, m.created_by, m.updated_by, + o.name as org_name, o.slug as org_slug, + (count(*) over ())::integer as total_count + from memories m + left join orgs o on o.id = m.org_id + where ( + ($1 is null and auth.role() = 'service_role') + or m.user_id = $1 + or m.org_id in (select lorekit_member_org_ids($1)) + ) + and lorekit_api_token_scope_allowed($32, m.scope) + and lorekit_api_token_org_allowed($33, $34, m.org_id) + and ( + case + when $2 then m.archived_at is not null + else m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + end + ) + and ($3 is null or m.scope = $3) + and ($4 is null or m.key = $4) + and ($5 is null or m.key ilike $5 || '%%') + and ($6 is null or m.key ilike '%%' || $6 || '%%' or m.value ilike '%%' || $6 || '%%') + and ($7 is null or m.created_at >= $7) + and ($8 is null or m.created_at < $8) + and ($9 is null or m.expires_at > $9) + and ($10 is null or m.expires_at <= $10) + and lorekit_match_tags(m.tags, $11, $12) + and lorekit_match_text(m.source_agent, $13, $14) + and lorekit_match_text(m.trigger, $15, $16) + and lorekit_match_text(m.kind, $17, $18) + and lorekit_match_text(m.host, $19, $20) + and lorekit_match_text(m.origin_repo, $21, $22) + and lorekit_match_text(m.origin_branch, $23, $24) + and lorekit_match_int (m.origin_pr, $25, $26) + and ($27 is null or case coalesce($28, 'in') + when 'nin' then ( + (case when m.org_id is null then 'personal' else o.slug end) is not null + and (case when m.org_id is null then 'personal' else o.slug end) <> all($27) + ) + else ( + ('personal' = any($27) and m.org_id is null) + or (m.org_id is not null and o.slug = any($27)) + ) + end) + and ($35 is null or m.created_at <= now() - ($35 * interval '1 day')) + and ($36 is null or coalesce(m.last_opened_at, m.created_at) <= now() - ($36 * interval '1 day')) + and ($37 is null or m.seen_count <= $37) + and ($38 is null or m.read_count <= $38) + and ($39 is null or m.opened_count <= $39) + and ( + $29 is null + or m.%1$I < $29 + or (m.%1$I = $29 and m.id < $30) + ) + order by m.%1$I desc, m.id desc + limit $31 + $q$, v_sort) + using + v_actor, coalesce(p_archived, false), p_scope, p_key, p_key_prefix, p_q, + p_created_since, p_created_until, p_expires_after, p_expires_on_or_before, + p_tags, p_tags_mode, + p_source_agent, p_source_agent_mode, + p_trigger, p_trigger_mode, + p_kind, p_kind_mode, + p_host, p_host_mode, + p_origin_repo, p_origin_repo_mode, + p_origin_branch, p_origin_branch_mode, + v_origin_pr, p_origin_pr_mode, + p_owner, p_owner_mode, + p_cursor_ts, p_cursor_id, + greatest(coalesce(p_limit, 51), 1), + coalesce(p_key_scopes, '{}'::text[]), + coalesce(p_key_org_access, 'all'), + coalesce(p_key_org_ids, '{}'::uuid[]), + p_min_age_days, p_unseen_days, p_max_seen_count, p_max_read_count, + p_max_opened_count; +end; +$$; + +grant execute on function lorekit_memory_list( + uuid, boolean, text, text, text, text, timestamptz, timestamptz, timestamptz, + timestamptz, text[], text, text[], text, text[], text, text[], text, text[], + text, text[], text, text[], text, text[], text, text[], text, text, + timestamptz, uuid, integer, text[], text, uuid[], integer, integer, integer, + integer, integer +) to authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00105', 'max_opened_count_condition') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00106_memory_utility_grid.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- 00106 — the delivered × chosen grid, and what delivery costs. +-- +-- WHAT THIS REPLACES. `lorekit_memory_read_ranking` (00085) ranks by +-- `read_count` alone, and 99.80% of recorded reads are bulk ride-alongs in a +-- `memory.list`/`memory.search` page. So the "cold" end of that ranking — the +-- end the Insights page nominates for pruning — is populated by NARROW SCOPES, +-- not by unused lore: a `branch` lesson is delivered a handful of times however +-- useful it is, while a `global` one is delivered on every session however +-- useless. Ranking two lessons by a number that mostly encodes their scope is +-- the defect; dividing by the deliveries cancels it, because scope breadth +-- appears in both halves. +-- +-- 00085 IS NOT DROPPED. `/read-ranking` still answers "what is read most", a +-- real question, and it is a live REST route. This adds the grid beside it. +-- +-- TWO FUNCTIONS, TWO WINDOWS, DELIBERATELY. +-- +-- `lorekit_memory_utility_census` counts, per quadrant, over the LIFETIME +-- counters on `memories`. It must agree with the per-lesson chip the +-- dashboard renders from those same two columns, so it reads the same source. +-- +-- `lorekit_memory_delivery_cost` sums over `memory_read_daily`, which is the +-- only source that can be WINDOWED ("this month"). It reports reads and an +-- estimated token volume. +-- +-- Mixing them would produce a page where the headline and the grid silently +-- describe different periods. Each caller captions its own. +-- +-- NO THRESHOLD IS HARDCODED HERE. `p_min_deliveries` / `p_min_age_days` / +-- `p_chosen_pull_through` / `p_broad_reach` arrive as parameters from +-- `LESSON_UTILITY_THRESHOLDS` in `@lorekit/schemas`, which is also what the +-- TypeScript chip reads. This is the `lorekit_get_limit` posture: the numbers +-- live in ONE place and the SQL encodes only the SHAPE of the rule (four +-- quadrants from two booleans, plus an evidence floor). A SQL default is +-- supplied so a hand-run query is not unusable, but it is a fallback, never +-- the authority — a drift between the two would put a lesson in one quadrant +-- on the card and another in the census. +-- +-- TOKENS ARE ESTIMATED, never tokenized: `length(value) / 4`. The headline is +-- an order of magnitude, and shipping a BPE tokenizer into an edge function to +-- sharpen a number nobody acts on to the digit is the wrong trade. Every +-- surface that renders it says "estimated". +-- +-- VISIBILITY is the same org-shared + own-rows predicate every per-user +-- analytics RPC in this family uses (`lorekit_memory_scopes`, +-- `lorekit_memory_read_ranking`, `lorekit_read_activity`), with the same +-- service-role + NULL-actor escape hatch and the same +-- `lorekit_api_token_scope_allowed` narrowing for a scoped key (00068/00069). +-- +-- INDEXES: `memories_user_read_count_idx` (00085) and +-- `memories_user_opened_count_idx` (00104) already cover the census scan; +-- `memory_read_daily`'s primary key covers the cost sum's join. +-- ═════════════════════════════════════════════════════════════════════════ + +-- ── the quadrant expression, in ONE place ──────────────────────────────── +-- +-- Both functions below need "which quadrant is this row in", and the census +-- needs it twice (count and filter). An immutable helper keeps the rule from +-- being written out three times and drifting between them — the same reason +-- `isGroomCandidate` is one pure function rather than a predicate inlined per +-- caller. +-- +-- Returns the SAME five names `LessonUtilitySchema` declares, so a rename on +-- either side is caught by `migrations.test.sql` §104 rather than silently +-- producing a quadrant no client knows how to render. +create or replace function lorekit_lesson_utility( + p_read_count integer, + p_opened_count integer, + p_created_at timestamptz, + p_min_deliveries integer default 10, + p_min_age_days integer default 7, + p_chosen_pull_through numeric default 0.02, + p_broad_reach integer default 100, + p_now timestamptz default now() +) +returns text +language sql +immutable +as $$ + select case + -- The evidence floor comes FIRST, and both halves of it are needed: a + -- lesson delivered 5,000 times in its first day has volume but no chance + -- to have been chosen, and a 400-day-old lesson delivered 3 times has age + -- but no denominator. Either alone would hand out a confident verdict the + -- data cannot support. + when p_created_at > p_now - make_interval(days => p_min_age_days) then 'unproven' + when coalesce(p_read_count, 0) < p_min_deliveries then 'unproven' + when coalesce(p_opened_count, 0)::numeric / nullif(p_read_count, 0) >= p_chosen_pull_through then + case when p_read_count >= p_broad_reach then 'load-bearing' else 'specialist' end + else + case when p_read_count >= p_broad_reach then 'noise-tax' else 'dormant' end + end; +$$; + +comment on function lorekit_lesson_utility(integer, integer, timestamptz, integer, integer, numeric, integer, timestamptz) is + 'Where one lesson sits on the delivered x chosen grid: load-bearing / + specialist / noise-tax / dormant / unproven. Pull-through is + opened_count / read_count -- a proper fraction, so scope breadth cancels. + Thresholds are PARAMETERS (LESSON_UTILITY_THRESHOLDS in @lorekit/schemas is + the authority); the defaults here are a fallback for a hand-run query, not + a second source of truth.'; + +-- ── the census ─────────────────────────────────────────────────────────── +create or replace function lorekit_memory_utility_census( + p_user_id uuid, + p_scope text default null, + p_key_scopes text[] default '{}', + p_min_deliveries integer default 10, + p_min_age_days integer default 7, + p_chosen_pull_through numeric default 0.02, + p_broad_reach integer default 100 +) +returns table ( + utility text, + n bigint +) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column declare - v_id uuid; + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; begin - update memories - set archived_at = now() - where user_id = p_user_id - and scope = p_scope - and key = p_key - and archived_at is null - returning id into v_id; + return query + -- LEFT JOIN off the five names rather than grouping the rows: a quadrant + -- with no members must come back as 0, not be absent. An absent key reads + -- to a client as "not measured", and "you have no noise tax" is a very + -- different answer from "we did not look". + select q.utility, coalesce(c.n, 0)::bigint + from unnest(array['load-bearing', 'specialist', 'noise-tax', 'dormant', 'unproven']) as q(utility) + left join ( + select lorekit_lesson_utility( + m.read_count, m.opened_count, m.created_at, + p_min_deliveries, p_min_age_days, p_chosen_pull_through, p_broad_reach, now() + ) as utility, + count(*) as n + from memories m + where ( + (v_actor is null and auth.role() = 'service_role') + or m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) + ) + and m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + and (p_scope is null or m.scope = p_scope) + and lorekit_api_token_scope_allowed(p_key_scopes, m.scope) + group by 1 + ) c on c.utility = q.utility; +end; +$$; - return v_id; +revoke execute on function lorekit_memory_utility_census(uuid, text, text[], integer, integer, numeric, integer) from public, anon; +grant execute on function lorekit_memory_utility_census(uuid, text, text[], integer, integer, numeric, integer) to authenticated, service_role; + +comment on function lorekit_memory_utility_census(uuid, text, text[], integer, integer, numeric, integer) is + 'How many active lessons sit in each quadrant of the delivered x chosen grid, + account-wide and ALL-TIME (the lifetime counters on memories, the same two + columns the per-lesson chip reads, so the two cannot disagree). Every one of + the five names is always returned, 0 included -- an absent key would read as + "not measured".'; + +-- ── the rows for one quadrant ──────────────────────────────────────────── +create or replace function lorekit_memory_utility_rows( + p_user_id uuid, + p_utility text, + p_scope text default null, + p_limit integer default 20, + p_key_scopes text[] default '{}', + p_min_deliveries integer default 10, + p_min_age_days integer default 7, + p_chosen_pull_through numeric default 0.02, + p_broad_reach integer default 100 +) +returns table ( + id uuid, + scope text, + key text, + read_count integer, + opened_count integer, + last_opened_at timestamptz, + created_at timestamptz +) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + v_limit integer := least(greatest(coalesce(p_limit, 20), 1), 100); +begin + if p_utility is null + or p_utility not in ('load-bearing', 'specialist', 'noise-tax', 'dormant', 'unproven') then + raise exception 'invalid utility %, expected one of load-bearing, specialist, noise-tax, dormant, unproven', p_utility + using errcode = '22023'; + end if; + + return query + select m.id, m.scope, m.key, m.read_count, m.opened_count, m.last_opened_at, m.created_at + from memories m + where ( + (v_actor is null and auth.role() = 'service_role') + or m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) + ) + and m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + and (p_scope is null or m.scope = p_scope) + and lorekit_api_token_scope_allowed(p_key_scopes, m.scope) + and lorekit_lesson_utility( + m.read_count, m.opened_count, m.created_at, + p_min_deliveries, p_min_age_days, p_chosen_pull_through, p_broad_reach, now() + ) = p_utility + -- Most-delivered first WITHIN a quadrant, because that is the order the + -- action wants: the noise-tax row costing the most context is the one to + -- prune first, and the load-bearing row reaching the most sessions is the + -- one to promote first. `id desc` is the stable tiebreak every other + -- ranked read in this family uses. + order by m.read_count desc, m.opened_count desc, m.id desc + limit v_limit; +end; +$$; + +revoke execute on function lorekit_memory_utility_rows(uuid, text, text, integer, text[], integer, integer, numeric, integer) from public, anon; +grant execute on function lorekit_memory_utility_rows(uuid, text, text, integer, text[], integer, integer, numeric, integer) to authenticated, service_role; + +comment on function lorekit_memory_utility_rows(uuid, text, text, integer, text[], integer, integer, numeric, integer) is + 'The active lessons in one quadrant of the delivered x chosen grid, most- + delivered first. Same visibility and scoped-key narrowing as + lorekit_memory_read_ranking. p_limit is clamped to [1, 100]; an unknown + quadrant name raises 22023 rather than returning nothing.'; + +-- ── what delivery cost, over a window ──────────────────────────────────── +create or replace function lorekit_memory_delivery_cost( + p_user_id uuid, + p_since timestamptz default null, + p_until timestamptz default null, + p_scope text default null, + p_key_scopes text[] default '{}' +) +returns table ( + delivered_reads bigint, + chosen_reads bigint, + delivered_tokens bigint, + chosen_tokens bigint +) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; +begin + return query + -- `coalesce` on the aggregates, not on the row: `sum` over no rows is + -- NULL, and a client reading `delivered_tokens: null` as 0 is the kind of + -- accidental agreement that breaks the first time a caller is stricter. + -- An account with no recorded reads gets four honest zeroes. + select + coalesce(sum(d.count), 0)::bigint, + coalesce(sum(d.count) filter (where d.read_kind = 'targeted'), 0)::bigint, + coalesce(sum(d.count * ceil(length(m.value) / 4.0)), 0)::bigint, + coalesce(sum(d.count * ceil(length(m.value) / 4.0)) filter (where d.read_kind = 'targeted'), 0)::bigint + from memory_read_daily d + join memories m on m.id = d.memory_id + where ( + (v_actor is null and auth.role() = 'service_role') + or m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) + ) + -- Archived lore is deliberately INCLUDED: it was delivered while it was + -- live, and the point of this figure is what the reads actually cost. + -- Excluding it would make the bill drop retroactively every time + -- something is pruned, which is exactly backwards as a feedback signal. + and (p_scope is null or m.scope = p_scope) + and lorekit_api_token_scope_allowed(p_key_scopes, m.scope) + -- HALF-OPEN `[since, until)`, the same asymmetry `/activity` and + -- `/read-activity` use, so two adjacent windows partition the reads + -- instead of both claiming the boundary day. The comparison is at DAY + -- grain because `memory_read_daily` is a daily rollup and no finer + -- answer exists — which makes `until` exclusive of the whole day it + -- names. A caller wanting "through right now" therefore passes NO + -- `until` (the default) rather than `now()`, which would drop today. + and (p_since is null or d.day >= p_since::date) + and (p_until is null or d.day < p_until::date); end; $$; --- 00072 mirrored. The hosted function was recreated with the calling key's --- restriction appended and a `(restored, existed)` return, so the pre-00072 --- `returns uuid` form is DROPPED rather than replaced: PostgREST resolves by --- argument NAME and the edge now sends all six, so leaving the 3-arg overload --- behind would either shadow this one or fail to resolve at all. +revoke execute on function lorekit_memory_delivery_cost(uuid, timestamptz, timestamptz, text, text[]) from public, anon; +grant execute on function lorekit_memory_delivery_cost(uuid, timestamptz, timestamptz, text, text[]) to authenticated, service_role; + +comment on function lorekit_memory_delivery_cost(uuid, timestamptz, timestamptz, text, text[]) is + 'Reads and ESTIMATED tokens of lore delivered over a window, and how much of + that was a deliberate (targeted) fetch. Windowed over memory_read_daily -- + a DIFFERENT source from lorekit_memory_utility_census, which reads the + lifetime counters; each caller must caption its own window. The window is + half-open [since, until) at DAY grain, so p_until excludes the whole day it + names -- pass NULL, not now(), for "through right now". Tokens are + length(value)/4, never tokenized: render them as estimates. Archived lore is + included, because it cost context while it was live.'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00106', 'memory_utility_grid') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00107_memory_citations.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- 00107 — citations: the signal pull-through cannot give. -- --- BYOD is a personal install with no orgs, so `p_key_org_access` / `p_key_org_ids` --- are accepted and inert (same as `memory_delete` below); the scope allowlist is --- enforced, because that half is what scoping a key means here too. -drop function if exists restore_memory(uuid, text, text); +-- WHAT IS STILL MISSING AFTER 00104. `opened_count / read_count` measures +-- SELECTION — of all the times a lesson was surfaced, how often did something +-- deliberately reach for it. That is a real signal and it is the best one +-- available from read telemetry alone, but it systematically under-counts the +-- path lore is mostly consumed by: a lesson INJECTED at SessionStart is already +-- in the agent's context and never needs a second fetch to change what the +-- agent does. Its pull-through can be 0% while it is the single most +-- influential lesson in the store. +-- +-- Every surface built on pull-through says so out loud (the Insights cost line +-- prints "this measures whether lore is looked up on purpose, not whether it +-- changed an outcome"). This migration is the other half: a way for the only +-- party that KNOWS whether a lesson was applied — the agent — to say so. +-- +-- THE SHAPE. `memory.write` gains an optional `cited: string[]` of `scope::key` +-- references. The retrospective write an agent already makes at the end of a +-- run names the lessons that shaped it, and each name becomes a citation row +-- joined to the memory it cites, the memory it was written from, and the +-- `correlation_id` that groups the run (`usage_events` already carries that +-- key, and `/usage/runs` already enumerates it). One new optional field on an +-- existing verb, not a new tool: every entry in `tools/list` is context the +-- model pays for in every session, and this one has nothing to dispatch. +-- +-- WHY A TABLE AND NOT JUST A COUNTER. The counter (`memories.cited_count`) is +-- what the dashboard reads and what a `max_cited_count` retention condition +-- would read, and it could have been the whole feature. The rows earn their +-- keep by answering the two questions a counter cannot: WHICH lesson credited +-- this one (so a load-bearing rule can be traced to the retrospectives that +-- kept invoking it) and WHICH RUN (so a citation joins to the same +-- `correlation_id` grouping `/usage/runs` already shows). A bare counter also +-- has no idempotency key, which is the next paragraph. +-- +-- IDEMPOTENCY, AND WHY IT LEANS TOWARD UNDER-COUNTING. The unique key is +-- `(cited_memory_id, citing_memory_id, coalesce(correlation_id, ''))`: one +-- citation per cited lesson per citing lesson per run. A retried write inside +-- one run does not double-count. Two genuinely separate runs each count, which +-- is right — a lesson applied on Monday and again on Friday was applied twice. +-- A caller that sends NO correlation id collapses to a single citation for that +-- pair forever, which under-counts; that is the deliberate direction, because +-- an inflated influence number is worse than a conservative one on a surface +-- whose entire argument is that the existing numbers overstate. +-- +-- THE `updated_at` EXEMPTION IS LOAD-BEARING (00103/00104). `cited_count` and +-- `last_cited_at` are the fifth and sixth columns of the derived set a READ-ish +-- path writes. Without adding them to the trigger's mask, recording a citation +-- would restamp the cited lesson's `updated_at` — reintroducing exactly the +-- bug 00103 fixed, on a new path, and making the freshness column lie again. +-- migrations.test.sql §105 AC-2 is that assertion. +-- ═════════════════════════════════════════════════════════════════════════ -create or replace function restore_memory( - p_user_id uuid, - p_scope text default null, - p_key text default null, - p_key_scopes text[] default '{}', - p_key_org_access text default 'all', -- inert in BYOD (no orgs) - p_key_org_ids uuid[] default '{}' -- inert in BYOD (no orgs) +-- ── 1. the counters on `memories` ───────────────────────────────────────── +alter table memories add column if not exists cited_count integer not null default 0; +alter table memories add column if not exists last_cited_at timestamptz; + +alter table memories drop constraint if exists memories_cited_count_non_negative; +alter table memories add constraint memories_cited_count_non_negative + check (cited_count >= 0); + +comment on column memories.cited_count is + 'How many times an agent has explicitly CREDITED this lesson for shaping a run, + via memory.write''s `cited` array (00107). Distinct from opened_count (00104), + which counts deliberate FETCHES: a lesson injected at session start is already + in context and can be applied without ever being fetched, so a lesson can have + opened_count = 0 and a high cited_count. Never moved by any read. Not null, + defaults to 0.'; + +comment on column memories.last_cited_at is + 'When this lesson was last explicitly credited by an agent (00107). Null until + the first citation. Moved in the same statement as cited_count, off the same + gate, so a count and its timestamp cannot disagree.'; + +-- ── 2. teach the trigger to mask them — BEFORE anything writes them ─────── +-- 00103 exempted read_count/last_read_at/last_opened_at; 00104 added +-- opened_count. These are the fifth and sixth columns of the same derived set: +-- a citation is recorded ABOUT a lesson by a write to a DIFFERENT lesson, so +-- letting it restamp the cited row's `updated_at` would say "this lesson was +-- edited" when nobody touched its content. Asserted by §105 AC-2. +create or replace function lorekit_memories_set_updated_at() +returns trigger +language plpgsql +set search_path = public, extensions +as $$ +declare + v_new memories := new; + v_old memories := old; + v_embedding_changed boolean; + v_counters_changed boolean; +begin + -- 00062's exemption: an embedding write. + v_embedding_changed := + new.embedding is distinct from old.embedding + or new.embedding_model is distinct from old.embedding_model; + + -- 00103's exemption, extended by 00104 (opened_count) and 00107 + -- (cited_count/last_cited_at): a derived-counter write that is not an edit. + v_counters_changed := + new.read_count is distinct from old.read_count + or new.last_read_at is distinct from old.last_read_at + or new.last_opened_at is distinct from old.last_opened_at + or new.opened_count is distinct from old.opened_count + or new.cited_count is distinct from old.cited_count + or new.last_cited_at is distinct from old.last_cited_at; + + -- Mask the derived columns and the one being decided. + -- + -- `fts` MUST be masked too, for 00062's reason: it is a GENERATED column and + -- Postgres computes generated columns AFTER BEFORE-row triggers, so + -- `new.fts` is NULL here while `old.fts` holds the stored value. An unmasked + -- comparison finds every update "changed", which silently turns this whole + -- function back into `set_updated_at`. It fails in the safe direction (a bump + -- that should not happen, never a missing bump), which is exactly why it + -- needs a test rather than a reading — see migrations.test.sql §101/§105. + v_new.embedding := null; v_new.embedding_model := null; + v_new.read_count := null; v_new.last_read_at := null; v_new.last_opened_at := null; + v_new.opened_count := null; + v_new.cited_count := null; v_new.last_cited_at := null; + v_new.updated_at := null; v_new.fts := null; + + v_old.embedding := null; v_old.embedding_model := null; + v_old.read_count := null; v_old.last_read_at := null; v_old.last_opened_at := null; + v_old.opened_count := null; + v_old.cited_count := null; v_old.last_cited_at := null; + v_old.updated_at := null; v_old.fts := null; + + if (v_embedding_changed or v_counters_changed) + and v_new is not distinct from v_old then + new.updated_at := old.updated_at; + else + new.updated_at := now(); + end if; + + return new; +end; +$$; + +comment on function lorekit_memories_set_updated_at() is + 'BEFORE UPDATE trigger on memories. Stamps updated_at = now() for a real + content edit, and LEAVES IT ALONE for a write that only moves derived + columns: the embedding pair (00062), the four read counters (00099/00103/ + 00104) and the two citation counters (00107). Compares NEW to OLD with every + derived column plus the generated `fts` masked out, so the exemption is a + property of the whole row rather than a list of allowed callers.'; + +-- ── 3. the citation ledger ──────────────────────────────────────────────── +create table if not exists memory_citations ( + id uuid primary key default gen_random_uuid(), + user_id uuid not null, + cited_memory_id uuid not null references memories(id) on delete cascade, + -- SET NULL, not CASCADE: purging the retrospective that credited a lesson + -- must not erase the evidence that the lesson was credited. The counter on + -- `memories` would then disagree with the ledger, and the counter is the one + -- the product reads. + citing_memory_id uuid references memories(id) on delete set null, + correlation_id text, + created_at timestamptz not null default now() +); + +-- Same ceiling `usage_events.correlation_id` carries (00058), so a value that +-- groups a run there can always be stored here. +alter table memory_citations drop constraint if exists memory_citations_correlation_id_len; +alter table memory_citations add constraint memory_citations_correlation_id_len + check (correlation_id is null or char_length(correlation_id) <= 200); + +-- The idempotency key. `coalesce` because a NULL never equals a NULL in a +-- unique index, so without it an uncorrelated retry would insert a second row. +-- See the header for why collapsing the uncorrelated case is the deliberate +-- direction. +create unique index if not exists memory_citations_once_per_run_idx + on memory_citations (cited_memory_id, citing_memory_id, coalesce(correlation_id, '')); + +-- The two reads this table exists for: "who credited this lesson" and "what did +-- this run credit". +create index if not exists memory_citations_cited_idx + on memory_citations (cited_memory_id, created_at desc); +create index if not exists memory_citations_user_correlation_idx + on memory_citations (user_id, correlation_id) + where correlation_id is not null; + +alter table memory_citations enable row level security; +-- No policies, and the `memory_read_daily`/`usage_events` reasoning verbatim: +-- every access is through the SECURITY DEFINER function below. RLS on with zero +-- policies denies every direct PostgREST request, which is the backstop against +-- the one path that is neither service-role nor SECURITY DEFINER — an +-- anon/authenticated key querying the table directly. +revoke all on memory_citations from anon, authenticated; +grant select, insert on memory_citations to service_role; + +comment on table memory_citations is + 'One row per (cited lesson, citing lesson, run): an agent explicitly crediting + a lesson for shaping the run it just finished, recorded from memory.write''s + `cited` array (00107). The influence signal pull-through (00104) cannot give, + because a lesson injected at session start is applied without ever being + fetched. Unique on (cited_memory_id, citing_memory_id, coalesce(correlation_id, + '''')) so a retry inside one run counts once. Never read through PostgREST.'; + +-- ── 4. the writer ───────────────────────────────────────────────────────── +-- Takes the references as two PARALLEL ARRAYS rather than an array of composite +-- types, for the reason every other array-taking RPC here does: PostgREST maps +-- `text[]` cleanly from JSON and a composite type would need a matching +-- declared type on both sides of the wire. +create or replace function lorekit_record_memory_citations( + p_user_id uuid, + p_citing_memory_id uuid, + p_cited_scopes text[], + p_cited_keys text[], + p_correlation_id text default null ) -returns table (restored boolean, existed boolean) +returns integer language plpgsql security definer set search_path = public as $$ declare - v_count integer; - v_existed boolean; + v_inserted integer := 0; begin - if not lorekit_api_token_scope_allowed(p_key_scopes, p_scope) then - raise exception using errcode = 'LK002', - message = format('key_scope_denied: scope=%s', p_scope); + if p_user_id is null + or p_cited_scopes is null + or array_length(p_cited_scopes, 1) is null + or coalesce(array_length(p_cited_scopes, 1), 0) <> coalesce(array_length(p_cited_keys, 1), 0) + then + return 0; + end if; + + -- Resolve refs → ids under the CALLER'S OWN tenancy, and nothing wider. This + -- runs SECURITY DEFINER on a service-role connection where `auth.uid()` is + -- NULL for the api_key tier, so the `user_id` predicate is the only thing + -- standing between a citation and someone else's lore. A ref that resolves to + -- nothing is silently skipped: it names a lesson this account cannot see, or + -- one that has since been deleted, and neither is the write's problem. + -- + -- Org-owned lore is deliberately NOT resolvable here yet. `memories.user_id` + -- is the row's owner, and widening to `lorekit_member_org_ids` would let one + -- member's retrospective move a counter on a shared lesson that other members + -- read — a decision about shared state that belongs in its own change, with + -- its own audit story. Under-counting again, deliberately. + with refs as ( + select trim(s.scope) as scope, trim(k.key) as key + from unnest(p_cited_scopes) with ordinality as s(scope, n) + join unnest(p_cited_keys) with ordinality as k(key, n) using (n) + where trim(s.scope) <> '' and trim(k.key) <> '' + ), + resolved as ( + select distinct m.id + from refs r + join memories m + on m.scope = r.scope + and m.key = r.key + and m.user_id = p_user_id + and m.archived_at is null + -- A lesson cannot cite itself. A retrospective that names its own key is + -- a model echoing the write it is making, not evidence of anything. + where p_citing_memory_id is null or m.id <> p_citing_memory_id + ), + ins as ( + insert into memory_citations (user_id, cited_memory_id, citing_memory_id, correlation_id) + select p_user_id, resolved.id, p_citing_memory_id, p_correlation_id + from resolved + on conflict do nothing + returning cited_memory_id + ), + bumped as ( + -- Only the rows that ACTUALLY inserted move a counter, which is what makes + -- the counter and the ledger agree after a retry. + select cited_memory_id, count(*)::integer as n from ins group by cited_memory_id + ), + upd as ( + update memories m + set cited_count = m.cited_count + b.n, + last_cited_at = now() + from bumped b + where m.id = b.cited_memory_id + returning b.n + ) + select coalesce(sum(n), 0)::integer into v_inserted from upd; + + return v_inserted; +exception + when others then + -- Never let a citation break the write it accompanies. Same posture as + -- `lorekit_record_memory_reads` and `lorekit_record_usage_event`. + return 0; +end; +$$; + +comment on function lorekit_record_memory_citations(uuid, uuid, text[], text[], text) is + 'Records that the write identified by p_citing_memory_id credited each + (p_cited_scopes[i], p_cited_keys[i]) lesson for shaping the run p_correlation_id + groups, and bumps memories.cited_count/.last_cited_at for each citation that + was actually new. Resolves references ONLY within p_user_id''s own active lore + -- an unresolvable ref is skipped, and a self-citation is dropped. Returns how + many citations were newly recorded (0 on a duplicate run, 0 on any error). + Swallows its own errors: a citation must never break the write carrying it.'; + +grant execute on function lorekit_record_memory_citations(uuid, uuid, text[], text[], text) + to authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00107', 'memory_citations') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00108_retention_conditions_on_aggregates.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- 00108 — the retention conditions reach the AGGREGATE reads, not just the list. +-- +-- THE BUG. `lorekit_memory_list` has accepted the five retention thresholds +-- since 00105 (`p_min_age_days`, `p_unseen_days`, `p_max_seen_count`, +-- `p_max_read_count`, `p_max_opened_count`). The three functions that COUNT the +-- same population did not: +-- +-- lorekit_memory_facets the filter menu's per-value counts +-- lorekit_memory_activity the Explorer's five stat cards +-- lorekit_memory_pivot the Explorer's matrix +-- +-- So setting any threshold moved the list and left every number describing it +-- unchanged. A user filtering to "never opened, older than 5 days" saw a short +-- list above a menu still offering `source::manual 150` — a count for a +-- population that filter had already excluded. The dimension filters were +-- always mirrored (00057/00064/00069), which is why the discrepancy read as a +-- facet bug rather than a missing parameter: a label pill moved the counts, a +-- threshold did not. +-- +-- The `created_since`/`created_until` window had the identical gap and is fixed +-- with it. `lorekit_memory_activity` needs no new window — its own +-- `p_since`/`p_until` already bound `created_at`. +-- +-- `q`, `p_key` and `expiring_within_days` stay unmirrored, for the reason +-- `ListFacetsQuerySchema` records: mirroring `q` means a second LIKE-escaper +-- inside plpgsql and mirroring `expiring_within_days` a second `now`-relative +-- boundary. A filter value is encoded exactly one way in this repo. The +-- thresholds carry no such cost — see the helper below. +-- +-- ── DESIGN ─────────────────────────────────────────────────────────────── +-- +-- ONE HELPER, `lorekit_match_retention`, in 00066's inlinable-predicate family +-- (`lorekit_match_text` / `_tags` / `_int`). Four callers had five nearly +-- identical `and (p_x is null or …)` lines each; they now have one call each, +-- and migration 00100's never-opened rule — `coalesce(last_opened_at, +-- created_at)`, so a lesson nobody has EVER opened ages from its creation +-- rather than being immune — lives in exactly one place instead of four. +-- +-- IT TAKES CUTOFF TIMESTAMPS, NOT DAY COUNTS. `now() - (n * interval '1 day')` +-- inside the helper would make it `stable` rather than `immutable`, costing the +-- inlining that keeps callers on index access, and would read the clock once +-- per ROW instead of once per call. Each caller resolves its own cutoffs in its +-- declare block, so the `now`-relative boundary is still computed exactly once +-- per query — the same discipline the unmirrored `expiring_within_days` +-- argument above is protecting. +-- +-- IN THE `base` CTE's WHERE, NEVER AN `ok_*` SELF-EXCLUSION FLAG. `facets` and +-- `pivot` count each dimension with every OTHER filter applied but not its own, +-- so that a value's count is what selecting it would yield. That trick is +-- meaningful only for a dimension the caller can select a value FROM. A +-- threshold has no value catalog — "count this threshold with every filter but +-- itself" describes nothing — so it belongs in the row-visibility predicate +-- every facet value is derived from, alongside `p_scope` and `p_archived`. +-- +-- ── SIGNATURES ─────────────────────────────────────────────────────────── +-- +-- The new parameters are APPENDED, null-defaulted, following 00101/00105. That +-- changes each function's signature, and `create or replace` CANNOT change a +-- signature — it would leave the old overload in place beside the new one, and +-- two overloads make every PostgREST named-argument call to that name fail as +-- ambiguous (the footgun 00092's header documents). So each of the three is +-- dropped at its EXACT old signature first. +-- +-- `lorekit_memory_list` keeps its 00105 signature verbatim — only its body +-- changes, to compose the helper in place of its five inline lines — so it is a +-- true `create or replace` with no drop and its grants persist. +-- +-- §108 in `migrations.test.sql` is the executable proof, including the case +-- that motivated the whole change: a threshold that narrows the list must +-- narrow the facet counts by the same rows. +-- ═════════════════════════════════════════════════════════════════════════ + +-- ── 1. The predicate ──────────────────────────────────────────────────────── + +-- The five retention thresholds as one row-level test. +-- +-- Every parameter is independently optional: a null threshold is "not +-- filtered" and matches everything, so an untouched condition never narrows. +-- All five null (the overwhelmingly common case) is a constant `true` the +-- planner discards. +-- +-- `immutable` + `parallel safe` + `language sql`, exactly as 00066's three +-- helpers, and for the same two reasons: the result depends only on the +-- arguments, and that is what lets the planner fold the call into an +-- index-usable expression rather than treating it as an opaque filter. +create or replace function lorekit_match_retention( + -- The row's own counters. + p_created_at timestamptz, + p_last_opened_at timestamptz, + p_seen_count integer, + p_read_count integer, + p_opened_count integer, + -- The filter, with both day counts already resolved to absolute instants by + -- the caller. Null = that condition is not applied. + p_created_cutoff timestamptz, + p_unseen_cutoff timestamptz, + p_max_seen_count integer, + p_max_read_count integer, + p_max_opened_count integer +) +returns boolean +language sql +immutable +parallel safe +as $$ + select coalesce( + (p_created_cutoff is null or p_created_at <= p_created_cutoff) + -- 00100: a lesson NEVER opened ages from its creation date. Without the + -- coalesce a null `last_opened_at` makes the comparison NULL, which + -- excludes the row — so the lore nothing has ever opened would be the one + -- lore an "unopened for N days" filter could never find. + and (p_unseen_cutoff is null or coalesce(p_last_opened_at, p_created_at) <= p_unseen_cutoff) + -- `<=`, so a threshold of 0 is meaningful: `max_opened_count => 0` is + -- "nothing ever chose to open it" (00105), not "no filter". + and (p_max_seen_count is null or p_seen_count <= p_max_seen_count) + and (p_max_read_count is null or p_read_count <= p_max_read_count) + and (p_max_opened_count is null or p_opened_count <= p_max_opened_count), + false); +$$; + +comment on function lorekit_match_retention( + timestamptz, timestamptz, integer, integer, integer, + timestamptz, timestamptz, integer, integer, integer +) is + 'Does a memory satisfy a retention-policy condition set? The five thresholds + of GroomConditionsSchema, as one inlinable row test shared by + lorekit_memory_list, _facets, _activity and _pivot so a count cannot + disagree with the list it describes (00108). Takes CUTOFF INSTANTS rather + than day counts so it stays immutable and reads no clock: the caller + resolves now()-relative bounds once per query. A null threshold is "not + filtered". Holds migration 00100''s never-opened rule + (coalesce(last_opened_at, created_at)) in one place.'; + +-- The same revoke/grant pair 00066's three helpers carry. A `create or replace` +-- of a NEW function starts from the default ACL, which is EXECUTE to public — +-- so without this line the helper would be the one member of the +-- `lorekit_match_*` family reachable by `anon`. It reads no table and would +-- leak nothing, but a predicate helper with a different ACL from its three +-- siblings is a difference someone has to re-derive later. +revoke execute on function lorekit_match_retention( + timestamptz, timestamptz, integer, integer, integer, + timestamptz, timestamptz, integer, integer, integer +) from public, anon; +grant execute on function lorekit_match_retention( + timestamptz, timestamptz, integer, integer, integer, + timestamptz, timestamptz, integer, integer, integer +) to authenticated, service_role; + +-- ── 2. lorekit_memory_facets ──────────────────────────────────────────────── + +drop function if exists lorekit_memory_facets( + uuid, boolean, text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[] +); + +create or replace function lorekit_memory_facets( + p_user_id uuid, + p_archived boolean default false, + p_scope text default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in', + -- Owner dimension (00064). `personal` plus one slug per member org with + -- visible rows. All optional: null/absent = not filtered. + p_owner text[] default null, + p_owner_mode text default 'in', + -- The CALLING KEY's restriction (00068), defaulted to unrestricted. + p_key_scopes text[] default '{}', + p_key_org_access text default 'all', + p_key_org_ids uuid[] default '{}', + -- 00108: the created_at window and the five retention thresholds, so a count + -- describes the same rows the list shows. Appended, null-defaulted. + p_created_since timestamptz default null, + p_created_until timestamptz default null, + p_min_age_days integer default null, + p_unseen_days integer default null, + p_max_seen_count integer default null, + p_max_read_count integer default null, + p_max_opened_count integer default null +) +returns table (facet text, value text, count bigint) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + v_origin_pr integer[] := ( + select array_agg(x::integer) + from unnest(coalesce(p_origin_pr, '{}'::text[])) as x + where x ~ '^[0-9]+$' + ); + -- Resolved ONCE per call, so lorekit_match_retention can stay immutable and + -- the clock is read once rather than once per row (00108). + v_created_cutoff timestamptz := case + when p_min_age_days is null then null else now() - (p_min_age_days * interval '1 day') + end; + v_unseen_cutoff timestamptz := case + when p_unseen_days is null then null else now() - (p_unseen_days * interval '1 day') + end; +begin + return query + with base as ( + select + m.tags, m.source_agent, m.trigger, m.kind, m.host, + m.origin_repo, m.origin_branch, m.origin_pr, + m.org_id, o.slug as org_slug, + -- Per-dimension match flag, now from the shared predicates so it cannot + -- drift from lorekit_memory_activity's. A null filter is "not filtered" → + -- the helper returns true, so an untouched dimension never narrows. + lorekit_match_tags(m.tags, p_tags, p_tags_mode) as ok_tag, + lorekit_match_text(m.source_agent, p_source_agent, p_source_agent_mode) as ok_source_agent, + lorekit_match_text(m.trigger, p_trigger, p_trigger_mode) as ok_trigger, + lorekit_match_text(m.kind, p_kind, p_kind_mode) as ok_kind, + lorekit_match_text(m.host, p_host, p_host_mode) as ok_host, + lorekit_match_text(m.origin_repo, p_origin_repo, p_origin_repo_mode) as ok_origin_repo, + lorekit_match_text(m.origin_branch, p_origin_branch, p_origin_branch_mode) as ok_origin_branch, + lorekit_match_int (m.origin_pr, v_origin_pr, p_origin_pr_mode) as ok_origin_pr, + -- Owner: the computed identity is `personal` (org_id null) or the org slug. + -- Stays inline — it is not one of the three column helpers (00064). + (p_owner is null or case coalesce(p_owner_mode, 'in') + when 'nin' then ( + (case when m.org_id is null then 'personal' else o.slug end) is not null + and (case when m.org_id is null then 'personal' else o.slug end) <> all(p_owner) + ) + else ( + ('personal' = any(p_owner) and m.org_id is null) + or (m.org_id is not null and o.slug = any(p_owner)) + ) + end) as ok_owner + from memories m + -- A personal row has no org, so this is a LEFT join; org rows resolve to + -- their slug. Visible org rows are always the caller's own orgs (the + -- visibility predicate below admits them only via lorekit_member_org_ids), + -- so `o.slug` is never a slug the caller cannot see. + left join orgs o on o.id = m.org_id + where ( + (v_actor is null and auth.role() = 'service_role') + or m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) + ) + -- The calling key's restriction, applied ONCE here in the row-visibility + -- predicate every emitted facet value is derived from — `origin_repo` is + -- a repository name by construction, so an unnarrowed facet list leaks + -- exactly what the scope catalog hides. + and lorekit_api_token_scope_allowed(p_key_scopes, m.scope) + and lorekit_api_token_org_allowed(p_key_org_access, p_key_org_ids, m.org_id) + and ( + case + when p_archived then m.archived_at is not null + else m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + end + ) + and (p_scope is null or m.scope = p_scope) + -- 00108. Here rather than as an `ok_*` flag: these narrow the population + -- every facet value is drawn from, they are not a dimension that can be + -- self-excluded. Half-open window, matching lorekit_memory_list. + and (p_created_since is null or m.created_at >= p_created_since) + and (p_created_until is null or m.created_at < p_created_until) + and lorekit_match_retention( + m.created_at, m.last_opened_at, m.seen_count, m.read_count, m.opened_count, + v_created_cutoff, v_unseen_cutoff, p_max_seen_count, p_max_read_count, p_max_opened_count + ) + ), cells as ( + select 'tag'::text as facet, t.tag as value + from base b + cross join lateral unnest(b.tags) as t(tag) + where b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner + union all + select 'source_agent', b.source_agent from base b + where b.ok_tag and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner + union all + select 'trigger', b.trigger from base b + where b.ok_tag and b.ok_source_agent and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner + union all + select 'kind', b.kind from base b + where b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner + union all + select 'host', b.host from base b + where b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner + union all + select 'origin_repo', b.origin_repo from base b + where b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner + union all + select 'origin_branch', b.origin_branch from base b + where b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_pr and b.ok_owner + union all + select 'origin_pr', b.origin_pr::text from base b + where b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_owner + union all + -- Owner is the ONE dimension self-excluded here (every other flag, NOT + -- ok_owner), so a drilled-in owner still lists the alternative owner. + select 'owner', case when b.org_id is null then 'personal' else b.org_slug end from base b + where b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr + ) + select c.facet, c.value, count(*) as count + from cells c + where c.value is not null + and btrim(c.value) <> '' + group by c.facet, c.value + order by c.facet asc, count(*) desc, c.value asc; +end; +$$; + +revoke execute on function lorekit_memory_facets( + uuid, boolean, text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[], timestamptz, timestamptz, integer, integer, integer, + integer, integer +) from public, anon; +grant execute on function lorekit_memory_facets( + uuid, boolean, text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[], timestamptz, timestamptz, integer, integer, integer, + integer, integer +) to authenticated, service_role; + +comment on function lorekit_memory_facets( + uuid, boolean, text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[], timestamptz, timestamptz, integer, integer, integer, + integer, integer +) is + 'Value catalog with counts for every filterable memory dimension (tag, + source_agent, trigger, kind, host, origin_repo, origin_branch, origin_pr, + owner) over the partition selected by p_archived, visible to the EFFECTIVE + caller and narrowed by the calling key''s own scope/org restriction. Each + dimension is counted with every OTHER dimension filter applied but not its + own (drill-down self-exclusion, 00057). The created_at window and the five + retention thresholds (00108) are NOT self-excluded: they narrow the + population every facet value is drawn from, so a count matches the list + under the same parameters. q/key/expiring_within_days remain unmirrored, so + with those active a count is an upper bound.'; + +-- ── 3. lorekit_memory_activity ────────────────────────────────────────────── + +drop function if exists lorekit_memory_activity( + uuid, text, timestamptz, timestamptz, text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[] +); + +create or replace function lorekit_memory_activity( + p_user_id uuid, + p_bucket text default 'day', + p_since timestamptz default null, + p_until timestamptz default null, + p_scope text default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in', + p_owner text[] default null, + p_owner_mode text default 'in', + -- The CALLING KEY's restriction (00068), defaulted to unrestricted. + p_key_scopes text[] default '{}', + p_key_org_access text default 'all', + p_key_org_ids uuid[] default '{}', + -- 00108: the five retention thresholds. No created_at window is added — this + -- function's own p_since/p_until already bound created_at, and a second + -- window would be two ways to say one thing. + p_min_age_days integer default null, + p_unseen_days integer default null, + p_max_seen_count integer default null, + p_max_read_count integer default null, + p_max_opened_count integer default null +) +returns table (bucket timestamptz, scope text, count bigint) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + -- `origin_pr` is an INTEGER column: coerce the digits-only list numerically so + -- this route matches GET /memories on the same input (00057's rationale). A + -- non-numeric entry is dropped; a list reducing to empty applies no filter. + v_origin_pr integer[] := ( + select array_agg(x::integer) + from unnest(coalesce(p_origin_pr, '{}'::text[])) as x + where x ~ '^[0-9]+$' + ); + v_created_cutoff timestamptz := case + when p_min_age_days is null then null else now() - (p_min_age_days * interval '1 day') + end; + v_unseen_cutoff timestamptz := case + when p_unseen_days is null then null else now() - (p_unseen_days * interval '1 day') + end; +begin + if p_bucket is null or p_bucket not in ('hour', 'day') then + raise exception 'invalid bucket %, expected hour or day', p_bucket + using errcode = '22023'; end if; - -- A null p_user_id is the SERVICE tier (no user, no key) — the hosted - -- functions' rule, and the one the raw-query paths have always used. It must - -- not degrade to `user_id = null`, which matches nothing. - v_existed := exists( - select 1 from memories m - where m.scope = p_scope and m.key = p_key and m.archived_at is not null - and (p_user_id is null or m.user_id = p_user_id) + return query + select date_trunc(p_bucket, m.created_at at time zone 'UTC') at time zone 'UTC' as bucket, + m.scope, + count(*) as count + from memories m + -- LEFT join for the owner predicate — a personal row has no org (00064). + left join orgs o on o.id = m.org_id + where ( + (v_actor is null and auth.role() = 'service_role') + or m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) + ) + -- The calling key's own restriction, ANDed on top of the caller's + -- visibility and never instead of it (the lorekit_memory_scopes rule). + and lorekit_api_token_scope_allowed(p_key_scopes, m.scope) + and lorekit_api_token_org_allowed(p_key_org_access, p_key_org_ids, m.org_id) + and m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + and (p_since is null or m.created_at >= p_since) + and (p_until is null or m.created_at < p_until) + -- Scope is a hard filter, always applied (00057's treatment). + and (p_scope is null or m.scope = p_scope) + -- Dimension filters — AND across, OR within, from the shared predicates + -- (00066) so the semantics cannot drift from lorekit_memory_facets. No + -- self-exclusion here: a straight count applies every one directly. + and lorekit_match_tags(m.tags, p_tags, p_tags_mode) + and lorekit_match_text(m.source_agent, p_source_agent, p_source_agent_mode) + and lorekit_match_text(m.trigger, p_trigger, p_trigger_mode) + and lorekit_match_text(m.kind, p_kind, p_kind_mode) + and lorekit_match_text(m.host, p_host, p_host_mode) + and lorekit_match_text(m.origin_repo, p_origin_repo, p_origin_repo_mode) + and lorekit_match_text(m.origin_branch, p_origin_branch, p_origin_branch_mode) + and lorekit_match_int (m.origin_pr, v_origin_pr, p_origin_pr_mode) + -- 00108: the same thresholds the list applies, so the Explorer's stat + -- cards describe the rows underneath them. + and lorekit_match_retention( + m.created_at, m.last_opened_at, m.seen_count, m.read_count, m.opened_count, + v_created_cutoff, v_unseen_cutoff, p_max_seen_count, p_max_read_count, p_max_opened_count + ) + -- Owner: `personal` for org_id-null rows, else the org slug (00064). + and (p_owner is null or case coalesce(p_owner_mode, 'in') + when 'nin' then ( + (case when m.org_id is null then 'personal' else o.slug end) is not null + and (case when m.org_id is null then 'personal' else o.slug end) <> all(p_owner) + ) + else ( + ('personal' = any(p_owner) and m.org_id is null) + or (m.org_id is not null and o.slug = any(p_owner)) + ) + end) + group by 1, m.scope + order by 1 asc, m.scope asc; +end; +$$; + +revoke execute on function lorekit_memory_activity( + uuid, text, timestamptz, timestamptz, text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[], integer, integer, integer, integer, integer +) from public, anon; +grant execute on function lorekit_memory_activity( + uuid, text, timestamptz, timestamptz, text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[], integer, integer, integer, integer, integer +) to authenticated, service_role; + +comment on function lorekit_memory_activity( + uuid, text, timestamptz, timestamptz, text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[], integer, integer, integer, integer, integer +) is + 'Memories created per UTC hour/day per scope, visible to the EFFECTIVE caller + and narrowed by the calling key''s own scope/org restriction. Applies every + dimension filter directly (no self-exclusion — this is a straight count) and, + since 00108, the same five retention thresholds lorekit_memory_list applies, + so the Explorer''s stat cards describe the list beneath them. p_since/p_until + already bound created_at, so no separate window parameter exists.'; + +-- ── 4. lorekit_memory_pivot ───────────────────────────────────────────────── + +drop function if exists lorekit_memory_pivot( + text, text, uuid, boolean, text, integer, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[] +); + +create or replace function lorekit_memory_pivot( + p_row_facet text, + p_col_facet text, + p_user_id uuid default null, + p_archived boolean default false, + p_scope text default null, + p_limit integer default 400, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in', + p_owner text[] default null, + p_owner_mode text default 'in', + -- The CALLING KEY's restriction (00068), defaulted to unrestricted. + p_key_scopes text[] default '{}', + p_key_org_access text default 'all', + p_key_org_ids uuid[] default '{}', + -- 00108: the created_at window and the five retention thresholds. + p_created_since timestamptz default null, + p_created_until timestamptz default null, + p_min_age_days integer default null, + p_unseen_days integer default null, + p_max_seen_count integer default null, + p_max_read_count integer default null, + p_max_opened_count integer default null +) +returns table (row_value text, col_value text, count bigint) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + v_origin_pr integer[] := ( + select array_agg(x::integer) + from unnest(coalesce(p_origin_pr, '{}'::text[])) as x + where x ~ '^[0-9]+$' ); + v_created_cutoff timestamptz := case + when p_min_age_days is null then null else now() - (p_min_age_days * interval '1 day') + end; + v_unseen_cutoff timestamptz := case + when p_unseen_days is null then null else now() - (p_unseen_days * interval '1 day') + end; +begin + return query + with base as ( + select + m.tags, m.source_agent, m.trigger, m.kind, m.host, + m.origin_repo, m.origin_branch, m.origin_pr, + m.org_id, o.slug as org_slug, + lorekit_match_tags(m.tags, p_tags, p_tags_mode) as ok_tag, + lorekit_match_text(m.source_agent, p_source_agent, p_source_agent_mode) as ok_source_agent, + lorekit_match_text(m.trigger, p_trigger, p_trigger_mode) as ok_trigger, + lorekit_match_text(m.kind, p_kind, p_kind_mode) as ok_kind, + lorekit_match_text(m.host, p_host, p_host_mode) as ok_host, + lorekit_match_text(m.origin_repo, p_origin_repo, p_origin_repo_mode) as ok_origin_repo, + lorekit_match_text(m.origin_branch, p_origin_branch, p_origin_branch_mode) as ok_origin_branch, + lorekit_match_int (m.origin_pr, v_origin_pr, p_origin_pr_mode) as ok_origin_pr, + -- Owner stays inline for lorekit_memory_facets' reason: it is a + -- LEFT JOIN-computed identity, not one of the three column helpers. + (p_owner is null or case coalesce(p_owner_mode, 'in') + when 'nin' then ( + (case when m.org_id is null then 'personal' else o.slug end) is not null + and (case when m.org_id is null then 'personal' else o.slug end) <> all(p_owner) + ) + else ( + ('personal' = any(p_owner) and m.org_id is null) + or (m.org_id is not null and o.slug = any(p_owner)) + ) + end) as ok_owner + from memories m + left join orgs o on o.id = m.org_id + where ( + (v_actor is null and auth.role() = 'service_role') + or m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) + ) + and lorekit_api_token_scope_allowed(p_key_scopes, m.scope) + and lorekit_api_token_org_allowed(p_key_org_access, p_key_org_ids, m.org_id) + and ( + case + when p_archived then m.archived_at is not null + else m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + end + ) + and (p_scope is null or m.scope = p_scope) + -- 00108. Row-visibility, not an `ok_*` flag — see lorekit_memory_facets. + and (p_created_since is null or m.created_at >= p_created_since) + and (p_created_until is null or m.created_at < p_created_until) + and lorekit_match_retention( + m.created_at, m.last_opened_at, m.seen_count, m.read_count, m.opened_count, + v_created_cutoff, v_unseen_cutoff, p_max_seen_count, p_max_read_count, p_max_opened_count + ) + ) + select rv.value, cv.value, count(*)::bigint + from base b + cross join lateral lorekit_memory_facet_value( + p_row_facet, b.tags, b.source_agent, b.trigger, b.kind, b.host, + b.origin_repo, b.origin_branch, b.origin_pr, b.org_id, b.org_slug + ) as rv(value) + cross join lateral lorekit_memory_facet_value( + p_col_facet, b.tags, b.source_agent, b.trigger, b.kind, b.host, + b.origin_repo, b.origin_branch, b.origin_pr, b.org_id, b.org_slug + ) as cv(value) + -- Each dimension's own filter applies UNLESS it is one of the two axes — + -- see the self-exclusion note at the top. One line per dimension so adding + -- a tenth is one line here, not a re-derivation. + where (p_row_facet = 'tag' or p_col_facet = 'tag' or b.ok_tag) + and (p_row_facet = 'source_agent' or p_col_facet = 'source_agent' or b.ok_source_agent) + and (p_row_facet = 'trigger' or p_col_facet = 'trigger' or b.ok_trigger) + and (p_row_facet = 'kind' or p_col_facet = 'kind' or b.ok_kind) + and (p_row_facet = 'host' or p_col_facet = 'host' or b.ok_host) + and (p_row_facet = 'origin_repo' or p_col_facet = 'origin_repo' or b.ok_origin_repo) + and (p_row_facet = 'origin_branch' or p_col_facet = 'origin_branch' or b.ok_origin_branch) + and (p_row_facet = 'origin_pr' or p_col_facet = 'origin_pr' or b.ok_origin_pr) + and (p_row_facet = 'owner' or p_col_facet = 'owner' or b.ok_owner) + -- A null column value yields no cell, matching lorekit_memory_facets. + and rv.value is not null + and cv.value is not null + group by rv.value, cv.value + -- Densest cells first, so a caller that has to truncate keeps the ones + -- worth drawing. Deterministic tie-break so a page boundary is stable. + order by count(*) desc, rv.value asc, cv.value asc + limit greatest(p_limit, 0); +end; +$$; - update memories m - set archived_at = null - where m.scope = p_scope and m.key = p_key and m.archived_at is not null - and (p_user_id is null or m.user_id = p_user_id); - get diagnostics v_count = row_count; +revoke execute on function lorekit_memory_pivot( + text, text, uuid, boolean, text, integer, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[], timestamptz, timestamptz, integer, integer, integer, + integer, integer +) from public, anon; +grant execute on function lorekit_memory_pivot( + text, text, uuid, boolean, text, integer, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[], timestamptz, timestamptz, integer, integer, integer, + integer, integer +) to authenticated, service_role; - return query select (v_count > 0), (v_existed or v_count > 0); +comment on function lorekit_memory_pivot( + text, text, uuid, boolean, text, integer, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[], timestamptz, timestamptz, integer, integer, integer, + integer, integer +) is + 'Cross-tabulation of two facet dimensions with a count per cell, visible to + the EFFECTIVE caller and narrowed by the calling key''s own scope/org + restriction. BOTH axes are self-excluded from the dimension filters, so a + caller that turns a cell into a filter still gets every other cell back. The + created_at window and the five retention thresholds (00108) are not + self-excluded — they narrow the population, matching lorekit_memory_facets.'; + +-- ── 5. lorekit_memory_list — compose the helper, signature unchanged ──────── +-- +-- A true `create or replace`: the 00105 signature is kept VERBATIM, so no drop +-- is needed and the 00105 grants persist. Only the five inline threshold lines +-- change, into one call, which is what puts 00100's never-opened rule in one +-- place across all four functions. `$35`/`$36` now carry resolved CUTOFFS +-- rather than day counts — the parameter list and every position are +-- unchanged, so no caller moves. +create or replace function lorekit_memory_list( + p_user_id uuid, + p_archived boolean default false, + p_scope text default null, + p_key text default null, + p_key_prefix text default null, + p_q text default null, + p_created_since timestamptz default null, + p_created_until timestamptz default null, + p_expires_after timestamptz default null, + p_expires_on_or_before timestamptz default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in', + p_owner text[] default null, + p_owner_mode text default 'in', + p_sort text default 'updated_at', + p_cursor_ts timestamptz default null, + p_cursor_id uuid default null, + p_limit integer default 51, + p_key_scopes text[] default '{}', + p_key_org_access text default 'all', + p_key_org_ids uuid[] default '{}', + p_min_age_days integer default null, + p_unseen_days integer default null, + p_max_seen_count integer default null, + p_max_read_count integer default null, + p_max_opened_count integer default null +) +returns table ( + id uuid, + scope text, + key text, + value text, + tags text[], + source_agent text, + trigger text, + created_at timestamptz, + updated_at timestamptz, + expires_at timestamptz, + archived_at timestamptz, + origin_repo text, + origin_branch text, + origin_commit text, + origin_pr integer, + kind text, + host text, + seen_count integer, + read_count integer, + opened_count integer, + last_read_at timestamptz, + last_opened_at timestamptz, + org_id uuid, + created_by uuid, + updated_by uuid, + org_name text, + org_slug text, + total_count integer +) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + v_origin_pr integer[] := ( + select array_agg(x::integer) + from unnest(coalesce(p_origin_pr, '{}'::text[])) as x + where x ~ '^0*[0-9]{1,9}$' + ); + v_sort text := case when p_sort = 'created_at' then 'created_at' else 'updated_at' end; + -- Resolved once per call so lorekit_match_retention stays immutable (00108). + v_created_cutoff timestamptz := case + when p_min_age_days is null then null else now() - (p_min_age_days * interval '1 day') + end; + v_unseen_cutoff timestamptz := case + when p_unseen_days is null then null else now() - (p_unseen_days * interval '1 day') + end; +begin + return query execute format($q$ + select + m.id, m.scope, m.key, m.value, m.tags, m.source_agent, m.trigger, + m.created_at, m.updated_at, m.expires_at, m.archived_at, + m.origin_repo, m.origin_branch, m.origin_commit, m.origin_pr, + m.kind, m.host, m.seen_count, + m.read_count, m.opened_count, m.last_read_at, m.last_opened_at, + m.org_id, m.created_by, m.updated_by, + o.name as org_name, o.slug as org_slug, + (count(*) over ())::integer as total_count + from memories m + left join orgs o on o.id = m.org_id + where ( + ($1 is null and auth.role() = 'service_role') + or m.user_id = $1 + or m.org_id in (select lorekit_member_org_ids($1)) + ) + and lorekit_api_token_scope_allowed($32, m.scope) + and lorekit_api_token_org_allowed($33, $34, m.org_id) + and ( + case + when $2 then m.archived_at is not null + else m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + end + ) + and ($3 is null or m.scope = $3) + and ($4 is null or m.key = $4) + and ($5 is null or m.key ilike $5 || '%%') + and ($6 is null or m.key ilike '%%' || $6 || '%%' or m.value ilike '%%' || $6 || '%%') + and ($7 is null or m.created_at >= $7) + and ($8 is null or m.created_at < $8) + and ($9 is null or m.expires_at > $9) + and ($10 is null or m.expires_at <= $10) + and lorekit_match_tags(m.tags, $11, $12) + and lorekit_match_text(m.source_agent, $13, $14) + and lorekit_match_text(m.trigger, $15, $16) + and lorekit_match_text(m.kind, $17, $18) + and lorekit_match_text(m.host, $19, $20) + and lorekit_match_text(m.origin_repo, $21, $22) + and lorekit_match_text(m.origin_branch, $23, $24) + and lorekit_match_int (m.origin_pr, $25, $26) + and ($27 is null or case coalesce($28, 'in') + when 'nin' then ( + (case when m.org_id is null then 'personal' else o.slug end) is not null + and (case when m.org_id is null then 'personal' else o.slug end) <> all($27) + ) + else ( + ('personal' = any($27) and m.org_id is null) + or (m.org_id is not null and o.slug = any($27)) + ) + end) + and lorekit_match_retention( + m.created_at, m.last_opened_at, m.seen_count, m.read_count, m.opened_count, + $35, $36, $37, $38, $39 + ) + and ( + $29 is null + or m.%1$I < $29 + or (m.%1$I = $29 and m.id < $30) + ) + order by m.%1$I desc, m.id desc + limit $31 + $q$, v_sort) + using + v_actor, coalesce(p_archived, false), p_scope, p_key, p_key_prefix, p_q, + p_created_since, p_created_until, p_expires_after, p_expires_on_or_before, + p_tags, p_tags_mode, + p_source_agent, p_source_agent_mode, + p_trigger, p_trigger_mode, + p_kind, p_kind_mode, + p_host, p_host_mode, + p_origin_repo, p_origin_repo_mode, + p_origin_branch, p_origin_branch_mode, + v_origin_pr, p_origin_pr_mode, + p_owner, p_owner_mode, + p_cursor_ts, p_cursor_id, + greatest(coalesce(p_limit, 51), 1), + coalesce(p_key_scopes, '{}'::text[]), + coalesce(p_key_org_access, 'all'), + coalesce(p_key_org_ids, '{}'::uuid[]), + v_created_cutoff, v_unseen_cutoff, p_max_seen_count, p_max_read_count, + p_max_opened_count; end; $$; -create or replace function purge_archived_memories( - p_user_id uuid, - p_retention_days integer default 30 +grant execute on function lorekit_memory_list( + uuid, boolean, text, text, text, text, timestamptz, timestamptz, timestamptz, + timestamptz, text[], text, text[], text, text[], text, text[], text, text[], + text, text[], text, text[], text, text[], text, text[], text, text, + timestamptz, uuid, integer, text[], text, uuid[], integer, integer, integer, + integer, integer +) to authenticated, service_role; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00108', 'retention_conditions_on_aggregates') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00109_usage_events_read_activity_idx.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- 00109 — a covering index for lorekit_read_activity's own predicate. +-- +-- THE BUG. `lorekit_read_activity` (00080) filters `usage_events` on +-- `user_id = v_actor AND tool_name IN ('memory.read', 'memory.list', +-- 'memory.search', 'memory.list_archived') AND created_at >= p_since AND +-- created_at < p_until` — and `GET /memories/read-activity` defaults that +-- window to 200 days when the caller passes no `since`/`until` +-- (`DEFAULT_WINDOW_DAYS`, read-activity.ts). +-- +-- The only index that can serve that query is `usage_events_user_created_at_idx` +-- (00034) on `(user_id, created_at desc)` — it carries no `tool_name`, so +-- Postgres range-scans EVERY row the user wrote in the window and filters +-- `tool_name` afterward. `usage_events` is this schema's highest-volume table +-- by construction (00080's own comment: a single SessionStart hook can write +-- ~31 rows per call, ~16,000 times, in a 30-day sample) — so for an active +-- account a 200-day scan routinely touches hundreds of thousands of rows that +-- are never in the 4-tool read family at all (writes, org calls, etc.), all to +-- return a handful of read-family rows. +-- +-- Observed in production: `GET /memories/read-activity` canceling with +-- "statement timeout" after ~20s of pure DB wait +-- (`lorekit.io.wait_ms` == the request's full duration), concentrated on +-- heavier accounts — exactly the profile a full per-user scan predicts. +-- +-- THE FIX. A partial index scoped to the same 4-tool read family this +-- function already hardcodes, so Postgres can seek straight to the +-- `(user_id, created_at)` range within JUST those rows instead of scanning +-- the account's entire event history. Partial (not a plain 3-column index) +-- because the read family is a small, fixed slice of `tool_name`'s domain — +-- indexing the other tool names would bloat the index for no query that uses +-- them via this predicate. +-- ═════════════════════════════════════════════════════════════════════════ + +create index if not exists usage_events_user_tool_created_idx + on usage_events (user_id, tool_name, created_at desc) + where tool_name in ('memory.read', 'memory.list', 'memory.search', 'memory.list_archived'); + +comment on index usage_events_user_tool_created_idx is + 'Covers lorekit_read_activity''s (user_id, tool_name IN (4 read tools), created_at) + predicate (00080) so the 200-day default window (read-activity.ts) does not + force a full per-user scan of usage_events, this schema''s highest-volume + table. Fixes production statement-timeout cancellations on + GET /memories/read-activity for heavier accounts.'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00109', 'usage_events_read_activity_idx') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00110_facets_within_group_cooccurrence.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- 00110 — within-group facet counting: co-occurrence for AND mode, and a +-- stable (zero-stays) value set for every dimension. +-- +-- THE BUG (two, related). `lorekit_memory_facets` (00108) counts each +-- dimension against every OTHER dimension's filter, self-excluding its own — +-- the drill-down convention that lets you widen or switch within the +-- dimension you are standing in. Two consequences of doing this +-- unconditionally: +-- +-- 1. Selecting a `label` in `all` (AND / "includes all") mode still +-- self-excluded the tag dimension, so every OTHER label's count ignored +-- the selection entirely rather than answering "how many memories carry +-- BOTH the selected label(s) AND this one" — the question an AND +-- selection actually poses. +-- 2. A value that had ZERO matching rows under the active filters simply +-- never appeared: the old `cells` CTE filtered rows to the matching set +-- BEFORE grouping, so a value only visible on now-excluded rows vanished +-- from the menu instead of reading as `count: 0`. A vanishing option +-- reads as "this value no longer exists"; a `0` reads as "not with your +-- current selection" — the honest answer. +-- +-- ── DESIGN (Decision D5) ───────────────────────────────────────────────── +-- +-- `v_tag_cooccur := (p_tags_mode = 'all')` — co-occurrence is gated on AND +-- mode. It is meaningless for a scalar column (`host = X` intersected with +-- `host = Y` is always empty — a memory has one host) and wrong for OR +-- multi-select (there you want what ADDING a value yields, i.e. the current +-- self-exclusion), so every scalar dimension and every non-`all` tag mode +-- keep self-exclusion exactly as 00108 left it. Only the tag branch's OWN +-- flag (`b.ok_tag`) is conditionally folded back in. +-- +-- Every branch now enumerates its full candidate set from `base` — the +-- unfiltered-by-dimension population `base` already restricts to +-- visibility / archived / tenant-scope / created-window / retention — and +-- computes its count with `count(*) filter (where )`, so a value +-- with zero matching rows still gets a `group by` row (via the `cells` CTE's +-- unconditional membership) with `count = 0`, instead of being absent. +-- +-- Response shape, RPC signature and grants are UNCHANGED (Decision D6) — only +-- the `cells` CTE's body and the final aggregation change from +-- "filter-then-group" to "group-then-filter-count". `migrations.test.sql` +-- §110 is the executable proof, including the AND vs. OR distinction and the +-- cross-group AND that must still narrow every dimension's counts. +-- ═════════════════════════════════════════════════════════════════════════ + +create or replace function lorekit_memory_facets( + p_user_id uuid, + p_archived boolean default false, + p_scope text default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in', + -- Owner dimension (00064). `personal` plus one slug per member org with + -- visible rows. All optional: null/absent = not filtered. + p_owner text[] default null, + p_owner_mode text default 'in', + -- The CALLING KEY's restriction (00068), defaulted to unrestricted. + p_key_scopes text[] default '{}', + p_key_org_access text default 'all', + p_key_org_ids uuid[] default '{}', + -- 00108: the created_at window and the five retention thresholds, so a count + -- describes the same rows the list shows. Appended, null-defaulted. + p_created_since timestamptz default null, + p_created_until timestamptz default null, + p_min_age_days integer default null, + p_unseen_days integer default null, + p_max_seen_count integer default null, + p_max_read_count integer default null, + p_max_opened_count integer default null ) -returns integer +returns table (facet text, value text, count bigint) language plpgsql +stable security definer set search_path = public as $$ +#variable_conflict use_column declare - v_count integer; + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + v_origin_pr integer[] := ( + select array_agg(x::integer) + from unnest(coalesce(p_origin_pr, '{}'::text[])) as x + where x ~ '^[0-9]+$' + ); + v_created_cutoff timestamptz := case + when p_min_age_days is null then null else now() - (p_min_age_days * interval '1 day') + end; + v_unseen_cutoff timestamptz := case + when p_unseen_days is null then null else now() - (p_unseen_days * interval '1 day') + end; + -- 00110: only `label`/tag has an `all` (AND / set-containment) mode — see + -- the module header. Every other dimension, and the tag facet in `in`/`nin` + -- mode, keep self-exclusion (unchanged). + v_tag_cooccur boolean := (p_tags_mode = 'all'); begin - delete from memories - where user_id = p_user_id - and archived_at is not null - and archived_at < now() - (p_retention_days * interval '1 day') - ; - get diagnostics v_count = row_count; - return v_count; + return query + with base as ( + select + m.tags, m.source_agent, m.trigger, m.kind, m.host, + m.origin_repo, m.origin_branch, m.origin_pr, + m.org_id, o.slug as org_slug, + -- Per-dimension match flag, from the shared predicates so it cannot + -- drift from lorekit_memory_activity's. A null filter is "not filtered" → + -- the helper returns true, so an untouched dimension never narrows. + lorekit_match_tags(m.tags, p_tags, p_tags_mode) as ok_tag, + lorekit_match_text(m.source_agent, p_source_agent, p_source_agent_mode) as ok_source_agent, + lorekit_match_text(m.trigger, p_trigger, p_trigger_mode) as ok_trigger, + lorekit_match_text(m.kind, p_kind, p_kind_mode) as ok_kind, + lorekit_match_text(m.host, p_host, p_host_mode) as ok_host, + lorekit_match_text(m.origin_repo, p_origin_repo, p_origin_repo_mode) as ok_origin_repo, + lorekit_match_text(m.origin_branch, p_origin_branch, p_origin_branch_mode) as ok_origin_branch, + lorekit_match_int (m.origin_pr, v_origin_pr, p_origin_pr_mode) as ok_origin_pr, + -- Owner: the computed identity is `personal` (org_id null) or the org slug. + -- Stays inline — it is not one of the three column helpers (00064). + (p_owner is null or case coalesce(p_owner_mode, 'in') + when 'nin' then ( + (case when m.org_id is null then 'personal' else o.slug end) is not null + and (case when m.org_id is null then 'personal' else o.slug end) <> all(p_owner) + ) + else ( + ('personal' = any(p_owner) and m.org_id is null) + or (m.org_id is not null and o.slug = any(p_owner)) + ) + end) as ok_owner + from memories m + -- A personal row has no org, so this is a LEFT join; org rows resolve to + -- their slug. Visible org rows are always the caller's own orgs (the + -- visibility predicate below admits them only via lorekit_member_org_ids), + -- so `o.slug` is never a slug the caller cannot see. + left join orgs o on o.id = m.org_id + where ( + (v_actor is null and auth.role() = 'service_role') + or m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) + ) + -- The calling key's restriction, applied ONCE here in the row-visibility + -- predicate every emitted facet value is derived from — `origin_repo` is + -- a repository name by construction, so an unnarrowed facet list leaks + -- exactly what the scope catalog hides. + and lorekit_api_token_scope_allowed(p_key_scopes, m.scope) + and lorekit_api_token_org_allowed(p_key_org_access, p_key_org_ids, m.org_id) + and ( + case + when p_archived then m.archived_at is not null + else m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + end + ) + and (p_scope is null or m.scope = p_scope) + -- 00108. Here rather than as an `ok_*` flag: these narrow the population + -- every facet value is drawn from, they are not a dimension that can be + -- self-excluded. Half-open window, matching lorekit_memory_list. + and (p_created_since is null or m.created_at >= p_created_since) + and (p_created_until is null or m.created_at < p_created_until) + and lorekit_match_retention( + m.created_at, m.last_opened_at, m.seen_count, m.read_count, m.opened_count, + v_created_cutoff, v_unseen_cutoff, p_max_seen_count, p_max_read_count, p_max_opened_count + ) + ), cells as ( + -- 00110: every branch now scans the FULL `base` population (not just rows + -- that already satisfy the other flags) and carries a `matched` boolean + -- instead of a row-level WHERE — the `group by` below therefore enumerates + -- every value the dimension has, and the final `count` reflects only the + -- ones actually matching. This is what keeps a non-matching value at + -- `count = 0` instead of dropping it. + select 'tag'::text as facet, t.tag as value, + (b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner + -- Co-occurrence: fold the tag dimension's OWN flag back in only in + -- `all` mode, so another label's count becomes "carries the + -- selection AND this label" rather than the plain self-excluded + -- total. `not v_tag_cooccur` keeps `in`/`nin` mode's existing + -- self-exclusion untouched. + and (not v_tag_cooccur or b.ok_tag)) as matched + from base b + cross join lateral unnest(b.tags) as t(tag) + union all + select 'source_agent', b.source_agent, + (b.ok_tag and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner) + from base b + where b.source_agent is not null + union all + select 'trigger', b.trigger, + (b.ok_tag and b.ok_source_agent and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner) + from base b + where b.trigger is not null + union all + select 'kind', b.kind, + (b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner) + from base b + where b.kind is not null + union all + select 'host', b.host, + (b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner) + from base b + where b.host is not null + union all + select 'origin_repo', b.origin_repo, + (b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner) + from base b + where b.origin_repo is not null + union all + select 'origin_branch', b.origin_branch, + (b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_pr and b.ok_owner) + from base b + where b.origin_branch is not null + union all + select 'origin_pr', b.origin_pr::text, + (b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_owner) + from base b + where b.origin_pr is not null + union all + -- Owner is the ONE dimension self-excluded here (every other flag, NOT + -- ok_owner), so a drilled-in owner still lists the alternative owner. + select 'owner', case when b.org_id is null then 'personal' else b.org_slug end, + (b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr) + from base b + ) + select c.facet, c.value, count(*) filter (where c.matched) as count + from cells c + where c.value is not null + and btrim(c.value) <> '' + group by c.facet, c.value + order by c.facet asc, count(*) filter (where c.matched) desc, c.value asc; end; $$; -create or replace function purge_expired_memories( - p_user_id uuid -) -returns integer +-- Signature is IDENTICAL to 00108's — a true `create or replace`, no drop +-- needed, and the 00108 grants persist untouched. Comment refreshed to +-- describe the new within-group semantics. +comment on function lorekit_memory_facets( + uuid, boolean, text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[], timestamptz, timestamptz, integer, integer, integer, + integer, integer +) is + 'Value catalog with counts for every filterable memory dimension (tag, + source_agent, trigger, kind, host, origin_repo, origin_branch, origin_pr, + owner) over the partition selected by p_archived, visible to the EFFECTIVE + caller and narrowed by the calling key''s own scope/org restriction. Each + dimension is counted with every OTHER dimension filter applied (cross-group + AND). WITHIN a dimension: self-exclusion (drill-down, 00057) for every + scalar dimension and for label/tag in `in`/`nin` (OR) mode; CO-OCCURRENCE + (intersection with the tag selection) for label/tag in `all` (AND) mode + (00110). Every dimension enumerates its FULL value set so a value with zero + matches under the active filters still returns `count = 0` instead of being + omitted (00110). The created_at window and the five retention thresholds + (00108) are NOT self-excluded: they narrow the population every facet value + is drawn from, so a count matches the list under the same parameters. + q/key/expiring_within_days remain unmirrored, so with those active a count + is an upper bound.'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00110', 'facets_within_group_cooccurrence') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00111_wildcard_scope_org_bindings.sql +-- ───────────────────────────────────────────────────────────────────────────── + +-- ═════════════════════════════════════════════════════════════════════════ +-- Wildcard scope → org bindings: bind a WHOLE prefix (`repo::owner/*`, +-- `branch::owner/repo::*`) to an org instead of one concrete scope at a time, +-- and resolve writes under a MOST-SPECIFIC-WINS rule when several bindings +-- could match. +-- +-- ── What changes, and why ─────────────────────────────────────────────── +-- +-- 1. `lorekit_scope_bind` gains a SQL-side grammar gate: the pattern must +-- satisfy `lorekit_api_token_scopes_valid` (00068) — the SAME charset and +-- wildcard-position rule already enforced for API-token scope allowlists, +-- reused rather than forked so the two authorities can never drift. The +-- gate sits AFTER the `manage_scopes` authorization check, so an +-- authorized owner submitting a malformed pattern is refused by the +-- grammar, not the auth layer — and a non-admin is still refused by auth +-- first, unchanged. `lorekit_scope_unbind` needs no grammar gate: deleting +-- a row by its exact stored string can't insert a bad one. +-- +-- 2. `memory_write`'s no-explicit-org branch changes from a single-row +-- equijoin (`where b.scope = p_scope`) to a wildcard-aware match: +-- +-- where o.deleted_at is null +-- and (b.scope = p_scope +-- or (right(b.scope, 1) = '*' +-- and p_scope like replace(left(b.scope, -1), '_', '\_') || '%')) +-- order by (right(b.scope, 1) = '*') asc, length(b.scope) desc, b.scope asc +-- limit 1 +-- +-- Precedence, in the ORDER BY: an EXACT string always beats ANY wildcard +-- (`false < true` on the first key); among wildcards, the LONGER literal +-- prefix wins (more specific); ties break on the scope string itself, +-- ascending, purely for determinism — two distinct wildcard prefixes of +-- EQUAL length cannot both match one `p_scope` (prefix containment forces +-- different lengths), and two identical exact strings can't coexist (the +-- unique index on `scope`), so the third key is a documented safeguard, +-- never a reachable branch. The LIKE escaping mirrors +-- `lorekit_api_token_scope_allowed` (00068) byte-for-byte — the pattern's +-- own CHECK already excludes `%` and `\`, so the escape only neutralises a +-- literal `_`. +-- +-- Every other branch of `memory_write` (explicit `p_org_slug`, the +-- service-role write, the personal fallback, the returned columns) is +-- copied VERBATIM from 00075 — the current definition (confirmed by +-- grepping every `memory_write` definer) — so this migration's diff is +-- exactly the matcher above. +-- +-- 3. Overlapping bindings are ALLOWED — an exact scope and a wildcard, or two +-- wildcards of different depth, may coexist, even bound to different +-- orgs. Only an IDENTICAL pattern STRING bound to a second org still +-- raises `scope_bound_elsewhere` (the existing unique index on `scope` is +-- unchanged — it was never a uniqueness-of-match constraint, only a +-- uniqueness-of-literal-row one). +-- +-- No REST route, OpenAPI path, `rest-tool-name` mapping, or new audit action +-- is added — bindings stay the dashboard-RPC-only exempt surface, and +-- wildcards reuse the existing `scope.bind` / `scope.unbind` audit actions. +-- ═════════════════════════════════════════════════════════════════════════ + +-- 0. `org_scope_bindings.scope` gets the SAME shape gate at the table level, +-- not only inside `lorekit_scope_bind`. Without this, a row inserted +-- before this migration under the old (non-empty-only) validation — or by +-- any future direct INSERT that bypasses the RPC — could hold a scope +-- string ending in `*` with no charset guarantee, and the LIKE-escaping in +-- `memory_write`'s matcher below is only injection-safe because it relies +-- on this exact shape (no `%`/`\` in a valid pattern). A CHECK makes any +-- such row fail this migration LOUDLY at deploy time instead of silently +-- starting to behave as a live wildcard the moment 00111 ships. +alter table org_scope_bindings + drop constraint if exists org_scope_bindings_scope_shape; +alter table org_scope_bindings + add constraint org_scope_bindings_scope_shape + check (lorekit_api_token_scopes_valid(array[scope])); + +-- 1. `lorekit_scope_bind` — add the grammar gate after the auth gate. +create or replace function lorekit_scope_bind(p_org_id uuid, p_scope text) +returns uuid language plpgsql security definer set search_path = public as $$ declare - v_count integer; + v_actor uuid := auth.uid(); + v_existing_org uuid; + v_id uuid; begin - delete from memories - where user_id = p_user_id - and expires_at is not null - and expires_at < now() - and archived_at is null - ; - get diagnostics v_count = row_count; - return v_count; + if p_scope is null or length(trim(p_scope)) = 0 then + raise exception using errcode = 'P0001', message = 'scope is required'; + end if; + + if not lorekit_org_can(v_actor, p_org_id, 'manage_scopes') then + raise exception using errcode = 'LK002', + message = format('org_permission_denied: org=%s capability=manage_scopes', p_org_id); + end if; + + -- Grammar gate, deliberately AFTER the authorization gate above so the two + -- layers are independently provable: an unauthorized actor is refused by + -- auth regardless of pattern shape, and an authorized actor with a bad + -- pattern is refused here. Reuses 00068's shape validator verbatim — this + -- is a SHAPE check (charset + wildcard position), not full canonical-scope + -- validation; the dashboard's `validateScopeBindingPattern` does the + -- stronger check for a non-wildcard string before this RPC is ever called. + if not lorekit_api_token_scopes_valid(array[p_scope]) then + raise exception using errcode = 'P0001', + message = format('invalid_scope_pattern: %s', p_scope); + end if; + + select org_id, id into v_existing_org, v_id from org_scope_bindings where scope = p_scope; + if v_existing_org = p_org_id then + -- Already bound to this org — idempotent no-op; return the existing id. + return v_id; + elsif v_existing_org is not null then + raise exception using errcode = 'P0001', + message = format('scope_bound_elsewhere: %s', p_scope); + end if; + + insert into org_scope_bindings (org_id, scope, created_by) + values (p_org_id, p_scope, v_actor) + returning id into v_id; + return v_id; end; $$; --- ───────────────────────────────────────────────────────────────────────────── --- 7. memory_write RPC --- --- Accepts the same parameters as the hosted memory_write (migration 00031) --- so no tool-handler code needs to change. --- --- BYOD note: p_org_slug is accepted for parameter-signature compatibility --- but is silently ignored — org ownership requires the hosted product's org --- management RPCs and orgs table. Writes always land in the personal --- (user-scoped) partition. A comment in the return is added for callers that --- inspect org_routed. --- --- The three key-restriction parameters (00068/00069) are NOT optional --- cosmetics. PostgREST resolves an RPC by argument NAME, so `create.ts` --- sending p_key_scopes / p_key_org_access / p_key_org_ids at a BYOD install --- whose function lacks them misses the function entirely (PGRST202) and --- surfaces as an opaque 500. And the scope allowlist is the LAST gate on the --- write path a caller cannot route around — the edge holds the service-role --- key, so the dispatcher's refusal is advisory by construction. Without it a --- BYOD install got the columns and the CHECKs with no SQL-layer enforcement --- at all. Guard kept byte-identical to 00069 §1. --- ───────────────────────────────────────────────────────────────────────────── - --- Every earlier signature is dropped, newest first: `create or replace` keys on --- the argument list, so growing the parameter list leaves the previous one --- behind as an overload that PostgREST would still resolve for a caller sending --- the old argument names. -drop function if exists memory_write(uuid, text, text, text, text[], text, text, timestamptz, text, integer, boolean); -drop function if exists memory_write(uuid, text, text, text, text[], text, text, timestamptz, text, integer); -drop function if exists memory_write(uuid, text, text, text, text[], text, text, timestamptz); -drop function if exists memory_write(uuid, text, text, text, text[], text, text); +comment on function lorekit_scope_bind(uuid, text) is + 'Bind a scope pattern (exact or owner wildcard, e.g. repo::owner/*) to an ' + 'org. Admin/owner only (manage_scopes); pattern must pass ' + 'lorekit_api_token_scopes_valid or raises invalid_scope_pattern.'; +-- 2. `memory_write` — wildcard-aware most-specific-wins binding resolution. +-- Signature and RETURNS TABLE shape are UNCHANGED from 00075, so no `drop`. create or replace function memory_write( - p_user_id uuid, - p_scope text, - p_key text, - p_value text, - p_tags text[] default '{}', - p_source_agent text default null, - p_trigger text default null, - p_created_at timestamptz default null, - p_org_slug text default null, -- accepted but ignored in BYOD (no orgs table) - p_ttl_days integer default null, - p_clear_ttl boolean default false, - -- The CALLING KEY's restriction (00068/00069), defaulted to unrestricted so - -- every existing BYOD caller keeps its behaviour with no call-site change. - p_key_scopes text[] default '{}', - -- The tenancy pair is accepted for signature compatibility and is INERT here: - -- BYOD has no orgs, so every row is personal and - -- `lorekit_api_token_org_allowed` returns true regardless of the tenancy. - -- Accepting it is what stops PostgREST's by-name resolution from missing this - -- function; ignoring it is correct rather than lax. - p_key_org_access text default 'all', - p_key_org_ids uuid[] default '{}' + p_user_id uuid, + p_scope text, + p_key text, + p_value text, + p_tags text[] default '{}', + p_source_agent text default null, + p_trigger text default null, + p_created_at timestamptz default null, + p_org_slug text default null, + p_ttl_seconds integer default null, + p_clear_ttl boolean default false, + p_origin_repo text default null, + p_origin_branch text default null, + p_origin_commit text default null, + p_origin_pr integer default null, + p_kind text default null, + p_host text default null, + p_key_scopes text[] default '{}', + p_key_org_access text default 'all', + p_key_org_ids uuid[] default '{}' ) returns table ( id uuid, @@ -679,25 +20855,38 @@ returns table ( inserted boolean, org_routed boolean, binding_org_slug text, - expires_at timestamptz + expires_at timestamptz, + scope text, + key text, + value text, + tags text[], + source_agent text, + trigger text, + updated_at timestamptz, + archived_at timestamptz, + origin_repo text, + origin_branch text, + origin_commit text, + origin_pr integer, + kind text, + host text, + seen_count integer ) language plpgsql +security definer set search_path = public as $$ +-- See 00075's comment for why this directive is required (the OUT columns +-- `scope`/`key` collide with the bare column names in the `on conflict` +-- arbiter lists below). +#variable_conflict use_column declare - v_expires_at timestamptz; - -- Tri-state for the expires_at UPDATE branch: - -- 'clear' → set to NULL - -- 'set' → set to v_expires_at - -- 'keep' → leave unchanged - v_ttl_action text := 'keep'; + v_org_id uuid; + v_binding_org uuid; + v_binding_slug text; + v_expires_at timestamptz; + v_ttl_action text := 'keep'; begin - -- p_org_slug is intentionally ignored in BYOD. - -- Org-owned writes require the hosted LoreKit product. - - -- The scope allowlist, checked FIRST and for every branch — 00069 §1 - -- verbatim. LK002 is the code `translateDbError` already maps to a 403 on - -- REST and a forbidden error on MCP, so no second mapping is needed. if not lorekit_api_token_scope_allowed(p_key_scopes, p_scope) then raise exception using errcode = 'LK002', message = format('key_scope_denied: scope=%s', p_scope); @@ -705,181 +20894,1463 @@ begin if p_clear_ttl then v_ttl_action := 'clear'; - elsif p_ttl_days is not null then - if p_ttl_days < 1 or p_ttl_days > 365 then + elsif p_ttl_seconds is not null then + if p_ttl_seconds < 1 or p_ttl_seconds > 31536000 then raise exception using errcode = 'P0001', - message = format('ttl_days must be between 1 and 365, got %s', p_ttl_days); + message = format('ttl_seconds must be between 1 and 31536000, got %s', p_ttl_seconds); end if; - v_expires_at := now() + (p_ttl_days * interval '1 day'); + v_expires_at := now() + (p_ttl_seconds * interval '1 second'); v_ttl_action := 'set'; end if; - if p_user_id is null then - -- service-role / CI writes: (scope, key) partial index for null user_id + if p_org_slug is not null then + select o.id into v_org_id from orgs o where o.slug = p_org_slug and o.deleted_at is null; + if v_org_id is null then + raise exception using errcode = 'P0001', message = format('unknown_org: %s', p_org_slug); + end if; + if not lorekit_org_can(p_user_id, v_org_id, 'write') then + raise exception using errcode = 'LK002', message = format('org_permission_denied: org=%s', p_org_slug); + end if; + if not lorekit_api_token_org_allowed(p_key_org_access, p_key_org_ids, v_org_id) then + raise exception using errcode = 'LK002', + message = format('key_org_denied: org=%s', p_org_slug); + end if; + else + -- Wildcard-aware, most-specific-wins resolution. A binding matches + -- p_scope either by exact string equality, or — when it ends in '*' — as + -- an owner-wildcard prefix (the same LIKE-escaping `lorekit_api_token_ + -- scope_allowed` (00068) uses, since `%` and `\` cannot occur in a + -- pattern that already passed the grammar gate above). Among matches: + -- exact (is-wildcard = false) always sorts before any wildcard; among + -- wildcards, the LONGER literal prefix (length desc) sorts first as the + -- more specific match; the trailing `scope asc` is a determinism + -- safeguard for a tie that provably cannot occur (see the header + -- comment), not a reachable precedence branch. + select b.org_id, o.slug into v_binding_org, v_binding_slug + from org_scope_bindings b + join orgs o on o.id = b.org_id + where o.deleted_at is null + and ( + b.scope = p_scope + or ( + right(b.scope, 1) = '*' + and p_scope like replace(left(b.scope, -1), '_', '\_') || '%' + ) + ) + order by (right(b.scope, 1) = '*') asc, length(b.scope) desc, b.scope asc + limit 1; + + if v_binding_org is not null + and p_user_id is not null + and lorekit_org_can(p_user_id, v_binding_org, 'write') + and lorekit_api_token_org_allowed(p_key_org_access, p_key_org_ids, v_binding_org) then + v_org_id := v_binding_org; + end if; + end if; + + if v_org_id is not null then + return query + insert into memories ( + user_id, org_id, scope, key, value, tags, source_agent, trigger, + created_at, updated_at, created_by, updated_by, expires_at, + origin_repo, origin_branch, origin_commit, origin_pr, kind, host, seen_count + ) + values ( + null, v_org_id, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, + coalesce(p_created_at, now()), coalesce(p_created_at, now()), + p_user_id, p_user_id, v_expires_at, + p_origin_repo, p_origin_branch, p_origin_commit, p_origin_pr, p_kind, p_host, 1 + ) + on conflict (org_id, scope, key) where org_id is not null and archived_at is null + do update set + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + updated_by = p_user_id, + seen_count = memories.seen_count + 1, + origin_repo = coalesce(excluded.origin_repo, memories.origin_repo), + origin_branch = coalesce(excluded.origin_branch, memories.origin_branch), + origin_commit = coalesce(excluded.origin_commit, memories.origin_commit), + origin_pr = coalesce(excluded.origin_pr, memories.origin_pr), + kind = coalesce(excluded.kind, memories.kind), + host = coalesce(excluded.host, memories.host), + expires_at = case v_ttl_action + when 'clear' then null + when 'set' then v_expires_at + else memories.expires_at + end + returning + memories.id, memories.created_at, (xmax::text = '0') as inserted, + true as org_routed, v_binding_slug as binding_org_slug, memories.expires_at, + memories.scope, memories.key, memories.value, memories.tags, + memories.source_agent, memories.trigger, memories.updated_at, memories.archived_at, + memories.origin_repo, memories.origin_branch, memories.origin_commit, memories.origin_pr, + memories.kind, memories.host, memories.seen_count; + + elsif p_user_id is null then return query insert into memories ( user_id, scope, key, value, tags, source_agent, trigger, - created_at, updated_at, created_by, updated_by, expires_at + created_at, updated_at, created_by, updated_by, expires_at, + origin_repo, origin_branch, origin_commit, origin_pr, kind, host, seen_count ) values ( null, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, coalesce(p_created_at, now()), coalesce(p_created_at, now()), - null, null, v_expires_at + null, null, v_expires_at, + p_origin_repo, p_origin_branch, p_origin_commit, p_origin_pr, p_kind, p_host, 1 ) on conflict (scope, key) where org_id is null and user_id is null and archived_at is null do update set - value = excluded.value, - tags = excluded.tags, - source_agent = excluded.source_agent, - trigger = excluded.trigger, - updated_at = now(), - expires_at = case v_ttl_action - when 'clear' then null - when 'set' then v_expires_at - else memories.expires_at - end + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + seen_count = memories.seen_count + 1, + origin_repo = coalesce(excluded.origin_repo, memories.origin_repo), + origin_branch = coalesce(excluded.origin_branch, memories.origin_branch), + origin_commit = coalesce(excluded.origin_commit, memories.origin_commit), + origin_pr = coalesce(excluded.origin_pr, memories.origin_pr), + kind = coalesce(excluded.kind, memories.kind), + host = coalesce(excluded.host, memories.host), + expires_at = case v_ttl_action + when 'clear' then null + when 'set' then v_expires_at + else memories.expires_at + end returning memories.id, memories.created_at, (xmax::text = '0') as inserted, - false as org_routed, null::text as binding_org_slug, memories.expires_at; + false as org_routed, v_binding_slug as binding_org_slug, memories.expires_at, + memories.scope, memories.key, memories.value, memories.tags, + memories.source_agent, memories.trigger, memories.updated_at, memories.archived_at, + memories.origin_repo, memories.origin_branch, memories.origin_commit, memories.origin_pr, + memories.kind, memories.host, memories.seen_count; else - -- user-scoped writes: (user_id, scope, key) partial index return query insert into memories ( user_id, scope, key, value, tags, source_agent, trigger, - created_at, updated_at, created_by, updated_by, expires_at + created_at, updated_at, created_by, updated_by, expires_at, + origin_repo, origin_branch, origin_commit, origin_pr, kind, host, seen_count ) values ( p_user_id, p_scope, p_key, p_value, p_tags, p_source_agent, p_trigger, coalesce(p_created_at, now()), coalesce(p_created_at, now()), - p_user_id, p_user_id, v_expires_at + p_user_id, p_user_id, v_expires_at, + p_origin_repo, p_origin_branch, p_origin_commit, p_origin_pr, p_kind, p_host, 1 ) on conflict (user_id, scope, key) where org_id is null and user_id is not null and archived_at is null do update set - value = excluded.value, - tags = excluded.tags, - source_agent = excluded.source_agent, - trigger = excluded.trigger, - updated_at = now(), - updated_by = p_user_id, - expires_at = case v_ttl_action - when 'clear' then null - when 'set' then v_expires_at - else memories.expires_at - end + value = excluded.value, + tags = excluded.tags, + source_agent = excluded.source_agent, + trigger = excluded.trigger, + updated_at = now(), + updated_by = p_user_id, + seen_count = memories.seen_count + 1, + origin_repo = coalesce(excluded.origin_repo, memories.origin_repo), + origin_branch = coalesce(excluded.origin_branch, memories.origin_branch), + origin_commit = coalesce(excluded.origin_commit, memories.origin_commit), + origin_pr = coalesce(excluded.origin_pr, memories.origin_pr), + kind = coalesce(excluded.kind, memories.kind), + host = coalesce(excluded.host, memories.host), + expires_at = case v_ttl_action + when 'clear' then null + when 'set' then v_expires_at + else memories.expires_at + end returning memories.id, memories.created_at, (xmax::text = '0') as inserted, - false as org_routed, null::text as binding_org_slug, memories.expires_at; + false as org_routed, v_binding_slug as binding_org_slug, memories.expires_at, + memories.scope, memories.key, memories.value, memories.tags, + memories.source_agent, memories.trigger, memories.updated_at, memories.archived_at, + memories.origin_repo, memories.origin_branch, memories.origin_commit, memories.origin_pr, + memories.kind, memories.host, memories.seen_count; end if; end; $$; -grant execute on function memory_write(uuid, text, text, text, text[], text, text, timestamptz, text, integer, boolean, text[], text, uuid[]) - to anon, authenticated, service_role; +grant execute on function memory_write( + uuid, text, text, text, text[], text, text, timestamptz, text, + integer, boolean, text, text, text, integer, text, text, text[], text, uuid[] +) to anon, authenticated, service_role; +insert into supabase_migrations.schema_migrations (version, name) +values ('00111', 'wildcard_scope_org_bindings') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00112_relevant_ranking.sql -- ───────────────────────────────────────────────────────────────────────────── --- 8. memory_delete RPC + +-- ═════════════════════════════════════════════════════════════════════════ +-- lorekit_memory_relevant_candidates — the candidate set behind +-- `GET /memories/relevant`, selected and GRADED by Postgres. +-- +-- WHAT THIS REPLACES, and why it had to become an RPC. The handler used to +-- build the candidate fetch with PostgREST: +-- +-- .textSearch('fts', q, { type: 'websearch', config: 'english' }) +-- .order('updated_at', { ascending: false }) +-- .limit(CANDIDATE_LIMIT) +-- +-- which buys two honest limits the handler documented and could not fix from +-- the TypeScript side: +-- +-- 1. RELEVANCE WAS BINARY. `ts_rank`/`ts_rank_cd` is not projectable through +-- PostgREST's query grammar, so a row that matched scored 1 and a row +-- that did not was never returned — nothing scored between. Ordering +-- among matches was decided entirely by recency and salience. +-- 2. THE CANDIDATE WINDOW WAS CUT BY RECENCY, NOT BY RANK. On a store with +-- more matching rows than the cap, the best-matching lesson could simply +-- fall outside the window and never be scored at all. +-- +-- Both are the same fix: choose and grade the candidates IN Postgres. This +-- function returns the route's narrow projection (`scope, key, value, +-- seen_count, updated_at, tags, origin_pr`) plus a `relevance` column +-- = `ts_rank_cd(fts, websearch_to_tsquery('english', p_q))`, and — when a +-- query is supplied — cuts the window in RANK order instead of recency order. +-- +-- DETERMINISTIC AND EMBEDDING-FREE, BY CONSTRUCTION. `ts_rank_cd` is a pure, +-- reproducible Postgres function over the same `fts` column `POST +-- /memories/search` already matches on. There is no model, no vector index and +-- no external call anywhere on this path: the same request against the same +-- rows returns the same rows in the same order, run to run. That is a +-- deliberate product decision, not an implementation detail — see +-- docs/decisions.md → "Graded relevance is lexical `ts_rank_cd`, never a +-- vector score". +-- +-- WHY `ts_rank_cd` AND NOT `ts_rank`. Cover density rewards query terms that +-- appear CLOSE TOGETHER, which is the signal that separates a lesson actually +-- about "migration backfill" from one that mentions migrations in its first +-- paragraph and backfills in its last. `ts_rank` counts occurrences without +-- regard to proximity and would rank the second one just as highly. -- --- Personal-only variant (no org gating — BYOD has no orgs table). --- p_org_slug is accepted for signature compatibility but ignored. +-- THE SCORER STILL LIVES IN TYPESCRIPT. This function does NOT rank — it +-- SELECTS and GRADES. The composite score stays in the shared +-- `lesson-rank.ts`/`lessons-pure.mjs` pair, because it is set-relative +-- (salience normalises against the most-recurring candidate) and must agree +-- exactly with the CLI hook's ordering, an agreement `lesson-rank-parity.spec.ts` +-- can only hold two TypeScript-family implementations to. A plpgsql third copy +-- would be unguardable. The split is unchanged: Postgres selects, TypeScript +-- orders — this migration only makes the SELECT side say how well each row +-- matched instead of merely that it did. +-- +-- TENANCY IS COMPOSED, NEVER RE-DERIVED. Three predicates, each the same +-- function its TypeScript counterpart mirrors: +-- * `lorekit_member_org_ids` — the single tenant-visibility source (00014), +-- exactly as `rls_read` (00015) and every sibling analytics RPC +-- (`lorekit_memory_read_ranking` 00085, `lorekit_read_activity`) ask it. +-- * `lorekit_api_token_org_allowed` (00068) — the SQL twin of +-- `effectiveOrgIds`/`ownRowsFragment` in `tenant-scope.ts`. Membership AND +-- the key's tenancy, which is the intersection the TypeScript filter +-- computes; a personal row (`org_id is null`) stays reachable under every +-- tenancy, and the owner's OWN org-owned rows drop out under a restricted +-- key — the case a bare `user_id = v_actor` disjunct would wrongly admit. +-- * `lorekit_api_token_scope_allowed` (00068/00069) — the SQL twin of +-- `keyScopeFilter`. A malformed stored pattern is dropped on both sides, +-- so a key can only ever be narrowed by one. +-- `security definer` (00085's shape) rather than invoker: the api_key tier +-- calls this on the SERVICE-ROLE client, where invoker would mean no RLS and +-- no predicate at all. Definer gives ONE predicate serving both auth tiers +-- instead of a predicate for one and RLS for the other. +-- +-- INDEXES: no new index. The `q` path is an `fts @@ tsquery` match served by +-- `memories_fts_idx` (00001, GIN); the no-`q` path is `updated_at desc, id +-- desc` served by `memories_user_updated_at_id_idx` (00096). `ts_rank_cd` is +-- evaluated only over rows the index already returned — it is a recheck on a +-- bounded set, never a scan driver. +-- ═════════════════════════════════════════════════════════════════════════ + +create or replace function lorekit_memory_relevant_candidates( + p_user_id uuid, + p_q text default null, + p_scopes text[] default '{}', + p_key_scopes text[] default '{}', + p_key_org_access text default 'all', + p_key_org_ids uuid[] default '{}', + p_limit integer default 200 +) +returns table ( + scope text, + key text, + value text, + seen_count integer, + updated_at timestamptz, + tags text[], + origin_pr integer, + relevance double precision +) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + -- Clamped the same way every sibling analytics RPC clamps: the caller names + -- the candidate cap, but the cap is this function's cost ceiling, not the + -- caller's to raise without bound. + v_limit integer := least(greatest(coalesce(p_limit, 200), 1), 1000); + -- NULL and '' are BOTH "no query", matching the handler's `if (params.q)` + -- falsiness exactly so the pre-RPC behaviour is preserved byte for byte. + -- + -- A non-empty `p_q` whose tsquery comes back EMPTY (every term a stopword, + -- e.g. 'the') is deliberately NOT folded into the no-query path: an empty + -- tsquery matches nothing, so such a request returns no candidates — which + -- is exactly what the PostgREST `.textSearch` form did. Folding it into "no + -- query" would silently turn a request that asked a question into the + -- unfiltered recency list, which is a different answer, not a better one. + v_query tsquery := case + when p_q is null or p_q = '' then null + else websearch_to_tsquery('english', p_q) + end; +begin + return query + select + m.scope, + m.key, + m.value, + m.seen_count, + m.updated_at, + m.tags, + m.origin_pr, + -- NULL when no query was asked — the absence of a relevance opinion, + -- which the handler maps to 0 exactly as the binary form did. `greatest( + -- …, 0)` is a floor, not a clamp: ts_rank_cd is non-negative by + -- definition, and the floor documents that an authorization-adjacent + -- numeric never leaves this function negative. + case + when v_query is null then null::double precision + else greatest(ts_rank_cd(m.fts, v_query), 0)::double precision + end as relevance + from memories m + where ( + -- Service-role with no actor (CI) reads everything, the same + -- escape hatch 00085 carries. + (v_actor is null and auth.role() = 'service_role') + or m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) + ) + -- Active lore only — the partition every read path applies. An archived + -- or expired lesson is not a candidate for "what should I read". + and m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + and lorekit_api_token_org_allowed( + coalesce(p_key_org_access, 'all'), + coalesce(p_key_org_ids, '{}'::uuid[]), + m.org_id + ) + and lorekit_api_token_scope_allowed(p_key_scopes, m.scope) + and (coalesce(cardinality(p_scopes), 0) = 0 or m.scope = any(p_scopes)) + and (v_query is null or m.fts @@ v_query) + order by + -- RANK FIRST when there is a rank to sort by — this is the half that + -- closes the "recency-windowed, not global" caveat. With no query every + -- row's sort key is NULL, so the expression collapses and the ordering + -- is the pre-RPC `updated_at desc, id desc` exactly. + case when v_query is null then null else ts_rank_cd(m.fts, v_query) end + desc nulls last, + m.updated_at desc, + -- The total-order tie-break. Without it the set that survives the cap + -- could differ between two identical requests, which is the one way + -- this endpoint could be non-deterministic without any caller being + -- able to see why. + m.id desc + limit v_limit; +end; +$$; + +revoke execute on function lorekit_memory_relevant_candidates(uuid, text, text[], text[], text, uuid[], integer) + from public, anon; +grant execute on function lorekit_memory_relevant_candidates(uuid, text, text[], text[], text, uuid[], integer) + to authenticated, service_role; + +comment on function lorekit_memory_relevant_candidates(uuid, text, text[], text[], text, uuid[], integer) is + 'Candidate rows for GET /memories/relevant, graded by ts_rank_cd over the + memories.fts column. With p_q set the window is cut in RANK order (closing + the recency-window caveat) and relevance is the raw, unbounded ts_rank_cd + value -- the caller normalises it into [0,1]. With p_q null or empty the + window is cut in updated_at desc order and relevance is NULL. Active rows + only (non-archived, non-expired). Visibility is lorekit_member_org_ids + narrowed by lorekit_api_token_org_allowed and lorekit_api_token_scope_allowed, + so it can never be wider than the RLS read policy or the calling key. + Deterministic and embedding-free: no model, no vector index, no external + call. p_limit is clamped to [1, 1000].'; + +insert into supabase_migrations.schema_migrations (version, name) +values ('00112', 'relevant_ranking') +on conflict (version) do nothing; + +-- ───────────────────────────────────────────────────────────────────────────── +-- 00113_explorer_reverse_activity_filters_and_opened_sort.sql -- ───────────────────────────────────────────────────────────────────────────── --- The pre-00069 signature is DROPPED rather than replaced: `create or replace` --- keys on the argument list, so adding parameters would leave two overloads and --- PostgREST would resolve the old one for a caller that omits them. -drop function if exists memory_delete(uuid, text, text, text, boolean); --- 00071 mirrored: the return gained `existed`, and `create or replace` cannot --- change a return type, so the 00069-shaped 8-arg form is dropped as well. -drop function if exists memory_delete(uuid, text, text, text, boolean, text[], text, uuid[]); +-- ═════════════════════════════════════════════════════════════════════════ +-- 00113 — the Explorer's reverse directions, and a "most chosen" sort. +-- +-- THE GAP. Every age/activity condition the four readers accept points ONE +-- way, because they were built for retention policies: `max_seen_count`, +-- `max_read_count`, `max_opened_count` ("N or fewer"), and `unseen_days` +-- ("not opened for N days"). Those are exactly the questions a groomer asks +-- before archiving. They cannot ask the opposite question — "which lessons are +-- used the MOST?", "what did an agent open this week?" — so the Explorer could +-- show you the dead weight but never the load-bearing lore. +-- +-- THE CHANGE. Four more conditions, each the mirror of one that exists: +-- +-- min_seen_count written at least N times (↔ max_seen_count) +-- min_read_count delivered at least N times (↔ max_read_count) +-- min_opened_count deliberately fetched ≥ N times (↔ max_opened_count) +-- opened_within_days last fetched within N days (↔ unseen_days) +-- +-- plus `p_sort => 'opened_count'` on `lorekit_memory_list`, so "most chosen +-- first" is an ORDER rather than a threshold the reader has to guess. +-- +-- They are READ conditions only. `GroomConditionsSchema` does not gain them and +-- no groom RPC is touched: archiving "everything chosen at least 10 times" is +-- not a retention policy anyone wants, and a policy that could express it would +-- be one careless edit away from archiving the most useful lore in the store. +-- +-- ── DESIGN ─────────────────────────────────────────────────────────────── +-- +-- SAME HELPER, WIDENED. The four new conditions go into +-- `lorekit_match_retention` rather than a second helper, for the reason 00108 +-- created it: every reader composes ONE predicate, so a count cannot disagree +-- with the list it describes (docs/decisions.md, "A COUNT must describe the +-- rows its LIST returns"). Its signature changes, so the 14-argument form is +-- created first, every caller is recreated against it, and the 10-argument +-- form is dropped last. +-- +-- `opened_within_days` IS NOT `unseen_days` NEGATED. `unseen_days` ages a +-- never-opened lesson from `created_at` (00100) so that "unopened for 90 days" +-- can find the lore nothing ever touched. Applying that same fallback here +-- would make a lesson written yesterday and never opened match "opened within +-- 7 days" — a claim about an agent's behaviour that no agent made. So a null +-- `last_opened_at` never satisfies it. +-- +-- THE FLOORS USE `>=` AND START AT THE FIRST VALUE THAT NARROWS at the API +-- (the schemas' bound): 1 for the read counters, since `>= 0` matches every +-- row, and 2 for `min_seen_count`, since `seen_count >= 1` is a table +-- constraint (00059). Accepting a no-op floor would put a pill on screen that +-- narrows nothing. +-- +-- THE COUNT SORT GETS ITS OWN CURSOR PARAMETER. The keyset predicate compares +-- the sort column against the last row's value; the date sorts carry it in +-- `p_cursor_ts timestamptz`, and an integer column needs an integer bound, so +-- `p_cursor_count` is appended and the format() splices in whichever +-- placeholder ($29 or $44) the sort needs. Ties break on `id desc`, exactly as +-- the date sorts do, so a page boundary inside a run of equal counts is stable. +-- +-- ── SIGNATURES ─────────────────────────────────────────────────────────── +-- +-- Appended, null-defaulted parameters on all four readers, so every existing +-- named-argument caller is unaffected. `create or replace` cannot change a +-- signature (00108's header), so each reader is dropped at its EXACT previous +-- argument list first — §108's AC-7 and §113 assert that exactly one overload +-- of each survives. +-- +-- §113 in `migrations.test.sql` is the executable proof. +-- ═════════════════════════════════════════════════════════════════════════ -create or replace function memory_delete( - p_user_id uuid, - p_org_slug text default null, -- accepted but ignored in BYOD - p_scope text default null, - p_key text default null, - p_force boolean default false, - -- Same three as memory_write, for the same two reasons: PostgREST resolves by - -- argument NAME (remove.ts sends all three), and this RPC chooses its own - -- rows, so `applyKeyScopeFilter` never sees them and the allowlist has - -- nowhere else to be enforced. - p_key_scopes text[] default '{}', - p_key_org_access text default 'all', -- inert in BYOD (no orgs) - p_key_org_ids uuid[] default '{}' -- inert in BYOD (no orgs) +-- ── 1. The predicate, widened ─────────────────────────────────────────────── + +-- The five retention thresholds plus the Explorer's four reverse directions, as +-- one row-level test. Every parameter is independently optional (null = not +-- filtered), and all null is a constant `true` the planner discards. +-- +-- `immutable` + `parallel safe` + `language sql` for 00108's reason: the result +-- depends only on the arguments, which is what lets the planner fold the call +-- into an index-usable expression. It still reads no clock — the caller +-- resolves `opened_within_days` to an instant exactly as it does the two +-- existing day counts. +create or replace function lorekit_match_retention( + -- The row's own counters. + p_created_at timestamptz, + p_last_opened_at timestamptz, + p_seen_count integer, + p_read_count integer, + p_opened_count integer, + -- The policy-expressible conditions (00108), day counts already resolved. + p_created_cutoff timestamptz, + p_unseen_cutoff timestamptz, + p_max_seen_count integer, + p_max_read_count integer, + p_max_opened_count integer, + -- 00113: the Explorer's reverse directions. Never carried by a policy. + p_opened_since timestamptz, + p_min_seen_count integer, + p_min_read_count integer, + p_min_opened_count integer ) -returns table (deleted boolean, archived boolean, existed boolean) +returns boolean +language sql +immutable +parallel safe +as $$ + select coalesce( + (p_created_cutoff is null or p_created_at <= p_created_cutoff) + -- 00100: a lesson NEVER opened ages from its creation date, so the lore + -- nothing ever opened is findable by an "unopened for N days" filter. + and (p_unseen_cutoff is null or coalesce(p_last_opened_at, p_created_at) <= p_unseen_cutoff) + -- `<=`, so a threshold of 0 is meaningful (00105). + and (p_max_seen_count is null or p_seen_count <= p_max_seen_count) + and (p_max_read_count is null or p_read_count <= p_max_read_count) + and (p_max_opened_count is null or p_opened_count <= p_max_opened_count) + -- 00113: deliberately WITHOUT 00100's fallback — a never-opened lesson was + -- not "opened within N days", however recently it was written. Spelled out + -- rather than left to the null comparison so the exclusion is visibly the + -- intent and not an accident of three-valued logic. + and (p_opened_since is null + or (p_last_opened_at is not null and p_last_opened_at >= p_opened_since)) + and (p_min_seen_count is null or p_seen_count >= p_min_seen_count) + and (p_min_read_count is null or p_read_count >= p_min_read_count) + and (p_min_opened_count is null or p_opened_count >= p_min_opened_count), + false); +$$; + +comment on function lorekit_match_retention( + timestamptz, timestamptz, integer, integer, integer, + timestamptz, timestamptz, integer, integer, integer, + timestamptz, integer, integer, integer +) is + 'Does a memory satisfy an age/activity condition set? The five thresholds of + GroomConditionsSchema (00108) plus the Explorer''s four reverse directions + (00113: opened-since cutoff, min seen/read/opened counts), as one inlinable + row test shared by lorekit_memory_list, _facets, _activity and _pivot so a + count cannot disagree with the list it describes. Takes CUTOFF INSTANTS + rather than day counts so it stays immutable and reads no clock. A null + argument is "not filtered". Holds 00100''s never-opened rule for unseen + days, and deliberately NOT for opened-since: a never-opened lesson was not + opened recently.'; + +-- The family's ACL (00066/00108): a NEW function starts from EXECUTE-to-public, +-- and Supabase's default privileges grant `anon` separately (00077). +revoke execute on function lorekit_match_retention( + timestamptz, timestamptz, integer, integer, integer, + timestamptz, timestamptz, integer, integer, integer, + timestamptz, integer, integer, integer +) from public, anon; +grant execute on function lorekit_match_retention( + timestamptz, timestamptz, integer, integer, integer, + timestamptz, timestamptz, integer, integer, integer, + timestamptz, integer, integer, integer +) to authenticated, service_role; + +-- ── 2. lorekit_memory_facets (00110 body) ───────────────────────────────── + +drop function if exists lorekit_memory_facets( + uuid, boolean, text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[], timestamptz, timestamptz, integer, integer, integer, + integer, integer +); + +create or replace function lorekit_memory_facets( + p_user_id uuid, + p_archived boolean default false, + p_scope text default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in', + -- Owner dimension (00064). `personal` plus one slug per member org with + -- visible rows. All optional: null/absent = not filtered. + p_owner text[] default null, + p_owner_mode text default 'in', + -- The CALLING KEY's restriction (00068), defaulted to unrestricted. + p_key_scopes text[] default '{}', + p_key_org_access text default 'all', + p_key_org_ids uuid[] default '{}', + -- 00108: the created_at window and the five retention thresholds, so a count + -- describes the same rows the list shows. Appended, null-defaulted. + p_created_since timestamptz default null, + p_created_until timestamptz default null, + p_min_age_days integer default null, + p_unseen_days integer default null, + p_max_seen_count integer default null, + p_max_read_count integer default null, + p_max_opened_count integer default null, + -- 00113: the Explorer's reverse directions — "opened within N days" and the + -- three "at least N" floors. Appended, null-defaulted. Read routes only: a + -- retention policy cannot carry them. + p_opened_within_days integer default null, + p_min_seen_count integer default null, + p_min_read_count integer default null, + p_min_opened_count integer default null +) +returns table (facet text, value text, count bigint) language plpgsql +stable security definer set search_path = public as $$ +#variable_conflict use_column declare - v_count integer; - v_existed boolean; + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + v_origin_pr integer[] := ( + select array_agg(x::integer) + from unnest(coalesce(p_origin_pr, '{}'::text[])) as x + where x ~ '^[0-9]+$' + ); + v_created_cutoff timestamptz := case + when p_min_age_days is null then null else now() - (p_min_age_days * interval '1 day') + end; + v_unseen_cutoff timestamptz := case + when p_unseen_days is null then null else now() - (p_unseen_days * interval '1 day') + end; + -- 00113: the "opened within N days" lower bound, resolved once per call for + -- the same reason as the two cutoffs above. + v_opened_since timestamptz := case + when p_opened_within_days is null then null else now() - (p_opened_within_days * interval '1 day') + end; + -- 00110: only `label`/tag has an `all` (AND / set-containment) mode — see + -- the module header. Every other dimension, and the tag facet in `in`/`nin` + -- mode, keep self-exclusion (unchanged). + v_tag_cooccur boolean := (p_tags_mode = 'all'); begin - -- p_org_slug is intentionally ignored in BYOD. - -- Org-gated deletes require the hosted LoreKit product. + return query + with base as ( + select + m.tags, m.source_agent, m.trigger, m.kind, m.host, + m.origin_repo, m.origin_branch, m.origin_pr, + m.org_id, o.slug as org_slug, + -- Per-dimension match flag, from the shared predicates so it cannot + -- drift from lorekit_memory_activity's. A null filter is "not filtered" → + -- the helper returns true, so an untouched dimension never narrows. + lorekit_match_tags(m.tags, p_tags, p_tags_mode) as ok_tag, + lorekit_match_text(m.source_agent, p_source_agent, p_source_agent_mode) as ok_source_agent, + lorekit_match_text(m.trigger, p_trigger, p_trigger_mode) as ok_trigger, + lorekit_match_text(m.kind, p_kind, p_kind_mode) as ok_kind, + lorekit_match_text(m.host, p_host, p_host_mode) as ok_host, + lorekit_match_text(m.origin_repo, p_origin_repo, p_origin_repo_mode) as ok_origin_repo, + lorekit_match_text(m.origin_branch, p_origin_branch, p_origin_branch_mode) as ok_origin_branch, + lorekit_match_int (m.origin_pr, v_origin_pr, p_origin_pr_mode) as ok_origin_pr, + -- Owner: the computed identity is `personal` (org_id null) or the org slug. + -- Stays inline — it is not one of the three column helpers (00064). + (p_owner is null or case coalesce(p_owner_mode, 'in') + when 'nin' then ( + (case when m.org_id is null then 'personal' else o.slug end) is not null + and (case when m.org_id is null then 'personal' else o.slug end) <> all(p_owner) + ) + else ( + ('personal' = any(p_owner) and m.org_id is null) + or (m.org_id is not null and o.slug = any(p_owner)) + ) + end) as ok_owner + from memories m + -- A personal row has no org, so this is a LEFT join; org rows resolve to + -- their slug. Visible org rows are always the caller's own orgs (the + -- visibility predicate below admits them only via lorekit_member_org_ids), + -- so `o.slug` is never a slug the caller cannot see. + left join orgs o on o.id = m.org_id + where ( + (v_actor is null and auth.role() = 'service_role') + or m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) + ) + -- The calling key's restriction, applied ONCE here in the row-visibility + -- predicate every emitted facet value is derived from — `origin_repo` is + -- a repository name by construction, so an unnarrowed facet list leaks + -- exactly what the scope catalog hides. + and lorekit_api_token_scope_allowed(p_key_scopes, m.scope) + and lorekit_api_token_org_allowed(p_key_org_access, p_key_org_ids, m.org_id) + and ( + case + when p_archived then m.archived_at is not null + else m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + end + ) + and (p_scope is null or m.scope = p_scope) + -- 00108. Here rather than as an `ok_*` flag: these narrow the population + -- every facet value is drawn from, they are not a dimension that can be + -- self-excluded. Half-open window, matching lorekit_memory_list. + and (p_created_since is null or m.created_at >= p_created_since) + and (p_created_until is null or m.created_at < p_created_until) + and lorekit_match_retention( + m.created_at, m.last_opened_at, m.seen_count, m.read_count, m.opened_count, + v_created_cutoff, v_unseen_cutoff, p_max_seen_count, p_max_read_count, p_max_opened_count, + v_opened_since, p_min_seen_count, p_min_read_count, p_min_opened_count + ) + ), cells as ( + -- 00110: every branch now scans the FULL `base` population (not just rows + -- that already satisfy the other flags) and carries a `matched` boolean + -- instead of a row-level WHERE — the `group by` below therefore enumerates + -- every value the dimension has, and the final `count` reflects only the + -- ones actually matching. This is what keeps a non-matching value at + -- `count = 0` instead of dropping it. + select 'tag'::text as facet, t.tag as value, + (b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner + -- Co-occurrence: fold the tag dimension's OWN flag back in only in + -- `all` mode, so another label's count becomes "carries the + -- selection AND this label" rather than the plain self-excluded + -- total. `not v_tag_cooccur` keeps `in`/`nin` mode's existing + -- self-exclusion untouched. + and (not v_tag_cooccur or b.ok_tag)) as matched + from base b + cross join lateral unnest(b.tags) as t(tag) + union all + select 'source_agent', b.source_agent, + (b.ok_tag and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner) + from base b + where b.source_agent is not null + union all + select 'trigger', b.trigger, + (b.ok_tag and b.ok_source_agent and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner) + from base b + where b.trigger is not null + union all + select 'kind', b.kind, + (b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner) + from base b + where b.kind is not null + union all + select 'host', b.host, + (b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner) + from base b + where b.host is not null + union all + select 'origin_repo', b.origin_repo, + (b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_branch and b.ok_origin_pr and b.ok_owner) + from base b + where b.origin_repo is not null + union all + select 'origin_branch', b.origin_branch, + (b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_pr and b.ok_owner) + from base b + where b.origin_branch is not null + union all + select 'origin_pr', b.origin_pr::text, + (b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_owner) + from base b + where b.origin_pr is not null + union all + -- Owner is the ONE dimension self-excluded here (every other flag, NOT + -- ok_owner), so a drilled-in owner still lists the alternative owner. + select 'owner', case when b.org_id is null then 'personal' else b.org_slug end, + (b.ok_tag and b.ok_source_agent and b.ok_trigger and b.ok_kind and b.ok_host + and b.ok_origin_repo and b.ok_origin_branch and b.ok_origin_pr) + from base b + ) + select c.facet, c.value, count(*) filter (where c.matched) as count + from cells c + where c.value is not null + and btrim(c.value) <> '' + group by c.facet, c.value + order by c.facet asc, count(*) filter (where c.matched) desc, c.value asc; +end; +$$; - -- 00069 §7 verbatim: the allowlist, before either branch. - if not lorekit_api_token_scope_allowed(p_key_scopes, p_scope) then - raise exception using errcode = 'LK002', - message = format('key_scope_denied: scope=%s', p_scope); +revoke execute on function lorekit_memory_facets( + uuid, boolean, text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[], timestamptz, timestamptz, integer, integer, integer, + integer, integer, + integer, integer, integer, integer +) from public, anon; +grant execute on function lorekit_memory_facets( + uuid, boolean, text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[], timestamptz, timestamptz, integer, integer, integer, + integer, integer, + integer, integer, integer, integer +) to authenticated, service_role; + +comment on function lorekit_memory_facets( + uuid, boolean, text, text[], text, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[], timestamptz, timestamptz, integer, integer, integer, + integer, integer, + integer, integer, integer, integer +) is + 'Value catalog with counts for every filterable memory dimension (tag, + source_agent, trigger, kind, host, origin_repo, origin_branch, origin_pr, + owner) over the partition selected by p_archived, visible to the EFFECTIVE + caller and narrowed by the calling key''s own scope/org restriction. Each + dimension is counted with every OTHER dimension filter applied (cross-group + AND). WITHIN a dimension: self-exclusion (drill-down, 00057) for every + scalar dimension and for label/tag in `in`/`nin` (OR) mode; CO-OCCURRENCE + for label/tag in `all` (AND) mode (00110). Every dimension enumerates its + FULL value set so a value with zero matches still returns `count = 0` + (00110). The created_at window, the five retention thresholds (00108) and + the Explorer''s reverse directions (00113: opened_within_days and the three + min_* floors) are NOT self-excluded: they narrow the population every facet + value is drawn from, so a count matches the list under the same parameters. + q/key/expiring_within_days remain unmirrored, so with those active a count + is an upper bound.'; + +-- ── 3. lorekit_memory_activity (00108 body) ─────────────────────────────── + +drop function if exists lorekit_memory_activity( + uuid, text, timestamptz, timestamptz, text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[], integer, integer, integer, integer, integer +); + +create or replace function lorekit_memory_activity( + p_user_id uuid, + p_bucket text default 'day', + p_since timestamptz default null, + p_until timestamptz default null, + p_scope text default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in', + p_owner text[] default null, + p_owner_mode text default 'in', + -- The CALLING KEY's restriction (00068), defaulted to unrestricted. + p_key_scopes text[] default '{}', + p_key_org_access text default 'all', + p_key_org_ids uuid[] default '{}', + -- 00108: the five retention thresholds. No created_at window is added — this + -- function's own p_since/p_until already bound created_at, and a second + -- window would be two ways to say one thing. + p_min_age_days integer default null, + p_unseen_days integer default null, + p_max_seen_count integer default null, + p_max_read_count integer default null, + p_max_opened_count integer default null, + -- 00113: the Explorer's reverse directions — "opened within N days" and the + -- three "at least N" floors. Appended, null-defaulted. Read routes only: a + -- retention policy cannot carry them. + p_opened_within_days integer default null, + p_min_seen_count integer default null, + p_min_read_count integer default null, + p_min_opened_count integer default null +) +returns table (bucket timestamptz, scope text, count bigint) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + -- `origin_pr` is an INTEGER column: coerce the digits-only list numerically so + -- this route matches GET /memories on the same input (00057's rationale). A + -- non-numeric entry is dropped; a list reducing to empty applies no filter. + v_origin_pr integer[] := ( + select array_agg(x::integer) + from unnest(coalesce(p_origin_pr, '{}'::text[])) as x + where x ~ '^[0-9]+$' + ); + v_created_cutoff timestamptz := case + when p_min_age_days is null then null else now() - (p_min_age_days * interval '1 day') + end; + v_unseen_cutoff timestamptz := case + when p_unseen_days is null then null else now() - (p_unseen_days * interval '1 day') + end; + -- 00113: the "opened within N days" lower bound, resolved once per call for + -- the same reason as the two cutoffs above. + v_opened_since timestamptz := case + when p_opened_within_days is null then null else now() - (p_opened_within_days * interval '1 day') + end; +begin + if p_bucket is null or p_bucket not in ('hour', 'day') then + raise exception 'invalid bucket %, expected hour or day', p_bucket + using errcode = '22023'; end if; - -- 00071 mirrored: `existed` is what lets the edge answer 404 (nothing there) - -- vs 403 (present, but this call removed nothing) instead of collapsing both - -- into not_found. And a null p_user_id is the SERVICE tier (no user, no key), - -- never "the user with no id" — see restore_memory above. - v_existed := exists( - select 1 from memories m - where m.scope = p_scope and m.key = p_key - and (p_user_id is null or m.user_id = p_user_id) + return query + select date_trunc(p_bucket, m.created_at at time zone 'UTC') at time zone 'UTC' as bucket, + m.scope, + count(*) as count + from memories m + -- LEFT join for the owner predicate — a personal row has no org (00064). + left join orgs o on o.id = m.org_id + where ( + (v_actor is null and auth.role() = 'service_role') + or m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) + ) + -- The calling key's own restriction, ANDed on top of the caller's + -- visibility and never instead of it (the lorekit_memory_scopes rule). + and lorekit_api_token_scope_allowed(p_key_scopes, m.scope) + and lorekit_api_token_org_allowed(p_key_org_access, p_key_org_ids, m.org_id) + and m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + and (p_since is null or m.created_at >= p_since) + and (p_until is null or m.created_at < p_until) + -- Scope is a hard filter, always applied (00057's treatment). + and (p_scope is null or m.scope = p_scope) + -- Dimension filters — AND across, OR within, from the shared predicates + -- (00066) so the semantics cannot drift from lorekit_memory_facets. No + -- self-exclusion here: a straight count applies every one directly. + and lorekit_match_tags(m.tags, p_tags, p_tags_mode) + and lorekit_match_text(m.source_agent, p_source_agent, p_source_agent_mode) + and lorekit_match_text(m.trigger, p_trigger, p_trigger_mode) + and lorekit_match_text(m.kind, p_kind, p_kind_mode) + and lorekit_match_text(m.host, p_host, p_host_mode) + and lorekit_match_text(m.origin_repo, p_origin_repo, p_origin_repo_mode) + and lorekit_match_text(m.origin_branch, p_origin_branch, p_origin_branch_mode) + and lorekit_match_int (m.origin_pr, v_origin_pr, p_origin_pr_mode) + -- 00108: the same thresholds the list applies, so the Explorer's stat + -- cards describe the rows underneath them. + and lorekit_match_retention( + m.created_at, m.last_opened_at, m.seen_count, m.read_count, m.opened_count, + v_created_cutoff, v_unseen_cutoff, p_max_seen_count, p_max_read_count, p_max_opened_count, + v_opened_since, p_min_seen_count, p_min_read_count, p_min_opened_count + ) + -- Owner: `personal` for org_id-null rows, else the org slug (00064). + and (p_owner is null or case coalesce(p_owner_mode, 'in') + when 'nin' then ( + (case when m.org_id is null then 'personal' else o.slug end) is not null + and (case when m.org_id is null then 'personal' else o.slug end) <> all(p_owner) + ) + else ( + ('personal' = any(p_owner) and m.org_id is null) + or (m.org_id is not null and o.slug = any(p_owner)) + ) + end) + group by 1, m.scope + order by 1 asc, m.scope asc; +end; +$$; + +revoke execute on function lorekit_memory_activity( + uuid, text, timestamptz, timestamptz, text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[], integer, integer, integer, integer, integer, + integer, integer, integer, integer +) from public, anon; +grant execute on function lorekit_memory_activity( + uuid, text, timestamptz, timestamptz, text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[], integer, integer, integer, integer, integer, + integer, integer, integer, integer +) to authenticated, service_role; + +comment on function lorekit_memory_activity( + uuid, text, timestamptz, timestamptz, text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[], integer, integer, integer, integer, integer, + integer, integer, integer, integer +) is + 'Memories created per UTC hour/day per scope, visible to the EFFECTIVE caller + and narrowed by the calling key''s own scope/org restriction. Applies every + dimension filter directly (no self-exclusion — this is a straight count), + the five retention thresholds (00108) and the Explorer''s reverse directions + (00113), so the Explorer''s stat cards describe the list beneath them. + p_since/p_until already bound created_at, so no separate window parameter + exists.'; + +-- ── 4. lorekit_memory_pivot (00108 body) ────────────────────────────────── + +drop function if exists lorekit_memory_pivot( + text, text, uuid, boolean, text, integer, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[], timestamptz, timestamptz, integer, integer, integer, + integer, integer +); + +create or replace function lorekit_memory_pivot( + p_row_facet text, + p_col_facet text, + p_user_id uuid default null, + p_archived boolean default false, + p_scope text default null, + p_limit integer default 400, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in', + p_owner text[] default null, + p_owner_mode text default 'in', + -- The CALLING KEY's restriction (00068), defaulted to unrestricted. + p_key_scopes text[] default '{}', + p_key_org_access text default 'all', + p_key_org_ids uuid[] default '{}', + -- 00108: the created_at window and the five retention thresholds. + p_created_since timestamptz default null, + p_created_until timestamptz default null, + p_min_age_days integer default null, + p_unseen_days integer default null, + p_max_seen_count integer default null, + p_max_read_count integer default null, + p_max_opened_count integer default null, + -- 00113: the Explorer's reverse directions — "opened within N days" and the + -- three "at least N" floors. Appended, null-defaulted. Read routes only: a + -- retention policy cannot carry them. + p_opened_within_days integer default null, + p_min_seen_count integer default null, + p_min_read_count integer default null, + p_min_opened_count integer default null +) +returns table (row_value text, col_value text, count bigint) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + v_origin_pr integer[] := ( + select array_agg(x::integer) + from unnest(coalesce(p_origin_pr, '{}'::text[])) as x + where x ~ '^[0-9]+$' ); + v_created_cutoff timestamptz := case + when p_min_age_days is null then null else now() - (p_min_age_days * interval '1 day') + end; + v_unseen_cutoff timestamptz := case + when p_unseen_days is null then null else now() - (p_unseen_days * interval '1 day') + end; + -- 00113: the "opened within N days" lower bound, resolved once per call for + -- the same reason as the two cutoffs above. + v_opened_since timestamptz := case + when p_opened_within_days is null then null else now() - (p_opened_within_days * interval '1 day') + end; +begin + return query + with base as ( + select + m.tags, m.source_agent, m.trigger, m.kind, m.host, + m.origin_repo, m.origin_branch, m.origin_pr, + m.org_id, o.slug as org_slug, + lorekit_match_tags(m.tags, p_tags, p_tags_mode) as ok_tag, + lorekit_match_text(m.source_agent, p_source_agent, p_source_agent_mode) as ok_source_agent, + lorekit_match_text(m.trigger, p_trigger, p_trigger_mode) as ok_trigger, + lorekit_match_text(m.kind, p_kind, p_kind_mode) as ok_kind, + lorekit_match_text(m.host, p_host, p_host_mode) as ok_host, + lorekit_match_text(m.origin_repo, p_origin_repo, p_origin_repo_mode) as ok_origin_repo, + lorekit_match_text(m.origin_branch, p_origin_branch, p_origin_branch_mode) as ok_origin_branch, + lorekit_match_int (m.origin_pr, v_origin_pr, p_origin_pr_mode) as ok_origin_pr, + -- Owner stays inline for lorekit_memory_facets' reason: it is a + -- LEFT JOIN-computed identity, not one of the three column helpers. + (p_owner is null or case coalesce(p_owner_mode, 'in') + when 'nin' then ( + (case when m.org_id is null then 'personal' else o.slug end) is not null + and (case when m.org_id is null then 'personal' else o.slug end) <> all(p_owner) + ) + else ( + ('personal' = any(p_owner) and m.org_id is null) + or (m.org_id is not null and o.slug = any(p_owner)) + ) + end) as ok_owner + from memories m + left join orgs o on o.id = m.org_id + where ( + (v_actor is null and auth.role() = 'service_role') + or m.user_id = v_actor + or m.org_id in (select lorekit_member_org_ids(v_actor)) + ) + and lorekit_api_token_scope_allowed(p_key_scopes, m.scope) + and lorekit_api_token_org_allowed(p_key_org_access, p_key_org_ids, m.org_id) + and ( + case + when p_archived then m.archived_at is not null + else m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + end + ) + and (p_scope is null or m.scope = p_scope) + -- 00108. Row-visibility, not an `ok_*` flag — see lorekit_memory_facets. + and (p_created_since is null or m.created_at >= p_created_since) + and (p_created_until is null or m.created_at < p_created_until) + and lorekit_match_retention( + m.created_at, m.last_opened_at, m.seen_count, m.read_count, m.opened_count, + v_created_cutoff, v_unseen_cutoff, p_max_seen_count, p_max_read_count, p_max_opened_count, + v_opened_since, p_min_seen_count, p_min_read_count, p_min_opened_count + ) + ) + select rv.value, cv.value, count(*)::bigint + from base b + cross join lateral lorekit_memory_facet_value( + p_row_facet, b.tags, b.source_agent, b.trigger, b.kind, b.host, + b.origin_repo, b.origin_branch, b.origin_pr, b.org_id, b.org_slug + ) as rv(value) + cross join lateral lorekit_memory_facet_value( + p_col_facet, b.tags, b.source_agent, b.trigger, b.kind, b.host, + b.origin_repo, b.origin_branch, b.origin_pr, b.org_id, b.org_slug + ) as cv(value) + -- Each dimension's own filter applies UNLESS it is one of the two axes — + -- see the self-exclusion note at the top. One line per dimension so adding + -- a tenth is one line here, not a re-derivation. + where (p_row_facet = 'tag' or p_col_facet = 'tag' or b.ok_tag) + and (p_row_facet = 'source_agent' or p_col_facet = 'source_agent' or b.ok_source_agent) + and (p_row_facet = 'trigger' or p_col_facet = 'trigger' or b.ok_trigger) + and (p_row_facet = 'kind' or p_col_facet = 'kind' or b.ok_kind) + and (p_row_facet = 'host' or p_col_facet = 'host' or b.ok_host) + and (p_row_facet = 'origin_repo' or p_col_facet = 'origin_repo' or b.ok_origin_repo) + and (p_row_facet = 'origin_branch' or p_col_facet = 'origin_branch' or b.ok_origin_branch) + and (p_row_facet = 'origin_pr' or p_col_facet = 'origin_pr' or b.ok_origin_pr) + and (p_row_facet = 'owner' or p_col_facet = 'owner' or b.ok_owner) + -- A null column value yields no cell, matching lorekit_memory_facets. + and rv.value is not null + and cv.value is not null + group by rv.value, cv.value + -- Densest cells first, so a caller that has to truncate keeps the ones + -- worth drawing. Deterministic tie-break so a page boundary is stable. + order by count(*) desc, rv.value asc, cv.value asc + limit greatest(p_limit, 0); +end; +$$; - if p_force then - delete from memories m - where m.scope = p_scope and m.key = p_key - and (p_user_id is null or m.user_id = p_user_id); - get diagnostics v_count = row_count; +revoke execute on function lorekit_memory_pivot( + text, text, uuid, boolean, text, integer, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[], timestamptz, timestamptz, integer, integer, integer, + integer, integer, + integer, integer, integer, integer +) from public, anon; +grant execute on function lorekit_memory_pivot( + text, text, uuid, boolean, text, integer, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[], timestamptz, timestamptz, integer, integer, integer, + integer, integer, + integer, integer, integer, integer +) to authenticated, service_role; - return query select (v_count > 0), false, (v_existed or v_count > 0); - else - update memories m - set archived_at = now() - where m.scope = p_scope and m.key = p_key and m.archived_at is null - and (p_user_id is null or m.user_id = p_user_id); - get diagnostics v_count = row_count; +comment on function lorekit_memory_pivot( + text, text, uuid, boolean, text, integer, text[], text, text[], text, text[], text, + text[], text, text[], text, text[], text, text[], text, text[], text, text[], text, + text[], text, uuid[], timestamptz, timestamptz, integer, integer, integer, + integer, integer, + integer, integer, integer, integer +) is + 'Cross-tabulation of two facet dimensions with a count per cell, visible to + the EFFECTIVE caller and narrowed by the calling key''s own scope/org + restriction. BOTH axes are self-excluded from the dimension filters. The + created_at window, the five retention thresholds (00108) and the Explorer''s + reverse directions (00113) are not self-excluded — they narrow the + population, matching lorekit_memory_facets.'; - return query select false, (v_count > 0), (v_existed or v_count > 0); - end if; +-- ── 5. lorekit_memory_list (00108 body) ─────────────────────────────────── +-- +-- The signature changes (four filters + the integer cursor), so unlike 00108 +-- this is a drop-and-create. 00105 recreated it with a bare `grant`, which +-- left the default-privilege EXECUTE for `anon` in place (00077's subject); +-- the revoke below restores what 00092/00094 established and what its three +-- siblings carry. It returns nothing to anon either way — the visibility +-- predicate admits no row for a null actor — but a security-definer reader +-- reachable from the anonymous surface is a difference nobody should have to +-- re-derive. + +drop function if exists lorekit_memory_list( + uuid, boolean, text, text, text, text, timestamptz, timestamptz, timestamptz, + timestamptz, text[], text, text[], text, text[], text, text[], text, text[], + text, text[], text, text[], text, text[], text, text[], text, text, + timestamptz, uuid, integer, text[], text, uuid[], integer, integer, integer, + integer, integer +); + +create or replace function lorekit_memory_list( + p_user_id uuid, + p_archived boolean default false, + p_scope text default null, + p_key text default null, + p_key_prefix text default null, + p_q text default null, + p_created_since timestamptz default null, + p_created_until timestamptz default null, + p_expires_after timestamptz default null, + p_expires_on_or_before timestamptz default null, + p_tags text[] default null, + p_tags_mode text default 'any', + p_source_agent text[] default null, + p_source_agent_mode text default 'in', + p_trigger text[] default null, + p_trigger_mode text default 'in', + p_kind text[] default null, + p_kind_mode text default 'in', + p_host text[] default null, + p_host_mode text default 'in', + p_origin_repo text[] default null, + p_origin_repo_mode text default 'in', + p_origin_branch text[] default null, + p_origin_branch_mode text default 'in', + p_origin_pr text[] default null, + p_origin_pr_mode text default 'in', + p_owner text[] default null, + p_owner_mode text default 'in', + p_sort text default 'updated_at', + p_cursor_ts timestamptz default null, + p_cursor_id uuid default null, + p_limit integer default 51, + p_key_scopes text[] default '{}', + p_key_org_access text default 'all', + p_key_org_ids uuid[] default '{}', + p_min_age_days integer default null, + p_unseen_days integer default null, + p_max_seen_count integer default null, + p_max_read_count integer default null, + p_max_opened_count integer default null, + -- 00113: the Explorer's reverse directions, appended and null-defaulted. + p_opened_within_days integer default null, + p_min_seen_count integer default null, + p_min_read_count integer default null, + p_min_opened_count integer default null, + -- 00113: the keyset cursor for `p_sort => 'opened_count'`. An integer sort + -- needs an integer cursor; `p_cursor_ts` keeps serving the two date sorts. + p_cursor_count integer default null +) +returns table ( + id uuid, + scope text, + key text, + value text, + tags text[], + source_agent text, + trigger text, + created_at timestamptz, + updated_at timestamptz, + expires_at timestamptz, + archived_at timestamptz, + origin_repo text, + origin_branch text, + origin_commit text, + origin_pr integer, + kind text, + host text, + seen_count integer, + read_count integer, + opened_count integer, + last_read_at timestamptz, + last_opened_at timestamptz, + org_id uuid, + created_by uuid, + updated_by uuid, + org_name text, + org_slug text, + total_count integer +) +language plpgsql +stable +security definer +set search_path = public +as $$ +#variable_conflict use_column +declare + v_actor uuid := case + when auth.role() = 'service_role' then coalesce(p_user_id, auth.uid()) + else auth.uid() + end; + v_origin_pr integer[] := ( + select array_agg(x::integer) + from unnest(coalesce(p_origin_pr, '{}'::text[])) as x + where x ~ '^0*[0-9]{1,9}$' + ); + -- 00113: `opened_count` joins the two date sorts. Anything else still falls + -- back to `updated_at`, so an unknown value cannot reach the format() below. + v_sort text := case + when p_sort in ('created_at', 'opened_count') then p_sort else 'updated_at' + end; + -- Which bound parameter carries the keyset value: the integer cursor ($44) + -- for the count sort, the timestamp cursor ($29) for the date sorts. A fixed + -- placeholder chosen here, never caller text, so splicing it is safe. + v_cursor text := case when v_sort = 'opened_count' then '$44' else '$29' end; + -- Resolved once per call so lorekit_match_retention stays immutable (00108). + v_created_cutoff timestamptz := case + when p_min_age_days is null then null else now() - (p_min_age_days * interval '1 day') + end; + v_unseen_cutoff timestamptz := case + when p_unseen_days is null then null else now() - (p_unseen_days * interval '1 day') + end; + -- 00113: the "opened within N days" lower bound, resolved once per call for + -- the same reason as the two cutoffs above. + v_opened_since timestamptz := case + when p_opened_within_days is null then null else now() - (p_opened_within_days * interval '1 day') + end; +begin + return query execute format($q$ + select + m.id, m.scope, m.key, m.value, m.tags, m.source_agent, m.trigger, + m.created_at, m.updated_at, m.expires_at, m.archived_at, + m.origin_repo, m.origin_branch, m.origin_commit, m.origin_pr, + m.kind, m.host, m.seen_count, + m.read_count, m.opened_count, m.last_read_at, m.last_opened_at, + m.org_id, m.created_by, m.updated_by, + o.name as org_name, o.slug as org_slug, + (count(*) over ())::integer as total_count + from memories m + left join orgs o on o.id = m.org_id + where ( + ($1 is null and auth.role() = 'service_role') + or m.user_id = $1 + or m.org_id in (select lorekit_member_org_ids($1)) + ) + and lorekit_api_token_scope_allowed($32, m.scope) + and lorekit_api_token_org_allowed($33, $34, m.org_id) + and ( + case + when $2 then m.archived_at is not null + else m.archived_at is null + and (m.expires_at is null or m.expires_at > now()) + end + ) + and ($3 is null or m.scope = $3) + and ($4 is null or m.key = $4) + and ($5 is null or m.key ilike $5 || '%%') + and ($6 is null or m.key ilike '%%' || $6 || '%%' or m.value ilike '%%' || $6 || '%%') + and ($7 is null or m.created_at >= $7) + and ($8 is null or m.created_at < $8) + and ($9 is null or m.expires_at > $9) + and ($10 is null or m.expires_at <= $10) + and lorekit_match_tags(m.tags, $11, $12) + and lorekit_match_text(m.source_agent, $13, $14) + and lorekit_match_text(m.trigger, $15, $16) + and lorekit_match_text(m.kind, $17, $18) + and lorekit_match_text(m.host, $19, $20) + and lorekit_match_text(m.origin_repo, $21, $22) + and lorekit_match_text(m.origin_branch, $23, $24) + and lorekit_match_int (m.origin_pr, $25, $26) + and ($27 is null or case coalesce($28, 'in') + when 'nin' then ( + (case when m.org_id is null then 'personal' else o.slug end) is not null + and (case when m.org_id is null then 'personal' else o.slug end) <> all($27) + ) + else ( + ('personal' = any($27) and m.org_id is null) + or (m.org_id is not null and o.slug = any($27)) + ) + end) + and lorekit_match_retention( + m.created_at, m.last_opened_at, m.seen_count, m.read_count, m.opened_count, + $35, $36, $37, $38, $39, + $40, $41, $42, $43 + ) + and ( + %2$s is null + or m.%1$I < %2$s + or (m.%1$I = %2$s and m.id < $30) + ) + order by m.%1$I desc, m.id desc + limit $31 + $q$, v_sort, v_cursor) + using + v_actor, coalesce(p_archived, false), p_scope, p_key, p_key_prefix, p_q, + p_created_since, p_created_until, p_expires_after, p_expires_on_or_before, + p_tags, p_tags_mode, + p_source_agent, p_source_agent_mode, + p_trigger, p_trigger_mode, + p_kind, p_kind_mode, + p_host, p_host_mode, + p_origin_repo, p_origin_repo_mode, + p_origin_branch, p_origin_branch_mode, + v_origin_pr, p_origin_pr_mode, + p_owner, p_owner_mode, + p_cursor_ts, p_cursor_id, + greatest(coalesce(p_limit, 51), 1), + coalesce(p_key_scopes, '{}'::text[]), + coalesce(p_key_org_access, 'all'), + coalesce(p_key_org_ids, '{}'::uuid[]), + v_created_cutoff, v_unseen_cutoff, p_max_seen_count, p_max_read_count, + p_max_opened_count, + v_opened_since, p_min_seen_count, p_min_read_count, p_min_opened_count, + p_cursor_count; end; $$; -grant execute on function memory_delete(uuid, text, text, text, boolean, text[], text, uuid[]) to anon, authenticated, service_role; +revoke execute on function lorekit_memory_list( + uuid, boolean, text, text, text, text, timestamptz, timestamptz, timestamptz, + timestamptz, text[], text, text[], text, text[], text, text[], text, text[], + text, text[], text, text[], text, text[], text, text[], text, text, + timestamptz, uuid, integer, text[], text, uuid[], integer, integer, integer, + integer, integer, + integer, integer, integer, integer, integer +) from public, anon; +grant execute on function lorekit_memory_list( + uuid, boolean, text, text, text, text, timestamptz, timestamptz, timestamptz, + timestamptz, text[], text, text[], text, text[], text, text[], text, text[], + text, text[], text, text[], text, text[], text, text[], text, text, + timestamptz, uuid, integer, text[], text, uuid[], integer, integer, integer, + integer, integer, + integer, integer, integer, integer, integer +) to authenticated, service_role; --- ───────────────────────────────────────────────────────────────────────────── --- 9. Grants --- ───────────────────────────────────────────────────────────────────────────── +comment on function lorekit_memory_list( + uuid, boolean, text, text, text, text, timestamptz, timestamptz, timestamptz, + timestamptz, text[], text, text[], text, text[], text, text[], text, text[], + text, text[], text, text[], text, text[], text, text[], text, text, + timestamptz, uuid, integer, text[], text, uuid[], integer, integer, integer, + integer, integer, + integer, integer, integer, integer, integer +) is + 'The keyset-paginated memory list behind GET /memories and POST /memories/list. + Sorts by updated_at (default), created_at, or opened_count (00113) — each + descending with id as the tie-break; p_cursor_ts carries the date sorts'' + keyset value and p_cursor_count the count sort''s. Applies the five retention + thresholds and the Explorer''s reverse directions through the shared + lorekit_match_retention, so its total_count agrees with _facets, _activity + and _pivot under the same parameters.'; -grant usage on schema public to anon, authenticated, service_role; -grant all on table memories to anon, authenticated, service_role; -grant all on table api_tokens to anon, authenticated, service_role; -grant all on table user_limits to anon, authenticated, service_role; -grant all on table rate_limit_counters to anon, authenticated, service_role; +-- ── 6. Retire the 10-argument helper ───────────────────────────────────── +-- +-- Last, once every caller above composes the 14-argument form. The callers +-- are plpgsql, whose bodies are resolved at call time and carry no catalog +-- dependency, so nothing blocks the drop; leaving it would be a second +-- predicate for someone to call by mistake. +drop function if exists lorekit_match_retention( + timestamptz, timestamptz, integer, integer, integer, + timestamptz, timestamptz, integer, integer, integer +); -grant execute on function lorekit_default_limit(text) to anon, authenticated, service_role; -grant execute on function lorekit_get_limit(uuid, text) to anon, authenticated, service_role; -grant execute on function lorekit_check_rate_limit(uuid, integer) to anon, authenticated, service_role; -grant execute on function lorekit_purge_rate_limit_counters(interval) to anon, authenticated, service_role; -grant execute on function archive_memory(uuid, text, text) to anon, authenticated, service_role; -grant execute on function restore_memory(uuid, text, text, text[], text, uuid[]) to anon, authenticated, service_role; -grant execute on function purge_archived_memories(uuid, integer) to anon, authenticated, service_role; -grant execute on function purge_expired_memories(uuid) to authenticated, service_role; +insert into supabase_migrations.schema_migrations (version, name) +values ('00113', 'explorer_reverse_activity_filters_and_opened_sort') +on conflict (version) do nothing; -comment on table memories is - 'LoreKit agent memories. BYOD install — personal rows only (no org sharing).'; -comment on column memories.expires_at is - 'Optional timestamp after which the row is treated as invisible by active-read - paths. NULL means the row never expires. Physically deleted by - purge_expired_memories() or the MCP memory.purge tool.'; -comment on column memories.org_id is - 'Accepted for hosted-schema compatibility; org-owned writes require the hosted - LoreKit product. In BYOD this column is always NULL.'; +commit; From 9e12c0132b15677634e87f33793f4d6896998cf8 Mon Sep 17 00:00:00 2001 From: "mads.thines" Date: Wed, 14 Oct 2026 22:18:56 +0000 Subject: [PATCH 02/16] fix(byod): gate bootstrap drift through supabase:test, probe storage without supabase-js Move the generator to supabase/byod/ and add a supabase:test target, so the check job's nx affected -t test runs the drift spec on any change under supabase/ without a workflow edit. doctor and bootstrap imported @supabase/supabase-js, which the published zero-dependency CLI does not ship, so the BYOD check failed for every npm install. Replace it with one fetch-based PostgREST probe. bootstrap no longer prints the service-role key as a Postgres password (it is not one), and its help no longer claims it creates the schema. Co-authored-by: dash0-dev[bot] <257284812+dash0-dev[bot]@users.noreply.github.com> --- .gitattributes | 3 + docs/deployment.md | 12 +++ docs/key-files.md | 1 + packages/cli/bin/lorekit.mjs | 19 ++-- packages/cli/src/commands/bootstrap.mjs | 86 ++++++++----------- packages/cli/src/commands/doctor.mjs | 19 ++-- packages/cli/src/shared/byod.mjs | 57 ++++++++++++ packages/cli/test/byod.test.mjs | 70 +++++++++++++++ supabase/byod/bootstrap.sql | 2 +- .../byod/generate.mjs | 8 +- .../byod/generate.test.mjs | 6 +- supabase/project.json | 8 ++ 12 files changed, 209 insertions(+), 82 deletions(-) create mode 100644 .gitattributes create mode 100644 packages/cli/src/shared/byod.mjs create mode 100644 packages/cli/test/byod.test.mjs rename scripts/codegen/gen-byod-bootstrap.mjs => supabase/byod/generate.mjs (95%) rename scripts/codegen/gen-byod-bootstrap.test.mjs => supabase/byod/generate.test.mjs (94%) diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..167b849dd --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# Generated from supabase/migrations/ by `node supabase/byod/generate.mjs`. +# Collapsed in GitHub diffs; review the migrations, not this projection. +supabase/byod/bootstrap.sql linguist-generated=true diff --git a/docs/deployment.md b/docs/deployment.md index 273fddb25..36ce1ac43 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -1062,6 +1062,18 @@ The collision-proof end state is Supabase's **timestamp filenames** (`YYYYMMDDHHMMSS_…`), which are monotonic and parallel-safe — a larger, rename-everything change to consider separately. +#### Every migration regenerates the BYOD bootstrap + +`supabase/byod/bootstrap.sql` — the single file a BYOD user applies to their own +database ([byod.md](./byod.md)) — is **generated** from `supabase/migrations/` +by `node supabase/byod/generate.mjs`. After adding or editing a migration, run +it and commit the result. The `supabase` project's `test` target +(`supabase/byod/generate.test.mjs`) fails when the committed file and the +migrations disagree, and the `check` job's `nx affected -t test` runs it on any +change under `supabase/`. The file used to be a hand-written subset, which +drifted until BYOD's `memory_write` could no longer be resolved by the edge +function's calls — a generated projection cannot drift. + --- ## Prerequisites diff --git a/docs/key-files.md b/docs/key-files.md index 43664fa57..13d07f01d 100644 --- a/docs/key-files.md +++ b/docs/key-files.md @@ -81,6 +81,7 @@ else. Grouped loosely by subsystem in migration/PR order. | `supabase/migrations/00042_audit_log_github_app_action.sql` | Forward-only drop+re-add of the `audit_log.action` CHECK, widened to all 24 actions — admits `github_app.installation_linked`, which `handleSetupReturn` had been emitting into a rejecting CHECK since the GitHub App shipped (silently, via non-throwing `recordAuditEvent`) | | `packages/mcp-core/src/rest/cors-origins.ts` | Pure CORS origin matching — `expandOriginSiblings`/`expandAllowedOrigins` (apex ⇄ `www.` allowlist widening), `isOriginAllowed`, and `corsResponseHeaders` (emits `Access-Control-Allow-Origin` only for an allowed origin, never an empty one). `isOriginAllowed` ALSO always admits two classes of origin regardless of `ALLOWED_ORIGINS`, both justified by Bearer-auth-not-CORS: **loopback** dev hosts, and the project's own **Vercel deployments** — `isVercelPreviewOrigin` matches a single DNS label before exactly `.vercel.app` that BOTH starts with `lorekit-` AND ends with the Vercel account scope `-mads-thines-projects` (so `lorekit-git--mads-thines-projects.vercel.app` and the `lorekit-mads-thines-projects.vercel.app` alias match, while a nested `lorekit-x.attacker.vercel.app`, a `lorekit`-prefixed different word like `lorekitten`, and — crucially — a third-party `lorekit-x-.vercel.app` are all rejected; the scope suffix is the half a third party cannot forge). Update `VERCEL_SCOPE` if the project moves Vercel accounts. Preview deployments get a per-branch/per-commit hostname (`lorekit-git--.vercel.app`) that can't be enumerated into a static allowlist, so they were CORS-blocked once the dashboard moved off PostgREST onto the REST edge functions (PR #311). Lifted out of `supabase/functions/_shared/api/cors.ts` so it has a vitest home (`cors-origins.spec.ts`); mirrored verbatim into `supabase/functions/_shared/api/cors-origins.ts` and guarded by `edge-parity.spec.ts` | | `packages/mcp-core/src/rest/rest-tool-name.ts` | Pure `restToolName`/`REST_TOOL_NAMES` — the REST route → `usage_events.tool_name` mapping, aligning each route with the MCP tool it is the equivalent of so both surfaces aggregate as one series. Import-free, mirrored verbatim into `supabase/functions/_shared/rest/rest-tool-name.ts` and guarded by `edge-parity.spec.ts`; a route registered without a mapping fails `rest-tool-name.spec.ts` | +| `supabase/byod/generate.mjs` + `generate.test.mjs` | Generates `supabase/byod/bootstrap.sql` (BYOD's one-file schema) from `supabase/migrations/` — verbatim, in order, in ONE transaction, behind a guard that refuses an already-initialised database, recording every version in `supabase_migrations.schema_migrations` so the database later upgrades with `supabase db push`. `--check` and the spec (the `supabase:test` target) fail on drift. Replaced a hand-written subset that had fallen 42 RPCs and a `memory_write` signature behind the edge | | `supabase/tests/bare-postgres-bootstrap.sql` | Makes `migrations.test.sql` runnable WITHOUT Docker: supplies the Supabase-platform pieces the migrations reference (the three `auth.*` claim readers, `auth.users`, `auth.identities`, the anon/authenticated/service_role roles — `service_role` BYPASSRLS as in production) so the migrations apply to a bare `initdb` cluster. Carries its own runbook and, prominently, its own fidelity caveat: a FAILURE is strong evidence, a PASS is not a substitute for CI's `Integration smoke` job. Exists because `supabase start` needs a Docker socket, which cloud sandboxes and agent sessions do not have — previously making the one file that exercises raw migration logic verifiable only by pushing and reading a CI log | | `packages/mcp-core/src/telemetry/telemetry-vocabulary.ts` | The CLOSED set of names `usage_events.tool_name` may carry: `TELEMETRY_OP_NAMES` = the catalog's tool names (DERIVED from `MCP_TOOLS`, never restated) ∪ `NON_CATALOG_OPS` — the REST operations with no MCP tool, each carrying the reason it has none. Closes the hole left by `restToolName`'s deliberate `.unmapped` fallback, which is right at runtime and silent in development. The ten dashboard analytics reads carry a `restOnly` reason (`REST_ONLY_OP_NAMES`), which is how the "REST-only on purpose" decision is machine-checked rather than only written down. **Node-side only — deliberately NOT mirrored:** nothing in the edge tree imports it, so unlike `account-wide-tools.ts` it is free to import the catalog. Guarded by `telemetry-vocabulary.spec.ts` (closure both ways, the ten pinned by name, and a scan proving no CLI command calls the ten while `/memories/relevant` still does) | | `supabase/functions/_shared/telemetry/usage.ts` | `recordUsageEvent` + `getUserPlanName` — the single edge usage-event writer, called once per MCP tool call (`mcp/mcp-handler.ts`) and once per REST route (`_shared/api/router.ts`). `mcp/limits.ts` re-exports it | diff --git a/packages/cli/bin/lorekit.mjs b/packages/cli/bin/lorekit.mjs index b1d4af638..caff2ff9a 100755 --- a/packages/cli/bin/lorekit.mjs +++ b/packages/cli/bin/lorekit.mjs @@ -120,7 +120,7 @@ ${c.bold('Commands')} regardless of policy. protect [--off] to unprotect. Remote only. --json. pin / unpin Shorthand for protect / protect --off. Remote only. --json. - bootstrap Apply the BYOD schema to a user-supplied Supabase database. + bootstrap Check a BYOD Supabase database and print how to apply the schema. Only needed when using LOREKIT_STORAGE_URL / LOREKIT_STORAGE_ANON_KEY. See docs/byod.md for setup instructions. migrate Relocate a LoreKit-format local store into the current layout, @@ -856,19 +856,20 @@ ${c.bold('Options')} --json Machine-readable result `, - bootstrap: `${c.bold('lorekit bootstrap')} — apply the LoreKit schema to your own Supabase database + bootstrap: `${c.bold('lorekit bootstrap')} — check your own Supabase database and show how to apply the LoreKit schema ${c.bold('Usage')} lorekit bootstrap [--yes] -For BYOD ("bring your own database") only: creates the tables, functions and -policies LoreKit needs in a Supabase project you control, so lore never leaves -your infrastructure. You only need this if you are pointing the CLI at your own -database via ${c.cyan('LOREKIT_STORAGE_URL')} / ${c.cyan('LOREKIT_STORAGE_ANON_KEY')} — the hosted -service and the offline store both need nothing here. +For BYOD ("bring your own database") only. Probes the Supabase project named by +${c.cyan('LOREKIT_STORAGE_URL')} (with ${c.cyan('LOREKIT_STORAGE_ANON_KEY')} or ${c.cyan('LOREKIT_STORAGE_SERVICE_KEY')}), +reports whether the LoreKit schema is already there, and prints the commands +that apply it: ${c.cyan('psql')} with supabase/byod/bootstrap.sql, the SQL Editor, or +${c.cyan('supabase db push')}. It does not run DDL itself — a Supabase API key cannot. +The hosted service and the offline store both need nothing here. -See ${c.cyan('docs/byod.md')} for the full setup, including which key to use and what the -schema contains. +See ${c.cyan('docs/byod.md')} for the full setup, including which key to use and how +upgrades work. ${c.bold('Options')} -y, --yes Apply without prompting diff --git a/packages/cli/src/commands/bootstrap.mjs b/packages/cli/src/commands/bootstrap.mjs index ed63ffecc..9582df03b 100644 --- a/packages/cli/src/commands/bootstrap.mjs +++ b/packages/cli/src/commands/bootstrap.mjs @@ -1,18 +1,20 @@ -// `lorekit bootstrap` — apply the BYOD schema to a user-supplied Supabase database. +// `lorekit bootstrap` — check a BYOD Supabase database and print how to apply +// the LoreKit schema to it. // -// Reads LOREKIT_STORAGE_URL and LOREKIT_STORAGE_SERVICE_KEY from the environment. -// If neither is set, prints a helpful message and exits 0 — bootstrap is only -// needed for BYOD (Bring Your Own Database) setups. +// Reads LOREKIT_STORAGE_URL and LOREKIT_STORAGE_ANON_KEY / LOREKIT_STORAGE_SERVICE_KEY +// from the environment. If none is set, prints a helpful message and exits 0 — +// bootstrap is only needed for BYOD (Bring Your Own Database) setups. // -// Because Supabase's JS client does not expose a raw SQL execution method for DDL, -// this command instructs the user to run the SQL file directly with psql when no -// direct execution path is available. It validates connectivity using the anon key -// and confirms the bootstrap.sql path for the user. +// It does NOT apply the schema itself: PostgREST (the only thing a Supabase API +// key reaches) cannot run DDL. It probes the project, locates +// supabase/byod/bootstrap.sql (generated from the migrations), and prints the +// psql / SQL Editor / `supabase db push` routes. import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import process from 'node:process'; import { log, err, heading, status, c } from '../shared/util.mjs'; +import { probeByodStorage } from '../shared/byod.mjs'; // The bootstrap.sql file is at /supabase/byod/bootstrap.sql. // This file lives at packages/cli/src/commands/bootstrap.mjs, so the relative @@ -74,59 +76,41 @@ export async function bootstrap(_args) { status('pass', 'bootstrap.sql', sqlPath); status('info', 'storage url', storageUrl); - // Test connectivity using the anon key if available (non-DDL path). - if (storageAnonKey) { - try { - const { createClient } = await import('@supabase/supabase-js'); - const db = createClient(storageUrl, storageAnonKey, { - auth: { persistSession: false, autoRefreshToken: false }, - }); - // Simple ping: list memories (will return 0 rows or error if schema not applied yet). - const { error } = await db.from('memories').select('id').limit(1); - if (error && error.code === '42P01') { - // Table does not exist yet — that's expected before bootstrap. - status('info', 'connectivity', 'connected — schema not yet applied (run bootstrap)'); - } else if (error) { - status('warn', 'connectivity', `connected but got: ${error.message}`); - } else { - status('pass', 'connectivity', 'connected and schema already present'); - } - } catch (e) { - status('warn', 'connectivity', `could not verify: ${e && e.message ? e.message : String(e)}`); + // Probe the project with whichever key is set (the anon key when both are, + // so the check runs with the least privilege that can answer it). + const probeKey = storageAnonKey || storageServiceKey; + if (probeKey) { + const probe = await probeByodStorage({ url: storageUrl, key: probeKey }); + if (probe.state === 'ready') { + status('pass', 'connectivity', 'connected — LoreKit schema already present'); + log(''); + log(' bootstrap.sql only initialises an EMPTY database. To upgrade this one, run'); + log(` from a LoreKit checkout: ${c.cyan('supabase link --project-ref')} ${parseRef(storageUrl)} && ${c.cyan('supabase db push')}`); + log(''); + return 0; } + if (probe.state === 'no-schema') status('info', 'connectivity', 'connected — schema not applied yet'); + else status('warn', 'connectivity', probe.detail); } else { - status('info', 'connectivity', 'skipped — LOREKIT_STORAGE_ANON_KEY not set'); + status('info', 'connectivity', 'skipped — no LOREKIT_STORAGE_ANON_KEY / LOREKIT_STORAGE_SERVICE_KEY set'); } - // Supabase JS client cannot execute raw DDL SQL directly. - // The correct path is psql or the Supabase dashboard SQL editor. + // PostgREST cannot run DDL, so the schema goes in over a Postgres connection + // (psql) or the SQL Editor. Never print a key here: the connection string's + // password is the DATABASE password, which is not the service-role API key. log(''); log(` ${c.bold('To apply the schema, run:')}`); log(''); - log(` ${c.cyan('psql "$DATABASE_URL" -f')} ${sqlPath}`); + log(` ${c.cyan('psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -f')} ${sqlPath}`); + log(''); + log(` ${c.dim('$DATABASE_URL is the connection string from the Supabase dashboard → Connect;')}`); + log(` ${c.dim('its password is your database password, not an API key.')}`); log(''); log(` Or paste the contents of ${c.dim(sqlPath)}`); - log(' into the Supabase dashboard → SQL Editor.'); + log(' into the Supabase dashboard → SQL Editor and run it.'); + log(''); + log(` Or, from a LoreKit checkout: ${c.cyan('supabase link --project-ref')} ${parseRef(storageUrl)} && ${c.cyan('supabase db push')}`); log(''); - - if (storageServiceKey) { - status( - 'info', - 'service key', - 'LOREKIT_STORAGE_SERVICE_KEY is set — use it as $DATABASE_URL password with psql', - ); - log(''); - log(` ${c.dim('Example:')}`); - log( - ` ${c.cyan('DATABASE_URL')}="postgresql://postgres.${parseRef(storageUrl)}:${storageServiceKey}@aws-0-us-east-1.pooler.supabase.com:6543/postgres"`, - ); - log(` ${c.cyan('psql "$DATABASE_URL"')} -f ${sqlPath}`); - log(''); - log( - ` ${c.dim('(Replace the host/port above with the exact connection string from your')}`, - ); - log(` ${c.dim('Supabase dashboard → Settings → Database → Connection string.)')}`); - } return 0; } diff --git a/packages/cli/src/commands/doctor.mjs b/packages/cli/src/commands/doctor.mjs index 9b5c4660f..0c43564c0 100644 --- a/packages/cli/src/commands/doctor.mjs +++ b/packages/cli/src/commands/doctor.mjs @@ -27,6 +27,7 @@ import { describeIdentity } from '../telemetry/telemetry-identity.mjs'; import { deriveScope } from '../shared/scope.mjs'; import { loadControl, HOOK_INSTRUCTION_EVENTS } from '../shared/control.mjs'; import { createStore } from '../store/index.mjs'; +import { probeByodStorage } from '../shared/byod.mjs'; import { log, heading, status, c } from '../shared/util.mjs'; const AUTH_CODES = new Set([401, 403, -32001]); @@ -644,20 +645,10 @@ async function checkBYODStorage(record) { return; } - try { - const { createClient } = await import('@supabase/supabase-js'); - const db = createClient(storageUrl, storageAnonKey, { - auth: { persistSession: false, autoRefreshToken: false }, - }); - const { error } = await db.from('memories').select('id').limit(1); - if (error && error.code !== '42P01') { - record('fail', 'byod storage', `connectivity error: ${error.message}`); - } else { - record('pass', 'byod storage', `ok — ${storageUrl}`); - } - } catch (e) { - record('fail', 'byod storage', `could not connect: ${e && e.message ? e.message : String(e)}`); - } + const probe = await probeByodStorage({ url: storageUrl, key: storageAnonKey }); + if (probe.state === 'ready') record('pass', 'byod storage', `ok — ${storageUrl}`); + else if (probe.state === 'no-schema') record('fail', 'byod storage', `${probe.detail} — see docs/byod.md`); + else record('fail', 'byod storage', probe.detail); } function gitTracked(root, dir) { diff --git a/packages/cli/src/shared/byod.mjs b/packages/cli/src/shared/byod.mjs new file mode 100644 index 000000000..51c014889 --- /dev/null +++ b/packages/cli/src/shared/byod.mjs @@ -0,0 +1,57 @@ +// BYOD ("bring your own database") storage probe — one PostgREST round-trip +// that says whether a Supabase project is reachable and holds the LoreKit +// schema. Shared by `lorekit doctor` and `lorekit bootstrap`. +// +// Zero-dependency on purpose. Both callers used to `import('@supabase/supabase-js')`, +// which the published CLI does not ship (it has no dependencies), so for every +// npm install the probe failed before sending a request and `doctor` reported +// "could not connect" whatever state the database was in. PostgREST is plain +// HTTP; `fetch` is all it needs. + +const DEFAULT_TIMEOUT_MS = 10_000; + +// PostgREST's answer for a table it cannot see. 42P01 is Postgres's +// undefined_table, surfaced by older PostgREST; PGRST205 is the schema-cache +// miss newer PostgREST returns for the same situation. +const MISSING_TABLE_CODES = new Set(['42P01', 'PGRST205']); + +/** + * Probe a BYOD Supabase project. + * @param {{ url: string, key: string, fetchImpl?: typeof fetch, timeoutMs?: number }} opts + * @returns {Promise<{ state: 'ready' } | { state: 'no-schema', detail: string } | { state: 'error', detail: string }>} + * ready — reachable, and the `memories` table answers + * no-schema — reachable, but the LoreKit schema has not been applied + * error — unreachable, rejected the key, or answered unexpectedly + */ +export async function probeByodStorage({ url, key, fetchImpl = globalThis.fetch, timeoutMs = DEFAULT_TIMEOUT_MS }) { + let endpoint; + try { + endpoint = new URL('rest/v1/memories?select=id&limit=1', url.endsWith('/') ? url : `${url}/`); + } catch { + return { state: 'error', detail: `not a valid URL: ${url}` }; + } + + let res; + try { + res = await fetchImpl(endpoint, { + headers: { apikey: key, Authorization: `Bearer ${key}`, Accept: 'application/json' }, + signal: AbortSignal.timeout(timeoutMs), + }); + } catch (e) { + return { state: 'error', detail: `could not connect: ${e && e.message ? e.message : String(e)}` }; + } + + if (res.ok) return { state: 'ready' }; + + let body = null; + try { + body = await res.json(); + } catch { + // Non-JSON error body (a proxy page, an HTML 502): reported by status alone. + } + if (body && MISSING_TABLE_CODES.has(body.code)) { + return { state: 'no-schema', detail: 'connected, but the LoreKit schema is not applied' }; + } + const message = body && body.message ? `: ${body.message}` : ''; + return { state: 'error', detail: `HTTP ${res.status}${message}` }; +} diff --git a/packages/cli/test/byod.test.mjs b/packages/cli/test/byod.test.mjs new file mode 100644 index 000000000..f7b6b3194 --- /dev/null +++ b/packages/cli/test/byod.test.mjs @@ -0,0 +1,70 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { probeByodStorage } from '../src/shared/byod.mjs'; + +// A fetch stand-in: records the request and answers with `status` + `body`. +function fakeFetch(status, body, calls = []) { + return async (url, init) => { + calls.push({ url: String(url), init }); + return { + ok: status >= 200 && status < 300, + status, + json: async () => { + if (body === undefined) throw new SyntaxError('not json'); + return body; + }, + }; + }; +} + +const URL_ = 'https://abcdefgh.supabase.co'; + +test('a 200 from the memories table is ready, and the key rides both headers', async () => { + const calls = []; + const r = await probeByodStorage({ url: URL_, key: 'k', fetchImpl: fakeFetch(200, [], calls) }); + assert.deepEqual(r, { state: 'ready' }); + assert.equal(calls[0].url, `${URL_}/rest/v1/memories?select=id&limit=1`); + assert.equal(calls[0].init.headers.apikey, 'k'); + assert.equal(calls[0].init.headers.Authorization, 'Bearer k'); +}); + +test('a trailing slash on the URL does not double up', async () => { + const calls = []; + await probeByodStorage({ url: `${URL_}/`, key: 'k', fetchImpl: fakeFetch(200, [], calls) }); + assert.equal(calls[0].url, `${URL_}/rest/v1/memories?select=id&limit=1`); +}); + +test('a missing table is no-schema under both PostgREST codes', async () => { + for (const code of ['42P01', 'PGRST205']) { + const r = await probeByodStorage({ url: URL_, key: 'k', fetchImpl: fakeFetch(404, { code, message: 'x' }) }); + assert.equal(r.state, 'no-schema', code); + } +}); + +test('a rejected key is an error carrying the status and message', async () => { + const r = await probeByodStorage({ url: URL_, key: 'bad', fetchImpl: fakeFetch(401, { message: 'Invalid API key' }) }); + assert.deepEqual(r, { state: 'error', detail: 'HTTP 401: Invalid API key' }); +}); + +test('a non-JSON error body is reported by status alone', async () => { + const r = await probeByodStorage({ url: URL_, key: 'k', fetchImpl: fakeFetch(502, undefined) }); + assert.deepEqual(r, { state: 'error', detail: 'HTTP 502' }); +}); + +test('a network failure is an error, never a throw', async () => { + const r = await probeByodStorage({ + url: URL_, + key: 'k', + fetchImpl: async () => { + throw new Error('getaddrinfo ENOTFOUND'); + }, + }); + assert.deepEqual(r, { state: 'error', detail: 'could not connect: getaddrinfo ENOTFOUND' }); +}); + +test('an unparseable URL is an error without a request', async () => { + const calls = []; + const r = await probeByodStorage({ url: 'not a url', key: 'k', fetchImpl: fakeFetch(200, [], calls) }); + assert.equal(r.state, 'error'); + assert.equal(calls.length, 0); +}); diff --git a/supabase/byod/bootstrap.sql b/supabase/byod/bootstrap.sql index e70071838..19d1a770b 100644 --- a/supabase/byod/bootstrap.sql +++ b/supabase/byod/bootstrap.sql @@ -1,7 +1,7 @@ -- LoreKit BYOD bootstrap schema -- -- ⚠ GENERATED FILE — DO NOT EDIT BY HAND. --- Generated by `node scripts/codegen/gen-byod-bootstrap.mjs` from the +-- Generated by `node supabase/byod/generate.mjs` from the -- 108 migrations in supabase/migrations/ (newest: 00113_explorer_reverse_activity_filters_and_opened_sort.sql). -- CI fails when this file and the migrations disagree; edit a migration (or add -- one) and regenerate instead. diff --git a/scripts/codegen/gen-byod-bootstrap.mjs b/supabase/byod/generate.mjs similarity index 95% rename from scripts/codegen/gen-byod-bootstrap.mjs rename to supabase/byod/generate.mjs index e6ea43d6d..d11f32ecb 100644 --- a/scripts/codegen/gen-byod-bootstrap.mjs +++ b/supabase/byod/generate.mjs @@ -3,8 +3,8 @@ // Source of truth: supabase/migrations/*.sql (applied in filename order) // Generated file: supabase/byod/bootstrap.sql // -// node scripts/codegen/gen-byod-bootstrap.mjs regenerate bootstrap.sql -// node scripts/codegen/gen-byod-bootstrap.mjs --check exit 1 if it is stale (CI) +// node supabase/byod/generate.mjs regenerate bootstrap.sql +// node supabase/byod/generate.mjs --check exit 1 if it is stale (CI) // // Why generated: bootstrap.sql used to be a hand-written SUBSET of the schema. // The edge function kept moving with the migrations and the subset did not — @@ -70,7 +70,7 @@ export function renderBootstrap(migrations) { const header = `-- LoreKit BYOD bootstrap schema -- -- ⚠ GENERATED FILE — DO NOT EDIT BY HAND. --- Generated by \`node scripts/codegen/gen-byod-bootstrap.mjs\` from the +-- Generated by \`node supabase/byod/generate.mjs\` from the -- ${parsed.length} migrations in supabase/migrations/ (newest: ${newest}). -- CI fails when this file and the migrations disagree; edit a migration (or add -- one) and regenerate instead. @@ -168,7 +168,7 @@ function main() { } if (actual !== expected) { console.error(`${rel} is out of date with supabase/migrations/.`); - console.error('Run: node scripts/codegen/gen-byod-bootstrap.mjs'); + console.error('Run: node supabase/byod/generate.mjs'); process.exit(1); } console.log(`${rel} matches supabase/migrations/.`); diff --git a/scripts/codegen/gen-byod-bootstrap.test.mjs b/supabase/byod/generate.test.mjs similarity index 94% rename from scripts/codegen/gen-byod-bootstrap.test.mjs rename to supabase/byod/generate.test.mjs index 074d33fcc..2c579d2cc 100644 --- a/scripts/codegen/gen-byod-bootstrap.test.mjs +++ b/supabase/byod/generate.test.mjs @@ -1,5 +1,5 @@ // Unit tests for the BYOD bootstrap generator's pure logic. -// node --test scripts/codegen/gen-byod-bootstrap.test.mjs +// node --test supabase/byod/generate.test.mjs import { test } from 'node:test'; import assert from 'node:assert/strict'; import fs from 'node:fs'; @@ -8,7 +8,7 @@ import { renderBootstrap, readMigrations, OUTPUT_PATH, -} from './gen-byod-bootstrap.mjs'; +} from './generate.mjs'; test('parseMigrationName follows the Supabase CLI grammar', () => { assert.deepEqual(parseMigrationName('00001_memories.sql'), { version: '00001', name: 'memories' }); @@ -57,6 +57,6 @@ test('the committed bootstrap.sql is generated from the current migrations', () assert.equal( fs.readFileSync(OUTPUT_PATH, 'utf8'), renderBootstrap(readMigrations()), - 'run: node scripts/codegen/gen-byod-bootstrap.mjs', + 'run: node supabase/byod/generate.mjs', ); }); diff --git a/supabase/project.json b/supabase/project.json index aecbd4ec4..3d7e8e57e 100644 --- a/supabase/project.json +++ b/supabase/project.json @@ -4,6 +4,14 @@ "projectType": "application", "tags": ["type:infra"], "targets": { + "test": { + "executor": "nx:run-commands", + "options": { + "command": "node --test byod/generate.test.mjs", + "cwd": "supabase" + } + }, + "start": { "executor": "nx:run-commands", "options": { From 8875f976d2dba6260bd8212316d46221af07a587 Mon Sep 17 00:00:00 2001 From: "mads.thines" Date: Wed, 14 Oct 2026 22:22:21 +0000 Subject: [PATCH 03/16] docs(byod): correct how BYOD is wired, and propose a local SQLite mode byod.md: BYOD is read from the mcp function's own secrets, so it needs a self-deployed function; an env block in .mcp.json never reaches the hosted one. Document the generated bootstrap.sql, upgrades via supabase db push, and the limits found while fixing it: only the mcp function has a BYOD mode, and a token's user must exist in the storage project's auth.users. tasks/local-sqlite-mode.md: a phased proposal for running the dashboard and usage views on SQLite without Supabase, by implementing the REST contract locally rather than porting the 48 RPCs. Co-authored-by: dash0-dev[bot] <257284812+dash0-dev[bot]@users.noreply.github.com> --- docs/byod.md | 158 ++++++++++++++++++++++---------- docs/tasks/local-sqlite-mode.md | 154 +++++++++++++++++++++++++++++++ 2 files changed, 264 insertions(+), 48 deletions(-) create mode 100644 docs/tasks/local-sqlite-mode.md diff --git a/docs/byod.md b/docs/byod.md index 9ad050ce3..12294fbd6 100644 --- a/docs/byod.md +++ b/docs/byod.md @@ -1,80 +1,142 @@ # Bring Your Own Database (BYOD) -LoreKit's memory tools can point at a Supabase project you control instead of -the hosted LoreKit backend. This is useful when you have data residency -requirements, an existing Supabase subscription, or want full control over -your memory data. +BYOD stores memories in a Supabase project you control, separate from the +project that runs LoreKit's `mcp` Edge Function. Use it when memory rows must +live in a specific database — data residency, an existing Supabase +subscription, or full control over the data. + +It covers the `memory.*` MCP tools. Read [How BYOD is wired](#how-byod-is-wired) +before setting it up: what reaches the storage project, and what does not, is +the part most likely to surprise you. + +## How BYOD is wired + +- **It needs your own deployment of the `mcp` function.** The function reads + `LOREKIT_STORAGE_URL`, `LOREKIT_STORAGE_ANON_KEY` and the optional + `LOREKIT_STORAGE_SERVICE_KEY` from its **own Edge Function secrets**. The + hosted endpoint (`https://pqokxlhvnosogizsjztg.supabase.co/functions/v1/mcp`) + has no BYOD storage configured, and an `env` block in your `.mcp.json` does + not reach it — those variables only reach the local `mcp-remote` process on + your machine. Deploy the function yourself first: [install.md](./install.md). +- **Two projects, two jobs.** API tokens and users are resolved in the + function's own project; memory rows are read from and written to the + storage project. +- **Only the `mcp` function has a BYOD mode.** The `memories` REST function — + which backs the web dashboard, the CLI's remote commands and the hook + engine's lesson injection — always uses the function's own project. See + [Limitations](#limitations). ## Setup -### 1. Create a Supabase project +### 1. Create the storage project -Create a free project at [supabase.com](https://supabase.com). Note your -project URL and API keys from **Settings → API**. +Create a project at [supabase.com](https://supabase.com). Note its URL and API +keys (**Project Settings → API**) and its database connection string +(**Connect**). ### 2. Apply the LoreKit schema -Run the bootstrap command to create the required tables and functions: +The storage project needs the full LoreKit schema — the same migrations the +hosted project runs. Pick one route. + +**Supabase CLI (recommended — the same command upgrades it later):** ```bash -LOREKIT_STORAGE_URL=https://your-project.supabase.co \ -LOREKIT_STORAGE_SERVICE_KEY=your-service-role-key \ -lorekit bootstrap +git clone https://github.com/mthines/lorekit.git +cd lorekit +supabase link --project-ref +supabase db push ``` -If the CLI is not installed, apply the schema directly with psql: +**One SQL file (empty database only):** ```bash -psql "$DATABASE_URL" -f path/to/bootstrap.sql +psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -f supabase/byod/bootstrap.sql ``` -Or copy the raw SQL from the [LoreKit repository](https://github.com/mthines/lorekit/blob/main/supabase/byod/bootstrap.sql) and paste it into the Supabase dashboard → SQL Editor. - -### 3. Configure your MCP server +or paste [`supabase/byod/bootstrap.sql`](https://github.com/mthines/lorekit/blob/main/supabase/byod/bootstrap.sql) +into the Supabase dashboard → **SQL Editor** and run it. `$DATABASE_URL` is the +connection string from **Connect**; its password is your **database password**, +not an API key. -In your `.mcp.json` or equivalent MCP config, add the env vars: +`bootstrap.sql` is generated from `supabase/migrations/` (CI fails when the two +disagree). It applies every migration in one transaction, so a failure leaves +the database untouched; it refuses to run against a database that already has +the schema; and it records each migration in Supabase's migration history, so +the CLI route above upgrades the database later. -```json -{ - "env": { - "LOREKIT_STORAGE_URL": "https://your-project.supabase.co", - "LOREKIT_STORAGE_ANON_KEY": "your-anon-key" - } -} -``` +`lorekit bootstrap`, run with `LOREKIT_STORAGE_URL` and a key in your shell, +checks whether the project is reachable and already has the schema, and prints +these commands. It does not run any DDL itself — a Supabase API key cannot. -Or set them in your shell before starting the MCP server. +### 3. Point your `mcp` function at it -## Billing +Set the storage secrets on the project that runs your `mcp` function: -Memories stored in your own database are **not counted** against LoreKit's -hosted memory-count billing. LoreKit has no visibility into your private -database and cannot meter memories stored there. +```bash +supabase secrets set --project-ref \ + LOREKIT_STORAGE_URL=https://.supabase.co \ + LOREKIT_STORAGE_ANON_KEY= +``` -BYOD users are on a flat-rate or open-source tier — no per-memory billing. +`LOREKIT_STORAGE_SERVICE_KEY` is optional. When it is set, the function uses it +instead of the anon key, and it **bypasses row-level security** in the storage +project: data isolation then rests on the `user_id` filters the function +applies, not on RLS policies. -You are responsible for configuring memory limits and rate limiting in your -own Supabase project. The bootstrap schema ships with a default 5,000-memory -cap trigger, which you can adjust by editing the `lorekit_default_limit` -function in your database. +### 4. Check it -## Limitations +```bash +LOREKIT_STORAGE_URL=https://.supabase.co \ +LOREKIT_STORAGE_ANON_KEY= \ +lorekit doctor +``` -The following LoreKit features are **not available** with BYOD: +The `byod storage` line passes when the project is reachable and holds the +schema, and says so when the schema has not been applied yet. -- **Org/scope sharing** — requires the hosted backend's org management RPCs -- **Web dashboard** — tightly coupled to hosted Supabase Auth -- **Audit logs** — not included in the BYOD schema -- **Webhook secrets** — not included in the BYOD schema -- **Automatic schema upgrades** — when LoreKit ships schema changes, you - must apply them manually. Run `lorekit doctor` to check compatibility. +## Upgrading -## Checking connectivity +When a LoreKit release adds migrations, upgrade the storage project from a +checkout of that release: ```bash -LOREKIT_STORAGE_URL=https://your-project.supabase.co \ -LOREKIT_STORAGE_ANON_KEY=your-anon-key \ -lorekit doctor +supabase link --project-ref +supabase db push ``` -The output will include a `[byod] storage ok` line if the database is reachable. +`db push` applies only the migrations the database has not recorded, which +works the same whether you set the database up with `db push` or with +`bootstrap.sql`. Upgrade the storage project before deploying the matching +`mcp` function: the function calls the RPCs those migrations define. + +## Billing + +Memories stored in your own database are **not counted** against LoreKit's +hosted memory-count billing. LoreKit has no visibility into your private +database and cannot meter memories stored there. + +The memory cap (default 5,000 active memories per user) and the limits table +live in your database: adjust them through `lorekit_default_limit` and +`user_limits` there. In BYOD mode the function skips the hosted rate limiter +and does not record hosted usage events. + +## Limitations + +- **Only the MCP tools use the storage project.** The web dashboard, the REST + API, the CLI's remote commands (`list`, `search`, `show`, `write` against the + remote store) and the hook engine's lesson injection go through the `memories` + function, which has no BYOD mode. They read and write the function's own + project and do not see memories stored through BYOD. +- **A token's user must also exist in the storage project.** `memories.user_id` + references the storage project's `auth.users`. A write made with an `lk_*` + token carries the token owner's id from the function's project, and it fails + with `memories_user_id_fkey` unless a user with that id also exists in the + storage project. Writes made with the service-role key carry no user id and + are not affected. +- **No per-user RLS in the storage project.** The function talks to it with the + service or anon key, never with the caller's JWT. +- **Tested surface.** The BYOD smoke test covers write, read, list, search and + delete across the four scope types. The schema now also contains the org, + audit, retention and webhook tables and RPCs, but BYOD has not been exercised + for those features — treat them as unsupported. diff --git a/docs/tasks/local-sqlite-mode.md b/docs/tasks/local-sqlite-mode.md new file mode 100644 index 000000000..36cd425e0 --- /dev/null +++ b/docs/tasks/local-sqlite-mode.md @@ -0,0 +1,154 @@ +# Proposal: local mode on SQLite — dashboard and usage data without Supabase + +> Status: **proposal / not started** · Area: CLI, web, REST contract · Size: **L** (phased) +> Companion to the BYOD fix that generated `supabase/byod/bootstrap.sql` from the +> migrations. Nothing in this document is built yet; it is here to be argued with +> before any code lands. + +## Problem + +There are two ways to run LoreKit today, and neither gives one person a +dashboard without Supabase: + +- **Offline store** — markdown files under `~/.lorekit` and `/.lorekit` + (`packages/cli/src/store/local.mjs`), served to agents by the local stdio MCP + server `lorekit mcp`. No account, no network — but no dashboard, no read + counters and no usage events, so `/insights` has nothing to show. +- **Hosted or self-hosted** — the dashboard, REST API and MCP server all sit on + Supabase: Postgres, Supabase Auth and Edge Functions. Self-hosting means your + own Supabase and Vercel projects ([install.md](../install.md)); BYOD only moves + memory rows to a second Supabase project ([byod.md](../byod.md)). + +The goal: run LoreKit on one machine, browse lore in the dashboard and read +usage data, with no Supabase — for local development and single-user +self-hosting. + +## What makes this hard + +The server logic lives in Postgres, not in TypeScript: + +| Fact (measured on `main` at 00113) | Consequence for SQLite | +|---|---| +| 108 migrations define 87 SQL functions; the edge functions call **48** distinct RPCs | Each RPC a route needs must be re-implemented | +| The Explorer's routes rest on ~1,200 lines of PL/pgSQL (`lorekit_memory_list` 206, `_facets` 218, `_activity` 136, `memory_write` 245, `memory_delete` 157, …); Insights on ~690 more (`lorekit_usage_stats`, `_usage_runs`, `_read_activity`, `_memory_pivot` 158, `_utility_*`, …) | That is the size of a second implementation | +| `lorekit_memory_list`, `_facets`, `_activity` and `_pivot` were all redefined in the newest migration (00113) | A second implementation drifts at that rate unless something forces parity | +| Full-text search is `tsvector` + `websearch_to_tsquery`; graded relevance is `ts_rank_cd` by decision | SQLite FTS5 ranks with `bm25` — ordering will differ, and must be documented as a local-mode difference, not relitigated for hosted | +| Labels are `text[]` with containment/overlap; RLS and `auth.uid()` scope every read | JSON arrays + `json_each`; single user, so tenant scoping collapses to one fixed id | +| Edge handlers speak PostgREST's query builder and `.rpc()`, not a repository interface | There is no seam to swap a SQLite client in behind the existing handlers | +| The dashboard reads memories and analytics through the REST API (`packages/web/src/lib/api/`, base URL derived from `NEXT_PUBLIC_SUPABASE_URL` in `rest.ts`), but signs in with Supabase Auth (`middleware.ts`, `@supabase/ssr`) and still has **40** direct supabase-js call sites for orgs, invites, API tokens, audit log, scope bindings and GitHub installations | The REST contract is the seam; the supabase-js surfaces stay hosted-only | + +The [BYOD drift](../byod.md) is the cautionary example: a hand-written second +copy of the schema fell 42 RPCs and a `memory_write` signature behind before +anyone noticed. Full SQLite parity would be that, at the scale of the whole +server. So this proposal does **not** port the server. It implements the REST +contract the dashboard already consumes, for one user, and holds it to the +Postgres implementation with shared contract tests. + +## Proposal + +```text +agent ──stdio──▶ lorekit mcp ──────────┐ +CLI / hooks ───────────────────────────┼──▶ SQLite store (~/.lorekit/lorekit.db) +dashboard ──HTTP──▶ lorekit serve ─────┘ node:sqlite, FTS5 + (REST subset of @lorekit/schemas) +``` + +### 1. A SQLite store behind the existing store contract + +`createSqliteStore(path)` beside `createLocalStore`/`createRemoteStore` +(`packages/cli/src/store/`), implementing the same contract, so `lorekit mcp`, +the hook engine and the read commands work unchanged with `--store sqlite`. + +- **Engine: `node:sqlite`**, built into Node — no npm dependency, so the CLI + stays zero-dependency. Verified in a sandbox on Node 24.21 (SQLite 3.53.4): + FTS5 with the `porter unicode61` tokenizer (`retry` matches `retried`) and + JSON1 both work. It needs Node 22.13+, where the module no longer sits behind + a flag; the CLI declares `node >=18`, so the SQLite store is opt-in and fails + with a clear message on older Node instead of raising the floor for everyone. +- **Schema**: one `memories` table carrying the columns `@lorekit/schemas` + responses need (scope, key, value, labels, kind, host, origin, TTL, archive, + protection, seen/read/opened counters, timestamps), an FTS5 external-content + table kept in sync by triggers, `usage_events`, and the read ledger the + read-activity view aggregates. + +| Postgres | SQLite | +|---|---| +| `uuid`, `timestamptz` | `TEXT` (UUID string; ISO-8601 UTC, so text order is time order) | +| `text[]` + `@>` / `&&` | JSON array + `json_each` | +| `tsvector`, `websearch_to_tsquery`, `ts_rank_cd` | FTS5 `MATCH`, `bm25()` | +| Partial unique indexes per ownership kind | Partial unique index on `(scope, key)` among active rows — one owner | +| `updated_at` and memory-cap triggers | SQLite triggers | +| `pg_cron` purges (expired, archived, old usage events) | Run in-process on start and hourly | +| RLS, `auth.uid()`, orgs | Not applicable — single local user | + +`lorekit migrate --to sqlite` imports the markdown store. Markdown stays the +default offline store; SQLite is what you switch to when you want the dashboard. + +### 2. `lorekit serve` — the REST subset, locally + +A Node HTTP server that answers the routes the dashboard calls, with the +request and response shapes from `@lorekit/schemas`, against the SQLite store. + +| Phase | Routes | Unlocks | +|---|---|---| +| Explorer | `POST /memories/list`, `GET/PATCH/DELETE /memories/:id`, `POST /memories`, `/memories/facets`, `/memories/scopes`, `/memories/activity`, `/memories/restore`, `/memories/protect` | Browse, filter, search, edit, archive | +| Insights | `/memories/usage`, `/memories/usage/runs`, `/memories/read-activity`, `/memories/read-ranking`, `/memories/utility`, `/memories/pivot`, `/memories/clusters` | Usage data | +| Later | `/memories/policies`, `/memories/groom/*`, `/memories/purge` | Grooming | + +Every other route answers `501` with a stable `local_mode_unsupported` error +code that the dashboard renders as "not available in local mode". + +Usage data comes from the same places as hosted: `lorekit mcp` and `serve` +record a `usage_events` row per operation, using the closed `tool_name` +vocabulary in `packages/mcp-core/src/telemetry/telemetry-vocabulary.ts`, and +bump read counters on reads — so `/insights` renders from local activity. + +### 3. Dashboard local mode + +- `NEXT_PUBLIC_LOREKIT_API_URL` overrides the REST base URL that `rest.ts` + derives from `NEXT_PUBLIC_SUPABASE_URL` today. +- `LOREKIT_LOCAL_MODE=1` makes `middleware.ts` skip the Supabase session and + use a fixed local identity, and hides the navigation entries backed by direct + supabase-js calls (orgs, invites, API tokens, audit log, scope bindings, + GitHub installations). +- Run it as `lorekit serve` plus the dashboard's dev server for development. + How to ship it to self-hosters is open question 2. + +### 4. Keeping two backends honest + +The REST contract is the single source of truth; both backends are tested +against it. One fixture suite — seed rows, request, expected response — runs +against the Postgres handlers (the existing `memories-api.integration.spec.ts` +stack) and against `serve`. A route whose two answers disagree fails CI. That is +the mechanism that stops the BYOD-style drift; without it, this proposal +should not ship. + +## Phases + +1. SQLite store + `lorekit mcp --store sqlite` + `migrate --to sqlite`. Usable + alone: agents on SQLite, no dashboard yet. +2. `lorekit serve` with the Explorer routes, dashboard local mode, the shared + contract suite. +3. Usage events, read counters and the Insights routes. +4. Optional: grooming routes. + +Each phase is its own PR, and each updates the public docs and `llms.txt` +(the repo's docs rule). + +## Open questions + +1. **Where `serve` lives.** The CLI is zero-dependency and cannot import + `@lorekit/schemas` (zod). Either a new package (`@lorekit/local`) that the + CLI delegates to when installed, or generated plain-JS validators committed + into the CLI the way `surfaces.generated.mjs` is. +2. **How self-hosters get the dashboard.** A prebuilt Next.js standalone bundle + inside an npm package is heavy; a container image is lighter to run but adds + a release artifact; the dev server is fine only for contributors. +3. **Which Insights instruments are in scope.** `utility`, `pivot` and + `clusters` are the most SQL-heavy; they could follow the core three. +4. **Teams.** SQLite is one writer on one machine. For a team self-hosting + without Supabase Cloud, Postgres via self-hosted Supabase (Docker) keeps + today's schema and RPCs untouched — worth evaluating separately rather than + stretching SQLite to multi-user. +5. **Node floor.** Opt-in on Node 22.13+, or raise `engines` when the SQLite + store becomes the offline default. From 06f42ffc48226f8af12ef559a3f97ccb5e5a6f6c Mon Sep 17 00:00:00 2001 From: "mads.thines" Date: Wed, 14 Oct 2026 22:35:24 +0000 Subject: [PATCH 04/16] =?UTF-8?q?fix(byod):=20address=20review=20=E2=80=94?= =?UTF-8?q?=20anon-key=20reads,=20old=20bootstrap=20databases,=20npm=20ins?= =?UTF-8?q?talls?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - byod.md: the service key is needed for reads (the read policies need auth.uid(), which an API key never carries); say what JWT callers see; add the path for databases set up from the old hand-written file; enable pg_cron first. - bootstrap: probe first and print the download URL when supabase/ is not in the install, instead of exiting 1 for every npm user. - generator: Supabase CLI filename grammar, CRLF/BOM normalisation, fail on duplicate versions, terminate each migration's last statement. - cli.md: document bootstrap and doctor's byod storage check. - local-sqlite-mode.md: onboarding and plan usage also read memories directly; they need REST routes before local mode. Co-authored-by: dash0-dev[bot] <257284812+dash0-dev[bot]@users.noreply.github.com> --- .gitattributes | 5 + docs/byod.md | 55 ++++-- docs/cli.md | 16 +- docs/tasks/local-sqlite-mode.md | 8 +- packages/cli/src/commands/bootstrap.mjs | 54 +++--- packages/cli/src/shared/byod.mjs | 6 +- supabase/byod/bootstrap.sql | 226 ++++++++++++------------ supabase/byod/generate.mjs | 28 ++- supabase/byod/generate.test.mjs | 17 ++ 9 files changed, 248 insertions(+), 167 deletions(-) diff --git a/.gitattributes b/.gitattributes index 167b849dd..931cc7d1d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,8 @@ # Generated from supabase/migrations/ by `node supabase/byod/generate.mjs`. # Collapsed in GitHub diffs; review the migrations, not this projection. supabase/byod/bootstrap.sql linguist-generated=true + +# The generator's drift check compares bytes. Keep both sides LF on every +# checkout, so core.autocrlf on Windows cannot turn a clean tree red. +supabase/migrations/*.sql text eol=lf +supabase/byod/bootstrap.sql text eol=lf diff --git a/docs/byod.md b/docs/byod.md index 12294fbd6..ea944bb23 100644 --- a/docs/byod.md +++ b/docs/byod.md @@ -5,7 +5,7 @@ project that runs LoreKit's `mcp` Edge Function. Use it when memory rows must live in a specific database — data residency, an existing Supabase subscription, or full control over the data. -It covers the `memory.*` MCP tools. Read [How BYOD is wired](#how-byod-is-wired) +It covers the MCP server's tools. Read [How BYOD is wired](#how-byod-is-wired) before setting it up: what reaches the storage project, and what does not, is the part most likely to surprise you. @@ -32,7 +32,9 @@ the part most likely to surprise you. Create a project at [supabase.com](https://supabase.com). Note its URL and API keys (**Project Settings → API**) and its database connection string -(**Connect**). +(**Connect**). Enable the `pg_cron` extension (**Database → Extensions**) +before applying the schema: the purge jobs are scheduled only when it is +already enabled. ### 2. Apply the LoreKit schema @@ -55,9 +57,9 @@ psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -f supabase/byod/bootstrap.sql ``` or paste [`supabase/byod/bootstrap.sql`](https://github.com/mthines/lorekit/blob/main/supabase/byod/bootstrap.sql) -into the Supabase dashboard → **SQL Editor** and run it. `$DATABASE_URL` is the -connection string from **Connect**; its password is your **database password**, -not an API key. +into the Supabase dashboard → **SQL Editor** and run it (the file is about 1 MB; +if the editor refuses it, use `psql`). `$DATABASE_URL` is the connection string +from **Connect**; its password is your **database password**, not an API key. `bootstrap.sql` is generated from `supabase/migrations/` (CI fails when the two disagree). It applies every migration in one transaction, so a failure leaves @@ -76,13 +78,20 @@ Set the storage secrets on the project that runs your `mcp` function: ```bash supabase secrets set --project-ref \ LOREKIT_STORAGE_URL=https://.supabase.co \ - LOREKIT_STORAGE_ANON_KEY= + LOREKIT_STORAGE_ANON_KEY= \ + LOREKIT_STORAGE_SERVICE_KEY= ``` -`LOREKIT_STORAGE_SERVICE_KEY` is optional. When it is set, the function uses it -instead of the anon key, and it **bypasses row-level security** in the storage -project: data isolation then rests on the `user_id` filters the function -applies, not on RLS policies. +**Set the service key.** The function reads with whichever key it is given, +and the storage project's read policies require `auth.uid()`, which an API key +never carries. With only the anon key, writes succeed (they go through a +`SECURITY DEFINER` function) but every read returns no rows. + +The service key **bypasses row-level security**. Reads made with an `lk_*` +token are still filtered to that token's user by the function. Reads from a +caller authenticated with a Supabase session (JWT) are not filtered, because +the function leaves RLS to do that, and so they see every row in the storage +project. ### 4. Check it @@ -93,7 +102,8 @@ lorekit doctor ``` The `byod storage` line passes when the project is reachable and holds the -schema, and says so when the schema has not been applied yet. +schema. It fails when the project is unreachable, rejects the key, or does not +have the schema yet, and says which. ## Upgrading @@ -110,6 +120,18 @@ works the same whether you set the database up with `db push` or with `bootstrap.sql`. Upgrade the storage project before deploying the matching `mcp` function: the function calls the RPCs those migrations define. +### Databases set up from the old `bootstrap.sql` + +`bootstrap.sql` used to be a hand-written subset of the schema. A database set +up from that version has no migration history — `select count(*) from +supabase_migrations.schema_migrations` errors or returns 0 — so `db push` would +try to replay every migration from the first and fail, and its schema is +missing most of the functions the current `mcp` function calls. Move to a fresh storage +project instead: apply the current schema to it as above, copy the `memories` +rows across (for example `pg_dump --data-only --table=public.memories` from the +old database, loaded with `psql` into the new one), then point +`LOREKIT_STORAGE_URL` at the new project. + ## Billing Memories stored in your own database are **not counted** against LoreKit's @@ -135,8 +157,9 @@ and does not record hosted usage events. storage project. Writes made with the service-role key carry no user id and are not affected. - **No per-user RLS in the storage project.** The function talks to it with the - service or anon key, never with the caller's JWT. -- **Tested surface.** The BYOD smoke test covers write, read, list, search and - delete across the four scope types. The schema now also contains the org, - audit, retention and webhook tables and RPCs, but BYOD has not been exercised - for those features — treat them as unsupported. + service or anon key, never with the caller's JWT — see step 3 for what that + means for reads. +- **Tested surface.** The BYOD smoke test exercises write, read, list, search + and delete, and runs only where its secrets are configured. The schema now + also contains the org, audit, retention and webhook tables and RPCs, but BYOD + has not been exercised for those features — treat them as unsupported. diff --git a/docs/cli.md b/docs/cli.md index 3c740ddc5..bcc023d4f 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -55,7 +55,10 @@ frontmatter, checked in BOTH the project and global install locations): a `curre `pass` naming its version, an `outdated` or `unknown` (legacy, predates version stamping) install is a `warn` — never a `fail`, the skill still works — naming the installed and shipped versions and pointing at `lorekit update --`, the same scope-aware wording the hooks-upgrade line above it -uses. +uses. When `LOREKIT_STORAGE_URL` is set, a `byod storage` check probes that project with +`LOREKIT_STORAGE_ANON_KEY` (`probeByodStorage` in `shared/byod.mjs` — one PostgREST +`GET /rest/v1/memories?limit=1`, plain `fetch`): it passes when the `memories` table answers and +FAILS when the project is unreachable, rejects the key, or has no LoreKit schema yet. ### `update` @@ -297,6 +300,17 @@ every param `encodeURIComponent(JSON.stringify(value))` and omits any param equa previously raw-encoded `?scope=`, which the dashboard's `JSON.parse` rejected into an all-scopes fallback; write-confirmations now deep-link to the exact lesson +### `bootstrap` + +BYOD only ([byod.md](./byod.md)). Probes the project named by `LOREKIT_STORAGE_URL` with +`LOREKIT_STORAGE_ANON_KEY` or `LOREKIT_STORAGE_SERVICE_KEY` (the same `probeByodStorage` as +`doctor`), reports whether the LoreKit schema is already there, and prints the three routes that +apply it: `psql -v ON_ERROR_STOP=1 -f supabase/byod/bootstrap.sql`, the SQL Editor, or +`supabase link` + `supabase db push`. It runs no DDL — PostgREST, the only thing a Supabase API key +reaches, cannot. The published package does not ship `supabase/`, so outside a LoreKit checkout it +prints the file's download URL instead of a local path. It never prints a key: the `psql` +connection string's password is the database password, not the service-role key. + ### `migrate` Relocate a LoreKit-format local store, or push one to the hosted store — `--from ` is required diff --git a/docs/tasks/local-sqlite-mode.md b/docs/tasks/local-sqlite-mode.md index 36cd425e0..185714ac5 100644 --- a/docs/tasks/local-sqlite-mode.md +++ b/docs/tasks/local-sqlite-mode.md @@ -29,13 +29,13 @@ The server logic lives in Postgres, not in TypeScript: | Fact (measured on `main` at 00113) | Consequence for SQLite | |---|---| -| 108 migrations define 87 SQL functions; the edge functions call **48** distinct RPCs | Each RPC a route needs must be re-implemented | +| 108 migrations define 87 SQL functions; the edge functions make string-literal `.rpc('…')` calls to **48** distinct ones | Each RPC a route needs must be re-implemented | | The Explorer's routes rest on ~1,200 lines of PL/pgSQL (`lorekit_memory_list` 206, `_facets` 218, `_activity` 136, `memory_write` 245, `memory_delete` 157, …); Insights on ~690 more (`lorekit_usage_stats`, `_usage_runs`, `_read_activity`, `_memory_pivot` 158, `_utility_*`, …) | That is the size of a second implementation | | `lorekit_memory_list`, `_facets`, `_activity` and `_pivot` were all redefined in the newest migration (00113) | A second implementation drifts at that rate unless something forces parity | | Full-text search is `tsvector` + `websearch_to_tsquery`; graded relevance is `ts_rank_cd` by decision | SQLite FTS5 ranks with `bm25` — ordering will differ, and must be documented as a local-mode difference, not relitigated for hosted | | Labels are `text[]` with containment/overlap; RLS and `auth.uid()` scope every read | JSON arrays + `json_each`; single user, so tenant scoping collapses to one fixed id | | Edge handlers speak PostgREST's query builder and `.rpc()`, not a repository interface | There is no seam to swap a SQLite client in behind the existing handlers | -| The dashboard reads memories and analytics through the REST API (`packages/web/src/lib/api/`, base URL derived from `NEXT_PUBLIC_SUPABASE_URL` in `rest.ts`), but signs in with Supabase Auth (`middleware.ts`, `@supabase/ssr`) and still has **40** direct supabase-js call sites for orgs, invites, API tokens, audit log, scope bindings and GitHub installations | The REST contract is the seam; the supabase-js surfaces stay hosted-only | +| The dashboard reads memories and analytics through the REST API (`packages/web/src/lib/api/`, base URL derived from `NEXT_PUBLIC_SUPABASE_URL` in `rest.ts`), but signs in with Supabase Auth (`middleware.ts`, `@supabase/ssr`) and still has about **40** direct supabase-js call sites — mostly orgs, invites, API tokens, audit log, scope bindings and GitHub installations, but also direct memory reads in onboarding (`lib/onboarding-server.ts`), the plan-usage figure (`lib/plan.ts`, `lorekit_memory_count`), org and scope-binding counts | The REST contract is the seam; the org/token/audit surfaces stay hosted-only, and the onboarding and plan reads need REST equivalents first | The [BYOD drift](../byod.md) is the cautionary example: a hand-written second copy of the schema fell 42 RPCs and a `memory_write` signature behind before @@ -111,6 +111,10 @@ bump read counters on reads — so `/insights` renders from local activity. use a fixed local identity, and hides the navigation entries backed by direct supabase-js calls (orgs, invites, API tokens, audit log, scope bindings, GitHub installations). +- The direct memory reads that are not behind a hidden entry — onboarding's + memory count and the plan-usage figure — move onto REST routes first, in the + hosted dashboard too. That is the existing rule ("a capability the dashboard + needs becomes part of the public contract"), applied ahead of local mode. - Run it as `lorekit serve` plus the dashboard's dev server for development. How to ship it to self-hosters is open question 2. diff --git a/packages/cli/src/commands/bootstrap.mjs b/packages/cli/src/commands/bootstrap.mjs index 9582df03b..155de497f 100644 --- a/packages/cli/src/commands/bootstrap.mjs +++ b/packages/cli/src/commands/bootstrap.mjs @@ -23,6 +23,15 @@ import { probeByodStorage } from '../shared/byod.mjs'; const BOOTSTRAP_SQL_PATH = fileURLToPath( new URL('../../../../supabase/byod/bootstrap.sql', import.meta.url), ); +const BOOTSTRAP_SQL_URL = 'https://raw.githubusercontent.com/mthines/lorekit/main/supabase/byod/bootstrap.sql'; + +// The checkout copy next to this file, else one under the current directory +// (running from a LoreKit checkout), else null (an npm install). +function locateBootstrapSql() { + if (fs.existsSync(BOOTSTRAP_SQL_PATH)) return BOOTSTRAP_SQL_PATH; + const cwdPath = path.resolve(process.cwd(), 'supabase/byod/bootstrap.sql'); + return fs.existsSync(cwdPath) ? cwdPath : null; +} export async function bootstrap(_args) { heading('LoreKit bootstrap'); @@ -32,48 +41,31 @@ export async function bootstrap(_args) { const storageAnonKey = process.env['LOREKIT_STORAGE_ANON_KEY']; // If no BYOD env vars are set, this command is a no-op — it's only for BYOD. - if (!storageUrl && !storageServiceKey) { + if (!storageUrl && !storageServiceKey && !storageAnonKey) { log(''); log( ` ${c.cyan('•')} No BYOD storage configured — ${c.dim('bootstrap is only needed for custom databases.')}`, ); log(''); - log(' Set LOREKIT_STORAGE_URL and LOREKIT_STORAGE_SERVICE_KEY to use your own'); - log(' Supabase project, then re-run this command.'); + log(' Set LOREKIT_STORAGE_URL and LOREKIT_STORAGE_ANON_KEY (or LOREKIT_STORAGE_SERVICE_KEY)'); + log(' for your own Supabase project, then re-run this command.'); log(''); log(' See docs/byod.md for setup instructions.'); return 0; } if (!storageUrl) { - err(`${c.red('Error:')} LOREKIT_STORAGE_SERVICE_KEY is set but LOREKIT_STORAGE_URL is missing.`); - err(' Both variables are required. Set LOREKIT_STORAGE_URL and try again.'); + err(`${c.red('Error:')} a LOREKIT_STORAGE_*_KEY is set but LOREKIT_STORAGE_URL is missing.`); + err(' Set LOREKIT_STORAGE_URL to the storage project\'s URL and try again.'); return 1; } - // Locate the bootstrap SQL file. - let sqlPath = BOOTSTRAP_SQL_PATH; - - // When running from a published npm package, the SQL file is not bundled - // with the CLI. Fall back to looking for it relative to CWD (for dev use). - if (!fs.existsSync(sqlPath)) { - const cwdPath = path.resolve(process.cwd(), 'supabase/byod/bootstrap.sql'); - if (fs.existsSync(cwdPath)) { - sqlPath = cwdPath; - } else { - err(`${c.red('Error:')} bootstrap.sql not found at expected path:`); - err(` ${sqlPath}`); - err(''); - err(' Apply the schema manually with psql:'); - err(' psql "$DATABASE_URL" -f supabase/byod/bootstrap.sql'); - err(''); - err(' Or download it from the LoreKit repository:'); - err(' https://github.com/mthines/lorekit/blob/main/supabase/byod/bootstrap.sql'); - return 1; - } - } - - status('pass', 'bootstrap.sql', sqlPath); + // The published npm package does not ship supabase/, so outside a LoreKit + // checkout there is no local file — that is the normal case, not an error: + // probe anyway and point at the file's download URL. + const sqlPath = locateBootstrapSql(); + if (sqlPath) status('pass', 'bootstrap.sql', sqlPath); + else status('info', 'bootstrap.sql', `not in this install — download: ${BOOTSTRAP_SQL_URL}`); status('info', 'storage url', storageUrl); // Probe the project with whichever key is set (the anon key when both are, @@ -98,15 +90,17 @@ export async function bootstrap(_args) { // PostgREST cannot run DDL, so the schema goes in over a Postgres connection // (psql) or the SQL Editor. Never print a key here: the connection string's // password is the DATABASE password, which is not the service-role API key. + const sqlArg = sqlPath ?? 'bootstrap.sql'; log(''); log(` ${c.bold('To apply the schema, run:')}`); log(''); - log(` ${c.cyan('psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -f')} ${sqlPath}`); + if (!sqlPath) log(` ${c.cyan('curl -fsSLO')} ${BOOTSTRAP_SQL_URL}`); + log(` ${c.cyan('psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -f')} ${sqlArg}`); log(''); log(` ${c.dim('$DATABASE_URL is the connection string from the Supabase dashboard → Connect;')}`); log(` ${c.dim('its password is your database password, not an API key.')}`); log(''); - log(` Or paste the contents of ${c.dim(sqlPath)}`); + log(` Or paste the contents of ${c.dim(sqlArg)}`); log(' into the Supabase dashboard → SQL Editor and run it.'); log(''); log(` Or, from a LoreKit checkout: ${c.cyan('supabase link --project-ref')} ${parseRef(storageUrl)} && ${c.cyan('supabase db push')}`); diff --git a/packages/cli/src/shared/byod.mjs b/packages/cli/src/shared/byod.mjs index 51c014889..97df36ca1 100644 --- a/packages/cli/src/shared/byod.mjs +++ b/packages/cli/src/shared/byod.mjs @@ -38,7 +38,11 @@ export async function probeByodStorage({ url, key, fetchImpl = globalThis.fetch, signal: AbortSignal.timeout(timeoutMs), }); } catch (e) { - return { state: 'error', detail: `could not connect: ${e && e.message ? e.message : String(e)}` }; + // Node's fetch reports every network failure as "fetch failed"; the + // reason (ENOTFOUND, ECONNREFUSED, a TLS error) is on `cause`. + const reason = e && e.cause ? e.cause.code || e.cause.message : null; + const message = e && e.message ? e.message : String(e); + return { state: 'error', detail: `could not connect: ${message}${reason ? ` (${reason})` : ''}` }; } if (res.ok) return { state: 'ready' }; diff --git a/supabase/byod/bootstrap.sql b/supabase/byod/bootstrap.sql index 19d1a770b..41d556c69 100644 --- a/supabase/byod/bootstrap.sql +++ b/supabase/byod/bootstrap.sql @@ -28,8 +28,12 @@ -- -- Billing: memories stored in your own database are NOT counted against -- LoreKit's hosted memory-count billing — LoreKit has no visibility into it. --- The memory cap and rate limit are read from `lorekit_default_limit` / --- `user_limits` in YOUR database; adjust them there. +-- The memory cap is read from `lorekit_default_limit` / `user_limits` in +-- YOUR database; adjust it there. (In BYOD mode the mcp function skips the +-- rate limiter.) +-- +-- pg_cron: the purge jobs are scheduled only when the pg_cron extension is +-- already enabled (Database → Extensions). Enable it before applying. -- -- See docs/byod.md for the full setup. @@ -43,7 +47,7 @@ begin if to_regclass('public.memories') is not null then raise exception using message = 'LoreKit schema already present: bootstrap.sql only initialises an empty database.', - hint = 'Upgrade an existing BYOD database with `supabase db push` (see docs/byod.md).'; + hint = 'Upgrade a database set up from this file with `supabase db push`. One set up from the older hand-written bootstrap.sql has no migration history: see docs/byod.md.'; end if; end $guard$; @@ -141,7 +145,7 @@ $$; create or replace trigger memories_updated_at before update on memories for each row execute function set_updated_at(); - +; insert into supabase_migrations.schema_migrations (version, name) values ('00001', 'memories') on conflict (version) do nothing; @@ -193,7 +197,7 @@ create policy "rls_api_tokens_delete" -- Service role can look up tokens for auth validation (bypasses RLS) -- No update policy needed — last_used_at is updated via service role - +; insert into supabase_migrations.schema_migrations (version, name) values ('00002', 'api_tokens') on conflict (version) do nothing; @@ -345,7 +349,7 @@ begin return v_count; end; $$; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00003', 'archive') on conflict (version) do nothing; @@ -568,7 +572,7 @@ begin end if; end; $$; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00004', 'limits') on conflict (version) do nothing; @@ -619,7 +623,7 @@ create policy "rls_webhook_secrets_update" -- Service role reads the active secret for HMAC verification on each webhook -- call (bypasses RLS — no additional policy needed for service role). - +; insert into supabase_migrations.schema_migrations (version, name) values ('00005', 'webhook_secrets') on conflict (version) do nothing; @@ -650,7 +654,7 @@ grant select, insert, update, delete on all tables in schema public alter default privileges in schema public grant select, insert, update, delete on tables to anon, authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00006', 'grants') on conflict (version) do nothing; @@ -723,7 +727,7 @@ $$; grant execute on function memory_write(uuid, text, text, text, text[], text, text) to anon, authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00007', 'memory_write_rpc') on conflict (version) do nothing; @@ -767,7 +771,7 @@ create unique index webhook_secrets_user_repo_active_uidx create index webhook_secrets_repo_active_idx on webhook_secrets (repo) where active = true and repo is not null; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00008', 'webhook_secrets_repo') on conflict (version) do nothing; @@ -848,7 +852,7 @@ $$; grant execute on function memory_write(uuid, text, text, text, text[], text, text, timestamptz) to anon, authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00009', 'memory_write_created_at') on conflict (version) do nothing; @@ -980,7 +984,7 @@ $$; create or replace trigger user_limits_audit after insert or update or delete on user_limits for each row execute function audit_user_limits(); - +; insert into supabase_migrations.schema_migrations (version, name) values ('00010', 'audit_log') on conflict (version) do nothing; @@ -1065,7 +1069,7 @@ $$; grant execute on function memory_write(uuid, text, text, text, text[], text, text, timestamptz) to anon, authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00011', 'memory_write_inserted') on conflict (version) do nothing; @@ -1099,7 +1103,7 @@ create index if not exists audit_log_target_trgm_idx create index if not exists audit_log_user_created_id_idx on audit_log (user_id, created_at desc, id); - +; insert into supabase_migrations.schema_migrations (version, name) values ('00012', 'audit_log_search') on conflict (version) do nothing; @@ -1131,7 +1135,7 @@ create index if not exists memories_key_trgm_idx create index if not exists memories_value_trgm_idx on memories using gin (value gin_trgm_ops); - +; insert into supabase_migrations.schema_migrations (version, name) values ('00013', 'memory_keyset_index') on conflict (version) do nothing; @@ -1218,7 +1222,7 @@ $$; -- membership via PostgREST RPC. RLS policies that call it run as the -- querying role, and only authenticated/service_role ever read `memories`. grant execute on function lorekit_member_org_ids(uuid) to authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00014', 'orgs') on conflict (version) do nothing; @@ -1283,7 +1287,7 @@ create policy "rls_read_archived" or org_id in (select lorekit_member_org_ids(auth.uid())) ) ); - +; insert into supabase_migrations.schema_migrations (version, name) values ('00015', 'memories_org_fk') on conflict (version) do nothing; @@ -1387,7 +1391,7 @@ $$; grant execute on function memory_write(uuid, text, text, text, text[], text, text, timestamptz) to anon, authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00016', 'memories_org_unique_arbiter') on conflict (version) do nothing; @@ -1477,7 +1481,7 @@ end; $$; grant execute on function lorekit_org_can(uuid, uuid, text) to anon, authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00017', 'org_roles_and_author') on conflict (version) do nothing; @@ -1606,7 +1610,7 @@ begin return new; end; $$; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00018', 'org_limits') on conflict (version) do nothing; @@ -1739,7 +1743,7 @@ $$; grant execute on function memory_write(uuid, text, text, text, text[], text, text, timestamptz, text) to anon, authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00019', 'memory_write_org') on conflict (version) do nothing; @@ -1840,7 +1844,7 @@ end; $$; grant execute on function memory_delete(uuid, text, text, text, boolean) to anon, authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00020', 'memory_delete_org') on conflict (version) do nothing; @@ -1923,7 +1927,7 @@ create policy "rls_org_invites_select_invitee" on org_invites for select drop policy "rls_org_members_select" on org_members; create policy "rls_org_members_select" on org_members for select using (lorekit_org_role(auth.uid(), org_id) is not null); - +; insert into supabase_migrations.schema_migrations (version, name) values ('00021', 'org_invites') on conflict (version) do nothing; @@ -2324,7 +2328,7 @@ end; $$; grant execute on function lorekit_org_leave(uuid) to authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00022', 'org_management_rpcs') on conflict (version) do nothing; @@ -2364,7 +2368,7 @@ alter table audit_log add constraint audit_log_action_check check (action in ( 'member.role_change', 'member.leave' )); - +; insert into supabase_migrations.schema_migrations (version, name) values ('00023', 'audit_log_org_actions') on conflict (version) do nothing; @@ -2441,7 +2445,7 @@ $$; -- for other users, so it is authenticated-only, defense in depth beyond the -- membership gate above. grant execute on function lorekit_org_members_list(uuid) to authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00024', 'org_member_identities') on conflict (version) do nothing; @@ -2560,7 +2564,7 @@ end; $$; grant execute on function lorekit_org_purge(uuid) to authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00025', 'safe_org_deletion') on conflict (version) do nothing; @@ -2826,7 +2830,7 @@ $$; grant execute on function memory_write(uuid, text, text, text, text[], text, text, timestamptz, text) to anon, authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00026', 'scope_org_bindings') on conflict (version) do nothing; @@ -2869,7 +2873,7 @@ alter table audit_log add constraint audit_log_action_check check (action in ( 'scope.bind', 'scope.unbind' )); - +; insert into supabase_migrations.schema_migrations (version, name) values ('00027', 'audit_log_scope_actions') on conflict (version) do nothing; @@ -2940,7 +2944,7 @@ $$; -- the same defense-in-depth posture as `lorekit_org_members_list` (00024) -- and every other PII-bearing definer function in this schema. grant execute on function lorekit_invite_org_details(uuid) to authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00028', 'invite_org_details') on conflict (version) do nothing; @@ -3164,7 +3168,7 @@ comment on column memories.expires_at is 'Optional timestamp after which the row is treated as invisible by active-read paths. NULL means the row never expires. Physically deleted by purge_expired_memories() or the MCP memory.purge tool.'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00030', 'memory_ttl') on conflict (version) do nothing; @@ -3346,7 +3350,7 @@ $$; grant execute on function memory_write(uuid, text, text, text, text[], text, text, timestamptz, text, integer, boolean) to anon, authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00031', 'memory_ttl_clear') on conflict (version) do nothing; @@ -3508,7 +3512,7 @@ grant select on table plans to anon, authenticated, service_role; grant select on table user_plans to anon, authenticated, service_role; grant insert, update on table user_plans to service_role; grant insert, update on table user_limits to service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00032', 'plans') on conflict (version) do nothing; @@ -3618,7 +3622,7 @@ begin end if; end; $$; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00033', 'performance_indexes') on conflict (version) do nothing; @@ -3766,7 +3770,7 @@ $$; grant execute on function lorekit_record_usage_event(uuid, uuid, text, text, text, text, text, integer, integer) to anon, authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00034', 'usage_events') on conflict (version) do nothing; @@ -3831,7 +3835,7 @@ $$; -- user JWT (supabase-js createServerClient uses the user's JWT session). grant execute on function lorekit_memory_count(uuid) to anon, authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00035', 'memory_count') on conflict (version) do nothing; @@ -3927,7 +3931,7 @@ $$; grant execute on function lorekit_memory_count(uuid) to anon, authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00036', 'fix_memory_count') on conflict (version) do nothing; @@ -4178,7 +4182,7 @@ $$; grant execute on function lorekit_find_user_by_github_id(text) to service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00037', 'github_installations') on conflict (version) do nothing; @@ -4370,7 +4374,7 @@ $$; grant execute on function memory_write(uuid, text, text, text, text[], text, text, timestamptz, text, integer, boolean) to anon, authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00038', 'memory_ttl_seconds') on conflict (version) do nothing; @@ -4460,7 +4464,7 @@ $$; -- RPC call. `authenticated` is safe because the edge functions resolve -- p_user_id from the verified credential, never from request input. grant execute on function lorekit_memory_scopes(uuid) to authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00039', 'memory_scopes') on conflict (version) do nothing; @@ -4974,7 +4978,7 @@ $$; -- membership gate above. revoke execute on function lorekit_org_members_list(uuid, uuid) from public; grant execute on function lorekit_org_members_list(uuid, uuid) to authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00041', 'org_actor_override') on conflict (version) do nothing; @@ -5035,7 +5039,7 @@ alter table audit_log add constraint audit_log_action_check check (action in ( 'scope.unbind', 'github_app.installation_linked' )); - +; insert into supabase_migrations.schema_migrations (version, name) values ('00042', 'audit_log_github_app_action') on conflict (version) do nothing; @@ -5126,7 +5130,7 @@ $$; -- input; `service_role` is the CI escape hatch. grant execute on function lorekit_usage_stats(uuid, timestamptz, timestamptz) to authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00043', 'usage_stats') on conflict (version) do nothing; @@ -5275,7 +5279,7 @@ $$; revoke execute on function lorekit_usage_stats(uuid, timestamptz, timestamptz, text) from public; grant execute on function lorekit_usage_stats(uuid, timestamptz, timestamptz, text) to authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00044', 'usage_event_dimensions') on conflict (version) do nothing; @@ -5342,7 +5346,7 @@ begin return v_count; end; $$; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00045', 'memory_expiry_events') on conflict (version) do nothing; @@ -5710,7 +5714,7 @@ comment on function memory_delete(uuid, text, text, text, boolean) is wins — scoping BOTH the personal user_id filter and the org capability check to the real caller. SECURITY DEFINER bypasses RLS; the actor is the gate. Fails closed on a NULL actor.'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00046', 'memory_rpc_actor_guard') on conflict (version) do nothing; @@ -5878,7 +5882,7 @@ revoke execute on function lorekit_installation_remove_repos(bigint, text[]) revoke execute on function lorekit_installation_remove(bigint) from public, anon, authenticated; revoke execute on function lorekit_purge_old_usage_events(interval) from public, anon, authenticated; revoke execute on function lorekit_purge_all_expired_memories() from public, anon, authenticated; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00047', 'readfn_grant_hardening') on conflict (version) do nothing; @@ -6160,7 +6164,7 @@ $$; grant execute on function memory_write(uuid, text, text, text, text[], text, text, timestamptz, text, integer, boolean, text, text, text, integer) to anon, authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00048', 'memory_origin') on conflict (version) do nothing; @@ -6246,7 +6250,7 @@ comment on function lorekit_memory_scopes(uuid) is auth.uid() wins — so an authenticated caller can never enumerate another user''s scope names. p_user_id IS NULL under service-role is the CI escape hatch. last_activity is max(created_at) over the same counted rows.'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00049', 'memory_scopes_last_activity') on conflict (version) do nothing; @@ -6341,7 +6345,7 @@ comment on function lorekit_memory_tags(uuid, boolean) is selected by p_archived, visible to the EFFECTIVE caller. Same service-role-gated actor rule and tenant predicate as lorekit_memory_scopes. Ordered count desc, tag asc.'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00050', 'memory_tags') on conflict (version) do nothing; @@ -6434,7 +6438,7 @@ comment on function lorekit_memory_activity(uuid, text, timestamptz, timestamptz [p_since, p_until) window, visible to the EFFECTIVE caller. Same service-role-gated actor rule and tenant predicate as lorekit_memory_scopes. p_bucket is validated against (hour, day).'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00051', 'memory_activity') on conflict (version) do nothing; @@ -6551,7 +6555,7 @@ comment on function lorekit_memory_facets(uuid, boolean) is partition selected by p_archived, visible to the EFFECTIVE caller. Same service-role-gated actor rule, tenant predicate and ordering as lorekit_memory_tags.'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00052', 'memory_facets') on conflict (version) do nothing; @@ -6654,7 +6658,7 @@ comment on function lorekit_read_activity(uuid, text, timestamptz, timestamptz) service-role + NULL escape hatch as lorekit_usage_stats — usage is a per-user ledger and is never org-shared. p_bucket is validated against (hour, day). Buckets with no records read are omitted.'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00053', 'read_activity') on conflict (version) do nothing; @@ -6849,7 +6853,7 @@ comment on function lorekit_read_activity(uuid, text, timestamptz, timestamptz) NULL escape hatch as lorekit_usage_stats — usage is a per-user ledger and is never org-shared. p_bucket is validated against (hour, day). Buckets with no records read are omitted.'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00054', 'usage_event_client') on conflict (version) do nothing; @@ -6950,7 +6954,7 @@ comment on function lorekit_blog_like(text, int) is the new global total. SECURITY DEFINER + granted to anon so an unauthenticated blog visitor can like a post. Validates the slug shape (lowercase kebab, <=128) so no junk rows can be created.'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00055', 'blog_post_likes') on conflict (version) do nothing; @@ -7263,7 +7267,7 @@ $$; grant execute on function lorekit_record_usage_event(uuid, uuid, text, text, text, text, text, integer, integer, integer, text, text, text, text) to anon, authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00056', 'memory_kind_host') on conflict (version) do nothing; @@ -7479,7 +7483,7 @@ comment on function lorekit_memory_facets( what adding that value would yield. With no filters supplied the result is 00052''s global catalog. Same service-role-gated actor rule and ordering as lorekit_memory_tags.'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00057', 'memory_facets_drilldown') on conflict (version) do nothing; @@ -7717,7 +7721,7 @@ comment on function lorekit_read_activity(uuid, text, timestamptz, timestamptz, NULL escape hatch as lorekit_usage_stats — usage is a per-user ledger and is never org-shared. p_bucket is validated against (hour, day). Buckets with no records read are omitted.'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00058', 'usage_event_scope') on conflict (version) do nothing; @@ -7970,7 +7974,7 @@ $$; grant execute on function memory_write(uuid, text, text, text, text[], text, text, timestamptz, text, integer, boolean, text, text, text, integer, text, text) to anon, authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00059', 'memory_seen_count') on conflict (version) do nothing; @@ -8079,7 +8083,7 @@ create index if not exists memories_embedding_hnsw_idx create index if not exists memories_embedding_pending_idx on memories (created_at desc) where embedding is null; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00060', 'memory_embeddings') on conflict (version) do nothing; @@ -8164,7 +8168,7 @@ on conflict (version) do nothing; create index if not exists memories_scope_updated_at_id_idx on memories (scope, updated_at desc, id desc) where archived_at is null; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00061', 'memories_keyset_index') on conflict (version) do nothing; @@ -8450,7 +8454,7 @@ comment on function lorekit_memory_set_embedding(uuid, uuid, text, text) is 'Returns true when a row was written, false when none matched — the write path records that ' 'miss on its span rather than dropping the embedding silently, which is what rls_update did ' 'for org-owned memories before 00062.'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00062', 'memory_embedding_write') on conflict (version) do nothing; @@ -8613,7 +8617,7 @@ comment on function lorekit_memory_activity( optional scope + dimension filters (00063) — the SAME predicate as GET /memories and lorekit_memory_facets, so the stat header agrees with the list. With no filters supplied the result is byte-for-byte 00051''s.'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00063', 'activity_filters') on conflict (version) do nothing; @@ -8999,7 +9003,7 @@ comment on function lorekit_memory_activity( `personal` / org slug) — the SAME predicate as GET /memories and lorekit_memory_facets, so the stat header agrees with the list. With no filters supplied the result is byte-for-byte 00051''s.'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00064', 'owner_filter') on conflict (version) do nothing; @@ -9081,7 +9085,7 @@ comment on function lorekit_memory_scopes(uuid) is auth.uid() wins — so an authenticated caller can never enumerate another user''s scope names. p_user_id IS NULL under service-role is the CI escape hatch. last_activity is max(created_at) over the same counted rows.'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00065', 'memory_scopes_order_by_count') on conflict (version) do nothing; @@ -9554,7 +9558,7 @@ comment on function lorekit_memory_activity( lorekit_memory_facets composes and the SAME behaviour GET /memories has, so the stat header agrees with the list. With no filters supplied the result is byte-for-byte 00051''s.'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00066', 'dimension_filter_helpers') on conflict (version) do nothing; @@ -9875,7 +9879,7 @@ comment on function lorekit_memory_list( already computed, so neither escaping nor the now()-relative boundary is implemented twice. Callers pass limit + 1 and derive hasMore from the overflow row.'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00067', 'memory_list_rpc') on conflict (version) do nothing; @@ -10307,7 +10311,7 @@ revoke execute on function lorekit_api_token_set_scoping(uuid, text[], text, uui from public, anon; grant execute on function lorekit_api_token_set_scoping(uuid, text[], text, uuid[]) to authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00068', 'api_token_scoping') on conflict (version) do nothing; @@ -11687,7 +11691,7 @@ comment on function lorekit_memory_list( narrowed by the CALLING KEY''s scope allowlist and tenancy (00069). Otherwise identical to the function 00067 introduced; the key parameters default to unrestricted, so a non-key caller is unaffected.'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00069', 'api_token_scoping_enforcement') on conflict (version) do nothing; @@ -11751,7 +11755,7 @@ alter table audit_log add constraint audit_log_action_check check (action in ( 'scope.unbind', 'github_app.installation_linked' )); - +; insert into supabase_migrations.schema_migrations (version, name) values ('00070', 'audit_log_api_key_scope_change') on conflict (version) do nothing; @@ -11986,7 +11990,7 @@ comment on function memory_delete(uuid, text, text, text, boolean, text[], text, (service-role connection, no p_user_id — no user and no key) is unpinned: it has no actor to be pinned to. Org removals remain gated on the archive / hard_delete capability.'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00071', 'api_token_scope_authorized_removal') on conflict (version) do nothing; @@ -12107,7 +12111,7 @@ comment on function restore_memory(uuid, text, text, text[], text, uuid[]) is their own (and member-org) rows — the 00046/00071 actor guard, applied to restore. The service tier (service-role connection, no p_user_id — no user and no key) is unpinned: it has no actor to be pinned to.'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00072', 'api_token_scope_authorized_restore') on conflict (version) do nothing; @@ -12386,7 +12390,7 @@ grant execute on function memory_write( uuid, text, text, text, text[], text, text, timestamptz, text, integer, boolean, text, text, text, integer, text, text, text[], text, uuid[] ) to anon, authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00073', 'memory_write_security_definer') on conflict (version) do nothing; @@ -12651,7 +12655,7 @@ begin end if; end; $$; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00074', 'db_query_stats') on conflict (version) do nothing; @@ -12930,7 +12934,7 @@ grant execute on function memory_write( uuid, text, text, text, text[], text, text, timestamptz, text, integer, boolean, text, text, text, integer, text, text, text[], text, uuid[] ) to anon, authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00075', 'memory_write_returns_full_row') on conflict (version) do nothing; @@ -12987,7 +12991,7 @@ comment on index memories_tags_idx is 'GIN index supporting tags @> / <@ / && containment queries (e.g. ' 'onboarding-server.ts''s source::pr-webhook check), avoiding a sequential ' 'scan on every /overview render.'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00076', 'memories_tags_gin_index') on conflict (version) do nothing; @@ -13051,7 +13055,7 @@ revoke execute on function purge_archived_memories(uuid, integer) from public, a revoke execute on function purge_expired_memories(uuid) from public, anon; revoke execute on function lorekit_usage_stats(uuid, timestamptz, timestamptz, text) from public, anon; revoke execute on function lorekit_memory_set_embedding(uuid, uuid, text, text) from public, anon; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00077', 'org_actor_functions_anon_revoke') on conflict (version) do nothing; @@ -13185,7 +13189,7 @@ $$; grant execute on function lorekit_record_usage_event(uuid, uuid, text, text, text, text, text, integer, integer, integer, text, text, text, text, text, integer) to anon, authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00078', 'usage_event_scope_count') on conflict (version) do nothing; @@ -13316,7 +13320,7 @@ comment on function lorekit_usage_stats(uuid, timestamptz, timestamptz, text) is top 20 by event count; anything else collapses to ''other'' (NULL host stays NULL). Summing event_count over every row reproduces the pre-00079 total — the extra dimensions only refine existing buckets, never duplicate rows.'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00079', 'usage_stats_client_kind_host') on conflict (version) do nothing; @@ -13437,7 +13441,7 @@ comment on function lorekit_read_activity(uuid, text, timestamptz, timestamptz, allowlist (00068/00069). One row per (bucket, scope, read_kind); retrieved + opened sum to the same total the pre-00080 function gave for the same window. Visibility is SELF-ONLY with the service-role + NULL escape hatch.'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00080', 'read_activity_read_kind') on conflict (version) do nothing; @@ -13529,7 +13533,7 @@ comment on function lorekit_usage_memory_count_peak(uuid, timestamptz, timestamp events at all. Self-only: v_actor is auth.uid() for a JWT caller and the verified service-role''s own p_user_id otherwise -- an authenticated caller can never pass another user''s id through p_user_id.'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00081', 'usage_memory_count_peak') on conflict (version) do nothing; @@ -13636,7 +13640,7 @@ $$; grant execute on function lorekit_record_usage_event(uuid, uuid, text, text, text, text, text, integer, integer, integer, text, text, text, text, text, integer, text) to anon, authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00082', 'usage_event_session_kind') on conflict (version) do nothing; @@ -13784,7 +13788,7 @@ comment on function lorekit_usage_runs(uuid, timestamptz, timestamptz, timestamp caller supplies the previous page''s last row as the cursor. Self-only visibility, same escape hatch as lorekit_usage_stats. No org axis -- usage_events is a per-user ledger.'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00083', 'usage_runs') on conflict (version) do nothing; @@ -13970,7 +13974,7 @@ comment on function lorekit_record_memory_reads(uuid[], text) is definition, not usage-stats.ts''s broader READ_TOOL_NAMES. Fail-safe: never throws, so a counter-write failure cannot fail the read it measures. No-op on a null/empty array.'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00084', 'memory_read_counters') on conflict (version) do nothing; @@ -14089,7 +14093,7 @@ comment on function lorekit_memory_read_ranking(uuid, text, text, integer, text[ counting", never "never read" -- the calling handler must stamp and every consumer must render the counting_since qualifier. p_limit is clamped to [1, 100].'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00085', 'memory_read_ranking') on conflict (version) do nothing; @@ -14172,7 +14176,7 @@ comment on index memories_tags_idx is 'scan on every /overview render. Rebuilt + ANALYZEd by 00086 after the ' 'same seq-scan signature reappeared four days after 00076 first added it ' '(Dash0 issues 8667967338373977430 / 9019610890880570001).'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00086', 'reindex_memories_tags_idx') on conflict (version) do nothing; @@ -14524,7 +14528,7 @@ begin end if; end; $$; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00088', 'retention_policies') on conflict (version) do nothing; @@ -14587,7 +14591,7 @@ alter table audit_log add constraint audit_log_action_check check (action in ( 'policy.delete', 'memory.protect' )); - +; insert into supabase_migrations.schema_migrations (version, name) values ('00089', 'audit_log_groom_actions') on conflict (version) do nothing; @@ -14854,7 +14858,7 @@ comment on function lorekit_memory_pivot( lorekit_memory_facets, with BOTH axes self-excluded from the filters so a drilled-in matrix still shows every cell you could move to. Ordered count desc, row asc, col asc and bounded by p_limit.'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00090', 'memory_pivot') on conflict (version) do nothing; @@ -15086,7 +15090,7 @@ grant execute on function lorekit_memory_list( text, text[], text, text[], text, text[], text, text[], text, text, timestamptz, uuid, integer, text[], text, uuid[], integer, integer, integer ) to authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00092', 'list_retention_conditions') on conflict (version) do nothing; @@ -15425,7 +15429,7 @@ as $$ $$; grant execute on function lorekit_policy_update(uuid, uuid, jsonb) to authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00093', 'policy_dimension_filters') on conflict (version) do nothing; @@ -15646,7 +15650,7 @@ grant execute on function lorekit_memory_list( text, text[], text, text[], text, text[], text, text[], text, text, timestamptz, uuid, integer, text[], text, uuid[], integer, integer, integer ) to authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00094', 'memory_list_total_count') on conflict (version) do nothing; @@ -15849,7 +15853,7 @@ begin end if; end; $$; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00095', 'groom_sweep_telemetry') on conflict (version) do nothing; @@ -15901,7 +15905,7 @@ on conflict (version) do nothing; create index if not exists memories_user_updated_at_id_idx on memories (user_id, updated_at desc, id desc) where archived_at is null; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00096', 'memories_user_updated_at_id_idx') on conflict (version) do nothing; @@ -16191,7 +16195,7 @@ grant execute on function lorekit_memory_list( -- after the two replacements above — it was never wired to anything in the -- first place. Dropping it rather than leaving it as inert dead weight. alter table memories drop column if exists last_seen_at; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00098', 'groom_unseen_days_last_read_at') on conflict (version) do nothing; @@ -16555,7 +16559,7 @@ grant execute on function lorekit_memory_list( text, text[], text, text[], text, text[], text, text[], text, text, timestamptz, uuid, integer, text[], text, uuid[], integer, integer, integer ) to authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00099', 'last_opened_at_targeted_agent_reads') on conflict (version) do nothing; @@ -16848,7 +16852,7 @@ grant execute on function lorekit_memory_list( text, text[], text, text[], text, text[], text, text[], text, text, timestamptz, uuid, integer, text[], text, uuid[], integer, integer, integer ) to authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00100', 'unseen_days_never_opened_uses_created_at') on conflict (version) do nothing; @@ -17375,7 +17379,7 @@ grant execute on function lorekit_memory_list( timestamptz, uuid, integer, text[], text, uuid[], integer, integer, integer, integer ) to authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00101', 'max_read_count_condition') on conflict (version) do nothing; @@ -17681,7 +17685,7 @@ revoke execute on function lorekit_relevance_config_deactivate(bigint, text) from public, anon; grant execute on function lorekit_relevance_config_deactivate(bigint, text) to authenticated; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00102', 'github_relevance_configs') on conflict (version) do nothing; @@ -17811,7 +17815,7 @@ comment on function lorekit_memories_set_updated_at() is 'index, lesson-rank and BOTH default sorts (Explorer, memory.list order=recency) all read. Before ' '00103 a single bulk memory.list restamped every row on the page, so updated_at reported ' 'last-READ under a last-WRITTEN name and recency ordering was effectively arbitrary.'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00103', 'reads_must_not_restamp_updated_at') on conflict (version) do nothing; @@ -18247,7 +18251,7 @@ grant execute on function lorekit_memory_list( timestamptz, uuid, integer, text[], text, uuid[], integer, integer, integer, integer ) to authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00104', 'opened_count_pull_through') on conflict (version) do nothing; @@ -18814,7 +18818,7 @@ grant execute on function lorekit_memory_list( timestamptz, uuid, integer, text[], text, uuid[], integer, integer, integer, integer, integer ) to authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00105', 'max_opened_count_condition') on conflict (version) do nothing; @@ -19138,7 +19142,7 @@ comment on function lorekit_memory_delivery_cost(uuid, timestamptz, timestamptz, names -- pass NULL, not now(), for "through right now". Tokens are length(value)/4, never tokenized: render them as estimates. Archived lore is included, because it cost context while it was live.'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00106', 'memory_utility_grid') on conflict (version) do nothing; @@ -19445,7 +19449,7 @@ comment on function lorekit_record_memory_citations(uuid, uuid, text[], text[], grant execute on function lorekit_record_memory_citations(uuid, uuid, text[], text[], text) to authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00107', 'memory_citations') on conflict (version) do nothing; @@ -20351,7 +20355,7 @@ grant execute on function lorekit_memory_list( timestamptz, uuid, integer, text[], text, uuid[], integer, integer, integer, integer, integer ) to authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00108', 'retention_conditions_on_aggregates') on conflict (version) do nothing; @@ -20404,7 +20408,7 @@ comment on index usage_events_user_tool_created_idx is force a full per-user scan of usage_events, this schema''s highest-volume table. Fixes production statement-timeout cancellations on GET /memories/read-activity for heavier accounts.'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00109', 'usage_events_read_activity_idx') on conflict (version) do nothing; @@ -20689,7 +20693,7 @@ comment on function lorekit_memory_facets( is drawn from, so a count matches the list under the same parameters. q/key/expiring_within_days remain unmirrored, so with those active a count is an upper bound.'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00110', 'facets_within_group_cooccurrence') on conflict (version) do nothing; @@ -21077,7 +21081,7 @@ grant execute on function memory_write( uuid, text, text, text, text[], text, text, timestamptz, text, integer, boolean, text, text, text, integer, text, text, text[], text, uuid[] ) to anon, authenticated, service_role; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00111', 'wildcard_scope_org_bindings') on conflict (version) do nothing; @@ -21284,7 +21288,7 @@ comment on function lorekit_memory_relevant_candidates(uuid, text, text[], text[ so it can never be wider than the RLS read policy or the calling key. Deterministic and embedding-free: no model, no vector index, no external call. p_limit is clamped to [1, 1000].'; - +; insert into supabase_migrations.schema_migrations (version, name) values ('00112', 'relevant_ranking') on conflict (version) do nothing; @@ -22348,7 +22352,7 @@ drop function if exists lorekit_match_retention( timestamptz, timestamptz, integer, integer, integer, timestamptz, timestamptz, integer, integer, integer ); - +; insert into supabase_migrations.schema_migrations (version, name) values ('00113', 'explorer_reverse_activity_filters_and_opened_sort') on conflict (version) do nothing; diff --git a/supabase/byod/generate.mjs b/supabase/byod/generate.mjs index d11f32ecb..7e273b76d 100644 --- a/supabase/byod/generate.mjs +++ b/supabase/byod/generate.mjs @@ -32,7 +32,7 @@ export const MIGRATIONS_DIR = path.join(ROOT, 'supabase/migrations'); export const OUTPUT_PATH = path.join(ROOT, 'supabase/byod/bootstrap.sql'); // The Supabase CLI's own migration filename grammar: `_.sql`. -const MIGRATION_FILE = /^(\d+)_(.+)\.sql$/; +const MIGRATION_FILE = /^(\d+)_(.*)\.sql$/; /** * Parse a migration filename into the (version, name) pair the Supabase CLI @@ -60,10 +60,18 @@ function banner(text) { */ export function renderBootstrap(migrations) { if (migrations.length === 0) throw new Error('no migrations to render'); + const seen = new Set(); const parsed = migrations.map(({ file, sql }) => { const id = parseMigrationName(file); if (!id) throw new Error(`not a migration filename: ${file}`); - return { file, sql: sql.endsWith('\n') ? sql : `${sql}\n`, ...id }; + // Two files with one version (parallel PRs both taking the next number) + // would otherwise be hidden by the history insert's `on conflict do nothing`. + if (seen.has(id.version)) throw new Error(`duplicate migration version ${id.version}: ${file}`); + seen.add(id.version); + // Normalise a BOM and CRLF line endings, so a Windows checkout with + // core.autocrlf renders byte-identical output and the drift check holds. + const text = sql.replace(/^\uFEFF/, '').replace(/\r\n/g, '\n'); + return { file, sql: text.endsWith('\n') ? text : `${text}\n`, ...id }; }); const newest = parsed[parsed.length - 1].file; @@ -97,8 +105,12 @@ export function renderBootstrap(migrations) { -- -- Billing: memories stored in your own database are NOT counted against -- LoreKit's hosted memory-count billing — LoreKit has no visibility into it. --- The memory cap and rate limit are read from \`lorekit_default_limit\` / --- \`user_limits\` in YOUR database; adjust them there. +-- The memory cap is read from \`lorekit_default_limit\` / \`user_limits\` in +-- YOUR database; adjust it there. (In BYOD mode the mcp function skips the +-- rate limiter.) +-- +-- pg_cron: the purge jobs are scheduled only when the pg_cron extension is +-- already enabled (Database → Extensions). Enable it before applying. -- -- See docs/byod.md for the full setup. @@ -112,7 +124,7 @@ begin if to_regclass('public.memories') is not null then raise exception using message = 'LoreKit schema already present: bootstrap.sql only initialises an empty database.', - hint = 'Upgrade an existing BYOD database with \`supabase db push\` (see docs/byod.md).'; + hint = 'Upgrade a database set up from this file with \`supabase db push\`. One set up from the older hand-written bootstrap.sql has no migration history: see docs/byod.md.'; end if; end $guard$; @@ -128,12 +140,16 @@ create table if not exists supabase_migrations.schema_migrations ( ); `; + // The bare `;` after each body: a migration's last statement need not end in + // one for the Supabase CLI, which splits statements itself, so terminate it + // here rather than let it run into the history insert. An empty statement is + // a no-op when the last statement did end in `;`. const body = parsed .map( ({ file, sql, version, name }) => ` ${banner(file)} -${sql} +${sql}; insert into supabase_migrations.schema_migrations (version, name) values (${sqlLiteral(version)}, ${sqlLiteral(name)}) on conflict (version) do nothing; diff --git a/supabase/byod/generate.test.mjs b/supabase/byod/generate.test.mjs index 2c579d2cc..5c927147d 100644 --- a/supabase/byod/generate.test.mjs +++ b/supabase/byod/generate.test.mjs @@ -15,6 +15,7 @@ test('parseMigrationName follows the Supabase CLI grammar', () => { assert.deepEqual(parseMigrationName('00113_a_b_c.sql'), { version: '00113', name: 'a_b_c' }); assert.equal(parseMigrationName('README.md'), null); assert.equal(parseMigrationName('memories.sql'), null); + assert.deepEqual(parseMigrationName('00002_.sql'), { version: '00002', name: '' }, 'CLI accepts an empty name'); }); test('renderBootstrap keeps migrations verbatim, in order, inside one transaction', () => { @@ -46,6 +47,22 @@ test('renderBootstrap guards against re-running on an initialised database', () assert.ok(guard > 0 && guard < out.indexOf('select 1;'), 'guard runs before any migration'); }); +test('renderBootstrap normalises CRLF and a BOM, and terminates an unterminated last statement', () => { + const out = renderBootstrap([{ file: '00001_first.sql', sql: '\uFEFFselect 1;\r\nselect 2' }]); + assert.ok(!out.includes('\r') && !out.includes('\uFEFF')); + assert.match(out, /select 2\n;\ninsert into supabase_migrations/); +}); + +test('renderBootstrap rejects two migrations with one version', () => { + assert.throws( + () => renderBootstrap([ + { file: '00114_a.sql', sql: 'select 1;' }, + { file: '00114_b.sql', sql: 'select 2;' }, + ]), + /duplicate migration version 00114/, + ); +}); + test('renderBootstrap is deterministic and rejects bad input', () => { const input = [{ file: '00001_first.sql', sql: 'select 1;' }]; assert.equal(renderBootstrap(input), renderBootstrap(input)); From a64ffd32a0beacc93b9549462d2c36b790b57c4e Mon Sep 17 00:00:00 2001 From: "mads.thines" Date: Fri, 16 Oct 2026 13:53:50 +0000 Subject: [PATCH 05/16] fix(byod): require the service key, scope session callers, drop user FKs in storage Three BYOD limits documented in the previous commits, fixed: - An anon-key storage client wrote but read back nothing (the read policies need auth.uid()). resolveStorageAdapter now requires LOREKIT_STORAGE_SERVICE_KEY and says why. - With the service key, a Supabase-session (JWT) caller reached the tools with no user id, so it read every row in the storage project and wrote ownerless rows. storageUserId scopes it by its resolved id, the path an lk_* token already takes, and refuses one with no id. - Every lk_* write failed on memories_user_id_fkey: the storage project's auth.users never holds the function project's users. storage-overlay.sql drops every public FK into auth.users (by query, idempotent); bootstrap.sql now ends with it and the db push route runs it after each push. doctor accepts either storage key. Co-authored-by: dash0-dev[bot] <257284812+dash0-dev[bot]@users.noreply.github.com> --- docs/byod.md | 66 +++++++++++-------- docs/cli.md | 5 +- docs/key-files.md | 2 +- packages/cli/src/commands/bootstrap.mjs | 5 +- packages/cli/src/commands/doctor.mjs | 14 ++-- .../mcp-core/src/db/storage-adapter.spec.ts | 54 ++++++++++++--- packages/mcp-core/src/db/storage-adapter.ts | 44 +++++++++++-- supabase/byod/bootstrap.sql | 53 ++++++++++++++- supabase/byod/generate.mjs | 24 +++++-- supabase/byod/generate.test.mjs | 24 ++++++- supabase/byod/storage-overlay.sql | 39 +++++++++++ supabase/functions/mcp/mcp-handler.ts | 32 ++++----- supabase/functions/mcp/storage-adapter.ts | 38 +++++++++-- 13 files changed, 314 insertions(+), 86 deletions(-) create mode 100644 supabase/byod/storage-overlay.sql diff --git a/docs/byod.md b/docs/byod.md index ea944bb23..7ad885918 100644 --- a/docs/byod.md +++ b/docs/byod.md @@ -20,7 +20,9 @@ the part most likely to surprise you. your machine. Deploy the function yourself first: [install.md](./install.md). - **Two projects, two jobs.** API tokens and users are resolved in the function's own project; memory rows are read from and written to the - storage project. + storage project. The storage project's `auth.users` never holds those users, + which is why the storage schema drops its foreign keys into `auth.users` + (step 2). - **Only the `mcp` function has a BYOD mode.** The `memories` REST function — which backs the web dashboard, the CLI's remote commands and the hook engine's lesson injection — always uses the function's own project. See @@ -41,15 +43,22 @@ already enabled. The storage project needs the full LoreKit schema — the same migrations the hosted project runs. Pick one route. -**Supabase CLI (recommended — the same command upgrades it later):** +**Supabase CLI (recommended — the same commands upgrade it later):** ```bash git clone https://github.com/mthines/lorekit.git cd lorekit supabase link --project-ref supabase db push +psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -f supabase/byod/storage-overlay.sql ``` +The last line applies the **storage overlay**: it drops every foreign key from +a `public` table into `auth.users`, because the users live in the function's +project, not this one. Without it, every write made with an `lk_*` token fails +with `memories_user_id_fkey`. It is idempotent; re-run it after every +`db push`. Never apply it to the project that runs the `mcp` function. + **One SQL file (empty database only):** ```bash @@ -62,10 +71,11 @@ if the editor refuses it, use `psql`). `$DATABASE_URL` is the connection string from **Connect**; its password is your **database password**, not an API key. `bootstrap.sql` is generated from `supabase/migrations/` (CI fails when the two -disagree). It applies every migration in one transaction, so a failure leaves -the database untouched; it refuses to run against a database that already has -the schema; and it records each migration in Supabase's migration history, so -the CLI route above upgrades the database later. +disagree) and ends with the storage overlay. It applies everything in one +transaction, so a failure leaves the database untouched; it refuses to run +against a database that already has the schema; and it records each migration +in Supabase's migration history, so the CLI route above upgrades the database +later. `lorekit bootstrap`, run with `LOREKIT_STORAGE_URL` and a key in your shell, checks whether the project is reachable and already has the schema, and prints @@ -82,16 +92,16 @@ supabase secrets set --project-ref \ LOREKIT_STORAGE_SERVICE_KEY= ``` -**Set the service key.** The function reads with whichever key it is given, -and the storage project's read policies require `auth.uid()`, which an API key -never carries. With only the anon key, writes succeed (they go through a -`SECURITY DEFINER` function) but every read returns no rows. +The service key is **required**: without it, every call to the function fails +with an error naming `LOREKIT_STORAGE_SERVICE_KEY`. The storage project's read policies need `auth.uid()`, which an API +key never carries, so an anon-key client could write but would read back +nothing. `LOREKIT_STORAGE_ANON_KEY` is accepted but unused once the service key +is set. -The service key **bypasses row-level security**. Reads made with an `lk_*` -token are still filtered to that token's user by the function. Reads from a -caller authenticated with a Supabase session (JWT) are not filtered, because -the function leaves RLS to do that, and so they see every row in the storage -project. +The service key bypasses row-level security, so the function scopes every call +itself: reads and writes made with an `lk_*` token, or from a Supabase session, +are filtered to and stamped with that caller's user id. Only the service-role +tier (CI) sees every row. ### 4. Check it @@ -113,12 +123,14 @@ checkout of that release: ```bash supabase link --project-ref supabase db push +psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -f supabase/byod/storage-overlay.sql ``` `db push` applies only the migrations the database has not recorded, which works the same whether you set the database up with `db push` or with -`bootstrap.sql`. Upgrade the storage project before deploying the matching -`mcp` function: the function calls the RPCs those migrations define. +`bootstrap.sql`. Re-run the overlay after it, in case a new migration added a +foreign key into `auth.users`. Upgrade the storage project before deploying the +matching `mcp` function: the function calls the RPCs those migrations define. ### Databases set up from the old `bootstrap.sql` @@ -126,8 +138,8 @@ works the same whether you set the database up with `db push` or with up from that version has no migration history — `select count(*) from supabase_migrations.schema_migrations` errors or returns 0 — so `db push` would try to replay every migration from the first and fail, and its schema is -missing most of the functions the current `mcp` function calls. Move to a fresh storage -project instead: apply the current schema to it as above, copy the `memories` +missing most of the functions the current `mcp` function calls. Move to a +fresh storage project instead: apply the current schema to it as above, copy the `memories` rows across (for example `pg_dump --data-only --table=public.memories` from the old database, loaded with `psql` into the new one), then point `LOREKIT_STORAGE_URL` at the new project. @@ -149,16 +161,12 @@ and does not record hosted usage events. API, the CLI's remote commands (`list`, `search`, `show`, `write` against the remote store) and the hook engine's lesson injection go through the `memories` function, which has no BYOD mode. They read and write the function's own - project and do not see memories stored through BYOD. -- **A token's user must also exist in the storage project.** `memories.user_id` - references the storage project's `auth.users`. A write made with an `lk_*` - token carries the token owner's id from the function's project, and it fails - with `memories_user_id_fkey` unless a user with that id also exists in the - storage project. Writes made with the service-role key carry no user id and - are not affected. -- **No per-user RLS in the storage project.** The function talks to it with the - service or anon key, never with the caller's JWT — see step 3 for what that - means for reads. + project and do not see memories stored through BYOD. To run the dashboard on + a database you control, self-host the whole stack instead + ([install.md](./install.md)). +- **No row-level security in the storage project.** The function talks to it + with the service key and scopes every call in its own code (step 3); RLS + policies there do not apply to it. - **Tested surface.** The BYOD smoke test exercises write, read, list, search and delete, and runs only where its secrets are configured. The schema now also contains the org, audit, retention and webhook tables and RPCs, but BYOD diff --git a/docs/cli.md b/docs/cli.md index bcc023d4f..1e7c88568 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -56,7 +56,7 @@ frontmatter, checked in BOTH the project and global install locations): a `curre a `warn` — never a `fail`, the skill still works — naming the installed and shipped versions and pointing at `lorekit update --`, the same scope-aware wording the hooks-upgrade line above it uses. When `LOREKIT_STORAGE_URL` is set, a `byod storage` check probes that project with -`LOREKIT_STORAGE_ANON_KEY` (`probeByodStorage` in `shared/byod.mjs` — one PostgREST +`LOREKIT_STORAGE_ANON_KEY` or, failing that, `LOREKIT_STORAGE_SERVICE_KEY` (`probeByodStorage` in `shared/byod.mjs` — one PostgREST `GET /rest/v1/memories?limit=1`, plain `fetch`): it passes when the `memories` table answers and FAILS when the project is unreachable, rejects the key, or has no LoreKit schema yet. @@ -306,7 +306,8 @@ BYOD only ([byod.md](./byod.md)). Probes the project named by `LOREKIT_STORAGE_U `LOREKIT_STORAGE_ANON_KEY` or `LOREKIT_STORAGE_SERVICE_KEY` (the same `probeByodStorage` as `doctor`), reports whether the LoreKit schema is already there, and prints the three routes that apply it: `psql -v ON_ERROR_STOP=1 -f supabase/byod/bootstrap.sql`, the SQL Editor, or -`supabase link` + `supabase db push`. It runs no DDL — PostgREST, the only thing a Supabase API key +`supabase link` + `supabase db push` followed by `supabase/byod/storage-overlay.sql` (which +`bootstrap.sql` already includes). It runs no DDL — PostgREST, the only thing a Supabase API key reaches, cannot. The published package does not ship `supabase/`, so outside a LoreKit checkout it prints the file's download URL instead of a local path. It never prints a key: the `psql` connection string's password is the database password, not the service-role key. diff --git a/docs/key-files.md b/docs/key-files.md index 13d07f01d..66f4330c8 100644 --- a/docs/key-files.md +++ b/docs/key-files.md @@ -81,7 +81,7 @@ else. Grouped loosely by subsystem in migration/PR order. | `supabase/migrations/00042_audit_log_github_app_action.sql` | Forward-only drop+re-add of the `audit_log.action` CHECK, widened to all 24 actions — admits `github_app.installation_linked`, which `handleSetupReturn` had been emitting into a rejecting CHECK since the GitHub App shipped (silently, via non-throwing `recordAuditEvent`) | | `packages/mcp-core/src/rest/cors-origins.ts` | Pure CORS origin matching — `expandOriginSiblings`/`expandAllowedOrigins` (apex ⇄ `www.` allowlist widening), `isOriginAllowed`, and `corsResponseHeaders` (emits `Access-Control-Allow-Origin` only for an allowed origin, never an empty one). `isOriginAllowed` ALSO always admits two classes of origin regardless of `ALLOWED_ORIGINS`, both justified by Bearer-auth-not-CORS: **loopback** dev hosts, and the project's own **Vercel deployments** — `isVercelPreviewOrigin` matches a single DNS label before exactly `.vercel.app` that BOTH starts with `lorekit-` AND ends with the Vercel account scope `-mads-thines-projects` (so `lorekit-git--mads-thines-projects.vercel.app` and the `lorekit-mads-thines-projects.vercel.app` alias match, while a nested `lorekit-x.attacker.vercel.app`, a `lorekit`-prefixed different word like `lorekitten`, and — crucially — a third-party `lorekit-x-.vercel.app` are all rejected; the scope suffix is the half a third party cannot forge). Update `VERCEL_SCOPE` if the project moves Vercel accounts. Preview deployments get a per-branch/per-commit hostname (`lorekit-git--.vercel.app`) that can't be enumerated into a static allowlist, so they were CORS-blocked once the dashboard moved off PostgREST onto the REST edge functions (PR #311). Lifted out of `supabase/functions/_shared/api/cors.ts` so it has a vitest home (`cors-origins.spec.ts`); mirrored verbatim into `supabase/functions/_shared/api/cors-origins.ts` and guarded by `edge-parity.spec.ts` | | `packages/mcp-core/src/rest/rest-tool-name.ts` | Pure `restToolName`/`REST_TOOL_NAMES` — the REST route → `usage_events.tool_name` mapping, aligning each route with the MCP tool it is the equivalent of so both surfaces aggregate as one series. Import-free, mirrored verbatim into `supabase/functions/_shared/rest/rest-tool-name.ts` and guarded by `edge-parity.spec.ts`; a route registered without a mapping fails `rest-tool-name.spec.ts` | -| `supabase/byod/generate.mjs` + `generate.test.mjs` | Generates `supabase/byod/bootstrap.sql` (BYOD's one-file schema) from `supabase/migrations/` — verbatim, in order, in ONE transaction, behind a guard that refuses an already-initialised database, recording every version in `supabase_migrations.schema_migrations` so the database later upgrades with `supabase db push`. `--check` and the spec (the `supabase:test` target) fail on drift. Replaced a hand-written subset that had fallen 42 RPCs and a `memory_write` signature behind the edge | +| `supabase/byod/generate.mjs` + `generate.test.mjs` | Generates `supabase/byod/bootstrap.sql` (BYOD's one-file schema) from `supabase/migrations/` — verbatim, in order, in ONE transaction, behind a guard that refuses an already-initialised database, recording every version in `supabase_migrations.schema_migrations` so the database later upgrades with `supabase db push`, and ending with `storage-overlay.sql` — an idempotent DO block that drops every `public` foreign key into `auth.users`, found by query not name, because a BYOD storage project's `auth.users` never holds the function project's users. `--check` and the spec (the `supabase:test` target) fail on drift. Replaced a hand-written subset that had fallen 42 RPCs and a `memory_write` signature behind the edge | | `supabase/tests/bare-postgres-bootstrap.sql` | Makes `migrations.test.sql` runnable WITHOUT Docker: supplies the Supabase-platform pieces the migrations reference (the three `auth.*` claim readers, `auth.users`, `auth.identities`, the anon/authenticated/service_role roles — `service_role` BYPASSRLS as in production) so the migrations apply to a bare `initdb` cluster. Carries its own runbook and, prominently, its own fidelity caveat: a FAILURE is strong evidence, a PASS is not a substitute for CI's `Integration smoke` job. Exists because `supabase start` needs a Docker socket, which cloud sandboxes and agent sessions do not have — previously making the one file that exercises raw migration logic verifiable only by pushing and reading a CI log | | `packages/mcp-core/src/telemetry/telemetry-vocabulary.ts` | The CLOSED set of names `usage_events.tool_name` may carry: `TELEMETRY_OP_NAMES` = the catalog's tool names (DERIVED from `MCP_TOOLS`, never restated) ∪ `NON_CATALOG_OPS` — the REST operations with no MCP tool, each carrying the reason it has none. Closes the hole left by `restToolName`'s deliberate `.unmapped` fallback, which is right at runtime and silent in development. The ten dashboard analytics reads carry a `restOnly` reason (`REST_ONLY_OP_NAMES`), which is how the "REST-only on purpose" decision is machine-checked rather than only written down. **Node-side only — deliberately NOT mirrored:** nothing in the edge tree imports it, so unlike `account-wide-tools.ts` it is free to import the catalog. Guarded by `telemetry-vocabulary.spec.ts` (closure both ways, the ten pinned by name, and a scan proving no CLI command calls the ten while `/memories/relevant` still does) | | `supabase/functions/_shared/telemetry/usage.ts` | `recordUsageEvent` + `getUserPlanName` — the single edge usage-event writer, called once per MCP tool call (`mcp/mcp-handler.ts`) and once per REST route (`_shared/api/router.ts`). `mcp/limits.ts` re-exports it | diff --git a/packages/cli/src/commands/bootstrap.mjs b/packages/cli/src/commands/bootstrap.mjs index 155de497f..cee21897b 100644 --- a/packages/cli/src/commands/bootstrap.mjs +++ b/packages/cli/src/commands/bootstrap.mjs @@ -78,6 +78,7 @@ export async function bootstrap(_args) { log(''); log(' bootstrap.sql only initialises an EMPTY database. To upgrade this one, run'); log(` from a LoreKit checkout: ${c.cyan('supabase link --project-ref')} ${parseRef(storageUrl)} && ${c.cyan('supabase db push')}`); + log(` then re-apply the storage overlay: ${c.cyan('psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -f')} supabase/byod/storage-overlay.sql`); log(''); return 0; } @@ -103,7 +104,9 @@ export async function bootstrap(_args) { log(` Or paste the contents of ${c.dim(sqlArg)}`); log(' into the Supabase dashboard → SQL Editor and run it.'); log(''); - log(` Or, from a LoreKit checkout: ${c.cyan('supabase link --project-ref')} ${parseRef(storageUrl)} && ${c.cyan('supabase db push')}`); + log(` Or, from a LoreKit checkout: ${c.cyan('supabase link --project-ref')} ${parseRef(storageUrl)} && ${c.cyan('supabase db push')},`); + log(` then ${c.cyan('psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -f')} supabase/byod/storage-overlay.sql`); + log(` ${c.dim('(bootstrap.sql already includes the overlay; the db push route needs it after every push)')}`); log(''); return 0; diff --git a/packages/cli/src/commands/doctor.mjs b/packages/cli/src/commands/doctor.mjs index 0c43564c0..b90252c2f 100644 --- a/packages/cli/src/commands/doctor.mjs +++ b/packages/cli/src/commands/doctor.mjs @@ -629,23 +629,23 @@ function cliVersion() { async function checkBYODStorage(record) { const storageUrl = process.env['LOREKIT_STORAGE_URL']; - const storageAnonKey = process.env['LOREKIT_STORAGE_ANON_KEY']; + const storageKey = process.env['LOREKIT_STORAGE_ANON_KEY'] || process.env['LOREKIT_STORAGE_SERVICE_KEY']; - if (!storageUrl && !storageAnonKey) { + if (!storageUrl && !storageKey) { return; // No BYOD configured — skip silently. } - if (storageUrl && !storageAnonKey) { - record('fail', 'byod storage', 'LOREKIT_STORAGE_URL is set but LOREKIT_STORAGE_ANON_KEY is missing'); + if (!storageUrl) { + record('fail', 'byod storage', 'a LOREKIT_STORAGE_*_KEY is set but LOREKIT_STORAGE_URL is missing'); return; } - if (!storageUrl && storageAnonKey) { - record('fail', 'byod storage', 'LOREKIT_STORAGE_ANON_KEY is set but LOREKIT_STORAGE_URL is missing'); + if (!storageKey) { + record('fail', 'byod storage', 'LOREKIT_STORAGE_URL is set but neither LOREKIT_STORAGE_ANON_KEY nor LOREKIT_STORAGE_SERVICE_KEY is'); return; } - const probe = await probeByodStorage({ url: storageUrl, key: storageAnonKey }); + const probe = await probeByodStorage({ url: storageUrl, key: storageKey }); if (probe.state === 'ready') record('pass', 'byod storage', `ok — ${storageUrl}`); else if (probe.state === 'no-schema') record('fail', 'byod storage', `${probe.detail} — see docs/byod.md`); else record('fail', 'byod storage', probe.detail); diff --git a/packages/mcp-core/src/db/storage-adapter.spec.ts b/packages/mcp-core/src/db/storage-adapter.spec.ts index d49237c32..1c1db919e 100644 --- a/packages/mcp-core/src/db/storage-adapter.spec.ts +++ b/packages/mcp-core/src/db/storage-adapter.spec.ts @@ -1,5 +1,5 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; -import { resolveStorageAdapter, createHostedAdapter, createBYODAdapter } from './storage-adapter.js'; +import { resolveStorageAdapter, createHostedAdapter, createBYODAdapter, storageUserId } from './storage-adapter.js'; // These are pure unit tests that verify adapter metadata flags only. // Mock createClient so the real Supabase client (which needs native WebSocket @@ -99,19 +99,31 @@ describe('resolveStorageAdapter', () => { expect(adapter.mode).toBe('hosted'); }); - it('returns BYOD adapter when LOREKIT_STORAGE_URL and LOREKIT_STORAGE_ANON_KEY are both set', () => { + it('throws naming LOREKIT_STORAGE_SERVICE_KEY when URL and only the anon key are set', () => { + // An anon-key client writes (memory_write is SECURITY DEFINER) but reads + // back nothing (the read policies need auth.uid()), so this must fail at + // configuration rather than serve empty reads that look healthy. process.env['LOREKIT_STORAGE_URL'] = 'https://example.supabase.co'; process.env['LOREKIT_STORAGE_ANON_KEY'] = 'test-anon-key'; - const adapter = resolveStorageAdapter(); - expect(adapter.mode).toBe('byod'); - expect(adapter.supportsHostedBilling).toBe(false); - expect(adapter.supportsRateLimit).toBe(false); + delete process.env['LOREKIT_STORAGE_SERVICE_KEY']; + expect(() => resolveStorageAdapter()).toThrow('LOREKIT_STORAGE_SERVICE_KEY'); }); - it('throws with message naming LOREKIT_STORAGE_ANON_KEY when URL is set without it', () => { + it('throws naming LOREKIT_STORAGE_SERVICE_KEY when URL is set without any key', () => { process.env['LOREKIT_STORAGE_URL'] = 'https://example.supabase.co'; delete process.env['LOREKIT_STORAGE_ANON_KEY']; - expect(() => resolveStorageAdapter()).toThrow('LOREKIT_STORAGE_ANON_KEY'); + delete process.env['LOREKIT_STORAGE_SERVICE_KEY']; + expect(() => resolveStorageAdapter()).toThrow('LOREKIT_STORAGE_SERVICE_KEY'); + }); + + it('returns a BYOD adapter from the URL and service key alone', () => { + process.env['LOREKIT_STORAGE_URL'] = 'https://example.supabase.co'; + delete process.env['LOREKIT_STORAGE_ANON_KEY']; + process.env['LOREKIT_STORAGE_SERVICE_KEY'] = 'test-service-key'; + const adapter = resolveStorageAdapter(); + expect(adapter.mode).toBe('byod'); + expect(adapter.supportsHostedBilling).toBe(false); + expect(adapter.supportsRateLimit).toBe(false); }); it('passes LOREKIT_STORAGE_SERVICE_KEY through to BYOD adapter when set', () => { @@ -125,3 +137,29 @@ describe('resolveStorageAdapter', () => { expect(adapter.supportsRateLimit).toBe(false); }); }); + +describe('storageUserId', () => { + const USER = '6f1c2a9e-0000-4000-8000-000000000001'; + + it('scopes an api_key caller by its id in both modes', () => { + expect(storageUserId('hosted', { type: 'api_key', userId: USER })).toBe(USER); + expect(storageUserId('byod', { type: 'api_key', userId: USER })).toBe(USER); + }); + + it('leaves a hosted JWT caller to RLS', () => { + expect(storageUserId('hosted', { type: 'user', userId: USER })).toBeNull(); + }); + + it('scopes a BYOD JWT caller by its id — the storage client carries no JWT for RLS to read', () => { + expect(storageUserId('byod', { type: 'user', userId: USER })).toBe(USER); + }); + + it('refuses a BYOD JWT caller with no resolved id instead of letting it through unscoped', () => { + expect(() => storageUserId('byod', { type: 'user' })).toThrow('BYOD'); + }); + + it('leaves the service tier unscoped in both modes', () => { + expect(storageUserId('hosted', { type: 'service' })).toBeNull(); + expect(storageUserId('byod', { type: 'service' })).toBeNull(); + }); +}); diff --git a/packages/mcp-core/src/db/storage-adapter.ts b/packages/mcp-core/src/db/storage-adapter.ts index 2fcb548b6..64075542b 100644 --- a/packages/mcp-core/src/db/storage-adapter.ts +++ b/packages/mcp-core/src/db/storage-adapter.ts @@ -94,11 +94,11 @@ export function createBYODAdapter(url: string, anonKey: string, serviceKey?: str * Resolve the correct StorageAdapter from environment variables or config. * * Resolution order: - * 1. LOREKIT_STORAGE_URL + LOREKIT_STORAGE_ANON_KEY → BYOD adapter - * (optionally uses LOREKIT_STORAGE_SERVICE_KEY for privileged access) + * 1. LOREKIT_STORAGE_URL + LOREKIT_STORAGE_SERVICE_KEY → BYOD adapter + * (LOREKIT_STORAGE_ANON_KEY is accepted but unused once the service key is set) * 2. SUPABASE_URL + SUPABASE_ANON_KEY → hosted adapter (default) * - * Throws if LOREKIT_STORAGE_URL is set without LOREKIT_STORAGE_ANON_KEY. + * Throws if LOREKIT_STORAGE_URL is set without LOREKIT_STORAGE_SERVICE_KEY. */ export function resolveStorageAdapter(jwt?: string): StorageAdapter { const storageUrl = process.env['LOREKIT_STORAGE_URL']; @@ -106,14 +106,44 @@ export function resolveStorageAdapter(jwt?: string): StorageAdapter { const storageServiceKey = process.env['LOREKIT_STORAGE_SERVICE_KEY']; if (storageUrl) { - if (!storageAnonKey) { + // The service key is required, not optional: the storage project's read + // policies need auth.uid(), which an API key never carries, so an anon-key + // client writes (memory_write is SECURITY DEFINER) but reads back nothing. + // Fail at configuration rather than return empty reads that look healthy. + if (!storageServiceKey) { throw new Error( - 'LOREKIT_STORAGE_URL is set but LOREKIT_STORAGE_ANON_KEY is missing. ' + - 'Both variables are required to use a custom database.', + 'LOREKIT_STORAGE_URL is set but LOREKIT_STORAGE_SERVICE_KEY is missing. ' + + 'BYOD needs the storage project\'s service-role key: with only the anon key every read returns no rows.', ); } - return createBYODAdapter(storageUrl, storageAnonKey, storageServiceKey); + return createBYODAdapter(storageUrl, storageAnonKey ?? '', storageServiceKey); } return createHostedAdapter(jwt); } + +/** + * The user id a tool call filters its reads and stamps its writes with. + * + * Hosted: an `api_key` caller runs on the service-role client, so it is scoped + * in app code by its id; a JWT (`user`) caller gets a client carrying its own + * JWT, so RLS scopes it and the id stays null; `service` sees everything. + * + * BYOD: the storage client is built from the storage project's SERVICE key and + * never carries the caller's JWT, so RLS cannot scope a JWT caller there. Left + * at null it would read every row in the storage project and write rows with + * no owner. It is scoped in app code by its resolved id instead — the same path + * an `api_key` caller takes. A JWT caller with no resolved id is refused rather + * than allowed through unscoped. + */ +export function storageUserId( + mode: StorageAdapter['mode'], + auth: { type: 'user' | 'service' | 'api_key'; userId?: string | null }, +): string | null { + if (auth.type === 'api_key') return auth.userId ?? null; + if (auth.type === 'user' && mode === 'byod') { + if (!auth.userId) throw new Error('BYOD: the session has no resolved user id, so its reads cannot be scoped.'); + return auth.userId; + } + return null; +} diff --git a/supabase/byod/bootstrap.sql b/supabase/byod/bootstrap.sql index 41d556c69..26836bd21 100644 --- a/supabase/byod/bootstrap.sql +++ b/supabase/byod/bootstrap.sql @@ -7,8 +7,11 @@ -- one) and regenerate instead. -- -- What it does: applies the FULL LoreKit schema — the same migrations the hosted --- project runs — to an empty Supabase project, in one transaction, and records --- each migration in Supabase's migration history. +-- project runs — to an empty Supabase project, in one transaction, records each +-- migration in Supabase's migration history, and ends with the BYOD storage +-- overlay (storage-overlay.sql), which drops the foreign keys into auth.users: +-- this project's auth.users never holds the users of the project that runs the +-- mcp function. Never apply this file to the project that runs the function. -- -- Fresh database only. A guard below aborts (and rolls everything back) when the -- LoreKit schema is already present. Apply with ON_ERROR_STOP so psql stops at @@ -25,6 +28,7 @@ -- -- supabase link --project-ref -- supabase db push +-- psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -f supabase/byod/storage-overlay.sql -- -- Billing: memories stored in your own database are NOT counted against -- LoreKit's hosted memory-count billing — LoreKit has no visibility into it. @@ -22357,4 +22361,49 @@ insert into supabase_migrations.schema_migrations (version, name) values ('00113', 'explorer_reverse_activity_filters_and_opened_sort') on conflict (version) do nothing; +-- ───────────────────────────────────────────────────────────────────────────── +-- storage-overlay.sql — BYOD only: drop foreign keys into auth.users +-- ───────────────────────────────────────────────────────────────────────────── + +-- LoreKit BYOD storage overlay +-- +-- Apply to a BYOD STORAGE database only — the separate Supabase project that +-- holds memory rows — after the schema, and again after every +-- `supabase db push` upgrade. Idempotent. bootstrap.sql already ends with it. +-- +-- psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -f supabase/byod/storage-overlay.sql +-- +-- Why: in BYOD, users and API tokens live in the project that runs the `mcp` +-- function; the storage project's auth.users never holds them. Every foreign +-- key from a public table into auth.users therefore rejects legitimate rows — +-- memory_write fails with memories_user_id_fkey for every lk_* token. Dropping +-- those keys is the overlay's only change. The ON DELETE CASCADE / SET NULL +-- they carried pointed at users this database never has; deleting a user in +-- the function's project does not remove their rows here. +-- +-- Never apply it to the project that RUNS the mcp function (hosted or +-- self-hosted): there auth.users is the real user table and the cascades +-- matter. +-- +-- The keys are found by query, not by name, so a migration that adds another +-- user foreign key is covered by re-running this after the upgrade. +do $overlay$ +declare + fk record; +begin + for fk in + select con.conrelid::regclass as tbl, con.conname + from pg_constraint con + join pg_class c on c.oid = con.conrelid + join pg_namespace n on n.oid = c.relnamespace + where con.contype = 'f' + and con.confrelid = 'auth.users'::regclass + and n.nspname = 'public' + loop + execute format('alter table %s drop constraint %I', fk.tbl, fk.conname); + end loop; +end +$overlay$; +; + commit; diff --git a/supabase/byod/generate.mjs b/supabase/byod/generate.mjs index 7e273b76d..477938cb1 100644 --- a/supabase/byod/generate.mjs +++ b/supabase/byod/generate.mjs @@ -1,6 +1,7 @@ #!/usr/bin/env node // Generate the BYOD ("bring your own database") bootstrap from the migrations. // Source of truth: supabase/migrations/*.sql (applied in filename order) +// + the BYOD storage overlay: supabase/byod/storage-overlay.sql // Generated file: supabase/byod/bootstrap.sql // // node supabase/byod/generate.mjs regenerate bootstrap.sql @@ -30,6 +31,7 @@ import { fileURLToPath } from 'node:url'; const ROOT = fileURLToPath(new URL('../../', import.meta.url)); export const MIGRATIONS_DIR = path.join(ROOT, 'supabase/migrations'); export const OUTPUT_PATH = path.join(ROOT, 'supabase/byod/bootstrap.sql'); +export const OVERLAY_PATH = path.join(ROOT, 'supabase/byod/storage-overlay.sql'); // The Supabase CLI's own migration filename grammar: `_.sql`. const MIGRATION_FILE = /^(\d+)_(.*)\.sql$/; @@ -54,11 +56,14 @@ function banner(text) { } /** - * Render bootstrap.sql from an ordered list of migrations. + * Render bootstrap.sql from an ordered list of migrations, followed by the + * BYOD storage overlay (storage-overlay.sql), all in one transaction. * @param {{ file: string, sql: string }[]} migrations already sorted by filename + * @param {string} overlay the storage overlay's SQL * @returns {string} */ -export function renderBootstrap(migrations) { +export function renderBootstrap(migrations, overlay) { + if (typeof overlay !== 'string' || overlay.trim() === '') throw new Error('no storage overlay to render'); if (migrations.length === 0) throw new Error('no migrations to render'); const seen = new Set(); const parsed = migrations.map(({ file, sql }) => { @@ -84,8 +89,11 @@ export function renderBootstrap(migrations) { -- one) and regenerate instead. -- -- What it does: applies the FULL LoreKit schema — the same migrations the hosted --- project runs — to an empty Supabase project, in one transaction, and records --- each migration in Supabase's migration history. +-- project runs — to an empty Supabase project, in one transaction, records each +-- migration in Supabase's migration history, and ends with the BYOD storage +-- overlay (storage-overlay.sql), which drops the foreign keys into auth.users: +-- this project's auth.users never holds the users of the project that runs the +-- mcp function. Never apply this file to the project that runs the function. -- -- Fresh database only. A guard below aborts (and rolls everything back) when the -- LoreKit schema is already present. Apply with ON_ERROR_STOP so psql stops at @@ -102,6 +110,7 @@ export function renderBootstrap(migrations) { -- -- supabase link --project-ref -- supabase db push +-- psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -f supabase/byod/storage-overlay.sql -- -- Billing: memories stored in your own database are NOT counted against -- LoreKit's hosted memory-count billing — LoreKit has no visibility into it. @@ -157,7 +166,12 @@ on conflict (version) do nothing; ) .join(''); + const overlaySql = overlay.replace(/^\uFEFF/, '').replace(/\r\n/g, '\n'); return `${header}${body} +${banner('storage-overlay.sql — BYOD only: drop foreign keys into auth.users')} + +${overlaySql.endsWith('\n') ? overlaySql : `${overlaySql}\n`}; + commit; `; } @@ -172,7 +186,7 @@ export function readMigrations(dir = MIGRATIONS_DIR) { } function main() { - const expected = renderBootstrap(readMigrations()); + const expected = renderBootstrap(readMigrations(), fs.readFileSync(OVERLAY_PATH, 'utf8')); const rel = path.relative(ROOT, OUTPUT_PATH); if (process.argv.includes('--check')) { diff --git a/supabase/byod/generate.test.mjs b/supabase/byod/generate.test.mjs index 5c927147d..b8b096fe5 100644 --- a/supabase/byod/generate.test.mjs +++ b/supabase/byod/generate.test.mjs @@ -5,11 +5,15 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; import { parseMigrationName, - renderBootstrap, + renderBootstrap as render, readMigrations, OUTPUT_PATH, + OVERLAY_PATH, } from './generate.mjs'; +const OVERLAY = '-- overlay\ndo $o$ begin perform 1; end $o$;\n'; +const renderBootstrap = (migrations, overlay = OVERLAY) => render(migrations, overlay); + test('parseMigrationName follows the Supabase CLI grammar', () => { assert.deepEqual(parseMigrationName('00001_memories.sql'), { version: '00001', name: 'memories' }); assert.deepEqual(parseMigrationName('00113_a_b_c.sql'), { version: '00113', name: 'a_b_c' }); @@ -63,6 +67,22 @@ test('renderBootstrap rejects two migrations with one version', () => { ); }); +test('renderBootstrap ends with the storage overlay, after every migration and inside the transaction', () => { + const out = renderBootstrap([{ file: '00001_first.sql', sql: 'select 1;' }]); + const history = out.lastIndexOf("values ('00001', 'first')"); + const overlay = out.indexOf('do $o$ begin perform 1; end $o$;'); + assert.ok(overlay > history, 'overlay runs after the last migration is recorded'); + assert.ok(overlay < out.lastIndexOf('commit;'), 'overlay is inside the transaction'); + assert.throws(() => render([{ file: '00001_first.sql', sql: 'select 1;' }], ''), /no storage overlay/); +}); + +test('the overlay finds the foreign keys by query, never by a hard-coded name', () => { + const overlay = fs.readFileSync(OVERLAY_PATH, 'utf8').replace(/^\s*--.*$/gm, ''); + assert.match(overlay, /confrelid = 'auth\.users'::regclass/); + assert.match(overlay, /nspname = 'public'/); + assert.doesNotMatch(overlay, /_fkey/, 'a named constraint would miss the next migration\'s user FK'); +}); + test('renderBootstrap is deterministic and rejects bad input', () => { const input = [{ file: '00001_first.sql', sql: 'select 1;' }]; assert.equal(renderBootstrap(input), renderBootstrap(input)); @@ -73,7 +93,7 @@ test('renderBootstrap is deterministic and rejects bad input', () => { test('the committed bootstrap.sql is generated from the current migrations', () => { assert.equal( fs.readFileSync(OUTPUT_PATH, 'utf8'), - renderBootstrap(readMigrations()), + render(readMigrations(), fs.readFileSync(OVERLAY_PATH, 'utf8')), 'run: node supabase/byod/generate.mjs', ); }); diff --git a/supabase/byod/storage-overlay.sql b/supabase/byod/storage-overlay.sql new file mode 100644 index 000000000..4ae8b469a --- /dev/null +++ b/supabase/byod/storage-overlay.sql @@ -0,0 +1,39 @@ +-- LoreKit BYOD storage overlay +-- +-- Apply to a BYOD STORAGE database only — the separate Supabase project that +-- holds memory rows — after the schema, and again after every +-- `supabase db push` upgrade. Idempotent. bootstrap.sql already ends with it. +-- +-- psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -f supabase/byod/storage-overlay.sql +-- +-- Why: in BYOD, users and API tokens live in the project that runs the `mcp` +-- function; the storage project's auth.users never holds them. Every foreign +-- key from a public table into auth.users therefore rejects legitimate rows — +-- memory_write fails with memories_user_id_fkey for every lk_* token. Dropping +-- those keys is the overlay's only change. The ON DELETE CASCADE / SET NULL +-- they carried pointed at users this database never has; deleting a user in +-- the function's project does not remove their rows here. +-- +-- Never apply it to the project that RUNS the mcp function (hosted or +-- self-hosted): there auth.users is the real user table and the cascades +-- matter. +-- +-- The keys are found by query, not by name, so a migration that adds another +-- user foreign key is covered by re-running this after the upgrade. +do $overlay$ +declare + fk record; +begin + for fk in + select con.conrelid::regclass as tbl, con.conname + from pg_constraint con + join pg_class c on c.oid = con.conrelid + join pg_namespace n on n.oid = c.relnamespace + where con.contype = 'f' + and con.confrelid = 'auth.users'::regclass + and n.nspname = 'public' + loop + execute format('alter table %s drop constraint %I', fk.tbl, fk.conname); + end loop; +end +$overlay$; diff --git a/supabase/functions/mcp/mcp-handler.ts b/supabase/functions/mcp/mcp-handler.ts index 126f31a59..8320c484b 100644 --- a/supabase/functions/mcp/mcp-handler.ts +++ b/supabase/functions/mcp/mcp-handler.ts @@ -3,9 +3,9 @@ * Handles initialize, tools/list, and tools/call. */ -import { type AuthContext, getDb, canWrite, canRead, getUserId, keyRestriction } from './auth.ts'; +import { type AuthContext, getDb, canWrite, canRead, keyRestriction } from './auth.ts'; import { scopeAllowedByKey, KeyScopeDeniedError } from '../_shared/schemas/api-key.ts'; -import { type StorageAdapter } from './storage-adapter.ts'; +import { type StorageAdapter, storageUserId } from './storage-adapter.ts'; import { UserInputError, safeValidateScope, parseMemoryRefs } from '../_shared/scope/scope.ts'; import { scopeTypeAttribute } from '../_shared/scope/scope-type-attribute.ts'; import { OrgPermissionError, UnknownOrgError } from './org-permissions.ts'; @@ -332,22 +332,17 @@ export async function handleMcp(req: Request, auth: AuthContext, span: Span, ada // own Supabase project. For hosted mode, derive a correctly-scoped client // from the auth context (JWT → user RLS, api_key → service-role). // - // LIMITATION — BYOD + JWT auth: when BYOD mode is active the adapter.db - // client is built at startup with the LOREKIT_STORAGE_SERVICE_KEY (or the - // anon key as fallback). A BYOD caller that authenticates via a Supabase JWT - // does NOT get a per-user-scoped client — the JWT is consumed by *this* - // hosted function for rate-limiting / auth, but the downstream BYOD database - // receives the service-key (or anon-key) client, not a JWT-forwarded one. - // RLS on the BYOD database therefore runs as service role (all rows visible) - // or anon (policy-dependent), never as the individual JWT user. - // BYOD users must rely on application-level data isolation in their own - // Supabase project. If per-user RLS is required, configure the BYOD database - // to treat service-role writes as trusted and enforce isolation via a - // user_id column rather than auth.uid()-based policies. + // BYOD + JWT auth: adapter.db is built from the storage project's SERVICE + // key and never carries the caller's JWT, so RLS on the storage project + // cannot scope a JWT caller. `storageUserId` scopes it in app code by its + // resolved id instead — the path an api_key caller already takes — so it + // neither reads other users' rows nor writes ownerless ones. const db = adapter.mode === 'byod' ? adapter.db : getDb(auth); - // toolUserId: null for JWT auth (RLS handles scoping), api_key userId otherwise. - // analyticsUserId: the resolved user ID for usage-event annotation regardless of auth type. - const toolUserId = getUserId(auth); + // toolUserId: who the tool scopes reads and stamps writes by — the api_key + // userId; null for a hosted JWT caller (RLS scopes it); the JWT caller's own + // id in BYOD mode (see above). analyticsUserId: the resolved user ID for + // usage-event annotation regardless of auth type. + const toolUserId = storageUserId(adapter.mode, auth); const analyticsUserId = toolUserId ?? (auth.type === 'user' ? auth.userId : null) ?? null; // Resolve plan name for usage-event annotation (fails open — null → 'free'). @@ -411,7 +406,8 @@ export async function handleMcp(req: Request, auth: AuthContext, span: Span, ada result = await RETENTION_TOOLS[toolName as keyof typeof RETENTION_TOOLS](db, toolArgs, analyticsUserId, toolSpan, keyRestriction(auth)); } else { // memory.* tools: (db, args, toolUserId, span, keyScoping, correlationId) - // toolUserId is null for JWT auth — RLS handles scoping on the DB side. + // toolUserId is null for hosted JWT auth — RLS handles scoping on the DB + // side — and the caller's own id in BYOD mode (`storageUserId`). // // The trailing `correlationId` is read by `memory.write` alone, for its // `cited` array (00107): a citation joins to the run `usage_events` diff --git a/supabase/functions/mcp/storage-adapter.ts b/supabase/functions/mcp/storage-adapter.ts index 8b96bc542..38be07aa6 100644 --- a/supabase/functions/mcp/storage-adapter.ts +++ b/supabase/functions/mcp/storage-adapter.ts @@ -77,14 +77,44 @@ export function resolveStorageAdapter(jwt?: string): StorageAdapter { const storageServiceKey = Deno.env.get('LOREKIT_STORAGE_SERVICE_KEY'); if (storageUrl) { - if (!storageAnonKey) { + // The service key is required, not optional: the storage project's read + // policies need auth.uid(), which an API key never carries, so an anon-key + // client writes (memory_write is SECURITY DEFINER) but reads back nothing. + // Fail at configuration rather than return empty reads that look healthy. + if (!storageServiceKey) { throw new Error( - 'LOREKIT_STORAGE_URL is set but LOREKIT_STORAGE_ANON_KEY is missing. ' + - 'Both variables are required to use a custom database.', + 'LOREKIT_STORAGE_URL is set but LOREKIT_STORAGE_SERVICE_KEY is missing. ' + + 'BYOD needs the storage project\'s service-role key: with only the anon key every read returns no rows.', ); } - return createBYODAdapter(storageUrl, storageAnonKey, storageServiceKey); + return createBYODAdapter(storageUrl, storageAnonKey ?? '', storageServiceKey); } return createHostedAdapter(jwt); } + +/** + * The user id a tool call filters its reads and stamps its writes with. + * + * Hosted: an `api_key` caller runs on the service-role client, so it is scoped + * in app code by its id; a JWT (`user`) caller gets a client carrying its own + * JWT, so RLS scopes it and the id stays null; `service` sees everything. + * + * BYOD: the storage client is built from the storage project's SERVICE key and + * never carries the caller's JWT, so RLS cannot scope a JWT caller there. Left + * at null it would read every row in the storage project and write rows with + * no owner. It is scoped in app code by its resolved id instead — the same path + * an `api_key` caller takes. A JWT caller with no resolved id is refused rather + * than allowed through unscoped. + */ +export function storageUserId( + mode: StorageAdapter['mode'], + auth: { type: 'user' | 'service' | 'api_key'; userId?: string | null }, +): string | null { + if (auth.type === 'api_key') return auth.userId ?? null; + if (auth.type === 'user' && mode === 'byod') { + if (!auth.userId) throw new Error('BYOD: the session has no resolved user id, so its reads cannot be scoped.'); + return auth.userId; + } + return null; +} From 59af8778d85c85bb7f5520502004ff62df44ba14 Mon Sep 17 00:00:00 2001 From: "mads.thines" Date: Fri, 16 Oct 2026 13:55:30 +0000 Subject: [PATCH 06/16] docs(tasks): replace the SQLite draft with a Docker self-host proposal Teams are a requirement, and team features (orgs, invites, tokens, RLS) live in ~50 Postgres RPCs, so SQLite would mean a second implementation. Propose one compose bundle of Supabase's open-source services plus the dashboard, scaffolded by a small lorekit selfhost init, with the CLI size measurements and the work items a proof of concept must settle. Co-authored-by: dash0-dev[bot] <257284812+dash0-dev[bot]@users.noreply.github.com> --- docs/tasks/local-sqlite-mode.md | 158 ------------------ .../tasks/self-host-without-supabase-cloud.md | 126 ++++++++++++++ 2 files changed, 126 insertions(+), 158 deletions(-) delete mode 100644 docs/tasks/local-sqlite-mode.md create mode 100644 docs/tasks/self-host-without-supabase-cloud.md diff --git a/docs/tasks/local-sqlite-mode.md b/docs/tasks/local-sqlite-mode.md deleted file mode 100644 index 185714ac5..000000000 --- a/docs/tasks/local-sqlite-mode.md +++ /dev/null @@ -1,158 +0,0 @@ -# Proposal: local mode on SQLite — dashboard and usage data without Supabase - -> Status: **proposal / not started** · Area: CLI, web, REST contract · Size: **L** (phased) -> Companion to the BYOD fix that generated `supabase/byod/bootstrap.sql` from the -> migrations. Nothing in this document is built yet; it is here to be argued with -> before any code lands. - -## Problem - -There are two ways to run LoreKit today, and neither gives one person a -dashboard without Supabase: - -- **Offline store** — markdown files under `~/.lorekit` and `/.lorekit` - (`packages/cli/src/store/local.mjs`), served to agents by the local stdio MCP - server `lorekit mcp`. No account, no network — but no dashboard, no read - counters and no usage events, so `/insights` has nothing to show. -- **Hosted or self-hosted** — the dashboard, REST API and MCP server all sit on - Supabase: Postgres, Supabase Auth and Edge Functions. Self-hosting means your - own Supabase and Vercel projects ([install.md](../install.md)); BYOD only moves - memory rows to a second Supabase project ([byod.md](../byod.md)). - -The goal: run LoreKit on one machine, browse lore in the dashboard and read -usage data, with no Supabase — for local development and single-user -self-hosting. - -## What makes this hard - -The server logic lives in Postgres, not in TypeScript: - -| Fact (measured on `main` at 00113) | Consequence for SQLite | -|---|---| -| 108 migrations define 87 SQL functions; the edge functions make string-literal `.rpc('…')` calls to **48** distinct ones | Each RPC a route needs must be re-implemented | -| The Explorer's routes rest on ~1,200 lines of PL/pgSQL (`lorekit_memory_list` 206, `_facets` 218, `_activity` 136, `memory_write` 245, `memory_delete` 157, …); Insights on ~690 more (`lorekit_usage_stats`, `_usage_runs`, `_read_activity`, `_memory_pivot` 158, `_utility_*`, …) | That is the size of a second implementation | -| `lorekit_memory_list`, `_facets`, `_activity` and `_pivot` were all redefined in the newest migration (00113) | A second implementation drifts at that rate unless something forces parity | -| Full-text search is `tsvector` + `websearch_to_tsquery`; graded relevance is `ts_rank_cd` by decision | SQLite FTS5 ranks with `bm25` — ordering will differ, and must be documented as a local-mode difference, not relitigated for hosted | -| Labels are `text[]` with containment/overlap; RLS and `auth.uid()` scope every read | JSON arrays + `json_each`; single user, so tenant scoping collapses to one fixed id | -| Edge handlers speak PostgREST's query builder and `.rpc()`, not a repository interface | There is no seam to swap a SQLite client in behind the existing handlers | -| The dashboard reads memories and analytics through the REST API (`packages/web/src/lib/api/`, base URL derived from `NEXT_PUBLIC_SUPABASE_URL` in `rest.ts`), but signs in with Supabase Auth (`middleware.ts`, `@supabase/ssr`) and still has about **40** direct supabase-js call sites — mostly orgs, invites, API tokens, audit log, scope bindings and GitHub installations, but also direct memory reads in onboarding (`lib/onboarding-server.ts`), the plan-usage figure (`lib/plan.ts`, `lorekit_memory_count`), org and scope-binding counts | The REST contract is the seam; the org/token/audit surfaces stay hosted-only, and the onboarding and plan reads need REST equivalents first | - -The [BYOD drift](../byod.md) is the cautionary example: a hand-written second -copy of the schema fell 42 RPCs and a `memory_write` signature behind before -anyone noticed. Full SQLite parity would be that, at the scale of the whole -server. So this proposal does **not** port the server. It implements the REST -contract the dashboard already consumes, for one user, and holds it to the -Postgres implementation with shared contract tests. - -## Proposal - -```text -agent ──stdio──▶ lorekit mcp ──────────┐ -CLI / hooks ───────────────────────────┼──▶ SQLite store (~/.lorekit/lorekit.db) -dashboard ──HTTP──▶ lorekit serve ─────┘ node:sqlite, FTS5 - (REST subset of @lorekit/schemas) -``` - -### 1. A SQLite store behind the existing store contract - -`createSqliteStore(path)` beside `createLocalStore`/`createRemoteStore` -(`packages/cli/src/store/`), implementing the same contract, so `lorekit mcp`, -the hook engine and the read commands work unchanged with `--store sqlite`. - -- **Engine: `node:sqlite`**, built into Node — no npm dependency, so the CLI - stays zero-dependency. Verified in a sandbox on Node 24.21 (SQLite 3.53.4): - FTS5 with the `porter unicode61` tokenizer (`retry` matches `retried`) and - JSON1 both work. It needs Node 22.13+, where the module no longer sits behind - a flag; the CLI declares `node >=18`, so the SQLite store is opt-in and fails - with a clear message on older Node instead of raising the floor for everyone. -- **Schema**: one `memories` table carrying the columns `@lorekit/schemas` - responses need (scope, key, value, labels, kind, host, origin, TTL, archive, - protection, seen/read/opened counters, timestamps), an FTS5 external-content - table kept in sync by triggers, `usage_events`, and the read ledger the - read-activity view aggregates. - -| Postgres | SQLite | -|---|---| -| `uuid`, `timestamptz` | `TEXT` (UUID string; ISO-8601 UTC, so text order is time order) | -| `text[]` + `@>` / `&&` | JSON array + `json_each` | -| `tsvector`, `websearch_to_tsquery`, `ts_rank_cd` | FTS5 `MATCH`, `bm25()` | -| Partial unique indexes per ownership kind | Partial unique index on `(scope, key)` among active rows — one owner | -| `updated_at` and memory-cap triggers | SQLite triggers | -| `pg_cron` purges (expired, archived, old usage events) | Run in-process on start and hourly | -| RLS, `auth.uid()`, orgs | Not applicable — single local user | - -`lorekit migrate --to sqlite` imports the markdown store. Markdown stays the -default offline store; SQLite is what you switch to when you want the dashboard. - -### 2. `lorekit serve` — the REST subset, locally - -A Node HTTP server that answers the routes the dashboard calls, with the -request and response shapes from `@lorekit/schemas`, against the SQLite store. - -| Phase | Routes | Unlocks | -|---|---|---| -| Explorer | `POST /memories/list`, `GET/PATCH/DELETE /memories/:id`, `POST /memories`, `/memories/facets`, `/memories/scopes`, `/memories/activity`, `/memories/restore`, `/memories/protect` | Browse, filter, search, edit, archive | -| Insights | `/memories/usage`, `/memories/usage/runs`, `/memories/read-activity`, `/memories/read-ranking`, `/memories/utility`, `/memories/pivot`, `/memories/clusters` | Usage data | -| Later | `/memories/policies`, `/memories/groom/*`, `/memories/purge` | Grooming | - -Every other route answers `501` with a stable `local_mode_unsupported` error -code that the dashboard renders as "not available in local mode". - -Usage data comes from the same places as hosted: `lorekit mcp` and `serve` -record a `usage_events` row per operation, using the closed `tool_name` -vocabulary in `packages/mcp-core/src/telemetry/telemetry-vocabulary.ts`, and -bump read counters on reads — so `/insights` renders from local activity. - -### 3. Dashboard local mode - -- `NEXT_PUBLIC_LOREKIT_API_URL` overrides the REST base URL that `rest.ts` - derives from `NEXT_PUBLIC_SUPABASE_URL` today. -- `LOREKIT_LOCAL_MODE=1` makes `middleware.ts` skip the Supabase session and - use a fixed local identity, and hides the navigation entries backed by direct - supabase-js calls (orgs, invites, API tokens, audit log, scope bindings, - GitHub installations). -- The direct memory reads that are not behind a hidden entry — onboarding's - memory count and the plan-usage figure — move onto REST routes first, in the - hosted dashboard too. That is the existing rule ("a capability the dashboard - needs becomes part of the public contract"), applied ahead of local mode. -- Run it as `lorekit serve` plus the dashboard's dev server for development. - How to ship it to self-hosters is open question 2. - -### 4. Keeping two backends honest - -The REST contract is the single source of truth; both backends are tested -against it. One fixture suite — seed rows, request, expected response — runs -against the Postgres handlers (the existing `memories-api.integration.spec.ts` -stack) and against `serve`. A route whose two answers disagree fails CI. That is -the mechanism that stops the BYOD-style drift; without it, this proposal -should not ship. - -## Phases - -1. SQLite store + `lorekit mcp --store sqlite` + `migrate --to sqlite`. Usable - alone: agents on SQLite, no dashboard yet. -2. `lorekit serve` with the Explorer routes, dashboard local mode, the shared - contract suite. -3. Usage events, read counters and the Insights routes. -4. Optional: grooming routes. - -Each phase is its own PR, and each updates the public docs and `llms.txt` -(the repo's docs rule). - -## Open questions - -1. **Where `serve` lives.** The CLI is zero-dependency and cannot import - `@lorekit/schemas` (zod). Either a new package (`@lorekit/local`) that the - CLI delegates to when installed, or generated plain-JS validators committed - into the CLI the way `surfaces.generated.mjs` is. -2. **How self-hosters get the dashboard.** A prebuilt Next.js standalone bundle - inside an npm package is heavy; a container image is lighter to run but adds - a release artifact; the dev server is fine only for contributors. -3. **Which Insights instruments are in scope.** `utility`, `pivot` and - `clusters` are the most SQL-heavy; they could follow the core three. -4. **Teams.** SQLite is one writer on one machine. For a team self-hosting - without Supabase Cloud, Postgres via self-hosted Supabase (Docker) keeps - today's schema and RPCs untouched — worth evaluating separately rather than - stretching SQLite to multi-user. -5. **Node floor.** Opt-in on Node 22.13+, or raise `engines` when the SQLite - store becomes the offline default. diff --git a/docs/tasks/self-host-without-supabase-cloud.md b/docs/tasks/self-host-without-supabase-cloud.md new file mode 100644 index 000000000..4e59268f1 --- /dev/null +++ b/docs/tasks/self-host-without-supabase-cloud.md @@ -0,0 +1,126 @@ +# Proposal: self-host LoreKit without Supabase Cloud — one Docker bundle for one person or a team + +> Status: **proposal / not started** · Area: deployment, web, CLI · Size: **M–L** (phased) +> Replaces this PR's earlier "local mode on SQLite" draft. It rests on three +> maintainer decisions: teams must be supported; anything big stays out of the +> CLI; the distribution should be the easiest and most flexible one. + +## Problem + +Seeing the dashboard and usage data today requires Supabase Cloud — the hosted +instance, or your own Supabase and Vercel projects ([install.md](../install.md)). +The offline markdown store needs no account but has no dashboard, and BYOD +([byod.md](../byod.md)) moves only memory rows. The goal: the dashboard and +usage data on infrastructure you run, with no Supabase account, for one person +on a laptop or a team on a server. + +## Decision 1 — Postgres, not SQLite + +Team features are implemented in Postgres, not TypeScript: + +| Fact (measured on `main` at 00113) | Consequence | +|---|---| +| 108 migrations define 87 SQL functions; the edge functions make string-literal `.rpc('…')` calls to 48 of them, the org, invite, token and retention RPCs among them | A SQLite backend re-implements each one a route needs | +| The Explorer's RPCs alone are ~1,200 lines of PL/pgSQL, Insights ~690 more | That is the size of the second implementation before any team feature | +| `lorekit_memory_list`, `_facets`, `_activity` and `_pivot` were all redefined in the newest migration | A second implementation drifts at that rate | +| Org visibility is one `SECURITY DEFINER` predicate (`lorekit_member_org_ids`) shared by RLS and the edge | Porting it means porting RLS semantics too | + +BYOD's hand-written copy of the schema is what that drift looks like: it fell +42 RPCs and a `memory_write` signature behind before anyone noticed. Postgres +keeps one implementation for hosted, self-hosted, team and solo use. + +## Decision 2 — Run Supabase's open-source services, not Supabase Cloud + +LoreKit's server side is four open-source components, each published as a +Docker image: Postgres (`supabase/postgres`, which ships pg_cron, pgvector and +pg_trgm), Auth (GoTrue), PostgREST, and Edge Functions (`supabase/edge-runtime`, +Deno). Kong routes `/auth/v1`, `/rest/v1` and `/functions/v1` on one origin, the +shape the dashboard (`rest.ts` derives `${url}/functions/v1`) and the CLI +already expect. The server code does not change, and the migrations apply the +way they do in CI. + +Supabase's full self-host stack adds Studio, Realtime, Storage, imgproxy and +analytics; LoreKit uses none of them, so the bundle leaves them out. + +## Decision 3 — Distribution: one compose bundle, scaffolded by the CLI + +| Option | Footprint | Teams | Verdict | +|---|---|---|---| +| **Compose bundle with prebuilt images** (GHCR) | Images pulled on first run; CLI unchanged except a small scaffold command | Yes | **Recommended** | +| Everything in one npm package | Needs the GoTrue, PostgREST and Deno binaries; the dashboard alone is a Next.js server | Yes | Not practical without containers | +| SQLite server inside the CLI | ~+100 KB on a 385 KB package; 0 ms startup cost if loaded only by its command | No | Single user only, and a second implementation | +| PGlite (Postgres in WASM) server | 34 MB unpacked — about 28× the CLI, so a separate package | No | Reuses the SQL, but PostgREST and the handlers would still need a port | + +Measured for the CLI-size question: `@lorekit/cli` is 385 KB packed, 1.2 MB +unpacked, and starts in ~130 ms (bare Node: ~45 ms) because `commands.mjs` +imports all 27 commands eagerly. `node:sqlite` is built into Node and costs +nothing to load. Under the recommended plan nothing big enters the CLI. + +The CLI's part is scaffolding: + +- `lorekit selfhost init [dir]` writes `compose.yaml` and a `.env` with + generated secrets (JWT secret, anon and service keys, Postgres password) — a + template plus a short command. +- `lorekit install --endpoint http://:8000/functions/v1/mcp` already + points agents at the result; `LOREKIT_APP_URL` already points deep links at + a self-hosted dashboard. + +## The bundle + +| Service | Image | Job | +|---|---|---| +| `db` | `supabase/postgres` | Data, in a volume | +| `migrate` | LoreKit image | One-shot: applies `supabase/migrations/` on first run and on upgrade | +| `auth` | `supabase/gotrue` | Email + password (auto-confirm, so no SMTP needed); GitHub OAuth optional | +| `rest` | `postgrest/postgrest` | PostgREST, which the edge functions and dashboard call | +| `functions` | LoreKit image on `supabase/edge-runtime` | `mcp`, `memories`, `orgs`, … with their `npm:` dependencies pre-cached | +| `kong` | `kong` | One origin (`:8000`) for auth, rest and functions | +| `web` | LoreKit image | The dashboard, a Next.js standalone build | + +- **One person:** `lorekit selfhost init && docker compose up -d` on a laptop. +- **A team:** the same on a VM behind TLS, with `SITE_URL` and `API_URL` set; + teammates sign up and are invited through the existing org feature. +- **Upgrades:** pull the new images; `migrate` applies new migrations. +- **Backups:** `pg_dump` of the `db` volume. + +## Work items the proof of concept must settle + +1. **Dashboard runtime config.** `NEXT_PUBLIC_SUPABASE_URL` (17 reads) and + `NEXT_PUBLIC_SUPABASE_ANON_KEY` are inlined at build time, so a prebuilt + image is tied to one backend. Serve them as runtime config through + `lib/supabase/config.ts`, already the single resolver, and add + `output: 'standalone'` to `next.config.ts`. +2. **Edge functions offline.** Their `npm:` imports resolve when the function + boots; bake a Deno cache into the image, and confirm `EdgeRuntime.waitUntil` + (used for token touch-ups and telemetry flushes) behaves under edge-runtime. +3. **Vercel assumptions.** `NEXT_PUBLIC_VERCEL_ENV` (13 reads) feeds the + deployment environment; self-host needs an explicit override, as the edge + already has. +4. **Optional telemetry.** With no Dash0 endpoint configured, exporters stay off. +5. **CI.** A job that boots the bundle and runs `packages/smoke-tests` against + it. Adding it needs a workflow edit by a maintainer. + +## Phases + +1. Compose bundle built from source; the smoke tests pass against it. +2. Runtime config, images on GHCR, `lorekit selfhost init`, and an install.md + Docker path (README "Run your own instance" too). +3. External managed Postgres and a Helm chart, if asked for. + +A Docker-free single-user mode (SQLite in the CLI, loaded only by its command) +stays possible later; it is not needed for this goal. + +## Relationship to BYOD + +BYOD remains the option for keeping memory rows in a separate Supabase project +while using a LoreKit `mcp` function — MCP tools only. The bundle is the path +for running the dashboard and usage views on your own infrastructure. + +## Open questions + +1. **Supabase software, no Supabase account — acceptable?** If the goal is no + Supabase components at all, the alternative is a LoreKit Node server on + plain Postgres that reuses the SQL but replaces GoTrue, PostgREST and the + edge runtime: weeks, not days. +2. **Registry.** GHCR under `mthines/lorekit`? +3. **Default sign-in.** Email + password only, or GitHub OAuth required? From da441cc2931a13151d936e96ea1f27e5cfc6b4b9 Mon Sep 17 00:00:00 2001 From: mthines Date: Sat, 17 Oct 2026 09:05:32 +0200 Subject: [PATCH 07/16] fix(mcp): return a BYOD storage misconfiguration in-band, not as a bare 500 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit resolveStorageAdapter() throws when LOREKIT_STORAGE_URL is set without LOREKIT_STORAGE_SERVICE_KEY. Called bare, the throw escaped Deno.serve, so every MCP call (initialize and tools/list included) got a bodyless "500 Internal Server Error" and the reason reached only the function log, although docs/byod.md promises an error naming the missing variable. Catch it and answer with jsonrpcError(-32603), the in-band shape the auth failure above already uses; mark the root span ERROR (error.type storage_config). Only authenticated callers reach it; the message names env vars, never values. A source-scan guard in mcp-guards/ pins it. local-verify F-1 (AC-6): AC6-nosvc-* probes returned HTTP 500 "Internal Server Error"; after the fix the same probes return {"code":-32603,"message":"LOREKIT_STORAGE_URL is set but LOREKIT_STORAGE_SERVICE_KEY is missing. …"}, unauthenticated calls still get -32001, and BYOD reads/writes with the service key are unchanged. --- .../mcp-storage-config-error.spec.ts | 45 +++++++++++++++++++ supabase/functions/mcp/index.ts | 15 ++++++- 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 packages/mcp-core/src/mcp-guards/mcp-storage-config-error.spec.ts diff --git a/packages/mcp-core/src/mcp-guards/mcp-storage-config-error.spec.ts b/packages/mcp-core/src/mcp-guards/mcp-storage-config-error.spec.ts new file mode 100644 index 000000000..c34e583a2 --- /dev/null +++ b/packages/mcp-core/src/mcp-guards/mcp-storage-config-error.spec.ts @@ -0,0 +1,45 @@ +import { describe, it, expect } from 'vitest'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import path from 'node:path'; + +/** + * Source-scan guard for the MCP transport (`supabase/functions/mcp/index.ts`), + * the `mcp-preamble-usage.spec.ts` pattern: the Deno edge tree has no test + * harness, so this property is asserted from here or not at all. + * + * `resolveStorageAdapter()` throws on a BYOD misconfiguration (a storage URL + * without the service key). Called bare, the throw escapes `Deno.serve` and + * every MCP call — `initialize` included — gets a bodyless `500 Internal Server + * Error`; the reason reaches only the function log. docs/byod.md promises the + * caller an error naming the missing variable, so the call must sit in a + * try/catch that answers in-band with `jsonrpcError`. + */ + +const here = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.resolve(here, '../../../..'); +const source = readFileSync(path.join(repoRoot, 'supabase', 'functions', 'mcp', 'index.ts'), 'utf8'); + +const COMMENT_PREFIXES = ['//', '*', '/*', '*/']; +const executable = source + .split('\n') + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .filter((line) => !COMMENT_PREFIXES.some((prefix) => line.startsWith(prefix))) + .join('\n'); + +describe('mcp transport — storage configuration errors', () => { + it('resolves the storage adapter inside a try block', () => { + expect(executable).toMatch(/try\s*\{\s*adapter\s*=\s*resolveStorageAdapter\(\);\s*\}\s*catch/); + }); + + it('answers a configuration error in-band with jsonrpcError, not a bare 500', () => { + const catchBlock = /adapter\s*=\s*resolveStorageAdapter\(\);\s*\}\s*catch\s*\([^)]*\)\s*\{([\s\S]*?)\n\}/.exec(executable); + expect(catchBlock?.[1]).toMatch(/return\s+jsonrpcError\(/); + }); + + it('never calls resolveStorageAdapter outside that guarded assignment', () => { + const calls = executable.match(/resolveStorageAdapter\(\)/g) ?? []; + expect(calls).toHaveLength(1); + }); +}); diff --git a/supabase/functions/mcp/index.ts b/supabase/functions/mcp/index.ts index 4ac4f6edb..19da59b70 100644 --- a/supabase/functions/mcp/index.ts +++ b/supabase/functions/mcp/index.ts @@ -169,7 +169,20 @@ Deno.serve(async (req: Request) => { ...(auth.userId ? { 'auth.user_id': auth.userId } : {}), }); - const adapter = resolveStorageAdapter(); + // A BYOD misconfiguration (e.g. LOREKIT_STORAGE_URL without the service + // key) throws here. Uncaught, the runtime answers every MCP call with a + // bare `500 Internal Server Error`, so the operator never sees the reason. + // Deliver it in-band like the auth failure above; the message names env + // vars, never their values, and only authenticated callers reach it. + let adapter: ReturnType; + try { + adapter = resolveStorageAdapter(); + } catch (err) { + const message = (err as Error).message; + console.error(`[mcp] storage configuration error: ${message}`); + span.error(message, 'storage_config'); + return jsonrpcError(await peekRequestId(req), -32603, message); + } // Per-user request rate limit — transport layer, all MCP methods. // Service-role (CI/internal) is exempt; unauthenticated requests never From 7bdf0e0d29e8072575976b6770fd12f7de3543da Mon Sep 17 00:00:00 2001 From: mthines Date: Sat, 17 Oct 2026 09:05:33 +0200 Subject: [PATCH 08/16] docs(byod): call the storage service key required, not optional MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 'How BYOD is wired' summary still listed LOREKIT_STORAGE_SERVICE_KEY as optional while §3 (and resolveStorageAdapter) require it; the anon key is the optional one. local-verify F-2: docs/byod.md:15-16 contradicted docs/byod.md:94 and the AC-6 probe, which refuses a storage URL without the service key. --- docs/byod.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/byod.md b/docs/byod.md index 7ad885918..ea54e0eb8 100644 --- a/docs/byod.md +++ b/docs/byod.md @@ -12,8 +12,8 @@ the part most likely to surprise you. ## How BYOD is wired - **It needs your own deployment of the `mcp` function.** The function reads - `LOREKIT_STORAGE_URL`, `LOREKIT_STORAGE_ANON_KEY` and the optional - `LOREKIT_STORAGE_SERVICE_KEY` from its **own Edge Function secrets**. The + `LOREKIT_STORAGE_URL`, the required `LOREKIT_STORAGE_SERVICE_KEY` and the + optional `LOREKIT_STORAGE_ANON_KEY` from its **own Edge Function secrets**. The hosted endpoint (`https://pqokxlhvnosogizsjztg.supabase.co/functions/v1/mcp`) has no BYOD storage configured, and an `env` block in your `.mcp.json` does not reach it — those variables only reach the local `mcp-remote` process on From 1c85def7bea22154e18c709709c488c0137358bc Mon Sep 17 00:00:00 2001 From: mthines Date: Sat, 17 Oct 2026 09:09:36 +0200 Subject: [PATCH 09/16] fix(mcp): stamp hosted session-JWT tool calls with the caller's id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A hosted MCP call authenticated by a Supabase session JWT ran with toolUserId = null on the assumption that RLS scopes it. True for reads, not for writes: memory_write reads p_user_id = null as an ownerless row, so the caller's own memory.list never returned it, and recordAudit then failed RLS on audit_log. The same null made memory.purge, policy.*, groom.* and memory.protect refuse session callers ("requires a user_id"). storageUserId now returns the resolved id for a JWT caller in both modes, the value the REST create handler already passes (auth.userId). RPCs that take p_user_id still pin non-service callers to auth.uid(), so the id cannot widen access. A hosted session with no resolved id still falls back to RLS; BYOD still refuses it. local-verify F-3 (EC-13): before, EC13-hosted-jwt-write-B stored user_id NULL, the caller's list was empty, and the log showed "[recordAudit] insert failed … row-level security policy for table audit_log". After: the row is owned by the caller, the audit row is written, read/list return it, user A cannot read it, policy.list answers, and every probe window is free of errors. --- packages/mcp-core/src/db/storage-adapter.spec.ts | 8 ++++++-- packages/mcp-core/src/db/storage-adapter.ts | 15 ++++++++++----- supabase/functions/mcp/mcp-handler.ts | 12 ++++++------ supabase/functions/mcp/storage-adapter.ts | 15 ++++++++++----- 4 files changed, 32 insertions(+), 18 deletions(-) diff --git a/packages/mcp-core/src/db/storage-adapter.spec.ts b/packages/mcp-core/src/db/storage-adapter.spec.ts index 1c1db919e..fe362f95e 100644 --- a/packages/mcp-core/src/db/storage-adapter.spec.ts +++ b/packages/mcp-core/src/db/storage-adapter.spec.ts @@ -146,8 +146,12 @@ describe('storageUserId', () => { expect(storageUserId('byod', { type: 'api_key', userId: USER })).toBe(USER); }); - it('leaves a hosted JWT caller to RLS', () => { - expect(storageUserId('hosted', { type: 'user', userId: USER })).toBeNull(); + it('stamps a hosted JWT caller with its id — memory_write reads p_user_id = null as an ownerless row', () => { + expect(storageUserId('hosted', { type: 'user', userId: USER })).toBe(USER); + }); + + it('leaves a hosted JWT caller with no resolved id to RLS rather than refusing it', () => { + expect(storageUserId('hosted', { type: 'user' })).toBeNull(); }); it('scopes a BYOD JWT caller by its id — the storage client carries no JWT for RLS to read', () => { diff --git a/packages/mcp-core/src/db/storage-adapter.ts b/packages/mcp-core/src/db/storage-adapter.ts index 64075542b..5fd2fb13c 100644 --- a/packages/mcp-core/src/db/storage-adapter.ts +++ b/packages/mcp-core/src/db/storage-adapter.ts @@ -126,8 +126,11 @@ export function resolveStorageAdapter(jwt?: string): StorageAdapter { * The user id a tool call filters its reads and stamps its writes with. * * Hosted: an `api_key` caller runs on the service-role client, so it is scoped - * in app code by its id; a JWT (`user`) caller gets a client carrying its own - * JWT, so RLS scopes it and the id stays null; `service` sees everything. + * in app code by its id. A JWT (`user`) caller gets a client carrying its own + * JWT, so RLS already scopes its reads — but writes still need the id: the + * `memory_write` RPC treats `p_user_id = null` as an ownerless row, which the + * caller could then not read back. So it carries its id too, as the REST + * create handler does (`auth.userId`). `service` sees everything. * * BYOD: the storage client is built from the storage project's SERVICE key and * never carries the caller's JWT, so RLS cannot scope a JWT caller there. Left @@ -141,9 +144,11 @@ export function storageUserId( auth: { type: 'user' | 'service' | 'api_key'; userId?: string | null }, ): string | null { if (auth.type === 'api_key') return auth.userId ?? null; - if (auth.type === 'user' && mode === 'byod') { - if (!auth.userId) throw new Error('BYOD: the session has no resolved user id, so its reads cannot be scoped.'); - return auth.userId; + if (auth.type === 'user') { + if (!auth.userId && mode === 'byod') { + throw new Error('BYOD: the session has no resolved user id, so its reads cannot be scoped.'); + } + return auth.userId ?? null; } return null; } diff --git a/supabase/functions/mcp/mcp-handler.ts b/supabase/functions/mcp/mcp-handler.ts index 8320c484b..625bddf6d 100644 --- a/supabase/functions/mcp/mcp-handler.ts +++ b/supabase/functions/mcp/mcp-handler.ts @@ -338,10 +338,10 @@ export async function handleMcp(req: Request, auth: AuthContext, span: Span, ada // resolved id instead — the path an api_key caller already takes — so it // neither reads other users' rows nor writes ownerless ones. const db = adapter.mode === 'byod' ? adapter.db : getDb(auth); - // toolUserId: who the tool scopes reads and stamps writes by — the api_key - // userId; null for a hosted JWT caller (RLS scopes it); the JWT caller's own - // id in BYOD mode (see above). analyticsUserId: the resolved user ID for - // usage-event annotation regardless of auth type. + // toolUserId: who the tool scopes reads and stamps writes by — the caller's + // resolved id for api_key and JWT callers alike (a hosted JWT caller is also + // RLS-scoped; the id is what makes its writes owned), null for service. + // analyticsUserId: the resolved user ID for usage-event annotation. const toolUserId = storageUserId(adapter.mode, auth); const analyticsUserId = toolUserId ?? (auth.type === 'user' ? auth.userId : null) ?? null; @@ -406,8 +406,8 @@ export async function handleMcp(req: Request, auth: AuthContext, span: Span, ada result = await RETENTION_TOOLS[toolName as keyof typeof RETENTION_TOOLS](db, toolArgs, analyticsUserId, toolSpan, keyRestriction(auth)); } else { // memory.* tools: (db, args, toolUserId, span, keyScoping, correlationId) - // toolUserId is null for hosted JWT auth — RLS handles scoping on the DB - // side — and the caller's own id in BYOD mode (`storageUserId`). + // toolUserId is the caller's own id for api_key and JWT auth, null for + // service (`storageUserId`). // // The trailing `correlationId` is read by `memory.write` alone, for its // `cited` array (00107): a citation joins to the run `usage_events` diff --git a/supabase/functions/mcp/storage-adapter.ts b/supabase/functions/mcp/storage-adapter.ts index 38be07aa6..4bfcf7097 100644 --- a/supabase/functions/mcp/storage-adapter.ts +++ b/supabase/functions/mcp/storage-adapter.ts @@ -97,8 +97,11 @@ export function resolveStorageAdapter(jwt?: string): StorageAdapter { * The user id a tool call filters its reads and stamps its writes with. * * Hosted: an `api_key` caller runs on the service-role client, so it is scoped - * in app code by its id; a JWT (`user`) caller gets a client carrying its own - * JWT, so RLS scopes it and the id stays null; `service` sees everything. + * in app code by its id. A JWT (`user`) caller gets a client carrying its own + * JWT, so RLS already scopes its reads — but writes still need the id: the + * `memory_write` RPC treats `p_user_id = null` as an ownerless row, which the + * caller could then not read back. So it carries its id too, as the REST + * create handler does (`auth.userId`). `service` sees everything. * * BYOD: the storage client is built from the storage project's SERVICE key and * never carries the caller's JWT, so RLS cannot scope a JWT caller there. Left @@ -112,9 +115,11 @@ export function storageUserId( auth: { type: 'user' | 'service' | 'api_key'; userId?: string | null }, ): string | null { if (auth.type === 'api_key') return auth.userId ?? null; - if (auth.type === 'user' && mode === 'byod') { - if (!auth.userId) throw new Error('BYOD: the session has no resolved user id, so its reads cannot be scoped.'); - return auth.userId; + if (auth.type === 'user') { + if (!auth.userId && mode === 'byod') { + throw new Error('BYOD: the session has no resolved user id, so its reads cannot be scoped.'); + } + return auth.userId ?? null; } return null; } From 4666d9fd950d05e8155cc600a1bbdc2329253dfe Mon Sep 17 00:00:00 2001 From: mthines Date: Sat, 17 Oct 2026 09:11:31 +0200 Subject: [PATCH 10/16] fix(mcp): redact URLs from server-fault (-32603) tool error messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A transport failure's message carries the request URL, and a PostgREST URL carries every filter value. With the BYOD storage project unreachable, the -32603 sent to the client read "error sending request for url (http:///rest/v1/memories?…&user_id=eq.)". Spans already redact it via Span.error → redactUrls; the response now uses the same helper and keeps the reason ("Connection refused"). local-verify F-4 (EC-7): F4-reverify-write/-list now answer "error sending request for url (): … Connection refused". The 7 s (write) / 14 s (list) latency is postgrest-js's default network retry (3 retries, exponential backoff, per query), shared with hosted mode, and is left as is: disabling it trades transient-error resilience for faster failure on a misconfiguration `lorekit doctor` already diagnoses in milliseconds. --- .../mcp-core/src/mcp-guards/mcp-tool-error-shape.spec.ts | 6 ++++++ supabase/functions/mcp/mcp-handler.ts | 6 +++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/mcp-core/src/mcp-guards/mcp-tool-error-shape.spec.ts b/packages/mcp-core/src/mcp-guards/mcp-tool-error-shape.spec.ts index f623d4d9a..4befa9e1b 100644 --- a/packages/mcp-core/src/mcp-guards/mcp-tool-error-shape.spec.ts +++ b/packages/mcp-core/src/mcp-guards/mcp-tool-error-shape.spec.ts @@ -111,6 +111,12 @@ describe('tool-originated failures use isError', () => { expect(toolCatch).toMatch(/jsonrpcError\(id,\s*-32603/); }); + it('a server fault reaches the client with its URLs redacted', () => { + // A transport failure's message carries the PostgREST URL and with it every + // filter value (`user_id=eq.`); spans redact it, so must the response. + expect(toolCatch).toMatch(/jsonrpcError\(id,\s*-32603,\s*redactUrls\(/); + }); + it('the isError text is the bare message, not the class-qualified span text', () => { // `msg` is `Name: message`, which is useful on a span and noise to a model. expect(toolCatch).toMatch(/text:\s*\(err as Error\)\.message/); diff --git a/supabase/functions/mcp/mcp-handler.ts b/supabase/functions/mcp/mcp-handler.ts index 625bddf6d..b24b3d4f5 100644 --- a/supabase/functions/mcp/mcp-handler.ts +++ b/supabase/functions/mcp/mcp-handler.ts @@ -22,6 +22,7 @@ import { type Params } from './tools.ts'; // the try/catch, the usage events — stays here and stays hand-written. import { MEMORY_TOOLS, ORG_TOOLS, RETENTION_TOOLS, ALL_TOOL_NAMES } from './tool-dispatch.generated.ts'; import { type Span } from '../_shared/telemetry/otel.ts'; +import { redactUrls } from '../_shared/telemetry/span-semconv.ts'; import { LimitError, recordUsageEvent, getUserPlanName } from './limits.ts'; import { toolRequires } from './permissions.ts'; import { isRefusedForScopedKey, accountWideRefusalMessage } from '../_shared/auth/account-wide-tools.ts'; @@ -579,7 +580,10 @@ export async function handleMcp(req: Request, auth: AuthContext, span: Span, ada isError: true, }); } - return jsonrpcError(id, -32603, (err as Error).message); + // A transport failure's message carries the request URL, and a PostgREST + // URL carries every filter value (`user_id=eq.`, `key=eq.`). + // Spans already redact it (`Span.error`); the client message must too. + return jsonrpcError(id, -32603, redactUrls(String((err as Error).message ?? err))); } } From 1bab44285a94214013bb007f8fe95fb8ce4cc327 Mon Sep 17 00:00:00 2001 From: mthines Date: Sat, 17 Oct 2026 09:11:54 +0200 Subject: [PATCH 11/16] docs: a token-permission denial is -32003, not -32001 CLAUDE.md and llms.txt (template.md) said a read-only token on a write, or a write-only token on a read, gets -32001. The edge answers those with JSONRPC_FORBIDDEN (-32003, mcp-handler.ts) since the in-band authz split; -32001 is only for a missing, invalid or rotated token. docs/mcp-tools.md already had it right. llms.txt regenerated with nx generate:llms. local-verify F-5 (EC-3/EC-4): lk_ro write and lk_wo read both returned -32003. --- CLAUDE.md | 2 +- packages/schemas/src/llms/template.md | 4 ++-- packages/web/public/llms.txt | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 3ef9391af..738e24733 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -254,7 +254,7 @@ Single `:` → 400 error. All segments lowercased. See [docs/scope-format.md](./ **Critical:** `api_key` auth uses service-role. ALL queries must `.eq('user_id', userId)`. Write tools require write permission (`lk_rw_*` / `lk_wo_*`); read tools require read permission (`lk_rw_*` / `lk_ro_*`). `lk_ro_*` is denied on write tools; `lk_wo_*` is denied -on read tools — both with the standard `-32001` permission-denied error. The gating logic +on read tools — both with the `-32003` forbidden error (`-32001` is unauthenticated). The gating logic (`READ_TOOLS`/`WRITE_TOOLS`/`toolRequires`/`tokenPrefixFor`) is a shared pure module, `packages/mcp-core/src/auth/permissions.ts`, mirrored self-contained into `supabase/functions/mcp/permissions.ts` (the `limits.ts` pattern). diff --git a/packages/schemas/src/llms/template.md b/packages/schemas/src/llms/template.md index b926a73ca..f8989b108 100644 --- a/packages/schemas/src/llms/template.md +++ b/packages/schemas/src/llms/template.md @@ -392,8 +392,8 @@ these as successes. | JSON-RPC code | Meaning | |---------------|---------| -| `-32001` | Unauthorized — missing, invalid, expired, or wrong-permission-tier token (e.g. read-only token on a write, write-only token on a read) | -| `-32003` | Forbidden — authenticated, but not permitted for this tool (e.g. an `lk_*` token calling an `org.*` tool) | +| `-32001` | Unauthorized — missing, invalid, expired, or rotated token | +| `-32003` | Forbidden — authenticated, but not permitted for this tool: a wrong-permission-tier token (read-only token on a write, write-only token on a read), an account-wide tool on a scoped key, or a scope outside the key's allowlist | | `-32603` | Server fault — a DB outage or unexpected exception. Safe to retry | | `-32700` | Parse error — malformed JSON body | | `-32601` | Unknown tool name | diff --git a/packages/web/public/llms.txt b/packages/web/public/llms.txt index 2fbb18c8a..bc4ef9fcb 100644 --- a/packages/web/public/llms.txt +++ b/packages/web/public/llms.txt @@ -919,8 +919,8 @@ these as successes. | JSON-RPC code | Meaning | |---------------|---------| -| `-32001` | Unauthorized — missing, invalid, expired, or wrong-permission-tier token (e.g. read-only token on a write, write-only token on a read) | -| `-32003` | Forbidden — authenticated, but not permitted for this tool (e.g. an `lk_*` token calling an `org.*` tool) | +| `-32001` | Unauthorized — missing, invalid, expired, or rotated token | +| `-32003` | Forbidden — authenticated, but not permitted for this tool: a wrong-permission-tier token (read-only token on a write, write-only token on a read), an account-wide tool on a scoped key, or a scope outside the key's allowlist | | `-32603` | Server fault — a DB outage or unexpected exception. Safe to retry | | `-32700` | Parse error — malformed JSON body | | `-32601` | Unknown tool name | From ee7cf30fed49ad51614b4c5f84adb94d2e534686 Mon Sep 17 00:00:00 2001 From: mthines Date: Sat, 17 Oct 2026 10:14:46 +0200 Subject: [PATCH 12/16] address review comment: correct stale comments: toolUserId is the JWT caller's id, not null Addresses @mthines's comment: https://github.com/mthines/lorekit/pull/690#discussion_r4302006253 Refs: https://github.com/mthines/lorekit/pull/690 --- supabase/functions/mcp/mcp-handler.ts | 19 +++++++++---------- supabase/functions/mcp/tools.ts | 18 ++++++++++-------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/supabase/functions/mcp/mcp-handler.ts b/supabase/functions/mcp/mcp-handler.ts index b24b3d4f5..90430f229 100644 --- a/supabase/functions/mcp/mcp-handler.ts +++ b/supabase/functions/mcp/mcp-handler.ts @@ -385,10 +385,11 @@ export async function handleMcp(req: Request, auth: AuthContext, span: Span, ada if (isOrgTool) { // org.* tools: (db, args, toolUserId, span) — the same shape as the // memory family, so both maps look alike and this dispatcher threads - // the actor one way. null for a JWT caller (auth.uid() applies inside - // the RPCs); the token owner for an api_key caller, forwarded as - // `p_actor_user_id` and honoured only on a verified service_role - // connection. + // the actor one way. `toolUserId` is the caller's own id for JWT and + // api_key callers alike (null only for service auth). It is forwarded + // as `p_actor_user_id`, which the RPCs honour only on a verified + // service_role connection (00041) — on a JWT connection they ignore it + // and resolve the actor from `auth.uid()`. result = await ORG_TOOLS[toolName as keyof typeof ORG_TOOLS](db, toolArgs, toolUserId, toolSpan); } else if (isRetentionTool) { // policy.*/groom.* tools: (db, args, userId, span, keyScoping) — but @@ -396,12 +397,10 @@ export async function handleMcp(req: Request, auth: AuthContext, span: Span, ada // `analyticsUserId` (the RESOLVED owner), never `toolUserId`. The // underlying lorekit_policy_*/lorekit_groom_* RPCs take `p_user_id` // explicitly and throw a UserInputError on null — there is no - // `auth.uid()` fallback the way the org RPCs have. `toolUserId` is - // null for every JWT caller, so passing it here would break - // policy/groom over MCP for exactly the dashboard-JWT path the REST - // `requireUserId(auth)` helper already resolves correctly. - // `analyticsUserId` is that same resolved owner for both JWT and - // api_key auth. `keyRestriction(auth)` is the same call the memory + // `auth.uid()` fallback the way the org RPCs have. `analyticsUserId` + // is the resolved owner for both JWT and api_key auth — the same id + // the REST `requireUserId(auth)` helper resolves. + // `keyRestriction(auth)` is the same call the memory // family makes below — each retention handler gates the scope it // resolves against it (see `assertScopeAllowed` in `tools.ts`). result = await RETENTION_TOOLS[toolName as keyof typeof RETENTION_TOOLS](db, toolArgs, analyticsUserId, toolSpan, keyRestriction(auth)); diff --git a/supabase/functions/mcp/tools.ts b/supabase/functions/mcp/tools.ts index 12674f677..b6236bcaa 100644 --- a/supabase/functions/mcp/tools.ts +++ b/supabase/functions/mcp/tools.ts @@ -1,9 +1,9 @@ /** * MCP tool handlers — one function per memory.* and org.* tool. * - * SECURITY: When userId is provided (api_key auth), every query MUST include - * .eq('user_id', userId). The service-role client bypasses RLS — without this - * filter, users could access each other's memories. + * SECURITY: When userId is provided (api_key and JWT auth), every query MUST + * include .eq('user_id', userId). The service-role client bypasses RLS — + * without this filter, users could access each other's memories. * * org.* tools accept BOTH auth tiers. A JWT caller resolves inside the SECURITY * DEFINER RPCs from auth.uid() as before; an api_key caller has no auth.uid() @@ -1039,14 +1039,16 @@ export async function toolPurge( // RPCs, which resolve the actor one of two ways: from `auth.uid()` on a JWT // connection, or from the explicit `p_actor_user_id` an api_key caller passes // (honoured only on a verified service_role connection — 00041). So the -// dispatcher resolves the caller and passes `userId`: null for a JWT caller, -// where auth.uid() applies, and the token's owner for an api_key caller. +// dispatcher resolves the caller and passes `userId` — the caller's own id for +// both tiers. The RPCs honour it as `p_actor_user_id` only on a verified +// service_role connection; on a JWT connection they resolve the actor from +// auth.uid(). // // A NULL actor fails closed inside the RPCs rather than defaulting to anyone. /** * Resolve an org's UUID from its slug. Throws if the org does not exist, is - * soft-deleted, or (on the api_key path) the caller is not a member. + * soft-deleted, or (whenever a caller id is passed) the caller is not a member. * * The membership join is NOT redundant with the role check inside the RPCs. On * a JWT connection RLS on `orgs` already restricts this read to the caller's @@ -1062,7 +1064,7 @@ async function resolveOrgId( slug: string, userId: string | null, ): Promise { - // JWT path: unchanged, RLS-scoped. + // No caller id (service auth — JWT and api_key callers pass their own id). if (!userId) { const { data: org, error } = await tracedDb .from('orgs') @@ -1076,7 +1078,7 @@ async function resolveOrgId( return (org as { id: string }).id; } - // api_key path: service-role, so the tenant predicate has to be explicit. + // Caller path (api_key or JWT): on service-role the tenant predicate has to be explicit. const { data: row, error } = await tracedDb .from('org_members') .select('org_id, orgs!inner(id, slug, deleted_at)') From 69b41d1d9d768cfb11c9dbbe95c9e310a765fca2 Mon Sep 17 00:00:00 2001 From: mthines Date: Sat, 17 Oct 2026 10:15:01 +0200 Subject: [PATCH 13/16] address review comment: docs: a token-permission denial is -32003 in the matrix and troubleshooting too Addresses @mthines's comment: https://github.com/mthines/lorekit/pull/690#discussion_r4302006255 Refs: https://github.com/mthines/lorekit/pull/690 --- docs/api-tokens.md | 28 ++++++++++++++-------------- docs/install.md | 5 +++-- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/docs/api-tokens.md b/docs/api-tokens.md index 9442cd0c2..7ad7d88cc 100644 --- a/docs/api-tokens.md +++ b/docs/api-tokens.md @@ -129,20 +129,20 @@ routed into an org the token was never granted. | Tool | Read + Write (`lk_rw_`) | Read only (`lk_ro_`) | Write only (`lk_wo_`) | |------|------------------------|---------------------|------------------------| -| `memory.write` | ✓ | ✗ (returns -32001) | ✓ | -| `memory.read` | ✓ | ✓ | ✗ (returns -32001) | -| `memory.list` | ✓ | ✓ | ✗ (returns -32001) | -| `memory.delete` | ✓ | ✗ (returns -32001) | ✓ | -| `memory.search` | ✓ | ✓ | ✗ (returns -32001) | -| `memory.archive` | ✓ | ✗ (returns -32001) | ✓ | -| `memory.restore` | ✓ | ✗ (returns -32001) | ✓ | -| `memory.purge` | ✓ | ✗ (returns -32001) | ✓ | -| `memory.purge_expired` | ✓ | ✗ (returns -32001) | ✓ | -| `memory.list_archived` | ✓ | ✓ | ✗ (returns -32001) | -| `org.list` | ✓ | ✓ | ✗ (returns -32001) | -| `org.create` | ✓ | ✗ (returns -32001) | ✓ | -| `org.rename` | ✓ | ✗ (returns -32001) | ✓ | -| `org.delete` | ✓ | ✗ (returns -32001) | ✓ | +| `memory.write` | ✓ | ✗ (returns -32003) | ✓ | +| `memory.read` | ✓ | ✓ | ✗ (returns -32003) | +| `memory.list` | ✓ | ✓ | ✗ (returns -32003) | +| `memory.delete` | ✓ | ✗ (returns -32003) | ✓ | +| `memory.search` | ✓ | ✓ | ✗ (returns -32003) | +| `memory.archive` | ✓ | ✗ (returns -32003) | ✓ | +| `memory.restore` | ✓ | ✗ (returns -32003) | ✓ | +| `memory.purge` | ✓ | ✗ (returns -32003) | ✓ | +| `memory.purge_expired` | ✓ | ✗ (returns -32003) | ✓ | +| `memory.list_archived` | ✓ | ✓ | ✗ (returns -32003) | +| `org.list` | ✓ | ✓ | ✗ (returns -32003) | +| `org.create` | ✓ | ✗ (returns -32003) | ✓ | +| `org.rename` | ✓ | ✗ (returns -32003) | ✓ | +| `org.delete` | ✓ | ✗ (returns -32003) | ✓ | The four `org.*` rows are new: those tools used to be dashboard-JWT-only on the MCP surface, while the REST `/orgs` routes already served `lk_*` tokens. Both diff --git a/docs/install.md b/docs/install.md index 695325a4c..8b7e49021 100644 --- a/docs/install.md +++ b/docs/install.md @@ -265,8 +265,9 @@ Store a read+write token as `LOREKIT_TOKEN` in your repo secrets, then use it in | Symptom | Likely cause | Fix | |---------|-------------|-----| -| `{"code":-32001}` on write | Read-only (`lk_ro_`) token | Generate a `lk_rw_` or `lk_wo_` token | -| `{"code":-32001}` on read | Write-only (`lk_wo_`) token, or missing/wrong token | Generate a `lk_rw_` or `lk_ro_` token; check the `Authorization` header | +| `{"code":-32003}` on write | Read-only (`lk_ro_`) token | Generate a `lk_rw_` or `lk_wo_` token | +| `{"code":-32003}` on read | Write-only (`lk_wo_`) token | Generate a `lk_rw_` or `lk_ro_` token | +| `{"code":-32001}` | Missing, invalid, expired or revoked token | Check the `Authorization` header; generate a new token | | `{"code":-32603}` | Scope validation error | Use `::` as separator; check scope format | | `{"db":false}` on health check | DB not reachable from Edge Function | Verify Supabase secrets in Step 5 | | Dashboard auth loop | Redirect URL not set | Add your URL in Supabase Auth settings (Step 8) | From 6c63753aa712fe542369b0d01855b9304793c328 Mon Sep 17 00:00:00 2001 From: mthines Date: Sat, 17 Oct 2026 10:15:16 +0200 Subject: [PATCH 14/16] address review comment: docs: -32603 also reports a BYOD storage misconfiguration, which a retry won't clear Addresses @mthines's comment: pr-reviewer report https://github.com/mthines/lorekit/pull/690#issuecomment-6231586869 Refs: https://github.com/mthines/lorekit/pull/690 --- docs/mcp-tools.md | 2 +- packages/schemas/src/llms/template.md | 2 +- packages/web/public/llms.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/mcp-tools.md b/docs/mcp-tools.md index 61be517dc..02ed50ab7 100644 --- a/docs/mcp-tools.md +++ b/docs/mcp-tools.md @@ -787,7 +787,7 @@ no `error` member. The `lorekit` CLI's own client checks `result.isError` | JSON-RPC code | Meaning | |---------------|---------| | `-32001` | Unauthorized — missing, invalid, or expired token | -| `-32603` | Server fault — a DB outage or an unexpected exception. Not something the model can self-correct from, and safe for a client to retry | +| `-32603` | Server fault — a DB outage or an unexpected exception. Not something the model can self-correct from, and safe for a client to retry. A BYOD storage misconfiguration (`LOREKIT_STORAGE_URL` set without `LOREKIT_STORAGE_SERVICE_KEY`) also returns `-32603`, with a message naming the missing variable — that one does not clear on retry; fix the function's secrets | | `-32700` | Parse error — malformed JSON body | | `-32601` | Unknown method or tool name | | `-32003` | Forbidden (authenticated, but not permitted) — a read-only token attempted a write (or vice versa), an account-wide tool on a scoped key, or a scope outside the key's allowlist. The last one covers `memory.*` and, as of this change, `policy.*`/`groom.*` too: `policy.create`/`groom.*`'s target-or-resolved scope, and `policy.update`/`policy.delete`'s STORED scope. | diff --git a/packages/schemas/src/llms/template.md b/packages/schemas/src/llms/template.md index f8989b108..620788a76 100644 --- a/packages/schemas/src/llms/template.md +++ b/packages/schemas/src/llms/template.md @@ -394,7 +394,7 @@ these as successes. |---------------|---------| | `-32001` | Unauthorized — missing, invalid, expired, or rotated token | | `-32003` | Forbidden — authenticated, but not permitted for this tool: a wrong-permission-tier token (read-only token on a write, write-only token on a read), an account-wide tool on a scoped key, or a scope outside the key's allowlist | -| `-32603` | Server fault — a DB outage or unexpected exception. Safe to retry | +| `-32603` | Server fault — a DB outage or unexpected exception. Safe to retry. A BYOD storage misconfiguration (`LOREKIT_STORAGE_URL` set without `LOREKIT_STORAGE_SERVICE_KEY`) also returns `-32603`, with a message naming the missing variable — that one does not clear on retry; fix the function's secrets | | `-32700` | Parse error — malformed JSON body | | `-32601` | Unknown tool name | diff --git a/packages/web/public/llms.txt b/packages/web/public/llms.txt index bc4ef9fcb..155b68913 100644 --- a/packages/web/public/llms.txt +++ b/packages/web/public/llms.txt @@ -921,7 +921,7 @@ these as successes. |---------------|---------| | `-32001` | Unauthorized — missing, invalid, expired, or rotated token | | `-32003` | Forbidden — authenticated, but not permitted for this tool: a wrong-permission-tier token (read-only token on a write, write-only token on a read), an account-wide tool on a scoped key, or a scope outside the key's allowlist | -| `-32603` | Server fault — a DB outage or unexpected exception. Safe to retry | +| `-32603` | Server fault — a DB outage or unexpected exception. Safe to retry. A BYOD storage misconfiguration (`LOREKIT_STORAGE_URL` set without `LOREKIT_STORAGE_SERVICE_KEY`) also returns `-32603`, with a message naming the missing variable — that one does not clear on retry; fix the function's secrets | | `-32700` | Parse error — malformed JSON body | | `-32601` | Unknown tool name | From ac170119dd2f4c0e70196cc6df83b503febbfec6 Mon Sep 17 00:00:00 2001 From: mthines Date: Sat, 17 Oct 2026 10:18:59 +0200 Subject: [PATCH 15/16] address review comment: cli: bootstrap help names a storage key, not only the anon key Addresses @mthines's comment: pr-reviewer report https://github.com/mthines/lorekit/pull/690#issuecomment-6231586869 Refs: https://github.com/mthines/lorekit/pull/690 --- packages/cli/bin/lorekit.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/bin/lorekit.mjs b/packages/cli/bin/lorekit.mjs index caff2ff9a..7107227e0 100755 --- a/packages/cli/bin/lorekit.mjs +++ b/packages/cli/bin/lorekit.mjs @@ -121,7 +121,7 @@ ${c.bold('Commands')} Remote only. --json. pin / unpin Shorthand for protect / protect --off. Remote only. --json. bootstrap Check a BYOD Supabase database and print how to apply the schema. - Only needed when using LOREKIT_STORAGE_URL / LOREKIT_STORAGE_ANON_KEY. + Only needed when using LOREKIT_STORAGE_URL with a storage key. See docs/byod.md for setup instructions. migrate Relocate a LoreKit-format local store into the current layout, or push it up to the hosted store with --to remote. From 086ee01ad17c0d08b1bddb91792c36b289769b04 Mon Sep 17 00:00:00 2001 From: mthines Date: Sat, 17 Oct 2026 10:29:05 +0200 Subject: [PATCH 16/16] address review comment: install.md -32603 row named a scope error; split into BYOD misconfig, server fault and isError scope rows Addresses pr-reviewer report finding (docs/install.md:271): https://github.com/mthines/lorekit/pull/690#issuecomment-6231586869 Refs: https://github.com/mthines/lorekit/pull/690 --- docs/install.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/install.md b/docs/install.md index 8b7e49021..212bf93f7 100644 --- a/docs/install.md +++ b/docs/install.md @@ -268,7 +268,9 @@ Store a read+write token as `LOREKIT_TOKEN` in your repo secrets, then use it in | `{"code":-32003}` on write | Read-only (`lk_ro_`) token | Generate a `lk_rw_` or `lk_wo_` token | | `{"code":-32003}` on read | Write-only (`lk_wo_`) token | Generate a `lk_rw_` or `lk_ro_` token | | `{"code":-32001}` | Missing, invalid, expired or revoked token | Check the `Authorization` header; generate a new token | -| `{"code":-32603}` | Scope validation error | Use `::` as separator; check scope format | +| `{"code":-32603}` naming `LOREKIT_STORAGE_SERVICE_KEY` | BYOD storage URL set without the service key | Set `LOREKIT_STORAGE_SERVICE_KEY` on the function project (see [byod.md](./byod.md)); retrying will not help | +| `{"code":-32603}` otherwise | Server fault — DB unreachable or an unexpected exception | Retry; if it persists, check the function logs | +| Tool result with `isError: true` mentioning the scope | Invalid scope format | Use `::` as separator; check scope format | | `{"db":false}` on health check | DB not reachable from Edge Function | Verify Supabase secrets in Step 5 | | Dashboard auth loop | Redirect URL not set | Add your URL in Supabase Auth settings (Step 8) |