From 7e01c62f2df5b53c1b1671d7c50ca9f8431ef33c Mon Sep 17 00:00:00 2001 From: "mads.thines" Date: Sat, 17 Oct 2026 14:31:44 +0000 Subject: [PATCH 01/28] test(web-e2e): add a Playwright smoke suite for every page of the website One manifest entry per page (src/routes.ts) generates the page tests; a vitest guard fails when a page.tsx under packages/web has no entry. Public pages run signed out against a local build; dashboard pages also get a signed-out redirect check, and render checks when a test user is configured. Co-authored-by: dash0-dev[bot] <257284812+dash0-dev[bot]@users.noreply.github.com> --- packages/web-e2e/.gitignore | 5 + packages/web-e2e/package.json | 15 ++ packages/web-e2e/playwright.config.ts | 90 ++++++++++ packages/web-e2e/project.json | 38 +++++ packages/web-e2e/src/auth.setup.ts | 28 +++ packages/web-e2e/src/journeys/api-docs.e2e.ts | 22 +++ .../web-e2e/src/journeys/dashboard.e2e.ts | 39 +++++ packages/web-e2e/src/journeys/login.e2e.ts | 30 ++++ .../web-e2e/src/journeys/navigation.e2e.ts | 38 +++++ packages/web-e2e/src/journeys/sign-out.e2e.ts | 23 +++ packages/web-e2e/src/pages.e2e.ts | 66 ++++++++ packages/web-e2e/src/routes.test.ts | 75 +++++++++ packages/web-e2e/src/routes.ts | 159 ++++++++++++++++++ packages/web-e2e/src/support/auth.ts | 52 ++++++ packages/web-e2e/src/support/env.ts | 121 +++++++++++++ packages/web-e2e/src/support/fixtures.ts | 95 +++++++++++ packages/web-e2e/tsconfig.json | 18 ++ packages/web-e2e/vitest.config.ts | 15 ++ pnpm-lock.yaml | 44 +++-- 19 files changed, 959 insertions(+), 14 deletions(-) create mode 100644 packages/web-e2e/.gitignore create mode 100644 packages/web-e2e/package.json create mode 100644 packages/web-e2e/playwright.config.ts create mode 100644 packages/web-e2e/project.json create mode 100644 packages/web-e2e/src/auth.setup.ts create mode 100644 packages/web-e2e/src/journeys/api-docs.e2e.ts create mode 100644 packages/web-e2e/src/journeys/dashboard.e2e.ts create mode 100644 packages/web-e2e/src/journeys/login.e2e.ts create mode 100644 packages/web-e2e/src/journeys/navigation.e2e.ts create mode 100644 packages/web-e2e/src/journeys/sign-out.e2e.ts create mode 100644 packages/web-e2e/src/pages.e2e.ts create mode 100644 packages/web-e2e/src/routes.test.ts create mode 100644 packages/web-e2e/src/routes.ts create mode 100644 packages/web-e2e/src/support/auth.ts create mode 100644 packages/web-e2e/src/support/env.ts create mode 100644 packages/web-e2e/src/support/fixtures.ts create mode 100644 packages/web-e2e/tsconfig.json create mode 100644 packages/web-e2e/vitest.config.ts diff --git a/packages/web-e2e/.gitignore b/packages/web-e2e/.gitignore new file mode 100644 index 000000000..9d181cf2b --- /dev/null +++ b/packages/web-e2e/.gitignore @@ -0,0 +1,5 @@ +# Saved sign-in session (contains auth cookies) — never commit. +.auth/ +playwright-report/ +test-results/ +blob-report/ diff --git a/packages/web-e2e/package.json b/packages/web-e2e/package.json new file mode 100644 index 000000000..f66b58a17 --- /dev/null +++ b/packages/web-e2e/package.json @@ -0,0 +1,15 @@ +{ + "name": "@lorekit/web-e2e", + "version": "0.0.1", + "private": true, + "description": "Playwright end-to-end smoke tests for the LoreKit website (@lorekit/web). One manifest entry per page in src/routes.ts; a vitest guard fails when a page has no entry.", + "scripts": { + "e2e": "playwright test", + "e2e:public": "playwright test --project=public", + "e2e:ui": "playwright test --ui", + "report": "playwright show-report" + }, + "devDependencies": { + "@playwright/test": "1.56.0" + } +} diff --git a/packages/web-e2e/playwright.config.ts b/packages/web-e2e/playwright.config.ts new file mode 100644 index 000000000..d225dfc75 --- /dev/null +++ b/packages/web-e2e/playwright.config.ts @@ -0,0 +1,90 @@ +import { defineConfig, devices } from '@playwright/test'; +import { STORAGE_STATE } from './src/support/auth'; +import { BASE_URL, E2E_PORT, STARTS_LOCAL_SERVER } from './src/support/env'; + +const CI = Boolean(process.env.CI); + +/** + * Projects, in the order they run: + * + * setup signs in once through the login form and saves the session + * (skipped, with the reason, when no test user is configured). + * public everything a signed-out visitor sees. Needs no credentials and + * no backend, so it runs on every PR against a local build. + * signed-in tests tagged @signed-in, reusing the saved session. + * sign-out tagged @sign-out; runs last because signing out revokes every + * session of the account, including the one `signed-in` uses. + * + * `PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH` points Chromium at a browser you + * already have (a sandbox that cannot reach Playwright's download host, for + * example). Unset — the normal case, CI included — Playwright uses its own. + */ +const executablePath = process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH || undefined; +const desktop = { + ...devices['Desktop Chrome'], + ...(executablePath ? { launchOptions: { executablePath } } : {}), +}; + +export default defineConfig({ + testDir: './src', + // `*.e2e.ts` (and the setup project's `*.setup.ts`) are Playwright's; + // `*.test.ts` is vitest's (the route coverage guard), so the two runners + // never pick up each other's files. + testMatch: /.*\.e2e\.ts$/, + outputDir: './test-results', + fullyParallel: true, + forbidOnly: CI, + retries: CI ? 2 : 0, + workers: CI ? 2 : undefined, + timeout: 45_000, + expect: { timeout: 15_000 }, + reporter: CI + ? [['github'], ['list'], ['html', { open: 'never', outputFolder: 'playwright-report' }]] + : [['list'], ['html', { open: 'never', outputFolder: 'playwright-report' }]], + + use: { + baseURL: BASE_URL, + trace: 'retain-on-failure', + screenshot: 'only-on-failure', + // Production builds register /sw.js, which serves pages cache-first; a + // smoke test must see what the server returns now. + serviceWorkers: 'block', + }, + + projects: [ + { name: 'setup', testMatch: /.*\.setup\.ts$/, use: desktop }, + { name: 'public', grepInvert: /@signed-in|@sign-out/, use: desktop }, + { + name: 'signed-in', + grep: /@signed-in/, + dependencies: ['setup'], + use: { ...desktop, storageState: STORAGE_STATE }, + }, + { + name: 'sign-out', + grep: /@sign-out/, + dependencies: ['signed-in'], + use: { ...desktop, storageState: STORAGE_STATE }, + }, + ], + + // No E2E_BASE_URL → serve a production build of packages/web locally. Build + // it first with `pnpm nx run web-e2e:build-web` (NEXT_PUBLIC_* values are + // inlined at build time, so the build — not this server — decides which + // Supabase the site talks to). E2E_WEB_COMMAND swaps the command, e.g. for + // `next dev` while iterating. + webServer: STARTS_LOCAL_SERVER + ? { + command: process.env.E2E_WEB_COMMAND || `pnpm --filter @lorekit/web exec next start --port ${E2E_PORT}`, + url: BASE_URL, + reuseExistingServer: !CI, + timeout: 180_000, + stdout: 'pipe', + stderr: 'pipe', + env: { + NEXT_PUBLIC_USE_LOCAL_SUPABASE: 'true', + NEXT_TELEMETRY_DISABLED: '1', + }, + } + : undefined, +}); diff --git a/packages/web-e2e/project.json b/packages/web-e2e/project.json new file mode 100644 index 000000000..f01357516 --- /dev/null +++ b/packages/web-e2e/project.json @@ -0,0 +1,38 @@ +{ + "name": "web-e2e", + "$schema": "../../node_modules/nx/schemas/project-schema.json", + "sourceRoot": "packages/web-e2e/src", + "projectType": "application", + "tags": ["type:e2e", "platform:web"], + "implicitDependencies": ["web"], + "targets": { + "typecheck": { + "executor": "nx:run-commands", + "cache": true, + "options": { + "command": "tsc --noEmit -p packages/web-e2e/tsconfig.json", + "cwd": "{workspaceRoot}" + } + }, + "build-web": { + "executor": "nx:run-commands", + "cache": false, + "options": { + "cwd": "packages/web", + "command": "next build", + "env": { + "NEXT_PUBLIC_USE_LOCAL_SUPABASE": "true", + "NEXT_TELEMETRY_DISABLED": "1" + } + } + }, + "e2e": { + "executor": "nx:run-commands", + "cache": false, + "options": { + "cwd": "packages/web-e2e", + "command": "playwright test" + } + } + } +} diff --git a/packages/web-e2e/src/auth.setup.ts b/packages/web-e2e/src/auth.setup.ts new file mode 100644 index 000000000..7287a6bc3 --- /dev/null +++ b/packages/web-e2e/src/auth.setup.ts @@ -0,0 +1,28 @@ +import { mkdirSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; +import { EMPTY_STORAGE_STATE, STORAGE_STATE, provisionUser, signInThroughTheForm } from './support/auth'; +import { NO_USER_REASON, USER_SOURCE } from './support/env'; +import { expect, test as setup } from './support/fixtures'; + +/** + * Sign in once, through the real form, and save the session for the + * `signed-in` project (playwright.config.ts). Every signed-in test reuses it. + */ +setup('sign in through the login form', async ({ page }) => { + mkdirSync(path.dirname(STORAGE_STATE), { recursive: true }); + // Always leave a file behind: the `signed-in` project loads this path, and + // its tests skip themselves when there is no user. + writeFileSync(STORAGE_STATE, JSON.stringify(EMPTY_STORAGE_STATE)); + setup.skip(!USER_SOURCE, NO_USER_REASON); + + const user = USER_SOURCE!.kind === 'fixed' ? USER_SOURCE! : await provisionUser(USER_SOURCE!); + await signInThroughTheForm(page, user.email, user.password); + + // Plain /login lands on DEFAULT_POST_LOGIN_PATH (lib/auth-redirect.ts). + await expect(page).toHaveURL((url) => url.pathname === '/lore', { timeout: 30_000 }); + // An authenticated-only control, so a route that also renders signed out + // can never be mistaken for a session. + await expect(page.getByRole('button', { name: 'Sign out', exact: true })).toBeVisible(); + + await page.context().storageState({ path: STORAGE_STATE }); +}); diff --git a/packages/web-e2e/src/journeys/api-docs.e2e.ts b/packages/web-e2e/src/journeys/api-docs.e2e.ts new file mode 100644 index 000000000..8d7122e42 --- /dev/null +++ b/packages/web-e2e/src/journeys/api-docs.e2e.ts @@ -0,0 +1,22 @@ +import { expect, originHeaders, test } from '../support/fixtures'; + +/** + * /api-docs is a route handler, not a page: it returns a Scalar HTML shell that + * draws itself from a CDN script, so the browser view depends on the internet. + * Assert what the SERVER returns instead — the shell and the OpenAPI spec. + */ + +test('the API reference shell is served', async ({ request }) => { + const res = await request.get('/api-docs', { headers: originHeaders() }); + expect(res.status()).toBe(200); + expect(res.headers()['content-type']).toContain('text/html'); + expect(await res.text()).toContain('LoreKit REST API'); +}); + +test('the OpenAPI spec is served as JSON', async ({ request }) => { + const res = await request.get('/api-docs/spec', { headers: originHeaders() }); + expect(res.status()).toBe(200); + const spec = await res.json(); + expect(spec.info?.title).toBe('LoreKit REST API'); + expect(Object.keys(spec.paths ?? {}).length).toBeGreaterThan(0); +}); diff --git a/packages/web-e2e/src/journeys/dashboard.e2e.ts b/packages/web-e2e/src/journeys/dashboard.e2e.ts new file mode 100644 index 000000000..a32eeda09 --- /dev/null +++ b/packages/web-e2e/src/journeys/dashboard.e2e.ts @@ -0,0 +1,39 @@ +import { NO_USER_REASON, USER_SOURCE } from '../support/env'; +import { expect, test } from '../support/fixtures'; + +/** Signed-in journeys through the app shell. Runs in the `signed-in` project. */ + +test.describe('the signed-in app shell', () => { + test.skip(!USER_SOURCE, NO_USER_REASON); + + test('keeps the session across a reload', { tag: '@signed-in' }, async ({ page }) => { + await page.goto('/lore'); + await expect(page.getByRole('button', { name: 'Sign out', exact: true })).toBeVisible(); + await page.reload(); + await expect(page).toHaveURL((url) => url.pathname === '/lore'); + await expect(page.getByRole('button', { name: 'Sign out', exact: true })).toBeVisible(); + }); + + test('navigates between Explorer, Insights and Settings', { tag: '@signed-in' }, async ({ page }) => { + await page.goto('/lore'); + const nav = page.getByRole('navigation', { name: 'Main navigation' }).first(); + + await nav.getByRole('link', { name: 'Insights', exact: true }).click(); + await expect(page).toHaveURL((url) => url.pathname === '/insights'); + await expect(page.getByRole('heading', { level: 1, name: 'Insights', exact: true })).toBeVisible(); + + await nav.getByRole('link', { name: 'Settings', exact: true }).click(); + await expect(page).toHaveURL((url) => url.pathname.startsWith('/settings')); + await expect(page.getByRole('heading', { level: 1, name: 'Settings', exact: true })).toBeVisible(); + + await nav.getByRole('link', { name: 'Explorer', exact: true }).click(); + await expect(page).toHaveURL((url) => url.pathname === '/lore'); + await expect(page.getByRole('heading', { level: 1, name: 'Lore Explorer', exact: true })).toBeVisible(); + }); + + test('sends a signed-in visitor away from /login', { tag: '@signed-in' }, async ({ page }) => { + await page.goto('/login?next=%2Fsettings%2Fuser'); + await expect(page).toHaveURL((url) => url.pathname === '/settings/user'); + await expect(page.getByRole('heading', { level: 2, name: 'User', exact: true })).toBeVisible(); + }); +}); diff --git a/packages/web-e2e/src/journeys/login.e2e.ts b/packages/web-e2e/src/journeys/login.e2e.ts new file mode 100644 index 000000000..ef9992ddf --- /dev/null +++ b/packages/web-e2e/src/journeys/login.e2e.ts @@ -0,0 +1,30 @@ +import { signInThroughTheForm } from '../support/auth'; +import { HAS_AUTH_BACKEND } from '../support/env'; +import { expect, test } from '../support/fixtures'; + +test('the login page offers GitHub, email and magic-link sign-in', async ({ page }) => { + await page.goto('/login'); + await expect(page.getByRole('button', { name: 'Continue with GitHub', exact: true })).toBeVisible(); + await expect(page.getByRole('button', { name: 'Or email me a magic link', exact: true })).toBeVisible(); + + await page.getByRole('button', { name: 'Continue with email', exact: true }).click(); + await expect(page.locator('#lk-email')).toBeVisible(); + await expect(page.locator('#lk-password')).toBeVisible(); + await expect(page.getByRole('link', { name: 'Forgot password?', exact: true })).toBeVisible(); + + await page.getByRole('button', { name: 'Back', exact: true }).click(); + await expect(page.getByRole('button', { name: 'Continue with email', exact: true })).toBeVisible(); +}); + +test.describe('signing in with a wrong password', () => { + // The auth endpoint answers 400 for bad credentials; Chromium logs that + // failed request as a console error. + test.use({ allowedConsoleErrors: [/status of 400/, /auth\/v1\/token/] }); + + test('shows an error and stays on /login', async ({ page }) => { + test.skip(!HAS_AUTH_BACKEND, 'a local build with no Supabase behind it cannot answer a sign-in'); + await signInThroughTheForm(page, `nobody-${Date.now()}@lorekit.test`, 'definitely-not-the-password-1A!'); + await expect(page.locator('form').getByRole('alert')).toBeVisible(); + await expect(page).toHaveURL((url) => url.pathname === '/login'); + }); +}); diff --git a/packages/web-e2e/src/journeys/navigation.e2e.ts b/packages/web-e2e/src/journeys/navigation.e2e.ts new file mode 100644 index 000000000..7f427accd --- /dev/null +++ b/packages/web-e2e/src/journeys/navigation.e2e.ts @@ -0,0 +1,38 @@ +import { expect, test } from '../support/fixtures'; + +/** + * Signed-out journeys across pages — the paths a first-time visitor takes. + * Per-page rendering is pages.e2e.ts's job; these check that pages connect. + */ + +test('the landing page header leads to the docs and the blog', async ({ page }) => { + await page.goto('/'); + const site = page.getByRole('navigation', { name: 'Site' }); + + await site.getByRole('link', { name: 'Docs', exact: true }).click(); + await expect(page).toHaveURL((url) => url.pathname === '/docs/setup'); + await expect(page.getByRole('heading', { level: 1, name: 'Getting started', exact: true })).toBeVisible(); + + await page.goBack(); + await page.getByRole('navigation', { name: 'Site' }).getByRole('link', { name: 'Blog', exact: true }).click(); + await expect(page).toHaveURL((url) => url.pathname === '/blog'); + await expect(page.getByRole('heading', { level: 1, name: 'Blog', exact: true })).toBeVisible(); +}); + +test('the landing page header leads to sign-in', async ({ page }) => { + await page.goto('/'); + await page.getByRole('navigation', { name: 'Site' }).getByRole('link', { name: 'Sign in', exact: true }).click(); + await expect(page).toHaveURL((url) => url.pathname === '/login'); + await expect(page.getByRole('heading', { level: 1, name: 'Sign in to LoreKit', exact: true })).toBeVisible(); +}); + +test.describe('an unknown URL', () => { + // Chromium logs the 404 document itself as a failed resource load. + test.use({ allowedConsoleErrors: [/status of 404/] }); + + test('shows the not-found page with a 404 status', async ({ page }) => { + const response = await page.goto('/this-page-does-not-exist-e2e'); + expect(response?.status()).toBe(404); + await expect(page.getByText('Page not found', { exact: true })).toBeVisible(); + }); +}); diff --git a/packages/web-e2e/src/journeys/sign-out.e2e.ts b/packages/web-e2e/src/journeys/sign-out.e2e.ts new file mode 100644 index 000000000..da0f4fe4f --- /dev/null +++ b/packages/web-e2e/src/journeys/sign-out.e2e.ts @@ -0,0 +1,23 @@ +import { NO_USER_REASON, TARGET, USER_SOURCE } from '../support/env'; +import { expect, test } from '../support/fixtures'; + +/** + * Signing out revokes EVERY session of the account — SignOutButton calls + * `supabase.auth.signOut()`, whose default scope is global. So this runs in its + * own project, after the `signed-in` project has finished with the shared + * session (playwright.config.ts), and never against production, where the same + * test account may be signed in elsewhere. + */ +test.describe('signing out', () => { + test.skip(!USER_SOURCE, NO_USER_REASON); + test.skip(TARGET === 'production', 'signing out revokes every session of the shared test account'); + + test('ends the session', { tag: '@sign-out' }, async ({ page }) => { + await page.goto('/lore'); + await page.getByRole('button', { name: 'Sign out', exact: true }).click(); + await expect(page.getByRole('button', { name: 'Sign out', exact: true })).toHaveCount(0); + + await page.goto('/lore'); + await expect(page).toHaveURL((url) => url.pathname === '/login'); + }); +}); diff --git a/packages/web-e2e/src/pages.e2e.ts b/packages/web-e2e/src/pages.e2e.ts new file mode 100644 index 000000000..08c6762df --- /dev/null +++ b/packages/web-e2e/src/pages.e2e.ts @@ -0,0 +1,66 @@ +import type { Page } from '@playwright/test'; +import { ROUTES, visitPath, type RouteCheck } from './routes'; +import { NO_USER_REASON, TARGET, USER_SOURCE } from './support/env'; +import { expect, test } from './support/fixtures'; + +/** + * One block of tests per entry in routes.ts. Nothing to edit here when a page + * is added — add the entry to routes.ts instead. + */ + +function heading(page: Page, r: RouteCheck) { + return page.getByRole('heading', { + level: r.heading.level, + name: r.heading.name, + exact: typeof r.heading.name === 'string', + }); +} + +async function expectPageRenders(page: Page, r: RouteCheck) { + const response = await page.goto(visitPath(r)); + expect(response, `no response for ${visitPath(r)}`).not.toBeNull(); + expect(response!.status(), `HTTP status for ${visitPath(r)}`).toBeLessThan(400); + if (r.landsOn) { + const landsOn = r.landsOn; + await expect(page).toHaveURL((url) => url.pathname === landsOn); + } + await expect(heading(page, r)).toBeVisible(); +} + +for (const r of ROUTES) { + const skipReason = r.skipOn?.[TARGET]; + + test.describe(`page ${r.route}`, () => { + test.use({ allowedConsoleErrors: r.allowConsoleErrors ?? [] }); + + if (r.access === 'public') { + test('renders for a signed-out visitor', async ({ page }) => { + test.skip(Boolean(skipReason), skipReason); + await expectPageRenders(page, r); + }); + return; + } + + // Free coverage of the auth gate for every dashboard page: no credentials + // needed, so this runs on every PR. + test('sends a signed-out visitor to /login', async ({ page }) => { + const path = visitPath(r); + await page.goto(path); + await expect(page).toHaveURL( + (url) => url.pathname === '/login' && url.searchParams.get('next') === path, + ); + await expect(page.getByRole('heading', { level: 1, name: 'Sign in to LoreKit', exact: true })).toBeVisible(); + }); + + test.describe('signed in', () => { + // Describe-level, so it is decided before a browser context (and the + // saved session it loads) is created. + test.skip(!USER_SOURCE, NO_USER_REASON); + + test('renders for a signed-in user', { tag: '@signed-in' }, async ({ page }) => { + test.skip(Boolean(skipReason), skipReason); + await expectPageRenders(page, r); + }); + }); + }); +} diff --git a/packages/web-e2e/src/routes.test.ts b/packages/web-e2e/src/routes.test.ts new file mode 100644 index 000000000..39c70ae49 --- /dev/null +++ b/packages/web-e2e/src/routes.test.ts @@ -0,0 +1,75 @@ +import { readdirSync, statSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { ROUTES, visitPath } from './routes'; + +/** + * The guard that keeps the page smoke suite complete. + * + * It walks packages/web/src/app for `page.tsx` files and compares the routes + * they define with routes.ts. A new page with no entry fails here — on the PR + * that adds it, in the `check` job, without a browser — and so does an entry + * for a page that was deleted or moved. + */ + +const APP_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../web/src/app'); + +/** `(dashboard)/settings/api-keys` → `/settings/api-keys`. */ +function toRoute(relativeDir: string): string { + const segments = relativeDir + .split(path.sep) + .filter((s) => s !== '' && !/^\(.*\)$/.test(s)); // route groups do not appear in the URL + return `/${segments.join('/')}`; +} + +function appRoutes(dir = APP_DIR, rel = ''): string[] { + const found: string[] = []; + for (const name of readdirSync(dir)) { + const full = path.join(dir, name); + if (statSync(full).isDirectory()) { + // `_private` folders and `@slot` parallel routes are not URL segments of their own. + if (name.startsWith('_') || name.startsWith('@')) continue; + found.push(...appRoutes(full, path.join(rel, name))); + } else if (name === 'page.tsx' || name === 'page.ts' || name === 'page.mdx') { + found.push(toRoute(rel)); + } + } + return found; +} + +const pages = appRoutes().sort(); +const covered = ROUTES.map((r) => r.route); + +describe('web e2e route coverage', () => { + it('finds the app directory', () => { + expect(pages.length, `no page.tsx found under ${APP_DIR}`).toBeGreaterThan(0); + }); + + it('has an entry in routes.ts for every page', () => { + const missing = pages.filter((p) => !covered.includes(p)); + expect( + missing, + `These pages have no smoke test. Add an entry for each to packages/web-e2e/src/routes.ts ` + + `(access + the heading a visitor sees):\n ${missing.join('\n ')}`, + ).toEqual([]); + }); + + it('has no entry for a page that no longer exists', () => { + const stale = covered.filter((r) => !pages.includes(r)); + expect( + stale, + `routes.ts names routes with no page.tsx behind them — remove or rename:\n ${stale.join('\n ')}`, + ).toEqual([]); + }); + + it('lists each route once', () => { + const dupes = covered.filter((r, i) => covered.indexOf(r) !== i); + expect(dupes, `duplicate entries in routes.ts: ${dupes.join(', ')}`).toEqual([]); + }); + + it('gives every dynamic route a concrete path to visit', () => { + const unvisitable = ROUTES.filter((r) => /\[/.test(visitPath(r))).map((r) => r.route); + expect(unvisitable, `set \`path\` to a real URL for: ${unvisitable.join(', ')}`).toEqual([]); + }); +}); diff --git a/packages/web-e2e/src/routes.ts b/packages/web-e2e/src/routes.ts new file mode 100644 index 000000000..d427e87aa --- /dev/null +++ b/packages/web-e2e/src/routes.ts @@ -0,0 +1,159 @@ +import type { TargetKind } from './support/env'; + +/** + * Every page of the LoreKit website, and what a visitor must see on it. + * + * This table IS the page smoke suite: `pages.e2e.ts` turns each entry into + * tests, and `routes.test.ts` fails the PR when a `page.tsx` under + * packages/web/src/app has no entry here (or an entry names a page that no + * longer exists). Adding a page therefore means adding one object below. + * + * What each entry produces: + * + * access: 'public' → "renders for a signed-out visitor" + * access: 'signed-in' → "sends a signed-out visitor to /login?next=" + * (runs everywhere, no credentials needed), and + * "renders for a signed-in user" (runs when a test + * user is configured — see support/env.ts) + * + * Assert on what a person sees: the page's heading, exactly as rendered. No + * page carries a data-testid today, and a heading changing is a real change + * worth a one-line update here. + */ + +export type Access = 'public' | 'signed-in'; + +export interface Heading { + /** Heading level. Settings pages share the h1 "Settings", so they assert their h2. */ + level: 1 | 2; + /** Exact accessible name, or a pattern when the heading is split across elements. */ + name: string | RegExp; +} + +export interface RouteCheck { + /** + * The route as the app router names it: the directory path of a `page.tsx` + * under packages/web/src/app, with route groups like `(dashboard)` removed. + * `/docs/[slug]`, not `/docs/offline`. The coverage guard matches on this. + */ + route: string; + /** The URL to visit. Defaults to `route`; required for a dynamic route. */ + path?: string; + access: Access; + /** What the visitor must see once the page has settled. */ + heading: Heading; + /** The pathname the visit ends on, when the page redirects (e.g. `/docs` → `/docs/setup`). */ + landsOn?: string; + /** + * Known, harmless problems to tolerate on this page only (see + * support/fixtures.ts for the line format). Say why in a comment. + */ + allowConsoleErrors?: RegExp[]; + /** Skip on one kind of target, with the reason. The entry still counts as covered. */ + skipOn?: Partial>; +} + +export const ROUTES: RouteCheck[] = [ + // ── Marketing, auth and content (public) ───────────────────────────────── + { + route: '/', + access: 'public', + // Two block spans inside one h1, so match on the first sentence. + heading: { level: 1, name: /Your coding agents forget\./ }, + }, + { route: '/login', access: 'public', heading: { level: 1, name: 'Sign in to LoreKit' } }, + { route: '/forgot-password', access: 'public', heading: { level: 1, name: 'Reset your password' } }, + // Signed out, this also shows "This reset link is invalid…" — expected. + { route: '/update-password', access: 'public', heading: { level: 1, name: 'Set a new password' } }, + { route: '/welcome', access: 'public', heading: { level: 1, name: 'Welcome to LoreKit' } }, + { route: '/blog', access: 'public', heading: { level: 1, name: 'Blog' } }, + { + route: '/blog/[slug]', + path: '/blog/self-healing-agents', + access: 'public', + heading: { level: 1, name: 'Self-healing agents are just a loop you forgot to build' }, + allowConsoleErrors: [ + // The like counter fetches /functions/v1/blog/likes and + // swallows a failure by design (PostLikes.tsx); a local build with no + // Supabase behind it cannot reach it. Likes are not core — the post is. + /blog\/likes/, + ], + }, + { route: '/docs', access: 'public', landsOn: '/docs/setup', heading: { level: 1, name: 'Getting started' } }, + { route: '/docs/[slug]', path: '/docs/offline', access: 'public', heading: { level: 1, name: 'Offline storage' } }, + { + route: '/learn/[[...slug]]', + path: '/learn/remote', + access: 'public', + landsOn: '/docs/remote', + heading: { level: 1, name: 'Remote storage' }, + }, + + // ── Dashboard (signed in) ──────────────────────────────────────────────── + // Note: opening /lore as a user with no API tokens creates one ("My first + // token") — onboarding-steps.tsx. Harmless for a test account. + { route: '/lore', access: 'signed-in', heading: { level: 1, name: 'Lore Explorer' } }, + { route: '/insights', access: 'signed-in', heading: { level: 1, name: 'Insights' } }, + { + route: '/onboarding', + access: 'signed-in', + landsOn: '/docs/setup', + heading: { level: 1, name: 'Getting started' }, + }, + + // Settings: every page shares the h1 "Settings"; each section is its h2. + { route: '/settings/api-keys', access: 'signed-in', heading: { level: 2, name: 'API keys' } }, + { route: '/settings/audit', access: 'signed-in', heading: { level: 2, name: 'Audit Logs' } }, + { + route: '/settings/developer', + access: 'signed-in', + heading: { level: 2, name: 'Developer' }, + skipOn: { + production: 'answers 404 in production for any account not in DEVELOPER_EMAILS (lib/developer-users.ts)', + }, + }, + // Behind the `retention-policies` flag (default on). + { route: '/settings/grooming', access: 'signed-in', heading: { level: 2, name: 'Retention Policies' } }, + // The nav says "Integrations"; the section heading is "GitHub App". + { route: '/settings/integrations', access: 'signed-in', heading: { level: 2, name: 'GitHub App' } }, + { route: '/settings/organization', access: 'signed-in', heading: { level: 2, name: 'Organization' } }, + { route: '/settings/plan', access: 'signed-in', heading: { level: 2, name: 'Plan' } }, + { route: '/settings/user', access: 'signed-in', heading: { level: 2, name: 'User' } }, + + // Tutorials moved to /docs; each old URL forwards to its doc. + { route: '/tutorials', access: 'signed-in', landsOn: '/docs/offline', heading: { level: 1, name: 'Offline storage' } }, + { + route: '/tutorials/offline', + access: 'signed-in', + landsOn: '/docs/offline', + heading: { level: 1, name: 'Offline storage' }, + }, + { + route: '/tutorials/organization', + access: 'signed-in', + landsOn: '/docs/organization', + heading: { level: 1, name: 'Team sharing' }, + }, + { + route: '/tutorials/private', + access: 'signed-in', + landsOn: '/docs/private', + heading: { level: 1, name: 'Private lore' }, + }, + { + route: '/tutorials/remote', + access: 'signed-in', + landsOn: '/docs/remote', + heading: { level: 1, name: 'Remote storage' }, + }, + { route: '/tutorials/tags', access: 'signed-in', landsOn: '/docs/tags', heading: { level: 1, name: 'Tags & scopes' } }, + { + route: '/tutorials/use-cases', + access: 'signed-in', + landsOn: '/docs/use-cases', + heading: { level: 1, name: 'Use cases' }, + }, +]; + +/** The URL a check visits. */ +export const visitPath = (r: RouteCheck): string => r.path ?? r.route; diff --git a/packages/web-e2e/src/support/auth.ts b/packages/web-e2e/src/support/auth.ts new file mode 100644 index 000000000..7b5f1add0 --- /dev/null +++ b/packages/web-e2e/src/support/auth.ts @@ -0,0 +1,52 @@ +import { randomUUID } from 'node:crypto'; +import path from 'node:path'; +import type { Page } from '@playwright/test'; +import type { ProvisionedUser } from './env'; + +/** Where the setup project saves the signed-in session for the `signed-in` project. */ +export const STORAGE_STATE = path.join(__dirname, '..', '..', '.auth', 'user.json'); + +/** A storage state with nothing in it — written when there is no user, so the path always exists. */ +export const EMPTY_STORAGE_STATE = { cookies: [], origins: [] }; + +/** + * Create a confirmed throwaway user on a LOCAL Supabase stack through the Auth + * admin API. `env.ts` refuses a non-local URL before this is ever reached. + * Nothing to clean up: the stack is torn down with the job. + */ +export async function provisionUser(source: ProvisionedUser): Promise<{ email: string; password: string }> { + // Unique per run so concurrent runs never collide on the users_email_key. + const email = `e2e-${Date.now()}-${Math.random().toString(36).slice(2, 8)}@lorekit.test`; + const password = `E2e-${randomUUID()}-Aa1!`; + const res = await fetch(`${source.supabaseUrl}/auth/v1/admin/users`, { + method: 'POST', + headers: { + apikey: source.serviceRoleKey, + Authorization: `Bearer ${source.serviceRoleKey}`, + 'Content-Type': 'application/json', + }, + // email_confirm skips the confirmation mail — the user can sign in at once. + body: JSON.stringify({ email, password, email_confirm: true }), + }); + if (!res.ok) { + throw new Error(`could not provision a test user (HTTP ${res.status}): ${(await res.text()).slice(0, 300)}`); + } + return { email, password }; +} + +/** + * Sign in through the real login form — the path a person takes, not an API + * shortcut, so a broken form fails here first. + * + * Keep in step with scripts/auth-bootstrap-credentials.mjs, which drives the + * same form for aw-tester: the email form is hidden behind "Continue with + * email", and two buttons read "Sign in" (the header's GitHub button and the + * form submit), so the submit is scoped to the form. + */ +export async function signInThroughTheForm(page: Page, email: string, password: string): Promise { + await page.goto('/login'); + await page.getByRole('button', { name: 'Continue with email', exact: true }).click(); + await page.locator('#lk-email').fill(email); + await page.locator('#lk-password').fill(password); + await page.locator('form').getByRole('button', { name: 'Sign in', exact: true }).click(); +} diff --git a/packages/web-e2e/src/support/env.ts b/packages/web-e2e/src/support/env.ts new file mode 100644 index 000000000..d993e6e4d --- /dev/null +++ b/packages/web-e2e/src/support/env.ts @@ -0,0 +1,121 @@ +/** + * Everything the suite reads from the environment, resolved in ONE place so a + * spec never reaches for `process.env` itself. + * + * ## Where the suite points + * + * E2E_BASE_URL the site under test. Unset → Playwright boots a local + * `next start` on E2E_PORT (see playwright.config.ts) against + * a build made with NEXT_PUBLIC_USE_LOCAL_SUPABASE=true. + * E2E_TARGET local | preview | production. Inferred from the URL when + * unset (localhost → local, lorekit.io → production, anything + * else → preview). Set it explicitly when the URL does not say + * which backend it talks to — a STAGED production build on a + * *.vercel.app URL is `production`. + * + * ## Getting through Vercel Deployment Protection + * + * VERCEL_AUTOMATION_BYPASS_SECRET sent as `x-vercel-protection-bypass` on + * requests to the site's own origin only (never to Supabase, + * where the extra header would fail the CORS preflight). + * + * ## Signing in (the `signed-in` project) + * + * The first source that is set wins: + * + * 1. E2E_USER_EMAIL + E2E_USER_PASSWORD — a fixed, pre-existing test account. + * LOREKIT_TESTING_USER_NAME + LOREKIT_TESTING_USER_PW are accepted as + * aliases, so the credentials aw-tester already uses work unchanged. + * 2. E2E_SUPABASE_URL + E2E_SUPABASE_SERVICE_ROLE_KEY — provision a throwaway + * confirmed user through the Auth admin API. LOCAL STACKS ONLY: anything + * that is not localhost/127.0.0.1 is refused, so a run can never create + * users in a real tenant (the same rule scripts/smoke/mint-smoke-jwt.mjs + * follows). + * 3. Neither → every signed-in test is skipped, and the reason is printed. + */ + +export type TargetKind = 'local' | 'preview' | 'production'; + +const env = (name: string): string | undefined => { + const v = process.env[name]; + return v && v.trim() !== '' ? v.trim() : undefined; +}; + +export const E2E_PORT = Number(env('E2E_PORT') ?? 3100); + +/** The origin under test, without a trailing slash. */ +export const BASE_URL = (env('E2E_BASE_URL') ?? `http://127.0.0.1:${E2E_PORT}`).replace(/\/+$/, ''); + +/** True when Playwright must start the site itself. */ +export const STARTS_LOCAL_SERVER = !env('E2E_BASE_URL'); + +function inferTarget(url: string): TargetKind { + const host = new URL(url).hostname; + if (host === 'localhost' || host === '127.0.0.1') return 'local'; + if (host === 'lorekit.io' || host.endsWith('.lorekit.io')) return 'production'; + return 'preview'; +} + +export const TARGET: TargetKind = (() => { + const explicit = env('E2E_TARGET'); + if (explicit === 'local' || explicit === 'preview' || explicit === 'production') return explicit; + if (explicit) throw new Error(`E2E_TARGET must be local | preview | production, got "${explicit}"`); + return inferTarget(BASE_URL); +})(); + +export const VERCEL_BYPASS_SECRET = env('VERCEL_AUTOMATION_BYPASS_SECRET'); + +export interface FixedUser { + kind: 'fixed'; + email: string; + password: string; +} +export interface ProvisionedUser { + kind: 'provision'; + supabaseUrl: string; + serviceRoleKey: string; +} +export type UserSource = FixedUser | ProvisionedUser; + +const LOCAL_HOSTS = new Set(['localhost', '127.0.0.1', '::1']); + +function resolveUserSource(): { source: UserSource | null; reason: string } { + const email = env('E2E_USER_EMAIL') ?? env('LOREKIT_TESTING_USER_NAME'); + const password = env('E2E_USER_PASSWORD') ?? env('LOREKIT_TESTING_USER_PW'); + if (email && password) return { source: { kind: 'fixed', email, password }, reason: '' }; + + const supabaseUrl = env('E2E_SUPABASE_URL'); + const serviceRoleKey = env('E2E_SUPABASE_SERVICE_ROLE_KEY'); + if (supabaseUrl && serviceRoleKey) { + const host = new URL(supabaseUrl).hostname; + if (!LOCAL_HOSTS.has(host)) { + throw new Error( + `E2E_SUPABASE_URL points at ${host}: throwaway users are only provisioned on a local stack. ` + + 'Use E2E_USER_EMAIL / E2E_USER_PASSWORD for a real project.', + ); + } + return { source: { kind: 'provision', supabaseUrl: supabaseUrl.replace(/\/+$/, ''), serviceRoleKey }, reason: '' }; + } + + return { + source: null, + reason: + 'no test user: set E2E_USER_EMAIL + E2E_USER_PASSWORD (or LOREKIT_TESTING_USER_NAME + LOREKIT_TESTING_USER_PW), ' + + 'or E2E_SUPABASE_URL + E2E_SUPABASE_SERVICE_ROLE_KEY for a local stack', + }; +} + +const resolved = resolveUserSource(); + +/** How the `signed-in` project gets a user, or null when it cannot. */ +export const USER_SOURCE: UserSource | null = resolved.source; + +/** Why signed-in tests are skipped (empty when they run). */ +export const NO_USER_REASON = resolved.reason; + +/** + * True when the site under test has a reachable auth backend. A local build + * with no `supabase start` behind it (the default PR job) has none, so tests + * that submit credentials skip there instead of asserting a network error. + */ +export const HAS_AUTH_BACKEND = USER_SOURCE !== null || TARGET !== 'local'; diff --git a/packages/web-e2e/src/support/fixtures.ts b/packages/web-e2e/src/support/fixtures.ts new file mode 100644 index 000000000..f380663bb --- /dev/null +++ b/packages/web-e2e/src/support/fixtures.ts @@ -0,0 +1,95 @@ +import { test as base, expect } from '@playwright/test'; +import { BASE_URL, VERCEL_BYPASS_SECRET } from './env'; + +const ORIGIN = new URL(BASE_URL).origin; + +/** + * Console errors ANY page may emit without failing its test. Every entry needs + * a reason — an allowlist nobody can explain is how real errors get hidden. + * Prefer a per-route `allowConsoleErrors` in routes.ts over growing this list. + */ +const GLOBAL_ALLOWED_PROBLEMS: RegExp[] = []; + +export interface PageHealthOptions { + /** + * Extra problems to tolerate for the current test, matched against the + * rendered problem line (`console.error: ()`, + * `uncaught exception: `, `HTTP `). + */ + allowedConsoleErrors: RegExp[]; +} + +/** + * The suite's `test`: Playwright's, plus two things every spec gets for free. + * + * • A health check. Each test records uncaught page exceptions, console + * errors and same-origin 5xx responses, and fails when any were seen — so + * a hydration error or a crashing component fails the smoke even when the + * heading still rendered. Allow a known one with `allowConsoleErrors` on the + * route (routes.ts) or `test.use({ allowedConsoleErrors: [...] })`. + * • The Vercel bypass header on requests to the site's OWN origin when + * VERCEL_AUTOMATION_BYPASS_SECRET is set. Only that origin: adding it to + * every request (`extraHTTPHeaders`) would put an unexpected header on the + * app's Supabase calls and fail their CORS preflight. + */ +export const test = base.extend({ + allowedConsoleErrors: [[], { option: true }], + + context: async ({ context }, use) => { + const secret = VERCEL_BYPASS_SECRET; + if (secret) { + await context.route( + (url) => url.origin === ORIGIN, + (route) => + route.continue({ + headers: { ...route.request().headers(), 'x-vercel-protection-bypass': secret }, + }), + ); + } + await use(context); + }, + + pageProblems: [ + async ({ page, allowedConsoleErrors }, use, testInfo) => { + const problems: string[] = []; + const allowed = [...GLOBAL_ALLOWED_PROBLEMS, ...allowedConsoleErrors]; + const record = (line: string) => { + if (!allowed.some((re) => re.test(line))) problems.push(line); + }; + + page.on('pageerror', (err) => record(`uncaught exception: ${err.message}`)); + page.on('console', (msg) => { + if (msg.type() !== 'error') return; + const where = msg.location()?.url; + record(`console.error: ${msg.text()}${where ? ` (${where})` : ''}`); + }); + page.on('response', (res) => { + if (res.status() < 500) return; + if (new URL(res.url()).origin !== ORIGIN) return; + record(`HTTP ${res.status()} ${res.request().method()} ${res.url()}`); + }); + + await use(problems); + + if (problems.length === 0) return; + await testInfo.attach('page-problems', { body: problems.join('\n'), contentType: 'text/plain' }); + // Only assert on an otherwise-passing test: a failed test already says + // what went wrong, and the attachment carries the rest. + if (testInfo.status === testInfo.expectedStatus) { + expect( + problems, + 'The page reported errors (listed above). Fix them, or — if one is known and harmless — allow it ' + + 'with `allowConsoleErrors` on the route in src/routes.ts.', + ).toEqual([]); + } + }, + { auto: true }, + ], +}); + +export { expect }; + +/** Headers for `request.*` calls that must get through Vercel Deployment Protection. */ +export function originHeaders(): Record { + return VERCEL_BYPASS_SECRET ? { 'x-vercel-protection-bypass': VERCEL_BYPASS_SECRET } : {}; +} diff --git a/packages/web-e2e/tsconfig.json b/packages/web-e2e/tsconfig.json new file mode 100644 index 000000000..8afcd534a --- /dev/null +++ b/packages/web-e2e/tsconfig.json @@ -0,0 +1,18 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "composite": false, + "declaration": false, + "declarationMap": false, + "emitDeclarationOnly": false, + "noEmit": true, + "module": "esnext", + "moduleResolution": "bundler", + "lib": ["ES2022", "DOM"], + "types": ["node"], + "strict": true, + "noUnusedLocals": true, + "skipLibCheck": true + }, + "include": ["playwright.config.ts", "vitest.config.ts", "src/**/*.ts"] +} diff --git a/packages/web-e2e/vitest.config.ts b/packages/web-e2e/vitest.config.ts new file mode 100644 index 000000000..c05522ba9 --- /dev/null +++ b/packages/web-e2e/vitest.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from 'vitest/config'; + +// Vitest runs ONLY the coverage guard (`src/**/*.test.ts`) — the static check +// that every page in packages/web has an entry in src/routes.ts. It needs no +// browser and no server, so it rides the `check` job's `nx affected -t test` +// on every PR that touches packages/web (see `implicitDependencies` in +// project.json). The browser specs are `*.e2e.ts` and belong to Playwright. +export default defineConfig({ + test: { + reporters: + process.env.GITHUB_ACTIONS === 'true' ? ['default', 'github-actions'] : ['default'], + environment: 'node', + include: ['src/**/*.test.ts'], + }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4e0a5c693..fba1017a8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -189,7 +189,7 @@ importers: version: 1.43.0 '@scalar/nextjs-api-reference': specifier: ^0.11.11 - version: 0.11.11(next@15.5.21(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(sass@1.69.0))(react@19.2.8) + version: 0.11.11(next@15.5.21(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(@playwright/test@1.56.0)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(sass@1.69.0))(react@19.2.8) '@shikijs/rehype': specifier: ^4.4.1 version: 4.4.1 @@ -222,7 +222,7 @@ importers: version: 12.42.2(react-dom@19.2.8(react@19.2.8))(react@19.2.8) next: specifier: ^15.3.8 - version: 15.5.21(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(sass@1.69.0) + version: 15.5.21(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(@playwright/test@1.56.0)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(sass@1.69.0) next-mdx-remote: specifier: ^6.0.0 version: 6.0.0(@types/react@19.2.17)(react@19.2.8) @@ -262,13 +262,13 @@ importers: version: 15.5.21 '@nx/next': specifier: 22.4.0 - version: 22.4.0(@babel/core@7.29.7)(@babel/traverse@7.29.7)(@rspack/core@1.6.8(@swc/helpers@0.5.23))(@swc-node/register@1.9.2(@swc/core@1.5.29(@swc/helpers@0.5.23))(@swc/types@0.1.27)(typescript@5.9.3))(@swc/core@1.5.29(@swc/helpers@0.5.23))(@swc/helpers@0.5.23)(@types/babel__core@7.20.5)(@zkochan/js-yaml@0.0.7)(esbuild@0.28.1)(eslint@8.57.1)(lightningcss@1.32.0)(next@15.5.21(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(sass@1.69.0))(nx@22.4.0(@swc-node/register@1.9.2(@swc/core@1.5.29(@swc/helpers@0.5.23))(@swc/types@0.1.27)(typescript@5.9.3))(@swc/core@1.5.29(@swc/helpers@0.5.23)))(postcss@8.4.30)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(typescript@5.9.3)(vite@7.3.6(@types/node@20.19.9)(jiti@2.7.0)(less@4.7.0)(lightningcss@1.32.0)(sass-embedded@1.100.0)(sass@1.69.0)(terser@5.49.0)(yaml@2.9.0))(vitest@4.0.9)(webpack@5.108.4(@swc/core@1.5.29(@swc/helpers@0.5.23))(esbuild@0.28.1)(lightningcss@1.32.0)(postcss@8.4.30)) + version: 22.4.0(0c83fb54c6721d3f2fb47dce4a18a4be) '@storybook/addon-vitest': specifier: ^10.5.5 version: 10.5.5(@vitest/browser-playwright@4.0.9)(@vitest/browser@4.0.9(msw@2.15.0(@types/node@20.19.9)(typescript@5.9.3))(vite@7.3.6(@types/node@20.19.9)(jiti@2.7.0)(less@4.7.0)(lightningcss@1.32.0)(sass-embedded@1.100.0)(sass@1.69.0)(terser@5.49.0)(yaml@2.9.0))(vitest@4.0.9))(@vitest/runner@4.0.9)(react@19.2.8)(storybook@10.5.5(@types/react@19.2.17)(prettier@3.6.2)(react@19.2.8))(vitest@4.0.9) '@storybook/nextjs-vite': specifier: ^10.5.5 - version: 10.5.5(@babel/core@7.29.7)(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(esbuild@0.28.1)(next@15.5.21(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(sass@1.69.0))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(rollup@4.62.2)(storybook@10.5.5(@types/react@19.2.17)(prettier@3.6.2)(react@19.2.8))(typescript@5.9.3)(vite@7.3.6(@types/node@20.19.9)(jiti@2.7.0)(less@4.7.0)(lightningcss@1.32.0)(sass-embedded@1.100.0)(sass@1.69.0)(terser@5.49.0)(yaml@2.9.0))(webpack@5.108.4(@swc/core@1.5.29(@swc/helpers@0.5.23))(esbuild@0.28.1)(lightningcss@1.32.0)(postcss@8.4.30)) + version: 10.5.5(@babel/core@7.29.7)(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(esbuild@0.28.1)(next@15.5.21(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(@playwright/test@1.56.0)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(sass@1.69.0))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(rollup@4.62.2)(storybook@10.5.5(@types/react@19.2.17)(prettier@3.6.2)(react@19.2.8))(typescript@5.9.3)(vite@7.3.6(@types/node@20.19.9)(jiti@2.7.0)(less@4.7.0)(lightningcss@1.32.0)(sass-embedded@1.100.0)(sass@1.69.0)(terser@5.49.0)(yaml@2.9.0))(webpack@5.108.4(@swc/core@1.5.29(@swc/helpers@0.5.23))(esbuild@0.28.1)(lightningcss@1.32.0)(postcss@8.4.30)) '@storybook/react': specifier: ^10.5.5 version: 10.5.5(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(storybook@10.5.5(@types/react@19.2.17)(prettier@3.6.2)(react@19.2.8))(typescript@5.9.3) @@ -315,6 +315,12 @@ importers: specifier: ^4.1.11 version: 4.3.3 + packages/web-e2e: + devDependencies: + '@playwright/test': + specifier: 1.56.0 + version: 1.56.0 + packages: '@adobe/css-tools@4.5.0': @@ -2214,6 +2220,11 @@ packages: peerDependencies: typescript: ^3 || ^4 || ^5 + '@playwright/test@1.56.0': + resolution: {integrity: sha512-Tzh95Twig7hUwwNe381/K3PggZBZblKUe2wv25oIpzWLr6Z0m4KgV1ZVIjnR6GM9ANEqjZD7XsZEa6JL/7YEgg==} + engines: {node: '>=18'} + hasBin: true + '@polka/url@1.0.0-next.29': resolution: {integrity: sha512-wwQAWhWSuHaag8c4q/KN/vCoeOJYshAIvMQwD4GpSb3OiZklFfvAgmj0VCBBImRpuF/aFgIRzllXlVX93Jevww==} @@ -9467,7 +9478,7 @@ snapshots: - vue-tsc - webpack-cli - '@nx/next@22.4.0(@babel/core@7.29.7)(@babel/traverse@7.29.7)(@rspack/core@1.6.8(@swc/helpers@0.5.23))(@swc-node/register@1.9.2(@swc/core@1.5.29(@swc/helpers@0.5.23))(@swc/types@0.1.27)(typescript@5.9.3))(@swc/core@1.5.29(@swc/helpers@0.5.23))(@swc/helpers@0.5.23)(@types/babel__core@7.20.5)(@zkochan/js-yaml@0.0.7)(esbuild@0.28.1)(eslint@8.57.1)(lightningcss@1.32.0)(next@15.5.21(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(sass@1.69.0))(nx@22.4.0(@swc-node/register@1.9.2(@swc/core@1.5.29(@swc/helpers@0.5.23))(@swc/types@0.1.27)(typescript@5.9.3))(@swc/core@1.5.29(@swc/helpers@0.5.23)))(postcss@8.4.30)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(typescript@5.9.3)(vite@7.3.6(@types/node@20.19.9)(jiti@2.7.0)(less@4.7.0)(lightningcss@1.32.0)(sass-embedded@1.100.0)(sass@1.69.0)(terser@5.49.0)(yaml@2.9.0))(vitest@4.0.9)(webpack@5.108.4(@swc/core@1.5.29(@swc/helpers@0.5.23))(esbuild@0.28.1)(lightningcss@1.32.0)(postcss@8.4.30))': + '@nx/next@22.4.0(0c83fb54c6721d3f2fb47dce4a18a4be)': dependencies: '@babel/plugin-proposal-decorators': 7.29.7(@babel/core@7.29.7) '@nx/devkit': 22.4.0(nx@22.4.0(@swc-node/register@1.9.2(@swc/core@1.5.29(@swc/helpers@0.5.23))(@swc/types@0.1.27)(typescript@5.9.3))(@swc/core@1.5.29(@swc/helpers@0.5.23))) @@ -9481,7 +9492,7 @@ snapshots: copy-webpack-plugin: 10.2.4(webpack@5.108.4(@swc/core@1.5.29(@swc/helpers@0.5.23))(esbuild@0.28.1)(lightningcss@1.32.0)(postcss@8.4.30)) file-loader: 6.2.0(webpack@5.108.4(@swc/core@1.5.29(@swc/helpers@0.5.23))(esbuild@0.28.1)(lightningcss@1.32.0)(postcss@8.4.30)) ignore: 5.3.2 - next: 15.5.21(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(sass@1.69.0) + next: 15.5.21(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(@playwright/test@1.56.0)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(sass@1.69.0) semver: 7.8.5 tslib: 2.8.1 webpack-merge: 5.10.0 @@ -10131,6 +10142,10 @@ snapshots: esquery: 1.7.0 typescript: 5.9.3 + '@playwright/test@1.56.0': + dependencies: + playwright: 1.56.0 + '@polka/url@1.0.0-next.29': {} '@rollup/plugin-babel@6.1.0(@babel/core@7.29.7)(@types/babel__core@7.20.5)(rollup@4.62.2)': @@ -10338,10 +10353,10 @@ snapshots: '@scalar/helpers@0.9.2': {} - '@scalar/nextjs-api-reference@0.11.11(next@15.5.21(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(sass@1.69.0))(react@19.2.8)': + '@scalar/nextjs-api-reference@0.11.11(next@15.5.21(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(@playwright/test@1.56.0)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(sass@1.69.0))(react@19.2.8)': dependencies: '@scalar/client-side-rendering': 0.3.4 - next: 15.5.21(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(sass@1.69.0) + next: 15.5.21(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(@playwright/test@1.56.0)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(sass@1.69.0) react: 19.2.8 '@scalar/schemas@0.7.4': @@ -10453,18 +10468,18 @@ snapshots: dependencies: react: 19.2.8 - '@storybook/nextjs-vite@10.5.5(@babel/core@7.29.7)(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(esbuild@0.28.1)(next@15.5.21(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(sass@1.69.0))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(rollup@4.62.2)(storybook@10.5.5(@types/react@19.2.17)(prettier@3.6.2)(react@19.2.8))(typescript@5.9.3)(vite@7.3.6(@types/node@20.19.9)(jiti@2.7.0)(less@4.7.0)(lightningcss@1.32.0)(sass-embedded@1.100.0)(sass@1.69.0)(terser@5.49.0)(yaml@2.9.0))(webpack@5.108.4(@swc/core@1.5.29(@swc/helpers@0.5.23))(esbuild@0.28.1)(lightningcss@1.32.0)(postcss@8.4.30))': + '@storybook/nextjs-vite@10.5.5(@babel/core@7.29.7)(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(esbuild@0.28.1)(next@15.5.21(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(@playwright/test@1.56.0)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(sass@1.69.0))(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(rollup@4.62.2)(storybook@10.5.5(@types/react@19.2.17)(prettier@3.6.2)(react@19.2.8))(typescript@5.9.3)(vite@7.3.6(@types/node@20.19.9)(jiti@2.7.0)(less@4.7.0)(lightningcss@1.32.0)(sass-embedded@1.100.0)(sass@1.69.0)(terser@5.49.0)(yaml@2.9.0))(webpack@5.108.4(@swc/core@1.5.29(@swc/helpers@0.5.23))(esbuild@0.28.1)(lightningcss@1.32.0)(postcss@8.4.30))': dependencies: '@storybook/builder-vite': 10.5.5(esbuild@0.28.1)(rollup@4.62.2)(storybook@10.5.5(@types/react@19.2.17)(prettier@3.6.2)(react@19.2.8))(vite@7.3.6(@types/node@20.19.9)(jiti@2.7.0)(less@4.7.0)(lightningcss@1.32.0)(sass-embedded@1.100.0)(sass@1.69.0)(terser@5.49.0)(yaml@2.9.0))(webpack@5.108.4(@swc/core@1.5.29(@swc/helpers@0.5.23))(esbuild@0.28.1)(lightningcss@1.32.0)(postcss@8.4.30)) '@storybook/react': 10.5.5(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(storybook@10.5.5(@types/react@19.2.17)(prettier@3.6.2)(react@19.2.8))(typescript@5.9.3) '@storybook/react-vite': 10.5.5(@types/react-dom@19.2.3(@types/react@19.2.17))(@types/react@19.2.17)(esbuild@0.28.1)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(rollup@4.62.2)(storybook@10.5.5(@types/react@19.2.17)(prettier@3.6.2)(react@19.2.8))(typescript@5.9.3)(vite@7.3.6(@types/node@20.19.9)(jiti@2.7.0)(less@4.7.0)(lightningcss@1.32.0)(sass-embedded@1.100.0)(sass@1.69.0)(terser@5.49.0)(yaml@2.9.0))(webpack@5.108.4(@swc/core@1.5.29(@swc/helpers@0.5.23))(esbuild@0.28.1)(lightningcss@1.32.0)(postcss@8.4.30)) - next: 15.5.21(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(sass@1.69.0) + next: 15.5.21(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(@playwright/test@1.56.0)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(sass@1.69.0) react: 19.2.8 react-dom: 19.2.8(react@19.2.8) storybook: 10.5.5(@types/react@19.2.17)(prettier@3.6.2)(react@19.2.8) styled-jsx: 5.1.6(@babel/core@7.29.7)(react@19.2.8) vite: 7.3.6(@types/node@20.19.9)(jiti@2.7.0)(less@4.7.0)(lightningcss@1.32.0)(sass-embedded@1.100.0)(sass@1.69.0)(terser@5.49.0)(yaml@2.9.0) - vite-plugin-storybook-nextjs: 3.3.1(next@15.5.21(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(sass@1.69.0))(storybook@10.5.5(@types/react@19.2.17)(prettier@3.6.2)(react@19.2.8))(typescript@5.9.3)(vite@7.3.6(@types/node@20.19.9)(jiti@2.7.0)(less@4.7.0)(lightningcss@1.32.0)(sass-embedded@1.100.0)(sass@1.69.0)(terser@5.49.0)(yaml@2.9.0)) + vite-plugin-storybook-nextjs: 3.3.1(next@15.5.21(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(@playwright/test@1.56.0)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(sass@1.69.0))(storybook@10.5.5(@types/react@19.2.17)(prettier@3.6.2)(react@19.2.8))(typescript@5.9.3)(vite@7.3.6(@types/node@20.19.9)(jiti@2.7.0)(less@4.7.0)(lightningcss@1.32.0)(sass-embedded@1.100.0)(sass@1.69.0)(terser@5.49.0)(yaml@2.9.0)) optionalDependencies: '@types/react': 19.2.17 '@types/react-dom': 19.2.3(@types/react@19.2.17) @@ -14099,7 +14114,7 @@ snapshots: - '@types/react' - supports-color - next@15.5.21(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(sass@1.69.0): + next@15.5.21(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(@playwright/test@1.56.0)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(sass@1.69.0): dependencies: '@next/env': 15.5.21 '@swc/helpers': 0.5.15 @@ -14118,6 +14133,7 @@ snapshots: '@next/swc-win32-arm64-msvc': 15.5.21 '@next/swc-win32-x64-msvc': 15.5.21 '@opentelemetry/api': 1.9.1 + '@playwright/test': 1.56.0 babel-plugin-react-compiler: 1.0.0 sass: 1.69.0 sharp: 0.34.5 @@ -15901,13 +15917,13 @@ snapshots: '@types/unist': 3.0.3 vfile-message: 4.0.3 - vite-plugin-storybook-nextjs@3.3.1(next@15.5.21(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(sass@1.69.0))(storybook@10.5.5(@types/react@19.2.17)(prettier@3.6.2)(react@19.2.8))(typescript@5.9.3)(vite@7.3.6(@types/node@20.19.9)(jiti@2.7.0)(less@4.7.0)(lightningcss@1.32.0)(sass-embedded@1.100.0)(sass@1.69.0)(terser@5.49.0)(yaml@2.9.0)): + vite-plugin-storybook-nextjs@3.3.1(next@15.5.21(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(@playwright/test@1.56.0)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(sass@1.69.0))(storybook@10.5.5(@types/react@19.2.17)(prettier@3.6.2)(react@19.2.8))(typescript@5.9.3)(vite@7.3.6(@types/node@20.19.9)(jiti@2.7.0)(less@4.7.0)(lightningcss@1.32.0)(sass-embedded@1.100.0)(sass@1.69.0)(terser@5.49.0)(yaml@2.9.0)): dependencies: '@next/env': 16.0.0 image-size: 2.0.2 magic-string: 0.30.21 module-alias: 2.3.4 - next: 15.5.21(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(sass@1.69.0) + next: 15.5.21(@babel/core@7.29.7)(@opentelemetry/api@1.9.1)(@playwright/test@1.56.0)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8)(sass@1.69.0) storybook: 10.5.5(@types/react@19.2.17)(prettier@3.6.2)(react@19.2.8) ts-dedent: 2.3.0 vite: 7.3.6(@types/node@20.19.9)(jiti@2.7.0)(less@4.7.0)(lightningcss@1.32.0)(sass-embedded@1.100.0)(sass@1.69.0)(terser@5.49.0)(yaml@2.9.0) From e37d85f57ceb09a2de9aa3403d5dbc0b03db15d7 Mon Sep 17 00:00:00 2001 From: "mads.thines" Date: Sat, 17 Oct 2026 14:45:48 +0000 Subject: [PATCH 02/28] test(cli): smoke every CLI command and MCP tool, and fail a PR that adds one without a check - packages/cli/test/smoke: a case manifest that runs the real lorekit binary for all 32 commands, in an offline lane (throwaway HOME, no network) and a remote lane (live endpoint, per-run scope, self-cleaning, refuses prod). - cli-smoke.test.mjs runs the offline lane inside cli:test and guards coverage: every command, alias, local stdio MCP tool and hosted MCP tool needs a check. - smoke-mcp-tools.mjs: add the missing memory.protect, policy.update and groom.run coverage, and report which tools each run exercised. Co-authored-by: dash0-dev[bot] <257284812+dash0-dev[bot]@users.noreply.github.com> --- packages/cli/test/cli-smoke.test.mjs | 120 ++++++++ packages/cli/test/smoke/cases.mjs | 404 +++++++++++++++++++++++++++ packages/cli/test/smoke/harness.mjs | 260 +++++++++++++++++ packages/cli/test/smoke/run.mjs | 97 +++++++ scripts/smoke/smoke-mcp-tools.mjs | 56 ++++ 5 files changed, 937 insertions(+) create mode 100644 packages/cli/test/cli-smoke.test.mjs create mode 100644 packages/cli/test/smoke/cases.mjs create mode 100644 packages/cli/test/smoke/harness.mjs create mode 100644 packages/cli/test/smoke/run.mjs diff --git a/packages/cli/test/cli-smoke.test.mjs b/packages/cli/test/cli-smoke.test.mjs new file mode 100644 index 000000000..fb66e90f6 --- /dev/null +++ b/packages/cli/test/cli-smoke.test.mjs @@ -0,0 +1,120 @@ +// The CLI + MCP smoke gate, run on every PR as part of `cli:test`. +// +// Two jobs: +// +// 1. COVERAGE. Every command in src/commands.mjs, every alias, every tool the +// local stdio server dispatches, and every tool the hosted server serves +// must be exercised by a smoke check — or carry a written exemption. A new +// command or catalog tool with no smoke fails here, on the PR that adds it. +// 2. THE OFFLINE LANE. Runs test/smoke/cases.mjs against the real binary in a +// throwaway HOME and project (no network), one subtest per case. +// +// The remote lane (the same cases against a live endpoint) runs at deploy time: +// `node packages/cli/test/smoke/run.mjs --remote `. +// +// WHY HERE. Adding a command touches src/commands.mjs and adding a catalog tool +// regenerates src/surfaces.generated.mjs — both in this package — so this guard +// is always affected by the change it polices (see surface-parity.test.mjs for +// what goes wrong when a guard lives elsewhere). + +import { test, describe } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; + +import { MCP_TOOL_NAMES } from '../src/surfaces.generated.mjs'; +import { CASES, LOCAL_MCP_CALLED, LOCAL_MCP_TOOLS, NOT_SMOKED } from './smoke/cases.mjs'; +import { runSmoke } from './smoke/run.mjs'; + +const read = (rel) => readFileSync(fileURLToPath(new URL(rel, import.meta.url)), 'utf8'); + +/** + * `{ name, aliases }` for every entry of `COMMANDS` in src/commands.mjs. A + * source scan, like surface-parity.test.mjs: importing the registry would pull + * the whole CLI module graph into this process. + */ +function parseCommands(source) { + const block = /export const COMMANDS = \[([\s\S]*?)\n\];/.exec(source); + assert.ok(block, 'could not find `export const COMMANDS = [` in src/commands.mjs'); + return [...block[1].matchAll(/\{\s*name:\s*'([^']+)'([^\n]*)\}/g)].map(([, name, rest]) => ({ + name, + aliases: [...(/aliases:\s*\[([^\]]*)\]/.exec(rest)?.[1] ?? '').matchAll(/'([^']+)'/g)].map((m) => m[1]), + })); +} + +const commands = parseCommands(read('../src/commands.mjs')); +const casesSource = read('./smoke/cases.mjs'); +const smoked = new Set(CASES.map((c) => c.command)); + +describe('CLI smoke coverage', () => { + test('finds the command registry', () => { + assert.ok(commands.length >= 20, `parsed only ${commands.length} commands from src/commands.mjs`); + }); + + test('every command has a smoke case or a written exemption', () => { + const missing = commands.map((c) => c.name).filter((n) => !smoked.has(n) && !NOT_SMOKED[n]); + assert.deepEqual( + missing, + [], + `No smoke case for: ${missing.join(', ')}. Add one to packages/cli/test/smoke/cases.mjs ` + + '(or, if it genuinely cannot be smoked, an entry with the reason in NOT_SMOKED).', + ); + }); + + test('every case and exemption names a real command', () => { + const names = new Set(commands.map((c) => c.name)); + const unknown = [...smoked, ...Object.keys(NOT_SMOKED)].filter((n) => !names.has(n)); + assert.deepEqual(unknown, [], `cases.mjs names commands that do not exist: ${unknown.join(', ')}`); + for (const [name, reason] of Object.entries(NOT_SMOKED)) { + assert.ok(String(reason).trim().length > 0, `NOT_SMOKED.${name} needs a reason`); + assert.ok(!smoked.has(name), `${name} is both smoked and exempt — drop the exemption`); + } + }); + + test('every alias is invoked by a case', () => { + const aliases = commands.flatMap((c) => c.aliases); + const missing = aliases.filter((a) => !casesSource.includes(`'${a}'`)); + assert.deepEqual(missing, [], `aliases no case invokes: ${missing.join(', ')}`); + }); + + test('every case runs in at least one lane', () => { + const laneless = CASES.filter((c) => c.lanes.length === 0).map((c) => `${c.command} — ${c.name}`); + assert.deepEqual(laneless, []); + }); +}); + +describe('MCP smoke coverage', () => { + test('the offline `mcp` case calls every tool the local stdio server dispatches', () => { + assert.deepEqual( + [...LOCAL_MCP_CALLED].sort(), + [...LOCAL_MCP_TOOLS].sort(), + 'LOCAL_MCP_CALLED in smoke/cases.mjs drifted from the catalog (MCP_TOOL_NAMES minus LOCAL_MCP_EXEMPT)', + ); + const notInSession = LOCAL_MCP_CALLED.filter((t) => !casesSource.includes(`name: '${t}'`)); + assert.deepEqual(notInSession, [], `listed in LOCAL_MCP_CALLED but never called: ${notInSession.join(', ')}`); + }); + + test('scripts/smoke/smoke-mcp-tools.mjs calls every tool the hosted server serves', () => { + // The hosted smoke runs at deploy time (deploy.yml smoke-preview, preview.yml) + // against a real project. This makes its coverage a PR-time guarantee. + const hosted = read('../../../scripts/smoke/smoke-mcp-tools.mjs'); + const missing = MCP_TOOL_NAMES.filter( + (t) => !new RegExp(`(call\\(|name: )'${t.replace('.', '\\.')}'`).test(hosted), + ); + assert.deepEqual( + missing, + [], + `No hosted smoke check calls: ${missing.join(', ')}. Add a check() to scripts/smoke/smoke-mcp-tools.mjs.`, + ); + }); +}); + +test('CLI smoke — offline lane against the real binary', async (t) => { + const { results } = await runSmoke({ lane: 'offline' }); + assert.ok(results.length > 0, 'no offline cases ran'); + for (const r of results) { + await t.test(r.label, () => { + if (r.outcome === 'fail') assert.fail(r.detail); + }); + } +}); diff --git a/packages/cli/test/smoke/cases.mjs b/packages/cli/test/smoke/cases.mjs new file mode 100644 index 000000000..a1e4e7fd3 --- /dev/null +++ b/packages/cli/test/smoke/cases.mjs @@ -0,0 +1,404 @@ +// The CLI smoke cases — one scripted session per lane that runs the REAL +// `lorekit` binary for every command it ships. +// +// offline a throwaway HOME + project, LOREKIT_MODE=local, no network. Runs on +// every PR inside `cli:test` (test/cli-smoke.test.mjs). +// remote the same binary against a live endpoint + token (the preview +// deploy). Writes only under the run's own project scope and +// hard-deletes what it wrote (run.mjs). Refuses production. +// +// ## Adding a command +// +// `cli-smoke.test.mjs` fails when a command in `src/commands.mjs` has no case +// here and no entry in NOT_SMOKED. Add a `smoke('', '', +// [lanes], async ({ cli, ctx }) => { ... })` below, in session order. Assert on +// behaviour (what changed, what came back), not just "exit 0". +// +// Cases run in order and may build on each other (a lesson written early is +// read later) — the session is the realistic shape, and every assertion names +// the exact command that broke. + +import assert from 'node:assert/strict'; +import { existsSync, readFileSync } from 'node:fs'; +import path from 'node:path'; + +import { LOCAL_MCP_EXEMPT, MCP_TOOL_NAMES } from '../../src/surfaces.generated.mjs'; +import { runMcp, trackRemoteWrite } from './harness.mjs'; + +/** @typedef {'offline' | 'remote'} Lane */ + +/** @type {{ command: string, name: string, lanes: Lane[], fn: Function }[]} */ +export const CASES = []; + +/** + * Commands with no smoke case, and why. A declared exemption, so a gap is a + * reviewable decision rather than an oversight. Empty today — keep it that way. + * @type {Record} + */ +export const NOT_SMOKED = {}; + +/** Register a case. `command` is the canonical name from src/commands.mjs (aliases are credited to it). */ +function smoke(command, name, lanes, fn) { + CASES.push({ command, name, lanes, fn }); +} + +const OFFLINE = /** @type {Lane[]} */ (['offline']); +const REMOTE = /** @type {Lane[]} */ (['remote']); +const BOTH = /** @type {Lane[]} */ (['offline', 'remote']); + +/** The tools the local stdio server should advertise: the catalog minus its declared exemptions. */ +export const LOCAL_MCP_TOOLS = MCP_TOOL_NAMES.filter((t) => !(t in LOCAL_MCP_EXEMPT)); + +/** The tools the offline `mcp` case calls — the coverage guard checks this against LOCAL_MCP_TOOLS. */ +export const LOCAL_MCP_CALLED = [ + 'memory.write', + 'memory.read', + 'memory.list', + 'memory.search', + 'memory.scopes', + 'memory.archive', + 'memory.restore', + 'memory.delete', + 'org.create', + 'org.list', + 'org.rename', + 'org.delete', +]; + +// ── helpers ────────────────────────────────────────────────────────────────── + +/** The store block a lane reads back from: `offline` locally, `remote` against an endpoint. */ +const storeOf = (ctx) => (ctx.lane === 'remote' ? 'remote' : 'offline'); + +/** Every `key` under a JSON block, however the command groups its entries. */ +function keysIn(value) { + const keys = []; + const walk = (v) => { + if (Array.isArray(v)) v.forEach(walk); + else if (v && typeof v === 'object') { + if (typeof v.key === 'string') keys.push(v.key); + Object.values(v).forEach(walk); + } + }; + walk(value); + return [...new Set(keys)]; +} + +const alphaValue = (ctx) => `The CLI smoke suite wrote this lesson — marker zebra-${ctx.runId}.`; + +/** Write through the CLI and record it for the remote lane's cleanup. */ +function writeLesson(cli, ctx, key, value, extra = []) { + trackRemoteWrite(ctx, ctx.scope, key); + return cli.json(['write', `${ctx.scope}::${key}`, value, ...extra]); +} + +// ── setup: install, health ─────────────────────────────────────────────────── + +smoke('install', 'scaffolds the skills for this project without prompting', OFFLINE, async ({ cli, ctx }) => { + cli.ok(['install', '--project', '--no-hooks', '--yes']); + assert.ok( + existsSync(path.join(ctx.dir, '.claude/skills/lorekit-memory/SKILL.md')), + 'install did not write .claude/skills/lorekit-memory/SKILL.md', + ); +}); + +smoke('install', 'wires the given endpoint into the project MCP config', REMOTE, async ({ cli, ctx }) => { + cli.ok(['install', '--project', '--no-hooks', '--yes', '--endpoint', ctx.endpoint, '--token', ctx.token]); + const mcp = readFileSync(path.join(ctx.dir, '.mcp.json'), 'utf8'); + assert.ok(mcp.includes(new URL(ctx.endpoint).host), `.mcp.json does not name ${ctx.endpoint}:\n${mcp}`); +}); + +smoke('update', 'reports freshly installed skills as up to date', OFFLINE, async ({ cli }) => { + assert.match(cli.ok(['update', '--check']), /already (at the shipped version|up to date)/); +}); + +smoke('doctor', 'passes on a fresh local install', OFFLINE, async ({ cli }) => { + const out = cli.ok(['doctor', '--mode', 'local']); + assert.match(out, /PASS home store/); + assert.doesNotMatch(out, /\bFAIL\b/); +}); + +smoke('doctor', 'completes a write → read → delete round-trip against the endpoint', REMOTE, async ({ cli }) => { + const out = cli.ok(['doctor', '--deep']); + assert.doesNotMatch(out, /\bFAIL\b/); +}); + +// ── write and read back ────────────────────────────────────────────────────── + +smoke('write', 'stores a lesson from a positional value', BOTH, async ({ cli, ctx }) => { + const out = writeLesson(cli, ctx, 'alpha', alphaValue(ctx), ['--tags', 'smoke,cli']); + assert.equal(out.store, ctx.lane === 'remote' ? 'remote' : 'local'); + assert.equal(out.scope, ctx.scope); + assert.equal(out.key, 'alpha'); + assert.deepEqual(out.tags, ['smoke', 'cli']); +}); + +smoke('write', 'stores a lesson piped on stdin', BOTH, async ({ cli, ctx }) => { + trackRemoteWrite(ctx, ctx.scope, 'beta'); + const out = cli.json(['write', `${ctx.scope}::beta`], { input: 'short body\n' }); + assert.equal(out.value, 'short body'); +}); + +smoke('show', 'returns the full lesson that was written', BOTH, async ({ cli, ctx }) => { + const out = cli.json(['show', `${ctx.scope}::alpha`]); + const block = out[storeOf(ctx)]; + assert.equal(block.found, true, `alpha not found in the ${storeOf(ctx)} store`); + assert.equal(block.record.value, alphaValue(ctx)); +}); + +smoke('list', 'lists the lessons that apply to this directory (also as `ls`)', BOTH, async ({ cli, ctx }) => { + for (const cmd of ctx.lane === 'offline' ? ['list', 'ls'] : ['list']) { + const out = cli.json([cmd]); + const keys = keysIn(out[storeOf(ctx)]); + assert.ok(keys.includes('alpha') && keys.includes('beta'), `\`${cmd}\` is missing alpha/beta: ${keys}`); + } +}); + +smoke('search', 'finds a lesson by a word in its body (also as `grep`)', BOTH, async ({ cli, ctx }) => { + for (const cmd of ctx.lane === 'offline' ? ['search', 'grep'] : ['search']) { + const out = cli.json([cmd, `zebra-${ctx.runId}`]); + assert.deepEqual(keysIn(out[storeOf(ctx)]), ['alpha'], `\`${cmd}\` should match alpha only`); + } +}); + +smoke('stats', 'counts the lessons per scope', BOTH, async ({ cli, ctx }) => { + const out = cli.json(['stats']); + assert.ok(out[storeOf(ctx)].total >= 2, `expected at least 2 lessons, got ${out[storeOf(ctx)].total}`); +}); + +smoke('scopes', 'inventories the scopes that hold lessons', BOTH, async ({ cli, ctx }) => { + const out = cli.json(['scopes']); + if (ctx.lane === 'offline') { + assert.ok(JSON.stringify(out.offline.scopes).includes(ctx.scope), `${ctx.scope} missing from scopes`); + } else { + // The remote cannot enumerate scopes; the command says so instead of failing. + assert.equal(typeof out.remote, 'object'); + } +}); + +smoke('tree', 'shows which lesson wins for this directory (also as `resolve`)', BOTH, async ({ cli, ctx }) => { + for (const cmd of ctx.lane === 'offline' ? ['tree', 'resolve'] : ['tree']) { + const out = cli.json([cmd]); + assert.ok(keysIn(out[storeOf(ctx)].winners).includes('alpha'), `\`${cmd}\` lists no winning alpha`); + } +}); + +smoke('lint', 'fails the gate on a too-short lesson and names it', BOTH, async ({ cli, ctx }) => { + // beta's body is deliberately short — lint is documented as a CI gate, so it must exit non-zero. + const out = cli.json(['lint'], { code: 1 }); + const findings = JSON.stringify(out[storeOf(ctx)]); + assert.match(findings, /"key":"beta"[^}]*"rule":"short-value"/, `no short-value finding for beta: ${findings}`); +}); + +smoke('dedupe', 'clusters a near-duplicate lesson', BOTH, async ({ cli, ctx }) => { + writeLesson(cli, ctx, 'alpha-copy', alphaValue(ctx)); + const out = cli.json(['dedupe']); + const clusters = JSON.stringify(out[storeOf(ctx)].clusters); + assert.ok(clusters.includes('alpha-copy') && clusters.includes('"alpha"'), `alpha/alpha-copy not clustered: ${clusters}`); +}); + +smoke('delete', 'hard-deletes with --force (also as `rm`)', BOTH, async ({ cli, ctx }) => { + const cmd = ctx.lane === 'offline' ? 'rm' : 'delete'; + const out = cli.json([cmd, `${ctx.scope}::alpha-copy`, '--force']); + assert.equal(out.ok, true); + assert.equal(out.force, true); + // `show` exits 1 for a lesson that is not there — the deletion is the point. + assert.equal(cli.json(['show', `${ctx.scope}::alpha-copy`], { code: 1 })[storeOf(ctx)].found, false); +}); + +smoke('diff', 'reports offline-vs-remote divergence only when both stores are readable', BOTH, async ({ cli, ctx }) => { + const out = cli.json(['diff']); + assert.equal(out.comparable, ctx.lane === 'remote', JSON.stringify(out).slice(0, 400)); +}); + +// ── archive lifecycle ──────────────────────────────────────────────────────── + +smoke('archive', 'hides a lesson from the default listing', BOTH, async ({ cli, ctx }) => { + cli.ok(['archive', `${ctx.scope}::beta`]); + assert.ok(!keysIn(cli.json(['list'])[storeOf(ctx)]).includes('beta'), 'beta still listed after archive'); +}); + +smoke('restore', 'brings an archived lesson back', BOTH, async ({ cli, ctx }) => { + cli.ok(['restore', `${ctx.scope}::beta`]); + assert.ok(keysIn(cli.json(['list'])[storeOf(ctx)]).includes('beta'), 'beta not listed after restore'); +}); + +// ── tools that read the store but change nothing ───────────────────────────── + +smoke('obligations', 'maps a changed file to the partners it obliges', BOTH, async ({ cli }) => { + const out = cli.json(['obligations', 'packages/schemas/src/shared/tool-catalog.ts']); + assert.ok(Array.isArray(out.matched) && out.matched.length > 0, 'tool-catalog.ts matched no obligation'); +}); + +smoke('invariants', 'scans for compile-pipeline candidates', BOTH, async ({ cli, ctx }) => { + const out = cli.json(['invariants', 'candidates']); + assert.equal(out[storeOf(ctx)].available, true); + assert.ok(Array.isArray(out[storeOf(ctx)].candidates)); +}); + +smoke('link', 'builds a dashboard deep link to one lesson (also as `url`)', OFFLINE, async ({ cli, ctx }) => { + for (const cmd of ['link', 'url']) { + const out = cli.json([cmd, `${ctx.scope}::alpha`]); + assert.ok(out.url.startsWith('https://lorekit.io/lore?'), out.url); + assert.deepEqual(out.params.lesson, { scope: ctx.scope, key: 'alpha' }); + } +}); + +smoke('migrate', 'previews a move between stores without writing', BOTH, async ({ cli, ctx }) => { + const from = path.join(ctx.home, '.lorekit'); + const to = ctx.lane === 'remote' ? 'remote' : 'project'; + if (ctx.lane === 'remote') { + // Give the dry run something to report: one lesson in the local home tier. + cli.ok(['write', `${ctx.scope}::local-only`, 'A lesson that only exists offline.', '--local']); + } + const out = cli.ok(['migrate', '--from', from, '--to', to]); + assert.match(out, /dry-run/); +}); + +smoke('bootstrap', 'explains that it only applies to a bring-your-own database', OFFLINE, async ({ cli }) => { + assert.match(cli.ok(['bootstrap', '--yes']), /No BYOD storage configured/); +}); + +smoke('completion', 'prints zsh and fish completion scripts', OFFLINE, async ({ cli }) => { + assert.match(cli.ok(['completion', 'zsh']), /^#compdef lorekit/); + assert.match(cli.ok(['completion', 'fish']), /complete -c lorekit/); +}); + +smoke('hook', 'injects the applicable lessons at SessionStart', BOTH, async ({ cli, ctx }) => { + const payload = JSON.stringify({ hook_event_name: 'SessionStart', cwd: ctx.dir, session_id: `smoke-${ctx.runId}` }); + const out = cli.ok(['hook', '--adapter', 'claude', '--event', 'SessionStart'], { input: payload }); + const hook = JSON.parse(out); + assert.equal(hook.hookSpecificOutput?.hookEventName, 'SessionStart'); + assert.ok(hook.hookSpecificOutput.additionalContext.includes(`zebra-${ctx.runId}`), 'alpha was not injected'); +}); + +// ── the local stdio MCP server ─────────────────────────────────────────────── + +smoke('mcp', 'serves the catalog over stdio and answers every memory tool', OFFLINE, async ({ ctx }) => { + const s = ctx.scope; + const session = runMcp(ctx, [ + { name: 'memory.write', arguments: { scope: s, key: 'gamma', value: 'Written over the stdio MCP server.' } }, + { name: 'memory.read', arguments: { scope: s, key: 'gamma' } }, + { name: 'memory.list', arguments: { scope: s } }, + // The local server searches the scopes it is given (mcp-server.mjs: list and + // search "need a scope (or a scope list) up front"). + { name: 'memory.search', arguments: { q: 'stdio MCP server', scopes: [s] } }, + { name: 'memory.scopes', arguments: {} }, + { name: 'memory.archive', arguments: { scope: s, key: 'gamma' } }, + { name: 'memory.restore', arguments: { scope: s, key: 'gamma' } }, + { name: 'memory.delete', arguments: { scope: s, key: 'gamma', force: true } }, + // org.* needs a remote; offline they must refuse and say how to fix it. + { name: 'org.create', arguments: { slug: 'smoke-org', name: 'Smoke' } }, + { name: 'org.list', arguments: {} }, + { name: 'org.rename', arguments: { slug: 'smoke-org', name: 'Renamed' } }, + { name: 'org.delete', arguments: { slug: 'smoke-org' } }, + ]); + assert.equal(session.code, 0, `lorekit mcp exited ${session.code}: ${session.stderr}`); + assert.equal(session.serverInfo?.name, 'lorekit-local'); + assert.deepEqual([...session.tools].sort(), [...LOCAL_MCP_TOOLS].sort(), 'advertised tools drifted from the catalog'); + + const [write, read, list, search, scopes, archive, restore, del, ...orgs] = session.results; + for (const [name, r] of Object.entries({ write, read, list, search, scopes, archive, restore, delete: del })) { + assert.equal(r.ok, true, `memory.${name} was refused: ${JSON.stringify(r.error)}`); + } + assert.ok(JSON.stringify(read.value).includes('Written over the stdio MCP server.'), 'memory.read lost the body'); + assert.ok(keysIn(list.value).includes('gamma'), 'memory.list is missing gamma'); + assert.ok(keysIn(search.value).includes('gamma'), 'memory.search did not find gamma'); + assert.ok(JSON.stringify(scopes.value).includes(s), 'memory.scopes is missing the run scope'); + for (const r of orgs) { + assert.equal(r.ok, false, 'an org.* tool succeeded with no remote'); + assert.match(String(r.error), /remote LoreKit endpoint/); + } +}); + +smoke('mcp', 'passes tools through to the endpoint over stdio', REMOTE, async ({ ctx }) => { + const session = runMcp(ctx, [{ name: 'memory.read', arguments: { scope: ctx.scope, key: 'alpha' } }]); + assert.equal(session.code, 0, `lorekit mcp exited ${session.code}: ${session.stderr}`); + assert.ok(session.tools.includes('memory.read'), `tools/list: ${session.tools}`); + assert.equal(session.results[0].ok, true, JSON.stringify(session.results[0].error)); + assert.ok(JSON.stringify(session.results[0].value).includes(`zebra-${ctx.runId}`), 'memory.read lost the body'); +}); + +// ── remote-only commands ───────────────────────────────────────────────────── + +// Offline, each must refuse with an actionable message rather than pretend to +// work. Against an endpoint, each must actually do its job. +const REMOTE_ONLY = [ + ['protect', (ctx) => ['protect', `${ctx.scope}::alpha`]], + ['pin', (ctx) => ['pin', `${ctx.scope}::alpha`]], + ['unpin', (ctx) => ['unpin', `${ctx.scope}::alpha`]], + ['policy', () => ['policy', 'list']], + ['groom', (ctx) => ['groom', '--scope', ctx.scope]], + ['purge', () => ['purge', '--yes']], + ['purge-expired', () => ['purge-expired', '--yes']], +]; +for (const [command, args] of REMOTE_ONLY) { + smoke(command, 'refuses offline and names what is missing', OFFLINE, async ({ cli, ctx }) => { + cli.fails(args(ctx), /remote store is not configured/); + }); +} + +smoke('protect', 'protects and unprotects a lesson', REMOTE, async ({ cli, ctx }) => { + assert.equal(cli.json(['protect', `${ctx.scope}::alpha`]).protected, true); + assert.equal(cli.json(['protect', `${ctx.scope}::alpha`, '--off']).protected, false); +}); + +smoke('pin', 'is shorthand for protect', REMOTE, async ({ cli, ctx }) => { + assert.equal(cli.json(['pin', `${ctx.scope}::alpha`]).protected, true); +}); + +smoke('policy', 'creates, lists, updates and deletes a retention policy', REMOTE, async ({ cli, ctx }) => { + const created = cli.json(['policy', 'create', '--scope', ctx.scope, '--name', `cli smoke ${ctx.runId}`, '--tags', 'groom-me']); + const id = created.policy?.id; + assert.ok(id, `policy create returned no id: ${JSON.stringify(created)}`); + ctx.policies.add(id); + + assert.ok(cli.json(['policy', 'list']).entries.some((p) => p.id === id), 'the new policy is not listed'); + const renamed = `cli smoke ${ctx.runId} renamed`; + assert.equal(cli.json(['policy', 'update', id, '--name', renamed]).policy?.name, renamed); + ctx.state.policyId = id; +}); + +smoke('groom', 'archives what a policy matches, but never a protected lesson', REMOTE, async ({ cli, ctx }) => { + writeLesson(cli, ctx, 'groom-target', 'A lesson the smoke policy should archive.', ['--tags', 'groom-me']); + writeLesson(cli, ctx, 'groom-keeper', 'A protected lesson the smoke policy must not touch.', ['--tags', 'groom-me']); + cli.json(['protect', `${ctx.scope}::groom-keeper`]); + + const preview = cli.json(['groom', '--policy-id', ctx.state.policyId]); + assert.deepEqual(keysIn(preview.keys), ['groom-target'], `preview: ${JSON.stringify(preview.keys)}`); + + const run = cli.json(['groom', '--policy-id', ctx.state.policyId, '--run', '--yes']); + assert.equal(run.archived, 1); + assert.equal(cli.json(['show', `${ctx.scope}::groom-keeper`]).remote.found, true, 'groom archived a protected lesson'); +}); + +smoke('unpin', 'is shorthand for protect --off', REMOTE, async ({ cli, ctx }) => { + assert.equal(cli.json(['unpin', `${ctx.scope}::groom-keeper`]).protected, false); + // Delete the policy here rather than leave it to cleanup, so `policy delete` is exercised too. + assert.equal(cli.json(['policy', 'delete', ctx.state.policyId, '--yes']).deleted, true); + ctx.policies.delete(ctx.state.policyId); +}); + +smoke('purge-expired', 'sweeps expired lessons for this account', REMOTE, async ({ cli, ctx }) => { + if (ctx.tier !== 'api_key') return 'skipped: account-wide sweep needs a user-scoped lk_* token'; + assert.equal(cli.json(['purge-expired', '--yes']).ok, true); +}); + +smoke('purge', 'purges archived lessons past the retention window', REMOTE, async ({ cli, ctx }) => { + if (ctx.tier !== 'api_key') return 'skipped: account-wide sweep needs a user-scoped lk_* token'; + // 365 days: only archived rows older than a year — nothing this run created. + assert.equal(cli.json(['purge', '--retention-days', '365', '--yes']).ok, true); +}); + +// ── teardown ───────────────────────────────────────────────────────────────── + +smoke('delete', 'archives without --force', BOTH, async ({ cli, ctx }) => { + const out = cli.json(['delete', `${ctx.scope}::beta`]); + assert.equal(out.ok, true); + assert.equal(out.force, false); +}); + +smoke('uninstall', 'removes what install wrote', BOTH, async ({ cli, ctx }) => { + cli.ok(['uninstall', '--project', '--yes']); + assert.ok(!existsSync(path.join(ctx.dir, '.claude/skills/lorekit-memory')), 'the skill is still installed'); +}); diff --git a/packages/cli/test/smoke/harness.mjs b/packages/cli/test/smoke/harness.mjs new file mode 100644 index 000000000..425063c02 --- /dev/null +++ b/packages/cli/test/smoke/harness.mjs @@ -0,0 +1,260 @@ +// The CLI smoke harness: a throwaway sandbox per run, a way to run the REAL +// `bin/lorekit.mjs` in it, a stdio-MCP driver, and the remote cleanup ledger. +// +// Nothing here knows which commands exist — that is `cases.mjs`'s job. Not a +// test file (no `.test.mjs`), so `node --test test/*.test.mjs` never runs it on +// its own; `cli-smoke.test.mjs` and `run.mjs` import it. +// +// Zero dependencies, like the CLI it drives. + +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdirSync, mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; + +export const CLI_BIN = fileURLToPath(new URL('../../bin/lorekit.mjs', import.meta.url)); + +/** The production Supabase project (CLAUDE.md → Endpoints). The remote lane WRITES, so it refuses this. */ +export const PRODUCTION_REF = 'pqokxlhvnosogizsjztg'; + +// Operator configuration that must never leak into a smoke run: an ambient +// LOREKIT_TOKEN / LOREKIT_MCP_URL usually points at production, and a BYOD +// LOREKIT_STORAGE_* would retarget `bootstrap`. Telemetry knobs +// (LOREKIT_TELEMETRY_TOKEN, LOREKIT_CORRELATION_ID, DEPLOYMENT_ENVIRONMENT) are +// kept, so a deploy-time run stays observable the way the other smokes are. +const STRIPPED_ENV = /^(LOREKIT_(MCP_URL|TOKEN|MODE|HOME|STORE|APP_URL|STORAGE_.*)|CLAUDE_.*)$/; + +function baseEnv() { + return Object.fromEntries(Object.entries(process.env).filter(([k]) => !STRIPPED_ENV.test(k))); +} + +/** + * Which auth tier a credential belongs to, from the string alone — the same + * rule scripts/smoke/smoke-mcp-tools.mjs uses. `purge` / `purge-expired` are + * account-wide sweeps: scoped to ONE user on an `lk_*` token, but to nobody on a + * service-role key, where they would reach every account on a shared project. + */ +export function credentialTier(token) { + if (!token) return 'none'; + if (token.startsWith('lk_')) return 'api_key'; + try { + const claims = JSON.parse(Buffer.from(token.split('.')[1], 'base64url').toString('utf8')); + return claims.role === 'service_role' ? 'service' : 'user'; + } catch { + return 'service'; + } +} + +/** + * A fresh sandbox: its own HOME (so `~/.claude`, `~/.lorekit` and shell rc files + * are throwaway) and a git-initialised project directory whose name gives the + * run a unique project scope, `project::cli-smoke-`. + */ +export function createContext({ lane, endpoint = null, token = null }) { + const runId = `${Date.now().toString(36)}${Math.random().toString(36).slice(2, 6)}`; + const root = mkdtempSync(path.join(tmpdir(), 'lorekit-cli-smoke-')); + const home = path.join(root, 'home'); + const dir = path.join(root, `cli-smoke-${runId}`); + mkdirSync(home, { recursive: true }); + mkdirSync(dir, { recursive: true }); + spawnSync('git', ['init', '-q'], { cwd: dir }); + return { + lane, + runId, + root, + home, + dir, + scope: `project::cli-smoke-${runId}`, + endpoint, + token, + tier: credentialTier(token), + /** Values a case hands to a later case (a policy id, say). */ + state: {}, + /** [scope, key] pairs written to the REMOTE store — hard-deleted in cleanup. */ + written: new Map(), + /** Retention-policy ids created on the remote — deleted in cleanup. */ + policies: new Set(), + }; +} + +export function disposeContext(ctx) { + rmSync(ctx.root, { recursive: true, force: true }); +} + +/** Run `lorekit ` in the sandbox. Never throws on a non-zero exit. */ +export function runCli(ctx, args, { input, env = {} } = {}) { + const remote = ctx.lane === 'remote'; + const res = spawnSync(process.execPath, [CLI_BIN, ...args], { + cwd: ctx.dir, + input, + encoding: 'utf8', + timeout: 90_000, + env: { + ...baseEnv(), + HOME: ctx.home, + USERPROFILE: ctx.home, + LOREKIT_HOME: path.join(ctx.home, '.lorekit'), + LOREKIT_MODE: remote ? 'remote' : 'local', + LOREKIT_MCP_URL: remote ? ctx.endpoint : '', + LOREKIT_TOKEN: remote ? ctx.token : '', + CI: '1', + NO_COLOR: '1', + ...env, + }, + }); + return { + args, + code: res.status, + stdout: res.stdout ?? '', + stderr: res.stderr ?? '', + output: `${res.stdout ?? ''}${res.stderr ?? ''}`, + error: res.error, + }; +} + +function describe(r) { + const tail = r.output.length > 1500 ? `…${r.output.slice(-1500)}` : r.output; + return `\`lorekit ${r.args.join(' ')}\` exited ${r.code}${r.error ? ` (${r.error.message})` : ''}\n${tail}`; +} + +/** + * The assertion helpers every case receives. Each one runs the CLI, checks the + * exit code, and says exactly which command did what when it does not hold. + */ +export function cliHelpers(ctx) { + const run = (args, opts) => runCli(ctx, args, opts); + return { + run, + /** Exit 0; returns stdout + stderr. */ + ok(args, opts) { + const r = run(args, opts); + assert.equal(r.code, 0, describe(r)); + return r.output; + }, + /** Exit `code` (default 0) with `--json` appended; returns the parsed stdout. */ + json(args, { code = 0, ...opts } = {}) { + const r = run([...args, '--json'], opts); + assert.equal(r.code, code, describe(r)); + try { + return JSON.parse(r.stdout); + } catch { + assert.fail(`stdout is not JSON — ${describe(r)}`); + } + }, + /** A non-zero exit whose output matches `pattern`: the command refused, and said why. */ + fails(args, pattern, opts) { + const r = run(args, opts); + assert.notEqual(r.code, 0, `expected a non-zero exit — ${describe(r)}`); + assert.match(r.output, pattern, describe(r)); + return r.output; + }, + }; +} + +/** + * Drive `lorekit mcp` (the local stdio MCP server) through one session: + * initialize, tools/list, then each call in order. Returns the advertised tool + * names and each call's unwrapped result — `{ ok, value }` mirroring how + * smoke-mcp-tools.mjs reads a `tools/call` answer (an `isError` result and a + * JSON-RPC error both count as refused). + */ +export function runMcp(ctx, calls) { + const frames = [ + { jsonrpc: '2.0', id: 1, method: 'initialize', params: {} }, + { jsonrpc: '2.0', id: 2, method: 'tools/list' }, + ...calls.map((c, i) => ({ + jsonrpc: '2.0', + id: i + 3, + method: 'tools/call', + params: { name: c.name, arguments: c.arguments ?? {} }, + })), + ]; + const r = runCli(ctx, ['mcp'], { input: `${frames.map((f) => JSON.stringify(f)).join('\n')}\n` }); + const byId = new Map(); + for (const line of r.stdout.split('\n')) { + if (!line.trim()) continue; + try { + const msg = JSON.parse(line); + if (msg.id !== undefined) byId.set(msg.id, msg); + } catch { + // stdout must carry ONLY JSON-RPC frames; anything else breaks every client. + assert.fail(`\`lorekit mcp\` wrote a non-JSON line to stdout: ${line.slice(0, 200)}`); + } + } + const unwrap = (msg) => { + if (!msg) return { ok: false, value: null, error: 'no response' }; + if (msg.error) return { ok: false, value: null, error: msg.error.message }; + const text = msg.result?.content?.[0]?.text; + let value = text; + try { + value = JSON.parse(text); + } catch { + /* non-JSON text is returned verbatim */ + } + const refused = msg.result?.isError === true || (value && typeof value === 'object' && value.ok === false); + return { ok: !refused, value, error: refused ? (value?.error ?? text) : null }; + }; + return { + code: r.code, + stderr: r.stderr, + serverInfo: byId.get(1)?.result?.serverInfo ?? null, + tools: (byId.get(2)?.result?.tools ?? []).map((t) => t.name), + results: calls.map((_, i) => unwrap(byId.get(i + 3))), + }; +} + +// ── remote cleanup ─────────────────────────────────────────────────────────── + +async function rpc(ctx, name, args) { + const res = await fetch(ctx.endpoint, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json, text/event-stream', + Authorization: `Bearer ${ctx.token}`, + }, + body: JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'tools/call', params: { name, arguments: args } }), + signal: AbortSignal.timeout(20_000), + }); + const json = await res.json().catch(() => null); + const text = json?.result?.content?.[0]?.text; + try { + return { ok: !json?.error && !json?.result?.isError, value: JSON.parse(text) }; + } catch { + return { ok: false, value: text ?? json }; + } +} + +/** Record a remote write so cleanup can hard-delete it whatever the assertions did. */ +export function trackRemoteWrite(ctx, scope, key) { + if (ctx.lane === 'remote') ctx.written.set(JSON.stringify([scope, key]), { scope, key }); +} + +/** + * Hard-delete everything the remote lane wrote, and delete its policies. + * Returns what could not be removed, so the summary can name it. + */ +export async function cleanupRemote(ctx) { + const residue = []; + if (ctx.lane !== 'remote') return residue; + for (const { scope, key } of ctx.written.values()) { + try { + const r = await rpc(ctx, 'memory.delete', { scope, key, force: true }); + if (!r.ok) residue.push(`memory ${scope}::${key} (${JSON.stringify(r.value).slice(0, 120)})`); + } catch (err) { + residue.push(`memory ${scope}::${key} (${err.name})`); + } + } + for (const id of ctx.policies) { + try { + const r = await rpc(ctx, 'policy.delete', { id }); + if (!r.ok) residue.push(`policy ${id}`); + } catch (err) { + residue.push(`policy ${id} (${err.name})`); + } + } + return residue; +} diff --git a/packages/cli/test/smoke/run.mjs b/packages/cli/test/smoke/run.mjs new file mode 100644 index 000000000..5a9667d6d --- /dev/null +++ b/packages/cli/test/smoke/run.mjs @@ -0,0 +1,97 @@ +#!/usr/bin/env node +// Run the CLI smoke cases (cases.mjs) against the real `lorekit` binary. +// +// node packages/cli/test/smoke/run.mjs # offline lane +// node packages/cli/test/smoke/run.mjs --remote +// node packages/cli/test/smoke/run.mjs --remote "$URL" "$TOKEN" --only groom +// +// The offline lane needs nothing: a throwaway HOME and project, no network. It +// also runs on every PR inside `cli:test` (test/cli-smoke.test.mjs). +// +// The remote lane WRITES: every lesson goes under the run's own project scope +// (`project::cli-smoke-`) and is hard-deleted at the end, together with +// the retention policy it creates; anything it could not remove is named in the +// summary. It refuses the production project unless --allow-production is +// passed, matching the repo's rule that write-heavy smokes run against preview +// only (.github/workflows/deploy.yml). +// +// Exit code: 0 when every case passed or was skipped, 1 otherwise. + +import process from 'node:process'; +import { pathToFileURL } from 'node:url'; + +import { CASES } from './cases.mjs'; +import { PRODUCTION_REF, cleanupRemote, cliHelpers, createContext, disposeContext } from './harness.mjs'; + +/** + * Run every case for one lane, in order. Never throws: returns the per-case + * outcomes plus anything the remote cleanup could not remove. + */ +export async function runSmoke({ lane = 'offline', endpoint = null, token = null, only = null, log = () => {} } = {}) { + const ctx = createContext({ lane, endpoint, token }); + const cli = cliHelpers(ctx); + const results = []; + try { + for (const c of CASES) { + if (!c.lanes.includes(lane)) continue; + const label = `${c.command} — ${c.name}`; + if (only && !label.includes(only)) continue; + try { + const skipped = await c.fn({ cli, ctx }); + if (typeof skipped === 'string') { + results.push({ label, command: c.command, outcome: 'skip', detail: skipped }); + log(` – ${label} (${skipped})`); + } else { + results.push({ label, command: c.command, outcome: 'pass' }); + log(` ✓ ${label}`); + } + } catch (err) { + results.push({ label, command: c.command, outcome: 'fail', detail: err.message }); + log(` ✗ ${label}\n${String(err.message).replace(/^/gm, ' ')}`); + } + } + } finally { + var residue = await cleanupRemote(ctx).catch((err) => [`cleanup crashed: ${err.message}`]); + disposeContext(ctx); + } + return { ctx: { lane, runId: ctx.runId, scope: ctx.scope }, results, residue }; +} + +async function main(argv) { + const flags = new Set(argv.filter((a) => a.startsWith('--'))); + const positional = argv.filter((a) => !a.startsWith('--')); + const onlyIdx = argv.indexOf('--only'); + const only = onlyIdx >= 0 ? argv[onlyIdx + 1] : null; + const remote = flags.has('--remote'); + const endpoint = remote ? (positional.find((p) => p !== only) ?? process.env.LOREKIT_MCP_URL) : null; + const token = remote ? (positional.filter((p) => p !== only)[1] ?? process.env.LOREKIT_TOKEN) : null; + + const unknown = [...flags].filter((f) => !['--remote', '--allow-production', '--only'].includes(f)); + if (unknown.length) { + console.error(`error: unknown option(s): ${unknown.join(', ')}`); + return 2; + } + if (remote && (!endpoint || !token)) { + console.error('usage: node packages/cli/test/smoke/run.mjs --remote [--only ] [--allow-production]'); + return 2; + } + if (remote && endpoint.includes(PRODUCTION_REF) && !flags.has('--allow-production')) { + console.error(`refusing to run against production (${PRODUCTION_REF}): the remote lane writes. Use the preview project.`); + return 2; + } + + const lane = remote ? 'remote' : 'offline'; + console.log(`CLI smoke — ${lane} lane${remote ? ` against ${new URL(endpoint).host}` : ''}`); + const { ctx, results, residue } = await runSmoke({ lane, endpoint, token, only, log: (l) => console.log(l) }); + + const count = (o) => results.filter((r) => r.outcome === o).length; + console.log(`\n${count('pass')} passed, ${count('fail')} failed, ${count('skip')} skipped — scope ${ctx.scope}`); + if (residue.length) { + console.log(`left behind (remove by hand):\n ${residue.join('\n ')}`); + } + return count('fail') > 0 ? 1 : 0; +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + main(process.argv.slice(2)).then((code) => process.exit(code)); +} diff --git a/scripts/smoke/smoke-mcp-tools.mjs b/scripts/smoke/smoke-mcp-tools.mjs index a89ba9f60..e0c644d62 100644 --- a/scripts/smoke/smoke-mcp-tools.mjs +++ b/scripts/smoke/smoke-mcp-tools.mjs @@ -234,6 +234,7 @@ async function rpc(body, { auth = token, retryOn429 = true, attempt = 0 } = {}) * `scripts/load-test/load-test-lib.mjs`, and both integration smoke specs. */ async function call(name, args = {}) { + exercised.add(name); const { status, json } = await rpc({ jsonrpc: '2.0', id: ++rpcId, @@ -273,6 +274,11 @@ const writtenIds = new Set(); // row ids — cleared by DELETE /memories/:id const createdOrgs = new Set(); const createdPolicies = new Set(); // policy ids — cleared by policy.delete const residue = []; +// Every tool a check reached this run. Static coverage (each catalog tool has +// a check) is enforced on every PR by packages/cli/test/cli-smoke.test.mjs; this +// reports which ones the CURRENT credential actually exercised, since tier-gated +// checks skip. +const exercised = new Set(); /** Record a write so the `finally` can undo it, whatever the assertion did. */ function tracked(scope, key, value) { @@ -851,6 +857,50 @@ check('policy.create → policy.list → groom.preview(policy_id) → policy.del ); }); +// ── 4c. memory.protect + policy.update + groom.run ────────────────────────── +// +// The three tools no check reached, so a regression in any of them could ship +// through a green deploy. One behaviour ties them together: grooming archives +// what a policy matches EXCEPT a protected lesson. Runs under its own scope so +// the archive can only ever touch rows this check wrote. +check('memory.protect shields a lesson from policy.update → groom.run', async () => { + requireUserScoped('a retention-policy owner'); + const scope = `project::smoke-groom-${RUN_ID}`; + const tag = `smoke-groom-${RUN_ID}`; + ok((await write(scope, 'target', { tags: [tag] })).ok, 'write target failed'); + ok((await write(scope, 'keeper', { tags: [tag] })).ok, 'write keeper failed'); + + const protect = await call('memory.protect', { scope, key: 'keeper', protected: true }); + ok(protect.ok, `memory.protect failed: ${JSON.stringify(protect.error ?? protect.value)}`); + eq(protect.value?.protected, true, 'memory.protect result'); + + // Created matching NOTHING (a tag no row carries), then pointed at the run's + // tag by policy.update — so the archive below proves the update took effect. + const create = await call('policy.create', { scope, name: `smoke groom ${RUN_ID}`, tags: [`${tag}-none`] }); + ok(create.ok, `policy.create failed: ${JSON.stringify(create.error ?? create.value)}`); + const policyId = create.value?.id; + ok(policyId, 'policy.create did not return an id'); + createdPolicies.add(policyId); + + const update = await call('policy.update', { id: policyId, name: `smoke groom ${RUN_ID} v2`, tags: [tag] }); + ok(update.ok, `policy.update failed: ${JSON.stringify(update.error ?? update.value)}`); + eq(update.value?.id, policyId, 'policy.update id'); + eq(update.value?.name, `smoke groom ${RUN_ID} v2`, 'policy.update name'); + + const run = await call('groom.run', { policy_id: policyId }); + ok(run.ok, `groom.run failed: ${JSON.stringify(run.error ?? run.value)}`); + eq(run.value?.keys?.map((k) => k.key), ['target'], 'groom.run archived keys'); + eq(run.value?.archived, 1, 'groom.run archived count'); + + const keeper = await call('memory.read', { scope, key: 'keeper' }); + ok(keeper.ok && keeper.value !== null, 'groom.run archived a protected lesson'); + const target = await call('memory.read', { scope, key: 'target' }); + ok(target.ok && target.value === null, 'groom.run reported target archived, but it is still readable'); + + const unprotect = await call('memory.protect', { scope, key: 'keeper', protected: false }); + eq(unprotect.value?.protected, false, 'memory.protect(false) result'); +}); + // ── 5. known defects, observed live (run with --probe) ─────────────────────── known( @@ -1127,6 +1177,12 @@ console.log( `\n${results.pass} passed · ${results.fail} failed · ${results.known} known · ` + `${results.fixed} fixed · ${results.skip} skipped`, ); +const unexercised = MCP_TOOL_NAMES.filter((t) => !exercised.has(t)); +console.log( + unexercised.length + ? `tools not exercised this run (credential tier ${TIER}): ${unexercised.join(', ')}` + : `all ${MCP_TOOL_NAMES.length} catalog tools exercised`, +); if (residue.length) { console.log('\nleft behind (delete by hand if it matters):'); for (const item of residue) console.log(` · ${item}`); From dbad003e02de7f32e8bfe89ccfd14d3cb4f5fe0f Mon Sep 17 00:00:00 2001 From: "mads.thines" Date: Sat, 17 Oct 2026 14:50:51 +0000 Subject: [PATCH 03/28] docs(testing): document the smoke and e2e suites and how to extend them Co-authored-by: dash0-dev[bot] <257284812+dash0-dev[bot]@users.noreply.github.com> --- CLAUDE.md | 3 + docs/README.md | 1 + docs/adding-an-operation.md | 7 ++ docs/smoke-tests.md | 122 +++++++++++++++++++++++++++++ packages/web-e2e/src/auth.setup.ts | 6 +- packages/web-e2e/src/pages.e2e.ts | 4 +- 6 files changed, 139 insertions(+), 4 deletions(-) create mode 100644 docs/smoke-tests.md diff --git a/CLAUDE.md b/CLAUDE.md index e3949be52..d6a27089c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -21,6 +21,7 @@ lets humans browse, search, and manage those lessons. | `plugins/` | `plugins/` | Per-framework deterministic bundles: `lorekit-claude` (marketplace plugin: skill + hooks + MCP), `lorekit-cursor` (rule + `stop` hook), `lorekit-codex` (feature-flagged hooks + `AGENTS.md` fallback, experimental). Root `.claude-plugin/marketplace.json` lists the Claude plugin. | | `supabase` | `supabase/` | Edge Functions (production MCP server), migrations, NX targets | | `@lorekit/smoke-tests` | `packages/smoke-tests/` | Live-endpoint integration/smoke tests against the deployed Edge Functions (memories, orgs, MCP, BYOD) — no application code, self-skips when its env vars are absent | +| `@lorekit/web-e2e` | `packages/web-e2e/` | Playwright smoke suite for every page of the website. One entry per page in `src/routes.ts`; a vitest guard fails the PR when a `page.tsx` has none. Runs per PR (local build + local Supabase), on the staged production build before promotion, and against `lorekit.io` after it — see [docs/smoke-tests.md](./docs/smoke-tests.md) | The **production MCP server** is `supabase/functions/mcp/index.ts` (Deno, self-contained). There is no other MCP server implementation — a prior Node.js/Fly.io variant (`packages/mcp-server/`) @@ -152,6 +153,8 @@ npx vitest run --config vitest.storybook.config.ts -u # update base - CI runs these in the `web-test` job (browser job, gated by `changes.web`) — NOT part of `check`'s `nx affected -t test`. - Full-page stories mock Supabase REST with **MSW** so real React Query hooks resolve against a stable dataset; server-component pages story their largest client subtree, never refactored to client. Storybook deploys as its OWN Vercel project. Full runbook + MSW / mixed-rendering / deploy detail: [docs/storybook.md](./docs/storybook.md). +**Smoke + e2e coverage is table-driven and guarded.** A new page needs an entry in `packages/web-e2e/src/routes.ts`; a new CLI command a case in `packages/cli/test/smoke/cases.mjs`; a new MCP tool a call in `scripts/smoke/smoke-mcp-tools.mjs` (and in the offline `mcp` case unless it is in `LOCAL_MCP_EXEMPT`). `routes.test.ts` and `cli-smoke.test.mjs` fail the PR otherwise. What runs where: [docs/smoke-tests.md](./docs/smoke-tests.md). + --- ## User-facing docs (mandatory on every change) diff --git a/docs/README.md b/docs/README.md index 358a4d7be..a32d7e936 100644 --- a/docs/README.md +++ b/docs/README.md @@ -21,6 +21,7 @@ | [telemetry-quality-review.md](./telemetry-quality-review.md) | Developers | Cross-service `traceparent` correlation contract, telemetry-quality review vs OTel semantic conventions, and the tests that guard it | | [deployment.md](./deployment.md) | Developers | Step-by-step deployment for all three pieces | | [storybook.md](./storybook.md) | Developers | Web Storybook: MSW-mocked full-page stories, determinism, and deploying Storybook as a second Vercel project | +| [smoke-tests.md](./smoke-tests.md) | Developers | Website e2e (Playwright), CLI and MCP smoke suites: what runs on each PR and deploy, and how to add a page, command or tool | | [releasing.md](./releasing.md) | Developers | Automated `@lorekit/cli` npm releases via release-please + conventional commits | | [adding-an-operation.md](./adding-an-operation.md) | Developers | Step-by-step checklist for adding a new MCP tool / CLI command / REST route across the catalog, both permission mirrors, the audit vocabulary, and docs | | [sandbox.md](./sandbox.md) | Developers | Fresh sandbox/container baseline — the six traps that make a red gate lie, and how to prove a failure pre-existing (full runbook behind the CLAUDE.md summary) | diff --git a/docs/adding-an-operation.md b/docs/adding-an-operation.md index efa075c26..ca36d5978 100644 --- a/docs/adding-an-operation.md +++ b/docs/adding-an-operation.md @@ -132,6 +132,12 @@ catch a mistake at that step. `packages/web/src/lib/docs/sections.ts` entry (`sections.spec.ts` guards drift), and update `docs/README.md`'s index table and `README.md`'s package map if the capability is user-facing. +11. **Smoke checks** — add a `check(...)` that calls the tool to + `scripts/smoke/smoke-mcp-tools.mjs` (the hosted server, run at deploy time), + and, unless it is in `LOCAL_MCP_EXEMPT`, call it in the offline `mcp` case in + `packages/cli/test/smoke/cases.mjs` (plus `LOCAL_MCP_CALLED`). A new CLI + command also needs a case there. `cli-smoke.test.mjs` fails the PR on any + of the three gaps. How-to: [smoke-tests.md](./smoke-tests.md). ## 3. Gates that catch a missed step @@ -145,6 +151,7 @@ failure is cheaper to fix: | `pnpm nx test schemas` (`surface-generator.spec.ts`, `render.spec.ts`) | Generated artifacts + `llms.txt` drift | | `pnpm nx test mcp-core` (`tool-catalog-parity.spec.ts`, `permissions.spec.ts`, `audit-vocabulary.spec.ts`, `edge-parity.spec.ts`, `edge-schema-parity.spec.ts`, `dry-run-coverage.spec.ts`, `audit-coverage.spec.ts`, `mcp-authz-status.spec.ts`) | Permission-mirror drift, audit vocabulary drift, pure-module mirror drift, missing dry-run/audit wiring | | `pnpm nx test cli` (`surface-parity.test.mjs`, `frameworks.test.mjs`) | CLI registry vs catalog binding drift | +| `pnpm nx test cli` (`cli-smoke.test.mjs`) | A tool or command with no smoke check; a broken command in the offline CLI/MCP session | | `node scripts/ci/deno-check-functions.mjs --node-modules-dir=none` | Type errors in the edge tree — **nothing else typechecks it** | | `pnpm nx test web` (`sections.spec.ts`) | Missing/malformed docs registry entry | diff --git a/docs/smoke-tests.md b/docs/smoke-tests.md new file mode 100644 index 000000000..762963278 --- /dev/null +++ b/docs/smoke-tests.md @@ -0,0 +1,122 @@ +# Smoke and end-to-end tests + +Three suites answer one question: **after a merge, does the core of LoreKit +still work?** Each is a table you add a row to, and each has a guard that fails +the PR when something new ships without a row. + +| Suite | What it drives | Where it lives | Guard | +|-------|----------------|----------------|-------| +| Website e2e | Every page of `@lorekit/web` in a real browser (Playwright) | `packages/web-e2e/` | `routes.test.ts` — every `page.tsx` needs an entry in `src/routes.ts` | +| CLI smoke | Every `lorekit` command, through the real binary | `packages/cli/test/smoke/` | `cli-smoke.test.mjs` — every command and alias needs a case | +| MCP smoke | Every MCP tool: the local stdio server and the hosted server | `packages/cli/test/smoke/cases.mjs` (`mcp` case) and `scripts/smoke/smoke-mcp-tools.mjs` | `cli-smoke.test.mjs` — every catalog tool needs a call in both | + +The guards need no browser, no network and no secrets, so they run in the +`check` job (`nx affected -t test`) on every PR. + +## What runs where + +| When | Website e2e | CLI smoke | MCP smoke | +|------|-------------|-----------|-----------| +| **Every PR** (`ci.yml`) | `web-e2e` job: a local production build against a local Supabase stack — public pages, the signed-out redirect of every dashboard page, and every dashboard page signed in as a throwaway user | offline lane, inside `cli:test` | local stdio server, inside `cli:test` | +| **`/preview` on a PR** (`preview.yml`) | — | remote lane against the preview project | hosted server against the preview project | +| **Merge → before production** (`deploy.yml`) | `e2e-web-staged`: the staged production build, before its domain is assigned. Gates `promote-web-production` | remote lane in `smoke-preview`. Gates `deploy-production` | hosted server in `smoke-preview`. Gates `deploy-production` | +| **Merge → after production** (`deploy.yml`) | `smoke-production`: `https://lorekit.io`, public + signed in. A failure rolls both halves back | — (it writes; production is never written to by a smoke) | stdio handshake only (read-only) | + +## Adding a page + +Add one object to `ROUTES` in `packages/web-e2e/src/routes.ts`: + +```ts +{ route: '/settings/billing', access: 'signed-in', heading: { level: 2, name: 'Billing' } }, +``` + +- `route` is the directory of the `page.tsx` with route groups removed + (`(dashboard)/settings/billing/page.tsx` → `/settings/billing`). For a dynamic + route, keep the pattern in `route` and put a real URL in `path`. +- `access: 'public'` → one test: it renders for a signed-out visitor. + `access: 'signed-in'` → two: a signed-out visitor is sent to + `/login?next=`, and a signed-in user sees the heading. +- `heading` is what a person sees — the exact accessible name. Use the h2 on + settings pages (they share the h1 "Settings"). +- `landsOn` when the page redirects; `skipOn` (with a reason) when one kind of + target cannot show it; `allowConsoleErrors` (with a comment) for a known, + harmless console error. + +Every test also fails on an uncaught exception, a `console.error` or a +same-origin 5xx, so a page that renders its heading but throws underneath still +fails. + +A journey that spans pages (sign in, navigate, sign out) goes in +`packages/web-e2e/src/journeys/` — tag it `@signed-in` if it needs a session. + +## Adding a CLI command + +Add a case to `packages/cli/test/smoke/cases.mjs`, in session order: + +```js +smoke('billing', 'shows the current plan', BOTH, async ({ cli, ctx }) => { + const out = cli.json(['billing']); + assert.equal(out.ok, true); +}); +``` + +- `cli.json(args)` appends `--json`, asserts exit 0 (or `{ code }`) and returns + the parsed output; `cli.ok(args)` returns the text; `cli.fails(args, /why/)` + asserts a non-zero exit with that message. +- Lanes: `offline` (throwaway HOME + project, `LOREKIT_MODE=local`, no network), + `remote` (a live endpoint), or both. A remote-only command gets an offline case + that asserts it refuses cleanly, and a remote case that asserts it works. +- Anything the remote lane writes must go under `ctx.scope` and through + `writeLesson(...)` / `trackRemoteWrite(...)`, so cleanup hard-deletes it. +- A command that genuinely cannot be smoked goes in `NOT_SMOKED` with the reason. + +## Adding an MCP tool + +After adding the operation (see [adding-an-operation.md](./adding-an-operation.md)): + +1. **Hosted:** add a `check(...)` to `scripts/smoke/smoke-mcp-tools.mjs` that + calls it and asserts what it did, writing only under the run's scope. +2. **Local stdio**, unless the tool is in `LOCAL_MCP_EXEMPT`: call it in the + offline `mcp` case in `cases.mjs` and add it to `LOCAL_MCP_CALLED`. + +## Running locally + +```bash +# Guards (fast, no browser) +pnpm nx test web-e2e +node --test packages/cli/test/cli-smoke.test.mjs # guards + the offline CLI/MCP lane + +# CLI remote lane against preview (writes; refuses production) +node packages/cli/test/smoke/run.mjs --remote "$PREVIEW_MCP_URL" "$LOREKIT_TOKEN" + +# Website e2e against a local build +pnpm nx run web-e2e:build-web # NEXT_PUBLIC_USE_LOCAL_SUPABASE=true +cd packages/web-e2e && npx playwright install chromium +npx playwright test --project=public # signed-out only, no backend needed +npx playwright test --ui # watch it run + +# …with signed-in pages, against `supabase start` +E2E_SUPABASE_URL=http://127.0.0.1:54321 \ +E2E_SUPABASE_SERVICE_ROLE_KEY="$(supabase status -o json | jq -r .SERVICE_ROLE_KEY)" \ + npx playwright test + +# …or against a deployed site +E2E_BASE_URL=https://lorekit.io E2E_USER_EMAIL=… E2E_USER_PASSWORD=… npx playwright test +``` + +`E2E_WEB_COMMAND="pnpm --filter @lorekit/web exec next dev --port 3100"` swaps +the build for the dev server while iterating. Every variable is documented at +the top of `packages/web-e2e/src/support/env.ts`. + +## Secrets the deploy-time runs use + +| Secret | Used by | Without it | +|--------|---------|------------| +| `VERCEL_AUTOMATION_BYPASS_SECRET` (repo) | `e2e-web-staged` — gets through Vercel Deployment Protection on the staged URL | the job skips with a warning and promotion proceeds as before | +| `LOREKIT_TESTING_USER_NAME`, `LOREKIT_TESTING_USER_PW` (repo) | signed-in pages on the staged build and on `lorekit.io` | signed-in tests skip, announced; public tests still run | +| `LOREKIT_SMOKE_TOKEN` (preview env, existing) | CLI remote lane, hosted MCP smoke | — (already required) | + +The test account must be a normal user (not in `DEVELOPER_EMAILS`); opening +`/lore` as a user with no API tokens creates one, so expect a "My first token". +The sign-out journey never runs against production: signing out revokes every +session of the account. diff --git a/packages/web-e2e/src/auth.setup.ts b/packages/web-e2e/src/auth.setup.ts index 7287a6bc3..bdaf08a5f 100644 --- a/packages/web-e2e/src/auth.setup.ts +++ b/packages/web-e2e/src/auth.setup.ts @@ -13,9 +13,11 @@ setup('sign in through the login form', async ({ page }) => { // Always leave a file behind: the `signed-in` project loads this path, and // its tests skip themselves when there is no user. writeFileSync(STORAGE_STATE, JSON.stringify(EMPTY_STORAGE_STATE)); - setup.skip(!USER_SOURCE, NO_USER_REASON); + const source = USER_SOURCE; + setup.skip(!source, NO_USER_REASON); + if (!source) return; - const user = USER_SOURCE!.kind === 'fixed' ? USER_SOURCE! : await provisionUser(USER_SOURCE!); + const user = source.kind === 'fixed' ? source : await provisionUser(source); await signInThroughTheForm(page, user.email, user.password); // Plain /login lands on DEFAULT_POST_LOGIN_PATH (lib/auth-redirect.ts). diff --git a/packages/web-e2e/src/pages.e2e.ts b/packages/web-e2e/src/pages.e2e.ts index 08c6762df..73f7ca94b 100644 --- a/packages/web-e2e/src/pages.e2e.ts +++ b/packages/web-e2e/src/pages.e2e.ts @@ -18,8 +18,8 @@ function heading(page: Page, r: RouteCheck) { async function expectPageRenders(page: Page, r: RouteCheck) { const response = await page.goto(visitPath(r)); - expect(response, `no response for ${visitPath(r)}`).not.toBeNull(); - expect(response!.status(), `HTTP status for ${visitPath(r)}`).toBeLessThan(400); + expect(response?.status() ?? 0, `HTTP status for ${visitPath(r)} (0 = no response)`).toBeGreaterThan(0); + expect(response?.status() ?? 0, `HTTP status for ${visitPath(r)}`).toBeLessThan(400); if (r.landsOn) { const landsOn = r.landsOn; await expect(page).toHaveURL((url) => url.pathname === landsOn); From 23d4e461c2b2052bcda67406663b55a4f82558ab Mon Sep 17 00:00:00 2001 From: "mads.thines" Date: Sat, 17 Oct 2026 14:51:25 +0000 Subject: [PATCH 04/28] ci: add the workflow wiring as a patch for a maintainer to apply The Dash0 GitHub App cannot push .github/workflows/* (no workflows permission). Apply with: git am .github/pending-workflow-changes.patch Co-authored-by: dash0-dev[bot] <257284812+dash0-dev[bot]@users.noreply.github.com> --- .github/pending-workflow-changes.patch | 434 +++++++++++++++++++++++++ 1 file changed, 434 insertions(+) create mode 100644 .github/pending-workflow-changes.patch diff --git a/.github/pending-workflow-changes.patch b/.github/pending-workflow-changes.patch new file mode 100644 index 000000000..3c130d724 --- /dev/null +++ b/.github/pending-workflow-changes.patch @@ -0,0 +1,434 @@ +From 4ced360342b29d9b355299282dd1a47c24513c5d Mon Sep 17 00:00:00 2001 +From: "mads.thines" +Date: Sat, 17 Oct 2026 14:50:55 +0000 +Subject: [PATCH] ci: run the e2e and smoke suites on every PR and gate deploys + on them + +- ci.yml: new web-e2e job (local build + local Supabase, every page signed + out and signed in) in the CI Summary gate; the web filter covers web-e2e. +- deploy.yml: CLI remote lane in smoke-preview; e2e-web-staged runs Playwright + on the staged production build and gates promote-web-production; + smoke-production runs Playwright against lorekit.io (failure rolls back). +- preview.yml: CLI remote lane, so /preview validates it before merge. + +Co-authored-by: dash0-dev[bot] <257284812+dash0-dev[bot]@users.noreply.github.com> +--- + .github/workflows/ci.yml | 128 ++++++++++++++++++++++++++- + .github/workflows/deploy.yml | 159 ++++++++++++++++++++++++++++++++-- + .github/workflows/preview.yml | 9 ++ + 3 files changed, 287 insertions(+), 9 deletions(-) + +diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml +index 6e640e2c..7d942e92 100644 +--- a/.github/workflows/ci.yml ++++ b/.github/workflows/ci.yml +@@ -25,7 +25,8 @@ jobs: + # `code` — any non-docs file touched → gates the `check` job. + # `api` — API/backend paths → gates the (expensive) `integration` job. + # `plugin` — Claude-plugin paths → gates the `plugin` smoke job. +- # `web` — dashboard paths (broad) → gates the `web-test` Storybook job. ++ # `web` — dashboard paths (broad) → gates the `web-test` Storybook job ++ # and the `web-e2e` Playwright job. + # `web_preview` — the narrower, Vercel-quota-metered subset → gates + # `web-preview`. Two filters, not one, because the test job is + # free CI compute and the preview is a metered deployment. +@@ -114,7 +115,10 @@ jobs: + # is precisely what the visual baselines are for. `ci.yml` is here for + # the same reason as in every sibling gate: so an edit to this job runs + # the job. +- if printf '%s\n' "$CHANGED" | grep -qE '^(packages/web/|packages/schemas/|package\.json$|pnpm-lock\.yaml$|nx\.json$|\.github/workflows/ci\.yml$)'; then ++ # ++ # `packages/web-e2e/` rides along: a change to the e2e suite itself must ++ # run the `web-e2e` job that executes it. ++ if printf '%s\n' "$CHANGED" | grep -qE '^(packages/web/|packages/web-e2e/|packages/schemas/|package\.json$|pnpm-lock\.yaml$|nx\.json$|\.github/workflows/ci\.yml$)'; then + echo "web=true" >> "$GITHUB_OUTPUT" + else + echo "web=false" >> "$GITHUB_OUTPUT" +@@ -361,6 +365,124 @@ jobs: + BASE: ${{ github.event.pull_request.base.sha }} + run: npx vitest run --config vitest.storybook.config.ts --changed="$BASE" --passWithNoTests + ++ # ── Web e2e — Playwright against a local production build ──────────────────── ++ # The page smoke suite (packages/web-e2e; docs/smoke-tests.md). The site is ++ # built with NEXT_PUBLIC_USE_LOCAL_SUPABASE=true and a local Supabase stack ++ # runs behind it, so the job needs NO secrets (fork PRs included) and ++ # touches nothing real. It proves that every public page renders, that every ++ # dashboard page sends a signed-out visitor to /login, and — signed in as a ++ # throwaway user the suite provisions on the local stack — that every ++ # dashboard page renders. Any uncaught exception, console error or 5xx on ++ # a page fails its test, not just a missing heading. ++ # ++ # The same suite runs again at deploy time: against the staged production ++ # build before its domain is assigned, and against lorekit.io after ++ # (deploy.yml `e2e-web-staged` / `smoke-production`). ++ # ++ # `npx playwright`, not `pnpm exec` / `nx run`, for the stdio reason given ++ # on web-test above. ++ web-e2e: ++ name: Web — Playwright e2e (local build + local Supabase) ++ runs-on: ubuntu-latest ++ needs: [changes] ++ if: needs.changes.outputs.web == 'true' ++ timeout-minutes: 30 ++ permissions: ++ contents: read ++ steps: ++ - name: Checkout repository ++ uses: actions/checkout@v7.0.1 ++ ++ - name: Set up pnpm ++ uses: pnpm/action-setup@v6 ++ ++ - name: Set up Node.js ++ uses: actions/setup-node@v7.1.0 ++ with: ++ node-version: 20 ++ cache: pnpm ++ ++ - name: Install dependencies ++ run: pnpm install --frozen-lockfile ++ ++ # Same key as web-test, so both jobs share one browser cache. ++ - name: Resolve Playwright version ++ id: pw ++ working-directory: packages/web-e2e ++ run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> "$GITHUB_OUTPUT" ++ ++ - name: Cache Playwright browsers ++ uses: actions/cache@v6 ++ with: ++ path: ~/.cache/ms-playwright ++ key: playwright-${{ runner.os }}-${{ steps.pw.outputs.version }} ++ ++ - name: Install Playwright Chromium (+ OS deps) ++ run: pnpm --filter @lorekit/web-e2e exec playwright install --with-deps chromium ++ ++ - name: Install Supabase CLI ++ uses: supabase/setup-cli@v3 ++ with: ++ # Same pin as the integration job — see the reason there. ++ version: 2.115.0 ++ ++ - name: Start local Supabase (auth + edge functions behind the dashboard) ++ run: supabase start ++ ++ - name: Capture local service-role key ++ id: supabase ++ run: | ++ supabase status -o json > /tmp/status.json ++ KEY="$(jq -r '.SERVICE_ROLE_KEY // .SECRET_KEY // .service_role_key // .Secret // empty' /tmp/status.json)" ++ if [ -z "$KEY" ]; then ++ echo "::error::No service-role/secret key in supabase status. Available keys:" ++ jq -r 'keys[]' /tmp/status.json ++ exit 1 ++ fi ++ echo "::add-mask::$KEY" ++ echo "service_role_key=$KEY" >> "$GITHUB_OUTPUT" ++ ++ - name: Cache the Next.js build cache ++ uses: actions/cache@v6 ++ with: ++ path: packages/web/.next/cache ++ key: web-e2e-next-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }}-${{ hashFiles('packages/web/src/**', 'packages/schemas/src/**') }} ++ restore-keys: | ++ web-e2e-next-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }}- ++ ++ # NEXT_PUBLIC_* values are inlined at build time, so THIS step decides ++ # that the site talks to the local stack and nothing else. ++ - name: Build the website (pointed at the local stack) ++ working-directory: packages/web ++ env: ++ NEXT_PUBLIC_USE_LOCAL_SUPABASE: 'true' ++ NEXT_TELEMETRY_DISABLED: '1' ++ run: npx next build ++ ++ # Playwright starts `next start` itself (playwright.config.ts webServer). ++ # E2E_SUPABASE_* lets the setup project create a throwaway confirmed user ++ # on the local stack (refused for any non-local URL — src/support/env.ts). ++ - name: Playwright — every page, signed out and signed in ++ working-directory: packages/web-e2e ++ env: ++ E2E_SUPABASE_URL: http://127.0.0.1:54321 ++ E2E_SUPABASE_SERVICE_ROLE_KEY: ${{ steps.supabase.outputs.service_role_key }} ++ run: npx playwright test ++ ++ - name: Upload the Playwright report (traces + screenshots) ++ if: failure() ++ uses: actions/upload-artifact@v7 ++ with: ++ name: web-e2e-report ++ path: | ++ packages/web-e2e/playwright-report ++ packages/web-e2e/test-results ++ retention-days: 14 ++ ++ - name: Stop local Supabase ++ if: always() ++ run: supabase stop --no-backup ++ + # ── Web preview — deploy a Vercel preview, ONLY when web paths changed ──────── + # Replaces Vercel's native Git integration for the dashboard project. That + # integration deployed a preview on EVERY PR/branch push — even docs- or +@@ -924,7 +1046,7 @@ jobs: + # `always()` so the gate still runs (and reports) even when an upstream job + # failed, was cancelled, or was skipped — otherwise it would be skipped too. + if: always() +- needs: [changes, check, plugin, integration, web-test, migration-order, deploy-scope, skill-version-guard, seed-tests, preview-filter, bench-tests, edge-typecheck] ++ needs: [changes, check, plugin, integration, web-test, web-e2e, migration-order, deploy-scope, skill-version-guard, seed-tests, preview-filter, bench-tests, edge-typecheck] + permissions: {} + steps: + - name: Aggregate job results into a pass/fail gate +diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml +index b0134efb..667ab538 100644 +--- a/.github/workflows/deploy.yml ++++ b/.github/workflows/deploy.yml +@@ -11,9 +11,9 @@ + # + # deploy-preview (API) ────────┐ + # deploy-web-preview (FE) ──────┤ +-# stage-web-production (FE build)┘ ++# stage-web-production (FE build)┘─▶ e2e-web-staged (Playwright) ──────────┐ + # └─▶ smoke-preview ─▶ deploy-production (API) ─▶ promote-web-production (FE) +-# └────────────────────────┴─▶ smoke-production ++# └────────────────────────┴─▶ smoke-production (+ Playwright) + # ├─▶ rollback-production (functions) + # └─▶ rollback-web-production (Vercel) + # any job fails ─▶ notify-failure (Discord + Dash0) +@@ -48,6 +48,12 @@ + # The fixed smoke user those secrets name must already exist on the project — seed it + # once with `node scripts/smoke/seed-smoke-user.mjs` (see docs/deployment.md → "Seed + # the orgs-smoke user"); the mint signs in as it, it is never auto-created. ++# Optional repo-level secrets for the website e2e suite (packages/web-e2e, ++# docs/smoke-tests.md): VERCEL_AUTOMATION_BYPASS_SECRET lets e2e-web-staged reach ++# the staged build through Vercel Deployment Protection (unset → that job skips, ++# announced, and promotion proceeds as before); LOREKIT_TESTING_USER_NAME + ++# LOREKIT_TESTING_USER_PW sign in a production test account for the signed-in ++# pages (unset → those tests skip, announced; public pages still run). + # Optional repo-level secret: DISCORD_WEBHOOK_URL — when set, notify-failure posts + # a message there on any deploy-pipeline failure. Unset → notify-failure no-ops. + # Add a required reviewer on the `production` environment for a manual gate. +@@ -395,6 +401,91 @@ jobs: + echo "### Web production (staged)" >> "$GITHUB_STEP_SUMMARY" + echo "- Built and uploaded: $URL (domain not yet applied)" >> "$GITHUB_STEP_SUMMARY" + ++ # ── 1d. Website e2e against the STAGED production build ─────────────────────── ++ # Playwright (packages/web-e2e) against the exact bundle promote-web-production ++ # is about to put on the production domain — so a broken page stops the flip ++ # instead of being rolled back after users saw it. The staged URL is behind ++ # Vercel Deployment Protection; VERCEL_AUTOMATION_BYPASS_SECRET gets through it ++ # and, while it is unset, this job skips (announced) and promotion proceeds ++ # exactly as before. ++ # ++ # Signed-in pages run here only when this merge leaves the API alone. When the ++ # API changed too, it reaches production only AFTER this job (deploy-production), ++ # so the staged bundle would be tested against an API it will never be served ++ # with; those pages run in smoke-production instead, once both halves are live. ++ # ++ # No `environment:` — a required reviewer on `production` must not gate a test. ++ e2e-web-staged: ++ name: Web e2e → staged production build (Playwright) ++ runs-on: ubuntu-latest ++ needs: [changes, stage-web-production] ++ if: needs.stage-web-production.result == 'success' ++ timeout-minutes: 20 ++ permissions: ++ contents: read ++ env: ++ E2E_BASE_URL: ${{ needs.stage-web-production.outputs.deployment_url }} ++ # The staged build talks to the PRODUCTION backend, whatever its URL says. ++ E2E_TARGET: production ++ VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} ++ E2E_USER_EMAIL: ${{ secrets.LOREKIT_TESTING_USER_NAME }} ++ E2E_USER_PASSWORD: ${{ secrets.LOREKIT_TESTING_USER_PW }} ++ API_CHANGED: ${{ needs.changes.outputs.api }} ++ steps: ++ - name: Skip without the Vercel bypass secret ++ id: gate ++ run: | ++ if [ -z "$VERCEL_AUTOMATION_BYPASS_SECRET" ]; then ++ echo "::warning::VERCEL_AUTOMATION_BYPASS_SECRET is not set — the staged build is behind Vercel Deployment Protection, so the website e2e gate is SKIPPED. Add the secret (Vercel ▸ Project ▸ Settings ▸ Deployment Protection ▸ Protection Bypass for Automation) to turn it on." ++ echo "run=false" >> "$GITHUB_OUTPUT" ++ else ++ echo "run=true" >> "$GITHUB_OUTPUT" ++ fi ++ ++ - name: Checkout repository ++ if: steps.gate.outputs.run == 'true' ++ uses: actions/checkout@v7.0.1 ++ ++ - name: Set up pnpm ++ if: steps.gate.outputs.run == 'true' ++ uses: pnpm/action-setup@v6 ++ ++ - name: Set up Node.js ++ if: steps.gate.outputs.run == 'true' ++ uses: actions/setup-node@v7.1.0 ++ with: ++ node-version: 20 ++ cache: pnpm ++ ++ - name: Install dependencies ++ if: steps.gate.outputs.run == 'true' ++ run: pnpm install --frozen-lockfile ++ ++ - name: Install Playwright Chromium (+ OS deps) ++ if: steps.gate.outputs.run == 'true' ++ run: pnpm --filter @lorekit/web-e2e exec playwright install --with-deps chromium ++ ++ - name: Playwright — staged build ++ if: steps.gate.outputs.run == 'true' ++ working-directory: packages/web-e2e ++ run: | ++ if [ "$API_CHANGED" = "true" ]; then ++ echo "::notice::This merge changes the API too — signed-in pages run in smoke-production, after both halves are live." ++ npx playwright test --project=public ++ else ++ npx playwright test --project=public --project=signed-in ++ fi ++ ++ - name: Upload the Playwright report (traces + screenshots) ++ if: failure() ++ uses: actions/upload-artifact@v7 ++ with: ++ name: web-e2e-staged-report ++ path: | ++ packages/web-e2e/playwright-report ++ packages/web-e2e/test-results ++ retention-days: 14 ++ + # ── 2. Integration test against PREVIEW (the gate before prod) ──────────────── + smoke-preview: + name: Smoke tests → preview +@@ -528,6 +619,29 @@ jobs: + URL: https://${{ secrets.SUPABASE_PROJECT_REF }}.supabase.co/functions/v1/mcp + TOKEN: ${{ secrets.LOREKIT_SMOKE_TOKEN }} + ++ # Every `lorekit` command, through the real binary, against the deploy that ++ # just landed (packages/cli/test/smoke, remote lane). The PR gate already ran ++ # the same cases offline; this proves them against a real backend. It ++ # writes only under its own `project::cli-smoke-` scope and hard-deletes ++ # what it wrote, and refuses production by itself — preview only, like the ++ # MCP surface smoke above. Retried once: the run is self-contained, so a ++ # re-run after a transient flake is safe and a real regression still fails. ++ - name: Smoke — every CLI command against preview ++ run: | ++ for attempt in 1 2; do ++ node packages/cli/test/smoke/run.mjs --remote "$URL" "$TOKEN" && break ++ rc=$? ++ if [ "$attempt" -eq 2 ]; then ++ echo "::error::CLI smoke failed after $attempt attempts" ++ exit "$rc" ++ fi ++ echo "::warning::CLI smoke failed (attempt $attempt/2) — retrying in 10s (transient live-endpoint flake)..." ++ sleep 10 ++ done ++ env: ++ URL: https://${{ secrets.SUPABASE_PROJECT_REF }}.supabase.co/functions/v1/mcp ++ TOKEN: ${{ secrets.LOREKIT_SMOKE_TOKEN }} ++ + # Mint a user JWT so smoke-rest can include the orgs suite. Fixed-user mode + # (LOREKIT_SMOKE_EMAIL/PASSWORD) — never provisions users in a real tenant, + # unlike CI's ephemeral mode against a throwaway local stack. Runs only when +@@ -711,13 +825,15 @@ jobs: + promote-web-production: + name: Promote web → production (Vercel) + runs-on: ubuntu-latest +- needs: [changes, deploy-production, stage-web-production] +- # Flip the web when it changed (stage-web-production ran) AND either the API ++ needs: [changes, deploy-production, stage-web-production, e2e-web-staged] ++ # Flip the web when it changed (stage-web-production ran), its staged build ++ # passed the website e2e (e2e-web-staged — green when it skips for a missing ++ # bypass secret), AND either the API + # did not change (nothing to wait for) or the API's prod deploy succeeded (so + # the API is live before the FE fronts it). `changes.outputs.api` disambiguates + # deploy-production's 'skipped' (API unchanged, fine) from a real upstream stop + # (API changed but its deploy failed / smoke-preview failed → not 'success'). +- if: always() && needs.stage-web-production.result == 'success' && (needs.changes.outputs.api == 'false' || needs.deploy-production.result == 'success') ++ if: always() && needs.stage-web-production.result == 'success' && needs.e2e-web-staged.result == 'success' && (needs.changes.outputs.api == 'false' || needs.deploy-production.result == 'success') + environment: production + permissions: + # `write` for this job's own deployed-SHA marker, nothing else. +@@ -917,6 +1033,35 @@ jobs: + exit 1 + fi + ++ # The website, in a browser, as users get it now that both halves are live: ++ # every public page, every dashboard page's signed-out redirect, and — with ++ # the test account secrets — every dashboard page signed in plus the shell ++ # journeys (packages/web-e2e). Read-only: page views and nothing else (the ++ # sign-out journey skips itself on production — it would revoke the shared ++ # account's other sessions). A failure here trips both rollbacks, like the ++ # curl above; the suite retries each test twice on CI before failing. ++ - name: Install Playwright Chromium (+ OS deps) ++ run: pnpm --filter @lorekit/web-e2e exec playwright install --with-deps chromium ++ ++ - name: Web e2e — every page against production (Playwright) ++ working-directory: packages/web-e2e ++ env: ++ E2E_BASE_URL: ${{ vars.WEB_PROD_URL || 'https://lorekit.io' }} ++ E2E_TARGET: production ++ E2E_USER_EMAIL: ${{ secrets.LOREKIT_TESTING_USER_NAME }} ++ E2E_USER_PASSWORD: ${{ secrets.LOREKIT_TESTING_USER_PW }} ++ run: npx playwright test --project=public --project=signed-in ++ ++ - name: Upload the Playwright report (traces + screenshots) ++ if: failure() ++ uses: actions/upload-artifact@v7 ++ with: ++ name: web-e2e-production-report ++ path: | ++ packages/web-e2e/playwright-report ++ packages/web-e2e/test-results ++ retention-days: 14 ++ + # Full authenticated onboarding + round-trip against the live production + # deploy, mirroring smoke-preview: scaffold skill + config, then + # `doctor --deep` (self-cleaning write→read→delete). Stronger than the +@@ -1222,7 +1367,7 @@ jobs: + notify-failure: + name: Notify on failure (Discord + Dash0) + runs-on: ubuntu-latest +- needs: [changes, deploy-preview, deploy-web-preview, stage-web-production, smoke-preview, deploy-production, promote-web-production, smoke-production, rollback-production, rollback-web-production] ++ needs: [changes, deploy-preview, deploy-web-preview, stage-web-production, e2e-web-staged, smoke-preview, deploy-production, promote-web-production, smoke-production, rollback-production, rollback-web-production] + if: always() && contains(needs.*.result, 'failure') + permissions: + contents: read # checkout, so the local composite action is available +@@ -1237,6 +1382,7 @@ jobs: + RESULT_DEPLOY_PREVIEW: ${{ needs.deploy-preview.result }} + RESULT_DEPLOY_WEB_PREVIEW: ${{ needs.deploy-web-preview.result }} + RESULT_STAGE_WEB_PRODUCTION: ${{ needs.stage-web-production.result }} ++ RESULT_E2E_WEB_STAGED: ${{ needs.e2e-web-staged.result }} + RESULT_SMOKE_PREVIEW: ${{ needs.smoke-preview.result }} + RESULT_DEPLOY_PRODUCTION: ${{ needs.deploy-production.result }} + RESULT_PROMOTE_WEB_PRODUCTION: ${{ needs.promote-web-production.result }} +@@ -1251,6 +1397,7 @@ jobs: + elif [ "$RESULT_DEPLOY_PREVIEW" = "failure" ]; then failed="deploy → preview" + elif [ "$RESULT_DEPLOY_WEB_PREVIEW" = "failure" ]; then failed="web deploy → preview" + elif [ "$RESULT_STAGE_WEB_PRODUCTION" = "failure" ]; then failed="web build (production staging)" ++ elif [ "$RESULT_E2E_WEB_STAGED" = "failure" ]; then failed="web e2e → staged production build" + elif [ "$RESULT_SMOKE_PREVIEW" = "failure" ]; then failed="smoke tests → preview" + elif [ "$RESULT_DEPLOY_PRODUCTION" = "failure" ]; then failed="deploy → production" + elif [ "$RESULT_PROMOTE_WEB_PRODUCTION" = "failure" ]; then failed="web promote → production" +diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml +index 5e940480..a1b523b0 100644 +--- a/.github/workflows/preview.yml ++++ b/.github/workflows/preview.yml +@@ -371,6 +371,15 @@ jobs: + URL: https://${{ secrets.SUPABASE_PROJECT_REF }}.supabase.co/functions/v1/mcp + TOKEN: ${{ secrets.LOREKIT_SMOKE_TOKEN }} + ++ # Every `lorekit` command against this PR's preview deploy — the same ++ # remote lane deploy.yml's smoke-preview runs after a merge, so `/preview` ++ # shows before merging whether the CLI still works against the PR's API. ++ - name: Smoke — every CLI command against preview ++ run: node packages/cli/test/smoke/run.mjs --remote "$URL" "$TOKEN" ++ env: ++ URL: https://${{ secrets.SUPABASE_PROJECT_REF }}.supabase.co/functions/v1/mcp ++ TOKEN: ${{ secrets.LOREKIT_SMOKE_TOKEN }} ++ + # Mint a user JWT so smoke-rest can include the orgs suite. Fixed-user mode + # (LOREKIT_SMOKE_EMAIL/PASSWORD) — never provisions users in a real tenant. + # Runs only when the optional credentials are configured; otherwise the +-- +2.39.5 + From b4af6070241c2c93a61b6b7aa66c5339e72b0f68 Mon Sep 17 00:00:00 2001 From: "mads.thines" Date: Sat, 17 Oct 2026 15:44:37 +0000 Subject: [PATCH 05/28] test(web-e2e): write e2e tests as Markdown specs in the ui-verify grammar MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adding a website test is now a Markdown block in packages/web-e2e/specs/, in aw-tester's WHEN/THEN grammar — the body of a PR block — instead of TypeScript. A zero-dep parser (src/spec/grammar.cjs) runs it deterministically, refusing what needs judgment (semantic:, intent specs, continues-from) by name. spec:import / spec:export move specs to and from ui-verify PR blocks and aw-tester's compiled routes. specs.test.ts parses every file and fails a PR when a page.tsx has no spec. Co-authored-by: dash0-dev[bot] <257284812+dash0-dev[bot]@users.noreply.github.com> --- CLAUDE.md | 4 +- docs/smoke-tests.md | 45 +- packages/web-e2e/package.json | 6 +- packages/web-e2e/scripts/ui-verify.cjs | 123 ++++ packages/web-e2e/specs/README.md | 110 ++++ packages/web-e2e/specs/journeys.md | 47 ++ packages/web-e2e/specs/pages.md | 213 +++++++ .../web-e2e/src/journeys/dashboard.e2e.ts | 11 +- packages/web-e2e/src/journeys/login.e2e.ts | 17 +- .../web-e2e/src/journeys/navigation.e2e.ts | 38 -- packages/web-e2e/src/pages.e2e.ts | 66 --- packages/web-e2e/src/routes.test.ts | 75 --- packages/web-e2e/src/routes.ts | 159 ------ packages/web-e2e/src/spec/grammar.cjs | 540 ++++++++++++++++++ packages/web-e2e/src/spec/load.ts | 16 + packages/web-e2e/src/spec/run.ts | 117 ++++ packages/web-e2e/src/specs.e2e.ts | 66 +++ packages/web-e2e/src/specs.test.ts | 237 ++++++++ packages/web-e2e/src/support/fixtures.ts | 16 +- packages/web-e2e/tsconfig.json | 5 +- packages/web-e2e/vitest.config.ts | 11 +- 21 files changed, 1530 insertions(+), 392 deletions(-) create mode 100644 packages/web-e2e/scripts/ui-verify.cjs create mode 100644 packages/web-e2e/specs/README.md create mode 100644 packages/web-e2e/specs/journeys.md create mode 100644 packages/web-e2e/specs/pages.md delete mode 100644 packages/web-e2e/src/journeys/navigation.e2e.ts delete mode 100644 packages/web-e2e/src/pages.e2e.ts delete mode 100644 packages/web-e2e/src/routes.test.ts delete mode 100644 packages/web-e2e/src/routes.ts create mode 100644 packages/web-e2e/src/spec/grammar.cjs create mode 100644 packages/web-e2e/src/spec/load.ts create mode 100644 packages/web-e2e/src/spec/run.ts create mode 100644 packages/web-e2e/src/specs.e2e.ts create mode 100644 packages/web-e2e/src/specs.test.ts diff --git a/CLAUDE.md b/CLAUDE.md index d6a27089c..b26ef43de 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -21,7 +21,7 @@ lets humans browse, search, and manage those lessons. | `plugins/` | `plugins/` | Per-framework deterministic bundles: `lorekit-claude` (marketplace plugin: skill + hooks + MCP), `lorekit-cursor` (rule + `stop` hook), `lorekit-codex` (feature-flagged hooks + `AGENTS.md` fallback, experimental). Root `.claude-plugin/marketplace.json` lists the Claude plugin. | | `supabase` | `supabase/` | Edge Functions (production MCP server), migrations, NX targets | | `@lorekit/smoke-tests` | `packages/smoke-tests/` | Live-endpoint integration/smoke tests against the deployed Edge Functions (memories, orgs, MCP, BYOD) — no application code, self-skips when its env vars are absent | -| `@lorekit/web-e2e` | `packages/web-e2e/` | Playwright smoke suite for every page of the website. One entry per page in `src/routes.ts`; a vitest guard fails the PR when a `page.tsx` has none. Runs per PR (local build + local Supabase), on the staged production build before promotion, and against `lorekit.io` after it — see [docs/smoke-tests.md](./docs/smoke-tests.md) | +| `@lorekit/web-e2e` | `packages/web-e2e/` | Playwright suite for every page and key journey of the website, written as Markdown specs in `specs/*.md` (aw-tester's `WHEN`/`THEN` grammar — the same body as a `` PR block; `pnpm spec:import`/`spec:export` move them). A vitest guard fails the PR when a spec does not parse or a `page.tsx` has no spec. Runs per PR (local build + local Supabase), on the staged production build before promotion, and against `lorekit.io` after it — see [docs/smoke-tests.md](./docs/smoke-tests.md) | The **production MCP server** is `supabase/functions/mcp/index.ts` (Deno, self-contained). There is no other MCP server implementation — a prior Node.js/Fly.io variant (`packages/mcp-server/`) @@ -153,7 +153,7 @@ npx vitest run --config vitest.storybook.config.ts -u # update base - CI runs these in the `web-test` job (browser job, gated by `changes.web`) — NOT part of `check`'s `nx affected -t test`. - Full-page stories mock Supabase REST with **MSW** so real React Query hooks resolve against a stable dataset; server-component pages story their largest client subtree, never refactored to client. Storybook deploys as its OWN Vercel project. Full runbook + MSW / mixed-rendering / deploy detail: [docs/storybook.md](./docs/storybook.md). -**Smoke + e2e coverage is table-driven and guarded.** A new page needs an entry in `packages/web-e2e/src/routes.ts`; a new CLI command a case in `packages/cli/test/smoke/cases.mjs`; a new MCP tool a call in `scripts/smoke/smoke-mcp-tools.mjs` (and in the offline `mcp` case unless it is in `LOCAL_MCP_EXEMPT`). `routes.test.ts` and `cli-smoke.test.mjs` fail the PR otherwise. What runs where: [docs/smoke-tests.md](./docs/smoke-tests.md). +**Smoke + e2e coverage is table-driven and guarded.** A new page needs a spec in `packages/web-e2e/specs/*.md` (Markdown, ui-verify grammar — `specs/README.md`); a new CLI command a case in `packages/cli/test/smoke/cases.mjs`; a new MCP tool a call in `scripts/smoke/smoke-mcp-tools.mjs` (and in the offline `mcp` case unless it is in `LOCAL_MCP_EXEMPT`). `specs.test.ts` and `cli-smoke.test.mjs` fail the PR otherwise. What runs where: [docs/smoke-tests.md](./docs/smoke-tests.md). --- diff --git a/docs/smoke-tests.md b/docs/smoke-tests.md index 762963278..e03aa1914 100644 --- a/docs/smoke-tests.md +++ b/docs/smoke-tests.md @@ -6,7 +6,7 @@ the PR when something new ships without a row. | Suite | What it drives | Where it lives | Guard | |-------|----------------|----------------|-------| -| Website e2e | Every page of `@lorekit/web` in a real browser (Playwright) | `packages/web-e2e/` | `routes.test.ts` — every `page.tsx` needs an entry in `src/routes.ts` | +| Website e2e | Every page and key journey of `@lorekit/web` in a real browser (Playwright) | `packages/web-e2e/specs/*.md` — Markdown, the ui-verify `WHEN`/`THEN` grammar | `specs.test.ts` — every spec parses, and every `page.tsx` is visited by one | | CLI smoke | Every `lorekit` command, through the real binary | `packages/cli/test/smoke/` | `cli-smoke.test.mjs` — every command and alias needs a case | | MCP smoke | Every MCP tool: the local stdio server and the hosted server | `packages/cli/test/smoke/cases.mjs` (`mcp` case) and `scripts/smoke/smoke-mcp-tools.mjs` | `cli-smoke.test.mjs` — every catalog tool needs a call in both | @@ -22,32 +22,41 @@ The guards need no browser, no network and no secrets, so they run in the | **Merge → before production** (`deploy.yml`) | `e2e-web-staged`: the staged production build, before its domain is assigned. Gates `promote-web-production` | remote lane in `smoke-preview`. Gates `deploy-production` | hosted server in `smoke-preview`. Gates `deploy-production` | | **Merge → after production** (`deploy.yml`) | `smoke-production`: `https://lorekit.io`, public + signed in. A failure rolls both halves back | — (it writes; production is never written to by a smoke) | stdio handshake only (read-only) | -## Adding a page +## Adding a website test -Add one object to `ROUTES` in `packages/web-e2e/src/routes.ts`: +Append a Markdown block to a file in `packages/web-e2e/specs/` — no TypeScript: -```ts -{ route: '/settings/billing', access: 'signed-in', heading: { level: 2, name: 'Billing' } }, +```markdown +## Spec 29: Settings › Billing renders +url: /settings/billing +preconditions: + - User is signed in (handled by the e2e auth setup) +flow: + - WHEN page loads + THEN {role: "heading", name: "Billing"} is visible ``` -- `route` is the directory of the `page.tsx` with route groups removed - (`(dashboard)/settings/billing/page.tsx` → `/settings/billing`). For a dynamic - route, keep the pattern in `route` and put a real URL in `path`. -- `access: 'public'` → one test: it renders for a signed-out visitor. - `access: 'signed-in'` → two: a signed-out visitor is sent to - `/login?next=`, and a signed-in user sees the heading. -- `heading` is what a person sees — the exact accessible name. Use the h2 on - settings pages (they share the h1 "Settings"). -- `landsOn` when the page redirects; `skipOn` (with a reason) when one kind of - target cannot show it; `allowConsoleErrors` (with a comment) for a known, - harmless console error. +- The format is aw-tester's `WHEN`/`THEN` grammar — the body of a + `` PR block — so a spec moves between a PR and this suite + unchanged. `pnpm spec:import --pr ` pulls a PR's ui-verify block in (an + intent block arrives as a draft with TODO lines to finish); `pnpm spec:export + ` prints a block to paste into a PR. Full guide, every supported phrase + and locator: [`packages/web-e2e/specs/README.md`](../packages/web-e2e/specs/README.md). +- `User is signed in …` in `preconditions:` runs the spec as the test user, and + also generates a check that a signed-out visitor to its url is sent to + `/login?next=`. `Not on production: ` skips it there. +- Assert what a person sees — the heading, exactly as rendered (settings pages + share the h1 "Settings", so use their section heading). +- A typo fails `nx test web-e2e` with the file and line, before CI boots a + browser; so does a new `page.tsx` that no spec visits. Every test also fails on an uncaught exception, a `console.error` or a same-origin 5xx, so a page that renders its heading but throws underneath still fails. -A journey that spans pages (sign in, navigate, sign out) goes in -`packages/web-e2e/src/journeys/` — tag it `@signed-in` if it needs a session. +What the grammar cannot express — a reload, a locator that needs scoping to one +container, a request-level check — goes in `packages/web-e2e/src/journeys/*.e2e.ts` +(tag it `@signed-in` if it needs a session). ## Adding a CLI command diff --git a/packages/web-e2e/package.json b/packages/web-e2e/package.json index f66b58a17..02aa386ad 100644 --- a/packages/web-e2e/package.json +++ b/packages/web-e2e/package.json @@ -2,12 +2,14 @@ "name": "@lorekit/web-e2e", "version": "0.0.1", "private": true, - "description": "Playwright end-to-end smoke tests for the LoreKit website (@lorekit/web). One manifest entry per page in src/routes.ts; a vitest guard fails when a page has no entry.", + "description": "Playwright end-to-end smoke tests for the LoreKit website (@lorekit/web). Tests are Markdown specs in specs/*.md, written in the aw-tester / ui-verify WHEN/THEN grammar; a vitest guard fails when a page has no spec.", "scripts": { "e2e": "playwright test", "e2e:public": "playwright test --project=public", "e2e:ui": "playwright test --ui", - "report": "playwright show-report" + "report": "playwright show-report", + "spec:import": "node scripts/ui-verify.cjs import", + "spec:export": "node scripts/ui-verify.cjs export" }, "devDependencies": { "@playwright/test": "1.56.0" diff --git a/packages/web-e2e/scripts/ui-verify.cjs b/packages/web-e2e/scripts/ui-verify.cjs new file mode 100644 index 000000000..ed2115d42 --- /dev/null +++ b/packages/web-e2e/scripts/ui-verify.cjs @@ -0,0 +1,123 @@ +#!/usr/bin/env node +'use strict'; +// Move e2e specs between this suite and /ui-verify. Usage: specs/README.md. +// +// node scripts/ui-verify.cjs import --pr [--out ] [--force] +// node scripts/ui-verify.cjs import --file [--out ] [--force] +// node scripts/ui-verify.cjs import --routes [--out ] [--force] +// node scripts/ui-verify.cjs export [ …] +// +// Every import is validated with the suite's own parser before it is reported +// ready; a draft with TODO lines is written, named as a draft, and fails +// `nx test web-e2e` until it is finished — it can never pass by accident. + +const { execFileSync } = require('node:child_process'); +const { existsSync, readFileSync, readdirSync, writeFileSync } = require('node:fs'); +const path = require('node:path'); +const { + SpecError, + extractUiVerifyBlock, + intentToDraft, + parseSpecFile, + routesToSpecs, + specsToUiVerifyBlock, +} = require('../src/spec/grammar.cjs'); + +const PKG = path.join(__dirname, '..'); + +function usage(msg) { + if (msg) console.error(`error: ${msg}\n`); + console.error( + [ + 'usage:', + ' ui-verify.cjs import --pr [--out ] [--force]', + ' ui-verify.cjs import --file [--out ] [--force]', + ' ui-verify.cjs import --routes [--out ] [--force]', + ' ui-verify.cjs export [ …]', + ].join('\n'), + ); + process.exit(2); +} + +function flag(args, name) { + const at = args.indexOf(name); + return at === -1 ? null : args[at + 1] ?? usage(`${name} needs a value`); +} + +/** A spec file body from ui-verify block content (or a whole spec file). */ +function fromBlockOrFile(text, origin) { + const block = extractUiVerifyBlock(text); + const content = block ? block.content : text; + const isIntent = block ? block.version === 'v2' : /^Format:\s*intent\s*$/m.test(content); + if (isIntent) { + const { text: draft, todos } = intentToDraft(content); + return { body: draft, note: `intent spec → draft with ${todos} TODO line(s) to finish` }; + } + // Grammar: keep everything from the first spec heading, drop the header. + const at = content.search(/^## Spec /m); + if (at === -1) throw new Error(`${origin}: no \`## Spec\` block found`); + return { body: content.slice(at).trim() + '\n', note: 'grammar spec → imported verbatim' }; +} + +function runImport(args) { + const pr = flag(args, '--pr'); + const file = flag(args, '--file'); + const routes = flag(args, '--routes'); + if ([pr, file, routes].filter(Boolean).length !== 1) usage('import takes exactly one of --pr, --file, --routes'); + + let body; + let notes = []; + let defaultName; + if (pr) { + const number = /(\d+)\/?$/.exec(pr)?.[1] ?? usage(`not a PR number or URL: ${pr}`); + const prBody = execFileSync('gh', ['pr', 'view', pr, '--json', 'body', '-q', '.body'], { encoding: 'utf8' }); + if (!extractUiVerifyBlock(prBody)) throw new Error(`PR ${pr} has no ui-verify block`); + ({ body, note: notes[0] } = fromBlockOrFile(prBody, `PR ${pr}`)); + defaultName = `pr-${number}.md`; + } else if (file) { + ({ body, note: notes[0] } = fromBlockOrFile(readFileSync(file, 'utf8'), file)); + defaultName = `${path.basename(file, path.extname(file))}.md`; + } else { + const files = readdirSync(routes) + .filter((f) => f.endsWith('.md')) + .sort() + .map((name) => ({ name, text: readFileSync(path.join(routes, name), 'utf8') })); + if (files.length === 0) throw new Error(`${routes}: no route files (*.md)`); + const out = routesToSpecs(files); + body = out.text; + notes = [`${files.length} compiled route(s) → imported`, ...out.warnings]; + defaultName = 'ui-verify-routes.md'; + } + + const out = path.resolve(flag(args, '--out') ?? path.join(PKG, 'specs', defaultName)); + if (existsSync(out) && !args.includes('--force')) throw new Error(`${out} exists — pass --force to overwrite`); + const text = `# Specs: imported from ui-verify\n\nTarget: local\n\n${body}`; + writeFileSync(out, text); + + for (const n of notes) console.log(`• ${n}`); + try { + const specs = parseSpecFile(text, path.relative(PKG, out)); + console.log(`✓ ${specs.length} spec(s) ready → ${path.relative(process.cwd(), out)}`); + } catch (err) { + if (!(err instanceof SpecError)) throw err; + console.log(`! draft written → ${path.relative(process.cwd(), out)}`); + console.log(` finish it before it can run: ${err.message}`); + } +} + +function runExport(args) { + const files = args.filter((a) => !a.startsWith('--')); + if (files.length === 0) usage('export needs at least one spec file'); + const specs = files.flatMap((f) => parseSpecFile(readFileSync(f, 'utf8'), f)); + process.stdout.write(`${specsToUiVerifyBlock(specs)}\n`); +} + +const [command, ...rest] = process.argv.slice(2); +try { + if (command === 'import') runImport(rest); + else if (command === 'export') runExport(rest); + else usage(command ? `unknown command "${command}"` : undefined); +} catch (err) { + console.error(`error: ${err.message}`); + process.exit(1); +} diff --git a/packages/web-e2e/specs/README.md b/packages/web-e2e/specs/README.md new file mode 100644 index 000000000..442f3ae4a --- /dev/null +++ b/packages/web-e2e/specs/README.md @@ -0,0 +1,110 @@ +# Website e2e specs + +Every `## Spec` block in this folder is a Playwright test. Write Markdown; no +TypeScript needed. The format is aw-tester's `WHEN`/`THEN` grammar, the same +one a `/ui-verify` PR block uses (``), so a spec moves +between a PR and this suite unchanged. + +## Add a test + +Append a block to any `*.md` file here (or add a new file): + +```markdown +## Spec 29: Settings › Billing renders +url: /settings/billing +preconditions: + - User is signed in (handled by the e2e auth setup) +flow: + - WHEN page loads + THEN {role: "heading", name: "Billing"} is visible +``` + +Then run it: + +```bash +cd packages/web-e2e +npx playwright test -g "Billing" # against a local build (see docs/smoke-tests.md) +pnpm nx test web-e2e # parse check + page coverage, no browser +``` + +A new page under `packages/web/src/app` with no spec fails `nx test web-e2e` +(and so the PR) until a spec visits it. + +## The grammar + +```text +## Spec : +url: +persist: critical-path (optional, ignored here) +preconditions: (optional) + - +flow: + - WHEN + THEN + AND + - CAPTURE "