diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e640e2ce..7d942e92c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,8 @@ jobs: # `code` — any non-docs file touched → gates the `check` job. # `api` — API/backend paths → gates the (expensive) `integration` job. # `plugin` — Claude-plugin paths → gates the `plugin` smoke job. - # `web` — dashboard paths (broad) → gates the `web-test` Storybook job. + # `web` — dashboard paths (broad) → gates the `web-test` Storybook job + # and the `web-e2e` Playwright job. # `web_preview` — the narrower, Vercel-quota-metered subset → gates # `web-preview`. Two filters, not one, because the test job is # free CI compute and the preview is a metered deployment. @@ -114,7 +115,10 @@ jobs: # is precisely what the visual baselines are for. `ci.yml` is here for # the same reason as in every sibling gate: so an edit to this job runs # the job. - if printf '%s\n' "$CHANGED" | grep -qE '^(packages/web/|packages/schemas/|package\.json$|pnpm-lock\.yaml$|nx\.json$|\.github/workflows/ci\.yml$)'; then + # + # `packages/web-e2e/` rides along: a change to the e2e suite itself must + # run the `web-e2e` job that executes it. + if printf '%s\n' "$CHANGED" | grep -qE '^(packages/web/|packages/web-e2e/|packages/schemas/|package\.json$|pnpm-lock\.yaml$|nx\.json$|\.github/workflows/ci\.yml$)'; then echo "web=true" >> "$GITHUB_OUTPUT" else echo "web=false" >> "$GITHUB_OUTPUT" @@ -361,6 +365,124 @@ jobs: BASE: ${{ github.event.pull_request.base.sha }} run: npx vitest run --config vitest.storybook.config.ts --changed="$BASE" --passWithNoTests + # ── Web e2e — Playwright against a local production build ──────────────────── + # The page smoke suite (packages/web-e2e; docs/smoke-tests.md). The site is + # built with NEXT_PUBLIC_USE_LOCAL_SUPABASE=true and a local Supabase stack + # runs behind it, so the job needs NO secrets (fork PRs included) and + # touches nothing real. It proves that every public page renders, that every + # dashboard page sends a signed-out visitor to /login, and — signed in as a + # throwaway user the suite provisions on the local stack — that every + # dashboard page renders. Any uncaught exception, console error or 5xx on + # a page fails its test, not just a missing heading. + # + # The same suite runs again at deploy time: against the staged production + # build before its domain is assigned, and against lorekit.io after + # (deploy.yml `e2e-web-staged` / `smoke-production`). + # + # `npx playwright`, not `pnpm exec` / `nx run`, for the stdio reason given + # on web-test above. + web-e2e: + name: Web — Playwright e2e (local build + local Supabase) + runs-on: ubuntu-latest + needs: [changes] + if: needs.changes.outputs.web == 'true' + timeout-minutes: 30 + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@v7.0.1 + + - name: Set up pnpm + uses: pnpm/action-setup@v6 + + - name: Set up Node.js + uses: actions/setup-node@v7.1.0 + with: + node-version: 20 + cache: pnpm + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + # Same key as web-test, so both jobs share one browser cache. + - name: Resolve Playwright version + id: pw + working-directory: packages/web-e2e + run: echo "version=$(node -p "require('@playwright/test/package.json').version")" >> "$GITHUB_OUTPUT" + + - name: Cache Playwright browsers + uses: actions/cache@v6 + with: + path: ~/.cache/ms-playwright + key: playwright-${{ runner.os }}-${{ steps.pw.outputs.version }} + + - name: Install Playwright Chromium (+ OS deps) + run: pnpm --filter @lorekit/web-e2e exec playwright install --with-deps chromium + + - name: Install Supabase CLI + uses: supabase/setup-cli@v3 + with: + # Same pin as the integration job — see the reason there. + version: 2.115.0 + + - name: Start local Supabase (auth + edge functions behind the dashboard) + run: supabase start + + - name: Capture local service-role key + id: supabase + run: | + supabase status -o json > /tmp/status.json + KEY="$(jq -r '.SERVICE_ROLE_KEY // .SECRET_KEY // .service_role_key // .Secret // empty' /tmp/status.json)" + if [ -z "$KEY" ]; then + echo "::error::No service-role/secret key in supabase status. Available keys:" + jq -r 'keys[]' /tmp/status.json + exit 1 + fi + echo "::add-mask::$KEY" + echo "service_role_key=$KEY" >> "$GITHUB_OUTPUT" + + - name: Cache the Next.js build cache + uses: actions/cache@v6 + with: + path: packages/web/.next/cache + key: web-e2e-next-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }}-${{ hashFiles('packages/web/src/**', 'packages/schemas/src/**') }} + restore-keys: | + web-e2e-next-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }}- + + # NEXT_PUBLIC_* values are inlined at build time, so THIS step decides + # that the site talks to the local stack and nothing else. + - name: Build the website (pointed at the local stack) + working-directory: packages/web + env: + NEXT_PUBLIC_USE_LOCAL_SUPABASE: 'true' + NEXT_TELEMETRY_DISABLED: '1' + run: npx next build + + # Playwright starts `next start` itself (playwright.config.ts webServer). + # E2E_SUPABASE_* lets the setup project create a throwaway confirmed user + # on the local stack (refused for any non-local URL — src/support/env.ts). + - name: Playwright — every page, signed out and signed in + working-directory: packages/web-e2e + env: + E2E_SUPABASE_URL: http://127.0.0.1:54321 + E2E_SUPABASE_SERVICE_ROLE_KEY: ${{ steps.supabase.outputs.service_role_key }} + run: npx playwright test + + - name: Upload the Playwright report (traces + screenshots) + if: failure() + uses: actions/upload-artifact@v7 + with: + name: web-e2e-report + path: | + packages/web-e2e/playwright-report + packages/web-e2e/test-results + retention-days: 14 + + - name: Stop local Supabase + if: always() + run: supabase stop --no-backup + # ── Web preview — deploy a Vercel preview, ONLY when web paths changed ──────── # Replaces Vercel's native Git integration for the dashboard project. That # integration deployed a preview on EVERY PR/branch push — even docs- or @@ -924,7 +1046,7 @@ jobs: # `always()` so the gate still runs (and reports) even when an upstream job # failed, was cancelled, or was skipped — otherwise it would be skipped too. if: always() - needs: [changes, check, plugin, integration, web-test, migration-order, deploy-scope, skill-version-guard, seed-tests, preview-filter, bench-tests, edge-typecheck] + needs: [changes, check, plugin, integration, web-test, web-e2e, migration-order, deploy-scope, skill-version-guard, seed-tests, preview-filter, bench-tests, edge-typecheck] permissions: {} steps: - name: Aggregate job results into a pass/fail gate diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index b0134efbb..9005aabee 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -11,9 +11,9 @@ # # deploy-preview (API) ────────┐ # deploy-web-preview (FE) ──────┤ -# stage-web-production (FE build)┘ +# stage-web-production (FE build)┘─▶ e2e-web-staged (Playwright) ──────────┐ # └─▶ smoke-preview ─▶ deploy-production (API) ─▶ promote-web-production (FE) -# └────────────────────────┴─▶ smoke-production +# └────────────────────────┴─▶ smoke-production (+ Playwright) # ├─▶ rollback-production (functions) # └─▶ rollback-web-production (Vercel) # any job fails ─▶ notify-failure (Discord + Dash0) @@ -37,7 +37,9 @@ # Repo-level Vercel secrets (used by every web job): VERCEL_TOKEN, VERCEL_ORG_ID, # VERCEL_PROJECT_ID — the same three preview.yml already uses. # Optional repo-level variable: WEB_PROD_URL (the production dashboard origin the -# production smoke curls; defaults to https://lorekit.io). +# production smoke curls; defaults to https://www.lorekit.io). It must be the +# origin that serves pages without a redirect: the e2e page-health check only +# counts same-origin responses. # Optional (preview only) — enables the orgs REST smoke suite in smoke-preview # (see its "Mint a user JWT" step); unset → that suite is announced-skipped and # the memories suite still runs. SUPABASE_ANON_KEY additionally points the @@ -48,6 +50,12 @@ # The fixed smoke user those secrets name must already exist on the project — seed it # once with `node scripts/smoke/seed-smoke-user.mjs` (see docs/deployment.md → "Seed # the orgs-smoke user"); the mint signs in as it, it is never auto-created. +# Optional repo-level secrets for the website e2e suite (packages/web-e2e, +# docs/smoke-tests.md): VERCEL_AUTOMATION_BYPASS_SECRET lets e2e-web-staged reach +# the staged build through Vercel Deployment Protection (unset → that job skips, +# announced, and promotion proceeds as before); LOREKIT_TESTING_USER_NAME + +# LOREKIT_TESTING_USER_PW sign in a production test account for the signed-in +# pages (unset → those tests skip, announced; public pages still run). # Optional repo-level secret: DISCORD_WEBHOOK_URL — when set, notify-failure posts # a message there on any deploy-pipeline failure. Unset → notify-failure no-ops. # Add a required reviewer on the `production` environment for a manual gate. @@ -395,6 +403,120 @@ jobs: echo "### Web production (staged)" >> "$GITHUB_STEP_SUMMARY" echo "- Built and uploaded: $URL (domain not yet applied)" >> "$GITHUB_STEP_SUMMARY" + # ── 1d. Website e2e against the STAGED production build ─────────────────────── + # Playwright (packages/web-e2e) against the exact bundle promote-web-production + # is about to put on the production domain — so a broken page stops the flip + # instead of being rolled back after users saw it. The staged URL is behind + # Vercel Deployment Protection; VERCEL_AUTOMATION_BYPASS_SECRET gets through it + # and, while it is unset, this job skips (announced) and promotion proceeds + # exactly as before. + # + # Signed-in pages run here only when this merge leaves the API alone. When the + # API changed too, it reaches production only AFTER this job (deploy-production), + # so the staged bundle would be tested against an API it will never be served + # with; those pages run in smoke-production instead, once both halves are live. + # + # No `environment:` — a required reviewer on `production` must not gate a test. + e2e-web-staged: + name: Web e2e → staged production build (Playwright) + runs-on: ubuntu-latest + needs: [changes, stage-web-production] + if: needs.stage-web-production.result == 'success' + timeout-minutes: 20 + permissions: + contents: read + # Secrets are set on the steps that use them, never job-wide: a job-level + # env is readable by every step, third-party actions and `pnpm install` included. + env: + E2E_BASE_URL: ${{ needs.stage-web-production.outputs.deployment_url }} + # The staged build talks to the PRODUCTION backend, whatever its URL says. + E2E_TARGET: production + API_CHANGED: ${{ needs.changes.outputs.api }} + steps: + - name: Skip without the Vercel bypass secret + id: gate + env: + HAS_BYPASS: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET != '' }} + run: | + if [ "$HAS_BYPASS" != "true" ]; then + echo "::warning::VERCEL_AUTOMATION_BYPASS_SECRET is not set — the staged build is behind Vercel Deployment Protection, so the website e2e gate is SKIPPED. Add the secret (Vercel ▸ Project ▸ Settings ▸ Deployment Protection ▸ Protection Bypass for Automation) to turn it on." + echo "run=false" >> "$GITHUB_OUTPUT" + else + echo "run=true" >> "$GITHUB_OUTPUT" + fi + + - name: Checkout repository + if: steps.gate.outputs.run == 'true' + uses: actions/checkout@v7.0.1 + + - name: Set up pnpm + if: steps.gate.outputs.run == 'true' + uses: pnpm/action-setup@v6 + + - name: Set up Node.js + if: steps.gate.outputs.run == 'true' + uses: actions/setup-node@v7.1.0 + with: + node-version: 20 + cache: pnpm + + - name: Install dependencies + if: steps.gate.outputs.run == 'true' + run: pnpm install --frozen-lockfile + + - name: Install Playwright Chromium (+ OS deps) + if: steps.gate.outputs.run == 'true' + run: pnpm --filter @lorekit/web-e2e exec playwright install --with-deps chromium + + - name: Playwright — public pages on the staged build + if: steps.gate.outputs.run == 'true' + working-directory: packages/web-e2e + env: + VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} + run: npx playwright test --project=public + + - name: Note — signed-in pages run after promotion this time + if: steps.gate.outputs.run == 'true' && env.API_CHANGED == 'true' + run: echo "::notice::This merge changes the API too — signed-in pages run in smoke-production, once both halves are live." + + # Sign in as its own step, allowed to fail: an expired test-account password + # or an auth rate limit is a credential problem, not a broken page, so it + # warns rather than blocking promotion. Pages that fail AFTER a successful + # sign-in do block it (next step). + - name: Sign in the e2e test account on the staged build + id: login + if: steps.gate.outputs.run == 'true' && env.API_CHANGED != 'true' + continue-on-error: true + working-directory: packages/web-e2e + env: + VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} + E2E_USER_EMAIL: ${{ secrets.LOREKIT_TESTING_USER_NAME }} + E2E_USER_PASSWORD: ${{ secrets.LOREKIT_TESTING_USER_PW }} + run: npx playwright test --project=setup + + - name: Playwright — signed-in pages on the staged build + if: steps.login.outcome == 'success' + working-directory: packages/web-e2e + env: + VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} + E2E_USER_EMAIL: ${{ secrets.LOREKIT_TESTING_USER_NAME }} + E2E_USER_PASSWORD: ${{ secrets.LOREKIT_TESTING_USER_PW }} + run: npx playwright test --project=signed-in --no-deps + + - name: Warn — the test account could not sign in + if: steps.login.outcome == 'failure' + run: echo "::warning::The e2e test account could not sign in to the staged build, so signed-in pages were NOT checked before promotion. Check LOREKIT_TESTING_USER_NAME / LOREKIT_TESTING_USER_PW." + + - name: Upload the Playwright report (screenshots — traces are off here) + if: failure() + uses: actions/upload-artifact@v7 + with: + name: web-e2e-staged-report + path: | + packages/web-e2e/playwright-report + packages/web-e2e/test-results + retention-days: 14 + # ── 2. Integration test against PREVIEW (the gate before prod) ──────────────── smoke-preview: name: Smoke tests → preview @@ -528,6 +650,29 @@ jobs: URL: https://${{ secrets.SUPABASE_PROJECT_REF }}.supabase.co/functions/v1/mcp TOKEN: ${{ secrets.LOREKIT_SMOKE_TOKEN }} + # Every `lorekit` command, through the real binary, against the deploy that + # just landed (packages/cli/test/smoke, remote lane). The PR gate already ran + # the same cases offline; this proves them against a real backend. It + # writes only under its own `project::cli-smoke-` scope and hard-deletes + # what it wrote, and refuses production by itself — preview only, like the + # MCP surface smoke above. Retried once: the run is self-contained, so a + # re-run after a transient flake is safe and a real regression still fails. + - name: Smoke — every CLI command against preview + run: | + for attempt in 1 2; do + node packages/cli/test/smoke/run.mjs --remote "$URL" "$TOKEN" && break + rc=$? + if [ "$attempt" -eq 2 ]; then + echo "::error::CLI smoke failed after $attempt attempts" + exit "$rc" + fi + echo "::warning::CLI smoke failed (attempt $attempt/2) — retrying in 10s (transient live-endpoint flake)..." + sleep 10 + done + env: + URL: https://${{ secrets.SUPABASE_PROJECT_REF }}.supabase.co/functions/v1/mcp + TOKEN: ${{ secrets.LOREKIT_SMOKE_TOKEN }} + # Mint a user JWT so smoke-rest can include the orgs suite. Fixed-user mode # (LOREKIT_SMOKE_EMAIL/PASSWORD) — never provisions users in a real tenant, # unlike CI's ephemeral mode against a throwaway local stack. Runs only when @@ -711,13 +856,15 @@ jobs: promote-web-production: name: Promote web → production (Vercel) runs-on: ubuntu-latest - needs: [changes, deploy-production, stage-web-production] - # Flip the web when it changed (stage-web-production ran) AND either the API + needs: [changes, deploy-production, stage-web-production, e2e-web-staged] + # Flip the web when it changed (stage-web-production ran), its staged build + # passed the website e2e (e2e-web-staged — green when it skips for a missing + # bypass secret), AND either the API # did not change (nothing to wait for) or the API's prod deploy succeeded (so # the API is live before the FE fronts it). `changes.outputs.api` disambiguates # deploy-production's 'skipped' (API unchanged, fine) from a real upstream stop # (API changed but its deploy failed / smoke-preview failed → not 'success'). - if: always() && needs.stage-web-production.result == 'success' && (needs.changes.outputs.api == 'false' || needs.deploy-production.result == 'success') + if: always() && needs.stage-web-production.result == 'success' && needs.e2e-web-staged.result == 'success' && (needs.changes.outputs.api == 'false' || needs.deploy-production.result == 'success') environment: production permissions: # `write` for this job's own deployed-SHA marker, nothing else. @@ -908,7 +1055,7 @@ jobs: # canonical dashboard domain. - name: Verify web dashboard responds (production) env: - URL: ${{ vars.WEB_PROD_URL || 'https://lorekit.io' }} + URL: ${{ vars.WEB_PROD_URL || 'https://www.lorekit.io' }} run: | CODE=$(curl -s -o /dev/null -w '%{http_code}' -L "$URL") echo "- Web dashboard: $URL (HTTP $CODE)" >> "$GITHUB_STEP_SUMMARY" @@ -917,6 +1064,61 @@ jobs: exit 1 fi + # The website, in a browser, as users get it now that both halves are live: + # every public page, every dashboard page's signed-out redirect, and — with + # the test account secrets — every dashboard page signed in plus the shell + # journeys (packages/web-e2e). Read-only: page views and nothing else (the + # sign-out and wrong-password journeys skip themselves on production). A + # failed page trips both rollbacks, like the curl above; the suite retries + # each test twice on CI before failing. + - name: Install Playwright Chromium (+ OS deps) + run: pnpm --filter @lorekit/web-e2e exec playwright install --with-deps chromium + + - name: Web e2e — public pages against production (Playwright) + working-directory: packages/web-e2e + env: + E2E_BASE_URL: ${{ vars.WEB_PROD_URL || 'https://www.lorekit.io' }} + E2E_TARGET: production + run: npx playwright test --project=public + + # Sign in as its own step, allowed to fail: an expired test-account password + # or an auth rate limit must never roll back a healthy deploy. A signed-in + # page that fails AFTER a successful sign-in does (next step). + - name: Web e2e — sign in the test account (production) + id: e2e_login + continue-on-error: true + working-directory: packages/web-e2e + env: + E2E_BASE_URL: ${{ vars.WEB_PROD_URL || 'https://www.lorekit.io' }} + E2E_TARGET: production + E2E_USER_EMAIL: ${{ secrets.LOREKIT_TESTING_USER_NAME }} + E2E_USER_PASSWORD: ${{ secrets.LOREKIT_TESTING_USER_PW }} + run: npx playwright test --project=setup + + - name: Web e2e — signed-in pages against production (Playwright) + if: steps.e2e_login.outcome == 'success' + working-directory: packages/web-e2e + env: + E2E_BASE_URL: ${{ vars.WEB_PROD_URL || 'https://www.lorekit.io' }} + E2E_TARGET: production + E2E_USER_EMAIL: ${{ secrets.LOREKIT_TESTING_USER_NAME }} + E2E_USER_PASSWORD: ${{ secrets.LOREKIT_TESTING_USER_PW }} + run: npx playwright test --project=signed-in --no-deps + + - name: Warn — the test account could not sign in (production) + if: steps.e2e_login.outcome == 'failure' + run: echo "::warning::The e2e test account could not sign in to production, so signed-in pages were NOT checked after this deploy. Check LOREKIT_TESTING_USER_NAME / LOREKIT_TESTING_USER_PW." + + - name: Upload the Playwright report (screenshots — traces are off here) + if: failure() + uses: actions/upload-artifact@v7 + with: + name: web-e2e-production-report + path: | + packages/web-e2e/playwright-report + packages/web-e2e/test-results + retention-days: 14 + # Full authenticated onboarding + round-trip against the live production # deploy, mirroring smoke-preview: scaffold skill + config, then # `doctor --deep` (self-cleaning write→read→delete). Stronger than the @@ -1222,7 +1424,7 @@ jobs: notify-failure: name: Notify on failure (Discord + Dash0) runs-on: ubuntu-latest - needs: [changes, deploy-preview, deploy-web-preview, stage-web-production, smoke-preview, deploy-production, promote-web-production, smoke-production, rollback-production, rollback-web-production] + needs: [changes, deploy-preview, deploy-web-preview, stage-web-production, e2e-web-staged, smoke-preview, deploy-production, promote-web-production, smoke-production, rollback-production, rollback-web-production] if: always() && contains(needs.*.result, 'failure') permissions: contents: read # checkout, so the local composite action is available @@ -1237,6 +1439,7 @@ jobs: RESULT_DEPLOY_PREVIEW: ${{ needs.deploy-preview.result }} RESULT_DEPLOY_WEB_PREVIEW: ${{ needs.deploy-web-preview.result }} RESULT_STAGE_WEB_PRODUCTION: ${{ needs.stage-web-production.result }} + RESULT_E2E_WEB_STAGED: ${{ needs.e2e-web-staged.result }} RESULT_SMOKE_PREVIEW: ${{ needs.smoke-preview.result }} RESULT_DEPLOY_PRODUCTION: ${{ needs.deploy-production.result }} RESULT_PROMOTE_WEB_PRODUCTION: ${{ needs.promote-web-production.result }} @@ -1251,6 +1454,7 @@ jobs: elif [ "$RESULT_DEPLOY_PREVIEW" = "failure" ]; then failed="deploy → preview" elif [ "$RESULT_DEPLOY_WEB_PREVIEW" = "failure" ]; then failed="web deploy → preview" elif [ "$RESULT_STAGE_WEB_PRODUCTION" = "failure" ]; then failed="web build (production staging)" + elif [ "$RESULT_E2E_WEB_STAGED" = "failure" ]; then failed="web e2e → staged production build" elif [ "$RESULT_SMOKE_PREVIEW" = "failure" ]; then failed="smoke tests → preview" elif [ "$RESULT_DEPLOY_PRODUCTION" = "failure" ]; then failed="deploy → production" elif [ "$RESULT_PROMOTE_WEB_PRODUCTION" = "failure" ]; then failed="web promote → production" diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml index 5e940480c..a1b523b01 100644 --- a/.github/workflows/preview.yml +++ b/.github/workflows/preview.yml @@ -371,6 +371,15 @@ jobs: URL: https://${{ secrets.SUPABASE_PROJECT_REF }}.supabase.co/functions/v1/mcp TOKEN: ${{ secrets.LOREKIT_SMOKE_TOKEN }} + # Every `lorekit` command against this PR's preview deploy — the same + # remote lane deploy.yml's smoke-preview runs after a merge, so `/preview` + # shows before merging whether the CLI still works against the PR's API. + - name: Smoke — every CLI command against preview + run: node packages/cli/test/smoke/run.mjs --remote "$URL" "$TOKEN" + env: + URL: https://${{ secrets.SUPABASE_PROJECT_REF }}.supabase.co/functions/v1/mcp + TOKEN: ${{ secrets.LOREKIT_SMOKE_TOKEN }} + # Mint a user JWT so smoke-rest can include the orgs suite. Fixed-user mode # (LOREKIT_SMOKE_EMAIL/PASSWORD) — never provisions users in a real tenant. # Runs only when the optional credentials are configured; otherwise the diff --git a/CLAUDE.md b/CLAUDE.md index e3949be52..7bb19b334 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -21,6 +21,7 @@ lets humans browse, search, and manage those lessons. | `plugins/` | `plugins/` | Per-framework deterministic bundles: `lorekit-claude` (marketplace plugin: skill + hooks + MCP), `lorekit-cursor` (rule + `stop` hook), `lorekit-codex` (feature-flagged hooks + `AGENTS.md` fallback, experimental). Root `.claude-plugin/marketplace.json` lists the Claude plugin. | | `supabase` | `supabase/` | Edge Functions (production MCP server), migrations, NX targets | | `@lorekit/smoke-tests` | `packages/smoke-tests/` | Live-endpoint integration/smoke tests against the deployed Edge Functions (memories, orgs, MCP, BYOD) — no application code, self-skips when its env vars are absent | +| `@lorekit/web-e2e` | `packages/web-e2e/` | Playwright suite for every page and key journey of the website, written as Markdown specs in `specs/*.md` (aw-tester's `WHEN`/`THEN` grammar — the same body as a `` PR block; `pnpm --filter @lorekit/web-e2e spec:import` / `spec:export` move them). A vitest guard fails the PR when a spec does not parse or a `page.tsx` has no spec. Runs per PR (local build + local Supabase), on the staged production build before promotion, and against `lorekit.io` after it — see [docs/smoke-tests.md](./docs/smoke-tests.md) | The **production MCP server** is `supabase/functions/mcp/index.ts` (Deno, self-contained). There is no other MCP server implementation — a prior Node.js/Fly.io variant (`packages/mcp-server/`) @@ -152,6 +153,8 @@ npx vitest run --config vitest.storybook.config.ts -u # update base - CI runs these in the `web-test` job (browser job, gated by `changes.web`) — NOT part of `check`'s `nx affected -t test`. - Full-page stories mock Supabase REST with **MSW** so real React Query hooks resolve against a stable dataset; server-component pages story their largest client subtree, never refactored to client. Storybook deploys as its OWN Vercel project. Full runbook + MSW / mixed-rendering / deploy detail: [docs/storybook.md](./docs/storybook.md). +**Smoke + e2e coverage is table-driven and guarded.** A new page needs a spec in `packages/web-e2e/specs/*.md` (Markdown, ui-verify grammar — `specs/README.md`); a new CLI command a case in `packages/cli/test/smoke/cases.mjs`; a new MCP tool a call in `scripts/smoke/smoke-mcp-tools.mjs` (and in the offline `mcp` case unless it is in `LOCAL_MCP_EXEMPT`). `specs.test.ts` and `cli-smoke.test.mjs` fail the PR otherwise. What runs where: [docs/smoke-tests.md](./docs/smoke-tests.md). + --- ## User-facing docs (mandatory on every change) diff --git a/docs/README.md b/docs/README.md index 358a4d7be..a32d7e936 100644 --- a/docs/README.md +++ b/docs/README.md @@ -21,6 +21,7 @@ | [telemetry-quality-review.md](./telemetry-quality-review.md) | Developers | Cross-service `traceparent` correlation contract, telemetry-quality review vs OTel semantic conventions, and the tests that guard it | | [deployment.md](./deployment.md) | Developers | Step-by-step deployment for all three pieces | | [storybook.md](./storybook.md) | Developers | Web Storybook: MSW-mocked full-page stories, determinism, and deploying Storybook as a second Vercel project | +| [smoke-tests.md](./smoke-tests.md) | Developers | Website e2e (Playwright), CLI and MCP smoke suites: what runs on each PR and deploy, and how to add a page, command or tool | | [releasing.md](./releasing.md) | Developers | Automated `@lorekit/cli` npm releases via release-please + conventional commits | | [adding-an-operation.md](./adding-an-operation.md) | Developers | Step-by-step checklist for adding a new MCP tool / CLI command / REST route across the catalog, both permission mirrors, the audit vocabulary, and docs | | [sandbox.md](./sandbox.md) | Developers | Fresh sandbox/container baseline — the six traps that make a red gate lie, and how to prove a failure pre-existing (full runbook behind the CLAUDE.md summary) | diff --git a/docs/adding-an-operation.md b/docs/adding-an-operation.md index efa075c26..ca36d5978 100644 --- a/docs/adding-an-operation.md +++ b/docs/adding-an-operation.md @@ -132,6 +132,12 @@ catch a mistake at that step. `packages/web/src/lib/docs/sections.ts` entry (`sections.spec.ts` guards drift), and update `docs/README.md`'s index table and `README.md`'s package map if the capability is user-facing. +11. **Smoke checks** — add a `check(...)` that calls the tool to + `scripts/smoke/smoke-mcp-tools.mjs` (the hosted server, run at deploy time), + and, unless it is in `LOCAL_MCP_EXEMPT`, call it in the offline `mcp` case in + `packages/cli/test/smoke/cases.mjs` (plus `LOCAL_MCP_CALLED`). A new CLI + command also needs a case there. `cli-smoke.test.mjs` fails the PR on any + of the three gaps. How-to: [smoke-tests.md](./smoke-tests.md). ## 3. Gates that catch a missed step @@ -145,6 +151,7 @@ failure is cheaper to fix: | `pnpm nx test schemas` (`surface-generator.spec.ts`, `render.spec.ts`) | Generated artifacts + `llms.txt` drift | | `pnpm nx test mcp-core` (`tool-catalog-parity.spec.ts`, `permissions.spec.ts`, `audit-vocabulary.spec.ts`, `edge-parity.spec.ts`, `edge-schema-parity.spec.ts`, `dry-run-coverage.spec.ts`, `audit-coverage.spec.ts`, `mcp-authz-status.spec.ts`) | Permission-mirror drift, audit vocabulary drift, pure-module mirror drift, missing dry-run/audit wiring | | `pnpm nx test cli` (`surface-parity.test.mjs`, `frameworks.test.mjs`) | CLI registry vs catalog binding drift | +| `pnpm nx test cli` (`cli-smoke.test.mjs`) | A tool or command with no smoke check; a broken command in the offline CLI/MCP session | | `node scripts/ci/deno-check-functions.mjs --node-modules-dir=none` | Type errors in the edge tree — **nothing else typechecks it** | | `pnpm nx test web` (`sections.spec.ts`) | Missing/malformed docs registry entry | diff --git a/docs/smoke-tests.md b/docs/smoke-tests.md new file mode 100644 index 000000000..5a0d000a2 --- /dev/null +++ b/docs/smoke-tests.md @@ -0,0 +1,143 @@ +# Smoke and end-to-end tests + +Three suites answer one question: **after a merge, does the core of LoreKit +still work?** Each is a table you add a row to, and each has a guard that fails +the PR when something new ships without a row. + +| Suite | What it drives | Where it lives | Guard | +|-------|----------------|----------------|-------| +| Website e2e | Every page and key journey of `@lorekit/web` in a real browser (Playwright) | `packages/web-e2e/specs/*.md` — Markdown, the ui-verify `WHEN`/`THEN` grammar | `specs.test.ts` — every spec parses, and every `page.tsx` is visited by one | +| CLI smoke | Every `lorekit` command, through the real binary | `packages/cli/test/smoke/` | `cli-smoke.test.mjs` — every command and alias needs a case | +| MCP smoke | Every MCP tool: the local stdio server and the hosted server | `packages/cli/test/smoke/cases.mjs` (`mcp` case) and `scripts/smoke/smoke-mcp-tools.mjs` | `cli-smoke.test.mjs` — every catalog tool needs a call in both | + +The guards need no browser, no network and no secrets, so they run in the +`check` job (`nx affected -t test`) on every PR. + +## What runs where + +| When | Website e2e | CLI smoke | MCP smoke | +|------|-------------|-----------|-----------| +| **Every PR** (`ci.yml`) | `web-e2e` job: a local production build against a local Supabase stack — public pages, the signed-out redirect of every dashboard page, and every dashboard page signed in as a throwaway user | offline lane, inside `cli:test` | local stdio server, inside `cli:test` | +| **`/preview` on a PR** (`preview.yml`) | — | remote lane against the preview project | hosted server against the preview project | +| **Merge → before production** (`deploy.yml`) | `e2e-web-staged`: the staged production build, before its domain is assigned (signed-in pages too when the merge leaves the API alone). Gates `promote-web-production` | remote lane in `smoke-preview`. Gates `deploy-production` | hosted server in `smoke-preview`. Gates `deploy-production` | +| **Merge → after production** (`deploy.yml`) | `smoke-production`: `https://www.lorekit.io`, public + signed in. A failed page rolls both halves back; a failed test-account sign-in only warns | — (it writes; production is never written to by a smoke) | stdio handshake only (read-only) | + +## Adding a website test + +Append a Markdown block to a file in `packages/web-e2e/specs/` — no TypeScript: + +```markdown +## Spec 29: Settings › Billing renders +url: /settings/billing +preconditions: + - User is signed in (handled by the e2e auth setup) +flow: + - WHEN page loads + THEN {role: "heading", name: "Billing"} is visible +``` + +- The format is aw-tester's `WHEN`/`THEN` grammar — the body of a + `` PR block — so a spec moves between a PR and this suite + unchanged. `pnpm --filter @lorekit/web-e2e spec:import --pr ` pulls a PR's + ui-verify block in (an intent block arrives as a draft with TODO lines to + finish); `pnpm --filter @lorekit/web-e2e spec:export ` prints a block to + paste into a PR. Full guide, every supported phrase and locator: + [`packages/web-e2e/specs/README.md`](../packages/web-e2e/specs/README.md). +- `User is signed in …` in `preconditions:` runs the spec as the test user, and + also generates a check that a signed-out visitor to its url is sent to + `/login?next=`. `Not on production: ` skips it there. +- Assert what a person sees — the heading, exactly as rendered (settings pages + share the h1 "Settings", so use their section heading). +- A typo fails `nx test web-e2e` with the file and line, before CI boots a + browser; so does a new `page.tsx` that no spec visits. + +Every test also fails on an uncaught exception, a `console.error` or a +same-origin 5xx, so a page that renders its heading but throws underneath still +fails. + +What the grammar cannot express — a reload, a locator that needs scoping to one +container, a request-level check — goes in `packages/web-e2e/src/journeys/*.e2e.ts` +(tag it `@signed-in` if it needs a session). + +## Adding a CLI command + +Add a case to `packages/cli/test/smoke/cases.mjs`, in session order: + +```js +smoke('billing', 'shows the current plan', BOTH, async ({ cli, ctx }) => { + const out = cli.json(['billing']); + assert.equal(out.ok, true); +}); +``` + +- `cli.json(args)` appends `--json`, asserts exit 0 (or `{ code }`) and returns + the parsed output; `cli.ok(args)` returns the text; `cli.fails(args, /why/)` + asserts a non-zero exit with that message. +- Lanes: `offline` (throwaway HOME + project, `LOREKIT_MODE=local`, no network), + `remote` (a live endpoint), or both. A remote-only command gets an offline case + that asserts it refuses cleanly, and a remote case that asserts it works. +- Anything the remote lane writes must go under `ctx.scope` and through + `writeLesson(...)` / `trackRemoteWrite(...)`, so cleanup hard-deletes it. +- A command that genuinely cannot be smoked goes in `NOT_SMOKED` with the reason. + +## Adding an MCP tool + +After adding the operation (see [adding-an-operation.md](./adding-an-operation.md)): + +1. **Hosted:** add a `check(...)` to `scripts/smoke/smoke-mcp-tools.mjs` that + calls it and asserts what it did, writing only under the run's scope. +2. **Local stdio**, unless the tool is in `LOCAL_MCP_EXEMPT`: call it in the + offline `mcp` case in `cases.mjs` and add it to `LOCAL_MCP_CALLED`. + +## Running locally + +```bash +# Guards (fast, no browser) +pnpm nx test web-e2e +node --test packages/cli/test/cli-smoke.test.mjs # guards + the offline CLI/MCP lane + +# CLI remote lane against preview (writes; refuses production) +node packages/cli/test/smoke/run.mjs --remote "$PREVIEW_MCP_URL" "$LOREKIT_TOKEN" + +# Website e2e against a local build +pnpm nx run web-e2e:build-web # NEXT_PUBLIC_USE_LOCAL_SUPABASE=true +cd packages/web-e2e && npx playwright install chromium +npx playwright test --project=public # signed-out only, no backend needed +npx playwright test --ui # watch it run + +# …with signed-in pages, against `supabase start` +E2E_SUPABASE_URL=http://127.0.0.1:54321 \ +E2E_SUPABASE_SERVICE_ROLE_KEY="$(supabase status -o json | jq -r '.SERVICE_ROLE_KEY // .SECRET_KEY // .service_role_key // .Secret // empty')" \ + npx playwright test + +# …or against a deployed site +E2E_BASE_URL=https://www.lorekit.io E2E_USER_EMAIL=… E2E_USER_PASSWORD=… npx playwright test +``` + +`E2E_WEB_COMMAND="pnpm --filter @lorekit/web exec next dev --port 3100"` swaps +the build for the dev server while iterating. Every variable is documented at +the top of `packages/web-e2e/src/support/env.ts`. + +## Secrets the deploy-time runs use + +| Secret | Used by | Without it | +|--------|---------|------------| +| `VERCEL_AUTOMATION_BYPASS_SECRET` (repo) | `e2e-web-staged` — gets through Vercel Deployment Protection on the staged URL | the job skips with a warning and promotion proceeds as before | +| `LOREKIT_TESTING_USER_NAME`, `LOREKIT_TESTING_USER_PW` (repo) | signed-in pages on the staged build and on `lorekit.io` | signed-in tests skip, announced; public tests still run | +| `LOREKIT_SMOKE_TOKEN` (preview env, existing) | CLI remote lane, hosted MCP smoke | — (already required) | + +Traces are off on any run that uses the bypass secret or targets production: a +trace records request headers and cookies, and this public repo's artifacts are +downloadable by anyone. Failure screenshots still upload. + +The sign-in is its own CI step in both deploy-time jobs: when the test account +cannot sign in (an expired password, an auth rate limit) the signed-in pages are +skipped with a warning, so a credential problem never blocks a promotion or +rolls back production — only a page that fails after a successful sign-in does. +The deliberately failed sign-in (`journeys/login.e2e.ts`) never targets +production. + +The test account must be a normal user (not in `DEVELOPER_EMAILS`); opening +`/lore` as a user with no API tokens creates one, so expect a "My first token". +The sign-out journey never runs against production: signing out revokes every +session of the account. diff --git a/packages/cli/test/cli-smoke.test.mjs b/packages/cli/test/cli-smoke.test.mjs new file mode 100644 index 000000000..fb66e90f6 --- /dev/null +++ b/packages/cli/test/cli-smoke.test.mjs @@ -0,0 +1,120 @@ +// The CLI + MCP smoke gate, run on every PR as part of `cli:test`. +// +// Two jobs: +// +// 1. COVERAGE. Every command in src/commands.mjs, every alias, every tool the +// local stdio server dispatches, and every tool the hosted server serves +// must be exercised by a smoke check — or carry a written exemption. A new +// command or catalog tool with no smoke fails here, on the PR that adds it. +// 2. THE OFFLINE LANE. Runs test/smoke/cases.mjs against the real binary in a +// throwaway HOME and project (no network), one subtest per case. +// +// The remote lane (the same cases against a live endpoint) runs at deploy time: +// `node packages/cli/test/smoke/run.mjs --remote `. +// +// WHY HERE. Adding a command touches src/commands.mjs and adding a catalog tool +// regenerates src/surfaces.generated.mjs — both in this package — so this guard +// is always affected by the change it polices (see surface-parity.test.mjs for +// what goes wrong when a guard lives elsewhere). + +import { test, describe } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; + +import { MCP_TOOL_NAMES } from '../src/surfaces.generated.mjs'; +import { CASES, LOCAL_MCP_CALLED, LOCAL_MCP_TOOLS, NOT_SMOKED } from './smoke/cases.mjs'; +import { runSmoke } from './smoke/run.mjs'; + +const read = (rel) => readFileSync(fileURLToPath(new URL(rel, import.meta.url)), 'utf8'); + +/** + * `{ name, aliases }` for every entry of `COMMANDS` in src/commands.mjs. A + * source scan, like surface-parity.test.mjs: importing the registry would pull + * the whole CLI module graph into this process. + */ +function parseCommands(source) { + const block = /export const COMMANDS = \[([\s\S]*?)\n\];/.exec(source); + assert.ok(block, 'could not find `export const COMMANDS = [` in src/commands.mjs'); + return [...block[1].matchAll(/\{\s*name:\s*'([^']+)'([^\n]*)\}/g)].map(([, name, rest]) => ({ + name, + aliases: [...(/aliases:\s*\[([^\]]*)\]/.exec(rest)?.[1] ?? '').matchAll(/'([^']+)'/g)].map((m) => m[1]), + })); +} + +const commands = parseCommands(read('../src/commands.mjs')); +const casesSource = read('./smoke/cases.mjs'); +const smoked = new Set(CASES.map((c) => c.command)); + +describe('CLI smoke coverage', () => { + test('finds the command registry', () => { + assert.ok(commands.length >= 20, `parsed only ${commands.length} commands from src/commands.mjs`); + }); + + test('every command has a smoke case or a written exemption', () => { + const missing = commands.map((c) => c.name).filter((n) => !smoked.has(n) && !NOT_SMOKED[n]); + assert.deepEqual( + missing, + [], + `No smoke case for: ${missing.join(', ')}. Add one to packages/cli/test/smoke/cases.mjs ` + + '(or, if it genuinely cannot be smoked, an entry with the reason in NOT_SMOKED).', + ); + }); + + test('every case and exemption names a real command', () => { + const names = new Set(commands.map((c) => c.name)); + const unknown = [...smoked, ...Object.keys(NOT_SMOKED)].filter((n) => !names.has(n)); + assert.deepEqual(unknown, [], `cases.mjs names commands that do not exist: ${unknown.join(', ')}`); + for (const [name, reason] of Object.entries(NOT_SMOKED)) { + assert.ok(String(reason).trim().length > 0, `NOT_SMOKED.${name} needs a reason`); + assert.ok(!smoked.has(name), `${name} is both smoked and exempt — drop the exemption`); + } + }); + + test('every alias is invoked by a case', () => { + const aliases = commands.flatMap((c) => c.aliases); + const missing = aliases.filter((a) => !casesSource.includes(`'${a}'`)); + assert.deepEqual(missing, [], `aliases no case invokes: ${missing.join(', ')}`); + }); + + test('every case runs in at least one lane', () => { + const laneless = CASES.filter((c) => c.lanes.length === 0).map((c) => `${c.command} — ${c.name}`); + assert.deepEqual(laneless, []); + }); +}); + +describe('MCP smoke coverage', () => { + test('the offline `mcp` case calls every tool the local stdio server dispatches', () => { + assert.deepEqual( + [...LOCAL_MCP_CALLED].sort(), + [...LOCAL_MCP_TOOLS].sort(), + 'LOCAL_MCP_CALLED in smoke/cases.mjs drifted from the catalog (MCP_TOOL_NAMES minus LOCAL_MCP_EXEMPT)', + ); + const notInSession = LOCAL_MCP_CALLED.filter((t) => !casesSource.includes(`name: '${t}'`)); + assert.deepEqual(notInSession, [], `listed in LOCAL_MCP_CALLED but never called: ${notInSession.join(', ')}`); + }); + + test('scripts/smoke/smoke-mcp-tools.mjs calls every tool the hosted server serves', () => { + // The hosted smoke runs at deploy time (deploy.yml smoke-preview, preview.yml) + // against a real project. This makes its coverage a PR-time guarantee. + const hosted = read('../../../scripts/smoke/smoke-mcp-tools.mjs'); + const missing = MCP_TOOL_NAMES.filter( + (t) => !new RegExp(`(call\\(|name: )'${t.replace('.', '\\.')}'`).test(hosted), + ); + assert.deepEqual( + missing, + [], + `No hosted smoke check calls: ${missing.join(', ')}. Add a check() to scripts/smoke/smoke-mcp-tools.mjs.`, + ); + }); +}); + +test('CLI smoke — offline lane against the real binary', async (t) => { + const { results } = await runSmoke({ lane: 'offline' }); + assert.ok(results.length > 0, 'no offline cases ran'); + for (const r of results) { + await t.test(r.label, () => { + if (r.outcome === 'fail') assert.fail(r.detail); + }); + } +}); diff --git a/packages/cli/test/smoke/cases.mjs b/packages/cli/test/smoke/cases.mjs new file mode 100644 index 000000000..7efc84e4b --- /dev/null +++ b/packages/cli/test/smoke/cases.mjs @@ -0,0 +1,410 @@ +// The CLI smoke cases — one scripted session per lane that runs the REAL +// `lorekit` binary for every command it ships. +// +// offline a throwaway HOME + project, LOREKIT_MODE=local, no network. Runs on +// every PR inside `cli:test` (test/cli-smoke.test.mjs). +// remote the same binary against a live endpoint + token (the preview +// deploy). Writes only under the run's own project scope and +// hard-deletes what it wrote (run.mjs). Refuses production. +// +// ## Adding a command +// +// `cli-smoke.test.mjs` fails when a command in `src/commands.mjs` has no case +// here and no entry in NOT_SMOKED. Add a `smoke('', '', +// [lanes], async ({ cli, ctx }) => { ... })` below, in session order. Assert on +// behaviour (what changed, what came back), not just "exit 0". +// +// Cases run in order and may build on each other (a lesson written early is +// read later) — the session is the realistic shape, and every assertion names +// the exact command that broke. + +import assert from 'node:assert/strict'; +import { existsSync, readFileSync } from 'node:fs'; +import path from 'node:path'; + +import { LOCAL_MCP_EXEMPT, MCP_TOOL_NAMES } from '../../src/surfaces.generated.mjs'; +import { runMcp, trackRemoteWrite } from './harness.mjs'; + +/** @typedef {'offline' | 'remote'} Lane */ + +/** @type {{ command: string, name: string, lanes: Lane[], fn: Function }[]} */ +export const CASES = []; + +/** + * Commands with no smoke case, and why. A declared exemption, so a gap is a + * reviewable decision rather than an oversight. Empty today — keep it that way. + * @type {Record} + */ +export const NOT_SMOKED = {}; + +/** Register a case. `command` is the canonical name from src/commands.mjs (aliases are credited to it). */ +function smoke(command, name, lanes, fn) { + CASES.push({ command, name, lanes, fn }); +} + +const OFFLINE = /** @type {Lane[]} */ (['offline']); +const REMOTE = /** @type {Lane[]} */ (['remote']); +const BOTH = /** @type {Lane[]} */ (['offline', 'remote']); + +/** The tools the local stdio server should advertise: the catalog minus its declared exemptions. */ +export const LOCAL_MCP_TOOLS = MCP_TOOL_NAMES.filter((t) => !(t in LOCAL_MCP_EXEMPT)); + +/** The tools the offline `mcp` case calls — the coverage guard checks this against LOCAL_MCP_TOOLS. */ +export const LOCAL_MCP_CALLED = [ + 'memory.write', + 'memory.read', + 'memory.list', + 'memory.search', + 'memory.scopes', + 'memory.archive', + 'memory.restore', + 'memory.delete', + 'org.create', + 'org.list', + 'org.rename', + 'org.delete', +]; + +// ── helpers ────────────────────────────────────────────────────────────────── + +/** The store block a lane reads back from: `offline` locally, `remote` against an endpoint. */ +const storeOf = (ctx) => (ctx.lane === 'remote' ? 'remote' : 'offline'); + +/** Every `key` under a JSON block, however the command groups its entries. */ +function keysIn(value) { + const keys = []; + const walk = (v) => { + if (Array.isArray(v)) v.forEach(walk); + else if (v && typeof v === 'object') { + if (typeof v.key === 'string') keys.push(v.key); + Object.values(v).forEach(walk); + } + }; + walk(value); + return [...new Set(keys)]; +} + +const alphaValue = (ctx) => `The CLI smoke suite wrote this lesson — marker zebra-${ctx.runId}.`; + +/** Write through the CLI and record it for the remote lane's cleanup. */ +function writeLesson(cli, ctx, name, value, extra = []) { + const key = ctx.key(name); + trackRemoteWrite(ctx, ctx.scope, key); + return cli.json(['write', `${ctx.scope}::${key}`, value, ...extra]); +} + +// ── setup: install, health ─────────────────────────────────────────────────── + +smoke('install', 'scaffolds the skills for this project without prompting', OFFLINE, async ({ cli, ctx }) => { + cli.ok(['install', '--project', '--no-hooks', '--yes']); + assert.ok( + existsSync(path.join(ctx.dir, '.claude/skills/lorekit-memory/SKILL.md')), + 'install did not write .claude/skills/lorekit-memory/SKILL.md', + ); +}); + +smoke('install', 'wires the given endpoint into the project MCP config', REMOTE, async ({ cli, ctx }) => { + cli.ok(['install', '--project', '--no-hooks', '--yes', '--endpoint', ctx.endpoint, '--token', ctx.token]); + const mcp = readFileSync(path.join(ctx.dir, '.mcp.json'), 'utf8'); + assert.ok(mcp.includes(new URL(ctx.endpoint).host), `.mcp.json does not name ${ctx.endpoint}:\n${mcp}`); +}); + +smoke('update', 'reports freshly installed skills as up to date', OFFLINE, async ({ cli }) => { + assert.match(cli.ok(['update', '--check']), /already (at the shipped version|up to date)/); +}); + +smoke('doctor', 'passes on a fresh local install', OFFLINE, async ({ cli }) => { + const out = cli.ok(['doctor', '--mode', 'local']); + assert.match(out, /PASS home store/); + assert.doesNotMatch(out, /\bFAIL\b/); +}); + +smoke('doctor', 'completes a write → read → delete round-trip against the endpoint', REMOTE, async ({ cli }) => { + const out = cli.ok(['doctor', '--deep']); + assert.doesNotMatch(out, /\bFAIL\b/); +}); + +// ── write and read back ────────────────────────────────────────────────────── + +smoke('write', 'stores a lesson from a positional value', BOTH, async ({ cli, ctx }) => { + const out = writeLesson(cli, ctx, 'alpha', alphaValue(ctx), ['--tags', 'smoke,cli']); + assert.equal(out.store, ctx.lane === 'remote' ? 'remote' : 'local'); + assert.equal(out.scope, ctx.scope); + assert.equal(out.key, ctx.key('alpha')); + assert.deepEqual(out.tags, ['smoke', 'cli']); +}); + +smoke('write', 'stores a lesson piped on stdin', BOTH, async ({ cli, ctx }) => { + trackRemoteWrite(ctx, ctx.scope, ctx.key('beta')); + const out = cli.json(['write', `${ctx.scope}::${ctx.key('beta')}`], { input: 'short body\n' }); + assert.equal(out.value, 'short body'); +}); + +smoke('show', 'returns the full lesson that was written', BOTH, async ({ cli, ctx }) => { + const out = cli.json(['show', `${ctx.scope}::${ctx.key('alpha')}`]); + const block = out[storeOf(ctx)]; + assert.equal(block.found, true, `alpha not found in the ${storeOf(ctx)} store`); + assert.equal(block.record.value, alphaValue(ctx)); +}); + +smoke('list', 'lists the lessons that apply to this directory (also as `ls`)', BOTH, async ({ cli, ctx }) => { + for (const cmd of ctx.lane === 'offline' ? ['list', 'ls'] : ['list']) { + const out = cli.json([cmd]); + const keys = keysIn(out[storeOf(ctx)]); + assert.ok(keys.includes(ctx.key('alpha')) && keys.includes(ctx.key('beta')), `\`${cmd}\` is missing alpha/beta: ${keys}`); + } +}); + +smoke('search', 'finds a lesson by a word in its body (also as `grep`)', BOTH, async ({ cli, ctx }) => { + for (const cmd of ctx.lane === 'offline' ? ['search', 'grep'] : ['search']) { + const out = cli.json([cmd, `zebra-${ctx.runId}`]); + assert.deepEqual(keysIn(out[storeOf(ctx)]), [ctx.key('alpha')], `\`${cmd}\` should match alpha only`); + } +}); + +smoke('stats', 'counts the lessons per scope', BOTH, async ({ cli, ctx }) => { + const out = cli.json(['stats']); + assert.ok(out[storeOf(ctx)].total >= 2, `expected at least 2 lessons, got ${out[storeOf(ctx)].total}`); +}); + +smoke('scopes', 'inventories the scopes that hold lessons', BOTH, async ({ cli, ctx }) => { + const out = cli.json(['scopes']); + const block = out[storeOf(ctx)]; + assert.equal(block.available, true, `the ${storeOf(ctx)} inventory is unavailable: ${block.reason}`); + assert.ok(JSON.stringify(block.scopes).includes(ctx.scope), `${ctx.scope} missing from scopes`); +}); + +smoke('tree', 'shows which lesson wins for this directory (also as `resolve`)', BOTH, async ({ cli, ctx }) => { + for (const cmd of ctx.lane === 'offline' ? ['tree', 'resolve'] : ['tree']) { + const out = cli.json([cmd]); + assert.ok(keysIn(out[storeOf(ctx)].winners).includes(ctx.key('alpha')), `\`${cmd}\` lists no winning alpha`); + } +}); + +smoke('lint', 'fails the gate on a too-short lesson and names it', BOTH, async ({ cli, ctx }) => { + // beta's body is deliberately short — lint is documented as a CI gate, so it must exit non-zero. + const out = cli.json(['lint'], { code: 1 }); + const findings = JSON.stringify(out[storeOf(ctx)]); + assert.match( + findings, + new RegExp(`"key":"${ctx.key('beta')}"[^}]*"rule":"short-value"`), + `no short-value finding for beta: ${findings}`, + ); +}); + +smoke('dedupe', 'clusters a near-duplicate lesson', BOTH, async ({ cli, ctx }) => { + writeLesson(cli, ctx, 'alpha-copy', alphaValue(ctx)); + const out = cli.json(['dedupe']); + const clusters = JSON.stringify(out[storeOf(ctx)].clusters); + assert.ok( + clusters.includes(`"${ctx.key('alpha-copy')}"`) && clusters.includes(`"${ctx.key('alpha')}"`), + `alpha/alpha-copy not clustered: ${clusters}`, + ); +}); + +smoke('delete', 'hard-deletes with --force (also as `rm`)', BOTH, async ({ cli, ctx }) => { + const cmd = ctx.lane === 'offline' ? 'rm' : 'delete'; + const out = cli.json([cmd, `${ctx.scope}::${ctx.key('alpha-copy')}`, '--force']); + assert.equal(out.ok, true); + assert.equal(out.force, true); + // `show` exits 1 for a lesson that is not there — the deletion is the point. + assert.equal(cli.json(['show', `${ctx.scope}::${ctx.key('alpha-copy')}`], { code: 1 })[storeOf(ctx)].found, false); +}); + +smoke('diff', 'reports offline-vs-remote divergence only when both stores are readable', BOTH, async ({ cli, ctx }) => { + const out = cli.json(['diff']); + assert.equal(out.comparable, ctx.lane === 'remote', JSON.stringify(out).slice(0, 400)); +}); + +// ── archive lifecycle ──────────────────────────────────────────────────────── + +smoke('archive', 'hides a lesson from the default listing', BOTH, async ({ cli, ctx }) => { + cli.ok(['archive', `${ctx.scope}::${ctx.key('beta')}`]); + assert.ok(!keysIn(cli.json(['list'])[storeOf(ctx)]).includes(ctx.key('beta')), 'beta still listed after archive'); +}); + +smoke('restore', 'brings an archived lesson back', BOTH, async ({ cli, ctx }) => { + cli.ok(['restore', `${ctx.scope}::${ctx.key('beta')}`]); + assert.ok(keysIn(cli.json(['list'])[storeOf(ctx)]).includes(ctx.key('beta')), 'beta not listed after restore'); +}); + +// ── tools that read the store but change nothing ───────────────────────────── + +smoke('obligations', 'maps a changed file to the partners it obliges', BOTH, async ({ cli }) => { + const out = cli.json(['obligations', 'packages/schemas/src/shared/tool-catalog.ts']); + assert.ok(Array.isArray(out.matched) && out.matched.length > 0, 'tool-catalog.ts matched no obligation'); +}); + +smoke('invariants', 'scans for compile-pipeline candidates', BOTH, async ({ cli, ctx }) => { + const out = cli.json(['invariants', 'candidates']); + assert.equal(out[storeOf(ctx)].available, true); + assert.ok(Array.isArray(out[storeOf(ctx)].candidates)); +}); + +smoke('link', 'builds a dashboard deep link to one lesson (also as `url`)', OFFLINE, async ({ cli, ctx }) => { + for (const cmd of ['link', 'url']) { + const out = cli.json([cmd, `${ctx.scope}::${ctx.key('alpha')}`]); + assert.ok(out.url.startsWith('https://lorekit.io/lore?'), out.url); + assert.deepEqual(out.params.lesson, { scope: ctx.scope, key: ctx.key('alpha') }); + } +}); + +smoke('migrate', 'previews a move between stores without writing', BOTH, async ({ cli, ctx }) => { + const from = path.join(ctx.home, '.lorekit'); + const to = ctx.lane === 'remote' ? 'remote' : 'project'; + if (ctx.lane === 'remote') { + // Give the dry run something to report: one lesson in the local home tier. + cli.ok(['write', `${ctx.scope}::${ctx.key('local-only')}`, 'A lesson that only exists offline.', '--local']); + } + const out = cli.ok(['migrate', '--from', from, '--to', to]); + assert.match(out, /dry-run/); +}); + +smoke('bootstrap', 'explains that it only applies to a bring-your-own database', OFFLINE, async ({ cli }) => { + assert.match(cli.ok(['bootstrap', '--yes']), /No BYOD storage configured/); +}); + +smoke('completion', 'prints zsh and fish completion scripts', OFFLINE, async ({ cli }) => { + assert.match(cli.ok(['completion', 'zsh']), /^#compdef lorekit/); + assert.match(cli.ok(['completion', 'fish']), /complete -c lorekit/); +}); + +smoke('hook', 'injects the applicable lessons at SessionStart', BOTH, async ({ cli, ctx }) => { + const payload = JSON.stringify({ hook_event_name: 'SessionStart', cwd: ctx.dir, session_id: `smoke-${ctx.runId}` }); + const out = cli.ok(['hook', '--adapter', 'claude', '--event', 'SessionStart'], { input: payload }); + const hook = JSON.parse(out); + assert.equal(hook.hookSpecificOutput?.hookEventName, 'SessionStart'); + assert.ok(hook.hookSpecificOutput.additionalContext.includes(`zebra-${ctx.runId}`), 'alpha was not injected'); +}); + +// ── the local stdio MCP server ─────────────────────────────────────────────── + +smoke('mcp', 'serves the catalog over stdio and answers every memory tool', OFFLINE, async ({ ctx }) => { + const s = ctx.scope; + const gamma = ctx.key('gamma'); + const session = runMcp(ctx, [ + { name: 'memory.write', arguments: { scope: s, key: gamma, value: 'Written over the stdio MCP server.' } }, + { name: 'memory.read', arguments: { scope: s, key: gamma } }, + { name: 'memory.list', arguments: { scope: s } }, + // The local server searches the scopes it is given (mcp-server.mjs: list and + // search "need a scope (or a scope list) up front"). + { name: 'memory.search', arguments: { q: 'stdio MCP server', scopes: [s] } }, + { name: 'memory.scopes', arguments: {} }, + { name: 'memory.archive', arguments: { scope: s, key: gamma } }, + { name: 'memory.restore', arguments: { scope: s, key: gamma } }, + { name: 'memory.delete', arguments: { scope: s, key: gamma, force: true } }, + // org.* needs a remote; offline they must refuse and say how to fix it. + { name: 'org.create', arguments: { slug: 'smoke-org', name: 'Smoke' } }, + { name: 'org.list', arguments: {} }, + { name: 'org.rename', arguments: { slug: 'smoke-org', name: 'Renamed' } }, + { name: 'org.delete', arguments: { slug: 'smoke-org' } }, + ]); + assert.equal(session.code, 0, `lorekit mcp exited ${session.code}: ${session.stderr}`); + assert.equal(session.serverInfo?.name, 'lorekit-local'); + assert.deepEqual([...session.tools].sort(), [...LOCAL_MCP_TOOLS].sort(), 'advertised tools drifted from the catalog'); + + const [write, read, list, search, scopes, archive, restore, del, ...orgs] = session.results; + for (const [name, r] of Object.entries({ write, read, list, search, scopes, archive, restore, delete: del })) { + assert.equal(r.ok, true, `memory.${name} was refused: ${JSON.stringify(r.error)}`); + } + assert.ok(JSON.stringify(read.value).includes('Written over the stdio MCP server.'), 'memory.read lost the body'); + assert.ok(keysIn(list.value).includes(gamma), 'memory.list is missing gamma'); + assert.ok(keysIn(search.value).includes(gamma), 'memory.search did not find gamma'); + assert.ok(JSON.stringify(scopes.value).includes(s), 'memory.scopes is missing the run scope'); + for (const r of orgs) { + assert.equal(r.ok, false, 'an org.* tool succeeded with no remote'); + assert.match(String(r.error), /remote LoreKit endpoint/); + } +}); + +smoke('mcp', 'passes tools through to the endpoint over stdio', REMOTE, async ({ ctx }) => { + const session = runMcp(ctx, [{ name: 'memory.read', arguments: { scope: ctx.scope, key: ctx.key('alpha') } }]); + assert.equal(session.code, 0, `lorekit mcp exited ${session.code}: ${session.stderr}`); + assert.ok(session.tools.includes('memory.read'), `tools/list: ${session.tools}`); + assert.equal(session.results[0].ok, true, JSON.stringify(session.results[0].error)); + assert.ok(JSON.stringify(session.results[0].value).includes(`zebra-${ctx.runId}`), 'memory.read lost the body'); +}); + +// ── remote-only commands ───────────────────────────────────────────────────── + +// Offline, each must refuse with an actionable message rather than pretend to +// work. Against an endpoint, each must actually do its job. +const REMOTE_ONLY = [ + ['protect', (ctx) => ['protect', `${ctx.scope}::${ctx.key('alpha')}`]], + ['pin', (ctx) => ['pin', `${ctx.scope}::${ctx.key('alpha')}`]], + ['unpin', (ctx) => ['unpin', `${ctx.scope}::${ctx.key('alpha')}`]], + ['policy', () => ['policy', 'list']], + ['groom', (ctx) => ['groom', '--scope', ctx.scope]], + ['purge', () => ['purge', '--yes']], + ['purge-expired', () => ['purge-expired', '--yes']], +]; +for (const [command, args] of REMOTE_ONLY) { + smoke(command, 'refuses offline and names what is missing', OFFLINE, async ({ cli, ctx }) => { + cli.fails(args(ctx), /remote store is not configured/); + }); +} + +smoke('protect', 'protects and unprotects a lesson', REMOTE, async ({ cli, ctx }) => { + assert.equal(cli.json(['protect', `${ctx.scope}::${ctx.key('alpha')}`]).protected, true); + assert.equal(cli.json(['protect', `${ctx.scope}::${ctx.key('alpha')}`, '--off']).protected, false); +}); + +smoke('pin', 'is shorthand for protect', REMOTE, async ({ cli, ctx }) => { + assert.equal(cli.json(['pin', `${ctx.scope}::${ctx.key('alpha')}`]).protected, true); +}); + +smoke('policy', 'creates, lists, updates and deletes a retention policy', REMOTE, async ({ cli, ctx }) => { + const created = cli.json(['policy', 'create', '--scope', ctx.scope, '--name', `cli smoke ${ctx.runId}`, '--tags', 'groom-me']); + const id = created.policy?.id; + assert.ok(id, `policy create returned no id: ${JSON.stringify(created)}`); + ctx.policies.add(id); + + assert.ok(cli.json(['policy', 'list']).entries.some((p) => p.id === id), 'the new policy is not listed'); + const renamed = `cli smoke ${ctx.runId} renamed`; + assert.equal(cli.json(['policy', 'update', id, '--name', renamed]).policy?.name, renamed); + ctx.state.policyId = id; +}); + +smoke('groom', 'archives what a policy matches, but never a protected lesson', REMOTE, async ({ cli, ctx }) => { + writeLesson(cli, ctx, 'groom-target', 'A lesson the smoke policy should archive.', ['--tags', 'groom-me']); + writeLesson(cli, ctx, 'groom-keeper', 'A protected lesson the smoke policy must not touch.', ['--tags', 'groom-me']); + cli.json(['protect', `${ctx.scope}::${ctx.key('groom-keeper')}`]); + + const preview = cli.json(['groom', '--policy-id', ctx.state.policyId]); + assert.deepEqual(keysIn(preview.keys), [ctx.key('groom-target')], `preview: ${JSON.stringify(preview.keys)}`); + + const run = cli.json(['groom', '--policy-id', ctx.state.policyId, '--run', '--yes']); + assert.equal(run.archived, 1); + assert.equal(cli.json(['show', `${ctx.scope}::${ctx.key('groom-keeper')}`]).remote.found, true, 'groom archived a protected lesson'); +}); + +smoke('unpin', 'is shorthand for protect --off', REMOTE, async ({ cli, ctx }) => { + assert.equal(cli.json(['unpin', `${ctx.scope}::${ctx.key('groom-keeper')}`]).protected, false); + // Delete the policy here rather than leave it to cleanup, so `policy delete` is exercised too. + assert.equal(cli.json(['policy', 'delete', ctx.state.policyId, '--yes']).deleted, true); + ctx.policies.delete(ctx.state.policyId); +}); + +smoke('purge-expired', 'sweeps expired lessons for this account', REMOTE, async ({ cli, ctx }) => { + if (ctx.tier !== 'api_key') return 'skipped: account-wide sweep needs a user-scoped lk_* token'; + assert.equal(cli.json(['purge-expired', '--yes']).ok, true); +}); + +smoke('purge', 'purges archived lessons past the retention window', REMOTE, async ({ cli, ctx }) => { + if (ctx.tier !== 'api_key') return 'skipped: account-wide sweep needs a user-scoped lk_* token'; + // 365 days: only archived rows older than a year — nothing this run created. + assert.equal(cli.json(['purge', '--retention-days', '365', '--yes']).ok, true); +}); + +// ── teardown ───────────────────────────────────────────────────────────────── + +smoke('delete', 'archives without --force', BOTH, async ({ cli, ctx }) => { + const out = cli.json(['delete', `${ctx.scope}::${ctx.key('beta')}`]); + assert.equal(out.ok, true); + assert.equal(out.force, false); +}); + +smoke('uninstall', 'removes what install wrote', BOTH, async ({ cli, ctx }) => { + cli.ok(['uninstall', '--project', '--yes']); + assert.ok(!existsSync(path.join(ctx.dir, '.claude/skills/lorekit-memory')), 'the skill is still installed'); +}); diff --git a/packages/cli/test/smoke/harness.mjs b/packages/cli/test/smoke/harness.mjs new file mode 100644 index 000000000..e2bdd7c74 --- /dev/null +++ b/packages/cli/test/smoke/harness.mjs @@ -0,0 +1,267 @@ +// The CLI smoke harness: a throwaway sandbox per run, a way to run the REAL +// `bin/lorekit.mjs` in it, a stdio-MCP driver, and the remote cleanup ledger. +// +// Nothing here knows which commands exist — that is `cases.mjs`'s job. Not a +// test file (no `.test.mjs`), so `node --test test/*.test.mjs` never runs it on +// its own; `cli-smoke.test.mjs` and `run.mjs` import it. +// +// Zero dependencies, like the CLI it drives. + +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdirSync, mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; + +export const CLI_BIN = fileURLToPath(new URL('../../bin/lorekit.mjs', import.meta.url)); + +/** The production Supabase project (CLAUDE.md → Endpoints). The remote lane WRITES, so it refuses this. */ +export const PRODUCTION_REF = 'pqokxlhvnosogizsjztg'; + +// Operator configuration that must never leak into a smoke run: an ambient +// LOREKIT_TOKEN / LOREKIT_MCP_URL usually points at production, and a BYOD +// LOREKIT_STORAGE_* would retarget `bootstrap`. Telemetry knobs +// (LOREKIT_TELEMETRY_TOKEN, LOREKIT_CORRELATION_ID, DEPLOYMENT_ENVIRONMENT) are +// kept, so a deploy-time run stays observable the way the other smokes are. +const STRIPPED_ENV = /^(LOREKIT_(MCP_URL|TOKEN|MODE|HOME|STORE|APP_URL|STORAGE_.*)|CLAUDE_.*)$/; + +function baseEnv() { + return Object.fromEntries(Object.entries(process.env).filter(([k]) => !STRIPPED_ENV.test(k))); +} + +/** + * Which auth tier a credential belongs to, from the string alone — the same + * rule scripts/smoke/smoke-mcp-tools.mjs uses. `purge` / `purge-expired` are + * account-wide sweeps: scoped to ONE user on an `lk_*` token, but to nobody on a + * service-role key, where they would reach every account on a shared project. + */ +export function credentialTier(token) { + if (!token) return 'none'; + if (token.startsWith('lk_')) return 'api_key'; + try { + const claims = JSON.parse(Buffer.from(token.split('.')[1], 'base64url').toString('utf8')); + return claims.role === 'service_role' ? 'service' : 'user'; + } catch { + return 'service'; + } +} + +/** + * A fresh sandbox: its own HOME (so `~/.claude`, `~/.lorekit` and shell rc files + * are throwaway) and a git-initialised project directory whose name gives the + * run a unique project scope, `project::cli-smoke-`. + */ +export function createContext({ lane, endpoint = null, token = null }) { + const runId = `${Date.now().toString(36)}${Math.random().toString(36).slice(2, 6)}`; + const root = mkdtempSync(path.join(tmpdir(), 'lorekit-cli-smoke-')); + const home = path.join(root, 'home'); + const dir = path.join(root, `cli-smoke-${runId}`); + mkdirSync(home, { recursive: true }); + mkdirSync(dir, { recursive: true }); + spawnSync('git', ['init', '-q'], { cwd: dir }); + // Lesson keys look like `smoke--`: the shape + // scripts/smoke/smoke-cleanup.mjs recognises (SMOKE_ARTEFACT_PATTERN, matched + // on the key), so a remote run killed before its cleanup leaves nothing the + // sweeper cannot find. + const stamp = Date.now(); + return { + lane, + runId, + /** @param {string} name */ + key: (name) => `smoke-${stamp}-${name}`, + root, + home, + dir, + scope: `project::cli-smoke-${runId}`, + endpoint, + token, + tier: credentialTier(token), + /** Values a case hands to a later case (a policy id, say). */ + state: {}, + /** [scope, key] pairs written to the REMOTE store — hard-deleted in cleanup. */ + written: new Map(), + /** Retention-policy ids created on the remote — deleted in cleanup. */ + policies: new Set(), + }; +} + +export function disposeContext(ctx) { + rmSync(ctx.root, { recursive: true, force: true }); +} + +/** Run `lorekit ` in the sandbox. Never throws on a non-zero exit. */ +export function runCli(ctx, args, { input, env = {} } = {}) { + const remote = ctx.lane === 'remote'; + const res = spawnSync(process.execPath, [CLI_BIN, ...args], { + cwd: ctx.dir, + input, + encoding: 'utf8', + timeout: 90_000, + env: { + ...baseEnv(), + HOME: ctx.home, + USERPROFILE: ctx.home, + LOREKIT_HOME: path.join(ctx.home, '.lorekit'), + LOREKIT_MODE: remote ? 'remote' : 'local', + LOREKIT_MCP_URL: remote ? ctx.endpoint : '', + LOREKIT_TOKEN: remote ? ctx.token : '', + CI: '1', + NO_COLOR: '1', + ...env, + }, + }); + return { + args, + code: res.status, + stdout: res.stdout ?? '', + stderr: res.stderr ?? '', + output: `${res.stdout ?? ''}${res.stderr ?? ''}`, + error: res.error, + }; +} + +function describe(r) { + const tail = r.output.length > 1500 ? `…${r.output.slice(-1500)}` : r.output; + return `\`lorekit ${r.args.join(' ')}\` exited ${r.code}${r.error ? ` (${r.error.message})` : ''}\n${tail}`; +} + +/** + * The assertion helpers every case receives. Each one runs the CLI, checks the + * exit code, and says exactly which command did what when it does not hold. + */ +export function cliHelpers(ctx) { + const run = (args, opts) => runCli(ctx, args, opts); + return { + run, + /** Exit 0; returns stdout + stderr. */ + ok(args, opts) { + const r = run(args, opts); + assert.equal(r.code, 0, describe(r)); + return r.output; + }, + /** Exit `code` (default 0) with `--json` appended; returns the parsed stdout. */ + json(args, { code = 0, ...opts } = {}) { + const r = run([...args, '--json'], opts); + assert.equal(r.code, code, describe(r)); + try { + return JSON.parse(r.stdout); + } catch { + assert.fail(`stdout is not JSON — ${describe(r)}`); + } + }, + /** A non-zero exit whose output matches `pattern`: the command refused, and said why. */ + fails(args, pattern, opts) { + const r = run(args, opts); + assert.notEqual(r.code, 0, `expected a non-zero exit — ${describe(r)}`); + assert.match(r.output, pattern, describe(r)); + return r.output; + }, + }; +} + +/** + * Drive `lorekit mcp` (the local stdio MCP server) through one session: + * initialize, tools/list, then each call in order. Returns the advertised tool + * names and each call's unwrapped result — `{ ok, value }` mirroring how + * smoke-mcp-tools.mjs reads a `tools/call` answer (an `isError` result and a + * JSON-RPC error both count as refused). + */ +export function runMcp(ctx, calls) { + const frames = [ + { jsonrpc: '2.0', id: 1, method: 'initialize', params: {} }, + { jsonrpc: '2.0', id: 2, method: 'tools/list' }, + ...calls.map((c, i) => ({ + jsonrpc: '2.0', + id: i + 3, + method: 'tools/call', + params: { name: c.name, arguments: c.arguments ?? {} }, + })), + ]; + const r = runCli(ctx, ['mcp'], { input: `${frames.map((f) => JSON.stringify(f)).join('\n')}\n` }); + const byId = new Map(); + for (const line of r.stdout.split('\n')) { + if (!line.trim()) continue; + try { + const msg = JSON.parse(line); + if (msg.id !== undefined) byId.set(msg.id, msg); + } catch { + // stdout must carry ONLY JSON-RPC frames; anything else breaks every client. + assert.fail(`\`lorekit mcp\` wrote a non-JSON line to stdout: ${line.slice(0, 200)}`); + } + } + const unwrap = (msg) => { + if (!msg) return { ok: false, value: null, error: 'no response' }; + if (msg.error) return { ok: false, value: null, error: msg.error.message }; + const text = msg.result?.content?.[0]?.text; + let value = text; + try { + value = JSON.parse(text); + } catch { + /* non-JSON text is returned verbatim */ + } + const refused = msg.result?.isError === true || (value && typeof value === 'object' && value.ok === false); + return { ok: !refused, value, error: refused ? (value?.error ?? text) : null }; + }; + return { + code: r.code, + stderr: r.stderr, + serverInfo: byId.get(1)?.result?.serverInfo ?? null, + tools: (byId.get(2)?.result?.tools ?? []).map((t) => t.name), + results: calls.map((_, i) => unwrap(byId.get(i + 3))), + }; +} + +// ── remote cleanup ─────────────────────────────────────────────────────────── + +async function rpc(ctx, name, args) { + const res = await fetch(ctx.endpoint, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json, text/event-stream', + Authorization: `Bearer ${ctx.token}`, + }, + body: JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'tools/call', params: { name, arguments: args } }), + signal: AbortSignal.timeout(20_000), + }); + const json = await res.json().catch(() => null); + const text = json?.result?.content?.[0]?.text; + try { + return { ok: !json?.error && !json?.result?.isError, value: JSON.parse(text) }; + } catch { + return { ok: false, value: text ?? json }; + } +} + +/** Record a remote write so cleanup can hard-delete it whatever the assertions did. */ +export function trackRemoteWrite(ctx, scope, key) { + if (ctx.lane === 'remote') ctx.written.set(JSON.stringify([scope, key]), { scope, key }); +} + +/** + * Hard-delete everything the remote lane wrote, and delete its policies. + * Returns what could not be removed, so the summary can name it. + */ +export async function cleanupRemote(ctx) { + const residue = []; + if (ctx.lane !== 'remote') return residue; + for (const { scope, key } of ctx.written.values()) { + try { + const r = await rpc(ctx, 'memory.delete', { scope, key, force: true }); + if (!r.ok) residue.push(`memory ${scope}::${key} (${JSON.stringify(r.value).slice(0, 120)})`); + } catch (err) { + residue.push(`memory ${scope}::${key} (${err.name})`); + } + } + for (const id of ctx.policies) { + try { + const r = await rpc(ctx, 'policy.delete', { id }); + if (!r.ok) residue.push(`policy ${id}`); + } catch (err) { + residue.push(`policy ${id} (${err.name})`); + } + } + return residue; +} diff --git a/packages/cli/test/smoke/run.mjs b/packages/cli/test/smoke/run.mjs new file mode 100644 index 000000000..5a9667d6d --- /dev/null +++ b/packages/cli/test/smoke/run.mjs @@ -0,0 +1,97 @@ +#!/usr/bin/env node +// Run the CLI smoke cases (cases.mjs) against the real `lorekit` binary. +// +// node packages/cli/test/smoke/run.mjs # offline lane +// node packages/cli/test/smoke/run.mjs --remote +// node packages/cli/test/smoke/run.mjs --remote "$URL" "$TOKEN" --only groom +// +// The offline lane needs nothing: a throwaway HOME and project, no network. It +// also runs on every PR inside `cli:test` (test/cli-smoke.test.mjs). +// +// The remote lane WRITES: every lesson goes under the run's own project scope +// (`project::cli-smoke-`) and is hard-deleted at the end, together with +// the retention policy it creates; anything it could not remove is named in the +// summary. It refuses the production project unless --allow-production is +// passed, matching the repo's rule that write-heavy smokes run against preview +// only (.github/workflows/deploy.yml). +// +// Exit code: 0 when every case passed or was skipped, 1 otherwise. + +import process from 'node:process'; +import { pathToFileURL } from 'node:url'; + +import { CASES } from './cases.mjs'; +import { PRODUCTION_REF, cleanupRemote, cliHelpers, createContext, disposeContext } from './harness.mjs'; + +/** + * Run every case for one lane, in order. Never throws: returns the per-case + * outcomes plus anything the remote cleanup could not remove. + */ +export async function runSmoke({ lane = 'offline', endpoint = null, token = null, only = null, log = () => {} } = {}) { + const ctx = createContext({ lane, endpoint, token }); + const cli = cliHelpers(ctx); + const results = []; + try { + for (const c of CASES) { + if (!c.lanes.includes(lane)) continue; + const label = `${c.command} — ${c.name}`; + if (only && !label.includes(only)) continue; + try { + const skipped = await c.fn({ cli, ctx }); + if (typeof skipped === 'string') { + results.push({ label, command: c.command, outcome: 'skip', detail: skipped }); + log(` – ${label} (${skipped})`); + } else { + results.push({ label, command: c.command, outcome: 'pass' }); + log(` ✓ ${label}`); + } + } catch (err) { + results.push({ label, command: c.command, outcome: 'fail', detail: err.message }); + log(` ✗ ${label}\n${String(err.message).replace(/^/gm, ' ')}`); + } + } + } finally { + var residue = await cleanupRemote(ctx).catch((err) => [`cleanup crashed: ${err.message}`]); + disposeContext(ctx); + } + return { ctx: { lane, runId: ctx.runId, scope: ctx.scope }, results, residue }; +} + +async function main(argv) { + const flags = new Set(argv.filter((a) => a.startsWith('--'))); + const positional = argv.filter((a) => !a.startsWith('--')); + const onlyIdx = argv.indexOf('--only'); + const only = onlyIdx >= 0 ? argv[onlyIdx + 1] : null; + const remote = flags.has('--remote'); + const endpoint = remote ? (positional.find((p) => p !== only) ?? process.env.LOREKIT_MCP_URL) : null; + const token = remote ? (positional.filter((p) => p !== only)[1] ?? process.env.LOREKIT_TOKEN) : null; + + const unknown = [...flags].filter((f) => !['--remote', '--allow-production', '--only'].includes(f)); + if (unknown.length) { + console.error(`error: unknown option(s): ${unknown.join(', ')}`); + return 2; + } + if (remote && (!endpoint || !token)) { + console.error('usage: node packages/cli/test/smoke/run.mjs --remote [--only ] [--allow-production]'); + return 2; + } + if (remote && endpoint.includes(PRODUCTION_REF) && !flags.has('--allow-production')) { + console.error(`refusing to run against production (${PRODUCTION_REF}): the remote lane writes. Use the preview project.`); + return 2; + } + + const lane = remote ? 'remote' : 'offline'; + console.log(`CLI smoke — ${lane} lane${remote ? ` against ${new URL(endpoint).host}` : ''}`); + const { ctx, results, residue } = await runSmoke({ lane, endpoint, token, only, log: (l) => console.log(l) }); + + const count = (o) => results.filter((r) => r.outcome === o).length; + console.log(`\n${count('pass')} passed, ${count('fail')} failed, ${count('skip')} skipped — scope ${ctx.scope}`); + if (residue.length) { + console.log(`left behind (remove by hand):\n ${residue.join('\n ')}`); + } + return count('fail') > 0 ? 1 : 0; +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + main(process.argv.slice(2)).then((code) => process.exit(code)); +} diff --git a/packages/web-e2e/.gitignore b/packages/web-e2e/.gitignore new file mode 100644 index 000000000..9d181cf2b --- /dev/null +++ b/packages/web-e2e/.gitignore @@ -0,0 +1,5 @@ +# Saved sign-in session (contains auth cookies) — never commit. +.auth/ +playwright-report/ +test-results/ +blob-report/ diff --git a/packages/web-e2e/package.json b/packages/web-e2e/package.json new file mode 100644 index 000000000..02aa386ad --- /dev/null +++ b/packages/web-e2e/package.json @@ -0,0 +1,17 @@ +{ + "name": "@lorekit/web-e2e", + "version": "0.0.1", + "private": true, + "description": "Playwright end-to-end smoke tests for the LoreKit website (@lorekit/web). Tests are Markdown specs in specs/*.md, written in the aw-tester / ui-verify WHEN/THEN grammar; a vitest guard fails when a page has no spec.", + "scripts": { + "e2e": "playwright test", + "e2e:public": "playwright test --project=public", + "e2e:ui": "playwright test --ui", + "report": "playwright show-report", + "spec:import": "node scripts/ui-verify.cjs import", + "spec:export": "node scripts/ui-verify.cjs export" + }, + "devDependencies": { + "@playwright/test": "1.56.0" + } +} diff --git a/packages/web-e2e/playwright.config.ts b/packages/web-e2e/playwright.config.ts new file mode 100644 index 000000000..f71f8c242 --- /dev/null +++ b/packages/web-e2e/playwright.config.ts @@ -0,0 +1,93 @@ +import { defineConfig, devices } from '@playwright/test'; +import { STORAGE_STATE } from './src/support/auth'; +import { BASE_URL, E2E_PORT, STARTS_LOCAL_SERVER, TARGET, VERCEL_BYPASS_SECRET } from './src/support/env'; + +const CI = Boolean(process.env.CI); + +/** + * Projects, in the order they run: + * + * setup signs in once through the login form and saves the session + * (skipped, with the reason, when no test user is configured). + * public everything a signed-out visitor sees. Needs no credentials and + * no backend, so it runs on every PR against a local build. + * signed-in tests tagged @signed-in, reusing the saved session. + * sign-out tagged @sign-out; runs last because signing out revokes every + * session of the account, including the one `signed-in` uses. + * + * `PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH` points Chromium at a browser you + * already have (a sandbox that cannot reach Playwright's download host, for + * example). Unset — the normal case, CI included — Playwright uses its own. + */ +const executablePath = process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH || undefined; +const desktop = { + ...devices['Desktop Chrome'], + ...(executablePath ? { launchOptions: { executablePath } } : {}), +}; + +export default defineConfig({ + testDir: './src', + // `*.e2e.ts` (and the setup project's `*.setup.ts`) are Playwright's; + // `*.test.ts` is vitest's (the route coverage guard), so the two runners + // never pick up each other's files. + testMatch: /.*\.e2e\.ts$/, + outputDir: './test-results', + fullyParallel: true, + forbidOnly: CI, + retries: CI ? 2 : 0, + workers: CI ? 2 : undefined, + timeout: 45_000, + expect: { timeout: 15_000 }, + reporter: CI + ? [['github'], ['list'], ['html', { open: 'never', outputFolder: 'playwright-report' }]] + : [['list'], ['html', { open: 'never', outputFolder: 'playwright-report' }]], + + use: { + baseURL: BASE_URL, + // A trace stores request headers and cookies verbatim, and CI uploads it from a + // public repo — so never record one where a bypass secret or a production + // session is in play. The PR job (local stack, throwaway user) keeps traces. + trace: VERCEL_BYPASS_SECRET || TARGET === 'production' ? 'off' : 'retain-on-failure', + screenshot: 'only-on-failure', + // Production builds register /sw.js, which serves pages cache-first; a + // smoke test must see what the server returns now. + serviceWorkers: 'block', + }, + + projects: [ + { name: 'setup', testMatch: /.*\.setup\.ts$/, use: desktop }, + { name: 'public', grepInvert: /@signed-in|@sign-out/, use: desktop }, + { + name: 'signed-in', + grep: /@signed-in/, + dependencies: ['setup'], + use: { ...desktop, storageState: STORAGE_STATE }, + }, + { + name: 'sign-out', + grep: /@sign-out/, + dependencies: ['signed-in'], + use: { ...desktop, storageState: STORAGE_STATE }, + }, + ], + + // No E2E_BASE_URL → serve a production build of packages/web locally. Build + // it first with `pnpm nx run web-e2e:build-web` (NEXT_PUBLIC_* values are + // inlined at build time, so the build — not this server — decides which + // Supabase the site talks to). E2E_WEB_COMMAND swaps the command, e.g. for + // `next dev` while iterating. + webServer: STARTS_LOCAL_SERVER + ? { + command: process.env.E2E_WEB_COMMAND || `pnpm --filter @lorekit/web exec next start --port ${E2E_PORT}`, + url: BASE_URL, + reuseExistingServer: !CI, + timeout: 180_000, + stdout: 'pipe', + stderr: 'pipe', + env: { + NEXT_PUBLIC_USE_LOCAL_SUPABASE: 'true', + NEXT_TELEMETRY_DISABLED: '1', + }, + } + : undefined, +}); diff --git a/packages/web-e2e/project.json b/packages/web-e2e/project.json new file mode 100644 index 000000000..be1dd8899 --- /dev/null +++ b/packages/web-e2e/project.json @@ -0,0 +1,41 @@ +{ + "name": "web-e2e", + "$schema": "../../node_modules/nx/schemas/project-schema.json", + "sourceRoot": "packages/web-e2e/src", + "projectType": "application", + "tags": ["type:e2e", "platform:web"], + "implicitDependencies": ["web"], + "targets": { + "test": { + "dependsOn": [] + }, + "typecheck": { + "executor": "nx:run-commands", + "cache": true, + "options": { + "command": "tsc --noEmit -p packages/web-e2e/tsconfig.json", + "cwd": "{workspaceRoot}" + } + }, + "build-web": { + "executor": "nx:run-commands", + "cache": false, + "options": { + "cwd": "packages/web", + "command": "next build", + "env": { + "NEXT_PUBLIC_USE_LOCAL_SUPABASE": "true", + "NEXT_TELEMETRY_DISABLED": "1" + } + } + }, + "e2e": { + "executor": "nx:run-commands", + "cache": false, + "options": { + "cwd": "packages/web-e2e", + "command": "playwright test" + } + } + } +} diff --git a/packages/web-e2e/scripts/ui-verify.cjs b/packages/web-e2e/scripts/ui-verify.cjs new file mode 100644 index 000000000..ed2115d42 --- /dev/null +++ b/packages/web-e2e/scripts/ui-verify.cjs @@ -0,0 +1,123 @@ +#!/usr/bin/env node +'use strict'; +// Move e2e specs between this suite and /ui-verify. Usage: specs/README.md. +// +// node scripts/ui-verify.cjs import --pr [--out ] [--force] +// node scripts/ui-verify.cjs import --file [--out ] [--force] +// node scripts/ui-verify.cjs import --routes [--out ] [--force] +// node scripts/ui-verify.cjs export [ …] +// +// Every import is validated with the suite's own parser before it is reported +// ready; a draft with TODO lines is written, named as a draft, and fails +// `nx test web-e2e` until it is finished — it can never pass by accident. + +const { execFileSync } = require('node:child_process'); +const { existsSync, readFileSync, readdirSync, writeFileSync } = require('node:fs'); +const path = require('node:path'); +const { + SpecError, + extractUiVerifyBlock, + intentToDraft, + parseSpecFile, + routesToSpecs, + specsToUiVerifyBlock, +} = require('../src/spec/grammar.cjs'); + +const PKG = path.join(__dirname, '..'); + +function usage(msg) { + if (msg) console.error(`error: ${msg}\n`); + console.error( + [ + 'usage:', + ' ui-verify.cjs import --pr [--out ] [--force]', + ' ui-verify.cjs import --file [--out ] [--force]', + ' ui-verify.cjs import --routes [--out ] [--force]', + ' ui-verify.cjs export [ …]', + ].join('\n'), + ); + process.exit(2); +} + +function flag(args, name) { + const at = args.indexOf(name); + return at === -1 ? null : args[at + 1] ?? usage(`${name} needs a value`); +} + +/** A spec file body from ui-verify block content (or a whole spec file). */ +function fromBlockOrFile(text, origin) { + const block = extractUiVerifyBlock(text); + const content = block ? block.content : text; + const isIntent = block ? block.version === 'v2' : /^Format:\s*intent\s*$/m.test(content); + if (isIntent) { + const { text: draft, todos } = intentToDraft(content); + return { body: draft, note: `intent spec → draft with ${todos} TODO line(s) to finish` }; + } + // Grammar: keep everything from the first spec heading, drop the header. + const at = content.search(/^## Spec /m); + if (at === -1) throw new Error(`${origin}: no \`## Spec\` block found`); + return { body: content.slice(at).trim() + '\n', note: 'grammar spec → imported verbatim' }; +} + +function runImport(args) { + const pr = flag(args, '--pr'); + const file = flag(args, '--file'); + const routes = flag(args, '--routes'); + if ([pr, file, routes].filter(Boolean).length !== 1) usage('import takes exactly one of --pr, --file, --routes'); + + let body; + let notes = []; + let defaultName; + if (pr) { + const number = /(\d+)\/?$/.exec(pr)?.[1] ?? usage(`not a PR number or URL: ${pr}`); + const prBody = execFileSync('gh', ['pr', 'view', pr, '--json', 'body', '-q', '.body'], { encoding: 'utf8' }); + if (!extractUiVerifyBlock(prBody)) throw new Error(`PR ${pr} has no ui-verify block`); + ({ body, note: notes[0] } = fromBlockOrFile(prBody, `PR ${pr}`)); + defaultName = `pr-${number}.md`; + } else if (file) { + ({ body, note: notes[0] } = fromBlockOrFile(readFileSync(file, 'utf8'), file)); + defaultName = `${path.basename(file, path.extname(file))}.md`; + } else { + const files = readdirSync(routes) + .filter((f) => f.endsWith('.md')) + .sort() + .map((name) => ({ name, text: readFileSync(path.join(routes, name), 'utf8') })); + if (files.length === 0) throw new Error(`${routes}: no route files (*.md)`); + const out = routesToSpecs(files); + body = out.text; + notes = [`${files.length} compiled route(s) → imported`, ...out.warnings]; + defaultName = 'ui-verify-routes.md'; + } + + const out = path.resolve(flag(args, '--out') ?? path.join(PKG, 'specs', defaultName)); + if (existsSync(out) && !args.includes('--force')) throw new Error(`${out} exists — pass --force to overwrite`); + const text = `# Specs: imported from ui-verify\n\nTarget: local\n\n${body}`; + writeFileSync(out, text); + + for (const n of notes) console.log(`• ${n}`); + try { + const specs = parseSpecFile(text, path.relative(PKG, out)); + console.log(`✓ ${specs.length} spec(s) ready → ${path.relative(process.cwd(), out)}`); + } catch (err) { + if (!(err instanceof SpecError)) throw err; + console.log(`! draft written → ${path.relative(process.cwd(), out)}`); + console.log(` finish it before it can run: ${err.message}`); + } +} + +function runExport(args) { + const files = args.filter((a) => !a.startsWith('--')); + if (files.length === 0) usage('export needs at least one spec file'); + const specs = files.flatMap((f) => parseSpecFile(readFileSync(f, 'utf8'), f)); + process.stdout.write(`${specsToUiVerifyBlock(specs)}\n`); +} + +const [command, ...rest] = process.argv.slice(2); +try { + if (command === 'import') runImport(rest); + else if (command === 'export') runExport(rest); + else usage(command ? `unknown command "${command}"` : undefined); +} catch (err) { + console.error(`error: ${err.message}`); + process.exit(1); +} diff --git a/packages/web-e2e/specs/README.md b/packages/web-e2e/specs/README.md new file mode 100644 index 000000000..442f3ae4a --- /dev/null +++ b/packages/web-e2e/specs/README.md @@ -0,0 +1,110 @@ +# Website e2e specs + +Every `## Spec` block in this folder is a Playwright test. Write Markdown; no +TypeScript needed. The format is aw-tester's `WHEN`/`THEN` grammar, the same +one a `/ui-verify` PR block uses (``), so a spec moves +between a PR and this suite unchanged. + +## Add a test + +Append a block to any `*.md` file here (or add a new file): + +```markdown +## Spec 29: Settings › Billing renders +url: /settings/billing +preconditions: + - User is signed in (handled by the e2e auth setup) +flow: + - WHEN page loads + THEN {role: "heading", name: "Billing"} is visible +``` + +Then run it: + +```bash +cd packages/web-e2e +npx playwright test -g "Billing" # against a local build (see docs/smoke-tests.md) +pnpm nx test web-e2e # parse check + page coverage, no browser +``` + +A new page under `packages/web/src/app` with no spec fails `nx test web-e2e` +(and so the PR) until a spec visits it. + +## The grammar + +```text +## Spec : +url: +persist: critical-path (optional, ignored here) +preconditions: (optional) + - +flow: + - WHEN + THEN + AND + - CAPTURE "