Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 8 additions & 35 deletions .github/workflows/deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,11 +51,9 @@
# once with `node scripts/smoke/seed-smoke-user.mjs` (see docs/deployment.md → "Seed
# the orgs-smoke user"); the mint signs in as it, it is never auto-created.
# Optional repo-level secrets for the website e2e suite (packages/web-e2e,
# docs/smoke-tests.md): VERCEL_AUTOMATION_BYPASS_SECRET lets e2e-web-staged reach
# the staged build through Vercel Deployment Protection (unset → that job skips,
# announced, and promotion proceeds as before); LOREKIT_TESTING_USER_NAME +
# LOREKIT_TESTING_USER_PW sign in a production test account for the signed-in
# pages (unset → those tests skip, announced; public pages still run).
# docs/smoke-tests.md): LOREKIT_TESTING_USER_NAME + LOREKIT_TESTING_USER_PW sign
# in a production test account for the signed-in pages (unset → those tests
# skip, announced; public pages still run).
# Optional repo-level secret: DISCORD_WEBHOOK_URL — when set, notify-failure posts
# a message there on any deploy-pipeline failure. Unset → notify-failure no-ops.
# Add a required reviewer on the `production` environment for a manual gate.
Expand Down Expand Up @@ -406,10 +404,8 @@ jobs:
# ── 1d. Website e2e against the STAGED production build ───────────────────────
# Playwright (packages/web-e2e) against the exact bundle promote-web-production
# is about to put on the production domain — so a broken page stops the flip
# instead of being rolled back after users saw it. The staged URL is behind
# Vercel Deployment Protection; VERCEL_AUTOMATION_BYPASS_SECRET gets through it
# and, while it is unset, this job skips (announced) and promotion proceeds
# exactly as before.
# instead of being rolled back after users saw it. The staged URL is public
# (Vercel Deployment Protection is off for this project), so no bypass secret.
#
# Signed-in pages run here only when this merge leaves the API alone. When the
# API changed too, it reaches production only AFTER this job (deploy-production),
Expand All @@ -433,50 +429,30 @@ jobs:
E2E_TARGET: production
API_CHANGED: ${{ needs.changes.outputs.api }}
steps:
- name: Skip without the Vercel bypass secret
id: gate
env:
HAS_BYPASS: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET != '' }}
run: |
if [ "$HAS_BYPASS" != "true" ]; then
echo "::warning::VERCEL_AUTOMATION_BYPASS_SECRET is not set — the staged build is behind Vercel Deployment Protection, so the website e2e gate is SKIPPED. Add the secret (Vercel ▸ Project ▸ Settings ▸ Deployment Protection ▸ Protection Bypass for Automation) to turn it on."
echo "run=false" >> "$GITHUB_OUTPUT"
else
echo "run=true" >> "$GITHUB_OUTPUT"
fi

- name: Checkout repository
if: steps.gate.outputs.run == 'true'
uses: actions/[email protected]

- name: Set up pnpm
if: steps.gate.outputs.run == 'true'
uses: pnpm/action-setup@v6

- name: Set up Node.js
if: steps.gate.outputs.run == 'true'
uses: actions/[email protected]
with:
node-version: 20
cache: pnpm

- name: Install dependencies
if: steps.gate.outputs.run == 'true'
run: pnpm install --frozen-lockfile

- name: Install Playwright Chromium (+ OS deps)
if: steps.gate.outputs.run == 'true'
run: pnpm --filter @lorekit/web-e2e exec playwright install --with-deps chromium

- name: Playwright — public pages on the staged build
if: steps.gate.outputs.run == 'true'
working-directory: packages/web-e2e
env:
VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }}
run: npx playwright test --project=public

- name: Note — signed-in pages run after promotion this time
if: steps.gate.outputs.run == 'true' && env.API_CHANGED == 'true'
if: env.API_CHANGED == 'true'
run: echo "::notice::This merge changes the API too — signed-in pages run in smoke-production, once both halves are live."

# Sign in as its own step, allowed to fail: an expired test-account password
Expand All @@ -485,11 +461,10 @@ jobs:
# sign-in do block it (next step).
- name: Sign in the e2e test account on the staged build
id: login
if: steps.gate.outputs.run == 'true' && env.API_CHANGED != 'true'
if: env.API_CHANGED != 'true'
continue-on-error: true
working-directory: packages/web-e2e
env:
VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }}
E2E_USER_EMAIL: ${{ secrets.LOREKIT_TESTING_USER_NAME }}
E2E_USER_PASSWORD: ${{ secrets.LOREKIT_TESTING_USER_PW }}
run: npx playwright test --project=setup
Expand All @@ -498,7 +473,6 @@ jobs:
if: steps.login.outcome == 'success'
working-directory: packages/web-e2e
env:
VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }}
E2E_USER_EMAIL: ${{ secrets.LOREKIT_TESTING_USER_NAME }}
E2E_USER_PASSWORD: ${{ secrets.LOREKIT_TESTING_USER_PW }}
run: npx playwright test --project=signed-in --no-deps
Expand Down Expand Up @@ -858,8 +832,7 @@ jobs:
runs-on: ubuntu-latest
needs: [changes, deploy-production, stage-web-production, e2e-web-staged]
# Flip the web when it changed (stage-web-production ran), its staged build
# passed the website e2e (e2e-web-staged — green when it skips for a missing
# bypass secret), AND either the API
# passed the website e2e (e2e-web-staged), AND either the API
# did not change (nothing to wait for) or the API's prod deploy succeeded (so
# the API is live before the FE fronts it). `changes.outputs.api` disambiguates
# deploy-production's 'skipped' (API unchanged, fine) from a real upstream stop
Expand Down
3 changes: 1 addition & 2 deletions docs/smoke-tests.md
Original file line number Diff line number Diff line change
Expand Up @@ -122,11 +122,10 @@ the top of `packages/web-e2e/src/support/env.ts`.

| Secret | Used by | Without it |
|--------|---------|------------|
| `VERCEL_AUTOMATION_BYPASS_SECRET` (repo) | `e2e-web-staged` — gets through Vercel Deployment Protection on the staged URL | the job skips with a warning and promotion proceeds as before |
| `LOREKIT_TESTING_USER_NAME`, `LOREKIT_TESTING_USER_PW` (repo) | signed-in pages on the staged build and on `lorekit.io` | signed-in tests skip, announced; public tests still run |
| `LOREKIT_SMOKE_TOKEN` (preview env, existing) | CLI remote lane, hosted MCP smoke | — (already required) |

Traces are off on any run that uses the bypass secret or targets production: a
Traces are off on any run that targets production (the staged build included): a
trace records request headers and cookies, and this public repo's artifacts are
downloadable by anyone. Failure screenshots still upload.

Expand Down
10 changes: 10 additions & 0 deletions packages/cli/test/smoke/cases.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -344,16 +344,24 @@ for (const [command, args] of REMOTE_ONLY) {
});
}

// protect / pin / unpin / policy / groom are owner-keyed: the server refuses a
// service-role credential (it has no user to own the row), as the hosted MCP
// smoke does for the same tools. Preview's smoke token is service-role.
const OWNERLESS = 'skipped: owner-keyed — a service-role credential has no owner';

smoke('protect', 'protects and unprotects a lesson', REMOTE, async ({ cli, ctx }) => {
if (ctx.tier === 'service') return OWNERLESS;
assert.equal(cli.json(['protect', `${ctx.scope}::${ctx.key('alpha')}`]).protected, true);
assert.equal(cli.json(['protect', `${ctx.scope}::${ctx.key('alpha')}`, '--off']).protected, false);
});

smoke('pin', 'is shorthand for protect', REMOTE, async ({ cli, ctx }) => {
if (ctx.tier === 'service') return OWNERLESS;
assert.equal(cli.json(['pin', `${ctx.scope}::${ctx.key('alpha')}`]).protected, true);
});

smoke('policy', 'creates, lists, updates and deletes a retention policy', REMOTE, async ({ cli, ctx }) => {
if (ctx.tier === 'service') return OWNERLESS;
const created = cli.json(['policy', 'create', '--scope', ctx.scope, '--name', `cli smoke ${ctx.runId}`, '--tags', 'groom-me']);
const id = created.policy?.id;
assert.ok(id, `policy create returned no id: ${JSON.stringify(created)}`);
Expand All @@ -366,6 +374,7 @@ smoke('policy', 'creates, lists, updates and deletes a retention policy', REMOTE
});

smoke('groom', 'archives what a policy matches, but never a protected lesson', REMOTE, async ({ cli, ctx }) => {
if (ctx.tier === 'service') return OWNERLESS;
writeLesson(cli, ctx, 'groom-target', 'A lesson the smoke policy should archive.', ['--tags', 'groom-me']);
writeLesson(cli, ctx, 'groom-keeper', 'A protected lesson the smoke policy must not touch.', ['--tags', 'groom-me']);
cli.json(['protect', `${ctx.scope}::${ctx.key('groom-keeper')}`]);
Expand All @@ -379,6 +388,7 @@ smoke('groom', 'archives what a policy matches, but never a protected lesson', R
});

smoke('unpin', 'is shorthand for protect --off', REMOTE, async ({ cli, ctx }) => {
if (ctx.tier === 'service') return OWNERLESS;
assert.equal(cli.json(['unpin', `${ctx.scope}::${ctx.key('groom-keeper')}`]).protected, false);
// Delete the policy here rather than leave it to cleanup, so `policy delete` is exercised too.
assert.equal(cli.json(['policy', 'delete', ctx.state.policyId, '--yes']).deleted, true);
Expand Down
8 changes: 4 additions & 4 deletions packages/web-e2e/playwright.config.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { defineConfig, devices } from '@playwright/test';
import { STORAGE_STATE } from './src/support/auth';
import { BASE_URL, E2E_PORT, STARTS_LOCAL_SERVER, TARGET, VERCEL_BYPASS_SECRET } from './src/support/env';
import { BASE_URL, E2E_PORT, STARTS_LOCAL_SERVER, TARGET } from './src/support/env';

const CI = Boolean(process.env.CI);

Expand Down Expand Up @@ -45,9 +45,9 @@ export default defineConfig({
use: {
baseURL: BASE_URL,
// A trace stores request headers and cookies verbatim, and CI uploads it from a
// public repo — so never record one where a bypass secret or a production
// session is in play. The PR job (local stack, throwaway user) keeps traces.
trace: VERCEL_BYPASS_SECRET || TARGET === 'production' ? 'off' : 'retain-on-failure',
// public repo — so never record one where a production session is in play.
// The PR job (local stack, throwaway user) keeps traces.
trace: TARGET === 'production' ? 'off' : 'retain-on-failure',
screenshot: 'only-on-failure',
// Production builds register /sw.js, which serves pages cache-first; a
// smoke test must see what the server returns now.
Expand Down
6 changes: 3 additions & 3 deletions packages/web-e2e/src/journeys/api-docs.e2e.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { expect, originHeaders, test } from '../support/fixtures';
import { expect, test } from '../support/fixtures';

/**
* /api-docs is a route handler, not a page: it returns a Scalar HTML shell that
Expand All @@ -7,14 +7,14 @@ import { expect, originHeaders, test } from '../support/fixtures';
*/

test('the API reference shell is served', async ({ request }) => {
const res = await request.get('/api-docs', { headers: originHeaders() });
const res = await request.get('/api-docs');
expect(res.status()).toBe(200);
expect(res.headers()['content-type']).toContain('text/html');
expect(await res.text()).toContain('<title>LoreKit REST API</title>');
});

test('the OpenAPI spec is served as JSON', async ({ request }) => {
const res = await request.get('/api-docs/spec', { headers: originHeaders() });
const res = await request.get('/api-docs/spec');
expect(res.status()).toBe(200);
const spec = await res.json();
expect(spec.info?.title).toBe('LoreKit REST API');
Expand Down
8 changes: 0 additions & 8 deletions packages/web-e2e/src/support/env.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,6 @@
* STAGED production build on a *.vercel.app URL is
* `production`.
*
* ## Getting through Vercel Deployment Protection
*
* VERCEL_AUTOMATION_BYPASS_SECRET sent as `x-vercel-protection-bypass` on
* requests to the site's own origin only (never to Supabase,
* where the extra header would fail the CORS preflight).
*
* ## Signing in (the `signed-in` project)
*
* The first source that is set wins:
Expand Down Expand Up @@ -66,8 +60,6 @@ export const TARGET: TargetKind = (() => {
return inferTarget(BASE_URL);
})();

export const VERCEL_BYPASS_SECRET = env('VERCEL_AUTOMATION_BYPASS_SECRET');

export interface FixedUser {
kind: 'fixed';
email: string;
Expand Down
42 changes: 9 additions & 33 deletions packages/web-e2e/src/support/fixtures.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { test as base, expect } from '@playwright/test';
import { BASE_URL, VERCEL_BYPASS_SECRET } from './env';
import { BASE_URL } from './env';

const ORIGIN = new URL(BASE_URL).origin;

Expand All @@ -24,37 +24,18 @@ export interface PageHealthOptions {
}

/**
* The suite's `test`: Playwright's, plus two things every spec gets for free.
*
* • A health check. Each test records uncaught page exceptions, console
* errors and same-origin 5xx responses, and fails when any were seen — so
* a hydration error or a crashing component fails the smoke even when the
* heading still rendered. A spec that asserts its own error status
* (`network: GET /x returned 404`) tolerates that status's console line;
* anything else known and harmless goes in GLOBAL_ALLOWED_PROBLEMS with a
* reason, or `test.use({ allowedConsoleErrors: allowing([...]) })` in a TS journey.
* • The Vercel bypass header on requests to the site's OWN origin when
* VERCEL_AUTOMATION_BYPASS_SECRET is set. Only that origin: adding it to
* every request (`extraHTTPHeaders`) would put an unexpected header on the
* app's Supabase calls and fail their CORS preflight.
* The suite's `test`: Playwright's, plus a health check every spec gets for
* free. Each test records uncaught page exceptions, console errors and
* same-origin 5xx responses, and fails when any were seen — so a hydration
* error or a crashing component fails the smoke even when the heading still
* rendered. A spec that asserts its own error status (`network: GET /x
* returned 404`) tolerates that status's console line; anything else known and
* harmless goes in GLOBAL_ALLOWED_PROBLEMS with a reason, or
* `test.use({ allowedConsoleErrors: allowing([...]) })` in a TS journey.
*/
export const test = base.extend<PageHealthOptions & { pageProblems: string[] }>({
allowedConsoleErrors: [[], { option: true }],

context: async ({ context }, use) => {
const secret = VERCEL_BYPASS_SECRET;
if (secret) {
await context.route(
(url) => url.origin === ORIGIN,
(route) =>
route.continue({
headers: { ...route.request().headers(), 'x-vercel-protection-bypass': secret },
}),
);
}
await use(context);
},

pageProblems: [
async ({ page, allowedConsoleErrors }, use, testInfo) => {
const problems: string[] = [];
Expand Down Expand Up @@ -101,8 +82,3 @@ export { expect };
* `[value, options]` tuple, so `[/a/, /b/]` would become the value `/a/`.
*/
export const allowing = (patterns: RegExp[]): [RegExp[], { scope: 'test' }] => [patterns, { scope: 'test' }];

/** Headers for `request.*` calls that must get through Vercel Deployment Protection. */
export function originHeaders(): Record<string, string> {
return VERCEL_BYPASS_SECRET ? { 'x-vercel-protection-bypass': VERCEL_BYPASS_SECRET } : {};
}
Loading