diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7d942e92c..d14cb44d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -462,11 +462,15 @@ jobs: # Playwright starts `next start` itself (playwright.config.ts webServer). # E2E_SUPABASE_* lets the setup project create a throwaway confirmed user # on the local stack (refused for any non-local URL — src/support/env.ts). + # LOREKIT_TELEMETRY_TOKEN lets src/support/otel-reporter.ts ship each test + # as a trace (service.name=e2e); empty on a fork PR, where it sends nothing. - name: Playwright — every page, signed out and signed in working-directory: packages/web-e2e env: E2E_SUPABASE_URL: http://127.0.0.1:54321 E2E_SUPABASE_SERVICE_ROLE_KEY: ${{ steps.supabase.outputs.service_role_key }} + LOREKIT_TELEMETRY_TOKEN: ${{ secrets.LOREKIT_TELEMETRY_TOKEN }} + LOREKIT_CORRELATION_ID: web-e2e-${{ github.run_id }}-${{ github.run_attempt }} run: npx playwright test - name: Upload the Playwright report (traces + screenshots) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 24230cd0f..a4830d369 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -428,6 +428,9 @@ jobs: # The staged build talks to the PRODUCTION backend, whatever its URL says. E2E_TARGET: production API_CHANGED: ${{ needs.changes.outputs.api }} + # The otel reporter (packages/web-e2e/src/support/otel-reporter.ts) ships + # each test as a trace; LOREKIT_TELEMETRY_TOKEN is set on the Playwright steps. + LOREKIT_CORRELATION_ID: e2e-web-staged-${{ github.run_id }}-${{ github.run_attempt }} steps: - name: Checkout repository uses: actions/checkout@v7.0.1 @@ -449,6 +452,8 @@ jobs: - name: Playwright — public pages on the staged build working-directory: packages/web-e2e + env: + LOREKIT_TELEMETRY_TOKEN: ${{ secrets.LOREKIT_TELEMETRY_TOKEN }} run: npx playwright test --project=public - name: Note — signed-in pages run after promotion this time @@ -467,6 +472,7 @@ jobs: env: E2E_USER_EMAIL: ${{ secrets.LOREKIT_TESTING_USER_NAME }} E2E_USER_PASSWORD: ${{ secrets.LOREKIT_TESTING_USER_PW }} + LOREKIT_TELEMETRY_TOKEN: ${{ secrets.LOREKIT_TELEMETRY_TOKEN }} run: npx playwright test --project=setup - name: Playwright — signed-in pages on the staged build @@ -475,6 +481,7 @@ jobs: env: E2E_USER_EMAIL: ${{ secrets.LOREKIT_TESTING_USER_NAME }} E2E_USER_PASSWORD: ${{ secrets.LOREKIT_TESTING_USER_PW }} + LOREKIT_TELEMETRY_TOKEN: ${{ secrets.LOREKIT_TELEMETRY_TOKEN }} run: npx playwright test --project=signed-in --no-deps - name: Warn — the test account could not sign in diff --git a/CLAUDE.md b/CLAUDE.md index 7bb19b334..2fc83f459 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -21,7 +21,7 @@ lets humans browse, search, and manage those lessons. | `plugins/` | `plugins/` | Per-framework deterministic bundles: `lorekit-claude` (marketplace plugin: skill + hooks + MCP), `lorekit-cursor` (rule + `stop` hook), `lorekit-codex` (feature-flagged hooks + `AGENTS.md` fallback, experimental). Root `.claude-plugin/marketplace.json` lists the Claude plugin. | | `supabase` | `supabase/` | Edge Functions (production MCP server), migrations, NX targets | | `@lorekit/smoke-tests` | `packages/smoke-tests/` | Live-endpoint integration/smoke tests against the deployed Edge Functions (memories, orgs, MCP, BYOD) — no application code, self-skips when its env vars are absent | -| `@lorekit/web-e2e` | `packages/web-e2e/` | Playwright suite for every page and key journey of the website, written as Markdown specs in `specs/*.md` (aw-tester's `WHEN`/`THEN` grammar — the same body as a `` PR block; `pnpm --filter @lorekit/web-e2e spec:import` / `spec:export` move them). A vitest guard fails the PR when a spec does not parse or a `page.tsx` has no spec. Runs per PR (local build + local Supabase), on the staged production build before promotion, and against `lorekit.io` after it — see [docs/smoke-tests.md](./docs/smoke-tests.md) | +| `@lorekit/web-e2e` | `packages/web-e2e/` | Playwright suite for every page and key journey of the website, written as Markdown specs in `specs/*.md` (aw-tester's `WHEN`/`THEN` grammar — the same body as a `` PR block; `pnpm --filter @lorekit/web-e2e spec:import` / `spec:export` move them). A vitest guard fails the PR when a spec does not parse or a `page.tsx` has no spec. Runs per PR (local build + local Supabase), on the staged production build before promotion, and against `lorekit.io` after it — see [docs/smoke-tests.md](./docs/smoke-tests.md). Each test attempt ships to Dash0 as a trace (`service.name=e2e`, `src/support/otel-reporter.ts`; typed values redacted) whenever the CLI's `resolveTelemetryConfig` enables export (`LOREKIT_TELEMETRY_TOKEN`, or the `OTEL_EXPORTER_OTLP_*` variables — an explicit endpoint needs no credential) | The **production MCP server** is `supabase/functions/mcp/index.ts` (Deno, self-contained). There is no other MCP server implementation — a prior Node.js/Fly.io variant (`packages/mcp-server/`) diff --git a/docs/otel.md b/docs/otel.md index ce7ea7afc..c00869d2d 100644 --- a/docs/otel.md +++ b/docs/otel.md @@ -341,6 +341,51 @@ itself forever. --- +## Website e2e runs (`service.name=e2e`) + +The Playwright suite (`packages/web-e2e`, [smoke-tests.md](./smoke-tests.md)) +ships every test attempt to Dash0 as its own trace, through a custom reporter, +`src/support/otel-reporter.ts`. Its span model is ported from the Dash0 repo's +`tests/e2e` reporter: + +| Span | Carries | +|---|---| +| Root, one per test attempt (its own trace) | `test.case.name`, `test.case.result.status` (`pass`/`fail`, absent on a skip), `test.suite.name`, `code.file.path`/`code.line.number`, and `lorekit.e2e.{project,status,expected_status,retry,flaky,worker_index,duration_ms,tags,skip_reason}`. ERROR with an `exception` event when the attempt ended unexpectedly | +| Child, one per step | `test.step`s (the spec grammar's `WHEN`/`THEN`), Playwright actions, `expect`s and hooks, nested as Playwright nests them; `lorekit.e2e.step.category`. Fixture steps are folded away. Capped at 250 per test (`lorekit.e2e.steps_dropped` counts the rest) | + +A retried test emits one trace per attempt, so a flake is two traces, the +second with `lorekit.e2e.flaky=true`. Run-wide values ride the resource: +`lorekit.e2e.target`, `lorekit.e2e.base_url`, `lorekit.correlation_id` (the +same run key the CLI, REST and MCP smokes send as the `X-LoreKit-Correlation-Id` +header, recorded on `usage_events` rather than on a span), and the +`cicd.pipeline.*` of the GitHub run. + +**What it never sends:** the value of any typing step. Playwright titles +`locator.fill(x)` as `Fill "x"`, which for the sign-in step is the test +account's password, so `Fill`/`Type`/`Insert`/`Press`/`Key up`/`Key down` +become `Fill "…"`. The same values are scrubbed from error text, where a failed +action's call log echoes them as `fill("x")`, and the configured secrets +(`E2E_USER_PASSWORD`, `LOREKIT_TESTING_USER_PW`, `E2E_SUPABASE_SERVICE_ROLE_KEY`) +are blanked from it whatever shape they arrive in. It also drops stdout, stderr, attachment paths, username and +hostname, which the Dash0 original records, and it rewrites absolute paths in +error messages and stack traces (repo root → relative, home directory → `~`) so +a failed local run does not ship the username. `otel-spans.test.ts` pins this. + +Export reuses `scripts/telemetry/otlp-export.mjs` and the CLI's +`resolveTelemetryConfig`, exactly as the eval exporter does: same token +priority, `Dash0-Dataset` routing, opt-outs and the hard-coded +`deployment.environment.name=test`. Nothing is sent unless that resolver +enables export — a credential (`LOREKIT_TELEMETRY_TOKEN`, +`OTEL_EXPORTER_OTLP_HEADERS`) or an explicit `OTEL_EXPORTER_OTLP_ENDPOINT` — so a +local shell that exports any of those sends, as the CLI does. Spans are sent once, from `onEnd`, and a failed export prints a line +without failing the run. `E2E_OTEL_OUT=` writes the payload to a file, +sent or not. + +In CI the Playwright steps of `web-e2e` (ci.yml), `e2e-web-staged` and +`smoke-production` (deploy.yml) carry the existing `LOREKIT_TELEMETRY_TOKEN`. +Compare runs on `service.name=e2e`, keyed by `vcs.ref.head.revision` or +`cicd.pipeline.run.id`. + ## Eval runs (`service.name=eval`) Full runbook, arms and how to read a summary: [`packages/evals/README.md`](../packages/evals/README.md). @@ -1557,11 +1602,12 @@ unique. | `sweep` | The row-scaling benchmark ([above](#the-row-scaling-sweep)) | `scripts/telemetry/sweep-telemetry.mjs` | | `load` | The load generator | `scripts/telemetry/load-telemetry.mjs` | | `eval` | The eval harness ([above](#eval-runs-servicenameeval)) | `scripts/telemetry/eval-telemetry.mjs` | +| `e2e` | The website Playwright suite ([above](#website-e2e-runs-servicenamee2e)) | `packages/web-e2e/src/support/otel-reporter.ts` | -- **The last three are synthetic by construction** and each gets its own name for - the same reason: a benchmark, a load test and an eval all emit traffic nobody +- **The last four are synthetic by construction** and each gets its own name for + the same reason: a benchmark, a load test, an eval and a browser test all emit traffic nobody asked for, and folding any of them into `api`/`cli`/`web` would corrupt the - very production numbers they exist to measure against. All three hard-code + very production numbers they exist to measure against. All four hard-code `deployment.environment.name=test` and are keyed by `vcs.ref.head.revision`. - **The edge functions are one service, not five.** They share a deployment, a database and a lifecycle; each function is an *operation* on `api`, not a separate service. Splitting them diff --git a/docs/smoke-tests.md b/docs/smoke-tests.md index f426974e0..38b0fe261 100644 --- a/docs/smoke-tests.md +++ b/docs/smoke-tests.md @@ -118,12 +118,24 @@ E2E_BASE_URL=https://www.lorekit.io E2E_USER_EMAIL=… E2E_USER_PASSWORD=… npx the build for the dev server while iterating. Every variable is documented at the top of `packages/web-e2e/src/support/env.ts`. +## Telemetry + +Every Playwright test attempt is also a trace in Dash0 (`service.name=e2e`), +shipped by `packages/web-e2e/src/support/otel-reporter.ts` whenever the CLI's +`resolveTelemetryConfig` enables export (`LOREKIT_TELEMETRY_TOKEN`, or the +standard `OTEL_EXPORTER_OTLP_*` variables — an explicit +`OTEL_EXPORTER_OTLP_ENDPOINT` needs no credential): a root span per test, a child per step, with +the status, retry, flakiness and duration. Typed values (the sign-in password) +are redacted. A run the resolver does not enable sends nothing (export it locally to send +from your machine too); `E2E_OTEL_OUT=` writes the payload to a file. Span reference: [otel.md](./otel.md#website-e2e-runs-servicenamee2e). + ## Secrets the deploy-time runs use | Secret | Used by | Without it | |--------|---------|------------| | `LOREKIT_TESTING_USER_NAME`, `LOREKIT_TESTING_USER_PW` (repo) | signed-in pages on the staged build and on `lorekit.io` | signed-in tests skip, announced; public tests still run | | `LOREKIT_SMOKE_TOKEN` (preview env, existing) | CLI remote lane, hosted MCP smoke | — (already required) | +| `LOREKIT_TELEMETRY_TOKEN` (repo, existing) | the e2e OTel reporter, on every Playwright step | tests run as before; no traces are sent | Traces are off on any run that targets production (the staged build included): a trace records request headers and cookies, and this public repo's artifacts are diff --git a/packages/web-e2e/playwright.config.ts b/packages/web-e2e/playwright.config.ts index 968240046..e983bd6f7 100644 --- a/packages/web-e2e/playwright.config.ts +++ b/packages/web-e2e/playwright.config.ts @@ -38,9 +38,12 @@ export default defineConfig({ workers: CI ? 2 : undefined, timeout: 45_000, expect: { timeout: 15_000 }, + // otel-reporter ships each test as a trace (service.name=e2e) when the CLI's + // resolveTelemetryConfig enables export (LOREKIT_TELEMETRY_TOKEN in CI, or the + // OTEL_EXPORTER_OTLP_* variables) and is a no-op otherwise. reporter: CI - ? [['github'], ['list'], ['html', { open: 'never', outputFolder: 'playwright-report' }]] - : [['list'], ['html', { open: 'never', outputFolder: 'playwright-report' }]], + ? [['github'], ['list'], ['html', { open: 'never', outputFolder: 'playwright-report' }], ['./src/support/otel-reporter.ts']] + : [['list'], ['html', { open: 'never', outputFolder: 'playwright-report' }], ['./src/support/otel-reporter.ts']], use: { baseURL: BASE_URL, diff --git a/packages/web-e2e/project.json b/packages/web-e2e/project.json index be1dd8899..814994a5d 100644 --- a/packages/web-e2e/project.json +++ b/packages/web-e2e/project.json @@ -3,11 +3,30 @@ "$schema": "../../node_modules/nx/schemas/project-schema.json", "sourceRoot": "packages/web-e2e/src", "projectType": "application", - "tags": ["type:e2e", "platform:web"], - "implicitDependencies": ["web"], + "tags": [ + "type:e2e", + "platform:web" + ], + "implicitDependencies": [ + "web", + "cli" + ], "targets": { "test": { - "dependsOn": [] + "dependsOn": [], + "inputs": [ + "default", + "^production", + { + "externalDependencies": [ + "vitest" + ] + }, + { + "env": "CI" + }, + "{workspaceRoot}/scripts/telemetry/**/*" + ] }, "typecheck": { "executor": "nx:run-commands", diff --git a/packages/web-e2e/src/otel-spans.test.ts b/packages/web-e2e/src/otel-spans.test.ts new file mode 100644 index 000000000..2aefd00da --- /dev/null +++ b/packages/web-e2e/src/otel-spans.test.ts @@ -0,0 +1,213 @@ +import { beforeAll, describe, expect, it } from 'vitest'; +import { loadOtlpExport, loadTelemetryConfig } from './support/otel-modules'; +import { buildRedactions, buildTestSpans, type OtlpEncoder, MAX_STEP_SPANS, stepSpanName, type ResultLike, type StepLike, type TestLike } from './support/otel-spans'; + +const T0 = new Date('2026-10-18T12:00:00.000Z'); + +const test: TestLike = { + titlePath: ['the signed-in app shell', 'keeps the session across a reload'], + projectName: 'signed-in', + file: 'journeys/dashboard.e2e.ts', + line: 13, + tags: ['@signed-in'], + expectedStatus: 'passed', +}; + +const step = (title: string, category: string, steps: StepLike[] = [], error?: string): StepLike => ({ + title, + category, + startTime: T0, + duration: 10, + steps, + ...(error ? { error: { message: error } } : {}), +}); + +const result = (over: Partial = {}): ResultLike => ({ + status: 'passed', + startTime: T0, + duration: 1500, + retry: 0, + workerIndex: 1, + steps: [], + ...over, +}); + +let encoder: OtlpEncoder; +beforeAll(async () => { + encoder = await loadOtlpExport(); +}); + +let counter = 0; +const newId = (bytes: 8 | 16) => String(++counter).padStart(bytes * 2, '0'); +const build = (r: ResultLike, t: TestLike = test) => buildTestSpans(t, r, newId, encoder); +const attrs = (span: { attributes: { key: string; value: Record }[] }) => + Object.fromEntries(span.attributes.map((a) => [a.key, Object.values(a.value)[0]])); + +describe('the reused CLI telemetry config', () => { + // The reporter calls this at runtime only, from onEnd, where a break is just a + // log line — so pin the contract it relies on here. + it('resolves an endpoint and headers with a token, and disables on opt-out', async () => { + const resolveTelemetryConfig = await loadTelemetryConfig(); + const on = resolveTelemetryConfig({ LOREKIT_TELEMETRY_TOKEN: 'test-token' }); + expect(on).toMatchObject({ enabled: true, endpoint: expect.stringMatching(/^https?:\/\//), headers: expect.any(Object) }); + expect(resolveTelemetryConfig({ LOREKIT_TELEMETRY_TOKEN: 'test-token', LOREKIT_TELEMETRY: 'off' })).toMatchObject({ + enabled: false, + reason: expect.any(String), + }); + }); +}); + +describe('buildRedactions', () => { + it('blanks configured secrets first, then makes repo paths relative, then folds home to ~', () => { + const redactions = buildRedactions( + { E2E_USER_PASSWORD: ' hunter2 ', LOREKIT_TESTING_USER_PW: '', E2E_SUPABASE_SERVICE_ROLE_KEY: 'sb_secret_x', OTHER: 'kept' }, + '/home/ci/repo/', + '/home/ci', + ); + expect(redactions).toEqual([ + ['hunter2', '…'], + ['sb_secret_x', '…'], + ['/home/ci/repo/', ''], + ['/home/ci', '~'], + ]); + const [root] = buildTestSpans( + test, + result({ status: 'failed', error: { message: 'hunter2 at /home/ci/repo/packages/a.ts and /home/ci/.cache/b.js' } }), + newId, + encoder, + redactions, + ); + expect(root.status.message).toBe('… at packages/a.ts and ~/.cache/b.js'); + }); +}); + +describe('stepSpanName', () => { + it.each([ + ['Fill "hunter2"', 'Fill "…"'], + ['Type "secret text"', 'Type "…"'], + ['Insert "x"', 'Insert "…"'], + ['Press "Enter"', 'Press "…"'], + ['Key down "Shift"', 'Key down "…"'], + ])('redacts the typed value of %s', (title, expected) => { + expect(stepSpanName('pw:api', title)).toBe(expected); + }); + + it('keeps titles that carry no typed value', () => { + expect(stepSpanName('pw:api', 'Navigate to "/login"')).toBe('Navigate to "/login"'); + expect(stepSpanName('test.step', 'Fill "the form"')).toBe('Fill "the form"'); + }); +}); + +describe('buildTestSpans', () => { + it('emits one root per attempt, in its own trace, with the test.* conventions', () => { + const [root, ...rest] = build(result()); + expect(rest).toEqual([]); + expect(root.parentSpanId).toBeUndefined(); + expect(root.traceId).toHaveLength(32); + expect(root.name).toBe('the signed-in app shell › keeps the session across a reload'); + expect(root.status).toEqual({ code: 0 }); + expect(attrs(root)).toMatchObject({ + 'test.case.name': 'journeys/dashboard.e2e.ts › the signed-in app shell › keeps the session across a reload', + 'test.case.result.status': 'pass', + 'lorekit.e2e.project': 'signed-in', + 'lorekit.e2e.retry': '0', + 'lorekit.e2e.flaky': false, + 'lorekit.e2e.tags': '@signed-in', + }); + expect(root.startTimeUnixNano).toBe(`${T0.getTime()}000000`); + expect(root.endTimeUnixNano).toBe(`${T0.getTime() + 1500}000000`); + }); + + it('marks an unexpected failure as an ERROR with an exception event', () => { + const [root] = build(result({ status: 'failed', error: { message: '\u001b[31mexpected 200\u001b[39m', stack: 'at x' } })); + expect(root.status).toEqual({ code: 2, message: 'expected 200' }); + expect(attrs(root)['test.case.result.status']).toBe('fail'); + const [exception] = root.events ?? []; + expect(exception.name).toBe('exception'); + expect(attrs(exception)).toMatchObject({ 'exception.message': 'expected 200', 'exception.stacktrace': 'at x' }); + }); + + it('rewrites absolute paths in error text so no home directory leaves the machine', () => { + const stack = 'Error: boom\n at /Users/someone/repo/packages/web-e2e/journeys/a.e2e.ts:3:1\n at /Users/someone/.cache/x.js:1:1'; + const [root] = buildTestSpans(test, result({ status: 'failed', error: { message: 'boom in /Users/someone/repo/x.ts', stack } }), newId, encoder, [ + ['/Users/someone/repo/packages/web-e2e/', ''], + ['/Users/someone', '~'], + ]); + expect(JSON.stringify(root)).not.toContain('someone'); + const [exception] = root.events ?? []; + expect(attrs(exception)['exception.stacktrace']).toContain('at journeys/a.e2e.ts:3:1'); + expect(attrs(exception)['exception.stacktrace']).toContain('at ~/.cache/x.js:1:1'); + expect(root.status.message).toBe('boom in ~/repo/x.ts'); + }); + + it('never ships a typed value or a known secret through error text', () => { + const callLog = [ + 'TimeoutError: locator.fill: Timeout 5000ms exceeded.', + 'Call log:', + ' - Fill "hunter2" with timeout 5000ms', + ' - waiting for getByLabel(\'Password\')', + ' - fill("hunter2")', + ' - elementHandle.type("pa"ss")', + ].join('\n'); + const spans = buildTestSpans( + test, + result({ + status: 'failed', + error: { message: callLog, stack: `${callLog}\n at x (s3cr3t-key)` }, + steps: [step('Fill "hunter2"', 'pw:api', [], callLog)], + }), + newId, + encoder, + [['s3cr3t-key', '…']], + ); + const json = JSON.stringify(spans); + for (const leaked of ['hunter2', 'pa\\"ss', 's3cr3t-key']) expect(json).not.toContain(leaked); + expect(spans[0].status.message).toContain('Fill "…" with timeout 5000ms'); + expect(spans[0].status.message).toContain('fill("…")'); + expect(spans[0].status.message).toContain("getByLabel('Password')"); + }); + + it('treats an expected failure (test.fail) as a pass', () => { + const [root] = build(result({ status: 'failed' }), { ...test, expectedStatus: 'failed' }); + expect(root.status.code).toBe(0); + expect(attrs(root)['test.case.result.status']).toBe('pass'); + }); + + it('flags a pass on a retry as flaky', () => { + expect(attrs(build(result({ retry: 1 }))[0])['lorekit.e2e.flaky']).toBe(true); + }); + + it('records a skip without a pass/fail verdict, keeping the reason', () => { + const [root] = build(result({ status: 'skipped', duration: 0 }), { ...test, expectedStatus: 'skipped', skipReason: 'no test user' }); + expect(attrs(root)['test.case.result.status']).toBeUndefined(); + expect(attrs(root)['lorekit.e2e.skip_reason']).toBe('no test user'); + }); + + it('nests steps, folds fixtures away, and never leaks a typed password', () => { + const spans = build( + result({ + steps: [ + step('Fixture "page"', 'fixture', [step('Create context', 'pw:api')]), + step('WHEN the visitor signs in', 'test.step', [ + step('Fill "hunter2"', 'pw:api'), + step('Expect "toHaveURL"', 'expect', [], 'timed out'), + ]), + ], + }), + ); + const [root, create, when, fill, expectStep] = spans; + expect(spans.map((s) => s.name)).toEqual([root.name, 'Create context', 'WHEN the visitor signs in', 'Fill "…"', 'Expect "toHaveURL"']); + expect(create.parentSpanId).toBe(root.spanId); // re-parented past the fixture + expect(fill.parentSpanId).toBe(when.spanId); + expect(expectStep.status).toEqual({ code: 2, message: 'timed out' }); + expect(JSON.stringify(spans)).not.toContain('hunter2'); + expect(new Set(spans.map((s) => s.traceId)).size).toBe(1); + }); + + it(`caps a test at ${MAX_STEP_SPANS} step spans and counts the rest`, () => { + const steps = Array.from({ length: MAX_STEP_SPANS + 5 }, (_, i) => step(`Click ${i}`, 'pw:api')); + const [root, ...stepSpans] = build(result({ steps })); + expect(stepSpans).toHaveLength(MAX_STEP_SPANS); + expect(attrs(root)['lorekit.e2e.steps_dropped']).toBe('5'); + }); +}); diff --git a/packages/web-e2e/src/support/otel-modules.ts b/packages/web-e2e/src/support/otel-modules.ts new file mode 100644 index 000000000..f99451e9e --- /dev/null +++ b/packages/web-e2e/src/support/otel-modules.ts @@ -0,0 +1,35 @@ +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import type { OtlpEncoder, OtlpSpan } from './otel-spans'; + +/** + * The two ES modules the e2e telemetry reuses rather than copies: + * + * scripts/telemetry/otlp-export.mjs the OTLP/JSON encoder + POST shared by + * the sweep/load/eval exporters + * packages/cli/src/telemetry/…mjs `resolveTelemetryConfig` — endpoint, + * credential, dataset, opt-outs + * + * Loaded with `import()` because Playwright compiles this package to CommonJS, + * which cannot statically import an ES module. + */ +const REPO_ROOT = resolve(__dirname, '../../../..'); +const moduleUrl = (path: string) => pathToFileURL(resolve(REPO_ROOT, path)).href; + +export interface OtlpExport extends OtlpEncoder { + spansEnvelope(serviceName: string, spans: OtlpSpan[], extraResource?: Record): unknown; + post(url: string, headers: Record, payload: unknown): Promise<{ ok: boolean; error?: string }>; +} + +export type TelemetryConfig = + | { enabled: true; endpoint: string; headers: Record } + | { enabled: false; reason: string }; + +export async function loadOtlpExport(): Promise { + return import(moduleUrl('scripts/telemetry/otlp-export.mjs')); +} + +export async function loadTelemetryConfig(): Promise<(env: NodeJS.ProcessEnv) => TelemetryConfig> { + const { resolveTelemetryConfig } = await import(moduleUrl('packages/cli/src/telemetry/telemetry.mjs')); + return resolveTelemetryConfig; +} diff --git a/packages/web-e2e/src/support/otel-reporter.ts b/packages/web-e2e/src/support/otel-reporter.ts new file mode 100644 index 000000000..2ba04165f --- /dev/null +++ b/packages/web-e2e/src/support/otel-reporter.ts @@ -0,0 +1,115 @@ +import { randomBytes } from 'node:crypto'; +import { writeFileSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { dirname, relative, resolve } from 'node:path'; +import type { FullConfig, Reporter, TestCase, TestResult } from '@playwright/test/reporter'; +import { BASE_URL, TARGET } from './env'; +import { loadOtlpExport, loadTelemetryConfig } from './otel-modules'; +import { buildRedactions, buildTestSpans, type OtlpEncoder, type OtlpSpan } from './otel-spans'; + +/** + * Ships every test attempt to Dash0 as a trace (`service.name=e2e`), so a red — + * or slowly greying — deploy gate is diagnosable from telemetry, and flakes and + * durations can be charted across runs. Span shape: `otel-spans.ts`. + * + * Export is the repo's existing one: the CLI's `resolveTelemetryConfig` decides + * endpoint, credential, dataset and opt-outs (`LOREKIT_TELEMETRY_TOKEN` in CI; + * `LOREKIT_TELEMETRY=off` / `DO_NOT_TRACK=1` opt out), and `otlp-export.mjs` + * encodes and POSTs. When that resolver does not enable export, nothing is sent. + * + * Results are collected during the run and sent ONCE, from `onEnd`. A failed + * export prints a line and never fails a run: the tests are the gate, their + * telemetry is not. + * + * E2E_OTEL_OUT= also writes the payload there, sent or not — the way to + * see exactly what would leave the machine. + */ + +export default class OtelReporter implements Reporter { + private readonly finished: [TestCase, TestResult][] = []; + /** The monorepo root — playwright.config.ts sits in packages/web-e2e/. */ + private repoRoot = process.cwd(); + + printsToStdio(): boolean { + return false; + } + + onBegin(config: FullConfig): void { + if (config.configFile) this.repoRoot = resolve(dirname(config.configFile), '../..'); + } + + onTestEnd(test: TestCase, result: TestResult): void { + this.finished.push([test, result]); + } + + async onEnd(): Promise { + if (this.finished.length === 0) return; + try { + await this.export(); + } catch (err) { + console.log(`otel: e2e span export skipped — ${(err as Error).message}`); + } + } + + private async export(): Promise { + const otlp = await loadOtlpExport(); + const spans = this.finished.flatMap(([test, result]) => this.spansFor(test, result, otlp)); + const payload = otlp.spansEnvelope('e2e', spans, runResource(process.env)); + + const out = process.env['E2E_OTEL_OUT']; + if (out) writeFileSync(out, JSON.stringify(payload, null, 2)); + + const resolveTelemetryConfig = await loadTelemetryConfig(); + const cfg = resolveTelemetryConfig(process.env); + if (!cfg.enabled) { + if (process.env['CI']) console.log(`otel: e2e spans not exported (${cfg.reason})`); + return; + } + const res = await otlp.post(`${cfg.endpoint}/v1/traces`, cfg.headers, payload); + console.log( + res.ok + ? `otel: exported ${spans.length} e2e spans for ${this.finished.length} test attempts (service.name=e2e)` + : `otel: e2e span export failed — ${res.error}`, + ); + } + + private spansFor(test: TestCase, result: TestResult, encoder: OtlpEncoder): OtlpSpan[] { + // titlePath(): ['', project, file, ...describes, title] + const [, , , ...titlePath] = test.titlePath(); + return buildTestSpans( + { + titlePath, + projectName: test.parent.project()?.name ?? '', + file: relative(this.repoRoot, test.location.file), + line: test.location.line, + tags: test.tags, + expectedStatus: test.expectedStatus, + skipReason: test.annotations.find((a) => a.type === 'skip')?.description, + }, + result, + (bytes) => randomBytes(bytes).toString('hex'), + encoder, + buildRedactions(process.env, this.repoRoot, homedir()), + ); + } +} + +/** Run-wide resource attributes: what was tested, and the CI run that tested it. */ +export function runResource(env: NodeJS.ProcessEnv): Record { + const github = env['GITHUB_ACTIONS'] === 'true'; + const correlation = env['LOREKIT_CORRELATION_ID']?.trim(); + return { + 'lorekit.e2e.target': TARGET, + 'lorekit.e2e.base_url': BASE_URL, + // Same bounds as smoke-telemetry.ts's header: the one key a deploy run's + // CLI, REST, MCP and browser smokes share. + 'lorekit.correlation_id': + correlation && correlation.length <= 200 && /^[A-Za-z0-9_\-./:#@]+$/.test(correlation) ? correlation : undefined, + 'cicd.pipeline.name': github ? env['GITHUB_WORKFLOW'] : undefined, + 'cicd.pipeline.task.name': github ? env['GITHUB_JOB'] : undefined, + 'cicd.pipeline.run.id': github ? env['GITHUB_RUN_ID'] : undefined, + 'cicd.pipeline.run.url.full': github + ? `${env['GITHUB_SERVER_URL']}/${env['GITHUB_REPOSITORY']}/actions/runs/${env['GITHUB_RUN_ID']}` + : undefined, + }; +} diff --git a/packages/web-e2e/src/support/otel-spans.ts b/packages/web-e2e/src/support/otel-spans.ts new file mode 100644 index 000000000..9fa2a77fa --- /dev/null +++ b/packages/web-e2e/src/support/otel-spans.ts @@ -0,0 +1,257 @@ +/** + * Playwright results → OTLP/JSON spans, for `otel-reporter.ts` to ship. + * + * Pure: no I/O, no clock, ids injected — so `otel-spans.test.ts` can pin the + * shape. The span model is ported from dash0's `tests/e2e` reporter: + * + * • one ROOT span per test attempt, each its own trace. A run of 70 tests is + * 70 small traces you can filter, not one trace nobody can read. A retried + * test emits one root per attempt, so a flake shows up as two traces. + * • a child span per step — `test.step`, Playwright actions, `expect`s and + * hooks. Fixture setup is plumbing: its spans are dropped and their children + * re-parented to the nearest kept ancestor. + * + * What it deliberately does NOT carry, unlike the dash0 original: stdout, + * stderr, attachment paths, username and hostname — and it rewrites the absolute + * paths in error messages and stacks (which name the home directory) as + * relative ones. And it REDACTS every typed value — Playwright titles + * `locator.fill(x)` as `Fill "x"`, and a failed action's error carries a call + * log with `fill("x")` in it, so the test account's password would otherwise + * ride a span name or an exception into Dash0. Known secret values are blanked + * from error text on top of that, whatever shape they arrive in. + */ + +/** + * The repo's ONE OTLP/JSON encoder — scripts/telemetry/otlp-export.mjs, shared + * with the sweep/load/eval exporters (int64 as a string, nanosecond strings) — + * passed in rather than imported: Playwright compiles this file to CommonJS, + * which cannot statically import an ES module, so the reporter loads it with + * `import()` and hands it over. + */ +export interface OtlpEncoder { + toOtlpAttributes(attrs: Record): OtlpAttribute[]; + toUnixNano(ms: number): string; +} +/** `{key, value: AnyValue}` — AnyValue being e.g. `{ stringValue }` or `{ intValue }`. */ +export interface OtlpAttribute { + key: string; + value: Record; +} + +/** OTLP span kind INTERNAL: a test acts for nobody. */ +const SPAN_KIND_INTERNAL = 1; +const STATUS_UNSET = 0; +const STATUS_ERROR = 2; + +/** Step categories that become spans. The rest (`fixture`, `attach`, …) are folded away. */ +const KEPT_STEP_CATEGORIES = new Set(['test.step', 'pw:api', 'expect', 'hook']); +/** A runaway loop of actions must not become a 10k-span trace. */ +export const MAX_STEP_SPANS = 250; + +const NAME_MAX = 200; +const MESSAGE_MAX = 1024; +const STACK_MAX = 4096; + +/** Playwright API titles that interpolate what the test TYPED (protocolMetainfo.js). */ +const TYPING_STEP = /^(Fill|Type|Insert|Press|Key up|Key down) "/; +/** + * The same typed values inside error text: a call-log title (`Fill "x"`) or a + * call-log action (`fill("x")`, `elementHandle.type("x")`). Playwright does not + * escape the value, so match greedily to the line's last quote. + */ +const TYPED_IN_ERROR = [ + /\b(Fill|Type|Insert|Press|Key up|Key down) ".*"/g, + /\b(fill|type|press|insertText)\(".*"\)/g, +] as const; + +export interface StepLike { + title: string; + category: string; + startTime: Date; + duration: number; + error?: { message?: string }; + steps: StepLike[]; +} + +export interface TestLike { + /** describe blocks + title, without the root/project/file entries. */ + titlePath: string[]; + projectName: string; + /** Repo-relative path, e.g. `packages/web-e2e/src/journeys/login.e2e.ts`. */ + file: string; + line: number; + tags: string[]; + expectedStatus: string; + /** The `test.skip(cond, reason)` reason, when there was one. */ + skipReason?: string; +} + +export interface ResultLike { + status: 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted'; + startTime: Date; + duration: number; + retry: number; + workerIndex: number; + error?: { message?: string; stack?: string }; + steps: StepLike[]; +} + +export interface OtlpSpan { + traceId: string; + spanId: string; + parentSpanId?: string; + name: string; + kind: number; + startTimeUnixNano: string; + endTimeUnixNano: string; + attributes: OtlpAttribute[]; + events?: { name: string; timeUnixNano: string; attributes: OtlpAttribute[] }[]; + status: { code: number; message?: string }; +} + +/** `bytes` random bytes as lowercase hex — 16 for a trace id, 8 for a span id. */ +export type NewId = (bytes: 8 | 16) => string; + +/** + * `[literal, replacement]` pairs applied to error text, in order — known secret + * values → `…`, then the repo root → `''`, then the home directory → `~`. A + * stack frame `/Users//…` would otherwise ship the username. + */ +export type Redactions = readonly (readonly [string, string])[]; + +/** Env vars whose values must never appear in exported error text. */ +export const SECRET_ENV = ['E2E_USER_PASSWORD', 'LOREKIT_TESTING_USER_PW', 'E2E_SUPABASE_SERVICE_ROLE_KEY'] as const; + +/** Secret values blanked first, then repo-relative paths, then `~` for the rest of home. */ +export function buildRedactions(env: Record, repoRoot: string, home: string): Redactions { + const secrets = SECRET_ENV.map((name) => env[name]?.trim()).filter((v): v is string => !!v); + return [...secrets.map((secret) => [secret, '…'] as const), [`${repoRoot.replace(/\/+$/, '')}/`, ''], [home, '~']]; +} + +function redact(text: string | undefined, redactions: Redactions): string | undefined { + if (!text) return text; + const literal = redactions.reduce((out, [from, to]) => (from ? out.split(from).join(to) : out), text); + return TYPED_IN_ERROR.reduce( + (out, pattern) => out.replace(pattern, (match) => match.replace(/(\(?)".*"(\)?)$/, '$1"…"$2')), + literal, + ); +} + +// eslint-disable-next-line no-control-regex -- matching the ESC control character is the point +const ANSI = /\u001B\[[0-9;]*[A-Za-z]/g; + +function clean(text: string | undefined, max: number): string | undefined { + if (!text) return undefined; + const plain = text.replace(ANSI, ''); + return plain.length > max ? `${plain.slice(0, max - 1)}…` : plain; +} + +/** A step's span name, with any typed value replaced. */ +export function stepSpanName(category: string, title: string): string { + if (category === 'pw:api') { + const typing = TYPING_STEP.exec(title); + if (typing) return `${typing[1]} "…"`; + } + return clean(title, NAME_MAX) ?? category; +} + +/** Did the attempt end the way the test said it would (`test.fail()` included)? */ +function outcome(test: TestLike, result: ResultLike): 'pass' | 'fail' | 'skipped' { + if (result.status === 'skipped') return 'skipped'; + return result.status === test.expectedStatus ? 'pass' : 'fail'; +} + +/** The spans for one test attempt: its root first, then its steps in start order. */ +export function buildTestSpans( + test: TestLike, + result: ResultLike, + newId: NewId, + { toOtlpAttributes, toUnixNano }: OtlpEncoder, + redactions: Redactions = [], +): OtlpSpan[] { + const errorText = (text: string | undefined, max: number) => clean(redact(text, redactions), max); + const traceId = newId(16); + const rootId = newId(8); + const startMs = result.startTime.getTime(); + const verdict = outcome(test, result); + const failed = verdict === 'fail'; + + const stepSpans: OtlpSpan[] = []; + let droppedSteps = 0; + const walk = (steps: StepLike[], parentSpanId: string) => { + for (const step of steps) { + if (!KEPT_STEP_CATEGORIES.has(step.category)) { + walk(step.steps, parentSpanId); + continue; + } + if (stepSpans.length >= MAX_STEP_SPANS) { + droppedSteps += 1 + countKept(step.steps); + continue; + } + const spanId = newId(8); + const stepStart = step.startTime.getTime(); + const message = errorText(step.error?.message, MESSAGE_MAX); + stepSpans.push({ + traceId, + spanId, + parentSpanId, + name: stepSpanName(step.category, step.title), + kind: SPAN_KIND_INTERNAL, + startTimeUnixNano: toUnixNano(stepStart), + endTimeUnixNano: toUnixNano(stepStart + step.duration), + attributes: toOtlpAttributes({ 'lorekit.e2e.step.category': step.category }), + status: step.error ? { code: STATUS_ERROR, message } : { code: STATUS_UNSET }, + }); + walk(step.steps, spanId); + } + }; + walk(result.steps, rootId); + + const endMs = startMs + result.duration; + const message = failed ? (errorText(result.error?.message, MESSAGE_MAX) ?? `test ${result.status}`) : undefined; + const root: OtlpSpan = { + traceId, + spanId: rootId, + name: clean(test.titlePath.join(' › '), NAME_MAX) ?? 'test', + kind: SPAN_KIND_INTERNAL, + startTimeUnixNano: toUnixNano(startMs), + endTimeUnixNano: toUnixNano(endMs), + attributes: toOtlpAttributes({ + // OTel semantic conventions (test.*) first, then this suite's own. + 'test.case.name': [test.file, ...test.titlePath].join(' › '), + 'test.case.result.status': verdict === 'skipped' ? undefined : verdict, + 'test.suite.name': test.file, + 'lorekit.e2e.project': test.projectName, + 'lorekit.e2e.status': result.status, + 'lorekit.e2e.expected_status': test.expectedStatus, + 'lorekit.e2e.retry': result.retry, + 'lorekit.e2e.flaky': verdict === 'pass' && result.retry > 0, + 'lorekit.e2e.worker_index': result.workerIndex, + 'lorekit.e2e.duration_ms': result.duration, + 'lorekit.e2e.tags': test.tags.length ? test.tags.join(',') : undefined, + 'lorekit.e2e.skip_reason': verdict === 'skipped' ? test.skipReason : undefined, + 'lorekit.e2e.steps_dropped': droppedSteps || undefined, + 'code.file.path': test.file, + 'code.line.number': test.line, + }), + status: failed ? { code: STATUS_ERROR, message } : { code: STATUS_UNSET }, + }; + if (failed && result.error) { + root.events = [ + { + name: 'exception', + timeUnixNano: toUnixNano(endMs), + attributes: toOtlpAttributes({ + 'exception.type': `playwright.${result.status}`, + 'exception.message': message, + 'exception.stacktrace': errorText(result.error.stack, STACK_MAX), + }), + }, + ]; + } + return [root, ...stepSpans]; +} + +function countKept(steps: StepLike[]): number { + return steps.reduce((n, s) => n + (KEPT_STEP_CATEGORIES.has(s.category) ? 1 : 0) + countKept(s.steps), 0); +}