test(web-e2e): ship every Playwright test as a trace to Dash0

.github/workflows/ci.yml

@@ -462,11 +462,15 @@ jobs:
       # Playwright starts `next start` itself (playwright.config.ts webServer).
       # E2E_SUPABASE_* lets the setup project create a throwaway confirmed user
       # on the local stack (refused for any non-local URL — src/support/env.ts).
+      # LOREKIT_TELEMETRY_TOKEN lets src/support/otel-reporter.ts ship each test
+      # as a trace (service.name=e2e); empty on a fork PR, where it sends nothing.
       - name: Playwright — every page, signed out and signed in
         working-directory: packages/web-e2e
         env:
           E2E_SUPABASE_URL: http://127.0.0.1:54321
           E2E_SUPABASE_SERVICE_ROLE_KEY: ${{ steps.supabase.outputs.service_role_key }}
+          LOREKIT_TELEMETRY_TOKEN: ${{ secrets.LOREKIT_TELEMETRY_TOKEN }}
+          LOREKIT_CORRELATION_ID: web-e2e-${{ github.run_id }}-${{ github.run_attempt }}
         run: npx playwright test
 
       - name: Upload the Playwright report (traces + screenshots)

.github/workflows/deploy.yml

@@ -428,6 +428,9 @@ jobs:
       # The staged build talks to the PRODUCTION backend, whatever its URL says.
       E2E_TARGET: production
       API_CHANGED: ${{ needs.changes.outputs.api }}
+      # The otel reporter (packages/web-e2e/src/support/otel-reporter.ts) ships
+      # each test as a trace; LOREKIT_TELEMETRY_TOKEN is set on the Playwright steps.
+      LOREKIT_CORRELATION_ID: e2e-web-staged-${{ github.run_id }}-${{ github.run_attempt }}
     steps:
       - name: Checkout repository
         uses: actions/[email protected]
@@ -449,6 +452,8 @@ jobs:
 
       - name: Playwright — public pages on the staged build
         working-directory: packages/web-e2e
+        env:
+          LOREKIT_TELEMETRY_TOKEN: ${{ secrets.LOREKIT_TELEMETRY_TOKEN }}
         run: npx playwright test --project=public
 
       - name: Note — signed-in pages run after promotion this time
@@ -467,6 +472,7 @@ jobs:
         env:
           E2E_USER_EMAIL: ${{ secrets.LOREKIT_TESTING_USER_NAME }}
           E2E_USER_PASSWORD: ${{ secrets.LOREKIT_TESTING_USER_PW }}
+          LOREKIT_TELEMETRY_TOKEN: ${{ secrets.LOREKIT_TELEMETRY_TOKEN }}
         run: npx playwright test --project=setup
 
       - name: Playwright — signed-in pages on the staged build
@@ -475,6 +481,7 @@ jobs:
         env:
           E2E_USER_EMAIL: ${{ secrets.LOREKIT_TESTING_USER_NAME }}
           E2E_USER_PASSWORD: ${{ secrets.LOREKIT_TESTING_USER_PW }}
+          LOREKIT_TELEMETRY_TOKEN: ${{ secrets.LOREKIT_TELEMETRY_TOKEN }}
         run: npx playwright test --project=signed-in --no-deps
 
       - name: Warn — the test account could not sign in

CLAUDE.md

@@ -21,7 +21,7 @@ lets humans browse, search, and manage those lessons.
 | `plugins/` | `plugins/` | Per-framework deterministic bundles: `lorekit-claude` (marketplace plugin: skill + hooks + MCP), `lorekit-cursor` (rule + `stop` hook), `lorekit-codex` (feature-flagged hooks + `AGENTS.md` fallback, experimental). Root `.claude-plugin/marketplace.json` lists the Claude plugin. |
 | `supabase` | `supabase/` | Edge Functions (production MCP server), migrations, NX targets |
 | `@lorekit/smoke-tests` | `packages/smoke-tests/` | Live-endpoint integration/smoke tests against the deployed Edge Functions (memories, orgs, MCP, BYOD) — no application code, self-skips when its env vars are absent |
-| `@lorekit/web-e2e` | `packages/web-e2e/` | Playwright suite for every page and key journey of the website, written as Markdown specs in `specs/*.md` (aw-tester's `WHEN`/`THEN` grammar — the same body as a `<!-- ui-verify:v1 -->` PR block; `pnpm --filter @lorekit/web-e2e spec:import` / `spec:export` move them). A vitest guard fails the PR when a spec does not parse or a `page.tsx` has no spec. Runs per PR (local build + local Supabase), on the staged production build before promotion, and against `lorekit.io` after it — see [docs/smoke-tests.md](./docs/smoke-tests.md) |
+| `@lorekit/web-e2e` | `packages/web-e2e/` | Playwright suite for every page and key journey of the website, written as Markdown specs in `specs/*.md` (aw-tester's `WHEN`/`THEN` grammar — the same body as a `<!-- ui-verify:v1 -->` PR block; `pnpm --filter @lorekit/web-e2e spec:import` / `spec:export` move them). A vitest guard fails the PR when a spec does not parse or a `page.tsx` has no spec. Runs per PR (local build + local Supabase), on the staged production build before promotion, and against `lorekit.io` after it — see [docs/smoke-tests.md](./docs/smoke-tests.md). Each test attempt ships to Dash0 as a trace (`service.name=e2e`, `src/support/otel-reporter.ts`; typed values redacted) whenever the CLI's `resolveTelemetryConfig` enables export (`LOREKIT_TELEMETRY_TOKEN`, or the `OTEL_EXPORTER_OTLP_*` variables — an explicit endpoint needs no credential) |
 
 The **production MCP server** is `supabase/functions/mcp/index.ts` (Deno, self-contained). There
 is no other MCP server implementation — a prior Node.js/Fly.io variant (`packages/mcp-server/`)

docs/otel.md

@@ -341,6 +341,51 @@ itself forever.
 
 ---
 
+## Website e2e runs (`service.name=e2e`)
+
+The Playwright suite (`packages/web-e2e`, [smoke-tests.md](./smoke-tests.md))
+ships every test attempt to Dash0 as its own trace, through a custom reporter,
+`src/support/otel-reporter.ts`. Its span model is ported from the Dash0 repo's
+`tests/e2e` reporter:
+
+| Span | Carries |
+|---|---|
+| Root, one per test attempt (its own trace) | `test.case.name`, `test.case.result.status` (`pass`/`fail`, absent on a skip), `test.suite.name`, `code.file.path`/`code.line.number`, and `lorekit.e2e.{project,status,expected_status,retry,flaky,worker_index,duration_ms,tags,skip_reason}`. ERROR with an `exception` event when the attempt ended unexpectedly |
+| Child, one per step | `test.step`s (the spec grammar's `WHEN`/`THEN`), Playwright actions, `expect`s and hooks, nested as Playwright nests them; `lorekit.e2e.step.category`. Fixture steps are folded away. Capped at 250 per test (`lorekit.e2e.steps_dropped` counts the rest) |
+
+A retried test emits one trace per attempt, so a flake is two traces, the
+second with `lorekit.e2e.flaky=true`. Run-wide values ride the resource:
+`lorekit.e2e.target`, `lorekit.e2e.base_url`, `lorekit.correlation_id` (the
+same run key the CLI, REST and MCP smokes send as the `X-LoreKit-Correlation-Id`
+header, recorded on `usage_events` rather than on a span), and the
+`cicd.pipeline.*` of the GitHub run.
+
+**What it never sends:** the value of any typing step. Playwright titles
+`locator.fill(x)` as `Fill "x"`, which for the sign-in step is the test
+account's password, so `Fill`/`Type`/`Insert`/`Press`/`Key up`/`Key down`
+become `Fill "…"`. The same values are scrubbed from error text, where a failed
+action's call log echoes them as `fill("x")`, and the configured secrets
+(`E2E_USER_PASSWORD`, `LOREKIT_TESTING_USER_PW`, `E2E_SUPABASE_SERVICE_ROLE_KEY`)
+are blanked from it whatever shape they arrive in. It also drops stdout, stderr, attachment paths, username and
+hostname, which the Dash0 original records, and it rewrites absolute paths in
+error messages and stack traces (repo root → relative, home directory → `~`) so
+a failed local run does not ship the username. `otel-spans.test.ts` pins this.
+
+Export reuses `scripts/telemetry/otlp-export.mjs` and the CLI's
+`resolveTelemetryConfig`, exactly as the eval exporter does: same token
+priority, `Dash0-Dataset` routing, opt-outs and the hard-coded
+`deployment.environment.name=test`. Nothing is sent unless that resolver
+enables export — a credential (`LOREKIT_TELEMETRY_TOKEN`,
+`OTEL_EXPORTER_OTLP_HEADERS`) or an explicit `OTEL_EXPORTER_OTLP_ENDPOINT` — so a
+local shell that exports any of those sends, as the CLI does. Spans are sent once, from `onEnd`, and a failed export prints a line
+without failing the run. `E2E_OTEL_OUT=<file>` writes the payload to a file,
+sent or not.
+
+In CI the Playwright steps of `web-e2e` (ci.yml), `e2e-web-staged` and
+`smoke-production` (deploy.yml) carry the existing `LOREKIT_TELEMETRY_TOKEN`.
+Compare runs on `service.name=e2e`, keyed by `vcs.ref.head.revision` or
+`cicd.pipeline.run.id`.
+
 ## Eval runs (`service.name=eval`)
 
 Full runbook, arms and how to read a summary: [`packages/evals/README.md`](../packages/evals/README.md).
@@ -1557,11 +1602,12 @@ unique.
 | `sweep` | The row-scaling benchmark ([above](#the-row-scaling-sweep)) | `scripts/telemetry/sweep-telemetry.mjs` |
 | `load` | The load generator | `scripts/telemetry/load-telemetry.mjs` |
 | `eval` | The eval harness ([above](#eval-runs-servicenameeval)) | `scripts/telemetry/eval-telemetry.mjs` |
+| `e2e` | The website Playwright suite ([above](#website-e2e-runs-servicenamee2e)) | `packages/web-e2e/src/support/otel-reporter.ts` |
 
-- **The last three are synthetic by construction** and each gets its own name for
-  the same reason: a benchmark, a load test and an eval all emit traffic nobody
+- **The last four are synthetic by construction** and each gets its own name for
+  the same reason: a benchmark, a load test, an eval and a browser test all emit traffic nobody
   asked for, and folding any of them into `api`/`cli`/`web` would corrupt the
-  very production numbers they exist to measure against. All three hard-code
+  very production numbers they exist to measure against. All four hard-code
   `deployment.environment.name=test` and are keyed by `vcs.ref.head.revision`.
 - **The edge functions are one service, not five.** They share a deployment, a database and a
   lifecycle; each function is an *operation* on `api`, not a separate service. Splitting them

docs/smoke-tests.md

@@ -118,12 +118,24 @@ E2E_BASE_URL=https://www.lorekit.io E2E_USER_EMAIL=… E2E_USER_PASSWORD=… npx
 the build for the dev server while iterating. Every variable is documented at
 the top of `packages/web-e2e/src/support/env.ts`.
 
+## Telemetry
+
+Every Playwright test attempt is also a trace in Dash0 (`service.name=e2e`),
+shipped by `packages/web-e2e/src/support/otel-reporter.ts` whenever the CLI's
+`resolveTelemetryConfig` enables export (`LOREKIT_TELEMETRY_TOKEN`, or the
+standard `OTEL_EXPORTER_OTLP_*` variables — an explicit
+`OTEL_EXPORTER_OTLP_ENDPOINT` needs no credential): a root span per test, a child per step, with
+the status, retry, flakiness and duration. Typed values (the sign-in password)
+are redacted. A run the resolver does not enable sends nothing (export it locally to send
+from your machine too); `E2E_OTEL_OUT=<file>` writes the payload to a file. Span reference: [otel.md](./otel.md#website-e2e-runs-servicenamee2e).
+
 ## Secrets the deploy-time runs use
 
 | Secret | Used by | Without it |
 |--------|---------|------------|
 | `LOREKIT_TESTING_USER_NAME`, `LOREKIT_TESTING_USER_PW` (repo) | signed-in pages on the staged build and on `lorekit.io` | signed-in tests skip, announced; public tests still run |
 | `LOREKIT_SMOKE_TOKEN` (preview env, existing) | CLI remote lane, hosted MCP smoke | — (already required) |
+| `LOREKIT_TELEMETRY_TOKEN` (repo, existing) | the e2e OTel reporter, on every Playwright step | tests run as before; no traces are sent |
 
 Traces are off on any run that targets production (the staged build included): a
 trace records request headers and cookies, and this public repo's artifacts are

packages/web-e2e/playwright.config.ts

@@ -38,9 +38,12 @@ export default defineConfig({
   workers: CI ? 2 : undefined,
   timeout: 45_000,
   expect: { timeout: 15_000 },
+  // otel-reporter ships each test as a trace (service.name=e2e) when the CLI's
+  // resolveTelemetryConfig enables export (LOREKIT_TELEMETRY_TOKEN in CI, or the
+  // OTEL_EXPORTER_OTLP_* variables) and is a no-op otherwise.
   reporter: CI
-    ? [['github'], ['list'], ['html', { open: 'never', outputFolder: 'playwright-report' }]]
-    : [['list'], ['html', { open: 'never', outputFolder: 'playwright-report' }]],
+    ? [['github'], ['list'], ['html', { open: 'never', outputFolder: 'playwright-report' }], ['./src/support/otel-reporter.ts']]
+    : [['list'], ['html', { open: 'never', outputFolder: 'playwright-report' }], ['./src/support/otel-reporter.ts']],
 
   use: {
     baseURL: BASE_URL,

packages/web-e2e/project.json

@@ -3,11 +3,30 @@
   "$schema": "../../node_modules/nx/schemas/project-schema.json",
   "sourceRoot": "packages/web-e2e/src",
   "projectType": "application",
-  "tags": ["type:e2e", "platform:web"],
-  "implicitDependencies": ["web"],
+  "tags": [
+    "type:e2e",
+    "platform:web"
+  ],
+  "implicitDependencies": [
+    "web",
+    "cli"
+  ],
   "targets": {
     "test": {
-      "dependsOn": []
+      "dependsOn": [],
+      "inputs": [
+        "default",
+        "^production",
+        {
+          "externalDependencies": [
+            "vitest"
+          ]
+        },
+        {
+          "env": "CI"
+        },
+        "{workspaceRoot}/scripts/telemetry/**/*"
+      ]
     },
     "typecheck": {
       "executor": "nx:run-commands",