From a418b3870c635c62adcddc61985ea4bcb643adde Mon Sep 17 00:00:00 2001 From: Torey Scheer Date: Wed, 16 Sep 2026 12:28:33 -0400 Subject: [PATCH 01/24] ci(ui): run the live Playwright suite against the built UI image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ui-tests` proves the UI is self-consistent, not that it works: it runs on the in-browser mock, so a backend contract change cannot fail it. That is how #2638 emptied three Substrate fields with every check green. Closes #2657. The lane runs in `test-e2e`, after the Go end-to-end tests and on the same cluster, reusing what that job already stands up — including the `smoke` agent the schedules spec needs. The UI image joins the fan-out, `ui.replicas` is no longer zeroed, and the address is the `kagent-ui` service's MetalLB IP, the chart already publishing it as a LoadBalancer. `UI_LOOP_LIVE_URL` points the suite at an app it did not start, so what runs is the shipped artifact: nginx, the SPA fallback, and the `env-config.js` rendered from the pod's environment at start, none of which exists in front of `yarn dev`. Two invariants worth stating, both learned the hard way: - `build-ui` and `build-golang-adk` both depend on `proto-generate`, and `buf generate` writes `ui/src/generated` as well as `go/api/gen`. Run in parallel, one make writes into the directory the other is tarring as a build context, so it runs once up front with `-o proto-generate` on the fan-out. - A green live run has to have been live. Against the image there is no build-time `VITE_API_MODE` pin, so `globalSetup` asserts `/mockServiceWorker.js` 404s — which says both that fixtures cannot be served and that this is the built artifact rather than a dev server that would pass every spec. Turning the lane on meant fixing the suite it runs, because nothing had run it and every spec had rotted. `agent-lifecycle.spec.ts` drove `/agents/new`, a page removed long before, for an agent nobody creates — replaced by `agent-templates.spec.ts`, which asserts the property only a cluster can settle: admission is read from the template's status and cannot be computed in the browser. `schedules.spec.ts` drove a dialog for an editor that is a page, and named CI's fixture rather than `setup-cluster.sh`'s, so it could only pass where nothing ran it. `substrate.spec.ts` read a tile once, catching the em-dash drawn before the data lands. Two harness faults came out of the same run: `selectFirstOption` could resolve an option from a dropdown that had just been dismissed, and the live project now runs `workers: 1`, these specs sharing one cluster where every mock test owns a backend in its own page's memory. Verified against a Kind cluster running the image built from this checkout: 7 live specs pass, and the mock suite is unchanged at 159. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Torey Scheer --- .github/workflows/ci.yaml | 88 +++++++++- ui/playwright.config.ts | 88 ++++++---- ui/playwright/DEFERRED.md | 26 ++- ui/playwright/README.md | 71 ++++++-- ui/playwright/globalSetup.ts | 24 ++- ui/playwright/helpers/app.ts | 2 - ui/playwright/helpers/resource.ts | 25 ++- ui/playwright/live/agent-lifecycle.spec.ts | 108 ------------ ui/playwright/live/agent-templates.spec.ts | 181 ++++++++++++++++++++ ui/playwright/live/helpers/live.ts | 14 +- ui/playwright/live/pages.spec.ts | 2 +- ui/playwright/live/schedules.spec.ts | 186 +++++++++++++++------ ui/playwright/live/substrate.spec.ts | 17 +- 13 files changed, 605 insertions(+), 227 deletions(-) delete mode 100644 ui/playwright/live/agent-lifecycle.spec.ts create mode 100644 ui/playwright/live/agent-templates.spec.ts diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 07dd4ac054..cd654654ba 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -44,7 +44,9 @@ jobs: VERSION: v0.0.1-test SUBSTRATE_VERSION: 0.0.29 runs-on: blacksmith-4vcpu-ubuntu-2404 - timeout-minutes: 30 + # 45, not 30: runs took 15-18 minutes, and the UI image and browser suite below add + # roughly ten more — close enough that a slow day would have read as a hang. + timeout-minutes: 45 steps: - name: Checkout repository uses: actions/checkout@v6 @@ -112,7 +114,6 @@ jobs: KMCP_ENABLED: "false" KAGENT_HELM_EXTRA_ARGS: >- --cleanup-on-fail=false - --set ui.replicas=0 --set kagent-tools.enabled=false --set grafana-mcp.enabled=false --set controller.substrate.enabled=true @@ -125,13 +126,20 @@ jobs: run: | echo "Cache key: ${{ needs.setup.outputs.cache-key }}" # Once, before the fan-out below. Every `build-*` target depends on - # `buildx-create`, so four of them starting at the same instant on a runner + # `buildx-create`, so several of them starting at the same instant on a runner # with no builder yet all try to create it and all but one fail. make buildx-create - printf '%s\n' controller golang-adk claude-harness codex-harness byo-a2a | xargs -P4 -n1 bash -c ' + # Likewise once, and for a sharper reason: `build-ui` and `build-golang-adk` both + # depend on it, and `buf generate` writes `ui/src/generated` as well as + # `go/api/gen` — so in parallel one make would write into the directory the + # other is tarring as a build context. `-o proto-generate` below says it is done. + make proto-generate + # `ui` is here so the browser suite below runs against the image an operator + # gets: nginx, the SPA fallback, and the env-config.js rendered at pod start. + printf '%s\n' controller ui golang-adk claude-harness codex-harness byo-a2a | xargs -P5 -n1 bash -c ' image="$1" DOCKER_BUILD_ARGS="--cache-from=type=gha,scope=${{ needs.setup.outputs.cache-key }}-e2e-${image} --cache-from=type=gha,scope=${{ env.CACHE_KEY_PREFIX }}-main-e2e-${image} --cache-to=type=gha,scope=${{ needs.setup.outputs.cache-key }}-e2e-${image},mode=max --platform=linux/amd64 --push" \ - make GIT_COMMIT=e2e BUILD_DATE=1970-01-01 "build-${image}" + make -o proto-generate GIT_COMMIT=e2e BUILD_DATE=1970-01-01 "build-${image}" ' _ make helm-install-provider kubectl rollout status deployment/kagent-controller -n kagent --timeout=120s @@ -167,6 +175,72 @@ jobs: export KAGENT_E2E_CLI="$PWD/core/bin/kagent-local" go test -v github.com/kagent-dev/kagent/go/core/test/e2e -failfast -shuffle=on -parallel 4 + # The UI, against this same cluster and the image built above. `ui-tests` runs on + # the in-browser mock, so a backend contract change cannot fail it — which is how + # #2638 emptied three Substrate fields with every check green. + # + # After the Go tests, not beside them: both create and delete resources in the same + # namespace, and these journeys read the lists back. + # + # `steps.install-kagent.outcome` rather than `success()`, so one failed Go test + # under `-failfast` does not also cost every UI signal. No cluster, no run. + - name: Setup Node.js + if: ${{ !cancelled() && steps.install-kagent.outcome == 'success' }} + uses: actions/setup-node@v7 + with: + node-version-file: ui/.nvmrc + + # Before the cache step, as in `ui-tests`: package.json pins Yarn 4, and the + # runner's own shim cannot read this lock file. + - name: Enable Corepack + if: ${{ !cancelled() && steps.install-kagent.outcome == 'success' }} + run: corepack enable + + - name: Cache Yarn downloads + if: ${{ !cancelled() && steps.install-kagent.outcome == 'success' }} + uses: actions/cache@v6 + with: + path: ui/.yarn/cache + key: yarn-${{ runner.os }}-${{ hashFiles('ui/yarn.lock') }} + restore-keys: yarn-${{ runner.os }}- + + - name: Prepare the live browser suite + if: ${{ !cancelled() && steps.install-kagent.outcome == 'success' }} + working-directory: ./ui + # Chromium only: the live suite declares one project. The mock suite's second + # engine is there to disagree about layout, which no backend contract rests on. + run: | + yarn install --immutable + yarn playwright install --with-deps chromium + + - name: Run live browser tests + if: ${{ !cancelled() && steps.install-kagent.outcome == 'success' }} + working-directory: ./ui + run: | + kubectl -n kagent rollout status deploy/kagent-ui --timeout=5m + # Already a LoadBalancer on a MetalLB cluster, so this is the address the + # controller e2e step takes — not a port-forward to keep alive for the run. + UI_IP="$(kubectl get svc -n kagent kagent-ui -o jsonpath='{.status.loadBalancer.ingress[0].ip}')" + test -n "$UI_IP" + export UI_LOOP_LIVE_URL="http://${UI_IP}:8080" + echo "UI_LOOP_LIVE_URL: $UI_LOOP_LIVE_URL" + # `globalSetup` refuses the run if that address answers with fixtures, or is a + # dev server rather than the built image. + yarn test:pw:live + + - name: Upload live browser report + if: ${{ failure() && steps.install-kagent.outcome == 'success' }} + uses: actions/upload-artifact@v5 + with: + name: ui-live-playwright-report + # The trace is the only record of what the cluster answered — there is no fixed + # fixture to re-read afterwards, as there would be for the mock suite. + path: | + ui/playwright-report + ui/test-results + retention-days: 7 + if-no-files-found: ignore + - name: fail print info if: failure() run: | @@ -179,6 +253,10 @@ jobs: kubectl get harnesses,agenttemplates -n kagent echo "::error::Kubectl logs -n kagent deployment/kagent-controller" kubectl logs -n kagent deployment/kagent-controller + # nginx logs every proxied request, so a browser failure that was really a + # backend failure says so here, as does an init.sh that rejected a value. + echo "::error::Kubectl logs -n kagent deployment/kagent-ui" + kubectl logs -n kagent deployment/kagent-ui --tail=200 || true kubectl get pods,actortemplates,workerpools -A echo "::error::Substrate logs" kubectl logs -n ate-system -l app --all-containers --prefix --tail=200 || true diff --git a/ui/playwright.config.ts b/ui/playwright.config.ts index ece9ddc181..ae749e9717 100644 --- a/ui/playwright.config.ts +++ b/ui/playwright.config.ts @@ -76,31 +76,61 @@ const LIVE = process.env.UI_LOOP_LIVE === "true"; * server is about to want, or vice versa. */ const LIVE_PORT = Number(process.env.UI_LOOP_LIVE_PORT ?? 8301); -const LIVE_BASE_URL = `http://localhost:${LIVE_PORT}`; + +/** + * An app already running — in CI, the `kagent-ui` service — tested instead of one + * this config starts. That is the deployed image, so nginx, the SPA fallback and the + * `env-config.js` rendered at pod start are under test rather than approximated by + * Vite. A developer still gets the dev server by default; the alternative is building + * an image to run a test. + */ +const LIVE_EXTERNAL_URL = process.env.UI_LOOP_LIVE_URL; + +const LIVE_BASE_URL = LIVE_EXTERNAL_URL ?? `http://localhost:${LIVE_PORT}`; /** Read by `playwright/globalSetup.ts` to decide what to verify about a server. */ export const LIVE_PROJECT = "chromium-live"; /** - * A live run reaches the backend through Vite's proxy, exactly as a deployed - * build reaches it through nginx — so the app uses the same relative URLs either - * way and this mode tests the addressing a real deployment uses. - * - * `VITE_API_MODE` is pinned as well as the runtime flag: the build-time pin is - * the one thing an inherited `.env` cannot override, and a live suite that - * silently answered from fixtures would be worse than a red one. + * Whether this live run is against a deployed app rather than a dev server, which + * `globalSetup` checks one more thing for — see `verifyLiveWiring`. */ -const LIVE_APP = { VITE_API_MODE: "live", ENABLE_MOCK_UI: "false" }; +export const LIVE_IS_DEPLOYED = LIVE_EXTERNAL_URL !== undefined; /** - * How the live server is started. - * - * Named for the same reason the three env pins above are: a branch whose backend - * needs more than a dev server — a credential minted per run, a port-forward - * probed before Vite starts — replaces this line rather than the block below. + * How the *dev server* is configured for a live run; a deployment is configured by its + * chart instead. It proxies `/api` the way nginx does in a cluster, so the app uses the + * same relative URLs either way — standing in for nginx rather than being it, which is + * the gap `UI_LOOP_LIVE_URL` closes. `VITE_API_MODE` is pinned at build time as well as + * at runtime, being the one thing an inherited `.env` cannot override. */ +const LIVE_APP = { VITE_API_MODE: "live", ENABLE_MOCK_UI: "false" }; + +/** How the live server is started. Unused when `UI_LOOP_LIVE_URL` names one already. */ const LIVE_COMMAND = `yarn dev --port ${LIVE_PORT}`; +/** + * The servers a live run starts, which is none when it was handed one: there is no + * process to own — the app is a pod. `globalSetup` checks the address serves the app. + */ +const LIVE_WEB_SERVERS = LIVE_EXTERNAL_URL + ? [] + : [ + { + command: LIVE_COMMAND, + url: LIVE_BASE_URL, + reuseExistingServer: false, + timeout: 120_000, + // Whatever starts the live server is the most useful output a failed + // live run has — something that cannot reach the backend says so there, + // and Playwright discards a web server's stdout unless asked to pass it + // through. + stdout: "pipe" as const, + stderr: "pipe" as const, + env: LIVE_APP, + }, + ]; + export default defineConfig({ testDir: "./playwright/tests", // Both servers have to be rendering, not merely listening, before any test @@ -143,7 +173,19 @@ export default defineConfig({ * and a mock-backed suite that needs more than thirty seconds for one test is saying * something is stuck, which is worth hearing rather than absorbing. */ - ...(LIVE ? { timeout: 120_000, expect: { timeout: 30_000 } } : {}), + ...(LIVE + ? { + timeout: 120_000, + expect: { timeout: 30_000 }, + /* + * One at a time. Every mock test owns a backend in its own page's memory; these + * share a cluster, so a spec creating a resource while another counts them is a + * failure with no defect behind it. Four files, seconds to run — it costs little. + */ + workers: 1, + fullyParallel: false, + } + : {}), use: { trace: "on-first-retry", screenshot: "only-on-failure", @@ -200,21 +242,7 @@ export default defineConfig({ // loud startup error instead; set UI_LOOP_PORT / UI_LOOP_EXTENSION_PORT to run // alongside a dev server you want to keep. webServer: LIVE - ? [ - { - command: LIVE_COMMAND, - url: LIVE_BASE_URL, - reuseExistingServer: false, - timeout: 120_000, - // Whatever starts the live server is the most useful output a failed - // live run has — something that cannot reach the backend says so there, - // and Playwright discards a web server's stdout unless asked to pass it - // through. - stdout: "pipe", - stderr: "pipe", - env: LIVE_APP, - }, - ] + ? LIVE_WEB_SERVERS : [ { command: `yarn dev --port ${PORT}`, diff --git a/ui/playwright/DEFERRED.md b/ui/playwright/DEFERRED.md index 4d8a758be7..de443b3297 100644 --- a/ui/playwright/DEFERRED.md +++ b/ui/playwright/DEFERRED.md @@ -15,7 +15,31 @@ missing is the page. | Old spec | Blocked on | Already available | |---|---|---| | `onboarding/onboarding.spec.ts` | No onboarding wizard exists on this architecture | — nothing; drop it unless the flow is rebuilt | -| `cleanup.spec.ts` | Not applicable while the suite runs on the mock backend: each test gets a fresh browser context, so there is nothing to sweep. Revisit if the suite gains a live-backend mode. | — | +| `cleanup.spec.ts` | Still not a spec. The mock suite gives each test a fresh browser context, so there is nothing to sweep; the live suite creates real resources and each spec removes what it made in a `finally`. What neither covers is a run *killed* between the two — see below. | — | + +## Now running in CI: the live suite + +`playwright/live/` runs in the `test-e2e` job, against the image built from +`ui/Dockerfile` on the same cluster as the Go end-to-end tests. `README.md` has the +wiring. Two things follow for this file. + +**The cleanup entry above is now about litter, not contexts.** A live spec deletes what +it created; a run killed between the two cannot. That is why `throwawayName` puts the +process and a timestamp in every name — anything matching `e2e-live-*` in `kagent` is +litter and safe to remove. A sweep spec stays the wrong shape for it, being one bad +selector away from deleting somebody's work. + +**Every live spec was broken the first time one was run against a cluster**, each in a +way the mock suite structurally could not show: `agent-lifecycle.spec.ts` drove +`/agents/new`, a page long removed, for an agent nobody creates; `schedules.spec.ts` +drove a dialog for an editor that is a page, and named CI's fixture rather than +`setup-cluster.sh`'s; `substrate.spec.ts` read a tile once, catching the em-dash it +draws before the data lands. Two harness faults came out of the same run and were fixed +in `helpers/resource.ts`. + +The specs were written, they were correct, nothing ran them, and they decayed into files +testing pages that had been removed. A stale entry costs more than no entry; a stale +*spec* costs more again. ## Ported since: chat diff --git a/ui/playwright/README.md b/ui/playwright/README.md index 42cf3794d8..d5dbb9675a 100644 --- a/ui/playwright/README.md +++ b/ui/playwright/README.md @@ -13,8 +13,9 @@ UI_LOOP_PORT=8012 yarn test:pw # when something else owns the default port Nothing else is needed — no cluster, no port-forward, no provider key. -There is a second suite that does need all three; see -[Live runs](#live-runs-against-a-real-backend) at the foot of this file. +There is a second suite that does need a cluster, and that CI runs against the +built UI image; see [Live runs](#live-runs-against-a-real-backend) at the foot of +this file. ## What changed from the old suite @@ -32,8 +33,8 @@ served. Worth stating plainly because it is a change in how contributors work, and because it is a trade: the suite no longer exercises the real controller, so it proves the UI behaves, not that the backend contract still holds. Contract drift -is caught by the Go tests and by whatever runs against a live cluster in CI — not -here. +is caught by the Go tests and by the live suite, which CI runs on the same cluster +as the Go end-to-end tests — not here. ## What it runs against @@ -212,15 +213,40 @@ conventions below: the shared fixture import, and antd's class names. cd ui yarn test:pw:live UI_LOOP_LIVE_PORT=8312 yarn test:pw:live # to run beside something on 8301 + +# Against an app that is already serving, rather than a dev server this starts: +UI_LOOP_LIVE_URL=http://127.0.0.1:8080 yarn test:pw:live ``` -Unlike `yarn test:pw`, this one **does** need a cluster, with the controller -port-forwarded. It is not run in CI. The specs live in `playwright/live/`, and the -coverage deliberately left out of it is in `DEFERRED.md`. +Unlike `yarn test:pw`, this one **does** need a cluster. The specs live in +`playwright/live/`, and the coverage deliberately left out of it is in +`DEFERRED.md`. + +**Two things can be at the other end.** Without `UI_LOOP_LIVE_URL` the suite starts +`yarn dev` and proxies to the controller, which needs nothing built and is what a +developer wants. With it, it drives an app already serving — in CI the image from +`ui/Dockerfile`, where nginx, the SPA fallback and the `env-config.js` rendered at pod +start are real rather than approximated by Vite. + +### In CI + +The `test-e2e` job runs it, after the Go end-to-end tests and on the same cluster, +reusing what that job stands up rather than building a second path — including the +`smoke` agent from `lifecycle.yaml.tmpl`. The address is the `kagent-ui` service's +MetalLB IP, the chart already publishing it as a LoadBalancer. + +After the Go tests, not beside them, and at `workers: 1`: these journeys create real +resources and read lists back, where `go test -parallel 4` — and each other — would be +writing to the same namespace at the same time. -A live run reaches the controller through Vite's proxy, exactly as a deployed -build reaches it through nginx, so the app uses the same relative URLs either way -and this mode tests the addressing a real deployment uses. +**A live spec has to work on both shapes of cluster.** `setup-cluster.sh` installs one +harness and an `assistant` agent; CI's fixture installs five and a `smoke`. Not +cosmetic: the template form applies a harness's labels unasked when there is only one. +So a spec takes whatever the cluster offers and reads the state it is in. + +**Chat stays on the mock backend.** Its journeys need deterministic streaming deltas, +tool ordering, cancellation and a failed turn with retry — none of which a real model +gives reliably, all of which the mock suite already asserts. **Why a separate mode rather than a third project.** `UI_LOOP_LIVE=true` swaps the whole `projects`/`webServer` pair in `playwright.config.ts` instead of appending to @@ -232,11 +258,20 @@ would cost every live run the time to boot Vite twice for nothing. The two runs are disjoint. The live project also gets its own port, 8301, far from the mock servers' 8001/8051 for the same reason those two are 50 apart. -**A green live run has to have been live.** `VITE_API_MODE` is pinned at build -time as well as at runtime, because a build-time pin is the one thing an inherited -`.env` cannot override — and a live suite that quietly answered from fixtures -would be worse than a red one, since a green one gets taken as evidence the -cluster works. `globalSetup` asks the page what settings it was actually handed -and refuses the run if they are not the live ones. Traces are kept on failure: -unlike the mock suite there is no fixed fixture to re-read afterwards, so the -trace is the only record of what the cluster answered. +**A green live run has to have been live.** A live suite that quietly answered from +fixtures would be worse than a red one, since a green one gets taken as evidence +the cluster works — so `globalSetup` asks the page what settings it was actually +handed, and refuses the run if they are not the live ones. + +The guarantee is made twice, the two ends needing different arguments: + +- **Against the dev server,** `VITE_API_MODE` is pinned at build time as well as at + runtime — the one thing an inherited `.env` cannot override. +- **Against a deployment** there is no such pin, but the mock backend is not in the + image at all: the build deletes `dist/mockServiceWorker.js` and nginx 404s the path. + `globalSetup` asserts that 404, which says both that fixtures cannot be served and + that this is the built artifact rather than a `yarn dev` that would pass every spec. + +Traces are kept on failure — there is no fixed fixture to re-read afterwards, so the +trace is the only record of what the cluster answered. CI uploads them as +`ui-live-playwright-report`. diff --git a/ui/playwright/globalSetup.ts b/ui/playwright/globalSetup.ts index 0d5b103e20..0ba0a79d1c 100644 --- a/ui/playwright/globalSetup.ts +++ b/ui/playwright/globalSetup.ts @@ -1,5 +1,5 @@ import { chromium, type FullConfig, type Page } from "@playwright/test"; -import { LIVE_PROJECT } from "../playwright.config"; +import { LIVE_IS_DEPLOYED, LIVE_PROJECT } from "../playwright.config"; /** * Waits until each server actually renders the app, not merely answers on its @@ -121,4 +121,26 @@ async function verifyLiveWiring(page: Page, baseUrl: string): Promise { `is a service worker, so a live run cannot be trusted while one is installed.`, ); } + + if (!LIVE_IS_DEPLOYED) return; + + /* + * The third way, and one only a deployment can be asked. The checks above lean on a + * build-time `VITE_API_MODE` pin, which the image has not got — what it has instead is + * no mock backend in it at all. So a 404 says both that fixtures cannot be served and + * that this is the built artifact rather than a dev server, which answers 200 and + * would pass every spec while testing none of the things the image is the point of. + */ + const worker = await page.request.get(`${baseUrl}/mockServiceWorker.js`, { + failOnStatusCode: false, + }); + if (worker.status() !== 404) { + throw new Error( + `${baseUrl}/mockServiceWorker.js answered ${worker.status()}, expected 404. ` + + `A built image does not contain that file and its nginx refuses the path, so ` + + `this address is serving something else — most likely a dev server, which ` + + `would pass these specs without testing nginx, the SPA fallback, or the ` + + `env-config.js rendered at pod start.`, + ); + } } diff --git a/ui/playwright/helpers/app.ts b/ui/playwright/helpers/app.ts index 7f983645d8..a6e7487288 100644 --- a/ui/playwright/helpers/app.ts +++ b/ui/playwright/helpers/app.ts @@ -13,14 +13,12 @@ export const routes = { dashboard: "/", login: "/login", agents: "/agents", - agentNew: "/agents/new", models: "/models", modelNew: "/models/new", mcpServers: "/mcp", mcpServerNew: "/mcp/new", prompts: "/prompts", promptNew: "/prompts/new", - snapshots: "/snapshots", substrate: "/substrate", /* The templates list is a tab of the agents page now. The old address still resolves — it redirects here — but a test should go where the reader goes. */ diff --git a/ui/playwright/helpers/resource.ts b/ui/playwright/helpers/resource.ts index 35f3e2f261..e40cc689f7 100644 --- a/ui/playwright/helpers/resource.ts +++ b/ui/playwright/helpers/resource.ts @@ -161,6 +161,15 @@ export function optionNamed(page: Page, label?: string): Locator { ); } +/** + * The dropdown on screen, whichever select opened it. By Playwright's own visibility + * rather than antd's `ant-select-dropdown-hidden`, which a dismissed dropdown does not + * carry until its close animation ends — lagging at exactly the wrong moment. + */ +function openDropdown(page: Page): Locator { + return page.locator(".ant-select-dropdown").filter({ visible: true }); +} + /** Opens a Select by its test id and picks one option by the label a reader sees. */ export async function selectOption( page: Page, @@ -172,12 +181,22 @@ export async function selectOption( } /** - * The same, where any option will do — a field the form requires but the assertion - * does not care about, like the namespace on a harness whose step is about the image. + * The same, where any option will do — a required field the assertion does not care + * about, or a model on a cluster whose models the spec did not install. + * + * It cannot name what it wants, so it has to be sure *which* dropdown it reads: it + * waits out any still animating away, then scopes to the one on screen. A live run + * spent its whole budget clicking an `ate-system` option from a dismissed select. */ export async function selectFirstOption(page: Page, testId: string): Promise { + await expect(openDropdown(page)).toHaveCount(0); await page.getByTestId(testId).click(); - await optionNamed(page).first().click(); + + const option = openDropdown(page).locator(".ant-select-item-option").first(); + await expect(option, `the select "${testId}" offered no options`).toBeVisible({ + timeout: 30_000, + }); + await option.click(); } /** diff --git a/ui/playwright/live/agent-lifecycle.spec.ts b/ui/playwright/live/agent-lifecycle.spec.ts deleted file mode 100644 index 45038048ab..0000000000 --- a/ui/playwright/live/agent-lifecycle.spec.ts +++ /dev/null @@ -1,108 +0,0 @@ -import { test, expect } from "@playwright/test"; -import { - dataRows, - expectNoLoadFailure, - liveRoutes, - loadLive, - rowNamed, - throwawayName, -} from "./helpers/live"; - -/** - * Creating and deleting an agent, on a real cluster, through the UI. - * - * This is the journey the mock suite cannot vouch for. Both halves of it were - * broken against a real controller while the mock suite was green: the form sent - * the model as a `namespace/name` ref where the controller wanted a bare name, so - * every create was rejected; and the page read the created agent out of a wrapper - * the create response does not have, so a create that *had* worked reported failure - * and stayed on the form. Neither could be seen without a cluster. - * - * The agent is deleted in teardown as well as in the spec body, because a run that - * dies midway would otherwise leave a real resource behind. - */ - -const AGENT = throwawayName("agent"); -const NAMESPACE = "kagent"; - -test.afterEach(async ({ request, baseURL }) => { - // Deleting through the API rather than the UI: teardown runs after a failure, when - // the page may be anywhere at all, and a cleanup that depends on the UI working is - // exactly the cleanup that fails when the UI does not. - const response = await request.delete( - `${baseURL}/api/agents/${NAMESPACE}/${AGENT}`, - { failOnStatusCode: false }, - ); - // 404 is the goal state: either the spec deleted it, or it was never created. - expect( - [200, 204, 404], - `teardown could not remove ${NAMESPACE}/${AGENT} (${response.status()})`, - ).toContain(response.status()); -}); - -test("live: an agent can be created and deleted through the UI", async ({ page }) => { - await test.step("1. the create form opens with the cluster's models offered", async () => { - await loadLive(page, liveRoutes.agentNew); - await expectNoLoadFailure(page); - - await page.getByTestId("agent-form-name").fill(AGENT); - await page.getByTestId("agent-form-namespace").fill(NAMESPACE); - await page - .getByTestId("agent-form-description") - .fill("Created by the live end-to-end suite; safe to delete."); - // Required, and the form says so rather than letting a create fail at the API — - // which is the client-side validation `playwright/DEFERRED.md` records as lost - // coverage. It is not lost; this spec relies on it. - await page - .getByTestId("agent-form-system-message") - .fill("You are a test agent created by an end-to-end run. Answer briefly."); - - // The options come from the cluster's own ModelConfigs, so an empty list here is - // a real failure rather than a slow render: the install ships one. - await page.getByTestId("agent-form-model").click(); - const option = page.locator(".ant-select-item-option").first(); - await expect(option, "the cluster offered no model configurations").toBeVisible({ - timeout: 30_000, - }); - await option.click(); - }); - - await test.step("2. submitting reaches the controller and reports success", async () => { - await page.getByTestId("agent-form-submit").click(); - - // Success is leaving the form. A create that failed keeps the user on it with an - // error, which is the shape the earlier defect produced for an agent that had in - // fact been created. - await expect(page).toHaveURL(/\/agents$/, { timeout: 60_000 }); - }); - - await test.step("3. the agent is listed, read back from the cluster", async () => { - await expectNoLoadFailure(page); - await expect(rowNamed(page, AGENT)).toHaveCount(1, { timeout: 60_000 }); - }); - - await test.step("4. deleting it asks first, and names what it will delete", async () => { - const before = await dataRows(page).count(); - - await page.getByTestId(`delete-${AGENT}`).click(); - const confirm = page.locator(".ant-popconfirm").filter({ hasText: AGENT }); - await expect(confirm, "the confirmation did not name the agent").toBeVisible(); - - await page.getByRole("button", { name: "Delete", exact: true }).last().click(); - - await test.step("and the row is gone from the list a reader looks at", async () => { - await expect(rowNamed(page, AGENT)).toHaveCount(0, { timeout: 60_000 }); - await expect(dataRows(page)).toHaveCount(before - 1); - }); - }); - - await test.step("5. the cluster agrees it is gone", async () => { - // The list could be stale; the controller cannot be. This is what makes the - // previous step evidence of a delete rather than of a re-render. - const response = await page.request.get( - `/api/agents/${NAMESPACE}/${AGENT}`, - { failOnStatusCode: false }, - ); - expect(response.status()).toBe(404); - }); -}); diff --git a/ui/playwright/live/agent-templates.spec.ts b/ui/playwright/live/agent-templates.spec.ts new file mode 100644 index 0000000000..a730fe975c --- /dev/null +++ b/ui/playwright/live/agent-templates.spec.ts @@ -0,0 +1,181 @@ +import { test, expect } from "../fixtures/test"; +import { + confirmation, + pressOnce, + selectFirstOption, + selectOption, +} from "../helpers/resource"; +import { + dataRows, + expectNoLoadFailure, + liveRoutes, + loadLive, + rowNamed, + throwawayName, +} from "./helpers/live"; + +/** + * Creating and deleting an agent template, on a real cluster, through the UI. + * + * The property it exists for: **admission is the controller's answer, not the form's.** + * `admittingHarnesses` is read from the template's *status* and cannot be computed in + * the browser, so a fixture can return any value it likes and the page will draw it. + * + * It replaces `agent-lifecycle.spec.ts`, which drove `/agents/new` — a page removed + * long before, for an agent nobody creates. Nothing ran the suite, so nothing said so. + */ + +/** The one this journey makes and removes. Carries the run, for litter left by a kill. */ +const TEMPLATE = throwawayName("template"); +const NAMESPACE = "kagent"; + +test("live: an agent template is created, admitted and deleted through the UI", async ({ + page, +}) => { + let created = false; + /** Which harness this install offered, read off the button in step 2. */ + let harness = ""; + + try { + await test.step("1. the form offers the cluster's own model configurations", async () => { + await loadLive(page, liveRoutes.agentTemplateNew); + await expectNoLoadFailure(page); + + await selectOption(page, "template-form-namespace", NAMESPACE); + await page.getByTestId("template-form-name").fill(TEMPLATE); + + // `spec.modelConfig` is the one field the CRD requires, and the options are the + // cluster's own ModelConfigs. Whichever one this install ships, rather than a + // name — the assertion is that the cluster answered, not which model it named, + // and `selectFirstOption` fails with that message if the list is empty. + await selectFirstOption(page, "template-form-model"); + }); + + await test.step("2. and the cluster's own harnesses, one of which makes it usable", async () => { + /* + * Two states, and which one appears is a fact about the cluster. With one harness + * the form applies its labels unasked, there being no decision to make; with + * several it warns until told. A `setup-cluster` cluster has one, CI's fixture + * five, so a spec that knew only the second would fail on every laptop. + */ + const admission = page.getByTestId("template-form-admission"); + const buttons = page.locator('[data-testid^="template-form-admit-"]'); + await expect(buttons.first(), "the cluster offered no harnesses").toBeVisible({ + timeout: 30_000, + }); + const offered = (await buttons.allTextContents()) + .map((name) => name.trim()) + .filter(Boolean); + + const admissionText = (await admission.textContent()) ?? ""; + harness = + offered.find((name) => admissionText.includes(`admitted by ${name}`)) ?? ""; + + if (harness === "") { + await expect(admission).toContainText("No harness will run this template"); + + // Enabled ones only: the form disables a harness whose selector is empty, since + // such a harness admits nothing and has no labels to copy. Clicking one would + // spend the step's budget on a button that was never going to answer. + const admit = page + .locator('[data-testid^="template-form-admit-"]:not([disabled])') + .first(); + await expect(admit, "no harness on the cluster admits anything").toBeVisible(); + harness = ((await admit.textContent()) ?? "").trim(); + // The button applies whatever labels that harness's selector matches on, which + // is the step a reader is most likely to miss and the one that makes the + // template mean anything. + await admit.click(); + } + + expect(harness, "no harness name could be read from the form").not.toBe(""); + await expect(admission).toContainText(`admitted by ${harness}`); + }); + + await test.step("3. submitting reaches the controller and lands on the list", async () => { + await expect(page.getByTestId("template-submit")).toBeEnabled(); + await page.getByTestId("template-submit").click(); + + // Success is leaving the form. A create the controller refused keeps the reader on + // it with `template-create-error` — which is the shape the defect this suite was + // written for produced for a template that had in fact been created. + await page.waitForURL(/\/agents\?.*tab=templates/, { timeout: 60_000 }); + created = true; + await expect(page).toHaveURL(new RegExp(`[?&]ns=${NAMESPACE}(&|$)`)); + }); + + await test.step("4. the row is read back from the cluster", async () => { + await expectNoLoadFailure(page); + await expect(rowNamed(page, TEMPLATE)).toHaveCount(1, { timeout: 60_000 }); + /* + * Deliberately not asserting the harness here. The row carries the namespace too, + * and on every cluster this runs against both are `kagent` — so the assertion + * would pass on the namespace whatever the admission column said. It belongs on an + * element holding the harness and nothing else, which is the next step. + */ + }); + + await test.step("5. the controller agrees a harness admits it", async () => { + await page.getByTestId(`template-link-${TEMPLATE}`).click(); + await page.waitForURL( + new RegExp(`/agent-templates/${NAMESPACE}/${TEMPLATE}`), + { timeout: 60_000 }, + ); + // The claim this journey exists for, on an element that holds "Runs on" and the + // admitting harnesses and nothing else. `admittingHarnesses` comes from the + // template's *status*, so the harness appearing here means the controller observed + // the labels the form applied and agreed — not that the form echoed itself back. + await expect(page.getByTestId("template-admission-status")).toContainText(harness, { + timeout: 60_000, + }); + await expect(page.getByTestId("template-admission-status")).not.toContainText( + "No harness", + ); + }); + + await test.step("6. deleting says what it costs, against the cluster's own count", async () => { + const remove = page.getByTestId(`delete-${TEMPLATE}`); + await expect(remove).toContainText("Delete template"); + await remove.click(); + + /* + * Either branch is legitimate: the count comes from `status.harnesses` while step 5 + * waited on `admittingHarnesses`, two fields filled by different work. The mock + * suite pins the wording; what a cluster shows is that the sentence is computed + * from real state at all, rather than coming back blank. + */ + await expect(page.getByTestId("template-delete-consequence")).toContainText( + /built from this template|no agent was ever built from it/, + { timeout: 30_000 }, + ); + }); + + await test.step("7. confirming removes it, and the re-read list agrees", async () => { + // Scoped to the visible popconfirm, and pressed once it has stopped arriving — + // see `helpers/resource` for what each of those is protecting against. + await pressOnce(confirmation(page).getByRole("button", { name: "Delete" })); + await page.waitForURL(/\/agents\?.*tab=templates/, { timeout: 60_000 }); + created = false; + + await expectNoLoadFailure(page); + // The rest of the list is still there — the cluster installs templates of its own + // — so "gone" names that one template rather than a read that returned nothing. + // That distinction is the whole reason `expectNoLoadFailure` exists, and an empty + // table is exactly how a failed list would look. + await expect(dataRows(page).first()).toBeVisible({ timeout: 60_000 }); + await expect(rowNamed(page, TEMPLATE)).toHaveCount(0, { timeout: 60_000 }); + }); + } finally { + /* + * A real resource on a real cluster, so a run that dies midway takes it with it. + * Through the UI because the app speaks gRPC-Web — there is no REST endpoint to + * call, though the previous version of this file believed there was. + */ + if (created) { + await page.goto(`/agent-templates/${NAMESPACE}/${TEMPLATE}`); + await page.getByTestId(`delete-${TEMPLATE}`).click(); + await pressOnce(confirmation(page).getByRole("button", { name: "Delete" })); + await page.waitForURL(/\/agents\?.*tab=templates/, { timeout: 60_000 }); + } + } +}); diff --git a/ui/playwright/live/helpers/live.ts b/ui/playwright/live/helpers/live.ts index c7abe1c2e6..897fc79bb0 100644 --- a/ui/playwright/live/helpers/live.ts +++ b/ui/playwright/live/helpers/live.ts @@ -9,14 +9,24 @@ import { expect, type Page } from "@playwright/test"; * are the parts of that worth sharing. */ -/** Where each page lives, so a renamed route breaks in one place. */ +/** + * Where each page lives, so a renamed route breaks in one place. Mirrors + * `src/router/routes.ts` rather than importing it, like the mock suite's table: a spec + * that reads the app's own constant follows a rename silently. The copy can rot instead + * — this one carried `/agents/new` for a form that had been deleted, and `agentDetail` + * (`/agents/:id`) swallowed the address so it was not even a 404. + */ export const liveRoutes = { dashboard: "/", agents: "/agents", - agentNew: "/agents/new", + /* A tab of the agents page. `/agent-templates` still redirects here, but a spec + should go where the reader goes. */ + agentTemplates: "/agents?tab=templates", + agentTemplateNew: "/agent-templates/new", models: "/models", mcpServers: "/mcp", prompts: "/prompts", + schedules: "/schedules", substrate: "/substrate", } as const; diff --git a/ui/playwright/live/pages.spec.ts b/ui/playwright/live/pages.spec.ts index 51d951ee02..6b89089a64 100644 --- a/ui/playwright/live/pages.spec.ts +++ b/ui/playwright/live/pages.spec.ts @@ -25,7 +25,7 @@ test("live: every page loads against the cluster and reports no failure", async page, }) => { for (const [name, path] of Object.entries(liveRoutes)) { - if (name === "agentNew") continue; // A form, covered by the lifecycle spec. + if (name === "agentTemplateNew") continue; // A form, covered by the lifecycle spec. await test.step(`${name} (${path})`, async () => { await loadLive(page, path); diff --git a/ui/playwright/live/schedules.spec.ts b/ui/playwright/live/schedules.spec.ts index 6b5dcc50aa..2cf24bfab6 100644 --- a/ui/playwright/live/schedules.spec.ts +++ b/ui/playwright/live/schedules.spec.ts @@ -1,60 +1,150 @@ import { test, expect } from "../fixtures/test"; -import { loadLive, throwawayName } from "./helpers/live"; +import { liveRoutes, loadLive, throwawayName } from "./helpers/live"; import { tick } from "../helpers/controls"; +import { optionNamed, pressUntil } from "../helpers/resource"; -// Requires the lifecycle fixture's ready kagent/smoke agent. Keep it paused: -// runtime execution is covered by the Go scheduling E2Es with a controlled model. -test("live: schedule configuration persists through the browser and controller", async ({ page }) => { +/** + * A schedule's configuration, round-tripped through a real controller. Kept paused — + * execution is covered by the Go scheduling E2Es with a controlled model. + * + * The values are the ones a backend drops or rounds: a fractional timeout, a named IANA + * zone, a weekday set. A fixture hands back whatever it was given, so the mock suite + * cannot fail on a controller that truncates `90.001` to `90`; this can. + * + * It also drove `getByRole("dialog")` for an editor that is a page, and named CI's + * `smoke` agent rather than `setup-cluster.sh`'s — so it could only pass where nothing + * ran it. Both found the first time it was pointed at a cluster. + */ +test("live: schedule configuration persists through the browser and controller", async ({ + page, +}) => { const name = throwawayName("schedule"); let detailURL: string | undefined; + try { - await loadLive(page, "/schedules"); - await page.getByRole("button", { name: "New Schedule", exact: true }).click(); - const editor = page.getByRole("dialog"); - await editor.getByLabel("Agent", { exact: true }).click(); - await page.getByTitle("kagent/smoke on kagent", { exact: true }).click(); - await editor.getByLabel("Schedule Name", { exact: true }).fill(name); - await editor.getByLabel("Prompt", { exact: true }).fill("Report cluster health."); - await editor.getByLabel("Repeat", { exact: true }).click(); - await page.getByTitle("Weekly", { exact: true }).click(); - for (const day of ["Tuesday", "Wednesday", "Thursday", "Friday"]) { - await tick(editor.getByLabel(day, { exact: true })); - } - await editor.getByLabel("At time", { exact: true }).fill("09:00"); - await editor.getByLabel("Time zone", { exact: true }).fill("America/New_York"); - await editor.getByLabel("Execution timeout (seconds)", { exact: true }).fill("90.001"); - await editor.getByLabel("Enable Schedule", { exact: true }).uncheck(); - await editor.getByRole("button", { name: "Create schedule", exact: true }).click(); - await expect(page.getByRole("heading", { name, exact: true })).toBeVisible(); - detailURL = page.url(); - await page.reload(); - await expect(page.getByRole("button", { name: "Resume", exact: true })).toBeEnabled(); - await expect(page.getByText("Weekdays at 09:00", { exact: true })).toBeVisible(); - await expect(page.getByText("90.001 seconds", { exact: true })).toBeVisible(); - await expect(page.getByText("America/New_York", { exact: true })).toBeVisible(); - await expect(page.getByText("No executions yet", { exact: true })).toBeVisible(); - - await page.getByRole("button", { name: "Edit", exact: true }).click(); - await expect(editor.getByLabel("Enable Schedule", { exact: true })).not.toBeChecked(); - await expect(editor.getByText("This schedule will not run automatically after it is saved.")).toBeVisible(); - await editor.getByLabel("Time zone", { exact: true }).fill(""); - await expect(editor.getByRole("status")).toHaveText("Weekdays at 09:00 (UTC)"); - await editor.getByLabel("Time zone", { exact: true }).fill("America/New_York"); - await editor.getByLabel("Prompt", { exact: true }).fill("Report unhealthy workloads only."); - await editor.getByRole("button", { name: "Save changes", exact: true }).click(); - await expect(editor).toBeHidden(); - await page.reload(); - await expect(page.getByText("Report unhealthy workloads only.", { exact: true })).toBeVisible(); - await expect(page.getByText("90.001 seconds", { exact: true })).toBeVisible(); + await test.step("1. the form offers the cluster's own agents", async () => { + await loadLive(page, liveRoutes.schedules); + await page.getByTestId("schedules-new").click(); + await expect(page).toHaveURL(/\/schedules\/new$/); + + await page.getByTestId("schedule-agent").click(); + // Whichever agent this install has: which one has nothing to do with the claim. + const agent = optionNamed(page).first(); + await expect(agent, "the cluster offered no agents to schedule").toBeVisible({ + timeout: 30_000, + }); + await agent.click(); + }); + + await test.step("2. a weekly, zoned, fractionally-timed schedule is described", async () => { + await page.getByTestId("schedule-name").fill(name); + + await page.getByTestId("schedule-frequency").click(); + // Pressed until the cadence actually changes: the weekday checkboxes only exist + // once the frequency is weekly, so a dropdown click swallowed by the animation + // leaves the next line waiting for controls that are never coming. + await pressUntil(optionNamed(page, "Weekly"), () => + expect(page.getByTestId("schedule-days")).toBeVisible(), + ); + // Monday is already on, so these four make it the whole working week — which the + // app states back as "Weekdays", and which is the reading asserted below. + for (const day of ["Tuesday", "Wednesday", "Thursday", "Friday"]) { + await tick(page.getByLabel(day, { exact: true })); + } + + await page.getByTestId("schedule-time").fill("09:00"); + // The time zone is an AutoComplete, so its id is on the wrapper and the caret goes + // in the input inside it. Escape dismisses the zone list, which otherwise sits + // over the fields below. + await page.getByTestId("schedule-timezone").locator("input").fill("America/New_York"); + await page.keyboard.press("Escape"); + + await page.getByTestId("schedule-prompt").fill("Report cluster health."); + await page.getByTestId("schedule-timeout").fill("90.001"); + + await page.getByTestId("schedule-enabled").uncheck(); + await expect(page.getByTestId("schedule-enabled-note")).toContainText( + "will not run automatically after it is created", + ); + }); + + await test.step("3. creating it reaches the controller and lands on its page", async () => { + await page.getByTestId("schedule-submit").click(); + await expect(page.getByRole("heading", { name, exact: true })).toBeVisible({ + timeout: 60_000, + }); + detailURL = page.url(); + await expect(page).toHaveURL(/\/schedules\/[0-9a-f-]+$/); + }); + + await test.step("4. a reload reads it back from the cluster, unchanged", async () => { + // The reload is the point. Everything above could be the form showing itself its + // own draft; only a re-read says the controller stored it. + await page.reload(); + + // Created paused, so the one control whose label flips offers to resume it. + await expect(page.getByTestId("schedule-pause")).toHaveText("Resume", { + timeout: 60_000, + }); + await expect(page.getByTestId("schedule-meta")).toContainText("Weekdays at 09:00"); + await expect(page.getByTestId("schedule-meta")).toContainText("America/New_York"); + // The fractional second survived. `90.001` is stored as seconds plus nanos, so a + // controller that kept only the seconds would read back "90 seconds" here. + await expect(page.getByTestId("schedule-detail")).toContainText("90.001 seconds"); + await expect(page.getByTestId("schedule-detail")).toContainText( + "Report cluster health.", + ); + }); + + await test.step("5. the edit form opens on the stored values, not on defaults", async () => { + await page.getByTestId("schedule-edit").click(); + await expect(page.getByTestId("schedule-time")).toHaveValue("09:00", { + timeout: 60_000, + }); + await expect(page.getByTestId("schedule-timeout")).toHaveValue("90.001"); + await expect(page.getByTestId("schedule-timezone").locator("input")).toHaveValue( + "America/New_York", + ); + await expect(page.getByTestId("schedule-enabled")).not.toBeChecked(); + // Both ends of the weekday set, so a picker that kept only the last day chosen + // would not pass on one assertion. + await expect(page.getByLabel("Monday", { exact: true })).toBeChecked(); + await expect(page.getByLabel("Friday", { exact: true })).toBeChecked(); + }); + + await test.step("6. an edit is saved and read back", async () => { + await page.getByTestId("schedule-prompt").fill("Report unhealthy workloads only."); + await page.getByTestId("schedule-submit").click(); + await expect(page.getByRole("heading", { name, exact: true })).toBeVisible({ + timeout: 60_000, + }); + + await page.reload(); + await expect(page.getByTestId("schedule-detail")).toContainText( + "Report unhealthy workloads only.", + { timeout: 60_000 }, + ); + // And the update did not quietly reset what it was not asked to change. + await expect(page.getByTestId("schedule-detail")).toContainText("90.001 seconds"); + }); } finally { + /* + * A real schedule on a real cluster, so a run that dies midway takes it with it. + * Through the UI, the app speaking gRPC-Web with no REST endpoint to call instead. + */ if (detailURL) { await page.goto(detailURL); - await page.getByRole("button", { name: `Delete schedule ${name}`, exact: true }).click(); - await page.getByRole("dialog").getByRole("button", { name: "Delete", exact: true }).click(); - await expect(page.getByText("This schedule was deleted. Its execution history is retained.")).toBeVisible(); - await expect(page.getByRole("button", { name: "Run", exact: true })).toBeDisabled(); - await page.getByRole("link", { name: "Back", exact: true }).click(); - await expect(page.getByRole("link", { name, exact: true })).toHaveCount(0); + const remove = page + .getByTestId("schedule-danger") + .getByRole("button", { name: `Delete schedule ${name}`, exact: true }); + await remove.click(); + // Pressed until it takes: a dropped Delete reports as "the page never navigated" + // rather than as a missed click. See `pressUntil`. + await pressUntil( + page.getByRole("dialog", { name: `Delete schedule ${name}?`, exact: true }) + .getByRole("button", { name: "Delete", exact: true }), + () => expect(page).toHaveURL(/\/schedules$/), + ); } } }); diff --git a/ui/playwright/live/substrate.spec.ts b/ui/playwright/live/substrate.spec.ts index 4498fababd..b2b3564a84 100644 --- a/ui/playwright/live/substrate.spec.ts +++ b/ui/playwright/live/substrate.spec.ts @@ -27,15 +27,16 @@ test("live: the substrate page renders the cluster's own inventory", async ({ pa await test.step("2. the tiles report a real count, not zero", async () => { /* - * The assertion "no load failure" cannot make. A page that reached the controller - * and understood none of the answer draws the same tiles with nothing in them, and - * this cluster is running a worker pool — so a zero here is a decode problem, not - * an empty cluster. + * The assertion "no load failure" cannot make: a page that understood none of the + * answer draws the same tiles, with an em-dash where each number goes. Retrying, + * because that em-dash is also what shows while the read is in flight — read once, + * this failed reporting "Actors running—" against a cluster that said "0/4" a moment + * later. Against the mock that gap is a millisecond, so only a cluster showed it. */ - const actors = page.getByTestId("substrate-stat-actors"); - await expect(actors).toBeVisible(); - const text = (await actors.textContent()) ?? ""; - expect(text, "the actor tile should report a count").toMatch(/\d/); + await expect( + page.getByTestId("substrate-stat-actors-value"), + "the actor tile should report a count", + ).toHaveText(/\d/, { timeout: 60_000 }); }); await test.step("3. the worker table holds rows the cluster returned", async () => { From 78531bf433d14b7a3f7b716ccea31ccbe874f6f0 Mon Sep 17 00:00:00 2001 From: Torey Scheer Date: Wed, 16 Sep 2026 15:12:35 -0400 Subject: [PATCH 02/24] test(ui): run each resource's write journey against both backends MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A create, an edit and a delete are where a fixture and a controller most easily disagree — a create wrapped one way in the fixtures and another by the API, a name sent where a ref belonged. Those journeys ran only against the fixtures, which answer whatever they were taught. `playwright/shared/` is claimed by every project, mock and live alike, and holds the portable half: the write journey for models, prompts and harnesses, plus two properties true whatever the backend holds — no conversation listed by a bare id, and a deep link resolving on a cold load. It is laid out like `tests/`, one folder per resource and app-wide specs at the root, and `conventions.test.ts` enforces that along with the rule that makes it work: a shared spec may not reach for `?mock=`, `withScenario` or `helpers/mockCalls`. Three invariants earned the hard way, each recorded where it applies: - **Counts are relative, never absolute.** The fixtures seed four models and a cluster seeds whatever it was installed with, so `before + 1` is the only form of the claim that survives both — and it is exactly as strong. - **antd draws its loading placeholder as a `tbody tr`.** Counting those read a seeded set of one for a table that had not loaded; data rows are `tbody tr.ant-table-row`. - **A one-shot read is not an assertion.** Two specs failed reading a value while it was still arriving, in opposite directions — the dashboard card against the mock, a substrate tile against the cluster. Both poll now. The duplicated driving is gone from `tests/`: those specs keep what only fixed data can settle — the seeded rows, the refresh counts, the required-field marks, and the empty and failure states no cluster can be asked for — and their titles say so. `routing` and `dashboard` are split the same way, with the SPA-fallback steps moving to `shared/` because nginx and Vite answer a deep link differently and only one of them ships. MCP servers are deliberately absent: that page cannot read its own writes against a real backend (#2849), and a shared spec would have had to press Refresh to get past it. `DEFERRED.md` records the deferral and what closing it looks like. `live/helpers/live.ts` keeps only what is live-specific. `loadApp`, `expectNoLoadFailure`, `rowNamed`, `dataRows` and `throwawayName` moved to `helpers/app.ts`; the first adds the mock scenario only where there is a mock backend to read it, and the two row helpers had drifted to different selectors in the two suites for no reason anybody could name. Verified against a kind cluster running the image built from this tree: 12 live specs and 169 mock specs pass, and the cluster is left with no stray resources. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Torey Scheer --- ui/playwright.config.ts | 16 +- ui/playwright/DEFERRED.md | 35 ++++ ui/playwright/README.md | 28 +++ ui/playwright/conventions.test.ts | 69 ++++++++ ui/playwright/helpers/app.ts | 49 +++++- ui/playwright/live/agent-templates.spec.ts | 8 +- ui/playwright/live/helpers/live.ts | 74 ++------ ui/playwright/live/pages.spec.ts | 16 +- ui/playwright/live/schedules.spec.ts | 5 +- ui/playwright/live/substrate.spec.ts | 5 +- ui/playwright/shared/dashboard.spec.ts | 64 +++++++ .../shared/harnesses/harnesses.spec.ts | 111 ++++++++++++ ui/playwright/shared/models/models.spec.ts | 161 ++++++++++++++++++ ui/playwright/shared/prompts/prompts.spec.ts | 130 ++++++++++++++ ui/playwright/shared/routing.spec.ts | 74 ++++++++ ui/playwright/tests/dashboard.spec.ts | 43 ++--- .../tests/harnesses/harnesses.spec.ts | 59 ++----- ui/playwright/tests/models/models.spec.ts | 135 ++------------- ui/playwright/tests/prompts/prompts.spec.ts | 48 +++--- ui/playwright/tests/routing.spec.ts | 73 ++------ 20 files changed, 829 insertions(+), 374 deletions(-) create mode 100644 ui/playwright/shared/dashboard.spec.ts create mode 100644 ui/playwright/shared/harnesses/harnesses.spec.ts create mode 100644 ui/playwright/shared/models/models.spec.ts create mode 100644 ui/playwright/shared/prompts/prompts.spec.ts create mode 100644 ui/playwright/shared/routing.spec.ts diff --git a/ui/playwright.config.ts b/ui/playwright.config.ts index ae749e9717..fadab2d99a 100644 --- a/ui/playwright.config.ts +++ b/ui/playwright.config.ts @@ -27,6 +27,16 @@ const EXTENSION_BASE_URL = `http://localhost:${EXTENSION_PORT}`; /** Specs that need an extension installed opt in by filename. */ const EXTENSION_SPECS = /\.withExtension\.spec\.ts$/; +/** + * Specs that run in both suites, claimed by every project below. + * + * Only what holds on either backend — no `?mock=` scenario, no fixture by name. That + * narrowness is the point and the risk: assertions true of both are the weakest ones, + * so this folder stays a smoke sweep rather than growing lifecycles. + * `conventions.test.ts` fails a spec here that reaches for a scenario. + */ +const SHARED_SPECS = /shared\/.*\.spec\.ts$/; + /** * The suite is the acceptance bar, so what it runs against cannot depend on the * shell it was started from: both servers are pinned to the in-browser mock @@ -132,7 +142,7 @@ const LIVE_WEB_SERVERS = LIVE_EXTERNAL_URL ]; export default defineConfig({ - testDir: "./playwright/tests", + testDir: "./playwright", // Both servers have to be rendering, not merely listening, before any test // navigates — see the file for what goes wrong otherwise. globalSetup: "./playwright/globalSetup.ts", @@ -195,7 +205,7 @@ export default defineConfig({ ? [ { name: LIVE_PROJECT, - testDir: "./playwright/live", + testMatch: [/live\/.*\.spec\.ts$/, SHARED_SPECS], use: { ...devices["Desktop Chrome"], baseURL: LIVE_BASE_URL, @@ -209,6 +219,7 @@ export default defineConfig({ : [ { name: "chromium", + testMatch: [/tests\/.*\.spec\.ts$/, SHARED_SPECS], testIgnore: EXTENSION_SPECS, use: { ...devices["Desktop Chrome"], baseURL: BASE_URL }, }, @@ -224,6 +235,7 @@ export default defineConfig({ // The extension split below is a build-time difference, not a browser one, // so it stays on one engine rather than doubling for no new signal. name: "firefox", + testMatch: [/tests\/.*\.spec\.ts$/, SHARED_SPECS], testIgnore: EXTENSION_SPECS, use: { ...devices["Desktop Firefox"], baseURL: BASE_URL }, }, diff --git a/ui/playwright/DEFERRED.md b/ui/playwright/DEFERRED.md index de443b3297..7b53349c39 100644 --- a/ui/playwright/DEFERRED.md +++ b/ui/playwright/DEFERRED.md @@ -41,6 +41,41 @@ The specs were written, they were correct, nothing ran them, and they decayed in testing pages that had been removed. A stale entry costs more than no entry; a stale *spec* costs more again. +## Deferred: MCP servers stay mock-only — the list cannot read its own writes + +Every other resource has its create/read/change/delete journey in `shared/`, running +against both backends. MCP servers do not, and the reason is a defect rather than an +awkward fixture: **#2849**. + +The page's two halves use different stores. `CreateToolServer` writes a Kubernetes +`RemoteMCPServer`; `ListToolServers` reads the PostgreSQL `toolserver` table; the only +writer of that table is the reconciler, after it has tried to connect to the server. So +the list lags a create by however long discovery takes. Measured on a cluster: + +``` +CreateToolServer OK +ListToolServers OK <- 57ms later; the new server is not in it + (nothing further) +ListToolServers OK <- a fresh page load a minute on; now present +``` + +Delete has the same shape in reverse, and `DeleteToolServer` resolves the server's kind +from that same projection — so while a new server is invisible it is also undeletable. + +A shared spec was written and did pass, by pressing **Refresh** after the create and +after the delete. It was withdrawn rather than landed: a spec that presses through a +defect to stay green is how the defect stops being noticed, and the press would have +needed removing anyway. The mock lifecycle in `tests/mcp-servers/mcp-servers.spec.ts` +keeps its full coverage meanwhile. + +**Revisit when #2849 lands.** The spec is a short port of the mock one — create with a +URL the cluster can resolve, read the row back, delete it — and the acceptance test is +that it needs no Refresh. + +Worth recording for its own sake: the fixtures cannot show this class of bug at all. They +answer from the page's own memory and are therefore always immediately consistent, so a +mock backend has no write that is not yet a read. Only a cluster has one. + ## Ported since: chat `chat/chat.spec.ts` and `chat/chat-errors.spec.ts` are live. The chat page was diff --git a/ui/playwright/README.md b/ui/playwright/README.md index d5dbb9675a..c64725a582 100644 --- a/ui/playwright/README.md +++ b/ui/playwright/README.md @@ -72,9 +72,37 @@ playwright/ chat, controls, style, mockCalls fixtures/test.ts import { test, expect } from here — never @playwright/test live/ the live suite: specs, plus helpers/ of its own + shared/ specs that run in both suites — laid out like tests/, one folder + per resource and app-wide specs at the root — see below DEFERRED.md the specs not yet portable, and what each one is waiting on ``` +**`shared/` runs in every project**, mock and live alike. What goes in it is narrow: no +`?mock=` scenario, no fixture named, nothing assuming a populated backend. +`conventions.test.ts` fails a spec here that reaches for one. + +It holds two kinds of spec. A property true whatever the backend holds — no conversation +is listed by a bare id, a deep link renders on a cold load. And **the write path of each +resource**: create, read back, change, delete. That second kind is where a fixture and a +controller most easily disagree, and it is the reason `models`, `prompts` and +`harnesses` have a spec here as well as in `tests/`. The names are `throwawayName`d and +every one cleans up in a `finally`, because live they are real. + +It is laid out like `tests/`: one folder per resource holding one spec holding one test, +titled for its folder, and app-wide specs (`dashboard`, `routing`) at the root. +`conventions.test.ts` checks that too — all of it except the empty-and-error rule, which +needs the `?mock=` a spec here may not touch. + +**MCP servers are deliberately not among them.** That page cannot read its own writes +against a real backend — see `DEFERRED.md` and #2849 — so a shared spec would have had to +press Refresh to get past a defect, which is the sort of workaround that keeps one alive. +It stays mock-only until the fix lands. + +What stays in `tests/` for each of those is what needs the fixtures: the exact seeded +rows, the required-field marks, the refresh counts, and the empty and failure states no +cluster can be asked for. Navigate with `loadApp`, which adds the mock scenario only +where there is a mock backend to read it. + The resources with a lifecycle spec are **models**, **MCP servers**, **prompt libraries**, **agent templates**, **harnesses** and **schedules**. Two are narrower than CRUD because the product is: an MCP server's address is its identity, so diff --git a/ui/playwright/conventions.test.ts b/ui/playwright/conventions.test.ts index 29252746d8..0ed4b54389 100644 --- a/ui/playwright/conventions.test.ts +++ b/ui/playwright/conventions.test.ts @@ -22,6 +22,7 @@ import { describe, expect, it } from "vitest"; */ const TESTS = join(__dirname, "tests"); +const SHARED = join(__dirname, "shared"); /** * Folders whose subject is the application rather than a resource it manages. @@ -41,6 +42,29 @@ function specsIn(dir: string): string[] { return readdirSync(dir).filter((name) => name.endsWith(".spec.ts")); } +/** Folders under `shared/`, which are resources for the same reason `tests/` ones are. */ +function sharedFolders(): string[] { + return readdirSync(SHARED).filter( + (name) => !name.startsWith(".") && statSync(join(SHARED, name)).isDirectory(), + ); +} + +/** + * Every spec under `shared/`, nested ones included. + * + * Recursive deliberately: the resource journeys moved a directory down into + * `shared//`, and a top-level-only read would have quietly stopped applying + * the rule below to exactly the specs it matters most for. + */ +function sharedSpecs(): string[] { + return [ + ...specsIn(SHARED).map((name) => join(SHARED, name)), + ...sharedFolders().flatMap((folder) => + specsIn(join(SHARED, folder)).map((name) => join(SHARED, folder, name)), + ), + ]; +} + function folders(): string[] { return readdirSync(TESTS).filter( // Dot-directories are whatever a contributor's local tooling dropped here; they are @@ -136,6 +160,51 @@ describe("playwright layout", () => { ).toEqual([]); }); + it("a shared spec asserts nothing only one backend can answer", () => { + /* + * `shared/` runs in both suites, so a scenario query there is either ignored by a + * cluster or — worse — read as a claim the run cannot make. The narrowness is the + * folder's whole value: assertions true on both backends are the weakest ones, and + * a lifecycle drifting in here would quietly cost the mock suite its precision. + */ + for (const spec of sharedSpecs()) { + const source = readFileSync(spec, "utf8"); + expect( + /mock=|withScenario|scenario:|helpers\/mockCalls/.test(source), + `${spec} drives the mock backend, so it cannot run live`, + ).toBe(false); + } + }); + + it.each(sharedFolders())( + "shared/%s holds one spec, holding one test, titled for its folder", + (folder) => { + /* + * The same shape as a resource folder in `tests/`, and for the same reasons — one + * journey per resource, and a title that says where it lives so `--grep` can + * select an area. Not the empty-and-error rule, though: those need `?mock=`, which + * is the one thing a spec here may not touch. + */ + const specs = specsIn(join(SHARED, folder)); + expect(specs, `shared/${folder}/ should hold one spec`).toHaveLength(1); + + const source = readFileSync(join(SHARED, folder, specs[0]), "utf8"); + expect( + [...source.matchAll(/^test(\.skip)?\(/gm)], + `shared/${folder}/${specs[0]} should hold one test`, + ).toHaveLength(1); + + const expected = folder.replace(/-/g, " "); + for (const title of titles(join(SHARED, folder, specs[0]))) { + const prefix = title.split(":")[0]; + expect( + prefix === expected || prefix.startsWith(`${expected} `), + `shared/${folder}/${specs[0]}: "${title}" should begin with "${expected}"`, + ).toBe(true); + } + }, + ); + it("every spec outside a folder is about the application, not a resource", () => { // Top level means the shell, routing, the dashboard, theme contrast — things that // are about the app rather than about something it manages. A resource folder diff --git a/ui/playwright/helpers/app.ts b/ui/playwright/helpers/app.ts index a6e7487288..ce7ac7fbf4 100644 --- a/ui/playwright/helpers/app.ts +++ b/ui/playwright/helpers/app.ts @@ -6,7 +6,9 @@ * the app already ships for the purpose. */ -import { expect, type Locator, type Page } from "@playwright/test"; +import { expect, test, type Locator, type Page } from "@playwright/test"; + +import { LIVE_PROJECT } from "../../playwright.config"; /** Routes the suite drives. Mirrors `src/router/routes.ts`. */ export const routes = { @@ -158,6 +160,40 @@ export function dataRows(page: Page): Locator { /** A navigation-sized budget, for the app booting rather than for what it rendered. */ const APP_BOOT_TIMEOUT = 15_000; +/** + * Navigates, for a spec in `shared/` that runs against either backend. + * + * `loadPage` cannot: it appends a `?mock=` scenario, which is meaningless to a cluster + * and misleading in a live trace. So the scenario is added only where there is a mock + * backend to read it, and the wait is on the shell rather than on a heading, a live + * page taking longer to have one. + */ +export async function loadApp(page: Page, path: string): Promise { + const live = test.info().project.name === LIVE_PROJECT; + await page.goto(live ? path : withScenario(path, "ok"), { + waitUntil: "domcontentloaded", + }); + await expect(page.getByTestId("app-content")).toBeVisible({ + timeout: live ? 60_000 : APP_BOOT_TIMEOUT, + }); +} + +/** + * Fails when the page is reporting that it could not reach the backend. + * + * Worth calling before asserting on content: the alternative is a failure reading "the + * table is empty" when the truth is "the backend did not answer" — the same distinction + * the app itself is careful about. + */ +export async function expectNoLoadFailure(page: Page): Promise { + const alerts = page.locator('[data-testid$="-error"]'); + const count = await alerts.count(); + if (count === 0) return; + + const texts = await alerts.allInnerTexts(); + expect(count, `the page reported a failure to load: ${texts.join(" | ")}`).toBe(0); +} + /** * Resolves once the app is on screen and no loading indicator is left on it. * @@ -169,3 +205,14 @@ export async function expectSettled(page: Page): Promise { await expect(page.locator("#root")).not.toBeEmpty({ timeout: APP_BOOT_TIMEOUT }); await expect(page.locator(".ant-spin-spinning")).toHaveCount(0); } + +/** + * A name no human would choose, carrying the run that made it. + * + * A spec that creates on a real cluster deletes what it made, but a run killed between + * the two cannot — so the name has to be enough for a person to identify the litter + * without the harness. Unique per run on the fixtures too, where it costs nothing and + * keeps a shared spec reading the same on both backends. + */ +export const throwawayName = (label: string): string => + `e2e-live-${label}-${process.pid}-${Date.now().toString(36)}`; diff --git a/ui/playwright/live/agent-templates.spec.ts b/ui/playwright/live/agent-templates.spec.ts index a730fe975c..3bea16e32b 100644 --- a/ui/playwright/live/agent-templates.spec.ts +++ b/ui/playwright/live/agent-templates.spec.ts @@ -8,11 +8,11 @@ import { import { dataRows, expectNoLoadFailure, - liveRoutes, - loadLive, + loadApp, rowNamed, throwawayName, -} from "./helpers/live"; +} from "../helpers/app"; +import { liveRoutes } from "./helpers/live"; /** * Creating and deleting an agent template, on a real cluster, through the UI. @@ -38,7 +38,7 @@ test("live: an agent template is created, admitted and deleted through the UI", try { await test.step("1. the form offers the cluster's own model configurations", async () => { - await loadLive(page, liveRoutes.agentTemplateNew); + await loadApp(page, liveRoutes.agentTemplateNew); await expectNoLoadFailure(page); await selectOption(page, "template-form-namespace", NAMESPACE); diff --git a/ui/playwright/live/helpers/live.ts b/ui/playwright/live/helpers/live.ts index 897fc79bb0..b27d240f36 100644 --- a/ui/playwright/live/helpers/live.ts +++ b/ui/playwright/live/helpers/live.ts @@ -1,20 +1,23 @@ -import { expect, type Page } from "@playwright/test"; - /** * Helpers for the live suite. * - * The mock suite's helpers are built around `?mock=` scenarios, which is the one - * thing a real controller cannot be told to do — there is no way to ask a cluster - * for a 500. So the live specs assert what a cluster genuinely produces, and these - * are the parts of that worth sharing. + * Only what is genuinely live-specific. Anything that works on either backend lives in + * `helpers/app.ts` instead, so a spec in `shared/` can reach it — `loadApp`, + * `expectNoLoadFailure`, `rowNamed` and `dataRows` all moved there, and this file had + * its own slightly different copies of the last two for no reason anybody could name. */ /** - * Where each page lives, so a renamed route breaks in one place. Mirrors - * `src/router/routes.ts` rather than importing it, like the mock suite's table: a spec - * that reads the app's own constant follows a rename silently. The copy can rot instead - * — this one carried `/agents/new` for a form that had been deleted, and `agentDetail` - * (`/agents/:id`) swallowed the address so it was not even a 404. + * The pages the live sweep visits, which is not every route the app has. + * + * Its own table rather than `routes` from `helpers/app.ts`: that one is every address a + * spec might drive, including forms and detail pages that need an id. This is the set of + * *landing pages* worth loading against a cluster, and `pages.spec.ts` iterates it. + * + * Mirrors `src/router/routes.ts` rather than importing it, like the mock suite's table: + * a spec that reads the app's own constant follows a rename silently. The copy can rot + * instead — this one carried `/agents/new` for a form that had been deleted, and + * `agentDetail` (`/agents/:id`) swallowed the address so it was not even a 404. */ export const liveRoutes = { dashboard: "/", @@ -29,52 +32,3 @@ export const liveRoutes = { schedules: "/schedules", substrate: "/substrate", } as const; - -/** - * Loads a page and waits for the app shell, not for the network to fall quiet. - * - * `networkidle` is the wrong signal against a real backend: a page that polls, or - * an agent whose status is still reconciling, may never produce a quiet network, - * and the wait would time out on a page that rendered correctly seconds earlier. - */ -export async function loadLive(page: Page, path: string): Promise { - await page.goto(path, { waitUntil: "domcontentloaded" }); - await page.waitForSelector('[data-testid="app-content"]', { timeout: 60_000 }); -} - -/** - * Fails when the page is reporting that it could not reach the backend. - * - * Called by every spec before asserting on content, because the alternative is a - * failure that reads as "the table is empty" when the truth is "the cluster did not - * answer" — the same distinction the app itself is careful about. - */ -export async function expectNoLoadFailure(page: Page): Promise { - const alerts = page.locator('[data-testid$="-error"]'); - const count = await alerts.count(); - if (count === 0) return; - - const texts = await alerts.allInnerTexts(); - expect( - count, - `the page reported a failure to load: ${texts.join(" | ")}`, - ).toBe(0); -} - -/** A table row containing `text`. */ -export const rowNamed = (page: Page, text: string) => - page.locator("tbody tr").filter({ hasText: text }); - -/** Rows that carry data, excluding the placeholder antd renders when there are none. */ -export const dataRows = (page: Page) => - page.locator("tbody tr").filter({ hasNot: page.locator(".ant-table-placeholder") }); - -/** - * A name no human would choose, carrying the run that made it. - * - * These specs create real resources on a real cluster. They delete what they create, - * but a run killed between the two cannot, so the name has to be enough for a person - * to identify the litter without the harness. - */ -export const throwawayName = (label: string): string => - `e2e-live-${label}-${process.pid}-${Date.now().toString(36)}`; diff --git a/ui/playwright/live/pages.spec.ts b/ui/playwright/live/pages.spec.ts index 6b89089a64..597e1c9722 100644 --- a/ui/playwright/live/pages.spec.ts +++ b/ui/playwright/live/pages.spec.ts @@ -1,10 +1,6 @@ import { test, expect } from "@playwright/test"; -import { - dataRows, - expectNoLoadFailure, - liveRoutes, - loadLive, -} from "./helpers/live"; +import { dataRows, expectNoLoadFailure, loadApp } from "../helpers/app"; +import { liveRoutes } from "./helpers/live"; /** * Every page, against a real controller. @@ -28,7 +24,7 @@ test("live: every page loads against the cluster and reports no failure", async if (name === "agentTemplateNew") continue; // A form, covered by the lifecycle spec. await test.step(`${name} (${path})`, async () => { - await loadLive(page, path); + await loadApp(page, path); // The distinction worth keeping: a page that could not reach the controller // must not be read as a page with nothing on it. await expectNoLoadFailure(page); @@ -40,7 +36,7 @@ test("live: every page loads against the cluster and reports no failure", async test("live: the agents the cluster installed are listed with their model", async ({ page, }) => { - await loadLive(page, liveRoutes.agents); + await loadApp(page, liveRoutes.agents); await expectNoLoadFailure(page); await test.step("the install's own agents are present", async () => { @@ -71,7 +67,7 @@ test("live: the agents the cluster installed are listed with their model", async test("live: the models the cluster installed are listed with their provider", async ({ page, }) => { - await loadLive(page, liveRoutes.models); + await loadApp(page, liveRoutes.models); await expectNoLoadFailure(page); await expect(dataRows(page).first()).toBeVisible({ timeout: 60_000 }); @@ -80,7 +76,7 @@ test("live: the models the cluster installed are listed with their provider", as }); test("live: tool servers report the tools they discovered", async ({ page }) => { - await loadLive(page, liveRoutes.mcpServers); + await loadApp(page, liveRoutes.mcpServers); await expectNoLoadFailure(page); await test.step("the summary counts servers and tools", async () => { diff --git a/ui/playwright/live/schedules.spec.ts b/ui/playwright/live/schedules.spec.ts index 2cf24bfab6..4497782740 100644 --- a/ui/playwright/live/schedules.spec.ts +++ b/ui/playwright/live/schedules.spec.ts @@ -1,5 +1,6 @@ import { test, expect } from "../fixtures/test"; -import { liveRoutes, loadLive, throwawayName } from "./helpers/live"; +import { loadApp, throwawayName } from "../helpers/app"; +import { liveRoutes } from "./helpers/live"; import { tick } from "../helpers/controls"; import { optionNamed, pressUntil } from "../helpers/resource"; @@ -23,7 +24,7 @@ test("live: schedule configuration persists through the browser and controller", try { await test.step("1. the form offers the cluster's own agents", async () => { - await loadLive(page, liveRoutes.schedules); + await loadApp(page, liveRoutes.schedules); await page.getByTestId("schedules-new").click(); await expect(page).toHaveURL(/\/schedules\/new$/); diff --git a/ui/playwright/live/substrate.spec.ts b/ui/playwright/live/substrate.spec.ts index b2b3564a84..9ddb2b9d4a 100644 --- a/ui/playwright/live/substrate.spec.ts +++ b/ui/playwright/live/substrate.spec.ts @@ -1,5 +1,6 @@ import { test, expect } from "@playwright/test"; -import { expectNoLoadFailure, liveRoutes, loadLive } from "./helpers/live"; +import { expectNoLoadFailure, loadApp } from "../helpers/app"; +import { liveRoutes } from "./helpers/live"; /** * The substrate page, against what the controller actually sends. @@ -16,7 +17,7 @@ import { expectNoLoadFailure, liveRoutes, loadLive } from "./helpers/live"; * thing that can come back empty from a cluster with the tiles still drawing zeros. */ test("live: the substrate page renders the cluster's own inventory", async ({ page }) => { - await loadLive(page, liveRoutes.substrate); + await loadApp(page, liveRoutes.substrate); await expectNoLoadFailure(page); await test.step("1. the page is there rather than an error", async () => { diff --git a/ui/playwright/shared/dashboard.spec.ts b/ui/playwright/shared/dashboard.spec.ts new file mode 100644 index 0000000000..e447cdf261 --- /dev/null +++ b/ui/playwright/shared/dashboard.spec.ts @@ -0,0 +1,64 @@ +import { test, expect } from "../fixtures/test"; +import { expectNoLoadFailure, loadApp } from "../helpers/app"; + +/** + * The dashboard's recent list, asserted the same way on either backend. + * + * The claim is a property rather than a value: `conversationTitle` answers with the + * name somebody gave a conversation, the title derived from its first message, or + * "Untitled" beside the short id — never the bare id alone. That holds whatever is in + * the list, so it is one of the few things worth asserting twice. + * + * What stays in `tests/dashboard.spec.ts` is what only fixtures can settle: that + * Refresh re-reads, and that a conversation somebody named shows that name. + */ +test("dashboard: the recent list names conversations rather than showing ids", async ({ + page, +}) => { + await loadApp(page, "/"); + await expectNoLoadFailure(page); + + const card = page.getByTestId("dashboard-recent-card"); + await expect(card).toBeVisible({ timeout: 30_000 }); + await expect(card).toContainText("Recent agent conversations"); + + /* + * A cluster may genuinely have no conversations yet where the fixtures always do, so + * the count cannot be the claim. That one of the two good states is drawn can be: the + * card renders a list, an empty state, or the unavailable notice, and exactly one of + * the first two is what "the backend answered and was understood" looks like. + * + * Said this way rather than looping over whatever rows exist, because a shared spec + * that asserts nothing when the list is empty is the way this folder rots. + */ + await expect + .poll( + async () => { + if ((await page.getByTestId("recent-agents-unavailable").count()) > 0) + return "unavailable"; + if ((await page.getByTestId("recent-agents").count()) > 0) return "listed"; + if ((await page.getByTestId("recent-agents-empty").count()) > 0) return "empty"; + // The fourth state, and the reason this polls: while the read is in flight the + // card draws none of the three. Counted once instead, this failed on the mock + // backend and passed live, which is the wrong way round for a real defect. + return "loading"; + }, + { + message: "the recent card should settle on a list or an empty state", + timeout: 30_000, + }, + ) + .toMatch(/^(listed|empty)$/); + + const labels = await page + .getByTestId("recent-agent") + .locator("a") + .evaluateAll((links) => links.map((link) => link.textContent?.trim() ?? "")); + + for (const label of labels) { + expect(label, "a conversation should be listed by name, not by its id").not.toMatch( + /^[0-9a-f]{8}$/, + ); + expect(label, "a conversation should carry some label").not.toBe(""); + } +}); diff --git a/ui/playwright/shared/harnesses/harnesses.spec.ts b/ui/playwright/shared/harnesses/harnesses.spec.ts new file mode 100644 index 0000000000..45f04481b7 --- /dev/null +++ b/ui/playwright/shared/harnesses/harnesses.spec.ts @@ -0,0 +1,111 @@ +import { type Page } from "@playwright/test"; +import { test, expect } from "../../fixtures/test"; +import { loadApp, throwawayName } from "../../helpers/app"; +import { LIFECYCLE_TIMEOUT, confirmDelete, selectOption } from "../../helpers/resource"; + +/** + * A harness created, read back and deleted — on either backend. + * + * There is no update half: the tab offers create and delete and no edit. + * + * The claim worth running against a cluster is step 3. A newly created harness is + * "not ready yet" rather than broken — `ready: false` also covers one the controller + * has not observed — and that is a state only a real controller genuinely produces. + * A fixture answering "ready" would hide the one state a new harness is actually in. + */ + +const CREATED = throwawayName("harness"); + +/** Its rows, which is the surface that can say whether any of this happened. */ +const table = "harnesses-table"; + +/* + * Data rows only. antd draws its loading and empty placeholders as a `tbody tr` too, so + * a bare `tbody tr` counts one row for a table that has not loaded — which read as a + * seeded set of one here, and only failed because the count afterwards disagreed. + */ +const harnessRows = (page: Page) => + page.getByTestId(table).locator("tbody tr.ant-table-row"); + +test.describe.configure({ timeout: LIFECYCLE_TIMEOUT }); + +test("harnesses: a harness is created, read and deleted", async ({ page }) => { + let created = false; + /** What the tab held before this journey, so the counts below can be relative. */ + let before = 0; + + try { + await test.step("1. the form refuses an image that is not pinned", async () => { + // Counted first: an absolute count is the fixtures' to make, but "one more, then + // one fewer" holds on any cluster. + await loadApp(page, "/agents?tab=harnesses"); + await expect(harnessRows(page).first()).toBeVisible({ timeout: 60_000 }); + before = await harnessRows(page).count(); + + await loadApp(page, "/harnesses/new"); + + await selectOption(page, "harness-namespace", "kagent"); + await page.getByTestId("harness-name").fill(CREATED); + await page.getByTestId("harness-worker-pool").fill("kagent-default"); + + // The constraints here are the cluster's rather than this page's: a tag can move + // under a running agent, and the CRD refuses one. A form that accepted it would + // build a resource the cluster rejects. + await page.getByTestId("harness-image").fill("ghcr.io/example/runtime:latest"); + await expect(page.getByTestId("harness-create")).toBeDisabled(); + + // The CRD requires a snapshot location too, so it is still refused without one. + await page + .getByTestId("harness-image") + .fill(`ghcr.io/example/runtime@sha256:${"a".repeat(64)}`); + await expect(page.getByTestId("harness-create")).toBeDisabled(); + }); + + await test.step("2. pinned by digest and told where snapshots go, it is created", async () => { + await page.getByTestId("harness-snapshot").fill("s3://ate-snapshots/kagent"); + await page.getByTestId("harness-selector-key").fill("runtime"); + await page.getByTestId("harness-selector-value").fill(CREATED); + await expect(page.getByTestId("harness-admits-nothing")).toHaveCount(0); + + await expect(page.getByTestId("harness-create")).toBeEnabled(); + await page.getByTestId("harness-create").click(); + + // Back to the tab it came from, with the new harness in the list. Read back off + // the table rather than from a toast: "the create returned" and "the thing + // exists" are different claims, and only the list checks the second. + await page.waitForURL(/tab=harnesses/, { timeout: 60_000 }); + created = true; + await expect(page.getByTestId(table)).toContainText(CREATED, { timeout: 60_000 }); + await expect.poll(() => harnessRows(page).count(), { timeout: 60_000 }).toBe( + before + 1, + ); + }); + + await test.step("3. and it is not ready yet, which is what a cluster reports", async () => { + const row = page.getByTestId(table).locator("tr", { hasText: CREATED }); + await expect(row.getByTestId("harness-ready")).toContainText("Not ready yet", { + timeout: 60_000, + }); + }); + + await test.step("4. it is removed from the same tab, and the rest stays", async () => { + await confirmDelete(page, CREATED); + + await expect(page.getByTestId(table)).not.toContainText(CREATED, { + timeout: 60_000, + }); + created = false; + // One row went, not the table: "gone" has to mean that harness rather than a read + // that failed and left an empty list behind it. + await expect.poll(() => harnessRows(page).count(), { timeout: 60_000 }).toBe(before); + await expect(page.getByTestId("harnesses-delete-error")).toHaveCount(0); + }); + } finally { + if (created) { + await loadApp(page, "/agents?tab=harnesses"); + if ((await page.getByTestId(table).getByText(CREATED).count()) > 0) { + await confirmDelete(page, CREATED); + } + } + } +}); diff --git a/ui/playwright/shared/models/models.spec.ts b/ui/playwright/shared/models/models.spec.ts new file mode 100644 index 0000000000..77e0eecc44 --- /dev/null +++ b/ui/playwright/shared/models/models.spec.ts @@ -0,0 +1,161 @@ +import { test, expect } from "../../fixtures/test"; +import { + dataRows, + expectNoLoadFailure, + loadApp, + rowNamed, + throwawayName, +} from "../../helpers/app"; +import { + LIFECYCLE_TIMEOUT, + confirmDelete, + confirmation, + selectOption, +} from "../../helpers/resource"; + +/** + * A model configuration created, read back, changed and deleted — on either backend. + * + * The write path is where a fixture and a controller most easily disagree: a create + * wrapped one way in the fixtures and another by the API, a name sent where a ref + * belonged. The fixtures answer whatever they were taught, so only a cluster can + * settle it — and only this spec asks both the same question. + * + * What stays in `tests/models/models.spec.ts` is what needs the fixtures: the exact + * seeded rows, the required-field marks, the refresh count, and the empty and failure + * states that no cluster can be asked for. + */ + +/** The one this journey makes, changes and removes. Unique, so a killed run leaves litter a person can spot. */ +const CREATED = throwawayName("model"); +const SECRET = "kagent-shared-e2e-secret"; + +test.describe.configure({ timeout: LIFECYCLE_TIMEOUT }); + +test("models: a configuration is created, read, changed and deleted", async ({ + page, +}) => { + let created = false; + /** What the list held before this journey, so the counts below can be relative. */ + let before = 0; + + try { + await test.step("1. a filled-in configuration is created and appears on the list", async () => { + /* + * Counted first, and relative from here on. The fixtures seed four and a cluster + * seeds whatever it was installed with, so an absolute count is the one thing this + * spec cannot assert — but "one more than before" is exactly as strong, and it is + * what catches a create that wrote two rows or a delete that took a neighbour. + */ + await loadApp(page, "/models"); + await expect(dataRows(page).first()).toBeVisible({ timeout: 60_000 }); + before = await dataRows(page).count(); + + await page.getByTestId("models-new").click(); + await page.waitForURL(/\/models\/new(\?|$)/, { timeout: 60_000 }); + + // The provider list is the app's own enum rather than the backend's, so the name + // a reader sees is the same on either — `providerDisplayName` turns + // `AmazonBedrock` into "AWS Bedrock". + await selectOption(page, "model-provider", "Anthropic"); + + // An AutoComplete, not a Select: the id is on the wrapper and the caret goes in + // the input inside it. Typed rather than picked, the field existing to accept a + // model the catalogue has not heard of. + await page.getByTestId("model-model").locator("input").fill("claude-sonnet-4"); + + await page.getByTestId("model-name").fill(CREATED); + // The namespace is half the ref, so one created without it is addressed as + // `/name` and never appears on the list. `kagent` is where both backends put + // things. + await selectOption(page, "model-namespace", "kagent"); + await page.getByTestId("model-api-key").fill("sk-not-a-real-key"); + + await page.getByTestId("model-submit").click(); + await page.waitForURL(/\/models(\?|$)/, { timeout: 60_000 }); + created = true; + + // Read back off the list rather than from a toast or a closed form: those two + // only prove the app believes it worked. + await expectNoLoadFailure(page); + const row = rowNamed(page, CREATED); + await expect(row).toHaveCount(1, { timeout: 60_000 }); + await expect(row).toContainText("Anthropic"); + await expect(row).toContainText("claude-sonnet-4"); + await expect(dataRows(page)).toHaveCount(before + 1); + }); + + await test.step("2. the edit form opens on what was saved, not a blank draft", async () => { + await page.getByTestId(`edit-${CREATED}`).click(); + await page.waitForURL(new RegExp(`/models/kagent/${CREATED}/edit$`), { + timeout: 60_000, + }); + + await expect(page.getByTestId("model-name")).toHaveValue(CREATED, { + timeout: 60_000, + }); + /* + * The identity and the provider are the ref and what the ref means, so an edit + * changes neither. Asserted here because it is the boundary between "edit" and + * "make a new one", and a form that quietly allowed it would write a resource + * nothing else in the cluster points at. + */ + await expect(page.getByTestId("model-name")).toBeDisabled(); + await expect(page.getByTestId("model-model").locator("input")).toBeDisabled(); + }); + + await test.step("3. a change is saved, and the list shows it", async () => { + // The credential moves from one the backend minted to a secret it is told to + // read — the one change on this form the list has a column for, which is what + // makes the save checkable from outside the form. + await page + .getByTestId("model-auth-type") + .getByText("Existing secret", { exact: true }) + .click(); + await page.getByTestId("model-api-key-secret").fill(SECRET); + + await page.getByTestId("model-submit").click(); + await page.waitForURL(/\/models(\?|$)/, { timeout: 60_000 }); + + const row = rowNamed(page, CREATED); + await expect(row).toContainText(SECRET, { timeout: 60_000 }); + // Changed, not duplicated — which a create dressed as an update would be. + await expect(row).toHaveCount(1); + await expect(dataRows(page)).toHaveCount(before + 1); + }); + + await test.step("4. deleting asks first, and Keep leaves it alone", async () => { + await page.getByTestId(`delete-${CREATED}`).click(); + const prompt = confirmation(page); + // The confirmation names the row. "Delete this model configuration?" is no help + // in a table of five, and *which* is the one question the reader has. + await expect(prompt).toContainText(CREATED); + await prompt.getByRole("button", { name: "Keep" }).click(); + await expect(rowNamed(page, CREATED)).toHaveCount(1); + // Waited out rather than assumed gone: the dialog stays visible while it + // animates away, and the next step's click would land on it. + await expect(prompt).toHaveCount(0); + }); + + await test.step("5. confirming removes that row and leaves the rest", async () => { + await confirmDelete(page, CREATED); + await expect(rowNamed(page, CREATED)).toHaveCount(0, { timeout: 60_000 }); + created = false; + + // One row went, not several, and not the read: a list that failed to reload is + // also a list the row is missing from. + await expectNoLoadFailure(page); + await expect(dataRows(page)).toHaveCount(before, { timeout: 60_000 }); + }); + } finally { + // A real resource on a real cluster when this runs live, so a run that dies midway + // takes it with it. + if (created) { + await loadApp(page, "/models"); + if ((await rowNamed(page, CREATED).count()) > 0) { + await confirmDelete(page, CREATED); + await expect(rowNamed(page, CREATED)).toHaveCount(0, { timeout: 60_000 }); + } + } + } +}); diff --git a/ui/playwright/shared/prompts/prompts.spec.ts b/ui/playwright/shared/prompts/prompts.spec.ts new file mode 100644 index 0000000000..3bd2c2b4c4 --- /dev/null +++ b/ui/playwright/shared/prompts/prompts.spec.ts @@ -0,0 +1,130 @@ +import { type Page } from "@playwright/test"; +import { test, expect } from "../../fixtures/test"; +import { + dataRows, + expectNoLoadFailure, + loadApp, + rowNamed, + throwawayName, +} from "../../helpers/app"; +import { LIFECYCLE_TIMEOUT, confirmDelete } from "../../helpers/resource"; + +/** + * A prompt library created, read, changed and deleted — on either backend. + * + * A library is fragments keyed by name, and the write path is where a fixture and a + * controller most easily disagree about the shape of one. What stays in + * `tests/prompts/prompts.spec.ts` is the seeded libraries, the namespace filter, the + * discard prompts, and the empty and failure states. + */ + +const CREATED = throwawayName("library"); + +/** The nth fragment row's key and text boxes. */ +const fragmentKey = (page: Page, index: number) => + page.getByTestId("fragment-key").nth(index); +const fragmentValue = (page: Page, index: number) => + page.getByTestId("fragment-value").nth(index); + +test.describe.configure({ timeout: LIFECYCLE_TIMEOUT }); + +test("prompts: a library is created, read, changed and deleted", async ({ page }) => { + let created = false; + /** What the list held before this journey, so the counts below can be relative. */ + let before = 0; + + try { + await test.step("1. a library with one fragment is created and listed", async () => { + // Counted first, and relative from here on: the fixtures seed two and a cluster + // seeds whatever it was installed with, but "one more than before" is exactly as + // strong and catches a create that wrote twice. + await loadApp(page, "/prompts"); + await expect(dataRows(page).first()).toBeVisible({ timeout: 60_000 }); + before = await dataRows(page).count(); + + await page.getByTestId("prompts-new").click(); + await expect(page.getByTestId("prompt-submit")).toBeVisible({ timeout: 60_000 }); + + await page.getByTestId("prompt-name").fill(CREATED); + await page.getByTestId("prompt-namespace").fill("kagent"); + await fragmentKey(page, 0).fill("changelog"); + await fragmentValue(page, 0).fill("Group by user impact."); + + await page.getByTestId("prompt-submit").click(); + await expect(page).toHaveURL(/\/prompts$/, { timeout: 60_000 }); + created = true; + + // Read back off the list rather than from a toast or a closed form: those two + // only prove the app believes it worked. + await expectNoLoadFailure(page); + const row = rowNamed(page, CREATED); + await expect(row).toContainText("1 key", { timeout: 60_000 }); + await expect(row).toContainText("changelog"); + await expect(dataRows(page)).toHaveCount(before + 1); + }); + + await test.step("2. opening it shows the fragment and how to include it", async () => { + await rowNamed(page, CREATED).getByRole("link").first().click(); + await page.waitForURL(new RegExp(`/prompts/kagent/${CREATED}$`), { + timeout: 60_000, + }); + + const fragments = page.getByTestId("prompt-fragments"); + await expect(fragments).toContainText("changelog", { timeout: 60_000 }); + await expect(fragments).toContainText("Group by user impact."); + }); + + await test.step("3. a fragment is added, saved, and read back off the library", async () => { + await page.getByTestId("prompt-edit").click(); + await page.waitForURL(new RegExp(`/prompts/kagent/${CREATED}/edit$`), { + timeout: 60_000, + }); + // Seeded from the saved library, so the form and the page it came from agree. + await expect(fragmentKey(page, 0)).toHaveValue("changelog", { timeout: 60_000 }); + + await page.getByTestId("fragment-add").click(); + await fragmentKey(page, 1).fill("handoff"); + await fragmentValue(page, 1).fill("Name the next owner explicitly."); + // The include tag is what a fragment is for, and it is offered before the save + // rather than only after it. + await expect(page.getByTestId("fragment-include-preview").last()).toContainText( + `{{include "${CREATED}/handoff"}}`, + ); + + await page.getByTestId("prompt-submit").click(); + await expect(page).toHaveURL(new RegExp(`/prompts/kagent/${CREATED}$`), { + timeout: 60_000, + }); + + // Read back from the re-read library rather than from the draft: a save that + // never reached the backend would leave the old text here. + const fragments = page.getByTestId("prompt-fragments"); + await expect(fragments).toContainText("Name the next owner explicitly.", { + timeout: 60_000, + }); + await expect(page.getByTestId("prompt-detail-meta")).toContainText("2 fragments"); + }); + + await test.step("4. the list behind it shows the change too", async () => { + await page.getByRole("link", { name: "Back to libraries" }).click(); + await expect(rowNamed(page, CREATED)).toContainText("2 keys", { timeout: 60_000 }); + await expect(rowNamed(page, CREATED)).toContainText("handoff"); + }); + + await test.step("5. confirming a delete removes that row and leaves the rest", async () => { + await confirmDelete(page, CREATED); + await expect(rowNamed(page, CREATED)).toHaveCount(0, { timeout: 60_000 }); + created = false; + + // One row went, not several, and not the read: a list that failed to reload is + // also a list the row is missing from. + await expectNoLoadFailure(page); + await expect(dataRows(page)).toHaveCount(before, { timeout: 60_000 }); + }); + } finally { + if (created) { + await loadApp(page, "/prompts"); + if ((await rowNamed(page, CREATED).count()) > 0) await confirmDelete(page, CREATED); + } + } +}); diff --git a/ui/playwright/shared/routing.spec.ts b/ui/playwright/shared/routing.spec.ts new file mode 100644 index 0000000000..ca64a1e2bb --- /dev/null +++ b/ui/playwright/shared/routing.spec.ts @@ -0,0 +1,74 @@ +import { test, expect } from "../fixtures/test"; +import { expectPageTitle, loadApp, routes } from "../helpers/app"; +import { clickNav, expectShell } from "../helpers/nav"; + +/** + * Routing, which is the part of this app a *server* can still get wrong. + * + * Client-side routing means a deep link is not a file on disk, so something has to + * answer `/substrate` and `/no-such-page` with the app shell rather than a 404. In a + * cluster that something is nginx's `try_files $uri $uri/ /index.html`; under `yarn dev` + * it is Vite's own fallback, which is a different implementation of the same promise. + * Steps 4 and 5 are here rather than in `tests/` for exactly that reason: they are the + * one thing in this suite that can pass against a dev server and fail against the image. + * + * What stays in `tests/routing.spec.ts` is the two steps needing fixtures: a deep link + * carrying an instance id, and the standalone login route. + */ + +test("routing: in-app navigation, deep links and 404, on either backend", async ({ + page, +}) => { + await test.step("1. a sidebar click changes both the URL and the content", async () => { + await loadApp(page, routes.dashboard); + await expectPageTitle(page, "Dashboard"); + + await clickNav(page, "agents", /\/agents(\?|$)/); + await expectPageTitle(page, "Agents"); + + await clickNav(page, "prompts", /\/prompts(\?|$)/); + await expectPageTitle(page, "Prompts"); + }); + + await test.step("2. the sidebar marks the active destination", async () => { + await expect(page.getByTestId("nav-prompts")).toHaveClass(/ant-menu-item-selected/); + await expect(page.getByTestId("nav-agents")).not.toHaveClass( + /ant-menu-item-selected/, + ); + }); + + await test.step("3. browser history moves between routes", async () => { + await page.goBack(); + await page.waitForURL(/\/agents(\?|$)/); + await expectPageTitle(page, "Agents"); + + await page.goForward(); + await page.waitForURL(/\/prompts(\?|$)/); + await expectPageTitle(page, "Prompts"); + }); + + await test.step("4. a deep link renders that route on a cold load", async () => { + // A full page load, not a client-side transition: this is the link somebody pastes + // into chat, and the one a server that does not fall back to index.html would break. + await loadApp(page, routes.substrate); + await expectPageTitle(page, "Substrate"); + await expectShell(page); + await expect(page).toHaveURL(/\/substrate/); + }); + + await test.step("5. an unknown path renders 404 inside the shell", async () => { + await loadApp(page, "/no-such-page"); + // The address it tried, which is what a reader compares against the link they + // followed — "that page does not exist" told them nothing they could act on. + await expect(page.getByTestId("not-found-path")).toHaveText("/no-such-page"); + // And somewhere to go that is not just "back to the dashboard", which is the right + // destination only if that is where they were headed. + await expect(page.getByTestId("not-found-link-agents")).toBeVisible(); + // Still inside the app: a wrong URL should not strand the reader with no way back. + await expectShell(page); + + await page.getByTestId("not-found-dashboard").click(); + await page.waitForURL(/\/$/); + await expectPageTitle(page, "Dashboard"); + }); +}); diff --git a/ui/playwright/tests/dashboard.spec.ts b/ui/playwright/tests/dashboard.spec.ts index 4d477989a6..5f7bc9d08f 100644 --- a/ui/playwright/tests/dashboard.spec.ts +++ b/ui/playwright/tests/dashboard.spec.ts @@ -4,15 +4,19 @@ import { operationCalls, rpc } from "../helpers/mockCalls"; import { clickRefresh } from "../helpers/resource"; /** - * The dashboard's recent list, which is conversations and now reads like it. + * The dashboard's recent list, on the fixtures. * * The card was headed "Recently created agents" and listed `AgentInstance` rows, which * are conversations rather than agents — so the heading was wrong about what it held. * The rows were worse: each linked to a conversation under a bare eight-character id, - * on the reasoning that "an agent has no name". A conversation does have one, and this - * is the third surface to show it after the rail and the agent's own table. + * on the reasoning that "an agent has no name". + * + * What is left here is what only a fixed backend can settle. That no row is a bare id + * is a property of any backend, so it runs against both from `shared/dashboard.spec.ts`. */ -test("dashboard: recent conversations read as names, not as ids", async ({ page }) => { +test("dashboard: Refresh re-reads, and a named conversation shows its name", async ({ + page, +}) => { await loadPage(page, routes.dashboard); await expectSettled(page); @@ -32,33 +36,10 @@ test("dashboard: recent conversations read as names, not as ids", async ({ page .toBeGreaterThan(before); }); - await test.step("2. and no row is a bare id", async () => { - /* - * The property, rather than a fixture's particular name. - * - * `conversationTitle` answers with the name somebody gave it, the title derived - * from its first message, or "Untitled" beside the short id — never the short id - * alone. So a link whose whole text is eight hex characters is the old behaviour, - * whichever conversation happens to be recent enough to appear here. - */ - const rows = page.getByTestId("recent-agent"); - await expect(rows.first()).toBeVisible(); - const labels = await rows.locator("a").evaluateAll((links) => - links.map((link) => link.textContent?.trim() ?? ""), - ); - - expect(labels.length).toBeGreaterThan(0); - for (const label of labels) { - expect(label, "a conversation should be listed by name, not by its id").not.toMatch( - /^[0-9a-f]{8}$/, - ); - } - }); - - await test.step("3. and a conversation somebody named shows that name", async () => { - // The other half: "not an id" would also be satisfied by every row reading - // "Untitled", which is true and useless. At least one of the fixtures' recent - // conversations carries a name its reader chose. + await test.step("2. and a conversation somebody named shows that name", async () => { + // The other half of the shared spec's claim: "not an id" would also be satisfied + // by every row reading "Untitled", which is true and useless. Only fixtures can + // guarantee a conversation whose name its reader chose. const labels = await page .getByTestId("recent-agent") .locator("a") diff --git a/ui/playwright/tests/harnesses/harnesses.spec.ts b/ui/playwright/tests/harnesses/harnesses.spec.ts index d19852572d..c9d40e6273 100644 --- a/ui/playwright/tests/harnesses/harnesses.spec.ts +++ b/ui/playwright/tests/harnesses/harnesses.spec.ts @@ -2,7 +2,6 @@ import { test, expect } from "../../fixtures/test"; import { loadPage, routes } from "../../helpers/app"; import { LIFECYCLE_TIMEOUT, - confirmDelete, expectRequired, selectFirstOption, } from "../../helpers/resource"; @@ -13,9 +12,13 @@ import { * One test, because a video and a trace are recorded per *test* — see * `playwright/README.md`. * - * **There is no update half.** The tab offers create and delete and no edit, so the - * journey is create, read back, remove. That is narrower than `HarnessService`, which - * implements update too — this application has never called it. + * **There is no update half.** The tab offers create and delete and no edit. + * + * **Nor is the write journey here.** Creating a harness, reading back that it is "not + * ready yet" and removing it runs against both backends from + * `playwright/shared/harnesses/harnesses.spec.ts`. What stays is the reading — the seeded + * rows, the selector on the page, the narrowing — and the two refusals below, which are + * about the form enforcing the cluster's constraints rather than about creating anything. * * ## What the tab exists to say * @@ -39,7 +42,7 @@ import { * for real, which is why the fixture refuses it too. */ -/** The one this journey makes, reads back and removes. */ +/** The name the validation steps type in. Nothing is created here — see the note above. */ const CREATED = "made-here"; /** Its rows, which is the surface that can say whether any of this happened. */ @@ -52,7 +55,9 @@ const table = "harnesses-table"; */ test.describe.configure({ timeout: LIFECYCLE_TIMEOUT }); -test("harnesses: a harness is created, read and deleted", async ({ page }) => { +test("harnesses: the tab reads, and the form refuses what the CRD refuses", async ({ + page, +}) => { await test.step("1. the harnesses are listed, with what admits a template on the page", async () => { await loadPage(page, routes.harnesses, { title: "Agents" }); await expect(page.getByTestId(table)).toBeVisible({ timeout: 30_000 }); @@ -145,45 +150,7 @@ test("harnesses: a harness is created, read and deleted", async ({ page }) => { await expect(page.getByTestId("harness-create")).toBeDisabled(); }); - await test.step("7. pinned by digest and told where snapshots go, it is created", async () => { - await page.getByTestId("harness-snapshot").fill("s3://ate-snapshots/kagent"); - await page.getByTestId("harness-selector-key").fill("runtime"); - await page.getByTestId("harness-selector-value").fill(CREATED); - await expect(page.getByTestId("harness-admits-nothing")).toHaveCount(0); - - await expect(page.getByTestId("harness-create")).toBeEnabled(); - await page.getByTestId("harness-create").click(); - - // Back to the tab it came from, with the new harness in the list. Read back off the - // table rather than from a toast or a closed form: "the create returned" and "the - // thing exists" are different claims, and only the list checks the second. - await page.waitForURL(/tab=harnesses/); - await expect(page.getByTestId(table)).toContainText(CREATED, { timeout: 30_000 }); - }); - - await test.step("8. and it is not ready yet, which is what a cluster reports", async () => { - // The controller has not observed it. A fixture that answered "ready" would hide the - // one state a newly created harness is actually in. - const row = page.getByTestId(table).locator("tr", { hasText: CREATED }); - await expect(row.getByTestId("harness-ready")).toContainText("Not ready yet"); - }); - - await test.step("9. it is removed from the same tab, and the rest stays", async () => { - const rows = page.getByTestId(table).locator("tbody tr"); - const before = await rows.count(); - - await confirmDelete(page, CREATED); - - await expect(page.getByTestId(table)).not.toContainText(CREATED, { - timeout: 30_000, - }); - // One row went, not the table: "gone" has to mean that harness rather than a read - // that failed and left an empty list behind it. - await expect.poll(() => rows.count()).toBe(before - 1); - await expect(page.getByTestId("harnesses-delete-error")).toHaveCount(0); - }); - - await test.step("10. an empty result leaves the tab standing, with no rows", async () => { + await test.step("7. an empty result leaves the tab standing, with no rows", async () => { // Last, after the delete: reaching these needs the backend answering differently and // `?mock=` is per-navigation, which discards what the journey made. await loadPage(page, routes.harnesses, { scenario: "empty", title: "Agents" }); @@ -192,7 +159,7 @@ test("harnesses: a harness is created, read and deleted", async ({ page }) => { await expect(page.getByTestId("harnesses-error")).toHaveCount(0); }); - await test.step("11. a failed load is reported, not disguised as an empty tab", async () => { + await test.step("8. a failed load is reported, not disguised as an empty tab", async () => { await loadPage(page, routes.harnesses, { scenario: "error", title: "Agents" }); const alert = page.getByTestId("harnesses-error"); diff --git a/ui/playwright/tests/models/models.spec.ts b/ui/playwright/tests/models/models.spec.ts index 520999be23..822ad48c92 100644 --- a/ui/playwright/tests/models/models.spec.ts +++ b/ui/playwright/tests/models/models.spec.ts @@ -11,10 +11,7 @@ import { expectLoading, chooseFilter, clickRefresh, - confirmDelete, - confirmation, expectRequired, - selectOption, } from "../../helpers/resource"; import { operationCalls, rpc } from "../../helpers/mockCalls"; @@ -24,11 +21,12 @@ import { operationCalls, rpc } from "../../helpers/mockCalls"; * One test, because a video and a trace are recorded per *test* — see * `playwright/README.md` for the shape and the trade it makes. * - * The fixture backend records writes (`src/mocks/state.ts`) so the reads afterwards - * can contradict the form, which is what steps 6, 8 and 10 rely on. Three claims that - * would otherwise each cost their own page load are steps here rather than files of - * their own: the required-field marks (5), the refresh confirmation (3) and the - * filter in the address (4). + * **The write journey is not here.** Creating a configuration, reading it back, changing + * its credential and deleting it runs against both backends from + * `playwright/shared/models/models.spec.ts`, which counts rows relative to what the list + * already held. What is left here is what only fixed data can settle: the seeded rows and + * their refs taken apart, the refresh confirmation, the filter in the address, the + * required-field marks, and the empty and failure states. */ /** The four seeded configurations, which is what "nothing narrowed" has to mean. */ @@ -39,9 +37,6 @@ const SEEDED = [ "bedrock-haiku", ]; -/** The one this journey makes, reads, renames the model on, and removes. */ -const CREATED = "browser-made-model"; - /* * A lifecycle is longer than a journey, so it gets its own budget — see * `LIFECYCLE_TIMEOUT`. Set per file rather than across the suite, so the tight default @@ -49,7 +44,7 @@ const CREATED = "browser-made-model"; */ test.describe.configure({ timeout: LIFECYCLE_TIMEOUT }); -test("models: a configuration is created, read, changed and deleted", async ({ +test("models: the list reads, narrows, and says what the form requires", async ({ page, }) => { await test.step("1. a loading state precedes the data", async () => { @@ -166,123 +161,13 @@ test("models: a configuration is created, read, changed and deleted", async ({ }); }); - await test.step("6. a filled-in configuration is created and appears on the list", async () => { - // The provider is picked by the name a reader sees, which is not the enum the - // draft stores: `providerDisplayName` turns `AmazonBedrock` into "AWS Bedrock". - await selectOption(page, "model-provider", "Anthropic"); - - // The model is an AutoComplete, not a Select — the test id is on the wrapper and - // the caret goes in the input inside it. Typed rather than picked, because the - // field exists to accept a model the catalogue has not heard of. - const model = page.getByTestId("model-model").locator("input"); - await model.fill("claude-sonnet-4"); - - await page.getByTestId("model-name").fill(CREATED); - // The namespace is half the ref, so a configuration created without one is - // addressed as `/name` and never appears on the list — which is why the form - // marks it required and why this step chooses one rather than leaving the - // default. - await selectOption(page, "model-namespace", "kagent"); - await page.getByTestId("model-api-key").fill("sk-not-a-real-key"); - - await page.getByTestId("model-submit").click(); - await page.waitForURL(/\/models(\?|$)/, { timeout: 30_000 }); - - // Read back off the list rather than from a toast or a closed form: those two - // only prove the app believes it worked. - const row = rowNamed(page, CREATED); - await expect(row).toHaveCount(1, { timeout: 30_000 }); - await expect(row).toContainText("Anthropic"); - await expect(row).toContainText("claude-sonnet-4"); - await expect(dataRows(page)).toHaveCount(SEEDED.length + 1); - }); - - await test.step("7. the edit form opens on what was saved, not on a blank draft", async () => { - await page.getByTestId(`edit-${CREATED}`).click(); - await page.waitForURL(new RegExp(`/models/kagent/${CREATED}/edit$`)); - await expectSettled(page); - - await expect(page.getByTestId("model-name")).toHaveValue(CREATED); - /* - * The identity and the provider are the ref and what the ref means, so an edit - * changes neither: a configuration cannot be renamed, moved to another namespace, - * or repointed at a different provider's model. Asserted here because it is the - * boundary between "edit" and "make a new one", and a form that quietly allowed - * it would write a resource nothing else in the cluster is pointing at. - */ - await expect(page.getByTestId("model-name")).toBeDisabled(); - await expect(page.getByTestId("model-model").locator("input")).toBeDisabled(); - - /* - * And the key is not asked for again, because the cluster already holds it. The - * label says so in words; the mark has to agree, or the form is demanding a - * credential in order to change anything else. - * - * The radio is clicked first because the field is not on screen until it is: a - * key is write-only, so the configuration comes back carrying no credential to - * seed the draft from, and the form opens on the mode that matches what it was - * given rather than on the one it was created with. - */ - await page - .getByTestId("model-auth-type") - .getByText("API key", { exact: true }) - .click(); - await expectRequired(page, { - marked: ["Name", "Namespace"], - unmarked: ["API key (leave blank to keep existing)"], - }); - }); - - await test.step("8. a change is saved, and the list shows it", async () => { - // The credential moves from one the controller minted to a Secret the cluster - // already holds — the one change on this form the list has a column for, which is - // what makes the save checkable from outside the form. - await page - .getByTestId("model-auth-type") - .getByText("Existing secret", { exact: true }) - .click(); - await page.getByTestId("model-api-key-secret").fill("kagent-anthropic"); - - await page.getByTestId("model-submit").click(); - await page.waitForURL(/\/models(\?|$)/, { timeout: 30_000 }); - - const row = rowNamed(page, CREATED); - await expect(row).toContainText("kagent-anthropic", { timeout: 30_000 }); - // Changed, not duplicated — which a create dressed as an update would be. - await expect(row).toHaveCount(1); - await expect(dataRows(page)).toHaveCount(SEEDED.length + 1); - }); - - await test.step("9. deleting asks first, and Keep leaves it alone", async () => { - await page.getByTestId(`delete-${CREATED}`).click(); - const prompt = confirmation(page); - // The confirmation names the row. "Delete this model configuration?" is no help - // in a table of five of them, and *which* is the one question the reader has. - await expect(prompt).toContainText(CREATED); - await prompt.getByRole("button", { name: "Keep" }).click(); - await expect(rowNamed(page, CREATED)).toHaveCount(1); - // Waited out rather than assumed gone: the dialog stays visible while it animates - // away, and the next step's click would land on it. - await expect(prompt).toHaveCount(0); - }); - - await test.step("10. confirming removes that row and leaves the rest", async () => { - await confirmDelete(page, CREATED); - - await expect(rowNamed(page, CREATED)).toHaveCount(0, { timeout: 30_000 }); - // "Gone" has to mean that one rather than the read: a list that failed to reload - // is also a list the row is missing from. - await expect(dataRows(page)).toHaveCount(SEEDED.length); - await expect(rowNamed(page, "default-model-config")).toHaveCount(1); - }); - - await test.step("11. an empty result says so instead of showing a bare table", async () => { + await test.step("6. an empty result says so instead of showing a bare table", async () => { await loadPage(page, routes.models, { scenario: "empty", title: "Models" }); await expect(page.getByText("No model configurations yet.")).toBeVisible(); await expect(dataRows(page)).toHaveCount(0); }); - await test.step("12. a failed load is reported, not disguised as an empty list", async () => { + await test.step("7. a failed load is reported, not disguised as an empty list", async () => { await loadPage(page, routes.models, { scenario: "error", title: "Models" }); const alert = page.getByTestId("models-error"); @@ -301,7 +186,7 @@ test("models: a configuration is created, read, changed and deleted", async ({ await expect(dataRows(page)).toHaveCount(0); }); - await test.step("13. retrying asks the backend again, and it recovers", async () => { + await test.step("8. retrying asks the backend again, and it recovers", async () => { const before = await operationCalls(page, rpc.listModelConfigs); await page.getByRole("button", { name: "Try again" }).click(); diff --git a/ui/playwright/tests/prompts/prompts.spec.ts b/ui/playwright/tests/prompts/prompts.spec.ts index 618e641709..dc90db5928 100644 --- a/ui/playwright/tests/prompts/prompts.spec.ts +++ b/ui/playwright/tests/prompts/prompts.spec.ts @@ -13,7 +13,6 @@ import { anyDialog, chooseFilter, clickRefresh, - confirmDelete, expectRequired, pressUntil, } from "../../helpers/resource"; @@ -63,7 +62,7 @@ const fragmentValue = (page: Page, index: number) => */ test.describe.configure({ timeout: LIFECYCLE_TIMEOUT }); -test("prompts: a library is created, read, changed and deleted", async ({ +test("prompts: libraries are read, edited, and their failures reported", async ({ page, }) => { await test.step("1. a loading state precedes the data", async () => { @@ -183,17 +182,20 @@ test("prompts: a library is created, read, changed and deleted", async ({ await expect(page).toHaveURL(/\/prompts\/new$/); await expect(page.getByTestId("prompt-name")).toHaveValue(CREATED); - await page.getByTestId("fragment-key").first().fill("changelog"); - await page.getByTestId("fragment-value").first().fill("Group by user impact."); - await page.getByTestId("prompt-submit").click(); - await expect(page).toHaveURL(/\/prompts$/, { timeout: 30_000 }); - // Read back off the list rather than from a toast or a closed form: those two only - // prove the app believes it worked. - const row = rowNamed(page, CREATED); - await expect(row).toContainText("1 key", { timeout: 30_000 }); - await expect(row).toContainText("changelog"); - await expect(dataRows(page)).toHaveCount(SEEDED.length + 1); + /* + * And that is where this step stops. Creating the library, reading it back, adding a + * fragment and deleting it runs against both backends from + * `playwright/shared/prompts/prompts.spec.ts`. Left here it was the same journey + * driven twice, and the count it used to assert is stronger there, where it is + * relative to whatever the list already held. + * + * Navigated rather than submitted, so the steps below start on the list with a draft + * abandoned — which is also the state a reader leaves behind when they change their + * mind, and nothing else covers it. + */ + await loadPage(page, routes.prompts, { title: "Prompts" }); + await expect(dataRows(page)).toHaveCount(SEEDED.length); }); await test.step("7. the list's edit action opens the form, not the reading page", async () => { @@ -323,7 +325,7 @@ test("prompts: a library is created, read, changed and deleted", async ({ // The keys column too, which is what the search on this page also covers. await expect(row).toContainText("handoff"); - await expect(dataRows(page)).toHaveCount(SEEDED.length + 1); + await expect(dataRows(page)).toHaveCount(SEEDED.length); await expect(rowNamed(page, "incident-playbooks")).toContainText("2 keys"); }); @@ -348,20 +350,10 @@ test("prompts: a library is created, read, changed and deleted", async ({ await expect(page.getByTestId("prompt-discard-body")).toHaveCount(0); await page.getByRole("link", { name: "Back to libraries" }).click(); - await expect(rowNamed(page, CREATED)).toHaveCount(1, { timeout: 30_000 }); - }); - - await test.step("13. confirming a delete removes that row and leaves the rest", async () => { - await confirmDelete(page, CREATED); - - await expect(rowNamed(page, CREATED)).toHaveCount(0, { timeout: 30_000 }); - // "Gone" has to mean that one rather than the read: a list that failed to reload is - // also a list the row is missing from. - await expect(dataRows(page)).toHaveCount(SEEDED.length); - await expect(rowNamed(page, "shared-fragments")).toHaveCount(1); + await expect(rowNamed(page, "shared-fragments")).toHaveCount(1, { timeout: 30_000 }); }); - await test.step("14. a library that has gone says so instead of offering a form", async () => { + await test.step("13. a library that has gone says so instead of offering a form", async () => { // Deep-linked, the way a stale tab or a shared address arrives. An edit form over a // library the cluster does not have would take input for a save that cannot land. await page.goto("/prompts/kagent/not-a-library/edit?mock=ok"); @@ -378,13 +370,13 @@ test("prompts: a library is created, read, changed and deleted", async ({ await expect(page.getByTestId("prompt-detail-error")).toHaveCount(0); }); - await test.step("15. an empty result says so instead of showing a bare table", async () => { + await test.step("14. an empty result says so instead of showing a bare table", async () => { await loadPage(page, routes.prompts, { scenario: "empty", title: "Prompts" }); await expect(page.getByText("No prompt libraries yet.")).toBeVisible(); await expect(dataRows(page)).toHaveCount(0); }); - await test.step("16. a failed load is reported, not disguised as an empty list", async () => { + await test.step("15. a failed load is reported, not disguised as an empty list", async () => { await loadPage(page, routes.prompts, { scenario: "error", title: "Prompts" }); const alert = page.getByTestId("prompts-error"); @@ -408,7 +400,7 @@ test("prompts: a library is created, read, changed and deleted", async ({ await expect(page.getByTestId("prompt-fragments")).toHaveCount(0); }); - await test.step("17. retrying asks the backend again, and it recovers", async () => { + await test.step("16. retrying asks the backend again, and it recovers", async () => { await loadPage(page, routes.prompts, { scenario: "error", title: "Prompts" }); /* diff --git a/ui/playwright/tests/routing.spec.ts b/ui/playwright/tests/routing.spec.ts index 6714b2adc1..5e2ff2f39d 100644 --- a/ui/playwright/tests/routing.spec.ts +++ b/ui/playwright/tests/routing.spec.ts @@ -1,57 +1,21 @@ import { test, expect } from "../fixtures/test"; import { agentChat, instances, loadPage, expectPageTitle, routes } from "../helpers/app"; -import { clickNav, expectNoShell, expectShell } from "../helpers/nav"; +import { expectNoShell, expectShell } from "../helpers/nav"; /** - * Routing — new coverage for the thing this rewrite changed most. + * Routing, the two parts of it that need fixtures. * - * The old app routed on the server through Next's file-system router; this one - * is a single-page app with a client-side router, which puts four behaviours at - * risk that used to come for free: in-app navigation, deep linking straight to a - * route, an unknown path resolving to a 404 rather than a blank screen, and a - * standalone route rendering outside the shell. + * A deep link carrying an `AgentInstance` id has to name a conversation that exists, + * and the login route has to have something to log in to — neither of which a clean + * cluster supplies. The rest of the journey is backend-agnostic and runs against both + * from `shared/routing.spec.ts`, including the deep link and 404 steps, which are the + * ones a server can get wrong. */ -test("routing: in-app navigation, deep links, 404, and standalone routes", async ({ +test("routing: a deep link with params, and the standalone login route", async ({ page, }) => { - await test.step("1. a sidebar click changes both the URL and the content", async () => { - await loadPage(page, routes.dashboard, { title: "Dashboard" }); - - await clickNav(page, "agents", /\/agents(\?|$)/); - await expectPageTitle(page, "Agents"); - - await clickNav(page, "prompts", /\/prompts(\?|$)/); - await expectPageTitle(page, "Prompts"); - }); - - await test.step("2. the sidebar marks the active destination", async () => { - await expect(page.getByTestId("nav-prompts")).toHaveClass(/ant-menu-item-selected/); - await expect(page.getByTestId("nav-agents")).not.toHaveClass( - /ant-menu-item-selected/, - ); - }); - - await test.step("3. browser history moves between routes", async () => { - await page.goBack(); - await page.waitForURL(/\/agents(\?|$)/); - await expectPageTitle(page, "Agents"); - - await page.goForward(); - await page.waitForURL(/\/prompts(\?|$)/); - await expectPageTitle(page, "Prompts"); - }); - - await test.step("4. a deep link renders that route on a cold load", async () => { - // A full page load, not a client-side transition: this is the link someone - // pastes into chat, and the one a server that does not fall back to - // index.html would break. - await loadPage(page, routes.substrate, { title: "Substrate" }); - await expectShell(page); - await expect(page).toHaveURL(/\/substrate/); - }); - - await test.step("5. a deep link with route params renders too", async () => { + await test.step("1. a deep link with route params renders too", async () => { // Two params: the namespace and the AgentInstance id, which is how every agent // surface is addressed now. await loadPage(page, agentChat(instances.ready)); @@ -65,24 +29,7 @@ test("routing: in-app navigation, deep links, 404, and standalone routes", async await expect(page.getByTestId("chat-panel")).toBeVisible(); }); - await test.step("6. an unknown path renders 404 inside the shell", async () => { - await loadPage(page, "/no-such-page"); - // The address it tried, which is what a reader compares against the link they - // followed — "that page does not exist" told them nothing they could act on. - await expect(page.getByTestId("not-found-path")).toHaveText("/no-such-page"); - // And somewhere to go that is not just "back to the dashboard", which is the right - // destination only if that is where they were headed. - await expect(page.getByTestId("not-found-link-agents")).toBeVisible(); - // Still inside the app: a wrong URL should not strand the user with no - // way back. - await expectShell(page); - - await page.getByTestId("not-found-dashboard").click(); - await page.waitForURL(/\/$/); - await expectPageTitle(page, "Dashboard"); - }); - - await test.step("7. login renders standalone, outside the shell", async () => { + await test.step("2. login renders standalone, outside the shell", async () => { await loadPage(page, routes.login); await expect(page.getByTestId("login-page")).toBeVisible(); await expectNoShell(page); From de5a3a1d6bf6bc5ae5d580c86d283df0f0a84227 Mon Sep 17 00:00:00 2001 From: Torey Scheer Date: Wed, 16 Sep 2026 16:04:34 -0400 Subject: [PATCH 03/24] test(ui): share the last two write journeys, and split schedules on the reload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `agent-templates` and `schedules` were the two resources whose journey still ran twice — once against the fixtures in `tests/`, once against a cluster in `live/` — which is the duplication the previous commit removed everywhere else. Both move to `shared/`, and what is left behind in each place is what only that place can answer. **Schedules split on a reload.** Moving the live spec wholesale did not work, and the reason is worth keeping: step 4 reloads, and the fixture backend holds writes in the page's own memory, so a reload starts a backend that has never heard of the schedule. That reload is not incidental — it is the whole claim, since everything short of a re-read could be the form showing itself its own draft. So the journey is `shared/schedules/`, clicking through on either backend, and `live/schedules.spec.ts` keeps only the persistence: a fractional timeout stored as seconds plus nanos, and a named IANA zone, both still there after a reload. The general rule is now in the README — a spec whose claim is "the backend stored it" needs a backend that stores. **Agent templates gained the edit they never had.** The shared spec created and deleted but never updated, so no suite was asserting an update against a real controller. It now changes the description in place and reads it back off the page, and it picked up the namespace-narrowing assertion the mock create was making — the line that once carried two faults at once, a redirect with no query string and a caller sending `namespace` where the list narrows on `ns`. The mock specs keep what needs fixed data: the seeded rows, the cadence and history of a schedule and the pausing that is unique to it, the template list's narrowing and sorting, the form's refusal of an unusable template, both branches of the delete warning, and the empty and failure states. Their titles say what they now cover. One thing the trim exposed: the agent-template delete was doing double duty, since removing the template also navigated back to the list, and the step after it relied on that. The hop is explicit now. The create stays in the mock template spec as setup rather than as a claim. Removing it would mean pointing the read-and-edit steps at the only seeded template that has a harness, and a later step asserts that template's description unedited — the note on `CREATED` records that, so the trade is not rediscovered. Verified against a kind cluster running the image built from this tree: 13 live specs and 173 mock specs pass, and the cluster is left with no stray resources. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Torey Scheer --- ui/playwright/README.md | 17 +- ui/playwright/live/schedules.spec.ts | 122 ++++------- .../agent-templates}/agent-templates.spec.ts | 71 +++++- .../shared/schedules/schedules.spec.ts | 163 ++++++++++++++ .../agent-templates/agent-templates.spec.ts | 79 ++----- .../tests/schedules/schedules.spec.ts | 205 ++---------------- 6 files changed, 318 insertions(+), 339 deletions(-) rename ui/playwright/{live => shared/agent-templates}/agent-templates.spec.ts (70%) create mode 100644 ui/playwright/shared/schedules/schedules.spec.ts diff --git a/ui/playwright/README.md b/ui/playwright/README.md index c64725a582..c954390640 100644 --- a/ui/playwright/README.md +++ b/ui/playwright/README.md @@ -83,10 +83,19 @@ playwright/ It holds two kinds of spec. A property true whatever the backend holds — no conversation is listed by a bare id, a deep link renders on a cold load. And **the write path of each -resource**: create, read back, change, delete. That second kind is where a fixture and a -controller most easily disagree, and it is the reason `models`, `prompts` and -`harnesses` have a spec here as well as in `tests/`. The names are `throwawayName`d and -every one cleans up in a `finally`, because live they are real. +resource**: create, read back, change, delete. + +That second kind is where a fixture and a controller most easily disagree, and it is why +every resource that can have its journey here does: `models`, `prompts`, `harnesses`, +`agent-templates` and `schedules`. The names are `throwawayName`d and every one cleans up +in a `finally`, because live they are real. + +**One move rules a spec out of here: a reload.** The fixture backend keeps writes in the +page's own memory, so a reload starts a backend that has never heard of what was just +created. Where a claim needs one, it splits: `shared/schedules/` clicks through the whole +journey on either backend, and `live/schedules.spec.ts` keeps the one thing only a real +backend can answer — that the values survive a re-read, since everything short of that +could be the form showing itself its own draft. It is laid out like `tests/`: one folder per resource holding one spec holding one test, titled for its folder, and app-wide specs (`dashboard`, `routing`) at the root. diff --git a/ui/playwright/live/schedules.spec.ts b/ui/playwright/live/schedules.spec.ts index 4497782740..f9ae5fa5e4 100644 --- a/ui/playwright/live/schedules.spec.ts +++ b/ui/playwright/live/schedules.spec.ts @@ -1,151 +1,117 @@ import { test, expect } from "../fixtures/test"; import { loadApp, throwawayName } from "../helpers/app"; -import { liveRoutes } from "./helpers/live"; import { tick } from "../helpers/controls"; import { optionNamed, pressUntil } from "../helpers/resource"; +import { liveRoutes } from "./helpers/live"; /** - * A schedule's configuration, round-tripped through a real controller. Kept paused — - * execution is covered by the Go scheduling E2Es with a controlled model. + * A schedule survives a reload — which is the one claim the fixtures cannot answer. + * + * The journey itself is in `shared/schedules/schedules.spec.ts` and runs against both + * backends. What is left here is the part that has to be live: everything a create + * reports back could be the form showing itself its own draft, and only re-reading from + * a backend that actually stores can tell the two apart. The fixture backend keeps + * writes in the page's own memory, so a reload there starts a backend that has never + * heard of the schedule — the one move it cannot survive. * - * The values are the ones a backend drops or rounds: a fractional timeout, a named IANA - * zone, a weekday set. A fixture hands back whatever it was given, so the mock suite - * cannot fail on a controller that truncates `90.001` to `90`; this can. + * The values are chosen to be the ones a controller drops or rounds: a fractional + * timeout stored as seconds plus nanos, a named IANA zone, and a weekday set. * - * It also drove `getByRole("dialog")` for an editor that is a page, and named CI's - * `smoke` agent rather than `setup-cluster.sh`'s — so it could only pass where nothing - * ran it. Both found the first time it was pointed at a cluster. + * Kept paused: execution is covered by the Go scheduling E2Es with a controlled model. */ -test("live: schedule configuration persists through the browser and controller", async ({ - page, -}) => { - const name = throwawayName("schedule"); + +const CREATED = throwawayName("schedule"); + +test("live: a schedule's configuration survives a reload", async ({ page }) => { let detailURL: string | undefined; try { - await test.step("1. the form offers the cluster's own agents", async () => { + await test.step("1. a schedule is created with the values most easily lost", async () => { await loadApp(page, liveRoutes.schedules); await page.getByTestId("schedules-new").click(); - await expect(page).toHaveURL(/\/schedules\/new$/); await page.getByTestId("schedule-agent").click(); - // Whichever agent this install has: which one has nothing to do with the claim. const agent = optionNamed(page).first(); await expect(agent, "the cluster offered no agents to schedule").toBeVisible({ timeout: 30_000, }); await agent.click(); - }); - - await test.step("2. a weekly, zoned, fractionally-timed schedule is described", async () => { - await page.getByTestId("schedule-name").fill(name); + await page.getByTestId("schedule-name").fill(CREATED); await page.getByTestId("schedule-frequency").click(); - // Pressed until the cadence actually changes: the weekday checkboxes only exist - // once the frequency is weekly, so a dropdown click swallowed by the animation - // leaves the next line waiting for controls that are never coming. await pressUntil(optionNamed(page, "Weekly"), () => expect(page.getByTestId("schedule-days")).toBeVisible(), ); - // Monday is already on, so these four make it the whole working week — which the - // app states back as "Weekdays", and which is the reading asserted below. for (const day of ["Tuesday", "Wednesday", "Thursday", "Friday"]) { await tick(page.getByLabel(day, { exact: true })); } - await page.getByTestId("schedule-time").fill("09:00"); - // The time zone is an AutoComplete, so its id is on the wrapper and the caret goes - // in the input inside it. Escape dismisses the zone list, which otherwise sits - // over the fields below. await page.getByTestId("schedule-timezone").locator("input").fill("America/New_York"); await page.keyboard.press("Escape"); - await page.getByTestId("schedule-prompt").fill("Report cluster health."); await page.getByTestId("schedule-timeout").fill("90.001"); - await page.getByTestId("schedule-enabled").uncheck(); - await expect(page.getByTestId("schedule-enabled-note")).toContainText( - "will not run automatically after it is created", - ); - }); - await test.step("3. creating it reaches the controller and lands on its page", async () => { await page.getByTestId("schedule-submit").click(); - await expect(page.getByRole("heading", { name, exact: true })).toBeVisible({ + await expect(page.getByRole("heading", { name: CREATED, exact: true })).toBeVisible({ timeout: 60_000, }); detailURL = page.url(); - await expect(page).toHaveURL(/\/schedules\/[0-9a-f-]+$/); }); - await test.step("4. a reload reads it back from the cluster, unchanged", async () => { - // The reload is the point. Everything above could be the form showing itself its - // own draft; only a re-read says the controller stored it. + await test.step("2. a reload reads it back from the controller, unchanged", async () => { + // The whole spec. Everything above could be the page showing itself what it just + // sent; only a re-read says the controller stored it. await page.reload(); - // Created paused, so the one control whose label flips offers to resume it. await expect(page.getByTestId("schedule-pause")).toHaveText("Resume", { timeout: 60_000, }); await expect(page.getByTestId("schedule-meta")).toContainText("Weekdays at 09:00"); await expect(page.getByTestId("schedule-meta")).toContainText("America/New_York"); - // The fractional second survived. `90.001` is stored as seconds plus nanos, so a - // controller that kept only the seconds would read back "90 seconds" here. + // `90.001` is stored as seconds plus nanos, so a controller that kept only the + // seconds would read back "90 seconds" here. await expect(page.getByTestId("schedule-detail")).toContainText("90.001 seconds"); await expect(page.getByTestId("schedule-detail")).toContainText( "Report cluster health.", ); }); - await test.step("5. the edit form opens on the stored values, not on defaults", async () => { + await test.step("3. and so does an edit", async () => { await page.getByTestId("schedule-edit").click(); - await expect(page.getByTestId("schedule-time")).toHaveValue("09:00", { + await expect(page.getByTestId("schedule-timeout")).toHaveValue("90.001", { timeout: 60_000, }); - await expect(page.getByTestId("schedule-timeout")).toHaveValue("90.001"); - await expect(page.getByTestId("schedule-timezone").locator("input")).toHaveValue( - "America/New_York", - ); - await expect(page.getByTestId("schedule-enabled")).not.toBeChecked(); - // Both ends of the weekday set, so a picker that kept only the last day chosen - // would not pass on one assertion. - await expect(page.getByLabel("Monday", { exact: true })).toBeChecked(); - await expect(page.getByLabel("Friday", { exact: true })).toBeChecked(); - }); - - await test.step("6. an edit is saved and read back", async () => { await page.getByTestId("schedule-prompt").fill("Report unhealthy workloads only."); await page.getByTestId("schedule-submit").click(); - await expect(page.getByRole("heading", { name, exact: true })).toBeVisible({ + await expect(page.getByRole("heading", { name: CREATED, exact: true })).toBeVisible({ timeout: 60_000, }); await page.reload(); - await expect(page.getByTestId("schedule-detail")).toContainText( - "Report unhealthy workloads only.", - { timeout: 60_000 }, - ); - // And the update did not quietly reset what it was not asked to change. - await expect(page.getByTestId("schedule-detail")).toContainText("90.001 seconds"); + const detail = page.getByTestId("schedule-detail"); + await expect(detail).toContainText("Report unhealthy workloads only.", { + timeout: 60_000, + }); + // The update did not quietly reset what it was not asked to change. + await expect(detail).toContainText("90.001 seconds"); }); } finally { - /* - * A real schedule on a real cluster, so a run that dies midway takes it with it. - * Through the UI, the app speaking gRPC-Web with no REST endpoint to call instead. - */ + // A real schedule on a real cluster, so a run that dies midway takes it with it. if (detailURL) { await page.goto(detailURL); const remove = page .getByTestId("schedule-danger") - .getByRole("button", { name: `Delete schedule ${name}`, exact: true }); - await remove.click(); - // Pressed until it takes: a dropped Delete reports as "the page never navigated" - // rather than as a missed click. See `pressUntil`. - await pressUntil( - page.getByRole("dialog", { name: `Delete schedule ${name}?`, exact: true }) - .getByRole("button", { name: "Delete", exact: true }), - () => expect(page).toHaveURL(/\/schedules$/), - ); + .getByRole("button", { name: `Delete schedule ${CREATED}`, exact: true }); + if ((await remove.count()) > 0) { + await remove.click(); + await pressUntil( + page + .getByRole("dialog", { name: `Delete schedule ${CREATED}?`, exact: true }) + .getByRole("button", { name: "Delete", exact: true }), + () => expect(page).toHaveURL(/\/schedules$/), + ); + } } } }); diff --git a/ui/playwright/live/agent-templates.spec.ts b/ui/playwright/shared/agent-templates/agent-templates.spec.ts similarity index 70% rename from ui/playwright/live/agent-templates.spec.ts rename to ui/playwright/shared/agent-templates/agent-templates.spec.ts index 3bea16e32b..102feaa917 100644 --- a/ui/playwright/live/agent-templates.spec.ts +++ b/ui/playwright/shared/agent-templates/agent-templates.spec.ts @@ -1,35 +1,49 @@ -import { test, expect } from "../fixtures/test"; +import { test, expect } from "../../fixtures/test"; import { + LIFECYCLE_TIMEOUT, confirmation, pressOnce, selectFirstOption, selectOption, -} from "../helpers/resource"; +} from "../../helpers/resource"; import { dataRows, expectNoLoadFailure, loadApp, rowNamed, throwawayName, -} from "../helpers/app"; -import { liveRoutes } from "./helpers/live"; +} from "../../helpers/app"; /** - * Creating and deleting an agent template, on a real cluster, through the UI. + * Creating and deleting an agent template, through the UI, on either backend. * * The property it exists for: **admission is the controller's answer, not the form's.** * `admittingHarnesses` is read from the template's *status* and cannot be computed in - * the browser, so a fixture can return any value it likes and the page will draw it. + * the browser, so a fixture can return any value it likes and the page will draw it — + * which is why the claim is worth making against a cluster as well as against fixtures. * - * It replaces `agent-lifecycle.spec.ts`, which drove `/agents/new` — a page removed + * It replaces `live/agent-lifecycle.spec.ts`, which drove `/agents/new` — a page removed * long before, for an agent nobody creates. Nothing ran the suite, so nothing said so. + * + * What stays in `tests/agent-templates/` is the seeded rows, the filter, the sorting, + * editing a template in place, and the empty and failure states. */ /** The one this journey makes and removes. Carries the run, for litter left by a kill. */ const TEMPLATE = throwawayName("template"); const NAMESPACE = "kagent"; +/** What the edit moves the description to, read back off the page afterwards. */ +const DESCRIPTION = "Edited by the shared suite."; + +/* + * A lifecycle is longer than a journey, so it gets its own budget — see + * `LIFECYCLE_TIMEOUT`. It came free while this spec lived in `live/`, where the whole + * run is given two minutes for the cluster's sake; in `shared/` the mock projects apply + * the tight default, and a journey of this length is long rather than stuck. + */ +test.describe.configure({ timeout: LIFECYCLE_TIMEOUT }); -test("live: an agent template is created, admitted and deleted through the UI", async ({ +test("agent templates: one is created, admitted, edited and deleted", async ({ page, }) => { let created = false; @@ -38,7 +52,7 @@ test("live: an agent template is created, admitted and deleted through the UI", try { await test.step("1. the form offers the cluster's own model configurations", async () => { - await loadApp(page, liveRoutes.agentTemplateNew); + await loadApp(page, "/agent-templates/new"); await expectNoLoadFailure(page); await selectOption(page, "template-form-namespace", NAMESPACE); @@ -101,7 +115,17 @@ test("live: an agent template is created, admitted and deleted through the UI", // written for produced for a template that had in fact been created. await page.waitForURL(/\/agents\?.*tab=templates/, { timeout: 60_000 }); created = true; + /* + * And the list comes back narrowed to the namespace that was being worked in. + * Nothing asserted this once, which is how two faults sat on the one line that + * asks for it: `/agent-templates` is a redirect carrying no query string, and the + * list narrows on `ns` while the caller was sending `namespace`. Either alone + * loses the filter, and the page looks reasonable both ways. + */ await expect(page).toHaveURL(new RegExp(`[?&]ns=${NAMESPACE}(&|$)`)); + await expect( + page.getByTestId(`templates-filters-pill-ns-${NAMESPACE}`), + ).toBeVisible(); }); await test.step("4. the row is read back from the cluster", async () => { @@ -133,7 +157,32 @@ test("live: an agent template is created, admitted and deleted through the UI", ); }); - await test.step("6. deleting says what it costs, against the cluster's own count", async () => { + await test.step("6. an edit in place is saved and read back", async () => { + /* + * The write half the create cannot show. A template's name and namespace are its + * ref and cannot change, so the description is what an edit has to move — and + * reading it back off the page after the save is what separates "the backend + * stored it" from "the draft is still on screen". + * + * Editing is a mode of the reading page rather than a separate address, so the + * submit appearing is also the assertion that the same component serves both. + */ + await expect(page.getByTestId("template-submit")).toHaveCount(0); + await page.getByTestId("template-edit").click(); + await expect(page.getByTestId("template-submit")).toBeVisible({ timeout: 60_000 }); + + await page.getByTestId("template-form-description").fill(DESCRIPTION); + await page.getByTestId("template-submit").click(); + + // Back to reading, showing the saved value rather than the draft: a save that did + // not reach the backend would leave the old one here. + await expect(page.getByTestId("template-edit")).toBeVisible({ timeout: 60_000 }); + await expect(page.getByTestId("template-form-description")).toHaveValue( + DESCRIPTION, + ); + }); + + await test.step("7. deleting says what it costs, against the backend's own count", async () => { const remove = page.getByTestId(`delete-${TEMPLATE}`); await expect(remove).toContainText("Delete template"); await remove.click(); @@ -150,7 +199,7 @@ test("live: an agent template is created, admitted and deleted through the UI", ); }); - await test.step("7. confirming removes it, and the re-read list agrees", async () => { + await test.step("8. confirming removes it, and the re-read list agrees", async () => { // Scoped to the visible popconfirm, and pressed once it has stopped arriving — // see `helpers/resource` for what each of those is protecting against. await pressOnce(confirmation(page).getByRole("button", { name: "Delete" })); diff --git a/ui/playwright/shared/schedules/schedules.spec.ts b/ui/playwright/shared/schedules/schedules.spec.ts new file mode 100644 index 0000000000..7df08148a9 --- /dev/null +++ b/ui/playwright/shared/schedules/schedules.spec.ts @@ -0,0 +1,163 @@ +import { test, expect } from "../../fixtures/test"; +import { loadApp, throwawayName } from "../../helpers/app"; +import { tick } from "../../helpers/controls"; +import { + LIFECYCLE_TIMEOUT, + optionNamed, + pressUntil, +} from "../../helpers/resource"; + +/** + * A schedule created, read back, changed and deleted — on either backend. + * + * Kept paused throughout: execution is covered by the Go scheduling E2Es with a + * controlled model, and a suite that waited for a real agent to answer would be + * measuring the model rather than the schedule. + * + * **No reload anywhere in it, deliberately.** The fixture backend keeps writes in the + * page's own memory, so a reload starts a backend that has never heard of the schedule + * — which is why `live/schedules.spec.ts` exists alongside this and owns exactly that + * claim. Here every read is a click-through, which is what a reader does anyway. + */ + +const CREATED = throwawayName("schedule"); + +test.describe.configure({ timeout: LIFECYCLE_TIMEOUT }); + +test("schedules: one is created, read, changed and deleted", async ({ page }) => { + let detailURL: string | undefined; + + try { + await test.step("1. the form offers the backend's own agents", async () => { + await loadApp(page, "/schedules"); + await page.getByTestId("schedules-new").click(); + await expect(page).toHaveURL(/\/schedules\/new(\?|$)/); + + await page.getByTestId("schedule-agent").click(); + // Whichever agent this install has: which one has nothing to do with the claim. + const agent = optionNamed(page).first(); + await expect(agent, "no agents were offered to schedule").toBeVisible({ + timeout: 30_000, + }); + await agent.click(); + }); + + await test.step("2. a weekly, zoned, fractionally-timed schedule is described", async () => { + await page.getByTestId("schedule-name").fill(CREATED); + + await page.getByTestId("schedule-frequency").click(); + // Pressed until the cadence actually changes: the weekday checkboxes only exist + // once the frequency is weekly, so a dropdown click swallowed by the animation + // leaves the next line waiting for controls that are never coming. + await pressUntil(optionNamed(page, "Weekly"), () => + expect(page.getByTestId("schedule-days")).toBeVisible(), + ); + // Monday is already on, so these four make it the whole working week — which the + // app states back as "Weekdays", and which is the reading asserted below. + for (const day of ["Tuesday", "Wednesday", "Thursday", "Friday"]) { + await tick(page.getByLabel(day, { exact: true })); + } + + await page.getByTestId("schedule-time").fill("09:00"); + // The time zone is an AutoComplete, so its id is on the wrapper and the caret goes + // in the input inside it. Escape dismisses the zone list, which otherwise sits + // over the fields below. + await page.getByTestId("schedule-timezone").locator("input").fill("America/New_York"); + await page.keyboard.press("Escape"); + + await page.getByTestId("schedule-prompt").fill("Report cluster health."); + // A fractional timeout, because it is the value a backend most easily rounds off. + await page.getByTestId("schedule-timeout").fill("90.001"); + + await page.getByTestId("schedule-enabled").uncheck(); + await expect(page.getByTestId("schedule-enabled-note")).toContainText( + "will not run automatically after it is created", + ); + }); + + await test.step("3. creating it lands on its own page, showing what was asked for", async () => { + await page.getByTestId("schedule-submit").click(); + await expect(page.getByRole("heading", { name: CREATED, exact: true })).toBeVisible({ + timeout: 60_000, + }); + detailURL = page.url(); + await expect(page).toHaveURL(/\/schedules\/[0-9a-f-]+(\?|$)/); + + // Created paused, so the one control whose label flips offers to resume it. + await expect(page.getByTestId("schedule-pause")).toHaveText("Resume"); + await expect(page.getByTestId("schedule-meta")).toContainText("Weekdays at 09:00"); + await expect(page.getByTestId("schedule-meta")).toContainText("America/New_York"); + // The fractional second survived the round trip rather than being floored to 90. + await expect(page.getByTestId("schedule-detail")).toContainText("90.001 seconds"); + }); + + await test.step("4. the edit form opens on the stored values, not on defaults", async () => { + await page.getByTestId("schedule-edit").click(); + await expect(page.getByTestId("schedule-time")).toHaveValue("09:00", { + timeout: 60_000, + }); + await expect(page.getByTestId("schedule-timeout")).toHaveValue("90.001"); + await expect(page.getByTestId("schedule-timezone").locator("input")).toHaveValue( + "America/New_York", + ); + await expect(page.getByTestId("schedule-enabled")).not.toBeChecked(); + // Both ends of the weekday set, so a picker that kept only the last day chosen + // would not pass on one assertion. + await expect(page.getByLabel("Monday", { exact: true })).toBeChecked(); + await expect(page.getByLabel("Friday", { exact: true })).toBeChecked(); + }); + + await test.step("5. an edit is saved and read back", async () => { + await page.getByTestId("schedule-prompt").fill("Report unhealthy workloads only."); + await page.getByTestId("schedule-submit").click(); + await expect(page.getByRole("heading", { name: CREATED, exact: true })).toBeVisible({ + timeout: 60_000, + }); + + const detail = page.getByTestId("schedule-detail"); + await expect(detail).toContainText("Report unhealthy workloads only.", { + timeout: 60_000, + }); + // And the update did not quietly reset what it was not asked to change. + await expect(detail).toContainText("90.001 seconds"); + }); + + await test.step("6. deleting asks in a modal, and confirming leaves for the list", async () => { + const remove = page + .getByTestId("schedule-danger") + .getByRole("button", { name: `Delete schedule ${CREATED}`, exact: true }); + await remove.click(); + + // Pressed until it takes: a dropped Delete reports as "the page never navigated" + // rather than as a missed click. See `pressUntil`. + await pressUntil( + page + .getByRole("dialog", { name: `Delete schedule ${CREATED}?`, exact: true }) + .getByRole("button", { name: "Delete", exact: true }), + () => expect(page).toHaveURL(/\/schedules(\?|$)/), + ); + detailURL = undefined; + + await expect(page.getByRole("link", { name: CREATED, exact: true })).toHaveCount(0, { + timeout: 60_000, + }); + }); + } finally { + // Live these are real resources, so a run that dies midway takes its own with it. + if (detailURL) { + await page.goto(detailURL); + const remove = page + .getByTestId("schedule-danger") + .getByRole("button", { name: `Delete schedule ${CREATED}`, exact: true }); + if ((await remove.count()) > 0) { + await remove.click(); + await pressUntil( + page + .getByRole("dialog", { name: `Delete schedule ${CREATED}?`, exact: true }) + .getByRole("button", { name: "Delete", exact: true }), + () => expect(page).toHaveURL(/\/schedules(\?|$)/), + ); + } + } + } +}); diff --git a/ui/playwright/tests/agent-templates/agent-templates.spec.ts b/ui/playwright/tests/agent-templates/agent-templates.spec.ts index 93a3c8e45c..6850ac6c8f 100644 --- a/ui/playwright/tests/agent-templates/agent-templates.spec.ts +++ b/ui/playwright/tests/agent-templates/agent-templates.spec.ts @@ -2,7 +2,6 @@ import { test, expect } from "../../fixtures/test"; import { dataRows, expectSettled, loadPage, rowNamed, routes } from "../../helpers/app"; import { LIFECYCLE_TIMEOUT, - confirmation, expectRequired, pressOnce, selectOption, @@ -14,6 +13,12 @@ import { * One test, because a video and a trace are recorded per *test* — see * `playwright/README.md`. * + * **The write journey is not all here.** Creating a template and removing it is asserted + * against both backends from `shared/agent-templates/agent-templates.spec.ts`, along with + * the edit. What is left is the reading — the seeded rows, the narrowing, the sorting — + * the form's refusal of an unusable template, the two branches of the delete warning, and + * the empty and failure states. + * * ## The property this spec exists for * * **A template no harness admits cannot be used, and nothing about it looks wrong.** A @@ -48,7 +53,14 @@ import { * it; the recording shows where. */ -/** The one this journey makes, reads, edits and removes. */ +/** + * The one this journey makes and edits — setup rather than the claim. + * + * Creating and deleting a template is asserted against both backends from + * `shared/agent-templates/`, so the delete is gone from here. The create stays because + * steps 6 to 10 need a template of their own to read and edit: the only seeded one with + * a harness is `k8s-agent-7f3a91c`, and step 12 asserts its description unedited. + */ const CREATED = "browser-made"; /* @@ -58,7 +70,7 @@ const CREATED = "browser-made"; */ test.describe.configure({ timeout: LIFECYCLE_TIMEOUT }); -test("agent templates: a template is created, read, edited and deleted", async ({ +test("agent templates: the list reads, and a template is read and edited", async ({ page, }) => { await test.step("1. the list says which templates anything will actually run", async () => { @@ -287,58 +299,15 @@ test("agent templates: a template is created, read, edited and deleted", async ( ); }); - await test.step("11. deleting says what it costs, in the confirmation", async () => { - // Before opening it: the consequence is nowhere on the page. That is the half of - // this property the confirmation itself cannot demonstrate — a warning a reader can - // walk past on the way to the button is a warning they will walk past. - await expect(page.locator("body")).not.toContainText("keep working"); - - // In the header, beside Edit and Back, rather than at the foot of the page. A - // destructive action a reader only reaches by scrolling past everything else reads - // as a footnote. - const deleteButton = page.getByTestId(`delete-${CREATED}`); - await expect(deleteButton).toContainText("Delete template"); - await deleteButton.click(); - + await test.step("11. a template nothing runs says that instead", async () => { /* - * Measured against the controller, not read off the schema. A scratch template with - * a live pair was deleted over gRPC on a cluster: the call was accepted, the - * resource went, and the `agent_template_harness_pair` row survived in Postgres with - * `retired_at` set — retired, not removed. The revision collector skips any revision - * an `agent_instance.prepared_revision` points at before the `ON DELETE RESTRICT` on - * that column could fire, so an agent's revision is retained *for it*; and - * `GetLatestRuntimeRevisionForInstance` requires `retired_at IS NULL`, which is what - * stops anything new being cut from the template afterwards. - * - * **What it must not say is that the agents keep running.** A (template, harness) - * pair *is* an agent here, and deleting the template retires the pair — that is - * exactly the mechanism that stops new work. What survives is the conversations - * already open, each holding a revision retained for it. + * Back to the list first. The delete that used to sit here navigated back as a side + * effect of removing the template, and it moved to `shared/agent-templates/` — so + * the return trip is now this step's own business rather than something it inherits. */ - const consequence = page.getByTestId("template-delete-consequence"); - await expect(consequence).toContainText("1 agent is built from this template"); - await expect(consequence).toContainText( - "Conversations already open with it keep working", - ); - await expect(consequence).toContainText("no new one can be started"); - }); - - await test.step("12. confirming removes it, and the list that opens does not show it", async () => { - // Scoped to the visible popconfirm: every row's confirmation is in the DOM at once, - // so an unscoped Delete can answer a prompt nobody is looking at. - await confirmation(page).getByRole("button", { name: "Delete" }).click(); - await page.waitForURL(/\/agents\?.*tab=templates/, { timeout: 30_000 }); - - // The claim worth making. The list is cached, so landing on it without re-reading - // shows the template that was just removed — which reads as a delete that silently - // failed, and is the reason the page invalidates before navigating. - await expect(rowNamed(page, CREATED)).toHaveCount(0, { timeout: 30_000 }); - // And the rest of the list is intact, so "gone" means that one rather than the read. - await expect(rowNamed(page, "k8s-agent-7f3a91c")).toBeVisible(); - await expect(page).toHaveURL(/[?&]ns=kagent(&|$)/); - }); + await page.getByRole("button", { name: "Back to templates" }).click(); + await page.waitForURL(/\/agents\?.*tab=templates/); - await test.step("13. a template nothing runs says that instead", async () => { // The other branch of the same sentence. Telling a reader that conversations will // keep working when no harness ever admitted the template would be noise dressed as // care. @@ -352,7 +321,7 @@ test("agent templates: a template is created, read, edited and deleted", async ( ); }); - await test.step("14. an agent in the Agents tab opens that agent", async () => { + await test.step("12. an agent in the Agents tab opens that agent", async () => { // The tab answers "what is built from this template", and each answer is a // (template, harness) pair — which is what an agent is. Leaving the rows as text // made it a dead end: it named the thing the reader wanted and gave them no way to @@ -393,7 +362,7 @@ test("agent templates: a template is created, read, edited and deleted", async ( await expect(page.getByTestId("chat-new-session")).toBeVisible({ timeout: 30_000 }); }); - await test.step("15. an empty result says so instead of showing a bare table", async () => { + await test.step("13. an empty result says so instead of showing a bare table", async () => { // Last, after the delete, because reaching these needs the backend answering // differently and `?mock=` is per-navigation — which discards what the journey made. // By here there is nothing left to discard. @@ -402,7 +371,7 @@ test("agent templates: a template is created, read, edited and deleted", async ( await expect(dataRows(page)).toHaveCount(0); }); - await test.step("16. a failed load is reported, not disguised as an empty list", async () => { + await test.step("14. a failed load is reported, not disguised as an empty list", async () => { await loadPage(page, routes.agentTemplates, { scenario: "error", title: "Agents" }); const alert = page.getByTestId("templates-error"); diff --git a/ui/playwright/tests/schedules/schedules.spec.ts b/ui/playwright/tests/schedules/schedules.spec.ts index 335bbbabad..0742200cc6 100644 --- a/ui/playwright/tests/schedules/schedules.spec.ts +++ b/ui/playwright/tests/schedules/schedules.spec.ts @@ -1,9 +1,13 @@ import { test, expect } from "../../fixtures/test"; -import { tick } from "../../helpers/controls"; -import { LIFECYCLE_TIMEOUT, optionNamed, pressUntil } from "../../helpers/resource"; +import { LIFECYCLE_TIMEOUT, optionNamed } from "../../helpers/resource"; /** - * Schedules — the whole life of one, in a single journey. + * Schedules — reading one, running it, and the states around that. + * + * **The write journey is not here.** Creating a schedule, reading it back, changing it + * and deleting it runs against both backends from `shared/schedules/schedules.spec.ts`; + * the claim that it survives a reload — which the fixtures structurally cannot answer, + * keeping writes in the page's own memory — is `live/schedules.spec.ts`. * * One test, because a video and a trace are recorded per *test* — see * `playwright/README.md`. @@ -35,8 +39,8 @@ import { LIFECYCLE_TIMEOUT, optionNamed, pressUntil } from "../../helpers/resour * ## Why it clicks through rather than navigating * * The mock backend keeps writes in the page's own memory, so a `page.goto` starts a - * backend that has never heard of the schedule just made. Everything from step 4 onwards - * therefore clicks, and the created schedule survives to the delete at the end. + * backend that has never heard of a change just made. The steps below therefore click + * through rather than navigate wherever a write has to outlive the step that made it. */ /** `Daily cluster report`, the seeded schedule the read half is asserted against. */ @@ -45,10 +49,6 @@ const SEEDED = "c686bd1d-9124-4e96-8df7-000000000001"; /** A schedule deleted before the fixtures were written, kept for its history. */ const RETIRED = "c686bd1d-9124-4e96-8df7-000000000004"; -/** The one this journey makes, reads, renames and removes. */ -const CREATED = "Probe alpha"; -const RENAMED = "Probe beta"; - /* * A lifecycle is longer than a journey, so it gets its own budget — see * `LIFECYCLE_TIMEOUT`. Set per file rather than across the suite, so the tight default @@ -56,7 +56,7 @@ const RENAMED = "Probe beta"; */ test.describe.configure({ timeout: LIFECYCLE_TIMEOUT }); -test("schedules: a schedule is created, read, run, changed and deleted", async ({ +test("schedules: a schedule is read, run, paused, and its failures reported", async ({ page, }) => { const rows = page.getByRole("row"); @@ -255,184 +255,7 @@ test("schedules: a schedule is created, read, run, changed and deleted", async ( await expect(page.getByText("Choose an agent.", { exact: true })).toBeVisible(); }); - await test.step("11. a filled-in schedule is created and lands on its own page", async () => { - await page.getByTestId("schedule-agent").click(); - await optionNamed(page, "kagent/k8s-agent-7f3a91c on k8s-agent").click(); - await page.getByTestId("schedule-name").fill(CREATED); - - // The picker, not the raw expression: a weekly cadence is stated back in words, so a - // reader can tell the schedule they described from the one they got. - await expect(page.getByTestId("schedule-cron")).toHaveCount(0); - await page.getByTestId("schedule-frequency").click(); - // Pressed until the cadence actually changes: the weekday checkboxes only exist - // once the frequency is weekly, so a dropdown click swallowed by the animation - // leaves the next line waiting for a control that is never coming. - await pressUntil(optionNamed(page, "Weekly"), () => - expect(page.getByTestId("schedule-days")).toBeVisible(), - ); - await page.getByTestId("schedule-time").fill("08:00"); - await tick(page.getByLabel("Wednesday", { exact: true })); - await expect(page.getByTestId("schedule-cadence")).toHaveText( - "Weekly on Monday, Wednesday at 08:00 (UTC)", - ); - - await page.getByTestId("schedule-prompt").fill("Check the probe."); - await page.getByTestId("schedule-timeout").fill("120"); - - // Enabled by default, and the sentence underneath changes with it — which is the - // only thing on screen that says whether creating this starts it running. - await expect(page.getByTestId("schedule-enabled")).toBeChecked(); - await expect(page.getByTestId("schedule-enabled-note")).toContainText( - "will run automatically after it is created", - ); - await page.getByTestId("schedule-enabled").uncheck(); - await expect(page.getByTestId("schedule-enabled-note")).toContainText( - "will not run automatically after it is created", - ); - - await page.getByTestId("schedule-submit").click(); - await expect( - page.getByRole("heading", { name: CREATED, exact: true }), - ).toBeVisible(); - await expect(page.getByTestId("schedule-meta")).toContainText( - "Weekly on Monday, Wednesday at 08:00", - ); - // The record below the header, not the pills beside the name: `schedule-meta` - // carries the cadence and the clock, and the timeout is one of its fields. - await expect(page.getByTestId("schedule-detail")).toContainText("120 seconds"); - }); - - await test.step("12. the list is the proof, with one more row", async () => { - // A closed form and a redirect only prove the app believes it worked. - await page.getByTestId("schedule-back").click(); - await expect(rowNamed(CREATED)).toHaveCount(1); - await expect(rowNamed(CREATED)).toContainText("Weekly on Monday, Wednesday"); - }); - - await test.step("13. an edit from the list renames it, rather than duplicating it", async () => { - const before = await rows.count(); - - await page.getByTestId(`edit-${CREATED}`).click(); - await expect( - page.getByRole("heading", { name: `Edit ${CREATED}`, exact: true }), - ).toBeVisible(); - // The draft opens on what was saved, including the switch that was turned off and - // both chosen days — a picker that kept only the last one would look right here with - // one assertion. - await expect(page.getByTestId("schedule-enabled")).not.toBeChecked(); - await expect(page.getByTestId("schedule-time")).toHaveValue("08:00"); - await expect(page.getByLabel("Monday", { exact: true })).toBeChecked(); - await expect(page.getByLabel("Wednesday", { exact: true })).toBeChecked(); - - // And the sentence says "saved" here where the create form said "created". Same - // switch, different consequence, and the wording is the only thing on screen that - // distinguishes them. - await expect(page.getByTestId("schedule-enabled-note")).toContainText( - "will not run automatically after it is saved", - ); - await tick(page.getByTestId("schedule-enabled")); - await expect(page.getByTestId("schedule-enabled-note")).toContainText( - "will run automatically after it is saved", - ); - - await page.getByTestId("schedule-name").fill(RENAMED); - // The time zone is an AutoComplete, so its id is on the wrapper and the caret goes - // in the input inside it. Every other field here carries its id on the control. - await page.getByTestId("schedule-timezone").locator("input").fill("Europe/Berlin"); - // The zone list is an autocomplete; dismiss it so it is not over the form. - await page.keyboard.press("Escape"); - await page.getByTestId("schedule-submit").click(); - await expect( - page.getByRole("heading", { name: RENAMED, exact: true }), - ).toBeVisible(); - - await page.getByTestId("schedule-back").click(); - await expect(rowNamed(RENAMED)).toHaveCount(1); - await expect(rowNamed(RENAMED)).toContainText("Europe/Berlin"); - // Renamed, not duplicated. - await expect(rowNamed(CREATED)).toHaveCount(0); - await expect(rows).toHaveCount(before); - }); - - await test.step("14. deleting asks first, navigates nowhere, and Keep leaves it", async () => { - await page - .getByRole("button", { name: `Delete schedule ${RENAMED}`, exact: true }) - .click(); - await expect(page.getByRole("button", { name: "Keep", exact: true })).toBeVisible(); - await expect(page).toHaveURL(/\/schedules(\?.*)?$/); - await pressUntil(page.getByRole("button", { name: "Keep", exact: true }), () => - expect(page.getByRole("button", { name: "Keep", exact: true })).toBeHidden(), - ); - await expect(rowNamed(RENAMED)).toHaveCount(1); - }); - - await test.step("15. and the delete on its own page asks in a modal, which Keep dismisses", async () => { - /* - * The other delete surface, and a different control: the list asks in a popconfirm - * beside the row, while the page about one schedule asks in a modal from its danger - * zone. Both are reached here rather than only the list one, because the two are - * separate call sites and it is the page-level one that carries the sentence saying - * what deleting costs. - */ - await page.getByTestId(`schedule-link-${RENAMED}`).click(); - await expect(page).toHaveURL(/\/schedules\/[0-9a-f-]+$/); - - await page - .getByTestId("schedule-danger") - .getByRole("button", { name: `Delete schedule ${RENAMED}`, exact: true }) - .click(); - // Titled with the schedule's name: "Delete this schedule?" is no help to somebody - // who arrived here from a list of four of them. - const confirmation = page.getByRole("dialog", { - name: `Delete schedule ${RENAMED}?`, - exact: true, - }); - await expect(confirmation).toContainText("Stops future executions."); - await pressUntil(confirmation.getByRole("button", { name: "Keep", exact: true }), () => - expect(confirmation).toBeHidden(), - ); - // Still usable afterwards, so a dismissed confirmation leaves no disabled page. - await expect(page.getByTestId("schedule-run")).toBeEnabled(); - }); - - await test.step("16. confirming removes it, leaves for the list, and the rest stays", async () => { - await page - .getByTestId("schedule-danger") - .getByRole("button", { name: `Delete schedule ${RENAMED}`, exact: true }) - .click(); - /* - * Pressed until it takes: a Delete click dropped on Firefox reports as "the page - * never navigated" rather than as a missed click. See `pressUntil`. - * - * Deleting leaves for the list, which is where the reader can act next — this page - * is now about a schedule that is gone — so the navigation is what proves the press - * landed. - */ - await pressUntil( - page - .getByRole("dialog", { name: `Delete schedule ${RENAMED}?`, exact: true }) - .getByRole("button", { name: "Delete", exact: true }), - () => expect(page).toHaveURL(/\/schedules(\?.*)?$/), - ); - await expect(rowNamed(RENAMED)).toHaveCount(0); - // One row went, not the table. - await expect( - page.getByTestId("schedule-link-Daily cluster report"), - ).toBeVisible(); - }); - - /* - * The states that need the backend answering differently, folded in here rather than - * kept as a second test. - * - * They were split out on the reasoning that a `page.goto` resets the fixture backend - * and would throw away the schedule the lifecycle is holding. True, and beside the - * point once they run *last*: by here the schedule has been deleted and there is - * nothing left to lose. `models`, `mcp-servers` and `prompts` all end the same way, - * and this file reading differently from them was the contradiction rather than the - * reset. - */ - await test.step("17. a read failure is not an empty list", async () => { + await test.step("11. a read failure is not an empty list", async () => { await page.goto("/schedules?mock=error"); await expect(page.getByTestId("schedules-error")).toContainText( "Could not load schedules", @@ -442,7 +265,7 @@ test("schedules: a schedule is created, read, run, changed and deleted", async ( await expect(page.getByTestId("schedules-empty")).toHaveCount(0); }); - await test.step("18. and an empty list says so plainly, with nothing to scroll", async () => { + await test.step("12. and an empty list says so plainly, with nothing to scroll", async () => { await page.goto("/schedules?mock=empty"); await expect(page.getByTestId("schedules-empty")).toBeVisible(); @@ -455,7 +278,7 @@ test("schedules: a schedule is created, read, run, changed and deleted", async ( expect(overflows).toBe(false); }); - await test.step("19. a link held from before a delete still opens, and says what it is", async () => { + await test.step("13. a link held from before a delete still opens, and says what it is", async () => { // The executions are retained, so the address is not a 404 — and must not render as // a live schedule either, or a reader will try to act on one that is gone. await page.goto(`/schedules/${RETIRED}?mock=ok`); @@ -469,7 +292,7 @@ test("schedules: a schedule is created, read, run, changed and deleted", async ( await expect(page.getByTestId("schedule-meta")).toContainText("Deleted"); }); - await test.step("20. and it is not offered in the list it was removed from", async () => { + await test.step("14. and it is not offered in the list it was removed from", async () => { await page.goto("/schedules?mock=ok"); await expect( page.getByTestId("schedule-link-Retired sweep"), From 878a0e05b048794e3e031797b31544b11024789f Mon Sep 17 00:00:00 2001 From: Torey Scheer Date: Wed, 16 Sep 2026 16:42:42 -0400 Subject: [PATCH 04/24] docs(ui): correct and shorten the mock specs' headers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moving each write journey to `shared/` left five headers describing a spec that no longer exists, and three were stale beyond the opening line: - `agent-templates` said the created template "survives all the way to the delete that removes it" — that delete moved two commits ago. - `harnesses` carried both "There is no update half" and "Nor is the write journey here", two ways of saying the same thing once the trim had landed. - `routing` said the login route needs something to log in to. It does not: the route was tried against a deployment and passes. It stays on the fixtures for a different reason, which the header now gives — `shared/routing` already proves the SPA fallback on `/substrate` and an unknown path, so `/login` earns nothing by running twice. Shorter as well as truer, 130 lines out for 65 in. What each keeps is the knowledge that cannot be recovered from the code — why a prompt save replaces the whole ConfigMap `data` map, why `ready: false` must not be drawn as "broken", why a schedule's history outlives the schedule, and the cluster observation behind the agent-template admission property. What went was boilerplate: every one of them repeated that a video and a trace are recorded per test, which `README.md` says. Comments only; no assertion, selector or title changed. 173 mock specs pass. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Torey Scheer --- .../agent-templates/agent-templates.spec.ts | 58 ++++++------------- .../tests/harnesses/harnesses.spec.ts | 41 +++++-------- ui/playwright/tests/models/models.spec.ts | 16 ++--- ui/playwright/tests/prompts/prompts.spec.ts | 26 ++++----- ui/playwright/tests/routing.spec.ts | 10 ++-- .../tests/schedules/schedules.spec.ts | 44 +++++--------- 6 files changed, 65 insertions(+), 130 deletions(-) diff --git a/ui/playwright/tests/agent-templates/agent-templates.spec.ts b/ui/playwright/tests/agent-templates/agent-templates.spec.ts index 6850ac6c8f..56ed2877c6 100644 --- a/ui/playwright/tests/agent-templates/agent-templates.spec.ts +++ b/ui/playwright/tests/agent-templates/agent-templates.spec.ts @@ -8,49 +8,25 @@ import { } from "../../helpers/resource"; /** - * Agent templates — the whole life of one, in a single journey. + * Agent templates, on the fixtures. Creating, editing and removing one runs against both + * backends from `shared/agent-templates/`; what is left is the reading — the seeded rows, + * the narrowing, the sorting — the form's refusal of an unusable template, both branches + * of the delete warning, and the empty and failure states. The create below is setup + * rather than a claim; see the note on `CREATED`. * - * One test, because a video and a trace are recorded per *test* — see - * `playwright/README.md`. + * **The property this spec exists for: a template no harness admits cannot be used, and + * nothing about it looks wrong.** A harness admits through a label selector, so a template + * whose labels match nothing reaches no prepared revision and every `CreateAgentInstance` + * naming it is refused — while still having a model, a prompt, and a row in this list. + * Confirmed against a cluster before any of this was built: an unlabelled template sat at + * `status: {observedGeneration: 1}` with no harnesses at all, and adding the one label its + * harness selects on took it to a ready golden snapshot in about ten seconds. So the "Runs + * on" column, the warning in the form and the button that applies a harness's labels are + * the feature rather than decoration. * - * **The write journey is not all here.** Creating a template and removing it is asserted - * against both backends from `shared/agent-templates/agent-templates.spec.ts`, along with - * the edit. What is left is the reading — the seeded rows, the narrowing, the sorting — - * the form's refusal of an unusable template, the two branches of the delete warning, and - * the empty and failure states. - * - * ## The property this spec exists for - * - * **A template no harness admits cannot be used, and nothing about it looks wrong.** A - * `Harness` admits templates through a label selector, and the CRD is explicit that a - * harness with no selector admits none — so a template whose labels match nothing - * reaches no prepared revision and every `CreateAgentInstance` naming it is refused. It - * still has a model, a prompt, a row in this list. - * - * That was confirmed against a cluster before any of this was built: an unlabelled - * template sat at `status: {observedGeneration: 1}` with no harnesses at all, and adding - * the one label its harness selects on took it to *"ActorTemplate golden snapshot is - * ready"* in about ten seconds. - * - * So the "Runs on" column, the warning in the form and the button that applies a - * harness's labels are the feature, not decoration — and they are what steps 1, 4 and 5 - * cover. - * - * ## The second property - * - * **Reading a template is not the same act as changing one.** A row opens a details - * page with editing as a mode rather than a page of inputs with Save waiting, which is - * why steps 6 and - * 7 assert the *reading* state as well as the writing one — and why they assert both are - * the same component, since a separate read-only view is what would drift. - * - * ## One thing the ordering buys, and one it costs - * - * The mock backend keeps writes in the page's own memory, so a `page.goto` starts a - * backend that has never heard of the template just made. Everything from step 5 onwards - * therefore clicks through rather than navigating, and the created template survives all - * the way to the delete that removes it. The cost is that a failure stops the steps after - * it; the recording shows where. + * **Reading a template is not the same act as changing one.** A row opens a details page + * with editing as a mode rather than a page of inputs with Save waiting, and both states + * are asserted on the same component, since a separate read-only view is what would drift. */ /** diff --git a/ui/playwright/tests/harnesses/harnesses.spec.ts b/ui/playwright/tests/harnesses/harnesses.spec.ts index c9d40e6273..b66fb17a98 100644 --- a/ui/playwright/tests/harnesses/harnesses.spec.ts +++ b/ui/playwright/tests/harnesses/harnesses.spec.ts @@ -7,39 +7,24 @@ import { } from "../../helpers/resource"; /** - * Harnesses — the whole life of one, in a single journey. - * - * One test, because a video and a trace are recorded per *test* — see - * `playwright/README.md`. - * - * **There is no update half.** The tab offers create and delete and no edit. - * - * **Nor is the write journey here.** Creating a harness, reading back that it is "not - * ready yet" and removing it runs against both backends from - * `playwright/shared/harnesses/harnesses.spec.ts`. What stays is the reading — the seeded - * rows, the selector on the page, the narrowing — and the two refusals below, which are - * about the form enforcing the cluster's constraints rather than about creating anything. - * - * ## What the tab exists to say + * Harnesses, on the fixtures. The tab offers create and delete and no edit, and that + * journey runs against both backends from `shared/harnesses/`. What stays is the reading + * — the seeded rows, the selector on the page, the narrowing — and the two refusals + * below, which are the form enforcing the cluster's constraints rather than creating + * anything. * * **The admission selector has to be visible.** A harness admits templates through a - * label selector, and that selector is what decides whether a template ever becomes an - * agent at all. A template carrying no label it matches saves happily and then does - * nothing, with nothing on screen explaining why — so the selector is on the page rather - * than behind an expander. + * label selector, and that selector decides whether a template ever becomes an agent at + * all. One carrying no label it matches saves happily and then does nothing, with nothing + * on screen explaining why. * * **A harness must not be called broken.** `ready: false` also covers one the controller - * has not observed yet, which is a different thing from one that failed — and the - * `kagent` harness on the development cluster is exactly that: it runs agents and carries - * `status: null`. Calling that "broken" sends somebody debugging a harness that works. - * - * ## Why the form is short + * has not observed yet, which is a different thing from one that failed — the `kagent` + * harness on a development cluster runs agents and carries `status: null`. * - * The CRD is strict, and the constraints the form enforces are the cluster's rather than - * this page's: exactly one runtime adapter, an image pinned by digest, and a worker pool - * for the Substrate Actors to be scheduled onto. A form that accepted a tag would build a - * resource the cluster rejects — the failure that is invisible until somebody tries it - * for real, which is why the fixture refuses it too. + * **The form is short because the CRD is strict**: exactly one runtime adapter, an image + * pinned by digest, and a worker pool to schedule onto. A form that accepted a tag would + * build a resource the cluster rejects, which is why the fixture refuses it too. */ /** The name the validation steps type in. Nothing is created here — see the note above. */ diff --git a/ui/playwright/tests/models/models.spec.ts b/ui/playwright/tests/models/models.spec.ts index 822ad48c92..2944b9d6a2 100644 --- a/ui/playwright/tests/models/models.spec.ts +++ b/ui/playwright/tests/models/models.spec.ts @@ -16,17 +16,13 @@ import { import { operationCalls, rpc } from "../../helpers/mockCalls"; /** - * Model configurations — the whole life of one, in a single journey. + * Model configurations, on the fixtures. * - * One test, because a video and a trace are recorded per *test* — see - * `playwright/README.md` for the shape and the trade it makes. - * - * **The write journey is not here.** Creating a configuration, reading it back, changing - * its credential and deleting it runs against both backends from - * `playwright/shared/models/models.spec.ts`, which counts rows relative to what the list - * already held. What is left here is what only fixed data can settle: the seeded rows and - * their refs taken apart, the refresh confirmation, the filter in the address, the - * required-field marks, and the empty and failure states. + * The write journey is not here: creating one, reading it back, changing its credential + * and deleting it runs against both backends from `shared/models/`. What is left is what + * only fixed data can settle — the seeded rows and their refs taken apart, the refresh + * confirmation, the filter in the address, the required-field marks, and the empty and + * failure states. */ /** The four seeded configurations, which is what "nothing narrowed" has to mean. */ diff --git a/ui/playwright/tests/prompts/prompts.spec.ts b/ui/playwright/tests/prompts/prompts.spec.ts index dc90db5928..beeb5af8da 100644 --- a/ui/playwright/tests/prompts/prompts.spec.ts +++ b/ui/playwright/tests/prompts/prompts.spec.ts @@ -19,25 +19,21 @@ import { import { operationCallCounts, operationCalls, rpc } from "../../helpers/mockCalls"; /** - * Prompt libraries — the whole life of one, in a single journey. + * Prompt libraries, on the fixtures. The write journey is in `shared/prompts/`; what is + * left is the seeded libraries, the namespace filter, the discard prompts, and the empty + * and failure states. * - * One test, because a video and a trace are recorded per *test* — see - * `playwright/README.md`. - * - * ## The two things a fragment list has that other resources do not + * Two things a fragment list has that other resources do not: * * **A save replaces the library.** `UpdatePromptTemplate` assigns the ConfigMap's whole - * `data` map, so removing a row deletes a fragment and two rows sharing a key silently - * merge into one. Nothing on screen would tell a reader that, so the form says it and - * refuses both — asserted in step 8, before the save that would otherwise lose prose - * somebody wrote. + * `data` map, so removing a row deletes a fragment and two rows sharing a key merge + * silently. The form says so and refuses both — step 8, before the save that would lose + * prose somebody wrote. * - * **The reads are scoped per namespace.** `ListPromptTemplates` requires a namespace - * and offers no wildcard, so `usePrompts` fans out one call per namespace rather than - * narrowing something already fetched. That makes the filter in step 5 a claim about - * what was *asked for*, and it is also why the timeouts here are longer than the - * default: under the slow scenario the page waits on the namespace list and then on one - * call per namespace. + * **The reads are scoped per namespace.** `ListPromptTemplates` requires one and offers + * no wildcard, so `usePrompts` fans out a call per namespace rather than narrowing + * something already fetched. That makes step 5 a claim about what was *asked for*, and is + * why the timeouts here are longer than the default. */ /** The two seeded libraries. */ diff --git a/ui/playwright/tests/routing.spec.ts b/ui/playwright/tests/routing.spec.ts index 5e2ff2f39d..8166751215 100644 --- a/ui/playwright/tests/routing.spec.ts +++ b/ui/playwright/tests/routing.spec.ts @@ -3,13 +3,13 @@ import { agentChat, instances, loadPage, expectPageTitle, routes } from "../help import { expectNoShell, expectShell } from "../helpers/nav"; /** - * Routing, the two parts of it that need fixtures. + * Routing, the two parts of it kept off the shared suite. * * A deep link carrying an `AgentInstance` id has to name a conversation that exists, - * and the login route has to have something to log in to — neither of which a clean - * cluster supplies. The rest of the journey is backend-agnostic and runs against both - * from `shared/routing.spec.ts`, including the deep link and 404 steps, which are the - * ones a server can get wrong. + * which a clean cluster has not got. The login route is portable — it was tried against + * a deployment and passes — but `shared/routing.spec.ts` already proves the SPA fallback + * on `/substrate` and an unknown path, and `/login` is a third instance of that same + * claim, so it earns nothing by running twice. */ test("routing: a deep link with params, and the standalone login route", async ({ diff --git a/ui/playwright/tests/schedules/schedules.spec.ts b/ui/playwright/tests/schedules/schedules.spec.ts index 0742200cc6..6904fed1fd 100644 --- a/ui/playwright/tests/schedules/schedules.spec.ts +++ b/ui/playwright/tests/schedules/schedules.spec.ts @@ -4,43 +4,25 @@ import { LIFECYCLE_TIMEOUT, optionNamed } from "../../helpers/resource"; /** * Schedules — reading one, running it, and the states around that. * - * **The write journey is not here.** Creating a schedule, reading it back, changing it - * and deleting it runs against both backends from `shared/schedules/schedules.spec.ts`; - * the claim that it survives a reload — which the fixtures structurally cannot answer, - * keeping writes in the page's own memory — is `live/schedules.spec.ts`. + * The write journey runs against both backends from `shared/schedules/`, and the claim + * that a schedule survives a reload — which the fixtures structurally cannot answer, + * keeping writes in the page's own memory — is `live/schedules.spec.ts`. That same + * memory is why the steps below click through rather than navigate wherever a write has + * to outlive the step that made it. * - * One test, because a video and a trace are recorded per *test* — see - * `playwright/README.md`. + * **A schedule is the only resource here that runs**, so pausing one, invoking it by hand + * while paused, and reading the execution it produced are covered here and nowhere else. * - * ## What is distinctive about this resource, and therefore what is covered - * - * **A schedule is the only resource here that runs.** So the journey covers pausing one, - * invoking it by hand while paused, and reading the execution it produced — none of which - * any other resource has, and all of which is the reason a schedule exists. - * - * **Its cadence has two representations.** The form offers a repeat picker and an - * advanced cron expression, and an expression the picker cannot represent has to survive - * an edit to some other field. That is asserted because it is the one that silently + * **Its cadence has two representations.** An advanced cron expression the repeat picker + * cannot show has to survive an edit to some other field — the case that silently * destroys a reader's work. * * **Its history outlives it.** A deleted schedule still opens by address and says what it - * is, because the executions are retained — so the address is not a 404 and must not be - * rendered as a live schedule either. - * - * ## What is still read as prose, deliberately - * - * Five selectors, and each is the right tool rather than a leftover. Two are fixture - * *data* in a cell — an execution's failure reason, its task id — which is the thing - * under test and has no id to give it. One is a form rule's message. The last two are - * `getByLabel("Monday")` on the weekday checkboxes, which are genuinely labelled - * controls: a label is what a reader clicks and what a screen reader announces, so - * reaching for one is not the same as matching copy. - * - * ## Why it clicks through rather than navigating + * is, so that address is neither a 404 nor a live schedule. * - * The mock backend keeps writes in the page's own memory, so a `page.goto` starts a - * backend that has never heard of a change just made. The steps below therefore click - * through rather than navigate wherever a write has to outlive the step that made it. + * Five selectors read prose deliberately: two are fixture data in a cell with no id to + * give it, one is a form rule's message, and two are `getByLabel` on the weekday + * checkboxes, which are genuinely labelled controls. */ /** `Daily cluster report`, the seeded schedule the read half is asserted against. */ From 811447f7609ab7134521f359fe225a65967a3c85 Mon Sep 17 00:00:00 2001 From: Torey Scheer Date: Wed, 16 Sep 2026 23:34:12 -0400 Subject: [PATCH 05/24] test(ui): close the races the shared suite was carrying MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six defects, most of them only reachable against a backend slower than the fixtures — which is to say, only in CI. **Counts were of rows on screen.** The lists page at 25, so on a cluster holding more than a page the created row is on page two: present, correct, and invisible to a locator. The totals now come off the `-summary` line, and `searchList` narrows to the name this run invented so the row is found wherever it landed. The harness tab keeps counting rows, which is right there — it sets `pagination={false}`. **`LIFECYCLE_TIMEOUT` was the tightest budget in the live run.** A `describe.configure` overrides the per-test default, so a shared lifecycle asking for sixty seconds was undercutting live's own hundred and twenty. It is 180s live, 60s mock. **Cleanups skipped their deletes.** A `count()` taken straight after a navigation reads zero because the page has not answered yet, so "is it still there?" always said no and the resource stayed on the cluster. `appeared()` waits first, and returns `false` rather than throwing because a throw in a `finally` replaces the failure the test was reporting. It treats a timeout as an answer and anything else — a locator matching several, raising strict mode — as a broken check, said out loud rather than silently. A mistyped test id cannot be told apart and the comment says so. **The uploaded CI artifact was near-empty.** `github` writes annotations and no files, so the report path had nothing in it and `if-no-files-found: ignore` hid that. The config asks for `html` alongside it. **`LIVE_EXTERNAL_URL` was read three inconsistent ways** — `??`, `!== undefined` and truthiness — so an empty string meant "deployed" to one and "not deployed" to another: it would have pointed `baseURL` at `""`, checked that for nginx, *and* started a dev server nothing would visit. Normalised at the source. **The one-harness admission read was a race.** Those labels are applied in an effect that lands after the buttons first render, so a single read could take the pre-effect state and go down the "nothing admits it" branch on a cluster where something does. The button's disabled state is the settled signal in both directions. Two more worth stating. The live admission assertion leaned on the controller having reconciled before the navigation — that page fetches once on mount, so a longer timeout only re-read a DOM that was never going to change; the reload is the refetch now. And widening the ESLint spec rules from `tests/` to `{tests,shared,live}` found `live/pages` and `live/substrate` importing from `@playwright/test`, so neither had the console-error guard, in the only suite that talks to a cluster. CI gains a gate on the `smoke` template existing. It is applied part-way through the Go step, so a step that died before that apply would fail the browser journeys three times each against a cluster with no agent to schedule. Verified against a kind cluster running the image built from this tree: 13 live and 173 mock specs pass, with no stray resources left behind. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Torey Scheer --- .github/workflows/ci.yaml | 33 +++++-- ui/eslint.config.mjs | 7 +- ui/playwright.config.ts | 23 ++++- ui/playwright/helpers/app.ts | 77 ++++++++++++++- ui/playwright/helpers/resource.ts | 40 +++++++- ui/playwright/live/pages.spec.ts | 2 +- ui/playwright/live/schedules.spec.ts | 11 ++- ui/playwright/live/substrate.spec.ts | 2 +- .../agent-templates/agent-templates.spec.ts | 95 ++++++++++++++++--- .../shared/harnesses/harnesses.spec.ts | 12 ++- ui/playwright/shared/models/models.spec.ts | 27 +++++- ui/playwright/shared/prompts/prompts.spec.ts | 25 ++++- .../shared/schedules/schedules.spec.ts | 5 +- .../agent-templates/agent-templates.spec.ts | 7 +- .../tests/harnesses/harnesses.spec.ts | 5 +- 15 files changed, 319 insertions(+), 52 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index cd654654ba..4abf1836d3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -175,6 +175,19 @@ jobs: export KAGENT_E2E_CLI="$PWD/core/bin/kagent-local" go test -v github.com/kagent-dev/kagent/go/core/test/e2e -failfast -shuffle=on -parallel 4 + # The `smoke` template is applied part-way through the step above, after four + # digest lookups. The browser journeys below need an agent to schedule and a + # template to list, so a step that died before that apply would fail them for a + # reason that has nothing to do with the UI — three retries each, against an empty + # cluster. `continue-on-error`, because a missing template means the Go step has + # already failed the job and this only decides whether the UI steps are worth + # running: `outcome` still reports the failure the gate below reads. + - name: Check the cluster has the smoke agent + id: smoke-agent + if: ${{ !cancelled() && steps.install-kagent.outcome == 'success' }} + continue-on-error: true + run: kubectl get agenttemplate smoke -n kagent + # The UI, against this same cluster and the image built above. `ui-tests` runs on # the in-browser mock, so a backend contract change cannot fail it — which is how # #2638 emptied three Substrate fields with every check green. @@ -182,10 +195,11 @@ jobs: # After the Go tests, not beside them: both create and delete resources in the same # namespace, and these journeys read the lists back. # - # `steps.install-kagent.outcome` rather than `success()`, so one failed Go test - # under `-failfast` does not also cost every UI signal. No cluster, no run. + # Gated on the cluster being usable rather than on `success()`, so one failed Go + # test under `-failfast` does not also cost every UI signal. No cluster or no + # agent, no run. - name: Setup Node.js - if: ${{ !cancelled() && steps.install-kagent.outcome == 'success' }} + if: ${{ !cancelled() && steps.smoke-agent.outcome == 'success' }} uses: actions/setup-node@v7 with: node-version-file: ui/.nvmrc @@ -193,11 +207,11 @@ jobs: # Before the cache step, as in `ui-tests`: package.json pins Yarn 4, and the # runner's own shim cannot read this lock file. - name: Enable Corepack - if: ${{ !cancelled() && steps.install-kagent.outcome == 'success' }} + if: ${{ !cancelled() && steps.smoke-agent.outcome == 'success' }} run: corepack enable - name: Cache Yarn downloads - if: ${{ !cancelled() && steps.install-kagent.outcome == 'success' }} + if: ${{ !cancelled() && steps.smoke-agent.outcome == 'success' }} uses: actions/cache@v6 with: path: ui/.yarn/cache @@ -205,7 +219,7 @@ jobs: restore-keys: yarn-${{ runner.os }}- - name: Prepare the live browser suite - if: ${{ !cancelled() && steps.install-kagent.outcome == 'success' }} + if: ${{ !cancelled() && steps.smoke-agent.outcome == 'success' }} working-directory: ./ui # Chromium only: the live suite declares one project. The mock suite's second # engine is there to disagree about layout, which no backend contract rests on. @@ -214,7 +228,7 @@ jobs: yarn playwright install --with-deps chromium - name: Run live browser tests - if: ${{ !cancelled() && steps.install-kagent.outcome == 'success' }} + if: ${{ !cancelled() && steps.smoke-agent.outcome == 'success' }} working-directory: ./ui run: | kubectl -n kagent rollout status deploy/kagent-ui --timeout=5m @@ -234,7 +248,10 @@ jobs: with: name: ui-live-playwright-report # The trace is the only record of what the cluster answered — there is no fixed - # fixture to re-read afterwards, as there would be for the mock suite. + # fixture to re-read afterwards, as there would be for the mock suite. The HTML + # report is what links one to the other, and it exists because the config asks + # for `html` alongside `github`; `github` on its own writes annotations and no + # files, which is how this path came to be uploaded empty. path: | ui/playwright-report ui/test-results diff --git a/ui/eslint.config.mjs b/ui/eslint.config.mjs index 43205388c0..0db47dd822 100644 --- a/ui/eslint.config.mjs +++ b/ui/eslint.config.mjs @@ -44,9 +44,14 @@ export default tseslint.config( * * `playwright/README.md` states these; a convention nothing checks is one that * regrows as an exception. + * + * Every folder that holds specs, not only `tests/`: `shared/` runs in every project + * and `live/` is the only suite that talks to a cluster, so a spec there trusting its + * own green matters more rather than less. Scoping this to `tests/` alone left both + * outside the guard from the day they were added. */ { - files: ["playwright/tests/**/*.spec.ts"], + files: ["playwright/{tests,shared,live}/**/*.spec.ts"], rules: { "no-restricted-imports": [ "error", diff --git a/ui/playwright.config.ts b/ui/playwright.config.ts index fadab2d99a..a3958fc32c 100644 --- a/ui/playwright.config.ts +++ b/ui/playwright.config.ts @@ -93,8 +93,16 @@ const LIVE_PORT = Number(process.env.UI_LOOP_LIVE_PORT ?? 8301); * `env-config.js` rendered at pod start are under test rather than approximated by * Vite. A developer still gets the dev server by default; the alternative is building * an image to run a test. + * + * Empty counts as absent, and the `||` is what makes that one question rather than + * three. Three things below ask it — what `baseURL` is, whether `globalSetup` checks + * for a deployed image, whether a dev server is started — and they had asked it three + * ways: a `??`, a `!== undefined` and a truthiness test. An exported-but-empty + * variable, which is what a `kubectl get svc` that found nothing leaves behind, then + * pointed the run at `""`, checked that for nginx, *and* started a dev server nothing + * would ever visit. */ -const LIVE_EXTERNAL_URL = process.env.UI_LOOP_LIVE_URL; +const LIVE_EXTERNAL_URL = process.env.UI_LOOP_LIVE_URL || undefined; const LIVE_BASE_URL = LIVE_EXTERNAL_URL ?? `http://localhost:${LIVE_PORT}`; @@ -170,7 +178,15 @@ export default defineConfig({ * of the machine than the local runs that provoked it. */ workers: process.env.CI ? "50%" : undefined, - reporter: process.env.CI ? "github" : "list", + /* + * Both in CI: `github` writes the annotations that put a failure on the diff, and it + * writes no files at all — so the report CI uploads as an artifact has to come from + * somewhere, and for a live failure the trace is the only account of what the cluster + * answered. + */ + reporter: process.env.CI + ? [["github"], ["html", { open: "never" }]] + : [["list"]], /* * A real backend behind a port-forward answers in tens of seconds where the * in-browser mock answers in milliseconds, so the defaults that suit the mock @@ -190,7 +206,8 @@ export default defineConfig({ /* * One at a time. Every mock test owns a backend in its own page's memory; these * share a cluster, so a spec creating a resource while another counts them is a - * failure with no defect behind it. Four files, seconds to run — it costs little. + * failure with no defect behind it. A dozen tests, and one cluster to run them + * against — it costs little. */ workers: 1, fullyParallel: false, diff --git a/ui/playwright/helpers/app.ts b/ui/playwright/helpers/app.ts index ce7ac7fbf4..017edec725 100644 --- a/ui/playwright/helpers/app.ts +++ b/ui/playwright/helpers/app.ts @@ -160,6 +160,18 @@ export function dataRows(page: Page): Locator { /** A navigation-sized budget, for the app booting rather than for what it rendered. */ const APP_BOOT_TIMEOUT = 15_000; +/** + * Whether this run is against a cluster rather than the fixtures. + * + * For the few places where the two backends differ in kind and not merely in speed — + * a reload restarts the in-browser fixture backend, and a controller fills a status + * only on a real one. A spec branching on this is saying so out loud, which is better + * than a claim that quietly means something different on each. + */ +export function isLiveRun(): boolean { + return test.info().project.name === LIVE_PROJECT; +} + /** * Navigates, for a spec in `shared/` that runs against either backend. * @@ -169,7 +181,7 @@ const APP_BOOT_TIMEOUT = 15_000; * page taking longer to have one. */ export async function loadApp(page: Page, path: string): Promise { - const live = test.info().project.name === LIVE_PROJECT; + const live = isLiveRun(); await page.goto(live ? path : withScenario(path, "ok"), { waitUntil: "domcontentloaded", }); @@ -216,3 +228,66 @@ export async function expectSettled(page: Page): Promise { */ export const throwawayName = (label: string): string => `e2e-live-${label}-${process.pid}-${Date.now().toString(36)}`; + +/** + * Narrows a list page to one name, using the search box the page already offers. + * + * Needed by any spec that creates a row and then reads it back on a list it did not + * seed. These tables page at 25, so on a cluster whose list already fills a page the + * new row is on page two — present, correct, and invisible to a locator. Searching for + * a name only this run could have made puts it on screen wherever it landed, and is + * what a reader looking for their own resource would do. + * + * Client-side, over every row fetched, so it is not a second read that could disagree + * with the first. + * + * @param list the page's test-id prefix — `models` for `models-filters-search`. + */ +export async function searchList(page: Page, list: string, term: string): Promise { + await page.getByTestId(`${list}-filters-search`).fill(term); +} + +/** + * Asserts how many rows the whole list holds — which is not how many are on screen. + * + * Read off the `-summary` line ("3 of 27 configurations"), whose second number is + * computed from every row fetched rather than from the page being shown. Counting + * `dataRows` instead answers a different question on any list longer than 25, and + * answers it wrongly while `searchList` is narrowing the table to one row. + * + * The summary renders only after a successful load, so waiting for it to say a number + * also distinguishes "the list holds that many" from "the read failed". + */ +export async function expectListTotal( + page: Page, + list: string, + total: number, + timeout = 60_000, +): Promise { + await expect(page.getByTestId(`${list}-summary`)).toContainText( + new RegExp(`\\bof ${total}\\b`), + { timeout }, + ); +} + +/** + * Resolves once the list has answered, which is what makes a row count mean anything. + * + * A list still fetching has no rows either, so a `count()` taken too early reads zero + * and every question asked of it gets the answer "not there" — including a cleanup + * asking whether there is anything left to delete. + */ +export async function expectListLoaded(page: Page, list: string): Promise { + await expect(page.getByTestId(`${list}-summary`)).toContainText(/\bof \d+\b/, { + timeout: 60_000, + }); +} + +/** What `expectListTotal` would be reading now, for a count taken before a change. */ +export async function readListTotal(page: Page, list: string): Promise { + await expectListLoaded(page, list); + const text = (await page.getByTestId(`${list}-summary`).textContent()) ?? ""; + const total = /\bof (\d+)\b/.exec(text)?.[1]; + expect(total, `no total could be read from "${text}"`).toBeDefined(); + return Number(total); +} diff --git a/ui/playwright/helpers/resource.ts b/ui/playwright/helpers/resource.ts index e40cc689f7..bebe7abd48 100644 --- a/ui/playwright/helpers/resource.ts +++ b/ui/playwright/helpers/resource.ts @@ -33,8 +33,16 @@ import { withScenario } from "./app"; * contended run, tight enough that a journey which doubles in cost is still a * failure. The slowest is prompts at about forty seconds under full parallel load, * its list fanning out one call per namespace. + * + * **Live is a different budget, not a slower version of the same one.** Against a + * cluster every step is a round trip to an API server and some of them wait on a + * controller to reconcile, which is work no mock does at all — the config already + * doubles the per-test default for that reason, and a `describe.configure` overrides + * it, so a lifecycle asking for sixty here would have been the tightest budget in the + * live run rather than the loosest. */ -export const LIFECYCLE_TIMEOUT = 60_000; +export const LIFECYCLE_TIMEOUT = + process.env.UI_LOOP_LIVE === "true" ? 180_000 : 60_000; /** * Presses a dialog's button, once the dialog has stopped arriving. @@ -110,6 +118,36 @@ export async function pressUntil( }).toPass({ timeout }); } +/** + * Whether something turned up, for a cleanup that has to tell "gone" from "not yet". + * + * A `count()` taken straight after a navigation asks the wrong question: `goto` and + * `loadApp` both return before the page's own read has landed, so nothing is on screen + * yet and every "is it still there?" is answered no — which is how a cleanup came to + * skip its delete and leave a real resource on the cluster. Waiting first is what makes + * an absence mean something. + * + * It resolves `false` rather than throwing, because this is called from a `finally`: + * an assertion failing there would replace the failure the test was actually reporting. + */ +export async function appeared(locator: Locator, timeout = 60_000): Promise { + try { + await locator.waitFor({ state: "visible", timeout }); + return true; + } catch (error) { + /* + * A timeout means it is not there. Anything else — a locator matching several, say — + * is a broken check reaching the caller as the same `false`, which a cleanup reads as + * "nothing to remove" while the resource stays on the cluster. Still `false`, since + * throwing from a `finally` would replace the test's own failure, but not silently. + */ + if ((error as Error | undefined)?.name !== "TimeoutError") { + console.warn(`appeared() could not check this locator: ${String(error)}`); + } + return false; + } +} + /** * Shows that a list says it is loading, and hands back a responsive backend. * diff --git a/ui/playwright/live/pages.spec.ts b/ui/playwright/live/pages.spec.ts index 597e1c9722..e184435c84 100644 --- a/ui/playwright/live/pages.spec.ts +++ b/ui/playwright/live/pages.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from "@playwright/test"; +import { test, expect } from "../fixtures/test"; import { dataRows, expectNoLoadFailure, loadApp } from "../helpers/app"; import { liveRoutes } from "./helpers/live"; diff --git a/ui/playwright/live/schedules.spec.ts b/ui/playwright/live/schedules.spec.ts index f9ae5fa5e4..3f44885453 100644 --- a/ui/playwright/live/schedules.spec.ts +++ b/ui/playwright/live/schedules.spec.ts @@ -1,7 +1,7 @@ import { test, expect } from "../fixtures/test"; import { loadApp, throwawayName } from "../helpers/app"; import { tick } from "../helpers/controls"; -import { optionNamed, pressUntil } from "../helpers/resource"; +import { appeared, optionNamed, pressUntil } from "../helpers/resource"; import { liveRoutes } from "./helpers/live"; /** @@ -97,19 +97,22 @@ test("live: a schedule's configuration survives a reload", async ({ page }) => { await expect(detail).toContainText("90.001 seconds"); }); } finally { - // A real schedule on a real cluster, so a run that dies midway takes it with it. + // A real schedule on a real cluster, and this spec never deletes one in the body — + // so every run, passing or not, leaves through here. if (detailURL) { await page.goto(detailURL); const remove = page .getByTestId("schedule-danger") .getByRole("button", { name: `Delete schedule ${CREATED}`, exact: true }); - if ((await remove.count()) > 0) { + // Waited for, not counted once: `goto` resolves on load and the detail read has + // not landed, so the danger zone is not drawn yet. See `appeared`. + if (await appeared(remove)) { await remove.click(); await pressUntil( page .getByRole("dialog", { name: `Delete schedule ${CREATED}?`, exact: true }) .getByRole("button", { name: "Delete", exact: true }), - () => expect(page).toHaveURL(/\/schedules$/), + () => expect(page).toHaveURL(/\/schedules(\?|$)/), ); } } diff --git a/ui/playwright/live/substrate.spec.ts b/ui/playwright/live/substrate.spec.ts index 9ddb2b9d4a..a7329395b4 100644 --- a/ui/playwright/live/substrate.spec.ts +++ b/ui/playwright/live/substrate.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from "@playwright/test"; +import { test, expect } from "../fixtures/test"; import { expectNoLoadFailure, loadApp } from "../helpers/app"; import { liveRoutes } from "./helpers/live"; diff --git a/ui/playwright/shared/agent-templates/agent-templates.spec.ts b/ui/playwright/shared/agent-templates/agent-templates.spec.ts index 102feaa917..e837d030e9 100644 --- a/ui/playwright/shared/agent-templates/agent-templates.spec.ts +++ b/ui/playwright/shared/agent-templates/agent-templates.spec.ts @@ -1,6 +1,7 @@ import { test, expect } from "../../fixtures/test"; import { LIFECYCLE_TIMEOUT, + appeared, confirmation, pressOnce, selectFirstOption, @@ -9,6 +10,8 @@ import { import { dataRows, expectNoLoadFailure, + expectSettled, + isLiveRun, loadApp, rowNamed, throwawayName, @@ -81,6 +84,18 @@ test("agent templates: one is created, admitted, edited and deleted", async ({ .map((name) => name.trim()) .filter(Boolean); + /* + * The one-harness case applies those labels in an effect, which lands *after* the + * buttons first render — so the single read below is a race against it, and losing + * it sends this step down the "nothing admits it" branch of a cluster where + * something does. The button is the signal to wait on: the form disables a harness + * that already admits the draft, and disables one whose selector is empty from the + * first render, so disabled is the settled state either way. With several + * harnesses nothing is applied unasked and a fresh template carries no labels, so + * the warning below is the only state there is. + */ + if (offered.length === 1) await expect(buttons.first()).toBeDisabled(); + const admissionText = (await admission.textContent()) ?? ""; harness = offered.find((name) => admissionText.includes(`admitted by ${name}`)) ?? ""; @@ -145,16 +160,52 @@ test("agent templates: one is created, admitted, edited and deleted", async ({ new RegExp(`/agent-templates/${NAMESPACE}/${TEMPLATE}`), { timeout: 60_000 }, ); - // The claim this journey exists for, on an element that holds "Runs on" and the - // admitting harnesses and nothing else. `admittingHarnesses` comes from the - // template's *status*, so the harness appearing here means the controller observed - // the labels the form applied and agreed — not that the form echoed itself back. - await expect(page.getByTestId("template-admission-status")).toContainText(harness, { - timeout: 60_000, - }); - await expect(page.getByTestId("template-admission-status")).not.toContainText( - "No harness", - ); + /* + * The claim this journey exists for, on an element that holds "Runs on" and the + * admitting harnesses and nothing else. `admittingHarnesses` comes from the + * template's *status*, so the harness appearing here means the controller observed + * the labels the form applied and agreed — not that the form echoed itself back. + */ + const status = page.getByTestId("template-admission-status"); + + if (isLiveRun()) { + /* + * Live, the page cannot find this out by waiting. A controller fills that status + * some time after the create returns, and this page reads it through SWR with no + * refresh interval and no revalidation on focus — one fetch, on mount. A longer + * assertion timeout would re-read a DOM that was never going to change, leaving + * the claim resting on whether the cluster reconciled in the seconds before the + * navigation. So the reload is the refetch, and the poll is how many times. + * + * Only live: the fixture backend keeps its writes in the page's own memory, so a + * reload there starts a backend that has never heard of this template — see + * `shared/schedules/schedules.spec.ts`, which avoids reloading for that reason. + */ + await expect + .poll( + async () => { + const text = (await status.textContent({ timeout: 30_000 })) ?? ""; + if (text.includes(harness)) return text; + + // Re-read after the reload rather than returning what was on screen + // before it: the round that finally succeeds should be the one that says + // so, not the one after it. + await page.reload(); + await expectSettled(page); + return (await status.textContent({ timeout: 30_000 })) ?? ""; + }, + { + timeout: 90_000, + message: `${TEMPLATE} was never admitted: the controller did not name ${harness} in its status, over repeated re-reads`, + }, + ) + .toContain(harness); + } else { + // The fixtures answer from the create itself, so one read settles it. + await expect(status).toContainText(harness, { timeout: 30_000 }); + } + + await expect(status).not.toContainText("No harness"); }); await test.step("6. an edit in place is saved and read back", async () => { @@ -221,10 +272,26 @@ test("agent templates: one is created, admitted, edited and deleted", async ({ * call, though the previous version of this file believed there was. */ if (created) { - await page.goto(`/agent-templates/${NAMESPACE}/${TEMPLATE}`); - await page.getByTestId(`delete-${TEMPLATE}`).click(); - await pressOnce(confirmation(page).getByRole("button", { name: "Delete" })); - await page.waitForURL(/\/agents\?.*tab=templates/, { timeout: 60_000 }); + await loadApp(page, `/agent-templates/${NAMESPACE}/${TEMPLATE}`); + /* + * Guarded, like every other shared spec's cleanup. `created` says a create + * succeeded, not that the template is still there — and on the mock projects the + * navigation above restarts the in-browser backend, so after a failure midway it + * is reliably *not* there. An unguarded click then waits out the whole test + * budget on a button that will never appear, and the run reports a timeout in + * the cleanup instead of the assertion that actually failed. + */ + /* + * Waited for, not counted once. `expectSettled` vouches for the shell and for + * antd spinners, and this page loads behind a `Skeleton` instead — so the count + * lands before the read does, reads zero, and leaves the template on the cluster. + */ + const remove = page.getByTestId(`delete-${TEMPLATE}`); + if (await appeared(remove)) { + await remove.click(); + await pressOnce(confirmation(page).getByRole("button", { name: "Delete" })); + await page.waitForURL(/\/agents\?.*tab=templates/, { timeout: 60_000 }); + } } } }); diff --git a/ui/playwright/shared/harnesses/harnesses.spec.ts b/ui/playwright/shared/harnesses/harnesses.spec.ts index 45f04481b7..41ef33d3df 100644 --- a/ui/playwright/shared/harnesses/harnesses.spec.ts +++ b/ui/playwright/shared/harnesses/harnesses.spec.ts @@ -1,7 +1,12 @@ import { type Page } from "@playwright/test"; import { test, expect } from "../../fixtures/test"; import { loadApp, throwawayName } from "../../helpers/app"; -import { LIFECYCLE_TIMEOUT, confirmDelete, selectOption } from "../../helpers/resource"; +import { + LIFECYCLE_TIMEOUT, + appeared, + confirmDelete, + selectOption, +} from "../../helpers/resource"; /** * A harness created, read back and deleted — on either backend. @@ -103,7 +108,10 @@ test("harnesses: a harness is created, read and deleted", async ({ page }) => { } finally { if (created) { await loadApp(page, "/agents?tab=harnesses"); - if ((await page.getByTestId(table).getByText(CREATED).count()) > 0) { + // Waited for, not counted once: `loadApp` returns as soon as the shell is up, and + // a tab still fetching has no rows — which reads as "already gone" and leaves a + // real Harness on the cluster. See `appeared`. + if (await appeared(page.getByTestId(table).getByText(CREATED).first())) { await confirmDelete(page, CREATED); } } diff --git a/ui/playwright/shared/models/models.spec.ts b/ui/playwright/shared/models/models.spec.ts index 77e0eecc44..cfb1d2988d 100644 --- a/ui/playwright/shared/models/models.spec.ts +++ b/ui/playwright/shared/models/models.spec.ts @@ -1,9 +1,13 @@ import { test, expect } from "../../fixtures/test"; import { dataRows, + expectListLoaded, + expectListTotal, expectNoLoadFailure, loadApp, + readListTotal, rowNamed, + searchList, throwawayName, } from "../../helpers/app"; import { @@ -46,10 +50,13 @@ test("models: a configuration is created, read, changed and deleted", async ({ * seeds whatever it was installed with, so an absolute count is the one thing this * spec cannot assert — but "one more than before" is exactly as strong, and it is * what catches a create that wrote two rows or a delete that took a neighbour. + * + * Off the summary rather than by counting rows: the table pages at 25, and a + * cluster is free to hold more than that — see `readListTotal`. */ await loadApp(page, "/models"); await expect(dataRows(page).first()).toBeVisible({ timeout: 60_000 }); - before = await dataRows(page).count(); + before = await readListTotal(page, "models"); await page.getByTestId("models-new").click(); await page.waitForURL(/\/models\/new(\?|$)/, { timeout: 60_000 }); @@ -78,11 +85,14 @@ test("models: a configuration is created, read, changed and deleted", async ({ // Read back off the list rather than from a toast or a closed form: those two // only prove the app believes it worked. await expectNoLoadFailure(page); + // Narrowed to the one name this run invented, so the assertions below are about + // that row wherever the cluster's own configurations put it. + await searchList(page, "models", CREATED); const row = rowNamed(page, CREATED); await expect(row).toHaveCount(1, { timeout: 60_000 }); await expect(row).toContainText("Anthropic"); await expect(row).toContainText("claude-sonnet-4"); - await expect(dataRows(page)).toHaveCount(before + 1); + await expectListTotal(page, "models", before + 1); }); await test.step("2. the edit form opens on what was saved, not a blank draft", async () => { @@ -117,11 +127,13 @@ test("models: a configuration is created, read, changed and deleted", async ({ await page.getByTestId("model-submit").click(); await page.waitForURL(/\/models(\?|$)/, { timeout: 60_000 }); + // The search went with the form; the list is whole again on the way back. + await searchList(page, "models", CREATED); const row = rowNamed(page, CREATED); await expect(row).toContainText(SECRET, { timeout: 60_000 }); // Changed, not duplicated — which a create dressed as an update would be. await expect(row).toHaveCount(1); - await expect(dataRows(page)).toHaveCount(before + 1); + await expectListTotal(page, "models", before + 1); }); await test.step("4. deleting asks first, and Keep leaves it alone", async () => { @@ -143,15 +155,20 @@ test("models: a configuration is created, read, changed and deleted", async ({ created = false; // One row went, not several, and not the read: a list that failed to reload is - // also a list the row is missing from. + // also a list the row is missing from, and the summary the total is read off + // renders only for a load that succeeded. await expectNoLoadFailure(page); - await expect(dataRows(page)).toHaveCount(before, { timeout: 60_000 }); + await expectListTotal(page, "models", before); }); } finally { // A real resource on a real cluster when this runs live, so a run that dies midway // takes it with it. if (created) { await loadApp(page, "/models"); + await searchList(page, "models", CREATED); + // Counted only once the list has answered: a read still in flight has no rows + // either, and taking that for "already gone" would leave it on the cluster. + await expectListLoaded(page, "models"); if ((await rowNamed(page, CREATED).count()) > 0) { await confirmDelete(page, CREATED); await expect(rowNamed(page, CREATED)).toHaveCount(0, { timeout: 60_000 }); diff --git a/ui/playwright/shared/prompts/prompts.spec.ts b/ui/playwright/shared/prompts/prompts.spec.ts index 3bd2c2b4c4..e3f3e54381 100644 --- a/ui/playwright/shared/prompts/prompts.spec.ts +++ b/ui/playwright/shared/prompts/prompts.spec.ts @@ -2,9 +2,13 @@ import { type Page } from "@playwright/test"; import { test, expect } from "../../fixtures/test"; import { dataRows, + expectListLoaded, + expectListTotal, expectNoLoadFailure, loadApp, + readListTotal, rowNamed, + searchList, throwawayName, } from "../../helpers/app"; import { LIFECYCLE_TIMEOUT, confirmDelete } from "../../helpers/resource"; @@ -37,10 +41,11 @@ test("prompts: a library is created, read, changed and deleted", async ({ page } await test.step("1. a library with one fragment is created and listed", async () => { // Counted first, and relative from here on: the fixtures seed two and a cluster // seeds whatever it was installed with, but "one more than before" is exactly as - // strong and catches a create that wrote twice. + // strong and catches a create that wrote twice. Off the summary rather than by + // counting rows, the table paging at 25 — see `readListTotal`. await loadApp(page, "/prompts"); await expect(dataRows(page).first()).toBeVisible({ timeout: 60_000 }); - before = await dataRows(page).count(); + before = await readListTotal(page, "prompts"); await page.getByTestId("prompts-new").click(); await expect(page.getByTestId("prompt-submit")).toBeVisible({ timeout: 60_000 }); @@ -57,10 +62,13 @@ test("prompts: a library is created, read, changed and deleted", async ({ page } // Read back off the list rather than from a toast or a closed form: those two // only prove the app believes it worked. await expectNoLoadFailure(page); + // Narrowed to the one name this run invented, so the assertions below are about + // that row wherever the cluster's own libraries put it. + await searchList(page, "prompts", CREATED); const row = rowNamed(page, CREATED); await expect(row).toContainText("1 key", { timeout: 60_000 }); await expect(row).toContainText("changelog"); - await expect(dataRows(page)).toHaveCount(before + 1); + await expectListTotal(page, "prompts", before + 1); }); await test.step("2. opening it shows the fragment and how to include it", async () => { @@ -107,6 +115,8 @@ test("prompts: a library is created, read, changed and deleted", async ({ page } await test.step("4. the list behind it shows the change too", async () => { await page.getByRole("link", { name: "Back to libraries" }).click(); + // The search went with the detail page; the list is whole again on the way back. + await searchList(page, "prompts", CREATED); await expect(rowNamed(page, CREATED)).toContainText("2 keys", { timeout: 60_000 }); await expect(rowNamed(page, CREATED)).toContainText("handoff"); }); @@ -117,13 +127,18 @@ test("prompts: a library is created, read, changed and deleted", async ({ page } created = false; // One row went, not several, and not the read: a list that failed to reload is - // also a list the row is missing from. + // also a list the row is missing from, and the summary the total is read off + // renders only for a load that succeeded. await expectNoLoadFailure(page); - await expect(dataRows(page)).toHaveCount(before, { timeout: 60_000 }); + await expectListTotal(page, "prompts", before); }); } finally { if (created) { await loadApp(page, "/prompts"); + await searchList(page, "prompts", CREATED); + // Counted only once the list has answered: a read still in flight has no rows + // either, and taking that for "already gone" would leave it on the cluster. + await expectListLoaded(page, "prompts"); if ((await rowNamed(page, CREATED).count()) > 0) await confirmDelete(page, CREATED); } } diff --git a/ui/playwright/shared/schedules/schedules.spec.ts b/ui/playwright/shared/schedules/schedules.spec.ts index 7df08148a9..fd2759f74b 100644 --- a/ui/playwright/shared/schedules/schedules.spec.ts +++ b/ui/playwright/shared/schedules/schedules.spec.ts @@ -3,6 +3,7 @@ import { loadApp, throwawayName } from "../../helpers/app"; import { tick } from "../../helpers/controls"; import { LIFECYCLE_TIMEOUT, + appeared, optionNamed, pressUntil, } from "../../helpers/resource"; @@ -149,7 +150,9 @@ test("schedules: one is created, read, changed and deleted", async ({ page }) => const remove = page .getByTestId("schedule-danger") .getByRole("button", { name: `Delete schedule ${CREATED}`, exact: true }); - if ((await remove.count()) > 0) { + // Waited for, not counted once: `goto` resolves on load and the detail read has + // not landed, so the danger zone is not drawn yet. See `appeared`. + if (await appeared(remove)) { await remove.click(); await pressUntil( page diff --git a/ui/playwright/tests/agent-templates/agent-templates.spec.ts b/ui/playwright/tests/agent-templates/agent-templates.spec.ts index 56ed2877c6..82154f9a34 100644 --- a/ui/playwright/tests/agent-templates/agent-templates.spec.ts +++ b/ui/playwright/tests/agent-templates/agent-templates.spec.ts @@ -339,9 +339,10 @@ test("agent templates: the list reads, and a template is read and edited", async }); await test.step("13. an empty result says so instead of showing a bare table", async () => { - // Last, after the delete, because reaching these needs the backend answering - // differently and `?mock=` is per-navigation — which discards what the journey made. - // By here there is nothing left to discard. + // Last, and it has to be: reaching these needs the backend answering differently, + // and `?mock=` is per-navigation — so arriving here discards everything the steps + // above made. Nothing below wants it. (Step 11 opens the delete warning to read it + // and then keeps the template; the delete itself lives in `shared/agent-templates/`.) await loadPage(page, routes.agentTemplates, { scenario: "empty", title: "Agents" }); await expect(page.getByText("No agent templates yet.")).toBeVisible(); await expect(dataRows(page)).toHaveCount(0); diff --git a/ui/playwright/tests/harnesses/harnesses.spec.ts b/ui/playwright/tests/harnesses/harnesses.spec.ts index b66fb17a98..2781dc8fd3 100644 --- a/ui/playwright/tests/harnesses/harnesses.spec.ts +++ b/ui/playwright/tests/harnesses/harnesses.spec.ts @@ -136,8 +136,9 @@ test("harnesses: the tab reads, and the form refuses what the CRD refuses", asyn }); await test.step("7. an empty result leaves the tab standing, with no rows", async () => { - // Last, after the delete: reaching these needs the backend answering differently and - // `?mock=` is per-navigation, which discards what the journey made. + // Last, and it has to be: reaching these needs the backend answering differently, + // and `?mock=` is per-navigation — so arriving here discards everything the steps + // above made. (The delete this used to follow now lives in `shared/harnesses/`.) await loadPage(page, routes.harnesses, { scenario: "empty", title: "Agents" }); await expect(page.getByTestId(table)).toBeVisible({ timeout: 30_000 }); await expect(page.getByTestId(table).locator("tbody tr.ant-table-row")).toHaveCount(0); From 2b7a4199d3206ee10f84cd113bd557dd875f964d Mon Sep 17 00:00:00 2001 From: Torey Scheer Date: Thu, 17 Sep 2026 01:40:46 -0400 Subject: [PATCH 06/24] test(ui): make the assertions that passed on anything fail on something MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An audit of what each test actually asserts, rather than whether it is green. Seven passed while claiming more than they checked. The live sweep's error check ran before the page had asked the controller anything, so its claim was "the shell rendered". Gating on each page's summary was not enough either: it renders for a frame before the answer lands, and a read delayed 1.5s and then failed passed both orders while the page ended up saying "Could not load agents · HTTP 500". A networkidle gate never settles, SWR retrying the failed read. So the sweep watches responses instead — any non-2xx under /api/, asserted per page and again at the end, which is the one that catches a failure landing after its page was checked. Stale claims in the same file: `not.toContainText("__NS__")` under "resolves its model", a string this repo has never rendered, on a table that has had no model column since the rewrite; and a "tool count" step that only re-checked `agents-error`. They assert the harness each row runs on, the provider tag, and the summary's numbers rather than the static word "server". `shared/harnesses` asserted that an unpinned image is refused while the snapshot location was also empty — the button was disabled either way, as the next line proved by leaving it disabled with a valid digest. It now fills a complete draft, asserts the button is enabled, and breaks one field at a time. The "admits no templates" warning is asserted to appear, which nothing did. `shared/dashboard` ties its label loop to the state the poll settled on: a clean cluster has no conversations, so in CI the loop body never ran. Four guards named test ids that have never existed in src, and so could not fail however the control came back. They read the chrome's own ids, search the dialog by role, and assert there is no switch on the page. Substrate names its five tiles rather than one invented absence. Verified by mutation, each reverted: a create-looking id added to the header fails the shell guard, widening the dialog regex to a control that is there fails on count 1, and the induced late failure now fails the sweep in 2.2s. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Torey Scheer --- ui/playwright/live/pages.spec.ts | 168 ++++++++++++++---- ui/playwright/shared/dashboard.spec.ts | 12 ++ .../shared/harnesses/harnesses.spec.ts | 51 ++++-- ui/playwright/tests/agents/agent-page.spec.ts | 7 +- ui/playwright/tests/agents/agents.spec.ts | 5 +- ui/playwright/tests/app-shell.spec.ts | 15 +- ui/playwright/tests/chat/panels.spec.ts | 11 +- .../tests/substrate/inventory.spec.ts | 19 +- 8 files changed, 229 insertions(+), 59 deletions(-) diff --git a/ui/playwright/live/pages.spec.ts b/ui/playwright/live/pages.spec.ts index e184435c84..5cbe38454c 100644 --- a/ui/playwright/live/pages.spec.ts +++ b/ui/playwright/live/pages.spec.ts @@ -1,5 +1,6 @@ +import { type Locator, type Page } from "@playwright/test"; import { test, expect } from "../fixtures/test"; -import { dataRows, expectNoLoadFailure, loadApp } from "../helpers/app"; +import { dataRows, expectListLoaded, expectNoLoadFailure, loadApp } from "../helpers/app"; import { liveRoutes } from "./helpers/live"; /** @@ -17,50 +18,137 @@ import { liveRoutes } from "./helpers/live"; * sent where a bare name belonged. None of them were visible to a green mock suite. */ +/** + * What each page draws once it has an answer — success *or* failure, which `loadApp` + * does not wait for, it waiting for the shell. + * + * Without it the sweep asked its question before the page had asked the controller + * anything: nothing had rendered either way, so every page passed, and a read that + * failed a second later passed with it. Demonstrated by delaying one read and then + * failing it — the guard saw nothing and the alert arrived after the step had moved on. + * + * The failure states are in here beside the good ones deliberately. Gated on success + * alone this waits out sixty seconds and reports "no summary", where what happened is on + * screen and has a message on it: `expectNoLoadFailure` reads it out instead. + * + * Each list's summary is the signal rather than its rows, because it renders "N of M" + * for a read that succeeded — zero rows included — and nothing at all while one is in + * flight. + */ +/** Any of these ids being on screen, whichever page they belong to. */ +const anyOf = (page: Page, ...testIds: string[]) => + page.locator(testIds.map((id) => `[data-testid="${id}"]`).join(", ")); + +/** A list's summary, which only a successful read draws, or the alert a failure does. */ +const listAnswered = (page: Page, list: string) => + anyOf(page, `${list}-summary`, `${list}-error`); + +const answered: Record< + Exclude, + (page: Page) => Locator +> = { + dashboard: (page) => + anyOf( + page, + "recent-agents", + "recent-agents-empty", + "recent-agents-unavailable", + "dashboard-error", + ), + agents: (page) => listAnswered(page, "agents"), + agentTemplates: (page) => listAnswered(page, "templates"), + models: (page) => listAnswered(page, "models"), + mcpServers: (page) => listAnswered(page, "mcp-servers"), + prompts: (page) => listAnswered(page, "prompts"), + // No summary on this one, so its table says it: rows, or "No schedules were found." + schedules: (page) => + anyOf(page, "schedules-empty", "schedules-error").or( + page.locator('[data-testid="schedules-table"] tbody tr.ant-table-row'), + ), + // The tiles draw an em-dash where the number goes until the read lands, so a digit is + // what says it has. `live/substrate.spec.ts` owns what the page then reports. + substrate: (page) => + page + .getByTestId("substrate-stat-actors-value") + .filter({ hasText: /\d/ }) + .or(page.getByTestId("substrate-inventory-error")), +}; + test("live: every page loads against the cluster and reports no failure", async ({ page, }) => { + /* + * What the controller refused, as it arrives. + * + * The rendering is a second-hand account of a read and it lags in both directions: a + * page that has just asked draws neither its summary nor its error, and the summary + * can be on screen for a frame before the answer lands. A check made in that gap + * passes on a read that fails a moment later — measured here by delaying one read to + * 1.5s and failing it, which the shell-only order and the summary-gated order both + * passed while the page ended up saying "Could not load agents · HTTP 500". + * + * A response has no such gap. Asserted per page for what has landed by then, and + * again at the end — which is the one that catches the case above: measured, the + * agents page's gate resolved with nothing refused yet, and the two 500s were in here + * three seconds later. Each is labelled with the page in hand when it landed, which + * for a late one is the page after the one that asked. + */ + const refused: string[] = []; + let current = ""; + page.on("response", (response) => { + if (response.ok() || !response.url().includes("/api/")) return; + refused.push(`${current}: ${response.status()} ${new URL(response.url()).pathname}`); + }); + for (const [name, path] of Object.entries(liveRoutes)) { if (name === "agentTemplateNew") continue; // A form, covered by the lifecycle spec. await test.step(`${name} (${path})`, async () => { + current = name; await loadApp(page, path); - // The distinction worth keeping: a page that could not reach the controller - // must not be read as a page with nothing on it. + // The page drew something, either way — see `answered`. A route added to + // `liveRoutes` without an entry there fails to compile rather than sweeping past. + await expect( + answered[name as keyof typeof answered](page).first(), + `${name} never came back, either way`, + ).toBeVisible({ timeout: 60_000 }); + // And what it drew was not a failure. The distinction worth keeping: a page that + // could not reach the controller must not be read as a page with nothing on it. await expectNoLoadFailure(page); - await expect(page.locator('[data-testid="app-content"]')).toBeVisible(); + expect(refused, "the controller refused a read").toEqual([]); }); } + + // The reads that failed late, each named by the page that asked. + expect(refused, "a read failed after its page had been checked").toEqual([]); }); -test("live: the agents the cluster installed are listed with their model", async ({ +test("live: the agents the cluster installed are listed, each on its harness", async ({ page, }) => { await loadApp(page, liveRoutes.agents); + await expectListLoaded(page, "agents"); await expectNoLoadFailure(page); await test.step("the install's own agents are present", async () => { - // kagent installs a set of agents; asserting on the count rather than on names - // keeps this from breaking when the chart's default set changes, while still - // failing if the list is empty because nothing was read. - await expect(dataRows(page).first()).toBeVisible({ timeout: 60_000 }); - const count = await dataRows(page).count(); - expect(count, "the cluster reported no agents at all").toBeGreaterThan(0); - }); - - await test.step("each row resolves its model rather than showing a ref", async () => { - // The controller denormalises the referenced ModelConfig onto the row. A row - // showing a bare dash here means either the reference did not resolve or the UI - // read the wrong field — both of which a fixture would have hidden. - const first = dataRows(page).first(); - await expect(first).not.toContainText("__NS__"); + // Off the summary: it counts every row fetched rather than the 25 on screen, and + // "some" rather than a number keeps this from breaking when the chart's default + // set changes — while still failing if the list came back empty. + await expect(page.getByTestId("agents-summary")).toContainText(/\bof [1-9]\d*\b/, { + timeout: 60_000, + }); }); - await test.step("the tool count is a number, not a crash", async () => { - // `tools` arrives as JSON null for an agent with none, because Go marshals a nil - // slice that way. Reading `.length` off it took this whole page down against a - // real cluster once. - await expect(page.getByTestId("agents-error")).toHaveCount(0); + await test.step("each row names the harness its agent runs on", async () => { + // An agent is a (template, harness) pair the controller materialises, so a row + // naming no harness is a pair that did not resolve. Read off the element holding + // the harness and nothing else: the row carries the namespace too, and on every + // cluster this runs against both are `kagent`. + const harnesses = page.locator('[data-testid^="agent-harness-"]'); + await expect(harnesses.first()).toBeVisible({ timeout: 60_000 }); + for (const harness of await harnesses.allTextContents()) { + expect(harness.trim(), "an agent row named no harness").not.toBe(""); + } }); }); @@ -68,24 +156,36 @@ test("live: the models the cluster installed are listed with their provider", as page, }) => { await loadApp(page, liveRoutes.models); + await expectListLoaded(page, "models"); await expectNoLoadFailure(page); - await expect(dataRows(page).first()).toBeVisible({ timeout: 60_000 }); - const count = await dataRows(page).count(); - expect(count, "the cluster reported no model configurations").toBeGreaterThan(0); + await expect(page.getByTestId("models-summary")).toContainText(/\bof [1-9]\d*\b/, { + timeout: 60_000, + }); + + // The provider is the column most easily left blank: the row renders its tag whether + // or not `spec.provider` came back, so the claim is that the tag says something. + const provider = dataRows(page).first().locator(".ant-tag").first(); + await expect(provider, "the first model row named no provider").toHaveText(/\S/); }); test("live: tool servers report the tools they discovered", async ({ page }) => { await loadApp(page, liveRoutes.mcpServers); + await expectListLoaded(page, "mcp-servers"); await expectNoLoadFailure(page); await test.step("the summary counts servers and tools", async () => { - // Both counts are derived: the servers from the rows, the tools by summing what - // each discovered. A server that discovered none must still be counted, since a - // registered server reporting nothing is the one most likely to be misconfigured. - await expect(page.getByTestId("mcp-servers-summary")).toBeVisible({ - timeout: 60_000, - }); - await expect(page.getByTestId("mcp-servers-summary")).toContainText("server"); + /* + * The numbers, because the words are static: "N of M servers · K tools" says + * "server" on a page that counted nothing, which is what this asserted before. + * + * Both are derived — the servers from the rows, the tools by summing what each + * discovered — and only the server count is required to be non-zero. Discovery is + * asynchronous (#2849), so a freshly installed cluster reporting zero tools for a + * while is telling the truth. + */ + const summary = page.getByTestId("mcp-servers-summary"); + await expect(summary).toContainText(/\bof [1-9]\d* servers?\b/, { timeout: 60_000 }); + await expect(summary).toContainText(/·\s*\d+ tools?\b/); }); }); diff --git a/ui/playwright/shared/dashboard.spec.ts b/ui/playwright/shared/dashboard.spec.ts index e447cdf261..a00e373c5d 100644 --- a/ui/playwright/shared/dashboard.spec.ts +++ b/ui/playwright/shared/dashboard.spec.ts @@ -50,10 +50,22 @@ test("dashboard: the recent list names conversations rather than showing ids", a ) .toMatch(/^(listed|empty)$/); + /* + * Read once, which the poll above has earned, and tied to which state it settled on. + * The loop below asserts nothing at all on an empty card — and a clean cluster, which + * is what CI runs this against, has no conversations — so without this the claim in + * the title goes unexercised in the one place the live lane runs. + */ + const listed = (await page.getByTestId("recent-agents").count()) > 0; const labels = await page .getByTestId("recent-agent") .locator("a") .evaluateAll((links) => links.map((link) => link.textContent?.trim() ?? "")); + if (listed) { + expect(labels.length, "the card drew a list and named nothing in it").toBeGreaterThan( + 0, + ); + } for (const label of labels) { expect(label, "a conversation should be listed by name, not by its id").not.toMatch( diff --git a/ui/playwright/shared/harnesses/harnesses.spec.ts b/ui/playwright/shared/harnesses/harnesses.spec.ts index 41ef33d3df..aaa26489b6 100644 --- a/ui/playwright/shared/harnesses/harnesses.spec.ts +++ b/ui/playwright/shared/harnesses/harnesses.spec.ts @@ -21,6 +21,10 @@ import { const CREATED = throwawayName("harness"); +/** A draft the cluster would accept, so that each refusal below is about one field. */ +const PINNED = `ghcr.io/example/runtime@sha256:${"a".repeat(64)}`; +const SNAPSHOT = "s3://ate-snapshots/kagent"; + /** Its rows, which is the surface that can say whether any of this happened. */ const table = "harnesses-table"; @@ -40,7 +44,7 @@ test("harnesses: a harness is created, read and deleted", async ({ page }) => { let before = 0; try { - await test.step("1. the form refuses an image that is not pinned", async () => { + await test.step("1. each field the cluster would refuse, refused on its own", async () => { // Counted first: an absolute count is the fixtures' to make, but "one more, then // one fewer" holds on any cluster. await loadApp(page, "/agents?tab=harnesses"); @@ -48,30 +52,45 @@ test("harnesses: a harness is created, read and deleted", async ({ page }) => { before = await harnessRows(page).count(); await loadApp(page, "/harnesses/new"); + const create = page.getByTestId("harness-create"); await selectOption(page, "harness-namespace", "kagent"); await page.getByTestId("harness-name").fill(CREATED); await page.getByTestId("harness-worker-pool").fill("kagent-default"); + await page.getByTestId("harness-image").fill(PINNED); + await page.getByTestId("harness-snapshot").fill(SNAPSHOT); - // The constraints here are the cluster's rather than this page's: a tag can move - // under a running agent, and the CRD refuses one. A form that accepted it would - // build a resource the cluster rejects. - await page.getByTestId("harness-image").fill("ghcr.io/example/runtime:latest"); - await expect(page.getByTestId("harness-create")).toBeDisabled(); - - // The CRD requires a snapshot location too, so it is still refused without one. - await page - .getByTestId("harness-image") - .fill(`ghcr.io/example/runtime@sha256:${"a".repeat(64)}`); - await expect(page.getByTestId("harness-create")).toBeDisabled(); - }); - - await test.step("2. pinned by digest and told where snapshots go, it is created", async () => { - await page.getByTestId("harness-snapshot").fill("s3://ate-snapshots/kagent"); + // A harness with no selector admits nothing, and the form says so before it is + // asked to create one — the one state that is a warning rather than a refusal. + await expect(page.getByTestId("harness-admits-nothing")).toBeVisible(); await page.getByTestId("harness-selector-key").fill("runtime"); await page.getByTestId("harness-selector-value").fill(CREATED); await expect(page.getByTestId("harness-admits-nothing")).toHaveCount(0); + /* + * Enabled with a complete draft, and each field below then broken on its own. + * Asserted the other way round — a bad image on a half-filled form — both of + * these passed on a form that had never looked at the field in question: the + * empty snapshot location was disabling the button by itself. + */ + await expect(create).toBeEnabled(); + + // The image, which the cluster refuses as a tag because a tag can move under a + // running agent. The form says which of the two it is unhappy about. + await page.getByTestId("harness-image").fill("ghcr.io/example/runtime:latest"); + await expect(create).toBeDisabled(); + await expect(page.getByText(/Pin the image by digest/)).toBeVisible(); + await page.getByTestId("harness-image").fill(PINNED); + await expect(create).toBeEnabled(); + + // And the snapshot location, which the CRD requires and which the controller + // would otherwise reject as "Invalid Harness", naming no field. + await page.getByTestId("harness-snapshot").fill(""); + await expect(create).toBeDisabled(); + await page.getByTestId("harness-snapshot").fill(SNAPSHOT); + }); + + await test.step("2. a complete draft is created", async () => { await expect(page.getByTestId("harness-create")).toBeEnabled(); await page.getByTestId("harness-create").click(); diff --git a/ui/playwright/tests/agents/agent-page.spec.ts b/ui/playwright/tests/agents/agent-page.spec.ts index 3a4898bc0f..986dbd02f8 100644 --- a/ui/playwright/tests/agents/agent-page.spec.ts +++ b/ui/playwright/tests/agents/agent-page.spec.ts @@ -186,8 +186,11 @@ test("agents: somebody else's conversation is listed, and plainly cannot be open await expectSettled(page); await test.step("1. the toggle and its alert are gone", async () => { - await expect(page.getByTestId("instances-all-creators")).toHaveCount(0); - await expect(page.getByTestId("instances-own-only")).toHaveCount(0); + // Asserted as "there is no such control", not as "two ids are absent": the ids this + // used to name have never existed in `src`, so those two assertions could not fail + // however the toggle came back. + await expect(page.getByRole("switch")).toHaveCount(0); + await expect(page.getByText(/created by (others|someone else)/i)).toHaveCount(0); }); await test.step("2. everyone's conversations are listed", async () => { diff --git a/ui/playwright/tests/agents/agents.spec.ts b/ui/playwright/tests/agents/agents.spec.ts index ee8ea0c6b6..99db4ee820 100644 --- a/ui/playwright/tests/agents/agents.spec.ts +++ b/ui/playwright/tests/agents/agents.spec.ts @@ -63,7 +63,10 @@ test("agents: the list is agents, and an agent is a template paired with a harne // below for why that row is there and when it is not. await expect(dataRows(page)).toHaveCount(6); await expect(page.getByTestId("agents-table")).toContainText("analytics"); - await expect(page.getByTestId("instances-all-namespaces")).toHaveCount(0); + // "There is no such switch", rather than the absence of the id it used to carry — + // which has never existed in `src`, so that assertion could not fail whatever the + // page did. + await expect(page.getByRole("switch")).toHaveCount(0); }); await test.step("2. a template two harnesses admit is two agents, told apart by the harness", async () => { diff --git a/ui/playwright/tests/app-shell.spec.ts b/ui/playwright/tests/app-shell.spec.ts index 8c24e8f71d..e750d7173c 100644 --- a/ui/playwright/tests/app-shell.spec.ts +++ b/ui/playwright/tests/app-shell.spec.ts @@ -57,8 +57,19 @@ test("app shell: chrome, navigation entries, and where creation lives", async ({ await expect(page.locator('[data-testid="app-header"][data-shell-probe="1"]')).toHaveCount(1); }); - await test.step("4. the chrome offers no create menu of its own", async () => { - await expect(page.getByTestId("create-menu-trigger")).toHaveCount(0); + await test.step("4. the chrome offers no create control of its own", async () => { + // Read off the chrome rather than aimed at one name: asserting that a + // `create-menu-trigger` is absent passes just as well on a header that grew a + // `header-new` instead, which is how this claim would actually be broken. Creating + // belongs to the list you are looking at — step 5. + const ids = await page + .locator('[data-testid="app-header"], [data-testid="app-sidebar"]') + .locator("[data-testid]") + .evaluateAll((nodes) => + nodes.map((node) => node.getAttribute("data-testid") ?? ""), + ); + expect(ids.length, "no controls were read off the chrome").toBeGreaterThan(0); + expect(ids.filter((id) => /(^|-)(new|create)(-|$)/.test(id))).toEqual([]); }); await test.step("5. every list that can create one says so, and reaches its form", async () => { diff --git a/ui/playwright/tests/chat/panels.spec.ts b/ui/playwright/tests/chat/panels.spec.ts index 389e00430a..4d47411844 100644 --- a/ui/playwright/tests/chat/panels.spec.ts +++ b/ui/playwright/tests/chat/panels.spec.ts @@ -39,8 +39,15 @@ test("chat: a conversation's record is read without leaving the conversation", a // There is no Edit anywhere on it: an instance has no spec to change. What the agent // *is* lives on its AgentTemplate and how it *runs* on its Harness, so a control here - // would offer something that does not exist. - await expect(page.getByTestId("agent-details-edit")).toHaveCount(0); + // would offer something that does not exist. By what a reader would press rather than + // by a test id: a control added here would carry an id of its own, and a guard naming + // one would pass on any other. + // Found before it is asked anything: "no Edit inside the dialog" is also true of a + // dialog that is not there, which would be this assertion proving nothing at all. + const details = page.getByRole("dialog"); + await expect(details).toBeVisible(); + await expect(details.getByRole("button", { name: /edit/i })).toHaveCount(0); + await expect(details.getByRole("link", { name: /edit/i })).toHaveCount(0); }); test("chat: the agent panel says what the conversation cannot", async ({ page }) => { diff --git a/ui/playwright/tests/substrate/inventory.spec.ts b/ui/playwright/tests/substrate/inventory.spec.ts index ad4da2ea2c..41160a9ba0 100644 --- a/ui/playwright/tests/substrate/inventory.spec.ts +++ b/ui/playwright/tests/substrate/inventory.spec.ts @@ -262,8 +262,23 @@ test("substrate: an unconfigured ate-api is explained, not reported as broken", await expectSettled(page); // Said by the two tables it applies to, not by a tile: a tile is for a number that - // moves, and this one read `connected` above ate-api's own timeout banner. - await expect(page.getByTestId("substrate-stat-ateapi")).toHaveCount(0); + // moves, and an ate-api one read `connected` above that service's own timeout banner. + // The tiles are named rather than one absence asserted, an ate-api tile returning + // under any other id being the same regression. + const tiles = await page + .locator('[data-testid^="substrate-stat-"]') + .evaluateAll((nodes) => + nodes + .map((node) => node.getAttribute("data-testid") ?? "") + .filter((id) => !id.endsWith("-value")), + ); + expect(tiles.sort()).toEqual([ + "substrate-stat-actors", + "substrate-stat-pools", + "substrate-stat-scope", + "substrate-stat-templates", + "substrate-stat-workers", + ]); await expect(page.getByTestId("substrate-inventory-error")).toHaveCount(0); await expect(page.getByTestId("substrate-partial")).toHaveCount(0); From b9330b23cdf987a4b2095d48aaac5f967005f99f Mon Sep 17 00:00:00 2001 From: Torey Scheer Date: Thu, 17 Sep 2026 02:57:34 -0400 Subject: [PATCH 07/24] test(ui): fix what the review found, and prove it on a cluster built from nothing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six findings from the review of the previous commit, each verified before and after. The live sweep watched for a failure the controller does not produce. The app speaks gRPC-Web, so a refused read answers HTTP 200 and puts the reason in a `grpc-status` trailer — measured against a cluster, a missing library came back `200 ok=true grpc-status=5`. Only nginx's own 502s would ever have reached the non-2xx check, so it reads both now. `pressUntil`'s budget was shorter than one attempt in the live project, where `settled` is an assertion on a thirty-second `expect` timeout. Measured: a 20s attempt under a 15s budget runs exactly once, under 90s it runs five times. So every live Delete was a single press with no retry, which is the whole reason that helper exists, and a swallowed one left the resource on the cluster. Five cleanups called `loadApp`, `searchList` or `page.goto` inside a `finally`, all of which throw — replacing the failure the test was reporting and skipping the delete underneath. They go through `helpers/cleanup.ts`, which warns. `appeared` was written for this and the blocks around it were not. `live/schedules.spec.ts` is a journey on the global budget while every shared one takes `LIFECYCLE_TIMEOUT`; the agent-templates journey read its row off an unnarrowed list that pages at 25, which is what `searchList` exists for; and six `expectNoLoadFailure` calls sat where they always pass. The helper now says what it must be called after. Verified on a cluster built from nothing: 13 live, 173 mock, 498 unit. The fresh run exercised what the old one could not — the dashboard's empty branch (no conversations, so the naming claim is untested there and now says so), the schedules empty-state gate, and lists of one. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Torey Scheer --- ui/playwright/helpers/app.ts | 6 ++ ui/playwright/helpers/cleanup.ts | 56 +++++++++++++++++++ ui/playwright/helpers/resource.ts | 16 +++++- ui/playwright/live/pages.spec.ts | 34 +++++++---- ui/playwright/live/schedules.spec.ts | 50 +++++++++++------ ui/playwright/live/substrate.spec.ts | 4 +- .../agent-templates/agent-templates.spec.ts | 35 ++++++++---- ui/playwright/shared/dashboard.spec.ts | 5 +- ui/playwright/shared/models/models.spec.ts | 29 ++++------ ui/playwright/shared/prompts/prompts.spec.ts | 25 ++++----- .../shared/schedules/schedules.spec.ts | 35 +++++++----- 11 files changed, 208 insertions(+), 87 deletions(-) create mode 100644 ui/playwright/helpers/cleanup.ts diff --git a/ui/playwright/helpers/app.ts b/ui/playwright/helpers/app.ts index 017edec725..b4670fadc1 100644 --- a/ui/playwright/helpers/app.ts +++ b/ui/playwright/helpers/app.ts @@ -196,6 +196,12 @@ export async function loadApp(page: Page, path: string): Promise { * Worth calling before asserting on content: the alternative is a failure reading "the * table is empty" when the truth is "the backend did not answer" — the same distinction * the app itself is careful about. + * + * **Call it after something that proves the read landed**, never straight after a + * navigation. It is a count taken once, and `loadApp` returns on the shell: asked in + * that gap it passes on every page, including one whose read fails a moment later. A + * list's `expectListLoaded` or its own summary is the signal to put in front of it — + * `live/pages.spec.ts` keeps the table of what each page draws. */ export async function expectNoLoadFailure(page: Page): Promise { const alerts = page.locator('[data-testid$="-error"]'); diff --git a/ui/playwright/helpers/cleanup.ts b/ui/playwright/helpers/cleanup.ts new file mode 100644 index 0000000000..ccc07107a1 --- /dev/null +++ b/ui/playwright/helpers/cleanup.ts @@ -0,0 +1,56 @@ +import { expect, type Page } from "@playwright/test"; + +import { expectListLoaded, loadApp, rowNamed, searchList } from "./app"; +import { confirmDelete } from "./resource"; + +/** + * Removes a row this run made, from a `finally`, without ever throwing. + * + * Every move a cleanup makes can fail — the navigation, the list read, the delete — and + * each of them throws. Thrown from a `finally`, that replaces the failure the test was + * actually reporting, *and* skips the delete underneath it: the run reports a timeout in + * the cleanup while the resource stays on the cluster. `appeared` was written for + * exactly this reason and the rest of these blocks were not. + * + * So it warns and returns. A cleanup that could not run says so in the output, and the + * test still reports what it found. + */ +export async function sweepUp( + page: Page, + list: string, + name: string, + /** Where that list lives, for the two that do not share their test-id prefix. */ + path = `/${list}`, +): Promise { + try { + await loadApp(page, path); + await searchList(page, list, name); + // Asked only once the list has answered: a read still in flight has no rows either, + // and taking that for "already gone" would leave it on the cluster. + await expectListLoaded(page, list); + if ((await rowNamed(page, name).count()) === 0) return; + + await confirmDelete(page, name); + await expect(rowNamed(page, name)).toHaveCount(0, { timeout: 60_000 }); + } catch (error) { + console.warn(`cleanup: ${name} may still be on the cluster — ${String(error)}`); + } +} + +/** + * The same promise for a cleanup that is not a row on a list: run it, never throw. + * + * For the journeys whose resource is reached by its own address — a schedule's detail + * page, a template's — where what has to happen is a navigation and a confirm rather + * than a search. + */ +export async function sweepQuietly( + what: string, + remove: () => Promise, +): Promise { + try { + await remove(); + } catch (error) { + console.warn(`cleanup: ${what} may still be on the cluster — ${String(error)}`); + } +} diff --git a/ui/playwright/helpers/resource.ts b/ui/playwright/helpers/resource.ts index bebe7abd48..9d770e3a74 100644 --- a/ui/playwright/helpers/resource.ts +++ b/ui/playwright/helpers/resource.ts @@ -44,6 +44,18 @@ import { withScenario } from "./app"; export const LIFECYCLE_TIMEOUT = process.env.UI_LOOP_LIVE === "true" ? 180_000 : 60_000; +/** + * How long `pressUntil` has to land a press, which has to outlast one attempt at it. + * + * `toPass` checks its deadline *between* attempts, so a budget shorter than one attempt + * buys exactly one press — the retry this helper exists for never happens. An attempt is + * a click plus the caller's `settled`, and `settled` is an assertion on the live + * project's own thirty-second `expect` timeout: mock, fifteen leaves room for two; + * live, fifteen was less than one, so a swallowed Delete was reported as "the page never + * navigated" and the resource stayed on the cluster. + */ +const PRESS_TIMEOUT = process.env.UI_LOOP_LIVE === "true" ? 90_000 : 15_000; + /** * Presses a dialog's button, once the dialog has stopped arriving. * @@ -105,9 +117,7 @@ export async function pressOnce(button: Locator): Promise { export async function pressUntil( button: Locator, settled: () => Promise, - // Half the thirty-second test budget, so that when this is what failed, this is - // what says so: at thirty the test expired first and reported its own timeout. - timeout = 15_000, + timeout = PRESS_TIMEOUT, ): Promise { await expect(async () => { // Bounded, because `toPass` checks its deadline between attempts and no diff --git a/ui/playwright/live/pages.spec.ts b/ui/playwright/live/pages.spec.ts index 5cbe38454c..dcf75d324c 100644 --- a/ui/playwright/live/pages.spec.ts +++ b/ui/playwright/live/pages.spec.ts @@ -18,6 +18,14 @@ import { liveRoutes } from "./helpers/live"; * sent where a bare name belonged. None of them were visible to a green mock suite. */ +/** Any of these ids being on screen, whichever page they belong to. */ +const anyOf = (page: Page, ...testIds: string[]) => + page.locator(testIds.map((id) => `[data-testid="${id}"]`).join(", ")); + +/** A list's summary, which only a successful read draws, or the alert a failure does. */ +const listAnswered = (page: Page, list: string) => + anyOf(page, `${list}-summary`, `${list}-error`); + /** * What each page draws once it has an answer — success *or* failure, which `loadApp` * does not wait for, it waiting for the shell. @@ -35,14 +43,6 @@ import { liveRoutes } from "./helpers/live"; * for a read that succeeded — zero rows included — and nothing at all while one is in * flight. */ -/** Any of these ids being on screen, whichever page they belong to. */ -const anyOf = (page: Page, ...testIds: string[]) => - page.locator(testIds.map((id) => `[data-testid="${id}"]`).join(", ")); - -/** A list's summary, which only a successful read draws, or the alert a failure does. */ -const listAnswered = (page: Page, list: string) => - anyOf(page, `${list}-summary`, `${list}-error`); - const answered: Record< Exclude, (page: Page) => Locator @@ -96,8 +96,22 @@ test("live: every page loads against the cluster and reports no failure", async const refused: string[] = []; let current = ""; page.on("response", (response) => { - if (response.ok() || !response.url().includes("/api/")) return; - refused.push(`${current}: ${response.status()} ${new URL(response.url()).pathname}`); + if (!response.url().includes("/api/")) return; + // HTTP tells only half of it. The app speaks gRPC-Web, so a controller that refuses + // a read still answers 200 and puts the reason in a `grpc-status` trailer — measured + // against this cluster, a missing library came back `200 ok=true grpc-status=5`. + // Reading the status alone would have watched for a failure mode the controller does + // not have, leaving only nginx's own 502s. A refusal raised mid-response carries its + // status in the body instead, which this does not read. + // + // Every status counts, `NOT_FOUND` included: these are list pages, and none of them + // asks the controller for something it is allowed not to have. A page that did — + // a detail page reads one and renders a "no such library" state rather than an + // error — would need this to say which statuses it means. + const status = response.headers()["grpc-status"]; + if (response.ok() && (status === undefined || status === "0")) return; + const how = response.ok() ? `grpc-status ${status}` : `HTTP ${response.status()}`; + refused.push(`${current}: ${how} ${new URL(response.url()).pathname}`); }); for (const [name, path] of Object.entries(liveRoutes)) { diff --git a/ui/playwright/live/schedules.spec.ts b/ui/playwright/live/schedules.spec.ts index 3f44885453..6a900c9081 100644 --- a/ui/playwright/live/schedules.spec.ts +++ b/ui/playwright/live/schedules.spec.ts @@ -1,7 +1,13 @@ import { test, expect } from "../fixtures/test"; import { loadApp, throwawayName } from "../helpers/app"; import { tick } from "../helpers/controls"; -import { appeared, optionNamed, pressUntil } from "../helpers/resource"; +import { sweepQuietly } from "../helpers/cleanup"; +import { + LIFECYCLE_TIMEOUT, + appeared, + optionNamed, + pressUntil, +} from "../helpers/resource"; import { liveRoutes } from "./helpers/live"; /** @@ -22,6 +28,14 @@ import { liveRoutes } from "./helpers/live"; const CREATED = throwawayName("schedule"); +/* + * A journey's budget, not a page's. This creates on a cluster, reloads twice and deletes + * in a `finally` — the same shape as every `shared/` journey, which all take + * `LIFECYCLE_TIMEOUT`. Left on the global live budget it is the tightest one in the run, + * and a kill mid-cleanup leaves a real Schedule behind. + */ +test.describe.configure({ timeout: LIFECYCLE_TIMEOUT }); + test("live: a schedule's configuration survives a reload", async ({ page }) => { let detailURL: string | undefined; @@ -100,21 +114,25 @@ test("live: a schedule's configuration survives a reload", async ({ page }) => { // A real schedule on a real cluster, and this spec never deletes one in the body — // so every run, passing or not, leaves through here. if (detailURL) { - await page.goto(detailURL); - const remove = page - .getByTestId("schedule-danger") - .getByRole("button", { name: `Delete schedule ${CREATED}`, exact: true }); - // Waited for, not counted once: `goto` resolves on load and the detail read has - // not landed, so the danger zone is not drawn yet. See `appeared`. - if (await appeared(remove)) { - await remove.click(); - await pressUntil( - page - .getByRole("dialog", { name: `Delete schedule ${CREATED}?`, exact: true }) - .getByRole("button", { name: "Delete", exact: true }), - () => expect(page).toHaveURL(/\/schedules(\?|$)/), - ); - } + // Captured, because the closure below outlives the narrowing of a `let`. + const detail = detailURL; + await sweepQuietly(CREATED, async () => { + await page.goto(detail); + const remove = page + .getByTestId("schedule-danger") + .getByRole("button", { name: `Delete schedule ${CREATED}`, exact: true }); + // Waited for, not counted once: `goto` resolves on load and the detail read has + // not landed, so the danger zone is not drawn yet. See `appeared`. + if (await appeared(remove)) { + await remove.click(); + await pressUntil( + page + .getByRole("dialog", { name: `Delete schedule ${CREATED}?`, exact: true }) + .getByRole("button", { name: "Delete", exact: true }), + () => expect(page).toHaveURL(/\/schedules(\?|$)/), + ); + } + }); } } }); diff --git a/ui/playwright/live/substrate.spec.ts b/ui/playwright/live/substrate.spec.ts index a7329395b4..7bc9f3fa6e 100644 --- a/ui/playwright/live/substrate.spec.ts +++ b/ui/playwright/live/substrate.spec.ts @@ -18,12 +18,14 @@ import { liveRoutes } from "./helpers/live"; */ test("live: the substrate page renders the cluster's own inventory", async ({ page }) => { await loadApp(page, liveRoutes.substrate); - await expectNoLoadFailure(page); await test.step("1. the page is there rather than an error", async () => { await expect(page.getByTestId("substrate-actors-card")).toBeVisible({ timeout: 60_000, }); + // After the card, not after `loadApp`: the alerts are counted once, and a page that + // has not read anything yet has none — see `expectNoLoadFailure`. + await expectNoLoadFailure(page); }); await test.step("2. the tiles report a real count, not zero", async () => { diff --git a/ui/playwright/shared/agent-templates/agent-templates.spec.ts b/ui/playwright/shared/agent-templates/agent-templates.spec.ts index e837d030e9..5f68ee3424 100644 --- a/ui/playwright/shared/agent-templates/agent-templates.spec.ts +++ b/ui/playwright/shared/agent-templates/agent-templates.spec.ts @@ -9,13 +9,16 @@ import { } from "../../helpers/resource"; import { dataRows, + expectListLoaded, expectNoLoadFailure, expectSettled, isLiveRun, loadApp, rowNamed, + searchList, throwawayName, } from "../../helpers/app"; +import { sweepQuietly } from "../../helpers/cleanup"; /** * Creating and deleting an agent template, through the UI, on either backend. @@ -56,7 +59,6 @@ test("agent templates: one is created, admitted, edited and deleted", async ({ try { await test.step("1. the form offers the cluster's own model configurations", async () => { await loadApp(page, "/agent-templates/new"); - await expectNoLoadFailure(page); await selectOption(page, "template-form-namespace", NAMESPACE); await page.getByTestId("template-form-name").fill(TEMPLATE); @@ -66,6 +68,10 @@ test("agent templates: one is created, admitted, edited and deleted", async ({ // name — the assertion is that the cluster answered, not which model it named, // and `selectFirstOption` fails with that message if the list is empty. await selectFirstOption(page, "template-form-model"); + + // Asked here rather than after `loadApp`: the form's own reads had not gone out + // then, so there was nothing for it to find — see `expectNoLoadFailure`. + await expectNoLoadFailure(page); }); await test.step("2. and the cluster's own harnesses, one of which makes it usable", async () => { @@ -144,6 +150,12 @@ test("agent templates: one is created, admitted, edited and deleted", async ({ }); await test.step("4. the row is read back from the cluster", async () => { + // Narrowed first. This table pages at 25 like the others, so on a namespace that + // already fills a page the new row is on page two — present, correct, and + // invisible to the locator below. `models` and `prompts` search for that reason + // and this journey was the one that did not. + await searchList(page, "templates", TEMPLATE); + await expectListLoaded(page, "templates"); await expectNoLoadFailure(page); await expect(rowNamed(page, TEMPLATE)).toHaveCount(1, { timeout: 60_000 }); /* @@ -257,12 +269,13 @@ test("agent templates: one is created, admitted, edited and deleted", async ({ await page.waitForURL(/\/agents\?.*tab=templates/, { timeout: 60_000 }); created = false; - await expectNoLoadFailure(page); // The rest of the list is still there — the cluster installs templates of its own // — so "gone" names that one template rather than a read that returned nothing. // That distinction is the whole reason `expectNoLoadFailure` exists, and an empty - // table is exactly how a failed list would look. + // table is exactly how a failed list would look. Waited for before it is asked, + // the count being taken once. await expect(dataRows(page).first()).toBeVisible({ timeout: 60_000 }); + await expectNoLoadFailure(page); await expect(rowNamed(page, TEMPLATE)).toHaveCount(0, { timeout: 60_000 }); }); } finally { @@ -272,7 +285,8 @@ test("agent templates: one is created, admitted, edited and deleted", async ({ * call, though the previous version of this file believed there was. */ if (created) { - await loadApp(page, `/agent-templates/${NAMESPACE}/${TEMPLATE}`); + await sweepQuietly(TEMPLATE, async () => { + await loadApp(page, `/agent-templates/${NAMESPACE}/${TEMPLATE}`); /* * Guarded, like every other shared spec's cleanup. `created` says a create * succeeded, not that the template is still there — and on the mock projects the @@ -286,12 +300,13 @@ test("agent templates: one is created, admitted, edited and deleted", async ({ * antd spinners, and this page loads behind a `Skeleton` instead — so the count * lands before the read does, reads zero, and leaves the template on the cluster. */ - const remove = page.getByTestId(`delete-${TEMPLATE}`); - if (await appeared(remove)) { - await remove.click(); - await pressOnce(confirmation(page).getByRole("button", { name: "Delete" })); - await page.waitForURL(/\/agents\?.*tab=templates/, { timeout: 60_000 }); - } + const remove = page.getByTestId(`delete-${TEMPLATE}`); + if (await appeared(remove)) { + await remove.click(); + await pressOnce(confirmation(page).getByRole("button", { name: "Delete" })); + await page.waitForURL(/\/agents\?.*tab=templates/, { timeout: 60_000 }); + } + }); } } }); diff --git a/ui/playwright/shared/dashboard.spec.ts b/ui/playwright/shared/dashboard.spec.ts index a00e373c5d..84216de18a 100644 --- a/ui/playwright/shared/dashboard.spec.ts +++ b/ui/playwright/shared/dashboard.spec.ts @@ -16,7 +16,6 @@ test("dashboard: the recent list names conversations rather than showing ids", a page, }) => { await loadApp(page, "/"); - await expectNoLoadFailure(page); const card = page.getByTestId("dashboard-recent-card"); await expect(card).toBeVisible({ timeout: 30_000 }); @@ -50,6 +49,10 @@ test("dashboard: the recent list names conversations rather than showing ids", a ) .toMatch(/^(listed|empty)$/); + // Asked now rather than after `loadApp`, which returns on the shell: the poll above is + // what makes an absence of alerts mean anything. + await expectNoLoadFailure(page); + /* * Read once, which the poll above has earned, and tied to which state it settled on. * The loop below asserts nothing at all on an empty card — and a clean cluster, which diff --git a/ui/playwright/shared/models/models.spec.ts b/ui/playwright/shared/models/models.spec.ts index cfb1d2988d..9a784976f1 100644 --- a/ui/playwright/shared/models/models.spec.ts +++ b/ui/playwright/shared/models/models.spec.ts @@ -16,6 +16,7 @@ import { confirmation, selectOption, } from "../../helpers/resource"; +import { sweepUp } from "../../helpers/cleanup"; /** * A model configuration created, read back, changed and deleted — on either backend. @@ -83,11 +84,14 @@ test("models: a configuration is created, read, changed and deleted", async ({ created = true; // Read back off the list rather than from a toast or a closed form: those two - // only prove the app believes it worked. - await expectNoLoadFailure(page); - // Narrowed to the one name this run invented, so the assertions below are about - // that row wherever the cluster's own configurations put it. + // only prove the app believes it worked. Narrowed to the one name this run + // invented, so the assertions below are about that row wherever the cluster's own + // configurations put it. await searchList(page, "models", CREATED); + // The list has answered before its alerts are counted: `waitForURL` lands on a + // page that has not read anything yet, where there is nothing to count. + await expectListLoaded(page, "models"); + await expectNoLoadFailure(page); const row = rowNamed(page, CREATED); await expect(row).toHaveCount(1, { timeout: 60_000 }); await expect(row).toContainText("Anthropic"); @@ -156,23 +160,14 @@ test("models: a configuration is created, read, changed and deleted", async ({ // One row went, not several, and not the read: a list that failed to reload is // also a list the row is missing from, and the summary the total is read off - // renders only for a load that succeeded. - await expectNoLoadFailure(page); + // renders only for a load that succeeded — so it is what is waited for, and the + // alert count after it is what names a failure. await expectListTotal(page, "models", before); + await expectNoLoadFailure(page); }); } finally { // A real resource on a real cluster when this runs live, so a run that dies midway // takes it with it. - if (created) { - await loadApp(page, "/models"); - await searchList(page, "models", CREATED); - // Counted only once the list has answered: a read still in flight has no rows - // either, and taking that for "already gone" would leave it on the cluster. - await expectListLoaded(page, "models"); - if ((await rowNamed(page, CREATED).count()) > 0) { - await confirmDelete(page, CREATED); - await expect(rowNamed(page, CREATED)).toHaveCount(0, { timeout: 60_000 }); - } - } + if (created) await sweepUp(page, "models", CREATED); } }); diff --git a/ui/playwright/shared/prompts/prompts.spec.ts b/ui/playwright/shared/prompts/prompts.spec.ts index e3f3e54381..d44626ebf8 100644 --- a/ui/playwright/shared/prompts/prompts.spec.ts +++ b/ui/playwright/shared/prompts/prompts.spec.ts @@ -12,6 +12,7 @@ import { throwawayName, } from "../../helpers/app"; import { LIFECYCLE_TIMEOUT, confirmDelete } from "../../helpers/resource"; +import { sweepUp } from "../../helpers/cleanup"; /** * A prompt library created, read, changed and deleted — on either backend. @@ -60,11 +61,13 @@ test("prompts: a library is created, read, changed and deleted", async ({ page } created = true; // Read back off the list rather than from a toast or a closed form: those two - // only prove the app believes it worked. - await expectNoLoadFailure(page); - // Narrowed to the one name this run invented, so the assertions below are about - // that row wherever the cluster's own libraries put it. + // only prove the app believes it worked. Narrowed to the one name this run + // invented, so the assertions below are about that row wherever the cluster's own + // libraries put it. await searchList(page, "prompts", CREATED); + // The list has answered before its alerts are counted — see `expectNoLoadFailure`. + await expectListLoaded(page, "prompts"); + await expectNoLoadFailure(page); const row = rowNamed(page, CREATED); await expect(row).toContainText("1 key", { timeout: 60_000 }); await expect(row).toContainText("changelog"); @@ -128,18 +131,12 @@ test("prompts: a library is created, read, changed and deleted", async ({ page } // One row went, not several, and not the read: a list that failed to reload is // also a list the row is missing from, and the summary the total is read off - // renders only for a load that succeeded. - await expectNoLoadFailure(page); + // renders only for a load that succeeded — so it is what is waited for, and the + // alert count after it is what names a failure. await expectListTotal(page, "prompts", before); + await expectNoLoadFailure(page); }); } finally { - if (created) { - await loadApp(page, "/prompts"); - await searchList(page, "prompts", CREATED); - // Counted only once the list has answered: a read still in flight has no rows - // either, and taking that for "already gone" would leave it on the cluster. - await expectListLoaded(page, "prompts"); - if ((await rowNamed(page, CREATED).count()) > 0) await confirmDelete(page, CREATED); - } + if (created) await sweepUp(page, "prompts", CREATED); } }); diff --git a/ui/playwright/shared/schedules/schedules.spec.ts b/ui/playwright/shared/schedules/schedules.spec.ts index fd2759f74b..50bc29e530 100644 --- a/ui/playwright/shared/schedules/schedules.spec.ts +++ b/ui/playwright/shared/schedules/schedules.spec.ts @@ -1,6 +1,7 @@ import { test, expect } from "../../fixtures/test"; import { loadApp, throwawayName } from "../../helpers/app"; import { tick } from "../../helpers/controls"; +import { sweepQuietly } from "../../helpers/cleanup"; import { LIFECYCLE_TIMEOUT, appeared, @@ -146,21 +147,25 @@ test("schedules: one is created, read, changed and deleted", async ({ page }) => } finally { // Live these are real resources, so a run that dies midway takes its own with it. if (detailURL) { - await page.goto(detailURL); - const remove = page - .getByTestId("schedule-danger") - .getByRole("button", { name: `Delete schedule ${CREATED}`, exact: true }); - // Waited for, not counted once: `goto` resolves on load and the detail read has - // not landed, so the danger zone is not drawn yet. See `appeared`. - if (await appeared(remove)) { - await remove.click(); - await pressUntil( - page - .getByRole("dialog", { name: `Delete schedule ${CREATED}?`, exact: true }) - .getByRole("button", { name: "Delete", exact: true }), - () => expect(page).toHaveURL(/\/schedules(\?|$)/), - ); - } + // Captured, because the closure below outlives the narrowing of a `let`. + const detail = detailURL; + await sweepQuietly(CREATED, async () => { + await page.goto(detail); + const remove = page + .getByTestId("schedule-danger") + .getByRole("button", { name: `Delete schedule ${CREATED}`, exact: true }); + // Waited for, not counted once: `goto` resolves on load and the detail read has + // not landed, so the danger zone is not drawn yet. See `appeared`. + if (await appeared(remove)) { + await remove.click(); + await pressUntil( + page + .getByRole("dialog", { name: `Delete schedule ${CREATED}?`, exact: true }) + .getByRole("button", { name: "Delete", exact: true }), + () => expect(page).toHaveURL(/\/schedules(\?|$)/), + ); + } + }); } } }); From 5f93a57d3eeb73a0a552df2e2f0766ef9d604a80 Mon Sep 17 00:00:00 2001 From: Torey Scheer Date: Thu, 17 Sep 2026 10:46:03 -0400 Subject: [PATCH 08/24] test(ui): give the retrying parts of the suite something to retry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second review round, five findings, each checked before acting on it. The live admission poll could not retry. `expect.poll` calls its callback outside its own try/catch, so a throw ends the poll rather than failing one round — measured: a callback that throws twice then succeeds gives up after 4ms, where one that returns is retried to its deadline. Both `textContent` on a page still drawing its Skeleton and `expectSettled` after the reload throw, so the ninety-second budget and its message were decoration. The callback now returns the reason it could not answer. The harnesses cleanup was the last `finally` still calling `loadApp` and `confirmDelete` directly, which is what `helpers/cleanup.ts` exists to stop: thrown from there they replace the failure the test was reporting and skip the delete underneath. `expectNoLoadFailure` matched `-error` only, while four pages say a failed read with `-unavailable` instead — the dashboard's recent list, the schedules table, the tools chart, the schedule history. Each of those pages happens to draw its `-error` sibling from the same condition, so nothing was slipping through today; matching both means the claim no longer rests on that. Per-assertion budgets were sixty seconds against a mock `LIFECYCLE_TIMEOUT` of sixty, so an assertion could never exhaust its own budget and a broken one was reported as "Test timeout exceeded" rather than by name. `READ_TIMEOUT` sizes a read below the journey that contains it. And the CI gate: five UI steps hung on one un-retried `kubectl get`, where a transient hiccup skipped the whole browser lane with the job still green — the shape of #2638. It retries, and a lane that does not run says so with a warning annotation. Verified: 13 live against a cluster built from nothing, 173 mock, 31 conventions. A dashboard whose read was aborted now fails the sweep. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Torey Scheer --- .github/workflows/ci.yaml | 20 +++++++- ui/playwright/helpers/app.ts | 11 +++- ui/playwright/helpers/resource.ts | 12 +++++ .../agent-templates/agent-templates.spec.ts | 51 ++++++++++++------- .../shared/harnesses/harnesses.spec.ts | 35 ++++++++----- ui/playwright/shared/models/models.spec.ts | 19 +++---- ui/playwright/shared/prompts/prompts.spec.ts | 30 ++++++----- .../shared/schedules/schedules.spec.ts | 11 ++-- 8 files changed, 129 insertions(+), 60 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4abf1836d3..4d1d6ff9f6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -186,7 +186,25 @@ jobs: id: smoke-agent if: ${{ !cancelled() && steps.install-kagent.outcome == 'success' }} continue-on-error: true - run: kubectl get agenttemplate smoke -n kagent + # Retried, because everything below is gated on this one command. Asked once, a + # transient API-server or kubeconfig hiccup skips the whole browser lane and the + # job still reports green — which is the shape of #2638, a suite that measured + # nothing while every check passed. + run: | + for attempt in $(seq 1 10); do + if kubectl get agenttemplate smoke -n kagent; then exit 0; fi + echo "attempt ${attempt}: not there yet" + sleep 6 + done + kubectl get agenttemplate -A || true + exit 1 + + # And when it is genuinely absent, said out loud on the run. A skipped lane and a + # passing one look identical in the checks list otherwise. + - name: Say so if the UI suite was skipped + if: ${{ !cancelled() && steps.smoke-agent.outcome != 'success' }} + run: | + echo "::warning title=UI live suite skipped::No smoke agent on the cluster, so the live Playwright lane did not run." # The UI, against this same cluster and the image built above. `ui-tests` runs on # the in-browser mock, so a backend contract change cannot fail it — which is how diff --git a/ui/playwright/helpers/app.ts b/ui/playwright/helpers/app.ts index b4670fadc1..7aba53d4e4 100644 --- a/ui/playwright/helpers/app.ts +++ b/ui/playwright/helpers/app.ts @@ -204,7 +204,16 @@ export async function loadApp(page: Page, path: string): Promise { * `live/pages.spec.ts` keeps the table of what each page draws. */ export async function expectNoLoadFailure(page: Page): Promise { - const alerts = page.locator('[data-testid$="-error"]'); + /* + * Both families, because the app says it two ways. A page with room for an alert draws + * `-error`; one with only a line of copy where the data goes draws + * `-unavailable` — the dashboard's recent list, the schedules table, the tools + * chart, the schedule history, each rendered on `error` and nothing else. Matching the + * first alone, this passed on a dashboard that could not reach the controller. + */ + const alerts = page.locator( + '[data-testid$="-error"], [data-testid$="-unavailable"]', + ); const count = await alerts.count(); if (count === 0) return; diff --git a/ui/playwright/helpers/resource.ts b/ui/playwright/helpers/resource.ts index 9d770e3a74..6dd5b18513 100644 --- a/ui/playwright/helpers/resource.ts +++ b/ui/playwright/helpers/resource.ts @@ -56,6 +56,18 @@ export const LIFECYCLE_TIMEOUT = */ const PRESS_TIMEOUT = process.env.UI_LOOP_LIVE === "true" ? 90_000 : 15_000; +/** + * How long one read inside a journey may take, which is not how long the journey may. + * + * These specs asked for sixty seconds an assertion while the mock lane's whole + * `LIFECYCLE_TIMEOUT` is sixty — so an assertion could never exhaust its own budget, and + * a broken one was reported as "Test timeout of 60000ms exceeded" rather than by name. + * The numbers were sized for the live budget and inherited unchanged by the mock run. + * Same argument as `PRESS_TIMEOUT`: when this is what failed, this should be what says + * so. + */ +export const READ_TIMEOUT = process.env.UI_LOOP_LIVE === "true" ? 60_000 : 20_000; + /** * Presses a dialog's button, once the dialog has stopped arriving. * diff --git a/ui/playwright/shared/agent-templates/agent-templates.spec.ts b/ui/playwright/shared/agent-templates/agent-templates.spec.ts index 5f68ee3424..123e916459 100644 --- a/ui/playwright/shared/agent-templates/agent-templates.spec.ts +++ b/ui/playwright/shared/agent-templates/agent-templates.spec.ts @@ -1,6 +1,7 @@ import { test, expect } from "../../fixtures/test"; import { LIFECYCLE_TIMEOUT, + READ_TIMEOUT, appeared, confirmation, pressOnce, @@ -134,7 +135,7 @@ test("agent templates: one is created, admitted, edited and deleted", async ({ // Success is leaving the form. A create the controller refused keeps the reader on // it with `template-create-error` — which is the shape the defect this suite was // written for produced for a template that had in fact been created. - await page.waitForURL(/\/agents\?.*tab=templates/, { timeout: 60_000 }); + await page.waitForURL(/\/agents\?.*tab=templates/, { timeout: READ_TIMEOUT }); created = true; /* * And the list comes back narrowed to the namespace that was being worked in. @@ -157,7 +158,7 @@ test("agent templates: one is created, admitted, edited and deleted", async ({ await searchList(page, "templates", TEMPLATE); await expectListLoaded(page, "templates"); await expectNoLoadFailure(page); - await expect(rowNamed(page, TEMPLATE)).toHaveCount(1, { timeout: 60_000 }); + await expect(rowNamed(page, TEMPLATE)).toHaveCount(1, { timeout: READ_TIMEOUT }); /* * Deliberately not asserting the harness here. The row carries the namespace too, * and on every cluster this runs against both are `kagent` — so the assertion @@ -170,7 +171,7 @@ test("agent templates: one is created, admitted, edited and deleted", async ({ await page.getByTestId(`template-link-${TEMPLATE}`).click(); await page.waitForURL( new RegExp(`/agent-templates/${NAMESPACE}/${TEMPLATE}`), - { timeout: 60_000 }, + { timeout: READ_TIMEOUT }, ); /* * The claim this journey exists for, on an element that holds "Runs on" and the @@ -196,15 +197,31 @@ test("agent templates: one is created, admitted, edited and deleted", async ({ await expect .poll( async () => { - const text = (await status.textContent({ timeout: 30_000 })) ?? ""; - if (text.includes(harness)) return text; + /* + * Nothing in here may throw. `expect.poll` calls its callback outside its + * own try/catch, so a throw ends the poll rather than failing one round of + * it — measured, a callback that throws twice and then succeeds gives up + * after 4ms, where one that returns is retried. Two here can: `textContent` + * on a detail page still drawing its `Skeleton`, and `expectSettled`'s own + * assertions after the reload. Either turned the ninety seconds below into + * a single attempt and reported a raw locator timeout instead of the + * message this poll was given. + */ + try { + const text = (await status.textContent({ timeout: 30_000 })) ?? ""; + if (text.includes(harness)) return text; - // Re-read after the reload rather than returning what was on screen - // before it: the round that finally succeeds should be the one that says - // so, not the one after it. - await page.reload(); - await expectSettled(page); - return (await status.textContent({ timeout: 30_000 })) ?? ""; + // Re-read after the reload rather than returning what was on screen + // before it: the round that finally succeeds should be the one that says + // so, not the one after it. + await page.reload(); + await expectSettled(page); + return (await status.textContent({ timeout: 30_000 })) ?? ""; + } catch (error) { + // The reason this round could not answer, which is what the failure + // message should end on if none of them ever does. + return `could not be read: ${String(error).split("\n")[0]}`; + } }, { timeout: 90_000, @@ -232,14 +249,14 @@ test("agent templates: one is created, admitted, edited and deleted", async ({ */ await expect(page.getByTestId("template-submit")).toHaveCount(0); await page.getByTestId("template-edit").click(); - await expect(page.getByTestId("template-submit")).toBeVisible({ timeout: 60_000 }); + await expect(page.getByTestId("template-submit")).toBeVisible({ timeout: READ_TIMEOUT }); await page.getByTestId("template-form-description").fill(DESCRIPTION); await page.getByTestId("template-submit").click(); // Back to reading, showing the saved value rather than the draft: a save that did // not reach the backend would leave the old one here. - await expect(page.getByTestId("template-edit")).toBeVisible({ timeout: 60_000 }); + await expect(page.getByTestId("template-edit")).toBeVisible({ timeout: READ_TIMEOUT }); await expect(page.getByTestId("template-form-description")).toHaveValue( DESCRIPTION, ); @@ -266,7 +283,7 @@ test("agent templates: one is created, admitted, edited and deleted", async ({ // Scoped to the visible popconfirm, and pressed once it has stopped arriving — // see `helpers/resource` for what each of those is protecting against. await pressOnce(confirmation(page).getByRole("button", { name: "Delete" })); - await page.waitForURL(/\/agents\?.*tab=templates/, { timeout: 60_000 }); + await page.waitForURL(/\/agents\?.*tab=templates/, { timeout: READ_TIMEOUT }); created = false; // The rest of the list is still there — the cluster installs templates of its own @@ -274,9 +291,9 @@ test("agent templates: one is created, admitted, edited and deleted", async ({ // That distinction is the whole reason `expectNoLoadFailure` exists, and an empty // table is exactly how a failed list would look. Waited for before it is asked, // the count being taken once. - await expect(dataRows(page).first()).toBeVisible({ timeout: 60_000 }); + await expect(dataRows(page).first()).toBeVisible({ timeout: READ_TIMEOUT }); await expectNoLoadFailure(page); - await expect(rowNamed(page, TEMPLATE)).toHaveCount(0, { timeout: 60_000 }); + await expect(rowNamed(page, TEMPLATE)).toHaveCount(0, { timeout: READ_TIMEOUT }); }); } finally { /* @@ -304,7 +321,7 @@ test("agent templates: one is created, admitted, edited and deleted", async ({ if (await appeared(remove)) { await remove.click(); await pressOnce(confirmation(page).getByRole("button", { name: "Delete" })); - await page.waitForURL(/\/agents\?.*tab=templates/, { timeout: 60_000 }); + await page.waitForURL(/\/agents\?.*tab=templates/, { timeout: READ_TIMEOUT }); } }); } diff --git a/ui/playwright/shared/harnesses/harnesses.spec.ts b/ui/playwright/shared/harnesses/harnesses.spec.ts index aaa26489b6..8471d5dad3 100644 --- a/ui/playwright/shared/harnesses/harnesses.spec.ts +++ b/ui/playwright/shared/harnesses/harnesses.spec.ts @@ -1,8 +1,10 @@ import { type Page } from "@playwright/test"; import { test, expect } from "../../fixtures/test"; import { loadApp, throwawayName } from "../../helpers/app"; +import { sweepQuietly } from "../../helpers/cleanup"; import { LIFECYCLE_TIMEOUT, + READ_TIMEOUT, appeared, confirmDelete, selectOption, @@ -48,7 +50,7 @@ test("harnesses: a harness is created, read and deleted", async ({ page }) => { // Counted first: an absolute count is the fixtures' to make, but "one more, then // one fewer" holds on any cluster. await loadApp(page, "/agents?tab=harnesses"); - await expect(harnessRows(page).first()).toBeVisible({ timeout: 60_000 }); + await expect(harnessRows(page).first()).toBeVisible({ timeout: READ_TIMEOUT }); before = await harnessRows(page).count(); await loadApp(page, "/harnesses/new"); @@ -97,10 +99,10 @@ test("harnesses: a harness is created, read and deleted", async ({ page }) => { // Back to the tab it came from, with the new harness in the list. Read back off // the table rather than from a toast: "the create returned" and "the thing // exists" are different claims, and only the list checks the second. - await page.waitForURL(/tab=harnesses/, { timeout: 60_000 }); + await page.waitForURL(/tab=harnesses/, { timeout: READ_TIMEOUT }); created = true; - await expect(page.getByTestId(table)).toContainText(CREATED, { timeout: 60_000 }); - await expect.poll(() => harnessRows(page).count(), { timeout: 60_000 }).toBe( + await expect(page.getByTestId(table)).toContainText(CREATED, { timeout: READ_TIMEOUT }); + await expect.poll(() => harnessRows(page).count(), { timeout: READ_TIMEOUT }).toBe( before + 1, ); }); @@ -108,7 +110,7 @@ test("harnesses: a harness is created, read and deleted", async ({ page }) => { await test.step("3. and it is not ready yet, which is what a cluster reports", async () => { const row = page.getByTestId(table).locator("tr", { hasText: CREATED }); await expect(row.getByTestId("harness-ready")).toContainText("Not ready yet", { - timeout: 60_000, + timeout: READ_TIMEOUT, }); }); @@ -116,23 +118,28 @@ test("harnesses: a harness is created, read and deleted", async ({ page }) => { await confirmDelete(page, CREATED); await expect(page.getByTestId(table)).not.toContainText(CREATED, { - timeout: 60_000, + timeout: READ_TIMEOUT, }); created = false; // One row went, not the table: "gone" has to mean that harness rather than a read // that failed and left an empty list behind it. - await expect.poll(() => harnessRows(page).count(), { timeout: 60_000 }).toBe(before); + await expect.poll(() => harnessRows(page).count(), { timeout: READ_TIMEOUT }).toBe(before); await expect(page.getByTestId("harnesses-delete-error")).toHaveCount(0); }); } finally { if (created) { - await loadApp(page, "/agents?tab=harnesses"); - // Waited for, not counted once: `loadApp` returns as soon as the shell is up, and - // a tab still fetching has no rows — which reads as "already gone" and leaves a - // real Harness on the cluster. See `appeared`. - if (await appeared(page.getByTestId(table).getByText(CREATED).first())) { - await confirmDelete(page, CREATED); - } + // Through `sweepQuietly` like every sibling: `loadApp` and `confirmDelete` both + // throw, and thrown from here that replaces the failure the test was reporting and + // skips the delete under it. This was the one block still doing that. + await sweepQuietly(CREATED, async () => { + await loadApp(page, "/agents?tab=harnesses"); + // Waited for, not counted once: `loadApp` returns as soon as the shell is up, + // and a tab still fetching has no rows — which reads as "already gone" and + // leaves a real Harness on the cluster. See `appeared`. + if (await appeared(page.getByTestId(table).getByText(CREATED).first())) { + await confirmDelete(page, CREATED); + } + }); } } }); diff --git a/ui/playwright/shared/models/models.spec.ts b/ui/playwright/shared/models/models.spec.ts index 9a784976f1..2cfca7e0d6 100644 --- a/ui/playwright/shared/models/models.spec.ts +++ b/ui/playwright/shared/models/models.spec.ts @@ -12,6 +12,7 @@ import { } from "../../helpers/app"; import { LIFECYCLE_TIMEOUT, + READ_TIMEOUT, confirmDelete, confirmation, selectOption, @@ -56,11 +57,11 @@ test("models: a configuration is created, read, changed and deleted", async ({ * cluster is free to hold more than that — see `readListTotal`. */ await loadApp(page, "/models"); - await expect(dataRows(page).first()).toBeVisible({ timeout: 60_000 }); + await expect(dataRows(page).first()).toBeVisible({ timeout: READ_TIMEOUT }); before = await readListTotal(page, "models"); await page.getByTestId("models-new").click(); - await page.waitForURL(/\/models\/new(\?|$)/, { timeout: 60_000 }); + await page.waitForURL(/\/models\/new(\?|$)/, { timeout: READ_TIMEOUT }); // The provider list is the app's own enum rather than the backend's, so the name // a reader sees is the same on either — `providerDisplayName` turns @@ -80,7 +81,7 @@ test("models: a configuration is created, read, changed and deleted", async ({ await page.getByTestId("model-api-key").fill("sk-not-a-real-key"); await page.getByTestId("model-submit").click(); - await page.waitForURL(/\/models(\?|$)/, { timeout: 60_000 }); + await page.waitForURL(/\/models(\?|$)/, { timeout: READ_TIMEOUT }); created = true; // Read back off the list rather than from a toast or a closed form: those two @@ -93,7 +94,7 @@ test("models: a configuration is created, read, changed and deleted", async ({ await expectListLoaded(page, "models"); await expectNoLoadFailure(page); const row = rowNamed(page, CREATED); - await expect(row).toHaveCount(1, { timeout: 60_000 }); + await expect(row).toHaveCount(1, { timeout: READ_TIMEOUT }); await expect(row).toContainText("Anthropic"); await expect(row).toContainText("claude-sonnet-4"); await expectListTotal(page, "models", before + 1); @@ -102,11 +103,11 @@ test("models: a configuration is created, read, changed and deleted", async ({ await test.step("2. the edit form opens on what was saved, not a blank draft", async () => { await page.getByTestId(`edit-${CREATED}`).click(); await page.waitForURL(new RegExp(`/models/kagent/${CREATED}/edit$`), { - timeout: 60_000, + timeout: READ_TIMEOUT, }); await expect(page.getByTestId("model-name")).toHaveValue(CREATED, { - timeout: 60_000, + timeout: READ_TIMEOUT, }); /* * The identity and the provider are the ref and what the ref means, so an edit @@ -129,12 +130,12 @@ test("models: a configuration is created, read, changed and deleted", async ({ await page.getByTestId("model-api-key-secret").fill(SECRET); await page.getByTestId("model-submit").click(); - await page.waitForURL(/\/models(\?|$)/, { timeout: 60_000 }); + await page.waitForURL(/\/models(\?|$)/, { timeout: READ_TIMEOUT }); // The search went with the form; the list is whole again on the way back. await searchList(page, "models", CREATED); const row = rowNamed(page, CREATED); - await expect(row).toContainText(SECRET, { timeout: 60_000 }); + await expect(row).toContainText(SECRET, { timeout: READ_TIMEOUT }); // Changed, not duplicated — which a create dressed as an update would be. await expect(row).toHaveCount(1); await expectListTotal(page, "models", before + 1); @@ -155,7 +156,7 @@ test("models: a configuration is created, read, changed and deleted", async ({ await test.step("5. confirming removes that row and leaves the rest", async () => { await confirmDelete(page, CREATED); - await expect(rowNamed(page, CREATED)).toHaveCount(0, { timeout: 60_000 }); + await expect(rowNamed(page, CREATED)).toHaveCount(0, { timeout: READ_TIMEOUT }); created = false; // One row went, not several, and not the read: a list that failed to reload is diff --git a/ui/playwright/shared/prompts/prompts.spec.ts b/ui/playwright/shared/prompts/prompts.spec.ts index d44626ebf8..4a0d32a3f6 100644 --- a/ui/playwright/shared/prompts/prompts.spec.ts +++ b/ui/playwright/shared/prompts/prompts.spec.ts @@ -11,7 +11,11 @@ import { searchList, throwawayName, } from "../../helpers/app"; -import { LIFECYCLE_TIMEOUT, confirmDelete } from "../../helpers/resource"; +import { + LIFECYCLE_TIMEOUT, + READ_TIMEOUT, + confirmDelete, +} from "../../helpers/resource"; import { sweepUp } from "../../helpers/cleanup"; /** @@ -45,11 +49,11 @@ test("prompts: a library is created, read, changed and deleted", async ({ page } // strong and catches a create that wrote twice. Off the summary rather than by // counting rows, the table paging at 25 — see `readListTotal`. await loadApp(page, "/prompts"); - await expect(dataRows(page).first()).toBeVisible({ timeout: 60_000 }); + await expect(dataRows(page).first()).toBeVisible({ timeout: READ_TIMEOUT }); before = await readListTotal(page, "prompts"); await page.getByTestId("prompts-new").click(); - await expect(page.getByTestId("prompt-submit")).toBeVisible({ timeout: 60_000 }); + await expect(page.getByTestId("prompt-submit")).toBeVisible({ timeout: READ_TIMEOUT }); await page.getByTestId("prompt-name").fill(CREATED); await page.getByTestId("prompt-namespace").fill("kagent"); @@ -57,7 +61,7 @@ test("prompts: a library is created, read, changed and deleted", async ({ page } await fragmentValue(page, 0).fill("Group by user impact."); await page.getByTestId("prompt-submit").click(); - await expect(page).toHaveURL(/\/prompts$/, { timeout: 60_000 }); + await expect(page).toHaveURL(/\/prompts$/, { timeout: READ_TIMEOUT }); created = true; // Read back off the list rather than from a toast or a closed form: those two @@ -69,7 +73,7 @@ test("prompts: a library is created, read, changed and deleted", async ({ page } await expectListLoaded(page, "prompts"); await expectNoLoadFailure(page); const row = rowNamed(page, CREATED); - await expect(row).toContainText("1 key", { timeout: 60_000 }); + await expect(row).toContainText("1 key", { timeout: READ_TIMEOUT }); await expect(row).toContainText("changelog"); await expectListTotal(page, "prompts", before + 1); }); @@ -77,21 +81,21 @@ test("prompts: a library is created, read, changed and deleted", async ({ page } await test.step("2. opening it shows the fragment and how to include it", async () => { await rowNamed(page, CREATED).getByRole("link").first().click(); await page.waitForURL(new RegExp(`/prompts/kagent/${CREATED}$`), { - timeout: 60_000, + timeout: READ_TIMEOUT, }); const fragments = page.getByTestId("prompt-fragments"); - await expect(fragments).toContainText("changelog", { timeout: 60_000 }); + await expect(fragments).toContainText("changelog", { timeout: READ_TIMEOUT }); await expect(fragments).toContainText("Group by user impact."); }); await test.step("3. a fragment is added, saved, and read back off the library", async () => { await page.getByTestId("prompt-edit").click(); await page.waitForURL(new RegExp(`/prompts/kagent/${CREATED}/edit$`), { - timeout: 60_000, + timeout: READ_TIMEOUT, }); // Seeded from the saved library, so the form and the page it came from agree. - await expect(fragmentKey(page, 0)).toHaveValue("changelog", { timeout: 60_000 }); + await expect(fragmentKey(page, 0)).toHaveValue("changelog", { timeout: READ_TIMEOUT }); await page.getByTestId("fragment-add").click(); await fragmentKey(page, 1).fill("handoff"); @@ -104,14 +108,14 @@ test("prompts: a library is created, read, changed and deleted", async ({ page } await page.getByTestId("prompt-submit").click(); await expect(page).toHaveURL(new RegExp(`/prompts/kagent/${CREATED}$`), { - timeout: 60_000, + timeout: READ_TIMEOUT, }); // Read back from the re-read library rather than from the draft: a save that // never reached the backend would leave the old text here. const fragments = page.getByTestId("prompt-fragments"); await expect(fragments).toContainText("Name the next owner explicitly.", { - timeout: 60_000, + timeout: READ_TIMEOUT, }); await expect(page.getByTestId("prompt-detail-meta")).toContainText("2 fragments"); }); @@ -120,13 +124,13 @@ test("prompts: a library is created, read, changed and deleted", async ({ page } await page.getByRole("link", { name: "Back to libraries" }).click(); // The search went with the detail page; the list is whole again on the way back. await searchList(page, "prompts", CREATED); - await expect(rowNamed(page, CREATED)).toContainText("2 keys", { timeout: 60_000 }); + await expect(rowNamed(page, CREATED)).toContainText("2 keys", { timeout: READ_TIMEOUT }); await expect(rowNamed(page, CREATED)).toContainText("handoff"); }); await test.step("5. confirming a delete removes that row and leaves the rest", async () => { await confirmDelete(page, CREATED); - await expect(rowNamed(page, CREATED)).toHaveCount(0, { timeout: 60_000 }); + await expect(rowNamed(page, CREATED)).toHaveCount(0, { timeout: READ_TIMEOUT }); created = false; // One row went, not several, and not the read: a list that failed to reload is diff --git a/ui/playwright/shared/schedules/schedules.spec.ts b/ui/playwright/shared/schedules/schedules.spec.ts index 50bc29e530..7c61d17e5e 100644 --- a/ui/playwright/shared/schedules/schedules.spec.ts +++ b/ui/playwright/shared/schedules/schedules.spec.ts @@ -4,6 +4,7 @@ import { tick } from "../../helpers/controls"; import { sweepQuietly } from "../../helpers/cleanup"; import { LIFECYCLE_TIMEOUT, + READ_TIMEOUT, appeared, optionNamed, pressUntil, @@ -80,7 +81,7 @@ test("schedules: one is created, read, changed and deleted", async ({ page }) => await test.step("3. creating it lands on its own page, showing what was asked for", async () => { await page.getByTestId("schedule-submit").click(); await expect(page.getByRole("heading", { name: CREATED, exact: true })).toBeVisible({ - timeout: 60_000, + timeout: READ_TIMEOUT, }); detailURL = page.url(); await expect(page).toHaveURL(/\/schedules\/[0-9a-f-]+(\?|$)/); @@ -96,7 +97,7 @@ test("schedules: one is created, read, changed and deleted", async ({ page }) => await test.step("4. the edit form opens on the stored values, not on defaults", async () => { await page.getByTestId("schedule-edit").click(); await expect(page.getByTestId("schedule-time")).toHaveValue("09:00", { - timeout: 60_000, + timeout: READ_TIMEOUT, }); await expect(page.getByTestId("schedule-timeout")).toHaveValue("90.001"); await expect(page.getByTestId("schedule-timezone").locator("input")).toHaveValue( @@ -113,12 +114,12 @@ test("schedules: one is created, read, changed and deleted", async ({ page }) => await page.getByTestId("schedule-prompt").fill("Report unhealthy workloads only."); await page.getByTestId("schedule-submit").click(); await expect(page.getByRole("heading", { name: CREATED, exact: true })).toBeVisible({ - timeout: 60_000, + timeout: READ_TIMEOUT, }); const detail = page.getByTestId("schedule-detail"); await expect(detail).toContainText("Report unhealthy workloads only.", { - timeout: 60_000, + timeout: READ_TIMEOUT, }); // And the update did not quietly reset what it was not asked to change. await expect(detail).toContainText("90.001 seconds"); @@ -141,7 +142,7 @@ test("schedules: one is created, read, changed and deleted", async ({ page }) => detailURL = undefined; await expect(page.getByRole("link", { name: CREATED, exact: true })).toHaveCount(0, { - timeout: 60_000, + timeout: READ_TIMEOUT, }); }); } finally { From 8bae71804980c5757f7c1e05ce6a0b85bfc87b31 Mon Sep 17 00:00:00 2001 From: Torey Scheer Date: Thu, 17 Sep 2026 12:38:48 -0400 Subject: [PATCH 09/24] fix(ui): do not count a list before it has been read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every list page drew its summary on `!error && !isLoading`, and SWR reports `isLoading` false on the first paint — its fetcher runs in an effect, after it. So a page that had not yet asked the controller anything rendered "0 of 0 harnesses" and held it there. Measured on the harnesses tab at 300ms intervals: "0 of 0" for the first 600ms, then "4 of 4". A count of nothing is a claim, and until the read lands the page has not earned it — the same distinction this codebase already makes about a failed read, which must not be drawn as "there is nothing here". The summary now waits for `data`, so the six lists say nothing until they have something to say. Found by the live suite: a journey that counted the tab before creating a harness read 0 against a tab holding four, and then asserted "one more than nothing". Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Torey Scheer --- ui/src/pages/AgentTemplatesPage.tsx | 6 +++++- ui/src/pages/AgentsPage.tsx | 6 +++++- ui/src/pages/McpServersPage.tsx | 6 +++++- ui/src/pages/ModelsPage.tsx | 6 +++++- ui/src/pages/PromptsPage.tsx | 6 +++++- ui/src/pages/agents/HarnessesTab.tsx | 6 +++++- 6 files changed, 30 insertions(+), 6 deletions(-) diff --git a/ui/src/pages/AgentTemplatesPage.tsx b/ui/src/pages/AgentTemplatesPage.tsx index d0378a8e19..eafed76498 100644 --- a/ui/src/pages/AgentTemplatesPage.tsx +++ b/ui/src/pages/AgentTemplatesPage.tsx @@ -280,7 +280,11 @@ export function AgentTemplatesTab() { {/* Only a successful load can be counted: "0 of 0" because a request failed would be a claim the page cannot support. */} - {!loadFailure && !templates.isLoading ? ( + /* `data !== undefined` as well as `!isLoading`: SWR runs its fetcher in an effect, so + the first paint reports "not loading" on a page that has not asked yet, and this + counted "0 of 0" for the 600ms before the answer arrived. A count of nothing is a + claim, and until the read lands this page has not earned it. */ + {!loadFailure && !templates.isLoading && templates.data !== undefined ? ( {filtered.length} of {rows.length}{" "} {rows.length === 1 ? "template" : "templates"} diff --git a/ui/src/pages/AgentsPage.tsx b/ui/src/pages/AgentsPage.tsx index 62c80d020a..3fc1a37079 100644 --- a/ui/src/pages/AgentsPage.tsx +++ b/ui/src/pages/AgentsPage.tsx @@ -459,7 +459,11 @@ export function AgentsTab() { ]} trailing={ - {!loadFailure && !templates.isLoading ? ( + /* `data !== undefined` as well as `!isLoading`: SWR runs its fetcher in an effect, so + the first paint reports "not loading" on a page that has not asked yet, and this + counted "0 of 0" for the 600ms before the answer arrived. A count of nothing is a + claim, and until the read lands this page has not earned it. */ + {!loadFailure && !templates.isLoading && templates.data !== undefined ? ( {filtered.length} of {agents.length}{" "} {agents.length === 1 ? "agent" : "agents"} diff --git a/ui/src/pages/McpServersPage.tsx b/ui/src/pages/McpServersPage.tsx index eed3544e5c..480c2dec66 100644 --- a/ui/src/pages/McpServersPage.tsx +++ b/ui/src/pages/McpServersPage.tsx @@ -308,7 +308,11 @@ export function McpServersPage() { trailing={ /* Only a successful load can be counted. Saying "0 servers" because a request failed would be a claim the page cannot support. */ - !error && !isLoading ? ( + /* `data !== undefined` as well as `!isLoading`: SWR runs its fetcher in an effect, so + the first paint reports "not loading" on a page that has not asked yet, and this + counted "0 of 0" for the 600ms before the answer arrived. A count of nothing is a + claim, and until the read lands this page has not earned it. */ + !error && !isLoading && data !== undefined ? ( {filtered.length} of {models.length}{" "} {models.length === 1 ? "configuration" : "configurations"} diff --git a/ui/src/pages/PromptsPage.tsx b/ui/src/pages/PromptsPage.tsx index acf81e4e90..d94aa4f015 100644 --- a/ui/src/pages/PromptsPage.tsx +++ b/ui/src/pages/PromptsPage.tsx @@ -231,7 +231,11 @@ export function PromptsPage() { }, ]} trailing={ - !error && !isLoading ? ( + /* `data !== undefined` as well as `!isLoading`: SWR runs its fetcher in an effect, so + the first paint reports "not loading" on a page that has not asked yet, and this + counted "0 of 0" for the 600ms before the answer arrived. A count of nothing is a + claim, and until the read lands this page has not earned it. */ + !error && !isLoading && data !== undefined ? ( {filtered.length} of {libraries.length}{" "} {libraries.length === 1 ? "library" : "libraries"} diff --git a/ui/src/pages/agents/HarnessesTab.tsx b/ui/src/pages/agents/HarnessesTab.tsx index 03807f46c1..20173a20a5 100644 --- a/ui/src/pages/agents/HarnessesTab.tsx +++ b/ui/src/pages/agents/HarnessesTab.tsx @@ -243,7 +243,11 @@ export function HarnessesTab() { ]} trailing={ - {!harnesses.error && !harnesses.isLoading ? ( + /* `data !== undefined` as well as `!isLoading`: SWR runs its fetcher in an effect, so + the first paint reports "not loading" on a page that has not asked yet, and this + counted "0 of 0" for the 600ms before the answer arrived. A count of nothing is a + claim, and until the read lands this page has not earned it. */ + {!harnesses.error && !harnesses.isLoading && harnesses.data !== undefined ? ( {filtered.length} of {rows.length}{" "} {rows.length === 1 ? "harness" : "harnesses"} From eeacea6f997582ddf18f856534794b5ac5a1c3ab Mon Sep 17 00:00:00 2001 From: Torey Scheer Date: Thu, 17 Sep 2026 12:39:01 -0400 Subject: [PATCH 10/24] test(ui): clean up where a timed-out test still can, and stop asking CI for what it has not got MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third review round, four findings, each measured before acting. The tool-server assertion could not pass in CI. The e2e job installs with `--set kagent-tools.enabled=false --set grafana-mcp.enabled=false`, and the chart's only RemoteMCPServer is gated on the first, so that cluster has none — where `setup-cluster.sh` leaves the default and has two. "At least one server", added last round to stop the assertion being vacuous, would have failed every CI run while passing on a laptop. Both counts are now required to be numbers the page computed, and the per-server check runs only where there are servers. Cleanup moved out of `finally` and into `afterEach`. Measured with a four-second test: after a timeout the body's `finally` is refused with "Target page, context or browser has been closed" while a hook still drives the page. A timed-out journey is the likeliest live failure and was the one case these blocks could never handle — so every resource they were meant to remove stayed on the cluster. Three journeys waited for a row before counting, which quietly required the cluster to already own one of that resource; a shared spec may not assume that. They read the total off the summary, which renders at "0 of 0" for a read that succeeded — and which now only renders once the read has landed, the flash that exposed being fixed in the commit before this one. The total is still read twice and has to agree with itself. And `expectListTotal`, `expectListLoaded` and `appeared` kept their hardcoded sixty seconds while their callers moved to `READ_TIMEOUT`, so a stuck read still reported the test timeout rather than naming what it waited for. Verified: 173 mock across both engines, 13 live against the fresh cluster running an image rebuilt from this branch, 498 unit, 31 conventions. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Torey Scheer --- ui/playwright/helpers/app.ts | 58 +- ui/playwright/helpers/resource.ts | 16 +- ui/playwright/live/pages.spec.ts | 37 +- ui/playwright/live/schedules.spec.ts | 183 +++---- .../agent-templates/agent-templates.spec.ts | 497 +++++++++--------- .../shared/harnesses/harnesses.spec.ts | 214 ++++---- ui/playwright/shared/models/models.spec.ts | 264 +++++----- ui/playwright/shared/prompts/prompts.spec.ts | 207 ++++---- .../shared/schedules/schedules.spec.ts | 273 +++++----- 9 files changed, 926 insertions(+), 823 deletions(-) diff --git a/ui/playwright/helpers/app.ts b/ui/playwright/helpers/app.ts index 7aba53d4e4..9aa958f41f 100644 --- a/ui/playwright/helpers/app.ts +++ b/ui/playwright/helpers/app.ts @@ -157,6 +157,19 @@ export function dataRows(page: Page): Locator { return page.locator("tbody tr.ant-table-row"); } +/** + * How long one read inside a journey may take, which is not how long the journey may. + * + * These specs asked for sixty seconds an assertion while the mock lane's whole + * `LIFECYCLE_TIMEOUT` is sixty — so an assertion could never exhaust its own budget, and + * a broken one was reported as "Test timeout of 60000ms exceeded" rather than by name. + * The numbers were sized for the live budget and inherited unchanged by the mock run. + * Same argument as `PRESS_TIMEOUT`: when this is what failed, this should be what says + * so. + */ +export const READ_TIMEOUT = process.env.UI_LOOP_LIVE === "true" ? 60_000 : 20_000; + + /** A navigation-sized budget, for the app booting rather than for what it rendered. */ const APP_BOOT_TIMEOUT = 15_000; @@ -277,7 +290,7 @@ export async function expectListTotal( page: Page, list: string, total: number, - timeout = 60_000, + timeout = READ_TIMEOUT, ): Promise { await expect(page.getByTestId(`${list}-summary`)).toContainText( new RegExp(`\\bof ${total}\\b`), @@ -285,6 +298,39 @@ export async function expectListTotal( ); } +/** + * The list's total, once the summary has stopped moving. + * + * Read once, the summary is not the signal it looks like. Every list draws it on + * `!error && !isLoading`, and SWR reports `isLoading` false on the first paint — the + * fetcher runs in an effect, after it — so a page that has not asked anything yet draws + * "0 of 0" for a frame. Measured on the harnesses tab: a total of 0 read off a tab + * holding four, and the journey then asserted "one more than nothing". + * + * So it is read twice and has to agree with itself. A list still arriving disagrees, and + * one that is genuinely empty says 0 twice — which is the distinction "a list still + * fetching has no rows either" was always about. + */ +async function settledTotal(page: Page, list: string): Promise { + const summary = page.getByTestId(`${list}-summary`); + await expect(summary).toContainText(/\bof \d+\b/, { timeout: READ_TIMEOUT }); + + let previous: number | undefined; + let total = 0; + await expect(async () => { + const text = (await summary.textContent()) ?? ""; + const read = /\bof (\d+)\b/.exec(text)?.[1]; + expect(read, `no total could be read from "${text}"`).toBeDefined(); + const value = Number(read); + const agreed = previous === value; + previous = value; + total = value; + expect(agreed, `still moving, at ${value}`).toBe(true); + }).toPass({ timeout: READ_TIMEOUT, intervals: [250] }); + + return total; +} + /** * Resolves once the list has answered, which is what makes a row count mean anything. * @@ -293,16 +339,10 @@ export async function expectListTotal( * asking whether there is anything left to delete. */ export async function expectListLoaded(page: Page, list: string): Promise { - await expect(page.getByTestId(`${list}-summary`)).toContainText(/\bof \d+\b/, { - timeout: 60_000, - }); + await settledTotal(page, list); } /** What `expectListTotal` would be reading now, for a count taken before a change. */ export async function readListTotal(page: Page, list: string): Promise { - await expectListLoaded(page, list); - const text = (await page.getByTestId(`${list}-summary`).textContent()) ?? ""; - const total = /\bof (\d+)\b/.exec(text)?.[1]; - expect(total, `no total could be read from "${text}"`).toBeDefined(); - return Number(total); + return settledTotal(page, list); } diff --git a/ui/playwright/helpers/resource.ts b/ui/playwright/helpers/resource.ts index 6dd5b18513..f63a8c8b3b 100644 --- a/ui/playwright/helpers/resource.ts +++ b/ui/playwright/helpers/resource.ts @@ -13,7 +13,7 @@ import { expect, type Locator, type Page } from "@playwright/test"; -import { withScenario } from "./app"; +import { READ_TIMEOUT, withScenario } from "./app"; /** * How long one resource's whole lifecycle is allowed to take. @@ -56,18 +56,6 @@ export const LIFECYCLE_TIMEOUT = */ const PRESS_TIMEOUT = process.env.UI_LOOP_LIVE === "true" ? 90_000 : 15_000; -/** - * How long one read inside a journey may take, which is not how long the journey may. - * - * These specs asked for sixty seconds an assertion while the mock lane's whole - * `LIFECYCLE_TIMEOUT` is sixty — so an assertion could never exhaust its own budget, and - * a broken one was reported as "Test timeout of 60000ms exceeded" rather than by name. - * The numbers were sized for the live budget and inherited unchanged by the mock run. - * Same argument as `PRESS_TIMEOUT`: when this is what failed, this should be what says - * so. - */ -export const READ_TIMEOUT = process.env.UI_LOOP_LIVE === "true" ? 60_000 : 20_000; - /** * Presses a dialog's button, once the dialog has stopped arriving. * @@ -152,7 +140,7 @@ export async function pressUntil( * It resolves `false` rather than throwing, because this is called from a `finally`: * an assertion failing there would replace the failure the test was actually reporting. */ -export async function appeared(locator: Locator, timeout = 60_000): Promise { +export async function appeared(locator: Locator, timeout = READ_TIMEOUT): Promise { try { await locator.waitFor({ state: "visible", timeout }); return true; diff --git a/ui/playwright/live/pages.spec.ts b/ui/playwright/live/pages.spec.ts index dcf75d324c..25fb05e1b4 100644 --- a/ui/playwright/live/pages.spec.ts +++ b/ui/playwright/live/pages.spec.ts @@ -1,6 +1,12 @@ import { type Locator, type Page } from "@playwright/test"; import { test, expect } from "../fixtures/test"; -import { dataRows, expectListLoaded, expectNoLoadFailure, loadApp } from "../helpers/app"; +import { + READ_TIMEOUT, + dataRows, + expectListLoaded, + expectNoLoadFailure, + loadApp, +} from "../helpers/app"; import { liveRoutes } from "./helpers/live"; /** @@ -183,7 +189,9 @@ test("live: the models the cluster installed are listed with their provider", as await expect(provider, "the first model row named no provider").toHaveText(/\S/); }); -test("live: tool servers report the tools they discovered", async ({ page }) => { +test("live: the tool server list counts what the controller gave it", async ({ + page, +}) => { await loadApp(page, liveRoutes.mcpServers); await expectListLoaded(page, "mcp-servers"); await expectNoLoadFailure(page); @@ -193,13 +201,28 @@ test("live: tool servers report the tools they discovered", async ({ page }) => * The numbers, because the words are static: "N of M servers · K tools" says * "server" on a page that counted nothing, which is what this asserted before. * - * Both are derived — the servers from the rows, the tools by summing what each - * discovered — and only the server count is required to be non-zero. Discovery is - * asynchronous (#2849), so a freshly installed cluster reporting zero tools for a - * while is telling the truth. + * Neither count may be required to be non-zero, and that is a fact about the + * clusters rather than a weakening. CI installs with `kagent-tools.enabled=false` + * and `grafana-mcp.enabled=false`, and the chart's only RemoteMCPServer is gated on + * the first — so that cluster has none, where `setup-cluster.sh` leaves the default + * and has two. Asking for one passed on a laptop and would have failed every CI run. + * Tool discovery is asynchronous besides (#2849), so a server that has just been + * registered honestly reports none for a while. */ const summary = page.getByTestId("mcp-servers-summary"); - await expect(summary).toContainText(/\bof [1-9]\d* servers?\b/, { timeout: 60_000 }); + await expect(summary).toContainText(/\b\d+ of \d+ servers?\b/, { + timeout: READ_TIMEOUT, + }); await expect(summary).toContainText(/·\s*\d+ tools?\b/); }); + + await test.step("and each server it does have reports a tool count", async () => { + // Where the cluster has none this asserts nothing, which is why the summary above + // is the claim. `tools` arrives as JSON null for a server that discovered nothing, + // Go marshalling a nil slice that way, and reading `.length` off it took this page + // down against a real cluster once. + const rows = dataRows(page); + if ((await rows.count()) === 0) return; + await expect(rows.first()).toContainText(/\d/); + }); }); diff --git a/ui/playwright/live/schedules.spec.ts b/ui/playwright/live/schedules.spec.ts index 6a900c9081..98215a82f4 100644 --- a/ui/playwright/live/schedules.spec.ts +++ b/ui/playwright/live/schedules.spec.ts @@ -36,103 +36,110 @@ const CREATED = throwawayName("schedule"); */ test.describe.configure({ timeout: LIFECYCLE_TIMEOUT }); -test("live: a schedule's configuration survives a reload", async ({ page }) => { - let detailURL: string | undefined; +/** Where this run's schedule lives, so the hook below can remove it. */ +let detailURL: string | undefined; + +/* + * Cleanup in a hook, not in the body's `finally`. + * + * A timed-out test is the likeliest live failure — a controller that never reconciles — + * and it is exactly the one a `finally` cannot clean up after: Playwright has closed the + * page by then, so every call in it throws. Measured, with a four-second test: the + * `finally` was refused with "Target page, context or browser has been closed" while + * this hook still drove the page. Hooks get their own budget, which is the point. + */ +test.afterEach(async ({ page }) => { + if (detailURL === undefined) return; + const detail = detailURL; + detailURL = undefined; + await sweepQuietly(CREATED, async () => { + await page.goto(detail); + const remove = page + .getByTestId("schedule-danger") + .getByRole("button", { name: `Delete schedule ${CREATED}`, exact: true }); + // Waited for, not counted once: `goto` resolves on load and the detail read has not + // landed, so the danger zone is not drawn yet. See `appeared`. + if (await appeared(remove)) { + await remove.click(); + await pressUntil( + page + .getByRole("dialog", { name: `Delete schedule ${CREATED}?`, exact: true }) + .getByRole("button", { name: "Delete", exact: true }), + () => expect(page).toHaveURL(/\/schedules(\?|$)/), + ); + } + }); +}); - try { - await test.step("1. a schedule is created with the values most easily lost", async () => { - await loadApp(page, liveRoutes.schedules); - await page.getByTestId("schedules-new").click(); +test("live: a schedule's configuration survives a reload", async ({ page }) => { + await test.step("1. a schedule is created with the values most easily lost", async () => { + await loadApp(page, liveRoutes.schedules); + await page.getByTestId("schedules-new").click(); - await page.getByTestId("schedule-agent").click(); - const agent = optionNamed(page).first(); - await expect(agent, "the cluster offered no agents to schedule").toBeVisible({ - timeout: 30_000, - }); - await agent.click(); + await page.getByTestId("schedule-agent").click(); + const agent = optionNamed(page).first(); + await expect(agent, "the cluster offered no agents to schedule").toBeVisible({ + timeout: 30_000, + }); + await agent.click(); - await page.getByTestId("schedule-name").fill(CREATED); - await page.getByTestId("schedule-frequency").click(); - await pressUntil(optionNamed(page, "Weekly"), () => - expect(page.getByTestId("schedule-days")).toBeVisible(), - ); - for (const day of ["Tuesday", "Wednesday", "Thursday", "Friday"]) { - await tick(page.getByLabel(day, { exact: true })); - } - await page.getByTestId("schedule-time").fill("09:00"); - await page.getByTestId("schedule-timezone").locator("input").fill("America/New_York"); - await page.keyboard.press("Escape"); - await page.getByTestId("schedule-prompt").fill("Report cluster health."); - await page.getByTestId("schedule-timeout").fill("90.001"); - await page.getByTestId("schedule-enabled").uncheck(); + await page.getByTestId("schedule-name").fill(CREATED); + await page.getByTestId("schedule-frequency").click(); + await pressUntil(optionNamed(page, "Weekly"), () => + expect(page.getByTestId("schedule-days")).toBeVisible(), + ); + for (const day of ["Tuesday", "Wednesday", "Thursday", "Friday"]) { + await tick(page.getByLabel(day, { exact: true })); + } + await page.getByTestId("schedule-time").fill("09:00"); + await page.getByTestId("schedule-timezone").locator("input").fill("America/New_York"); + await page.keyboard.press("Escape"); + await page.getByTestId("schedule-prompt").fill("Report cluster health."); + await page.getByTestId("schedule-timeout").fill("90.001"); + await page.getByTestId("schedule-enabled").uncheck(); - await page.getByTestId("schedule-submit").click(); - await expect(page.getByRole("heading", { name: CREATED, exact: true })).toBeVisible({ - timeout: 60_000, - }); - detailURL = page.url(); + await page.getByTestId("schedule-submit").click(); + await expect(page.getByRole("heading", { name: CREATED, exact: true })).toBeVisible({ + timeout: 60_000, }); + detailURL = page.url(); + }); - await test.step("2. a reload reads it back from the controller, unchanged", async () => { - // The whole spec. Everything above could be the page showing itself what it just - // sent; only a re-read says the controller stored it. - await page.reload(); + await test.step("2. a reload reads it back from the controller, unchanged", async () => { + // The whole spec. Everything above could be the page showing itself what it just + // sent; only a re-read says the controller stored it. + await page.reload(); - await expect(page.getByTestId("schedule-pause")).toHaveText("Resume", { - timeout: 60_000, - }); - await expect(page.getByTestId("schedule-meta")).toContainText("Weekdays at 09:00"); - await expect(page.getByTestId("schedule-meta")).toContainText("America/New_York"); - // `90.001` is stored as seconds plus nanos, so a controller that kept only the - // seconds would read back "90 seconds" here. - await expect(page.getByTestId("schedule-detail")).toContainText("90.001 seconds"); - await expect(page.getByTestId("schedule-detail")).toContainText( - "Report cluster health.", - ); + await expect(page.getByTestId("schedule-pause")).toHaveText("Resume", { + timeout: 60_000, }); + await expect(page.getByTestId("schedule-meta")).toContainText("Weekdays at 09:00"); + await expect(page.getByTestId("schedule-meta")).toContainText("America/New_York"); + // `90.001` is stored as seconds plus nanos, so a controller that kept only the + // seconds would read back "90 seconds" here. + await expect(page.getByTestId("schedule-detail")).toContainText("90.001 seconds"); + await expect(page.getByTestId("schedule-detail")).toContainText( + "Report cluster health.", + ); + }); - await test.step("3. and so does an edit", async () => { - await page.getByTestId("schedule-edit").click(); - await expect(page.getByTestId("schedule-timeout")).toHaveValue("90.001", { - timeout: 60_000, - }); - await page.getByTestId("schedule-prompt").fill("Report unhealthy workloads only."); - await page.getByTestId("schedule-submit").click(); - await expect(page.getByRole("heading", { name: CREATED, exact: true })).toBeVisible({ - timeout: 60_000, - }); + await test.step("3. and so does an edit", async () => { + await page.getByTestId("schedule-edit").click(); + await expect(page.getByTestId("schedule-timeout")).toHaveValue("90.001", { + timeout: 60_000, + }); + await page.getByTestId("schedule-prompt").fill("Report unhealthy workloads only."); + await page.getByTestId("schedule-submit").click(); + await expect(page.getByRole("heading", { name: CREATED, exact: true })).toBeVisible({ + timeout: 60_000, + }); - await page.reload(); - const detail = page.getByTestId("schedule-detail"); - await expect(detail).toContainText("Report unhealthy workloads only.", { - timeout: 60_000, - }); - // The update did not quietly reset what it was not asked to change. - await expect(detail).toContainText("90.001 seconds"); + await page.reload(); + const detail = page.getByTestId("schedule-detail"); + await expect(detail).toContainText("Report unhealthy workloads only.", { + timeout: 60_000, }); - } finally { - // A real schedule on a real cluster, and this spec never deletes one in the body — - // so every run, passing or not, leaves through here. - if (detailURL) { - // Captured, because the closure below outlives the narrowing of a `let`. - const detail = detailURL; - await sweepQuietly(CREATED, async () => { - await page.goto(detail); - const remove = page - .getByTestId("schedule-danger") - .getByRole("button", { name: `Delete schedule ${CREATED}`, exact: true }); - // Waited for, not counted once: `goto` resolves on load and the detail read has - // not landed, so the danger zone is not drawn yet. See `appeared`. - if (await appeared(remove)) { - await remove.click(); - await pressUntil( - page - .getByRole("dialog", { name: `Delete schedule ${CREATED}?`, exact: true }) - .getByRole("button", { name: "Delete", exact: true }), - () => expect(page).toHaveURL(/\/schedules(\?|$)/), - ); - } - }); - } - } + // The update did not quietly reset what it was not asked to change. + await expect(detail).toContainText("90.001 seconds"); + }); }); diff --git a/ui/playwright/shared/agent-templates/agent-templates.spec.ts b/ui/playwright/shared/agent-templates/agent-templates.spec.ts index 123e916459..fb12147439 100644 --- a/ui/playwright/shared/agent-templates/agent-templates.spec.ts +++ b/ui/playwright/shared/agent-templates/agent-templates.spec.ts @@ -1,7 +1,6 @@ import { test, expect } from "../../fixtures/test"; import { LIFECYCLE_TIMEOUT, - READ_TIMEOUT, appeared, confirmation, pressOnce, @@ -9,7 +8,6 @@ import { selectOption, } from "../../helpers/resource"; import { - dataRows, expectListLoaded, expectNoLoadFailure, expectSettled, @@ -18,6 +16,7 @@ import { rowNamed, searchList, throwawayName, + READ_TIMEOUT, } from "../../helpers/app"; import { sweepQuietly } from "../../helpers/cleanup"; @@ -50,280 +49,284 @@ const DESCRIPTION = "Edited by the shared suite."; */ test.describe.configure({ timeout: LIFECYCLE_TIMEOUT }); +/** Whether this run has a template on the cluster, read by the hook below. */ +let created = false; + +/* + * Cleanup in a hook, not in the body's `finally`. + * + * A timed-out test is the likeliest live failure — a controller that never reconciles — + * and it is exactly the one a `finally` cannot clean up after: Playwright has closed the + * page by then, so every call in it throws. Measured, with a four-second test: the + * `finally` was refused with "Target page, context or browser has been closed" while + * this hook still drove the page. Hooks get their own budget, which is the point. + */ +test.afterEach(async ({ page }) => { + if (!created) return; + created = false; + /* + * Through the UI because the app speaks gRPC-Web — there is no REST endpoint to call, + * though a previous version of this file believed there was. Guarded, because `created` + * says a create succeeded rather than that the template is still there: on the mock + * projects the navigation restarts the in-browser backend, so after a failure midway it + * is reliably not there, and an unguarded click would wait out the whole budget. + */ + await sweepQuietly(TEMPLATE, async () => { + await loadApp(page, `/agent-templates/${NAMESPACE}/${TEMPLATE}`); + /* + * Waited for, not counted once. `expectSettled` vouches for the shell and for antd + * spinners, and this page loads behind a `Skeleton` instead — so the count lands + * before the read does, reads zero, and leaves the template on the cluster. + */ + const remove = page.getByTestId(`delete-${TEMPLATE}`); + if (await appeared(remove)) { + await remove.click(); + await pressOnce(confirmation(page).getByRole("button", { name: "Delete" })); + await page.waitForURL(/\/agents\?.*tab=templates/, { timeout: READ_TIMEOUT }); + } + }); +}); + test("agent templates: one is created, admitted, edited and deleted", async ({ page, }) => { - let created = false; /** Which harness this install offered, read off the button in step 2. */ let harness = ""; - try { - await test.step("1. the form offers the cluster's own model configurations", async () => { - await loadApp(page, "/agent-templates/new"); - - await selectOption(page, "template-form-namespace", NAMESPACE); - await page.getByTestId("template-form-name").fill(TEMPLATE); - - // `spec.modelConfig` is the one field the CRD requires, and the options are the - // cluster's own ModelConfigs. Whichever one this install ships, rather than a - // name — the assertion is that the cluster answered, not which model it named, - // and `selectFirstOption` fails with that message if the list is empty. - await selectFirstOption(page, "template-form-model"); + await test.step("1. the form offers the cluster's own model configurations", async () => { + await loadApp(page, "/agent-templates/new"); - // Asked here rather than after `loadApp`: the form's own reads had not gone out - // then, so there was nothing for it to find — see `expectNoLoadFailure`. - await expectNoLoadFailure(page); - }); - - await test.step("2. and the cluster's own harnesses, one of which makes it usable", async () => { - /* - * Two states, and which one appears is a fact about the cluster. With one harness - * the form applies its labels unasked, there being no decision to make; with - * several it warns until told. A `setup-cluster` cluster has one, CI's fixture - * five, so a spec that knew only the second would fail on every laptop. - */ - const admission = page.getByTestId("template-form-admission"); - const buttons = page.locator('[data-testid^="template-form-admit-"]'); - await expect(buttons.first(), "the cluster offered no harnesses").toBeVisible({ - timeout: 30_000, - }); - const offered = (await buttons.allTextContents()) - .map((name) => name.trim()) - .filter(Boolean); + await selectOption(page, "template-form-namespace", NAMESPACE); + await page.getByTestId("template-form-name").fill(TEMPLATE); - /* - * The one-harness case applies those labels in an effect, which lands *after* the - * buttons first render — so the single read below is a race against it, and losing - * it sends this step down the "nothing admits it" branch of a cluster where - * something does. The button is the signal to wait on: the form disables a harness - * that already admits the draft, and disables one whose selector is empty from the - * first render, so disabled is the settled state either way. With several - * harnesses nothing is applied unasked and a fresh template carries no labels, so - * the warning below is the only state there is. - */ - if (offered.length === 1) await expect(buttons.first()).toBeDisabled(); + // `spec.modelConfig` is the one field the CRD requires, and the options are the + // cluster's own ModelConfigs. Whichever one this install ships, rather than a + // name — the assertion is that the cluster answered, not which model it named, + // and `selectFirstOption` fails with that message if the list is empty. + await selectFirstOption(page, "template-form-model"); - const admissionText = (await admission.textContent()) ?? ""; - harness = - offered.find((name) => admissionText.includes(`admitted by ${name}`)) ?? ""; + // Asked here rather than after `loadApp`: the form's own reads had not gone out + // then, so there was nothing for it to find — see `expectNoLoadFailure`. + await expectNoLoadFailure(page); + }); - if (harness === "") { - await expect(admission).toContainText("No harness will run this template"); + await test.step("2. and the cluster's own harnesses, one of which makes it usable", async () => { + /* + * Two states, and which one appears is a fact about the cluster. With one harness + * the form applies its labels unasked, there being no decision to make; with + * several it warns until told. A `setup-cluster` cluster has one, CI's fixture + * five, so a spec that knew only the second would fail on every laptop. + */ + const admission = page.getByTestId("template-form-admission"); + const buttons = page.locator('[data-testid^="template-form-admit-"]'); + await expect(buttons.first(), "the cluster offered no harnesses").toBeVisible({ + timeout: 30_000, + }); + const offered = (await buttons.allTextContents()) + .map((name) => name.trim()) + .filter(Boolean); - // Enabled ones only: the form disables a harness whose selector is empty, since - // such a harness admits nothing and has no labels to copy. Clicking one would - // spend the step's budget on a button that was never going to answer. - const admit = page - .locator('[data-testid^="template-form-admit-"]:not([disabled])') - .first(); - await expect(admit, "no harness on the cluster admits anything").toBeVisible(); - harness = ((await admit.textContent()) ?? "").trim(); - // The button applies whatever labels that harness's selector matches on, which - // is the step a reader is most likely to miss and the one that makes the - // template mean anything. - await admit.click(); - } + /* + * The one-harness case applies those labels in an effect, which lands *after* the + * buttons first render — so the single read below is a race against it, and losing + * it sends this step down the "nothing admits it" branch of a cluster where + * something does. The button is the signal to wait on: the form disables a harness + * that already admits the draft, and disables one whose selector is empty from the + * first render, so disabled is the settled state either way. With several + * harnesses nothing is applied unasked and a fresh template carries no labels, so + * the warning below is the only state there is. + */ + if (offered.length === 1) await expect(buttons.first()).toBeDisabled(); - expect(harness, "no harness name could be read from the form").not.toBe(""); - await expect(admission).toContainText(`admitted by ${harness}`); - }); + const admissionText = (await admission.textContent()) ?? ""; + harness = + offered.find((name) => admissionText.includes(`admitted by ${name}`)) ?? ""; - await test.step("3. submitting reaches the controller and lands on the list", async () => { - await expect(page.getByTestId("template-submit")).toBeEnabled(); - await page.getByTestId("template-submit").click(); + if (harness === "") { + await expect(admission).toContainText("No harness will run this template"); - // Success is leaving the form. A create the controller refused keeps the reader on - // it with `template-create-error` — which is the shape the defect this suite was - // written for produced for a template that had in fact been created. - await page.waitForURL(/\/agents\?.*tab=templates/, { timeout: READ_TIMEOUT }); - created = true; - /* - * And the list comes back narrowed to the namespace that was being worked in. - * Nothing asserted this once, which is how two faults sat on the one line that - * asks for it: `/agent-templates` is a redirect carrying no query string, and the - * list narrows on `ns` while the caller was sending `namespace`. Either alone - * loses the filter, and the page looks reasonable both ways. - */ - await expect(page).toHaveURL(new RegExp(`[?&]ns=${NAMESPACE}(&|$)`)); - await expect( - page.getByTestId(`templates-filters-pill-ns-${NAMESPACE}`), - ).toBeVisible(); - }); + // Enabled ones only: the form disables a harness whose selector is empty, since + // such a harness admits nothing and has no labels to copy. Clicking one would + // spend the step's budget on a button that was never going to answer. + const admit = page + .locator('[data-testid^="template-form-admit-"]:not([disabled])') + .first(); + await expect(admit, "no harness on the cluster admits anything").toBeVisible(); + harness = ((await admit.textContent()) ?? "").trim(); + // The button applies whatever labels that harness's selector matches on, which + // is the step a reader is most likely to miss and the one that makes the + // template mean anything. + await admit.click(); + } - await test.step("4. the row is read back from the cluster", async () => { - // Narrowed first. This table pages at 25 like the others, so on a namespace that - // already fills a page the new row is on page two — present, correct, and - // invisible to the locator below. `models` and `prompts` search for that reason - // and this journey was the one that did not. - await searchList(page, "templates", TEMPLATE); - await expectListLoaded(page, "templates"); - await expectNoLoadFailure(page); - await expect(rowNamed(page, TEMPLATE)).toHaveCount(1, { timeout: READ_TIMEOUT }); - /* - * Deliberately not asserting the harness here. The row carries the namespace too, - * and on every cluster this runs against both are `kagent` — so the assertion - * would pass on the namespace whatever the admission column said. It belongs on an - * element holding the harness and nothing else, which is the next step. - */ - }); + expect(harness, "no harness name could be read from the form").not.toBe(""); + await expect(admission).toContainText(`admitted by ${harness}`); + }); - await test.step("5. the controller agrees a harness admits it", async () => { - await page.getByTestId(`template-link-${TEMPLATE}`).click(); - await page.waitForURL( - new RegExp(`/agent-templates/${NAMESPACE}/${TEMPLATE}`), - { timeout: READ_TIMEOUT }, - ); - /* - * The claim this journey exists for, on an element that holds "Runs on" and the - * admitting harnesses and nothing else. `admittingHarnesses` comes from the - * template's *status*, so the harness appearing here means the controller observed - * the labels the form applied and agreed — not that the form echoed itself back. - */ - const status = page.getByTestId("template-admission-status"); + await test.step("3. submitting reaches the controller and lands on the list", async () => { + await expect(page.getByTestId("template-submit")).toBeEnabled(); + await page.getByTestId("template-submit").click(); - if (isLiveRun()) { - /* - * Live, the page cannot find this out by waiting. A controller fills that status - * some time after the create returns, and this page reads it through SWR with no - * refresh interval and no revalidation on focus — one fetch, on mount. A longer - * assertion timeout would re-read a DOM that was never going to change, leaving - * the claim resting on whether the cluster reconciled in the seconds before the - * navigation. So the reload is the refetch, and the poll is how many times. - * - * Only live: the fixture backend keeps its writes in the page's own memory, so a - * reload there starts a backend that has never heard of this template — see - * `shared/schedules/schedules.spec.ts`, which avoids reloading for that reason. - */ - await expect - .poll( - async () => { - /* - * Nothing in here may throw. `expect.poll` calls its callback outside its - * own try/catch, so a throw ends the poll rather than failing one round of - * it — measured, a callback that throws twice and then succeeds gives up - * after 4ms, where one that returns is retried. Two here can: `textContent` - * on a detail page still drawing its `Skeleton`, and `expectSettled`'s own - * assertions after the reload. Either turned the ninety seconds below into - * a single attempt and reported a raw locator timeout instead of the - * message this poll was given. - */ - try { - const text = (await status.textContent({ timeout: 30_000 })) ?? ""; - if (text.includes(harness)) return text; + // Success is leaving the form. A create the controller refused keeps the reader on + // it with `template-create-error` — which is the shape the defect this suite was + // written for produced for a template that had in fact been created. + await page.waitForURL(/\/agents\?.*tab=templates/, { timeout: READ_TIMEOUT }); + created = true; + /* + * And the list comes back narrowed to the namespace that was being worked in. + * Nothing asserted this once, which is how two faults sat on the one line that + * asks for it: `/agent-templates` is a redirect carrying no query string, and the + * list narrows on `ns` while the caller was sending `namespace`. Either alone + * loses the filter, and the page looks reasonable both ways. + */ + await expect(page).toHaveURL(new RegExp(`[?&]ns=${NAMESPACE}(&|$)`)); + await expect( + page.getByTestId(`templates-filters-pill-ns-${NAMESPACE}`), + ).toBeVisible(); + }); - // Re-read after the reload rather than returning what was on screen - // before it: the round that finally succeeds should be the one that says - // so, not the one after it. - await page.reload(); - await expectSettled(page); - return (await status.textContent({ timeout: 30_000 })) ?? ""; - } catch (error) { - // The reason this round could not answer, which is what the failure - // message should end on if none of them ever does. - return `could not be read: ${String(error).split("\n")[0]}`; - } - }, - { - timeout: 90_000, - message: `${TEMPLATE} was never admitted: the controller did not name ${harness} in its status, over repeated re-reads`, - }, - ) - .toContain(harness); - } else { - // The fixtures answer from the create itself, so one read settles it. - await expect(status).toContainText(harness, { timeout: 30_000 }); - } + await test.step("4. the row is read back from the cluster", async () => { + // Narrowed first. This table pages at 25 like the others, so on a namespace that + // already fills a page the new row is on page two — present, correct, and + // invisible to the locator below. `models` and `prompts` search for that reason + // and this journey was the one that did not. + await searchList(page, "templates", TEMPLATE); + await expectListLoaded(page, "templates"); + await expectNoLoadFailure(page); + await expect(rowNamed(page, TEMPLATE)).toHaveCount(1, { timeout: READ_TIMEOUT }); + /* + * Deliberately not asserting the harness here. The row carries the namespace too, + * and on every cluster this runs against both are `kagent` — so the assertion + * would pass on the namespace whatever the admission column said. It belongs on an + * element holding the harness and nothing else, which is the next step. + */ + }); - await expect(status).not.toContainText("No harness"); - }); + await test.step("5. the controller agrees a harness admits it", async () => { + await page.getByTestId(`template-link-${TEMPLATE}`).click(); + await page.waitForURL( + new RegExp(`/agent-templates/${NAMESPACE}/${TEMPLATE}`), + { timeout: READ_TIMEOUT }, + ); + /* + * The claim this journey exists for, on an element that holds "Runs on" and the + * admitting harnesses and nothing else. `admittingHarnesses` comes from the + * template's *status*, so the harness appearing here means the controller observed + * the labels the form applied and agreed — not that the form echoed itself back. + */ + const status = page.getByTestId("template-admission-status"); - await test.step("6. an edit in place is saved and read back", async () => { + if (isLiveRun()) { /* - * The write half the create cannot show. A template's name and namespace are its - * ref and cannot change, so the description is what an edit has to move — and - * reading it back off the page after the save is what separates "the backend - * stored it" from "the draft is still on screen". + * Live, the page cannot find this out by waiting. A controller fills that status + * some time after the create returns, and this page reads it through SWR with no + * refresh interval and no revalidation on focus — one fetch, on mount. A longer + * assertion timeout would re-read a DOM that was never going to change, leaving + * the claim resting on whether the cluster reconciled in the seconds before the + * navigation. So the reload is the refetch, and the poll is how many times. * - * Editing is a mode of the reading page rather than a separate address, so the - * submit appearing is also the assertion that the same component serves both. + * Only live: the fixture backend keeps its writes in the page's own memory, so a + * reload there starts a backend that has never heard of this template — see + * `shared/schedules/schedules.spec.ts`, which avoids reloading for that reason. */ - await expect(page.getByTestId("template-submit")).toHaveCount(0); - await page.getByTestId("template-edit").click(); - await expect(page.getByTestId("template-submit")).toBeVisible({ timeout: READ_TIMEOUT }); + await expect + .poll( + async () => { + /* + * Nothing in here may throw. `expect.poll` calls its callback outside its + * own try/catch, so a throw ends the poll rather than failing one round of + * it — measured, a callback that throws twice and then succeeds gives up + * after 4ms, where one that returns is retried. Two here can: `textContent` + * on a detail page still drawing its `Skeleton`, and `expectSettled`'s own + * assertions after the reload. Either turned the ninety seconds below into + * a single attempt and reported a raw locator timeout instead of the + * message this poll was given. + */ + try { + const text = (await status.textContent({ timeout: 30_000 })) ?? ""; + if (text.includes(harness)) return text; - await page.getByTestId("template-form-description").fill(DESCRIPTION); - await page.getByTestId("template-submit").click(); + // Re-read after the reload rather than returning what was on screen + // before it: the round that finally succeeds should be the one that says + // so, not the one after it. + await page.reload(); + await expectSettled(page); + return (await status.textContent({ timeout: 30_000 })) ?? ""; + } catch (error) { + // The reason this round could not answer, which is what the failure + // message should end on if none of them ever does. + return `could not be read: ${String(error).split("\n")[0]}`; + } + }, + { + timeout: 90_000, + message: `${TEMPLATE} was never admitted: the controller did not name ${harness} in its status, over repeated re-reads`, + }, + ) + .toContain(harness); + } else { + // The fixtures answer from the create itself, so one read settles it. + await expect(status).toContainText(harness, { timeout: 30_000 }); + } - // Back to reading, showing the saved value rather than the draft: a save that did - // not reach the backend would leave the old one here. - await expect(page.getByTestId("template-edit")).toBeVisible({ timeout: READ_TIMEOUT }); - await expect(page.getByTestId("template-form-description")).toHaveValue( - DESCRIPTION, - ); - }); + await expect(status).not.toContainText("No harness"); + }); - await test.step("7. deleting says what it costs, against the backend's own count", async () => { - const remove = page.getByTestId(`delete-${TEMPLATE}`); - await expect(remove).toContainText("Delete template"); - await remove.click(); + await test.step("6. an edit in place is saved and read back", async () => { + /* + * The write half the create cannot show. A template's name and namespace are its + * ref and cannot change, so the description is what an edit has to move — and + * reading it back off the page after the save is what separates "the backend + * stored it" from "the draft is still on screen". + * + * Editing is a mode of the reading page rather than a separate address, so the + * submit appearing is also the assertion that the same component serves both. + */ + await expect(page.getByTestId("template-submit")).toHaveCount(0); + await page.getByTestId("template-edit").click(); + await expect(page.getByTestId("template-submit")).toBeVisible({ timeout: READ_TIMEOUT }); - /* - * Either branch is legitimate: the count comes from `status.harnesses` while step 5 - * waited on `admittingHarnesses`, two fields filled by different work. The mock - * suite pins the wording; what a cluster shows is that the sentence is computed - * from real state at all, rather than coming back blank. - */ - await expect(page.getByTestId("template-delete-consequence")).toContainText( - /built from this template|no agent was ever built from it/, - { timeout: 30_000 }, - ); - }); + await page.getByTestId("template-form-description").fill(DESCRIPTION); + await page.getByTestId("template-submit").click(); - await test.step("8. confirming removes it, and the re-read list agrees", async () => { - // Scoped to the visible popconfirm, and pressed once it has stopped arriving — - // see `helpers/resource` for what each of those is protecting against. - await pressOnce(confirmation(page).getByRole("button", { name: "Delete" })); - await page.waitForURL(/\/agents\?.*tab=templates/, { timeout: READ_TIMEOUT }); - created = false; + // Back to reading, showing the saved value rather than the draft: a save that did + // not reach the backend would leave the old one here. + await expect(page.getByTestId("template-edit")).toBeVisible({ timeout: READ_TIMEOUT }); + await expect(page.getByTestId("template-form-description")).toHaveValue( + DESCRIPTION, + ); + }); + + await test.step("7. deleting says what it costs, against the backend's own count", async () => { + const remove = page.getByTestId(`delete-${TEMPLATE}`); + await expect(remove).toContainText("Delete template"); + await remove.click(); - // The rest of the list is still there — the cluster installs templates of its own - // — so "gone" names that one template rather than a read that returned nothing. - // That distinction is the whole reason `expectNoLoadFailure` exists, and an empty - // table is exactly how a failed list would look. Waited for before it is asked, - // the count being taken once. - await expect(dataRows(page).first()).toBeVisible({ timeout: READ_TIMEOUT }); - await expectNoLoadFailure(page); - await expect(rowNamed(page, TEMPLATE)).toHaveCount(0, { timeout: READ_TIMEOUT }); - }); - } finally { /* - * A real resource on a real cluster, so a run that dies midway takes it with it. - * Through the UI because the app speaks gRPC-Web — there is no REST endpoint to - * call, though the previous version of this file believed there was. + * Either branch is legitimate: the count comes from `status.harnesses` while step 5 + * waited on `admittingHarnesses`, two fields filled by different work. The mock + * suite pins the wording; what a cluster shows is that the sentence is computed + * from real state at all, rather than coming back blank. */ - if (created) { - await sweepQuietly(TEMPLATE, async () => { - await loadApp(page, `/agent-templates/${NAMESPACE}/${TEMPLATE}`); - /* - * Guarded, like every other shared spec's cleanup. `created` says a create - * succeeded, not that the template is still there — and on the mock projects the - * navigation above restarts the in-browser backend, so after a failure midway it - * is reliably *not* there. An unguarded click then waits out the whole test - * budget on a button that will never appear, and the run reports a timeout in - * the cleanup instead of the assertion that actually failed. - */ - /* - * Waited for, not counted once. `expectSettled` vouches for the shell and for - * antd spinners, and this page loads behind a `Skeleton` instead — so the count - * lands before the read does, reads zero, and leaves the template on the cluster. - */ - const remove = page.getByTestId(`delete-${TEMPLATE}`); - if (await appeared(remove)) { - await remove.click(); - await pressOnce(confirmation(page).getByRole("button", { name: "Delete" })); - await page.waitForURL(/\/agents\?.*tab=templates/, { timeout: READ_TIMEOUT }); - } - }); - } - } + await expect(page.getByTestId("template-delete-consequence")).toContainText( + /built from this template|no agent was ever built from it/, + { timeout: 30_000 }, + ); + }); + + await test.step("8. confirming removes it, and the re-read list agrees", async () => { + // Scoped to the visible popconfirm, and pressed once it has stopped arriving — + // see `helpers/resource` for what each of those is protecting against. + await pressOnce(confirmation(page).getByRole("button", { name: "Delete" })); + await page.waitForURL(/\/agents\?.*tab=templates/, { timeout: READ_TIMEOUT }); + created = false; + + // "Gone" has to name that one template rather than a read that returned nothing, + // and an empty table is exactly how a failed list would look. Said by the summary, + // which only a successful read draws: a row would do too, but only on a cluster + // that owns a template besides the one just deleted. + await expectListLoaded(page, "templates"); + await expectNoLoadFailure(page); + await expect(rowNamed(page, TEMPLATE)).toHaveCount(0, { timeout: READ_TIMEOUT }); + }); }); diff --git a/ui/playwright/shared/harnesses/harnesses.spec.ts b/ui/playwright/shared/harnesses/harnesses.spec.ts index 8471d5dad3..98187ce8ce 100644 --- a/ui/playwright/shared/harnesses/harnesses.spec.ts +++ b/ui/playwright/shared/harnesses/harnesses.spec.ts @@ -1,10 +1,14 @@ -import { type Page } from "@playwright/test"; import { test, expect } from "../../fixtures/test"; -import { loadApp, throwawayName } from "../../helpers/app"; +import { + READ_TIMEOUT, + expectListTotal, + loadApp, + readListTotal, + throwawayName, +} from "../../helpers/app"; import { sweepQuietly } from "../../helpers/cleanup"; import { LIFECYCLE_TIMEOUT, - READ_TIMEOUT, appeared, confirmDelete, selectOption, @@ -30,116 +34,122 @@ const SNAPSHOT = "s3://ate-snapshots/kagent"; /** Its rows, which is the surface that can say whether any of this happened. */ const table = "harnesses-table"; +test.describe.configure({ timeout: LIFECYCLE_TIMEOUT }); + +/** Whether this run has a harness on the cluster, read by the hook below. */ +let created = false; + /* - * Data rows only. antd draws its loading and empty placeholders as a `tbody tr` too, so - * a bare `tbody tr` counts one row for a table that has not loaded — which read as a - * seeded set of one here, and only failed because the count afterwards disagreed. + * Cleanup in a hook, not in the body's `finally`. + * + * A timed-out test is the likeliest live failure — a controller that never reconciles — + * and it is exactly the one a `finally` cannot clean up after: Playwright has closed the + * page by then, so every call in it throws. Measured, with a four-second test: the + * `finally` was refused with "Target page, context or browser has been closed" while + * this hook still drove the page. Hooks get their own budget, which is the point. */ -const harnessRows = (page: Page) => - page.getByTestId(table).locator("tbody tr.ant-table-row"); - -test.describe.configure({ timeout: LIFECYCLE_TIMEOUT }); +test.afterEach(async ({ page }) => { + if (!created) return; + created = false; + await sweepQuietly(CREATED, async () => { + await loadApp(page, "/agents?tab=harnesses"); + // Waited for, not counted once: `loadApp` returns as soon as the shell is up, and a + // tab still fetching has no rows — which reads as "already gone" and leaves a real + // Harness on the cluster. See `appeared`. + if (await appeared(page.getByTestId(table).getByText(CREATED).first())) { + await confirmDelete(page, CREATED); + } + }); +}); test("harnesses: a harness is created, read and deleted", async ({ page }) => { - let created = false; /** What the tab held before this journey, so the counts below can be relative. */ let before = 0; - try { - await test.step("1. each field the cluster would refuse, refused on its own", async () => { - // Counted first: an absolute count is the fixtures' to make, but "one more, then - // one fewer" holds on any cluster. - await loadApp(page, "/agents?tab=harnesses"); - await expect(harnessRows(page).first()).toBeVisible({ timeout: READ_TIMEOUT }); - before = await harnessRows(page).count(); - - await loadApp(page, "/harnesses/new"); - const create = page.getByTestId("harness-create"); - - await selectOption(page, "harness-namespace", "kagent"); - await page.getByTestId("harness-name").fill(CREATED); - await page.getByTestId("harness-worker-pool").fill("kagent-default"); - await page.getByTestId("harness-image").fill(PINNED); - await page.getByTestId("harness-snapshot").fill(SNAPSHOT); - - // A harness with no selector admits nothing, and the form says so before it is - // asked to create one — the one state that is a warning rather than a refusal. - await expect(page.getByTestId("harness-admits-nothing")).toBeVisible(); - await page.getByTestId("harness-selector-key").fill("runtime"); - await page.getByTestId("harness-selector-value").fill(CREATED); - await expect(page.getByTestId("harness-admits-nothing")).toHaveCount(0); - - /* - * Enabled with a complete draft, and each field below then broken on its own. - * Asserted the other way round — a bad image on a half-filled form — both of - * these passed on a form that had never looked at the field in question: the - * empty snapshot location was disabling the button by itself. - */ - await expect(create).toBeEnabled(); - - // The image, which the cluster refuses as a tag because a tag can move under a - // running agent. The form says which of the two it is unhappy about. - await page.getByTestId("harness-image").fill("ghcr.io/example/runtime:latest"); - await expect(create).toBeDisabled(); - await expect(page.getByText(/Pin the image by digest/)).toBeVisible(); - await page.getByTestId("harness-image").fill(PINNED); - await expect(create).toBeEnabled(); - - // And the snapshot location, which the CRD requires and which the controller - // would otherwise reject as "Invalid Harness", naming no field. - await page.getByTestId("harness-snapshot").fill(""); - await expect(create).toBeDisabled(); - await page.getByTestId("harness-snapshot").fill(SNAPSHOT); + await test.step("1. each field the cluster would refuse, refused on its own", async () => { + // Counted first: an absolute count is the fixtures' to make, but "one more, then + // one fewer" holds on any cluster. + await loadApp(page, "/agents?tab=harnesses"); + /* + * Off the summary, like every other list here. Counting rows instead needed a row to + * wait for, which demanded that the cluster already own a harness — and reading the + * count straight after the summary does not work either: the summary renders a frame + * before the rows do, so the count came back 0 against a tab holding four. + */ + before = await readListTotal(page, "harnesses"); + + await loadApp(page, "/harnesses/new"); + const create = page.getByTestId("harness-create"); + + await selectOption(page, "harness-namespace", "kagent"); + await page.getByTestId("harness-name").fill(CREATED); + await page.getByTestId("harness-worker-pool").fill("kagent-default"); + await page.getByTestId("harness-image").fill(PINNED); + await page.getByTestId("harness-snapshot").fill(SNAPSHOT); + + // A harness with no selector admits nothing, and the form says so before it is + // asked to create one — the one state that is a warning rather than a refusal. + await expect(page.getByTestId("harness-admits-nothing")).toBeVisible(); + await page.getByTestId("harness-selector-key").fill("runtime"); + await page.getByTestId("harness-selector-value").fill(CREATED); + await expect(page.getByTestId("harness-admits-nothing")).toHaveCount(0); + + /* + * Enabled with a complete draft, and each field below then broken on its own. + * Asserted the other way round — a bad image on a half-filled form — both of + * these passed on a form that had never looked at the field in question: the + * empty snapshot location was disabling the button by itself. + */ + await expect(create).toBeEnabled(); + + // The image, which the cluster refuses as a tag because a tag can move under a + // running agent. The form says which of the two it is unhappy about. + await page.getByTestId("harness-image").fill("ghcr.io/example/runtime:latest"); + await expect(create).toBeDisabled(); + await expect(page.getByText(/Pin the image by digest/)).toBeVisible(); + await page.getByTestId("harness-image").fill(PINNED); + await expect(create).toBeEnabled(); + + // And the snapshot location, which the CRD requires and which the controller + // would otherwise reject as "Invalid Harness", naming no field. + await page.getByTestId("harness-snapshot").fill(""); + await expect(create).toBeDisabled(); + await page.getByTestId("harness-snapshot").fill(SNAPSHOT); + }); + + await test.step("2. a complete draft is created", async () => { + await expect(page.getByTestId("harness-create")).toBeEnabled(); + await page.getByTestId("harness-create").click(); + + // Back to the tab it came from, with the new harness in the list. Read back off + // the table rather than from a toast: "the create returned" and "the thing + // exists" are different claims, and only the list checks the second. + await page.waitForURL(/tab=harnesses/, { timeout: READ_TIMEOUT }); + created = true; + await expect(page.getByTestId(table)).toContainText(CREATED, { + timeout: READ_TIMEOUT, }); + await expectListTotal(page, "harnesses", before + 1); + }); - await test.step("2. a complete draft is created", async () => { - await expect(page.getByTestId("harness-create")).toBeEnabled(); - await page.getByTestId("harness-create").click(); - - // Back to the tab it came from, with the new harness in the list. Read back off - // the table rather than from a toast: "the create returned" and "the thing - // exists" are different claims, and only the list checks the second. - await page.waitForURL(/tab=harnesses/, { timeout: READ_TIMEOUT }); - created = true; - await expect(page.getByTestId(table)).toContainText(CREATED, { timeout: READ_TIMEOUT }); - await expect.poll(() => harnessRows(page).count(), { timeout: READ_TIMEOUT }).toBe( - before + 1, - ); + await test.step("3. and it is not ready yet, which is what a cluster reports", async () => { + const row = page.getByTestId(table).locator("tr", { hasText: CREATED }); + await expect(row.getByTestId("harness-ready")).toContainText("Not ready yet", { + timeout: READ_TIMEOUT, }); + }); - await test.step("3. and it is not ready yet, which is what a cluster reports", async () => { - const row = page.getByTestId(table).locator("tr", { hasText: CREATED }); - await expect(row.getByTestId("harness-ready")).toContainText("Not ready yet", { - timeout: READ_TIMEOUT, - }); - }); - - await test.step("4. it is removed from the same tab, and the rest stays", async () => { - await confirmDelete(page, CREATED); + await test.step("4. it is removed from the same tab, and the rest stays", async () => { + await confirmDelete(page, CREATED); - await expect(page.getByTestId(table)).not.toContainText(CREATED, { - timeout: READ_TIMEOUT, - }); - created = false; - // One row went, not the table: "gone" has to mean that harness rather than a read - // that failed and left an empty list behind it. - await expect.poll(() => harnessRows(page).count(), { timeout: READ_TIMEOUT }).toBe(before); - await expect(page.getByTestId("harnesses-delete-error")).toHaveCount(0); + await expect(page.getByTestId(table)).not.toContainText(CREATED, { + timeout: READ_TIMEOUT, }); - } finally { - if (created) { - // Through `sweepQuietly` like every sibling: `loadApp` and `confirmDelete` both - // throw, and thrown from here that replaces the failure the test was reporting and - // skips the delete under it. This was the one block still doing that. - await sweepQuietly(CREATED, async () => { - await loadApp(page, "/agents?tab=harnesses"); - // Waited for, not counted once: `loadApp` returns as soon as the shell is up, - // and a tab still fetching has no rows — which reads as "already gone" and - // leaves a real Harness on the cluster. See `appeared`. - if (await appeared(page.getByTestId(table).getByText(CREATED).first())) { - await confirmDelete(page, CREATED); - } - }); - } - } + created = false; + // One row went, not the table: "gone" has to mean that harness rather than a read + // that failed and left an empty list behind it, and the summary is drawn only for a + // read that succeeded. + await expectListTotal(page, "harnesses", before); + await expect(page.getByTestId("harnesses-delete-error")).toHaveCount(0); + }); }); diff --git a/ui/playwright/shared/models/models.spec.ts b/ui/playwright/shared/models/models.spec.ts index 2cfca7e0d6..61baceeec9 100644 --- a/ui/playwright/shared/models/models.spec.ts +++ b/ui/playwright/shared/models/models.spec.ts @@ -1,6 +1,5 @@ import { test, expect } from "../../fixtures/test"; import { - dataRows, expectListLoaded, expectListTotal, expectNoLoadFailure, @@ -9,10 +8,10 @@ import { rowNamed, searchList, throwawayName, + READ_TIMEOUT, } from "../../helpers/app"; import { LIFECYCLE_TIMEOUT, - READ_TIMEOUT, confirmDelete, confirmation, selectOption, @@ -38,137 +37,150 @@ const SECRET = "kagent-shared-e2e-secret"; test.describe.configure({ timeout: LIFECYCLE_TIMEOUT }); +/** Whether this run has a configuration on the cluster, read by the hook below. */ +let created = false; + +/* + * Cleanup in a hook, not in the body's `finally`. + * + * A timed-out test is the likeliest live failure — a controller that never reconciles — + * and it is exactly the one a `finally` cannot clean up after: Playwright has closed the + * page by then, so every call in it throws. Measured, with a four-second test: the + * `finally` was refused with "Target page, context or browser has been closed" while + * this hook still drove the page. Hooks get their own budget, which is the point. + */ +test.afterEach(async ({ page }) => { + if (!created) return; + created = false; + await sweepUp(page, "models", CREATED); +}); + test("models: a configuration is created, read, changed and deleted", async ({ page, }) => { - let created = false; /** What the list held before this journey, so the counts below can be relative. */ let before = 0; - try { - await test.step("1. a filled-in configuration is created and appears on the list", async () => { - /* - * Counted first, and relative from here on. The fixtures seed four and a cluster - * seeds whatever it was installed with, so an absolute count is the one thing this - * spec cannot assert — but "one more than before" is exactly as strong, and it is - * what catches a create that wrote two rows or a delete that took a neighbour. - * - * Off the summary rather than by counting rows: the table pages at 25, and a - * cluster is free to hold more than that — see `readListTotal`. - */ - await loadApp(page, "/models"); - await expect(dataRows(page).first()).toBeVisible({ timeout: READ_TIMEOUT }); - before = await readListTotal(page, "models"); - - await page.getByTestId("models-new").click(); - await page.waitForURL(/\/models\/new(\?|$)/, { timeout: READ_TIMEOUT }); - - // The provider list is the app's own enum rather than the backend's, so the name - // a reader sees is the same on either — `providerDisplayName` turns - // `AmazonBedrock` into "AWS Bedrock". - await selectOption(page, "model-provider", "Anthropic"); - - // An AutoComplete, not a Select: the id is on the wrapper and the caret goes in - // the input inside it. Typed rather than picked, the field existing to accept a - // model the catalogue has not heard of. - await page.getByTestId("model-model").locator("input").fill("claude-sonnet-4"); - - await page.getByTestId("model-name").fill(CREATED); - // The namespace is half the ref, so one created without it is addressed as - // `/name` and never appears on the list. `kagent` is where both backends put - // things. - await selectOption(page, "model-namespace", "kagent"); - await page.getByTestId("model-api-key").fill("sk-not-a-real-key"); - - await page.getByTestId("model-submit").click(); - await page.waitForURL(/\/models(\?|$)/, { timeout: READ_TIMEOUT }); - created = true; - - // Read back off the list rather than from a toast or a closed form: those two - // only prove the app believes it worked. Narrowed to the one name this run - // invented, so the assertions below are about that row wherever the cluster's own - // configurations put it. - await searchList(page, "models", CREATED); - // The list has answered before its alerts are counted: `waitForURL` lands on a - // page that has not read anything yet, where there is nothing to count. - await expectListLoaded(page, "models"); - await expectNoLoadFailure(page); - const row = rowNamed(page, CREATED); - await expect(row).toHaveCount(1, { timeout: READ_TIMEOUT }); - await expect(row).toContainText("Anthropic"); - await expect(row).toContainText("claude-sonnet-4"); - await expectListTotal(page, "models", before + 1); - }); - - await test.step("2. the edit form opens on what was saved, not a blank draft", async () => { - await page.getByTestId(`edit-${CREATED}`).click(); - await page.waitForURL(new RegExp(`/models/kagent/${CREATED}/edit$`), { - timeout: READ_TIMEOUT, - }); - - await expect(page.getByTestId("model-name")).toHaveValue(CREATED, { - timeout: READ_TIMEOUT, - }); - /* - * The identity and the provider are the ref and what the ref means, so an edit - * changes neither. Asserted here because it is the boundary between "edit" and - * "make a new one", and a form that quietly allowed it would write a resource - * nothing else in the cluster points at. - */ - await expect(page.getByTestId("model-name")).toBeDisabled(); - await expect(page.getByTestId("model-model").locator("input")).toBeDisabled(); - }); - - await test.step("3. a change is saved, and the list shows it", async () => { - // The credential moves from one the backend minted to a secret it is told to - // read — the one change on this form the list has a column for, which is what - // makes the save checkable from outside the form. - await page - .getByTestId("model-auth-type") - .getByText("Existing secret", { exact: true }) - .click(); - await page.getByTestId("model-api-key-secret").fill(SECRET); - - await page.getByTestId("model-submit").click(); - await page.waitForURL(/\/models(\?|$)/, { timeout: READ_TIMEOUT }); - - // The search went with the form; the list is whole again on the way back. - await searchList(page, "models", CREATED); - const row = rowNamed(page, CREATED); - await expect(row).toContainText(SECRET, { timeout: READ_TIMEOUT }); - // Changed, not duplicated — which a create dressed as an update would be. - await expect(row).toHaveCount(1); - await expectListTotal(page, "models", before + 1); - }); - - await test.step("4. deleting asks first, and Keep leaves it alone", async () => { - await page.getByTestId(`delete-${CREATED}`).click(); - const prompt = confirmation(page); - // The confirmation names the row. "Delete this model configuration?" is no help - // in a table of five, and *which* is the one question the reader has. - await expect(prompt).toContainText(CREATED); - await prompt.getByRole("button", { name: "Keep" }).click(); - await expect(rowNamed(page, CREATED)).toHaveCount(1); - // Waited out rather than assumed gone: the dialog stays visible while it - // animates away, and the next step's click would land on it. - await expect(prompt).toHaveCount(0); + await test.step("1. a filled-in configuration is created and appears on the list", async () => { + /* + * Counted first, and relative from here on. The fixtures seed four and a cluster + * seeds whatever it was installed with, so an absolute count is the one thing this + * spec cannot assert — but "one more than before" is exactly as strong, and it is + * what catches a create that wrote two rows or a delete that took a neighbour. + * + * Off the summary rather than by counting rows: the table pages at 25, and a + * cluster is free to hold more than that — see `readListTotal`. + */ + await loadApp(page, "/models"); + // Off the summary alone: `readListTotal` waits for it, and it renders at "0 of 0" + // for a read that succeeded. Waiting for a row first demanded that the cluster + // already own a configuration, which is not something a shared spec may assume. + before = await readListTotal(page, "models"); + + await page.getByTestId("models-new").click(); + await page.waitForURL(/\/models\/new(\?|$)/, { timeout: READ_TIMEOUT }); + + // The provider list is the app's own enum rather than the backend's, so the name + // a reader sees is the same on either — `providerDisplayName` turns + // `AmazonBedrock` into "AWS Bedrock". + await selectOption(page, "model-provider", "Anthropic"); + + // An AutoComplete, not a Select: the id is on the wrapper and the caret goes in + // the input inside it. Typed rather than picked, the field existing to accept a + // model the catalogue has not heard of. + await page.getByTestId("model-model").locator("input").fill("claude-sonnet-4"); + + await page.getByTestId("model-name").fill(CREATED); + // The namespace is half the ref, so one created without it is addressed as + // `/name` and never appears on the list. `kagent` is where both backends put + // things. + await selectOption(page, "model-namespace", "kagent"); + await page.getByTestId("model-api-key").fill("sk-not-a-real-key"); + + await page.getByTestId("model-submit").click(); + await page.waitForURL(/\/models(\?|$)/, { timeout: READ_TIMEOUT }); + created = true; + + // Read back off the list rather than from a toast or a closed form: those two + // only prove the app believes it worked. Narrowed to the one name this run + // invented, so the assertions below are about that row wherever the cluster's own + // configurations put it. + await searchList(page, "models", CREATED); + // The list has answered before its alerts are counted: `waitForURL` lands on a + // page that has not read anything yet, where there is nothing to count. + await expectListLoaded(page, "models"); + await expectNoLoadFailure(page); + const row = rowNamed(page, CREATED); + await expect(row).toHaveCount(1, { timeout: READ_TIMEOUT }); + await expect(row).toContainText("Anthropic"); + await expect(row).toContainText("claude-sonnet-4"); + await expectListTotal(page, "models", before + 1); + }); + + await test.step("2. the edit form opens on what was saved, not a blank draft", async () => { + await page.getByTestId(`edit-${CREATED}`).click(); + await page.waitForURL(new RegExp(`/models/kagent/${CREATED}/edit$`), { + timeout: READ_TIMEOUT, }); - await test.step("5. confirming removes that row and leaves the rest", async () => { - await confirmDelete(page, CREATED); - await expect(rowNamed(page, CREATED)).toHaveCount(0, { timeout: READ_TIMEOUT }); - created = false; - - // One row went, not several, and not the read: a list that failed to reload is - // also a list the row is missing from, and the summary the total is read off - // renders only for a load that succeeded — so it is what is waited for, and the - // alert count after it is what names a failure. - await expectListTotal(page, "models", before); - await expectNoLoadFailure(page); + await expect(page.getByTestId("model-name")).toHaveValue(CREATED, { + timeout: READ_TIMEOUT, }); - } finally { - // A real resource on a real cluster when this runs live, so a run that dies midway - // takes it with it. - if (created) await sweepUp(page, "models", CREATED); - } + /* + * The identity and the provider are the ref and what the ref means, so an edit + * changes neither. Asserted here because it is the boundary between "edit" and + * "make a new one", and a form that quietly allowed it would write a resource + * nothing else in the cluster points at. + */ + await expect(page.getByTestId("model-name")).toBeDisabled(); + await expect(page.getByTestId("model-model").locator("input")).toBeDisabled(); + }); + + await test.step("3. a change is saved, and the list shows it", async () => { + // The credential moves from one the backend minted to a secret it is told to + // read — the one change on this form the list has a column for, which is what + // makes the save checkable from outside the form. + await page + .getByTestId("model-auth-type") + .getByText("Existing secret", { exact: true }) + .click(); + await page.getByTestId("model-api-key-secret").fill(SECRET); + + await page.getByTestId("model-submit").click(); + await page.waitForURL(/\/models(\?|$)/, { timeout: READ_TIMEOUT }); + + // The search went with the form; the list is whole again on the way back. + await searchList(page, "models", CREATED); + const row = rowNamed(page, CREATED); + await expect(row).toContainText(SECRET, { timeout: READ_TIMEOUT }); + // Changed, not duplicated — which a create dressed as an update would be. + await expect(row).toHaveCount(1); + await expectListTotal(page, "models", before + 1); + }); + + await test.step("4. deleting asks first, and Keep leaves it alone", async () => { + await page.getByTestId(`delete-${CREATED}`).click(); + const prompt = confirmation(page); + // The confirmation names the row. "Delete this model configuration?" is no help + // in a table of five, and *which* is the one question the reader has. + await expect(prompt).toContainText(CREATED); + await prompt.getByRole("button", { name: "Keep" }).click(); + await expect(rowNamed(page, CREATED)).toHaveCount(1); + // Waited out rather than assumed gone: the dialog stays visible while it + // animates away, and the next step's click would land on it. + await expect(prompt).toHaveCount(0); + }); + + await test.step("5. confirming removes that row and leaves the rest", async () => { + await confirmDelete(page, CREATED); + await expect(rowNamed(page, CREATED)).toHaveCount(0, { timeout: READ_TIMEOUT }); + created = false; + + // One row went, not several, and not the read: a list that failed to reload is + // also a list the row is missing from, and the summary the total is read off + // renders only for a load that succeeded — so it is what is waited for, and the + // alert count after it is what names a failure. + await expectListTotal(page, "models", before); + await expectNoLoadFailure(page); + }); }); diff --git a/ui/playwright/shared/prompts/prompts.spec.ts b/ui/playwright/shared/prompts/prompts.spec.ts index 4a0d32a3f6..d96a795ade 100644 --- a/ui/playwright/shared/prompts/prompts.spec.ts +++ b/ui/playwright/shared/prompts/prompts.spec.ts @@ -1,7 +1,6 @@ import { type Page } from "@playwright/test"; import { test, expect } from "../../fixtures/test"; import { - dataRows, expectListLoaded, expectListTotal, expectNoLoadFailure, @@ -10,10 +9,10 @@ import { rowNamed, searchList, throwawayName, + READ_TIMEOUT, } from "../../helpers/app"; import { LIFECYCLE_TIMEOUT, - READ_TIMEOUT, confirmDelete, } from "../../helpers/resource"; import { sweepUp } from "../../helpers/cleanup"; @@ -37,110 +36,124 @@ const fragmentValue = (page: Page, index: number) => test.describe.configure({ timeout: LIFECYCLE_TIMEOUT }); +/** Whether this run has a library on the cluster, read by the hook below. */ +let created = false; + +/* + * Cleanup in a hook, not in the body's `finally`. + * + * A timed-out test is the likeliest live failure — a controller that never reconciles — + * and it is exactly the one a `finally` cannot clean up after: Playwright has closed the + * page by then, so every call in it throws. Measured, with a four-second test: the + * `finally` was refused with "Target page, context or browser has been closed" while + * this hook still drove the page. Hooks get their own budget, which is the point. + */ +test.afterEach(async ({ page }) => { + if (!created) return; + created = false; + await sweepUp(page, "prompts", CREATED); +}); + test("prompts: a library is created, read, changed and deleted", async ({ page }) => { - let created = false; /** What the list held before this journey, so the counts below can be relative. */ let before = 0; - try { - await test.step("1. a library with one fragment is created and listed", async () => { - // Counted first, and relative from here on: the fixtures seed two and a cluster - // seeds whatever it was installed with, but "one more than before" is exactly as - // strong and catches a create that wrote twice. Off the summary rather than by - // counting rows, the table paging at 25 — see `readListTotal`. - await loadApp(page, "/prompts"); - await expect(dataRows(page).first()).toBeVisible({ timeout: READ_TIMEOUT }); - before = await readListTotal(page, "prompts"); - - await page.getByTestId("prompts-new").click(); - await expect(page.getByTestId("prompt-submit")).toBeVisible({ timeout: READ_TIMEOUT }); - - await page.getByTestId("prompt-name").fill(CREATED); - await page.getByTestId("prompt-namespace").fill("kagent"); - await fragmentKey(page, 0).fill("changelog"); - await fragmentValue(page, 0).fill("Group by user impact."); - - await page.getByTestId("prompt-submit").click(); - await expect(page).toHaveURL(/\/prompts$/, { timeout: READ_TIMEOUT }); - created = true; - - // Read back off the list rather than from a toast or a closed form: those two - // only prove the app believes it worked. Narrowed to the one name this run - // invented, so the assertions below are about that row wherever the cluster's own - // libraries put it. - await searchList(page, "prompts", CREATED); - // The list has answered before its alerts are counted — see `expectNoLoadFailure`. - await expectListLoaded(page, "prompts"); - await expectNoLoadFailure(page); - const row = rowNamed(page, CREATED); - await expect(row).toContainText("1 key", { timeout: READ_TIMEOUT }); - await expect(row).toContainText("changelog"); - await expectListTotal(page, "prompts", before + 1); + await test.step("1. a library with one fragment is created and listed", async () => { + // Counted first, and relative from here on: the fixtures seed two and a cluster + // seeds whatever it was installed with, but "one more than before" is exactly as + // strong and catches a create that wrote twice. Off the summary rather than by + // counting rows, the table paging at 25 — see `readListTotal`. + await loadApp(page, "/prompts"); + // Off the summary alone — see `models`: waiting for a row demanded a library the + // cluster need not have. + before = await readListTotal(page, "prompts"); + + await page.getByTestId("prompts-new").click(); + await expect(page.getByTestId("prompt-submit")).toBeVisible({ timeout: READ_TIMEOUT }); + + await page.getByTestId("prompt-name").fill(CREATED); + await page.getByTestId("prompt-namespace").fill("kagent"); + await fragmentKey(page, 0).fill("changelog"); + await fragmentValue(page, 0).fill("Group by user impact."); + + await page.getByTestId("prompt-submit").click(); + await expect(page).toHaveURL(/\/prompts$/, { timeout: READ_TIMEOUT }); + created = true; + + // Read back off the list rather than from a toast or a closed form: those two + // only prove the app believes it worked. Narrowed to the one name this run + // invented, so the assertions below are about that row wherever the cluster's own + // libraries put it. + await searchList(page, "prompts", CREATED); + // The list has answered before its alerts are counted — see `expectNoLoadFailure`. + await expectListLoaded(page, "prompts"); + await expectNoLoadFailure(page); + const row = rowNamed(page, CREATED); + await expect(row).toContainText("1 key", { timeout: READ_TIMEOUT }); + await expect(row).toContainText("changelog"); + await expectListTotal(page, "prompts", before + 1); + }); + + await test.step("2. opening it shows the fragment and how to include it", async () => { + await rowNamed(page, CREATED).getByRole("link").first().click(); + await page.waitForURL(new RegExp(`/prompts/kagent/${CREATED}$`), { + timeout: READ_TIMEOUT, }); - await test.step("2. opening it shows the fragment and how to include it", async () => { - await rowNamed(page, CREATED).getByRole("link").first().click(); - await page.waitForURL(new RegExp(`/prompts/kagent/${CREATED}$`), { - timeout: READ_TIMEOUT, - }); + const fragments = page.getByTestId("prompt-fragments"); + await expect(fragments).toContainText("changelog", { timeout: READ_TIMEOUT }); + await expect(fragments).toContainText("Group by user impact."); + }); - const fragments = page.getByTestId("prompt-fragments"); - await expect(fragments).toContainText("changelog", { timeout: READ_TIMEOUT }); - await expect(fragments).toContainText("Group by user impact."); + await test.step("3. a fragment is added, saved, and read back off the library", async () => { + await page.getByTestId("prompt-edit").click(); + await page.waitForURL(new RegExp(`/prompts/kagent/${CREATED}/edit$`), { + timeout: READ_TIMEOUT, }); - - await test.step("3. a fragment is added, saved, and read back off the library", async () => { - await page.getByTestId("prompt-edit").click(); - await page.waitForURL(new RegExp(`/prompts/kagent/${CREATED}/edit$`), { - timeout: READ_TIMEOUT, - }); - // Seeded from the saved library, so the form and the page it came from agree. - await expect(fragmentKey(page, 0)).toHaveValue("changelog", { timeout: READ_TIMEOUT }); - - await page.getByTestId("fragment-add").click(); - await fragmentKey(page, 1).fill("handoff"); - await fragmentValue(page, 1).fill("Name the next owner explicitly."); - // The include tag is what a fragment is for, and it is offered before the save - // rather than only after it. - await expect(page.getByTestId("fragment-include-preview").last()).toContainText( - `{{include "${CREATED}/handoff"}}`, - ); - - await page.getByTestId("prompt-submit").click(); - await expect(page).toHaveURL(new RegExp(`/prompts/kagent/${CREATED}$`), { - timeout: READ_TIMEOUT, - }); - - // Read back from the re-read library rather than from the draft: a save that - // never reached the backend would leave the old text here. - const fragments = page.getByTestId("prompt-fragments"); - await expect(fragments).toContainText("Name the next owner explicitly.", { - timeout: READ_TIMEOUT, - }); - await expect(page.getByTestId("prompt-detail-meta")).toContainText("2 fragments"); - }); - - await test.step("4. the list behind it shows the change too", async () => { - await page.getByRole("link", { name: "Back to libraries" }).click(); - // The search went with the detail page; the list is whole again on the way back. - await searchList(page, "prompts", CREATED); - await expect(rowNamed(page, CREATED)).toContainText("2 keys", { timeout: READ_TIMEOUT }); - await expect(rowNamed(page, CREATED)).toContainText("handoff"); + // Seeded from the saved library, so the form and the page it came from agree. + await expect(fragmentKey(page, 0)).toHaveValue("changelog", { timeout: READ_TIMEOUT }); + + await page.getByTestId("fragment-add").click(); + await fragmentKey(page, 1).fill("handoff"); + await fragmentValue(page, 1).fill("Name the next owner explicitly."); + // The include tag is what a fragment is for, and it is offered before the save + // rather than only after it. + await expect(page.getByTestId("fragment-include-preview").last()).toContainText( + `{{include "${CREATED}/handoff"}}`, + ); + + await page.getByTestId("prompt-submit").click(); + await expect(page).toHaveURL(new RegExp(`/prompts/kagent/${CREATED}$`), { + timeout: READ_TIMEOUT, }); - await test.step("5. confirming a delete removes that row and leaves the rest", async () => { - await confirmDelete(page, CREATED); - await expect(rowNamed(page, CREATED)).toHaveCount(0, { timeout: READ_TIMEOUT }); - created = false; - - // One row went, not several, and not the read: a list that failed to reload is - // also a list the row is missing from, and the summary the total is read off - // renders only for a load that succeeded — so it is what is waited for, and the - // alert count after it is what names a failure. - await expectListTotal(page, "prompts", before); - await expectNoLoadFailure(page); + // Read back from the re-read library rather than from the draft: a save that + // never reached the backend would leave the old text here. + const fragments = page.getByTestId("prompt-fragments"); + await expect(fragments).toContainText("Name the next owner explicitly.", { + timeout: READ_TIMEOUT, }); - } finally { - if (created) await sweepUp(page, "prompts", CREATED); - } + await expect(page.getByTestId("prompt-detail-meta")).toContainText("2 fragments"); + }); + + await test.step("4. the list behind it shows the change too", async () => { + await page.getByRole("link", { name: "Back to libraries" }).click(); + // The search went with the detail page; the list is whole again on the way back. + await searchList(page, "prompts", CREATED); + await expect(rowNamed(page, CREATED)).toContainText("2 keys", { timeout: READ_TIMEOUT }); + await expect(rowNamed(page, CREATED)).toContainText("handoff"); + }); + + await test.step("5. confirming a delete removes that row and leaves the rest", async () => { + await confirmDelete(page, CREATED); + await expect(rowNamed(page, CREATED)).toHaveCount(0, { timeout: READ_TIMEOUT }); + created = false; + + // One row went, not several, and not the read: a list that failed to reload is + // also a list the row is missing from, and the summary the total is read off + // renders only for a load that succeeded — so it is what is waited for, and the + // alert count after it is what names a failure. + await expectListTotal(page, "prompts", before); + await expectNoLoadFailure(page); + }); }); diff --git a/ui/playwright/shared/schedules/schedules.spec.ts b/ui/playwright/shared/schedules/schedules.spec.ts index 7c61d17e5e..f7c6b0645e 100644 --- a/ui/playwright/shared/schedules/schedules.spec.ts +++ b/ui/playwright/shared/schedules/schedules.spec.ts @@ -1,10 +1,9 @@ import { test, expect } from "../../fixtures/test"; -import { loadApp, throwawayName } from "../../helpers/app"; +import { READ_TIMEOUT, loadApp, throwawayName } from "../../helpers/app"; import { tick } from "../../helpers/controls"; import { sweepQuietly } from "../../helpers/cleanup"; import { LIFECYCLE_TIMEOUT, - READ_TIMEOUT, appeared, optionNamed, pressUntil, @@ -27,146 +26,154 @@ const CREATED = throwawayName("schedule"); test.describe.configure({ timeout: LIFECYCLE_TIMEOUT }); -test("schedules: one is created, read, changed and deleted", async ({ page }) => { - let detailURL: string | undefined; - - try { - await test.step("1. the form offers the backend's own agents", async () => { - await loadApp(page, "/schedules"); - await page.getByTestId("schedules-new").click(); - await expect(page).toHaveURL(/\/schedules\/new(\?|$)/); - - await page.getByTestId("schedule-agent").click(); - // Whichever agent this install has: which one has nothing to do with the claim. - const agent = optionNamed(page).first(); - await expect(agent, "no agents were offered to schedule").toBeVisible({ - timeout: 30_000, - }); - await agent.click(); - }); - - await test.step("2. a weekly, zoned, fractionally-timed schedule is described", async () => { - await page.getByTestId("schedule-name").fill(CREATED); - - await page.getByTestId("schedule-frequency").click(); - // Pressed until the cadence actually changes: the weekday checkboxes only exist - // once the frequency is weekly, so a dropdown click swallowed by the animation - // leaves the next line waiting for controls that are never coming. - await pressUntil(optionNamed(page, "Weekly"), () => - expect(page.getByTestId("schedule-days")).toBeVisible(), - ); - // Monday is already on, so these four make it the whole working week — which the - // app states back as "Weekdays", and which is the reading asserted below. - for (const day of ["Tuesday", "Wednesday", "Thursday", "Friday"]) { - await tick(page.getByLabel(day, { exact: true })); - } - - await page.getByTestId("schedule-time").fill("09:00"); - // The time zone is an AutoComplete, so its id is on the wrapper and the caret goes - // in the input inside it. Escape dismisses the zone list, which otherwise sits - // over the fields below. - await page.getByTestId("schedule-timezone").locator("input").fill("America/New_York"); - await page.keyboard.press("Escape"); - - await page.getByTestId("schedule-prompt").fill("Report cluster health."); - // A fractional timeout, because it is the value a backend most easily rounds off. - await page.getByTestId("schedule-timeout").fill("90.001"); - - await page.getByTestId("schedule-enabled").uncheck(); - await expect(page.getByTestId("schedule-enabled-note")).toContainText( - "will not run automatically after it is created", - ); - }); - - await test.step("3. creating it lands on its own page, showing what was asked for", async () => { - await page.getByTestId("schedule-submit").click(); - await expect(page.getByRole("heading", { name: CREATED, exact: true })).toBeVisible({ - timeout: READ_TIMEOUT, - }); - detailURL = page.url(); - await expect(page).toHaveURL(/\/schedules\/[0-9a-f-]+(\?|$)/); - - // Created paused, so the one control whose label flips offers to resume it. - await expect(page.getByTestId("schedule-pause")).toHaveText("Resume"); - await expect(page.getByTestId("schedule-meta")).toContainText("Weekdays at 09:00"); - await expect(page.getByTestId("schedule-meta")).toContainText("America/New_York"); - // The fractional second survived the round trip rather than being floored to 90. - await expect(page.getByTestId("schedule-detail")).toContainText("90.001 seconds"); - }); - - await test.step("4. the edit form opens on the stored values, not on defaults", async () => { - await page.getByTestId("schedule-edit").click(); - await expect(page.getByTestId("schedule-time")).toHaveValue("09:00", { - timeout: READ_TIMEOUT, - }); - await expect(page.getByTestId("schedule-timeout")).toHaveValue("90.001"); - await expect(page.getByTestId("schedule-timezone").locator("input")).toHaveValue( - "America/New_York", - ); - await expect(page.getByTestId("schedule-enabled")).not.toBeChecked(); - // Both ends of the weekday set, so a picker that kept only the last day chosen - // would not pass on one assertion. - await expect(page.getByLabel("Monday", { exact: true })).toBeChecked(); - await expect(page.getByLabel("Friday", { exact: true })).toBeChecked(); - }); - - await test.step("5. an edit is saved and read back", async () => { - await page.getByTestId("schedule-prompt").fill("Report unhealthy workloads only."); - await page.getByTestId("schedule-submit").click(); - await expect(page.getByRole("heading", { name: CREATED, exact: true })).toBeVisible({ - timeout: READ_TIMEOUT, - }); - - const detail = page.getByTestId("schedule-detail"); - await expect(detail).toContainText("Report unhealthy workloads only.", { - timeout: READ_TIMEOUT, - }); - // And the update did not quietly reset what it was not asked to change. - await expect(detail).toContainText("90.001 seconds"); - }); +/** Where this run's schedule lives, so the hook below can remove it. */ +let detailURL: string | undefined; - await test.step("6. deleting asks in a modal, and confirming leaves for the list", async () => { - const remove = page - .getByTestId("schedule-danger") - .getByRole("button", { name: `Delete schedule ${CREATED}`, exact: true }); +/* + * Cleanup in a hook, not in the body's `finally`. + * + * A timed-out test is the likeliest live failure — a controller that never reconciles — + * and it is exactly the one a `finally` cannot clean up after: Playwright has closed the + * page by then, so every call in it throws. Measured, with a four-second test: the + * `finally` was refused with "Target page, context or browser has been closed" while + * this hook still drove the page. Hooks get their own budget, which is the point. + */ +test.afterEach(async ({ page }) => { + if (detailURL === undefined) return; + const detail = detailURL; + detailURL = undefined; + await sweepQuietly(CREATED, async () => { + await page.goto(detail); + const remove = page + .getByTestId("schedule-danger") + .getByRole("button", { name: `Delete schedule ${CREATED}`, exact: true }); + // Waited for, not counted once: `goto` resolves on load and the detail read has not + // landed, so the danger zone is not drawn yet. See `appeared`. + if (await appeared(remove)) { await remove.click(); - - // Pressed until it takes: a dropped Delete reports as "the page never navigated" - // rather than as a missed click. See `pressUntil`. await pressUntil( page .getByRole("dialog", { name: `Delete schedule ${CREATED}?`, exact: true }) .getByRole("button", { name: "Delete", exact: true }), () => expect(page).toHaveURL(/\/schedules(\?|$)/), ); - detailURL = undefined; + } + }); +}); - await expect(page.getByRole("link", { name: CREATED, exact: true })).toHaveCount(0, { - timeout: READ_TIMEOUT, - }); +test("schedules: one is created, read, changed and deleted", async ({ page }) => { + await test.step("1. the form offers the backend's own agents", async () => { + await loadApp(page, "/schedules"); + await page.getByTestId("schedules-new").click(); + await expect(page).toHaveURL(/\/schedules\/new(\?|$)/); + + await page.getByTestId("schedule-agent").click(); + // Whichever agent this install has: which one has nothing to do with the claim. + const agent = optionNamed(page).first(); + await expect(agent, "no agents were offered to schedule").toBeVisible({ + timeout: 30_000, }); - } finally { - // Live these are real resources, so a run that dies midway takes its own with it. - if (detailURL) { - // Captured, because the closure below outlives the narrowing of a `let`. - const detail = detailURL; - await sweepQuietly(CREATED, async () => { - await page.goto(detail); - const remove = page - .getByTestId("schedule-danger") - .getByRole("button", { name: `Delete schedule ${CREATED}`, exact: true }); - // Waited for, not counted once: `goto` resolves on load and the detail read has - // not landed, so the danger zone is not drawn yet. See `appeared`. - if (await appeared(remove)) { - await remove.click(); - await pressUntil( - page - .getByRole("dialog", { name: `Delete schedule ${CREATED}?`, exact: true }) - .getByRole("button", { name: "Delete", exact: true }), - () => expect(page).toHaveURL(/\/schedules(\?|$)/), - ); - } - }); + await agent.click(); + }); + + await test.step("2. a weekly, zoned, fractionally-timed schedule is described", async () => { + await page.getByTestId("schedule-name").fill(CREATED); + + await page.getByTestId("schedule-frequency").click(); + // Pressed until the cadence actually changes: the weekday checkboxes only exist + // once the frequency is weekly, so a dropdown click swallowed by the animation + // leaves the next line waiting for controls that are never coming. + await pressUntil(optionNamed(page, "Weekly"), () => + expect(page.getByTestId("schedule-days")).toBeVisible(), + ); + // Monday is already on, so these four make it the whole working week — which the + // app states back as "Weekdays", and which is the reading asserted below. + for (const day of ["Tuesday", "Wednesday", "Thursday", "Friday"]) { + await tick(page.getByLabel(day, { exact: true })); } - } + + await page.getByTestId("schedule-time").fill("09:00"); + // The time zone is an AutoComplete, so its id is on the wrapper and the caret goes + // in the input inside it. Escape dismisses the zone list, which otherwise sits + // over the fields below. + await page.getByTestId("schedule-timezone").locator("input").fill("America/New_York"); + await page.keyboard.press("Escape"); + + await page.getByTestId("schedule-prompt").fill("Report cluster health."); + // A fractional timeout, because it is the value a backend most easily rounds off. + await page.getByTestId("schedule-timeout").fill("90.001"); + + await page.getByTestId("schedule-enabled").uncheck(); + await expect(page.getByTestId("schedule-enabled-note")).toContainText( + "will not run automatically after it is created", + ); + }); + + await test.step("3. creating it lands on its own page, showing what was asked for", async () => { + await page.getByTestId("schedule-submit").click(); + await expect(page.getByRole("heading", { name: CREATED, exact: true })).toBeVisible({ + timeout: READ_TIMEOUT, + }); + detailURL = page.url(); + await expect(page).toHaveURL(/\/schedules\/[0-9a-f-]+(\?|$)/); + + // Created paused, so the one control whose label flips offers to resume it. + await expect(page.getByTestId("schedule-pause")).toHaveText("Resume"); + await expect(page.getByTestId("schedule-meta")).toContainText("Weekdays at 09:00"); + await expect(page.getByTestId("schedule-meta")).toContainText("America/New_York"); + // The fractional second survived the round trip rather than being floored to 90. + await expect(page.getByTestId("schedule-detail")).toContainText("90.001 seconds"); + }); + + await test.step("4. the edit form opens on the stored values, not on defaults", async () => { + await page.getByTestId("schedule-edit").click(); + await expect(page.getByTestId("schedule-time")).toHaveValue("09:00", { + timeout: READ_TIMEOUT, + }); + await expect(page.getByTestId("schedule-timeout")).toHaveValue("90.001"); + await expect(page.getByTestId("schedule-timezone").locator("input")).toHaveValue( + "America/New_York", + ); + await expect(page.getByTestId("schedule-enabled")).not.toBeChecked(); + // Both ends of the weekday set, so a picker that kept only the last day chosen + // would not pass on one assertion. + await expect(page.getByLabel("Monday", { exact: true })).toBeChecked(); + await expect(page.getByLabel("Friday", { exact: true })).toBeChecked(); + }); + + await test.step("5. an edit is saved and read back", async () => { + await page.getByTestId("schedule-prompt").fill("Report unhealthy workloads only."); + await page.getByTestId("schedule-submit").click(); + await expect(page.getByRole("heading", { name: CREATED, exact: true })).toBeVisible({ + timeout: READ_TIMEOUT, + }); + + const detail = page.getByTestId("schedule-detail"); + await expect(detail).toContainText("Report unhealthy workloads only.", { + timeout: READ_TIMEOUT, + }); + // And the update did not quietly reset what it was not asked to change. + await expect(detail).toContainText("90.001 seconds"); + }); + + await test.step("6. deleting asks in a modal, and confirming leaves for the list", async () => { + const remove = page + .getByTestId("schedule-danger") + .getByRole("button", { name: `Delete schedule ${CREATED}`, exact: true }); + await remove.click(); + + // Pressed until it takes: a dropped Delete reports as "the page never navigated" + // rather than as a missed click. See `pressUntil`. + await pressUntil( + page + .getByRole("dialog", { name: `Delete schedule ${CREATED}?`, exact: true }) + .getByRole("button", { name: "Delete", exact: true }), + () => expect(page).toHaveURL(/\/schedules(\?|$)/), + ); + detailURL = undefined; + + await expect(page.getByRole("link", { name: CREATED, exact: true })).toHaveCount(0, { + timeout: READ_TIMEOUT, + }); + }); }); From 9d61a3a0b97f53b19c3a758f6ab2467b8dafef44 Mon Sep 17 00:00:00 2001 From: Torey Scheer Date: Thu, 17 Sep 2026 15:40:15 -0400 Subject: [PATCH 11/24] fix(ui): stop three list pages printing their own source at the reader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A `/* … */` written among a JSX element's children is a text node, not a comment. The gating comment added in 8bae7180 landed inside `trailing={…}` on three pages, where it is a comment, and among a ``'s children on the other three, where it is not — so Agents, Templates and Harnesses each rendered a paragraph about SWR into their filter bars. Nothing caught it: `tsc` and eslint both accept it, and no assertion in this suite read that region. 173 mock, 13 live and 498 unit tests all passed over it. So the pages are now asked directly, in `app-shell`, over all six surfaces including the two tabs — two of the three were tabs. The check waits for each page's table before asserting the absence, which is the difference between a check and a formality: asked on arrival it passed over a harnesses tab broken on purpose, because "there is no source on this page" is true of a page that has not drawn yet. With the wait it fails, naming the route. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Torey Scheer --- .../agent-templates/agent-templates.spec.ts | 2 +- ui/playwright/shared/models/models.spec.ts | 2 +- ui/playwright/shared/prompts/prompts.spec.ts | 2 +- ui/playwright/tests/app-shell.spec.ts | 34 +++++++++++++++++++ ui/src/pages/AgentTemplatesPage.tsx | 4 +-- ui/src/pages/AgentsPage.tsx | 4 +-- ui/src/pages/agents/HarnessesTab.tsx | 4 +-- 7 files changed, 43 insertions(+), 9 deletions(-) diff --git a/ui/playwright/shared/agent-templates/agent-templates.spec.ts b/ui/playwright/shared/agent-templates/agent-templates.spec.ts index fb12147439..0e37a01cf7 100644 --- a/ui/playwright/shared/agent-templates/agent-templates.spec.ts +++ b/ui/playwright/shared/agent-templates/agent-templates.spec.ts @@ -8,6 +8,7 @@ import { selectOption, } from "../../helpers/resource"; import { + READ_TIMEOUT, expectListLoaded, expectNoLoadFailure, expectSettled, @@ -16,7 +17,6 @@ import { rowNamed, searchList, throwawayName, - READ_TIMEOUT, } from "../../helpers/app"; import { sweepQuietly } from "../../helpers/cleanup"; diff --git a/ui/playwright/shared/models/models.spec.ts b/ui/playwright/shared/models/models.spec.ts index 61baceeec9..90afe3696d 100644 --- a/ui/playwright/shared/models/models.spec.ts +++ b/ui/playwright/shared/models/models.spec.ts @@ -1,5 +1,6 @@ import { test, expect } from "../../fixtures/test"; import { + READ_TIMEOUT, expectListLoaded, expectListTotal, expectNoLoadFailure, @@ -8,7 +9,6 @@ import { rowNamed, searchList, throwawayName, - READ_TIMEOUT, } from "../../helpers/app"; import { LIFECYCLE_TIMEOUT, diff --git a/ui/playwright/shared/prompts/prompts.spec.ts b/ui/playwright/shared/prompts/prompts.spec.ts index d96a795ade..ecf0a1b39a 100644 --- a/ui/playwright/shared/prompts/prompts.spec.ts +++ b/ui/playwright/shared/prompts/prompts.spec.ts @@ -1,6 +1,7 @@ import { type Page } from "@playwright/test"; import { test, expect } from "../../fixtures/test"; import { + READ_TIMEOUT, expectListLoaded, expectListTotal, expectNoLoadFailure, @@ -9,7 +10,6 @@ import { rowNamed, searchList, throwawayName, - READ_TIMEOUT, } from "../../helpers/app"; import { LIFECYCLE_TIMEOUT, diff --git a/ui/playwright/tests/app-shell.spec.ts b/ui/playwright/tests/app-shell.spec.ts index e750d7173c..092182103f 100644 --- a/ui/playwright/tests/app-shell.spec.ts +++ b/ui/playwright/tests/app-shell.spec.ts @@ -110,6 +110,40 @@ test("app shell: chrome, navigation entries, and where creation lives", async ({ await expectShell(page); } }); + + await test.step("6. no page is printing its own source at the reader", async () => { + /* + * A `/* … *\/` written among a JSX element's children rather than inside `{…}` is a + * text node, not a comment, and it renders. Three of these pages shipped a paragraph + * about SWR into their filter bars that way, and nothing caught it: `tsc` and eslint + * both accept it, and no assertion in this suite read that region. So the pages are + * asked directly, tabs included — two of the three were tabs. + */ + const surfaces = [ + [routes.agents, "agents-table"], + [`${routes.agents}?tab=templates`, "templates-table"], + [`${routes.agents}?tab=harnesses`, "harnesses-table"], + [routes.models, "models-table"], + [routes.prompts, "prompts-table"], + [routes.mcpServers, "mcp-servers-table"], + ] as const; + + for (const [route, table] of surfaces) { + await loadPage(page, route); + /* + * The table first, and that is the whole assertion working. "There is no source on + * this page" is true of a page that has not drawn yet, so asked on arrival it + * passed over a genuinely broken harnesses tab — checked by breaking one on + * purpose. The absence only means something once there is something to be absent + * from. + */ + await expect(page.getByTestId(table)).toBeVisible(); + await expect( + page.locator("body"), + `${route} is printing source commentary at the reader`, + ).not.toContainText("*/"); + } + }); }); /** diff --git a/ui/src/pages/AgentTemplatesPage.tsx b/ui/src/pages/AgentTemplatesPage.tsx index eafed76498..b25238a31b 100644 --- a/ui/src/pages/AgentTemplatesPage.tsx +++ b/ui/src/pages/AgentTemplatesPage.tsx @@ -280,10 +280,10 @@ export function AgentTemplatesTab() { {/* Only a successful load can be counted: "0 of 0" because a request failed would be a claim the page cannot support. */} - /* `data !== undefined` as well as `!isLoading`: SWR runs its fetcher in an effect, so + {/* `data !== undefined` as well as `!isLoading`: SWR runs its fetcher in an effect, so the first paint reports "not loading" on a page that has not asked yet, and this counted "0 of 0" for the 600ms before the answer arrived. A count of nothing is a - claim, and until the read lands this page has not earned it. */ + claim, and until the read lands this page has not earned it. */} {!loadFailure && !templates.isLoading && templates.data !== undefined ? ( {filtered.length} of {rows.length}{" "} diff --git a/ui/src/pages/AgentsPage.tsx b/ui/src/pages/AgentsPage.tsx index 3fc1a37079..962ad65d7e 100644 --- a/ui/src/pages/AgentsPage.tsx +++ b/ui/src/pages/AgentsPage.tsx @@ -459,10 +459,10 @@ export function AgentsTab() { ]} trailing={ - /* `data !== undefined` as well as `!isLoading`: SWR runs its fetcher in an effect, so + {/* `data !== undefined` as well as `!isLoading`: SWR runs its fetcher in an effect, so the first paint reports "not loading" on a page that has not asked yet, and this counted "0 of 0" for the 600ms before the answer arrived. A count of nothing is a - claim, and until the read lands this page has not earned it. */ + claim, and until the read lands this page has not earned it. */} {!loadFailure && !templates.isLoading && templates.data !== undefined ? ( {filtered.length} of {agents.length}{" "} diff --git a/ui/src/pages/agents/HarnessesTab.tsx b/ui/src/pages/agents/HarnessesTab.tsx index 20173a20a5..8a36e97ec0 100644 --- a/ui/src/pages/agents/HarnessesTab.tsx +++ b/ui/src/pages/agents/HarnessesTab.tsx @@ -243,10 +243,10 @@ export function HarnessesTab() { ]} trailing={ - /* `data !== undefined` as well as `!isLoading`: SWR runs its fetcher in an effect, so + {/* `data !== undefined` as well as `!isLoading`: SWR runs its fetcher in an effect, so the first paint reports "not loading" on a page that has not asked yet, and this counted "0 of 0" for the 600ms before the answer arrived. A count of nothing is a - claim, and until the read lands this page has not earned it. */ + claim, and until the read lands this page has not earned it. */} {!harnesses.error && !harnesses.isLoading && harnesses.data !== undefined ? ( {filtered.length} of {rows.length}{" "} From 2036c9f2a81f93afdf9d55a3809cf66be93caac0 Mon Sep 17 00:00:00 2001 From: Torey Scheer Date: Thu, 17 Sep 2026 18:19:54 -0400 Subject: [PATCH 12/24] test(ui): use the primitive rather than defending against it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two of the fixes in this branch reached around a tool instead of for it. `expect.poll` ends on a throwing callback, so the live admission check wrapped its callback in a try/catch that returned a string to keep the poll alive. `toPass` is the primitive for retrying a callback that throws — `pressUntil` was already using it two files away. The failure is now the assertion's own rather than a sentence composed to survive the wrong mechanism. And the list total was read twice and required to agree, because every summary drew "0 of 0" for the 600ms before its read landed. That is fixed where it belongs now, so the double read was a test carrying a workaround for a bug that no longer exists. One read again; a return of the flash fails the journeys that count before and after a create, which is how it was found. Two rules added to the README for the pattern behind most of this branch's defects: never assert an absence before the thing could appear, and clean up in a hook rather than a `finally`. Verified: 173 mock, 13 live, 31 conventions. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Torey Scheer --- ui/playwright/README.md | 6 ++ ui/playwright/helpers/app.ts | 33 +++-------- .../agent-templates/agent-templates.spec.ts | 55 +++++++------------ 3 files changed, 35 insertions(+), 59 deletions(-) diff --git a/ui/playwright/README.md b/ui/playwright/README.md index c954390640..662909991f 100644 --- a/ui/playwright/README.md +++ b/ui/playwright/README.md @@ -204,6 +204,12 @@ conventions below: the shared fixture import, and antd's class names. The trade is deliberate: a failed step stops the ones after it, so a broken create hides whether delete works. That is the right way round — a resource whose create is broken is broken, and the recording shows where it stopped. +- **Never assert an absence before the thing could appear.** "No error", "no rows", + "no source text on the page" are all true of a page that has not drawn yet, so each + one needs a positive signal in front of it — a summary, a table, a settled state. + Four defects on this suite were that shape, every one of them green. +- **Clean up in a hook, never in a `finally`.** A timed-out test has a closed page, so + everything in its `finally` throws and the resource stays on the cluster. - **Keep the writes in one browsing context.** The fixture backend keeps writes in the page's own memory, so a `page.goto` starts a backend that has never heard of the thing just created, and the failure reads as "the create did not stick" when nothing diff --git a/ui/playwright/helpers/app.ts b/ui/playwright/helpers/app.ts index 9aa958f41f..df424b0bd5 100644 --- a/ui/playwright/helpers/app.ts +++ b/ui/playwright/helpers/app.ts @@ -299,36 +299,21 @@ export async function expectListTotal( } /** - * The list's total, once the summary has stopped moving. + * The list's total, off the summary that only a landed read draws. * - * Read once, the summary is not the signal it looks like. Every list draws it on - * `!error && !isLoading`, and SWR reports `isLoading` false on the first paint — the - * fetcher runs in an effect, after it — so a page that has not asked anything yet draws - * "0 of 0" for a frame. Measured on the harnesses tab: a total of 0 read off a tab - * holding four, and the journey then asserted "one more than nothing". - * - * So it is read twice and has to agree with itself. A list still arriving disagrees, and - * one that is genuinely empty says 0 twice — which is the distinction "a list still - * fetching has no rows either" was always about. + * It read the summary twice and required the two to agree, because every list drew + * "0 of 0" for the 600ms before its read arrived. That is fixed where it belongs now — + * the summary waits for `data` — so this is one read again, and a return of the flash + * fails the journeys that count before and after a create. */ async function settledTotal(page: Page, list: string): Promise { const summary = page.getByTestId(`${list}-summary`); await expect(summary).toContainText(/\bof \d+\b/, { timeout: READ_TIMEOUT }); - let previous: number | undefined; - let total = 0; - await expect(async () => { - const text = (await summary.textContent()) ?? ""; - const read = /\bof (\d+)\b/.exec(text)?.[1]; - expect(read, `no total could be read from "${text}"`).toBeDefined(); - const value = Number(read); - const agreed = previous === value; - previous = value; - total = value; - expect(agreed, `still moving, at ${value}`).toBe(true); - }).toPass({ timeout: READ_TIMEOUT, intervals: [250] }); - - return total; + const text = (await summary.textContent()) ?? ""; + const read = /\bof (\d+)\b/.exec(text)?.[1]; + expect(read, `no total could be read from "${text}"`).toBeDefined(); + return Number(read); } /** diff --git a/ui/playwright/shared/agent-templates/agent-templates.spec.ts b/ui/playwright/shared/agent-templates/agent-templates.spec.ts index 0e37a01cf7..ce99cd72db 100644 --- a/ui/playwright/shared/agent-templates/agent-templates.spec.ts +++ b/ui/playwright/shared/agent-templates/agent-templates.spec.ts @@ -229,41 +229,26 @@ test("agent templates: one is created, admitted, edited and deleted", async ({ * reload there starts a backend that has never heard of this template — see * `shared/schedules/schedules.spec.ts`, which avoids reloading for that reason. */ - await expect - .poll( - async () => { - /* - * Nothing in here may throw. `expect.poll` calls its callback outside its - * own try/catch, so a throw ends the poll rather than failing one round of - * it — measured, a callback that throws twice and then succeeds gives up - * after 4ms, where one that returns is retried. Two here can: `textContent` - * on a detail page still drawing its `Skeleton`, and `expectSettled`'s own - * assertions after the reload. Either turned the ninety seconds below into - * a single attempt and reported a raw locator timeout instead of the - * message this poll was given. - */ - try { - const text = (await status.textContent({ timeout: 30_000 })) ?? ""; - if (text.includes(harness)) return text; - - // Re-read after the reload rather than returning what was on screen - // before it: the round that finally succeeds should be the one that says - // so, not the one after it. - await page.reload(); - await expectSettled(page); - return (await status.textContent({ timeout: 30_000 })) ?? ""; - } catch (error) { - // The reason this round could not answer, which is what the failure - // message should end on if none of them ever does. - return `could not be read: ${String(error).split("\n")[0]}`; - } - }, - { - timeout: 90_000, - message: `${TEMPLATE} was never admitted: the controller did not name ${harness} in its status, over repeated re-reads`, - }, - ) - .toContain(harness); + await expect(async () => { + /* + * `toPass`, not `expect.poll`. The poll calls its callback outside its own + * try/catch, so a throw ends it rather than failing one round — which two things + * in here do: `textContent` on a page still drawing its `Skeleton`, and + * `expectSettled` after the reload. This is the primitive for retrying a callback + * that throws, so the failure is the assertion's own rather than a string + * composed to keep the poll alive. + */ + if (!((await status.textContent()) ?? "").includes(harness)) { + // Re-read after the reload, not before it: the round that finally succeeds + // should be the one that says so. + await page.reload(); + await expectSettled(page); + } + await expect( + status, + `${TEMPLATE} was never admitted: the controller did not name ${harness} in its status`, + ).toContainText(harness); + }).toPass({ timeout: 90_000 }); } else { // The fixtures answer from the create itself, so one read settles it. await expect(status).toContainText(harness, { timeout: 30_000 }); From b7460524ad411a9f6382a0689bac954e5cb0ed1f Mon Sep 17 00:00:00 2001 From: Torey Scheer Date: Thu, 17 Sep 2026 22:34:38 -0400 Subject: [PATCH 13/24] test(ui): keep the two claims main's live schedules spec was making MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The merge took this branch's `live/schedules.spec.ts` over main's, which asserted two things nothing else here did. Both are answerable on either backend, so they land in the shared journey rather than where they came from. A blank time zone is not blank — the form stores `timeZone || "UTC"` — so the cadence line has to say which it is. Filled, cleared, and put back, one field at a time. With the fallback removed the line reads "Weekdays at 09:00 ()", which is what a reader would have been left to interpret. And a schedule that has never run says so. The page keeps three states apart — no executions, no search match, and a failed read — and the mock spec could only ever assert the absence of the first; a journey that creates a paused schedule is the one place it can be produced. Verified on a cluster rebuilt on substrate 0.2.0-beta2, which is what main now installs: 13 live, 183 mock, no resources left behind. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Torey Scheer --- .../shared/schedules/schedules.spec.ts | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/ui/playwright/shared/schedules/schedules.spec.ts b/ui/playwright/shared/schedules/schedules.spec.ts index f7c6b0645e..58b477f637 100644 --- a/ui/playwright/shared/schedules/schedules.spec.ts +++ b/ui/playwright/shared/schedules/schedules.spec.ts @@ -96,9 +96,25 @@ test("schedules: one is created, read, changed and deleted", async ({ page }) => // The time zone is an AutoComplete, so its id is on the wrapper and the caret goes // in the input inside it. Escape dismisses the zone list, which otherwise sits // over the fields below. - await page.getByTestId("schedule-timezone").locator("input").fill("America/New_York"); + const zone = page.getByTestId("schedule-timezone").locator("input"); + await zone.fill("America/New_York"); await page.keyboard.press("Escape"); + /* + * The cadence line says the zone back, and says "UTC" when the field is empty rather + * than leaving it blank — which is what the form will store for a blank one. A reader + * who cleared it is otherwise told nothing about what they have just chosen. + * + * One field at a time: cleared, then put back. + */ + const cadence = page.getByTestId("schedule-cadence"); + await expect(cadence).toContainText("(America/New_York)"); + await zone.fill(""); + await expect(cadence).toContainText("(UTC)"); + await zone.fill("America/New_York"); + await page.keyboard.press("Escape"); + await expect(cadence).toContainText("(America/New_York)"); + await page.getByTestId("schedule-prompt").fill("Report cluster health."); // A fractional timeout, because it is the value a backend most easily rounds off. await page.getByTestId("schedule-timeout").fill("90.001"); @@ -119,6 +135,10 @@ test("schedules: one is created, read, changed and deleted", async ({ page }) => // Created paused, so the one control whose label flips offers to resume it. await expect(page.getByTestId("schedule-pause")).toHaveText("Resume"); + // And it has never run. Its own state, not "no search matched" and not "the read + // failed" — three things the page keeps apart and a new schedule is the only one of + // them this journey can produce. + await expect(page.getByTestId("schedule-history-empty")).toBeVisible(); await expect(page.getByTestId("schedule-meta")).toContainText("Weekdays at 09:00"); await expect(page.getByTestId("schedule-meta")).toContainText("America/New_York"); // The fractional second survived the round trip rather than being floored to 90. From bfe32c4680731102b936ce10c8ae62b2eef79f57 Mon Sep 17 00:00:00 2001 From: Torey Scheer Date: Thu, 17 Sep 2026 22:58:30 -0400 Subject: [PATCH 14/24] docs(ui): say what is actually blocking the deferred chat coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Read against #2828, which tracks these entries and asks that each be closed or re-argued when its blocker moves. This branch moves one of them. Two entries said the missing coverage "belongs in `playwright/live/`". That lane now exists and runs in CI, so on a plain reading they are ready to write — and whoever tried would find the real obstacle underneath: an `AgentInstance` exists only once a message has been sent, a streamed reply needs a model that answers, and neither cluster has one. CI installs with `OPENAI_API_KEY: fake` and `setup-cluster.sh` sets no key at all. Both entries now say so. And the cleanup note claimed each live spec deletes what it made in a `finally`, which is both out of date and the opposite of what was measured: a timed-out test has a closed page, so a `finally` cannot clean up after the failure most likely to leave something behind. They are hooks now. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Torey Scheer --- ui/playwright/DEFERRED.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/ui/playwright/DEFERRED.md b/ui/playwright/DEFERRED.md index 6576732361..f3edd851d4 100644 --- a/ui/playwright/DEFERRED.md +++ b/ui/playwright/DEFERRED.md @@ -24,7 +24,9 @@ missing is the page. wiring. Two things follow for this file. **The cleanup entry above is now about litter, not contexts.** A live spec deletes what -it created; a run killed between the two cannot. That is why `throwawayName` puts the +it created, in a `test.afterEach` rather than a `finally` — measured, a timed-out test has +a closed page and everything in its `finally` throws, which is the failure most likely to +leave something behind. A run killed outright still cannot. That is why `throwawayName` puts the process and a timestamp in every name — anything matching `e2e-live-*` in `kagent` is litter and safe to remove. A sweep spec stays the wrong shape for it, being one bad selector away from deleting somebody's work. @@ -215,7 +217,10 @@ The reading itself is covered exhaustively in `src/components/chat/lifecycleRead since a substrate agent really does suspend itself then and nothing in the API reports it. What is missing is a browser journey that suspends an instance from the agents list while a chat page is open on it and watches the indicator follow. That belongs in `playwright/live/`, -where the operation is real. +where the operation is real — and that lane now exists and runs in CI, so the blocker has +moved rather than gone: an `AgentInstance` exists only once a message has been sent, and +neither cluster has a model that can answer one. CI installs with `OPENAI_API_KEY: fake`, +and `setup-cluster.sh` sets no key at all. ### Streaming, end to end, against a controller @@ -232,6 +237,9 @@ artifact path. Teaching the fixture to emit artifact frames would mean it stoppe `ChatClient` and started being an A2A server, which is the wrong seam — the transport is already covered by unit tests over real bytes. The browser-level gap is a `playwright/live/` spec that sends a message and asserts the reply grows on screen before the turn completes. +The lane is no longer what blocks it: a reply that streams needs a model that answers, and +both clusters this suite runs against carry a fake key. Reachable today only by a developer +with their own. A related gap worth naming rather than leaving implicit: the mock backend serves one instance per conversation and never *changes* an instance's `operation`, so the lifecycle From 7a3950eabd8a1899dda730eecc4e3f41bb49b93b Mon Sep 17 00:00:00 2001 From: Torey Scheer Date: Fri, 18 Sep 2026 00:07:11 -0400 Subject: [PATCH 15/24] test(ui): drive the tool approval controls, which shipped without any MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Read against #2828. The entry deferring this said the approval controls needed a product decision. That decision landed in #2714 and the controls shipped with it — per-tool approve and reject, a reason on a rejection, a submit — and `chat-approval` appeared nowhere in `playwright/`. Not a deferred gap: an untested surface behind a note that had gone out of date. Two scenarios carry it, both parking the turn at the `ChatClient` boundary the `asks` scenarios already park it at. That is the seam the entry argued for keeping; what stays out is wire frames, which unit tests cover over real bytes. `approves` offers two tools so one can be allowed and the other denied in a single submission — a single-tool fixture cannot show that the controls belong to their row rather than to the form. The fixture reads the decisions out of the payload it received and says them back, so the spec asserts what went to the agent rather than what the form looked like. Checked by breaking it both ways: sending every decision as an approval fails on the rejected tool, and dropping the rejection reason fails on the reason. `asks-unknown` parks on a request this build cannot render — a turn started without the HITL extension, carrying prose and no correlation id. The page says so and offers only the discard, and that sentence was untested too. Verified: 186 mock, 530 unit, 31 conventions. One firefox failure per full run, a different test each time and passing 6/6 in isolation, at load average 50-58 with the kind cluster running; it happens with these changes stashed, so it is the contention the config documents rather than these. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Torey Scheer --- ui/playwright/DEFERRED.md | 14 ++-- ui/playwright/tests/chat/approvals.spec.ts | 97 ++++++++++++++++++++++ ui/src/api/chat/mockChatClient.ts | 91 +++++++++++++++++++- ui/src/mocks/scenario.ts | 23 ++++- 4 files changed, 218 insertions(+), 7 deletions(-) create mode 100644 ui/playwright/tests/chat/approvals.spec.ts diff --git a/ui/playwright/DEFERRED.md b/ui/playwright/DEFERRED.md index f3edd851d4..bb915128de 100644 --- a/ui/playwright/DEFERRED.md +++ b/ui/playwright/DEFERRED.md @@ -255,16 +255,20 @@ plainly rather than guessing at. **A `tool_approval_request`** carries `tools[]` and a `hint` and is answered with `tool_approval_response` / `approvals[]` — a different payload, and a different control: -per-tool approve or reject, with a rejection reason. The prompt names the tools and -offers only the discard, which is honest. Building the approval controls needs the -product decision about what a reader is being asked to vouch for, and it should not be -guessed from the shape of the payload. +per-tool approve or reject, with a rejection reason. **No longer deferred.** The product +decision landed in #2714 and the controls shipped with it; they had no browser coverage +until `tests/chat/approvals.spec.ts`, which decides two tools opposite ways in one +submission and reads the decisions back off the reply rather than off the form — so it +fails if the page sends both as approvals or pairs a reason with the wrong tool. The +fixture parks on the request at the `ChatClient` boundary, which is what `asks` already +does; what stays out is wire frames, and that stays out. **A turn parked without the HITL extension activated** has no payload at all — the question exists only as prose and carries no correlation id, so no answer can be routed to it. The prompt says so and offers the discard. This build always activates the extension, so it can only arise from a turn started by something else (a `kubectl`-driven -send, an older client). It is not worth engineering around; it is worth not lying about. +send, an older client). It is not worth engineering around; it is worth not lying about — +and the sentence that does the not-lying is covered in the same spec. **The `ask_user` payload still renders as JSON in the transcript**, beside the answerable prompt — the tool call and its result are structured data and are shown as such. That is diff --git a/ui/playwright/tests/chat/approvals.spec.ts b/ui/playwright/tests/chat/approvals.spec.ts new file mode 100644 index 0000000000..f0b8ca903f --- /dev/null +++ b/ui/playwright/tests/chat/approvals.spec.ts @@ -0,0 +1,97 @@ +import { test, expect } from "../../fixtures/test"; +import { agentChat, instances } from "../../helpers/app"; + +/** + * A turn parked on something other than a question. + * + * `questions.spec.ts` covers `ask_user`, which is answerable. These are its two + * neighbours: a tool approval, which this build asks the reader to vouch for tool by + * tool, and a request it does not recognise, which it says plainly it cannot answer. + * + * Both were deferred as "blocked on a product decision" while the controls were being + * argued about. The decision landed in #2714 and the controls shipped with it — approve, + * reject, a reason per rejection — and nothing in the browser had driven them since. + */ + +const AGENT_CHAT = agentChat(instances.ready); + +test("chat: a tool approval is decided per tool, and the decisions are what go back", async ({ + page, +}) => { + await page.goto(`${AGENT_CHAT}?chat=approves`); + await page.getByTestId("chat-input").fill("Tidy the cluster up."); + await page.getByTestId("chat-send").click(); + + const prompt = page.getByTestId("chat-awaiting-reply"); + await expect(prompt).toBeVisible({ timeout: 30_000 }); + await expect(prompt).toHaveAttribute("data-kind", "tool_approval"); + + await test.step("1. it names what it is asking to run, and why", async () => { + // The tools by name, because "the agent wants to run some tools" is not something a + // reader can vouch for. The hint is the runtime's own sentence about the risk. + const tools = page.getByTestId("chat-approval-tool"); + await expect(tools).toHaveCount(2); + await expect(tools.first()).toHaveText("kubectl_apply"); + await expect(tools.last()).toHaveText("shell_exec"); + await expect(prompt).toContainText("Approve only what you recognise"); + }); + + await test.step("2. one allowed and one denied, in the same submission", async () => { + // Per row, not per form: the decision belongs to the tool, and a control wired to + // the form would send the same answer for both. + const rows = page.getByTestId("chat-approval-tool"); + await rows.first().locator("..").getByRole("button", { name: "Allow" }).click(); + await rows.last().locator("..").getByRole("button", { name: "Deny" }).click(); + + // The reason appears with the rejection and belongs to that tool alone — keyed on + // the tool's id, so a form that kept one reason for the page would fail here. + const reason = page.getByTestId("chat-approval-reason-call-2"); + await expect(reason).toBeVisible(); + await expect(page.getByTestId("chat-approval-reason-call-1")).toHaveCount(0); + await reason.fill("It deletes a cache I still need."); + }); + + await test.step("3. the agent is told which was which", async () => { + await page.getByTestId("chat-approval-submit").click(); + + /* + * Read back off the reply rather than off the form: the fixture reads the decisions + * out of the payload it received and says them, so this fails if the page sent both + * as approvals, paired the reason with the wrong tool, or dropped the reason. + */ + const transcript = page.getByTestId("chat-transcript"); + await expect(transcript).toContainText("call-1 approved", { timeout: 30_000 }); + await expect(transcript).toContainText("call-2 rejected"); + await expect(transcript).toContainText("It deletes a cache I still need."); + // And the turn is no longer parked, so the conversation is usable again. + await expect(prompt).toHaveCount(0); + }); +}); + +test("chat: a question this build cannot answer says so, rather than guessing", async ({ + page, +}) => { + /* + * A turn started without the HITL extension carries its question as prose and no + * correlation id, so there is nothing to answer against. It arises from a send this + * build did not make — a `kubectl`-driven one, an older client — and the only honest + * thing the page can do is say which of the two it is looking at. + */ + await page.goto(`${AGENT_CHAT}?chat=asks-unknown`); + await page.getByTestId("chat-input").fill("Do the thing."); + await page.getByTestId("chat-send").click(); + + const prompt = page.getByTestId("chat-awaiting-reply"); + await expect(prompt).toBeVisible({ timeout: 30_000 }); + await expect(prompt).toHaveAttribute("data-kind", "unknown"); + await expect(prompt).toContainText("without the extension that carries them"); + + // No controls invented for it: there is nothing to answer, so the only way out is to + // let the turn go. Asserted as an absence *after* the prompt is on screen, which is + // what makes the absence mean anything. + await expect(page.getByTestId("chat-approval-submit")).toHaveCount(0); + await expect(page.getByTestId("chat-answer-text-0")).toHaveCount(0); + + await page.getByTestId("chat-dismiss-question").click(); + await expect(prompt).toHaveCount(0); +}); diff --git a/ui/src/api/chat/mockChatClient.ts b/ui/src/api/chat/mockChatClient.ts index dbbd2c3f68..2f2262eb12 100644 --- a/ui/src/api/chat/mockChatClient.ts +++ b/ui/src/api/chat/mockChatClient.ts @@ -35,6 +35,8 @@ const TIMING = { error: { step: 300, word: 45 }, asks: { step: 300, word: 45 }, "asks-text": { step: 300, word: 45 }, + approves: { step: 300, word: 45 }, + "asks-unknown": { step: 300, word: 45 }, } as const; /** @@ -53,6 +55,19 @@ const NOTE_QUESTION = "What should I put on the order note?"; /** The correlation id, which a real answer echoes verbatim. */ const REQUEST_ID = "adk-mock-ask-1"; +/** + * The tools the scripted approval turn asks to run. + * + * Two, because the decision is per tool: one approved and one rejected in the same + * submission is the case a single-tool fixture cannot produce, and it is the one that + * says the controls are wired to their own row rather than to the form. + */ +const APPROVAL_TOOLS = [ + { id: "call-1", name: "kubectl_apply", args: { manifest: "deployment.yaml" } }, + { id: "call-2", name: "shell_exec", args: { command: "rm -rf /tmp//cache" } }, +]; +const APPROVAL_HINT = "These change the cluster. Approve only what you recognise."; + /** Where the scripted turn gives up when the scenario asks it to fail. */ const FAILURE_MESSAGE = "The agent stopped responding. The connection to the runtime was lost."; @@ -229,12 +244,29 @@ export class MockChatClient implements ChatClient { * as though it worked. So the acknowledgement here says which happened — that * silent failure is the reason this fixture bothers to check. */ + /* + * What the agent understood of an approval, said tool by tool. A fixture that + * acknowledged "noted" either way would pass a UI that sent every decision as an + * approval, or paired the reasons with the wrong rows. + */ + const decisions = parked.kind === "tool_approval" ? readApproval(input.hitl) : undefined; + const approvalReply = + decisions && + decisions + .map( + (decision) => + `${decision.id} ${decision.approved ? "approved" : `rejected (${decision.reason ?? "no reason given"})`}`, + ) + .join("; "); + const acknowledgement = message( `${parked.taskId}-ack`, "agent", structured && parked.kind === "ask_user" && structured.id === parked.requestId ? `Noted: **${structured.answers.map((a) => a.join(", ")).join("; ")}**.` - : `I did not catch a choice in that.`, + : approvalReply + ? `Noted: **${approvalReply}**.` + : `I did not catch a choice in that.`, parked.taskId, ); answered.push(acknowledgement); @@ -369,6 +401,34 @@ export class MockChatClient implements ChatClient { return; } + if (scenario === "approves") { + // The same park, a different request: tools to vouch for rather than a question + // to answer. `hint` is the runtime's own sentence about why it is asking. + this.persist(sessionId); + const request: PendingRequest = { + kind: "tool_approval", + taskId, + tools: APPROVAL_TOOLS, + hint: APPROVAL_HINT, + }; + saveParked(sessionId, request); + yield { type: "status", state: "input_required", taskId, awaiting: request }; + return; + } + + if (scenario === "asks-unknown") { + /* + * Parked on something this build cannot render: a turn started without the + * extension carries its question as prose and no correlation id, so there is + * nothing to answer against. `unknown` is what `readRequest` returns for it. + */ + this.persist(sessionId); + const request: PendingRequest = { kind: "unknown", taskId }; + saveParked(sessionId, request); + yield { type: "status", state: "input_required", taskId, awaiting: request }; + return; + } + yield { type: "status", state: "completed", taskId }; } @@ -464,6 +524,35 @@ function readAnswer( return { id: body.id, answers }; } +/** + * The decisions carried back by a tool approval, read the way the runtime reads them. + * + * `rejection_reason` is snake_case on the wire and optional: a rejection may carry one + * and an approval never does. Read here so the acknowledgement can say which tools were + * approved — a fixture that answered "noted" either way would let a UI that sent the + * decisions the wrong way round pass. + */ +function readApproval( + hitl: Record | undefined, +): { id: string; approved: boolean; reason?: string }[] | undefined { + const payload = hitl?.[HITL_EXTENSION_URI]; + if (typeof payload !== "object" || payload === null) return undefined; + const body = payload as Record; + if (body.type !== "tool_approval_response" || !Array.isArray(body.approvals)) { + return undefined; + } + return body.approvals.filter(isRecord).map((entry) => ({ + id: typeof entry.id === "string" ? entry.id : "", + approved: entry.approved === true, + reason: + typeof entry.rejection_reason === "string" ? entry.rejection_reason : undefined, + })); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + function clearParked(sessionId: string): void { try { window.sessionStorage.removeItem(PARKED_PREFIX + sessionId); diff --git a/ui/src/mocks/scenario.ts b/ui/src/mocks/scenario.ts index 14d82de7b1..409ed2bce8 100644 --- a/ui/src/mocks/scenario.ts +++ b/ui/src/mocks/scenario.ts @@ -93,7 +93,28 @@ function isScenario(value: string | null): value is MockScenario { * `asks-text` offers one prose field — the only shape where the field can take the * caret on arrival and Enter can mean "send". */ -export const CHAT_SCENARIOS = ["ok", "error", "slow", "asks", "asks-text"] as const; +/** + * `approves` parks the turn on a tool approval request instead of a question: a + * different payload, and a different set of controls — approve or reject each tool, + * with a reason on a rejection. It is a third variation of the parked turn above rather + * than a new seam, because this client already yields the request at the `ChatClient` + * boundary; what it must not do is start emitting wire frames, which is a transport + * shape covered by unit tests over real bytes. + * + * `asks-unknown` parks it on a request this build does not recognise, which is what a + * turn started without the HITL extension looks like: prose, no correlation id, nothing + * to answer against. The page says so and offers only the discard, and that sentence is + * the only thing standing between a reader and a question they cannot see is unanswerable. + */ +export const CHAT_SCENARIOS = [ + "ok", + "error", + "slow", + "asks", + "asks-text", + "approves", + "asks-unknown", +] as const; export type ChatScenario = (typeof CHAT_SCENARIOS)[number]; From 792e5b06374417e9fb519a0a8bb2d02c6303806f Mon Sep 17 00:00:00 2001 From: Torey Scheer Date: Fri, 18 Sep 2026 01:44:39 -0400 Subject: [PATCH 16/24] test(ui): drive the one-tool approval too, and shorten what is now covered MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One tool is a different prompt rather than a shorter one — Approve and Reject on the prompt itself, a rejection opening its reason with the caret in it — and the two-tool fixture cannot reach any of it. A second scenario covers it, and the approval surface now has no undriven control. Then the file itself. Three sections recited what the chat, MCP server, prompt and form-validation specs cover, which those specs describe better; they are one paragraph now, keeping the lesson they carried — two of them had been listed as blocked on pages that already existed, and a stale "blocked on" costs more than no entry. The approval entry keeps its argument and loses its recital. One correction rather than a compression: the entry on what was lost when agents became AgentInstances says no single surface shows an agent's readiness reason end to end. True, and neither half is covered either — `AgentDetailsPage`, the record with `instance-state`, `instance-operation` and `instance-failure`, has no spec at all. Found by listing every test id in `src` against every one the suite references: 188 of 490 are never driven, and that page is the largest of them. Verified: 36 chat specs, 530 unit, 31 conventions. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Torey Scheer --- ui/playwright/DEFERRED.md | 80 ++++++---------------- ui/playwright/tests/chat/approvals.spec.ts | 29 ++++++++ ui/src/api/chat/mockChatClient.ts | 5 +- ui/src/mocks/scenario.ts | 6 ++ 4 files changed, 60 insertions(+), 60 deletions(-) diff --git a/ui/playwright/DEFERRED.md b/ui/playwright/DEFERRED.md index bb915128de..7f22d2b881 100644 --- a/ui/playwright/DEFERRED.md +++ b/ui/playwright/DEFERRED.md @@ -78,60 +78,24 @@ Worth recording for its own sake: the fixtures cannot show this class of bug at answer from the page's own memory and are therefore always immediately consistent, so a mock backend has no write that is not yet a read. Only a cluster has one. -## Ported since: chat +## Ported since, and no longer worth describing here -`chat/chat.spec.ts` and `chat/chat-errors.spec.ts` are live. The chat page was -rebuilt on the `ChatClient` port, so both journeys assert against the real page: -history, sending, streaming deltas, tool call and result rendering, a failed -turn with retry, cancelling mid-stream, and the session list failing on its own. +Chat and its error journeys, the chat-message extension point, MCP servers, prompt +libraries, and form validation for every resource that has a form. Each is a spec now, +and a spec describes itself better than a list of what it covers. -The chat-message extension point is covered too, now that the example mounts a -component there: `extension-points.withExtension.spec.ts` asserts one slot per message -and four *distinguishable* contributions, so per-message context is proven rather -than assumed. Every extension point the app declares now has a runtime -assertion. +**Two of those had been listed as blocked on pages that already existed.** That is the +lesson worth keeping: a stale "blocked on" entry costs more than no entry, because it +stops somebody porting work that is already possible. -## Covered: form validation, and every resource's lifecycle - -Each resource's lifecycle spec asserts its own form's gate in the create step — the -submit refused while a required field is empty, the refusal naming the field, the -address staying on the form — and checks the required *marks* against that gate with -`expectRequired`. antd draws the mark from `required` on a `Form.Item` while these -forms gate their submit in code, so the two are separate statements about the same -field and only a test keeps them agreeing. - -**What is still missing is the agent form, because there is no agent form.** An agent -is an `AgentTemplate` paired with a `Harness` and is not created, so the old -"declarative agent create blocks submit" assertion has no page to run against. If a -create-an-agent surface lands, its validation belongs in that change. +**What is still missing is the agent form, because there is no agent form.** An agent is +an `AgentTemplate` paired with a `Harness` and is not created. If a create-an-agent +surface lands, its validation belongs in that change. ## Not started by request -App extension-point specs. The framework is still being edited and its -contract is not frozen; the team lead will ask for these once it lands. - -## Ported since: MCP servers and prompt libraries - -`mcp-servers/mcp-servers.spec.ts` and `prompts/prompts.spec.ts` are live: one lifecycle -spec apiece, with the failure states among the steps. - -**These were listed above as blocked on pages that did not exist. The pages did -exist** — `McpServersPage`, `PromptsPage` and `PromptDetailPage` are all real, and were -before the specs were written. The entries were simply stale, which is worth recording: -this file is only useful while it is true, and a stale "blocked on" entry costs more -than no entry at all, because it stops somebody porting work that is already possible. - -The specs cover the list, the per-server tool count including a server that discovered -none, the filter, the step through to a library's fragments and the include expression -a reader copies, and both failure journeys. The detail page's two failure states are -asserted apart — a library that could not be loaded and a library that does not exist -lead to different actions, and the page distinguishes them. - -One thing they needed from the harness: a spec can now declare console output it -provokes on purpose, with `test.use({ expectedNoise: [...] })`. The not-found journey -makes the browser log a 404, and forgiving 404s for the whole suite would have blunted -the guard — a 404 is also what a missing asset looks like, and this repository has -shipped one to production that way before. +App extension-point specs. The framework is still being edited and its contract is not +frozen; the team lead will ask for these once it lands. ## Lost with the REST path tests, and where it went instead @@ -195,7 +159,10 @@ landing page carries all three as tabs, and a conversation's record links out to template and to the agent rather than duplicating either. What is still not covered in a browser is that an agent's readiness *reason* is readable end to end, because the `AgentInstance` record reports a failure message and the template reports a condition, and -no single surface shows both. +no single surface shows both. Nor is either half: `AgentDetailsPage` — the record itself, +with `instance-state`, `instance-operation`, `instance-failure` and the sentence saying +what a state means — has no spec at all. A gap rather than a deferral, and worth taking +before the end-to-end one. ## What the chat fixes could not be covered against @@ -253,15 +220,12 @@ answer names the parked turn and carries the extension payload, and the agent us What is left are the two neighbouring cases, both of which the UI *recognises* and says plainly rather than guessing at. -**A `tool_approval_request`** carries `tools[]` and a `hint` and is answered with -`tool_approval_response` / `approvals[]` — a different payload, and a different control: -per-tool approve or reject, with a rejection reason. **No longer deferred.** The product -decision landed in #2714 and the controls shipped with it; they had no browser coverage -until `tests/chat/approvals.spec.ts`, which decides two tools opposite ways in one -submission and reads the decisions back off the reply rather than off the form — so it -fails if the page sends both as approvals or pairs a reason with the wrong tool. The -fixture parks on the request at the `ChatClient` boundary, which is what `asks` already -does; what stays out is wire frames, and that stays out. +**A `tool_approval_request`** is **no longer deferred.** The product decision landed in +#2714 and the controls shipped with it; they had no browser coverage until +`tests/chat/approvals.spec.ts`, which drives both shapes — several tools decided +independently behind a Submit, and a single tool decided on the prompt itself — and reads +the decisions back off the reply rather than off the form, so it fails if the page sends +both as approvals or pairs a reason with the wrong tool. **A turn parked without the HITL extension activated** has no payload at all — the question exists only as prose and carries no correlation id, so no answer can be routed diff --git a/ui/playwright/tests/chat/approvals.spec.ts b/ui/playwright/tests/chat/approvals.spec.ts index f0b8ca903f..f6e965d8d2 100644 --- a/ui/playwright/tests/chat/approvals.spec.ts +++ b/ui/playwright/tests/chat/approvals.spec.ts @@ -68,6 +68,35 @@ test("chat: a tool approval is decided per tool, and the decisions are what go b }); }); +test("chat: one tool is approved or rejected on the prompt itself", async ({ page }) => { + /* + * One tool is a different prompt, not a shorter one: there is nothing to decide + * between, so the decision is the prompt's own Approve and Reject rather than a row's + * Allow and Deny behind a Submit. Both paths send the same payload, and only this one + * has ever been reachable by a reader with a single-tool agent. + */ + await page.goto(`${AGENT_CHAT}?chat=approves-one`); + await page.getByTestId("chat-input").fill("Clear the cache."); + await page.getByTestId("chat-send").click(); + + const prompt = page.getByTestId("chat-awaiting-reply"); + await expect(prompt).toBeVisible({ timeout: 30_000 }); + await expect(page.getByTestId("chat-approval-tool")).toHaveCount(1); + + // Rejecting asks why, with the caret already in the field — the one place it can be, + // there being no other tool to choose between. + await page.getByTestId("chat-approval-reject").click(); + const reason = page.getByTestId("chat-approval-reason-call-2"); + await expect(reason).toBeFocused(); + await page.keyboard.type("Not on a Friday."); + + await page.getByTestId("chat-approval-submit").click(); + const transcript = page.getByTestId("chat-transcript"); + await expect(transcript).toContainText("call-2 rejected", { timeout: 30_000 }); + await expect(transcript).toContainText("Not on a Friday."); + await expect(prompt).toHaveCount(0); +}); + test("chat: a question this build cannot answer says so, rather than guessing", async ({ page, }) => { diff --git a/ui/src/api/chat/mockChatClient.ts b/ui/src/api/chat/mockChatClient.ts index 2f2262eb12..7bfef7b414 100644 --- a/ui/src/api/chat/mockChatClient.ts +++ b/ui/src/api/chat/mockChatClient.ts @@ -36,6 +36,7 @@ const TIMING = { asks: { step: 300, word: 45 }, "asks-text": { step: 300, word: 45 }, approves: { step: 300, word: 45 }, + "approves-one": { step: 300, word: 45 }, "asks-unknown": { step: 300, word: 45 }, } as const; @@ -401,14 +402,14 @@ export class MockChatClient implements ChatClient { return; } - if (scenario === "approves") { + if (scenario === "approves" || scenario === "approves-one") { // The same park, a different request: tools to vouch for rather than a question // to answer. `hint` is the runtime's own sentence about why it is asking. this.persist(sessionId); const request: PendingRequest = { kind: "tool_approval", taskId, - tools: APPROVAL_TOOLS, + tools: scenario === "approves-one" ? [APPROVAL_TOOLS[1]] : APPROVAL_TOOLS, hint: APPROVAL_HINT, }; saveParked(sessionId, request); diff --git a/ui/src/mocks/scenario.ts b/ui/src/mocks/scenario.ts index 409ed2bce8..bc12936faf 100644 --- a/ui/src/mocks/scenario.ts +++ b/ui/src/mocks/scenario.ts @@ -101,6 +101,11 @@ function isScenario(value: string | null): value is MockScenario { * boundary; what it must not do is start emitting wire frames, which is a transport * shape covered by unit tests over real bytes. * + * `approves-one` is the same request with a single tool, which is a different set of + * controls rather than a shorter list of the same ones: Approve and Reject on the prompt + * itself, and a rejection that opens its reason with the caret already in it. A fixture + * with two tools cannot reach either. + * * `asks-unknown` parks it on a request this build does not recognise, which is what a * turn started without the HITL extension looks like: prose, no correlation id, nothing * to answer against. The page says so and offers only the discard, and that sentence is @@ -113,6 +118,7 @@ export const CHAT_SCENARIOS = [ "asks", "asks-text", "approves", + "approves-one", "asks-unknown", ] as const; From e2c88b804977ebc8a6afc4ac8ae33d0e0d185bc5 Mon Sep 17 00:00:00 2001 From: Torey Scheer Date: Fri, 18 Sep 2026 10:58:38 -0400 Subject: [PATCH 17/24] docs(ui): correct what an audit of DEFERRED.md found MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A file whose only value is being true, checked claim by claim: every path, identifier, spec name and request shape it names, then the prose. Most of it holds. `ListModelConfigsRequest` really is empty and `ListPromptTemplatesRequest` really does take only a namespace, so the paging table is accurate; `GetSubstrateStatus` really is gone; `ListTasks` is there to carry the auto-titling argument; the MCP entry's two `Refresh` presses are still in the spec, so #2849's acceptance test still reads. Of the three files it names that do not exist, two are the point — old specs it records as lost. Three corrections: - "Not started by request: app extension-point specs" said the opposite of the paragraph two above it, which lists them as covered. Both specs exist and run, and between them assert every point the app declares. - The cleanup row still said each live spec removes what it made in a `finally`, corrected everywhere else in this file already. - `agent-lifecycle.spec.ts` is named for a fault it had; it is gone, replaced by `shared/agent-templates/`, and now says so. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Torey Scheer --- ui/playwright/DEFERRED.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/ui/playwright/DEFERRED.md b/ui/playwright/DEFERRED.md index 7f22d2b881..9885f68185 100644 --- a/ui/playwright/DEFERRED.md +++ b/ui/playwright/DEFERRED.md @@ -15,7 +15,7 @@ missing is the page. | Old spec | Blocked on | Already available | |---|---|---| | `onboarding/onboarding.spec.ts` | No onboarding wizard exists on this architecture | — nothing; drop it unless the flow is rebuilt | -| `cleanup.spec.ts` | Still not a spec. The mock suite gives each test a fresh browser context, so there is nothing to sweep; the live suite creates real resources and each spec removes what it made in a `finally`. What neither covers is a run *killed* between the two — see below. | — | +| `cleanup.spec.ts` | Still not a spec. The mock suite gives each test a fresh browser context, so there is nothing to sweep; the live suite creates real resources and each spec removes what it made in a `test.afterEach`. What neither covers is a run *killed* between the two — see below. | — | ## Now running in CI: the live suite @@ -32,7 +32,8 @@ litter and safe to remove. A sweep spec stays the wrong shape for it, being one selector away from deleting somebody's work. **Every live spec was broken the first time one was run against a cluster**, each in a -way the mock suite structurally could not show: `agent-lifecycle.spec.ts` drove +way the mock suite structurally could not show: `agent-lifecycle.spec.ts` — since +replaced by `shared/agent-templates/` — drove `/agents/new`, a page long removed, for an agent nobody creates; `schedules.spec.ts` drove a dialog for an editor that is a page, and named CI's fixture rather than `setup-cluster.sh`'s; `substrate.spec.ts` read a tile once, catching the em-dash it @@ -92,10 +93,14 @@ stops somebody porting work that is already possible. an `AgentTemplate` paired with a `Harness` and is not created. If a create-an-agent surface lands, its validation belongs in that change. -## Not started by request +## Was "not started by request", and is not that any more -App extension-point specs. The framework is still being edited and its contract is not -frozen; the team lead will ask for these once it lands. +App extension-point specs were held back while the framework's contract was unfrozen. +`extension-points.withExtension.spec.ts` and `extension-points-absent.spec.ts` both exist +and run, and between them assert every point the app declares — so this entry described +the tree accurately for about as long as it took the contract to settle, and then said +the opposite of the paragraph above it for longer. Anything still wanted here has to be +named; "the specs" is done. ## Lost with the REST path tests, and where it went instead From af726c36412517d0a67370fb8de877010edd904a Mon Sep 17 00:00:00 2001 From: Torey Scheer Date: Fri, 18 Sep 2026 11:08:00 -0400 Subject: [PATCH 18/24] test(ui): drive the conversation record, which had no spec at all MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by listing every test id in `src` against every one the suite references: `AgentDetailsPage` was the largest undriven surface in the app. Its fixtures were already there — `instances.failed` is commented "with a reason the conversation's record page shows" — so the fixture was written for this page and the spec never followed. Two tests. The record reports its state in words rather than in the controller's vocabulary, says what that state means, and links to the agent and the template instead of restating what they hold — which is the claim `DEFERRED.md` makes for why replacing the old agent details page was not a reduction: an instance has no spec. Then the three readings the page must keep apart: a failure carries the reason it failed, somebody else's conversation reads as not found because an instance is read as its creator, and a read that failed offers a retry rather than a way back to the list. Checked by breaking both substantive claims: pointing the agent link at the list fails the href, and showing the wire value in the state tag fails on "ready" where a word was expected. Verified: 191 mock, 31 conventions. The two firefox checkpoints failures in the full run pass 6/6 in isolation and were already shown to fail with these changes stashed — the contention the config documents. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Torey Scheer --- .../tests/agents/agent-details.spec.ts | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 ui/playwright/tests/agents/agent-details.spec.ts diff --git a/ui/playwright/tests/agents/agent-details.spec.ts b/ui/playwright/tests/agents/agent-details.spec.ts new file mode 100644 index 0000000000..84b602cd8f --- /dev/null +++ b/ui/playwright/tests/agents/agent-details.spec.ts @@ -0,0 +1,107 @@ +import { test, expect } from "../../fixtures/test"; +import { + agentDetail, + agentPage, + agents, + instances, + loadPage, + withScenario, +} from "../../helpers/app"; + +/** + * A conversation's own record — what the control plane knows about one `AgentInstance`. + * + * This is the page that replaced the old agent details page when agents became + * instances, and `DEFERRED.md` records why the replacement is not a reduction: an + * instance has no spec. What it has is a state, an operation, the pair it was cut from + * and a failure, and the configuration lives on the template and the harness it links + * to. Nothing had driven it in a browser until this spec. + */ + +test("agents: a conversation's record reports its state and links to what configures it", async ({ + page, +}) => { + await loadPage(page, agentDetail(instances.ready)); + + await test.step("1. the state, in words, and what the word means", async () => { + await expect(page.getByTestId("instance-status-card")).toBeVisible(); + // Read to the operator rather than passed through: `ACTOR_STATE_RUNNING` is the + // controller's vocabulary, and a reader should not have to learn it here. + const state = page.getByTestId("instance-state"); + await expect(state).toHaveText(/^[A-Z][a-z]/); + await expect(state).not.toContainText("_"); + // The sentence beside it, which is the part a state tag cannot carry: what being in + // this state means for what the reader can do next. + await expect(page.getByTestId("instance-state-meaning")).not.toBeEmpty(); + }); + + await test.step("2. it links to its agent and its template rather than restating them", async () => { + /* + * The claim the page exists to make. An instance carries no model, no prompt and no + * tools, so the record links to the two surfaces that do — and a link is checked by + * where it goes, a label being the easy half to get right. + */ + await expect(page.getByTestId("instance-agent-link")).toHaveAttribute( + "href", + agentPage(agents.k8s), + ); + await expect(page.getByTestId("instance-template-link")).toHaveAttribute( + "href", + /\/agent-templates\//, + ); + // And it says why the template link is the one that matters, since editing it + // changes every agent cut from it rather than this conversation alone. + await expect(page.getByTestId("instance-template-note")).toContainText( + "not only this one", + ); + }); + + await test.step("3. the record itself, which is what the controller stored", async () => { + await expect(page.getByTestId("instance-details")).toBeVisible(); + await expect(page.getByTestId("instance-details")).toContainText(instances.ready); + }); +}); + +test("agents: a record the reader cannot have says which of the two it is", async ({ + page, +}) => { + await test.step("1. a failed conversation reports the reason it failed", async () => { + /* + * The half of "an agent's readiness reason, end to end" that this page owns. A + * failure with no message renders a sentence saying the record held none — so an + * empty alert here would be a rendering fault rather than a quiet controller, and + * the fixture carries a real message to tell those apart. + */ + await loadPage(page, agentDetail(instances.failed)); + const failure = page.getByTestId("instance-failure"); + await expect(failure).toBeVisible(); + await expect(failure).toContainText("cannot be resumed"); + }); + + await test.step("2. somebody else's reads as not found, in the controller's own words", async () => { + /* + * Not a bug and not a euphemism: an instance is read as its creator, so the + * controller genuinely answers `NotFound` for one that is not yours. The page says + * both readings out loud rather than picking one, because a reader who knows the + * conversation exists would otherwise think the record was lost. + */ + await loadPage(page, agentDetail(instances.someoneElses)); + const notFound = page.getByTestId("instance-not-found"); + await expect(notFound).toBeVisible(); + await expect(notFound).toContainText("started by somebody else"); + // No half-drawn record behind the notice. + await expect(page.getByTestId("instance-details")).toHaveCount(0); + }); + + await test.step("3. a read that failed is not a record that is missing", async () => { + // The distinction this suite keeps everywhere: "we could not find out" must not + // render as "there is nothing here". Different id, different action — retry rather + // than a way back to the list. + await loadPage(page, withScenario(agentDetail(instances.ready), "error")); + await expect(page.getByTestId("instance-error")).toBeVisible(); + await expect(page.getByTestId("instance-not-found")).toHaveCount(0); + await expect( + page.getByTestId("instance-error").getByRole("button", { name: "Try again" }), + ).toBeVisible(); + }); +}); From 68da1a73f0ce9497c9405f613c403d40f1662ae7 Mon Sep 17 00:00:00 2001 From: Torey Scheer Date: Fri, 18 Sep 2026 14:04:43 -0400 Subject: [PATCH 19/24] test(ui): close the last review round, and verify on a cluster built from nothing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Seven findings, none blocking, two with teeth. `detailURL` was recorded after the heading assertion, so a create that reached the controller while the detail page was slow to draw left the hook with no address — a real Schedule on the cluster, which is the litter the hook exists to prevent. Both schedules specs record the address first now. And the fixture had grown a second payload parser that drifted permissive: it took an empty `approvals` array and an empty `rejection_reason` where the app's own reader rejects both, so it would have acknowledged payloads the app refuses. It uses `readToolApprovalResponse` now. The rest: the live sweep walks eight pages on one budget and had the project default, so a slow cluster reported "Test timeout exceeded" rather than naming the page; `agent-details` applied its mock scenario twice and worked only because `URLSearchParams.get` returns the first; an orphaned comment; and two documents each contradicting something true in the same file — the README said `finally` eleven lines above the rule saying never `finally`, and a CI comment claimed `buildx-create` runs once when nothing passes it `-o`. Verified: 193 mock across both engines, 13 live against a cluster deleted and rebuilt from nothing on substrate 0.2.0-beta2 with images from this tree, 530 unit, 31 conventions. No `e2e-live-*` left on the cluster afterwards. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Torey Scheer --- .github/workflows/ci.yaml | 4 +- ui/playwright/README.md | 3 +- ui/playwright/live/pages.spec.ts | 9 +++ ui/playwright/live/schedules.spec.ts | 9 ++- .../shared/schedules/schedules.spec.ts | 9 ++- .../tests/agents/agent-details.spec.ts | 3 +- ui/src/api/chat/mockChatClient.ts | 64 +++++++------------ 7 files changed, 53 insertions(+), 48 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 933cfaa980..a842945e40 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -133,7 +133,9 @@ jobs: BUILDX_BUILDER_NAME=$(docker buildx inspect | awk '$1 == "Name:" { print $2; exit }') test -n "$BUILDX_BUILDER_NAME" export BUILDX_BUILDER_NAME - # Once, before the fan-out: every `build-*` depends on it. + # Before the fan-out. Each `build-*` runs it again — it is not passed `-o` the + # way `proto-generate` is below — which is harmless only because the target + # documents itself as safe to run concurrently. make buildx-create # Likewise once, and for a sharper reason: `build-ui` and `build-golang-adk` both # depend on it, and `buf generate` writes `ui/src/generated` as well as diff --git a/ui/playwright/README.md b/ui/playwright/README.md index 662909991f..39f40c4874 100644 --- a/ui/playwright/README.md +++ b/ui/playwright/README.md @@ -88,7 +88,8 @@ resource**: create, read back, change, delete. That second kind is where a fixture and a controller most easily disagree, and it is why every resource that can have its journey here does: `models`, `prompts`, `harnesses`, `agent-templates` and `schedules`. The names are `throwawayName`d and every one cleans up -in a `finally`, because live they are real. +in a `test.afterEach`, because live they are real — see the rule below for why not a +`finally`. **One move rules a spec out of here: a reload.** The fixture backend keeps writes in the page's own memory, so a reload starts a backend that has never heard of what was just diff --git a/ui/playwright/live/pages.spec.ts b/ui/playwright/live/pages.spec.ts index 25fb05e1b4..6716687f58 100644 --- a/ui/playwright/live/pages.spec.ts +++ b/ui/playwright/live/pages.spec.ts @@ -7,6 +7,7 @@ import { expectNoLoadFailure, loadApp, } from "../helpers/app"; +import { LIFECYCLE_TIMEOUT } from "../helpers/resource"; import { liveRoutes } from "./helpers/live"; /** @@ -80,6 +81,14 @@ const answered: Record< .or(page.getByTestId("substrate-inventory-error")), }; +/* + * Eight pages, each with a load and a read behind its own sixty seconds, in one test. + * On the live project's 120s default two slow ones exhaust it and the failure reads + * "Test timeout exceeded" rather than naming the page — the reporting loss that + * `READ_TIMEOUT` and `PRESS_TIMEOUT` exist to avoid. + */ +test.describe.configure({ timeout: LIFECYCLE_TIMEOUT }); + test("live: every page loads against the cluster and reports no failure", async ({ page, }) => { diff --git a/ui/playwright/live/schedules.spec.ts b/ui/playwright/live/schedules.spec.ts index 98215a82f4..6082c88b71 100644 --- a/ui/playwright/live/schedules.spec.ts +++ b/ui/playwright/live/schedules.spec.ts @@ -99,10 +99,17 @@ test("live: a schedule's configuration survives a reload", async ({ page }) => { await page.getByTestId("schedule-enabled").uncheck(); await page.getByTestId("schedule-submit").click(); + /* + * The address first, then what is on it. Recorded after the heading, a create that + * reached the controller while the detail page was slow to draw left the hook with + * no URL to clean up — a real Schedule on the cluster, which is the one thing the + * hook exists to prevent. + */ + await page.waitForURL(/\/schedules\/[0-9a-f-]+(\?|$)/, { timeout: 60_000 }); + detailURL = page.url(); await expect(page.getByRole("heading", { name: CREATED, exact: true })).toBeVisible({ timeout: 60_000, }); - detailURL = page.url(); }); await test.step("2. a reload reads it back from the controller, unchanged", async () => { diff --git a/ui/playwright/shared/schedules/schedules.spec.ts b/ui/playwright/shared/schedules/schedules.spec.ts index 58b477f637..2b9ea085e3 100644 --- a/ui/playwright/shared/schedules/schedules.spec.ts +++ b/ui/playwright/shared/schedules/schedules.spec.ts @@ -127,10 +127,17 @@ test("schedules: one is created, read, changed and deleted", async ({ page }) => await test.step("3. creating it lands on its own page, showing what was asked for", async () => { await page.getByTestId("schedule-submit").click(); + /* + * The address first, then what is on it. Recorded after the heading, a create that + * reached the controller while the detail page was slow to draw left the hook with + * no URL to clean up — a real Schedule on the cluster, which is the one thing the + * hook exists to prevent. + */ + await page.waitForURL(/\/schedules\/[0-9a-f-]+(\?|$)/, { timeout: READ_TIMEOUT }); + detailURL = page.url(); await expect(page.getByRole("heading", { name: CREATED, exact: true })).toBeVisible({ timeout: READ_TIMEOUT, }); - detailURL = page.url(); await expect(page).toHaveURL(/\/schedules\/[0-9a-f-]+(\?|$)/); // Created paused, so the one control whose label flips offers to resume it. diff --git a/ui/playwright/tests/agents/agent-details.spec.ts b/ui/playwright/tests/agents/agent-details.spec.ts index 84b602cd8f..99e0458a5d 100644 --- a/ui/playwright/tests/agents/agent-details.spec.ts +++ b/ui/playwright/tests/agents/agent-details.spec.ts @@ -5,7 +5,6 @@ import { agents, instances, loadPage, - withScenario, } from "../../helpers/app"; /** @@ -97,7 +96,7 @@ test("agents: a record the reader cannot have says which of the two it is", asyn // The distinction this suite keeps everywhere: "we could not find out" must not // render as "there is nothing here". Different id, different action — retry rather // than a way back to the list. - await loadPage(page, withScenario(agentDetail(instances.ready), "error")); + await loadPage(page, agentDetail(instances.ready), { scenario: "error" }); await expect(page.getByTestId("instance-error")).toBeVisible(); await expect(page.getByTestId("instance-not-found")).toHaveCount(0); await expect( diff --git a/ui/src/api/chat/mockChatClient.ts b/ui/src/api/chat/mockChatClient.ts index 7bfef7b414..80f1c876ad 100644 --- a/ui/src/api/chat/mockChatClient.ts +++ b/ui/src/api/chat/mockChatClient.ts @@ -16,7 +16,11 @@ import { currentChatScenario } from "@/mocks/scenario"; import { allAgentInstances, instanceShareForToken } from "@/mocks/state"; import { ApiError } from "../ApiError"; import { agentInstanceShareToken } from "../shareToken"; -import { HITL_EXTENSION_URI, type PendingRequest } from "./hitl"; +import { + HITL_EXTENSION_URI, + readToolApprovalResponse, + type PendingRequest, +} from "./hitl"; import { conversationKey } from "./types"; import type { ChatClient, @@ -239,24 +243,29 @@ export class MockChatClient implements ChatClient { /* * What the agent understood, which is not the same as what it received. * - * The runtime reads the structured answer only from a message that both - * declares the extension and carries the payload under its URI; anything else - * reaches the agent as ordinary prose, the turn resumes, and the reply reads - * as though it worked. So the acknowledgement here says which happened — that - * silent failure is the reason this fixture bothers to check. - */ - /* - * What the agent understood of an approval, said tool by tool. A fixture that - * acknowledged "noted" either way would pass a UI that sent every decision as an - * approval, or paired the reasons with the wrong rows. + * The runtime reads a structured answer only from a message that both declares the + * extension and carries the payload under its URI; anything else reaches the agent + * as ordinary prose, the turn resumes, and the reply reads as though it worked. So + * the acknowledgement below says which happened, tool by tool for an approval — + * that silent failure is the reason this fixture bothers to check. + * + * Through the app's own parser rather than a second one: a copy here drifted + * permissive, taking an empty `approvals` array and an empty `rejection_reason` + * where the real reader rejects both, and a fixture that accepts more than the code + * it stands in for acknowledges payloads the app would refuse. The extension list + * is asserted rather than read — the port carries the payload directly, having no + * wire to declare it on. */ - const decisions = parked.kind === "tool_approval" ? readApproval(input.hitl) : undefined; + const decisions = + parked.kind === "tool_approval" + ? readToolApprovalResponse(input.hitl, [HITL_EXTENSION_URI]) + : undefined; const approvalReply = decisions && decisions .map( (decision) => - `${decision.id} ${decision.approved ? "approved" : `rejected (${decision.reason ?? "no reason given"})`}`, + `${decision.id} ${decision.approved ? "approved" : `rejected (${decision.rejectionReason ?? "no reason given"})`}`, ) .join("; "); @@ -525,35 +534,6 @@ function readAnswer( return { id: body.id, answers }; } -/** - * The decisions carried back by a tool approval, read the way the runtime reads them. - * - * `rejection_reason` is snake_case on the wire and optional: a rejection may carry one - * and an approval never does. Read here so the acknowledgement can say which tools were - * approved — a fixture that answered "noted" either way would let a UI that sent the - * decisions the wrong way round pass. - */ -function readApproval( - hitl: Record | undefined, -): { id: string; approved: boolean; reason?: string }[] | undefined { - const payload = hitl?.[HITL_EXTENSION_URI]; - if (typeof payload !== "object" || payload === null) return undefined; - const body = payload as Record; - if (body.type !== "tool_approval_response" || !Array.isArray(body.approvals)) { - return undefined; - } - return body.approvals.filter(isRecord).map((entry) => ({ - id: typeof entry.id === "string" ? entry.id : "", - approved: entry.approved === true, - reason: - typeof entry.rejection_reason === "string" ? entry.rejection_reason : undefined, - })); -} - -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null; -} - function clearParked(sessionId: string): void { try { window.sessionStorage.removeItem(PARKED_PREFIX + sessionId); From 3892ef9c7a23d7fbe60aa95a0767b1f85b72b7cc Mon Sep 17 00:00:00 2001 From: Torey Scheer Date: Tue, 22 Sep 2026 12:48:04 -0400 Subject: [PATCH 20/24] docs(ui): close what is no longer deferred, and say what the job costs DEFERRED.md had grown into a record of work already done. Everything settled moves to one Closed section, a line each, so the file is only what is still blocked: the agent form (there is none, and routes.ts records why), onboarding, a cleanup spec, the REST path tests, the extension-point specs, tool approval, the mock ports, the live suite running in CI, and AgentDetailsPage - which the file still called uncovered after this branch covered it. Two specs cited sections this removes. The reason AgentDetailsPage's record is not a reduction stays, so that citation resolves. The other cited a sharing gap that was never written down at all; it is real, so it is an entry now. The test-e2e comment reasoned from a runtime the job no longer has. It takes around ten minutes with the image build and the browser suite in it. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Torey Scheer --- .github/workflows/ci.yaml | 3 +- ui/playwright/DEFERRED.md | 347 +++++++++++++------------------------- ui/playwright/README.md | 2 +- 3 files changed, 117 insertions(+), 235 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a842945e40..1ccab9d322 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -42,7 +42,8 @@ jobs: VERSION: v0.0.1-test SUBSTRATE_VERSION: 0.2.0-beta2 runs-on: blacksmith-4vcpu-ubuntu-2404 - # 50: main's own 40, plus the ten or so the UI image and the browser suite below add. + # 50: main's own 40, kept as headroom. The job itself takes around 10 minutes, + # UI image and browser suite included, on a warm buildx cache. timeout-minutes: 50 steps: - name: Checkout repository diff --git a/ui/playwright/DEFERRED.md b/ui/playwright/DEFERRED.md index 9885f68185..9cdca572b2 100644 --- a/ui/playwright/DEFERRED.md +++ b/ui/playwright/DEFERRED.md @@ -1,48 +1,55 @@ # Deferred specs -The old suite had 13 specs. Everything it covered that still has a page is ported, -plus a good deal it did not have: a lifecycle spec per resource, routing, auth, the -two extension-point specs. The rest is listed here rather than committed as skipped -tests, because a skipped or vacuous spec reads as coverage and this list does not. - -Where an entry below names a spec that no longer exists under that filename, the -coverage moved rather than went; `README.md` has the current layout. - -Each entry names the surface that has to exist before the spec can assert -anything real. In every case the data layer is already in place — what is -missing is the page. - -| Old spec | Blocked on | Already available | -|---|---|---| -| `onboarding/onboarding.spec.ts` | No onboarding wizard exists on this architecture | — nothing; drop it unless the flow is rebuilt | -| `cleanup.spec.ts` | Still not a spec. The mock suite gives each test a fresh browser context, so there is nothing to sweep; the live suite creates real resources and each spec removes what it made in a `test.afterEach`. What neither covers is a run *killed* between the two — see below. | — | - -## Now running in CI: the live suite - -`playwright/live/` runs in the `test-e2e` job, against the image built from -`ui/Dockerfile` on the same cluster as the Go end-to-end tests. `README.md` has the -wiring. Two things follow for this file. - -**The cleanup entry above is now about litter, not contexts.** A live spec deletes what -it created, in a `test.afterEach` rather than a `finally` — measured, a timed-out test has -a closed page and everything in its `finally` throws, which is the failure most likely to -leave something behind. A run killed outright still cannot. That is why `throwawayName` puts the -process and a timestamp in every name — anything matching `e2e-live-*` in `kagent` is -litter and safe to remove. A sweep spec stays the wrong shape for it, being one bad -selector away from deleting somebody's work. - -**Every live spec was broken the first time one was run against a cluster**, each in a -way the mock suite structurally could not show: `agent-lifecycle.spec.ts` — since -replaced by `shared/agent-templates/` — drove -`/agents/new`, a page long removed, for an agent nobody creates; `schedules.spec.ts` -drove a dialog for an editor that is a page, and named CI's fixture rather than -`setup-cluster.sh`'s; `substrate.spec.ts` read a tile once, catching the em-dash it -draws before the data lands. Two harness faults came out of the same run and were fixed -in `helpers/resource.ts`. - -The specs were written, they were correct, nothing ran them, and they decayed into files -testing pages that had been removed. A stale entry costs more than no entry; a stale -*spec* costs more again. +Coverage this suite does not have, and what each one is waiting on. Kept as prose +rather than as skipped tests, because a skipped spec reads as coverage and this list +does not. + +Two rules for editing it, both learned the hard way here: + +- **A stale entry costs more than no entry.** Several entries were once listed as + blocked on pages that already existed, which stops somebody porting work that is + already possible. When something lands, close it in the same change. +- **Close, do not archive.** An entry describing a page that no longer exists, or a + gap since covered, belongs in *Closed* below as one line — or deleted. + +## Closed + +Nothing below is a gap. They are listed only so they are not looked for again. + +- **Onboarding.** There is no onboarding wizard on this architecture and none planned. +- **A cleanup spec.** Each mock test gets a fresh browser context, and each live spec + deletes what it made in a `test.afterEach`. A run killed outright still leaks, which + is why `throwawayName` puts the process and a timestamp in every name: anything + matching `e2e-live-*` in `kagent` is litter and safe to remove by hand. A sweep spec + stays the wrong shape for that, being one bad selector away from deleting real work. +- **Agent-create validation.** There is no agent form. An agent is an `AgentTemplate` + paired with a `Harness`, materialised by admission — `router/routes.ts` records why + there is no `agentNew` and no `agentEdit`. Nothing creates one, so there is nothing + to validate. The create-and-read-back property that lived in the removed agent-create + spec is now `tests/harnesses/harnesses.spec.ts`. +- **The REST path tests.** `src/api/{readPaths,writePaths}.test.ts` drove the client over + REST URLs that no longer exist; the controller serves gRPC-Web. `src/api/operations.test.ts` + replaces them against the real generated descriptors and covers strictly more. +- **The extension-point specs.** `extension-points-absent.spec.ts` and + `extension-points.withExtension.spec.ts` both run and between them assert every point + the app declares. +- **Tool approval, and a question asked without the extension.** Both shipped with #2714 + and are driven by `tests/chat/approvals.spec.ts` — several tools decided independently + behind a Submit, one tool decided on the prompt itself, and the unanswerable turn that + says so rather than inventing controls. The decisions are read back off the reply, not + off the form. +- **`AgentDetailsPage`.** Covered by `tests/agents/agent-details.spec.ts`: the state and + what it means, the links out to the template and the agent, the failure, and the + difference between a record that is missing and a read that failed. The page used to + show a `SandboxAgent`'s spec — its model, its tool bindings, its `Ready` condition — + and shows an `AgentInstance` record instead. That is not a reduction to restore: an + instance genuinely has no spec, and the configuration lives on the `AgentTemplate` and + `Harness` surfaces, which have their own specs. +- **The live suite running anywhere.** `playwright/live/` runs in the `test-e2e` job + against the image built from `ui/Dockerfile`. `README.md` has the wiring. +- **Chat and its error journeys, the chat-message extension point, MCP servers in mock, + prompt libraries, and form validation for every resource with a form.** Each is a spec + now, and a spec describes itself better than a list of what it covers. ## Deferred: MCP servers stay mock-only — the list cannot read its own writes @@ -79,170 +86,54 @@ Worth recording for its own sake: the fixtures cannot show this class of bug at answer from the page's own memory and are therefore always immediately consistent, so a mock backend has no write that is not yet a read. Only a cluster has one. -## Ported since, and no longer worth describing here - -Chat and its error journeys, the chat-message extension point, MCP servers, prompt -libraries, and form validation for every resource that has a form. Each is a spec now, -and a spec describes itself better than a list of what it covers. - -**Two of those had been listed as blocked on pages that already existed.** That is the -lesson worth keeping: a stale "blocked on" entry costs more than no entry, because it -stops somebody porting work that is already possible. - -**What is still missing is the agent form, because there is no agent form.** An agent is -an `AgentTemplate` paired with a `Harness` and is not created. If a create-an-agent -surface lands, its validation belongs in that change. - -## Was "not started by request", and is not that any more - -App extension-point specs were held back while the framework's contract was unfrozen. -`extension-points.withExtension.spec.ts` and `extension-points-absent.spec.ts` both exist -and run, and between them assert every point the app declares — so this entry described -the tree accurately for about as long as it took the contract to settle, and then said -the opposite of the paragraph above it for longer. Anything still wanted here has to be -named; "the specs" is done. - -## Lost with the REST path tests, and where it went instead - -`src/api/readPaths.test.ts` and `src/api/writePaths.test.ts` are gone. They drove the API -client over REST URLs against the MSW fixture backend, and neither the URLs nor that -backend's REST routes exist any more — the controller serves its application API as -gRPC-Web. `src/api/operations.test.ts` replaces them, against the real generated service -descriptors served in-process, and covers strictly more of what those two were for: which -RPC each operation invokes, with what identity in the request message, and what the -response converts to. - -**One property could not live there, and it now lives in a browser spec instead.** The old -write tests read each create *back through its list* — "the create returned 200" and "the -thing exists" are different claims, and only a stateful backend can check the second. The -in-process router is stateless per test, so `operations.test.ts` cannot. That property is now -`playwright/tests/harnesses/harnesses.spec.ts`: create a harness, land back on the tab it was -created from, and find it in the list — and find it reported "not ready yet", which is the -state a cluster reports for one the controller has not observed. Nothing about it is -deferred any more. - -It lived in an agent-create spec until that page was removed: an agent is not something -anybody creates, so the form that appeared to create one went, and the read-back property -moved to the nearest thing that is genuinely created. - -Two things worth keeping from writing it, because both cost time and neither is guessable: - -- **Stay inside one browsing context.** The fixture backend keeps writes in the page's own - memory, deliberately, so one spec's creates cannot leak into the next one's list. A - `page.goto` therefore starts a backend that has never heard of the thing just created, - and the failure reads as "the create did not stick" when nothing is wrong. Click through - from the list. -- **The second read is the point.** `chat-capabilities-toggle` is asserted rather than the - heading or the panel, because those render from the URL and would appear for an agent - that does not exist. That button renders only when the per-agent read resolved a row, so - it is what distinguishes "the list re-fetched" from "the thing exists". The weaker - version of this spec passes and proves less than it looks like it does. - -**Every resource reads its create back through its list now**, not only harnesses — see -*Covered: form validation, and every resource's lifecycle* above, which also records the create-cache defect -that used to make that impossible for three of them and how the fix was proved. - ---- - -## Lost when agents became AgentInstances - -One thing the suite used to cover no longer exists, and it should not be replaced by a -passing test of something adjacent. Two others that were listed here — the capabilities -panel and the sharing loop — have since come back and are covered. - -### An agent's own tools, model and readiness on its details page - -The details page showed a `SandboxAgent`'s spec: its model resolved from a `ModelConfig`, -its tool bindings, and its `Ready` condition with a reason. It now shows the -`AgentInstance` record instead — state, operation, the pair it was cut from, the prepared -revision, the A2A authority and the failure — which is the whole of what the API knows -about an instance. - -That is not a reduction to fix: an instance genuinely has no spec. The configuration -belongs on the `AgentTemplate` and `Harness` surfaces, and those exist now — the agents -landing page carries all three as tabs, and a conversation's record links out to the -template and to the agent rather than duplicating either. What is still not covered in a -browser is that an agent's readiness *reason* is readable end to end, because the -`AgentInstance` record reports a failure message and the template reports a condition, and -no single surface shows both. Nor is either half: `AgentDetailsPage` — the record itself, -with `instance-state`, `instance-operation`, `instance-failure` and the sentence saying -what a state means — has no spec at all. A gap rather than a deferral, and worth taking -before the end-to-end one. - -## What the chat fixes could not be covered against - -Three gaps left by the work on the reader's own message, the artifact-append streaming -and the lifecycle indicator. Each is a *mock* gap: the mock backend cannot produce the -state the assertion would need, and inventing one would make the fixture the thing being -tested. - -### The suspending stage of the lifecycle indicator - -`chat.spec.ts` drives the indicator through its resting reading and through `running`, -because a turn produces both. It never sees `resuming` or `suspending`: those come from -`AgentInstance.operation`, which the controller claims and clears as it works, and the -mock backend serves a static record. Faking one would prove only that a fixture can hold -a string. - -The reading itself is covered exhaustively in `src/components/chat/lifecycleReading.test.ts` -— including the case worth guarding hardest, that **no stage is claimed when a turn ends**, -since a substrate agent really does suspend itself then and nothing in the API reports it. -What is missing is a browser journey that suspends an instance from the agents list while a -chat page is open on it and watches the indicator follow. That belongs in `playwright/live/`, -where the operation is real — and that lane now exists and runs in CI, so the blocker has -moved rather than gone: an `AgentInstance` exists only once a message has been sent, and -neither cluster has a model that can answer one. CI installs with `OPENAI_API_KEY: fake`, -and `setup-cluster.sh` sets no key at all. - -### Streaming, end to end, against a controller - -The client now honours an artifact's `append` flag, which is how this runtime streams: one -`artifactId` for the reply, one frame per token, `append` on every frame after the first, -then a closing frame repeating the whole answer. That shape is pinned in -`src/api/chat/a2aGrpcChatClient.test.ts` against frames captured from the controller on -2026-08-24, and it was confirmed by hand — `grpcurl` at the gateway, and a throwaway -Playwright run against a live instance that rendered the reply. - -**The mock chat client does not reproduce that shape.** It streams with `delta` events, -which is the port's own vocabulary rather than the wire's, so no browser test exercises the -artifact path. Teaching the fixture to emit artifact frames would mean it stopped being a -`ChatClient` and started being an A2A server, which is the wrong seam — the transport is -already covered by unit tests over real bytes. The browser-level gap is a `playwright/live/` -spec that sends a message and asserts the reply grows on screen before the turn completes. -The lane is no longer what blocks it: a reply that streams needs a model that answers, and -both clusters this suite runs against carry a fake key. Reachable today only by a developer -with their own. - -A related gap worth naming rather than leaving implicit: the mock backend serves one -instance per conversation and never *changes* an instance's `operation`, so the lifecycle -indicator's `resuming` and `suspending` stages have no browser coverage either. Both -belong in the same live spec. - -### Tool approval, and a question asked without the extension - -`ask_user` is now answerable end to end: the question renders with its choices, the -answer names the parked turn and carries the extension payload, and the agent uses it. -What is left are the two neighbouring cases, both of which the UI *recognises* and says -plainly rather than guessing at. - -**A `tool_approval_request`** is **no longer deferred.** The product decision landed in -#2714 and the controls shipped with it; they had no browser coverage until -`tests/chat/approvals.spec.ts`, which drives both shapes — several tools decided -independently behind a Submit, and a single tool decided on the prompt itself — and reads -the decisions back off the reply rather than off the form, so it fails if the page sends -both as approvals or pairs a reason with the wrong tool. - -**A turn parked without the HITL extension activated** has no payload at all — the -question exists only as prose and carries no correlation id, so no answer can be routed -to it. The prompt says so and offers the discard. This build always activates the -extension, so it can only arise from a turn started by something else (a `kubectl`-driven -send, an older client). It is not worth engineering around; it is worth not lying about — -and the sentence that does the not-lying is covered in the same spec. - -**The `ask_user` payload still renders as JSON in the transcript**, beside the answerable -prompt — the tool call and its result are structured data and are shown as such. That is -now duplication rather than a defect, and collapsing it needs a decision about whether a -tool call that has an interactive rendering should still show its raw form at all. +## Deferred: the two chat gaps that need a model which answers + +Both belong in one `playwright/live/` spec, and neither cluster this suite runs against +can host it: CI installs with `OPENAI_API_KEY: fake`, and `setup-cluster.sh` sets no key +at all. An `AgentInstance` exists only once a message has been sent, so a turn that never +gets a reply produces neither state. Reachable today only by a developer with their own +key. + +**The `resuming` and `suspending` stages of the lifecycle indicator.** `chat.spec.ts` +drives the indicator at rest and through `running`, because a turn produces both. The +other two come from `AgentInstance.operation`, which the controller claims and clears as +it works; the mock backend serves a static record, and faking one would prove only that a +fixture can hold a string. The reading itself is covered exhaustively in +`src/components/chat/lifecycleReading.test.ts` — including the case worth guarding +hardest, that **no stage is claimed when a turn ends**, since a substrate agent really +does suspend itself then and nothing in the API reports it. What is missing is the +journey: suspend an instance from the agents list with a chat page open on it, and watch +the indicator follow. + +**Streaming, end to end.** The client honours an artifact's `append` flag, which is how +this runtime streams: one `artifactId` for the reply, one frame per token, `append` on +every frame after the first, then a closing frame repeating the whole answer. That shape +is pinned in `src/api/chat/a2aGrpcChatClient.test.ts` against frames captured from the +controller on 2026-08-24. The mock chat client streams `delta` events — the port's +vocabulary rather than the wire's — so no browser test exercises the artifact path. +Teaching the fixture to emit artifact frames would make it an A2A server rather than a +`ChatClient`, which is the wrong seam; the transport is already covered over real bytes. +The gap is a live spec that sends a message and watches the reply grow before the turn +completes. + +**Open, but a product decision rather than coverage:** the `ask_user` payload still +renders as JSON in the transcript beside the answerable prompt. That is duplication +rather than a defect, and collapsing it needs a decision about whether a tool call with +an interactive rendering should show its raw form at all. + +## Deferred: proving a share token actually travels + +`tests/chat/sharing.spec.ts` covers the loop — create a link, see it listed once, revoke +it, open one — and can prove the page spends a token and reports a refusal. It cannot +prove the header reaches a backend: chat in mock mode is served by a client-side fake +that builds no request, so the spec reads the registration directly rather than seeing +what travelled. + +Only a live spec closes it, and unlike the two chat gaps above this one needs no model +that answers — a share is over an instance, and an instance exists as soon as a +conversation is opened. What it needs is a second identity: the point of the check is +that the A2A gateway reads the instance as the share's *owner*, which a visitor who is +the same signed-in reader cannot demonstrate. ## Blocked on the API: server-side paging, searching and sorting — for every list @@ -274,16 +165,16 @@ Two capabilities remain deferred until Substrate supports them: **Which actor is on a worker is not deferred; it is not available.** ate-api's `Worker` carries capacity and allocation and no actor reference — the binding lives on the actor — so the workers table has no Actor column. `busyWorkerCount` counts workers with a positive -allocated actor count reported by Substrate. A column would need the -walk per page. +allocated actor count reported by Substrate. A column would need the walk per page. **A single-message read is defensible only while the message really holds everything.** `GetSubstrateStatus` is the read that failed this way once: a cluster of 410,110 actors produces a response gRPC refused to send, which is why the substrate page was split into -three reads in the first place. That endpoint has been removed from `SystemService`. For the three reads at the top of this table that do still answer with -everything, **the moment one starts paging — or starts truncating to survive — its -client-side search and sort must be labelled or removed in the same change**, because an -unlabelled filter over a page reports "no matches" about a row on page nine. +three reads in the first place. That endpoint has been removed from `SystemService`. For +the three reads at the top of this table that do still answer with everything, **the +moment one starts paging — or starts truncating to survive — its client-side search and +sort must be labelled or removed in the same change**, because an unlabelled filter over +a page reports "no matches" about a row on page nine. The prompts page is a partial exception worth not losing: `ListPromptTemplates` takes a namespace, so `usePrompts` fans out one call per namespace and its **namespace filter is @@ -301,8 +192,6 @@ pages, and none of their totals is `rows.length`. Counting what arrived and call total is the lie a separate summary read exists to prevent, which is what `GetSubstrateSummary` is for. ---- - ## Auto-titling costs a read per row, so the table still does not do it A conversation is named by the reader, and an unnamed one can be titled from its first @@ -310,16 +199,13 @@ message — `ListTasks{ContextID: instanceId}` returns the history. That is **fr chat page**, which has already read the transcript because it is rendering it. The **rail** now pays for the rest, bounded at thirty: every row but the open one used to -read `Untitled · 50b46891`, which made the list very nearly unusable — the one row a -reader could identify was the one they were already looking at. Thirty reads for a rail -somebody is navigating by is a trade worth making; failures are per-row and silent, -because a title is a convenience over an id that already identifies the row. +read `Untitled · 50b46891`, which made the list very nearly unusable. Failures are per-row +and silent, because a title is a convenience over an id that already identifies the row. The agent's conversation **table** still falls back to `Untitled · `, and that is a decision rather than an omission: it is the surface that could hold hundreds of rows, and one read per row to put a label on them is the cost the rail's budget of thirty -exists to bound. What narrows it is described two sections down — the read is paged and -this client follows every page, and the search and sort are the browser's. +exists to bound. Two ways it could stop being a trade-off, both server-side and neither invented here: @@ -329,10 +215,9 @@ Two ways it could stop being a trade-off, both server-side and neither invented - **`ListAgentInstances` gains a field mask** for it, so callers that want it pay and callers that do not are unaffected. -Either would let a list show what the chat page already shows. Until then, what a list -renders for an unnamed conversation is pinned by `agents/agent-page.spec.ts` — -both that it is never a bare UUID, and that the derived title appears where the -transcript is in hand. +Until then, what a list renders for an unnamed conversation is pinned by +`agents/agent-page.spec.ts` — both that it is never a bare UUID, and that the derived +title appears where the transcript is in hand. ## An agent's conversation search is over what was fetched, and the page-following is why @@ -345,9 +230,7 @@ sorts run in the browser. That is honest here for a reason worth stating, because it is the one read on the list above that is paged at all: the client follows every page token before rendering anything (`INSTANCE_PAGE_LIMIT` in `api/grpc/operations.ts`), so what is in the browser is every -conversation with that agent rather than the first fifty. The page used to say so under -the table; that note was removed as commentary a reader has no use for, which leaves this -file as the record. +conversation with that agent rather than the first fifty. **If that page-following is ever removed** — and it should be, once an agent can have thousands of conversations — the search and the sort must go server-side in the same @@ -376,11 +259,9 @@ harness that will run it, when the cluster has exactly one. **Why:** the fixtures carry more than one harness on purpose — one of them exists specifically so a template can be admitted by *two*, which is what makes an agent list show two rows for one template. A single-harness cluster is therefore not a state these -fixtures can be in, and the default correctly does nothing against them. - -The opposite half *is* covered: with several harnesses nothing is chosen for the reader, -and a template no harness admits says so ("creating one, and being told when nothing -will run it"). +fixtures can be in, and the default correctly does nothing against them. The opposite +half *is* covered: with several harnesses nothing is chosen for the reader, and a +template no harness admits says so. **How it was checked instead:** against the live cluster, which has one harness (`kagent`) — the same shape the default exists for. diff --git a/ui/playwright/README.md b/ui/playwright/README.md index 39f40c4874..ed84918a0f 100644 --- a/ui/playwright/README.md +++ b/ui/playwright/README.md @@ -74,7 +74,7 @@ playwright/ live/ the live suite: specs, plus helpers/ of its own shared/ specs that run in both suites — laid out like tests/, one folder per resource and app-wide specs at the root — see below - DEFERRED.md the specs not yet portable, and what each one is waiting on + DEFERRED.md coverage this suite does not have, and what each gap waits on ``` **`shared/` runs in every project**, mock and live alike. What goes in it is narrow: no From dfc488ef961f6994fe14b594cc507bd8c1c2f4ca Mon Sep 17 00:00:00 2001 From: Torey Scheer Date: Tue, 22 Sep 2026 13:19:44 -0400 Subject: [PATCH 21/24] docs(ui): decide two entries instead of deferring them The suspending and resuming lifecycle stages are not planned. The reading has exhaustive unit coverage where the logic lives, including the case that matters most - no stage claimed when a turn ends - so a browser test would add that a controller string reaches an element, for two stages of four, and would be the only spec here needing its own API key. Recorded as a decision so it is not re-argued each time somebody notices the gap. The broken-create entry is dropped: it contradicted README.md, which states the ordered lifecycle as a deliberate trade and the right way round. A file of gaps should not carry an open item against a decision already made. DEFERRED.md now says which of three states an entry is in - deferred, not planned, or closed - since that is most of what a reader needs from it. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Torey Scheer --- ui/playwright/DEFERRED.md | 92 +++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 47 deletions(-) diff --git a/ui/playwright/DEFERRED.md b/ui/playwright/DEFERRED.md index 9cdca572b2..34c718f6a2 100644 --- a/ui/playwright/DEFERRED.md +++ b/ui/playwright/DEFERRED.md @@ -1,16 +1,22 @@ # Deferred specs -Coverage this suite does not have, and what each one is waiting on. Kept as prose +Coverage this suite does not have, and what each gap is waiting on. Kept as prose rather than as skipped tests, because a skipped spec reads as coverage and this list does not. -Two rules for editing it, both learned the hard way here: +An entry is in one of three states, and saying which is most of the value here: +**deferred** (blocked on something nameable), **not planned** (a decision, so it is not +re-argued every time somebody notices the gap), or **closed**. + +Three rules for editing it, all learned the hard way here: - **A stale entry costs more than no entry.** Several entries were once listed as blocked on pages that already existed, which stops somebody porting work that is already possible. When something lands, close it in the same change. - **Close, do not archive.** An entry describing a page that no longer exists, or a gap since covered, belongs in *Closed* below as one line — or deleted. +- **Do not defer a decision.** If the suite is not going to cover something, say so and + say why. An entry that reads as queued is one somebody will pick up. ## Closed @@ -86,28 +92,38 @@ Worth recording for its own sake: the fixtures cannot show this class of bug at answer from the page's own memory and are therefore always immediately consistent, so a mock backend has no write that is not yet a read. Only a cluster has one. -## Deferred: the two chat gaps that need a model which answers - -Both belong in one `playwright/live/` spec, and neither cluster this suite runs against -can host it: CI installs with `OPENAI_API_KEY: fake`, and `setup-cluster.sh` sets no key -at all. An `AgentInstance` exists only once a message has been sent, so a turn that never -gets a reply produces neither state. Reachable today only by a developer with their own -key. - -**The `resuming` and `suspending` stages of the lifecycle indicator.** `chat.spec.ts` -drives the indicator at rest and through `running`, because a turn produces both. The -other two come from `AgentInstance.operation`, which the controller claims and clears as -it works; the mock backend serves a static record, and faking one would prove only that a -fixture can hold a string. The reading itself is covered exhaustively in -`src/components/chat/lifecycleReading.test.ts` — including the case worth guarding -hardest, that **no stage is claimed when a turn ends**, since a substrate agent really -does suspend itself then and nothing in the API reports it. What is missing is the -journey: suspend an instance from the agents list with a chat page open on it, and watch -the indicator follow. - -**Streaming, end to end.** The client honours an artifact's `append` flag, which is how -this runtime streams: one `artifactId` for the reply, one frame per token, `append` on -every frame after the first, then a closing frame repeating the whole answer. That shape +## Not planned: the `resuming` and `suspending` lifecycle stages + +A decision rather than a queue entry, recorded so it is not re-argued each time +somebody notices the gap. + +`chat.spec.ts` drives the lifecycle indicator at rest and through `running`, because a +turn produces both. The other two stages come from `AgentInstance.operation`, which the +controller claims and clears as it works. The mock backend serves a static record, and +faking one would prove only that a fixture can hold a string; a live journey — suspend an +instance from the agents list with a chat page open on it, and watch the indicator follow +— needs a model that can answer, which neither cluster this suite runs against has. + +**What tips it from deferred to not planned** is that the part with the logic in it is +already covered, and it is covered where the logic lives: +`src/components/chat/lifecycleReading.test.ts` exercises every reading exhaustively, +including the case worth guarding hardest — that **no stage is claimed when a turn ends**, +since a substrate agent really does suspend itself then and nothing in the API reports it. +What a browser test would add is that a string the controller sets reaches an element, +for two stages out of four, at the cost of the only spec in this suite needing its own +API key. + +Revisit if the indicator grows behaviour of its own, rather than reading a field. + +## Deferred: streaming, end to end + +The one chat gap still worth a spec, and it needs a cluster with a model that answers: +CI installs with `OPENAI_API_KEY: fake` and `setup-cluster.sh` sets no key at all, so it +is reachable today only by a developer with their own. + +The client honours an artifact's `append` flag, which is how this runtime streams: one +`artifactId` for the reply, one frame per token, `append` on every frame after the +first, then a closing frame repeating the whole answer. That shape is pinned in `src/api/chat/a2aGrpcChatClient.test.ts` against frames captured from the controller on 2026-08-24. The mock chat client streams `delta` events — the port's vocabulary rather than the wire's — so no browser test exercises the artifact path. @@ -116,10 +132,11 @@ Teaching the fixture to emit artifact frames would make it an A2A server rather The gap is a live spec that sends a message and watches the reply grow before the turn completes. -**Open, but a product decision rather than coverage:** the `ask_user` payload still -renders as JSON in the transcript beside the answerable prompt. That is duplication -rather than a defect, and collapsing it needs a decision about whether a tool call with -an interactive rendering should show its raw form at all. +## Open, but a product decision rather than coverage + +The `ask_user` payload still renders as JSON in the transcript beside the answerable +prompt. That is duplication rather than a defect, and collapsing it needs a decision +about whether a tool call with an interactive rendering should show its raw form at all. ## Deferred: proving a share token actually travels @@ -269,22 +286,3 @@ template no harness admits says so. **What would close it:** a fixture scenario with a single harness. Worth doing when something else needs one; a scenario knob added for one assertion is a second fixture backend to keep honest. - -## A broken create takes that resource's failure states with it - -Each resource spec runs its empty, failure and retry states after the lifecycle, and a -journey is ordered — so a create that breaks aborts the three steps least likely to be -broken by the same change. `agent-templates` did exactly that during this port: step 10 -failed and steps 11 to 14 never ran. - -The README justifies the position by the fixture reset — reaching those states needs -`?mock=`, which is per-navigation and discards what the lifecycle created. That is a -reason they cannot sit in the *middle*; it does not choose an end, because the reload -starts a fresh backend whichever end they are at. - -Moving them first is not the fix either: they would then run against a pristine backend, -which is not the state they are about, and the lifecycle would start from one a -navigation had just reset. What actually removes the coupling is a second `test` in the -same file — one recording for the lifecycle, one for the states, neither able to abort -the other. That costs `conventions.test.ts` its "one spec, one test" rule, so it is an -amendment to the convention rather than a reshuffle, and belongs in its own change. From a04d005a1bf9bec440e84fbc1f3821778913a663 Mon Sep 17 00:00:00 2001 From: Torey Scheer Date: Tue, 22 Sep 2026 13:32:33 -0400 Subject: [PATCH 22/24] fix(ui): gate the last two reads on data, not on isLoading alone A sweep for the summary-flash pattern this branch already fixed on six pages found two more call sites it had missed. AgentPage's conversation count used !error && !isLoading, the same expression, so it can report "0 of 0 conversations" for a page that has not yet asked the controller anything. AppDetailPage is the worse of the two: it announces "No such app" on the same gate, which asserts an absence before the read that would contradict it - under a comment claiming absence is only meaningful once the read finished. Not measured. The 600ms flash was measured against a real backend, and the mock backend answers from the page's own memory, so neither the slow scenario nor a delayed route reproduces a loading window - a reverted control page does not flash either. These two are fixed for parity with the six and because the absence rule in playwright/README.md is unambiguous, not on fresh evidence. Also corrected a stale comment in AgentPage: it credited a note under the table with saying the search is client-side, and that note was removed some time ago. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Torey Scheer --- ui/src/pages/AgentPage.tsx | 14 +++++++++++--- ui/src/pages/AppDetailPage.tsx | 11 +++++++++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/ui/src/pages/AgentPage.tsx b/ui/src/pages/AgentPage.tsx index 4cb1eecf83..e8b78eec83 100644 --- a/ui/src/pages/AgentPage.tsx +++ b/ui/src/pages/AgentPage.tsx @@ -68,8 +68,10 @@ const PAGE_SIZE = 25; * resolving through the revision rather than through labels selects conversations * stored before the fields existed, with no migration and no backfill. * - * Search and sort are still the browser's, over whatever pages have been read, and - * the note under the table says so rather than implying otherwise. + * Search and sort are still the browser's. That is honest only because this client + * follows every page token before rendering, so what it searches is every + * conversation with this agent rather than the first page; `playwright/DEFERRED.md` + * records what has to change if the page-following ever goes. * * ## Somebody else's conversation is listed and cannot be opened * @@ -569,7 +571,13 @@ export function AgentPage() { }, ]} trailing={ - !conversations.error && !conversations.isLoading ? ( + /* `data !== undefined` as well as `!isLoading`: SWR runs its fetcher in an + effect, so the first paint reports "not loading" with nothing read yet, + and this count renders "0 of 0 conversations" for a page that has not + asked the controller anything. */ + !conversations.error && + !conversations.isLoading && + conversations.data !== undefined ? ( ) : null} - {/* Absence is only meaningful once the read finished and succeeded. */} - {appName && !servers.error && !servers.isLoading && matches.length === 0 ? ( + {/* Absence is only meaningful once the read finished and succeeded — and + `!isLoading` alone does not mean that. SWR runs its fetcher in an effect, + so the first paint reports "not loading" with nothing read, and this + announced "No such app" about a server list it had not yet asked for. */} + {appName && + !servers.error && + !servers.isLoading && + servers.data !== undefined && + matches.length === 0 ? ( Date: Tue, 22 Sep 2026 13:35:47 -0400 Subject: [PATCH 23/24] docs(ui): say which of the two API entries is a defect and which is not Both were listed as plainly open, which read as queued coverage work. Neither is: what each waits on is a proto change, and the behaviour as it stands is already pinned by agent-page.spec.ts. Auto-titling is a cost decision. Conversation search is a tripwire rather than a gap - the client follows every page token, so a browser-side search covers every conversation with that agent, and it stops being honest only when the page-following goes. Both now record what the proto actually says, checked there rather than here: AgentInstance gained name and context_id and carries nothing derived from the transcript, and ListAgentInstancesRequest is all_creators, page, agent_template and harness with no filter or sort field. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Torey Scheer --- ui/playwright/DEFERRED.md | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/ui/playwright/DEFERRED.md b/ui/playwright/DEFERRED.md index 34c718f6a2..aa4affaf74 100644 --- a/ui/playwright/DEFERRED.md +++ b/ui/playwright/DEFERRED.md @@ -209,7 +209,12 @@ pages, and none of their totals is `rows.length`. Counting what arrived and call total is the lie a separate summary read exists to prevent, which is what `GetSubstrateSummary` is for. -## Auto-titling costs a read per row, so the table still does not do it +## Not deferred coverage: auto-titling costs a read per row + +A cost decision with a server-side fix, rather than a spec somebody owes. What the +table renders today is pinned by `agents/agent-page.spec.ts` — that it is never a bare +UUID, and that the derived title appears where the transcript is in hand — so the +behaviour is covered; what is open is making a better behaviour possible. A conversation is named by the reader, and an unnamed one can be titled from its first message — `ListTasks{ContextID: instanceId}` returns the history. That is **free on the @@ -232,17 +237,22 @@ Two ways it could stop being a trade-off, both server-side and neither invented - **`ListAgentInstances` gains a field mask** for it, so callers that want it pay and callers that do not are unaffected. -Until then, what a list renders for an unnamed conversation is pinned by -`agents/agent-page.spec.ts` — both that it is never a bare UUID, and that the derived -title appears where the transcript is in hand. +Neither has landed, checked at the source rather than here: `AgentInstance` in +`proto/kagent/api/v1alpha1/agent_instances.proto` gained `name` (field 13, the +reader-supplied title) and `context_id`, and carries nothing derived from the +transcript. + +## Not a defect yet: an agent's conversation search is over what was fetched -## An agent's conversation search is over what was fetched, and the page-following is why +A tripwire rather than a gap, and the distinction is the whole entry: the search is +honest today and stops being honest on a change somebody will make for other reasons. `ListAgentInstances` narrows to one agent **on the server**: it takes `agent_template` and `harness` and resolves them through the prepared revision. That is the narrowing that matters, because it is the one the paging is applied after. What the request does -**not** carry is a search term or a sort field, so the agent page's search box and column -sorts run in the browser. +**not** carry is a search term or a sort field — `ListAgentInstancesRequest` is +`all_creators`, `page`, `agent_template` and `harness`, and nothing else — so the agent +page's search box and column sorts run in the browser. That is honest here for a reason worth stating, because it is the one read on the list above that is paged at all: the client follows every page token before rendering anything From 6c8873c4cfea7a522eefb08271bb511948131818 Mon Sep 17 00:00:00 2001 From: Torey Scheer Date: Tue, 22 Sep 2026 14:12:10 -0400 Subject: [PATCH 24/24] fix(ui): resolve the review on the live-suite branch The summary flash was diagnosed wrong. It is not SWR running its fetcher in an effect: useApiResource reports isLoading false for a held-back key, and the namespace-gated hooks joined [] to "" and held back forever - so the data gate this branch added would have hidden those summaries permanently on a cluster with no visible namespaces. Both hooks now key on the array, so a known-empty set resolves as the answer it is. The explanation lives in useApiResource, once, instead of pasted into seven pages with the wrong cause in it. Also from the review: - tests/agent-templates asserts the populated delete warning again - the count, what keeps working and what cannot be started. Only the empty branch was left, so the shared spec's claim that the mock suite pins the wording was false. - tests/models checks that editing does not demand the API key again. - tests/schedules ports the fixture claims dropped with the old steps: the row's derived cadence, both confirmation shapes and their Keep, and that a delete takes one row and leaves the others. Rename-from-list is not ported: there is no rename on this page. - The created flag is set before each submit, not after the redirect, so a slow redirect no longer fails the test and leaks the resource. - playwright.config matches by glob. Playwright matches a testMatch regex against the absolute path, so a checkout under a directory called tests/ or live/ put every spec in the wrong lane. Both lanes verified unchanged: 96 mock, 13 live. - conventions.test.ts catches a shared spec importing a named fixture, which is the half the scenario check missed and the half the README already claimed. - The live report uploads on any finished run: with two retries, a test that fails then passes left no trace. - live/substrate checks for a load failure at the end, not after a card the page draws before it has read anything, and shared/schedules waits for the list before asserting a row is gone. - The prompts abandoned-draft claim is dropped - a goto restarts the mock backend, so a draft that wrongly saved would be wiped and the test still pass. - DEFERRED.md cited tests/harnesses for a create it does not do. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Torey Scheer --- .github/workflows/ci.yaml | 6 +- ui/playwright.config.ts | 13 ++-- ui/playwright/DEFERRED.md | 3 +- ui/playwright/conventions.test.ts | 25 ++++++++ ui/playwright/helpers/cleanup.ts | 10 +-- ui/playwright/live/substrate.spec.ts | 11 +++- .../agent-templates/agent-templates.spec.ts | 6 +- .../shared/harnesses/harnesses.spec.ts | 6 +- ui/playwright/shared/models/models.spec.ts | 6 +- ui/playwright/shared/prompts/prompts.spec.ts | 6 +- .../shared/schedules/schedules.spec.ts | 12 ++++ .../agent-templates/agent-templates.spec.ts | 24 ++++++- ui/playwright/tests/models/models.spec.ts | 40 +++++++++++- ui/playwright/tests/prompts/prompts.spec.ts | 8 ++- .../tests/schedules/schedules.spec.ts | 64 ++++++++++++++++++- ui/src/api/hooks/useAgentBuildingBlocks.ts | 14 +++- ui/src/api/hooks/useApiResource.ts | 14 +++- ui/src/pages/AgentPage.tsx | 5 +- ui/src/pages/AgentTemplatesPage.tsx | 5 +- ui/src/pages/AgentsPage.tsx | 5 +- ui/src/pages/AppDetailPage.tsx | 8 +-- ui/src/pages/McpServersPage.tsx | 5 +- ui/src/pages/ModelsPage.tsx | 5 +- ui/src/pages/PromptsPage.tsx | 5 +- ui/src/pages/agents/HarnessesTab.tsx | 5 +- 25 files changed, 248 insertions(+), 63 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1ccab9d322..21c6aa5786 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -275,7 +275,11 @@ jobs: yarn test:pw:live - name: Upload live browser report - if: ${{ failure() && steps.install-kagent.outcome == 'success' }} + # Any finished run, not just a failed one: CI retries twice, so a spec that + # fails and then passes leaves a green job and no trace of the failure — which + # is the run whose trace is worth the most. A clean run uploads the HTML report + # and nothing else, since `test-results` only has content when something failed. + if: ${{ !cancelled() && steps.install-kagent.outcome == 'success' }} uses: actions/upload-artifact@v5 with: name: ui-live-playwright-report diff --git a/ui/playwright.config.ts b/ui/playwright.config.ts index a3958fc32c..e473ff34d1 100644 --- a/ui/playwright.config.ts +++ b/ui/playwright.config.ts @@ -34,8 +34,13 @@ const EXTENSION_SPECS = /\.withExtension\.spec\.ts$/; * narrowness is the point and the risk: assertions true of both are the weakest ones, * so this folder stays a smoke sweep rather than growing lifecycles. * `conventions.test.ts` fails a spec here that reaches for a scenario. + * + * A glob, not a regex: Playwright matches a `testMatch` regex against the *absolute* + * path, so `/shared\//` also matches every spec in a checkout that happens to live + * under a directory of that name. A glob resolves against `testDir`, which is the + * thing actually meant here. */ -const SHARED_SPECS = /shared\/.*\.spec\.ts$/; +const SHARED_SPECS = "shared/**/*.spec.ts"; /** * The suite is the acceptance bar, so what it runs against cannot depend on the @@ -222,7 +227,7 @@ export default defineConfig({ ? [ { name: LIVE_PROJECT, - testMatch: [/live\/.*\.spec\.ts$/, SHARED_SPECS], + testMatch: ["live/**/*.spec.ts", SHARED_SPECS], use: { ...devices["Desktop Chrome"], baseURL: LIVE_BASE_URL, @@ -236,7 +241,7 @@ export default defineConfig({ : [ { name: "chromium", - testMatch: [/tests\/.*\.spec\.ts$/, SHARED_SPECS], + testMatch: ["tests/**/*.spec.ts", SHARED_SPECS], testIgnore: EXTENSION_SPECS, use: { ...devices["Desktop Chrome"], baseURL: BASE_URL }, }, @@ -252,7 +257,7 @@ export default defineConfig({ // The extension split below is a build-time difference, not a browser one, // so it stays on one engine rather than doubling for no new signal. name: "firefox", - testMatch: [/tests\/.*\.spec\.ts$/, SHARED_SPECS], + testMatch: ["tests/**/*.spec.ts", SHARED_SPECS], testIgnore: EXTENSION_SPECS, use: { ...devices["Desktop Firefox"], baseURL: BASE_URL }, }, diff --git a/ui/playwright/DEFERRED.md b/ui/playwright/DEFERRED.md index aa4affaf74..2a2e95b26b 100644 --- a/ui/playwright/DEFERRED.md +++ b/ui/playwright/DEFERRED.md @@ -32,7 +32,8 @@ Nothing below is a gap. They are listed only so they are not looked for again. paired with a `Harness`, materialised by admission — `router/routes.ts` records why there is no `agentNew` and no `agentEdit`. Nothing creates one, so there is nothing to validate. The create-and-read-back property that lived in the removed agent-create - spec is now `tests/harnesses/harnesses.spec.ts`. + spec is now `shared/harnesses/` and `shared/agent-templates/`, which create against + either backend — `tests/harnesses/` only checks when the button is enabled. - **The REST path tests.** `src/api/{readPaths,writePaths}.test.ts` drove the client over REST URLs that no longer exist; the controller serves gRPC-Web. `src/api/operations.test.ts` replaces them against the real generated descriptors and covers strictly more. diff --git a/ui/playwright/conventions.test.ts b/ui/playwright/conventions.test.ts index 0ed4b54389..ab8ab0196d 100644 --- a/ui/playwright/conventions.test.ts +++ b/ui/playwright/conventions.test.ts @@ -2,6 +2,15 @@ import { readdirSync, readFileSync, statSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; +/** + * The exports of `helpers/app` that are seeded mock rows rather than helpers. + * + * A list because the distinction is not visible in the name: `routes` and `loadApp` + * hold on either backend, `instances` and `agents` are ids only the fixture backend + * has. + */ +const FIXTURE_EXPORTS = ["instances", "agents"]; + /** * The layout rules from `README.md`, checked. * @@ -173,6 +182,22 @@ describe("playwright layout", () => { /mock=|withScenario|scenario:|helpers\/mockCalls/.test(source), `${spec} drives the mock backend, so it cannot run live`, ).toBe(false); + + /* + * And the other half, which a scenario check misses entirely: a seeded row named + * by id. `instances.ready` is a UUID out of `src/mocks/fixtures.ts` — legal + * TypeScript, green in the mock lane, and a 404 against any cluster. The README + * has always said this folder may not name a fixture; until this ran, nothing + * checked the half that is not a query parameter. + */ + const imported = [...source.matchAll(/import\s*\{([^}]*)\}\s*from\s*"[^"]*helpers\/app"/g)] + .flatMap((match) => match[1].split(",")) + .map((binding) => binding.trim().split(/\s+as\s+/)[0].trim()); + const fixtures = imported.filter((binding) => FIXTURE_EXPORTS.includes(binding)); + expect( + fixtures, + `${spec} names seeded fixtures (${fixtures.join(", ")}), which no cluster has`, + ).toEqual([]); } }); diff --git a/ui/playwright/helpers/cleanup.ts b/ui/playwright/helpers/cleanup.ts index ccc07107a1..b5e5404ae5 100644 --- a/ui/playwright/helpers/cleanup.ts +++ b/ui/playwright/helpers/cleanup.ts @@ -4,13 +4,15 @@ import { expectListLoaded, loadApp, rowNamed, searchList } from "./app"; import { confirmDelete } from "./resource"; /** - * Removes a row this run made, from a `finally`, without ever throwing. + * Removes a row this run made, from a `test.afterEach`, without ever throwing. * * Every move a cleanup makes can fail — the navigation, the list read, the delete — and - * each of them throws. Thrown from a `finally`, that replaces the failure the test was + * each of them throws. Thrown from a cleanup, that replaces the failure the test was * actually reporting, *and* skips the delete underneath it: the run reports a timeout in - * the cleanup while the resource stays on the cluster. `appeared` was written for - * exactly this reason and the rest of these blocks were not. + * the cleanup while the resource stays on the cluster. + * + * A hook rather than a `finally`, which is the other half: a timed-out test has a closed + * page, so everything in its `finally` throws before it can delete anything. * * So it warns and returns. A cleanup that could not run says so in the output, and the * test still reports what it found. diff --git a/ui/playwright/live/substrate.spec.ts b/ui/playwright/live/substrate.spec.ts index 28bf146dd1..5cd822b3e1 100644 --- a/ui/playwright/live/substrate.spec.ts +++ b/ui/playwright/live/substrate.spec.ts @@ -25,9 +25,6 @@ test("live: the substrate page renders the cluster's own inventory", async ({ pa await expect(page.getByTestId("substrate-actors-card")).toBeVisible({ timeout: 60_000, }); - // After the card, not after `loadApp`: the alerts are counted once, and a page that - // has not read anything yet has none — see `expectNoLoadFailure`. - await expectNoLoadFailure(page); }); await test.step("2. the tiles report a real count, not zero", async () => { @@ -53,5 +50,13 @@ test("live: the substrate page renders the cluster's own inventory", async ({ pa await expect(workers.locator(".ant-table-row").first()).toBeVisible({ timeout: 60_000, }); + + /* + * Last, and not after the card in step 1: the card is a shell the page draws + * before it has read anything, so an alert counted then is counted before any read + * could have failed — and `expectNoLoadFailure` reads once rather than retrying, so + * it would never look again. Here, three reads have demonstrably landed. + */ + await expectNoLoadFailure(page); }); }); diff --git a/ui/playwright/shared/agent-templates/agent-templates.spec.ts b/ui/playwright/shared/agent-templates/agent-templates.spec.ts index ce99cd72db..900ba54a1e 100644 --- a/ui/playwright/shared/agent-templates/agent-templates.spec.ts +++ b/ui/playwright/shared/agent-templates/agent-templates.spec.ts @@ -165,13 +165,17 @@ test("agent templates: one is created, admitted, edited and deleted", async ({ await test.step("3. submitting reaches the controller and lands on the list", async () => { await expect(page.getByTestId("template-submit")).toBeEnabled(); + // Set before the submit, not after the redirect: a create the controller accepted + // but whose redirect was slow would otherwise fail the test with the flag still + // false, and the cleanup would skip a resource that really is on the cluster. The + // sweep looks for the row, so claiming one that was never made costs nothing. + created = true; await page.getByTestId("template-submit").click(); // Success is leaving the form. A create the controller refused keeps the reader on // it with `template-create-error` — which is the shape the defect this suite was // written for produced for a template that had in fact been created. await page.waitForURL(/\/agents\?.*tab=templates/, { timeout: READ_TIMEOUT }); - created = true; /* * And the list comes back narrowed to the namespace that was being worked in. * Nothing asserted this once, which is how two faults sat on the one line that diff --git a/ui/playwright/shared/harnesses/harnesses.spec.ts b/ui/playwright/shared/harnesses/harnesses.spec.ts index 98187ce8ce..87862ddd56 100644 --- a/ui/playwright/shared/harnesses/harnesses.spec.ts +++ b/ui/playwright/shared/harnesses/harnesses.spec.ts @@ -119,13 +119,17 @@ test("harnesses: a harness is created, read and deleted", async ({ page }) => { await test.step("2. a complete draft is created", async () => { await expect(page.getByTestId("harness-create")).toBeEnabled(); + // Set before the submit, not after the redirect: a create the controller accepted + // but whose redirect was slow would otherwise fail the test with the flag still + // false, and the cleanup would skip a resource that really is on the cluster. The + // sweep looks for the row, so claiming one that was never made costs nothing. + created = true; await page.getByTestId("harness-create").click(); // Back to the tab it came from, with the new harness in the list. Read back off // the table rather than from a toast: "the create returned" and "the thing // exists" are different claims, and only the list checks the second. await page.waitForURL(/tab=harnesses/, { timeout: READ_TIMEOUT }); - created = true; await expect(page.getByTestId(table)).toContainText(CREATED, { timeout: READ_TIMEOUT, }); diff --git a/ui/playwright/shared/models/models.spec.ts b/ui/playwright/shared/models/models.spec.ts index 90afe3696d..2be4778d42 100644 --- a/ui/playwright/shared/models/models.spec.ts +++ b/ui/playwright/shared/models/models.spec.ts @@ -97,9 +97,13 @@ test("models: a configuration is created, read, changed and deleted", async ({ await selectOption(page, "model-namespace", "kagent"); await page.getByTestId("model-api-key").fill("sk-not-a-real-key"); + // Set before the submit, not after the redirect: a create the controller accepted + // but whose redirect was slow would otherwise fail the test with the flag still + // false, and the cleanup would skip a resource that really is on the cluster. The + // sweep looks for the row, so claiming one that was never made costs nothing. + created = true; await page.getByTestId("model-submit").click(); await page.waitForURL(/\/models(\?|$)/, { timeout: READ_TIMEOUT }); - created = true; // Read back off the list rather than from a toast or a closed form: those two // only prove the app believes it worked. Narrowed to the one name this run diff --git a/ui/playwright/shared/prompts/prompts.spec.ts b/ui/playwright/shared/prompts/prompts.spec.ts index ecf0a1b39a..09331ce78d 100644 --- a/ui/playwright/shared/prompts/prompts.spec.ts +++ b/ui/playwright/shared/prompts/prompts.spec.ts @@ -76,9 +76,13 @@ test("prompts: a library is created, read, changed and deleted", async ({ page } await fragmentKey(page, 0).fill("changelog"); await fragmentValue(page, 0).fill("Group by user impact."); + // Set before the submit, not after the redirect: a create the controller accepted + // but whose redirect was slow would otherwise fail the test with the flag still + // false, and the cleanup would skip a resource that really is on the cluster. The + // sweep looks for the row, so claiming one that was never made costs nothing. + created = true; await page.getByTestId("prompt-submit").click(); await expect(page).toHaveURL(/\/prompts$/, { timeout: READ_TIMEOUT }); - created = true; // Read back off the list rather than from a toast or a closed form: those two // only prove the app believes it worked. Narrowed to the one name this run diff --git a/ui/playwright/shared/schedules/schedules.spec.ts b/ui/playwright/shared/schedules/schedules.spec.ts index 2b9ea085e3..0e0233661d 100644 --- a/ui/playwright/shared/schedules/schedules.spec.ts +++ b/ui/playwright/shared/schedules/schedules.spec.ts @@ -199,6 +199,18 @@ test("schedules: one is created, read, changed and deleted", async ({ page }) => ); detailURL = undefined; + /* + * Wait for the list to draw before asserting the row is gone. Zero rows is also + * what a list that has not rendered yet looks like, so without this a delete the + * controller refused still passes — either the empty state or a first row, then + * the absence. + */ + await expect( + page + .getByTestId("schedules-empty") + .or(page.locator('[data-testid="schedules-table"] tbody tr.ant-table-row')) + .first(), + ).toBeVisible({ timeout: READ_TIMEOUT }); await expect(page.getByRole("link", { name: CREATED, exact: true })).toHaveCount(0, { timeout: READ_TIMEOUT, }); diff --git a/ui/playwright/tests/agent-templates/agent-templates.spec.ts b/ui/playwright/tests/agent-templates/agent-templates.spec.ts index 0bd3094ff3..6dc3f417c7 100644 --- a/ui/playwright/tests/agent-templates/agent-templates.spec.ts +++ b/ui/playwright/tests/agent-templates/agent-templates.spec.ts @@ -285,15 +285,37 @@ test("agent templates: the list reads, and a template is read and edited", async ); }); - await test.step("11. a template nothing runs says that instead", async () => { + await test.step("11. deleting counts what is built from the template, both ways", async () => { /* * Back to the list first. The delete that used to sit here navigated back as a side * effect of removing the template, and it moved to `shared/agent-templates/` — so * the return trip is now this step's own business rather than something it inherits. + * + * Both branches are asserted here because only the fixtures can hold both at once: + * the shared spec deletes a template it created seconds earlier, which can only ever + * be the "nothing is built from it" branch, and it says so by matching either + * sentence. The seeded wording is this suite's to pin. */ await page.getByRole("button", { name: "Back to templates" }).click(); await page.waitForURL(/\/agents\?.*tab=templates/); + // A template an agent *is* built from: the count, and the two things that follow + // from it — what survives the delete and what cannot be started after it. This is + // the branch a reader is most likely to be reading before they decide. + await page.getByTestId("templates-filters-search").fill("k8s-agent-7f3a91c"); + await page.getByTestId("template-link-k8s-agent-7f3a91c").click(); + await page.waitForURL(/\/agent-templates\/kagent\/k8s-agent-7f3a91c/); + + await page.getByTestId("delete-k8s-agent-7f3a91c").click(); + const populated = page.getByTestId("template-delete-consequence"); + await expect(populated).toContainText("1 agent is built from this template"); + await expect(populated).toContainText("keep working"); + await expect(populated).toContainText("no new one can be started"); + + await page.keyboard.press("Escape"); + await page.getByRole("button", { name: "Back to templates" }).click(); + await page.waitForURL(/\/agents\?.*tab=templates/); + // The other branch of the same sentence. Telling a reader that conversations will // keep working when no harness ever admitted the template would be noise dressed as // care. diff --git a/ui/playwright/tests/models/models.spec.ts b/ui/playwright/tests/models/models.spec.ts index 2944b9d6a2..9db93fdf0a 100644 --- a/ui/playwright/tests/models/models.spec.ts +++ b/ui/playwright/tests/models/models.spec.ts @@ -157,13 +157,47 @@ test("models: the list reads, narrows, and says what the form requires", async ( }); }); - await test.step("6. an empty result says so instead of showing a bare table", async () => { + await test.step("6. editing does not ask for the API key again", async () => { + /* + * The other half of step 5, and the half a reader notices: a stored key is write-only + * — the controller never sends it back — so an edit form that marked the field + * required would demand the secret again to change a display name, and there would be + * nowhere to read it from. + * + * The label carries the promise ("leave blank to keep existing") and `expectRequired` + * checks that the mark agrees with it. Asserted as a pair for the same reason step 5 + * is: a form that marked nothing would pass a check that only looked at the unmarked + * list. + */ + await loadPage(page, routes.models, { title: "Models" }); + await page.getByTestId("edit-default-model-config").click(); + await page.waitForURL(/\/models\/kagent\/default-model-config\/edit$/); + await expectSettled(page); + + // Every seeded configuration authenticates by secret reference, so the inline field + // has to be asked for. Switching to it is also the case that matters: the reader is + // replacing how this model authenticates, and still should not have to retype a key + // to do it. + // The label, not the input: antd's button-style radio hides the input itself, so a + // click on the role never lands. + await page + .getByTestId("model-auth-type") + .getByText("API key", { exact: true }) + .click(); + + await expectRequired(page, { + marked: ["Provider", "Model", "Name", "Namespace"], + unmarked: ["Authentication", "API key (leave blank to keep existing)"], + }); + }); + + await test.step("7. an empty result says so instead of showing a bare table", async () => { await loadPage(page, routes.models, { scenario: "empty", title: "Models" }); await expect(page.getByText("No model configurations yet.")).toBeVisible(); await expect(dataRows(page)).toHaveCount(0); }); - await test.step("7. a failed load is reported, not disguised as an empty list", async () => { + await test.step("8. a failed load is reported, not disguised as an empty list", async () => { await loadPage(page, routes.models, { scenario: "error", title: "Models" }); const alert = page.getByTestId("models-error"); @@ -182,7 +216,7 @@ test("models: the list reads, narrows, and says what the form requires", async ( await expect(dataRows(page)).toHaveCount(0); }); - await test.step("8. retrying asks the backend again, and it recovers", async () => { + await test.step("9. retrying asks the backend again, and it recovers", async () => { const before = await operationCalls(page, rpc.listModelConfigs); await page.getByRole("button", { name: "Try again" }).click(); diff --git a/ui/playwright/tests/prompts/prompts.spec.ts b/ui/playwright/tests/prompts/prompts.spec.ts index beeb5af8da..98d5aa54da 100644 --- a/ui/playwright/tests/prompts/prompts.spec.ts +++ b/ui/playwright/tests/prompts/prompts.spec.ts @@ -186,9 +186,11 @@ test("prompts: libraries are read, edited, and their failures reported", async ( * driven twice, and the count it used to assert is stronger there, where it is * relative to whatever the list already held. * - * Navigated rather than submitted, so the steps below start on the list with a draft - * abandoned — which is also the state a reader leaves behind when they change their - * mind, and nothing else covers it. + * Navigated rather than submitted, so the steps below start on the list. + * + * Not coverage of an abandoned draft, though it looks like it: `loadPage` is a + * `page.goto`, which restarts the module-level mock backend, so a draft that had + * wrongly saved itself would be wiped by the navigation and this would still pass. */ await loadPage(page, routes.prompts, { title: "Prompts" }); await expect(dataRows(page)).toHaveCount(SEEDED.length); diff --git a/ui/playwright/tests/schedules/schedules.spec.ts b/ui/playwright/tests/schedules/schedules.spec.ts index 6904fed1fd..bfdf163fed 100644 --- a/ui/playwright/tests/schedules/schedules.spec.ts +++ b/ui/playwright/tests/schedules/schedules.spec.ts @@ -1,5 +1,10 @@ import { test, expect } from "../../fixtures/test"; -import { LIFECYCLE_TIMEOUT, optionNamed } from "../../helpers/resource"; +import { + LIFECYCLE_TIMEOUT, + confirmation, + optionNamed, + pressOnce, +} from "../../helpers/resource"; /** * Schedules — reading one, running it, and the states around that. @@ -52,6 +57,16 @@ test("schedules: a schedule is read, run, paused, and its failures reported", as // No pagination over a list this size: a control that pages nothing is a control // that implies there is more to see. await expect(page.getByTestId("schedules-pages")).toHaveCount(0); + + /* + * The row reads the cron rather than printing it. `0 9 * * *` is a field the + * controller stores and not something to put in front of a reader, and the column + * is the only place the derived reading is shown — `scheduleTiming.test.ts` covers + * every shape the reading takes, including the weekly ones no fixture here has. + */ + const row = rowNamed("Daily cluster report"); + await expect(row).toContainText("Every day at 09:00"); + await expect(row).not.toContainText("* * *"); }); await test.step("2. a row opens its schedule, and its buttons still do their own job", async () => { @@ -260,7 +275,50 @@ test("schedules: a schedule is read, run, paused, and its failures reported", as expect(overflows).toBe(false); }); - await test.step("13. a link held from before a delete still opens, and says what it is", async () => { + await test.step("13. delete asks twice over, and Keep leaves the row where it was", async () => { + await page.goto("/schedules?mock=ok"); + + /* + * The list confirms in a popconfirm and the detail page in a modal — two shapes, + * one copy, and the cancel on each reads "Keep" rather than "Cancel" because the + * reader is choosing between two outcomes rather than dismissing a dialog. + * + * Both paths matter: the sentence is what a reader decides on, and it is the part + * that would go stale silently if only one of the two were driven. + */ + await page.getByTestId("delete-Schedule 3").click(); + const popconfirm = confirmation(page); + await expect(popconfirm).toContainText("Stops future executions."); + await expect(popconfirm).toContainText("history and conversations are retained"); + await pressOnce(popconfirm.getByRole("button", { name: "Keep", exact: true })); + await expect(page.getByTestId("schedule-link-Schedule 3")).toBeVisible(); + + // And the same on the detail page's own delete, which is a modal. + await page.getByTestId("schedule-link-Daily cluster report").click(); + await page.waitForURL(new RegExp(`/schedules/${SEEDED}$`)); + await page.getByTestId("delete-Daily cluster report").click(); + const modal = page.getByRole("dialog"); + await expect(modal).toContainText("Stops future executions."); + await pressOnce(modal.getByRole("button", { name: "Keep", exact: true })); + await expect(page).toHaveURL(new RegExp(`/schedules/${SEEDED}$`)); + }); + + await test.step("14. a delete takes one row and leaves the others", async () => { + // The claim a delete test usually forgets: that it removed the row it was asked + // for and not the list. Only the seeded fixtures can say this, since a live journey + // deletes the one thing it made and has nothing else of its own to count. + await page.goto("/schedules?mock=ok"); + await page.getByTestId("delete-Schedule 3").click(); + await pressOnce( + confirmation(page).getByRole("button", { name: "Delete", exact: true }), + ); + + await expect(page.getByTestId("schedule-link-Schedule 3")).toHaveCount(0); + await expect(page.getByTestId("schedule-link-Daily cluster report")).toBeVisible(); + await expect(page.getByTestId("schedule-link-Schedule 2")).toBeVisible(); + }); + + await test.step("15. a link held from before a delete still opens, and says what it is", async () => { // The executions are retained, so the address is not a 404 — and must not render as // a live schedule either, or a reader will try to act on one that is gone. await page.goto(`/schedules/${RETIRED}?mock=ok`); @@ -274,7 +332,7 @@ test("schedules: a schedule is read, run, paused, and its failures reported", as await expect(page.getByTestId("schedule-meta")).toContainText("Deleted"); }); - await test.step("14. and it is not offered in the list it was removed from", async () => { + await test.step("16. and it is not offered in the list it was removed from", async () => { await page.goto("/schedules?mock=ok"); await expect( page.getByTestId("schedule-link-Retired sweep"), diff --git a/ui/src/api/hooks/useAgentBuildingBlocks.ts b/ui/src/api/hooks/useAgentBuildingBlocks.ts index f1157cf0a6..a63c5f841d 100644 --- a/ui/src/api/hooks/useAgentBuildingBlocks.ts +++ b/ui/src/api/hooks/useAgentBuildingBlocks.ts @@ -48,9 +48,17 @@ export function useAgentTemplates(namespace?: string): ApiResource { + /* + * Keyed on `namespaces` rather than on the joined string, because `[]` joins to + * `""` — and holding the read back for that conflates "we do not know the + * namespaces yet" with "we know, and there are none". The second is an answer: it + * resolves to an empty list, so a page can say "0 of 0" and mean it. Held back, the + * key stays null, `isLoading` is reported false (idle, not loading) and `data` never + * arrives, so a summary gated on `data` would never render at all. + */ const key = namespaces ? [...namespaces].sort().join(",") : undefined; - return useApiResource(key ? ["harnesses.listAll", key] : null, async () => { + return useApiResource(namespaces ? ["harnesses.listAll", key] : null, async () => { const names = key ? key.split(",").filter(Boolean) : []; const settled = await Promise.allSettled( names.map((namespace) => apiClient.agentBuildingBlocks.harnesses(namespace)), @@ -105,10 +113,12 @@ export function useAgentTemplatesAcrossNamespaces( ): ApiResource { // Sorted into the key, so the same set in a different order is the same read rather // than a cache miss that refetches everything. + // `namespaces`, not `key`: see `useHarnessesAcrossNamespaces` — `[]` joins to `""`, + // and a known-empty set is an answer rather than a reason to hold the read back. const key = namespaces ? [...namespaces].sort().join(",") : undefined; return useApiResource( - key ? ["agentTemplates.listAll", key] : null, + namespaces ? ["agentTemplates.listAll", key] : null, async () => { const names = key ? key.split(",").filter(Boolean) : []; const settled = await Promise.allSettled( diff --git a/ui/src/api/hooks/useApiResource.ts b/ui/src/api/hooks/useApiResource.ts index d0aa2052bb..4e1b26c4fc 100644 --- a/ui/src/api/hooks/useApiResource.ts +++ b/ui/src/api/hooks/useApiResource.ts @@ -42,8 +42,18 @@ export function useApiResource( return { data, - // A held-back request (`key === null`) is idle, not loading — otherwise a - // page waiting on a route param renders a spinner that never resolves. + /* + * A held-back request (`key === null`) is idle, not loading — otherwise a page + * waiting on a route param renders a spinner that never resolves. + * + * The consequence every caller has to handle: while a read is held back this + * reports `isLoading: false` with `data` undefined, so `!error && !isLoading` is + * true for a page that has asked nothing. A count rendered on that gate claims + * "0 of 0" about a backend it has not called. Gate a summary on `data !== undefined` + * as well — and if the key is held back on another read's result, make sure the + * empty case resolves rather than holding back forever, or the summary never + * renders at all. + */ isLoading: key !== null && isLoading, isValidating, error: error ?? undefined, diff --git a/ui/src/pages/AgentPage.tsx b/ui/src/pages/AgentPage.tsx index e8b78eec83..50a58faf14 100644 --- a/ui/src/pages/AgentPage.tsx +++ b/ui/src/pages/AgentPage.tsx @@ -571,10 +571,7 @@ export function AgentPage() { }, ]} trailing={ - /* `data !== undefined` as well as `!isLoading`: SWR runs its fetcher in an - effect, so the first paint reports "not loading" with nothing read yet, - and this count renders "0 of 0 conversations" for a page that has not - asked the controller anything. */ + /* `data !== undefined`: an idle read reports `isLoading: false` with nothing in it — see `useApiResource`. */ !conversations.error && !conversations.isLoading && conversations.data !== undefined ? ( diff --git a/ui/src/pages/AgentTemplatesPage.tsx b/ui/src/pages/AgentTemplatesPage.tsx index b25238a31b..2e6da86969 100644 --- a/ui/src/pages/AgentTemplatesPage.tsx +++ b/ui/src/pages/AgentTemplatesPage.tsx @@ -280,10 +280,7 @@ export function AgentTemplatesTab() { {/* Only a successful load can be counted: "0 of 0" because a request failed would be a claim the page cannot support. */} - {/* `data !== undefined` as well as `!isLoading`: SWR runs its fetcher in an effect, so - the first paint reports "not loading" on a page that has not asked yet, and this - counted "0 of 0" for the 600ms before the answer arrived. A count of nothing is a - claim, and until the read lands this page has not earned it. */} + {/* `data !== undefined`: an idle read reports `isLoading: false` with nothing in it — see `useApiResource`. */} {!loadFailure && !templates.isLoading && templates.data !== undefined ? ( {filtered.length} of {rows.length}{" "} diff --git a/ui/src/pages/AgentsPage.tsx b/ui/src/pages/AgentsPage.tsx index 962ad65d7e..9f827569cb 100644 --- a/ui/src/pages/AgentsPage.tsx +++ b/ui/src/pages/AgentsPage.tsx @@ -459,10 +459,7 @@ export function AgentsTab() { ]} trailing={ - {/* `data !== undefined` as well as `!isLoading`: SWR runs its fetcher in an effect, so - the first paint reports "not loading" on a page that has not asked yet, and this - counted "0 of 0" for the 600ms before the answer arrived. A count of nothing is a - claim, and until the read lands this page has not earned it. */} + {/* `data !== undefined`: an idle read reports `isLoading: false` with nothing in it — see `useApiResource`. */} {!loadFailure && !templates.isLoading && templates.data !== undefined ? ( {filtered.length} of {agents.length}{" "} diff --git a/ui/src/pages/AppDetailPage.tsx b/ui/src/pages/AppDetailPage.tsx index 27b232e002..d085a8a4e6 100644 --- a/ui/src/pages/AppDetailPage.tsx +++ b/ui/src/pages/AppDetailPage.tsx @@ -120,10 +120,10 @@ export function AppDetailPage() { /> ) : null} - {/* Absence is only meaningful once the read finished and succeeded — and - `!isLoading` alone does not mean that. SWR runs its fetcher in an effect, - so the first paint reports "not loading" with nothing read, and this - announced "No such app" about a server list it had not yet asked for. */} + {/* Absence is only meaningful once the read finished and succeeded, and + `!isLoading` alone does not mean that: an idle read reports it false with + nothing in it, so this announced "No such app" about a list it had not + asked for. See `useApiResource`. */} {appName && !servers.error && !servers.isLoading && diff --git a/ui/src/pages/McpServersPage.tsx b/ui/src/pages/McpServersPage.tsx index 480c2dec66..79247224db 100644 --- a/ui/src/pages/McpServersPage.tsx +++ b/ui/src/pages/McpServersPage.tsx @@ -308,10 +308,7 @@ export function McpServersPage() { trailing={ /* Only a successful load can be counted. Saying "0 servers" because a request failed would be a claim the page cannot support. */ - /* `data !== undefined` as well as `!isLoading`: SWR runs its fetcher in an effect, so - the first paint reports "not loading" on a page that has not asked yet, and this - counted "0 of 0" for the 600ms before the answer arrived. A count of nothing is a - claim, and until the read lands this page has not earned it. */ + /* `data !== undefined`: an idle read reports `isLoading: false` with nothing in it — see `useApiResource`. */ !error && !isLoading && data !== undefined ? ( {filtered.length} of {models.length}{" "} diff --git a/ui/src/pages/PromptsPage.tsx b/ui/src/pages/PromptsPage.tsx index d94aa4f015..be14963bc9 100644 --- a/ui/src/pages/PromptsPage.tsx +++ b/ui/src/pages/PromptsPage.tsx @@ -231,10 +231,7 @@ export function PromptsPage() { }, ]} trailing={ - /* `data !== undefined` as well as `!isLoading`: SWR runs its fetcher in an effect, so - the first paint reports "not loading" on a page that has not asked yet, and this - counted "0 of 0" for the 600ms before the answer arrived. A count of nothing is a - claim, and until the read lands this page has not earned it. */ + /* `data !== undefined`: an idle read reports `isLoading: false` with nothing in it — see `useApiResource`. */ !error && !isLoading && data !== undefined ? ( {filtered.length} of {libraries.length}{" "} diff --git a/ui/src/pages/agents/HarnessesTab.tsx b/ui/src/pages/agents/HarnessesTab.tsx index 8a36e97ec0..a50501eb5b 100644 --- a/ui/src/pages/agents/HarnessesTab.tsx +++ b/ui/src/pages/agents/HarnessesTab.tsx @@ -243,10 +243,7 @@ export function HarnessesTab() { ]} trailing={ - {/* `data !== undefined` as well as `!isLoading`: SWR runs its fetcher in an effect, so - the first paint reports "not loading" on a page that has not asked yet, and this - counted "0 of 0" for the 600ms before the answer arrived. A count of nothing is a - claim, and until the read lands this page has not earned it. */} + {/* `data !== undefined`: an idle read reports `isLoading: false` with nothing in it — see `useApiResource`. */} {!harnesses.error && !harnesses.isLoading && harnesses.data !== undefined ? ( {filtered.length} of {rows.length}{" "}