Bound WASM Worker startup separately from the policy deadline (#727)

.github/workflows/wasm-shared.yml

@@ -95,7 +95,9 @@ jobs:
     if: ${{ github.event_name != 'workflow_dispatch' || inputs.mode == 'qualify' }}
     name: Shared WASM source, model, package, and browser qualification
     runs-on: ubuntu-latest
-    timeout-minutes: 35
+    # Three browser engines add a system-package download whose hosted apt mirror
+    # throughput varies widely (41 s to 16 min observed); keep visible headroom.
+    timeout-minutes: 55
     env:
       QUINT_VERSION: 0.32.0
       RUST_VERSION: 1.90.0
@@ -176,12 +178,18 @@ jobs:
           set -euo pipefail
           scripts/wasm-release/qualify-supervisor-custody.sh --compile-installed artifacts/wasm-release "$RUNNER_TEMP/wasm-supervisor-evidence"
 
-      - name: Install pinned Playwright Chromium
+      - name: Install pinned Playwright browsers
         working-directory: tests/Wasm.Browser.Conformance
+        timeout-minutes: 20
         run: |
           npm ci
           test "$(node -p "require('@playwright/test/package.json').version")" = "$PLAYWRIGHT_VERSION"
-          npx playwright install --with-deps chromium
+          # One bounded retry for a transient mirror/download failure.
+          for attempt in 1 2; do
+            if timeout 540 npx playwright install --with-deps chromium firefox webkit; then break; fi
+            test "$attempt" -lt 2 || exit 1
+            echo "browser install attempt $attempt failed; retrying once" >&2
+          done
 
       - name: Rebuild pinned WebAssembly fixtures
         run: |
@@ -208,10 +216,21 @@ jobs:
           tests/Wasm.Browser.Conformance/run.sh 2>&1 | tee "$RUNNER_TEMP/wasm-custody-logs/browser.log"
 
       - name: Verify SDK archive and fresh package consumer
+        env:
+          WASM_COLD_LOAD_REPORT_DIR: ${{ runner.temp }}/wasm-cold-load
         run: |
           set -euo pipefail
           scripts/verify-wasm-package-consumer.sh --custody artifacts/wasm-release 2>&1 | tee "$RUNNER_TEMP/wasm-custody-logs/package-consumer.log"
 
+      - name: Retain cold-load browser timing evidence
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
+        with:
+          name: wasm-cold-load-${{ github.run_id }}-${{ github.run_attempt }}
+          retention-days: 30
+          if-no-files-found: warn
+          path: ${{ runner.temp }}/wasm-cold-load/
+
       - name: Qualify compatible supervisor source and installed custody
         run: |
           set -euo pipefail

docs/atlas/catalog.json

@@ -1 +1 @@
-{"records":["049f1d0d8155f65b6f7675ebdb200d4595570cc7d585dd74876e0285eceffed6","07ee12b5da1313c1b494217324fb4f4301c8f3ed2de82948b62fa04dfe9302ce","0863093e69a16355fa9b90f8e7e29156ea4ad88c4523bc4883568bde831bdf35","0b92e4db5e34990113bb43cb8cd58cf251d06083cf812398d6da9fdebb98e0f5","2056fa8b653c2240afdad2a16f572a07019109254847829610a9468bb2f95248","231df3792931ad2333d00970a5993b3e13fa9dc38f687899f9fa2c966ed9fbfa","24fdc3980640ba58bedcae6f16d5b467eb7383c31161fb9b6ff10fd2e667b0e5","264a2f7887a9b6473cbc24102e1bf9be70e57f6d7ba224138140efa094d541f4","2b7dfb3a2f76368cb603161cdb6f8fa8c2b8afc3e96659dd37c975921ba5ba66","33df463263533529cca24a0b09937573f836d197cc1a739e9960b1cdd4deb1fd","343077d02ddd266b21d8677f7c256854134f7e3e2d18137b55ebe3f32564c5c0","34391bda3fb56135a2bddf74ab790958c175bc6a69c38175e6f8db792cfd50a0","36a19fefd7024f54f3391ffed19019c10222a514719edd7a44f0cde2a1c48258","3d3a0a302c02cd0bfee86715d09b0211baa8dad3e47402c638fc95c7cfa02e87","3de8718fb3ca9b15212e8bd84fba205995d24636087090a61c63c28a96a745b1","3df4f31fda99ad137c651322ad29809e36380400516399274870b4a4332b4a92","3f09ee199e3c2e808e0f6e74f197f2fdd8158ae099a7195c498ce9466bb78b03","44bda82d6f9a9625a83283bf56a15a1831f287037214dc134eeeae87f1faa869","5466c617291eff27f59b569a2d42a18dc0c134d4935d52d802c5a71b8a594281","6c5b38ee260d1f15f8029d699ec1cebceafb5e1407a6341dc7a4b96f4d9d3521","71bc29a175940d22dd1d85148ee49f47aed714395fbd51d07b8751147815a55f","72742cbaa561237f10251e94aa5c61153ab5c7f14fc5771b7a5ebdece1e1cb11","7eba935f363db90d4b6d3fea2d226a7a58b08b88cf748e619ddb407ea83911e9","7f62ef13b4b46b4f23cae9536c977c15c85320a253ed2bc609eb7a1f0afdcf56","85f06991da74fa4e59ba73c7034dc53a35a4e90c5b64d83cc8956d3afd61d376","87d6de6f3730cecd75d9a4411bf4dbac4e058127a3ce9f46b07f6de110bc26de","8b1cf51c5f03c0ae19c46ca3ea4a1158acbb6e47bf109ca8c506c5417b51cf9f","8c514796f190fe6d4307be57983c84f92208f5d2f00fcbe3093b72618f2d7bc1","95115e1052b5ea4fcd393f5c9b9c6f0298657f0028eee84e16215ede97576cfa","9b6c05923a43ea37b10504396105d97ba51c0430e82fb3bb4953ee4256cdc1aa","a38737476856236f33e0aef1b97e2b900677cf99f1dbcfd381c708ceeca8b7f3","ab4502eff76606ac8e58b623878f3d76e1e0b18f30796aec7c446e2aab8b57c8","ab69ebe00eac40c68aec6131ef0559967b399266aae89f9673d3c30ce12f36b0","be492023350541da5256d6f653711b5f736d5216cf121a7e02ca43e1887215e4","c01c2c98f13a65b49113916a2f295e46a03fc4c185ece880267dd86c3f3dee8b","d557525e3ebfad227c8d6db9bb6350d42506fb02b1de8058cb443f4f008c0f67","d9718eb12e9ae9ea3a29a9904f131f53e99c584ef1b297aa05f1f8f6cdc6ab2a","fa8cd29fe962d7e853457e829c2d5927685f0ff374e6450fff5529f785db442d"],"schema":"fsgg.atlas/1"}
+{"records":["049f1d0d8155f65b6f7675ebdb200d4595570cc7d585dd74876e0285eceffed6","07ee12b5da1313c1b494217324fb4f4301c8f3ed2de82948b62fa04dfe9302ce","0863093e69a16355fa9b90f8e7e29156ea4ad88c4523bc4883568bde831bdf35","0b92e4db5e34990113bb43cb8cd58cf251d06083cf812398d6da9fdebb98e0f5","2056fa8b653c2240afdad2a16f572a07019109254847829610a9468bb2f95248","231df3792931ad2333d00970a5993b3e13fa9dc38f687899f9fa2c966ed9fbfa","24fdc3980640ba58bedcae6f16d5b467eb7383c31161fb9b6ff10fd2e667b0e5","264a2f7887a9b6473cbc24102e1bf9be70e57f6d7ba224138140efa094d541f4","2b7dfb3a2f76368cb603161cdb6f8fa8c2b8afc3e96659dd37c975921ba5ba66","33df463263533529cca24a0b09937573f836d197cc1a739e9960b1cdd4deb1fd","343077d02ddd266b21d8677f7c256854134f7e3e2d18137b55ebe3f32564c5c0","34391bda3fb56135a2bddf74ab790958c175bc6a69c38175e6f8db792cfd50a0","36a19fefd7024f54f3391ffed19019c10222a514719edd7a44f0cde2a1c48258","3d3a0a302c02cd0bfee86715d09b0211baa8dad3e47402c638fc95c7cfa02e87","3de8718fb3ca9b15212e8bd84fba205995d24636087090a61c63c28a96a745b1","3df4f31fda99ad137c651322ad29809e36380400516399274870b4a4332b4a92","3f09ee199e3c2e808e0f6e74f197f2fdd8158ae099a7195c498ce9466bb78b03","44bda82d6f9a9625a83283bf56a15a1831f287037214dc134eeeae87f1faa869","5466c617291eff27f59b569a2d42a18dc0c134d4935d52d802c5a71b8a594281","6c5b38ee260d1f15f8029d699ec1cebceafb5e1407a6341dc7a4b96f4d9d3521","71bc29a175940d22dd1d85148ee49f47aed714395fbd51d07b8751147815a55f","72742cbaa561237f10251e94aa5c61153ab5c7f14fc5771b7a5ebdece1e1cb11","7eba935f363db90d4b6d3fea2d226a7a58b08b88cf748e619ddb407ea83911e9","7f62ef13b4b46b4f23cae9536c977c15c85320a253ed2bc609eb7a1f0afdcf56","85f06991da74fa4e59ba73c7034dc53a35a4e90c5b64d83cc8956d3afd61d376","87d6de6f3730cecd75d9a4411bf4dbac4e058127a3ce9f46b07f6de110bc26de","8b1cf51c5f03c0ae19c46ca3ea4a1158acbb6e47bf109ca8c506c5417b51cf9f","8c514796f190fe6d4307be57983c84f92208f5d2f00fcbe3093b72618f2d7bc1","95115e1052b5ea4fcd393f5c9b9c6f0298657f0028eee84e16215ede97576cfa","9b6c05923a43ea37b10504396105d97ba51c0430e82fb3bb4953ee4256cdc1aa","a38737476856236f33e0aef1b97e2b900677cf99f1dbcfd381c708ceeca8b7f3","ab4502eff76606ac8e58b623878f3d76e1e0b18f30796aec7c446e2aab8b57c8","ab69ebe00eac40c68aec6131ef0559967b399266aae89f9673d3c30ce12f36b0","b091d93823405b0556d71f65027d5256170a3462bd33c0c80cfeecd278468104","be492023350541da5256d6f653711b5f736d5216cf121a7e02ca43e1887215e4","c01c2c98f13a65b49113916a2f295e46a03fc4c185ece880267dd86c3f3dee8b","d557525e3ebfad227c8d6db9bb6350d42506fb02b1de8058cb443f4f008c0f67","d9718eb12e9ae9ea3a29a9904f131f53e99c584ef1b297aa05f1f8f6cdc6ab2a","fa8cd29fe962d7e853457e829c2d5927685f0ff374e6450fff5529f785db442d"],"schema":"fsgg.atlas/1"}

docs/atlas/records/b091d93823405b0556d71f65027d5256170a3462bd33c0c80cfeecd278468104.json

@@ -0,0 +1 @@
+{"citations":[{"path":"src/Wasm.Browser/Lifecycle.fs","repository":"FS-GG/FS.GG.Game","revision":"7878a467b548f4701d1e1468e1d41f6289a38692","sha256":"d6d0086817727ea29b1629d3591445b61829b9c809696fbb40e4439b77bae9e0"},{"path":"src/Wasm.Browser/Host.fs","repository":"FS-GG/FS.GG.Game","revision":"7878a467b548f4701d1e1468e1d41f6289a38692","sha256":"ac936241d188418b7e30161b8acb61b69886f102ac38035364402e25ea63d88e"},{"path":"src/Wasm.Browser/RuntimeProtocol.fsi","repository":"FS-GG/FS.GG.Game","revision":"7878a467b548f4701d1e1468e1d41f6289a38692","sha256":"a0b5e3a4ed6e29baded5ca23d1ee45a05c45ed52882204ef25c6efc133fb1eb8"},{"path":"eng/wasm-shared/lifecycle.qnt","repository":"FS-GG/FS.GG.Game","revision":"7878a467b548f4701d1e1468e1d41f6289a38692","sha256":"2b9b891b4273f9b764a104d3d038d213fc19947682d11752920c717254c9e45c"},{"path":"eng/wasm-shared/event-boundary-qualification.qnt","repository":"FS-GG/FS.GG.Game","revision":"7878a467b548f4701d1e1468e1d41f6289a38692","sha256":"9c5201181102be946b36d8d1b9747ede7ee91804ef25a9253fc9371e34c892ce"},{"path":"tests/Wasm.Lifecycle.Tests/LifecycleTests.fs","repository":"FS-GG/FS.GG.Game","revision":"7878a467b548f4701d1e1468e1d41f6289a38692","sha256":"eca173449e9c781b5a8d8a439811c277dd7bdc918c6cff0ad7a67fb49941fa38"},{"path":"tests/Wasm.PackageConsumer/browser/cold-load.spec.mjs","repository":"FS-GG/FS.GG.Game","revision":"7878a467b548f4701d1e1468e1d41f6289a38692","sha256":"8d1d46d03ca903152567a8af191423d64594a961eed06cb1c7ed42d4843d0722"},{"path":"tests/Wasm.PackageConsumer/browser/package-consumer.spec.mjs","repository":"FS-GG/FS.GG.Game","revision":"7878a467b548f4701d1e1468e1d41f6289a38692","sha256":"4fd8e8e1800d1d6ce920dcaf46320e3f732c7f8202c8d7416592e9ea490ae41b"},{"path":"scripts/verify-wasm-supervisor.sh","repository":"FS-GG/FS.GG.Game","revision":"7878a467b548f4701d1e1468e1d41f6289a38692","sha256":"81509859dec1bef24ed76505aa41d92d9aff330995b92b694325ade03689aa3c"},{"path":"docs/wasm/runtime.md","repository":"FS-GG/FS.GG.Game","revision":"7878a467b548f4701d1e1468e1d41f6289a38692","sha256":"b544a2e7f47bfec67245d669e0f9c094ec7c64b9bd847ba97bf97c761c709c17"},{"path":"docs/roadmaps/wasm-shared-01.md","repository":"FS-GG/FS.GG.Game","revision":"7878a467b548f4701d1e1468e1d41f6289a38692","sha256":"7010ad685d9a8991972578f86ed907840addf32ceb529910c1442173103af00f"}],"conflictsWith":[],"id":"game/wasm-worker-startup-bound","kind":"decision","recordedAt":"2026-10-23T01:35:33Z","repository":"FS-GG/FS.GG.Game","schema":"fsgg.atlas/1","status":"recorded","summary":"Decision (owner fdev, FS-GG/FS.GG.Game#727): the profile policy deadline (250 ms) bounds guest policy work only. A load still creates the module Worker and posts its command at once; the Worker's first Compile receipt for that command is its readiness report. Before readiness the control is projected in phase 'start' under a separate startup bound (HostSettings default 5000 ms, selectable 1-60000 ms, capped by any caller enclosing deadline, so receivers wanting only the policy budget pass none for loads); expiry there is the distinct appended DeliveryReason.WorkerStartTimeout (Discarded, TimedOut/Compile, not dispatched, Worker terminated, no game authority). Readiness replaces the startup timer with the policy timer. Compilation, phase and digest receipts before readiness, repeated readiness and readiness after expiry are inert; Worker failure, termination or a protocol-violating invocation receipt before readiness still faults the starting Worker. Ready-Worker requests keep enqueue budgets. The canonical lifecycle model, schedules (default 58/selected 16), F# correspondence and unit tests agree; coherent source version 0.7.0, ApiCompat-compatible with published 0.6.0. Lesson: Firefox reduced timer precision lets setTimeout callbacks observe floor(performance.now()) <= due (measured 36/300; Chromium/WebKit 0/300). A connected-interpreter timer observation that expires nothing was never re-armed, so a looping guest could hang unretired (intermittent 15 s Firefox watchdog hangs); the interpreter now delivers timer observations only after the clock passes due and re-arms the remainder. Lesson: WebKit words wasm traps 'Unreachable code ...'; compare trap diagnostics case-insensitively. Sampling: requiring an explicit readiness step made the selected frozen-commit-expiry witness (a 1-in-1000 lucky sample on main, 0 at four other seeds) unreachable; an initialized-candidate sampled entry makes it structurally reachable (14/1000) instead of changing the pinned seed. Evidence at capture: hosted wasm-shared run 39881900333 on PR728 head 1f68f96 (4 vCPU, 20 fresh-profile cold loads per browser, artifact wasm-cold-load-39881900333-1) measured startup max Chromium 37 ms, Firefox 76 ms, WebKit 52 ms and policy-load max 5.7/8/9 ms; the same suite passed in all three browsers. These are finite samples on the bundled 0.7 Worker, not a future fit guarantee. Publication, Templates re-pin and installed acceptance are separate later boundaries.","supersedes":[],"tags":["wasm","worker","startup","deadline","firefox","webkit","timers","lifecycle","quint","browser","decision"],"title":"Worker startup is bounded separately from the WASM policy deadline; Firefox timers can fire before due"}

docs/roadmaps/wasm-shared-01.md

@@ -1,5 +1,56 @@
 # WASM-SHARED-01 — producer qualification successor
 
+## Current 0.7 source candidate: separate Worker startup bound
+
+Installed-C run [39874053570](https://github.com/FS-GG/FS.GG.Templates/actions/runs/39874053570)
+refused a correct cold Firefox load with `DeadlineExpired` on a hosted 4-vCPU
+runner: the 250 ms policy deadline was armed at enqueue, together with creating a
+fresh module Worker whose (then unbundled 0.3) module graph alone took about
+252 ms (FS-GG/FS.GG.Game#727). The owner direction is that the policy deadline
+bounds policy work only.
+
+The coherent `0.7.0` source candidate posts the load command at Worker creation,
+as before, but treats the Worker's first `Compile` receipt as its readiness report.
+Until readiness the control is in phase `start` under a separate startup bound
+(default 5,000 ms, selectable 1–60,000 ms, capped by any enclosing deadline);
+its expiry is the distinct, appended `DeliveryReason.WorkerStartTimeout` with no
+dispatch and no game authority. Readiness starts the unchanged 250 ms policy
+budget. Requests to ready Workers, disposal, replacement, candidate, freeze and
+retirement semantics are unchanged. Compilation, phase and digest receipts before
+readiness are inert; a Worker failure, termination or protocol-violating invocation
+receipt before readiness still faults it. A caller enclosing deadline on a load also caps startup, so receivers that
+want only the policy budget should not pass one for loads.
+
+The startup default is twenty policy budgets. It separates a Worker that never
+becomes ready (blocked asset, hung fetch) from ordinary cold starts with at least
+an order of magnitude margin over the measured hosted cold starts recorded by the
+cold-load qualification, and keeps a worst-case refusal visible within seconds.
+Products with slower deployment networks may select a larger bound.
+
+The canonical model adds the `start` phase, readiness as a `compile` phase
+observation, startup expiry as `settleStartTimedOut`/`WorkerStartTimeout`, and
+invariants that a starting Worker has no compiled or initialized authority, that
+startup and policy deadlines stay within their own bounds, and that startup expiry
+is distinct, undispatched and terminating. Correspondence schedules deliver each
+fresh Worker's readiness at its load time; five new deterministic schedules cover
+the startup boundary before/at/after, an early compile receipt and a candidate
+startup refusal (default 58, selected 16). The selected sampled run adds an
+initialized-candidate entry because the readiness step otherwise made the frozen
+commit expiry witness unreachable at the pinned seed (it was a 1-in-1,000 sample).
+
+The producer's package-consumer and raw conformance browser suites now run in
+Chromium, Firefox and WebKit. A cold-load case records per-browser startup and
+policy-load timings for 20 fresh-profile loads under a hosted-like 4-CPU budget
+(hosted runners have 4 vCPUs; larger hosts pin browsers to 4 CPUs) and retains
+them as the `wasm-cold-load-*` artifact. The 0.5 closed single-module Worker
+bundle already removed the 0.3 module waterfall; receivers adopt it by re-pinning.
+The three-engine install adds a system-package download whose hosted apt mirror
+throughput varied from 41 s to 16 min 28 s; that slow original cancelled run
+39884877948 at the former 35-minute limit after every earlier gate passed. The
+qualify job limit is now 55 minutes, with a 20-minute install step bound and one
+retry, against about 27 minutes of nominal work.
+Publication, receiver re-pin and installed acceptance remain separate boundaries.
+
 ## Current 0.6 publication and installed qualification
 
 The published current-only successor is coherent `0.6.0` for the pair and SDK.

docs/wasm/runtime.md

@@ -37,6 +37,25 @@ transactional candidate. A candidate needs successful initialization plus an
 exact transaction and generation validation before commit. Retiring workers
 continue to consume capacity until their termination is observed. Completions
 from a frozen or retired generation are historical and cannot become current.
+Worker infrastructure startup and guest policy work have separate bounds. A load
+creates a fresh module Worker and posts its command at once; the browser queues the
+command until the Worker's module graph has loaded. The Worker's first receipt for
+that command (its `Compile` phase observation) is its readiness report. Until then
+the load control is projected in phase `start` and only the startup bound applies:
+`HostSettings.defaultWorkerStartTimeoutMilliseconds` (5,000 ms), selectable in
+1–60,000 ms with `HostSettings.withWorkerStartTimeout`, or an earlier caller
+enclosing deadline. Expiry before readiness settles `Discarded` with the distinct
+reason `WorkerStartTimeout`, a `TimedOut`/`Compile`/not-dispatched outcome, and
+terminates the Worker; no guest policy was admitted and no game authority follows.
+Readiness replaces the startup timer with the policy timer: the profile budget
+(SC2 250 ms `EndToEndFromEnqueue`, BAR 250 ms phase watchdog) starts at the
+readiness report, capped by any enclosing deadline. Policy expiry after readiness
+remains `DeadlineExpired`. Compilation, phase and digest receipts before
+readiness, a repeated readiness report and readiness after expiry are inert; a
+Worker failure, Worker termination or protocol-violating invocation receipt before
+readiness still faults and retires the starting Worker. Requests to an
+already-ready Worker are unchanged: their budget still starts at enqueue.
+
 Every event checks host-observed monotonic time; a request may complete at its
 deadline and expires after it. The event boundary first clamps the original
 observation time, expires elapsed work, then handles the same original input.