From 96b3cc9cc139f0daec690a9e346046458a28c3ca Mon Sep 17 00:00:00 2001 From: Justin Persaud Date: Thu, 15 Oct 2026 11:26:50 -0400 Subject: [PATCH 01/11] Add a script that carries files forward from previous staging builds Each Worker version only contains the files uploaded with it, so tabs still running an older build would lose its chunks on every deploy. Files from the current build are never replaced. --- .../scripts/carryForwardPreviousBuilds.ts | 47 ++++++++++ .../scripts/carryForwardStagingBuilds.sh | 33 +++++++ .../tests/carryForwardPreviousBuilds.test.ts | 93 +++++++++++++++++++ 3 files changed, 173 insertions(+) create mode 100644 web/cloudflare-worker/scripts/carryForwardPreviousBuilds.ts create mode 100755 web/cloudflare-worker/scripts/carryForwardStagingBuilds.sh create mode 100644 web/cloudflare-worker/tests/carryForwardPreviousBuilds.test.ts diff --git a/web/cloudflare-worker/scripts/carryForwardPreviousBuilds.ts b/web/cloudflare-worker/scripts/carryForwardPreviousBuilds.ts new file mode 100644 index 000000000000..713aa718af16 --- /dev/null +++ b/web/cloudflare-worker/scripts/carryForwardPreviousBuilds.ts @@ -0,0 +1,47 @@ +/** + * Copies files from previous web builds into the current one when the current build doesn't have them, so tabs still + * running an older build can load its chunks after a deploy. Files with fixed names exist in every build and are + * never replaced, so in practice only content-hashed and versioned files are carried forward. + * + * node scripts/carryForwardPreviousBuilds.ts ... + * + * List previous builds newest first. + */ +import {copyFileSync, existsSync, mkdirSync, readdirSync} from 'node:fs'; +import path from 'node:path'; + +const UNPUBLISHED_FILE = /\.(?:br|map)$/; + +function carryForwardPreviousBuilds(currentBuildDir: string, previousBuildDirs: string[]): number { + let copiedCount = 0; + for (const previousBuildDir of previousBuildDirs) { + for (const relativePath of readdirSync(previousBuildDir, {recursive: true, withFileTypes: true})) { + if (!relativePath.isFile() || UNPUBLISHED_FILE.test(relativePath.name)) { + continue; + } + + const sourcePath = path.join(relativePath.parentPath, relativePath.name); + const targetPath = path.join(currentBuildDir, path.relative(previousBuildDir, sourcePath)); + if (existsSync(targetPath)) { + continue; + } + + mkdirSync(path.dirname(targetPath), {recursive: true}); + copyFileSync(sourcePath, targetPath); + copiedCount++; + } + } + return copiedCount; +} + +if (import.meta.main) { + const [currentBuildDir, ...previousBuildDirs] = process.argv.slice(2); + if (!currentBuildDir || previousBuildDirs.length === 0) { + console.error('Usage: node scripts/carryForwardPreviousBuilds.ts ...'); + process.exit(1); + } + const copiedCount = carryForwardPreviousBuilds(currentBuildDir, previousBuildDirs); + console.log(`Carried forward ${copiedCount} files from ${previousBuildDirs.length} previous builds.`); +} + +export default carryForwardPreviousBuilds; diff --git a/web/cloudflare-worker/scripts/carryForwardStagingBuilds.sh b/web/cloudflare-worker/scripts/carryForwardStagingBuilds.sh new file mode 100755 index 000000000000..81c9584e7c9a --- /dev/null +++ b/web/cloudflare-worker/scripts/carryForwardStagingBuilds.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Downloads the web builds of the last N staging releases and carries their files forward into dist/, so tabs still +# running one of those builds can load its chunks after this deploy. Needs `gh` with read access to Expensify/App. +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +RELEASE_COUNT="$1" +WORKER_DIR="$(cd "$(dirname "$0")/.." && pwd)" +DIST_DIR="$WORKER_DIR/../../dist" +DOWNLOAD_DIR="$(mktemp -d)" +trap 'rm -rf "$DOWNLOAD_DIR"' EXIT + +PREVIOUS_BUILDS=() +while read -r TAG; do + mkdir -p "$DOWNLOAD_DIR/$TAG" + if gh release download "$TAG" --repo Expensify/App --pattern webBuild.tar.gz --output - | tar -xz -C "$DOWNLOAD_DIR/$TAG"; then + PREVIOUS_BUILDS+=("$DOWNLOAD_DIR/$TAG/dist") + else + echo "::warning::Could not download the web build from release $TAG, so its files are not carried forward." + fi +done < <(gh release list --repo Expensify/App --exclude-drafts --limit 50 --json tagName \ + --jq "[.[] | select(.tagName | test(\"-staging$\"))][:$RELEASE_COUNT][].tagName") + +if [[ ${#PREVIOUS_BUILDS[@]} -eq 0 ]]; then + echo "::warning::No previous staging builds were downloaded, so open tabs may fail to load chunks after this deploy." + exit 0 +fi + +node "$WORKER_DIR/scripts/carryForwardPreviousBuilds.ts" "$DIST_DIR" "${PREVIOUS_BUILDS[@]}" diff --git a/web/cloudflare-worker/tests/carryForwardPreviousBuilds.test.ts b/web/cloudflare-worker/tests/carryForwardPreviousBuilds.test.ts new file mode 100644 index 000000000000..8d3209966fa5 --- /dev/null +++ b/web/cloudflare-worker/tests/carryForwardPreviousBuilds.test.ts @@ -0,0 +1,93 @@ +import assert from 'node:assert/strict'; +import {existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import path from 'node:path'; +import {afterEach, beforeEach, describe, test} from 'node:test'; + +import carryForwardPreviousBuilds from '../scripts/carryForwardPreviousBuilds.ts'; + +function writeFiles(rootDir: string, files: Record) { + for (const [filePath, contents] of Object.entries(files)) { + const absolutePath = path.join(rootDir, filePath); + mkdirSync(path.dirname(absolutePath), {recursive: true}); + writeFileSync(absolutePath, contents); + } +} + +describe('carryForwardPreviousBuilds', () => { + let tempDir: string; + let currentBuild: string; + let previousBuild: string; + + beforeEach(() => { + tempDir = mkdtempSync(path.join(tmpdir(), 'carry-forward-')); + currentBuild = path.join(tempDir, 'current'); + previousBuild = path.join(tempDir, 'previous'); + writeFiles(currentBuild, { + 'index.html': 'current shell', + 'main-1111111111111111.bundle.js': 'current chunk', + 'version.json': '{"version":"2"}', + }); + }); + + afterEach(() => { + rmSync(tempDir, {recursive: true, force: true}); + }); + + test('adds files from a previous build that the current build no longer has', () => { + // Given a previous build whose chunks an open tab may still request + writeFiles(previousBuild, { + 'main-0000000000000000.bundle.js': 'old chunk', + 'css/old.0123456789.css': 'old nested style', + }); + + // When it is carried forward + const copiedCount = carryForwardPreviousBuilds(currentBuild, [previousBuild]); + + // Then both files are added, so the tab can still load them after the deploy + assert.equal(copiedCount, 2); + assert.equal(readFileSync(path.join(currentBuild, 'main-0000000000000000.bundle.js'), 'utf8'), 'old chunk'); + assert.equal(readFileSync(path.join(currentBuild, 'css/old.0123456789.css'), 'utf8'), 'old nested style'); + }); + + test('never replaces a file from the current build', () => { + // Given a previous build with its own copies of files that keep the same name across builds + writeFiles(previousBuild, {'index.html': 'old shell', 'version.json': '{"version":"1"}'}); + + // When it is carried forward + const copiedCount = carryForwardPreviousBuilds(currentBuild, [previousBuild]); + + // Then the current versions stay, otherwise the deploy would serve the previous build's shell and version + assert.equal(copiedCount, 0); + assert.equal(readFileSync(path.join(currentBuild, 'index.html'), 'utf8'), 'current shell'); + assert.equal(readFileSync(path.join(currentBuild, 'version.json'), 'utf8'), '{"version":"2"}'); + }); + + test('skips Brotli twins and source maps', () => { + // Given a previous build containing files that are never published + writeFiles(previousBuild, {'main-0000000000000000.bundle.js.br': 'twin', 'main-0000000000000000.bundle.js.map': 'map'}); + + // When it is carried forward + const copiedCount = carryForwardPreviousBuilds(currentBuild, [previousBuild]); + + // Then they are left behind instead of being copied only to be ignored on upload + assert.equal(copiedCount, 0); + assert.equal(existsSync(path.join(currentBuild, 'main-0000000000000000.bundle.js.br')), false); + assert.equal(existsSync(path.join(currentBuild, 'main-0000000000000000.bundle.js.map')), false); + }); + + test('takes each file from the first build that has it', () => { + // Given two previous builds that both contain a file the current build lacks + const olderBuild = path.join(tempDir, 'older'); + writeFiles(previousBuild, {'module-a.0123456789.js': 'newer'}); + writeFiles(olderBuild, {'module-a.0123456789.js': 'older', 'module-b.0123456789.js': 'only in older'}); + + // When they are carried forward, newest first + const copiedCount = carryForwardPreviousBuilds(currentBuild, [previousBuild, olderBuild]); + + // Then the shared file comes from the newer build and is copied once + assert.equal(copiedCount, 2); + assert.equal(readFileSync(path.join(currentBuild, 'module-a.0123456789.js'), 'utf8'), 'newer'); + assert.equal(readFileSync(path.join(currentBuild, 'module-b.0123456789.js'), 'utf8'), 'only in older'); + }); +}); From f85b94bd7cabb811481e38d19e28001b35f63200 Mon Sep 17 00:00:00 2001 From: Justin Persaud Date: Thu, 15 Oct 2026 11:26:52 -0400 Subject: [PATCH 02/11] Include Storybook and previous builds' files in the test-staging deploy --- .github/workflows/deploy.yml | 17 +++++++++++++++-- web/cloudflare-worker/README.md | 7 +++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index eeb9592a36f2..12cbb01f809f 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -772,8 +772,8 @@ jobs: webDeployTestStaging: name: Deploy Web to test-staging.new.expensify.com - needs: [prep, webBuild] - if: ${{ needs.webBuild.result == 'success' && needs.prep.outputs.DEPLOY_ENV == 'staging' }} + needs: [prep, webBuild, buildStorybook] + if: ${{ always() && needs.webBuild.result == 'success' && needs.prep.outputs.DEPLOY_ENV == 'staging' }} # Trial hosting for the web build, so a failure here must not fail the staging deploy. continue-on-error: true runs-on: blacksmith-2vcpu-ubuntu-2404 @@ -801,6 +801,19 @@ jobs: - name: Extract web build run: tar -xzf "${{ needs.webBuild.outputs.TAR_FILENAME }}" + - name: Download storybook docs artifact + continue-on-error: true + # v7 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 + with: + name: storybook-docs-artifact + path: ./dist/docs + + - name: Carry forward files from previous staging builds + run: ./web/cloudflare-worker/scripts/carryForwardStagingBuilds.sh 10 + env: + GH_TOKEN: ${{ github.token }} + - name: Install Cloudflare Worker dependencies working-directory: web/cloudflare-worker run: npm ci diff --git a/web/cloudflare-worker/README.md b/web/cloudflare-worker/README.md index 7cbb1b0cefdd..858214475119 100644 --- a/web/cloudflare-worker/README.md +++ b/web/cloudflare-worker/README.md @@ -117,6 +117,13 @@ The script diffs status, caching and security headers for a fixed set of paths. npx wrangler deploy --env test-staging ``` +Each Worker version only contains the files uploaded with it, so before deploying, CI also adds: + +- **Files from the last 10 staging builds** that the current build no longer has (`scripts/carryForwardStagingBuilds.sh`). These are almost all old content-hashed chunks, which tabs still running an older build request after a deploy. Files from the current build are never replaced. +- **Storybook**, into `dist/docs`, so it is served at `/docs/index.html`. + +To carry forward previous builds by hand (needs `gh`), run `./scripts/carryForwardStagingBuilds.sh 10` after putting the current build in `dist/`. + Staging and production deploys will run `wrangler deploy --env staging|production` from CI, with `CLOUDFLARE_ACCOUNT_ID` and a scoped `CLOUDFLARE_API_TOKEN` supplied as environment secrets. Workers Logs are sampled in `staging` and `production` (`head_sampling_rate`), because the Worker runs for every asset request. `test-staging` logs every request. From 3ca6841f9cadbd95b1b2530838141ad1b4b82f8c Mon Sep 17 00:00:00 2001 From: Justin Persaud Date: Thu, 15 Oct 2026 11:29:29 -0400 Subject: [PATCH 03/11] Deploy each staging web build to the staging Static Assets Worker The test-staging job becomes a matrix over test-staging and staging. The staging Worker serves no traffic until staging.new.expensify.com is routed to it in Cloudflare, and stays continue-on-error until then. --- .github/workflows/deploy.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 12cbb01f809f..168d970bb413 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -770,12 +770,17 @@ jobs: env: HOST: ${{ env.DEPLOY_ENV == 'production' && vars.WEB_PRODUCTION_HOST || vars.WEB_STAGING_HOST }} - webDeployTestStaging: - name: Deploy Web to test-staging.new.expensify.com + webDeployCloudflareWorkers: + name: Deploy Web to Cloudflare Workers (${{ matrix.environment }}) needs: [prep, webBuild, buildStorybook] if: ${{ always() && needs.webBuild.result == 'success' && needs.prep.outputs.DEPLOY_ENV == 'staging' }} # Trial hosting for the web build, so a failure here must not fail the staging deploy. continue-on-error: true + strategy: + fail-fast: false + matrix: + # `staging` serves no traffic until staging.new.expensify.com is routed to its Worker in Cloudflare. + environment: [test-staging, staging] runs-on: blacksmith-2vcpu-ubuntu-2404 steps: - name: Checkout @@ -820,13 +825,14 @@ jobs: - name: Deploy to Cloudflare Workers working-directory: web/cloudflare-worker - run: npx wrangler deploy --env test-staging + run: npx wrangler deploy --env "${{ matrix.environment }}" env: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_WORKERS_DEPLOY_TOKEN }} CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} WRANGLER_SEND_METRICS: false - name: Verify deploy + if: ${{ matrix.environment == 'test-staging' }} run: | APP_VERSION=$(jq -r .version < package.json) ./.github/scripts/verifyDeploy.sh https://test-staging.new.expensify.com "$APP_VERSION" From 45ae241c9c841d4b9949fe73b1b5493e24ad80da Mon Sep 17 00:00:00 2001 From: Justin Persaud Date: Thu, 15 Oct 2026 11:29:31 -0400 Subject: [PATCH 04/11] Document the staging Worker deploy --- web/cloudflare-worker/README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/web/cloudflare-worker/README.md b/web/cloudflare-worker/README.md index 858214475119..8f192fe90883 100644 --- a/web/cloudflare-worker/README.md +++ b/web/cloudflare-worker/README.md @@ -108,10 +108,14 @@ The script diffs status, caching and security headers for a fixed set of paths. | Environment | Worker | Route | CSP | | --- | --- | --- | --- | | `test-staging` | `test-staging-new-expensify-assets` | `test-staging.new.expensify.com/*` (managed in Cloudflare, not in `wrangler.jsonc`) | Staging | -| `staging` | `staging-new-expensify-assets` | None until switchover, then `staging.new.expensify.com/*` | Staging | +| `staging` | `staging-new-expensify-assets` | None until switchover, then `staging.new.expensify.com/*` (managed in Cloudflare, not in `wrangler.jsonc`) | Staging | | `production` | `new-expensify-assets` | None until switchover, then `new.expensify.com/*` | Production | -`test-staging` serves a staging build on a hostname nobody else uses. Every staging deploy also deploys the same build there (the `webDeployTestStaging` job in `.github/workflows/deploy.yml`), using `CLOUDFLARE_WORKERS_DEPLOY_TOKEN`, which can only edit this Worker. That token can't read zones, so the route isn't in `wrangler.jsonc`: wrangler would try to sync it and fail. A failure there does not fail the staging deploy. To deploy it by hand, put a staging build in `dist/` and run: +Every staging deploy also deploys the same build to the `test-staging` and `staging` Workers (the `webDeployCloudflareWorkers` job in `.github/workflows/deploy.yml`). It uses `CLOUDFLARE_WORKERS_DEPLOY_TOKEN`, which can only edit those two Workers. That token can't read zones, so routes aren't in `wrangler.jsonc`: wrangler would try to sync them and fail. A failure there does not fail the staging deploy. + +`test-staging` serves a staging build on a hostname nobody else uses. `staging` serves nothing until `staging.new.expensify.com/*` is pointed at its Worker in Cloudflare. + +To deploy one by hand, put a staging build in `dist/` and run: ```bash npx wrangler deploy --env test-staging From 0e95c856d681bbfb1fb71187256034f77445df6f Mon Sep 17 00:00:00 2001 From: Justin Persaud Date: Thu, 15 Oct 2026 11:56:17 -0400 Subject: [PATCH 05/11] Exclude the release being deployed from carried-forward builds A retried deploy, or one whose release is published while the job runs, counted its own release as one of the previous 10, so only nine older builds were carried forward. The current release is identified from dist/version.json. --- web/cloudflare-worker/scripts/carryForwardStagingBuilds.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/web/cloudflare-worker/scripts/carryForwardStagingBuilds.sh b/web/cloudflare-worker/scripts/carryForwardStagingBuilds.sh index 81c9584e7c9a..c5d0103a3d4b 100755 --- a/web/cloudflare-worker/scripts/carryForwardStagingBuilds.sh +++ b/web/cloudflare-worker/scripts/carryForwardStagingBuilds.sh @@ -11,6 +11,8 @@ fi RELEASE_COUNT="$1" WORKER_DIR="$(cd "$(dirname "$0")/.." && pwd)" DIST_DIR="$WORKER_DIR/../../dist" +# A retried deploy, or one whose release is published while this runs, would otherwise count its own release. +CURRENT_TAG="$(jq -r .version "$DIST_DIR/version.json")-staging" DOWNLOAD_DIR="$(mktemp -d)" trap 'rm -rf "$DOWNLOAD_DIR"' EXIT @@ -23,7 +25,7 @@ while read -r TAG; do echo "::warning::Could not download the web build from release $TAG, so its files are not carried forward." fi done < <(gh release list --repo Expensify/App --exclude-drafts --limit 50 --json tagName \ - --jq "[.[] | select(.tagName | test(\"-staging$\"))][:$RELEASE_COUNT][].tagName") + --jq "[.[] | select((.tagName | test(\"-staging$\")) and .tagName != \"$CURRENT_TAG\")][:$RELEASE_COUNT][].tagName") if [[ ${#PREVIOUS_BUILDS[@]} -eq 0 ]]; then echo "::warning::No previous staging builds were downloaded, so open tabs may fail to load chunks after this deploy." From e171117be39376ed46164e22600edeb97573cdbc Mon Sep 17 00:00:00 2001 From: Justin Persaud Date: Thu, 15 Oct 2026 11:56:18 -0400 Subject: [PATCH 06/11] Explain route management for both staging Worker environments --- web/cloudflare-worker/wrangler.jsonc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/web/cloudflare-worker/wrangler.jsonc b/web/cloudflare-worker/wrangler.jsonc index 611081288528..74a7256e1e4f 100644 --- a/web/cloudflare-worker/wrangler.jsonc +++ b/web/cloudflare-worker/wrangler.jsonc @@ -26,16 +26,17 @@ "vars": { "ENVIRONMENT": "staging", }, + // Routes are managed in Cloudflare, not here: syncing routes needs zone access, and the CI deploy token can only + // edit the test-staging and staging Workers. "env": { - // Serves a staging build on a hostname nobody else uses, so it can be deployed and tested independently. - // Its route (test-staging.new.expensify.com/*) is managed in Cloudflare, not here: syncing routes needs zone - // access, and the CI deploy token can only edit this Worker. + // Serves a staging build on test-staging.new.expensify.com, a hostname nobody else uses. "test-staging": { "name": "test-staging-new-expensify-assets", "vars": { "ENVIRONMENT": "staging", }, }, + // Serves nothing until staging.new.expensify.com/* is routed to this Worker. "staging": { "name": "staging-new-expensify-assets", // The Worker runs for every asset, so logging every request would exceed the included Workers Logs volume. From 5e6140c1925fc3a56d42529614f5e2b05aab1615 Mon Sep 17 00:00:00 2001 From: Justin Persaud Date: Thu, 15 Oct 2026 15:05:20 -0400 Subject: [PATCH 07/11] Serve files from previous staging builds out of R2 A Worker version only contains the current build, so a tab running an older build gets a 404 for any chunk that has since changed. When the current build misses a file requested by a script or image, the staging Workers now look it up in the new-expensify-web-assets-staging bucket. The current build always takes priority, and navigations never read from the bucket. --- web/cloudflare-worker/src/index.ts | 43 ++++++++- .../tests/createWorkerHarness.ts | 23 +++++ web/cloudflare-worker/tests/worker.test.ts | 91 +++++++++++++++++++ web/cloudflare-worker/wrangler.jsonc | 5 + 4 files changed, 157 insertions(+), 5 deletions(-) diff --git a/web/cloudflare-worker/src/index.ts b/web/cloudflare-worker/src/index.ts index 015fa6612178..605990253d84 100644 --- a/web/cloudflare-worker/src/index.ts +++ b/web/cloudflare-worker/src/index.ts @@ -52,19 +52,52 @@ function setSecurityHeaders(headers: Headers, environment: Env['ENVIRONMENT'], n headers.set('Document-Policy', 'js-profiling'); } +function getPreviousBuildFileHeaders(object: R2Object): Headers { + const headers = new Headers({'Cache-Control': 'public, max-age=0, must-revalidate', ETag: object.httpEtag}); + object.writeHttpMetadata(headers); + return headers; +} + +/** A file uploaded by an earlier deploy, for tabs still running that build. Null when no build had it. */ +async function getPreviousBuildFile(env: Env, url: URL, method: string): Promise { + if (!env.PREVIOUS_BUILDS) { + return null; + } + + let key: string; + try { + key = decodeURIComponent(url.pathname.slice(1)); + } catch { + return null; + } + + if (method === 'HEAD') { + const object = await env.PREVIOUS_BUILDS.head(key); + return object ? new Response(null, {headers: getPreviousBuildFileHeaders(object)}) : null; + } + + const object = await env.PREVIOUS_BUILDS.get(key); + return object ? new Response(object.body, {headers: getPreviousBuildFileHeaders(object)}) : null; +} + export default { async fetch(request, env): Promise { const url = new URL(request.url); const nonce = generateNonce(); const isNavigation = request.headers.get('Sec-Fetch-Mode') === 'navigate'; const isAASAPath = url.pathname === AASA_PATH || url.pathname === LEGACY_AASA_PATH; - const assetResponse = await env.ASSETS.fetch(buildAssetRequest(request, url)); + let assetResponse = await env.ASSETS.fetch(buildAssetRequest(request, url)); - // The SPA fallback answers every miss with index.html. A browser asking for a missing chunk needs a real 404. + // The SPA fallback answers every miss with index.html. A browser asking for a missing chunk needs the file from + // an earlier build, or a real 404. if (isHTMLResponse(assetResponse) && (isAASAPath || (!isNavigation && STATIC_FILE_PATH.test(url.pathname)))) { - const notFound = new Response('Not Found', {status: 404, headers: {'Content-Type': 'text/plain; charset=utf-8'}}); - setSecurityHeaders(notFound.headers, env.ENVIRONMENT, nonce); - return notFound; + const previousBuildFile = isAASAPath ? null : await getPreviousBuildFile(env, url, request.method); + if (!previousBuildFile) { + const notFound = new Response('Not Found', {status: 404, headers: {'Content-Type': 'text/plain; charset=utf-8'}}); + setSecurityHeaders(notFound.headers, env.ENVIRONMENT, nonce); + return notFound; + } + assetResponse = previousBuildFile; } const response = new Response(assetResponse.body, assetResponse); diff --git a/web/cloudflare-worker/tests/createWorkerHarness.ts b/web/cloudflare-worker/tests/createWorkerHarness.ts index 74e6a7f39c5a..41e3fd4fb4b2 100644 --- a/web/cloudflare-worker/tests/createWorkerHarness.ts +++ b/web/cloudflare-worker/tests/createWorkerHarness.ts @@ -30,8 +30,21 @@ const FAKE_WEB_BUILD: Record = { '.well-known/apple-app-site-association': '{"applinks":{"details":[]}}', }; +type PreviousBuildFile = { + path: string; + contents: string; + contentType: string; +}; + +/** The part of the R2 binding the tests use; the full Workers types conflict with Node's globals in this project. */ +type PreviousBuildsEnv = { + PREVIOUS_BUILDS?: {put: (key: string, value: string, options: {httpMetadata: {contentType: string}}) => Promise}; +}; + type WorkerHarness = { server: ReturnType; + /** Stores files in the Worker's local PREVIOUS_BUILDS bucket, as a deploy of an earlier build would have. */ + addPreviousBuildFiles: (files: PreviousBuildFile[]) => Promise; cleanup: () => Promise; }; @@ -70,8 +83,18 @@ function createWorkerHarness(env?: 'test-staging' | 'staging' | 'production', {o ); const server = createTestHarness({workers: [{configPath, env}]}); + const workerName = (env ? rawConfig.env?.[env]?.name : undefined) ?? rawConfig.name ?? ''; return { server, + addPreviousBuildFiles: async (files) => { + const {PREVIOUS_BUILDS: previousBuilds} = await server.getWorker(workerName).getEnv(); + if (!previousBuilds) { + throw new Error(`The ${env ?? 'default'} environment has no PREVIOUS_BUILDS binding`); + } + for (const file of files) { + await previousBuilds.put(file.path, file.contents, {httpMetadata: {contentType: file.contentType}}); + } + }, cleanup: async () => { await server.close(); rmSync(tempDir, {recursive: true, force: true}); diff --git a/web/cloudflare-worker/tests/worker.test.ts b/web/cloudflare-worker/tests/worker.test.ts index 19f7a5abb043..09997a2c11b4 100644 --- a/web/cloudflare-worker/tests/worker.test.ts +++ b/web/cloudflare-worker/tests/worker.test.ts @@ -207,6 +207,88 @@ describe('staging worker', () => { }); }); +describe('staging worker with files from previous builds', () => { + const {server, addPreviousBuildFiles, cleanup} = createWorkerHarness('staging'); + const PREVIOUS_CHUNK_PATH = '/main-aaaaaaaaaaaaaaaa.bundle.js'; + + before(async () => { + await server.listen(); + await addPreviousBuildFiles([ + {path: PREVIOUS_CHUNK_PATH.slice(1), contents: 'window.previousBuildChunk = true;', contentType: 'text/javascript'}, + {path: 'canvaskit-0.40.0.wasm', contents: 'previous wasm', contentType: 'application/wasm'}, + {path: HASHED_BUNDLE_PATH.slice(1), contents: 'window.previousCopyOfCurrentChunk = true;', contentType: 'text/javascript'}, + {path: 'r/123/receipt.pdf', contents: 'not a page', contentType: 'application/pdf'}, + ]); + }); + + after(async () => { + await cleanup(); + }); + + test('serves a chunk the current build no longer has from a previous build', async () => { + // Given a tab still running an older build, whose chunk was uploaded to the bucket when that build deployed + // When the tab lazy-loads it after a newer deploy + const response = await server.fetch(PREVIOUS_CHUNK_PATH); + + // Then it loads, so the tab keeps working instead of reloading + assert.equal(response.status, 200); + assert.equal(await response.text(), 'window.previousBuildChunk = true;'); + assert.equal(response.headers.get('Content-Type'), 'text/javascript'); + assert.equal(response.headers.get('Cache-Control'), 'public, max-age=31536000, immutable'); + assertSecurityHeaders(response); + }); + + test('makes a previous build file without a content hash revalidate', async () => { + // Given a previous build file whose name has a version instead of a content hash + // When it is requested + const response = await server.fetch('/canvaskit-0.40.0.wasm'); + + // Then it is served, but not cached as immutable + assert.equal(response.status, 200); + assert.equal(response.headers.get('Content-Type'), 'application/wasm'); + assert.equal(response.headers.get('Cache-Control'), 'public, max-age=0, must-revalidate'); + }); + + test('answers HEAD for a previous build file without a body', async () => { + // Given a chunk that only exists in a previous build + // When it is requested with HEAD + const response = await server.fetch(PREVIOUS_CHUNK_PATH, {method: 'HEAD'}); + + // Then it reports the file exists, without sending it + assert.equal(response.status, 200); + assert.equal(response.headers.get('Content-Type'), 'text/javascript'); + assert.equal(await response.text(), ''); + }); + + test('serves the current build when a previous build has the same file', async () => { + // Given a file that is in both the current build and the bucket + // When it is requested + const response = await server.fetch(HASHED_BUNDLE_PATH); + + // Then the current build's copy wins, so the bucket can never replace what was just deployed + assert.match(await response.text(), /fixtureBundle/); + }); + + test('never serves a previous build file to a navigation', async () => { + // Given a route that looks like a file and happens to match a key in the bucket + // When the browser navigates to it + const response = await server.fetch('/r/123/receipt.pdf', {headers: NAVIGATION_HEADERS}); + + // Then it gets the app shell, because only script and image requests can need an older build's files + assert.equal(response.status, 200); + assert.match(await response.text(), /New Expensify fixture/); + }); + + test('returns 404 for a file no build has', async () => { + // Given a chunk that is in neither the current build nor the bucket + // When it is requested + const response = await server.fetch('/main-ffffffffffffffff.bundle.js'); + + // Then it is a 404, so the client's ChunkLoadError recovery runs + assert.equal(response.status, 404); + }); +}); + describe('worker with a build that has no apple-app-site-association file', () => { const {server, cleanup} = createWorkerHarness('staging', {omittedFiles: ['.well-known/apple-app-site-association']}); @@ -274,4 +356,13 @@ describe('production worker', () => { assert.doesNotMatch(csp, /new-expensify:\/\/staging/); assertTemplateInlineScriptsAllowed(csp); }); + + test('returns 404 for a missing chunk without a previous builds bucket', async () => { + // Given production has no PREVIOUS_BUILDS binding yet + // When a chunk the current build doesn't have is requested + const response = await server.fetch('/main-ffffffffffffffff.bundle.js'); + + // Then it is a 404 rather than an error from the missing binding + assert.equal(response.status, 404); + }); }); diff --git a/web/cloudflare-worker/wrangler.jsonc b/web/cloudflare-worker/wrangler.jsonc index 74a7256e1e4f..2c64b1669af8 100644 --- a/web/cloudflare-worker/wrangler.jsonc +++ b/web/cloudflare-worker/wrangler.jsonc @@ -23,6 +23,9 @@ // Every response needs security headers, and HTML needs a per-request CSP nonce, including SPA navigations. "run_worker_first": true, }, + // Files from previous staging builds, served when a tab running an older build requests a file the current build + // doesn't have. `wrangler dev` uses an empty local bucket. + "r2_buckets": [{"binding": "PREVIOUS_BUILDS", "bucket_name": "new-expensify-web-assets-staging"}], "vars": { "ENVIRONMENT": "staging", }, @@ -32,6 +35,7 @@ // Serves a staging build on test-staging.new.expensify.com, a hostname nobody else uses. "test-staging": { "name": "test-staging-new-expensify-assets", + "r2_buckets": [{"binding": "PREVIOUS_BUILDS", "bucket_name": "new-expensify-web-assets-staging"}], "vars": { "ENVIRONMENT": "staging", }, @@ -39,6 +43,7 @@ // Serves nothing until staging.new.expensify.com/* is routed to this Worker. "staging": { "name": "staging-new-expensify-assets", + "r2_buckets": [{"binding": "PREVIOUS_BUILDS", "bucket_name": "new-expensify-web-assets-staging"}], // The Worker runs for every asset, so logging every request would exceed the included Workers Logs volume. "observability": { "enabled": true, From 0cb2ad34ab27f96646ca550c33af3c17d94cce53 Mon Sep 17 00:00:00 2001 From: Justin Persaud Date: Thu, 15 Oct 2026 15:05:21 -0400 Subject: [PATCH 08/11] Upload each staging build to R2 instead of carrying builds forward Uploading only the new files of each build keeps CI time constant however much history the Workers can serve, and the bucket's 60-day lifecycle rule bounds that history. The upload is skipped with a warning until the R2 credentials are added. --- .github/workflows/deploy.yml | 26 +++++- .../scripts/carryForwardPreviousBuilds.ts | 47 ---------- .../scripts/carryForwardStagingBuilds.sh | 35 ------- .../tests/carryForwardPreviousBuilds.test.ts | 93 ------------------- 4 files changed, 21 insertions(+), 180 deletions(-) delete mode 100644 web/cloudflare-worker/scripts/carryForwardPreviousBuilds.ts delete mode 100755 web/cloudflare-worker/scripts/carryForwardStagingBuilds.sh delete mode 100644 web/cloudflare-worker/tests/carryForwardPreviousBuilds.test.ts diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 168d970bb413..25261bd29721 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -806,6 +806,27 @@ jobs: - name: Extract web build run: tar -xzf "${{ needs.webBuild.outputs.TAR_FILENAME }}" + # The Workers serve these to tabs still running this build after later deploys. Both Workers read the same bucket, + # so only one matrix leg uploads. + - name: Upload web build to the previous builds bucket + if: ${{ matrix.environment == 'staging' }} + continue-on-error: true + run: | + if [[ -z "$AWS_ACCESS_KEY_ID" ]]; then + echo "::warning::CLOUDFLARE_R2_WEB_ASSETS_ACCESS_KEY_ID is not set, so this build is not available to future deploys." + exit 0 + fi + aws configure set default.s3.preferred_transfer_client classic + aws s3 sync dist s3://new-expensify-web-assets-staging/ \ + --endpoint-url "https://$CLOUDFLARE_ACCOUNT_ID.r2.cloudflarestorage.com" \ + --size-only --no-progress \ + --exclude '*.br' --exclude '*.map' + env: + AWS_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_WEB_ASSETS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_WEB_ASSETS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: auto + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + - name: Download storybook docs artifact continue-on-error: true # v7 @@ -814,11 +835,6 @@ jobs: name: storybook-docs-artifact path: ./dist/docs - - name: Carry forward files from previous staging builds - run: ./web/cloudflare-worker/scripts/carryForwardStagingBuilds.sh 10 - env: - GH_TOKEN: ${{ github.token }} - - name: Install Cloudflare Worker dependencies working-directory: web/cloudflare-worker run: npm ci diff --git a/web/cloudflare-worker/scripts/carryForwardPreviousBuilds.ts b/web/cloudflare-worker/scripts/carryForwardPreviousBuilds.ts deleted file mode 100644 index 713aa718af16..000000000000 --- a/web/cloudflare-worker/scripts/carryForwardPreviousBuilds.ts +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copies files from previous web builds into the current one when the current build doesn't have them, so tabs still - * running an older build can load its chunks after a deploy. Files with fixed names exist in every build and are - * never replaced, so in practice only content-hashed and versioned files are carried forward. - * - * node scripts/carryForwardPreviousBuilds.ts ... - * - * List previous builds newest first. - */ -import {copyFileSync, existsSync, mkdirSync, readdirSync} from 'node:fs'; -import path from 'node:path'; - -const UNPUBLISHED_FILE = /\.(?:br|map)$/; - -function carryForwardPreviousBuilds(currentBuildDir: string, previousBuildDirs: string[]): number { - let copiedCount = 0; - for (const previousBuildDir of previousBuildDirs) { - for (const relativePath of readdirSync(previousBuildDir, {recursive: true, withFileTypes: true})) { - if (!relativePath.isFile() || UNPUBLISHED_FILE.test(relativePath.name)) { - continue; - } - - const sourcePath = path.join(relativePath.parentPath, relativePath.name); - const targetPath = path.join(currentBuildDir, path.relative(previousBuildDir, sourcePath)); - if (existsSync(targetPath)) { - continue; - } - - mkdirSync(path.dirname(targetPath), {recursive: true}); - copyFileSync(sourcePath, targetPath); - copiedCount++; - } - } - return copiedCount; -} - -if (import.meta.main) { - const [currentBuildDir, ...previousBuildDirs] = process.argv.slice(2); - if (!currentBuildDir || previousBuildDirs.length === 0) { - console.error('Usage: node scripts/carryForwardPreviousBuilds.ts ...'); - process.exit(1); - } - const copiedCount = carryForwardPreviousBuilds(currentBuildDir, previousBuildDirs); - console.log(`Carried forward ${copiedCount} files from ${previousBuildDirs.length} previous builds.`); -} - -export default carryForwardPreviousBuilds; diff --git a/web/cloudflare-worker/scripts/carryForwardStagingBuilds.sh b/web/cloudflare-worker/scripts/carryForwardStagingBuilds.sh deleted file mode 100755 index c5d0103a3d4b..000000000000 --- a/web/cloudflare-worker/scripts/carryForwardStagingBuilds.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -# Downloads the web builds of the last N staging releases and carries their files forward into dist/, so tabs still -# running one of those builds can load its chunks after this deploy. Needs `gh` with read access to Expensify/App. -set -euo pipefail - -if [[ $# -ne 1 ]]; then - echo "Usage: $0 " >&2 - exit 1 -fi - -RELEASE_COUNT="$1" -WORKER_DIR="$(cd "$(dirname "$0")/.." && pwd)" -DIST_DIR="$WORKER_DIR/../../dist" -# A retried deploy, or one whose release is published while this runs, would otherwise count its own release. -CURRENT_TAG="$(jq -r .version "$DIST_DIR/version.json")-staging" -DOWNLOAD_DIR="$(mktemp -d)" -trap 'rm -rf "$DOWNLOAD_DIR"' EXIT - -PREVIOUS_BUILDS=() -while read -r TAG; do - mkdir -p "$DOWNLOAD_DIR/$TAG" - if gh release download "$TAG" --repo Expensify/App --pattern webBuild.tar.gz --output - | tar -xz -C "$DOWNLOAD_DIR/$TAG"; then - PREVIOUS_BUILDS+=("$DOWNLOAD_DIR/$TAG/dist") - else - echo "::warning::Could not download the web build from release $TAG, so its files are not carried forward." - fi -done < <(gh release list --repo Expensify/App --exclude-drafts --limit 50 --json tagName \ - --jq "[.[] | select((.tagName | test(\"-staging$\")) and .tagName != \"$CURRENT_TAG\")][:$RELEASE_COUNT][].tagName") - -if [[ ${#PREVIOUS_BUILDS[@]} -eq 0 ]]; then - echo "::warning::No previous staging builds were downloaded, so open tabs may fail to load chunks after this deploy." - exit 0 -fi - -node "$WORKER_DIR/scripts/carryForwardPreviousBuilds.ts" "$DIST_DIR" "${PREVIOUS_BUILDS[@]}" diff --git a/web/cloudflare-worker/tests/carryForwardPreviousBuilds.test.ts b/web/cloudflare-worker/tests/carryForwardPreviousBuilds.test.ts deleted file mode 100644 index 8d3209966fa5..000000000000 --- a/web/cloudflare-worker/tests/carryForwardPreviousBuilds.test.ts +++ /dev/null @@ -1,93 +0,0 @@ -import assert from 'node:assert/strict'; -import {existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs'; -import {tmpdir} from 'node:os'; -import path from 'node:path'; -import {afterEach, beforeEach, describe, test} from 'node:test'; - -import carryForwardPreviousBuilds from '../scripts/carryForwardPreviousBuilds.ts'; - -function writeFiles(rootDir: string, files: Record) { - for (const [filePath, contents] of Object.entries(files)) { - const absolutePath = path.join(rootDir, filePath); - mkdirSync(path.dirname(absolutePath), {recursive: true}); - writeFileSync(absolutePath, contents); - } -} - -describe('carryForwardPreviousBuilds', () => { - let tempDir: string; - let currentBuild: string; - let previousBuild: string; - - beforeEach(() => { - tempDir = mkdtempSync(path.join(tmpdir(), 'carry-forward-')); - currentBuild = path.join(tempDir, 'current'); - previousBuild = path.join(tempDir, 'previous'); - writeFiles(currentBuild, { - 'index.html': 'current shell', - 'main-1111111111111111.bundle.js': 'current chunk', - 'version.json': '{"version":"2"}', - }); - }); - - afterEach(() => { - rmSync(tempDir, {recursive: true, force: true}); - }); - - test('adds files from a previous build that the current build no longer has', () => { - // Given a previous build whose chunks an open tab may still request - writeFiles(previousBuild, { - 'main-0000000000000000.bundle.js': 'old chunk', - 'css/old.0123456789.css': 'old nested style', - }); - - // When it is carried forward - const copiedCount = carryForwardPreviousBuilds(currentBuild, [previousBuild]); - - // Then both files are added, so the tab can still load them after the deploy - assert.equal(copiedCount, 2); - assert.equal(readFileSync(path.join(currentBuild, 'main-0000000000000000.bundle.js'), 'utf8'), 'old chunk'); - assert.equal(readFileSync(path.join(currentBuild, 'css/old.0123456789.css'), 'utf8'), 'old nested style'); - }); - - test('never replaces a file from the current build', () => { - // Given a previous build with its own copies of files that keep the same name across builds - writeFiles(previousBuild, {'index.html': 'old shell', 'version.json': '{"version":"1"}'}); - - // When it is carried forward - const copiedCount = carryForwardPreviousBuilds(currentBuild, [previousBuild]); - - // Then the current versions stay, otherwise the deploy would serve the previous build's shell and version - assert.equal(copiedCount, 0); - assert.equal(readFileSync(path.join(currentBuild, 'index.html'), 'utf8'), 'current shell'); - assert.equal(readFileSync(path.join(currentBuild, 'version.json'), 'utf8'), '{"version":"2"}'); - }); - - test('skips Brotli twins and source maps', () => { - // Given a previous build containing files that are never published - writeFiles(previousBuild, {'main-0000000000000000.bundle.js.br': 'twin', 'main-0000000000000000.bundle.js.map': 'map'}); - - // When it is carried forward - const copiedCount = carryForwardPreviousBuilds(currentBuild, [previousBuild]); - - // Then they are left behind instead of being copied only to be ignored on upload - assert.equal(copiedCount, 0); - assert.equal(existsSync(path.join(currentBuild, 'main-0000000000000000.bundle.js.br')), false); - assert.equal(existsSync(path.join(currentBuild, 'main-0000000000000000.bundle.js.map')), false); - }); - - test('takes each file from the first build that has it', () => { - // Given two previous builds that both contain a file the current build lacks - const olderBuild = path.join(tempDir, 'older'); - writeFiles(previousBuild, {'module-a.0123456789.js': 'newer'}); - writeFiles(olderBuild, {'module-a.0123456789.js': 'older', 'module-b.0123456789.js': 'only in older'}); - - // When they are carried forward, newest first - const copiedCount = carryForwardPreviousBuilds(currentBuild, [previousBuild, olderBuild]); - - // Then the shared file comes from the newer build and is copied once - assert.equal(copiedCount, 2); - assert.equal(readFileSync(path.join(currentBuild, 'module-a.0123456789.js'), 'utf8'), 'newer'); - assert.equal(readFileSync(path.join(currentBuild, 'module-b.0123456789.js'), 'utf8'), 'only in older'); - }); -}); From 011dc3a2f7e9524c2697c0fcb48f9583edb7fe93 Mon Sep 17 00:00:00 2001 From: Justin Persaud Date: Thu, 15 Oct 2026 15:05:47 -0400 Subject: [PATCH 09/11] Document serving previous builds from R2 --- web/cloudflare-worker/README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/web/cloudflare-worker/README.md b/web/cloudflare-worker/README.md index 8f192fe90883..136436993cc4 100644 --- a/web/cloudflare-worker/README.md +++ b/web/cloudflare-worker/README.md @@ -14,7 +14,7 @@ Every request runs the Worker first (`run_worker_first: true`), which reads the | HTML (`/`, `/index.html`, any SPA deep link) | Each `