[No QA] Deploy staging web builds to the staging Static Assets Worker

.github/workflows/deploy.yml

@@ -770,12 +770,17 @@ jobs:
         env:
           HOST: ${{ env.DEPLOY_ENV == 'production' && vars.WEB_PRODUCTION_HOST || vars.WEB_STAGING_HOST }}
 
-  webDeployTestStaging:
-    name: Deploy Web to test-staging.new.expensify.com
-    needs: [prep, webBuild]
-    if: ${{ needs.webBuild.result == 'success' && needs.prep.outputs.DEPLOY_ENV == 'staging' }}
+  webDeployCloudflareWorkers:
+    name: Deploy Web to Cloudflare Workers (${{ matrix.environment }})
+    needs: [prep, webBuild, buildStorybook]
+    if: ${{ always() && needs.webBuild.result == 'success' && needs.prep.outputs.DEPLOY_ENV == 'staging' }}
     # Trial hosting for the web build, so a failure here must not fail the staging deploy.
     continue-on-error: true
+    strategy:
+      fail-fast: false
+      matrix:
+        # `staging` serves no traffic until staging.new.expensify.com is routed to its Worker in Cloudflare.
+        environment: [test-staging, staging]
     runs-on: blacksmith-2vcpu-ubuntu-2404
     steps:
       - name: Checkout
@@ -801,19 +806,50 @@ jobs:
       - name: Extract web build
         run: tar -xzf "${{ needs.webBuild.outputs.TAR_FILENAME }}"
 
+      # The Workers serve these to tabs still running this build after later deploys. Both Workers read the same bucket,
+      # so only one matrix leg uploads.
+      - name: Upload web build to the previous builds bucket
+        if: ${{ matrix.environment == 'staging' }}
+        continue-on-error: true
+        run: |
+          if [[ -z "$AWS_ACCESS_KEY_ID" ]]; then
+            echo "::warning::CLOUDFLARE_R2_WEB_ASSETS_ACCESS_KEY_ID is not set, so this build is not available to future deploys."
+            exit 0
+          fi
+          aws configure set default.s3.preferred_transfer_client classic
+          # Re-uploads every file, so files that stay the same across builds don't hit the bucket's 60-day expiry while still in use.
+          aws s3 cp dist s3://new-expensify-web-assets-staging/ --recursive \
+            --endpoint-url "https://$CLOUDFLARE_ACCOUNT_ID.r2.cloudflarestorage.com" \
+            --no-progress \
+            --exclude '*.br' --exclude '*.map'
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.CLOUDFLARE_R2_WEB_ASSETS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CLOUDFLARE_R2_WEB_ASSETS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: auto
+          CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
+
+      - name: Download storybook docs artifact
+        continue-on-error: true
+        # v7
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131
+        with:
+          name: storybook-docs-artifact
+          path: ./dist/docs
+
       - name: Install Cloudflare Worker dependencies
         working-directory: web/cloudflare-worker
         run: npm ci
 
       - name: Deploy to Cloudflare Workers
         working-directory: web/cloudflare-worker
-        run: npx wrangler deploy --env test-staging
+        run: npx wrangler deploy --env "${{ matrix.environment }}"
         env:
           CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_WORKERS_DEPLOY_TOKEN }}
           CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
           WRANGLER_SEND_METRICS: false
 
       - name: Verify deploy
+        if: ${{ matrix.environment == 'test-staging' }}
         run: |
           APP_VERSION=$(jq -r .version < package.json)
           ./.github/scripts/verifyDeploy.sh https://test-staging.new.expensify.com "$APP_VERSION"

web/cloudflare-worker/README.md

@@ -14,7 +14,7 @@ Every request runs the Worker first (`run_worker_first: true`), which reads the
 | HTML (`/`, `/index.html`, any SPA deep link) | Each `<script nonce="nonce-random-value">` gets a per-request nonce that matches the CSP. `Cache-Control: no-cache`, with no `ETag` or `Last-Modified`, since a 304 would pair a cached body with a new nonce |
 | Content-hashed files (`main-<16 hex>.bundle.js`, `name.<10 hex>.css`) | `Cache-Control: public, max-age=31536000, immutable` |
 | Files with fixed names (`version.json`, `service-worker.js`, `manifest.json`, ...) | Default `public, max-age=0, must-revalidate`, with an `ETag` |
-| A file path that does not exist (`/main-<old hash>.bundle.js`) | `404`, so a stale tab gets a `ChunkLoadError` and recovers instead of parsing HTML as JavaScript |
+| A file the current build doesn't have (`/main-<old hash>.bundle.js`) | The file from a previous build, if the environment has a `PREVIOUS_BUILDS` bucket that holds it (see [Files from previous builds](#files-from-previous-builds)). Otherwise `404`, so a stale tab gets a `ChunkLoadError` and recovers instead of parsing HTML as JavaScript |
 | Any other path that does not exist (`/r/123`) | The app shell (`index.html`) with `200` |
 | `/.well-known/apple-app-site-association` and `/apple-app-site-association` | The same file with `Content-Type: application/json` |
 
@@ -108,15 +108,27 @@ The script diffs status, caching and security headers for a fixed set of paths.
 | Environment | Worker | Route | CSP |
 | --- | --- | --- | --- |
 | `test-staging` | `test-staging-new-expensify-assets` | `test-staging.new.expensify.com/*` (managed in Cloudflare, not in `wrangler.jsonc`) | Staging |
-| `staging` | `staging-new-expensify-assets` | None until switchover, then `staging.new.expensify.com/*` | Staging |
+| `staging` | `staging-new-expensify-assets` | None until switchover, then `staging.new.expensify.com/*` (managed in Cloudflare, not in `wrangler.jsonc`) | Staging |
 | `production` | `new-expensify-assets` | None until switchover, then `new.expensify.com/*` | Production |
 
-`test-staging` serves a staging build on a hostname nobody else uses. Every staging deploy also deploys the same build there (the `webDeployTestStaging` job in `.github/workflows/deploy.yml`), using `CLOUDFLARE_WORKERS_DEPLOY_TOKEN`, which can only edit this Worker. That token can't read zones, so the route isn't in `wrangler.jsonc`: wrangler would try to sync it and fail. A failure there does not fail the staging deploy. To deploy it by hand, put a staging build in `dist/` and run:
+Every staging deploy also deploys the same build to the `test-staging` and `staging` Workers (the `webDeployCloudflareWorkers` job in `.github/workflows/deploy.yml`). It uses `CLOUDFLARE_WORKERS_DEPLOY_TOKEN`, which can only edit those two Workers. That token can't read zones, so routes aren't in `wrangler.jsonc`: wrangler would try to sync them and fail. A failure there does not fail the staging deploy.
+
+`test-staging` serves a staging build on a hostname nobody else uses. `staging` serves nothing until `staging.new.expensify.com/*` is pointed at its Worker in Cloudflare.
+
+To deploy one by hand, put a staging build in `dist/` and run:
 
 ```bash
 npx wrangler deploy --env test-staging
 ```
 
+CI also adds Storybook to `dist/docs` before deploying, so it is served at `/docs/index.html`.
+
+### Files from previous builds
+
+Each Worker version only contains the files uploaded with it, but a tab still running an older build keeps requesting that build's chunks after a deploy. So every staging deploy also uploads its build (without `*.br`, `*.map` and Storybook) to the R2 bucket `new-expensify-web-assets-staging`. Every file is re-uploaded each time, so a file still used by recent builds never reaches the bucket's 60-day expiry. When the current build doesn't have a file that a script or image requests, the `test-staging` and `staging` Workers look it up in that bucket (the `PREVIOUS_BUILDS` binding) and return `404` only if it isn't there either. The current build always takes priority, and navigations never read from the bucket.
+
+The bucket and its 60-day lifecycle rule are managed in Terraform. The upload uses `CLOUDFLARE_R2_WEB_ASSETS_ACCESS_KEY_ID` and `CLOUDFLARE_R2_WEB_ASSETS_SECRET_ACCESS_KEY`, an R2 token that can only read and write that bucket. `wrangler dev` uses an empty local bucket.
+
 Staging and production deploys will run `wrangler deploy --env staging|production` from CI, with `CLOUDFLARE_ACCOUNT_ID` and a scoped `CLOUDFLARE_API_TOKEN` supplied as environment secrets.
 
 Workers Logs are sampled in `staging` and `production` (`head_sampling_rate`), because the Worker runs for every asset request. `test-staging` logs every request.

web/cloudflare-worker/src/index.ts

@@ -52,19 +52,52 @@ function setSecurityHeaders(headers: Headers, environment: Env['ENVIRONMENT'], n
     headers.set('Document-Policy', 'js-profiling');
 }
 
+function getPreviousBuildFileHeaders(object: R2Object): Headers {
+    const headers = new Headers({'Cache-Control': 'public, max-age=0, must-revalidate', ETag: object.httpEtag});
+    object.writeHttpMetadata(headers);
+    return headers;
+}
+
+/** A file uploaded by an earlier deploy, for tabs still running that build. Null when no build had it. */
+async function getPreviousBuildFile(env: Env, url: URL, method: string): Promise<Response | null> {
+    if (!env.PREVIOUS_BUILDS) {
+        return null;
+    }
+
+    let key: string;
+    try {
+        key = decodeURIComponent(url.pathname.slice(1));
+    } catch {
+        return null;
+    }
+
+    if (method === 'HEAD') {
+        const object = await env.PREVIOUS_BUILDS.head(key);
+        return object ? new Response(null, {headers: getPreviousBuildFileHeaders(object)}) : null;
+    }
+
+    const object = await env.PREVIOUS_BUILDS.get(key);
+    return object ? new Response(object.body, {headers: getPreviousBuildFileHeaders(object)}) : null;
+}
+
 export default {
     async fetch(request, env): Promise<Response> {
         const url = new URL(request.url);
         const nonce = generateNonce();
         const isNavigation = request.headers.get('Sec-Fetch-Mode') === 'navigate';
         const isAASAPath = url.pathname === AASA_PATH || url.pathname === LEGACY_AASA_PATH;
-        const assetResponse = await env.ASSETS.fetch(buildAssetRequest(request, url));
+        let assetResponse = await env.ASSETS.fetch(buildAssetRequest(request, url));
 
-        // The SPA fallback answers every miss with index.html. A browser asking for a missing chunk needs a real 404.
+        // The SPA fallback answers every miss with index.html. A browser asking for a missing chunk needs the file from
+        // an earlier build, or a real 404.
         if (isHTMLResponse(assetResponse) && (isAASAPath || (!isNavigation && STATIC_FILE_PATH.test(url.pathname)))) {
-            const notFound = new Response('Not Found', {status: 404, headers: {'Content-Type': 'text/plain; charset=utf-8'}});
-            setSecurityHeaders(notFound.headers, env.ENVIRONMENT, nonce);
-            return notFound;
+            const previousBuildFile = isAASAPath ? null : await getPreviousBuildFile(env, url, request.method);
+            if (!previousBuildFile) {
+                const notFound = new Response('Not Found', {status: 404, headers: {'Content-Type': 'text/plain; charset=utf-8'}});
+                setSecurityHeaders(notFound.headers, env.ENVIRONMENT, nonce);
+                return notFound;
+            }
+            assetResponse = previousBuildFile;
         }
 
         const response = new Response(assetResponse.body, assetResponse);

web/cloudflare-worker/tests/createWorkerHarness.ts

@@ -30,8 +30,21 @@ const FAKE_WEB_BUILD: Record<string, string> = {
     '.well-known/apple-app-site-association': '{"applinks":{"details":[]}}',
 };
 
+type PreviousBuildFile = {
+    path: string;
+    contents: string;
+    contentType: string;
+};
+
+/** The part of the R2 binding the tests use; the full Workers types conflict with Node's globals in this project. */
+type PreviousBuildsEnv = {
+    PREVIOUS_BUILDS?: {put: (key: string, value: string, options: {httpMetadata: {contentType: string}}) => Promise<unknown>};
+};
+
 type WorkerHarness = {
     server: ReturnType<typeof createTestHarness>;
+    /** Stores files in the Worker's local PREVIOUS_BUILDS bucket, as a deploy of an earlier build would have. */
+    addPreviousBuildFiles: (files: PreviousBuildFile[]) => Promise<void>;
     cleanup: () => Promise<void>;
 };
 
@@ -70,8 +83,18 @@ function createWorkerHarness(env?: 'test-staging' | 'staging' | 'production', {o
     );
 
     const server = createTestHarness({workers: [{configPath, env}]});
+    const workerName = (env ? rawConfig.env?.[env]?.name : undefined) ?? rawConfig.name ?? '';
     return {
         server,
+        addPreviousBuildFiles: async (files) => {
+            const {PREVIOUS_BUILDS: previousBuilds} = await server.getWorker<PreviousBuildsEnv>(workerName).getEnv();
+            if (!previousBuilds) {
+                throw new Error(`The ${env ?? 'default'} environment has no PREVIOUS_BUILDS binding`);
+            }
+            for (const file of files) {
+                await previousBuilds.put(file.path, file.contents, {httpMetadata: {contentType: file.contentType}});
+            }
+        },
         cleanup: async () => {
             await server.close();
             rmSync(tempDir, {recursive: true, force: true});

web/cloudflare-worker/tests/worker.test.ts

@@ -207,6 +207,88 @@ describe('staging worker', () => {
     });
 });
 
+describe('staging worker with files from previous builds', () => {
+    const {server, addPreviousBuildFiles, cleanup} = createWorkerHarness('staging');
+    const PREVIOUS_CHUNK_PATH = '/main-aaaaaaaaaaaaaaaa.bundle.js';
+
+    before(async () => {
+        await server.listen();
+        await addPreviousBuildFiles([
+            {path: PREVIOUS_CHUNK_PATH.slice(1), contents: 'window.previousBuildChunk = true;', contentType: 'text/javascript'},
+            {path: 'canvaskit-0.40.0.wasm', contents: 'previous wasm', contentType: 'application/wasm'},
+            {path: HASHED_BUNDLE_PATH.slice(1), contents: 'window.previousCopyOfCurrentChunk = true;', contentType: 'text/javascript'},
+            {path: 'r/123/receipt.pdf', contents: 'not a page', contentType: 'application/pdf'},
+        ]);
+    });
+
+    after(async () => {
+        await cleanup();
+    });
+
+    test('serves a chunk the current build no longer has from a previous build', async () => {
+        // Given a tab still running an older build, whose chunk was uploaded to the bucket when that build deployed
+        // When the tab lazy-loads it after a newer deploy
+        const response = await server.fetch(PREVIOUS_CHUNK_PATH);
+
+        // Then it loads, so the tab keeps working instead of reloading
+        assert.equal(response.status, 200);
+        assert.equal(await response.text(), 'window.previousBuildChunk = true;');
+        assert.equal(response.headers.get('Content-Type'), 'text/javascript');
+        assert.equal(response.headers.get('Cache-Control'), 'public, max-age=31536000, immutable');
+        assertSecurityHeaders(response);
+    });
+
+    test('makes a previous build file without a content hash revalidate', async () => {
+        // Given a previous build file whose name has a version instead of a content hash
+        // When it is requested
+        const response = await server.fetch('/canvaskit-0.40.0.wasm');
+
+        // Then it is served, but not cached as immutable
+        assert.equal(response.status, 200);
+        assert.equal(response.headers.get('Content-Type'), 'application/wasm');
+        assert.equal(response.headers.get('Cache-Control'), 'public, max-age=0, must-revalidate');
+    });
+
+    test('answers HEAD for a previous build file without a body', async () => {
+        // Given a chunk that only exists in a previous build
+        // When it is requested with HEAD
+        const response = await server.fetch(PREVIOUS_CHUNK_PATH, {method: 'HEAD'});
+
+        // Then it reports the file exists, without sending it
+        assert.equal(response.status, 200);
+        assert.equal(response.headers.get('Content-Type'), 'text/javascript');
+        assert.equal(await response.text(), '');
+    });
+
+    test('serves the current build when a previous build has the same file', async () => {
+        // Given a file that is in both the current build and the bucket
+        // When it is requested
+        const response = await server.fetch(HASHED_BUNDLE_PATH);
+
+        // Then the current build's copy wins, so the bucket can never replace what was just deployed
+        assert.match(await response.text(), /fixtureBundle/);
+    });
+
+    test('never serves a previous build file to a navigation', async () => {
+        // Given a route that looks like a file and happens to match a key in the bucket
+        // When the browser navigates to it
+        const response = await server.fetch('/r/123/receipt.pdf', {headers: NAVIGATION_HEADERS});
+
+        // Then it gets the app shell, because only script and image requests can need an older build's files
+        assert.equal(response.status, 200);
+        assert.match(await response.text(), /New Expensify fixture/);
+    });
+
+    test('returns 404 for a file no build has', async () => {
+        // Given a chunk that is in neither the current build nor the bucket
+        // When it is requested
+        const response = await server.fetch('/main-ffffffffffffffff.bundle.js');
+
+        // Then it is a 404, so the client's ChunkLoadError recovery runs
+        assert.equal(response.status, 404);
+    });
+});
+
 describe('worker with a build that has no apple-app-site-association file', () => {
     const {server, cleanup} = createWorkerHarness('staging', {omittedFiles: ['.well-known/apple-app-site-association']});
 
@@ -274,4 +356,13 @@ describe('production worker', () => {
         assert.doesNotMatch(csp, /new-expensify:\/\/staging/);
         assertTemplateInlineScriptsAllowed(csp);
     });
+
+    test('returns 404 for a missing chunk without a previous builds bucket', async () => {
+        // Given production has no PREVIOUS_BUILDS binding yet
+        // When a chunk the current build doesn't have is requested
+        const response = await server.fetch('/main-ffffffffffffffff.bundle.js');
+
+        // Then it is a 404 rather than an error from the missing binding
+        assert.equal(response.status, 404);
+    });
 });

web/cloudflare-worker/wrangler.jsonc

@@ -23,21 +23,27 @@
         // Every response needs security headers, and HTML needs a per-request CSP nonce, including SPA navigations.
         "run_worker_first": true,
     },
+    // Files from previous staging builds, served when a tab running an older build requests a file the current build
+    // doesn't have. `wrangler dev` uses an empty local bucket.
+    "r2_buckets": [{"binding": "PREVIOUS_BUILDS", "bucket_name": "new-expensify-web-assets-staging"}],
     "vars": {
         "ENVIRONMENT": "staging",
     },
+    // Routes are managed in Cloudflare, not here: syncing routes needs zone access, and the CI deploy token can only
+    // edit the test-staging and staging Workers.
     "env": {
-        // Serves a staging build on a hostname nobody else uses, so it can be deployed and tested independently.
-        // Its route (test-staging.new.expensify.com/*) is managed in Cloudflare, not here: syncing routes needs zone
-        // access, and the CI deploy token can only edit this Worker.
+        // Serves a staging build on test-staging.new.expensify.com, a hostname nobody else uses.
         "test-staging": {
             "name": "test-staging-new-expensify-assets",
+            "r2_buckets": [{"binding": "PREVIOUS_BUILDS", "bucket_name": "new-expensify-web-assets-staging"}],
             "vars": {
                 "ENVIRONMENT": "staging",
             },
         },
+        // Serves nothing until staging.new.expensify.com/* is routed to this Worker.
         "staging": {
             "name": "staging-new-expensify-assets",
+            "r2_buckets": [{"binding": "PREVIOUS_BUILDS", "bucket_name": "new-expensify-web-assets-staging"}],
             // The Worker runs for every asset, so logging every request would exceed the included Workers Logs volume.
             "observability": {
                 "enabled": true,