ci(root): add recovery-mode to beta publish for auto-retry on Rekor conflicts

.github/workflows/publish.yml

@@ -1,6 +1,18 @@
 name: Publish @bitgo-beta
 on:
   workflow_dispatch:
+    inputs:
+      recovery-mode:
+        description: |
+          Recover a stuck beta publish where sigstore/Rekor already logged
+          provenance for a package's currently-computed version/tarball
+          (registry write failed) but the package never landed on npm. When
+          enabled, the publish step automatically detects the Rekor
+          conflict per-package and bumps that package's version until it
+          finds one that publishes cleanly - no manual input needed.
+        type: boolean
+        required: false
+        default: false
   # push:
   #   branches:
   #     - master
@@ -89,6 +101,9 @@ jobs:
         run: git commit -am "Auto updated ${{ env.preid }} branch" --no-verify || echo "No changes to commit"
 
       - name: Lerna Publish
+        # In recovery mode a stuck package's Rekor conflict fails this step outright; let it
+        # continue so Verify Publish below still gets a chance to bump+retry that package.
+        continue-on-error: ${{ inputs.recovery-mode }}
         run: yarn lerna publish from-package --preid ${{ env.preid }} --dist-tag ${{ env.preid }} --force-publish --yes --loglevel silly
         env:
           NPM_CONFIG_PROVENANCE: true
@@ -97,3 +112,4 @@ jobs:
         run: npx tsx ./scripts/verify-release.ts ${{ env.preid }}
         env:
           NPM_CONFIG_PROVENANCE: true
+          RECOVERY_MODE: ${{ inputs.recovery-mode }}

scripts/verify-release.ts

@@ -1,15 +1,67 @@
+import assert from 'node:assert';
 import execa from 'execa';
-import { readFileSync } from 'fs';
+import { readFileSync, writeFileSync } from 'fs';
 import path from 'path';
+import { inc } from 'semver';
 import { getLernaModules, getDistTags } from './prepareRelease';
 
 let lernaModuleLocations: string[] = [];
 
+const RECOVERY_MODE = process.env.RECOVERY_MODE === 'true';
+const MAX_RECOVERY_ATTEMPTS = 5;
+
 async function getLernaModuleLocations(): Promise<void> {
   const modules = await getLernaModules();
   lernaModuleLocations = modules.map(({ location }) => location);
 }
 
+function isRekorConflict(output: string): boolean {
+  return (
+    output.includes('TLOG_CREATE_ENTRY_ERROR') ||
+    (output.includes('(409)') && output.includes('transparency log'))
+  );
+}
+
+// retries a stuck package's publish, bumping its version each time to sidestep an already-logged Rekor entry for that exact tarball digest
+async function publishWithRecovery(
+  cwd: string,
+  json: any,
+  preid: string,
+): Promise<{ stdout: string; exitCode: number }> {
+  const maxAttempts = RECOVERY_MODE ? MAX_RECOVERY_ATTEMPTS : 1;
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    try {
+      return await execa(
+        'npm',
+        ['publish', '--tag', preid, '--provenance'],
+        { cwd },
+      );
+    } catch (e: any) {
+      const output = `${e.stdout ?? ''}\n${e.stderr ?? ''}`;
+      if (!RECOVERY_MODE || !isRekorConflict(output)) {
+        throw e;
+      }
+      if (attempt === maxAttempts) {
+        throw new Error(
+          `${json.name}: still hitting a Rekor tarball-digest conflict after ${maxAttempts} version bumps`,
+        );
+      }
+      const next = inc(json.version, 'prerelease', undefined, preid);
+      assert(typeof next === 'string', `Failed to increment version for ${json.name}`);
+      json.version = next;
+      writeFileSync(
+        path.join(cwd, 'package.json'),
+        JSON.stringify(json, null, 2) + '\n',
+      );
+      console.warn(
+        `${json.name}: Rekor conflict, retrying with bumped version ${json.version} (attempt ${attempt + 1}/${maxAttempts})`,
+      );
+    }
+  }
+  // unreachable: the loop above always returns or throws
+  throw new Error(`${json.name}: exhausted publish attempts`);
+}
+
 async function verifyPackage(dir: string, preid = 'beta'): Promise<boolean> {
   const cwd = dir;
   const json = JSON.parse(
@@ -25,11 +77,7 @@ async function verifyPackage(dir: string, preid = 'beta'): Promise<boolean> {
       console.log(
         `${json.name} missing. Expected ${json.version}, latest is ${distTags[preid]}`,
       );
-      const { stdout, exitCode } = await execa(
-        'npm',
-        ['publish', '--tag', preid, '--provenance'],
-        { cwd },
-      );
+      const { stdout, exitCode } = await publishWithRecovery(cwd, json, preid);
       console.log(stdout);
       return exitCode === 0;
     } else {
@@ -44,13 +92,21 @@ async function verifyPackage(dir: string, preid = 'beta'): Promise<boolean> {
 
 async function verify(preid?: string) {
   await getLernaModuleLocations();
+  let anyFailed = false;
   for (let i = 0; i < lernaModuleLocations.length; i++) {
     const dir = lernaModuleLocations[i];
     if (!(await verifyPackage(dir, preid))) {
       console.error('Failed to verify outstanding packages.');
-      return;
+      if (!RECOVERY_MODE) {
+        return;
+      }
+      // keep bumping/retrying the rest of the packages instead of aborting on the first stuck one
+      anyFailed = true;
     }
   }
+  if (anyFailed) {
+    process.exitCode = 1;
+  }
 }
 
 // e.g. for alpha releases: `npx tsx ./scripts/verify-beta.ts alpha`