From d22e5ad35aad4b7fa3b6b9f49a4ff74c4c7254b6 Mon Sep 17 00:00:00 2001 From: Zahin Mohammad Date: Fri, 24 Jul 2026 14:43:43 -0400 Subject: [PATCH] ci(root): switch beta publish recovery to auto-retry with version bump Stuck beta publishes retried on publish.yml can hit TLOG_CREATE_ENTRY_ERROR (409) sigstore/Rekor collisions: Rekor entries are keyed on the tarball's sha256 digest, which embeds the package version, and prepare-release.ts recomputes an identical next version (and therefore an identical digest) on retry since the stuck package's npm dist-tag never advanced after the failed registry write. `recovery-mode` is a single boolean workflow_dispatch input; the bump amount is decided automatically at actual publish time rather than guessed up front. scripts/verify-release.ts, which already loops over packages checking each one's live dist-tag against its local version and publishes any that are missing, now retries a still-missing package's `npm publish` call when recovery-mode is enabled and the failure output matches the Rekor conflict signature (TLOG_CREATE_ENTRY_ERROR or a 409 "transparency log" error), bumping that package's prerelease version between attempts (capped at 5) so its tarball digest is new. This is strictly opt-in via RECOVERY_MODE; behavior is unchanged when it's off. Also mark the "Lerna Publish" workflow step continue-on-error when recovery-mode is enabled, since a stuck package's Rekor conflict fails that step outright and would otherwise prevent the "Verify Publish" step (and its new retry logic) from running at all. No Linear ticket exists for this CI-only infra fix; using the INF-000 junk-drawer reference per the WEB-000/BTC-000/CSHLD-000 convention documented in CLAUDE.md, since INF (Infrastructure) is the closest registered issue prefix for a non-coin, non-web CI change. TICKET: INF-000 Co-Authored-By: Claude Sonnet 5 --- .github/workflows/publish.yml | 16 ++++++++ scripts/verify-release.ts | 70 +++++++++++++++++++++++++++++++---- 2 files changed, 79 insertions(+), 7 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a6de32846c..7f07d072f9 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,6 +1,18 @@ name: Publish @bitgo-beta on: workflow_dispatch: + inputs: + recovery-mode: + description: | + Recover a stuck beta publish where sigstore/Rekor already logged + provenance for a package's currently-computed version/tarball + (registry write failed) but the package never landed on npm. When + enabled, the publish step automatically detects the Rekor + conflict per-package and bumps that package's version until it + finds one that publishes cleanly - no manual input needed. + type: boolean + required: false + default: false # push: # branches: # - master @@ -89,6 +101,9 @@ jobs: run: git commit -am "Auto updated ${{ env.preid }} branch" --no-verify || echo "No changes to commit" - name: Lerna Publish + # In recovery mode a stuck package's Rekor conflict fails this step outright; let it + # continue so Verify Publish below still gets a chance to bump+retry that package. + continue-on-error: ${{ inputs.recovery-mode }} run: yarn lerna publish from-package --preid ${{ env.preid }} --dist-tag ${{ env.preid }} --force-publish --yes --loglevel silly env: NPM_CONFIG_PROVENANCE: true @@ -97,3 +112,4 @@ jobs: run: npx tsx ./scripts/verify-release.ts ${{ env.preid }} env: NPM_CONFIG_PROVENANCE: true + RECOVERY_MODE: ${{ inputs.recovery-mode }} diff --git a/scripts/verify-release.ts b/scripts/verify-release.ts index c4e9358e86..97b217af50 100644 --- a/scripts/verify-release.ts +++ b/scripts/verify-release.ts @@ -1,15 +1,67 @@ +import assert from 'node:assert'; import execa from 'execa'; -import { readFileSync } from 'fs'; +import { readFileSync, writeFileSync } from 'fs'; import path from 'path'; +import { inc } from 'semver'; import { getLernaModules, getDistTags } from './prepareRelease'; let lernaModuleLocations: string[] = []; +const RECOVERY_MODE = process.env.RECOVERY_MODE === 'true'; +const MAX_RECOVERY_ATTEMPTS = 5; + async function getLernaModuleLocations(): Promise { const modules = await getLernaModules(); lernaModuleLocations = modules.map(({ location }) => location); } +function isRekorConflict(output: string): boolean { + return ( + output.includes('TLOG_CREATE_ENTRY_ERROR') || + (output.includes('(409)') && output.includes('transparency log')) + ); +} + +// retries a stuck package's publish, bumping its version each time to sidestep an already-logged Rekor entry for that exact tarball digest +async function publishWithRecovery( + cwd: string, + json: any, + preid: string, +): Promise<{ stdout: string; exitCode: number }> { + const maxAttempts = RECOVERY_MODE ? MAX_RECOVERY_ATTEMPTS : 1; + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + return await execa( + 'npm', + ['publish', '--tag', preid, '--provenance'], + { cwd }, + ); + } catch (e: any) { + const output = `${e.stdout ?? ''}\n${e.stderr ?? ''}`; + if (!RECOVERY_MODE || !isRekorConflict(output)) { + throw e; + } + if (attempt === maxAttempts) { + throw new Error( + `${json.name}: still hitting a Rekor tarball-digest conflict after ${maxAttempts} version bumps`, + ); + } + const next = inc(json.version, 'prerelease', undefined, preid); + assert(typeof next === 'string', `Failed to increment version for ${json.name}`); + json.version = next; + writeFileSync( + path.join(cwd, 'package.json'), + JSON.stringify(json, null, 2) + '\n', + ); + console.warn( + `${json.name}: Rekor conflict, retrying with bumped version ${json.version} (attempt ${attempt + 1}/${maxAttempts})`, + ); + } + } + // unreachable: the loop above always returns or throws + throw new Error(`${json.name}: exhausted publish attempts`); +} + async function verifyPackage(dir: string, preid = 'beta'): Promise { const cwd = dir; const json = JSON.parse( @@ -25,11 +77,7 @@ async function verifyPackage(dir: string, preid = 'beta'): Promise { console.log( `${json.name} missing. Expected ${json.version}, latest is ${distTags[preid]}`, ); - const { stdout, exitCode } = await execa( - 'npm', - ['publish', '--tag', preid, '--provenance'], - { cwd }, - ); + const { stdout, exitCode } = await publishWithRecovery(cwd, json, preid); console.log(stdout); return exitCode === 0; } else { @@ -44,13 +92,21 @@ async function verifyPackage(dir: string, preid = 'beta'): Promise { async function verify(preid?: string) { await getLernaModuleLocations(); + let anyFailed = false; for (let i = 0; i < lernaModuleLocations.length; i++) { const dir = lernaModuleLocations[i]; if (!(await verifyPackage(dir, preid))) { console.error('Failed to verify outstanding packages.'); - return; + if (!RECOVERY_MODE) { + return; + } + // keep bumping/retrying the rest of the packages instead of aborting on the first stuck one + anyFailed = true; } } + if (anyFailed) { + process.exitCode = 1; + } } // e.g. for alpha releases: `npx tsx ./scripts/verify-beta.ts alpha`