diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a6de32846c..7f07d072f9 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,6 +1,18 @@ name: Publish @bitgo-beta on: workflow_dispatch: + inputs: + recovery-mode: + description: | + Recover a stuck beta publish where sigstore/Rekor already logged + provenance for a package's currently-computed version/tarball + (registry write failed) but the package never landed on npm. When + enabled, the publish step automatically detects the Rekor + conflict per-package and bumps that package's version until it + finds one that publishes cleanly - no manual input needed. + type: boolean + required: false + default: false # push: # branches: # - master @@ -89,6 +101,9 @@ jobs: run: git commit -am "Auto updated ${{ env.preid }} branch" --no-verify || echo "No changes to commit" - name: Lerna Publish + # In recovery mode a stuck package's Rekor conflict fails this step outright; let it + # continue so Verify Publish below still gets a chance to bump+retry that package. + continue-on-error: ${{ inputs.recovery-mode }} run: yarn lerna publish from-package --preid ${{ env.preid }} --dist-tag ${{ env.preid }} --force-publish --yes --loglevel silly env: NPM_CONFIG_PROVENANCE: true @@ -97,3 +112,4 @@ jobs: run: npx tsx ./scripts/verify-release.ts ${{ env.preid }} env: NPM_CONFIG_PROVENANCE: true + RECOVERY_MODE: ${{ inputs.recovery-mode }} diff --git a/scripts/verify-release.ts b/scripts/verify-release.ts index c4e9358e86..97b217af50 100644 --- a/scripts/verify-release.ts +++ b/scripts/verify-release.ts @@ -1,15 +1,67 @@ +import assert from 'node:assert'; import execa from 'execa'; -import { readFileSync } from 'fs'; +import { readFileSync, writeFileSync } from 'fs'; import path from 'path'; +import { inc } from 'semver'; import { getLernaModules, getDistTags } from './prepareRelease'; let lernaModuleLocations: string[] = []; +const RECOVERY_MODE = process.env.RECOVERY_MODE === 'true'; +const MAX_RECOVERY_ATTEMPTS = 5; + async function getLernaModuleLocations(): Promise { const modules = await getLernaModules(); lernaModuleLocations = modules.map(({ location }) => location); } +function isRekorConflict(output: string): boolean { + return ( + output.includes('TLOG_CREATE_ENTRY_ERROR') || + (output.includes('(409)') && output.includes('transparency log')) + ); +} + +// retries a stuck package's publish, bumping its version each time to sidestep an already-logged Rekor entry for that exact tarball digest +async function publishWithRecovery( + cwd: string, + json: any, + preid: string, +): Promise<{ stdout: string; exitCode: number }> { + const maxAttempts = RECOVERY_MODE ? MAX_RECOVERY_ATTEMPTS : 1; + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + return await execa( + 'npm', + ['publish', '--tag', preid, '--provenance'], + { cwd }, + ); + } catch (e: any) { + const output = `${e.stdout ?? ''}\n${e.stderr ?? ''}`; + if (!RECOVERY_MODE || !isRekorConflict(output)) { + throw e; + } + if (attempt === maxAttempts) { + throw new Error( + `${json.name}: still hitting a Rekor tarball-digest conflict after ${maxAttempts} version bumps`, + ); + } + const next = inc(json.version, 'prerelease', undefined, preid); + assert(typeof next === 'string', `Failed to increment version for ${json.name}`); + json.version = next; + writeFileSync( + path.join(cwd, 'package.json'), + JSON.stringify(json, null, 2) + '\n', + ); + console.warn( + `${json.name}: Rekor conflict, retrying with bumped version ${json.version} (attempt ${attempt + 1}/${maxAttempts})`, + ); + } + } + // unreachable: the loop above always returns or throws + throw new Error(`${json.name}: exhausted publish attempts`); +} + async function verifyPackage(dir: string, preid = 'beta'): Promise { const cwd = dir; const json = JSON.parse( @@ -25,11 +77,7 @@ async function verifyPackage(dir: string, preid = 'beta'): Promise { console.log( `${json.name} missing. Expected ${json.version}, latest is ${distTags[preid]}`, ); - const { stdout, exitCode } = await execa( - 'npm', - ['publish', '--tag', preid, '--provenance'], - { cwd }, - ); + const { stdout, exitCode } = await publishWithRecovery(cwd, json, preid); console.log(stdout); return exitCode === 0; } else { @@ -44,13 +92,21 @@ async function verifyPackage(dir: string, preid = 'beta'): Promise { async function verify(preid?: string) { await getLernaModuleLocations(); + let anyFailed = false; for (let i = 0; i < lernaModuleLocations.length; i++) { const dir = lernaModuleLocations[i]; if (!(await verifyPackage(dir, preid))) { console.error('Failed to verify outstanding packages.'); - return; + if (!RECOVERY_MODE) { + return; + } + // keep bumping/retrying the rest of the packages instead of aborting on the first stuck one + anyFailed = true; } } + if (anyFailed) { + process.exitCode = 1; + } } // e.g. for alpha releases: `npx tsx ./scripts/verify-beta.ts alpha`