From cba791c1220c4f23b592fb213335ee3dd57aa4dd Mon Sep 17 00:00:00 2001 From: Roman Bakaleyko Date: Mon, 17 Aug 2026 13:21:29 +0300 Subject: [PATCH 1/7] DEVOPS-6131: route sbt resolution through virtana-zing AR mirror (PR pipeline) The delta-sharing sbt build fails on jenkins-eng: Maven Central (repo1.maven.org) 429-rate-limits the shared egress IP, so sbt can't download its own launcher (org.scala-sbt:sbt:1.9.9) or its plugins/deps. Fix (sbt analog of the mirrorMavenCentral shared step): before `make build`, point sbt's Central resolution at the virtana-zing Artifact Registry Maven proxy, authenticated with a short-lived OAuth token minted from gcr_push_key: - ~/.sbt/repositories maven-central -> us-maven.pkg.dev/.../virtana-zing - ~/.config/coursier/... launcher (boot) auth - credentials.sbt / project/credentials.sbt build + plugin (meta) auth - pre-fetch sbt-launch.jar through the mirror (bearer header) - -Dsbt.override.build.repos=true Verified green end-to-end on a trimmed clone (rbak-test-build): no 429, no unauthorized, no token leak, image builds. Co-Authored-By: Claude Opus 4.8 (1M context) --- ci/Jenkinsfile | 67 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index 68c98ebd2..523209c16 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -22,7 +22,70 @@ node('docker-big') { try { stage('Build image') { ansiColor('xterm') { - sh("${MAKE} build") + // DEVOPS-6131: route sbt's Maven-Central resolution through the virtana-zing + // Artifact Registry mirror. jenkins-eng's shared egress IP gets 429'd by + // Central's Cloudflare front-end, which breaks sbt's own launcher/dependency + // download. This is the sbt analog of the mirrorMavenCentral shared step + // (sbt reads ~/.sbt/repositories + its native `credentials`, not settings.xml). + // Auth: short-lived OAuth token minted from gcr_push_key (zing-gcr-push, has + // artifactregistry read on zing-registry-188222). set +x keeps it out of the log. + withCredentials([file(credentialsId: 'gcr_push_key', variable: 'AR_KEY_FILE')]) { + sh ''' + set +x + set -e + MIRROR="https://us-maven.pkg.dev/zing-registry-188222/virtana-zing" + SBT_VERSION="$(awk -F= '/sbt.version/{gsub(/ /,"",$2);print $2}' project/build.properties)" + + # Mint the token in a throwaway gcloud config so the agent's own + # active account is left untouched (agents are reused across builds). + AR_TOKEN="$( + CLOUDSDK_CONFIG="$(mktemp -d)"; export CLOUDSDK_CONFIG + gcloud auth activate-service-account --key-file="$AR_KEY_FILE" --quiet 1>&2 + gcloud auth print-access-token + rm -rf "$CLOUDSDK_CONFIG" + )" + export AR_TOKEN + + # 1) sbt boot + dependency resolution: maven-central -> mirror. + mkdir -p "$HOME/.sbt" "$HOME/.config/coursier" + cat > "$HOME/.sbt/repositories" < the sbt LAUNCHER's shaded coursier (boot) + # - credentials.sbt -> the main build (library deps) + # - project/credentials.sbt -> the meta build (sbt plugins) + # lm-coursier only honors sbt's native `credentials`, not the file/env. + cat > "$HOME/.config/coursier/credentials.properties" < credentials.sbt + echo "$CRED_LINE" > project/credentials.sbt + + # 3) pre-fetch the sbt launch jar through the mirror (the bare curl in + # build/sbt-launch-lib.bash can't send an auth header itself). + mkdir -p build + curl --fail --location --silent \ + -H "Authorization: Bearer ${AR_TOKEN}" \ + "${MIRROR}/org/scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch-${SBT_VERSION}.jar" \ + -o "build/sbt-launch-${SBT_VERSION}.jar" + + # 4) force sbt to use ONLY the mirror repositories (no fallback to Central). + export SBT_OPTS="-Dsbt.repository.config=$HOME/.sbt/repositories -Dsbt.override.build.repos=true" + + make -f ci/Makefile build + ''' + } } } @@ -42,6 +105,8 @@ node('docker-big') { } finally { stage ('Clean test environment') { + // remove the token-bearing coursier properties from the shared agent HOME + sh('rm -f "$HOME/.config/coursier/credentials.properties" || true') sh("${MAKE} clean") } } From a4be646c8c4e4359c292e9d86dcbd6b7fb9d0a14 Mon Sep 17 00:00:00 2001 From: Roman Bakaleyko Date: Mon, 17 Aug 2026 13:44:54 +0300 Subject: [PATCH 2/7] DEVOPS-6131: move sbt-mirror logic into ci/sbt-mirror templates + script Refactor the inline pipeline sh into repo-side config templates and one setup script, so ci/Jenkinsfile just binds the credential and calls it: ci/sbt-mirror/ repositories static repo list (maven-central -> mirror) credentials.properties.tmpl coursier boot creds (token populated on the fly) credentials.sbt lm-coursier creds for deps + plugins (reads ~/.sbt/.ar-token) setup.sh mints token, renders the above, pre-fetches sbt-launch.jar, writes .sbtopts (-Dsbt.override.build.repos=true) Build stage is now: withCredentials([file(credentialsId:'gcr_push_key', variable:'AR_KEY_FILE')]) { sh 'ci/sbt-mirror/setup.sh && make -f ci/Makefile build' } Same behavior as the previous commit; the shared script lets the inline `build` job reuse identical logic. Generated copies (credentials.sbt, project/credentials.sbt, .sbtopts) are gitignored. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitignore | 3 + ci/Jenkinsfile | 71 +++------------------- ci/sbt-mirror/credentials.properties.tmpl | 4 ++ ci/sbt-mirror/credentials.sbt | 13 ++++ ci/sbt-mirror/repositories | 6 ++ ci/sbt-mirror/setup.sh | 72 +++++++++++++++++++++++ 6 files changed, 105 insertions(+), 64 deletions(-) create mode 100644 ci/sbt-mirror/credentials.properties.tmpl create mode 100644 ci/sbt-mirror/credentials.sbt create mode 100644 ci/sbt-mirror/repositories create mode 100755 ci/sbt-mirror/setup.sh diff --git a/.gitignore b/.gitignore index e3b431ec2..85e120df6 100644 --- a/.gitignore +++ b/.gitignore @@ -116,3 +116,6 @@ spark-warehouse/ # For venv *.venv .venv/ +/credentials.sbt +/project/credentials.sbt +/.sbtopts diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index 523209c16..dacdddec5 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -22,69 +22,12 @@ node('docker-big') { try { stage('Build image') { ansiColor('xterm') { - // DEVOPS-6131: route sbt's Maven-Central resolution through the virtana-zing - // Artifact Registry mirror. jenkins-eng's shared egress IP gets 429'd by - // Central's Cloudflare front-end, which breaks sbt's own launcher/dependency - // download. This is the sbt analog of the mirrorMavenCentral shared step - // (sbt reads ~/.sbt/repositories + its native `credentials`, not settings.xml). - // Auth: short-lived OAuth token minted from gcr_push_key (zing-gcr-push, has - // artifactregistry read on zing-registry-188222). set +x keeps it out of the log. + // DEVOPS-6131: sbt can't reach Maven Central from jenkins-eng (shared egress IP + // is 429'd), so ci/sbt-mirror/setup.sh points sbt at the virtana-zing Artifact + // Registry proxy before the build. Auth = short-lived token minted from + // gcr_push_key. All the logic + config templates live in ci/sbt-mirror/. withCredentials([file(credentialsId: 'gcr_push_key', variable: 'AR_KEY_FILE')]) { - sh ''' - set +x - set -e - MIRROR="https://us-maven.pkg.dev/zing-registry-188222/virtana-zing" - SBT_VERSION="$(awk -F= '/sbt.version/{gsub(/ /,"",$2);print $2}' project/build.properties)" - - # Mint the token in a throwaway gcloud config so the agent's own - # active account is left untouched (agents are reused across builds). - AR_TOKEN="$( - CLOUDSDK_CONFIG="$(mktemp -d)"; export CLOUDSDK_CONFIG - gcloud auth activate-service-account --key-file="$AR_KEY_FILE" --quiet 1>&2 - gcloud auth print-access-token - rm -rf "$CLOUDSDK_CONFIG" - )" - export AR_TOKEN - - # 1) sbt boot + dependency resolution: maven-central -> mirror. - mkdir -p "$HOME/.sbt" "$HOME/.config/coursier" - cat > "$HOME/.sbt/repositories" < the sbt LAUNCHER's shaded coursier (boot) - # - credentials.sbt -> the main build (library deps) - # - project/credentials.sbt -> the meta build (sbt plugins) - # lm-coursier only honors sbt's native `credentials`, not the file/env. - cat > "$HOME/.config/coursier/credentials.properties" < credentials.sbt - echo "$CRED_LINE" > project/credentials.sbt - - # 3) pre-fetch the sbt launch jar through the mirror (the bare curl in - # build/sbt-launch-lib.bash can't send an auth header itself). - mkdir -p build - curl --fail --location --silent \ - -H "Authorization: Bearer ${AR_TOKEN}" \ - "${MIRROR}/org/scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch-${SBT_VERSION}.jar" \ - -o "build/sbt-launch-${SBT_VERSION}.jar" - - # 4) force sbt to use ONLY the mirror repositories (no fallback to Central). - export SBT_OPTS="-Dsbt.repository.config=$HOME/.sbt/repositories -Dsbt.override.build.repos=true" - - make -f ci/Makefile build - ''' + sh("ci/sbt-mirror/setup.sh && ${MAKE} build") } } } @@ -105,8 +48,8 @@ EOF } finally { stage ('Clean test environment') { - // remove the token-bearing coursier properties from the shared agent HOME - sh('rm -f "$HOME/.config/coursier/credentials.properties" || true') + // remove token-bearing files setup.sh wrote into the shared agent HOME + sh('rm -f "$HOME/.config/coursier/credentials.properties" "$HOME/.sbt/.ar-token" || true') sh("${MAKE} clean") } } diff --git a/ci/sbt-mirror/credentials.properties.tmpl b/ci/sbt-mirror/credentials.properties.tmpl new file mode 100644 index 000000000..cbace2718 --- /dev/null +++ b/ci/sbt-mirror/credentials.properties.tmpl @@ -0,0 +1,4 @@ +virtana-zing.host=us-maven.pkg.dev +virtana-zing.username=oauth2accesstoken +virtana-zing.password=${AR_TOKEN} +virtana-zing.auto=true diff --git a/ci/sbt-mirror/credentials.sbt b/ci/sbt-mirror/credentials.sbt new file mode 100644 index 000000000..cf68cba6e --- /dev/null +++ b/ci/sbt-mirror/credentials.sbt @@ -0,0 +1,13 @@ +// DEVOPS-6131 (generated copy - do not edit here; source: ci/sbt-mirror/credentials.sbt). +// sbt's lm-coursier only authenticates via the native `credentials` setting (not the coursier +// properties file / COURSIER_CREDENTIALS env). ci/sbt-mirror/setup.sh drops this at both the build +// root and project/ (meta build) so library AND plugin resolution can reach the virtana-zing mirror. +// The token is read from ~/.sbt/.ar-token (written by setup.sh) so no secret lives in this file; +// if the token file is absent (e.g. a local dev build not using the mirror), it adds no credentials. +credentials ++= { + val tokenFile = file(sys.props("user.home")) / ".sbt" / ".ar-token" + if (tokenFile.exists) + Seq(Credentials("Artifact Registry", "us-maven.pkg.dev", "oauth2accesstoken", IO.read(tokenFile).trim)) + else + Seq.empty +} diff --git a/ci/sbt-mirror/repositories b/ci/sbt-mirror/repositories new file mode 100644 index 000000000..1df7d6cf5 --- /dev/null +++ b/ci/sbt-mirror/repositories @@ -0,0 +1,6 @@ +[repositories] + local + maven-central: https://us-maven.pkg.dev/zing-registry-188222/virtana-zing + sbt-plugin-releases: https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/, [organization]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext] + typesafe-ivy-releases: https://repo.typesafe.com/typesafe/ivy-releases/, [organization]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly + sbt-ivy-snapshots: https://repo.scala-sbt.org/scalasbt/ivy-snapshots/, [organization]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly diff --git a/ci/sbt-mirror/setup.sh b/ci/sbt-mirror/setup.sh new file mode 100755 index 000000000..81a37b6c2 --- /dev/null +++ b/ci/sbt-mirror/setup.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# +# DEVOPS-6131 - configure sbt to resolve Maven Central through the virtana-zing Artifact Registry +# proxy instead of hitting repo1.maven.org directly (jenkins-eng's shared egress IP gets 429'd by +# Central's Cloudflare front-end, which breaks sbt's launcher + dependency/plugin downloads). +# +# This is the sbt analog of the mirrorMavenCentral shared step: sbt doesn't read Maven's +# settings.xml, so we render sbt's own config from the templates next to this script and populate +# the short-lived credential on the fly. +# +# Usage (from a pipeline, credential bound as AR_KEY_FILE): +# withCredentials([file(credentialsId: 'gcr_push_key', variable: 'AR_KEY_FILE')]) { +# sh 'ci/sbt-mirror/setup.sh && make -f ci/Makefile build' +# } +# +# AR_KEY_FILE must point at a GCP service-account key with artifactregistry.reader on +# zing-registry-188222. sbt here runs directly on the Jenkins agent, so config is written to the +# agent HOME - no container mount needed. (If a build ran sbt inside a container, mount $HOME/.sbt +# and $HOME/.config/coursier into it.) +# +# Cleanup of the token-bearing files ($HOME/.sbt/.ar-token, coursier credentials.properties) is the +# caller's responsibility - do it in the pipeline's finally block. + +set -eu + +: "${AR_KEY_FILE:?ci/sbt-mirror/setup.sh: AR_KEY_FILE (path to a GCP SA key) must be set}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +MIRROR="https://us-maven.pkg.dev/zing-registry-188222/virtana-zing" +SBT_VERSION="$(awk -F= '/sbt.version/{gsub(/ /,"",$2);print $2}' "${REPO_ROOT}/project/build.properties")" + +# Mint a short-lived AR token in a throwaway gcloud config dir so the agent's own active gcloud +# account is left untouched (agents are reused across builds). xtrace off so it never hits the log. +{ set +x; } 2>/dev/null +AR_TOKEN="$( + CLOUDSDK_CONFIG="$(mktemp -d)"; export CLOUDSDK_CONFIG + gcloud auth activate-service-account --key-file="${AR_KEY_FILE}" --quiet 1>&2 + gcloud auth print-access-token + rm -rf "${CLOUDSDK_CONFIG}" +)" + +# 1) repositories: maven-central -> mirror (static template, no secret). +mkdir -p "${HOME}/.sbt" "${HOME}/.config/coursier" +cp "${SCRIPT_DIR}/repositories" "${HOME}/.sbt/repositories" + +# 2) credentials for the mirror host: +# a) coursier properties (token populated from template) -> the sbt LAUNCHER's shaded coursier (boot); +# b) token file + credentials.sbt at build root AND project/ (meta) -> lm-coursier for deps + plugins. +while IFS= read -r line || [ -n "${line}" ]; do + printf '%s\n' "${line//'${AR_TOKEN}'/${AR_TOKEN}}" +done < "${SCRIPT_DIR}/credentials.properties.tmpl" > "${HOME}/.config/coursier/credentials.properties" + +printf '%s' "${AR_TOKEN}" > "${HOME}/.sbt/.ar-token" +chmod 600 "${HOME}/.sbt/.ar-token" "${HOME}/.config/coursier/credentials.properties" +cp "${SCRIPT_DIR}/credentials.sbt" "${REPO_ROOT}/credentials.sbt" +cp "${SCRIPT_DIR}/credentials.sbt" "${REPO_ROOT}/project/credentials.sbt" + +# 3) pre-fetch the sbt launch jar through the mirror (build/sbt-launch-lib.bash's bare curl can't auth). +mkdir -p "${REPO_ROOT}/build" +curl --fail --location --silent -H "Authorization: Bearer ${AR_TOKEN}" \ + "${MIRROR}/org/scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch-${SBT_VERSION}.jar" \ + -o "${REPO_ROOT}/build/sbt-launch-${SBT_VERSION}.jar" + +# 4) force sbt to use ONLY the mirror repositories (no fallback to Central). build/sbt reads .sbtopts +# from the repo root at launch, so no env needs to survive into the separate `make` process. +cat > "${REPO_ROOT}/.sbtopts" < Date: Mon, 17 Aug 2026 13:51:29 +0300 Subject: [PATCH 3/7] DEVOPS-6131: fix credential delivery in refactor (use AR_TOKEN env, not file read) The previous refactor had project/credentials.sbt read the token via IO.read of a file; that didn't reach the meta build, so plugin resolution 401'd. Revert to the proven mechanism: credentials.sbt reads AR_TOKEN from the environment. setup.sh writes a sourceable .ar-token.env, and the pipeline does `ci/sbt-mirror/setup.sh && . ./.ar-token.env && make build` so $AR_TOKEN reaches sbt (setup.sh runs in its own process). No secret is committed; .ar-token.env is gitignored and removed in the finally block. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitignore | 1 + ci/Jenkinsfile | 6 +++--- ci/sbt-mirror/credentials.sbt | 15 ++++++--------- ci/sbt-mirror/setup.sh | 17 ++++++++++++----- 4 files changed, 22 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index 85e120df6..c99949884 100644 --- a/.gitignore +++ b/.gitignore @@ -119,3 +119,4 @@ spark-warehouse/ /credentials.sbt /project/credentials.sbt /.sbtopts +/.ar-token.env diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index dacdddec5..8d7356516 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -27,7 +27,7 @@ node('docker-big') { // Registry proxy before the build. Auth = short-lived token minted from // gcr_push_key. All the logic + config templates live in ci/sbt-mirror/. withCredentials([file(credentialsId: 'gcr_push_key', variable: 'AR_KEY_FILE')]) { - sh("ci/sbt-mirror/setup.sh && ${MAKE} build") + sh("ci/sbt-mirror/setup.sh && . ./.ar-token.env && ${MAKE} build") } } } @@ -48,8 +48,8 @@ node('docker-big') { } finally { stage ('Clean test environment') { - // remove token-bearing files setup.sh wrote into the shared agent HOME - sh('rm -f "$HOME/.config/coursier/credentials.properties" "$HOME/.sbt/.ar-token" || true') + // remove token-bearing files setup.sh wrote (HOME + workspace) + sh('rm -f "$HOME/.config/coursier/credentials.properties" ./.ar-token.env || true') sh("${MAKE} clean") } } diff --git a/ci/sbt-mirror/credentials.sbt b/ci/sbt-mirror/credentials.sbt index cf68cba6e..26c2abf24 100644 --- a/ci/sbt-mirror/credentials.sbt +++ b/ci/sbt-mirror/credentials.sbt @@ -2,12 +2,9 @@ // sbt's lm-coursier only authenticates via the native `credentials` setting (not the coursier // properties file / COURSIER_CREDENTIALS env). ci/sbt-mirror/setup.sh drops this at both the build // root and project/ (meta build) so library AND plugin resolution can reach the virtana-zing mirror. -// The token is read from ~/.sbt/.ar-token (written by setup.sh) so no secret lives in this file; -// if the token file is absent (e.g. a local dev build not using the mirror), it adds no credentials. -credentials ++= { - val tokenFile = file(sys.props("user.home")) / ".sbt" / ".ar-token" - if (tokenFile.exists) - Seq(Credentials("Artifact Registry", "us-maven.pkg.dev", "oauth2accesstoken", IO.read(tokenFile).trim)) - else - Seq.empty -} +// The token comes from the AR_TOKEN env var (exported by sourcing .ar-token.env before the build), +// so no secret lives in this file; with AR_TOKEN unset (e.g. a local dev build not using the +// mirror) it adds no credentials. +credentials ++= sys.env.get("AR_TOKEN").filter(_.nonEmpty).map { token => + Credentials("Artifact Registry", "us-maven.pkg.dev", "oauth2accesstoken", token) +}.toSeq diff --git a/ci/sbt-mirror/setup.sh b/ci/sbt-mirror/setup.sh index 81a37b6c2..060bc5311 100755 --- a/ci/sbt-mirror/setup.sh +++ b/ci/sbt-mirror/setup.sh @@ -10,15 +10,16 @@ # # Usage (from a pipeline, credential bound as AR_KEY_FILE): # withCredentials([file(credentialsId: 'gcr_push_key', variable: 'AR_KEY_FILE')]) { -# sh 'ci/sbt-mirror/setup.sh && make -f ci/Makefile build' +# sh 'ci/sbt-mirror/setup.sh && . ./.ar-token.env && make -f ci/Makefile build' # } +# (source .ar-token.env so $AR_TOKEN reaches sbt; setup.sh runs in its own process.) # # AR_KEY_FILE must point at a GCP service-account key with artifactregistry.reader on # zing-registry-188222. sbt here runs directly on the Jenkins agent, so config is written to the # agent HOME - no container mount needed. (If a build ran sbt inside a container, mount $HOME/.sbt # and $HOME/.config/coursier into it.) # -# Cleanup of the token-bearing files ($HOME/.sbt/.ar-token, coursier credentials.properties) is the +# Cleanup of the token-bearing files (.ar-token.env, coursier credentials.properties) is the # caller's responsibility - do it in the pipeline's finally block. set -eu @@ -46,16 +47,22 @@ cp "${SCRIPT_DIR}/repositories" "${HOME}/.sbt/repositories" # 2) credentials for the mirror host: # a) coursier properties (token populated from template) -> the sbt LAUNCHER's shaded coursier (boot); -# b) token file + credentials.sbt at build root AND project/ (meta) -> lm-coursier for deps + plugins. +# b) credentials.sbt at build root AND project/ (meta) -> lm-coursier for deps + plugins. It reads +# the token from $AR_TOKEN, delivered by sourcing .ar-token.env just before `make` (lm-coursier +# honors only sbt's native `credentials`, not the coursier file/env). while IFS= read -r line || [ -n "${line}" ]; do printf '%s\n' "${line//'${AR_TOKEN}'/${AR_TOKEN}}" done < "${SCRIPT_DIR}/credentials.properties.tmpl" > "${HOME}/.config/coursier/credentials.properties" +chmod 600 "${HOME}/.config/coursier/credentials.properties" -printf '%s' "${AR_TOKEN}" > "${HOME}/.sbt/.ar-token" -chmod 600 "${HOME}/.sbt/.ar-token" "${HOME}/.config/coursier/credentials.properties" cp "${SCRIPT_DIR}/credentials.sbt" "${REPO_ROOT}/credentials.sbt" cp "${SCRIPT_DIR}/credentials.sbt" "${REPO_ROOT}/project/credentials.sbt" +# AR_TOKEN for lm-coursier: written to a sourceable env file rather than exported, because setup.sh +# runs as its own process - the caller does `. ./.ar-token.env` before make so it reaches sbt. +printf "export AR_TOKEN='%s'\n" "${AR_TOKEN}" > "${REPO_ROOT}/.ar-token.env" +chmod 600 "${REPO_ROOT}/.ar-token.env" + # 3) pre-fetch the sbt launch jar through the mirror (build/sbt-launch-lib.bash's bare curl can't auth). mkdir -p "${REPO_ROOT}/build" curl --fail --location --silent -H "Authorization: Bearer ${AR_TOKEN}" \ From 7163277ad0a91435a3f3e10fb2881bcd5ab05ac2 Mon Sep 17 00:00:00 2001 From: Roman Bakaleyko Date: Mon, 17 Aug 2026 13:56:52 +0300 Subject: [PATCH 4/7] DEVOPS-6131: prevent token echo (set +x) and restore proven credentials.sbt - Jenkins runs sh with -x, so sourcing .ar-token.env echoed the token; prefix the build step with `set +x`. - Revert credentials.sbt to the exact Credentials form proven to authenticate plugin + dependency resolution against the mirror. Co-Authored-By: Claude Opus 4.8 (1M context) --- ci/Jenkinsfile | 4 +++- ci/sbt-mirror/credentials.sbt | 8 +++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index 8d7356516..05cc576a5 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -27,7 +27,9 @@ node('docker-big') { // Registry proxy before the build. Auth = short-lived token minted from // gcr_push_key. All the logic + config templates live in ci/sbt-mirror/. withCredentials([file(credentialsId: 'gcr_push_key', variable: 'AR_KEY_FILE')]) { - sh("ci/sbt-mirror/setup.sh && . ./.ar-token.env && ${MAKE} build") + // set +x FIRST: sourcing .ar-token.env would otherwise echo the token + // (Jenkins runs sh with -x). Keep this leading `set +x`. + sh("set +x; ci/sbt-mirror/setup.sh && . ./.ar-token.env && ${MAKE} build") } } } diff --git a/ci/sbt-mirror/credentials.sbt b/ci/sbt-mirror/credentials.sbt index 26c2abf24..3b4b2f933 100644 --- a/ci/sbt-mirror/credentials.sbt +++ b/ci/sbt-mirror/credentials.sbt @@ -3,8 +3,6 @@ // properties file / COURSIER_CREDENTIALS env). ci/sbt-mirror/setup.sh drops this at both the build // root and project/ (meta build) so library AND plugin resolution can reach the virtana-zing mirror. // The token comes from the AR_TOKEN env var (exported by sourcing .ar-token.env before the build), -// so no secret lives in this file; with AR_TOKEN unset (e.g. a local dev build not using the -// mirror) it adds no credentials. -credentials ++= sys.env.get("AR_TOKEN").filter(_.nonEmpty).map { token => - Credentials("Artifact Registry", "us-maven.pkg.dev", "oauth2accesstoken", token) -}.toSeq +// so no secret lives in this file; with AR_TOKEN unset (local dev, mirror not used) the password is +// empty and this credential is simply never matched/used. +credentials += Credentials("", "us-maven.pkg.dev", "oauth2accesstoken", sys.env.getOrElse("AR_TOKEN", "")) From 82f7f3d0e25181e1440ad7eb2164cb3fa9163bb6 Mon Sep 17 00:00:00 2001 From: Roman Bakaleyko Date: Mon, 17 Aug 2026 14:11:15 +0300 Subject: [PATCH 5/7] DEVOPS-6131: set AR realm on credentials to silence Ivy "Unable to find credentials" sbt's Ivy code path (makePom/metadata) matches credentials by realm, not just host. With an empty realm it logged 4x "Unable to find credentials for [https://us-maven.pkg.dev @ us-maven.pkg.dev]" (non-fatal; coursier still resolved by host). Set the realm to exactly what AR advertises (WWW-Authenticate: Basic realm="https://us-maven.pkg.dev") so Ivy matches too. Co-Authored-By: Claude Opus 4.8 (1M context) --- ci/sbt-mirror/credentials.sbt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ci/sbt-mirror/credentials.sbt b/ci/sbt-mirror/credentials.sbt index 3b4b2f933..031c65325 100644 --- a/ci/sbt-mirror/credentials.sbt +++ b/ci/sbt-mirror/credentials.sbt @@ -5,4 +5,10 @@ // The token comes from the AR_TOKEN env var (exported by sourcing .ar-token.env before the build), // so no secret lives in this file; with AR_TOKEN unset (local dev, mirror not used) the password is // empty and this credential is simply never matched/used. -credentials += Credentials("", "us-maven.pkg.dev", "oauth2accesstoken", sys.env.getOrElse("AR_TOKEN", "")) +// +// The realm MUST be "https://us-maven.pkg.dev" (exactly what AR sends in its WWW-Authenticate +// header). coursier matches credentials by host alone, but sbt's Ivy code path matches by realm too; +// an empty realm makes Ivy log "Unable to find credentials for [https://us-maven.pkg.dev @ ...]" +// during makePom/metadata even though the build still succeeds via coursier. Matching the realm +// silences those. +credentials += Credentials("https://us-maven.pkg.dev", "us-maven.pkg.dev", "oauth2accesstoken", sys.env.getOrElse("AR_TOKEN", "")) From 487f0ccad7e03fb2f17719cc9f30463f2c70ea1f Mon Sep 17 00:00:00 2001 From: Roman Bakaleyko Date: Mon, 17 Aug 2026 14:25:36 +0300 Subject: [PATCH 6/7] DEVOPS-6131: scope AR credentials to ThisBuild (silence subproject Ivy warnings) delta-sharing is multi-project (server/client/spark). A plain `credentials += ...` is root-project-only, so Ivy resolution for the subprojects logged "Unable to find credentials for [https://us-maven.pkg.dev @ us-maven.pkg.dev]" (non-fatal; coursier still downloaded everything build-wide). Scope to `ThisBuild / credentials` so all subprojects have the credential. Co-Authored-By: Claude Opus 4.8 (1M context) --- ci/sbt-mirror/credentials.sbt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/sbt-mirror/credentials.sbt b/ci/sbt-mirror/credentials.sbt index 031c65325..b1ba78384 100644 --- a/ci/sbt-mirror/credentials.sbt +++ b/ci/sbt-mirror/credentials.sbt @@ -6,9 +6,9 @@ // so no secret lives in this file; with AR_TOKEN unset (local dev, mirror not used) the password is // empty and this credential is simply never matched/used. // -// The realm MUST be "https://us-maven.pkg.dev" (exactly what AR sends in its WWW-Authenticate -// header). coursier matches credentials by host alone, but sbt's Ivy code path matches by realm too; -// an empty realm makes Ivy log "Unable to find credentials for [https://us-maven.pkg.dev @ ...]" -// during makePom/metadata even though the build still succeeds via coursier. Matching the realm -// silences those. -credentials += Credentials("https://us-maven.pkg.dev", "us-maven.pkg.dev", "oauth2accesstoken", sys.env.getOrElse("AR_TOKEN", "")) +// Scope = ThisBuild so EVERY subproject (server/client/spark) sees the credential. A plain +// `credentials += ...` is root-project-only, so Ivy resolution for the subprojects (scalastyle, +// makePom) found no credential and logged "Unable to find credentials for [... @ us-maven.pkg.dev]" +// even though coursier (which aggregates credentials build-wide) still downloaded everything. +// Realm matches exactly what AR sends in WWW-Authenticate: Basic realm="https://us-maven.pkg.dev". +ThisBuild / credentials += Credentials("https://us-maven.pkg.dev", "us-maven.pkg.dev", "oauth2accesstoken", sys.env.getOrElse("AR_TOKEN", "")) From 079285dd7e03d00a7f362e282ddaf74026fcf5fe Mon Sep 17 00:00:00 2001 From: Roman Bakaleyko Date: Mon, 17 Aug 2026 14:39:33 +0300 Subject: [PATCH 7/7] DEVOPS-6131: consolidate into with-mirror.sh wrapper (in-process token, self-cleaning) Replace the setup.sh + .ar-token.env + pipeline-finally-cleanup dance with a single wrapper that runs the build with the mirror configured: ci/sbt-mirror/with-mirror.sh make -f ci/Makefile build - Token stays IN-PROCESS (exported to the child build only); no token is written to a file that gets passed around. The one transient file that must hold it (coursier boot creds) plus the rendered sbt config are removed by an EXIT trap, pass or fail. - gcloud auth already uses a throwaway CLOUDSDK_CONFIG dir, so the agent's own gcloud account is never activated/replaced (it is not `gcloud auth login`). - repositories now written workspace-local (.sbt-mirror-repositories) so we don't clobber a shared ~/.sbt/repositories on the agent. Pipeline Build stage is now a single line; no finally cleanup of auth files needed. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitignore | 2 +- ci/Jenkinsfile | 15 +++---- ci/sbt-mirror/setup.sh | 79 ---------------------------------- ci/sbt-mirror/with-mirror.sh | 82 ++++++++++++++++++++++++++++++++++++ 4 files changed, 90 insertions(+), 88 deletions(-) delete mode 100755 ci/sbt-mirror/setup.sh create mode 100755 ci/sbt-mirror/with-mirror.sh diff --git a/.gitignore b/.gitignore index c99949884..144de683f 100644 --- a/.gitignore +++ b/.gitignore @@ -119,4 +119,4 @@ spark-warehouse/ /credentials.sbt /project/credentials.sbt /.sbtopts -/.ar-token.env +/.sbt-mirror-repositories diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index 05cc576a5..cae8f61b2 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -23,13 +23,13 @@ node('docker-big') { stage('Build image') { ansiColor('xterm') { // DEVOPS-6131: sbt can't reach Maven Central from jenkins-eng (shared egress IP - // is 429'd), so ci/sbt-mirror/setup.sh points sbt at the virtana-zing Artifact - // Registry proxy before the build. Auth = short-lived token minted from - // gcr_push_key. All the logic + config templates live in ci/sbt-mirror/. + // is 429'd). ci/sbt-mirror/with-mirror.sh runs the build with sbt pointed at the + // virtana-zing Artifact Registry proxy: it mints a short-lived token from + // gcr_push_key (in-process; gcloud uses a throwaway config so the agent account + // is untouched) and cleans up its transient config on exit. Logic + templates + // all live in ci/sbt-mirror/. withCredentials([file(credentialsId: 'gcr_push_key', variable: 'AR_KEY_FILE')]) { - // set +x FIRST: sourcing .ar-token.env would otherwise echo the token - // (Jenkins runs sh with -x). Keep this leading `set +x`. - sh("set +x; ci/sbt-mirror/setup.sh && . ./.ar-token.env && ${MAKE} build") + sh("ci/sbt-mirror/with-mirror.sh ${MAKE} build") } } } @@ -50,8 +50,7 @@ node('docker-big') { } finally { stage ('Clean test environment') { - // remove token-bearing files setup.sh wrote (HOME + workspace) - sh('rm -f "$HOME/.config/coursier/credentials.properties" ./.ar-token.env || true') + // with-mirror.sh removes its own auth files via an EXIT trap; just clean the build. sh("${MAKE} clean") } } diff --git a/ci/sbt-mirror/setup.sh b/ci/sbt-mirror/setup.sh deleted file mode 100755 index 060bc5311..000000000 --- a/ci/sbt-mirror/setup.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env bash -# -# DEVOPS-6131 - configure sbt to resolve Maven Central through the virtana-zing Artifact Registry -# proxy instead of hitting repo1.maven.org directly (jenkins-eng's shared egress IP gets 429'd by -# Central's Cloudflare front-end, which breaks sbt's launcher + dependency/plugin downloads). -# -# This is the sbt analog of the mirrorMavenCentral shared step: sbt doesn't read Maven's -# settings.xml, so we render sbt's own config from the templates next to this script and populate -# the short-lived credential on the fly. -# -# Usage (from a pipeline, credential bound as AR_KEY_FILE): -# withCredentials([file(credentialsId: 'gcr_push_key', variable: 'AR_KEY_FILE')]) { -# sh 'ci/sbt-mirror/setup.sh && . ./.ar-token.env && make -f ci/Makefile build' -# } -# (source .ar-token.env so $AR_TOKEN reaches sbt; setup.sh runs in its own process.) -# -# AR_KEY_FILE must point at a GCP service-account key with artifactregistry.reader on -# zing-registry-188222. sbt here runs directly on the Jenkins agent, so config is written to the -# agent HOME - no container mount needed. (If a build ran sbt inside a container, mount $HOME/.sbt -# and $HOME/.config/coursier into it.) -# -# Cleanup of the token-bearing files (.ar-token.env, coursier credentials.properties) is the -# caller's responsibility - do it in the pipeline's finally block. - -set -eu - -: "${AR_KEY_FILE:?ci/sbt-mirror/setup.sh: AR_KEY_FILE (path to a GCP SA key) must be set}" - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" -MIRROR="https://us-maven.pkg.dev/zing-registry-188222/virtana-zing" -SBT_VERSION="$(awk -F= '/sbt.version/{gsub(/ /,"",$2);print $2}' "${REPO_ROOT}/project/build.properties")" - -# Mint a short-lived AR token in a throwaway gcloud config dir so the agent's own active gcloud -# account is left untouched (agents are reused across builds). xtrace off so it never hits the log. -{ set +x; } 2>/dev/null -AR_TOKEN="$( - CLOUDSDK_CONFIG="$(mktemp -d)"; export CLOUDSDK_CONFIG - gcloud auth activate-service-account --key-file="${AR_KEY_FILE}" --quiet 1>&2 - gcloud auth print-access-token - rm -rf "${CLOUDSDK_CONFIG}" -)" - -# 1) repositories: maven-central -> mirror (static template, no secret). -mkdir -p "${HOME}/.sbt" "${HOME}/.config/coursier" -cp "${SCRIPT_DIR}/repositories" "${HOME}/.sbt/repositories" - -# 2) credentials for the mirror host: -# a) coursier properties (token populated from template) -> the sbt LAUNCHER's shaded coursier (boot); -# b) credentials.sbt at build root AND project/ (meta) -> lm-coursier for deps + plugins. It reads -# the token from $AR_TOKEN, delivered by sourcing .ar-token.env just before `make` (lm-coursier -# honors only sbt's native `credentials`, not the coursier file/env). -while IFS= read -r line || [ -n "${line}" ]; do - printf '%s\n' "${line//'${AR_TOKEN}'/${AR_TOKEN}}" -done < "${SCRIPT_DIR}/credentials.properties.tmpl" > "${HOME}/.config/coursier/credentials.properties" -chmod 600 "${HOME}/.config/coursier/credentials.properties" - -cp "${SCRIPT_DIR}/credentials.sbt" "${REPO_ROOT}/credentials.sbt" -cp "${SCRIPT_DIR}/credentials.sbt" "${REPO_ROOT}/project/credentials.sbt" - -# AR_TOKEN for lm-coursier: written to a sourceable env file rather than exported, because setup.sh -# runs as its own process - the caller does `. ./.ar-token.env` before make so it reaches sbt. -printf "export AR_TOKEN='%s'\n" "${AR_TOKEN}" > "${REPO_ROOT}/.ar-token.env" -chmod 600 "${REPO_ROOT}/.ar-token.env" - -# 3) pre-fetch the sbt launch jar through the mirror (build/sbt-launch-lib.bash's bare curl can't auth). -mkdir -p "${REPO_ROOT}/build" -curl --fail --location --silent -H "Authorization: Bearer ${AR_TOKEN}" \ - "${MIRROR}/org/scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch-${SBT_VERSION}.jar" \ - -o "${REPO_ROOT}/build/sbt-launch-${SBT_VERSION}.jar" - -# 4) force sbt to use ONLY the mirror repositories (no fallback to Central). build/sbt reads .sbtopts -# from the repo root at launch, so no env needs to survive into the separate `make` process. -cat > "${REPO_ROOT}/.sbtopts" <" >&2; exit 2; } + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +MIRROR="https://us-maven.pkg.dev/zing-registry-188222/virtana-zing" +SBT_VERSION="$(awk -F= '/sbt.version/{gsub(/ /,"",$2);print $2}' "${REPO_ROOT}/project/build.properties")" + +COURSIER_CREDS="${HOME}/.config/coursier/credentials.properties" +SBT_REPOS="${REPO_ROOT}/.sbt-mirror-repositories" + +# Remove every transient auth/config file on exit - the token never outlives this build. +cleanup() { + rm -f "${COURSIER_CREDS}" "${SBT_REPOS}" "${REPO_ROOT}/.sbtopts" \ + "${REPO_ROOT}/credentials.sbt" "${REPO_ROOT}/project/credentials.sbt" +} +trap cleanup EXIT + +# Mint a short-lived AR token in a throwaway gcloud config dir (agent's own account untouched). +# xtrace guarded off so the token is never echoed even if the caller runs us under `bash -x`. +{ set +x; } 2>/dev/null +AR_TOKEN="$( + CLOUDSDK_CONFIG="$(mktemp -d)"; export CLOUDSDK_CONFIG + gcloud auth activate-service-account --key-file="${AR_KEY_FILE}" --quiet 1>&2 + gcloud auth print-access-token + rm -rf "${CLOUDSDK_CONFIG}" +)" +export AR_TOKEN + +# repositories: workspace-local (so we don't clobber a shared ~/.sbt/repositories on the agent). +cp "${SCRIPT_DIR}/repositories" "${SBT_REPOS}" + +# coursier boot creds (token populated from template) - for the sbt LAUNCHER's shaded coursier. +mkdir -p "$(dirname "${COURSIER_CREDS}")" +while IFS= read -r line || [ -n "${line}" ]; do + printf '%s\n' "${line//'${AR_TOKEN}'/${AR_TOKEN}}" +done < "${SCRIPT_DIR}/credentials.properties.tmpl" > "${COURSIER_CREDS}" +chmod 600 "${COURSIER_CREDS}" + +# lm-coursier + Ivy creds for deps + plugins - read $AR_TOKEN from the env (no secret on disk). +cp "${SCRIPT_DIR}/credentials.sbt" "${REPO_ROOT}/credentials.sbt" +cp "${SCRIPT_DIR}/credentials.sbt" "${REPO_ROOT}/project/credentials.sbt" + +# force sbt to use ONLY the mirror repositories (no fallback to Central). +cat > "${REPO_ROOT}/.sbtopts" < ${MIRROR} (sbt ${SBT_VERSION}); running: $*" +"$@"