diff --git a/.buildkite/pipelines/derive_qa_stack_env.sh b/.buildkite/pipelines/derive_qa_stack_env.sh new file mode 100755 index 000000000..f259b2bfb --- /dev/null +++ b/.buildkite/pipelines/derive_qa_stack_env.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. + +# Derive STACK_VERSION and ES_BRANCH for the Appex QA / PyTorch downstream +# trigger (appex-qa-stateful-custom-ml-cpp-build-testing, generated by +# elastic/qaf-tests). Without these the downstream falls back to its defaults +# (ES_BRANCH=main, STACK_VERSION=) and then tries to +# download an artifact named ml-cpp--SNAPSHOT-... from the parent +# build. On a release-branch / backport build the parent produced +# ml-cpp--SNAPSHOT-... instead, so the download misses +# ("No artifacts found") and the QA build fails spuriously. +# +# This is meant to be SOURCED by the run_*_tests.yml.sh generators. Those +# scripts pipe their stdout straight into `buildkite-agent pipeline upload`, so +# this helper MUST NOT print anything to stdout — all diagnostics go to stderr. +# Any values already present in the environment (e.g. from PR comment vars +# GITHUB_PR_COMMENT_VAR_BRANCH / _VERSION) are respected and never overwritten. + +# STACK_VERSION: the ml-cpp product version for this build. gradle.properties' +# elasticsearchVersion is exactly the artifact version (e.g. 9.5.0 on the 9.5 +# branch, 9.6.0 on main); qaf-tests strips/re-appends -SNAPSHOT itself. +if [[ -z "${STACK_VERSION:-}" && -f gradle.properties ]]; then + _ml_cpp_version=$(sed -n 's/^elasticsearchVersion=//p' gradle.properties | tr -d '[:space:]') + if [[ -n "${_ml_cpp_version}" ]]; then + STACK_VERSION="${_ml_cpp_version}" + fi +fi + +# ES_BRANCH: the Elasticsearch branch qaf-tests should build the custom +# distribution from. It is "main" for the current development line and the +# "." release branch otherwise. We decide by comparing this +# build's version to ml-cpp main's version (fetched the same way the downstream +# fetches ml-cpp/main assets), so there is no hard-coded "current dev minor" to +# keep updating at feature freeze. +if [[ -z "${ES_BRANCH:-}" && -n "${STACK_VERSION:-}" ]]; then + _main_version=$(python3 - <<'PY' 2>/dev/null +import re, sys, urllib.request +try: + with urllib.request.urlopen( + "https://raw.githubusercontent.com/elastic/ml-cpp/main/gradle.properties", + timeout=15, + ) as response: + text = response.read().decode("utf-8", "replace") + match = re.search(r"^elasticsearchVersion=(.+)$", text, re.M) + sys.stdout.write(match.group(1).strip() if match else "") +except Exception: + sys.stdout.write("") +PY +) + _version_no_snapshot="${STACK_VERSION%-SNAPSHOT}" + _main_no_snapshot="${_main_version%-SNAPSHOT}" + if [[ -n "${_main_no_snapshot}" && "${_version_no_snapshot}" == "${_main_no_snapshot}" ]]; then + ES_BRANCH="main" + else + # major.minor of this build's version, e.g. 9.5.0 -> 9.5. + ES_BRANCH=$(printf '%s' "${_version_no_snapshot}" | awk -F. 'NF>=2 {print $1"."$2}') + if [[ -z "${_main_no_snapshot}" ]]; then + # Could not determine main's version (e.g. transient network issue). + # Fall back to the branch name: only treat recognised release-line + # branches as releases, otherwise preserve the historical default of + # main so ordinary PRs are unaffected. + _norm_branch="${BUILDKITE_BRANCH#elastic+}" + case "${_norm_branch}" in + backport/*|[0-9]*.[0-9]*|ci/ml-cpp-version-bump-*|ci/ml-cpp-minor-freeze-main-*) + : # keep the derived major.minor + ;; + *) + ES_BRANCH="main" + ;; + esac + fi + if [[ -z "${ES_BRANCH}" ]]; then + ES_BRANCH="main" + fi + fi +fi + +export STACK_VERSION ES_BRANCH +echo "derive_qa_stack_env: STACK_VERSION='${STACK_VERSION:-}' ES_BRANCH='${ES_BRANCH:-}'" >&2 diff --git a/.buildkite/pipelines/run_pytorch_tests.yml.sh b/.buildkite/pipelines/run_pytorch_tests.yml.sh index 9aa367d5f..997538de6 100755 --- a/.buildkite/pipelines/run_pytorch_tests.yml.sh +++ b/.buildkite/pipelines/run_pytorch_tests.yml.sh @@ -10,6 +10,13 @@ SAFE_MESSAGE=$(printf '%s' "${BUILDKITE_MESSAGE}" | head -1 | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g') +# Derive STACK_VERSION / ES_BRANCH so release-branch and backport builds test +# against the matching stack version and ES branch instead of the qaf-tests +# defaults (main / current-dev SNAPSHOT). Silent on stdout by contract. +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/derive_qa_stack_env.sh" + cat < /dev/null && pwd ) +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/derive_qa_stack_env.sh" + cat <