From 3e0a7408c45878e90f5fb4f44b749d33cfd9e51d Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Mon, 27 Jul 2026 10:16:56 +1200 Subject: [PATCH] [ML] Make Appex QA/PyTorch tests version-aware on release branches The downstream appex-qa-stateful-custom-ml-cpp-build-testing pipeline (elastic/qaf-tests) defaults to ES_BRANCH=main and STACK_VERSION=, then downloads ml-cpp--SNAPSHOT-linux-x86_64.zip from the parent build. On a release-branch / backport build the parent produced ml-cpp--SNAPSHOT-... instead, so the download missed ('No artifacts found') and the QA/PyTorch build failed spuriously (e.g. a 9.5 backport looking for 9.6.0-SNAPSHOT). Derive STACK_VERSION from gradle.properties (the exact artifact version) and ES_BRANCH by comparing to ml-cpp main's version (main for the current dev line, else major.minor), and forward both through the QA and PyTorch downstream triggers. For main builds the derived values equal today's defaults, so only release-branch/backport builds change behaviour. No qaf-tests change is required; it already supports these overrides and lists the active release branches/versions. Co-authored-by: Cursor --- .buildkite/pipelines/derive_qa_stack_env.sh | 86 +++++++++++++++++++ .buildkite/pipelines/run_pytorch_tests.yml.sh | 19 ++++ .buildkite/pipelines/run_qa_tests.yml.sh | 8 ++ 3 files changed, 113 insertions(+) create mode 100755 .buildkite/pipelines/derive_qa_stack_env.sh diff --git a/.buildkite/pipelines/derive_qa_stack_env.sh b/.buildkite/pipelines/derive_qa_stack_env.sh new file mode 100755 index 0000000000..f259b2bfb0 --- /dev/null +++ b/.buildkite/pipelines/derive_qa_stack_env.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. + +# Derive STACK_VERSION and ES_BRANCH for the Appex QA / PyTorch downstream +# trigger (appex-qa-stateful-custom-ml-cpp-build-testing, generated by +# elastic/qaf-tests). Without these the downstream falls back to its defaults +# (ES_BRANCH=main, STACK_VERSION=) and then tries to +# download an artifact named ml-cpp--SNAPSHOT-... from the parent +# build. On a release-branch / backport build the parent produced +# ml-cpp--SNAPSHOT-... instead, so the download misses +# ("No artifacts found") and the QA build fails spuriously. +# +# This is meant to be SOURCED by the run_*_tests.yml.sh generators. Those +# scripts pipe their stdout straight into `buildkite-agent pipeline upload`, so +# this helper MUST NOT print anything to stdout — all diagnostics go to stderr. +# Any values already present in the environment (e.g. from PR comment vars +# GITHUB_PR_COMMENT_VAR_BRANCH / _VERSION) are respected and never overwritten. + +# STACK_VERSION: the ml-cpp product version for this build. gradle.properties' +# elasticsearchVersion is exactly the artifact version (e.g. 9.5.0 on the 9.5 +# branch, 9.6.0 on main); qaf-tests strips/re-appends -SNAPSHOT itself. +if [[ -z "${STACK_VERSION:-}" && -f gradle.properties ]]; then + _ml_cpp_version=$(sed -n 's/^elasticsearchVersion=//p' gradle.properties | tr -d '[:space:]') + if [[ -n "${_ml_cpp_version}" ]]; then + STACK_VERSION="${_ml_cpp_version}" + fi +fi + +# ES_BRANCH: the Elasticsearch branch qaf-tests should build the custom +# distribution from. It is "main" for the current development line and the +# "." release branch otherwise. We decide by comparing this +# build's version to ml-cpp main's version (fetched the same way the downstream +# fetches ml-cpp/main assets), so there is no hard-coded "current dev minor" to +# keep updating at feature freeze. +if [[ -z "${ES_BRANCH:-}" && -n "${STACK_VERSION:-}" ]]; then + _main_version=$(python3 - <<'PY' 2>/dev/null +import re, sys, urllib.request +try: + with urllib.request.urlopen( + "https://raw.githubusercontent.com/elastic/ml-cpp/main/gradle.properties", + timeout=15, + ) as response: + text = response.read().decode("utf-8", "replace") + match = re.search(r"^elasticsearchVersion=(.+)$", text, re.M) + sys.stdout.write(match.group(1).strip() if match else "") +except Exception: + sys.stdout.write("") +PY +) + _version_no_snapshot="${STACK_VERSION%-SNAPSHOT}" + _main_no_snapshot="${_main_version%-SNAPSHOT}" + if [[ -n "${_main_no_snapshot}" && "${_version_no_snapshot}" == "${_main_no_snapshot}" ]]; then + ES_BRANCH="main" + else + # major.minor of this build's version, e.g. 9.5.0 -> 9.5. + ES_BRANCH=$(printf '%s' "${_version_no_snapshot}" | awk -F. 'NF>=2 {print $1"."$2}') + if [[ -z "${_main_no_snapshot}" ]]; then + # Could not determine main's version (e.g. transient network issue). + # Fall back to the branch name: only treat recognised release-line + # branches as releases, otherwise preserve the historical default of + # main so ordinary PRs are unaffected. + _norm_branch="${BUILDKITE_BRANCH#elastic+}" + case "${_norm_branch}" in + backport/*|[0-9]*.[0-9]*|ci/ml-cpp-version-bump-*|ci/ml-cpp-minor-freeze-main-*) + : # keep the derived major.minor + ;; + *) + ES_BRANCH="main" + ;; + esac + fi + if [[ -z "${ES_BRANCH}" ]]; then + ES_BRANCH="main" + fi + fi +fi + +export STACK_VERSION ES_BRANCH +echo "derive_qa_stack_env: STACK_VERSION='${STACK_VERSION:-}' ES_BRANCH='${ES_BRANCH:-}'" >&2 diff --git a/.buildkite/pipelines/run_pytorch_tests.yml.sh b/.buildkite/pipelines/run_pytorch_tests.yml.sh index 9aa367d5f0..997538de65 100755 --- a/.buildkite/pipelines/run_pytorch_tests.yml.sh +++ b/.buildkite/pipelines/run_pytorch_tests.yml.sh @@ -10,6 +10,13 @@ SAFE_MESSAGE=$(printf '%s' "${BUILDKITE_MESSAGE}" | head -1 | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g') +# Derive STACK_VERSION / ES_BRANCH so release-branch and backport builds test +# against the matching stack version and ES branch instead of the qaf-tests +# defaults (main / current-dev SNAPSHOT). Silent on stdout by contract. +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/derive_qa_stack_env.sh" + cat < /dev/null && pwd ) +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/derive_qa_stack_env.sh" + cat <