diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 7b3ebce7..55513ec1 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -20,34 +20,48 @@ jobs: fetch-depth: 1 persist-credentials: false - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 - - name: Test release build-output materialization - run: | - ./tests/release_build_output_descriptors_test.sh - ./tests/release_build_output_prepare_test.sh - ./tests/release_build_output_test.sh - - name: Prepare release build-output dispatch smoke test + + test-release-catalog-actions: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false + - name: Test release catalog materialization + run: ./ci/run_release_catalog_tests.sh + - name: Prepare release catalog dispatch smoke test run: | - mkdir -p release-build-output-smoke - printf '%s\n' smoke >release-build-output-smoke/package.tar.gz - - name: Run release build-output dispatch smoke test - uses: ./release-build-output-dispatch + mkdir -p release-catalog-smoke + printf '%s\n' smoke >release-catalog-smoke/package.tar.gz + jq -n \ + '{ecosystem: "archive", name: "smoke", version: "1.0"}' \ + >release-catalog-smoke/package-identity.json + - name: Run release catalog dispatch smoke test + uses: ./release-catalog env: SHARED_ACTIONS_REPO: ${{ github.event.pull_request.head.repo.full_name }} SHARED_ACTIONS_REF: ${{ github.event.pull_request.head.sha }} with: - artifact-type: custom - output-directory: release-build-output-smoke - release-artifacts: '[{"path":"package.tar.gz"}]' - release-package: '{"ecosystem":"archive","name":"smoke","version":"1.0"}' - release-unit: archive:smoke - source-artifact-name: release-build-output-dispatch-smoke + config: >- + { + "release_catalog_key": "archive:smoke", + "artifact_directory": "release-catalog-smoke", + "artifacts": [{ + "path": "package.tar.gz", + "package_identity_file": "package-identity.json" + }] + } + source-artifact-name: release-catalog-smoke source-sha: ${{ github.event.pull_request.head.sha }} - - name: Verify release build-output dispatch smoke test + - name: Verify release catalog dispatch smoke test run: | jq -e ' - .artifacts[0].unit_id == "archive:smoke" - and .artifacts[0].path == "package.tar.gz" - ' release-build-output-smoke/release-build-output.json >/dev/null - jq -e ' - .metadata.artifacts == [{path: "package.tar.gz", sbom_kind: "generated-identity"}] - ' release-build-output-smoke/release-build-metadata.json >/dev/null + .producer == "rapidsai/shared-actions/release-catalog" + and .source.artifact == "release-catalog-smoke" + and .entries[0].release_catalog_key == "archive:smoke" + and .entries[0].path == "package.tar.gz" + and .entries[0].sbom_kind == "generated-identity" + ' release-catalog-smoke/release-catalog-entries.json >/dev/null diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 799d7320..de5d229e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,13 @@ ci: autoupdate_schedule: quarterly - skip: [actionlint-docker] + # These hooks require tools unavailable in pre-commit.ci: + # + # * actionlint-docker requires Docker + # * release-catalog-config requires jq + # + # Both run in other CI jobs with controlled runtime dependencies. + skip: [actionlint-docker, release-catalog-config] repos: - repo: https://github.com/pre-commit/pre-commit-hooks @@ -11,6 +17,7 @@ repos: hooks: - id: trailing-whitespace - id: check-added-large-files + - id: check-json - id: check-yaml - id: end-of-file-fixer - repo: https://github.com/zizmorcore/zizmor-pre-commit @@ -52,3 +59,25 @@ repos: hooks: - id: yamllint additional_dependencies: [pyyaml] + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.37.2 + hooks: + - id: check-jsonschema + name: validate release catalog config schema fixtures + args: [--schemafile, release-catalog/config.schema.json] + files: ^tests/release-catalog-config/valid/.*\.json$ + - repo: local + hooks: + - id: release-catalog-config + name: validate release catalog configuration + entry: ./tests/release_catalog_config_test.sh + language: system + pass_filenames: false + files: ^(release-catalog/(config\.schema\.json|validate-config\.sh)|tests/release-catalog-config/.*\.json|tests/release_catalog_config_test\.sh)$ + - id: release-catalog-schemas + name: validate release catalog schema contracts + entry: python ci/test_release_catalog_schemas.py + language: python + additional_dependencies: [jsonschema==4.25.1] + pass_filenames: false + files: ^(release-catalog/(config|entries)\.schema\.json|release-catalog/examples/cuvs-java/release-catalog-entries\.json|tests/release-catalog-config/.*\.json|ci/test_release_catalog_schemas\.py)$ diff --git a/README.md b/README.md index 4ea1b0ca..6ead1668 100644 --- a/README.md +++ b/README.md @@ -9,53 +9,27 @@ A dispatch action is one that: * clones the shared-actions repository (repo/ref changeable using env vars) * runs (dispatches to) another action within the clone, using a relative path -## Release build-output companions - -`release-build-output-dispatch` validates a producer's local build artifact -directory and uploads a companion artifact named -`release-build-output-`. The companion contains -`release-build-output.json`, `release-build-metadata.json`, provenance, and an -SBOM record for every primary artifact. - -Conda and wheel jobs can set `artifact-type` to `conda` or `wheel` and omit -`release-artifacts`; the implementation reads package metadata from the built -files. Custom bundles provide explicit artifact descriptors and either inline -package identity or a producer-created package JSON file. - -```yaml -- name: Create release build-output companion - uses: rapidsai/shared-actions/release-build-output-dispatch@main - with: - artifact-type: wheel - output-directory: ${{ steps.package-name.outputs.WHEEL_OUTPUT_DIR }} - release-unit: wheel:example - source-artifact-name: ${{ steps.package-name.outputs.RAPIDS_PACKAGE_NAME }} - source-sha: ${{ github.sha }} -``` - -A descriptor-selected producer SBOM is classified as `producer-dependency`. -When no SBOM is supplied, the action generates an SPDX artifact-identity -envelope and classifies it as `generated-identity`. The generated envelope -contains the primary artifact's identity and SHA-256 but no dependency -inventory; it must not be treated as dependency coverage. - -The dispatch wrapper honors `SHARED_ACTIONS_REPO` and `SHARED_ACTIONS_REF`. -When neither is set, it checks out the same repository and ref used to invoke -the wrapper, which allows a feature-branch wrapper to dispatch to its matching -implementation during canary testing. - -There can be more complicated arrangements of more actions, but the idea is to -have the local clone of the shared-actions repository be the first step of an action. - -Actions that refer to each other assume that they have been checked out to the -./shared-actions folder. This *should* be the root of the GitHub Actions workspace. -This assumption is what allow code reuse between actions. - -Actions that use this pattern should include "dispatch" in their folder name, so -that they can be readily distinguished from any actions that are either -standalone or otherwise implementations that assume that the ./shared-actions -folder is already cloned, so that they can use relative paths to reference other -actions and files. +The checkout is the important part of the pattern. It gives every subsequent +step a stable `./shared-actions` path and lets a caller test changes by setting +`SHARED_ACTIONS_REPO` and `SHARED_ACTIONS_REF` without changing the caller's +`uses:` line. When those variables are absent, dispatch actions normally use +`rapidsai/shared-actions` at `main`. + +The public action and its implementation may use either of two layouts: + +* A small dispatch action can invoke a separate implementation action under the + checked-out `./shared-actions` directory. This is useful when several public + actions share one implementation. +* A self-dispatching action can keep its `action.yml`, scripts, schemas, and + documentation in one folder. After checkout, its shell steps invoke the files + in the corresponding `./shared-actions/` folder. This is simpler + when the implementation has no independent callers. + +In both layouts, actions must use files from the checked-out `./shared-actions` +tree rather than from the revision that initially loaded the public +`action.yml`. Otherwise `SHARED_ACTIONS_REPO` and `SHARED_ACTIONS_REF` would +select only the wrapper while the implementation continued to come from a +different revision. ## Example dispatch action @@ -72,9 +46,10 @@ runs: - name: Clone shared-actions repo uses: actions/checkout@v4 with: - repository: ${{ env.SHARED_ACTIONS_REPO }} - ref: ${{ env.SHARED_ACTIONS_REF }} + repository: ${{ env.SHARED_ACTIONS_REPO || 'rapidsai/shared-actions' }} + ref: ${{ env.SHARED_ACTIONS_REF || 'main' }} path: ./shared-actions + persist-credentials: false - name: Run local implementation action uses: ./shared-actions/impls/example-action ``` diff --git a/ci/run_release_catalog_tests.sh b/ci/run_release_catalog_tests.sh new file mode 100755 index 00000000..5df0f660 --- /dev/null +++ b/ci/run_release_catalog_tests.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +set -euo pipefail + +repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +"${repository_root}/tests/release_catalog_config_test.sh" +"${repository_root}/tests/release_catalog_discovery_test.sh" +"${repository_root}/tests/release_catalog_error_test.sh" +"${repository_root}/tests/release_catalog_package_parsing_test.sh" +"${repository_root}/tests/release_catalog_test.sh" diff --git a/ci/test_release_catalog_schemas.py b/ci/test_release_catalog_schemas.py new file mode 100644 index 00000000..a03695c5 --- /dev/null +++ b/ci/test_release_catalog_schemas.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +"""Verify valid and invalid release-catalog contracts against JSON Schema.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from jsonschema import Draft202012Validator + +ROOT = Path(__file__).parents[1] + + +def _load(path: Path) -> dict[str, object]: + value = json.loads(path.read_text()) + if not isinstance(value, dict): + raise ValueError(f"schema fixture must be a JSON object: {path}") + return value + + +def _validator(path: Path) -> Draft202012Validator: + schema = _load(path) + Draft202012Validator.check_schema(schema) + return Draft202012Validator(schema) + + +def main() -> None: + config = _validator(ROOT / "release-catalog/config.schema.json") + fixtures = ROOT / "tests/release-catalog-config" + for path in sorted((fixtures / "valid").glob("*.json")): + errors = list(config.iter_errors(_load(path))) + if errors: + raise ValueError(f"valid fixture rejected: {path}: {errors[0].message}") + for path in sorted((fixtures / "invalid").glob("*.json")): + if not list(config.iter_errors(_load(path))): + raise ValueError(f"invalid fixture accepted: {path}") + + entries = _validator(ROOT / "release-catalog/entries.schema.json") + example = _load(ROOT / "release-catalog/examples/cuvs-java/release-catalog-entries.json") + errors = list(entries.iter_errors(example)) + if errors: + raise ValueError(f"generated entries example rejected: {errors[0].message}") + wrong_producer = {**example, "producer": "some-other-tool"} + if not list(entries.iter_errors(wrong_producer)): + raise ValueError("entries schema accepted an unknown producer") + + +if __name__ == "__main__": + main() diff --git a/release-build-output-dispatch/action.yml b/release-build-output-dispatch/action.yml deleted file mode 100644 index 154ba3d7..00000000 --- a/release-build-output-dispatch/action.yml +++ /dev/null @@ -1,72 +0,0 @@ -name: Dispatch release build output -description: Check out the selected shared-actions revision and create a release build-output companion. - -inputs: - artifact-type: - description: One of conda, wheel, or custom. - required: true - release-unit: - description: Release-platform unit ID for every primary artifact in this bundle. - required: true - release-package: - description: JSON package fields shared by the bundle. - required: false - release-package-file: - description: Relative path to producer-created package JSON inside output-directory. - required: false - release-artifacts: - description: JSON artifact and evidence descriptors relative to output-directory. - required: false - output-directory: - description: Directory containing the primary files and any producer-supplied evidence. - required: true - manifest-name: - description: Filename to write inside output-directory. - required: false - default: release-build-output.json - metadata-name: - description: Filename for the build metadata envelope. - required: false - default: release-build-metadata.json - source-artifact-name: - description: Name of the GitHub Actions artifact bundle containing this output. - required: true - source-sha: - description: Source revision built by the producing job. - required: false - -outputs: - manifest-path: - description: Absolute path to the generated manifest. - value: ${{ steps.release-build-output.outputs.manifest-path }} - metadata-path: - description: Absolute path to the build metadata envelope. - value: ${{ steps.release-build-output.outputs.metadata-path }} - manifest-artifact-name: - description: Name of the uploaded GitHub Actions companion artifact. - value: ${{ steps.release-build-output.outputs.manifest-artifact-name }} - -runs: - using: composite - steps: - - name: Check out shared-actions implementation - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ env.SHARED_ACTIONS_REPO || github.action_repository || 'rapidsai/shared-actions' }} - ref: ${{ env.SHARED_ACTIONS_REF || github.action_ref || 'main' }} - path: ./shared-actions - persist-credentials: false - - id: release-build-output - name: Create release build-output companion - uses: ./shared-actions/release-build-output - with: - artifact-type: ${{ inputs.artifact-type }} - release-unit: ${{ inputs.release-unit }} - release-package: ${{ inputs.release-package }} - release-package-file: ${{ inputs.release-package-file }} - release-artifacts: ${{ inputs.release-artifacts }} - output-directory: ${{ inputs.output-directory }} - manifest-name: ${{ inputs.manifest-name }} - metadata-name: ${{ inputs.metadata-name }} - source-artifact-name: ${{ inputs.source-artifact-name }} - source-sha: ${{ inputs.source-sha }} diff --git a/release-build-output/action.yml b/release-build-output/action.yml deleted file mode 100644 index 7c6965e3..00000000 --- a/release-build-output/action.yml +++ /dev/null @@ -1,90 +0,0 @@ -name: Create release build output companion -description: Validate a build artifact bundle and upload its release-platform manifest and evidence companion. - -inputs: - artifact-type: - description: One of conda, wheel, or custom. Conda and wheel descriptors are derived when release-artifacts is omitted. - required: true - release-unit: - description: Release-platform unit ID for every primary artifact in this bundle. - required: true - release-package: - description: JSON package fields shared by the bundle. Provide this or release-package-file for custom artifacts. - required: false - release-package-file: - description: Relative path to producer-created package JSON inside output-directory. - required: false - release-artifacts: - description: JSON artifact and evidence descriptors relative to output-directory. Required for custom artifacts. - required: false - output-directory: - description: Directory containing the primary files and any producer-supplied evidence. - required: true - manifest-name: - description: Filename to write inside output-directory. - required: false - default: release-build-output.json - metadata-name: - description: Filename for the build-environment and SBOM-classification envelope. - required: false - default: release-build-metadata.json - source-artifact-name: - description: Name of the GitHub Actions artifact bundle containing this output. - required: true - source-sha: - description: Source revision built by the producing job. Defaults to the current workflow SHA. - required: false - -outputs: - manifest-path: - description: Absolute path to the generated manifest. - value: ${{ steps.materialize.outputs.manifest-path }} - metadata-path: - description: Absolute path to the build metadata envelope. - value: ${{ steps.materialize.outputs.metadata-path }} - manifest-artifact-name: - description: Name of the uploaded GitHub Actions companion artifact. - value: ${{ steps.companion-name.outputs.name }} - -runs: - using: composite - steps: - - id: prepare - name: Describe release artifacts - shell: bash - env: - RELEASE_ARTIFACTS: ${{ inputs.release-artifacts }} - RELEASE_ARTIFACT_TYPE: ${{ inputs.artifact-type }} - RELEASE_OUTPUT_DIRECTORY: ${{ inputs.output-directory }} - RELEASE_PACKAGE: ${{ inputs.release-package }} - RELEASE_PACKAGE_FILE: ${{ inputs.release-package-file }} - run: ./shared-actions/release-build-output/prepare.sh - - id: materialize - name: Materialize release build-output records - shell: bash - env: - RELEASE_ARTIFACTS: ${{ steps.prepare.outputs.artifacts }} - RELEASE_MANIFEST_NAME: ${{ inputs.manifest-name }} - RELEASE_METADATA_NAME: ${{ inputs.metadata-name }} - RELEASE_OUTPUT_DIRECTORY: ${{ inputs.output-directory }} - RELEASE_PACKAGE: ${{ steps.prepare.outputs.package }} - RELEASE_PACKAGE_FILE: ${{ inputs.release-package-file }} - RELEASE_SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} - RELEASE_SOURCE_SHA: ${{ inputs.source-sha || github.sha }} - RELEASE_UNIT: ${{ inputs.release-unit }} - run: ./shared-actions/release-build-output/materialize.sh - - id: companion-name - name: Set companion artifact name - shell: bash - env: - SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} - run: echo "name=release-build-output-${SOURCE_ARTIFACT_NAME}" >>"${GITHUB_OUTPUT}" - - name: Upload release build-output companion - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - if-no-files-found: error - name: ${{ steps.companion-name.outputs.name }} - path: | - ${{ inputs.output-directory }}/${{ inputs.manifest-name }} - ${{ inputs.output-directory }}/${{ inputs.metadata-name }} - ${{ inputs.output-directory }}/release-evidence/** diff --git a/release-build-output/describe-conda.sh b/release-build-output/describe-conda.sh deleted file mode 100755 index 6449addd..00000000 --- a/release-build-output/describe-conda.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - -set -euo pipefail - -if [[ "$#" -ne 1 || ! -d "$1" ]]; then - echo "usage: $0 CONDA_OUTPUT_DIRECTORY" >&2 - exit 1 -fi - -output_directory="$(realpath "$1")" -descriptors='[]' -package_count=0 - -while IFS= read -r package_path; do - package_path="$(realpath "${package_path}")" - if [[ "${package_path}" != "${output_directory}"/* ]]; then - echo "Conda package must resolve inside output directory: ${package_path}" >&2 - exit 1 - fi - relative_path="${package_path#"${output_directory}/"}" - - case "${package_path}" in - *.conda) - info_members=() - while IFS= read -r info_member; do - info_members+=("${info_member}") - done < <(unzip -Z1 "${package_path}" | awk '/^info-.*\.tar\.zst$/') - if [[ "${#info_members[@]}" -ne 1 ]]; then - echo ".conda package must contain exactly one info-*.tar.zst member: ${relative_path}" >&2 - exit 1 - fi - index_json="$(unzip -p "${package_path}" "${info_members[0]}" | zstd -dc | tar -xOf - info/index.json)" - ;; - *.tar.bz2) - index_json="$(tar -xOjf "${package_path}" info/index.json)" - ;; - *) - echo "unsupported Conda package extension: ${relative_path}" >&2 - exit 1 - ;; - esac - - if ! jq -e ' - type == "object" - and (.name | type == "string" and length > 0) - and (.version | type == "string" and length > 0) - and (.build | type == "string" and length > 0) - and (.subdir | type == "string" and length > 0) - ' <<<"${index_json}" >/dev/null; then - echo "Conda info/index.json must contain exact name, version, build, and subdir fields: ${relative_path}" >&2 - exit 1 - fi - - package="$(jq -c '{ecosystem: "conda", name, version, build, platform: .subdir}' <<<"${index_json}")" - descriptors="$(jq -cn \ - --arg path "${relative_path}" \ - --argjson package "${package}" \ - --argjson current "${descriptors}" \ - '$current + [{path: $path, package: $package}]')" - package_count=$((package_count + 1)) -done < <(find "${output_directory}" -type f \( -name '*.conda' -o -name '*.tar.bz2' \) -print | sort) - -if [[ "${package_count}" -eq 0 ]]; then - echo "Conda output directory contains no .conda or .tar.bz2 packages: ${output_directory}" >&2 - exit 1 -fi - -printf '%s\n' "${descriptors}" diff --git a/release-build-output/describe-wheels.sh b/release-build-output/describe-wheels.sh deleted file mode 100755 index bc6b925e..00000000 --- a/release-build-output/describe-wheels.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - -set -euo pipefail - -if [[ "$#" -ne 1 || ! -d "$1" ]]; then - echo "usage: $0 WHEEL_OUTPUT_DIRECTORY" >&2 - exit 1 -fi - -output_directory="$(realpath "$1")" -descriptors='[]' -wheel_count=0 - -while IFS= read -r wheel_path; do - wheel_path="$(realpath "${wheel_path}")" - if [[ "${wheel_path}" != "${output_directory}"/* ]]; then - echo "wheel must resolve inside output directory: ${wheel_path}" >&2 - exit 1 - fi - relative_path="${wheel_path#"${output_directory}/"}" - - metadata_members=() - while IFS= read -r metadata_member; do - metadata_members+=("${metadata_member}") - done < <(unzip -Z1 "${wheel_path}" | awk '/\.dist-info\/METADATA$/') - if [[ "${#metadata_members[@]}" -ne 1 ]]; then - echo "wheel must contain exactly one .dist-info/METADATA file: ${relative_path}" >&2 - exit 1 - fi - - metadata="$(unzip -p "${wheel_path}" "${metadata_members[0]}")" - package_name="$(awk 'tolower($0) ~ /^name:[[:space:]]*/ {sub(/^[^:]*:[[:space:]]*/, ""); sub(/\r$/, ""); print; exit}' <<<"${metadata}")" - package_version="$(awk 'tolower($0) ~ /^version:[[:space:]]*/ {sub(/^[^:]*:[[:space:]]*/, ""); sub(/\r$/, ""); print; exit}' <<<"${metadata}")" - if [[ -z "${package_name}" || -z "${package_version}" ]]; then - echo "wheel metadata must contain non-empty Name and Version fields: ${relative_path}" >&2 - exit 1 - fi - - descriptors="$(jq -cn \ - --arg path "${relative_path}" \ - --arg name "${package_name}" \ - --arg version "${package_version}" \ - --argjson current "${descriptors}" \ - '$current + [{path: $path, package: {ecosystem: "wheel", name: $name, version: $version}}]')" - wheel_count=$((wheel_count + 1)) -done < <(find "${output_directory}" -type f -name '*.whl' -print | sort) - -if [[ "${wheel_count}" -eq 0 ]]; then - echo "wheel output directory contains no .whl files: ${output_directory}" >&2 - exit 1 -fi - -printf '%s\n' "${descriptors}" diff --git a/release-build-output/materialize.sh b/release-build-output/materialize.sh deleted file mode 100755 index 673dd6ba..00000000 --- a/release-build-output/materialize.sh +++ /dev/null @@ -1,367 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - -set -euo pipefail - -require_nonempty() { - local name="$1" - local value="$2" - if [[ -z "${value}" ]]; then - echo "${name} must be a non-empty string" >&2 - exit 1 - fi -} - -require_nonempty "RELEASE_UNIT" "${RELEASE_UNIT:-}" -require_nonempty "RELEASE_OUTPUT_DIRECTORY" "${RELEASE_OUTPUT_DIRECTORY:-}" -require_nonempty "RELEASE_MANIFEST_NAME" "${RELEASE_MANIFEST_NAME:-}" -require_nonempty "RELEASE_METADATA_NAME" "${RELEASE_METADATA_NAME:-}" -require_nonempty "RELEASE_ARTIFACTS" "${RELEASE_ARTIFACTS:-}" -require_nonempty "RELEASE_SOURCE_ARTIFACT_NAME" "${RELEASE_SOURCE_ARTIFACT_NAME:-}" - -if [[ -n "${RELEASE_PACKAGE:-}" && -n "${RELEASE_PACKAGE_FILE:-}" ]]; then - echo "release-package and release-package-file are mutually exclusive" >&2 - exit 1 -fi -if [[ -z "${RELEASE_PACKAGE:-}" && -z "${RELEASE_PACKAGE_FILE:-}" ]]; then - echo "one of release-package or release-package-file is required" >&2 - exit 1 -fi - -source_sha="${RELEASE_SOURCE_SHA:-${GITHUB_SHA:-}}" -require_nonempty "RELEASE_SOURCE_SHA or GITHUB_SHA" "${source_sha}" - -require_plain_filename() { - local label="$1" - local filename="$2" - if [[ "${filename}" == */* || "${filename}" == .* || "${filename}" == *".."* ]]; then - echo "${label} must be a plain filename" >&2 - exit 1 - fi -} - -require_plain_filename "manifest-name" "${RELEASE_MANIFEST_NAME}" -require_plain_filename "metadata-name" "${RELEASE_METADATA_NAME}" -if [[ "${RELEASE_MANIFEST_NAME}" == "${RELEASE_METADATA_NAME}" ]]; then - echo "manifest-name and metadata-name must differ" >&2 - exit 1 -fi - -if [[ ! -d "${RELEASE_OUTPUT_DIRECTORY}" ]]; then - echo "output-directory does not exist or is not a directory: ${RELEASE_OUTPUT_DIRECTORY}" >&2 - exit 1 -fi - -ensure_relative_pattern() { - local field="$1" - local pattern="$2" - if [[ "${pattern}" == /* || "${pattern}" == */../* || "${pattern}" == ../* || "${pattern}" == *"/.." ]]; then - echo "${field} must be a relative path inside output-directory: ${pattern}" >&2 - exit 1 - fi -} - -output_directory="$(realpath "${RELEASE_OUTPUT_DIRECTORY}")" -if [[ -n "${RELEASE_PACKAGE_FILE:-}" ]]; then - ensure_relative_pattern "release-package-file" "${RELEASE_PACKAGE_FILE}" - package_file_path="$(realpath "${output_directory}/${RELEASE_PACKAGE_FILE}")" - if [[ "${package_file_path}" != "${output_directory}"/* || ! -f "${package_file_path}" ]]; then - echo "release-package-file must resolve to one file inside output-directory: ${RELEASE_PACKAGE_FILE}" >&2 - exit 1 - fi - RELEASE_PACKAGE="$(jq -c . "${package_file_path}")" -fi - -if ! jq -e ' - type == "object" - and (keys - ["ecosystem", "name", "version", "build", "platform"] | length == 0) - and (.ecosystem | type == "string" and length > 0) - and (.name | type == "string" and length > 0) - and ((.version // "") | type == "string") - and ((.build // "") | type == "string") - and ((.platform // "") | type == "string") -' <<<"${RELEASE_PACKAGE}" >/dev/null; then - echo "release-package must be a package object with ecosystem and name; version may be supplied or derived per artifact" >&2 - exit 1 -fi - -if ! jq -e 'type == "array" and length > 0' <<<"${RELEASE_ARTIFACTS}" >/dev/null; then - echo "release-artifacts must be a non-empty JSON array" >&2 - exit 1 -fi - -manifest_path="${output_directory}/${RELEASE_MANIFEST_NAME}" -metadata_path="${output_directory}/${RELEASE_METADATA_NAME}" -temporary_manifest="$(mktemp "${output_directory}/.release-build-output.XXXXXX")" -trap 'rm -f "${temporary_manifest}"' EXIT - -printf '%s\n' '{"schema_version":1,"producer":"release-platform","artifacts":[]}' >"${temporary_manifest}" - -resolve_one_file() { - local field="$1" - local pattern="$2" - local -a matches=() - - ensure_relative_pattern "${field}" "${pattern}" - while IFS= read -r match; do - matches+=("${match}") - done < <(compgen -G "${output_directory}/${pattern}" || true) - if [[ "${#matches[@]}" -ne 1 || ! -f "${matches[0]:-}" ]]; then - echo "${field} pattern must resolve to exactly one file: ${pattern}" >&2 - exit 1 - fi - - local resolved - resolved="$(realpath "${matches[0]}")" - if [[ "${resolved}" != "${output_directory}"/* ]]; then - echo "${field} must resolve inside output-directory: ${pattern}" >&2 - exit 1 - fi - printf '%s\n' "${resolved#"${output_directory}/"}" -} - -derive_package_version() { - local ecosystem="$1" - local package_name="$2" - local artifact_path="$3" - local filename - filename="$(basename "${artifact_path}")" - - case "${ecosystem}" in - conda) - local conda_prefix="${package_name}-" - if [[ "${filename}" != "${conda_prefix}"* ]]; then - echo "Conda artifact filename does not start with package name '${package_name}': ${filename}" >&2 - exit 1 - fi - local conda_remainder="${filename#"${conda_prefix}"}" - local conda_version="${conda_remainder%%-*}" - if [[ -z "${conda_version}" || "${conda_version}" == "${conda_remainder}" ]]; then - echo "cannot derive Conda package version from artifact filename: ${filename}" >&2 - exit 1 - fi - printf '%s\n' "${conda_version}" - ;; - wheel) - local wheel_prefix="${package_name//-/_}-" - if [[ "${filename}" != "${wheel_prefix}"* ]]; then - echo "wheel artifact filename does not start with normalized package name '${package_name}': ${filename}" >&2 - exit 1 - fi - local wheel_remainder="${filename#"${wheel_prefix}"}" - local wheel_version="${wheel_remainder%%-*}" - if [[ -z "${wheel_version}" || "${wheel_version}" == "${wheel_remainder}" ]]; then - echo "cannot derive wheel package version from artifact filename: ${filename}" >&2 - exit 1 - fi - printf '%s\n' "${wheel_version}" - ;; - *) - echo "release-package version is required for ${ecosystem} artifacts" >&2 - exit 1 - ;; - esac -} - -generated_evidence_path() { - local primary_path="$1" - local kind="$2" - local artifact_digest - artifact_digest="$(sha256sum "${output_directory}/${primary_path}" | awk '{print $1}')" - local artifact_label="${primary_path//\//_}" - printf 'release-evidence/%s.%s.%s.json\n' "${artifact_label}" "${artifact_digest}" "${kind}" -} - -copy_supplied_evidence() { - local primary_path="$1" - local supplied_path="$2" - local kind="$3" - local artifact_digest - artifact_digest="$(sha256sum "${output_directory}/${primary_path}" | awk '{print $1}')" - local artifact_label="${primary_path//\//_}" - local destination - destination="release-evidence/${artifact_label}.${artifact_digest}/${kind}-$(basename "${supplied_path}")" - mkdir -p "$(dirname "${output_directory}/${destination}")" - cp "${output_directory}/${supplied_path}" "${output_directory}/${destination}" - printf '%s\n' "${destination}" -} - -write_generated_sbom() { - local primary_path="$1" - local package="$2" - local destination="$3" - local artifact_digest - artifact_digest="$(sha256sum "${output_directory}/${primary_path}" | awk '{print $1}')" - mkdir -p "$(dirname "${output_directory}/${destination}")" - jq -n -S \ - --arg artifact_digest "${artifact_digest}" \ - --arg artifact_path "${primary_path}" \ - --argjson package "${package}" \ - '{ - spdxVersion: "SPDX-2.3", - dataLicense: "CC0-1.0", - SPDXID: "SPDXRef-DOCUMENT", - name: ("RAPIDS release artifact " + $artifact_path), - documentNamespace: ("https://rapids.ai/release-platform/spdx/" + $artifact_digest), - creationInfo: { - creators: ["Tool: rapidsai/shared-workflows release-build-output"], - created: (now | strftime("%Y-%m-%dT%H:%M:%SZ")) - }, - documentDescribes: ["SPDXRef-Artifact"], - packages: [{ - SPDXID: "SPDXRef-Artifact", - name: $package.name, - versionInfo: $package.version, - downloadLocation: "NOASSERTION", - filesAnalyzed: false, - checksums: [{algorithm: "SHA256", checksumValue: $artifact_digest}] - }], - relationships: [{ - spdxElementId: "SPDXRef-DOCUMENT", - relationshipType: "DESCRIBES", - relatedSpdxElement: "SPDXRef-Artifact" - }], - comment: "Artifact-identity SBOM envelope. A producer-supplied dependency SBOM may replace this record." - }' >"${output_directory}/${destination}" -} - -write_generated_provenance() { - local primary_path="$1" - local package="$2" - local destination="$3" - local artifact_digest - artifact_digest="$(sha256sum "${output_directory}/${primary_path}" | awk '{print $1}')" - mkdir -p "$(dirname "${output_directory}/${destination}")" - jq -n -S \ - --arg artifact_digest "${artifact_digest}" \ - --arg artifact_path "${primary_path}" \ - --arg repository "${GITHUB_REPOSITORY:-}" \ - --arg run_attempt "${GITHUB_RUN_ATTEMPT:-}" \ - --arg run_id "${GITHUB_RUN_ID:-}" \ - --arg source_sha "${source_sha}" \ - --arg workflow_ref "${GITHUB_WORKFLOW_REF:-}" \ - --argjson package "${package}" \ - '{ - _type: "https://in-toto.io/Statement/v1", - subject: [{name: $artifact_path, digest: {sha256: $artifact_digest}}], - predicateType: "https://slsa.dev/provenance/v1", - predicate: { - buildDefinition: { - buildType: "https://rapids.ai/release-platform/build-output/v1", - externalParameters: {release_unit: env.RELEASE_UNIT, package: $package}, - resolvedDependencies: [{ - uri: ("git+https://github.com/" + $repository + "@" + $source_sha), - digest: {gitCommit: $source_sha} - }] - }, - runDetails: { - builder: {id: ("https://github.com/" + $workflow_ref)}, - metadata: {invocationId: ("https://github.com/" + $repository + "/actions/runs/" + $run_id + "/attempts/" + $run_attempt)} - } - } - }' >"${output_directory}/${destination}" -} - -shopt -s globstar nullglob -artifact_metadata='[]' -while IFS= read -r descriptor; do - if ! jq -e ' - type == "object" - and (keys - ["path", "sbom", "provenance", "signature", "package"] | length == 0) - and (.path | type == "string" and length > 0) - and ((.sbom // "") | type == "string") - and ((.provenance // "") | type == "string") - and ((.signature // "") | type == "string") - and ((.package // {}) | type == "object") - and ((.package // {} | keys - ["ecosystem", "name", "version", "build", "platform"]) | length == 0) - and ((.package // {} | to_entries | map(.value | type == "string" and length > 0) | all)) - ' <<<"${descriptor}" >/dev/null; then - echo "every release-artifacts entry must contain path and optional SBOM/provenance/signature/package overrides" >&2 - exit 1 - fi - - primary_path="$(resolve_one_file path "$(jq -r '.path' <<<"${descriptor}")")" - sbom_pattern="$(jq -r '.sbom // empty' <<<"${descriptor}")" - provenance_pattern="$(jq -r '.provenance // empty' <<<"${descriptor}")" - signature_pattern="$(jq -r '.signature // empty' <<<"${descriptor}")" - package_override="$(jq -c '.package // {}' <<<"${descriptor}")" - - package="$(jq -cn --argjson base "${RELEASE_PACKAGE}" --argjson override "${package_override}" '$base + $override')" - package_version="$(jq -r '.version // empty' <<<"${package}")" - if [[ -z "${package_version}" ]]; then - package_version="$(derive_package_version "$(jq -r '.ecosystem' <<<"${package}")" "$(jq -r '.name' <<<"${package}")" "${primary_path}")" - package="$(jq -c --arg version "${package_version}" '. + {version: $version}' <<<"${package}")" - fi - - if [[ -n "${sbom_pattern}" ]]; then - supplied_sbom_path="$(resolve_one_file sbom "${sbom_pattern}")" - sbom_path="$(copy_supplied_evidence "${primary_path}" "${supplied_sbom_path}" "sbom")" - sbom_kind="producer-dependency" - else - sbom_path="$(generated_evidence_path "${primary_path}" "spdx")" - write_generated_sbom "${primary_path}" "${package}" "${sbom_path}" - sbom_kind="generated-identity" - fi - if [[ -n "${provenance_pattern}" ]]; then - supplied_provenance_path="$(resolve_one_file provenance "${provenance_pattern}")" - provenance_path="$(copy_supplied_evidence "${primary_path}" "${supplied_provenance_path}" "provenance")" - else - provenance_path="$(generated_evidence_path "${primary_path}" "provenance")" - write_generated_provenance "${primary_path}" "${package}" "${provenance_path}" - fi - artifact="$(jq -cn \ - --arg unit_id "${RELEASE_UNIT}" \ - --arg path "${primary_path}" \ - --arg sbom "${sbom_path}" \ - --arg provenance "${provenance_path}" \ - --argjson package "${package}" \ - '{unit_id: $unit_id, path: $path, sbom: $sbom, provenance: $provenance, package: $package}')" - if [[ -n "${signature_pattern}" ]]; then - supplied_signature_path="$(resolve_one_file signature "${signature_pattern}")" - signature_path="$(copy_supplied_evidence "${primary_path}" "${supplied_signature_path}" "signature")" - artifact="$(jq -c --arg signature "${signature_path}" '. + {signature: $signature}' <<<"${artifact}")" - fi - - artifact_metadata="$(jq -cn \ - --arg path "${primary_path}" \ - --arg sbom_kind "${sbom_kind}" \ - --argjson artifacts "${artifact_metadata}" \ - '$artifacts + [{path: $path, sbom_kind: $sbom_kind}]')" - - jq --argjson artifact "${artifact}" '.artifacts += [$artifact]' "${temporary_manifest}" >"${temporary_manifest}.next" - mv "${temporary_manifest}.next" "${temporary_manifest}" -done < <(jq -c '.[]' <<<"${RELEASE_ARTIFACTS}") - -if ! jq -e '.artifacts as $items | ($items | map([.unit_id, .path] | join("\u0000")) | unique | length) == ($items | length)' "${temporary_manifest}" >/dev/null; then - echo "release-artifacts contains duplicate unit/path entries" >&2 - exit 1 -fi - -jq -S . "${temporary_manifest}" >"${manifest_path}" -jq -n -S \ - --arg artifact_name "${RELEASE_SOURCE_ARTIFACT_NAME}" \ - --arg manifest_name "${RELEASE_MANIFEST_NAME}" \ - --arg repository "${GITHUB_REPOSITORY:-}" \ - --arg run_attempt "${GITHUB_RUN_ATTEMPT:-}" \ - --arg run_id "${GITHUB_RUN_ID:-}" \ - --arg sha "${source_sha}" \ - --arg unit_id "${RELEASE_UNIT}" \ - --arg workflow_ref "${GITHUB_WORKFLOW_REF:-}" \ - --argjson artifact_metadata "${artifact_metadata}" \ - '{ - schema_version: 1, - producer: "shared-workflows", - release_unit: $unit_id, - source_artifact: $artifact_name, - build_output_manifest: $manifest_name, - build_environment: { - repository: $repository, - sha: $sha, - workflow_ref: $workflow_ref, - run_id: $run_id, - run_attempt: $run_attempt - }, - metadata: {artifacts: $artifact_metadata} - }' >"${metadata_path}" -echo "manifest-path=${manifest_path}" >>"${GITHUB_OUTPUT}" -echo "metadata-path=${metadata_path}" >>"${GITHUB_OUTPUT}" diff --git a/release-build-output/prepare.sh b/release-build-output/prepare.sh deleted file mode 100755 index f3aa97f4..00000000 --- a/release-build-output/prepare.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - -set -euo pipefail - -require_nonempty() { - local name="$1" - local value="$2" - if [[ -z "${value}" ]]; then - echo "${name} must be a non-empty string" >&2 - exit 1 - fi -} - -require_nonempty "RELEASE_ARTIFACT_TYPE" "${RELEASE_ARTIFACT_TYPE:-}" -require_nonempty "RELEASE_OUTPUT_DIRECTORY" "${RELEASE_OUTPUT_DIRECTORY:-}" - -script_directory="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -artifacts="${RELEASE_ARTIFACTS:-}" -package="${RELEASE_PACKAGE:-}" - -case "${RELEASE_ARTIFACT_TYPE}" in - conda) - if [[ -z "${artifacts}" ]]; then - artifacts="$("${script_directory}/describe-conda.sh" "${RELEASE_OUTPUT_DIRECTORY}")" - fi - if [[ -z "${package}" && -z "${RELEASE_PACKAGE_FILE:-}" ]]; then - package='{"ecosystem":"conda","name":"bundle"}' - fi - ;; - wheel) - if [[ -z "${artifacts}" ]]; then - artifacts="$("${script_directory}/describe-wheels.sh" "${RELEASE_OUTPUT_DIRECTORY}")" - fi - if [[ -z "${package}" && -z "${RELEASE_PACKAGE_FILE:-}" ]]; then - package='{"ecosystem":"wheel","name":"bundle"}' - fi - ;; - custom) - require_nonempty "RELEASE_ARTIFACTS" "${artifacts}" - ;; - *) - echo "artifact-type must be one of: conda, custom, wheel" >&2 - exit 1 - ;; -esac - -if [[ -z "${package}" && -z "${RELEASE_PACKAGE_FILE:-}" ]]; then - echo "one of release-package or release-package-file is required for custom artifacts" >&2 - exit 1 -fi - -printf 'artifacts=%s\n' "${artifacts}" >>"${GITHUB_OUTPUT}" -printf 'package=%s\n' "${package}" >>"${GITHUB_OUTPUT}" diff --git a/release-catalog/README.md b/release-catalog/README.md new file mode 100644 index 00000000..16a5ffa4 --- /dev/null +++ b/release-catalog/README.md @@ -0,0 +1,219 @@ +# Release catalog companions + +## Why this exists + +RAPIDS builds hundreds of binary artifacts (conda packages, wheels, JARs, and +others) across many repositories and CI jobs. Two things are hard to do with +those artifacts today: + +- **Assemble and test a release before tagging it.** We want to pick a + consistent set of artifacts, test them together as a candidate, and only + then publish. That requires a single inventory of what each job built. +- **Keep build-time evidence.** Facts such as which commit, workflow, and run + produced a wheel cannot be reliably reconstructed after the fact. They have + to be captured by the job that built the artifact. This captured metadata + can make security scans more accurate. + +This action solves both by having each build job write a small **companion** +next to its artifacts: a folder with an inventory of the files that the job +produced, plus provenance and SBOM documents for each one. The companion never +modifies the artifacts themselves. + +## Lifecycle + +1. A build job produces its artifacts as usual. +2. This action runs as a later step in that job and writes a companion into + the artifact directory. +3. The artifacts and their companion are uploaded to a private S3 bucket + (`upload-to-s3: true`). +4. The [RAPIDS release platform (in + development)](https://github.com/rapidsai/release-scripts/pull/113) reads the + companions from every job in a release train, merges them into the **release + catalog**, and uses the catalog to assemble and test a **release candidate**. + +"Release candidate" here means a set of artifacts that can be deployed to +create a release, not a release-candidate version of one artifact. + +## Glossary + +| Term | Meaning | +| ---- | ------- | +| **Artifact** | A file produced by a build job that we may publish: a `.conda` file, a wheel, a JAR, and so on. Sometimes called the *primary* artifact to distinguish it from its evidence files. | +| **Companion** | The set of files this action writes next to a job's artifacts: `release-catalog-entries.json` plus a `release-evidence/` directory. One companion per build job. | +| **Producer** | The tool that wrote the companion: this action, recorded as `rapidsai/shared-actions/release-catalog` in the `producer` field. The build job that created the artifacts is identified by the `source` block instead. | +| **Release catalog** | The merged inventory of all companions for a release, maintained by the release platform. | +| **Release train** | One planned release of RAPIDS, identified by an immutable JSON document that the [release platform](https://github.com/rapidsai/release-scripts/issues/102) maintains. Its SHA-256 is used to group all companions that belong to that release in S3. | +| **Release candidate** | A set of artifacts selected from the release catalog to be tested and, if it passes, published. | +| **Provenance** | A [SLSA](https://slsa.dev/spec/v1.2/attestation-model) statement describing how, where, and from what source an artifact was built. | +| **SBOM** | Software bill of materials. In general an SBOM lists a package's components by ecosystem, name, and version. The [CycloneDX](https://cyclonedx.org/docs/1.6/json/) SBOM this action generates lists only the artifact itself and its digest; listing dependencies is a future addition. | + +## Companion contents + +Each build job creates one companion with this shape: + +```text +. +├── release-catalog-entries.json +└── release-evidence + ├── ..provenance.json + ├── ..sbom.cdx.json + ├── ..provenance.json + ├── ..sbom.cdx.json + └── ... +``` + +Evidence filenames are derived from the artifact's path relative to +`artifact_directory`, with `/` replaced by `_`, so `linux-64/librmm-1.0.conda` +produces `release-evidence/linux-64_librmm-1.0.conda..provenance.json`. + +`release-catalog-entries.json` records the artifacts produced by the job +(`.conda`, `.whl`, or otherwise), the SHA-256 of each file, the package +identity of each one, the paths to its evidence files, and the source context +of the job (repository, commit, workflow, run). See +[`examples/cuvs-java`](examples/cuvs-java) for a complete companion. + +Generated entry documents conform to +[`entries.schema.json`](entries.schema.json). The schema fixes the versioned +output contract for producer identity, source context, package identity, and +evidence paths. + +For each artifact in its `entries` array, the action creates one provenance +file and one [CycloneDX 1.6](https://cyclonedx.org/docs/1.6/json/) SBOM. The SHA-256 in each evidence +filename is the digest of that artifact's contents, so evidence files can +never be confused across artifacts. The catalog entry lists the SBOM path +under `sbom`. + +Each entry also carries `sbom_kind: "generated-identity"`. This marks the +SBOM as an identity-only document generated by this action. On its own, each +identity SBOM binds a name, version, and digest in a format scanners already +consume. Merged across all jobs, they inventory every package a release ships. +They do not yet describe what is inside each package; that is what the future +dependency SBOM will add. That will be a new `sbom_kind` value and a new +`schema_version`, so existing consumers can tell the two apart. + +## Action inputs + +The action is used via `rapidsai/shared-actions/release-catalog`. + +Configuration, materialization, and upload failures appear as GitHub Actions +error annotations. When the scripts run outside GitHub Actions, the same errors +are written to stderr with the responsible script name. + +| Input | Required | Description | +| ----- | -------- | ----------- | +| `config` | yes | JSON object selecting the artifacts and their release catalog key. Schema: [`config.schema.json`](config.schema.json). Details below. | +| `source-sha` | yes | Full 40- or 64-character Git object ID of the commit that was checked out and built. Standard RAPIDS workflows pass `${{ env.RAPIDS_SHA }}` from `rapids-github-info`. | +| `source-artifact-name` | yes | Stable name for this job's output, for example the GitHub Actions artifact name. Used as the last path component in S3. | +| `upload-to-s3` | no | `"true"` to upload the artifacts and companion to S3. Defaults to `"false"`, which only writes the companion locally. | +| `candidate-train-sha256` | when uploading | SHA-256 of the release train JSON this build belongs to. All companions for one release share it. | +| `candidate-bucket` | no | S3 bucket for companions. Defaults to `rapids-release-candidates`. | +| `candidate-prefix` | no | Optional root prefix inside the bucket. Defaults to empty, placing each train directly below the bucket root. | + +When uploading, every file the companion lists (the artifacts, the evidence +files, and `release-catalog-entries.json` itself) lands under +`s3:///////`. +When `candidate-prefix` is set explicitly, it is inserted between the bucket +name and the train SHA-256. +Uploads are conditional: an existing object is accepted only if its bytes +match, so reruns are safe. + +## `config` contents + +The canonical schema and field documentation are in +[`config.schema.json`](config.schema.json). It is validated at runtime by +[validate-config.sh](validate-config.sh). + +### `release_catalog_key` + +This is a grouping label that represents a common "release policy." Artifacts +that share a `release_catalog_key` are versioned, validated, ordered, +published, and promoted together. For example, the `cudf` and `dask-cudf` +Conda packages both use `conda:cudf` because they share the same version, +validation, inter-package order, and publishing destination, and are +ultimately published together. + +The key does not need to be unique per artifact or per matrix variant. Use a +separate key only when the outputs intentionally follow a separate release +workflow. + +Standard RAPIDS Conda and wheel workflows use `:`, +such as `conda:cudf`, because those workflows apply a repository-level release +policy. A custom job should use `:`, such as +`maven:cuvs-java`, or simply `maven:cuvs`. + +### `artifact_directory` and `artifacts` + +`artifact_directory` is the required base directory containing the artifacts, +relative to the job working directory. The action does not scan the working +directory by default. +For standard Conda, wheel, and Maven JAR jobs, omit `artifacts`; the action +discovers supported artifacts below that directory and parses package identity +from their embedded metadata. + +```yaml +- name: Create wheel release catalog companion + uses: rapidsai/shared-actions/release-catalog@main + with: + config: >- + { + "release_catalog_key": "wheel:example", + "artifact_directory": ${{ toJSON(steps.package-name.outputs.WHEEL_OUTPUT_DIR) }} + } + source-artifact-name: ${{ steps.package-name.outputs.RAPIDS_PACKAGE_NAME }} + source-sha: ${{ env.RAPIDS_SHA }} +``` + +Other formats require an explicit `artifacts` list. Each dictionary in the list +corresponds to one artifact file. Wildcards are allowed, but a wildcard that +resolves to zero files or multiple files results in an error. + +```yaml +- name: Create custom release catalog companion + uses: rapidsai/shared-actions/release-catalog@main + with: + config: >- + { + "release_catalog_key": "maven:cuvs-java", + "artifact_directory": "java/cuvs-java/target", + "artifacts": [{ + "path": "cuvs-java-*-x86_64-cuda*.jar" + }] + } + source-artifact-name: cuvs-java + source-sha: ${{ env.RAPIDS_SHA }} +``` + +For a JAR, the action requires exactly one +`META-INF/maven///pom.properties` file and reads its +`groupId`, `artifactId`, and `version`. The directory coordinates must match +the property values. JARs without this metadata, or shaded JARs containing +multiple Maven descriptors, require `package_identity_file` so the action does +not guess which package the artifact represents. + +#### Package identity file + +`package_identity_file` is a path to a JSON file that contains at least +`ecosystem`, `name`, and `version` keys; `build` and `platform` are optional. +This path is relative to `artifact_directory`. + +```json +{ + "ecosystem": "maven", + "name": "ai.rapids:cuvs-java", + "version": "26.08.0" +} +``` + +Multiple artifacts in the list may reference the same identity file when the +files have the same package identity. + +## Design notes + +Provenance and the SBOM are kept as separate documents. Provenance describes +how a package was built and by whom; an SBOM identifies the package and, once +dependency SBOMs are added, what makes it up. They complement each other: an +SBOM without provenance may describe the wrong or an untrusted artifact, while +provenance without an SBOM cannot be matched to a package by name and version. +Although CycloneDX can carry some build metadata, embedding provenance in the +SBOM would not replace the in-toto/SLSA statement and would couple evidence +with different consumers and lifecycles. diff --git a/release-catalog/action.yml b/release-catalog/action.yml new file mode 100644 index 00000000..493fc3d7 --- /dev/null +++ b/release-catalog/action.yml @@ -0,0 +1,88 @@ +name: Release catalog +description: >- + Create a release catalog companion (artifact inventory, provenance, and CycloneDX SBOM) + for the artifacts a build job produced, and optionally upload it to S3. + See release-catalog/README.md. + +inputs: + config: + description: >- + JSON string with configuration for this action. Schema and field documentation: + https://github.com/rapidsai/shared-actions/blob/main/release-catalog/config.schema.json + required: true + source-sha: + description: >- + Full 40- or 64-character Git object ID of the commit that was checked out + and built. Standard RAPIDS workflows pass RAPIDS_SHA from rapids-github-info. + required: true + source-artifact-name: + description: >- + Stable name for this job's output, such as the GitHub Actions artifact + name. Used as the last path component in S3. + required: true + candidate-train-sha256: + description: >- + SHA-256 of the release train JSON this build belongs to. Required when + upload-to-s3 is true; ignored otherwise. + required: false + default: "" + candidate-bucket: + description: S3 bucket that receives the companion when upload-to-s3 is true. + required: false + default: rapids-release-candidates + candidate-prefix: + description: Optional root prefix inside candidate-bucket. Empty writes below the bucket root. + required: false + default: "" + upload-to-s3: + description: Whether to upload the companion to S3. When false, the companion is only written locally. + required: false + default: "false" +runs: + using: composite + steps: + - name: Clone shared-actions repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ env.SHARED_ACTIONS_REPO || 'rapidsai/shared-actions' }} + ref: ${{ env.SHARED_ACTIONS_REF || 'main' }} + path: ./shared-actions + persist-credentials: false + - id: implementation + name: Resolve release catalog implementation + shell: bash + run: | + implementation_path="${PWD}/shared-actions/release-catalog" + if [[ ! -d "${implementation_path}" ]]; then + echo "release-catalog implementation not found at ${implementation_path}" >&2 + exit 1 + fi + echo "path=${implementation_path}" >>"${GITHUB_OUTPUT}" + - id: configuration + name: Validate release catalog configuration + shell: bash + env: + IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} + RELEASE_CATALOG_CONFIG: ${{ inputs.config }} + run: "${IMPLEMENTATION_PATH}/validate-config.sh" + - name: Materialize release catalog entries + shell: bash + env: + IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} + RELEASE_ARTIFACTS: ${{ steps.configuration.outputs.artifacts }} + RELEASE_ARTIFACT_DIRECTORY: ${{ steps.configuration.outputs.artifact_directory }} + RELEASE_SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} + RELEASE_SOURCE_SHA: ${{ inputs.source-sha }} + RELEASE_CATALOG_KEY: ${{ steps.configuration.outputs.release_catalog_key }} + run: "${IMPLEMENTATION_PATH}/materialize.sh" + - name: Upload release catalog companion to S3 + if: ${{ inputs.upload-to-s3 == 'true' }} + shell: bash + env: + IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} + RELEASE_ARTIFACT_DIRECTORY: ${{ steps.configuration.outputs.artifact_directory }} + RELEASE_CANDIDATE_BUCKET: ${{ inputs.candidate-bucket }} + RELEASE_CANDIDATE_PREFIX: ${{ inputs.candidate-prefix }} + RELEASE_CANDIDATE_TRAIN_SHA256: ${{ inputs.candidate-train-sha256 }} + RELEASE_SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} + run: "${IMPLEMENTATION_PATH}/upload-s3.sh" diff --git a/release-catalog/config.schema.json b/release-catalog/config.schema.json new file mode 100644 index 00000000..2e2c4a7f --- /dev/null +++ b/release-catalog/config.schema.json @@ -0,0 +1,66 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://rapids.ai/schemas/release-catalog-config-v1.json", + "title": "Release catalog configuration", + "description": "Caller configuration for one release catalog companion.", + "type": "object", + "additionalProperties": false, + "required": [ + "release_catalog_key", + "artifact_directory" + ], + "properties": { + "release_catalog_key": { + "description": "Label for artifact group. Artifacts within a group share one release policy. This is not a UUID, is not generated by the action, and need not be unique per package. Standard workflows use :, such as conda:cudf; custom jobs use :, such as maven:cuvs-java. See https://github.com/rapidsai/shared-actions/blob/main/release-catalog/README.md#release_catalog_key", + "type": "string", + "minLength": 1, + "pattern": "^[^\\r\\n]+$", + "not": { + "pattern": "(^/)|((^|/)\\.\\.(/|$))" + } + }, + "artifact_directory": { + "description": "Base directory for artifact paths, relative to the job working directory.", + "type": "string", + "minLength": 1, + "pattern": "^[^\\r\\n]+$" + }, + "artifacts": { + "description": "Optional explicit artifact list. When omitted, Conda packages, wheels, and Maven JARs are discovered in artifact_directory.", + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/artifact" + } + } + }, + "$defs": { + "artifact": { + "type": "object", + "additionalProperties": false, + "required": [ + "path" + ], + "properties": { + "path": { + "description": "Artifact path or glob, relative to artifact_directory. Absolute paths and parent-directory traversal are forbidden.", + "type": "string", + "minLength": 1, + "pattern": "^[^\\r\\n]+$", + "not": { + "pattern": "(^/)|((^|/)\\.\\.(/|$))" + } + }, + "package_identity_file": { + "description": "Path, relative to artifact_directory, to package identity JSON file for this artifact. Required only when identity cannot be extracted from the artifact. See https://github.com/rapidsai/shared-actions/blob/main/release-catalog/README.md#package-identity-file", + "type": "string", + "minLength": 1, + "pattern": "^[^\\r\\n]+$", + "not": { + "pattern": "(^/)|((^|/)\\.\\.(/|$))" + } + } + } + } + } +} diff --git a/release-catalog/entries.schema.json b/release-catalog/entries.schema.json new file mode 100644 index 00000000..c51669d9 --- /dev/null +++ b/release-catalog/entries.schema.json @@ -0,0 +1,71 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://rapids.ai/schemas/release-catalog-entries-v1.json", + "title": "Release catalog companion entries", + "description": "Generated inventory and evidence references for one build job.", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "producer", "source", "entries"], + "properties": { + "schema_version": {"const": 1}, + "producer": { + "description": "The tool that generated this document. Matches metadata.tools in the CycloneDX SBOM.", + "const": "rapidsai/shared-actions/release-catalog" + }, + "source": {"$ref": "#/$defs/source"}, + "entries": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/entry"} + } + }, + "$defs": { + "relative_path": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\r\\n]+$", + "not": {"pattern": "(^/)|((^|/)\\.\\.(/|$))"} + }, + "source": { + "description": "GitHub Actions context for the job that produced this companion. It binds every entry to the built source revision and the workflow run that created it.", + "type": "object", + "additionalProperties": false, + "required": ["artifact", "repository", "sha", "workflow_ref", "run_id", "run_attempt"], + "properties": { + "artifact": {"type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$"}, + "repository": {"type": "string", "pattern": "^[^/\\r\\n]+/[^/\\r\\n]+$"}, + "sha": {"type": "string", "pattern": "^(?:[0-9a-fA-F]{40}|[0-9a-fA-F]{64})$"}, + "workflow_ref": {"type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$"}, + "run_id": {"type": "string", "pattern": "^[1-9][0-9]*$"}, + "run_attempt": {"type": "string", "pattern": "^[1-9][0-9]*$"} + } + }, + "package": { + "description": "Package identity used by release tooling and publication destinations. It is extracted from supported artifacts or supplied explicitly when the artifact format cannot provide it.", + "type": "object", + "additionalProperties": false, + "required": ["ecosystem", "name", "version"], + "properties": { + "ecosystem": {"type": "string", "minLength": 1}, + "name": {"type": "string", "minLength": 1}, + "version": {"type": "string", "minLength": 1}, + "build": {"type": "string", "minLength": 1}, + "platform": {"type": "string", "minLength": 1} + } + }, + "entry": { + "type": "object", + "additionalProperties": false, + "required": ["release_catalog_key", "path", "sha256", "package", "sbom", "sbom_kind", "provenance"], + "properties": { + "release_catalog_key": {"type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$"}, + "path": {"$ref": "#/$defs/relative_path"}, + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "package": {"$ref": "#/$defs/package"}, + "sbom": {"$ref": "#/$defs/relative_path"}, + "sbom_kind": {"const": "generated-identity"}, + "provenance": {"$ref": "#/$defs/relative_path"} + } + } + } +} diff --git a/release-catalog/error.sh b/release-catalog/error.sh new file mode 100644 index 00000000..d62d0434 --- /dev/null +++ b/release-catalog/error.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +# Shared error rendering for release-catalog scripts. GitHub Actions receives a +# workflow-command annotation; local callers receive a conventional stderr line. + +release_catalog_error() { + local message="$1" + local script_name="${RELEASE_CATALOG_SCRIPT_NAME:-release-catalog}" + local title="${RELEASE_CATALOG_ERROR_TITLE:-Release catalog error}" + + if [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then + message="${message//'%'/'%25'}" + message="${message//$'\r'/'%0D'}" + message="${message//$'\n'/'%0A'}" + printf '::error title=%s::%s\n' "${title}" "${message}" >&2 + else + printf '[%s] Error: %s\n' "${script_name}" "${message}" >&2 + fi +} diff --git a/release-catalog/examples/README.md b/release-catalog/examples/README.md new file mode 100644 index 00000000..acc8132f --- /dev/null +++ b/release-catalog/examples/README.md @@ -0,0 +1,16 @@ +# Example companion + +[`cuvs-java`](cuvs-java) is a complete, representative companion for one cuVS +Java build job. It was derived from the test fixture in +`tests/release_catalog_test.sh`; its source SHA and GitHub Actions run ID are +deliberately non-production example values. + +The JAR itself is an artifact, not part of the companion, and is therefore not +checked in. +Its example contents are `jar` followed by a newline, whose SHA-256 digest is +`fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9`. +That digest links the catalog entry, CycloneDX document, provenance +statement, and the artifact, which is uploaded to S3 alongside the companion. + +The three JSON files are about 3 KiB in total. Companion size grows linearly +with the number of artifacts in the job. diff --git a/release-catalog/examples/cuvs-java/release-catalog-entries.json b/release-catalog/examples/cuvs-java/release-catalog-entries.json new file mode 100644 index 00000000..93ca43cc --- /dev/null +++ b/release-catalog/examples/cuvs-java/release-catalog-entries.json @@ -0,0 +1,27 @@ +{ + "entries": [ + { + "package": { + "ecosystem": "maven", + "name": "ai.rapids:cuvs-java", + "version": "26.08.0" + }, + "path": "cuvs-java-26.08.0.jar", + "provenance": "release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.provenance.json", + "release_catalog_key": "maven:cuvs-java", + "sbom": "release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.cdx.json", + "sbom_kind": "generated-identity", + "sha256": "fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9" + } + ], + "producer": "rapidsai/shared-actions/release-catalog", + "schema_version": 1, + "source": { + "artifact": "cuvs-java-cuda12.9.1", + "repository": "NVIDIA/cuvs", + "run_attempt": "1", + "run_id": "1234", + "sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "workflow_ref": "NVIDIA/cuvs/.github/workflows/build.yaml@refs/heads/release/26.08" + } +} diff --git a/release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.provenance.json b/release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.provenance.json new file mode 100644 index 00000000..8b7189b8 --- /dev/null +++ b/release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.provenance.json @@ -0,0 +1,41 @@ +{ + "_type": "https://in-toto.io/Statement/v1", + "predicate": { + "buildDefinition": { + "buildType": "https://rapids.ai/release-platform/catalog-record/v1", + "externalParameters": { + "package": { + "ecosystem": "maven", + "name": "ai.rapids:cuvs-java", + "version": "26.08.0" + }, + "release_catalog_key": "maven:cuvs-java" + }, + "resolvedDependencies": [ + { + "digest": { + "gitCommit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + "uri": "git+https://github.com/NVIDIA/cuvs@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] + }, + "runDetails": { + "builder": { + "id": "https://github.com/NVIDIA/cuvs/.github/workflows/build.yaml@refs/heads/release/26.08" + }, + "metadata": { + "invocationId": "https://github.com/NVIDIA/cuvs/actions/runs/1234/attempts/1" + } + } + }, + "predicateType": "https://slsa.dev/provenance/v1", + "subject": [ + { + "digest": { + "sha256": "fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9" + }, + "name": "cuvs-java-26.08.0.jar" + } + ] +} diff --git a/release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.cdx.json b/release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.cdx.json new file mode 100644 index 00000000..8b237a4e --- /dev/null +++ b/release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.cdx.json @@ -0,0 +1,39 @@ +{ + "bomFormat": "CycloneDX", + "metadata": { + "component": { + "bom-ref": "urn:sha256:fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9", + "hashes": [ + { + "alg": "SHA-256", + "content": "fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9" + } + ], + "name": "ai.rapids:cuvs-java", + "properties": [ + { + "name": "rapids:artifact:path", + "value": "cuvs-java-26.08.0.jar" + }, + { + "name": "rapids:package:ecosystem", + "value": "maven" + } + ], + "type": "file", + "version": "26.08.0" + }, + "timestamp": "2026-08-21T00:00:00Z", + "tools": { + "components": [ + { + "group": "rapidsai", + "name": "shared-actions/release-catalog", + "type": "application" + } + ] + } + }, + "specVersion": "1.6", + "version": 1 +} diff --git a/release-catalog/materialize.sh b/release-catalog/materialize.sh new file mode 100755 index 00000000..7eb0baad --- /dev/null +++ b/release-catalog/materialize.sh @@ -0,0 +1,534 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +set -euo pipefail + +export RELEASE_CATALOG_SCRIPT_NAME="materialize.sh" +export RELEASE_CATALOG_ERROR_TITLE="Release catalog materialization failed" +# shellcheck source=release-catalog/error.sh +source "$(dirname "${BASH_SOURCE[0]}")/error.sh" + +# Record artifact metadata from completed build directory into a release catalog companion. +# +# Inputs come from release-catalog/action.yml. RELEASE_ARTIFACTS is +# either empty (discover Conda packages, wheels, and Maven JARs) or a validated JSON array +# selecting custom artifacts. GitHub Actions supplies the GITHUB_* build +# context, while RELEASE_SOURCE_SHA identifies the commit actually checked out. +# +# The script validates all context before touching outputs, resolves every +# artifact to exactly one file inside RELEASE_ARTIFACT_DIRECTORY, reads or +# extracts its package identity, generates an identity-only CycloneDX SBOM plus +# provenance evidence, and writes release-catalog-entries.json. It never +# modifies primary files. + +require_nonempty() { + local name="$1" + local value="$2" + if [[ -z "${value}" ]]; then + release_catalog_error "${name} must be a non-empty string" + exit 1 + fi +} + +require_single_line() { + local name="$1" + local value="$2" + require_nonempty "${name}" "${value}" + if [[ "${value}" == *$'\n'* || "${value}" == *$'\r'* ]]; then + release_catalog_error "${name} must be a single-line string" + exit 1 + fi +} + +require_positive_integer() { + local name="$1" + local value="$2" + if [[ ! "${value}" =~ ^[1-9][0-9]*$ ]]; then + release_catalog_error "${name} must be a positive integer" + exit 1 + fi +} + +# Fail before materialization when provenance would otherwise contain empty or +# malformed source fields. These values are guaranteed in GitHub Actions; a +# local caller must set explicit test values. +require_single_line "RELEASE_CATALOG_KEY" "${RELEASE_CATALOG_KEY:-}" +require_single_line "RELEASE_ARTIFACT_DIRECTORY" "${RELEASE_ARTIFACT_DIRECTORY:-}" +require_single_line "RELEASE_SOURCE_ARTIFACT_NAME" "${RELEASE_SOURCE_ARTIFACT_NAME:-}" + +source_sha="${RELEASE_SOURCE_SHA:-}" +require_single_line "RELEASE_SOURCE_SHA" "${source_sha}" +if [[ ! "${source_sha}" =~ ^[[:xdigit:]]{40}$ && ! "${source_sha}" =~ ^[[:xdigit:]]{64}$ ]]; then + release_catalog_error "RELEASE_SOURCE_SHA must be a 40- or 64-character hexadecimal Git object ID" + exit 1 +fi + +github_repository="${GITHUB_REPOSITORY:-}" +github_run_attempt="${GITHUB_RUN_ATTEMPT:-}" +github_run_id="${GITHUB_RUN_ID:-}" +github_workflow_ref="${GITHUB_WORKFLOW_REF:-}" +require_single_line "GITHUB_REPOSITORY" "${github_repository}" +require_single_line "GITHUB_RUN_ATTEMPT" "${github_run_attempt}" +require_single_line "GITHUB_RUN_ID" "${github_run_id}" +require_single_line "GITHUB_WORKFLOW_REF" "${github_workflow_ref}" +if [[ ! "${github_repository}" =~ ^[^/[:space:]]+/[^/[:space:]]+$ ]]; then + release_catalog_error "GITHUB_REPOSITORY must have owner/repository form" + exit 1 +fi +require_positive_integer "GITHUB_RUN_ATTEMPT" "${github_run_attempt}" +require_positive_integer "GITHUB_RUN_ID" "${github_run_id}" + +if [[ ! -d "${RELEASE_ARTIFACT_DIRECTORY}" ]]; then + release_catalog_error "artifact-directory does not exist or is not a directory: ${RELEASE_ARTIFACT_DIRECTORY}" + exit 1 +fi + +ensure_relative_pattern() { + local field="$1" + local pattern="$2" + if [[ "${pattern}" == /* || "${pattern}" == */../* || "${pattern}" == ../* || "${pattern}" == *"/.." ]]; then + release_catalog_error "${field} must be a relative path inside artifact-directory: ${pattern}" + exit 1 + fi +} + +artifact_directory="$(realpath "${RELEASE_ARTIFACT_DIRECTORY}")" + +# Artifact paths may contain globs for matrix-dependent filenames, but each +# descriptor must resolve to one regular file below the configured directory. +resolve_one_file() { + local field="$1" + local pattern="$2" + local -a matches=() + + ensure_relative_pattern "${field}" "${pattern}" + while IFS= read -r match; do + matches+=("${match}") + done < <(compgen -G "${artifact_directory}/${pattern}" || true) + if [[ "${#matches[@]}" -ne 1 || ! -f "${matches[0]:-}" ]]; then + release_catalog_error "${field} pattern must resolve to exactly one file: ${pattern}" + exit 1 + fi + + local resolved + resolved="$(realpath "${matches[0]}")" + if [[ "${resolved}" != "${artifact_directory}"/* ]]; then + release_catalog_error "${field} must resolve inside artifact-directory: ${pattern}" + exit 1 + fi + printf '%s\n' "${resolved#"${artifact_directory}/"}" +} + +# Normalize supported package formats into the common package identity stored +# in each catalog entry. Other formats supply the same fields via +# package_identity_file instead. + +# Wheels expose Core Metadata in .dist-info/METADATA. +describe_wheel_package() { + local wheel_path="$1" + local relative_path="${wheel_path#"${artifact_directory}/"}" + local -a metadata_members=() + local metadata_member + while IFS= read -r metadata_member; do + metadata_members+=("${metadata_member}") + done < <(unzip -Z1 "${wheel_path}" | awk '/\.dist-info\/METADATA$/') + if [[ "${#metadata_members[@]}" -ne 1 ]]; then + release_catalog_error "wheel must contain exactly one .dist-info/METADATA file: ${relative_path}" + exit 1 + fi + + local metadata package_name package_version + metadata="$(unzip -p "${wheel_path}" "${metadata_members[0]}")" + package_name="$(awk 'tolower($0) ~ /^name:[[:space:]]*/ {sub(/^[^:]*:[[:space:]]*/, ""); sub(/\r$/, ""); print; exit}' <<<"${metadata}")" + package_version="$(awk 'tolower($0) ~ /^version:[[:space:]]*/ {sub(/^[^:]*:[[:space:]]*/, ""); sub(/\r$/, ""); print; exit}' <<<"${metadata}")" + if [[ -z "${package_name}" || -z "${package_version}" ]]; then + release_catalog_error "wheel metadata must contain non-empty Name and Version fields: ${relative_path}" + exit 1 + fi + + jq -cn --arg name "${package_name}" --arg version "${package_version}" \ + '{ecosystem: "wheel", name: $name, version: $version}' +} + +# Maven-built JARs normally expose their coordinates in exactly one +# META-INF/maven///pom.properties file. Shaded JARs may +# contain several descriptors, so ambiguity is an error rather than a guess. +describe_maven_jar_package() { + local jar_path="$1" + local relative_path="${jar_path#"${artifact_directory}/"}" + local -a metadata_members=() + local metadata_member + while IFS= read -r metadata_member; do + metadata_members+=("${metadata_member}") + done < <(unzip -Z1 "${jar_path}" | awk '$0 ~ "^META-INF/maven/[^/]+/[^/]+/pom[.]properties$"') + if [[ "${#metadata_members[@]}" -ne 1 ]]; then + release_catalog_error "Maven JAR must contain exactly one META-INF/maven///pom.properties file: ${relative_path}" + return 1 + fi + + local metadata coordinates_path coordinate_group_id coordinate_artifact_id + local group_id artifact_id package_version + metadata="$(unzip -p "${jar_path}" "${metadata_members[0]}")" + coordinates_path="${metadata_members[0]#META-INF/maven/}" + coordinates_path="${coordinates_path%/pom.properties}" + coordinate_group_id="${coordinates_path%%/*}" + coordinate_artifact_id="${coordinates_path#*/}" + group_id="$(awk -F= '$1 == "groupId" {sub(/^[^=]*=/, ""); sub(/\r$/, ""); print; exit}' <<<"${metadata}")" + artifact_id="$(awk -F= '$1 == "artifactId" {sub(/^[^=]*=/, ""); sub(/\r$/, ""); print; exit}' <<<"${metadata}")" + package_version="$(awk -F= '$1 == "version" {sub(/^[^=]*=/, ""); sub(/\r$/, ""); print; exit}' <<<"${metadata}")" + if [[ -z "${group_id}" || -z "${artifact_id}" || -z "${package_version}" ]]; then + release_catalog_error "Maven pom.properties must contain non-empty groupId, artifactId, and version fields: ${relative_path}" + return 1 + fi + if [[ "${group_id}" != "${coordinate_group_id}" || "${artifact_id}" != "${coordinate_artifact_id}" ]]; then + release_catalog_error "Maven pom.properties coordinates do not match its META-INF path: ${relative_path}" + return 1 + fi + + jq -cn --arg name "${group_id}:${artifact_id}" --arg version "${package_version}" \ + '{ecosystem: "maven", name: $name, version: $version}' +} + +# Conda packages expose info/index.json. +describe_conda_package() { + local package_path="$1" + local relative_path="${package_path#"${artifact_directory}/"}" + local index_json + case "${package_path}" in + *.conda) + local -a info_members=() + local info_member + while IFS= read -r info_member; do + info_members+=("${info_member}") + done < <(unzip -Z1 "${package_path}" | awk '/^info-.*\.tar\.zst$/') + if [[ "${#info_members[@]}" -ne 1 ]]; then + release_catalog_error ".conda package must contain exactly one info-*.tar.zst member: ${relative_path}" + exit 1 + fi + index_json="$(unzip -p "${package_path}" "${info_members[0]}" | zstd -dc | tar -xOf - info/index.json)" + ;; + *.tar.bz2) + index_json="$(tar -xOjf "${package_path}" info/index.json)" + ;; + *) + release_catalog_error "unsupported Conda package extension: ${relative_path}" + exit 1 + ;; + esac + + if ! jq -e ' + type == "object" + and (.name | type == "string" and length > 0) + and (.version | type == "string" and length > 0) + and (.build | type == "string" and length > 0) + and (.subdir | type == "string" and length > 0) + ' <<<"${index_json}" >/dev/null; then + release_catalog_error "Conda info/index.json must contain exact name, version, build, and subdir fields: ${relative_path}" + exit 1 + fi + + jq -c '{ecosystem: "conda", name, version, build, platform: .subdir}' <<<"${index_json}" +} + +prepare_artifacts() { + local configured_artifacts="${RELEASE_ARTIFACTS:-}" + local prepared_artifacts='[]' + local conda_package descriptor package primary_path primary_file identity_file + + if [[ -z "${configured_artifacts}" ]]; then + local -a detected_files=() + while IFS= read -r primary_file; do + detected_files+=("${primary_file}") + done < <(find "${artifact_directory}" -type f \( -name '*.conda' -o -name '*.tar.bz2' -o -name '*.whl' -o -name '*.jar' \) -print | sort) + + for primary_file in "${detected_files[@]}"; do + primary_path="${primary_file#"${artifact_directory}/"}" + case "${primary_file}" in + *.whl) + if ! package="$(describe_wheel_package "${primary_file}")"; then + return 1 + fi + ;; + *.jar) + if ! package="$(describe_maven_jar_package "${primary_file}")"; then + return 1 + fi + ;; + *) + if ! package="$(describe_conda_package "${primary_file}")"; then + return 1 + fi + ;; + esac + descriptor="$(jq -cn --arg path "${primary_path}" --argjson package "${package}" '{path: $path, package: $package}')" + prepared_artifacts="$(jq -cn --argjson current "${prepared_artifacts}" --argjson descriptor "${descriptor}" '$current + [$descriptor]')" + done + else + if ! jq -e 'type == "array" and length > 0' <<<"${configured_artifacts}" >/dev/null; then + release_catalog_error "release-artifacts must be a non-empty JSON array when supplied" + exit 1 + fi + while IFS= read -r descriptor; do + if jq -e 'has("package")' <<<"${descriptor}" >/dev/null; then + release_catalog_error "package identity must not be supplied inline: ${descriptor}" + return 1 + fi + primary_path="$(resolve_one_file path "$(jq -r '.path' <<<"${descriptor}")")" + primary_file="${artifact_directory}/${primary_path}" + identity_file="$(jq -r '.package_identity_file // empty' <<<"${descriptor}")" + package='' + + case "${primary_file}" in + *.whl) + if [[ -n "${identity_file}" ]]; then + release_catalog_error "package_identity_file is not allowed when wheel identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" + exit 1 + fi + if ! package="$(describe_wheel_package "${primary_file}")"; then + return 1 + fi + ;; + *.conda) + if [[ -n "${identity_file}" ]]; then + release_catalog_error "package_identity_file is not allowed when Conda identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" + exit 1 + fi + if ! package="$(describe_conda_package "${primary_file}")"; then + return 1 + fi + ;; + *.jar) + if package="$(describe_maven_jar_package "${primary_file}" 2>/dev/null)"; then + if [[ -n "${identity_file}" ]]; then + release_catalog_error "package_identity_file is not allowed when Maven JAR identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" + exit 1 + fi + elif [[ -z "${identity_file}" ]]; then + release_catalog_error "JAR does not contain one unambiguous Maven package identity and requires package_identity_file: $(jq -r '.path' <<<"${descriptor}")" + exit 1 + fi + ;; + *.tar.bz2) + if conda_package="$(describe_conda_package "${primary_file}" 2>/dev/null)"; then + if [[ -n "${identity_file}" ]]; then + release_catalog_error "package_identity_file is not allowed when Conda identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" + exit 1 + fi + package="${conda_package}" + elif [[ -z "${identity_file}" ]]; then + release_catalog_error "artifact is not a valid Conda package and requires package_identity_file: $(jq -r '.path' <<<"${descriptor}")" + exit 1 + fi + ;; + *) + if [[ -z "${identity_file}" ]]; then + release_catalog_error "artifact identity cannot be extracted; package_identity_file is required: $(jq -r '.path' <<<"${descriptor}")" + exit 1 + fi + ;; + esac + + if [[ -n "${package}" ]]; then + descriptor="$(jq -c --argjson package "${package}" '. + {package: $package}' <<<"${descriptor}")" + fi + prepared_artifacts="$(jq -cn --argjson current "${prepared_artifacts}" --argjson descriptor "${descriptor}" '$current + [$descriptor]')" + done < <(jq -c '.[]' <<<"${configured_artifacts}") + fi + + if [[ "$(jq 'length' <<<"${prepared_artifacts}")" -eq 0 ]]; then + release_catalog_error "artifact-directory contains no detectable Conda, wheel, or Maven JAR artifacts; explicitly selected artifacts require package_identity_file when their identity cannot be parsed" + exit 1 + fi + printf '%s\n' "${prepared_artifacts}" +} + +# From here onward, every descriptor has an exact path and either extracted +# package identity or a caller-created package identity file. +artifacts="$(prepare_artifacts)" +# Fixed name: upload-s3.sh, the schema, and consumers all depend on it. +entries_path="${artifact_directory}/release-catalog-entries.json" +temporary_manifest="$(mktemp "${artifact_directory}/.release-catalog.XXXXXX")" +trap 'rm -f "${temporary_manifest}"' EXIT + +printf '%s\n' '{"entries":[]}' >"${temporary_manifest}" + +validate_package_identity() { + jq -e ' + type == "object" + and (keys - ["ecosystem", "name", "version", "build", "platform"] | length == 0) + and (.ecosystem | type == "string" and length > 0) + and (.name | type == "string" and length > 0) + and (.version | type == "string" and length > 0) + and ((has("build") | not) or (.build | type == "string" and length > 0)) + and ((has("platform") | not) or (.platform | type == "string" and length > 0)) + ' >/dev/null +} + +generated_evidence_path() { + local primary_path="$1" + local kind="$2" + local artifact_digest + artifact_digest="$(sha256sum "${artifact_directory}/${primary_path}" | awk '{print $1}')" + local artifact_label="${primary_path//\//_}" + printf 'release-evidence/%s.%s.%s.json\n' "${artifact_label}" "${artifact_digest}" "${kind}" +} + +# This initial CycloneDX implementation identifies the artifact only. Component +# inventory and the dependency graph are intentionally deferred until a future +# implementation can accept evidence captured by the producer during the build. +write_generated_cyclonedx_sbom() { + local primary_path="$1" + local package="$2" + local destination="$3" + local artifact_digest + artifact_digest="$(sha256sum "${artifact_directory}/${primary_path}" | awk '{print $1}')" + mkdir -p "$(dirname "${artifact_directory}/${destination}")" + jq -n -S \ + --arg artifact_digest "${artifact_digest}" \ + --arg artifact_path "${primary_path}" \ + --argjson package "${package}" \ + '{ + bomFormat: "CycloneDX", + specVersion: "1.6", + version: 1, + metadata: { + timestamp: (now | strftime("%Y-%m-%dT%H:%M:%SZ")), + tools: { + components: [{ + type: "application", + group: "rapidsai", + name: "shared-actions/release-catalog" + }] + }, + component: { + type: "file", + "bom-ref": ("urn:sha256:" + $artifact_digest), + name: $package.name, + version: $package.version, + hashes: [{alg: "SHA-256", content: $artifact_digest}], + properties: [ + {name: "rapids:artifact:path", value: $artifact_path}, + {name: "rapids:package:ecosystem", value: $package.ecosystem} + ] + } + } + }' >"${artifact_directory}/${destination}" +} + +write_generated_provenance() { + local primary_path="$1" + local package="$2" + local destination="$3" + local artifact_digest + artifact_digest="$(sha256sum "${artifact_directory}/${primary_path}" | awk '{print $1}')" + mkdir -p "$(dirname "${artifact_directory}/${destination}")" + jq -n -S \ + --arg artifact_digest "${artifact_digest}" \ + --arg artifact_path "${primary_path}" \ + --arg repository "${github_repository}" \ + --arg run_attempt "${github_run_attempt}" \ + --arg run_id "${github_run_id}" \ + --arg source_sha "${source_sha}" \ + --arg workflow_ref "${github_workflow_ref}" \ + --argjson package "${package}" \ + '{ + _type: "https://in-toto.io/Statement/v1", + subject: [{name: $artifact_path, digest: {sha256: $artifact_digest}}], + predicateType: "https://slsa.dev/provenance/v1", + predicate: { + buildDefinition: { + buildType: "https://rapids.ai/release-platform/catalog-record/v1", + externalParameters: {release_catalog_key: env.RELEASE_CATALOG_KEY, package: $package}, + resolvedDependencies: [{ + uri: ("git+https://github.com/" + $repository + "@" + $source_sha), + digest: {gitCommit: $source_sha} + }] + }, + runDetails: { + builder: {id: ("https://github.com/" + $workflow_ref)}, + metadata: {invocationId: ("https://github.com/" + $repository + "/actions/runs/" + $run_id + "/attempts/" + $run_attempt)} + } + } + }' >"${artifact_directory}/${destination}" +} + +shopt -s globstar nullglob +# Create one catalog entry, one identity-only SBOM document, and one provenance +# document per primary artifact. Caller-supplied dependency evidence is +# intentionally not accepted by this version of the schema. +while IFS= read -r descriptor; do + if ! jq -e ' + type == "object" + and (keys - ["path", "package_identity_file", "package"] | length == 0) + and (.path | type == "string" and length > 0) + and ((.package_identity_file // "") | type == "string") + and ([has("package"), has("package_identity_file")] | map(select(.)) | length == 1) + ' <<<"${descriptor}" >/dev/null; then + release_catalog_error "release artifact descriptor must contain path and exactly one package identity source: ${descriptor}" + exit 1 + fi + + primary_path="$(resolve_one_file path "$(jq -r '.path' <<<"${descriptor}")")" + package_identity_file="$(jq -r '.package_identity_file // empty' <<<"${descriptor}")" + if [[ -n "${package_identity_file}" ]]; then + package_identity_path="$(resolve_one_file package_identity_file "${package_identity_file}")" + if ! package="$(jq -ce . "${artifact_directory}/${package_identity_path}" 2>/dev/null)"; then + release_catalog_error "package_identity_file must contain valid JSON: ${package_identity_file}" + exit 1 + fi + else + package="$(jq -c '.package' <<<"${descriptor}")" + fi + if ! validate_package_identity <<<"${package}"; then + release_catalog_error "package identity for ${primary_path} must contain non-empty ecosystem, name, and version strings and only optional build or platform strings" + exit 1 + fi + + artifact_sha256="$(sha256sum "${artifact_directory}/${primary_path}" | awk '{print $1}')" + sbom_path="$(generated_evidence_path "${primary_path}" "sbom.cdx")" + write_generated_cyclonedx_sbom "${primary_path}" "${package}" "${sbom_path}" + provenance_path="$(generated_evidence_path "${primary_path}" "provenance")" + write_generated_provenance "${primary_path}" "${package}" "${provenance_path}" + + entry="$(jq -cn \ + --arg release_catalog_key "${RELEASE_CATALOG_KEY}" \ + --arg path "${primary_path}" \ + --arg sha256 "${artifact_sha256}" \ + --arg sbom "${sbom_path}" \ + --arg provenance "${provenance_path}" \ + --argjson package "${package}" \ + '{release_catalog_key: $release_catalog_key, path: $path, sha256: $sha256, sbom: $sbom, sbom_kind: "generated-identity", provenance: $provenance, package: $package}')" + + jq --argjson entry "${entry}" '.entries += [$entry]' "${temporary_manifest}" >"${temporary_manifest}.next" + mv "${temporary_manifest}.next" "${temporary_manifest}" +done < <(jq -c '.[]' <<<"${artifacts}") + +if ! jq -e '.entries as $items | ($items | map([.release_catalog_key, .path] | join("\u0000")) | unique | length) == ($items | length)' "${temporary_manifest}" >/dev/null; then + release_catalog_error "release-artifacts contains duplicate release_catalog_key/path entries" + exit 1 +fi + +# Store the job's source context (repository, commit, workflow, run) once at +# the top level rather than on every entry. The release platform keeps this +# association when it merges companions from many jobs into the catalog. +jq -n -S \ + --arg artifact_name "${RELEASE_SOURCE_ARTIFACT_NAME}" \ + --arg repository "${github_repository}" \ + --arg run_attempt "${github_run_attempt}" \ + --arg run_id "${github_run_id}" \ + --arg sha "${source_sha}" \ + --arg workflow_ref "${github_workflow_ref}" \ + --argjson entries "$(jq -c '.entries' "${temporary_manifest}")" \ + '{ + schema_version: 1, + producer: "rapidsai/shared-actions/release-catalog", + source: { + artifact: $artifact_name, + repository: $repository, + sha: $sha, + workflow_ref: $workflow_ref, + run_id: $run_id, + run_attempt: $run_attempt + }, + entries: $entries + }' >"${entries_path}" diff --git a/release-catalog/upload-s3.sh b/release-catalog/upload-s3.sh new file mode 100755 index 00000000..503eb2c9 --- /dev/null +++ b/release-catalog/upload-s3.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +# Upload the files declared by the generated catalog document. Uploading +# existing filenames with different content will raise an error. +set -euo pipefail + +export RELEASE_CATALOG_SCRIPT_NAME="upload-s3.sh" +export RELEASE_CATALOG_ERROR_TITLE="Release catalog upload failed" +# shellcheck source=release-catalog/error.sh +source "$(dirname "${BASH_SOURCE[0]}")/error.sh" + +require_value() { + local name="$1" + local value="$2" + if [[ -z "${value}" || "${value}" == *$'\n'* || "${value}" == *$'\r'* ]]; then + release_catalog_error "${name} must be a non-empty single-line string" + exit 1 + fi +} + +safe_prefix() { + local prefix="$1" + [[ "${prefix}" != /* && "${prefix}" != */ && "${prefix}" != *'//' && "${prefix}" != *'..'* ]] +} + +for value in RELEASE_ARTIFACT_DIRECTORY RELEASE_CANDIDATE_BUCKET RELEASE_CANDIDATE_TRAIN_SHA256 RELEASE_SOURCE_ARTIFACT_NAME GITHUB_REPOSITORY GITHUB_RUN_ID; do + require_value "${value}" "${!value:-}" +done +RELEASE_CANDIDATE_PREFIX="${RELEASE_CANDIDATE_PREFIX:-}" +if [[ ! "${RELEASE_CANDIDATE_TRAIN_SHA256}" =~ ^[[:xdigit:]]{64}$ ]]; then + release_catalog_error "RELEASE_CANDIDATE_TRAIN_SHA256 must be a SHA-256 hex digest" + exit 1 +fi +if [[ "${RELEASE_SOURCE_ARTIFACT_NAME}" == */* || "${RELEASE_SOURCE_ARTIFACT_NAME}" == *".."* ]]; then + release_catalog_error "RELEASE_SOURCE_ARTIFACT_NAME must be a plain name without path separators" + exit 1 +fi +if ! safe_prefix "${RELEASE_CANDIDATE_PREFIX}"; then + release_catalog_error "RELEASE_CANDIDATE_PREFIX must be a safe, relative S3 prefix" + exit 1 +fi + +companion_root="$(realpath "${RELEASE_ARTIFACT_DIRECTORY}")" +entries_path="${companion_root}/release-catalog-entries.json" +if [[ ! -f "${entries_path}" ]]; then + release_catalog_error "release catalog entries are missing: ${entries_path}" + exit 1 +fi + +# Catalog paths are relative to the companion root. Reject traversal before an +# S3 key is constructed, even though materialize.sh already validates them. +safe_relative_path() { + local path="$1" + [[ -n "${path}" && "${path}" != /* && "${path}" != ../* && "${path}" != */../* && "${path}" != *'/..' ]] +} + +base_key="${RELEASE_CANDIDATE_TRAIN_SHA256}/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID}/${RELEASE_SOURCE_ARTIFACT_NAME}" +if [[ -n "${RELEASE_CANDIDATE_PREFIX}" ]]; then + base_key="${RELEASE_CANDIDATE_PREFIX}/${base_key}" +fi +# Upload exactly the files the companion declares: the entries document, each +# artifact, and each artifact's evidence files. +mapfile -t companion_paths < <(jq -r '["release-catalog-entries.json"] + ([.entries[] | .path, .sbom, .provenance] | unique) | .[]' "${entries_path}") +head_metadata="$(mktemp)" +trap 'rm -f "${head_metadata}"' EXIT + +upload_one() { + local relative_path="$1" + local local_path="${companion_root}/${relative_path}" + local object_key="${base_key}/${relative_path}" + local checksum + checksum="$(openssl dgst -sha256 -binary "${local_path}" | base64)" + + if aws s3api head-object --bucket "${RELEASE_CANDIDATE_BUCKET}" --key "${object_key}" --checksum-mode ENABLED >"${head_metadata}" 2>/dev/null; then + if [[ "$(jq -r '.ChecksumSHA256 // empty' "${head_metadata}")" != "${checksum}" ]]; then + release_catalog_error "Filename already exists remotely and has different content from current file: ${object_key}" + exit 1 + fi + return + fi + + if ! aws s3api put-object --bucket "${RELEASE_CANDIDATE_BUCKET}" --key "${object_key}" --body "${local_path}" --checksum-algorithm SHA256 --checksum-sha256 "${checksum}" --if-none-match '*' --tagging 'release-candidate-status=candidate' >/dev/null 2>&1; then + aws s3api head-object --bucket "${RELEASE_CANDIDATE_BUCKET}" --key "${object_key}" --checksum-mode ENABLED >"${head_metadata}" + if [[ "$(jq -r '.ChecksumSHA256 // empty' "${head_metadata}")" != "${checksum}" ]]; then + release_catalog_error "candidate object could not be written immutably: ${object_key}" + exit 1 + fi + fi +} + +for relative_path in "${companion_paths[@]}"; do + if ! safe_relative_path "${relative_path}" || [[ ! -f "${companion_root}/${relative_path}" ]]; then + release_catalog_error "catalog declares a missing or unsafe candidate file: ${relative_path}" + exit 1 + fi + upload_one "${relative_path}" +done + +printf 'Release catalog companion: s3://%s/%s\n' "${RELEASE_CANDIDATE_BUCKET}" "${base_key}" >>"${GITHUB_STEP_SUMMARY}" diff --git a/release-catalog/validate-config.sh b/release-catalog/validate-config.sh new file mode 100755 index 00000000..56545dca --- /dev/null +++ b/release-catalog/validate-config.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +set -euo pipefail + +export RELEASE_CATALOG_SCRIPT_NAME="validate-config.sh" +export RELEASE_CATALOG_ERROR_TITLE="Invalid release catalog configuration" +# shellcheck source=release-catalog/error.sh +source "$(dirname "${BASH_SOURCE[0]}")/error.sh" + +# Validate the public JSON input before materialize.sh reads any build output. +# Structural errors are emitted as GitHub Actions annotations so callers see +# every invalid field in one run. Validated values are written to GITHUB_OUTPUT +# for the dispatch action's later materialization step. + +config="${RELEASE_CATALOG_CONFIG:-}" +if [[ -z "${config}" ]]; then + release_catalog_error "release catalog configuration must be a non-empty JSON object" + exit 1 +fi + +if ! compact_config="$(jq -ce . <<<"${config}" 2>/dev/null)"; then + parse_error="$(jq -ce . <<<"${config}" 2>&1 || true)" + release_catalog_error "release catalog configuration must be valid JSON: ${parse_error}" + exit 1 +fi + +validation_errors="$(jq -r ' + def single_line_string: + type == "string" and length > 0 and (test("[\\r\\n]") | not); + def relative_path: + single_line_string + and (startswith("/") | not) + and (split("/") | index("..") | not); + if type != "object" then + ["release catalog configuration must be a JSON object"] + else + (keys - ["release_catalog_key", "artifact_directory", "artifacts"]) as $unknown + | [ + if ($unknown | length) > 0 then + "unknown field(s): " + ($unknown | join(", ")) + else empty end, + if (.release_catalog_key | single_line_string) then empty + else "release_catalog_key must be a non-empty, single-line string" end, + if (.artifact_directory | relative_path) then empty + else "artifact_directory must be a non-empty relative path without parent traversal" end + ] + + if has("artifacts") then + [ + if (.artifacts | type == "array" and length > 0) then empty + else "artifacts must be a non-empty array when supplied" end + ] + else [] end + + if (.artifacts | type) == "array" then + [ + .artifacts | to_entries[] + | .key as $index + | .value as $artifact + | if ($artifact | type) != "object" then + "artifacts[\($index)] must be an object" + else + ($artifact | keys - ["path", "package_identity_file"]) as $artifact_unknown + | if ($artifact_unknown | length) > 0 then + "artifacts[\($index)] has unknown field(s): " + ($artifact_unknown | join(", ")) + else empty end, + if ($artifact.path | relative_path) then empty + else "artifacts[\($index)].path must be a non-empty relative path without parent traversal" end, + ($artifact | to_entries[] + | select(.key == "package_identity_file") + | select((.value | relative_path) | not) + | "artifacts[\($index)].\(.key) must be a non-empty relative path without parent traversal") + end + ] + else [] end + end + | .[] +' <<<"${compact_config}")" + +if [[ -n "${validation_errors}" ]]; then + while IFS= read -r validation_error; do + release_catalog_error "${validation_error}" + done <<<"${validation_errors}" + exit 1 +fi + +if [[ -z "${GITHUB_OUTPUT:-}" ]]; then + release_catalog_error "GITHUB_OUTPUT is required" + exit 1 +fi + +{ + printf 'release_catalog_key=%s\n' "$(jq -r '.release_catalog_key' <<<"${compact_config}")" + printf 'artifact_directory=%s\n' "$(jq -r '.artifact_directory' <<<"${compact_config}")" + printf 'artifacts=%s\n' "$(jq -c '.artifacts // empty' <<<"${compact_config}")" +} >>"${GITHUB_OUTPUT}" diff --git a/tests/release-catalog-config/invalid/absolute-artifact-path.json b/tests/release-catalog-config/invalid/absolute-artifact-path.json new file mode 100644 index 00000000..0a9fbd0b --- /dev/null +++ b/tests/release-catalog-config/invalid/absolute-artifact-path.json @@ -0,0 +1,4 @@ +{ + "release_catalog_key": "archive:smoke", + "artifacts": [{"path": "/tmp/smoke.tar.gz", "package_identity_file": "package.json"}] +} diff --git a/tests/release-catalog-config/invalid/absolute-identity-path.json b/tests/release-catalog-config/invalid/absolute-identity-path.json new file mode 100644 index 00000000..b4f8a39d --- /dev/null +++ b/tests/release-catalog-config/invalid/absolute-identity-path.json @@ -0,0 +1,4 @@ +{ + "release_catalog_key": "archive:smoke", + "artifacts": [{"path": "smoke.tar.gz", "package_identity_file": "/tmp/package.json"}] +} diff --git a/tests/release-catalog-config/invalid/parent-artifact-path.json b/tests/release-catalog-config/invalid/parent-artifact-path.json new file mode 100644 index 00000000..1d7bc648 --- /dev/null +++ b/tests/release-catalog-config/invalid/parent-artifact-path.json @@ -0,0 +1,4 @@ +{ + "release_catalog_key": "archive:smoke", + "artifacts": [{"path": "../smoke.tar.gz", "package_identity_file": "package.json"}] +} diff --git a/tests/release-catalog-config/invalid/unknown-field.json b/tests/release-catalog-config/invalid/unknown-field.json new file mode 100644 index 00000000..316cb46d --- /dev/null +++ b/tests/release-catalog-config/invalid/unknown-field.json @@ -0,0 +1,4 @@ +{ + "release_catalog_key": "archive:smoke", + "unexpected": true +} diff --git a/tests/release-catalog-config/valid/package-identity-file.json b/tests/release-catalog-config/valid/package-identity-file.json new file mode 100644 index 00000000..c3b55a05 --- /dev/null +++ b/tests/release-catalog-config/valid/package-identity-file.json @@ -0,0 +1,10 @@ +{ + "release_catalog_key": "maven:cuvs-java", + "artifact_directory": "java/cuvs-java/target", + "artifacts": [ + { + "path": "cuvs-java-*-x86_64-cuda*.jar", + "package_identity_file": "cuvs-java.release-package-identity.json" + } + ] +} diff --git a/tests/release_build_output_descriptors_test.sh b/tests/release_build_output_descriptors_test.sh deleted file mode 100755 index c6b6c46c..00000000 --- a/tests/release_build_output_descriptors_test.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - -set -euo pipefail - -repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -temporary_directory="$(mktemp -d)" -trap 'rm -rf "${temporary_directory}"' EXIT - -wheel_output_directory="${temporary_directory}/wheels" -wheel_staging_directory="${temporary_directory}/wheel-staging" -mkdir -p "${wheel_output_directory}" "${wheel_staging_directory}/libkvikio_cu12-26.8.0.dist-info" -printf '%s\n' \ - 'Metadata-Version: 2.1' \ - 'Name: libkvikio-cu12' \ - 'Version: 26.8.0a32' \ - >"${wheel_staging_directory}/libkvikio_cu12-26.8.0.dist-info/METADATA" -( - cd "${wheel_staging_directory}" - zip -qr "${wheel_output_directory}/libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl" . -) - -wheel_descriptors="$("${repository_root}/release-build-output/describe-wheels.sh" "${wheel_output_directory}")" -jq -e ' - . == [{ - path: "libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl", - package: {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"} - }] -' <<<"${wheel_descriptors}" >/dev/null - -conda_output_directory="${temporary_directory}/conda" -tar_bz2_staging_directory="${temporary_directory}/tar-bz2-staging" -conda_staging_directory="${temporary_directory}/conda-staging" -mkdir -p \ - "${conda_output_directory}/linux-64" \ - "${conda_output_directory}/noarch" \ - "${tar_bz2_staging_directory}/info" \ - "${conda_staging_directory}/info" - -jq -n \ - '{name: "rapids-dask-dependency", version: "26.08.0", build: "py_0", subdir: "noarch"}' \ - >"${tar_bz2_staging_directory}/info/index.json" -tar -cjf \ - "${conda_output_directory}/noarch/rapids-dask-dependency-26.08.0-py_0.tar.bz2" \ - -C "${tar_bz2_staging_directory}" \ - info/index.json - -jq -n \ - '{name: "librmm", version: "26.08.00a32", build: "cuda12_260714_2f567060", subdir: "linux-64"}' \ - >"${conda_staging_directory}/info/index.json" -tar -cf "${temporary_directory}/info-librmm.tar" -C "${conda_staging_directory}" info/index.json -zstd -q -f "${temporary_directory}/info-librmm.tar" -o "${temporary_directory}/info-librmm.tar.zst" -( - cd "${temporary_directory}" - zip -q \ - "${conda_output_directory}/linux-64/librmm-26.08.00a32-cuda12_260714_2f567060.conda" \ - info-librmm.tar.zst -) - -conda_descriptors="$("${repository_root}/release-build-output/describe-conda.sh" "${conda_output_directory}")" -jq -e ' - length == 2 - and any(.[]; - .path == "linux-64/librmm-26.08.00a32-cuda12_260714_2f567060.conda" - and .package == { - ecosystem: "conda", - name: "librmm", - version: "26.08.00a32", - build: "cuda12_260714_2f567060", - platform: "linux-64" - } - ) - and any(.[]; - .path == "noarch/rapids-dask-dependency-26.08.0-py_0.tar.bz2" - and .package == { - ecosystem: "conda", - name: "rapids-dask-dependency", - version: "26.08.0", - build: "py_0", - platform: "noarch" - } - ) -' <<<"${conda_descriptors}" >/dev/null diff --git a/tests/release_build_output_prepare_test.sh b/tests/release_build_output_prepare_test.sh deleted file mode 100755 index 3598e1e5..00000000 --- a/tests/release_build_output_prepare_test.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - -set -euo pipefail - -repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -temporary_directory="$(mktemp -d)" -trap 'rm -rf "${temporary_directory}"' EXIT - -mkdir -p "${temporary_directory}/wheel/libkvikio_cu12-26.8.0.dist-info" -printf '%s\n' \ - 'Metadata-Version: 2.1' \ - 'Name: libkvikio-cu12' \ - 'Version: 26.8.0' \ - >"${temporary_directory}/wheel/libkvikio_cu12-26.8.0.dist-info/METADATA" -( - cd "${temporary_directory}/wheel" - zip -qr "${temporary_directory}/libkvikio_cu12-26.8.0-py3-none-manylinux_2_28_x86_64.whl" . -) - -GITHUB_OUTPUT="${temporary_directory}/wheel-output" -RELEASE_ARTIFACTS='' -RELEASE_ARTIFACT_TYPE=wheel -RELEASE_OUTPUT_DIRECTORY="${temporary_directory}" -RELEASE_PACKAGE='' -RELEASE_PACKAGE_FILE='' -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE - -"${repository_root}/release-build-output/prepare.sh" - -grep -Fx 'package={"ecosystem":"wheel","name":"bundle"}' "${GITHUB_OUTPUT}" -artifacts="$(sed -n 's/^artifacts=//p' "${GITHUB_OUTPUT}")" -jq -e ' - . == [{ - path: "libkvikio_cu12-26.8.0-py3-none-manylinux_2_28_x86_64.whl", - package: {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0"} - }] -' <<<"${artifacts}" >/dev/null - -GITHUB_OUTPUT="${temporary_directory}/custom-output" -RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz", sbom: "bundle.spdx.json"}]')" -RELEASE_ARTIFACT_TYPE=custom -RELEASE_PACKAGE='' -RELEASE_PACKAGE_FILE=release-package.json -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_PACKAGE RELEASE_PACKAGE_FILE - -"${repository_root}/release-build-output/prepare.sh" - -grep -Fx 'artifacts=[{"path":"bundle.tar.gz","sbom":"bundle.spdx.json"}]' "${GITHUB_OUTPUT}" -grep -Fx 'package=' "${GITHUB_OUTPUT}" - -GITHUB_OUTPUT="${temporary_directory}/invalid-output" -RELEASE_ARTIFACT_TYPE=archive -export GITHUB_OUTPUT RELEASE_ARTIFACT_TYPE -if "${repository_root}/release-build-output/prepare.sh" 2>"${temporary_directory}/invalid-error"; then - echo "prepare.sh unexpectedly accepted an invalid artifact type" >&2 - exit 1 -fi -grep -Fx 'artifact-type must be one of: conda, custom, wheel' "${temporary_directory}/invalid-error" diff --git a/tests/release_build_output_test.sh b/tests/release_build_output_test.sh deleted file mode 100755 index 2a5dc6e1..00000000 --- a/tests/release_build_output_test.sh +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - -set -euo pipefail - -repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -temporary_directory="$(mktemp -d)" -trap 'rm -rf "${temporary_directory}"' EXIT - -bundle_directory="${temporary_directory}/bundle" -mkdir -p "${bundle_directory}" -printf '%s\n' jar >"${bundle_directory}/cuvs-java-26.08.0.jar" -printf '%s\n' sbom >"${bundle_directory}/cuvs-java-26.08.0.spdx.json" -printf '%s\n' provenance >"${bundle_directory}/cuvs-java-26.08.0.provenance.jsonl" -printf '%s\n' signature >"${bundle_directory}/cuvs-java-26.08.0.jar.asc" -jq -n \ - '{ecosystem: "maven", name: "ai.rapids:cuvs-java", version: "26.08.0"}' \ - >"${bundle_directory}/cuvs-java-package.json" - -GITHUB_OUTPUT="${temporary_directory}/github-output" -GITHUB_REPOSITORY="rapidsai/cuvs" -GITHUB_RUN_ATTEMPT="1" -GITHUB_RUN_ID="1234" -GITHUB_SHA="0123456789012345678901234567890123456789" -GITHUB_WORKFLOW_REF="rapidsai/cuvs/.github/workflows/build.yaml@refs/heads/release/26.08" -export GITHUB_OUTPUT -RELEASE_ARTIFACTS="$(jq -cn '[{path: "cuvs-java-*.jar", sbom: "cuvs-java-*.spdx.json", provenance: "cuvs-java-*.provenance.jsonl", signature: "cuvs-java-*.jar.asc"}]')" -RELEASE_MANIFEST_NAME="release-build-output.json" -RELEASE_METADATA_NAME="release-build-metadata.json" -RELEASE_OUTPUT_DIRECTORY="${bundle_directory}" -RELEASE_PACKAGE='' -RELEASE_PACKAGE_FILE="cuvs-java-package.json" -RELEASE_SOURCE_ARTIFACT_NAME="cuvs-java-cuda12.9.1" -RELEASE_SOURCE_SHA="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" -RELEASE_UNIT="maven:cuvs-java" -export GITHUB_REPOSITORY GITHUB_RUN_ATTEMPT GITHUB_RUN_ID GITHUB_SHA GITHUB_WORKFLOW_REF -export RELEASE_ARTIFACTS RELEASE_MANIFEST_NAME RELEASE_METADATA_NAME RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE -export RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_UNIT - -"${repository_root}/release-build-output/materialize.sh" - -canonical_bundle_directory="$(realpath "${bundle_directory}")" -manifest_path="${canonical_bundle_directory}/release-build-output.json" -metadata_path="${canonical_bundle_directory}/release-build-metadata.json" -jq -e ' - .schema_version == 1 - and .producer == "release-platform" - and (.artifacts | length == 1) - and .artifacts[0].unit_id == "maven:cuvs-java" - and .artifacts[0].path == "cuvs-java-26.08.0.jar" - and .artifacts[0].package.name == "ai.rapids:cuvs-java" -' "${manifest_path}" >/dev/null -jq -e ' - .schema_version == 1 - and .producer == "shared-workflows" - and .release_unit == "maven:cuvs-java" - and .source_artifact == "cuvs-java-cuda12.9.1" - and .build_output_manifest == "release-build-output.json" - and .build_environment.repository == "rapidsai/cuvs" - and .build_environment.sha == "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - and .metadata.artifacts == [{path: "cuvs-java-26.08.0.jar", sbom_kind: "producer-dependency"}] -' "${metadata_path}" >/dev/null -grep -Fx "manifest-path=${manifest_path}" "${GITHUB_OUTPUT}" -grep -Fx "metadata-path=${metadata_path}" "${GITHUB_OUTPUT}" - -supplied_sbom_path="$(jq -r '.artifacts[0].sbom' "${manifest_path}")" -supplied_provenance_path="$(jq -r '.artifacts[0].provenance' "${manifest_path}")" -supplied_signature_path="$(jq -r '.artifacts[0].signature' "${manifest_path}")" -[[ "${supplied_sbom_path}" == release-evidence/*/sbom-* ]] -[[ "${supplied_provenance_path}" == release-evidence/*/provenance-* ]] -[[ "${supplied_signature_path}" == release-evidence/*/signature-* ]] -grep -Fx sbom "${canonical_bundle_directory}/${supplied_sbom_path}" -grep -Fx provenance "${canonical_bundle_directory}/${supplied_provenance_path}" -grep -Fx signature "${canonical_bundle_directory}/${supplied_signature_path}" - -isolated_companion_directory="${temporary_directory}/isolated-companion" -mkdir -p "${isolated_companion_directory}" -cp "${manifest_path}" "${metadata_path}" "${isolated_companion_directory}/" -cp -R "${canonical_bundle_directory}/release-evidence" "${isolated_companion_directory}/" -while IFS= read -r evidence_path; do - test -f "${isolated_companion_directory}/${evidence_path}" -done < <(jq -r '.artifacts[] | .sbom, .provenance, (.signature // empty)' "${isolated_companion_directory}/release-build-output.json") - -generated_directory="${temporary_directory}/generated-bundle" -mkdir -p "${generated_directory}/linux-64" -printf '%s\n' conda >"${generated_directory}/linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.conda" - -RELEASE_ARTIFACTS="$(jq -cn '[{path: "linux-64/kvikio-*.conda"}]')" -RELEASE_OUTPUT_DIRECTORY="${generated_directory}" -RELEASE_PACKAGE="$(jq -cn '{ecosystem: "conda", name: "kvikio"}')" -RELEASE_PACKAGE_FILE='' -RELEASE_SOURCE_ARTIFACT_NAME="kvikio_conda_python_kvikio_x86_64_abi3_cu12" -RELEASE_SOURCE_SHA="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" -RELEASE_UNIT="conda:kvikio" -export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_UNIT - -"${repository_root}/release-build-output/materialize.sh" - -generated_manifest_path="${generated_directory}/release-build-output.json" -generated_metadata_path="${generated_directory}/release-build-metadata.json" -generated_sbom_path="$(jq -r '.artifacts[0].sbom' "${generated_manifest_path}")" -generated_provenance_path="$(jq -r '.artifacts[0].provenance' "${generated_manifest_path}")" -jq -e ' - .artifacts[0].unit_id == "conda:kvikio" - and .artifacts[0].path == "linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.conda" - and .artifacts[0].package == {ecosystem: "conda", name: "kvikio", version: "26.08.00a32"} -' "${generated_manifest_path}" >/dev/null -jq -e ' - .spdxVersion == "SPDX-2.3" - and .packages[0].name == "kvikio" - and .packages[0].versionInfo == "26.08.00a32" -' "${generated_directory}/${generated_sbom_path}" >/dev/null -jq -e ' - .predicateType == "https://slsa.dev/provenance/v1" - and .predicate.buildDefinition.externalParameters.release_unit == "conda:kvikio" - and .predicate.buildDefinition.resolvedDependencies[0].digest.gitCommit == "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" -' "${generated_directory}/${generated_provenance_path}" >/dev/null -jq -e ' - .release_unit == "conda:kvikio" - and .source_artifact == "kvikio_conda_python_kvikio_x86_64_abi3_cu12" - and .build_environment.sha == "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" - and .metadata.artifacts == [{path: "linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.conda", sbom_kind: "generated-identity"}] -' "${generated_metadata_path}" >/dev/null - -wheel_directory="${temporary_directory}/wheel-bundle" -mkdir -p "${wheel_directory}" -printf '%s\n' wheel >"${wheel_directory}/libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl" - -RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl"}]')" -RELEASE_OUTPUT_DIRECTORY="${wheel_directory}" -RELEASE_PACKAGE="$(jq -cn '{ecosystem: "wheel", name: "libkvikio-cu12"}')" -RELEASE_SOURCE_ARTIFACT_NAME="kvikio_wheel_cpp_libkvikio_x86_64_cu12" -RELEASE_SOURCE_SHA="cccccccccccccccccccccccccccccccccccccccc" -RELEASE_UNIT="wheel:kvikio" -export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_UNIT - -"${repository_root}/release-build-output/materialize.sh" - -jq -e ' - .artifacts[0].unit_id == "wheel:kvikio" - and .artifacts[0].path == "libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl" - and .artifacts[0].package == {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"} -' "${wheel_directory}/release-build-output.json" >/dev/null -jq -e ' - .metadata.artifacts == [{path: "libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl", sbom_kind: "generated-identity"}] -' "${wheel_directory}/release-build-metadata.json" >/dev/null - -missing_version_directory="${temporary_directory}/missing-version-bundle" -mkdir -p "${missing_version_directory}" -printf '%s\n' archive >"${missing_version_directory}/bundle.tar.gz" - -RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz"}]')" -RELEASE_OUTPUT_DIRECTORY="${missing_version_directory}" -RELEASE_PACKAGE="$(jq -cn '{ecosystem: "archive", name: "bundle"}')" -RELEASE_PACKAGE_FILE='' -RELEASE_SOURCE_ARTIFACT_NAME="bundle-archive" -RELEASE_SOURCE_SHA="dddddddddddddddddddddddddddddddddddddddd" -RELEASE_UNIT="archive:bundle" -export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_UNIT - -if "${repository_root}/release-build-output/materialize.sh" 2>"${temporary_directory}/missing-version-error"; then - echo "materialize.sh unexpectedly accepted a custom artifact without a package version" >&2 - exit 1 -fi -grep -Fx 'release-package version is required for archive artifacts' "${temporary_directory}/missing-version-error" diff --git a/tests/release_catalog_config_test.sh b/tests/release_catalog_config_test.sh new file mode 100755 index 00000000..2226feb7 --- /dev/null +++ b/tests/release_catalog_config_test.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +set -euo pipefail + +repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +validator="${repository_root}/release-catalog/validate-config.sh" +temporary_directory="$(mktemp -d)" +trap 'rm -rf "${temporary_directory}"' EXIT + +assert_invalid() { + local name="$1" + local config="$2" + local expected="$3" + local error_file="${temporary_directory}/${name}.error" + + if RELEASE_CATALOG_CONFIG="${config}" GITHUB_OUTPUT="${temporary_directory}/${name}.output" \ + "${validator}" 2>"${error_file}"; then + echo "validate-config.sh unexpectedly accepted ${name}" >&2 + exit 1 + fi + if ! grep -F -- "${expected}" "${error_file}" >/dev/null; then + echo "validate-config.sh did not explain ${name}; output was:" >&2 + sed 's/^/ /' "${error_file}" >&2 + exit 1 + fi +} + +valid_output="${temporary_directory}/valid.output" +RELEASE_CATALOG_CONFIG="$(<"${repository_root}/tests/release-catalog-config/valid/package-identity-file.json")" \ + GITHUB_OUTPUT="${valid_output}" "${validator}" + +grep -Fx 'release_catalog_key=maven:cuvs-java' "${valid_output}" +grep -Fx 'artifact_directory=java/cuvs-java/target' "${valid_output}" +grep -Fx 'artifacts=[{"path":"cuvs-java-*-x86_64-cuda*.jar","package_identity_file":"cuvs-java.release-package-identity.json"}]' "${valid_output}" + +assert_invalid \ + malformed-json \ + '{"release_catalog_key":' \ + 'release catalog configuration must be valid JSON' + +assert_invalid \ + not-an-object \ + '[]' \ + 'release catalog configuration must be a JSON object' + +assert_invalid \ + missing-fields \ + '{}' \ + 'release_catalog_key must be a non-empty, single-line string' + +assert_invalid \ + top-level-package-identity-file \ + '{"release_catalog_key":"archive:smoke","package_identity_file":"package.json"}' \ + 'unknown field(s): package_identity_file' + +assert_invalid \ + unknown-field \ + '{"release_catalog_key":"archive:smoke","artifacts":[{"path":"smoke.tar.gz","package_identity_file":"package.json"}],"component_id":"archive:smoke"}' \ + 'unknown field(s): component_id' + +assert_invalid \ + artifact-type \ + '{"artifact_type":"custom","release_catalog_key":"archive:smoke","artifacts":[{"path":"smoke.tar.gz","package_identity_file":"package.json"}]}' \ + 'unknown field(s): artifact_type' + +assert_invalid \ + inline-package \ + '{"release_catalog_key":"archive:smoke","package":{"ecosystem":"archive","name":"smoke","version":"1.0"},"artifacts":[{"path":"smoke.tar.gz","package_identity_file":"package.json"}]}' \ + 'unknown field(s): package' + +assert_invalid \ + per-artifact-package \ + '{"release_catalog_key":"archive:smoke","artifacts":[{"path":"smoke.tar.gz","package":{"ecosystem":"archive","name":"smoke","version":"1.0"}}]}' \ + 'artifacts[0] has unknown field(s): package' + +assert_invalid \ + malformed-artifact \ + '{"release_catalog_key":"archive:smoke","artifacts":[{"file":"smoke.tar.gz","package_identity_file":false,"sbom":false}]}' \ + 'artifacts[0] has unknown field(s): file, sbom' +grep -F 'artifacts[0].path must be a non-empty relative path without parent traversal' "${temporary_directory}/malformed-artifact.error" >/dev/null +grep -F 'artifacts[0].package_identity_file must be a non-empty relative path without parent traversal' "${temporary_directory}/malformed-artifact.error" >/dev/null + +assert_invalid \ + missing-artifact-directory \ + '{"release_catalog_key":"conda:smoke"}' \ + 'artifact_directory must be a non-empty relative path without parent traversal' + +assert_invalid \ + absolute-artifact-directory \ + '{"release_catalog_key":"conda:smoke","artifact_directory":"/tmp/artifacts"}' \ + 'artifact_directory must be a non-empty relative path without parent traversal' + +assert_invalid \ + parent-artifact-directory \ + '{"release_catalog_key":"conda:smoke","artifact_directory":"../artifacts"}' \ + 'artifact_directory must be a non-empty relative path without parent traversal' + +standard_output="${temporary_directory}/standard.output" +RELEASE_CATALOG_CONFIG='{ + "release_catalog_key": "wheel:kvikio", + "artifact_directory": "dist" +}' GITHUB_OUTPUT="${standard_output}" "${validator}" + +grep -Fx 'release_catalog_key=wheel:kvikio' "${standard_output}" +grep -Fx 'artifact_directory=dist' "${standard_output}" +grep -Fx 'artifacts=' "${standard_output}" + +for invalid_fixture in "${repository_root}"/tests/release-catalog-config/invalid/*.json; do + fixture_name="$(basename "${invalid_fixture}" .json)" + case "${fixture_name}" in + absolute-artifact-path | parent-artifact-path) + expected="artifacts[0].path must be a non-empty relative path without parent traversal" + ;; + absolute-identity-path) + expected="artifacts[0].package_identity_file must be a non-empty relative path without parent traversal" + ;; + unknown-field) + expected="unknown field(s): unexpected" + ;; + *) + echo "missing expected runtime error for ${fixture_name}" >&2 + exit 1 + ;; + esac + assert_invalid "fixture-${fixture_name}" "$(<"${invalid_fixture}")" "${expected}" +done diff --git a/tests/release_catalog_discovery_test.sh b/tests/release_catalog_discovery_test.sh new file mode 100755 index 00000000..72728211 --- /dev/null +++ b/tests/release_catalog_discovery_test.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +set -euo pipefail + +# Error output assertions below exercise the local rendering path. The +# GitHub-specific rendering path is covered by release_catalog_error_test.sh. +export GITHUB_ACTIONS="" + +repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +temporary_directory="$(mktemp -d)" +trap 'rm -rf "${temporary_directory}"' EXIT + +GITHUB_REPOSITORY="rapidsai/shared-actions" +GITHUB_RUN_ATTEMPT="1" +GITHUB_RUN_ID="1234" +GITHUB_WORKFLOW_REF="rapidsai/shared-actions/.github/workflows/pr.yml@refs/pull/136/merge" +RELEASE_CATALOG_KEY="test:discovery" +RELEASE_SOURCE_ARTIFACT_NAME="discovery-test" +RELEASE_SOURCE_SHA="0123456789012345678901234567890123456789" +export GITHUB_REPOSITORY GITHUB_RUN_ATTEMPT GITHUB_RUN_ID GITHUB_WORKFLOW_REF +export RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY RELEASE_SOURCE_ARTIFACT_NAME + +mkdir -p "${temporary_directory}/wheel/libkvikio_cu12-26.8.0.dist-info" +printf '%s\n' \ + 'Metadata-Version: 2.1' \ + 'Name: libkvikio-cu12' \ + 'Version: 26.8.0' \ + >"${temporary_directory}/wheel/libkvikio_cu12-26.8.0.dist-info/METADATA" +( + cd "${temporary_directory}/wheel" + zip -qr "${temporary_directory}/libkvikio_cu12-26.8.0-py3-none-manylinux_2_28_x86_64.whl" . +) + +RELEASE_ARTIFACTS='' +RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY + +"${repository_root}/release-catalog/materialize.sh" + +jq -e ' + [.entries[] | {path, package}] == [{ + path: "libkvikio_cu12-26.8.0-py3-none-manylinux_2_28_x86_64.whl", + package: {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0"} + }] +' "${temporary_directory}/release-catalog-entries.json" >/dev/null + +conda_output_directory="${temporary_directory}/conda" +conda_staging_directory="${temporary_directory}/conda-staging" +mkdir -p "${conda_output_directory}/noarch" "${conda_staging_directory}/info" +jq -n \ + '{name: "rapids-dask-dependency", version: "26.08.0", build: "py_0", subdir: "noarch"}' \ + >"${conda_staging_directory}/info/index.json" +tar -cjf \ + "${conda_output_directory}/noarch/rapids-dask-dependency-26.08.0-py_0.tar.bz2" \ + -C "${conda_staging_directory}" \ + info/index.json + +RELEASE_ARTIFACT_DIRECTORY="${conda_output_directory}" +export RELEASE_ARTIFACT_DIRECTORY + +"${repository_root}/release-catalog/materialize.sh" + +jq -e ' + [.entries[] | {path, package}] == [{ + path: "noarch/rapids-dask-dependency-26.08.0-py_0.tar.bz2", + package: { + ecosystem: "conda", + name: "rapids-dask-dependency", + version: "26.08.0", + build: "py_0", + platform: "noarch" + } + }] +' "${conda_output_directory}/release-catalog-entries.json" >/dev/null + +RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" +printf '%s\n' bundle >"${temporary_directory}/bundle.tar.gz" +jq -n '{ecosystem: "archive", name: "bundle", version: "1.0"}' \ + >"${temporary_directory}/release-package-identity.json" +RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz", package_identity_file: "release-package-identity.json"}]')" +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY + +"${repository_root}/release-catalog/materialize.sh" + +jq -e ' + .entries[0].path == "bundle.tar.gz" + and .entries[0].package == {ecosystem: "archive", name: "bundle", version: "1.0"} + and .entries[0].sbom_kind == "generated-identity" +' "${temporary_directory}/release-catalog-entries.json" >/dev/null + +RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" +RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl"}]')" +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY +"${repository_root}/release-catalog/materialize.sh" +jq -e ' + .entries[0].path == "libkvikio_cu12-26.8.0-py3-none-manylinux_2_28_x86_64.whl" + and .entries[0].package == {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0"} + and .entries[0].sbom_kind == "generated-identity" +' "${temporary_directory}/release-catalog-entries.json" >/dev/null + +mixed_directory="${temporary_directory}/mixed" +mkdir -p "${mixed_directory}" +cp "${temporary_directory}/libkvikio_cu12-26.8.0-py3-none-manylinux_2_28_x86_64.whl" "${mixed_directory}/" +cp "${conda_output_directory}/noarch/rapids-dask-dependency-26.08.0-py_0.tar.bz2" "${mixed_directory}/" +RELEASE_ARTIFACTS='' +RELEASE_ARTIFACT_DIRECTORY="${mixed_directory}" +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY +"${repository_root}/release-catalog/materialize.sh" +jq -e ' + (.entries | length == 2) + and ([.entries[].package.ecosystem] | sort) == ["conda", "wheel"] +' "${mixed_directory}/release-catalog-entries.json" >/dev/null + +empty_directory="${temporary_directory}/empty" +mkdir -p "${empty_directory}" +RELEASE_ARTIFACT_DIRECTORY="${empty_directory}" +export RELEASE_ARTIFACT_DIRECTORY +if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/empty-error"; then + echo "materialize.sh unexpectedly accepted an artifact directory without detectable artifacts" >&2 + exit 1 +fi +grep -Fx '[materialize.sh] Error: artifact-directory contains no detectable Conda, wheel, or Maven JAR artifacts; explicitly selected artifacts require package_identity_file when their identity cannot be parsed' "${temporary_directory}/empty-error" + +RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" +RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz"}]')" +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY +if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/unsupported-error"; then + echo "materialize.sh unexpectedly accepted an unsupported artifact without identity" >&2 + exit 1 +fi +grep -Fx '[materialize.sh] Error: artifact identity cannot be extracted; package_identity_file is required: bundle.tar.gz' "${temporary_directory}/unsupported-error" + +RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl", package_identity_file: "release-package-identity.json"}]')" +export RELEASE_ARTIFACTS +if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/conflicting-identity-error"; then + echo "materialize.sh unexpectedly accepted an identity file for a parseable artifact" >&2 + exit 1 +fi +grep -Fx '[materialize.sh] Error: package_identity_file is not allowed when wheel identity can be extracted: libkvikio_cu12-*.whl' "${temporary_directory}/conflicting-identity-error" diff --git a/tests/release_catalog_error_test.sh b/tests/release_catalog_error_test.sh new file mode 100755 index 00000000..1539f5b6 --- /dev/null +++ b/tests/release_catalog_error_test.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +set -euo pipefail + +repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +temporary_directory="$(mktemp -d)" +trap 'rm -rf "${temporary_directory}"' EXIT + +# shellcheck source=/dev/null +source "${repository_root}/release-catalog/error.sh" + +export RELEASE_CATALOG_SCRIPT_NAME="materialize.sh" +export GITHUB_ACTIONS="" +release_catalog_error "invalid artifact" 2>"${temporary_directory}/local-error" +grep -Fxq '[materialize.sh] Error: invalid artifact' "${temporary_directory}/local-error" + +export GITHUB_ACTIONS="true" +export RELEASE_CATALOG_ERROR_TITLE="Release catalog upload failed" +release_catalog_error $'invalid 100%\r\nartifact' 2>"${temporary_directory}/github-error" +grep -Fxq '::error title=Release catalog upload failed::invalid 100%25%0D%0Aartifact' \ + "${temporary_directory}/github-error" diff --git a/tests/release_catalog_package_parsing_test.sh b/tests/release_catalog_package_parsing_test.sh new file mode 100755 index 00000000..726f55f1 --- /dev/null +++ b/tests/release_catalog_package_parsing_test.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +set -euo pipefail + +repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +temporary_directory="$(mktemp -d)" +trap 'rm -rf "${temporary_directory}"' EXIT + +GITHUB_REPOSITORY="rapidsai/shared-actions" +GITHUB_RUN_ATTEMPT="1" +GITHUB_RUN_ID="1234" +GITHUB_WORKFLOW_REF="rapidsai/shared-actions/.github/workflows/pr.yml@refs/pull/136/merge" +RELEASE_ARTIFACTS='' +RELEASE_CATALOG_KEY="test:package-parsing" +RELEASE_SOURCE_ARTIFACT_NAME="package-parsing-test" +RELEASE_SOURCE_SHA="0123456789012345678901234567890123456789" +export GITHUB_REPOSITORY GITHUB_RUN_ATTEMPT GITHUB_RUN_ID GITHUB_WORKFLOW_REF +export RELEASE_SOURCE_SHA RELEASE_ARTIFACTS RELEASE_CATALOG_KEY RELEASE_SOURCE_ARTIFACT_NAME + +wheel_output_directory="${temporary_directory}/wheels" +wheel_staging_directory="${temporary_directory}/wheel-staging" +mkdir -p "${wheel_output_directory}" "${wheel_staging_directory}/libkvikio_cu12-26.8.0.dist-info" +printf '%s\n' \ + 'Metadata-Version: 2.1' \ + 'Name: libkvikio-cu12' \ + 'Version: 26.8.0a32' \ + >"${wheel_staging_directory}/libkvikio_cu12-26.8.0.dist-info/METADATA" +( + cd "${wheel_staging_directory}" + zip -qr "${wheel_output_directory}/libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl" . +) + +RELEASE_ARTIFACT_DIRECTORY="${wheel_output_directory}" +export RELEASE_ARTIFACT_DIRECTORY +"${repository_root}/release-catalog/materialize.sh" +jq -e ' + [.entries[] | {path, package}] == [{ + path: "libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl", + package: {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"} + }] +' "${wheel_output_directory}/release-catalog-entries.json" >/dev/null + +jar_output_directory="${temporary_directory}/jars" +jar_staging_directory="${temporary_directory}/jar-staging" +mkdir -p "${jar_output_directory}" "${jar_staging_directory}/META-INF/maven/ai.rapids/cuvs-java" +printf '%s\n' \ + 'artifactId=cuvs-java' \ + 'groupId=ai.rapids' \ + 'version=26.08.0' \ + >"${jar_staging_directory}/META-INF/maven/ai.rapids/cuvs-java/pom.properties" +( + cd "${jar_staging_directory}" + zip -qr "${jar_output_directory}/cuvs-java-26.08.0.jar" . +) + +RELEASE_ARTIFACT_DIRECTORY="${jar_output_directory}" +export RELEASE_ARTIFACT_DIRECTORY +"${repository_root}/release-catalog/materialize.sh" +jq -e ' + [.entries[] | {path, package}] == [{ + path: "cuvs-java-26.08.0.jar", + package: {ecosystem: "maven", name: "ai.rapids:cuvs-java", version: "26.08.0"} + }] +' "${jar_output_directory}/release-catalog-entries.json" >/dev/null + +ambiguous_jar_directory="${temporary_directory}/ambiguous-jar" +ambiguous_jar_staging_directory="${temporary_directory}/ambiguous-jar-staging" +mkdir -p \ + "${ambiguous_jar_directory}" \ + "${ambiguous_jar_staging_directory}/META-INF/maven/example/first" \ + "${ambiguous_jar_staging_directory}/META-INF/maven/example/second" +printf '%s\n' 'groupId=example' 'artifactId=first' 'version=1.0' \ + >"${ambiguous_jar_staging_directory}/META-INF/maven/example/first/pom.properties" +printf '%s\n' 'groupId=example' 'artifactId=second' 'version=1.0' \ + >"${ambiguous_jar_staging_directory}/META-INF/maven/example/second/pom.properties" +( + cd "${ambiguous_jar_staging_directory}" + zip -qr "${ambiguous_jar_directory}/shaded.jar" . +) +RELEASE_ARTIFACT_DIRECTORY="${ambiguous_jar_directory}" +export RELEASE_ARTIFACT_DIRECTORY +if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/ambiguous-jar-error"; then + echo "materialize.sh unexpectedly accepted a JAR with ambiguous Maven identity" >&2 + exit 1 +fi +grep -F 'Maven JAR must contain exactly one META-INF/maven///pom.properties file: shaded.jar' \ + "${temporary_directory}/ambiguous-jar-error" >/dev/null + +conda_output_directory="${temporary_directory}/conda" +tar_bz2_staging_directory="${temporary_directory}/tar-bz2-staging" +conda_staging_directory="${temporary_directory}/conda-staging" +mkdir -p \ + "${conda_output_directory}/linux-64" \ + "${conda_output_directory}/noarch" \ + "${tar_bz2_staging_directory}/info" \ + "${conda_staging_directory}/info" + +jq -n \ + '{name: "rapids-dask-dependency", version: "26.08.0", build: "py_0", subdir: "noarch"}' \ + >"${tar_bz2_staging_directory}/info/index.json" +tar -cjf \ + "${conda_output_directory}/noarch/rapids-dask-dependency-26.08.0-py_0.tar.bz2" \ + -C "${tar_bz2_staging_directory}" \ + info/index.json + +jq -n \ + '{name: "librmm", version: "26.08.00a32", build: "cuda12_260714_2f567060", subdir: "linux-64"}' \ + >"${conda_staging_directory}/info/index.json" +tar -cf "${temporary_directory}/info-librmm.tar" -C "${conda_staging_directory}" info/index.json +zstd -q -f "${temporary_directory}/info-librmm.tar" -o "${temporary_directory}/info-librmm.tar.zst" +( + cd "${temporary_directory}" + zip -q \ + "${conda_output_directory}/linux-64/librmm-26.08.00a32-cuda12_260714_2f567060.conda" \ + info-librmm.tar.zst +) + +RELEASE_ARTIFACT_DIRECTORY="${conda_output_directory}" +export RELEASE_ARTIFACT_DIRECTORY +"${repository_root}/release-catalog/materialize.sh" +jq -e ' + (.entries | length == 2) + and any(.entries[]; + .path == "linux-64/librmm-26.08.00a32-cuda12_260714_2f567060.conda" + and .package == { + ecosystem: "conda", + name: "librmm", + version: "26.08.00a32", + build: "cuda12_260714_2f567060", + platform: "linux-64" + } + ) + and any(.entries[]; + .path == "noarch/rapids-dask-dependency-26.08.0-py_0.tar.bz2" + and .package == { + ecosystem: "conda", + name: "rapids-dask-dependency", + version: "26.08.0", + build: "py_0", + platform: "noarch" + } + ) +' "${conda_output_directory}/release-catalog-entries.json" >/dev/null + +invalid_wheel_directory="${temporary_directory}/invalid-wheel" +mkdir -p "${invalid_wheel_directory}" +printf '%s\n' invalid >"${invalid_wheel_directory}/invalid.whl" +RELEASE_ARTIFACT_DIRECTORY="${invalid_wheel_directory}" +export RELEASE_ARTIFACT_DIRECTORY +if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/invalid-wheel-error"; then + echo "materialize.sh unexpectedly accepted an invalid wheel" >&2 + exit 1 +fi +grep -F 'wheel must contain exactly one .dist-info/METADATA file: invalid.whl' \ + "${temporary_directory}/invalid-wheel-error" >/dev/null diff --git a/tests/release_catalog_test.sh b/tests/release_catalog_test.sh new file mode 100755 index 00000000..f77d2faa --- /dev/null +++ b/tests/release_catalog_test.sh @@ -0,0 +1,241 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +set -euo pipefail + +# Error output assertions below exercise the local rendering path. The +# GitHub-specific rendering path is covered by release_catalog_error_test.sh. +export GITHUB_ACTIONS="" + +repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +temporary_directory="$(mktemp -d)" +trap 'rm -rf "${temporary_directory}"' EXIT + +bundle_directory="${temporary_directory}/bundle" +mkdir -p "${bundle_directory}" +printf '%s\n' jar >"${bundle_directory}/cuvs-java-26.08.0.jar" +jq -n \ + '{ecosystem: "maven", name: "ai.rapids:cuvs-java", version: "26.08.0"}' \ + >"${bundle_directory}/cuvs-java-package-identity.json" + +GITHUB_REPOSITORY="NVIDIA/cuvs" +GITHUB_RUN_ATTEMPT="1" +GITHUB_RUN_ID="1234" +GITHUB_SHA="0123456789012345678901234567890123456789" +GITHUB_WORKFLOW_REF="NVIDIA/cuvs/.github/workflows/build.yaml@refs/heads/release/26.08" +RELEASE_ARTIFACTS="$(jq -cn '[{path: "cuvs-java-*.jar", package_identity_file: "cuvs-java-package-identity.json"}]')" +RELEASE_ARTIFACT_DIRECTORY="${bundle_directory}" +RELEASE_SOURCE_ARTIFACT_NAME="cuvs-java-cuda12.9.1" +RELEASE_SOURCE_SHA="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +RELEASE_CATALOG_KEY="maven:cuvs-java" +export GITHUB_REPOSITORY GITHUB_RUN_ATTEMPT GITHUB_RUN_ID GITHUB_SHA GITHUB_WORKFLOW_REF +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY +export RELEASE_SOURCE_SHA RELEASE_SOURCE_ARTIFACT_NAME RELEASE_CATALOG_KEY + +"${repository_root}/release-catalog/materialize.sh" + +canonical_bundle_directory="$(realpath "${bundle_directory}")" +entries_path="${canonical_bundle_directory}/release-catalog-entries.json" +test ! -e "${canonical_bundle_directory}/release-catalog-metadata.json" +jq -e ' + .schema_version == 1 + and .producer == "rapidsai/shared-actions/release-catalog" + and .source.artifact == "cuvs-java-cuda12.9.1" + and .source.repository == "NVIDIA/cuvs" + and .source.sha == "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + and (.entries | length == 1) + and .entries[0].release_catalog_key == "maven:cuvs-java" + and .entries[0].path == "cuvs-java-26.08.0.jar" + and .entries[0].sha256 == "fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9" + and .entries[0].package.name == "ai.rapids:cuvs-java" + and .entries[0].sbom_kind == "generated-identity" + and (.entries[0].sbom | endswith(".sbom.cdx.json")) +' "${entries_path}" >/dev/null + +# Mirror the S3 uploader's declared-file selection without requiring AWS. +mapfile -t upload_paths < <(jq -r '["release-catalog-entries.json"] + ([.entries[] | .path, .sbom, .provenance] | unique) | .[]' "${entries_path}") +artifact_sha256="$(sha256sum "${canonical_bundle_directory}/cuvs-java-26.08.0.jar" | awk '{print $1}')" +test "${upload_paths[*]}" = "release-catalog-entries.json cuvs-java-26.08.0.jar release-evidence/cuvs-java-26.08.0.jar.${artifact_sha256}.provenance.json release-evidence/cuvs-java-26.08.0.jar.${artifact_sha256}.sbom.cdx.json" + +# Keep the checked-in example synchronized with the exact output owned by this +# action. Only the generated SBOM timestamps are normalized. +example_directory="${repository_root}/release-catalog/examples/cuvs-java" +diff -u "${example_directory}/release-catalog-entries.json" "${entries_path}" +example_cyclonedx_sbom_path="$(jq -r '.entries[0].sbom' "${entries_path}")" +example_provenance_path="$(jq -r '.entries[0].provenance' "${entries_path}")" +jq -S '.metadata.timestamp = "2026-08-21T00:00:00Z"' \ + "${canonical_bundle_directory}/${example_cyclonedx_sbom_path}" \ + >"${temporary_directory}/normalized-example.cdx.json" +diff -u \ + "${example_directory}/${example_cyclonedx_sbom_path}" \ + "${temporary_directory}/normalized-example.cdx.json" +diff -u \ + "${example_directory}/${example_provenance_path}" \ + "${canonical_bundle_directory}/${example_provenance_path}" + +isolated_companion_directory="${temporary_directory}/isolated-companion" +mkdir -p "${isolated_companion_directory}" +cp "${entries_path}" "${isolated_companion_directory}/" +cp -R "${canonical_bundle_directory}/release-evidence" "${isolated_companion_directory}/" +while IFS= read -r evidence_path; do + test -f "${isolated_companion_directory}/${evidence_path}" +done < <(jq -r '.entries[] | .sbom, .provenance' "${isolated_companion_directory}/release-catalog-entries.json") + +multiple_identity_directory="${temporary_directory}/multiple-identities" +mkdir -p "${multiple_identity_directory}" +printf '%s\n' first >"${multiple_identity_directory}/first.tar.gz" +printf '%s\n' second >"${multiple_identity_directory}/second.jar" +jq -n '{ecosystem: "archive", name: "first", version: "1.0"}' \ + >"${multiple_identity_directory}/first.identity.json" +jq -n '{ecosystem: "maven", name: "example:second", version: "2.0"}' \ + >"${multiple_identity_directory}/second.identity.json" +RELEASE_ARTIFACTS="$(jq -cn '[ + {path: "first.tar.gz", package_identity_file: "first.identity.json"}, + {path: "second.jar", package_identity_file: "second.identity.json"} +]')" +RELEASE_ARTIFACT_DIRECTORY="${multiple_identity_directory}" +RELEASE_SOURCE_ARTIFACT_NAME="multiple-identities" +RELEASE_SOURCE_SHA="eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" +RELEASE_CATALOG_KEY="archive:multiple" +export RELEASE_SOURCE_SHA RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_CATALOG_KEY +"${repository_root}/release-catalog/materialize.sh" +jq -e ' + (.entries | length == 2) + and .entries[0].package == {ecosystem: "archive", name: "first", version: "1.0"} + and .entries[1].package == {ecosystem: "maven", name: "example:second", version: "2.0"} +' "${multiple_identity_directory}/release-catalog-entries.json" >/dev/null +test "$(find "${multiple_identity_directory}/release-evidence" -type f | wc -l | tr -d ' ')" = "4" + +generated_directory="${temporary_directory}/generated-bundle" +mkdir -p "${generated_directory}/linux-64" +printf '%s\n' conda >"${generated_directory}/linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.bin" +jq -n '{ecosystem: "conda", name: "kvikio", version: "26.08.00a32"}' \ + >"${generated_directory}/kvikio.identity.json" + +RELEASE_ARTIFACTS="$(jq -cn '[{path: "linux-64/kvikio-*.bin", package_identity_file: "kvikio.identity.json"}]')" +RELEASE_ARTIFACT_DIRECTORY="${generated_directory}" +RELEASE_SOURCE_ARTIFACT_NAME="kvikio_conda_python_kvikio_x86_64_abi3_cu12" +RELEASE_SOURCE_SHA="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +RELEASE_CATALOG_KEY="conda:kvikio" +export RELEASE_SOURCE_SHA RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_CATALOG_KEY + +"${repository_root}/release-catalog/materialize.sh" + +generated_entries_path="${generated_directory}/release-catalog-entries.json" +generated_cyclonedx_sbom_path="$(jq -r '.entries[0].sbom' "${generated_entries_path}")" +generated_provenance_path="$(jq -r '.entries[0].provenance' "${generated_entries_path}")" +jq -e ' + .source.artifact == "kvikio_conda_python_kvikio_x86_64_abi3_cu12" + and .source.sha == "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + and .entries[0].release_catalog_key == "conda:kvikio" + and .entries[0].path == "linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.bin" + and .entries[0].package == {ecosystem: "conda", name: "kvikio", version: "26.08.00a32"} + and .entries[0].sbom_kind == "generated-identity" +' "${generated_entries_path}" >/dev/null +jq -e ' + .bomFormat == "CycloneDX" + and .specVersion == "1.6" + and .metadata.component.type == "file" + and .metadata.component.name == "kvikio" + and .metadata.component.version == "26.08.00a32" + and .metadata.component.hashes[0].alg == "SHA-256" +' "${generated_directory}/${generated_cyclonedx_sbom_path}" >/dev/null +jq -e ' + .predicateType == "https://slsa.dev/provenance/v1" + and .predicate.buildDefinition.externalParameters.release_catalog_key == "conda:kvikio" + and .predicate.buildDefinition.resolvedDependencies[0].digest.gitCommit == "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +' "${generated_directory}/${generated_provenance_path}" >/dev/null +wheel_directory="${temporary_directory}/wheel-bundle" +mkdir -p "${wheel_directory}" +printf '%s\n' wheel >"${wheel_directory}/libkvikio_cu12-26.8.0a32.bin" +jq -n '{ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"}' \ + >"${wheel_directory}/libkvikio.identity.json" + +RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.bin", package_identity_file: "libkvikio.identity.json"}]')" +RELEASE_ARTIFACT_DIRECTORY="${wheel_directory}" +RELEASE_SOURCE_ARTIFACT_NAME="kvikio_wheel_cpp_libkvikio_x86_64_cu12" +RELEASE_SOURCE_SHA="cccccccccccccccccccccccccccccccccccccccc" +RELEASE_CATALOG_KEY="wheel:kvikio" +export RELEASE_SOURCE_SHA RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_CATALOG_KEY + +"${repository_root}/release-catalog/materialize.sh" + +jq -e ' + .entries[0].release_catalog_key == "wheel:kvikio" + and .entries[0].path == "libkvikio_cu12-26.8.0a32.bin" + and .entries[0].package == {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"} + and .entries[0].sbom_kind == "generated-identity" +' "${wheel_directory}/release-catalog-entries.json" >/dev/null + +missing_version_directory="${temporary_directory}/missing-version-bundle" +mkdir -p "${missing_version_directory}" +printf '%s\n' archive >"${missing_version_directory}/bundle.tar.gz" +jq -n \ + '{ecosystem: "archive", name: "bundle"}' \ + >"${missing_version_directory}/bundle-package-identity.json" + +RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz", package_identity_file: "bundle-package-identity.json"}]')" +RELEASE_ARTIFACT_DIRECTORY="${missing_version_directory}" +RELEASE_SOURCE_ARTIFACT_NAME="bundle-archive" +RELEASE_SOURCE_SHA="dddddddddddddddddddddddddddddddddddddddd" +RELEASE_CATALOG_KEY="archive:bundle" +export RELEASE_SOURCE_SHA RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_CATALOG_KEY + +if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/missing-version-error"; then + echo "materialize.sh unexpectedly accepted a custom artifact without a package version" >&2 + exit 1 +fi +grep -Fx '[materialize.sh] Error: package identity for bundle.tar.gz must contain non-empty ecosystem, name, and version strings and only optional build or platform strings' "${temporary_directory}/missing-version-error" + +if (unset RELEASE_SOURCE_SHA; "${repository_root}/release-catalog/materialize.sh") 2>"${temporary_directory}/missing-source-sha-error"; then + echo "materialize.sh unexpectedly accepted a missing RELEASE_SOURCE_SHA" >&2 + exit 1 +fi +grep -Fx '[materialize.sh] Error: RELEASE_SOURCE_SHA must be a non-empty string' "${temporary_directory}/missing-source-sha-error" + +if RELEASE_SOURCE_SHA="not-a-git-object-id" \ + "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/invalid-rapids-sha-error"; then + echo "materialize.sh unexpectedly accepted an invalid RELEASE_SOURCE_SHA" >&2 + exit 1 +fi +grep -Fx '[materialize.sh] Error: RELEASE_SOURCE_SHA must be a 40- or 64-character hexadecimal Git object ID' \ + "${temporary_directory}/invalid-rapids-sha-error" + +assert_missing_build_context() { + local variable_name="$1" + local error_file="${temporary_directory}/missing-${variable_name}.error" + if (unset "${variable_name}"; "${repository_root}/release-catalog/materialize.sh") 2>"${error_file}"; then + echo "materialize.sh unexpectedly accepted a missing ${variable_name}" >&2 + exit 1 + fi + grep -Fx "[materialize.sh] Error: ${variable_name} must be a non-empty string" "${error_file}" +} + +assert_missing_build_context GITHUB_REPOSITORY +assert_missing_build_context GITHUB_RUN_ATTEMPT +assert_missing_build_context GITHUB_RUN_ID +assert_missing_build_context GITHUB_WORKFLOW_REF + +if GITHUB_REPOSITORY="cuvs" \ + "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/invalid-repository.error"; then + echo "materialize.sh unexpectedly accepted an invalid GITHUB_REPOSITORY" >&2 + exit 1 +fi +grep -Fx '[materialize.sh] Error: GITHUB_REPOSITORY must have owner/repository form' \ + "${temporary_directory}/invalid-repository.error" + +if GITHUB_RUN_ATTEMPT="0" \ + "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/invalid-run-attempt.error"; then + echo "materialize.sh unexpectedly accepted an invalid GITHUB_RUN_ATTEMPT" >&2 + exit 1 +fi +grep -Fx '[materialize.sh] Error: GITHUB_RUN_ATTEMPT must be a positive integer' \ + "${temporary_directory}/invalid-run-attempt.error" + +if GITHUB_RUN_ID="not-an-integer" \ + "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/invalid-run-id.error"; then + echo "materialize.sh unexpectedly accepted an invalid GITHUB_RUN_ID" >&2 + exit 1 +fi +grep -Fx '[materialize.sh] Error: GITHUB_RUN_ID must be a positive integer' \ + "${temporary_directory}/invalid-run-id.error"