From 6a4084216c53ef682a43ceba4bfe8d77c930b354 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 7 Aug 2026 13:39:40 -0500 Subject: [PATCH 01/36] Make release output dispatch container-safe --- .github/workflows/pr.yml | 14 +- .pre-commit-config.yaml | 21 +- README.md | 46 +++- release-build-output-dispatch/action.yml | 123 ++++++----- release-build-output/action.yml | 90 -------- release-build-output/config.schema.json | 206 ++++++++++++++++++ release-build-output/validate-config.sh | 119 ++++++++++ .../inline-package.json | 14 ++ .../package-file.json | 12 + tests/release_build_output_config_test.sh | 107 +++++++++ 10 files changed, 600 insertions(+), 152 deletions(-) delete mode 100644 release-build-output/action.yml create mode 100644 release-build-output/config.schema.json create mode 100755 release-build-output/validate-config.sh create mode 100644 tests/release-build-output-config/inline-package.json create mode 100644 tests/release-build-output-config/package-file.json create mode 100755 tests/release_build_output_config_test.sh diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 7b3ebce7..4c797303 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -22,6 +22,7 @@ jobs: - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 - name: Test release build-output materialization run: | + ./tests/release_build_output_config_test.sh ./tests/release_build_output_descriptors_test.sh ./tests/release_build_output_prepare_test.sh ./tests/release_build_output_test.sh @@ -35,11 +36,14 @@ jobs: SHARED_ACTIONS_REPO: ${{ github.event.pull_request.head.repo.full_name }} SHARED_ACTIONS_REF: ${{ github.event.pull_request.head.sha }} with: - artifact-type: custom - output-directory: release-build-output-smoke - release-artifacts: '[{"path":"package.tar.gz"}]' - release-package: '{"ecosystem":"archive","name":"smoke","version":"1.0"}' - release-unit: archive:smoke + config: >- + { + "artifact_type": "custom", + "component_id": "archive:smoke", + "output_directory": "release-build-output-smoke", + "package": {"ecosystem": "archive", "name": "smoke", "version": "1.0"}, + "artifacts": [{"path": "package.tar.gz"}] + } source-artifact-name: release-build-output-dispatch-smoke source-sha: ${{ github.event.pull_request.head.sha }} - name: Verify release build-output dispatch smoke test diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 799d7320..4a7f1db5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,10 @@ ci: autoupdate_schedule: quarterly - skip: [actionlint-docker] + # The runtime validator suite requires jq. pre-commit.ci validates the same + # valid fixtures against config.schema.json; GitHub Actions runs the full + # valid/invalid runtime suite on an Ubuntu runner with jq. + skip: [actionlint-docker, release-build-output-config] repos: - repo: https://github.com/pre-commit/pre-commit-hooks @@ -11,6 +14,7 @@ repos: hooks: - id: trailing-whitespace - id: check-added-large-files + - id: check-json - id: check-yaml - id: end-of-file-fixer - repo: https://github.com/zizmorcore/zizmor-pre-commit @@ -52,3 +56,18 @@ repos: hooks: - id: yamllint additional_dependencies: [pyyaml] + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.37.2 + hooks: + - id: check-jsonschema + name: validate release build-output config schema fixtures + args: [--schemafile, release-build-output/config.schema.json] + files: ^tests/release-build-output-config/.*\.json$ + - repo: local + hooks: + - id: release-build-output-config + name: validate release build-output configuration + entry: ./tests/release_build_output_config_test.sh + language: system + pass_filenames: false + files: ^(release-build-output/(config\.schema\.json|validate-config\.sh)|tests/release_build_output_config_test\.sh)$ diff --git a/README.md b/README.md index 4ea1b0ca..69f206a8 100644 --- a/README.md +++ b/README.md @@ -17,18 +17,50 @@ directory and uploads a companion artifact named `release-build-output.json`, `release-build-metadata.json`, provenance, and an SBOM record for every primary artifact. -Conda and wheel jobs can set `artifact-type` to `conda` or `wheel` and omit -`release-artifacts`; the implementation reads package metadata from the built -files. Custom bundles provide explicit artifact descriptors and either inline -package identity or a producer-created package JSON file. +Every producer passes one `config` JSON object. `artifact_type` is the +discriminator: `conda` and `wheel` derive artifact descriptors and package +metadata from the built files, while `custom` selects them explicitly. The +action validates the complete object before inspecting build outputs and +reports each invalid field as a pipeline error. Its schema is +[`release-build-output/config.schema.json`](release-build-output/config.schema.json). + +The custom configuration requires `component_id`, a non-empty `artifacts` +array, and exactly one of `package` or `package_file`. `output_directory` +defaults to `.`. A descriptor's `sbom`, `provenance`, and `signature` fields +select producer-supplied evidence; omitting `sbom` requests a generated +identity-only SPDX envelope, not a dependency SBOM. + +```yaml +- name: Create custom release build-output companion + uses: rapidsai/shared-actions/release-build-output-dispatch@main + with: + config: >- + { + "artifact_type": "custom", + "component_id": "maven:cuvs-java", + "output_directory": "java/cuvs-java/target", + "package_file": "cuvs-java.release-package.json", + "artifacts": [{"path": "cuvs-java-*-x86_64-cuda*.jar"}] + } + source-artifact-name: cuvs-java + source-sha: ${{ github.sha }} +``` + +The validator is also exercised by pre-commit against valid and invalid +fixtures. Pre-commit can validate the JSON structure, while the pipeline checks +that package files, primary artifacts, and evidence globs resolve after the +producer has run. ```yaml - name: Create release build-output companion uses: rapidsai/shared-actions/release-build-output-dispatch@main with: - artifact-type: wheel - output-directory: ${{ steps.package-name.outputs.WHEEL_OUTPUT_DIR }} - release-unit: wheel:example + config: >- + { + "artifact_type": "wheel", + "component_id": "wheel:example", + "output_directory": ${{ toJSON(steps.package-name.outputs.WHEEL_OUTPUT_DIR) }} + } source-artifact-name: ${{ steps.package-name.outputs.RAPIDS_PACKAGE_NAME }} source-sha: ${{ github.sha }} ``` diff --git a/release-build-output-dispatch/action.yml b/release-build-output-dispatch/action.yml index 154ba3d7..fc798f74 100644 --- a/release-build-output-dispatch/action.yml +++ b/release-build-output-dispatch/action.yml @@ -1,33 +1,12 @@ name: Dispatch release build output -description: Check out the selected shared-actions revision and create a release build-output companion. +description: Create a release build-output companion from the selected shared-actions revision. inputs: - artifact-type: - description: One of conda, wheel, or custom. + config: + description: >- + Release build-output JSON object containing artifact_type, component_id, output_directory, and custom artifact + selection and package identity when artifact_type is custom. required: true - release-unit: - description: Release-platform unit ID for every primary artifact in this bundle. - required: true - release-package: - description: JSON package fields shared by the bundle. - required: false - release-package-file: - description: Relative path to producer-created package JSON inside output-directory. - required: false - release-artifacts: - description: JSON artifact and evidence descriptors relative to output-directory. - required: false - output-directory: - description: Directory containing the primary files and any producer-supplied evidence. - required: true - manifest-name: - description: Filename to write inside output-directory. - required: false - default: release-build-output.json - metadata-name: - description: Filename for the build metadata envelope. - required: false - default: release-build-metadata.json source-artifact-name: description: Name of the GitHub Actions artifact bundle containing this output. required: true @@ -38,35 +17,81 @@ inputs: outputs: manifest-path: description: Absolute path to the generated manifest. - value: ${{ steps.release-build-output.outputs.manifest-path }} + value: ${{ steps.materialize.outputs.manifest-path }} metadata-path: description: Absolute path to the build metadata envelope. - value: ${{ steps.release-build-output.outputs.metadata-path }} + value: ${{ steps.materialize.outputs.metadata-path }} manifest-artifact-name: description: Name of the uploaded GitHub Actions companion artifact. - value: ${{ steps.release-build-output.outputs.manifest-artifact-name }} + value: ${{ steps.companion-name.outputs.name }} runs: using: composite steps: - - name: Check out shared-actions implementation - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ env.SHARED_ACTIONS_REPO || github.action_repository || 'rapidsai/shared-actions' }} - ref: ${{ env.SHARED_ACTIONS_REF || github.action_ref || 'main' }} - path: ./shared-actions - persist-credentials: false - - id: release-build-output - name: Create release build-output companion - uses: ./shared-actions/release-build-output + - id: implementation + name: Resolve release build-output implementation + shell: bash + env: + ACTION_PATH: ${{ github.action_path }} + HOST_WORKSPACE: ${{ github.workspace }} + run: | + host_root="$(dirname "$(dirname "${HOST_WORKSPACE}")")" + container_root="$(dirname "$(dirname "${PWD}")")" + action_path="${ACTION_PATH}" + if [[ ! -d "${action_path}" && "${action_path}" == "${host_root}/"* ]]; then + action_path="${container_root}/${action_path#"${host_root}/"}" + fi + implementation_path="$(dirname "${action_path}")/release-build-output" + if [[ ! -d "${implementation_path}" ]]; then + echo "release-build-output implementation not found at ${implementation_path}" >&2 + exit 1 + fi + echo "path=${implementation_path}" >>"${GITHUB_OUTPUT}" + - id: configuration + name: Validate release build-output configuration + shell: bash + env: + IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} + RELEASE_BUILD_OUTPUT_CONFIG: ${{ inputs.config }} + run: "${IMPLEMENTATION_PATH}/validate-config.sh" + - id: prepare + name: Describe release artifacts + shell: bash + env: + IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} + RELEASE_ARTIFACTS: ${{ steps.configuration.outputs.artifacts }} + RELEASE_ARTIFACT_TYPE: ${{ steps.configuration.outputs.artifact_type }} + RELEASE_OUTPUT_DIRECTORY: ${{ steps.configuration.outputs.output_directory }} + RELEASE_PACKAGE: ${{ steps.configuration.outputs.package }} + RELEASE_PACKAGE_FILE: ${{ steps.configuration.outputs.package_file }} + run: "${IMPLEMENTATION_PATH}/prepare.sh" + - id: materialize + name: Materialize release build-output records + shell: bash + env: + IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} + RELEASE_ARTIFACTS: ${{ steps.prepare.outputs.artifacts }} + RELEASE_MANIFEST_NAME: release-build-output.json + RELEASE_METADATA_NAME: release-build-metadata.json + RELEASE_OUTPUT_DIRECTORY: ${{ steps.configuration.outputs.output_directory }} + RELEASE_PACKAGE: ${{ steps.prepare.outputs.package }} + RELEASE_PACKAGE_FILE: ${{ steps.configuration.outputs.package_file }} + RELEASE_SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} + RELEASE_SOURCE_SHA: ${{ inputs.source-sha || github.sha }} + RELEASE_UNIT: ${{ steps.configuration.outputs.component_id }} + run: "${IMPLEMENTATION_PATH}/materialize.sh" + - id: companion-name + name: Set companion artifact name + shell: bash + env: + SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} + run: echo "name=release-build-output-${SOURCE_ARTIFACT_NAME}" >>"${GITHUB_OUTPUT}" + - name: Upload release build-output companion + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - artifact-type: ${{ inputs.artifact-type }} - release-unit: ${{ inputs.release-unit }} - release-package: ${{ inputs.release-package }} - release-package-file: ${{ inputs.release-package-file }} - release-artifacts: ${{ inputs.release-artifacts }} - output-directory: ${{ inputs.output-directory }} - manifest-name: ${{ inputs.manifest-name }} - metadata-name: ${{ inputs.metadata-name }} - source-artifact-name: ${{ inputs.source-artifact-name }} - source-sha: ${{ inputs.source-sha }} + if-no-files-found: error + name: ${{ steps.companion-name.outputs.name }} + path: | + ${{ steps.configuration.outputs.output_directory }}/release-build-output.json + ${{ steps.configuration.outputs.output_directory }}/release-build-metadata.json + ${{ steps.configuration.outputs.output_directory }}/release-evidence/** diff --git a/release-build-output/action.yml b/release-build-output/action.yml deleted file mode 100644 index 7c6965e3..00000000 --- a/release-build-output/action.yml +++ /dev/null @@ -1,90 +0,0 @@ -name: Create release build output companion -description: Validate a build artifact bundle and upload its release-platform manifest and evidence companion. - -inputs: - artifact-type: - description: One of conda, wheel, or custom. Conda and wheel descriptors are derived when release-artifacts is omitted. - required: true - release-unit: - description: Release-platform unit ID for every primary artifact in this bundle. - required: true - release-package: - description: JSON package fields shared by the bundle. Provide this or release-package-file for custom artifacts. - required: false - release-package-file: - description: Relative path to producer-created package JSON inside output-directory. - required: false - release-artifacts: - description: JSON artifact and evidence descriptors relative to output-directory. Required for custom artifacts. - required: false - output-directory: - description: Directory containing the primary files and any producer-supplied evidence. - required: true - manifest-name: - description: Filename to write inside output-directory. - required: false - default: release-build-output.json - metadata-name: - description: Filename for the build-environment and SBOM-classification envelope. - required: false - default: release-build-metadata.json - source-artifact-name: - description: Name of the GitHub Actions artifact bundle containing this output. - required: true - source-sha: - description: Source revision built by the producing job. Defaults to the current workflow SHA. - required: false - -outputs: - manifest-path: - description: Absolute path to the generated manifest. - value: ${{ steps.materialize.outputs.manifest-path }} - metadata-path: - description: Absolute path to the build metadata envelope. - value: ${{ steps.materialize.outputs.metadata-path }} - manifest-artifact-name: - description: Name of the uploaded GitHub Actions companion artifact. - value: ${{ steps.companion-name.outputs.name }} - -runs: - using: composite - steps: - - id: prepare - name: Describe release artifacts - shell: bash - env: - RELEASE_ARTIFACTS: ${{ inputs.release-artifacts }} - RELEASE_ARTIFACT_TYPE: ${{ inputs.artifact-type }} - RELEASE_OUTPUT_DIRECTORY: ${{ inputs.output-directory }} - RELEASE_PACKAGE: ${{ inputs.release-package }} - RELEASE_PACKAGE_FILE: ${{ inputs.release-package-file }} - run: ./shared-actions/release-build-output/prepare.sh - - id: materialize - name: Materialize release build-output records - shell: bash - env: - RELEASE_ARTIFACTS: ${{ steps.prepare.outputs.artifacts }} - RELEASE_MANIFEST_NAME: ${{ inputs.manifest-name }} - RELEASE_METADATA_NAME: ${{ inputs.metadata-name }} - RELEASE_OUTPUT_DIRECTORY: ${{ inputs.output-directory }} - RELEASE_PACKAGE: ${{ steps.prepare.outputs.package }} - RELEASE_PACKAGE_FILE: ${{ inputs.release-package-file }} - RELEASE_SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} - RELEASE_SOURCE_SHA: ${{ inputs.source-sha || github.sha }} - RELEASE_UNIT: ${{ inputs.release-unit }} - run: ./shared-actions/release-build-output/materialize.sh - - id: companion-name - name: Set companion artifact name - shell: bash - env: - SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} - run: echo "name=release-build-output-${SOURCE_ARTIFACT_NAME}" >>"${GITHUB_OUTPUT}" - - name: Upload release build-output companion - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - if-no-files-found: error - name: ${{ steps.companion-name.outputs.name }} - path: | - ${{ inputs.output-directory }}/${{ inputs.manifest-name }} - ${{ inputs.output-directory }}/${{ inputs.metadata-name }} - ${{ inputs.output-directory }}/release-evidence/** diff --git a/release-build-output/config.schema.json b/release-build-output/config.schema.json new file mode 100644 index 00000000..81d37d6f --- /dev/null +++ b/release-build-output/config.schema.json @@ -0,0 +1,206 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://rapids.ai/schemas/release-build-output-config-v1.json", + "title": "Release build-output configuration", + "description": "Producer configuration for one release build-output companion.", + "type": "object", + "additionalProperties": false, + "required": [ + "artifact_type", + "component_id" + ], + "properties": { + "artifact_type": { + "description": "How primary artifact descriptors and package identities are obtained.", + "type": "string", + "enum": [ + "conda", + "custom", + "wheel" + ] + }, + "component_id": { + "description": "Stable release component ID shared by the selected files and their matrix variants.", + "type": "string", + "minLength": 1, + "pattern": "^[^\\r\\n]+$" + }, + "output_directory": { + "description": "Base directory for artifact and evidence paths. Defaults to the job working directory.", + "type": "string", + "minLength": 1, + "pattern": "^[^\\r\\n]+$", + "default": "." + }, + "package": { + "$ref": "#/$defs/package" + }, + "package_file": { + "description": "Producer-created package identity JSON, relative to output_directory.", + "type": "string", + "minLength": 1, + "pattern": "^[^\\r\\n]+$" + }, + "artifacts": { + "description": "Primary release files and optional producer-supplied evidence sidecars.", + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/artifact" + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "artifact_type": { + "const": "custom" + } + }, + "required": [ + "artifact_type" + ] + }, + "then": { + "required": [ + "artifacts" + ], + "oneOf": [ + { + "required": [ + "package" + ], + "not": { + "required": [ + "package_file" + ] + } + }, + { + "required": [ + "package_file" + ], + "not": { + "required": [ + "package" + ] + } + } + ] + }, + "else": { + "required": [ + "output_directory" + ], + "not": { + "anyOf": [ + { + "required": [ + "artifacts" + ] + }, + { + "required": [ + "package" + ] + }, + { + "required": [ + "package_file" + ] + } + ] + } + } + } + ], + "$defs": { + "package": { + "type": "object", + "additionalProperties": false, + "required": [ + "ecosystem", + "name" + ], + "properties": { + "ecosystem": { + "type": "string", + "minLength": 1 + }, + "name": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string" + }, + "build": { + "type": "string" + }, + "platform": { + "type": "string" + } + } + }, + "artifact": { + "type": "object", + "additionalProperties": false, + "required": [ + "path" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\r\\n]+$" + }, + "sbom": { + "description": "Producer-supplied dependency SBOM path or glob.", + "type": "string", + "minLength": 1, + "pattern": "^[^\\r\\n]+$" + }, + "provenance": { + "description": "Producer-supplied provenance path or glob.", + "type": "string", + "minLength": 1, + "pattern": "^[^\\r\\n]+$" + }, + "signature": { + "description": "Producer-supplied signature path or glob.", + "type": "string", + "minLength": 1, + "pattern": "^[^\\r\\n]+$" + }, + "package": { + "description": "Per-artifact package identity overrides.", + "type": "object", + "additionalProperties": false, + "properties": { + "ecosystem": { + "type": "string", + "minLength": 1 + }, + "name": { + "type": "string", + "minLength": 1 + }, + "version": { + "type": "string", + "minLength": 1 + }, + "build": { + "type": "string", + "minLength": 1 + }, + "platform": { + "type": "string", + "minLength": 1 + } + } + } + } + } + } +} diff --git a/release-build-output/validate-config.sh b/release-build-output/validate-config.sh new file mode 100755 index 00000000..458f4e5e --- /dev/null +++ b/release-build-output/validate-config.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +set -euo pipefail + +emit_error() { + local message="$1" + printf '::error title=Invalid release-build-output configuration::%s\n' "${message}" >&2 +} + +config="${RELEASE_BUILD_OUTPUT_CONFIG:-}" +if [[ -z "${config}" ]]; then + emit_error "release-build-output must be a non-empty JSON object" + exit 1 +fi + +if ! compact_config="$(jq -ce . <<<"${config}" 2>/dev/null)"; then + parse_error="$(jq -ce . <<<"${config}" 2>&1 || true)" + emit_error "release-build-output must be valid JSON: ${parse_error}" + exit 1 +fi + +validation_errors="$(jq -r ' + def single_line_string: + type == "string" and length > 0 and (test("[\\r\\n]") | not); + def package_object: + type == "object" + and (keys - ["ecosystem", "name", "version", "build", "platform"] | length == 0) + and (.ecosystem | single_line_string) + and (.name | single_line_string) + and ((.version // "") | type == "string") + and ((.build // "") | type == "string") + and ((.platform // "") | type == "string"); + def artifact_package_object: + type == "object" + and (keys - ["ecosystem", "name", "version", "build", "platform"] | length == 0) + and (to_entries | map(.value | single_line_string) | all); + if type != "object" then + ["release-build-output must be a JSON object"] + else + (keys - ["artifact_type", "component_id", "output_directory", "package", "package_file", "artifacts"]) as $unknown + | [ + if ($unknown | length) > 0 then + "unknown field(s): " + ($unknown | join(", ")) + else empty end, + if (.artifact_type == "conda" or .artifact_type == "custom" or .artifact_type == "wheel") then empty + else "artifact_type must be one of: conda, custom, wheel" end, + if (.component_id | single_line_string) then empty + else "component_id must be a non-empty, single-line string" end, + if (has("output_directory") | not) or (.output_directory | single_line_string) then empty + else "output_directory must be a non-empty, single-line string when supplied" end + ] + + if .artifact_type == "custom" then + [ + if ([has("package"), has("package_file")] | map(select(.)) | length) == 1 then empty + else "exactly one of package or package_file is required for custom artifacts" end, + if (has("package") | not) or (.package | package_object) then empty + else "package must contain non-empty ecosystem and name strings and only version, build, or platform as optional string fields" end, + if (has("package_file") | not) or (.package_file | single_line_string) then empty + else "package_file must be a non-empty, single-line path relative to output_directory" end, + if (.artifacts | type == "array" and length > 0) then empty + else "artifacts must be a non-empty array for custom artifacts" end + ] + elif .artifact_type == "conda" or .artifact_type == "wheel" then + [ + if (.output_directory | single_line_string) then empty + else "output_directory is required for conda and wheel artifacts" end, + if has("artifacts") or has("package") or has("package_file") then + "artifacts, package, and package_file are only valid when artifact_type is custom" + else empty end + ] + else [] end + + if .artifact_type == "custom" and (.artifacts | type) == "array" then + [ + .artifacts | to_entries[] + | .key as $index + | .value as $artifact + | if ($artifact | type) != "object" then + "artifacts[\($index)] must be an object" + else + ($artifact | keys - ["path", "sbom", "provenance", "signature", "package"]) as $artifact_unknown + | if ($artifact_unknown | length) > 0 then + "artifacts[\($index)] has unknown field(s): " + ($artifact_unknown | join(", ")) + else empty end, + if ($artifact.path | single_line_string) then empty + else "artifacts[\($index)].path must be a non-empty, single-line string" end, + ($artifact | to_entries[] + | select(.key == "sbom" or .key == "provenance" or .key == "signature") + | select((.value | single_line_string) | not) + | "artifacts[\($index)].\(.key) must be a non-empty, single-line string"), + if ($artifact | has("package") | not) or ($artifact.package | artifact_package_object) then empty + else "artifacts[\($index)].package must contain only non-empty string identity overrides" end + end + ] + else [] end + end + | .[] +' <<<"${compact_config}")" + +if [[ -n "${validation_errors}" ]]; then + while IFS= read -r validation_error; do + emit_error "${validation_error}" + done <<<"${validation_errors}" + exit 1 +fi + +if [[ -z "${GITHUB_OUTPUT:-}" ]]; then + emit_error "GITHUB_OUTPUT is required" + exit 1 +fi + +{ + printf 'artifact_type=%s\n' "$(jq -r '.artifact_type' <<<"${compact_config}")" + printf 'component_id=%s\n' "$(jq -r '.component_id' <<<"${compact_config}")" + printf 'output_directory=%s\n' "$(jq -r '.output_directory // "."' <<<"${compact_config}")" + printf 'package=%s\n' "$(jq -c '.package // empty' <<<"${compact_config}")" + printf 'package_file=%s\n' "$(jq -r '.package_file // empty' <<<"${compact_config}")" + printf 'artifacts=%s\n' "$(jq -c '.artifacts // empty' <<<"${compact_config}")" +} >>"${GITHUB_OUTPUT}" diff --git a/tests/release-build-output-config/inline-package.json b/tests/release-build-output-config/inline-package.json new file mode 100644 index 00000000..3443ea3b --- /dev/null +++ b/tests/release-build-output-config/inline-package.json @@ -0,0 +1,14 @@ +{ + "artifact_type": "custom", + "component_id": "archive:smoke", + "package": { + "ecosystem": "archive", + "name": "smoke", + "version": "1.0" + }, + "artifacts": [ + { + "path": "smoke.tar.gz" + } + ] +} diff --git a/tests/release-build-output-config/package-file.json b/tests/release-build-output-config/package-file.json new file mode 100644 index 00000000..de25059e --- /dev/null +++ b/tests/release-build-output-config/package-file.json @@ -0,0 +1,12 @@ +{ + "artifact_type": "custom", + "component_id": "maven:cuvs-java", + "output_directory": "java/cuvs-java/target", + "package_file": "cuvs-java.release-package.json", + "artifacts": [ + { + "path": "cuvs-java-*-x86_64-cuda*.jar", + "sbom": "cuvs-java.spdx.json" + } + ] +} diff --git a/tests/release_build_output_config_test.sh b/tests/release_build_output_config_test.sh new file mode 100755 index 00000000..d77375e8 --- /dev/null +++ b/tests/release_build_output_config_test.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +set -euo pipefail + +repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +validator="${repository_root}/release-build-output/validate-config.sh" +temporary_directory="$(mktemp -d)" +trap 'rm -rf "${temporary_directory}"' EXIT + +assert_invalid() { + local name="$1" + local config="$2" + local expected="$3" + local error_file="${temporary_directory}/${name}.error" + + if RELEASE_BUILD_OUTPUT_CONFIG="${config}" GITHUB_OUTPUT="${temporary_directory}/${name}.output" \ + "${validator}" 2>"${error_file}"; then + echo "validate-config.sh unexpectedly accepted ${name}" >&2 + exit 1 + fi + if ! grep -F -- "${expected}" "${error_file}" >/dev/null; then + echo "validate-config.sh did not explain ${name}; output was:" >&2 + sed 's/^/ /' "${error_file}" >&2 + exit 1 + fi +} + +valid_output="${temporary_directory}/valid.output" +RELEASE_BUILD_OUTPUT_CONFIG="$(<"${repository_root}/tests/release-build-output-config/package-file.json")" \ + GITHUB_OUTPUT="${valid_output}" "${validator}" + +grep -Fx 'component_id=maven:cuvs-java' "${valid_output}" +grep -Fx 'artifact_type=custom' "${valid_output}" +grep -Fx 'output_directory=java/cuvs-java/target' "${valid_output}" +grep -Fx 'package=' "${valid_output}" +grep -Fx 'package_file=cuvs-java.release-package.json' "${valid_output}" +grep -Fx 'artifacts=[{"path":"cuvs-java-*-x86_64-cuda*.jar","sbom":"cuvs-java.spdx.json"}]' "${valid_output}" + +inline_output="${temporary_directory}/inline.output" +RELEASE_BUILD_OUTPUT_CONFIG="$(<"${repository_root}/tests/release-build-output-config/inline-package.json")" \ + GITHUB_OUTPUT="${inline_output}" "${validator}" + +grep -Fx 'output_directory=.' "${inline_output}" +grep -Fx 'package={"ecosystem":"archive","name":"smoke","version":"1.0"}' "${inline_output}" +grep -Fx 'package_file=' "${inline_output}" + +assert_invalid \ + malformed-json \ + '{"component_id":' \ + 'release-build-output must be valid JSON' + +assert_invalid \ + not-an-object \ + '[]' \ + 'release-build-output must be a JSON object' + +assert_invalid \ + missing-fields \ + '{}' \ + 'component_id must be a non-empty, single-line string' +grep -F 'artifact_type must be one of: conda, custom, wheel' "${temporary_directory}/missing-fields.error" >/dev/null + +assert_invalid \ + custom-missing-fields \ + '{"artifact_type":"custom","component_id":"archive:smoke"}' \ + 'exactly one of package or package_file is required for custom artifacts' +grep -F 'artifacts must be a non-empty array for custom artifacts' "${temporary_directory}/custom-missing-fields.error" >/dev/null + +assert_invalid \ + unknown-field \ + '{"artifact_type":"custom","component_id":"archive:smoke","package_file":"package.json","artifacts":[{"path":"smoke.tar.gz"}],"unit":"archive:smoke"}' \ + 'unknown field(s): unit' + +assert_invalid \ + conflicting-package-source \ + '{"artifact_type":"custom","component_id":"archive:smoke","package":{"ecosystem":"archive","name":"smoke"},"package_file":"package.json","artifacts":[{"path":"smoke.tar.gz"}]}' \ + 'exactly one of package or package_file is required for custom artifacts' + +assert_invalid \ + malformed-artifact \ + '{"artifact_type":"custom","component_id":"archive:smoke","package_file":"package.json","artifacts":[{"file":"smoke.tar.gz","sbom":false}]}' \ + 'artifacts[0] has unknown field(s): file' +grep -F 'artifacts[0].path must be a non-empty, single-line string' "${temporary_directory}/malformed-artifact.error" >/dev/null +grep -F 'artifacts[0].sbom must be a non-empty, single-line string' "${temporary_directory}/malformed-artifact.error" >/dev/null + +assert_invalid \ + standard-missing-output-directory \ + '{"artifact_type":"conda","component_id":"conda:smoke"}' \ + 'output_directory is required for conda and wheel artifacts' + +assert_invalid \ + standard-custom-fields \ + '{"artifact_type":"wheel","component_id":"wheel:smoke","output_directory":"dist","artifacts":[{"path":"smoke.whl"}]}' \ + 'artifacts, package, and package_file are only valid when artifact_type is custom' + +standard_output="${temporary_directory}/standard.output" +RELEASE_BUILD_OUTPUT_CONFIG='{ + "artifact_type": "wheel", + "component_id": "wheel:kvikio", + "output_directory": "dist" +}' GITHUB_OUTPUT="${standard_output}" "${validator}" + +grep -Fx 'artifact_type=wheel' "${standard_output}" +grep -Fx 'component_id=wheel:kvikio' "${standard_output}" +grep -Fx 'output_directory=dist' "${standard_output}" +grep -Fx 'artifacts=' "${standard_output}" From a14b8c572fad094379f0e4f4de92d295722c1865 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 17 Aug 2026 11:35:25 -0500 Subject: [PATCH 02/36] Link release output config schema --- release-build-output-dispatch/action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release-build-output-dispatch/action.yml b/release-build-output-dispatch/action.yml index fc798f74..85cf7e97 100644 --- a/release-build-output-dispatch/action.yml +++ b/release-build-output-dispatch/action.yml @@ -5,7 +5,8 @@ inputs: config: description: >- Release build-output JSON object containing artifact_type, component_id, output_directory, and custom artifact - selection and package identity when artifact_type is custom. + selection and package identity when artifact_type is custom. Schema and field documentation: + https://github.com/rapidsai/shared-actions/blob/main/release-build-output/config.schema.json required: true source-artifact-name: description: Name of the GitHub Actions artifact bundle containing this output. From 1ca50f81199a5d125892f1bf3e68050557ef1eb4 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 17 Aug 2026 20:22:48 -0500 Subject: [PATCH 03/36] Document release build output companions --- README.md | 60 ++------------------ release-build-output/README.md | 101 +++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 56 deletions(-) create mode 100644 release-build-output/README.md diff --git a/README.md b/README.md index 69f206a8..697fd7f8 100644 --- a/README.md +++ b/README.md @@ -13,63 +13,11 @@ A dispatch action is one that: `release-build-output-dispatch` validates a producer's local build artifact directory and uploads a companion artifact named -`release-build-output-`. The companion contains -`release-build-output.json`, `release-build-metadata.json`, provenance, and an -SBOM record for every primary artifact. - -Every producer passes one `config` JSON object. `artifact_type` is the -discriminator: `conda` and `wheel` derive artifact descriptors and package -metadata from the built files, while `custom` selects them explicitly. The -action validates the complete object before inspecting build outputs and -reports each invalid field as a pipeline error. Its schema is -[`release-build-output/config.schema.json`](release-build-output/config.schema.json). - -The custom configuration requires `component_id`, a non-empty `artifacts` -array, and exactly one of `package` or `package_file`. `output_directory` -defaults to `.`. A descriptor's `sbom`, `provenance`, and `signature` fields -select producer-supplied evidence; omitting `sbom` requests a generated -identity-only SPDX envelope, not a dependency SBOM. +`release-build-output-`. -```yaml -- name: Create custom release build-output companion - uses: rapidsai/shared-actions/release-build-output-dispatch@main - with: - config: >- - { - "artifact_type": "custom", - "component_id": "maven:cuvs-java", - "output_directory": "java/cuvs-java/target", - "package_file": "cuvs-java.release-package.json", - "artifacts": [{"path": "cuvs-java-*-x86_64-cuda*.jar"}] - } - source-artifact-name: cuvs-java - source-sha: ${{ github.sha }} -``` - -The validator is also exercised by pre-commit against valid and invalid -fixtures. Pre-commit can validate the JSON structure, while the pipeline checks -that package files, primary artifacts, and evidence globs resolve after the -producer has run. - -```yaml -- name: Create release build-output companion - uses: rapidsai/shared-actions/release-build-output-dispatch@main - with: - config: >- - { - "artifact_type": "wheel", - "component_id": "wheel:example", - "output_directory": ${{ toJSON(steps.package-name.outputs.WHEEL_OUTPUT_DIR) }} - } - source-artifact-name: ${{ steps.package-name.outputs.RAPIDS_PACKAGE_NAME }} - source-sha: ${{ github.sha }} -``` - -A descriptor-selected producer SBOM is classified as `producer-dependency`. -When no SBOM is supplied, the action generates an SPDX artifact-identity -envelope and classifies it as `generated-identity`. The generated envelope -contains the primary artifact's identity and SHA-256 but no dependency -inventory; it must not be treated as dependency coverage. +See the [`release-build-output` documentation](release-build-output/README.md) +for configuration, examples, source-revision handling, companion contents, and +evidence semantics. The dispatch wrapper honors `SHARED_ACTIONS_REPO` and `SHARED_ACTIONS_REF`. When neither is set, it checks out the same repository and ref used to invoke diff --git a/release-build-output/README.md b/release-build-output/README.md new file mode 100644 index 00000000..8f6d6935 --- /dev/null +++ b/release-build-output/README.md @@ -0,0 +1,101 @@ +# Release build-output companions + +`release-build-output-dispatch` records the exact files produced by a package +build and uploads a companion GitHub Actions artifact named +`release-build-output-`. Release tooling uses the +companion to associate primary artifacts with their build identity and +available evidence without reconstructing the build later. + +The companion contains: + +```text +. +├── release-build-metadata.json +├── release-build-output.json +└── release-evidence + ├── .provenance.json + └── .spdx.json +``` + +## Configuration + +Every producer passes one `config` JSON object. Its canonical schema and field +documentation are in [`config.schema.json`](config.schema.json). + +The `artifact_type` field determines how the action discovers primary files: + +- `conda` and `wheel` derive artifact descriptors and package identity from the + built packages; +- `custom` requires explicit artifact descriptors and exactly one of `package` + or `package_file`. + +`component_id` is a stable release-catalog identifier such as `conda:cudf` or +`maven:cuvs-java`; it is not a generated UUID. Standard RAPIDS Conda and wheel +workflows construct it as `:`. Custom producers +must supply the release unit selected for that package family. + +The action validates the complete configuration before inspecting build +outputs. It then verifies properties that depend on produced files, including +whether package, primary-artifact, and evidence paths resolve unambiguously. + +## Source revision + +`source-sha` must identify the repository revision actually checked out and +built. RAPIDS shared workflows pass `${{ env.RAPIDS_SHA }}`: + +- Conda build workflows set `RAPIDS_SHA` to `git rev-parse HEAD` immediately + after checkout. +- Wheel and custom workflows use `rapids-github-info`; it uses `inputs.sha` + when supplied and otherwise sets `RAPIDS_SHA` to `git rev-parse HEAD`. + +This distinction matters when a reusable workflow checks out a repository or +revision different from the workflow event. Direct callers should likewise +resolve the checked-out commit instead of assuming `${{ github.sha }}` names +the built source. + +## Standard package example + +```yaml +- name: Create wheel release build-output companion + uses: rapidsai/shared-actions/release-build-output-dispatch@main + with: + config: >- + { + "artifact_type": "wheel", + "component_id": "wheel:example", + "output_directory": ${{ toJSON(steps.package-name.outputs.WHEEL_OUTPUT_DIR) }} + } + source-artifact-name: ${{ steps.package-name.outputs.RAPIDS_PACKAGE_NAME }} + source-sha: ${{ env.RAPIDS_SHA }} +``` + +## Custom package example + +```yaml +- name: Create custom release build-output companion + uses: rapidsai/shared-actions/release-build-output-dispatch@main + with: + config: >- + { + "artifact_type": "custom", + "component_id": "maven:cuvs-java", + "output_directory": "java/cuvs-java/target", + "package_file": "cuvs-java.release-package.json", + "artifacts": [{"path": "cuvs-java-*-x86_64-cuda*.jar"}] + } + source-artifact-name: cuvs-java + source-sha: ${{ env.RAPIDS_SHA }} +``` + +## Evidence semantics + +A descriptor-selected producer SBOM is classified as `producer-dependency`. +It is evidence supplied by the producer and may contain a dependency inventory. + +When no SBOM is selected, the action generates an SPDX artifact-identity +envelope classified as `generated-identity`. It records package identity and +the primary artifact SHA-256, but contains no dependency or source-license +inventory. It must not be reported as producer-supplied dependency coverage. + +Producer-supplied SBOM, provenance, and signature sidecars are copied under +`release-evidence/` so the companion remains independently consumable. From 283e97d75c4eb4776156dff1804aac0c4e527b26 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 17 Aug 2026 21:20:24 -0500 Subject: [PATCH 04/36] Use release catalog key terminology --- .github/workflows/pr.yml | 4 +-- .pre-commit-config.yaml | 9 ++++-- release-build-output-dispatch/action.yml | 5 ++-- release-build-output/README.md | 14 ++++++---- release-build-output/config.schema.json | 6 ++-- release-build-output/materialize.sh | 16 +++++------ release-build-output/validate-config.sh | 8 +++--- .../inline-package.json | 2 +- .../package-file.json | 2 +- tests/release_build_output_config_test.sh | 24 ++++++++-------- tests/release_build_output_test.sh | 28 +++++++++---------- 11 files changed, 61 insertions(+), 57 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 4c797303..8cbc2db5 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -39,7 +39,7 @@ jobs: config: >- { "artifact_type": "custom", - "component_id": "archive:smoke", + "release_catalog_key": "archive:smoke", "output_directory": "release-build-output-smoke", "package": {"ecosystem": "archive", "name": "smoke", "version": "1.0"}, "artifacts": [{"path": "package.tar.gz"}] @@ -49,7 +49,7 @@ jobs: - name: Verify release build-output dispatch smoke test run: | jq -e ' - .artifacts[0].unit_id == "archive:smoke" + .artifacts[0].release_catalog_key == "archive:smoke" and .artifacts[0].path == "package.tar.gz" ' release-build-output-smoke/release-build-output.json >/dev/null jq -e ' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4a7f1db5..92f9986c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,9 +3,12 @@ ci: autoupdate_schedule: quarterly - # The runtime validator suite requires jq. pre-commit.ci validates the same - # valid fixtures against config.schema.json; GitHub Actions runs the full - # valid/invalid runtime suite on an Ubuntu runner with jq. + # These hooks require tools unavailable in pre-commit.ci: + # + # * actionlint-docker requires Docker + # * release-build-output-config requires jq + # + # Both run in other CI jobs with controlled runtime dependencies. skip: [actionlint-docker, release-build-output-config] repos: diff --git a/release-build-output-dispatch/action.yml b/release-build-output-dispatch/action.yml index 85cf7e97..9af3d5de 100644 --- a/release-build-output-dispatch/action.yml +++ b/release-build-output-dispatch/action.yml @@ -4,8 +4,7 @@ description: Create a release build-output companion from the selected shared-ac inputs: config: description: >- - Release build-output JSON object containing artifact_type, component_id, output_directory, and custom artifact - selection and package identity when artifact_type is custom. Schema and field documentation: + JSON string with configuration for this action. Schema and field documentation: https://github.com/rapidsai/shared-actions/blob/main/release-build-output/config.schema.json required: true source-artifact-name: @@ -79,7 +78,7 @@ runs: RELEASE_PACKAGE_FILE: ${{ steps.configuration.outputs.package_file }} RELEASE_SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} RELEASE_SOURCE_SHA: ${{ inputs.source-sha || github.sha }} - RELEASE_UNIT: ${{ steps.configuration.outputs.component_id }} + RELEASE_CATALOG_KEY: ${{ steps.configuration.outputs.release_catalog_key }} run: "${IMPLEMENTATION_PATH}/materialize.sh" - id: companion-name name: Set companion artifact name diff --git a/release-build-output/README.md b/release-build-output/README.md index 8f6d6935..0899b292 100644 --- a/release-build-output/README.md +++ b/release-build-output/README.md @@ -29,10 +29,12 @@ The `artifact_type` field determines how the action discovers primary files: - `custom` requires explicit artifact descriptors and exactly one of `package` or `package_file`. -`component_id` is a stable release-catalog identifier such as `conda:cudf` or -`maven:cuvs-java`; it is not a generated UUID. Standard RAPIDS Conda and wheel -workflows construct it as `:`. Custom producers -must supply the release unit selected for that package family. +`release_catalog_key` identifies the release-catalog entry that owns these +artifacts. Every file and matrix variant in the same publishable artifact set +uses the same key. Standard RAPIDS Conda and wheel workflows construct it as +`:`, such as `conda:cudf`; custom producers select +an existing catalog key, such as `maven:cuvs-java`. Do not generate a UUID or a +per-build value. The action validates the complete configuration before inspecting build outputs. It then verifies properties that depend on produced files, including @@ -62,7 +64,7 @@ the built source. config: >- { "artifact_type": "wheel", - "component_id": "wheel:example", + "release_catalog_key": "wheel:example", "output_directory": ${{ toJSON(steps.package-name.outputs.WHEEL_OUTPUT_DIR) }} } source-artifact-name: ${{ steps.package-name.outputs.RAPIDS_PACKAGE_NAME }} @@ -78,7 +80,7 @@ the built source. config: >- { "artifact_type": "custom", - "component_id": "maven:cuvs-java", + "release_catalog_key": "maven:cuvs-java", "output_directory": "java/cuvs-java/target", "package_file": "cuvs-java.release-package.json", "artifacts": [{"path": "cuvs-java-*-x86_64-cuda*.jar"}] diff --git a/release-build-output/config.schema.json b/release-build-output/config.schema.json index 81d37d6f..9efd2ba6 100644 --- a/release-build-output/config.schema.json +++ b/release-build-output/config.schema.json @@ -7,7 +7,7 @@ "additionalProperties": false, "required": [ "artifact_type", - "component_id" + "release_catalog_key" ], "properties": { "artifact_type": { @@ -19,8 +19,8 @@ "wheel" ] }, - "component_id": { - "description": "Stable release component ID shared by the selected files and their matrix variants.", + "release_catalog_key": { + "description": "Key of the release-catalog entry that owns these artifacts. Reuse it for every file and matrix variant in the same publishable artifact set. Standard RAPIDS workflows use :, such as conda:cudf; custom producers select an existing catalog key, such as maven:cuvs-java. This is not a generated UUID or per-build value.", "type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$" diff --git a/release-build-output/materialize.sh b/release-build-output/materialize.sh index 673dd6ba..a500e744 100755 --- a/release-build-output/materialize.sh +++ b/release-build-output/materialize.sh @@ -12,7 +12,7 @@ require_nonempty() { fi } -require_nonempty "RELEASE_UNIT" "${RELEASE_UNIT:-}" +require_nonempty "RELEASE_CATALOG_KEY" "${RELEASE_CATALOG_KEY:-}" require_nonempty "RELEASE_OUTPUT_DIRECTORY" "${RELEASE_OUTPUT_DIRECTORY:-}" require_nonempty "RELEASE_MANIFEST_NAME" "${RELEASE_MANIFEST_NAME:-}" require_nonempty "RELEASE_METADATA_NAME" "${RELEASE_METADATA_NAME:-}" @@ -248,7 +248,7 @@ write_generated_provenance() { predicate: { buildDefinition: { buildType: "https://rapids.ai/release-platform/build-output/v1", - externalParameters: {release_unit: env.RELEASE_UNIT, package: $package}, + externalParameters: {release_catalog_key: env.RELEASE_CATALOG_KEY, package: $package}, resolvedDependencies: [{ uri: ("git+https://github.com/" + $repository + "@" + $source_sha), digest: {gitCommit: $source_sha} @@ -310,12 +310,12 @@ while IFS= read -r descriptor; do write_generated_provenance "${primary_path}" "${package}" "${provenance_path}" fi artifact="$(jq -cn \ - --arg unit_id "${RELEASE_UNIT}" \ + --arg release_catalog_key "${RELEASE_CATALOG_KEY}" \ --arg path "${primary_path}" \ --arg sbom "${sbom_path}" \ --arg provenance "${provenance_path}" \ --argjson package "${package}" \ - '{unit_id: $unit_id, path: $path, sbom: $sbom, provenance: $provenance, package: $package}')" + '{release_catalog_key: $release_catalog_key, path: $path, sbom: $sbom, provenance: $provenance, package: $package}')" if [[ -n "${signature_pattern}" ]]; then supplied_signature_path="$(resolve_one_file signature "${signature_pattern}")" signature_path="$(copy_supplied_evidence "${primary_path}" "${supplied_signature_path}" "signature")" @@ -332,8 +332,8 @@ while IFS= read -r descriptor; do mv "${temporary_manifest}.next" "${temporary_manifest}" done < <(jq -c '.[]' <<<"${RELEASE_ARTIFACTS}") -if ! jq -e '.artifacts as $items | ($items | map([.unit_id, .path] | join("\u0000")) | unique | length) == ($items | length)' "${temporary_manifest}" >/dev/null; then - echo "release-artifacts contains duplicate unit/path entries" >&2 +if ! jq -e '.artifacts as $items | ($items | map([.release_catalog_key, .path] | join("\u0000")) | unique | length) == ($items | length)' "${temporary_manifest}" >/dev/null; then + echo "release-artifacts contains duplicate release_catalog_key/path entries" >&2 exit 1 fi @@ -345,13 +345,13 @@ jq -n -S \ --arg run_attempt "${GITHUB_RUN_ATTEMPT:-}" \ --arg run_id "${GITHUB_RUN_ID:-}" \ --arg sha "${source_sha}" \ - --arg unit_id "${RELEASE_UNIT}" \ + --arg release_catalog_key "${RELEASE_CATALOG_KEY}" \ --arg workflow_ref "${GITHUB_WORKFLOW_REF:-}" \ --argjson artifact_metadata "${artifact_metadata}" \ '{ schema_version: 1, producer: "shared-workflows", - release_unit: $unit_id, + release_catalog_key: $release_catalog_key, source_artifact: $artifact_name, build_output_manifest: $manifest_name, build_environment: { diff --git a/release-build-output/validate-config.sh b/release-build-output/validate-config.sh index 458f4e5e..bf7aa946 100755 --- a/release-build-output/validate-config.sh +++ b/release-build-output/validate-config.sh @@ -38,15 +38,15 @@ validation_errors="$(jq -r ' if type != "object" then ["release-build-output must be a JSON object"] else - (keys - ["artifact_type", "component_id", "output_directory", "package", "package_file", "artifacts"]) as $unknown + (keys - ["artifact_type", "release_catalog_key", "output_directory", "package", "package_file", "artifacts"]) as $unknown | [ if ($unknown | length) > 0 then "unknown field(s): " + ($unknown | join(", ")) else empty end, if (.artifact_type == "conda" or .artifact_type == "custom" or .artifact_type == "wheel") then empty else "artifact_type must be one of: conda, custom, wheel" end, - if (.component_id | single_line_string) then empty - else "component_id must be a non-empty, single-line string" end, + if (.release_catalog_key | single_line_string) then empty + else "release_catalog_key must be a non-empty, single-line string" end, if (has("output_directory") | not) or (.output_directory | single_line_string) then empty else "output_directory must be a non-empty, single-line string when supplied" end ] @@ -111,7 +111,7 @@ fi { printf 'artifact_type=%s\n' "$(jq -r '.artifact_type' <<<"${compact_config}")" - printf 'component_id=%s\n' "$(jq -r '.component_id' <<<"${compact_config}")" + printf 'release_catalog_key=%s\n' "$(jq -r '.release_catalog_key' <<<"${compact_config}")" printf 'output_directory=%s\n' "$(jq -r '.output_directory // "."' <<<"${compact_config}")" printf 'package=%s\n' "$(jq -c '.package // empty' <<<"${compact_config}")" printf 'package_file=%s\n' "$(jq -r '.package_file // empty' <<<"${compact_config}")" diff --git a/tests/release-build-output-config/inline-package.json b/tests/release-build-output-config/inline-package.json index 3443ea3b..80d6b6d3 100644 --- a/tests/release-build-output-config/inline-package.json +++ b/tests/release-build-output-config/inline-package.json @@ -1,6 +1,6 @@ { "artifact_type": "custom", - "component_id": "archive:smoke", + "release_catalog_key": "archive:smoke", "package": { "ecosystem": "archive", "name": "smoke", diff --git a/tests/release-build-output-config/package-file.json b/tests/release-build-output-config/package-file.json index de25059e..67776f18 100644 --- a/tests/release-build-output-config/package-file.json +++ b/tests/release-build-output-config/package-file.json @@ -1,6 +1,6 @@ { "artifact_type": "custom", - "component_id": "maven:cuvs-java", + "release_catalog_key": "maven:cuvs-java", "output_directory": "java/cuvs-java/target", "package_file": "cuvs-java.release-package.json", "artifacts": [ diff --git a/tests/release_build_output_config_test.sh b/tests/release_build_output_config_test.sh index d77375e8..2f451c4b 100755 --- a/tests/release_build_output_config_test.sh +++ b/tests/release_build_output_config_test.sh @@ -30,7 +30,7 @@ valid_output="${temporary_directory}/valid.output" RELEASE_BUILD_OUTPUT_CONFIG="$(<"${repository_root}/tests/release-build-output-config/package-file.json")" \ GITHUB_OUTPUT="${valid_output}" "${validator}" -grep -Fx 'component_id=maven:cuvs-java' "${valid_output}" +grep -Fx 'release_catalog_key=maven:cuvs-java' "${valid_output}" grep -Fx 'artifact_type=custom' "${valid_output}" grep -Fx 'output_directory=java/cuvs-java/target' "${valid_output}" grep -Fx 'package=' "${valid_output}" @@ -47,7 +47,7 @@ grep -Fx 'package_file=' "${inline_output}" assert_invalid \ malformed-json \ - '{"component_id":' \ + '{"release_catalog_key":' \ 'release-build-output must be valid JSON' assert_invalid \ @@ -58,50 +58,50 @@ assert_invalid \ assert_invalid \ missing-fields \ '{}' \ - 'component_id must be a non-empty, single-line string' + 'release_catalog_key must be a non-empty, single-line string' grep -F 'artifact_type must be one of: conda, custom, wheel' "${temporary_directory}/missing-fields.error" >/dev/null assert_invalid \ custom-missing-fields \ - '{"artifact_type":"custom","component_id":"archive:smoke"}' \ + '{"artifact_type":"custom","release_catalog_key":"archive:smoke"}' \ 'exactly one of package or package_file is required for custom artifacts' grep -F 'artifacts must be a non-empty array for custom artifacts' "${temporary_directory}/custom-missing-fields.error" >/dev/null assert_invalid \ unknown-field \ - '{"artifact_type":"custom","component_id":"archive:smoke","package_file":"package.json","artifacts":[{"path":"smoke.tar.gz"}],"unit":"archive:smoke"}' \ - 'unknown field(s): unit' + '{"artifact_type":"custom","release_catalog_key":"archive:smoke","package_file":"package.json","artifacts":[{"path":"smoke.tar.gz"}],"component_id":"archive:smoke"}' \ + 'unknown field(s): component_id' assert_invalid \ conflicting-package-source \ - '{"artifact_type":"custom","component_id":"archive:smoke","package":{"ecosystem":"archive","name":"smoke"},"package_file":"package.json","artifacts":[{"path":"smoke.tar.gz"}]}' \ + '{"artifact_type":"custom","release_catalog_key":"archive:smoke","package":{"ecosystem":"archive","name":"smoke"},"package_file":"package.json","artifacts":[{"path":"smoke.tar.gz"}]}' \ 'exactly one of package or package_file is required for custom artifacts' assert_invalid \ malformed-artifact \ - '{"artifact_type":"custom","component_id":"archive:smoke","package_file":"package.json","artifacts":[{"file":"smoke.tar.gz","sbom":false}]}' \ + '{"artifact_type":"custom","release_catalog_key":"archive:smoke","package_file":"package.json","artifacts":[{"file":"smoke.tar.gz","sbom":false}]}' \ 'artifacts[0] has unknown field(s): file' grep -F 'artifacts[0].path must be a non-empty, single-line string' "${temporary_directory}/malformed-artifact.error" >/dev/null grep -F 'artifacts[0].sbom must be a non-empty, single-line string' "${temporary_directory}/malformed-artifact.error" >/dev/null assert_invalid \ standard-missing-output-directory \ - '{"artifact_type":"conda","component_id":"conda:smoke"}' \ + '{"artifact_type":"conda","release_catalog_key":"conda:smoke"}' \ 'output_directory is required for conda and wheel artifacts' assert_invalid \ standard-custom-fields \ - '{"artifact_type":"wheel","component_id":"wheel:smoke","output_directory":"dist","artifacts":[{"path":"smoke.whl"}]}' \ + '{"artifact_type":"wheel","release_catalog_key":"wheel:smoke","output_directory":"dist","artifacts":[{"path":"smoke.whl"}]}' \ 'artifacts, package, and package_file are only valid when artifact_type is custom' standard_output="${temporary_directory}/standard.output" RELEASE_BUILD_OUTPUT_CONFIG='{ "artifact_type": "wheel", - "component_id": "wheel:kvikio", + "release_catalog_key": "wheel:kvikio", "output_directory": "dist" }' GITHUB_OUTPUT="${standard_output}" "${validator}" grep -Fx 'artifact_type=wheel' "${standard_output}" -grep -Fx 'component_id=wheel:kvikio' "${standard_output}" +grep -Fx 'release_catalog_key=wheel:kvikio' "${standard_output}" grep -Fx 'output_directory=dist' "${standard_output}" grep -Fx 'artifacts=' "${standard_output}" diff --git a/tests/release_build_output_test.sh b/tests/release_build_output_test.sh index 2a5dc6e1..bdbf8320 100755 --- a/tests/release_build_output_test.sh +++ b/tests/release_build_output_test.sh @@ -32,10 +32,10 @@ RELEASE_PACKAGE='' RELEASE_PACKAGE_FILE="cuvs-java-package.json" RELEASE_SOURCE_ARTIFACT_NAME="cuvs-java-cuda12.9.1" RELEASE_SOURCE_SHA="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" -RELEASE_UNIT="maven:cuvs-java" +RELEASE_CATALOG_KEY="maven:cuvs-java" export GITHUB_REPOSITORY GITHUB_RUN_ATTEMPT GITHUB_RUN_ID GITHUB_SHA GITHUB_WORKFLOW_REF export RELEASE_ARTIFACTS RELEASE_MANIFEST_NAME RELEASE_METADATA_NAME RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE -export RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_UNIT +export RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY "${repository_root}/release-build-output/materialize.sh" @@ -46,14 +46,14 @@ jq -e ' .schema_version == 1 and .producer == "release-platform" and (.artifacts | length == 1) - and .artifacts[0].unit_id == "maven:cuvs-java" + and .artifacts[0].release_catalog_key == "maven:cuvs-java" and .artifacts[0].path == "cuvs-java-26.08.0.jar" and .artifacts[0].package.name == "ai.rapids:cuvs-java" ' "${manifest_path}" >/dev/null jq -e ' .schema_version == 1 and .producer == "shared-workflows" - and .release_unit == "maven:cuvs-java" + and .release_catalog_key == "maven:cuvs-java" and .source_artifact == "cuvs-java-cuda12.9.1" and .build_output_manifest == "release-build-output.json" and .build_environment.repository == "rapidsai/cuvs" @@ -91,8 +91,8 @@ RELEASE_PACKAGE="$(jq -cn '{ecosystem: "conda", name: "kvikio"}')" RELEASE_PACKAGE_FILE='' RELEASE_SOURCE_ARTIFACT_NAME="kvikio_conda_python_kvikio_x86_64_abi3_cu12" RELEASE_SOURCE_SHA="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" -RELEASE_UNIT="conda:kvikio" -export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_UNIT +RELEASE_CATALOG_KEY="conda:kvikio" +export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY "${repository_root}/release-build-output/materialize.sh" @@ -101,7 +101,7 @@ generated_metadata_path="${generated_directory}/release-build-metadata.json" generated_sbom_path="$(jq -r '.artifacts[0].sbom' "${generated_manifest_path}")" generated_provenance_path="$(jq -r '.artifacts[0].provenance' "${generated_manifest_path}")" jq -e ' - .artifacts[0].unit_id == "conda:kvikio" + .artifacts[0].release_catalog_key == "conda:kvikio" and .artifacts[0].path == "linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.conda" and .artifacts[0].package == {ecosystem: "conda", name: "kvikio", version: "26.08.00a32"} ' "${generated_manifest_path}" >/dev/null @@ -112,11 +112,11 @@ jq -e ' ' "${generated_directory}/${generated_sbom_path}" >/dev/null jq -e ' .predicateType == "https://slsa.dev/provenance/v1" - and .predicate.buildDefinition.externalParameters.release_unit == "conda:kvikio" + and .predicate.buildDefinition.externalParameters.release_catalog_key == "conda:kvikio" and .predicate.buildDefinition.resolvedDependencies[0].digest.gitCommit == "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ' "${generated_directory}/${generated_provenance_path}" >/dev/null jq -e ' - .release_unit == "conda:kvikio" + .release_catalog_key == "conda:kvikio" and .source_artifact == "kvikio_conda_python_kvikio_x86_64_abi3_cu12" and .build_environment.sha == "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" and .metadata.artifacts == [{path: "linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.conda", sbom_kind: "generated-identity"}] @@ -131,13 +131,13 @@ RELEASE_OUTPUT_DIRECTORY="${wheel_directory}" RELEASE_PACKAGE="$(jq -cn '{ecosystem: "wheel", name: "libkvikio-cu12"}')" RELEASE_SOURCE_ARTIFACT_NAME="kvikio_wheel_cpp_libkvikio_x86_64_cu12" RELEASE_SOURCE_SHA="cccccccccccccccccccccccccccccccccccccccc" -RELEASE_UNIT="wheel:kvikio" -export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_UNIT +RELEASE_CATALOG_KEY="wheel:kvikio" +export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY "${repository_root}/release-build-output/materialize.sh" jq -e ' - .artifacts[0].unit_id == "wheel:kvikio" + .artifacts[0].release_catalog_key == "wheel:kvikio" and .artifacts[0].path == "libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl" and .artifacts[0].package == {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"} ' "${wheel_directory}/release-build-output.json" >/dev/null @@ -155,8 +155,8 @@ RELEASE_PACKAGE="$(jq -cn '{ecosystem: "archive", name: "bundle"}')" RELEASE_PACKAGE_FILE='' RELEASE_SOURCE_ARTIFACT_NAME="bundle-archive" RELEASE_SOURCE_SHA="dddddddddddddddddddddddddddddddddddddddd" -RELEASE_UNIT="archive:bundle" -export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_UNIT +RELEASE_CATALOG_KEY="archive:bundle" +export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY if "${repository_root}/release-build-output/materialize.sh" 2>"${temporary_directory}/missing-version-error"; then echo "materialize.sh unexpectedly accepted a custom artifact without a package version" >&2 From dee223bbb89d768cf821aec7951c96b851238c3f Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 17 Aug 2026 21:40:07 -0500 Subject: [PATCH 05/36] Rename release catalog feature --- .github/workflows/pr.yml | 30 +++++++++---------- .pre-commit-config.yaml | 18 +++++------ README.md | 8 ++--- .../action.yml | 30 +++++++++---------- .../README.md | 18 +++++------ .../config.schema.json | 6 ++-- .../describe-conda.sh | 0 .../describe-wheels.sh | 0 .../materialize.sh | 8 ++--- .../prepare.sh | 0 .../validate-config.sh | 10 +++---- .../inline-package.json | 0 .../package-file.json | 0 ...test.sh => release_catalog_config_test.sh} | 14 ++++----- ...sh => release_catalog_descriptors_test.sh} | 4 +-- ...est.sh => release_catalog_prepare_test.sh} | 6 ++-- ...output_test.sh => release_catalog_test.sh} | 28 ++++++++--------- 17 files changed, 90 insertions(+), 90 deletions(-) rename {release-build-output-dispatch => release-catalog-dispatch}/action.yml (81%) rename {release-build-output => release-catalog}/README.md (87%) rename {release-build-output => release-catalog}/config.schema.json (95%) rename {release-build-output => release-catalog}/describe-conda.sh (100%) rename {release-build-output => release-catalog}/describe-wheels.sh (100%) rename {release-build-output => release-catalog}/materialize.sh (98%) rename {release-build-output => release-catalog}/prepare.sh (100%) rename {release-build-output => release-catalog}/validate-config.sh (93%) rename tests/{release-build-output-config => release-catalog-config}/inline-package.json (100%) rename tests/{release-build-output-config => release-catalog-config}/package-file.json (100%) rename tests/{release_build_output_config_test.sh => release_catalog_config_test.sh} (88%) rename tests/{release_build_output_descriptors_test.sh => release_catalog_descriptors_test.sh} (92%) rename tests/{release_build_output_prepare_test.sh => release_catalog_prepare_test.sh} (90%) rename tests/{release_build_output_test.sh => release_catalog_test.sh} (89%) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 8cbc2db5..54a32531 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -20,18 +20,18 @@ jobs: fetch-depth: 1 persist-credentials: false - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 - - name: Test release build-output materialization + - name: Test release catalog materialization run: | - ./tests/release_build_output_config_test.sh - ./tests/release_build_output_descriptors_test.sh - ./tests/release_build_output_prepare_test.sh - ./tests/release_build_output_test.sh - - name: Prepare release build-output dispatch smoke test + ./tests/release_catalog_config_test.sh + ./tests/release_catalog_descriptors_test.sh + ./tests/release_catalog_prepare_test.sh + ./tests/release_catalog_test.sh + - name: Prepare release catalog dispatch smoke test run: | - mkdir -p release-build-output-smoke - printf '%s\n' smoke >release-build-output-smoke/package.tar.gz - - name: Run release build-output dispatch smoke test - uses: ./release-build-output-dispatch + mkdir -p release-catalog-smoke + printf '%s\n' smoke >release-catalog-smoke/package.tar.gz + - name: Run release catalog dispatch smoke test + uses: ./release-catalog-dispatch env: SHARED_ACTIONS_REPO: ${{ github.event.pull_request.head.repo.full_name }} SHARED_ACTIONS_REF: ${{ github.event.pull_request.head.sha }} @@ -40,18 +40,18 @@ jobs: { "artifact_type": "custom", "release_catalog_key": "archive:smoke", - "output_directory": "release-build-output-smoke", + "output_directory": "release-catalog-smoke", "package": {"ecosystem": "archive", "name": "smoke", "version": "1.0"}, "artifacts": [{"path": "package.tar.gz"}] } - source-artifact-name: release-build-output-dispatch-smoke + source-artifact-name: release-catalog-dispatch-smoke source-sha: ${{ github.event.pull_request.head.sha }} - - name: Verify release build-output dispatch smoke test + - name: Verify release catalog dispatch smoke test run: | jq -e ' .artifacts[0].release_catalog_key == "archive:smoke" and .artifacts[0].path == "package.tar.gz" - ' release-build-output-smoke/release-build-output.json >/dev/null + ' release-catalog-smoke/release-catalog-entries.json >/dev/null jq -e ' .metadata.artifacts == [{path: "package.tar.gz", sbom_kind: "generated-identity"}] - ' release-build-output-smoke/release-build-metadata.json >/dev/null + ' release-catalog-smoke/release-catalog-metadata.json >/dev/null diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 92f9986c..6cf849f3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,10 +6,10 @@ ci: # These hooks require tools unavailable in pre-commit.ci: # # * actionlint-docker requires Docker - # * release-build-output-config requires jq + # * release-catalog-config requires jq # # Both run in other CI jobs with controlled runtime dependencies. - skip: [actionlint-docker, release-build-output-config] + skip: [actionlint-docker, release-catalog-config] repos: - repo: https://github.com/pre-commit/pre-commit-hooks @@ -63,14 +63,14 @@ repos: rev: 0.37.2 hooks: - id: check-jsonschema - name: validate release build-output config schema fixtures - args: [--schemafile, release-build-output/config.schema.json] - files: ^tests/release-build-output-config/.*\.json$ + name: validate release catalog config schema fixtures + args: [--schemafile, release-catalog/config.schema.json] + files: ^tests/release-catalog-config/.*\.json$ - repo: local hooks: - - id: release-build-output-config - name: validate release build-output configuration - entry: ./tests/release_build_output_config_test.sh + - id: release-catalog-config + name: validate release catalog configuration + entry: ./tests/release_catalog_config_test.sh language: system pass_filenames: false - files: ^(release-build-output/(config\.schema\.json|validate-config\.sh)|tests/release_build_output_config_test\.sh)$ + files: ^(release-catalog/(config\.schema\.json|validate-config\.sh)|tests/release_catalog_config_test\.sh)$ diff --git a/README.md b/README.md index 697fd7f8..88248366 100644 --- a/README.md +++ b/README.md @@ -9,13 +9,13 @@ A dispatch action is one that: * clones the shared-actions repository (repo/ref changeable using env vars) * runs (dispatches to) another action within the clone, using a relative path -## Release build-output companions +## Release catalog companions -`release-build-output-dispatch` validates a producer's local build artifact +`release-catalog-dispatch` validates a producer's local build artifact directory and uploads a companion artifact named -`release-build-output-`. +`release-catalog-`. -See the [`release-build-output` documentation](release-build-output/README.md) +See the [release catalog documentation](release-catalog/README.md) for configuration, examples, source-revision handling, companion contents, and evidence semantics. diff --git a/release-build-output-dispatch/action.yml b/release-catalog-dispatch/action.yml similarity index 81% rename from release-build-output-dispatch/action.yml rename to release-catalog-dispatch/action.yml index 9af3d5de..b06951a5 100644 --- a/release-build-output-dispatch/action.yml +++ b/release-catalog-dispatch/action.yml @@ -1,11 +1,11 @@ -name: Dispatch release build output -description: Create a release build-output companion from the selected shared-actions revision. +name: Dispatch release catalog +description: Create a release catalog companion from the selected shared-actions revision. inputs: config: description: >- JSON string with configuration for this action. Schema and field documentation: - https://github.com/rapidsai/shared-actions/blob/main/release-build-output/config.schema.json + https://github.com/rapidsai/shared-actions/blob/main/release-catalog/config.schema.json required: true source-artifact-name: description: Name of the GitHub Actions artifact bundle containing this output. @@ -29,7 +29,7 @@ runs: using: composite steps: - id: implementation - name: Resolve release build-output implementation + name: Resolve release catalog implementation shell: bash env: ACTION_PATH: ${{ github.action_path }} @@ -41,18 +41,18 @@ runs: if [[ ! -d "${action_path}" && "${action_path}" == "${host_root}/"* ]]; then action_path="${container_root}/${action_path#"${host_root}/"}" fi - implementation_path="$(dirname "${action_path}")/release-build-output" + implementation_path="$(dirname "${action_path}")/release-catalog" if [[ ! -d "${implementation_path}" ]]; then - echo "release-build-output implementation not found at ${implementation_path}" >&2 + echo "release-catalog implementation not found at ${implementation_path}" >&2 exit 1 fi echo "path=${implementation_path}" >>"${GITHUB_OUTPUT}" - id: configuration - name: Validate release build-output configuration + name: Validate release catalog configuration shell: bash env: IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} - RELEASE_BUILD_OUTPUT_CONFIG: ${{ inputs.config }} + RELEASE_CATALOG_CONFIG: ${{ inputs.config }} run: "${IMPLEMENTATION_PATH}/validate-config.sh" - id: prepare name: Describe release artifacts @@ -66,13 +66,13 @@ runs: RELEASE_PACKAGE_FILE: ${{ steps.configuration.outputs.package_file }} run: "${IMPLEMENTATION_PATH}/prepare.sh" - id: materialize - name: Materialize release build-output records + name: Materialize release catalog records shell: bash env: IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} RELEASE_ARTIFACTS: ${{ steps.prepare.outputs.artifacts }} - RELEASE_MANIFEST_NAME: release-build-output.json - RELEASE_METADATA_NAME: release-build-metadata.json + RELEASE_MANIFEST_NAME: release-catalog-entries.json + RELEASE_METADATA_NAME: release-catalog-metadata.json RELEASE_OUTPUT_DIRECTORY: ${{ steps.configuration.outputs.output_directory }} RELEASE_PACKAGE: ${{ steps.prepare.outputs.package }} RELEASE_PACKAGE_FILE: ${{ steps.configuration.outputs.package_file }} @@ -85,13 +85,13 @@ runs: shell: bash env: SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} - run: echo "name=release-build-output-${SOURCE_ARTIFACT_NAME}" >>"${GITHUB_OUTPUT}" - - name: Upload release build-output companion + run: echo "name=release-catalog-${SOURCE_ARTIFACT_NAME}" >>"${GITHUB_OUTPUT}" + - name: Upload release catalog companion uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: if-no-files-found: error name: ${{ steps.companion-name.outputs.name }} path: | - ${{ steps.configuration.outputs.output_directory }}/release-build-output.json - ${{ steps.configuration.outputs.output_directory }}/release-build-metadata.json + ${{ steps.configuration.outputs.output_directory }}/release-catalog-entries.json + ${{ steps.configuration.outputs.output_directory }}/release-catalog-metadata.json ${{ steps.configuration.outputs.output_directory }}/release-evidence/** diff --git a/release-build-output/README.md b/release-catalog/README.md similarity index 87% rename from release-build-output/README.md rename to release-catalog/README.md index 0899b292..22cf77e2 100644 --- a/release-build-output/README.md +++ b/release-catalog/README.md @@ -1,8 +1,8 @@ -# Release build-output companions +# Release catalog companions -`release-build-output-dispatch` records the exact files produced by a package +`release-catalog-dispatch` records the exact files produced by a package build and uploads a companion GitHub Actions artifact named -`release-build-output-`. Release tooling uses the +`release-catalog-`. Release tooling uses the companion to associate primary artifacts with their build identity and available evidence without reconstructing the build later. @@ -10,8 +10,8 @@ The companion contains: ```text . -├── release-build-metadata.json -├── release-build-output.json +├── release-catalog-metadata.json +├── release-catalog-entries.json └── release-evidence ├── .provenance.json └── .spdx.json @@ -58,8 +58,8 @@ the built source. ## Standard package example ```yaml -- name: Create wheel release build-output companion - uses: rapidsai/shared-actions/release-build-output-dispatch@main +- name: Create wheel release catalog companion + uses: rapidsai/shared-actions/release-catalog-dispatch@main with: config: >- { @@ -74,8 +74,8 @@ the built source. ## Custom package example ```yaml -- name: Create custom release build-output companion - uses: rapidsai/shared-actions/release-build-output-dispatch@main +- name: Create custom release catalog companion + uses: rapidsai/shared-actions/release-catalog-dispatch@main with: config: >- { diff --git a/release-build-output/config.schema.json b/release-catalog/config.schema.json similarity index 95% rename from release-build-output/config.schema.json rename to release-catalog/config.schema.json index 9efd2ba6..6c6b77a3 100644 --- a/release-build-output/config.schema.json +++ b/release-catalog/config.schema.json @@ -1,8 +1,8 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://rapids.ai/schemas/release-build-output-config-v1.json", - "title": "Release build-output configuration", - "description": "Producer configuration for one release build-output companion.", + "$id": "https://rapids.ai/schemas/release-catalog-config-v1.json", + "title": "Release catalog configuration", + "description": "Producer configuration for one release catalog companion.", "type": "object", "additionalProperties": false, "required": [ diff --git a/release-build-output/describe-conda.sh b/release-catalog/describe-conda.sh similarity index 100% rename from release-build-output/describe-conda.sh rename to release-catalog/describe-conda.sh diff --git a/release-build-output/describe-wheels.sh b/release-catalog/describe-wheels.sh similarity index 100% rename from release-build-output/describe-wheels.sh rename to release-catalog/describe-wheels.sh diff --git a/release-build-output/materialize.sh b/release-catalog/materialize.sh similarity index 98% rename from release-build-output/materialize.sh rename to release-catalog/materialize.sh index a500e744..80b2b97a 100755 --- a/release-build-output/materialize.sh +++ b/release-catalog/materialize.sh @@ -92,7 +92,7 @@ fi manifest_path="${output_directory}/${RELEASE_MANIFEST_NAME}" metadata_path="${output_directory}/${RELEASE_METADATA_NAME}" -temporary_manifest="$(mktemp "${output_directory}/.release-build-output.XXXXXX")" +temporary_manifest="$(mktemp "${output_directory}/.release-catalog.XXXXXX")" trap 'rm -f "${temporary_manifest}"' EXIT printf '%s\n' '{"schema_version":1,"producer":"release-platform","artifacts":[]}' >"${temporary_manifest}" @@ -204,7 +204,7 @@ write_generated_sbom() { name: ("RAPIDS release artifact " + $artifact_path), documentNamespace: ("https://rapids.ai/release-platform/spdx/" + $artifact_digest), creationInfo: { - creators: ["Tool: rapidsai/shared-workflows release-build-output"], + creators: ["Tool: rapidsai/shared-workflows release catalog"], created: (now | strftime("%Y-%m-%dT%H:%M:%SZ")) }, documentDescribes: ["SPDXRef-Artifact"], @@ -247,7 +247,7 @@ write_generated_provenance() { predicateType: "https://slsa.dev/provenance/v1", predicate: { buildDefinition: { - buildType: "https://rapids.ai/release-platform/build-output/v1", + buildType: "https://rapids.ai/release-platform/catalog-record/v1", externalParameters: {release_catalog_key: env.RELEASE_CATALOG_KEY, package: $package}, resolvedDependencies: [{ uri: ("git+https://github.com/" + $repository + "@" + $source_sha), @@ -353,7 +353,7 @@ jq -n -S \ producer: "shared-workflows", release_catalog_key: $release_catalog_key, source_artifact: $artifact_name, - build_output_manifest: $manifest_name, + catalog_record_manifest: $manifest_name, build_environment: { repository: $repository, sha: $sha, diff --git a/release-build-output/prepare.sh b/release-catalog/prepare.sh similarity index 100% rename from release-build-output/prepare.sh rename to release-catalog/prepare.sh diff --git a/release-build-output/validate-config.sh b/release-catalog/validate-config.sh similarity index 93% rename from release-build-output/validate-config.sh rename to release-catalog/validate-config.sh index bf7aa946..12916159 100755 --- a/release-build-output/validate-config.sh +++ b/release-catalog/validate-config.sh @@ -5,18 +5,18 @@ set -euo pipefail emit_error() { local message="$1" - printf '::error title=Invalid release-build-output configuration::%s\n' "${message}" >&2 + printf '::error title=Invalid release catalog configuration::%s\n' "${message}" >&2 } -config="${RELEASE_BUILD_OUTPUT_CONFIG:-}" +config="${RELEASE_CATALOG_CONFIG:-}" if [[ -z "${config}" ]]; then - emit_error "release-build-output must be a non-empty JSON object" + emit_error "release catalog configuration must be a non-empty JSON object" exit 1 fi if ! compact_config="$(jq -ce . <<<"${config}" 2>/dev/null)"; then parse_error="$(jq -ce . <<<"${config}" 2>&1 || true)" - emit_error "release-build-output must be valid JSON: ${parse_error}" + emit_error "release catalog configuration must be valid JSON: ${parse_error}" exit 1 fi @@ -36,7 +36,7 @@ validation_errors="$(jq -r ' and (keys - ["ecosystem", "name", "version", "build", "platform"] | length == 0) and (to_entries | map(.value | single_line_string) | all); if type != "object" then - ["release-build-output must be a JSON object"] + ["release catalog configuration must be a JSON object"] else (keys - ["artifact_type", "release_catalog_key", "output_directory", "package", "package_file", "artifacts"]) as $unknown | [ diff --git a/tests/release-build-output-config/inline-package.json b/tests/release-catalog-config/inline-package.json similarity index 100% rename from tests/release-build-output-config/inline-package.json rename to tests/release-catalog-config/inline-package.json diff --git a/tests/release-build-output-config/package-file.json b/tests/release-catalog-config/package-file.json similarity index 100% rename from tests/release-build-output-config/package-file.json rename to tests/release-catalog-config/package-file.json diff --git a/tests/release_build_output_config_test.sh b/tests/release_catalog_config_test.sh similarity index 88% rename from tests/release_build_output_config_test.sh rename to tests/release_catalog_config_test.sh index 2f451c4b..36d92c7c 100755 --- a/tests/release_build_output_config_test.sh +++ b/tests/release_catalog_config_test.sh @@ -4,7 +4,7 @@ set -euo pipefail repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -validator="${repository_root}/release-build-output/validate-config.sh" +validator="${repository_root}/release-catalog/validate-config.sh" temporary_directory="$(mktemp -d)" trap 'rm -rf "${temporary_directory}"' EXIT @@ -14,7 +14,7 @@ assert_invalid() { local expected="$3" local error_file="${temporary_directory}/${name}.error" - if RELEASE_BUILD_OUTPUT_CONFIG="${config}" GITHUB_OUTPUT="${temporary_directory}/${name}.output" \ + if RELEASE_CATALOG_CONFIG="${config}" GITHUB_OUTPUT="${temporary_directory}/${name}.output" \ "${validator}" 2>"${error_file}"; then echo "validate-config.sh unexpectedly accepted ${name}" >&2 exit 1 @@ -27,7 +27,7 @@ assert_invalid() { } valid_output="${temporary_directory}/valid.output" -RELEASE_BUILD_OUTPUT_CONFIG="$(<"${repository_root}/tests/release-build-output-config/package-file.json")" \ +RELEASE_CATALOG_CONFIG="$(<"${repository_root}/tests/release-catalog-config/package-file.json")" \ GITHUB_OUTPUT="${valid_output}" "${validator}" grep -Fx 'release_catalog_key=maven:cuvs-java' "${valid_output}" @@ -38,7 +38,7 @@ grep -Fx 'package_file=cuvs-java.release-package.json' "${valid_output}" grep -Fx 'artifacts=[{"path":"cuvs-java-*-x86_64-cuda*.jar","sbom":"cuvs-java.spdx.json"}]' "${valid_output}" inline_output="${temporary_directory}/inline.output" -RELEASE_BUILD_OUTPUT_CONFIG="$(<"${repository_root}/tests/release-build-output-config/inline-package.json")" \ +RELEASE_CATALOG_CONFIG="$(<"${repository_root}/tests/release-catalog-config/inline-package.json")" \ GITHUB_OUTPUT="${inline_output}" "${validator}" grep -Fx 'output_directory=.' "${inline_output}" @@ -48,12 +48,12 @@ grep -Fx 'package_file=' "${inline_output}" assert_invalid \ malformed-json \ '{"release_catalog_key":' \ - 'release-build-output must be valid JSON' + 'release catalog configuration must be valid JSON' assert_invalid \ not-an-object \ '[]' \ - 'release-build-output must be a JSON object' + 'release catalog configuration must be a JSON object' assert_invalid \ missing-fields \ @@ -95,7 +95,7 @@ assert_invalid \ 'artifacts, package, and package_file are only valid when artifact_type is custom' standard_output="${temporary_directory}/standard.output" -RELEASE_BUILD_OUTPUT_CONFIG='{ +RELEASE_CATALOG_CONFIG='{ "artifact_type": "wheel", "release_catalog_key": "wheel:kvikio", "output_directory": "dist" diff --git a/tests/release_build_output_descriptors_test.sh b/tests/release_catalog_descriptors_test.sh similarity index 92% rename from tests/release_build_output_descriptors_test.sh rename to tests/release_catalog_descriptors_test.sh index c6b6c46c..1c608e51 100755 --- a/tests/release_build_output_descriptors_test.sh +++ b/tests/release_catalog_descriptors_test.sh @@ -20,7 +20,7 @@ printf '%s\n' \ zip -qr "${wheel_output_directory}/libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl" . ) -wheel_descriptors="$("${repository_root}/release-build-output/describe-wheels.sh" "${wheel_output_directory}")" +wheel_descriptors="$("${repository_root}/release-catalog/describe-wheels.sh" "${wheel_output_directory}")" jq -e ' . == [{ path: "libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl", @@ -57,7 +57,7 @@ zstd -q -f "${temporary_directory}/info-librmm.tar" -o "${temporary_directory}/i info-librmm.tar.zst ) -conda_descriptors="$("${repository_root}/release-build-output/describe-conda.sh" "${conda_output_directory}")" +conda_descriptors="$("${repository_root}/release-catalog/describe-conda.sh" "${conda_output_directory}")" jq -e ' length == 2 and any(.[]; diff --git a/tests/release_build_output_prepare_test.sh b/tests/release_catalog_prepare_test.sh similarity index 90% rename from tests/release_build_output_prepare_test.sh rename to tests/release_catalog_prepare_test.sh index 3598e1e5..a84eae03 100755 --- a/tests/release_build_output_prepare_test.sh +++ b/tests/release_catalog_prepare_test.sh @@ -26,7 +26,7 @@ RELEASE_PACKAGE='' RELEASE_PACKAGE_FILE='' export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE -"${repository_root}/release-build-output/prepare.sh" +"${repository_root}/release-catalog/prepare.sh" grep -Fx 'package={"ecosystem":"wheel","name":"bundle"}' "${GITHUB_OUTPUT}" artifacts="$(sed -n 's/^artifacts=//p' "${GITHUB_OUTPUT}")" @@ -44,7 +44,7 @@ RELEASE_PACKAGE='' RELEASE_PACKAGE_FILE=release-package.json export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_PACKAGE RELEASE_PACKAGE_FILE -"${repository_root}/release-build-output/prepare.sh" +"${repository_root}/release-catalog/prepare.sh" grep -Fx 'artifacts=[{"path":"bundle.tar.gz","sbom":"bundle.spdx.json"}]' "${GITHUB_OUTPUT}" grep -Fx 'package=' "${GITHUB_OUTPUT}" @@ -52,7 +52,7 @@ grep -Fx 'package=' "${GITHUB_OUTPUT}" GITHUB_OUTPUT="${temporary_directory}/invalid-output" RELEASE_ARTIFACT_TYPE=archive export GITHUB_OUTPUT RELEASE_ARTIFACT_TYPE -if "${repository_root}/release-build-output/prepare.sh" 2>"${temporary_directory}/invalid-error"; then +if "${repository_root}/release-catalog/prepare.sh" 2>"${temporary_directory}/invalid-error"; then echo "prepare.sh unexpectedly accepted an invalid artifact type" >&2 exit 1 fi diff --git a/tests/release_build_output_test.sh b/tests/release_catalog_test.sh similarity index 89% rename from tests/release_build_output_test.sh rename to tests/release_catalog_test.sh index bdbf8320..51d93836 100755 --- a/tests/release_build_output_test.sh +++ b/tests/release_catalog_test.sh @@ -25,8 +25,8 @@ GITHUB_SHA="0123456789012345678901234567890123456789" GITHUB_WORKFLOW_REF="rapidsai/cuvs/.github/workflows/build.yaml@refs/heads/release/26.08" export GITHUB_OUTPUT RELEASE_ARTIFACTS="$(jq -cn '[{path: "cuvs-java-*.jar", sbom: "cuvs-java-*.spdx.json", provenance: "cuvs-java-*.provenance.jsonl", signature: "cuvs-java-*.jar.asc"}]')" -RELEASE_MANIFEST_NAME="release-build-output.json" -RELEASE_METADATA_NAME="release-build-metadata.json" +RELEASE_MANIFEST_NAME="release-catalog-entries.json" +RELEASE_METADATA_NAME="release-catalog-metadata.json" RELEASE_OUTPUT_DIRECTORY="${bundle_directory}" RELEASE_PACKAGE='' RELEASE_PACKAGE_FILE="cuvs-java-package.json" @@ -37,11 +37,11 @@ export GITHUB_REPOSITORY GITHUB_RUN_ATTEMPT GITHUB_RUN_ID GITHUB_SHA GITHUB_WORK export RELEASE_ARTIFACTS RELEASE_MANIFEST_NAME RELEASE_METADATA_NAME RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE export RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY -"${repository_root}/release-build-output/materialize.sh" +"${repository_root}/release-catalog/materialize.sh" canonical_bundle_directory="$(realpath "${bundle_directory}")" -manifest_path="${canonical_bundle_directory}/release-build-output.json" -metadata_path="${canonical_bundle_directory}/release-build-metadata.json" +manifest_path="${canonical_bundle_directory}/release-catalog-entries.json" +metadata_path="${canonical_bundle_directory}/release-catalog-metadata.json" jq -e ' .schema_version == 1 and .producer == "release-platform" @@ -55,7 +55,7 @@ jq -e ' and .producer == "shared-workflows" and .release_catalog_key == "maven:cuvs-java" and .source_artifact == "cuvs-java-cuda12.9.1" - and .build_output_manifest == "release-build-output.json" + and .catalog_record_manifest == "release-catalog-entries.json" and .build_environment.repository == "rapidsai/cuvs" and .build_environment.sha == "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" and .metadata.artifacts == [{path: "cuvs-java-26.08.0.jar", sbom_kind: "producer-dependency"}] @@ -79,7 +79,7 @@ cp "${manifest_path}" "${metadata_path}" "${isolated_companion_directory}/" cp -R "${canonical_bundle_directory}/release-evidence" "${isolated_companion_directory}/" while IFS= read -r evidence_path; do test -f "${isolated_companion_directory}/${evidence_path}" -done < <(jq -r '.artifacts[] | .sbom, .provenance, (.signature // empty)' "${isolated_companion_directory}/release-build-output.json") +done < <(jq -r '.artifacts[] | .sbom, .provenance, (.signature // empty)' "${isolated_companion_directory}/release-catalog-entries.json") generated_directory="${temporary_directory}/generated-bundle" mkdir -p "${generated_directory}/linux-64" @@ -94,10 +94,10 @@ RELEASE_SOURCE_SHA="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" RELEASE_CATALOG_KEY="conda:kvikio" export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY -"${repository_root}/release-build-output/materialize.sh" +"${repository_root}/release-catalog/materialize.sh" -generated_manifest_path="${generated_directory}/release-build-output.json" -generated_metadata_path="${generated_directory}/release-build-metadata.json" +generated_manifest_path="${generated_directory}/release-catalog-entries.json" +generated_metadata_path="${generated_directory}/release-catalog-metadata.json" generated_sbom_path="$(jq -r '.artifacts[0].sbom' "${generated_manifest_path}")" generated_provenance_path="$(jq -r '.artifacts[0].provenance' "${generated_manifest_path}")" jq -e ' @@ -134,16 +134,16 @@ RELEASE_SOURCE_SHA="cccccccccccccccccccccccccccccccccccccccc" RELEASE_CATALOG_KEY="wheel:kvikio" export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY -"${repository_root}/release-build-output/materialize.sh" +"${repository_root}/release-catalog/materialize.sh" jq -e ' .artifacts[0].release_catalog_key == "wheel:kvikio" and .artifacts[0].path == "libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl" and .artifacts[0].package == {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"} -' "${wheel_directory}/release-build-output.json" >/dev/null +' "${wheel_directory}/release-catalog-entries.json" >/dev/null jq -e ' .metadata.artifacts == [{path: "libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl", sbom_kind: "generated-identity"}] -' "${wheel_directory}/release-build-metadata.json" >/dev/null +' "${wheel_directory}/release-catalog-metadata.json" >/dev/null missing_version_directory="${temporary_directory}/missing-version-bundle" mkdir -p "${missing_version_directory}" @@ -158,7 +158,7 @@ RELEASE_SOURCE_SHA="dddddddddddddddddddddddddddddddddddddddd" RELEASE_CATALOG_KEY="archive:bundle" export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY -if "${repository_root}/release-build-output/materialize.sh" 2>"${temporary_directory}/missing-version-error"; then +if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/missing-version-error"; then echo "materialize.sh unexpectedly accepted a custom artifact without a package version" >&2 exit 1 fi From 84d135e9782e142b1cd8f82e39d4ef4c786897ca Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 17 Aug 2026 22:07:44 -0500 Subject: [PATCH 06/36] clarify release catalog key uniqueness/reuse --- release-catalog/README.md | 23 ++++++++++++++++++----- release-catalog/config.schema.json | 2 +- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/release-catalog/README.md b/release-catalog/README.md index 22cf77e2..0a940f18 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -30,11 +30,24 @@ The `artifact_type` field determines how the action discovers primary files: or `package_file`. `release_catalog_key` identifies the release-catalog entry that owns these -artifacts. Every file and matrix variant in the same publishable artifact set -uses the same key. Standard RAPIDS Conda and wheel workflows construct it as -`:`, such as `conda:cudf`; custom producers select -an existing catalog key, such as `maven:cuvs-java`. Do not generate a UUID or a -per-build value. +artifacts. Every file from every matrix variant in the same publishable artifact +set uses the same key, and multiple files are aggregated. Standard RAPIDS Conda +and wheel workflows construct it as `:`, such as +`conda:cudf`; custom producers select an existing catalog key, such as +`maven:cuvs-java`. Do not generate a UUID or a per-build value. + +Use a distinct `release_catalog_key` only when the release catalog intentionally gives the +outputs different release policy. Examples include: + +* different versioning +* validation requirements +* dependency ordering +* publication destinations +* promotion strategy. + +Multiple package names from one repository do not by themselves justify separate +keys: for example, `cudf` and `dask-cudf` Conda packages remain part of +`conda:cudf` when they share one release policy. The action validates the complete configuration before inspecting build outputs. It then verifies properties that depend on produced files, including diff --git a/release-catalog/config.schema.json b/release-catalog/config.schema.json index 6c6b77a3..9883c7d7 100644 --- a/release-catalog/config.schema.json +++ b/release-catalog/config.schema.json @@ -20,7 +20,7 @@ ] }, "release_catalog_key": { - "description": "Key of the release-catalog entry that owns these artifacts. Reuse it for every file and matrix variant in the same publishable artifact set. Standard RAPIDS workflows use :, such as conda:cudf; custom producers select an existing catalog key, such as maven:cuvs-java. This is not a generated UUID or per-build value.", + "description": "Release-catalog entry name that owns these artifacts. Default for conda/wheels is :, such as conda:cudf. Custom producers specify their own, such as maven:cuvs-java.", "type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$" From 065ae84573e0e613d5473d8d3dfca4fa91188922 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 17 Aug 2026 23:07:38 -0500 Subject: [PATCH 07/36] merge release catalog metadata into entries --- .github/workflows/pr.yml | 10 ++-- release-catalog-dispatch/action.yml | 17 +++--- release-catalog/README.md | 9 +++- release-catalog/materialize.sh | 51 ++++++------------ tests/release_catalog_test.sh | 83 ++++++++++++----------------- 5 files changed, 68 insertions(+), 102 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 54a32531..2a578473 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -49,9 +49,9 @@ jobs: - name: Verify release catalog dispatch smoke test run: | jq -e ' - .artifacts[0].release_catalog_key == "archive:smoke" - and .artifacts[0].path == "package.tar.gz" + .producer == "shared-workflows" + and .source.artifact == "release-catalog-dispatch-smoke" + and .entries[0].release_catalog_key == "archive:smoke" + and .entries[0].path == "package.tar.gz" + and .entries[0].sbom_kind == "generated-identity" ' release-catalog-smoke/release-catalog-entries.json >/dev/null - jq -e ' - .metadata.artifacts == [{path: "package.tar.gz", sbom_kind: "generated-identity"}] - ' release-catalog-smoke/release-catalog-metadata.json >/dev/null diff --git a/release-catalog-dispatch/action.yml b/release-catalog-dispatch/action.yml index b06951a5..e95c3844 100644 --- a/release-catalog-dispatch/action.yml +++ b/release-catalog-dispatch/action.yml @@ -15,13 +15,10 @@ inputs: required: false outputs: - manifest-path: - description: Absolute path to the generated manifest. - value: ${{ steps.materialize.outputs.manifest-path }} - metadata-path: - description: Absolute path to the build metadata envelope. - value: ${{ steps.materialize.outputs.metadata-path }} - manifest-artifact-name: + entries-path: + description: Absolute path to the generated release catalog entries. + value: ${{ steps.materialize.outputs.entries-path }} + companion-artifact-name: description: Name of the uploaded GitHub Actions companion artifact. value: ${{ steps.companion-name.outputs.name }} @@ -66,13 +63,12 @@ runs: RELEASE_PACKAGE_FILE: ${{ steps.configuration.outputs.package_file }} run: "${IMPLEMENTATION_PATH}/prepare.sh" - id: materialize - name: Materialize release catalog records + name: Materialize release catalog entries shell: bash env: IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} RELEASE_ARTIFACTS: ${{ steps.prepare.outputs.artifacts }} - RELEASE_MANIFEST_NAME: release-catalog-entries.json - RELEASE_METADATA_NAME: release-catalog-metadata.json + RELEASE_ENTRIES_NAME: release-catalog-entries.json RELEASE_OUTPUT_DIRECTORY: ${{ steps.configuration.outputs.output_directory }} RELEASE_PACKAGE: ${{ steps.prepare.outputs.package }} RELEASE_PACKAGE_FILE: ${{ steps.configuration.outputs.package_file }} @@ -93,5 +89,4 @@ runs: name: ${{ steps.companion-name.outputs.name }} path: | ${{ steps.configuration.outputs.output_directory }}/release-catalog-entries.json - ${{ steps.configuration.outputs.output_directory }}/release-catalog-metadata.json ${{ steps.configuration.outputs.output_directory }}/release-evidence/** diff --git a/release-catalog/README.md b/release-catalog/README.md index 0a940f18..2c05c416 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -10,13 +10,18 @@ The companion contains: ```text . -├── release-catalog-metadata.json ├── release-catalog-entries.json └── release-evidence ├── .provenance.json └── .spdx.json ``` +`release-catalog-entries.json` is one atomic job-level envelope. Its `source` +object records the source artifact and build context, while its `entries` array +contains the release catalog entries produced by that job. The release platform +validates and aggregates entry arrays from selected builds into the release +catalog; there is no separate metadata document to keep synchronized. + ## Configuration Every producer passes one `config` JSON object. Its canonical schema and field @@ -29,7 +34,7 @@ The `artifact_type` field determines how the action discovers primary files: - `custom` requires explicit artifact descriptors and exactly one of `package` or `package_file`. -`release_catalog_key` identifies the release-catalog entry that owns these +`release_catalog_key` identifies the release catalog entry that owns these artifacts. Every file from every matrix variant in the same publishable artifact set uses the same key, and multiple files are aggregated. Standard RAPIDS Conda and wheel workflows construct it as `:`, such as diff --git a/release-catalog/materialize.sh b/release-catalog/materialize.sh index 80b2b97a..f800d03a 100755 --- a/release-catalog/materialize.sh +++ b/release-catalog/materialize.sh @@ -14,8 +14,7 @@ require_nonempty() { require_nonempty "RELEASE_CATALOG_KEY" "${RELEASE_CATALOG_KEY:-}" require_nonempty "RELEASE_OUTPUT_DIRECTORY" "${RELEASE_OUTPUT_DIRECTORY:-}" -require_nonempty "RELEASE_MANIFEST_NAME" "${RELEASE_MANIFEST_NAME:-}" -require_nonempty "RELEASE_METADATA_NAME" "${RELEASE_METADATA_NAME:-}" +require_nonempty "RELEASE_ENTRIES_NAME" "${RELEASE_ENTRIES_NAME:-}" require_nonempty "RELEASE_ARTIFACTS" "${RELEASE_ARTIFACTS:-}" require_nonempty "RELEASE_SOURCE_ARTIFACT_NAME" "${RELEASE_SOURCE_ARTIFACT_NAME:-}" @@ -40,12 +39,7 @@ require_plain_filename() { fi } -require_plain_filename "manifest-name" "${RELEASE_MANIFEST_NAME}" -require_plain_filename "metadata-name" "${RELEASE_METADATA_NAME}" -if [[ "${RELEASE_MANIFEST_NAME}" == "${RELEASE_METADATA_NAME}" ]]; then - echo "manifest-name and metadata-name must differ" >&2 - exit 1 -fi +require_plain_filename "entries-name" "${RELEASE_ENTRIES_NAME}" if [[ ! -d "${RELEASE_OUTPUT_DIRECTORY}" ]]; then echo "output-directory does not exist or is not a directory: ${RELEASE_OUTPUT_DIRECTORY}" >&2 @@ -90,12 +84,11 @@ if ! jq -e 'type == "array" and length > 0' <<<"${RELEASE_ARTIFACTS}" >/dev/null exit 1 fi -manifest_path="${output_directory}/${RELEASE_MANIFEST_NAME}" -metadata_path="${output_directory}/${RELEASE_METADATA_NAME}" +entries_path="${output_directory}/${RELEASE_ENTRIES_NAME}" temporary_manifest="$(mktemp "${output_directory}/.release-catalog.XXXXXX")" trap 'rm -f "${temporary_manifest}"' EXIT -printf '%s\n' '{"schema_version":1,"producer":"release-platform","artifacts":[]}' >"${temporary_manifest}" +printf '%s\n' '{"entries":[]}' >"${temporary_manifest}" resolve_one_file() { local field="$1" @@ -263,7 +256,6 @@ write_generated_provenance() { } shopt -s globstar nullglob -artifact_metadata='[]' while IFS= read -r descriptor; do if ! jq -e ' type == "object" @@ -309,59 +301,48 @@ while IFS= read -r descriptor; do provenance_path="$(generated_evidence_path "${primary_path}" "provenance")" write_generated_provenance "${primary_path}" "${package}" "${provenance_path}" fi - artifact="$(jq -cn \ + entry="$(jq -cn \ --arg release_catalog_key "${RELEASE_CATALOG_KEY}" \ --arg path "${primary_path}" \ --arg sbom "${sbom_path}" \ + --arg sbom_kind "${sbom_kind}" \ --arg provenance "${provenance_path}" \ --argjson package "${package}" \ - '{release_catalog_key: $release_catalog_key, path: $path, sbom: $sbom, provenance: $provenance, package: $package}')" + '{release_catalog_key: $release_catalog_key, path: $path, sbom: $sbom, sbom_kind: $sbom_kind, provenance: $provenance, package: $package}')" if [[ -n "${signature_pattern}" ]]; then supplied_signature_path="$(resolve_one_file signature "${signature_pattern}")" signature_path="$(copy_supplied_evidence "${primary_path}" "${supplied_signature_path}" "signature")" - artifact="$(jq -c --arg signature "${signature_path}" '. + {signature: $signature}' <<<"${artifact}")" + entry="$(jq -c --arg signature "${signature_path}" '. + {signature: $signature}' <<<"${entry}")" fi - artifact_metadata="$(jq -cn \ - --arg path "${primary_path}" \ - --arg sbom_kind "${sbom_kind}" \ - --argjson artifacts "${artifact_metadata}" \ - '$artifacts + [{path: $path, sbom_kind: $sbom_kind}]')" - - jq --argjson artifact "${artifact}" '.artifacts += [$artifact]' "${temporary_manifest}" >"${temporary_manifest}.next" + jq --argjson entry "${entry}" '.entries += [$entry]' "${temporary_manifest}" >"${temporary_manifest}.next" mv "${temporary_manifest}.next" "${temporary_manifest}" done < <(jq -c '.[]' <<<"${RELEASE_ARTIFACTS}") -if ! jq -e '.artifacts as $items | ($items | map([.release_catalog_key, .path] | join("\u0000")) | unique | length) == ($items | length)' "${temporary_manifest}" >/dev/null; then +if ! jq -e '.entries as $items | ($items | map([.release_catalog_key, .path] | join("\u0000")) | unique | length) == ($items | length)' "${temporary_manifest}" >/dev/null; then echo "release-artifacts contains duplicate release_catalog_key/path entries" >&2 exit 1 fi -jq -S . "${temporary_manifest}" >"${manifest_path}" jq -n -S \ --arg artifact_name "${RELEASE_SOURCE_ARTIFACT_NAME}" \ - --arg manifest_name "${RELEASE_MANIFEST_NAME}" \ --arg repository "${GITHUB_REPOSITORY:-}" \ --arg run_attempt "${GITHUB_RUN_ATTEMPT:-}" \ --arg run_id "${GITHUB_RUN_ID:-}" \ --arg sha "${source_sha}" \ - --arg release_catalog_key "${RELEASE_CATALOG_KEY}" \ --arg workflow_ref "${GITHUB_WORKFLOW_REF:-}" \ - --argjson artifact_metadata "${artifact_metadata}" \ + --argjson entries "$(jq -c '.entries' "${temporary_manifest}")" \ '{ schema_version: 1, producer: "shared-workflows", - release_catalog_key: $release_catalog_key, - source_artifact: $artifact_name, - catalog_record_manifest: $manifest_name, - build_environment: { + source: { + artifact: $artifact_name, repository: $repository, sha: $sha, workflow_ref: $workflow_ref, run_id: $run_id, run_attempt: $run_attempt }, - metadata: {artifacts: $artifact_metadata} - }' >"${metadata_path}" -echo "manifest-path=${manifest_path}" >>"${GITHUB_OUTPUT}" -echo "metadata-path=${metadata_path}" >>"${GITHUB_OUTPUT}" + entries: $entries + }' >"${entries_path}" +echo "entries-path=${entries_path}" >>"${GITHUB_OUTPUT}" diff --git a/tests/release_catalog_test.sh b/tests/release_catalog_test.sh index 51d93836..a390b98c 100755 --- a/tests/release_catalog_test.sh +++ b/tests/release_catalog_test.sh @@ -25,8 +25,7 @@ GITHUB_SHA="0123456789012345678901234567890123456789" GITHUB_WORKFLOW_REF="rapidsai/cuvs/.github/workflows/build.yaml@refs/heads/release/26.08" export GITHUB_OUTPUT RELEASE_ARTIFACTS="$(jq -cn '[{path: "cuvs-java-*.jar", sbom: "cuvs-java-*.spdx.json", provenance: "cuvs-java-*.provenance.jsonl", signature: "cuvs-java-*.jar.asc"}]')" -RELEASE_MANIFEST_NAME="release-catalog-entries.json" -RELEASE_METADATA_NAME="release-catalog-metadata.json" +RELEASE_ENTRIES_NAME="release-catalog-entries.json" RELEASE_OUTPUT_DIRECTORY="${bundle_directory}" RELEASE_PACKAGE='' RELEASE_PACKAGE_FILE="cuvs-java-package.json" @@ -34,38 +33,31 @@ RELEASE_SOURCE_ARTIFACT_NAME="cuvs-java-cuda12.9.1" RELEASE_SOURCE_SHA="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" RELEASE_CATALOG_KEY="maven:cuvs-java" export GITHUB_REPOSITORY GITHUB_RUN_ATTEMPT GITHUB_RUN_ID GITHUB_SHA GITHUB_WORKFLOW_REF -export RELEASE_ARTIFACTS RELEASE_MANIFEST_NAME RELEASE_METADATA_NAME RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE +export RELEASE_ARTIFACTS RELEASE_ENTRIES_NAME RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE export RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" canonical_bundle_directory="$(realpath "${bundle_directory}")" -manifest_path="${canonical_bundle_directory}/release-catalog-entries.json" -metadata_path="${canonical_bundle_directory}/release-catalog-metadata.json" -jq -e ' - .schema_version == 1 - and .producer == "release-platform" - and (.artifacts | length == 1) - and .artifacts[0].release_catalog_key == "maven:cuvs-java" - and .artifacts[0].path == "cuvs-java-26.08.0.jar" - and .artifacts[0].package.name == "ai.rapids:cuvs-java" -' "${manifest_path}" >/dev/null +entries_path="${canonical_bundle_directory}/release-catalog-entries.json" +test ! -e "${canonical_bundle_directory}/release-catalog-metadata.json" jq -e ' .schema_version == 1 and .producer == "shared-workflows" - and .release_catalog_key == "maven:cuvs-java" - and .source_artifact == "cuvs-java-cuda12.9.1" - and .catalog_record_manifest == "release-catalog-entries.json" - and .build_environment.repository == "rapidsai/cuvs" - and .build_environment.sha == "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - and .metadata.artifacts == [{path: "cuvs-java-26.08.0.jar", sbom_kind: "producer-dependency"}] -' "${metadata_path}" >/dev/null -grep -Fx "manifest-path=${manifest_path}" "${GITHUB_OUTPUT}" -grep -Fx "metadata-path=${metadata_path}" "${GITHUB_OUTPUT}" - -supplied_sbom_path="$(jq -r '.artifacts[0].sbom' "${manifest_path}")" -supplied_provenance_path="$(jq -r '.artifacts[0].provenance' "${manifest_path}")" -supplied_signature_path="$(jq -r '.artifacts[0].signature' "${manifest_path}")" + and .source.artifact == "cuvs-java-cuda12.9.1" + and .source.repository == "rapidsai/cuvs" + and .source.sha == "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + and (.entries | length == 1) + and .entries[0].release_catalog_key == "maven:cuvs-java" + and .entries[0].path == "cuvs-java-26.08.0.jar" + and .entries[0].package.name == "ai.rapids:cuvs-java" + and .entries[0].sbom_kind == "producer-dependency" +' "${entries_path}" >/dev/null +grep -Fx "entries-path=${entries_path}" "${GITHUB_OUTPUT}" + +supplied_sbom_path="$(jq -r '.entries[0].sbom' "${entries_path}")" +supplied_provenance_path="$(jq -r '.entries[0].provenance' "${entries_path}")" +supplied_signature_path="$(jq -r '.entries[0].signature' "${entries_path}")" [[ "${supplied_sbom_path}" == release-evidence/*/sbom-* ]] [[ "${supplied_provenance_path}" == release-evidence/*/provenance-* ]] [[ "${supplied_signature_path}" == release-evidence/*/signature-* ]] @@ -75,11 +67,11 @@ grep -Fx signature "${canonical_bundle_directory}/${supplied_signature_path}" isolated_companion_directory="${temporary_directory}/isolated-companion" mkdir -p "${isolated_companion_directory}" -cp "${manifest_path}" "${metadata_path}" "${isolated_companion_directory}/" +cp "${entries_path}" "${isolated_companion_directory}/" cp -R "${canonical_bundle_directory}/release-evidence" "${isolated_companion_directory}/" while IFS= read -r evidence_path; do test -f "${isolated_companion_directory}/${evidence_path}" -done < <(jq -r '.artifacts[] | .sbom, .provenance, (.signature // empty)' "${isolated_companion_directory}/release-catalog-entries.json") +done < <(jq -r '.entries[] | .sbom, .provenance, (.signature // empty)' "${isolated_companion_directory}/release-catalog-entries.json") generated_directory="${temporary_directory}/generated-bundle" mkdir -p "${generated_directory}/linux-64" @@ -96,15 +88,17 @@ export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAG "${repository_root}/release-catalog/materialize.sh" -generated_manifest_path="${generated_directory}/release-catalog-entries.json" -generated_metadata_path="${generated_directory}/release-catalog-metadata.json" -generated_sbom_path="$(jq -r '.artifacts[0].sbom' "${generated_manifest_path}")" -generated_provenance_path="$(jq -r '.artifacts[0].provenance' "${generated_manifest_path}")" +generated_entries_path="${generated_directory}/release-catalog-entries.json" +generated_sbom_path="$(jq -r '.entries[0].sbom' "${generated_entries_path}")" +generated_provenance_path="$(jq -r '.entries[0].provenance' "${generated_entries_path}")" jq -e ' - .artifacts[0].release_catalog_key == "conda:kvikio" - and .artifacts[0].path == "linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.conda" - and .artifacts[0].package == {ecosystem: "conda", name: "kvikio", version: "26.08.00a32"} -' "${generated_manifest_path}" >/dev/null + .source.artifact == "kvikio_conda_python_kvikio_x86_64_abi3_cu12" + and .source.sha == "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + and .entries[0].release_catalog_key == "conda:kvikio" + and .entries[0].path == "linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.conda" + and .entries[0].package == {ecosystem: "conda", name: "kvikio", version: "26.08.00a32"} + and .entries[0].sbom_kind == "generated-identity" +' "${generated_entries_path}" >/dev/null jq -e ' .spdxVersion == "SPDX-2.3" and .packages[0].name == "kvikio" @@ -115,13 +109,6 @@ jq -e ' and .predicate.buildDefinition.externalParameters.release_catalog_key == "conda:kvikio" and .predicate.buildDefinition.resolvedDependencies[0].digest.gitCommit == "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ' "${generated_directory}/${generated_provenance_path}" >/dev/null -jq -e ' - .release_catalog_key == "conda:kvikio" - and .source_artifact == "kvikio_conda_python_kvikio_x86_64_abi3_cu12" - and .build_environment.sha == "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" - and .metadata.artifacts == [{path: "linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.conda", sbom_kind: "generated-identity"}] -' "${generated_metadata_path}" >/dev/null - wheel_directory="${temporary_directory}/wheel-bundle" mkdir -p "${wheel_directory}" printf '%s\n' wheel >"${wheel_directory}/libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl" @@ -137,13 +124,11 @@ export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_SOURCE "${repository_root}/release-catalog/materialize.sh" jq -e ' - .artifacts[0].release_catalog_key == "wheel:kvikio" - and .artifacts[0].path == "libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl" - and .artifacts[0].package == {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"} + .entries[0].release_catalog_key == "wheel:kvikio" + and .entries[0].path == "libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl" + and .entries[0].package == {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"} + and .entries[0].sbom_kind == "generated-identity" ' "${wheel_directory}/release-catalog-entries.json" >/dev/null -jq -e ' - .metadata.artifacts == [{path: "libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl", sbom_kind: "generated-identity"}] -' "${wheel_directory}/release-catalog-metadata.json" >/dev/null missing_version_directory="${temporary_directory}/missing-version-bundle" mkdir -p "${missing_version_directory}" From 246ec8e545484c5890b1e81e90e07979943ecb86 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 18 Aug 2026 09:40:11 -0500 Subject: [PATCH 08/36] simplify custom package identity input --- release-catalog-dispatch/action.yml | 7 +- release-catalog/README.md | 36 ++++- release-catalog/config.schema.json | 105 ++------------ release-catalog/materialize.sh | 134 +++++++----------- release-catalog/prepare.sh | 14 +- release-catalog/validate-config.sh | 35 ++--- .../inline-package.json | 14 -- ...e-file.json => package-identity-file.json} | 2 +- tests/release_catalog_config_test.sh | 32 ++--- tests/release_catalog_prepare_test.sh | 12 +- tests/release_catalog_test.sh | 33 +++-- 11 files changed, 145 insertions(+), 279 deletions(-) delete mode 100644 tests/release-catalog-config/inline-package.json rename tests/release-catalog-config/{package-file.json => package-identity-file.json} (77%) diff --git a/release-catalog-dispatch/action.yml b/release-catalog-dispatch/action.yml index e95c3844..d53309ca 100644 --- a/release-catalog-dispatch/action.yml +++ b/release-catalog-dispatch/action.yml @@ -59,8 +59,7 @@ runs: RELEASE_ARTIFACTS: ${{ steps.configuration.outputs.artifacts }} RELEASE_ARTIFACT_TYPE: ${{ steps.configuration.outputs.artifact_type }} RELEASE_OUTPUT_DIRECTORY: ${{ steps.configuration.outputs.output_directory }} - RELEASE_PACKAGE: ${{ steps.configuration.outputs.package }} - RELEASE_PACKAGE_FILE: ${{ steps.configuration.outputs.package_file }} + RELEASE_PACKAGE_IDENTITY_FILE: ${{ steps.configuration.outputs.package_identity_file }} run: "${IMPLEMENTATION_PATH}/prepare.sh" - id: materialize name: Materialize release catalog entries @@ -68,10 +67,10 @@ runs: env: IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} RELEASE_ARTIFACTS: ${{ steps.prepare.outputs.artifacts }} + RELEASE_ARTIFACT_TYPE: ${{ steps.configuration.outputs.artifact_type }} RELEASE_ENTRIES_NAME: release-catalog-entries.json RELEASE_OUTPUT_DIRECTORY: ${{ steps.configuration.outputs.output_directory }} - RELEASE_PACKAGE: ${{ steps.prepare.outputs.package }} - RELEASE_PACKAGE_FILE: ${{ steps.configuration.outputs.package_file }} + RELEASE_PACKAGE_IDENTITY_FILE: ${{ steps.configuration.outputs.package_identity_file }} RELEASE_SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} RELEASE_SOURCE_SHA: ${{ inputs.source-sha || github.sha }} RELEASE_CATALOG_KEY: ${{ steps.configuration.outputs.release_catalog_key }} diff --git a/release-catalog/README.md b/release-catalog/README.md index 2c05c416..b1a7785c 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -31,8 +31,8 @@ The `artifact_type` field determines how the action discovers primary files: - `conda` and `wheel` derive artifact descriptors and package identity from the built packages; -- `custom` requires explicit artifact descriptors and exactly one of `package` - or `package_file`. +- `custom` requires explicit artifact descriptors and a + `package_identity_file` created by the producer during the build. `release_catalog_key` identifies the release catalog entry that owns these artifacts. Every file from every matrix variant in the same publishable artifact @@ -54,9 +54,10 @@ Multiple package names from one repository do not by themselves justify separate keys: for example, `cudf` and `dask-cudf` Conda packages remain part of `conda:cudf` when they share one release policy. -The action validates the complete configuration before inspecting build -outputs. It then verifies properties that depend on produced files, including -whether package, primary-artifact, and evidence paths resolve unambiguously. +The action validates the configuration before inspecting build outputs. It then +verifies properties that depend on produced files, including the package +identity contents and whether primary-artifact and evidence paths resolve +unambiguously. ## Source revision @@ -100,13 +101,26 @@ the built source. "artifact_type": "custom", "release_catalog_key": "maven:cuvs-java", "output_directory": "java/cuvs-java/target", - "package_file": "cuvs-java.release-package.json", + "package_identity_file": "cuvs-java.release-package-identity.json", "artifacts": [{"path": "cuvs-java-*-x86_64-cuda*.jar"}] } source-artifact-name: cuvs-java source-sha: ${{ env.RAPIDS_SHA }} ``` +Before the action runs, the producer creates +`java/cuvs-java/target/cuvs-java.release-package-identity.json`. The +`package_identity_file` value is the path to that file relative to +`output_directory`. For example: + +```json +{ + "ecosystem": "maven", + "name": "ai.rapids:cuvs-java", + "version": "26.08.0" +} +``` + ## Evidence semantics A descriptor-selected producer SBOM is classified as `producer-dependency`. @@ -119,3 +133,13 @@ inventory. It must not be reported as producer-supplied dependency coverage. Producer-supplied SBOM, provenance, and signature sidecars are copied under `release-evidence/` so the companion remains independently consumable. + +Concrete producer-supplied evidence examples include: + +- an official [SPDX 2.3 dependency SBOM](https://github.com/spdx/spdx-examples/blob/2181917ef6ff74de89252ee785583c27a38d6199/presentations/OSS-NA-2023/SPDXVersion2.3/03-SBOMwDependency.json); +- an official [SLSA provenance v1 statement](https://github.com/slsa-framework/github-actions-buildtypes/blob/5f855ef0106dad3ee0e0f1046dc31b3b65152956/workflow/v1/example.json); +- a Maven Central [detached ASCII-armored signature](https://repo1.maven.org/maven2/org/apache/commons/commons-lang3/3.17.0/commons-lang3-3.17.0.jar.asc). + +These examples illustrate the expected purpose of the files, not required +serialization formats. The action copies producer-supplied evidence as opaque +sidecars and does not validate their contents or require these formats. diff --git a/release-catalog/config.schema.json b/release-catalog/config.schema.json index 9883c7d7..5610a1d3 100644 --- a/release-catalog/config.schema.json +++ b/release-catalog/config.schema.json @@ -11,7 +11,7 @@ ], "properties": { "artifact_type": { - "description": "How primary artifact descriptors and package identities are obtained.", + "description": "Metadata can be extracted from the artifact for conda/wheel types. More info needed for custom type.", "type": "string", "enum": [ "conda", @@ -20,7 +20,7 @@ ] }, "release_catalog_key": { - "description": "Release-catalog entry name that owns these artifacts. Default for conda/wheels is :, such as conda:cudf. Custom producers specify their own, such as maven:cuvs-java.", + "description": "Release-catalog entry name that owns these artifacts. Default for conda & wheels is :, such as conda:cudf. Custom jobs specify their own, such as maven:cuvs-java.", "type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$" @@ -32,17 +32,14 @@ "pattern": "^[^\\r\\n]+$", "default": "." }, - "package": { - "$ref": "#/$defs/package" - }, - "package_file": { - "description": "Producer-created package identity JSON, relative to output_directory.", + "package_identity_file": { + "description": "Path, relative to output_directory, to producer-created package identity JSON. Required for custom artifacts; Conda and wheel identities are extracted from the artifacts. The file must contain ecosystem, name, and version and may contain build and platform.", "type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$" }, "artifacts": { - "description": "Primary release files and optional producer-supplied evidence sidecars.", + "description": "Primary release files and optional SBoM evidence sidecar files", "type": "array", "minItems": 1, "items": { @@ -64,29 +61,8 @@ }, "then": { "required": [ - "artifacts" - ], - "oneOf": [ - { - "required": [ - "package" - ], - "not": { - "required": [ - "package_file" - ] - } - }, - { - "required": [ - "package_file" - ], - "not": { - "required": [ - "package" - ] - } - } + "artifacts", + "package_identity_file" ] }, "else": { @@ -102,12 +78,7 @@ }, { "required": [ - "package" - ] - }, - { - "required": [ - "package_file" + "package_identity_file" ] } ] @@ -116,33 +87,6 @@ } ], "$defs": { - "package": { - "type": "object", - "additionalProperties": false, - "required": [ - "ecosystem", - "name" - ], - "properties": { - "ecosystem": { - "type": "string", - "minLength": 1 - }, - "name": { - "type": "string", - "minLength": 1 - }, - "version": { - "type": "string" - }, - "build": { - "type": "string" - }, - "platform": { - "type": "string" - } - } - }, "artifact": { "type": "object", "additionalProperties": false, @@ -156,49 +100,22 @@ "pattern": "^[^\\r\\n]+$" }, "sbom": { - "description": "Producer-supplied dependency SBOM path or glob.", + "description": "Dependency SBOM path or glob. Example file: https://github.com/spdx/spdx-examples/blob/2181917ef6ff74de89252ee785583c27a38d6199/presentations/OSS-NA-2023/SPDXVersion2.3/03-SBOMwDependency.json", "type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$" }, "provenance": { - "description": "Producer-supplied provenance path or glob.", + "description": "Provenance path or glob. Example file: https://github.com/slsa-framework/github-actions-buildtypes/blob/5f855ef0106dad3ee0e0f1046dc31b3b65152956/workflow/v1/example.json", "type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$" }, "signature": { - "description": "Producer-supplied signature path or glob.", + "description": "Signature path or glob. Example file: https://repo1.maven.org/maven2/org/apache/commons/commons-lang3/3.17.0/commons-lang3-3.17.0.jar.asc", "type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$" - }, - "package": { - "description": "Per-artifact package identity overrides.", - "type": "object", - "additionalProperties": false, - "properties": { - "ecosystem": { - "type": "string", - "minLength": 1 - }, - "name": { - "type": "string", - "minLength": 1 - }, - "version": { - "type": "string", - "minLength": 1 - }, - "build": { - "type": "string", - "minLength": 1 - }, - "platform": { - "type": "string", - "minLength": 1 - } - } } } } diff --git a/release-catalog/materialize.sh b/release-catalog/materialize.sh index f800d03a..24c45fcf 100755 --- a/release-catalog/materialize.sh +++ b/release-catalog/materialize.sh @@ -13,20 +13,12 @@ require_nonempty() { } require_nonempty "RELEASE_CATALOG_KEY" "${RELEASE_CATALOG_KEY:-}" +require_nonempty "RELEASE_ARTIFACT_TYPE" "${RELEASE_ARTIFACT_TYPE:-}" require_nonempty "RELEASE_OUTPUT_DIRECTORY" "${RELEASE_OUTPUT_DIRECTORY:-}" require_nonempty "RELEASE_ENTRIES_NAME" "${RELEASE_ENTRIES_NAME:-}" require_nonempty "RELEASE_ARTIFACTS" "${RELEASE_ARTIFACTS:-}" require_nonempty "RELEASE_SOURCE_ARTIFACT_NAME" "${RELEASE_SOURCE_ARTIFACT_NAME:-}" -if [[ -n "${RELEASE_PACKAGE:-}" && -n "${RELEASE_PACKAGE_FILE:-}" ]]; then - echo "release-package and release-package-file are mutually exclusive" >&2 - exit 1 -fi -if [[ -z "${RELEASE_PACKAGE:-}" && -z "${RELEASE_PACKAGE_FILE:-}" ]]; then - echo "one of release-package or release-package-file is required" >&2 - exit 1 -fi - source_sha="${RELEASE_SOURCE_SHA:-${GITHUB_SHA:-}}" require_nonempty "RELEASE_SOURCE_SHA or GITHUB_SHA" "${source_sha}" @@ -56,26 +48,40 @@ ensure_relative_pattern() { } output_directory="$(realpath "${RELEASE_OUTPUT_DIRECTORY}")" -if [[ -n "${RELEASE_PACKAGE_FILE:-}" ]]; then - ensure_relative_pattern "release-package-file" "${RELEASE_PACKAGE_FILE}" - package_file_path="$(realpath "${output_directory}/${RELEASE_PACKAGE_FILE}")" - if [[ "${package_file_path}" != "${output_directory}"/* || ! -f "${package_file_path}" ]]; then - echo "release-package-file must resolve to one file inside output-directory: ${RELEASE_PACKAGE_FILE}" >&2 +package_identity='' +case "${RELEASE_ARTIFACT_TYPE}" in + custom) + require_nonempty "RELEASE_PACKAGE_IDENTITY_FILE" "${RELEASE_PACKAGE_IDENTITY_FILE:-}" + ensure_relative_pattern "package-identity-file" "${RELEASE_PACKAGE_IDENTITY_FILE}" + package_identity_path="$(realpath "${output_directory}/${RELEASE_PACKAGE_IDENTITY_FILE}")" + if [[ "${package_identity_path}" != "${output_directory}"/* || ! -f "${package_identity_path}" ]]; then + echo "package-identity-file must resolve to one file inside output-directory: ${RELEASE_PACKAGE_IDENTITY_FILE}" >&2 + exit 1 + fi + package_identity="$(jq -c . "${package_identity_path}")" + ;; + conda | wheel) + if [[ -n "${RELEASE_PACKAGE_IDENTITY_FILE:-}" ]]; then + echo "package-identity-file is only valid for custom artifacts" >&2 + exit 1 + fi + ;; + *) + echo "artifact-type must be one of: conda, custom, wheel" >&2 exit 1 - fi - RELEASE_PACKAGE="$(jq -c . "${package_file_path}")" -fi + ;; +esac -if ! jq -e ' +if [[ "${RELEASE_ARTIFACT_TYPE}" == "custom" ]] && ! jq -e ' type == "object" and (keys - ["ecosystem", "name", "version", "build", "platform"] | length == 0) and (.ecosystem | type == "string" and length > 0) and (.name | type == "string" and length > 0) - and ((.version // "") | type == "string") - and ((.build // "") | type == "string") - and ((.platform // "") | type == "string") -' <<<"${RELEASE_PACKAGE}" >/dev/null; then - echo "release-package must be a package object with ecosystem and name; version may be supplied or derived per artifact" >&2 + and (.version | type == "string" and length > 0) + and ((has("build") | not) or (.build | type == "string" and length > 0)) + and ((has("platform") | not) or (.platform | type == "string" and length > 0)) +' <<<"${package_identity}" >/dev/null; then + echo "package identity file must contain non-empty ecosystem, name, and version strings and only optional build or platform strings" >&2 exit 1 fi @@ -113,49 +119,6 @@ resolve_one_file() { printf '%s\n' "${resolved#"${output_directory}/"}" } -derive_package_version() { - local ecosystem="$1" - local package_name="$2" - local artifact_path="$3" - local filename - filename="$(basename "${artifact_path}")" - - case "${ecosystem}" in - conda) - local conda_prefix="${package_name}-" - if [[ "${filename}" != "${conda_prefix}"* ]]; then - echo "Conda artifact filename does not start with package name '${package_name}': ${filename}" >&2 - exit 1 - fi - local conda_remainder="${filename#"${conda_prefix}"}" - local conda_version="${conda_remainder%%-*}" - if [[ -z "${conda_version}" || "${conda_version}" == "${conda_remainder}" ]]; then - echo "cannot derive Conda package version from artifact filename: ${filename}" >&2 - exit 1 - fi - printf '%s\n' "${conda_version}" - ;; - wheel) - local wheel_prefix="${package_name//-/_}-" - if [[ "${filename}" != "${wheel_prefix}"* ]]; then - echo "wheel artifact filename does not start with normalized package name '${package_name}': ${filename}" >&2 - exit 1 - fi - local wheel_remainder="${filename#"${wheel_prefix}"}" - local wheel_version="${wheel_remainder%%-*}" - if [[ -z "${wheel_version}" || "${wheel_version}" == "${wheel_remainder}" ]]; then - echo "cannot derive wheel package version from artifact filename: ${filename}" >&2 - exit 1 - fi - printf '%s\n' "${wheel_version}" - ;; - *) - echo "release-package version is required for ${ecosystem} artifacts" >&2 - exit 1 - ;; - esac -} - generated_evidence_path() { local primary_path="$1" local kind="$2" @@ -257,18 +220,34 @@ write_generated_provenance() { shopt -s globstar nullglob while IFS= read -r descriptor; do - if ! jq -e ' + if ! jq -e --arg artifact_type "${RELEASE_ARTIFACT_TYPE}" ' type == "object" - and (keys - ["path", "sbom", "provenance", "signature", "package"] | length == 0) + and ( + if $artifact_type == "custom" then + (keys - ["path", "sbom", "provenance", "signature"] | length == 0) + else + (keys - ["path", "sbom", "provenance", "signature", "package"] | length == 0) + end + ) and (.path | type == "string" and length > 0) and ((.sbom // "") | type == "string") and ((.provenance // "") | type == "string") and ((.signature // "") | type == "string") - and ((.package // {}) | type == "object") - and ((.package // {} | keys - ["ecosystem", "name", "version", "build", "platform"]) | length == 0) - and ((.package // {} | to_entries | map(.value | type == "string" and length > 0) | all)) + and ( + if $artifact_type == "custom" then + has("package") | not + else + (.package | type == "object") + and (.package | keys - ["ecosystem", "name", "version", "build", "platform"] | length == 0) + and (.package.ecosystem | type == "string" and length > 0) + and (.package.name | type == "string" and length > 0) + and (.package.version | type == "string" and length > 0) + and ((.package | has("build") | not) or (.package.build | type == "string" and length > 0)) + and ((.package | has("platform") | not) or (.package.platform | type == "string" and length > 0)) + end + ) ' <<<"${descriptor}" >/dev/null; then - echo "every release-artifacts entry must contain path and optional SBOM/provenance/signature/package overrides" >&2 + echo "release artifact descriptor is invalid for ${RELEASE_ARTIFACT_TYPE}: ${descriptor}" >&2 exit 1 fi @@ -276,13 +255,10 @@ while IFS= read -r descriptor; do sbom_pattern="$(jq -r '.sbom // empty' <<<"${descriptor}")" provenance_pattern="$(jq -r '.provenance // empty' <<<"${descriptor}")" signature_pattern="$(jq -r '.signature // empty' <<<"${descriptor}")" - package_override="$(jq -c '.package // {}' <<<"${descriptor}")" - - package="$(jq -cn --argjson base "${RELEASE_PACKAGE}" --argjson override "${package_override}" '$base + $override')" - package_version="$(jq -r '.version // empty' <<<"${package}")" - if [[ -z "${package_version}" ]]; then - package_version="$(derive_package_version "$(jq -r '.ecosystem' <<<"${package}")" "$(jq -r '.name' <<<"${package}")" "${primary_path}")" - package="$(jq -c --arg version "${package_version}" '. + {version: $version}' <<<"${package}")" + if [[ "${RELEASE_ARTIFACT_TYPE}" == "custom" ]]; then + package="${package_identity}" + else + package="$(jq -c '.package' <<<"${descriptor}")" fi if [[ -n "${sbom_pattern}" ]]; then diff --git a/release-catalog/prepare.sh b/release-catalog/prepare.sh index f3aa97f4..4a803d28 100755 --- a/release-catalog/prepare.sh +++ b/release-catalog/prepare.sh @@ -17,27 +17,21 @@ require_nonempty "RELEASE_OUTPUT_DIRECTORY" "${RELEASE_OUTPUT_DIRECTORY:-}" script_directory="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" artifacts="${RELEASE_ARTIFACTS:-}" -package="${RELEASE_PACKAGE:-}" case "${RELEASE_ARTIFACT_TYPE}" in conda) if [[ -z "${artifacts}" ]]; then artifacts="$("${script_directory}/describe-conda.sh" "${RELEASE_OUTPUT_DIRECTORY}")" fi - if [[ -z "${package}" && -z "${RELEASE_PACKAGE_FILE:-}" ]]; then - package='{"ecosystem":"conda","name":"bundle"}' - fi ;; wheel) if [[ -z "${artifacts}" ]]; then artifacts="$("${script_directory}/describe-wheels.sh" "${RELEASE_OUTPUT_DIRECTORY}")" fi - if [[ -z "${package}" && -z "${RELEASE_PACKAGE_FILE:-}" ]]; then - package='{"ecosystem":"wheel","name":"bundle"}' - fi ;; custom) require_nonempty "RELEASE_ARTIFACTS" "${artifacts}" + require_nonempty "RELEASE_PACKAGE_IDENTITY_FILE" "${RELEASE_PACKAGE_IDENTITY_FILE:-}" ;; *) echo "artifact-type must be one of: conda, custom, wheel" >&2 @@ -45,10 +39,4 @@ case "${RELEASE_ARTIFACT_TYPE}" in ;; esac -if [[ -z "${package}" && -z "${RELEASE_PACKAGE_FILE:-}" ]]; then - echo "one of release-package or release-package-file is required for custom artifacts" >&2 - exit 1 -fi - printf 'artifacts=%s\n' "${artifacts}" >>"${GITHUB_OUTPUT}" -printf 'package=%s\n' "${package}" >>"${GITHUB_OUTPUT}" diff --git a/release-catalog/validate-config.sh b/release-catalog/validate-config.sh index 12916159..9cbfafd1 100755 --- a/release-catalog/validate-config.sh +++ b/release-catalog/validate-config.sh @@ -23,22 +23,10 @@ fi validation_errors="$(jq -r ' def single_line_string: type == "string" and length > 0 and (test("[\\r\\n]") | not); - def package_object: - type == "object" - and (keys - ["ecosystem", "name", "version", "build", "platform"] | length == 0) - and (.ecosystem | single_line_string) - and (.name | single_line_string) - and ((.version // "") | type == "string") - and ((.build // "") | type == "string") - and ((.platform // "") | type == "string"); - def artifact_package_object: - type == "object" - and (keys - ["ecosystem", "name", "version", "build", "platform"] | length == 0) - and (to_entries | map(.value | single_line_string) | all); if type != "object" then ["release catalog configuration must be a JSON object"] else - (keys - ["artifact_type", "release_catalog_key", "output_directory", "package", "package_file", "artifacts"]) as $unknown + (keys - ["artifact_type", "release_catalog_key", "output_directory", "package_identity_file", "artifacts"]) as $unknown | [ if ($unknown | length) > 0 then "unknown field(s): " + ($unknown | join(", ")) @@ -52,12 +40,8 @@ validation_errors="$(jq -r ' ] + if .artifact_type == "custom" then [ - if ([has("package"), has("package_file")] | map(select(.)) | length) == 1 then empty - else "exactly one of package or package_file is required for custom artifacts" end, - if (has("package") | not) or (.package | package_object) then empty - else "package must contain non-empty ecosystem and name strings and only version, build, or platform as optional string fields" end, - if (has("package_file") | not) or (.package_file | single_line_string) then empty - else "package_file must be a non-empty, single-line path relative to output_directory" end, + if (.package_identity_file | single_line_string) then empty + else "package_identity_file is required for custom artifacts and must be a non-empty, single-line path relative to output_directory" end, if (.artifacts | type == "array" and length > 0) then empty else "artifacts must be a non-empty array for custom artifacts" end ] @@ -65,8 +49,8 @@ validation_errors="$(jq -r ' [ if (.output_directory | single_line_string) then empty else "output_directory is required for conda and wheel artifacts" end, - if has("artifacts") or has("package") or has("package_file") then - "artifacts, package, and package_file are only valid when artifact_type is custom" + if has("artifacts") or has("package_identity_file") then + "artifacts and package_identity_file are only valid when artifact_type is custom" else empty end ] else [] end @@ -78,7 +62,7 @@ validation_errors="$(jq -r ' | if ($artifact | type) != "object" then "artifacts[\($index)] must be an object" else - ($artifact | keys - ["path", "sbom", "provenance", "signature", "package"]) as $artifact_unknown + ($artifact | keys - ["path", "sbom", "provenance", "signature"]) as $artifact_unknown | if ($artifact_unknown | length) > 0 then "artifacts[\($index)] has unknown field(s): " + ($artifact_unknown | join(", ")) else empty end, @@ -87,9 +71,7 @@ validation_errors="$(jq -r ' ($artifact | to_entries[] | select(.key == "sbom" or .key == "provenance" or .key == "signature") | select((.value | single_line_string) | not) - | "artifacts[\($index)].\(.key) must be a non-empty, single-line string"), - if ($artifact | has("package") | not) or ($artifact.package | artifact_package_object) then empty - else "artifacts[\($index)].package must contain only non-empty string identity overrides" end + | "artifacts[\($index)].\(.key) must be a non-empty, single-line string") end ] else [] end @@ -113,7 +95,6 @@ fi printf 'artifact_type=%s\n' "$(jq -r '.artifact_type' <<<"${compact_config}")" printf 'release_catalog_key=%s\n' "$(jq -r '.release_catalog_key' <<<"${compact_config}")" printf 'output_directory=%s\n' "$(jq -r '.output_directory // "."' <<<"${compact_config}")" - printf 'package=%s\n' "$(jq -c '.package // empty' <<<"${compact_config}")" - printf 'package_file=%s\n' "$(jq -r '.package_file // empty' <<<"${compact_config}")" + printf 'package_identity_file=%s\n' "$(jq -r '.package_identity_file // empty' <<<"${compact_config}")" printf 'artifacts=%s\n' "$(jq -c '.artifacts // empty' <<<"${compact_config}")" } >>"${GITHUB_OUTPUT}" diff --git a/tests/release-catalog-config/inline-package.json b/tests/release-catalog-config/inline-package.json deleted file mode 100644 index 80d6b6d3..00000000 --- a/tests/release-catalog-config/inline-package.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "artifact_type": "custom", - "release_catalog_key": "archive:smoke", - "package": { - "ecosystem": "archive", - "name": "smoke", - "version": "1.0" - }, - "artifacts": [ - { - "path": "smoke.tar.gz" - } - ] -} diff --git a/tests/release-catalog-config/package-file.json b/tests/release-catalog-config/package-identity-file.json similarity index 77% rename from tests/release-catalog-config/package-file.json rename to tests/release-catalog-config/package-identity-file.json index 67776f18..0c6bacd8 100644 --- a/tests/release-catalog-config/package-file.json +++ b/tests/release-catalog-config/package-identity-file.json @@ -2,7 +2,7 @@ "artifact_type": "custom", "release_catalog_key": "maven:cuvs-java", "output_directory": "java/cuvs-java/target", - "package_file": "cuvs-java.release-package.json", + "package_identity_file": "cuvs-java.release-package-identity.json", "artifacts": [ { "path": "cuvs-java-*-x86_64-cuda*.jar", diff --git a/tests/release_catalog_config_test.sh b/tests/release_catalog_config_test.sh index 36d92c7c..ccfe3959 100755 --- a/tests/release_catalog_config_test.sh +++ b/tests/release_catalog_config_test.sh @@ -27,24 +27,15 @@ assert_invalid() { } valid_output="${temporary_directory}/valid.output" -RELEASE_CATALOG_CONFIG="$(<"${repository_root}/tests/release-catalog-config/package-file.json")" \ +RELEASE_CATALOG_CONFIG="$(<"${repository_root}/tests/release-catalog-config/package-identity-file.json")" \ GITHUB_OUTPUT="${valid_output}" "${validator}" grep -Fx 'release_catalog_key=maven:cuvs-java' "${valid_output}" grep -Fx 'artifact_type=custom' "${valid_output}" grep -Fx 'output_directory=java/cuvs-java/target' "${valid_output}" -grep -Fx 'package=' "${valid_output}" -grep -Fx 'package_file=cuvs-java.release-package.json' "${valid_output}" +grep -Fx 'package_identity_file=cuvs-java.release-package-identity.json' "${valid_output}" grep -Fx 'artifacts=[{"path":"cuvs-java-*-x86_64-cuda*.jar","sbom":"cuvs-java.spdx.json"}]' "${valid_output}" -inline_output="${temporary_directory}/inline.output" -RELEASE_CATALOG_CONFIG="$(<"${repository_root}/tests/release-catalog-config/inline-package.json")" \ - GITHUB_OUTPUT="${inline_output}" "${validator}" - -grep -Fx 'output_directory=.' "${inline_output}" -grep -Fx 'package={"ecosystem":"archive","name":"smoke","version":"1.0"}' "${inline_output}" -grep -Fx 'package_file=' "${inline_output}" - assert_invalid \ malformed-json \ '{"release_catalog_key":' \ @@ -64,22 +55,27 @@ grep -F 'artifact_type must be one of: conda, custom, wheel' "${temporary_direct assert_invalid \ custom-missing-fields \ '{"artifact_type":"custom","release_catalog_key":"archive:smoke"}' \ - 'exactly one of package or package_file is required for custom artifacts' + 'package_identity_file is required for custom artifacts' grep -F 'artifacts must be a non-empty array for custom artifacts' "${temporary_directory}/custom-missing-fields.error" >/dev/null assert_invalid \ unknown-field \ - '{"artifact_type":"custom","release_catalog_key":"archive:smoke","package_file":"package.json","artifacts":[{"path":"smoke.tar.gz"}],"component_id":"archive:smoke"}' \ + '{"artifact_type":"custom","release_catalog_key":"archive:smoke","package_identity_file":"package.json","artifacts":[{"path":"smoke.tar.gz"}],"component_id":"archive:smoke"}' \ 'unknown field(s): component_id' assert_invalid \ - conflicting-package-source \ - '{"artifact_type":"custom","release_catalog_key":"archive:smoke","package":{"ecosystem":"archive","name":"smoke"},"package_file":"package.json","artifacts":[{"path":"smoke.tar.gz"}]}' \ - 'exactly one of package or package_file is required for custom artifacts' + inline-package \ + '{"artifact_type":"custom","release_catalog_key":"archive:smoke","package_identity_file":"package.json","package":{"ecosystem":"archive","name":"smoke","version":"1.0"},"artifacts":[{"path":"smoke.tar.gz"}]}' \ + 'unknown field(s): package' + +assert_invalid \ + per-artifact-package \ + '{"artifact_type":"custom","release_catalog_key":"archive:smoke","package_identity_file":"package.json","artifacts":[{"path":"smoke.tar.gz","package":{"ecosystem":"archive","name":"smoke","version":"1.0"}}]}' \ + 'artifacts[0] has unknown field(s): package' assert_invalid \ malformed-artifact \ - '{"artifact_type":"custom","release_catalog_key":"archive:smoke","package_file":"package.json","artifacts":[{"file":"smoke.tar.gz","sbom":false}]}' \ + '{"artifact_type":"custom","release_catalog_key":"archive:smoke","package_identity_file":"package.json","artifacts":[{"file":"smoke.tar.gz","sbom":false}]}' \ 'artifacts[0] has unknown field(s): file' grep -F 'artifacts[0].path must be a non-empty, single-line string' "${temporary_directory}/malformed-artifact.error" >/dev/null grep -F 'artifacts[0].sbom must be a non-empty, single-line string' "${temporary_directory}/malformed-artifact.error" >/dev/null @@ -92,7 +88,7 @@ assert_invalid \ assert_invalid \ standard-custom-fields \ '{"artifact_type":"wheel","release_catalog_key":"wheel:smoke","output_directory":"dist","artifacts":[{"path":"smoke.whl"}]}' \ - 'artifacts, package, and package_file are only valid when artifact_type is custom' + 'artifacts and package_identity_file are only valid when artifact_type is custom' standard_output="${temporary_directory}/standard.output" RELEASE_CATALOG_CONFIG='{ diff --git a/tests/release_catalog_prepare_test.sh b/tests/release_catalog_prepare_test.sh index a84eae03..399df071 100755 --- a/tests/release_catalog_prepare_test.sh +++ b/tests/release_catalog_prepare_test.sh @@ -22,13 +22,11 @@ GITHUB_OUTPUT="${temporary_directory}/wheel-output" RELEASE_ARTIFACTS='' RELEASE_ARTIFACT_TYPE=wheel RELEASE_OUTPUT_DIRECTORY="${temporary_directory}" -RELEASE_PACKAGE='' -RELEASE_PACKAGE_FILE='' -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE +RELEASE_PACKAGE_IDENTITY_FILE='' +export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE_IDENTITY_FILE "${repository_root}/release-catalog/prepare.sh" -grep -Fx 'package={"ecosystem":"wheel","name":"bundle"}' "${GITHUB_OUTPUT}" artifacts="$(sed -n 's/^artifacts=//p' "${GITHUB_OUTPUT}")" jq -e ' . == [{ @@ -40,14 +38,12 @@ jq -e ' GITHUB_OUTPUT="${temporary_directory}/custom-output" RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz", sbom: "bundle.spdx.json"}]')" RELEASE_ARTIFACT_TYPE=custom -RELEASE_PACKAGE='' -RELEASE_PACKAGE_FILE=release-package.json -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_PACKAGE RELEASE_PACKAGE_FILE +RELEASE_PACKAGE_IDENTITY_FILE=release-package-identity.json +export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_PACKAGE_IDENTITY_FILE "${repository_root}/release-catalog/prepare.sh" grep -Fx 'artifacts=[{"path":"bundle.tar.gz","sbom":"bundle.spdx.json"}]' "${GITHUB_OUTPUT}" -grep -Fx 'package=' "${GITHUB_OUTPUT}" GITHUB_OUTPUT="${temporary_directory}/invalid-output" RELEASE_ARTIFACT_TYPE=archive diff --git a/tests/release_catalog_test.sh b/tests/release_catalog_test.sh index a390b98c..db347cf3 100755 --- a/tests/release_catalog_test.sh +++ b/tests/release_catalog_test.sh @@ -15,7 +15,7 @@ printf '%s\n' provenance >"${bundle_directory}/cuvs-java-26.08.0.provenance.json printf '%s\n' signature >"${bundle_directory}/cuvs-java-26.08.0.jar.asc" jq -n \ '{ecosystem: "maven", name: "ai.rapids:cuvs-java", version: "26.08.0"}' \ - >"${bundle_directory}/cuvs-java-package.json" + >"${bundle_directory}/cuvs-java-package-identity.json" GITHUB_OUTPUT="${temporary_directory}/github-output" GITHUB_REPOSITORY="rapidsai/cuvs" @@ -25,15 +25,15 @@ GITHUB_SHA="0123456789012345678901234567890123456789" GITHUB_WORKFLOW_REF="rapidsai/cuvs/.github/workflows/build.yaml@refs/heads/release/26.08" export GITHUB_OUTPUT RELEASE_ARTIFACTS="$(jq -cn '[{path: "cuvs-java-*.jar", sbom: "cuvs-java-*.spdx.json", provenance: "cuvs-java-*.provenance.jsonl", signature: "cuvs-java-*.jar.asc"}]')" +RELEASE_ARTIFACT_TYPE=custom RELEASE_ENTRIES_NAME="release-catalog-entries.json" RELEASE_OUTPUT_DIRECTORY="${bundle_directory}" -RELEASE_PACKAGE='' -RELEASE_PACKAGE_FILE="cuvs-java-package.json" +RELEASE_PACKAGE_IDENTITY_FILE="cuvs-java-package-identity.json" RELEASE_SOURCE_ARTIFACT_NAME="cuvs-java-cuda12.9.1" RELEASE_SOURCE_SHA="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" RELEASE_CATALOG_KEY="maven:cuvs-java" export GITHUB_REPOSITORY GITHUB_RUN_ATTEMPT GITHUB_RUN_ID GITHUB_SHA GITHUB_WORKFLOW_REF -export RELEASE_ARTIFACTS RELEASE_ENTRIES_NAME RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_ENTRIES_NAME RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE_IDENTITY_FILE export RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" @@ -77,14 +77,14 @@ generated_directory="${temporary_directory}/generated-bundle" mkdir -p "${generated_directory}/linux-64" printf '%s\n' conda >"${generated_directory}/linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.conda" -RELEASE_ARTIFACTS="$(jq -cn '[{path: "linux-64/kvikio-*.conda"}]')" +RELEASE_ARTIFACTS="$(jq -cn '[{path: "linux-64/kvikio-*.conda", package: {ecosystem: "conda", name: "kvikio", version: "26.08.00a32"}}]')" +RELEASE_ARTIFACT_TYPE=conda RELEASE_OUTPUT_DIRECTORY="${generated_directory}" -RELEASE_PACKAGE="$(jq -cn '{ecosystem: "conda", name: "kvikio"}')" -RELEASE_PACKAGE_FILE='' +RELEASE_PACKAGE_IDENTITY_FILE='' RELEASE_SOURCE_ARTIFACT_NAME="kvikio_conda_python_kvikio_x86_64_abi3_cu12" RELEASE_SOURCE_SHA="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" RELEASE_CATALOG_KEY="conda:kvikio" -export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE_IDENTITY_FILE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" @@ -113,13 +113,13 @@ wheel_directory="${temporary_directory}/wheel-bundle" mkdir -p "${wheel_directory}" printf '%s\n' wheel >"${wheel_directory}/libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl" -RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl"}]')" +RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl", package: {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"}}]')" +RELEASE_ARTIFACT_TYPE=wheel RELEASE_OUTPUT_DIRECTORY="${wheel_directory}" -RELEASE_PACKAGE="$(jq -cn '{ecosystem: "wheel", name: "libkvikio-cu12"}')" RELEASE_SOURCE_ARTIFACT_NAME="kvikio_wheel_cpp_libkvikio_x86_64_cu12" RELEASE_SOURCE_SHA="cccccccccccccccccccccccccccccccccccccccc" RELEASE_CATALOG_KEY="wheel:kvikio" -export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_OUTPUT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" @@ -133,18 +133,21 @@ jq -e ' missing_version_directory="${temporary_directory}/missing-version-bundle" mkdir -p "${missing_version_directory}" printf '%s\n' archive >"${missing_version_directory}/bundle.tar.gz" +jq -n \ + '{ecosystem: "archive", name: "bundle"}' \ + >"${missing_version_directory}/bundle-package-identity.json" RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz"}]')" +RELEASE_ARTIFACT_TYPE=custom RELEASE_OUTPUT_DIRECTORY="${missing_version_directory}" -RELEASE_PACKAGE="$(jq -cn '{ecosystem: "archive", name: "bundle"}')" -RELEASE_PACKAGE_FILE='' +RELEASE_PACKAGE_IDENTITY_FILE='bundle-package-identity.json' RELEASE_SOURCE_ARTIFACT_NAME="bundle-archive" RELEASE_SOURCE_SHA="dddddddddddddddddddddddddddddddddddddddd" RELEASE_CATALOG_KEY="archive:bundle" -export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE RELEASE_PACKAGE_FILE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE_IDENTITY_FILE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/missing-version-error"; then echo "materialize.sh unexpectedly accepted a custom artifact without a package version" >&2 exit 1 fi -grep -Fx 'release-package version is required for archive artifacts' "${temporary_directory}/missing-version-error" +grep -Fx 'package identity file must contain non-empty ecosystem, name, and version strings and only optional build or platform strings' "${temporary_directory}/missing-version-error" From 69e767c27e9c7bc3f8f9dc12b6431cee59965e28 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 18 Aug 2026 10:53:49 -0500 Subject: [PATCH 09/36] resolve release catalog identity per artifact --- release-catalog-dispatch/action.yml | 4 - release-catalog/README.md | 24 ++-- release-catalog/config.schema.json | 63 ++-------- release-catalog/describe-conda.sh | 17 ++- release-catalog/describe-wheels.sh | 17 ++- release-catalog/materialize.sh | 89 +++++-------- release-catalog/prepare.sh | 118 +++++++++++++++--- release-catalog/validate-config.sh | 26 +--- .../package-identity-file.json | 3 +- tests/release_catalog_config_test.sh | 43 +++---- tests/release_catalog_prepare_test.sh | 109 +++++++++++++--- tests/release_catalog_test.sh | 45 ++++--- 12 files changed, 324 insertions(+), 234 deletions(-) diff --git a/release-catalog-dispatch/action.yml b/release-catalog-dispatch/action.yml index d53309ca..c38cb790 100644 --- a/release-catalog-dispatch/action.yml +++ b/release-catalog-dispatch/action.yml @@ -57,9 +57,7 @@ runs: env: IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} RELEASE_ARTIFACTS: ${{ steps.configuration.outputs.artifacts }} - RELEASE_ARTIFACT_TYPE: ${{ steps.configuration.outputs.artifact_type }} RELEASE_OUTPUT_DIRECTORY: ${{ steps.configuration.outputs.output_directory }} - RELEASE_PACKAGE_IDENTITY_FILE: ${{ steps.configuration.outputs.package_identity_file }} run: "${IMPLEMENTATION_PATH}/prepare.sh" - id: materialize name: Materialize release catalog entries @@ -67,10 +65,8 @@ runs: env: IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} RELEASE_ARTIFACTS: ${{ steps.prepare.outputs.artifacts }} - RELEASE_ARTIFACT_TYPE: ${{ steps.configuration.outputs.artifact_type }} RELEASE_ENTRIES_NAME: release-catalog-entries.json RELEASE_OUTPUT_DIRECTORY: ${{ steps.configuration.outputs.output_directory }} - RELEASE_PACKAGE_IDENTITY_FILE: ${{ steps.configuration.outputs.package_identity_file }} RELEASE_SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} RELEASE_SOURCE_SHA: ${{ inputs.source-sha || github.sha }} RELEASE_CATALOG_KEY: ${{ steps.configuration.outputs.release_catalog_key }} diff --git a/release-catalog/README.md b/release-catalog/README.md index b1a7785c..8b765b2a 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -27,12 +27,12 @@ catalog; there is no separate metadata document to keep synchronized. Every producer passes one `config` JSON object. Its canonical schema and field documentation are in [`config.schema.json`](config.schema.json). -The `artifact_type` field determines how the action discovers primary files: - -- `conda` and `wheel` derive artifact descriptors and package identity from the - built packages; -- `custom` requires explicit artifact descriptors and a - `package_identity_file` created by the producer during the build. +When `artifacts` is omitted, the action discovers every Conda and wheel output +in `output_directory` and extracts identity from each package independently. +When `artifacts` is supplied, each descriptor is resolved independently: +supported Conda and wheel files are parsed, while any other artifact requires +its own `package_identity_file`. Evidence paths also belong to the individual +artifact descriptor. `release_catalog_key` identifies the release catalog entry that owns these artifacts. Every file from every matrix variant in the same publishable artifact @@ -82,7 +82,6 @@ the built source. with: config: >- { - "artifact_type": "wheel", "release_catalog_key": "wheel:example", "output_directory": ${{ toJSON(steps.package-name.outputs.WHEEL_OUTPUT_DIR) }} } @@ -98,11 +97,12 @@ the built source. with: config: >- { - "artifact_type": "custom", "release_catalog_key": "maven:cuvs-java", "output_directory": "java/cuvs-java/target", - "package_identity_file": "cuvs-java.release-package-identity.json", - "artifacts": [{"path": "cuvs-java-*-x86_64-cuda*.jar"}] + "artifacts": [{ + "path": "cuvs-java-*-x86_64-cuda*.jar", + "package_identity_file": "cuvs-java.release-package-identity.json" + }] } source-artifact-name: cuvs-java source-sha: ${{ env.RAPIDS_SHA }} @@ -110,8 +110,8 @@ the built source. Before the action runs, the producer creates `java/cuvs-java/target/cuvs-java.release-package-identity.json`. The -`package_identity_file` value is the path to that file relative to -`output_directory`. For example: +artifact descriptor's `package_identity_file` value is the path to that file +relative to `output_directory`. For example: ```json { diff --git a/release-catalog/config.schema.json b/release-catalog/config.schema.json index 5610a1d3..c8402e1d 100644 --- a/release-catalog/config.schema.json +++ b/release-catalog/config.schema.json @@ -6,19 +6,9 @@ "type": "object", "additionalProperties": false, "required": [ - "artifact_type", "release_catalog_key" ], "properties": { - "artifact_type": { - "description": "Metadata can be extracted from the artifact for conda/wheel types. More info needed for custom type.", - "type": "string", - "enum": [ - "conda", - "custom", - "wheel" - ] - }, "release_catalog_key": { "description": "Release-catalog entry name that owns these artifacts. Default for conda & wheels is :, such as conda:cudf. Custom jobs specify their own, such as maven:cuvs-java.", "type": "string", @@ -32,14 +22,8 @@ "pattern": "^[^\\r\\n]+$", "default": "." }, - "package_identity_file": { - "description": "Path, relative to output_directory, to producer-created package identity JSON. Required for custom artifacts; Conda and wheel identities are extracted from the artifacts. The file must contain ecosystem, name, and version and may contain build and platform.", - "type": "string", - "minLength": 1, - "pattern": "^[^\\r\\n]+$" - }, "artifacts": { - "description": "Primary release files and optional SBoM evidence sidecar files", + "description": "Optional explicit primary release files and their identity or evidence sidecars. When omitted, Conda and wheel artifacts are discovered in output_directory.", "type": "array", "minItems": 1, "items": { @@ -47,45 +31,6 @@ } } }, - "allOf": [ - { - "if": { - "properties": { - "artifact_type": { - "const": "custom" - } - }, - "required": [ - "artifact_type" - ] - }, - "then": { - "required": [ - "artifacts", - "package_identity_file" - ] - }, - "else": { - "required": [ - "output_directory" - ], - "not": { - "anyOf": [ - { - "required": [ - "artifacts" - ] - }, - { - "required": [ - "package_identity_file" - ] - } - ] - } - } - } - ], "$defs": { "artifact": { "type": "object", @@ -99,6 +44,12 @@ "minLength": 1, "pattern": "^[^\\r\\n]+$" }, + "package_identity_file": { + "description": "Path, relative to output_directory, to producer-created package identity JSON for this artifact. Required only when identity cannot be extracted from the artifact. The file must contain ecosystem, name, and version and may contain build and platform.", + "type": "string", + "minLength": 1, + "pattern": "^[^\\r\\n]+$" + }, "sbom": { "description": "Dependency SBOM path or glob. Example file: https://github.com/spdx/spdx-examples/blob/2181917ef6ff74de89252ee785583c27a38d6199/presentations/OSS-NA-2023/SPDXVersion2.3/03-SBOMwDependency.json", "type": "string", diff --git a/release-catalog/describe-conda.sh b/release-catalog/describe-conda.sh index 6449addd..23bdcf47 100755 --- a/release-catalog/describe-conda.sh +++ b/release-catalog/describe-conda.sh @@ -3,16 +3,25 @@ set -euo pipefail -if [[ "$#" -ne 1 || ! -d "$1" ]]; then - echo "usage: $0 CONDA_OUTPUT_DIRECTORY" >&2 +if [[ "$#" -lt 1 || ! -d "$1" ]]; then + echo "usage: $0 CONDA_OUTPUT_DIRECTORY [CONDA_PACKAGE ...]" >&2 exit 1 fi output_directory="$(realpath "$1")" descriptors='[]' package_count=0 +package_paths=() -while IFS= read -r package_path; do +if [[ "$#" -gt 1 ]]; then + package_paths=("${@:2}") +else + while IFS= read -r package_path; do + package_paths+=("${package_path}") + done < <(find "${output_directory}" -type f \( -name '*.conda' -o -name '*.tar.bz2' \) -print | sort) +fi + +for package_path in "${package_paths[@]}"; do package_path="$(realpath "${package_path}")" if [[ "${package_path}" != "${output_directory}"/* ]]; then echo "Conda package must resolve inside output directory: ${package_path}" >&2 @@ -59,7 +68,7 @@ while IFS= read -r package_path; do --argjson current "${descriptors}" \ '$current + [{path: $path, package: $package}]')" package_count=$((package_count + 1)) -done < <(find "${output_directory}" -type f \( -name '*.conda' -o -name '*.tar.bz2' \) -print | sort) +done if [[ "${package_count}" -eq 0 ]]; then echo "Conda output directory contains no .conda or .tar.bz2 packages: ${output_directory}" >&2 diff --git a/release-catalog/describe-wheels.sh b/release-catalog/describe-wheels.sh index bc6b925e..f440055f 100755 --- a/release-catalog/describe-wheels.sh +++ b/release-catalog/describe-wheels.sh @@ -3,16 +3,25 @@ set -euo pipefail -if [[ "$#" -ne 1 || ! -d "$1" ]]; then - echo "usage: $0 WHEEL_OUTPUT_DIRECTORY" >&2 +if [[ "$#" -lt 1 || ! -d "$1" ]]; then + echo "usage: $0 WHEEL_OUTPUT_DIRECTORY [WHEEL ...]" >&2 exit 1 fi output_directory="$(realpath "$1")" descriptors='[]' wheel_count=0 +wheel_paths=() + +if [[ "$#" -gt 1 ]]; then + wheel_paths=("${@:2}") +else + while IFS= read -r wheel_path; do + wheel_paths+=("${wheel_path}") + done < <(find "${output_directory}" -type f -name '*.whl' -print | sort) +fi -while IFS= read -r wheel_path; do +for wheel_path in "${wheel_paths[@]}"; do wheel_path="$(realpath "${wheel_path}")" if [[ "${wheel_path}" != "${output_directory}"/* ]]; then echo "wheel must resolve inside output directory: ${wheel_path}" >&2 @@ -44,7 +53,7 @@ while IFS= read -r wheel_path; do --argjson current "${descriptors}" \ '$current + [{path: $path, package: {ecosystem: "wheel", name: $name, version: $version}}]')" wheel_count=$((wheel_count + 1)) -done < <(find "${output_directory}" -type f -name '*.whl' -print | sort) +done if [[ "${wheel_count}" -eq 0 ]]; then echo "wheel output directory contains no .whl files: ${output_directory}" >&2 diff --git a/release-catalog/materialize.sh b/release-catalog/materialize.sh index 24c45fcf..ffc9d92b 100755 --- a/release-catalog/materialize.sh +++ b/release-catalog/materialize.sh @@ -13,7 +13,6 @@ require_nonempty() { } require_nonempty "RELEASE_CATALOG_KEY" "${RELEASE_CATALOG_KEY:-}" -require_nonempty "RELEASE_ARTIFACT_TYPE" "${RELEASE_ARTIFACT_TYPE:-}" require_nonempty "RELEASE_OUTPUT_DIRECTORY" "${RELEASE_OUTPUT_DIRECTORY:-}" require_nonempty "RELEASE_ENTRIES_NAME" "${RELEASE_ENTRIES_NAME:-}" require_nonempty "RELEASE_ARTIFACTS" "${RELEASE_ARTIFACTS:-}" @@ -48,42 +47,6 @@ ensure_relative_pattern() { } output_directory="$(realpath "${RELEASE_OUTPUT_DIRECTORY}")" -package_identity='' -case "${RELEASE_ARTIFACT_TYPE}" in - custom) - require_nonempty "RELEASE_PACKAGE_IDENTITY_FILE" "${RELEASE_PACKAGE_IDENTITY_FILE:-}" - ensure_relative_pattern "package-identity-file" "${RELEASE_PACKAGE_IDENTITY_FILE}" - package_identity_path="$(realpath "${output_directory}/${RELEASE_PACKAGE_IDENTITY_FILE}")" - if [[ "${package_identity_path}" != "${output_directory}"/* || ! -f "${package_identity_path}" ]]; then - echo "package-identity-file must resolve to one file inside output-directory: ${RELEASE_PACKAGE_IDENTITY_FILE}" >&2 - exit 1 - fi - package_identity="$(jq -c . "${package_identity_path}")" - ;; - conda | wheel) - if [[ -n "${RELEASE_PACKAGE_IDENTITY_FILE:-}" ]]; then - echo "package-identity-file is only valid for custom artifacts" >&2 - exit 1 - fi - ;; - *) - echo "artifact-type must be one of: conda, custom, wheel" >&2 - exit 1 - ;; -esac - -if [[ "${RELEASE_ARTIFACT_TYPE}" == "custom" ]] && ! jq -e ' - type == "object" - and (keys - ["ecosystem", "name", "version", "build", "platform"] | length == 0) - and (.ecosystem | type == "string" and length > 0) - and (.name | type == "string" and length > 0) - and (.version | type == "string" and length > 0) - and ((has("build") | not) or (.build | type == "string" and length > 0)) - and ((has("platform") | not) or (.platform | type == "string" and length > 0)) -' <<<"${package_identity}" >/dev/null; then - echo "package identity file must contain non-empty ecosystem, name, and version strings and only optional build or platform strings" >&2 - exit 1 -fi if ! jq -e 'type == "array" and length > 0' <<<"${RELEASE_ARTIFACTS}" >/dev/null; then echo "release-artifacts must be a non-empty JSON array" >&2 @@ -119,6 +82,18 @@ resolve_one_file() { printf '%s\n' "${resolved#"${output_directory}/"}" } +validate_package_identity() { + jq -e ' + type == "object" + and (keys - ["ecosystem", "name", "version", "build", "platform"] | length == 0) + and (.ecosystem | type == "string" and length > 0) + and (.name | type == "string" and length > 0) + and (.version | type == "string" and length > 0) + and ((has("build") | not) or (.build | type == "string" and length > 0)) + and ((has("platform") | not) or (.platform | type == "string" and length > 0)) + ' >/dev/null +} + generated_evidence_path() { local primary_path="$1" local kind="$2" @@ -220,34 +195,17 @@ write_generated_provenance() { shopt -s globstar nullglob while IFS= read -r descriptor; do - if ! jq -e --arg artifact_type "${RELEASE_ARTIFACT_TYPE}" ' + if ! jq -e ' type == "object" - and ( - if $artifact_type == "custom" then - (keys - ["path", "sbom", "provenance", "signature"] | length == 0) - else - (keys - ["path", "sbom", "provenance", "signature", "package"] | length == 0) - end - ) + and (keys - ["path", "package_identity_file", "sbom", "provenance", "signature", "package"] | length == 0) and (.path | type == "string" and length > 0) + and ((.package_identity_file // "") | type == "string") and ((.sbom // "") | type == "string") and ((.provenance // "") | type == "string") and ((.signature // "") | type == "string") - and ( - if $artifact_type == "custom" then - has("package") | not - else - (.package | type == "object") - and (.package | keys - ["ecosystem", "name", "version", "build", "platform"] | length == 0) - and (.package.ecosystem | type == "string" and length > 0) - and (.package.name | type == "string" and length > 0) - and (.package.version | type == "string" and length > 0) - and ((.package | has("build") | not) or (.package.build | type == "string" and length > 0)) - and ((.package | has("platform") | not) or (.package.platform | type == "string" and length > 0)) - end - ) + and ([has("package"), has("package_identity_file")] | map(select(.)) | length == 1) ' <<<"${descriptor}" >/dev/null; then - echo "release artifact descriptor is invalid for ${RELEASE_ARTIFACT_TYPE}: ${descriptor}" >&2 + echo "release artifact descriptor must contain path, exactly one package identity source, and optional evidence paths: ${descriptor}" >&2 exit 1 fi @@ -255,11 +213,20 @@ while IFS= read -r descriptor; do sbom_pattern="$(jq -r '.sbom // empty' <<<"${descriptor}")" provenance_pattern="$(jq -r '.provenance // empty' <<<"${descriptor}")" signature_pattern="$(jq -r '.signature // empty' <<<"${descriptor}")" - if [[ "${RELEASE_ARTIFACT_TYPE}" == "custom" ]]; then - package="${package_identity}" + package_identity_file="$(jq -r '.package_identity_file // empty' <<<"${descriptor}")" + if [[ -n "${package_identity_file}" ]]; then + package_identity_path="$(resolve_one_file package_identity_file "${package_identity_file}")" + if ! package="$(jq -ce . "${output_directory}/${package_identity_path}" 2>/dev/null)"; then + echo "package_identity_file must contain valid JSON: ${package_identity_file}" >&2 + exit 1 + fi else package="$(jq -c '.package' <<<"${descriptor}")" fi + if ! validate_package_identity <<<"${package}"; then + echo "package identity for ${primary_path} must contain non-empty ecosystem, name, and version strings and only optional build or platform strings" >&2 + exit 1 + fi if [[ -n "${sbom_pattern}" ]]; then supplied_sbom_path="$(resolve_one_file sbom "${sbom_pattern}")" diff --git a/release-catalog/prepare.sh b/release-catalog/prepare.sh index 4a803d28..9246974e 100755 --- a/release-catalog/prepare.sh +++ b/release-catalog/prepare.sh @@ -12,31 +12,109 @@ require_nonempty() { fi } -require_nonempty "RELEASE_ARTIFACT_TYPE" "${RELEASE_ARTIFACT_TYPE:-}" require_nonempty "RELEASE_OUTPUT_DIRECTORY" "${RELEASE_OUTPUT_DIRECTORY:-}" script_directory="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" artifacts="${RELEASE_ARTIFACTS:-}" +if [[ ! -d "${RELEASE_OUTPUT_DIRECTORY}" ]]; then + echo "output-directory does not exist or is not a directory: ${RELEASE_OUTPUT_DIRECTORY}" >&2 + exit 1 +fi +output_directory="$(realpath "${RELEASE_OUTPUT_DIRECTORY}")" -case "${RELEASE_ARTIFACT_TYPE}" in - conda) - if [[ -z "${artifacts}" ]]; then - artifacts="$("${script_directory}/describe-conda.sh" "${RELEASE_OUTPUT_DIRECTORY}")" - fi - ;; - wheel) - if [[ -z "${artifacts}" ]]; then - artifacts="$("${script_directory}/describe-wheels.sh" "${RELEASE_OUTPUT_DIRECTORY}")" - fi - ;; - custom) - require_nonempty "RELEASE_ARTIFACTS" "${artifacts}" - require_nonempty "RELEASE_PACKAGE_IDENTITY_FILE" "${RELEASE_PACKAGE_IDENTITY_FILE:-}" - ;; - *) - echo "artifact-type must be one of: conda, custom, wheel" >&2 +ensure_relative_pattern() { + local field="$1" + local pattern="$2" + if [[ "${pattern}" == /* || "${pattern}" == */../* || "${pattern}" == ../* || "${pattern}" == *"/.." ]]; then + echo "${field} must be a relative path inside output-directory: ${pattern}" >&2 + exit 1 + fi +} + +resolve_primary_file() { + local pattern="$1" + local -a matches=() + + ensure_relative_pattern "artifact path" "${pattern}" + while IFS= read -r match; do + matches+=("${match}") + done < <(compgen -G "${output_directory}/${pattern}" || true) + if [[ "${#matches[@]}" -ne 1 || ! -f "${matches[0]:-}" ]]; then + echo "artifact path must resolve to exactly one file: ${pattern}" >&2 + exit 1 + fi + + local resolved + resolved="$(realpath "${matches[0]}")" + if [[ "${resolved}" != "${output_directory}"/* ]]; then + echo "artifact path must resolve inside output-directory: ${pattern}" >&2 exit 1 - ;; -esac + fi + printf '%s\n' "${resolved}" +} + +if [[ -z "${artifacts}" ]]; then + conda_descriptors='[]' + wheel_descriptors='[]' + if [[ -n "$(find "${output_directory}" -type f \( -name '*.conda' -o -name '*.tar.bz2' \) -print -quit)" ]]; then + conda_descriptors="$("${script_directory}/describe-conda.sh" "${output_directory}")" + fi + if [[ -n "$(find "${output_directory}" -type f -name '*.whl' -print -quit)" ]]; then + wheel_descriptors="$("${script_directory}/describe-wheels.sh" "${output_directory}")" + fi + artifacts="$(jq -cn --argjson conda "${conda_descriptors}" --argjson wheel "${wheel_descriptors}" '$conda + $wheel')" + if [[ "$(jq 'length' <<<"${artifacts}")" -eq 0 ]]; then + echo "output-directory contains no detectable Conda or wheel artifacts; explicitly selected artifacts require package_identity_file when their identity cannot be parsed" >&2 + exit 1 + fi +else + prepared_artifacts='[]' + while IFS= read -r descriptor; do + primary_file="$(resolve_primary_file "$(jq -r '.path' <<<"${descriptor}")")" + identity_file="$(jq -r '.package_identity_file // empty' <<<"${descriptor}")" + package='' + + case "${primary_file}" in + *.whl) + if [[ -n "${identity_file}" ]]; then + echo "package_identity_file is not allowed when wheel identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" >&2 + exit 1 + fi + package="$("${script_directory}/describe-wheels.sh" "${output_directory}" "${primary_file}" | jq -c '.[0].package')" + ;; + *.conda) + if [[ -n "${identity_file}" ]]; then + echo "package_identity_file is not allowed when Conda identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" >&2 + exit 1 + fi + package="$("${script_directory}/describe-conda.sh" "${output_directory}" "${primary_file}" | jq -c '.[0].package')" + ;; + *.tar.bz2) + if conda_descriptor="$("${script_directory}/describe-conda.sh" "${output_directory}" "${primary_file}" 2>/dev/null)"; then + if [[ -n "${identity_file}" ]]; then + echo "package_identity_file is not allowed when Conda identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" >&2 + exit 1 + fi + package="$(jq -c '.[0].package' <<<"${conda_descriptor}")" + elif [[ -z "${identity_file}" ]]; then + echo "artifact is not a valid Conda package and requires package_identity_file: $(jq -r '.path' <<<"${descriptor}")" >&2 + exit 1 + fi + ;; + *) + if [[ -z "${identity_file}" ]]; then + echo "artifact identity cannot be extracted; package_identity_file is required: $(jq -r '.path' <<<"${descriptor}")" >&2 + exit 1 + fi + ;; + esac + + if [[ -n "${package}" ]]; then + descriptor="$(jq -c --argjson package "${package}" '. + {package: $package}' <<<"${descriptor}")" + fi + prepared_artifacts="$(jq -cn --argjson current "${prepared_artifacts}" --argjson descriptor "${descriptor}" '$current + [$descriptor]')" + done < <(jq -c '.[]' <<<"${artifacts}") + artifacts="${prepared_artifacts}" +fi printf 'artifacts=%s\n' "${artifacts}" >>"${GITHUB_OUTPUT}" diff --git a/release-catalog/validate-config.sh b/release-catalog/validate-config.sh index 9cbfafd1..042a424a 100755 --- a/release-catalog/validate-config.sh +++ b/release-catalog/validate-config.sh @@ -26,35 +26,23 @@ validation_errors="$(jq -r ' if type != "object" then ["release catalog configuration must be a JSON object"] else - (keys - ["artifact_type", "release_catalog_key", "output_directory", "package_identity_file", "artifacts"]) as $unknown + (keys - ["release_catalog_key", "output_directory", "artifacts"]) as $unknown | [ if ($unknown | length) > 0 then "unknown field(s): " + ($unknown | join(", ")) else empty end, - if (.artifact_type == "conda" or .artifact_type == "custom" or .artifact_type == "wheel") then empty - else "artifact_type must be one of: conda, custom, wheel" end, if (.release_catalog_key | single_line_string) then empty else "release_catalog_key must be a non-empty, single-line string" end, if (has("output_directory") | not) or (.output_directory | single_line_string) then empty else "output_directory must be a non-empty, single-line string when supplied" end ] - + if .artifact_type == "custom" then + + if has("artifacts") then [ - if (.package_identity_file | single_line_string) then empty - else "package_identity_file is required for custom artifacts and must be a non-empty, single-line path relative to output_directory" end, if (.artifacts | type == "array" and length > 0) then empty - else "artifacts must be a non-empty array for custom artifacts" end - ] - elif .artifact_type == "conda" or .artifact_type == "wheel" then - [ - if (.output_directory | single_line_string) then empty - else "output_directory is required for conda and wheel artifacts" end, - if has("artifacts") or has("package_identity_file") then - "artifacts and package_identity_file are only valid when artifact_type is custom" - else empty end + else "artifacts must be a non-empty array when supplied" end ] else [] end - + if .artifact_type == "custom" and (.artifacts | type) == "array" then + + if (.artifacts | type) == "array" then [ .artifacts | to_entries[] | .key as $index @@ -62,14 +50,14 @@ validation_errors="$(jq -r ' | if ($artifact | type) != "object" then "artifacts[\($index)] must be an object" else - ($artifact | keys - ["path", "sbom", "provenance", "signature"]) as $artifact_unknown + ($artifact | keys - ["path", "package_identity_file", "sbom", "provenance", "signature"]) as $artifact_unknown | if ($artifact_unknown | length) > 0 then "artifacts[\($index)] has unknown field(s): " + ($artifact_unknown | join(", ")) else empty end, if ($artifact.path | single_line_string) then empty else "artifacts[\($index)].path must be a non-empty, single-line string" end, ($artifact | to_entries[] - | select(.key == "sbom" or .key == "provenance" or .key == "signature") + | select(.key == "package_identity_file" or .key == "sbom" or .key == "provenance" or .key == "signature") | select((.value | single_line_string) | not) | "artifacts[\($index)].\(.key) must be a non-empty, single-line string") end @@ -92,9 +80,7 @@ if [[ -z "${GITHUB_OUTPUT:-}" ]]; then fi { - printf 'artifact_type=%s\n' "$(jq -r '.artifact_type' <<<"${compact_config}")" printf 'release_catalog_key=%s\n' "$(jq -r '.release_catalog_key' <<<"${compact_config}")" printf 'output_directory=%s\n' "$(jq -r '.output_directory // "."' <<<"${compact_config}")" - printf 'package_identity_file=%s\n' "$(jq -r '.package_identity_file // empty' <<<"${compact_config}")" printf 'artifacts=%s\n' "$(jq -c '.artifacts // empty' <<<"${compact_config}")" } >>"${GITHUB_OUTPUT}" diff --git a/tests/release-catalog-config/package-identity-file.json b/tests/release-catalog-config/package-identity-file.json index 0c6bacd8..46068c04 100644 --- a/tests/release-catalog-config/package-identity-file.json +++ b/tests/release-catalog-config/package-identity-file.json @@ -1,11 +1,10 @@ { - "artifact_type": "custom", "release_catalog_key": "maven:cuvs-java", "output_directory": "java/cuvs-java/target", - "package_identity_file": "cuvs-java.release-package-identity.json", "artifacts": [ { "path": "cuvs-java-*-x86_64-cuda*.jar", + "package_identity_file": "cuvs-java.release-package-identity.json", "sbom": "cuvs-java.spdx.json" } ] diff --git a/tests/release_catalog_config_test.sh b/tests/release_catalog_config_test.sh index ccfe3959..c0776a4d 100755 --- a/tests/release_catalog_config_test.sh +++ b/tests/release_catalog_config_test.sh @@ -31,10 +31,8 @@ RELEASE_CATALOG_CONFIG="$(<"${repository_root}/tests/release-catalog-config/pack GITHUB_OUTPUT="${valid_output}" "${validator}" grep -Fx 'release_catalog_key=maven:cuvs-java' "${valid_output}" -grep -Fx 'artifact_type=custom' "${valid_output}" grep -Fx 'output_directory=java/cuvs-java/target' "${valid_output}" -grep -Fx 'package_identity_file=cuvs-java.release-package-identity.json' "${valid_output}" -grep -Fx 'artifacts=[{"path":"cuvs-java-*-x86_64-cuda*.jar","sbom":"cuvs-java.spdx.json"}]' "${valid_output}" +grep -Fx 'artifacts=[{"path":"cuvs-java-*-x86_64-cuda*.jar","package_identity_file":"cuvs-java.release-package-identity.json","sbom":"cuvs-java.spdx.json"}]' "${valid_output}" assert_invalid \ malformed-json \ @@ -50,54 +48,53 @@ assert_invalid \ missing-fields \ '{}' \ 'release_catalog_key must be a non-empty, single-line string' -grep -F 'artifact_type must be one of: conda, custom, wheel' "${temporary_directory}/missing-fields.error" >/dev/null assert_invalid \ - custom-missing-fields \ - '{"artifact_type":"custom","release_catalog_key":"archive:smoke"}' \ - 'package_identity_file is required for custom artifacts' -grep -F 'artifacts must be a non-empty array for custom artifacts' "${temporary_directory}/custom-missing-fields.error" >/dev/null + top-level-package-identity-file \ + '{"release_catalog_key":"archive:smoke","package_identity_file":"package.json"}' \ + 'unknown field(s): package_identity_file' assert_invalid \ unknown-field \ - '{"artifact_type":"custom","release_catalog_key":"archive:smoke","package_identity_file":"package.json","artifacts":[{"path":"smoke.tar.gz"}],"component_id":"archive:smoke"}' \ + '{"release_catalog_key":"archive:smoke","artifacts":[{"path":"smoke.tar.gz","package_identity_file":"package.json"}],"component_id":"archive:smoke"}' \ 'unknown field(s): component_id' +assert_invalid \ + artifact-type \ + '{"artifact_type":"custom","release_catalog_key":"archive:smoke","artifacts":[{"path":"smoke.tar.gz","package_identity_file":"package.json"}]}' \ + 'unknown field(s): artifact_type' + assert_invalid \ inline-package \ - '{"artifact_type":"custom","release_catalog_key":"archive:smoke","package_identity_file":"package.json","package":{"ecosystem":"archive","name":"smoke","version":"1.0"},"artifacts":[{"path":"smoke.tar.gz"}]}' \ + '{"release_catalog_key":"archive:smoke","package":{"ecosystem":"archive","name":"smoke","version":"1.0"},"artifacts":[{"path":"smoke.tar.gz","package_identity_file":"package.json"}]}' \ 'unknown field(s): package' assert_invalid \ per-artifact-package \ - '{"artifact_type":"custom","release_catalog_key":"archive:smoke","package_identity_file":"package.json","artifacts":[{"path":"smoke.tar.gz","package":{"ecosystem":"archive","name":"smoke","version":"1.0"}}]}' \ + '{"release_catalog_key":"archive:smoke","artifacts":[{"path":"smoke.tar.gz","package":{"ecosystem":"archive","name":"smoke","version":"1.0"}}]}' \ 'artifacts[0] has unknown field(s): package' assert_invalid \ malformed-artifact \ - '{"artifact_type":"custom","release_catalog_key":"archive:smoke","package_identity_file":"package.json","artifacts":[{"file":"smoke.tar.gz","sbom":false}]}' \ + '{"release_catalog_key":"archive:smoke","artifacts":[{"file":"smoke.tar.gz","package_identity_file":false,"sbom":false}]}' \ 'artifacts[0] has unknown field(s): file' grep -F 'artifacts[0].path must be a non-empty, single-line string' "${temporary_directory}/malformed-artifact.error" >/dev/null +grep -F 'artifacts[0].package_identity_file must be a non-empty, single-line string' "${temporary_directory}/malformed-artifact.error" >/dev/null grep -F 'artifacts[0].sbom must be a non-empty, single-line string' "${temporary_directory}/malformed-artifact.error" >/dev/null -assert_invalid \ - standard-missing-output-directory \ - '{"artifact_type":"conda","release_catalog_key":"conda:smoke"}' \ - 'output_directory is required for conda and wheel artifacts' - -assert_invalid \ - standard-custom-fields \ - '{"artifact_type":"wheel","release_catalog_key":"wheel:smoke","output_directory":"dist","artifacts":[{"path":"smoke.whl"}]}' \ - 'artifacts and package_identity_file are only valid when artifact_type is custom' +default_output="${temporary_directory}/default.output" +RELEASE_CATALOG_CONFIG='{"release_catalog_key":"conda:smoke"}' \ + GITHUB_OUTPUT="${default_output}" "${validator}" +grep -Fx 'release_catalog_key=conda:smoke' "${default_output}" +grep -Fx 'output_directory=.' "${default_output}" +grep -Fx 'artifacts=' "${default_output}" standard_output="${temporary_directory}/standard.output" RELEASE_CATALOG_CONFIG='{ - "artifact_type": "wheel", "release_catalog_key": "wheel:kvikio", "output_directory": "dist" }' GITHUB_OUTPUT="${standard_output}" "${validator}" -grep -Fx 'artifact_type=wheel' "${standard_output}" grep -Fx 'release_catalog_key=wheel:kvikio' "${standard_output}" grep -Fx 'output_directory=dist' "${standard_output}" grep -Fx 'artifacts=' "${standard_output}" diff --git a/tests/release_catalog_prepare_test.sh b/tests/release_catalog_prepare_test.sh index 399df071..f83e90db 100755 --- a/tests/release_catalog_prepare_test.sh +++ b/tests/release_catalog_prepare_test.sh @@ -20,10 +20,8 @@ printf '%s\n' \ GITHUB_OUTPUT="${temporary_directory}/wheel-output" RELEASE_ARTIFACTS='' -RELEASE_ARTIFACT_TYPE=wheel RELEASE_OUTPUT_DIRECTORY="${temporary_directory}" -RELEASE_PACKAGE_IDENTITY_FILE='' -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE_IDENTITY_FILE +export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY "${repository_root}/release-catalog/prepare.sh" @@ -35,21 +33,104 @@ jq -e ' }] ' <<<"${artifacts}" >/dev/null +conda_output_directory="${temporary_directory}/conda" +conda_staging_directory="${temporary_directory}/conda-staging" +mkdir -p "${conda_output_directory}/noarch" "${conda_staging_directory}/info" +jq -n \ + '{name: "rapids-dask-dependency", version: "26.08.0", build: "py_0", subdir: "noarch"}' \ + >"${conda_staging_directory}/info/index.json" +tar -cjf \ + "${conda_output_directory}/noarch/rapids-dask-dependency-26.08.0-py_0.tar.bz2" \ + -C "${conda_staging_directory}" \ + info/index.json + +GITHUB_OUTPUT="${temporary_directory}/conda-output" +RELEASE_OUTPUT_DIRECTORY="${conda_output_directory}" +export GITHUB_OUTPUT RELEASE_OUTPUT_DIRECTORY + +"${repository_root}/release-catalog/prepare.sh" + +artifacts="$(sed -n 's/^artifacts=//p' "${GITHUB_OUTPUT}")" +jq -e ' + . == [{ + path: "noarch/rapids-dask-dependency-26.08.0-py_0.tar.bz2", + package: { + ecosystem: "conda", + name: "rapids-dask-dependency", + version: "26.08.0", + build: "py_0", + platform: "noarch" + } + }] +' <<<"${artifacts}" >/dev/null + GITHUB_OUTPUT="${temporary_directory}/custom-output" -RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz", sbom: "bundle.spdx.json"}]')" -RELEASE_ARTIFACT_TYPE=custom -RELEASE_PACKAGE_IDENTITY_FILE=release-package-identity.json -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_PACKAGE_IDENTITY_FILE +RELEASE_OUTPUT_DIRECTORY="${temporary_directory}" +printf '%s\n' bundle >"${temporary_directory}/bundle.tar.gz" +jq -n '{ecosystem: "archive", name: "bundle", version: "1.0"}' \ + >"${temporary_directory}/release-package-identity.json" +RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz", package_identity_file: "release-package-identity.json", sbom: "bundle.spdx.json"}]')" +export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY "${repository_root}/release-catalog/prepare.sh" -grep -Fx 'artifacts=[{"path":"bundle.tar.gz","sbom":"bundle.spdx.json"}]' "${GITHUB_OUTPUT}" +grep -Fx 'artifacts=[{"path":"bundle.tar.gz","package_identity_file":"release-package-identity.json","sbom":"bundle.spdx.json"}]' "${GITHUB_OUTPUT}" + +GITHUB_OUTPUT="${temporary_directory}/explicit-wheel-output" +RELEASE_OUTPUT_DIRECTORY="${temporary_directory}" +RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl", signature: "libkvikio.sig"}]')" +export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY +"${repository_root}/release-catalog/prepare.sh" +artifacts="$(sed -n 's/^artifacts=//p' "${GITHUB_OUTPUT}")" +jq -e ' + . == [{ + path: "libkvikio_cu12-*.whl", + signature: "libkvikio.sig", + package: {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0"} + }] +' <<<"${artifacts}" >/dev/null + +mixed_directory="${temporary_directory}/mixed" +mkdir -p "${mixed_directory}" +cp "${temporary_directory}/libkvikio_cu12-26.8.0-py3-none-manylinux_2_28_x86_64.whl" "${mixed_directory}/" +cp "${conda_output_directory}/noarch/rapids-dask-dependency-26.08.0-py_0.tar.bz2" "${mixed_directory}/" +GITHUB_OUTPUT="${temporary_directory}/mixed-output" +RELEASE_ARTIFACTS='' +RELEASE_OUTPUT_DIRECTORY="${mixed_directory}" +export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY +"${repository_root}/release-catalog/prepare.sh" +artifacts="$(sed -n 's/^artifacts=//p' "${GITHUB_OUTPUT}")" +jq -e ' + length == 2 + and ([.[].package.ecosystem] | sort) == ["conda", "wheel"] +' <<<"${artifacts}" >/dev/null + +empty_directory="${temporary_directory}/empty" +mkdir -p "${empty_directory}" +GITHUB_OUTPUT="${temporary_directory}/empty-output" +RELEASE_OUTPUT_DIRECTORY="${empty_directory}" +export GITHUB_OUTPUT RELEASE_OUTPUT_DIRECTORY +if "${repository_root}/release-catalog/prepare.sh" 2>"${temporary_directory}/empty-error"; then + echo "prepare.sh unexpectedly accepted an output directory without detectable artifacts" >&2 + exit 1 +fi +grep -Fx 'output-directory contains no detectable Conda or wheel artifacts; explicitly selected artifacts require package_identity_file when their identity cannot be parsed' "${temporary_directory}/empty-error" + +GITHUB_OUTPUT="${temporary_directory}/unsupported-output" +RELEASE_OUTPUT_DIRECTORY="${temporary_directory}" +RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz"}]')" +export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY +if "${repository_root}/release-catalog/prepare.sh" 2>"${temporary_directory}/unsupported-error"; then + echo "prepare.sh unexpectedly accepted an unsupported artifact without identity" >&2 + exit 1 +fi +grep -Fx 'artifact identity cannot be extracted; package_identity_file is required: bundle.tar.gz' "${temporary_directory}/unsupported-error" -GITHUB_OUTPUT="${temporary_directory}/invalid-output" -RELEASE_ARTIFACT_TYPE=archive -export GITHUB_OUTPUT RELEASE_ARTIFACT_TYPE -if "${repository_root}/release-catalog/prepare.sh" 2>"${temporary_directory}/invalid-error"; then - echo "prepare.sh unexpectedly accepted an invalid artifact type" >&2 +GITHUB_OUTPUT="${temporary_directory}/conflicting-identity-output" +RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl", package_identity_file: "release-package-identity.json"}]')" +export GITHUB_OUTPUT RELEASE_ARTIFACTS +if "${repository_root}/release-catalog/prepare.sh" 2>"${temporary_directory}/conflicting-identity-error"; then + echo "prepare.sh unexpectedly accepted an identity file for a parseable artifact" >&2 exit 1 fi -grep -Fx 'artifact-type must be one of: conda, custom, wheel' "${temporary_directory}/invalid-error" +grep -Fx 'package_identity_file is not allowed when wheel identity can be extracted: libkvikio_cu12-*.whl' "${temporary_directory}/conflicting-identity-error" diff --git a/tests/release_catalog_test.sh b/tests/release_catalog_test.sh index db347cf3..50badf06 100755 --- a/tests/release_catalog_test.sh +++ b/tests/release_catalog_test.sh @@ -24,16 +24,14 @@ GITHUB_RUN_ID="1234" GITHUB_SHA="0123456789012345678901234567890123456789" GITHUB_WORKFLOW_REF="rapidsai/cuvs/.github/workflows/build.yaml@refs/heads/release/26.08" export GITHUB_OUTPUT -RELEASE_ARTIFACTS="$(jq -cn '[{path: "cuvs-java-*.jar", sbom: "cuvs-java-*.spdx.json", provenance: "cuvs-java-*.provenance.jsonl", signature: "cuvs-java-*.jar.asc"}]')" -RELEASE_ARTIFACT_TYPE=custom +RELEASE_ARTIFACTS="$(jq -cn '[{path: "cuvs-java-*.jar", package_identity_file: "cuvs-java-package-identity.json", sbom: "cuvs-java-*.spdx.json", provenance: "cuvs-java-*.provenance.jsonl", signature: "cuvs-java-*.jar.asc"}]')" RELEASE_ENTRIES_NAME="release-catalog-entries.json" RELEASE_OUTPUT_DIRECTORY="${bundle_directory}" -RELEASE_PACKAGE_IDENTITY_FILE="cuvs-java-package-identity.json" RELEASE_SOURCE_ARTIFACT_NAME="cuvs-java-cuda12.9.1" RELEASE_SOURCE_SHA="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" RELEASE_CATALOG_KEY="maven:cuvs-java" export GITHUB_REPOSITORY GITHUB_RUN_ATTEMPT GITHUB_RUN_ID GITHUB_SHA GITHUB_WORKFLOW_REF -export RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_ENTRIES_NAME RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE_IDENTITY_FILE +export RELEASE_ARTIFACTS RELEASE_ENTRIES_NAME RELEASE_OUTPUT_DIRECTORY export RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" @@ -73,18 +71,40 @@ while IFS= read -r evidence_path; do test -f "${isolated_companion_directory}/${evidence_path}" done < <(jq -r '.entries[] | .sbom, .provenance, (.signature // empty)' "${isolated_companion_directory}/release-catalog-entries.json") +multiple_identity_directory="${temporary_directory}/multiple-identities" +mkdir -p "${multiple_identity_directory}" +printf '%s\n' first >"${multiple_identity_directory}/first.tar.gz" +printf '%s\n' second >"${multiple_identity_directory}/second.jar" +jq -n '{ecosystem: "archive", name: "first", version: "1.0"}' \ + >"${multiple_identity_directory}/first.identity.json" +jq -n '{ecosystem: "maven", name: "example:second", version: "2.0"}' \ + >"${multiple_identity_directory}/second.identity.json" +RELEASE_ARTIFACTS="$(jq -cn '[ + {path: "first.tar.gz", package_identity_file: "first.identity.json"}, + {path: "second.jar", package_identity_file: "second.identity.json"} +]')" +RELEASE_OUTPUT_DIRECTORY="${multiple_identity_directory}" +RELEASE_SOURCE_ARTIFACT_NAME="multiple-identities" +RELEASE_SOURCE_SHA="eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" +RELEASE_CATALOG_KEY="archive:multiple" +export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY +"${repository_root}/release-catalog/materialize.sh" +jq -e ' + (.entries | length == 2) + and .entries[0].package == {ecosystem: "archive", name: "first", version: "1.0"} + and .entries[1].package == {ecosystem: "maven", name: "example:second", version: "2.0"} +' "${multiple_identity_directory}/release-catalog-entries.json" >/dev/null + generated_directory="${temporary_directory}/generated-bundle" mkdir -p "${generated_directory}/linux-64" printf '%s\n' conda >"${generated_directory}/linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.conda" RELEASE_ARTIFACTS="$(jq -cn '[{path: "linux-64/kvikio-*.conda", package: {ecosystem: "conda", name: "kvikio", version: "26.08.00a32"}}]')" -RELEASE_ARTIFACT_TYPE=conda RELEASE_OUTPUT_DIRECTORY="${generated_directory}" -RELEASE_PACKAGE_IDENTITY_FILE='' RELEASE_SOURCE_ARTIFACT_NAME="kvikio_conda_python_kvikio_x86_64_abi3_cu12" RELEASE_SOURCE_SHA="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" RELEASE_CATALOG_KEY="conda:kvikio" -export RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE_IDENTITY_FILE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY +export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" @@ -114,12 +134,11 @@ mkdir -p "${wheel_directory}" printf '%s\n' wheel >"${wheel_directory}/libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl" RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl", package: {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"}}]')" -RELEASE_ARTIFACT_TYPE=wheel RELEASE_OUTPUT_DIRECTORY="${wheel_directory}" RELEASE_SOURCE_ARTIFACT_NAME="kvikio_wheel_cpp_libkvikio_x86_64_cu12" RELEASE_SOURCE_SHA="cccccccccccccccccccccccccccccccccccccccc" RELEASE_CATALOG_KEY="wheel:kvikio" -export RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_OUTPUT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY +export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" @@ -137,17 +156,15 @@ jq -n \ '{ecosystem: "archive", name: "bundle"}' \ >"${missing_version_directory}/bundle-package-identity.json" -RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz"}]')" -RELEASE_ARTIFACT_TYPE=custom +RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz", package_identity_file: "bundle-package-identity.json"}]')" RELEASE_OUTPUT_DIRECTORY="${missing_version_directory}" -RELEASE_PACKAGE_IDENTITY_FILE='bundle-package-identity.json' RELEASE_SOURCE_ARTIFACT_NAME="bundle-archive" RELEASE_SOURCE_SHA="dddddddddddddddddddddddddddddddddddddddd" RELEASE_CATALOG_KEY="archive:bundle" -export RELEASE_ARTIFACTS RELEASE_ARTIFACT_TYPE RELEASE_OUTPUT_DIRECTORY RELEASE_PACKAGE_IDENTITY_FILE RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY +export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/missing-version-error"; then echo "materialize.sh unexpectedly accepted a custom artifact without a package version" >&2 exit 1 fi -grep -Fx 'package identity file must contain non-empty ecosystem, name, and version strings and only optional build or platform strings' "${temporary_directory}/missing-version-error" +grep -Fx 'package identity for bundle.tar.gz must contain non-empty ecosystem, name, and version strings and only optional build or platform strings' "${temporary_directory}/missing-version-error" From d56661cb230b5f42407f95a26ebbbda0427d1880 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 18 Aug 2026 11:16:23 -0500 Subject: [PATCH 10/36] rename release catalog artifact directory --- release-catalog-dispatch/action.yml | 8 ++-- release-catalog/README.md | 8 ++-- release-catalog/config.schema.json | 6 +-- release-catalog/describe-conda.sh | 14 +++--- release-catalog/describe-wheels.sh | 14 +++--- release-catalog/materialize.sh | 44 +++++++++---------- release-catalog/prepare.sh | 32 +++++++------- release-catalog/validate-config.sh | 8 ++-- .../package-identity-file.json | 2 +- tests/release_catalog_config_test.sh | 8 ++-- tests/release_catalog_prepare_test.sh | 32 +++++++------- tests/release_catalog_test.sh | 20 ++++----- 12 files changed, 98 insertions(+), 98 deletions(-) diff --git a/release-catalog-dispatch/action.yml b/release-catalog-dispatch/action.yml index c38cb790..e2cd123b 100644 --- a/release-catalog-dispatch/action.yml +++ b/release-catalog-dispatch/action.yml @@ -57,7 +57,7 @@ runs: env: IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} RELEASE_ARTIFACTS: ${{ steps.configuration.outputs.artifacts }} - RELEASE_OUTPUT_DIRECTORY: ${{ steps.configuration.outputs.output_directory }} + RELEASE_ARTIFACT_DIRECTORY: ${{ steps.configuration.outputs.artifact_directory }} run: "${IMPLEMENTATION_PATH}/prepare.sh" - id: materialize name: Materialize release catalog entries @@ -66,7 +66,7 @@ runs: IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} RELEASE_ARTIFACTS: ${{ steps.prepare.outputs.artifacts }} RELEASE_ENTRIES_NAME: release-catalog-entries.json - RELEASE_OUTPUT_DIRECTORY: ${{ steps.configuration.outputs.output_directory }} + RELEASE_ARTIFACT_DIRECTORY: ${{ steps.configuration.outputs.artifact_directory }} RELEASE_SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} RELEASE_SOURCE_SHA: ${{ inputs.source-sha || github.sha }} RELEASE_CATALOG_KEY: ${{ steps.configuration.outputs.release_catalog_key }} @@ -83,5 +83,5 @@ runs: if-no-files-found: error name: ${{ steps.companion-name.outputs.name }} path: | - ${{ steps.configuration.outputs.output_directory }}/release-catalog-entries.json - ${{ steps.configuration.outputs.output_directory }}/release-evidence/** + ${{ steps.configuration.outputs.artifact_directory }}/release-catalog-entries.json + ${{ steps.configuration.outputs.artifact_directory }}/release-evidence/** diff --git a/release-catalog/README.md b/release-catalog/README.md index 8b765b2a..8e26c08e 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -28,7 +28,7 @@ Every producer passes one `config` JSON object. Its canonical schema and field documentation are in [`config.schema.json`](config.schema.json). When `artifacts` is omitted, the action discovers every Conda and wheel output -in `output_directory` and extracts identity from each package independently. +in `artifact_directory` and extracts identity from each package independently. When `artifacts` is supplied, each descriptor is resolved independently: supported Conda and wheel files are parsed, while any other artifact requires its own `package_identity_file`. Evidence paths also belong to the individual @@ -83,7 +83,7 @@ the built source. config: >- { "release_catalog_key": "wheel:example", - "output_directory": ${{ toJSON(steps.package-name.outputs.WHEEL_OUTPUT_DIR) }} + "artifact_directory": ${{ toJSON(steps.package-name.outputs.WHEEL_OUTPUT_DIR) }} } source-artifact-name: ${{ steps.package-name.outputs.RAPIDS_PACKAGE_NAME }} source-sha: ${{ env.RAPIDS_SHA }} @@ -98,7 +98,7 @@ the built source. config: >- { "release_catalog_key": "maven:cuvs-java", - "output_directory": "java/cuvs-java/target", + "artifact_directory": "java/cuvs-java/target", "artifacts": [{ "path": "cuvs-java-*-x86_64-cuda*.jar", "package_identity_file": "cuvs-java.release-package-identity.json" @@ -111,7 +111,7 @@ the built source. Before the action runs, the producer creates `java/cuvs-java/target/cuvs-java.release-package-identity.json`. The artifact descriptor's `package_identity_file` value is the path to that file -relative to `output_directory`. For example: +relative to `artifact_directory`. For example: ```json { diff --git a/release-catalog/config.schema.json b/release-catalog/config.schema.json index c8402e1d..0f099e1b 100644 --- a/release-catalog/config.schema.json +++ b/release-catalog/config.schema.json @@ -15,7 +15,7 @@ "minLength": 1, "pattern": "^[^\\r\\n]+$" }, - "output_directory": { + "artifact_directory": { "description": "Base directory for artifact and evidence paths. Defaults to the job working directory.", "type": "string", "minLength": 1, @@ -23,7 +23,7 @@ "default": "." }, "artifacts": { - "description": "Optional explicit primary release files and their identity or evidence sidecars. When omitted, Conda and wheel artifacts are discovered in output_directory.", + "description": "Optional explicit primary release files and their identity or evidence sidecars. When omitted, Conda and wheel artifacts are discovered in artifact_directory.", "type": "array", "minItems": 1, "items": { @@ -45,7 +45,7 @@ "pattern": "^[^\\r\\n]+$" }, "package_identity_file": { - "description": "Path, relative to output_directory, to producer-created package identity JSON for this artifact. Required only when identity cannot be extracted from the artifact. The file must contain ecosystem, name, and version and may contain build and platform.", + "description": "Path, relative to artifact_directory, to producer-created package identity JSON for this artifact. Required only when identity cannot be extracted from the artifact. The file must contain ecosystem, name, and version and may contain build and platform.", "type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$" diff --git a/release-catalog/describe-conda.sh b/release-catalog/describe-conda.sh index 23bdcf47..868175f6 100755 --- a/release-catalog/describe-conda.sh +++ b/release-catalog/describe-conda.sh @@ -4,11 +4,11 @@ set -euo pipefail if [[ "$#" -lt 1 || ! -d "$1" ]]; then - echo "usage: $0 CONDA_OUTPUT_DIRECTORY [CONDA_PACKAGE ...]" >&2 + echo "usage: $0 CONDA_ARTIFACT_DIRECTORY [CONDA_PACKAGE ...]" >&2 exit 1 fi -output_directory="$(realpath "$1")" +artifact_directory="$(realpath "$1")" descriptors='[]' package_count=0 package_paths=() @@ -18,16 +18,16 @@ if [[ "$#" -gt 1 ]]; then else while IFS= read -r package_path; do package_paths+=("${package_path}") - done < <(find "${output_directory}" -type f \( -name '*.conda' -o -name '*.tar.bz2' \) -print | sort) + done < <(find "${artifact_directory}" -type f \( -name '*.conda' -o -name '*.tar.bz2' \) -print | sort) fi for package_path in "${package_paths[@]}"; do package_path="$(realpath "${package_path}")" - if [[ "${package_path}" != "${output_directory}"/* ]]; then - echo "Conda package must resolve inside output directory: ${package_path}" >&2 + if [[ "${package_path}" != "${artifact_directory}"/* ]]; then + echo "Conda package must resolve inside artifact directory: ${package_path}" >&2 exit 1 fi - relative_path="${package_path#"${output_directory}/"}" + relative_path="${package_path#"${artifact_directory}/"}" case "${package_path}" in *.conda) @@ -71,7 +71,7 @@ for package_path in "${package_paths[@]}"; do done if [[ "${package_count}" -eq 0 ]]; then - echo "Conda output directory contains no .conda or .tar.bz2 packages: ${output_directory}" >&2 + echo "Conda artifact directory contains no .conda or .tar.bz2 packages: ${artifact_directory}" >&2 exit 1 fi diff --git a/release-catalog/describe-wheels.sh b/release-catalog/describe-wheels.sh index f440055f..8cf2e8ad 100755 --- a/release-catalog/describe-wheels.sh +++ b/release-catalog/describe-wheels.sh @@ -4,11 +4,11 @@ set -euo pipefail if [[ "$#" -lt 1 || ! -d "$1" ]]; then - echo "usage: $0 WHEEL_OUTPUT_DIRECTORY [WHEEL ...]" >&2 + echo "usage: $0 WHEEL_ARTIFACT_DIRECTORY [WHEEL ...]" >&2 exit 1 fi -output_directory="$(realpath "$1")" +artifact_directory="$(realpath "$1")" descriptors='[]' wheel_count=0 wheel_paths=() @@ -18,16 +18,16 @@ if [[ "$#" -gt 1 ]]; then else while IFS= read -r wheel_path; do wheel_paths+=("${wheel_path}") - done < <(find "${output_directory}" -type f -name '*.whl' -print | sort) + done < <(find "${artifact_directory}" -type f -name '*.whl' -print | sort) fi for wheel_path in "${wheel_paths[@]}"; do wheel_path="$(realpath "${wheel_path}")" - if [[ "${wheel_path}" != "${output_directory}"/* ]]; then - echo "wheel must resolve inside output directory: ${wheel_path}" >&2 + if [[ "${wheel_path}" != "${artifact_directory}"/* ]]; then + echo "wheel must resolve inside artifact directory: ${wheel_path}" >&2 exit 1 fi - relative_path="${wheel_path#"${output_directory}/"}" + relative_path="${wheel_path#"${artifact_directory}/"}" metadata_members=() while IFS= read -r metadata_member; do @@ -56,7 +56,7 @@ for wheel_path in "${wheel_paths[@]}"; do done if [[ "${wheel_count}" -eq 0 ]]; then - echo "wheel output directory contains no .whl files: ${output_directory}" >&2 + echo "wheel artifact directory contains no .whl files: ${artifact_directory}" >&2 exit 1 fi diff --git a/release-catalog/materialize.sh b/release-catalog/materialize.sh index ffc9d92b..57317441 100755 --- a/release-catalog/materialize.sh +++ b/release-catalog/materialize.sh @@ -13,7 +13,7 @@ require_nonempty() { } require_nonempty "RELEASE_CATALOG_KEY" "${RELEASE_CATALOG_KEY:-}" -require_nonempty "RELEASE_OUTPUT_DIRECTORY" "${RELEASE_OUTPUT_DIRECTORY:-}" +require_nonempty "RELEASE_ARTIFACT_DIRECTORY" "${RELEASE_ARTIFACT_DIRECTORY:-}" require_nonempty "RELEASE_ENTRIES_NAME" "${RELEASE_ENTRIES_NAME:-}" require_nonempty "RELEASE_ARTIFACTS" "${RELEASE_ARTIFACTS:-}" require_nonempty "RELEASE_SOURCE_ARTIFACT_NAME" "${RELEASE_SOURCE_ARTIFACT_NAME:-}" @@ -32,8 +32,8 @@ require_plain_filename() { require_plain_filename "entries-name" "${RELEASE_ENTRIES_NAME}" -if [[ ! -d "${RELEASE_OUTPUT_DIRECTORY}" ]]; then - echo "output-directory does not exist or is not a directory: ${RELEASE_OUTPUT_DIRECTORY}" >&2 +if [[ ! -d "${RELEASE_ARTIFACT_DIRECTORY}" ]]; then + echo "artifact-directory does not exist or is not a directory: ${RELEASE_ARTIFACT_DIRECTORY}" >&2 exit 1 fi @@ -41,20 +41,20 @@ ensure_relative_pattern() { local field="$1" local pattern="$2" if [[ "${pattern}" == /* || "${pattern}" == */../* || "${pattern}" == ../* || "${pattern}" == *"/.." ]]; then - echo "${field} must be a relative path inside output-directory: ${pattern}" >&2 + echo "${field} must be a relative path inside artifact-directory: ${pattern}" >&2 exit 1 fi } -output_directory="$(realpath "${RELEASE_OUTPUT_DIRECTORY}")" +artifact_directory="$(realpath "${RELEASE_ARTIFACT_DIRECTORY}")" if ! jq -e 'type == "array" and length > 0' <<<"${RELEASE_ARTIFACTS}" >/dev/null; then echo "release-artifacts must be a non-empty JSON array" >&2 exit 1 fi -entries_path="${output_directory}/${RELEASE_ENTRIES_NAME}" -temporary_manifest="$(mktemp "${output_directory}/.release-catalog.XXXXXX")" +entries_path="${artifact_directory}/${RELEASE_ENTRIES_NAME}" +temporary_manifest="$(mktemp "${artifact_directory}/.release-catalog.XXXXXX")" trap 'rm -f "${temporary_manifest}"' EXIT printf '%s\n' '{"entries":[]}' >"${temporary_manifest}" @@ -67,7 +67,7 @@ resolve_one_file() { ensure_relative_pattern "${field}" "${pattern}" while IFS= read -r match; do matches+=("${match}") - done < <(compgen -G "${output_directory}/${pattern}" || true) + done < <(compgen -G "${artifact_directory}/${pattern}" || true) if [[ "${#matches[@]}" -ne 1 || ! -f "${matches[0]:-}" ]]; then echo "${field} pattern must resolve to exactly one file: ${pattern}" >&2 exit 1 @@ -75,11 +75,11 @@ resolve_one_file() { local resolved resolved="$(realpath "${matches[0]}")" - if [[ "${resolved}" != "${output_directory}"/* ]]; then - echo "${field} must resolve inside output-directory: ${pattern}" >&2 + if [[ "${resolved}" != "${artifact_directory}"/* ]]; then + echo "${field} must resolve inside artifact-directory: ${pattern}" >&2 exit 1 fi - printf '%s\n' "${resolved#"${output_directory}/"}" + printf '%s\n' "${resolved#"${artifact_directory}/"}" } validate_package_identity() { @@ -98,7 +98,7 @@ generated_evidence_path() { local primary_path="$1" local kind="$2" local artifact_digest - artifact_digest="$(sha256sum "${output_directory}/${primary_path}" | awk '{print $1}')" + artifact_digest="$(sha256sum "${artifact_directory}/${primary_path}" | awk '{print $1}')" local artifact_label="${primary_path//\//_}" printf 'release-evidence/%s.%s.%s.json\n' "${artifact_label}" "${artifact_digest}" "${kind}" } @@ -108,12 +108,12 @@ copy_supplied_evidence() { local supplied_path="$2" local kind="$3" local artifact_digest - artifact_digest="$(sha256sum "${output_directory}/${primary_path}" | awk '{print $1}')" + artifact_digest="$(sha256sum "${artifact_directory}/${primary_path}" | awk '{print $1}')" local artifact_label="${primary_path//\//_}" local destination destination="release-evidence/${artifact_label}.${artifact_digest}/${kind}-$(basename "${supplied_path}")" - mkdir -p "$(dirname "${output_directory}/${destination}")" - cp "${output_directory}/${supplied_path}" "${output_directory}/${destination}" + mkdir -p "$(dirname "${artifact_directory}/${destination}")" + cp "${artifact_directory}/${supplied_path}" "${artifact_directory}/${destination}" printf '%s\n' "${destination}" } @@ -122,8 +122,8 @@ write_generated_sbom() { local package="$2" local destination="$3" local artifact_digest - artifact_digest="$(sha256sum "${output_directory}/${primary_path}" | awk '{print $1}')" - mkdir -p "$(dirname "${output_directory}/${destination}")" + artifact_digest="$(sha256sum "${artifact_directory}/${primary_path}" | awk '{print $1}')" + mkdir -p "$(dirname "${artifact_directory}/${destination}")" jq -n -S \ --arg artifact_digest "${artifact_digest}" \ --arg artifact_path "${primary_path}" \ @@ -153,7 +153,7 @@ write_generated_sbom() { relatedSpdxElement: "SPDXRef-Artifact" }], comment: "Artifact-identity SBOM envelope. A producer-supplied dependency SBOM may replace this record." - }' >"${output_directory}/${destination}" + }' >"${artifact_directory}/${destination}" } write_generated_provenance() { @@ -161,8 +161,8 @@ write_generated_provenance() { local package="$2" local destination="$3" local artifact_digest - artifact_digest="$(sha256sum "${output_directory}/${primary_path}" | awk '{print $1}')" - mkdir -p "$(dirname "${output_directory}/${destination}")" + artifact_digest="$(sha256sum "${artifact_directory}/${primary_path}" | awk '{print $1}')" + mkdir -p "$(dirname "${artifact_directory}/${destination}")" jq -n -S \ --arg artifact_digest "${artifact_digest}" \ --arg artifact_path "${primary_path}" \ @@ -190,7 +190,7 @@ write_generated_provenance() { metadata: {invocationId: ("https://github.com/" + $repository + "/actions/runs/" + $run_id + "/attempts/" + $run_attempt)} } } - }' >"${output_directory}/${destination}" + }' >"${artifact_directory}/${destination}" } shopt -s globstar nullglob @@ -216,7 +216,7 @@ while IFS= read -r descriptor; do package_identity_file="$(jq -r '.package_identity_file // empty' <<<"${descriptor}")" if [[ -n "${package_identity_file}" ]]; then package_identity_path="$(resolve_one_file package_identity_file "${package_identity_file}")" - if ! package="$(jq -ce . "${output_directory}/${package_identity_path}" 2>/dev/null)"; then + if ! package="$(jq -ce . "${artifact_directory}/${package_identity_path}" 2>/dev/null)"; then echo "package_identity_file must contain valid JSON: ${package_identity_file}" >&2 exit 1 fi diff --git a/release-catalog/prepare.sh b/release-catalog/prepare.sh index 9246974e..241e8724 100755 --- a/release-catalog/prepare.sh +++ b/release-catalog/prepare.sh @@ -12,21 +12,21 @@ require_nonempty() { fi } -require_nonempty "RELEASE_OUTPUT_DIRECTORY" "${RELEASE_OUTPUT_DIRECTORY:-}" +require_nonempty "RELEASE_ARTIFACT_DIRECTORY" "${RELEASE_ARTIFACT_DIRECTORY:-}" script_directory="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" artifacts="${RELEASE_ARTIFACTS:-}" -if [[ ! -d "${RELEASE_OUTPUT_DIRECTORY}" ]]; then - echo "output-directory does not exist or is not a directory: ${RELEASE_OUTPUT_DIRECTORY}" >&2 +if [[ ! -d "${RELEASE_ARTIFACT_DIRECTORY}" ]]; then + echo "artifact-directory does not exist or is not a directory: ${RELEASE_ARTIFACT_DIRECTORY}" >&2 exit 1 fi -output_directory="$(realpath "${RELEASE_OUTPUT_DIRECTORY}")" +artifact_directory="$(realpath "${RELEASE_ARTIFACT_DIRECTORY}")" ensure_relative_pattern() { local field="$1" local pattern="$2" if [[ "${pattern}" == /* || "${pattern}" == */../* || "${pattern}" == ../* || "${pattern}" == *"/.." ]]; then - echo "${field} must be a relative path inside output-directory: ${pattern}" >&2 + echo "${field} must be a relative path inside artifact-directory: ${pattern}" >&2 exit 1 fi } @@ -38,7 +38,7 @@ resolve_primary_file() { ensure_relative_pattern "artifact path" "${pattern}" while IFS= read -r match; do matches+=("${match}") - done < <(compgen -G "${output_directory}/${pattern}" || true) + done < <(compgen -G "${artifact_directory}/${pattern}" || true) if [[ "${#matches[@]}" -ne 1 || ! -f "${matches[0]:-}" ]]; then echo "artifact path must resolve to exactly one file: ${pattern}" >&2 exit 1 @@ -46,8 +46,8 @@ resolve_primary_file() { local resolved resolved="$(realpath "${matches[0]}")" - if [[ "${resolved}" != "${output_directory}"/* ]]; then - echo "artifact path must resolve inside output-directory: ${pattern}" >&2 + if [[ "${resolved}" != "${artifact_directory}"/* ]]; then + echo "artifact path must resolve inside artifact-directory: ${pattern}" >&2 exit 1 fi printf '%s\n' "${resolved}" @@ -56,15 +56,15 @@ resolve_primary_file() { if [[ -z "${artifacts}" ]]; then conda_descriptors='[]' wheel_descriptors='[]' - if [[ -n "$(find "${output_directory}" -type f \( -name '*.conda' -o -name '*.tar.bz2' \) -print -quit)" ]]; then - conda_descriptors="$("${script_directory}/describe-conda.sh" "${output_directory}")" + if [[ -n "$(find "${artifact_directory}" -type f \( -name '*.conda' -o -name '*.tar.bz2' \) -print -quit)" ]]; then + conda_descriptors="$("${script_directory}/describe-conda.sh" "${artifact_directory}")" fi - if [[ -n "$(find "${output_directory}" -type f -name '*.whl' -print -quit)" ]]; then - wheel_descriptors="$("${script_directory}/describe-wheels.sh" "${output_directory}")" + if [[ -n "$(find "${artifact_directory}" -type f -name '*.whl' -print -quit)" ]]; then + wheel_descriptors="$("${script_directory}/describe-wheels.sh" "${artifact_directory}")" fi artifacts="$(jq -cn --argjson conda "${conda_descriptors}" --argjson wheel "${wheel_descriptors}" '$conda + $wheel')" if [[ "$(jq 'length' <<<"${artifacts}")" -eq 0 ]]; then - echo "output-directory contains no detectable Conda or wheel artifacts; explicitly selected artifacts require package_identity_file when their identity cannot be parsed" >&2 + echo "artifact-directory contains no detectable Conda or wheel artifacts; explicitly selected artifacts require package_identity_file when their identity cannot be parsed" >&2 exit 1 fi else @@ -80,17 +80,17 @@ else echo "package_identity_file is not allowed when wheel identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" >&2 exit 1 fi - package="$("${script_directory}/describe-wheels.sh" "${output_directory}" "${primary_file}" | jq -c '.[0].package')" + package="$("${script_directory}/describe-wheels.sh" "${artifact_directory}" "${primary_file}" | jq -c '.[0].package')" ;; *.conda) if [[ -n "${identity_file}" ]]; then echo "package_identity_file is not allowed when Conda identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" >&2 exit 1 fi - package="$("${script_directory}/describe-conda.sh" "${output_directory}" "${primary_file}" | jq -c '.[0].package')" + package="$("${script_directory}/describe-conda.sh" "${artifact_directory}" "${primary_file}" | jq -c '.[0].package')" ;; *.tar.bz2) - if conda_descriptor="$("${script_directory}/describe-conda.sh" "${output_directory}" "${primary_file}" 2>/dev/null)"; then + if conda_descriptor="$("${script_directory}/describe-conda.sh" "${artifact_directory}" "${primary_file}" 2>/dev/null)"; then if [[ -n "${identity_file}" ]]; then echo "package_identity_file is not allowed when Conda identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" >&2 exit 1 diff --git a/release-catalog/validate-config.sh b/release-catalog/validate-config.sh index 042a424a..6a5496ec 100755 --- a/release-catalog/validate-config.sh +++ b/release-catalog/validate-config.sh @@ -26,15 +26,15 @@ validation_errors="$(jq -r ' if type != "object" then ["release catalog configuration must be a JSON object"] else - (keys - ["release_catalog_key", "output_directory", "artifacts"]) as $unknown + (keys - ["release_catalog_key", "artifact_directory", "artifacts"]) as $unknown | [ if ($unknown | length) > 0 then "unknown field(s): " + ($unknown | join(", ")) else empty end, if (.release_catalog_key | single_line_string) then empty else "release_catalog_key must be a non-empty, single-line string" end, - if (has("output_directory") | not) or (.output_directory | single_line_string) then empty - else "output_directory must be a non-empty, single-line string when supplied" end + if (has("artifact_directory") | not) or (.artifact_directory | single_line_string) then empty + else "artifact_directory must be a non-empty, single-line string when supplied" end ] + if has("artifacts") then [ @@ -81,6 +81,6 @@ fi { printf 'release_catalog_key=%s\n' "$(jq -r '.release_catalog_key' <<<"${compact_config}")" - printf 'output_directory=%s\n' "$(jq -r '.output_directory // "."' <<<"${compact_config}")" + printf 'artifact_directory=%s\n' "$(jq -r '.artifact_directory // "."' <<<"${compact_config}")" printf 'artifacts=%s\n' "$(jq -c '.artifacts // empty' <<<"${compact_config}")" } >>"${GITHUB_OUTPUT}" diff --git a/tests/release-catalog-config/package-identity-file.json b/tests/release-catalog-config/package-identity-file.json index 46068c04..99047495 100644 --- a/tests/release-catalog-config/package-identity-file.json +++ b/tests/release-catalog-config/package-identity-file.json @@ -1,6 +1,6 @@ { "release_catalog_key": "maven:cuvs-java", - "output_directory": "java/cuvs-java/target", + "artifact_directory": "java/cuvs-java/target", "artifacts": [ { "path": "cuvs-java-*-x86_64-cuda*.jar", diff --git a/tests/release_catalog_config_test.sh b/tests/release_catalog_config_test.sh index c0776a4d..8bfcce06 100755 --- a/tests/release_catalog_config_test.sh +++ b/tests/release_catalog_config_test.sh @@ -31,7 +31,7 @@ RELEASE_CATALOG_CONFIG="$(<"${repository_root}/tests/release-catalog-config/pack GITHUB_OUTPUT="${valid_output}" "${validator}" grep -Fx 'release_catalog_key=maven:cuvs-java' "${valid_output}" -grep -Fx 'output_directory=java/cuvs-java/target' "${valid_output}" +grep -Fx 'artifact_directory=java/cuvs-java/target' "${valid_output}" grep -Fx 'artifacts=[{"path":"cuvs-java-*-x86_64-cuda*.jar","package_identity_file":"cuvs-java.release-package-identity.json","sbom":"cuvs-java.spdx.json"}]' "${valid_output}" assert_invalid \ @@ -86,15 +86,15 @@ default_output="${temporary_directory}/default.output" RELEASE_CATALOG_CONFIG='{"release_catalog_key":"conda:smoke"}' \ GITHUB_OUTPUT="${default_output}" "${validator}" grep -Fx 'release_catalog_key=conda:smoke' "${default_output}" -grep -Fx 'output_directory=.' "${default_output}" +grep -Fx 'artifact_directory=.' "${default_output}" grep -Fx 'artifacts=' "${default_output}" standard_output="${temporary_directory}/standard.output" RELEASE_CATALOG_CONFIG='{ "release_catalog_key": "wheel:kvikio", - "output_directory": "dist" + "artifact_directory": "dist" }' GITHUB_OUTPUT="${standard_output}" "${validator}" grep -Fx 'release_catalog_key=wheel:kvikio' "${standard_output}" -grep -Fx 'output_directory=dist' "${standard_output}" +grep -Fx 'artifact_directory=dist' "${standard_output}" grep -Fx 'artifacts=' "${standard_output}" diff --git a/tests/release_catalog_prepare_test.sh b/tests/release_catalog_prepare_test.sh index f83e90db..f897badc 100755 --- a/tests/release_catalog_prepare_test.sh +++ b/tests/release_catalog_prepare_test.sh @@ -20,8 +20,8 @@ printf '%s\n' \ GITHUB_OUTPUT="${temporary_directory}/wheel-output" RELEASE_ARTIFACTS='' -RELEASE_OUTPUT_DIRECTORY="${temporary_directory}" -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY +RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" +export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY "${repository_root}/release-catalog/prepare.sh" @@ -45,8 +45,8 @@ tar -cjf \ info/index.json GITHUB_OUTPUT="${temporary_directory}/conda-output" -RELEASE_OUTPUT_DIRECTORY="${conda_output_directory}" -export GITHUB_OUTPUT RELEASE_OUTPUT_DIRECTORY +RELEASE_ARTIFACT_DIRECTORY="${conda_output_directory}" +export GITHUB_OUTPUT RELEASE_ARTIFACT_DIRECTORY "${repository_root}/release-catalog/prepare.sh" @@ -65,21 +65,21 @@ jq -e ' ' <<<"${artifacts}" >/dev/null GITHUB_OUTPUT="${temporary_directory}/custom-output" -RELEASE_OUTPUT_DIRECTORY="${temporary_directory}" +RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" printf '%s\n' bundle >"${temporary_directory}/bundle.tar.gz" jq -n '{ecosystem: "archive", name: "bundle", version: "1.0"}' \ >"${temporary_directory}/release-package-identity.json" RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz", package_identity_file: "release-package-identity.json", sbom: "bundle.spdx.json"}]')" -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY +export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY "${repository_root}/release-catalog/prepare.sh" grep -Fx 'artifacts=[{"path":"bundle.tar.gz","package_identity_file":"release-package-identity.json","sbom":"bundle.spdx.json"}]' "${GITHUB_OUTPUT}" GITHUB_OUTPUT="${temporary_directory}/explicit-wheel-output" -RELEASE_OUTPUT_DIRECTORY="${temporary_directory}" +RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl", signature: "libkvikio.sig"}]')" -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY +export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY "${repository_root}/release-catalog/prepare.sh" artifacts="$(sed -n 's/^artifacts=//p' "${GITHUB_OUTPUT}")" jq -e ' @@ -96,8 +96,8 @@ cp "${temporary_directory}/libkvikio_cu12-26.8.0-py3-none-manylinux_2_28_x86_64. cp "${conda_output_directory}/noarch/rapids-dask-dependency-26.08.0-py_0.tar.bz2" "${mixed_directory}/" GITHUB_OUTPUT="${temporary_directory}/mixed-output" RELEASE_ARTIFACTS='' -RELEASE_OUTPUT_DIRECTORY="${mixed_directory}" -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY +RELEASE_ARTIFACT_DIRECTORY="${mixed_directory}" +export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY "${repository_root}/release-catalog/prepare.sh" artifacts="$(sed -n 's/^artifacts=//p' "${GITHUB_OUTPUT}")" jq -e ' @@ -108,18 +108,18 @@ jq -e ' empty_directory="${temporary_directory}/empty" mkdir -p "${empty_directory}" GITHUB_OUTPUT="${temporary_directory}/empty-output" -RELEASE_OUTPUT_DIRECTORY="${empty_directory}" -export GITHUB_OUTPUT RELEASE_OUTPUT_DIRECTORY +RELEASE_ARTIFACT_DIRECTORY="${empty_directory}" +export GITHUB_OUTPUT RELEASE_ARTIFACT_DIRECTORY if "${repository_root}/release-catalog/prepare.sh" 2>"${temporary_directory}/empty-error"; then - echo "prepare.sh unexpectedly accepted an output directory without detectable artifacts" >&2 + echo "prepare.sh unexpectedly accepted an artifact directory without detectable artifacts" >&2 exit 1 fi -grep -Fx 'output-directory contains no detectable Conda or wheel artifacts; explicitly selected artifacts require package_identity_file when their identity cannot be parsed' "${temporary_directory}/empty-error" +grep -Fx 'artifact-directory contains no detectable Conda or wheel artifacts; explicitly selected artifacts require package_identity_file when their identity cannot be parsed' "${temporary_directory}/empty-error" GITHUB_OUTPUT="${temporary_directory}/unsupported-output" -RELEASE_OUTPUT_DIRECTORY="${temporary_directory}" +RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz"}]')" -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY +export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY if "${repository_root}/release-catalog/prepare.sh" 2>"${temporary_directory}/unsupported-error"; then echo "prepare.sh unexpectedly accepted an unsupported artifact without identity" >&2 exit 1 diff --git a/tests/release_catalog_test.sh b/tests/release_catalog_test.sh index 50badf06..5278994b 100755 --- a/tests/release_catalog_test.sh +++ b/tests/release_catalog_test.sh @@ -26,12 +26,12 @@ GITHUB_WORKFLOW_REF="rapidsai/cuvs/.github/workflows/build.yaml@refs/heads/relea export GITHUB_OUTPUT RELEASE_ARTIFACTS="$(jq -cn '[{path: "cuvs-java-*.jar", package_identity_file: "cuvs-java-package-identity.json", sbom: "cuvs-java-*.spdx.json", provenance: "cuvs-java-*.provenance.jsonl", signature: "cuvs-java-*.jar.asc"}]')" RELEASE_ENTRIES_NAME="release-catalog-entries.json" -RELEASE_OUTPUT_DIRECTORY="${bundle_directory}" +RELEASE_ARTIFACT_DIRECTORY="${bundle_directory}" RELEASE_SOURCE_ARTIFACT_NAME="cuvs-java-cuda12.9.1" RELEASE_SOURCE_SHA="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" RELEASE_CATALOG_KEY="maven:cuvs-java" export GITHUB_REPOSITORY GITHUB_RUN_ATTEMPT GITHUB_RUN_ID GITHUB_SHA GITHUB_WORKFLOW_REF -export RELEASE_ARTIFACTS RELEASE_ENTRIES_NAME RELEASE_OUTPUT_DIRECTORY +export RELEASE_ARTIFACTS RELEASE_ENTRIES_NAME RELEASE_ARTIFACT_DIRECTORY export RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" @@ -83,11 +83,11 @@ RELEASE_ARTIFACTS="$(jq -cn '[ {path: "first.tar.gz", package_identity_file: "first.identity.json"}, {path: "second.jar", package_identity_file: "second.identity.json"} ]')" -RELEASE_OUTPUT_DIRECTORY="${multiple_identity_directory}" +RELEASE_ARTIFACT_DIRECTORY="${multiple_identity_directory}" RELEASE_SOURCE_ARTIFACT_NAME="multiple-identities" RELEASE_SOURCE_SHA="eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" RELEASE_CATALOG_KEY="archive:multiple" -export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" jq -e ' (.entries | length == 2) @@ -100,11 +100,11 @@ mkdir -p "${generated_directory}/linux-64" printf '%s\n' conda >"${generated_directory}/linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.conda" RELEASE_ARTIFACTS="$(jq -cn '[{path: "linux-64/kvikio-*.conda", package: {ecosystem: "conda", name: "kvikio", version: "26.08.00a32"}}]')" -RELEASE_OUTPUT_DIRECTORY="${generated_directory}" +RELEASE_ARTIFACT_DIRECTORY="${generated_directory}" RELEASE_SOURCE_ARTIFACT_NAME="kvikio_conda_python_kvikio_x86_64_abi3_cu12" RELEASE_SOURCE_SHA="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" RELEASE_CATALOG_KEY="conda:kvikio" -export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" @@ -134,11 +134,11 @@ mkdir -p "${wheel_directory}" printf '%s\n' wheel >"${wheel_directory}/libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl" RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl", package: {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"}}]')" -RELEASE_OUTPUT_DIRECTORY="${wheel_directory}" +RELEASE_ARTIFACT_DIRECTORY="${wheel_directory}" RELEASE_SOURCE_ARTIFACT_NAME="kvikio_wheel_cpp_libkvikio_x86_64_cu12" RELEASE_SOURCE_SHA="cccccccccccccccccccccccccccccccccccccccc" RELEASE_CATALOG_KEY="wheel:kvikio" -export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" @@ -157,11 +157,11 @@ jq -n \ >"${missing_version_directory}/bundle-package-identity.json" RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz", package_identity_file: "bundle-package-identity.json"}]')" -RELEASE_OUTPUT_DIRECTORY="${missing_version_directory}" +RELEASE_ARTIFACT_DIRECTORY="${missing_version_directory}" RELEASE_SOURCE_ARTIFACT_NAME="bundle-archive" RELEASE_SOURCE_SHA="dddddddddddddddddddddddddddddddddddddddd" RELEASE_CATALOG_KEY="archive:bundle" -export RELEASE_ARTIFACTS RELEASE_OUTPUT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/missing-version-error"; then echo "materialize.sh unexpectedly accepted a custom artifact without a package version" >&2 From 99a77c5fcfce899e6a55ef9efbf1f5c2d85d5bf8 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 18 Aug 2026 11:53:25 -0500 Subject: [PATCH 11/36] consolidate release catalog implementation --- release-catalog-dispatch/action.yml | 10 +- release-catalog/describe-conda.sh | 78 -------- release-catalog/describe-wheels.sh | 63 ------ release-catalog/materialize.sh | 185 ++++++++++++++++-- release-catalog/prepare.sh | 120 ------------ ...t.sh => release_catalog_discovery_test.sh} | 64 +++--- ...> release_catalog_package_parsing_test.sh} | 42 +++- tests/release_catalog_test.sh | 16 +- 8 files changed, 252 insertions(+), 326 deletions(-) delete mode 100755 release-catalog/describe-conda.sh delete mode 100755 release-catalog/describe-wheels.sh delete mode 100755 release-catalog/prepare.sh rename tests/{release_catalog_prepare_test.sh => release_catalog_discovery_test.sh} (66%) rename tests/{release_catalog_descriptors_test.sh => release_catalog_package_parsing_test.sh} (60%) diff --git a/release-catalog-dispatch/action.yml b/release-catalog-dispatch/action.yml index e2cd123b..9c2e9257 100644 --- a/release-catalog-dispatch/action.yml +++ b/release-catalog-dispatch/action.yml @@ -51,20 +51,12 @@ runs: IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} RELEASE_CATALOG_CONFIG: ${{ inputs.config }} run: "${IMPLEMENTATION_PATH}/validate-config.sh" - - id: prepare - name: Describe release artifacts - shell: bash - env: - IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} - RELEASE_ARTIFACTS: ${{ steps.configuration.outputs.artifacts }} - RELEASE_ARTIFACT_DIRECTORY: ${{ steps.configuration.outputs.artifact_directory }} - run: "${IMPLEMENTATION_PATH}/prepare.sh" - id: materialize name: Materialize release catalog entries shell: bash env: IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} - RELEASE_ARTIFACTS: ${{ steps.prepare.outputs.artifacts }} + RELEASE_ARTIFACTS: ${{ steps.configuration.outputs.artifacts }} RELEASE_ENTRIES_NAME: release-catalog-entries.json RELEASE_ARTIFACT_DIRECTORY: ${{ steps.configuration.outputs.artifact_directory }} RELEASE_SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} diff --git a/release-catalog/describe-conda.sh b/release-catalog/describe-conda.sh deleted file mode 100755 index 868175f6..00000000 --- a/release-catalog/describe-conda.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - -set -euo pipefail - -if [[ "$#" -lt 1 || ! -d "$1" ]]; then - echo "usage: $0 CONDA_ARTIFACT_DIRECTORY [CONDA_PACKAGE ...]" >&2 - exit 1 -fi - -artifact_directory="$(realpath "$1")" -descriptors='[]' -package_count=0 -package_paths=() - -if [[ "$#" -gt 1 ]]; then - package_paths=("${@:2}") -else - while IFS= read -r package_path; do - package_paths+=("${package_path}") - done < <(find "${artifact_directory}" -type f \( -name '*.conda' -o -name '*.tar.bz2' \) -print | sort) -fi - -for package_path in "${package_paths[@]}"; do - package_path="$(realpath "${package_path}")" - if [[ "${package_path}" != "${artifact_directory}"/* ]]; then - echo "Conda package must resolve inside artifact directory: ${package_path}" >&2 - exit 1 - fi - relative_path="${package_path#"${artifact_directory}/"}" - - case "${package_path}" in - *.conda) - info_members=() - while IFS= read -r info_member; do - info_members+=("${info_member}") - done < <(unzip -Z1 "${package_path}" | awk '/^info-.*\.tar\.zst$/') - if [[ "${#info_members[@]}" -ne 1 ]]; then - echo ".conda package must contain exactly one info-*.tar.zst member: ${relative_path}" >&2 - exit 1 - fi - index_json="$(unzip -p "${package_path}" "${info_members[0]}" | zstd -dc | tar -xOf - info/index.json)" - ;; - *.tar.bz2) - index_json="$(tar -xOjf "${package_path}" info/index.json)" - ;; - *) - echo "unsupported Conda package extension: ${relative_path}" >&2 - exit 1 - ;; - esac - - if ! jq -e ' - type == "object" - and (.name | type == "string" and length > 0) - and (.version | type == "string" and length > 0) - and (.build | type == "string" and length > 0) - and (.subdir | type == "string" and length > 0) - ' <<<"${index_json}" >/dev/null; then - echo "Conda info/index.json must contain exact name, version, build, and subdir fields: ${relative_path}" >&2 - exit 1 - fi - - package="$(jq -c '{ecosystem: "conda", name, version, build, platform: .subdir}' <<<"${index_json}")" - descriptors="$(jq -cn \ - --arg path "${relative_path}" \ - --argjson package "${package}" \ - --argjson current "${descriptors}" \ - '$current + [{path: $path, package: $package}]')" - package_count=$((package_count + 1)) -done - -if [[ "${package_count}" -eq 0 ]]; then - echo "Conda artifact directory contains no .conda or .tar.bz2 packages: ${artifact_directory}" >&2 - exit 1 -fi - -printf '%s\n' "${descriptors}" diff --git a/release-catalog/describe-wheels.sh b/release-catalog/describe-wheels.sh deleted file mode 100755 index 8cf2e8ad..00000000 --- a/release-catalog/describe-wheels.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - -set -euo pipefail - -if [[ "$#" -lt 1 || ! -d "$1" ]]; then - echo "usage: $0 WHEEL_ARTIFACT_DIRECTORY [WHEEL ...]" >&2 - exit 1 -fi - -artifact_directory="$(realpath "$1")" -descriptors='[]' -wheel_count=0 -wheel_paths=() - -if [[ "$#" -gt 1 ]]; then - wheel_paths=("${@:2}") -else - while IFS= read -r wheel_path; do - wheel_paths+=("${wheel_path}") - done < <(find "${artifact_directory}" -type f -name '*.whl' -print | sort) -fi - -for wheel_path in "${wheel_paths[@]}"; do - wheel_path="$(realpath "${wheel_path}")" - if [[ "${wheel_path}" != "${artifact_directory}"/* ]]; then - echo "wheel must resolve inside artifact directory: ${wheel_path}" >&2 - exit 1 - fi - relative_path="${wheel_path#"${artifact_directory}/"}" - - metadata_members=() - while IFS= read -r metadata_member; do - metadata_members+=("${metadata_member}") - done < <(unzip -Z1 "${wheel_path}" | awk '/\.dist-info\/METADATA$/') - if [[ "${#metadata_members[@]}" -ne 1 ]]; then - echo "wheel must contain exactly one .dist-info/METADATA file: ${relative_path}" >&2 - exit 1 - fi - - metadata="$(unzip -p "${wheel_path}" "${metadata_members[0]}")" - package_name="$(awk 'tolower($0) ~ /^name:[[:space:]]*/ {sub(/^[^:]*:[[:space:]]*/, ""); sub(/\r$/, ""); print; exit}' <<<"${metadata}")" - package_version="$(awk 'tolower($0) ~ /^version:[[:space:]]*/ {sub(/^[^:]*:[[:space:]]*/, ""); sub(/\r$/, ""); print; exit}' <<<"${metadata}")" - if [[ -z "${package_name}" || -z "${package_version}" ]]; then - echo "wheel metadata must contain non-empty Name and Version fields: ${relative_path}" >&2 - exit 1 - fi - - descriptors="$(jq -cn \ - --arg path "${relative_path}" \ - --arg name "${package_name}" \ - --arg version "${package_version}" \ - --argjson current "${descriptors}" \ - '$current + [{path: $path, package: {ecosystem: "wheel", name: $name, version: $version}}]')" - wheel_count=$((wheel_count + 1)) -done - -if [[ "${wheel_count}" -eq 0 ]]; then - echo "wheel artifact directory contains no .whl files: ${artifact_directory}" >&2 - exit 1 -fi - -printf '%s\n' "${descriptors}" diff --git a/release-catalog/materialize.sh b/release-catalog/materialize.sh index 57317441..1a353a15 100755 --- a/release-catalog/materialize.sh +++ b/release-catalog/materialize.sh @@ -15,7 +15,6 @@ require_nonempty() { require_nonempty "RELEASE_CATALOG_KEY" "${RELEASE_CATALOG_KEY:-}" require_nonempty "RELEASE_ARTIFACT_DIRECTORY" "${RELEASE_ARTIFACT_DIRECTORY:-}" require_nonempty "RELEASE_ENTRIES_NAME" "${RELEASE_ENTRIES_NAME:-}" -require_nonempty "RELEASE_ARTIFACTS" "${RELEASE_ARTIFACTS:-}" require_nonempty "RELEASE_SOURCE_ARTIFACT_NAME" "${RELEASE_SOURCE_ARTIFACT_NAME:-}" source_sha="${RELEASE_SOURCE_SHA:-${GITHUB_SHA:-}}" @@ -48,17 +47,6 @@ ensure_relative_pattern() { artifact_directory="$(realpath "${RELEASE_ARTIFACT_DIRECTORY}")" -if ! jq -e 'type == "array" and length > 0' <<<"${RELEASE_ARTIFACTS}" >/dev/null; then - echo "release-artifacts must be a non-empty JSON array" >&2 - exit 1 -fi - -entries_path="${artifact_directory}/${RELEASE_ENTRIES_NAME}" -temporary_manifest="$(mktemp "${artifact_directory}/.release-catalog.XXXXXX")" -trap 'rm -f "${temporary_manifest}"' EXIT - -printf '%s\n' '{"entries":[]}' >"${temporary_manifest}" - resolve_one_file() { local field="$1" local pattern="$2" @@ -82,6 +70,175 @@ resolve_one_file() { printf '%s\n' "${resolved#"${artifact_directory}/"}" } +describe_wheel_package() { + local wheel_path="$1" + local relative_path="${wheel_path#"${artifact_directory}/"}" + local -a metadata_members=() + local metadata_member + while IFS= read -r metadata_member; do + metadata_members+=("${metadata_member}") + done < <(unzip -Z1 "${wheel_path}" | awk '/\.dist-info\/METADATA$/') + if [[ "${#metadata_members[@]}" -ne 1 ]]; then + echo "wheel must contain exactly one .dist-info/METADATA file: ${relative_path}" >&2 + exit 1 + fi + + local metadata package_name package_version + metadata="$(unzip -p "${wheel_path}" "${metadata_members[0]}")" + package_name="$(awk 'tolower($0) ~ /^name:[[:space:]]*/ {sub(/^[^:]*:[[:space:]]*/, ""); sub(/\r$/, ""); print; exit}' <<<"${metadata}")" + package_version="$(awk 'tolower($0) ~ /^version:[[:space:]]*/ {sub(/^[^:]*:[[:space:]]*/, ""); sub(/\r$/, ""); print; exit}' <<<"${metadata}")" + if [[ -z "${package_name}" || -z "${package_version}" ]]; then + echo "wheel metadata must contain non-empty Name and Version fields: ${relative_path}" >&2 + exit 1 + fi + + jq -cn --arg name "${package_name}" --arg version "${package_version}" \ + '{ecosystem: "wheel", name: $name, version: $version}' +} + +describe_conda_package() { + local package_path="$1" + local relative_path="${package_path#"${artifact_directory}/"}" + local index_json + case "${package_path}" in + *.conda) + local -a info_members=() + local info_member + while IFS= read -r info_member; do + info_members+=("${info_member}") + done < <(unzip -Z1 "${package_path}" | awk '/^info-.*\.tar\.zst$/') + if [[ "${#info_members[@]}" -ne 1 ]]; then + echo ".conda package must contain exactly one info-*.tar.zst member: ${relative_path}" >&2 + exit 1 + fi + index_json="$(unzip -p "${package_path}" "${info_members[0]}" | zstd -dc | tar -xOf - info/index.json)" + ;; + *.tar.bz2) + index_json="$(tar -xOjf "${package_path}" info/index.json)" + ;; + *) + echo "unsupported Conda package extension: ${relative_path}" >&2 + exit 1 + ;; + esac + + if ! jq -e ' + type == "object" + and (.name | type == "string" and length > 0) + and (.version | type == "string" and length > 0) + and (.build | type == "string" and length > 0) + and (.subdir | type == "string" and length > 0) + ' <<<"${index_json}" >/dev/null; then + echo "Conda info/index.json must contain exact name, version, build, and subdir fields: ${relative_path}" >&2 + exit 1 + fi + + jq -c '{ecosystem: "conda", name, version, build, platform: .subdir}' <<<"${index_json}" +} + +prepare_artifacts() { + local configured_artifacts="${RELEASE_ARTIFACTS:-}" + local prepared_artifacts='[]' + local conda_package descriptor package primary_path primary_file identity_file + + if [[ -z "${configured_artifacts}" ]]; then + local -a detected_files=() + while IFS= read -r primary_file; do + detected_files+=("${primary_file}") + done < <(find "${artifact_directory}" -type f \( -name '*.conda' -o -name '*.tar.bz2' -o -name '*.whl' \) -print | sort) + + for primary_file in "${detected_files[@]}"; do + primary_path="${primary_file#"${artifact_directory}/"}" + case "${primary_file}" in + *.whl) + if ! package="$(describe_wheel_package "${primary_file}")"; then + return 1 + fi + ;; + *) + if ! package="$(describe_conda_package "${primary_file}")"; then + return 1 + fi + ;; + esac + descriptor="$(jq -cn --arg path "${primary_path}" --argjson package "${package}" '{path: $path, package: $package}')" + prepared_artifacts="$(jq -cn --argjson current "${prepared_artifacts}" --argjson descriptor "${descriptor}" '$current + [$descriptor]')" + done + else + if ! jq -e 'type == "array" and length > 0' <<<"${configured_artifacts}" >/dev/null; then + echo "release-artifacts must be a non-empty JSON array when supplied" >&2 + exit 1 + fi + while IFS= read -r descriptor; do + if jq -e 'has("package")' <<<"${descriptor}" >/dev/null; then + echo "package identity must not be supplied inline: ${descriptor}" >&2 + return 1 + fi + primary_path="$(resolve_one_file path "$(jq -r '.path' <<<"${descriptor}")")" + primary_file="${artifact_directory}/${primary_path}" + identity_file="$(jq -r '.package_identity_file // empty' <<<"${descriptor}")" + package='' + + case "${primary_file}" in + *.whl) + if [[ -n "${identity_file}" ]]; then + echo "package_identity_file is not allowed when wheel identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" >&2 + exit 1 + fi + if ! package="$(describe_wheel_package "${primary_file}")"; then + return 1 + fi + ;; + *.conda) + if [[ -n "${identity_file}" ]]; then + echo "package_identity_file is not allowed when Conda identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" >&2 + exit 1 + fi + if ! package="$(describe_conda_package "${primary_file}")"; then + return 1 + fi + ;; + *.tar.bz2) + if conda_package="$(describe_conda_package "${primary_file}" 2>/dev/null)"; then + if [[ -n "${identity_file}" ]]; then + echo "package_identity_file is not allowed when Conda identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" >&2 + exit 1 + fi + package="${conda_package}" + elif [[ -z "${identity_file}" ]]; then + echo "artifact is not a valid Conda package and requires package_identity_file: $(jq -r '.path' <<<"${descriptor}")" >&2 + exit 1 + fi + ;; + *) + if [[ -z "${identity_file}" ]]; then + echo "artifact identity cannot be extracted; package_identity_file is required: $(jq -r '.path' <<<"${descriptor}")" >&2 + exit 1 + fi + ;; + esac + + if [[ -n "${package}" ]]; then + descriptor="$(jq -c --argjson package "${package}" '. + {package: $package}' <<<"${descriptor}")" + fi + prepared_artifacts="$(jq -cn --argjson current "${prepared_artifacts}" --argjson descriptor "${descriptor}" '$current + [$descriptor]')" + done < <(jq -c '.[]' <<<"${configured_artifacts}") + fi + + if [[ "$(jq 'length' <<<"${prepared_artifacts}")" -eq 0 ]]; then + echo "artifact-directory contains no detectable Conda or wheel artifacts; explicitly selected artifacts require package_identity_file when their identity cannot be parsed" >&2 + exit 1 + fi + printf '%s\n' "${prepared_artifacts}" +} + +artifacts="$(prepare_artifacts)" +entries_path="${artifact_directory}/${RELEASE_ENTRIES_NAME}" +temporary_manifest="$(mktemp "${artifact_directory}/.release-catalog.XXXXXX")" +trap 'rm -f "${temporary_manifest}"' EXIT + +printf '%s\n' '{"entries":[]}' >"${temporary_manifest}" + validate_package_identity() { jq -e ' type == "object" @@ -133,6 +290,8 @@ write_generated_sbom() { dataLicense: "CC0-1.0", SPDXID: "SPDXRef-DOCUMENT", name: ("RAPIDS release artifact " + $artifact_path), + # This is a unique identifier, not a retrieval URL; SPDX requires an + # absolute URI for the document namespace but does not require it to resolve. documentNamespace: ("https://rapids.ai/release-platform/spdx/" + $artifact_digest), creationInfo: { creators: ["Tool: rapidsai/shared-workflows release catalog"], @@ -260,7 +419,7 @@ while IFS= read -r descriptor; do jq --argjson entry "${entry}" '.entries += [$entry]' "${temporary_manifest}" >"${temporary_manifest}.next" mv "${temporary_manifest}.next" "${temporary_manifest}" -done < <(jq -c '.[]' <<<"${RELEASE_ARTIFACTS}") +done < <(jq -c '.[]' <<<"${artifacts}") if ! jq -e '.entries as $items | ($items | map([.release_catalog_key, .path] | join("\u0000")) | unique | length) == ($items | length)' "${temporary_manifest}" >/dev/null; then echo "release-artifacts contains duplicate release_catalog_key/path entries" >&2 diff --git a/release-catalog/prepare.sh b/release-catalog/prepare.sh deleted file mode 100755 index 241e8724..00000000 --- a/release-catalog/prepare.sh +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - -set -euo pipefail - -require_nonempty() { - local name="$1" - local value="$2" - if [[ -z "${value}" ]]; then - echo "${name} must be a non-empty string" >&2 - exit 1 - fi -} - -require_nonempty "RELEASE_ARTIFACT_DIRECTORY" "${RELEASE_ARTIFACT_DIRECTORY:-}" - -script_directory="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -artifacts="${RELEASE_ARTIFACTS:-}" -if [[ ! -d "${RELEASE_ARTIFACT_DIRECTORY}" ]]; then - echo "artifact-directory does not exist or is not a directory: ${RELEASE_ARTIFACT_DIRECTORY}" >&2 - exit 1 -fi -artifact_directory="$(realpath "${RELEASE_ARTIFACT_DIRECTORY}")" - -ensure_relative_pattern() { - local field="$1" - local pattern="$2" - if [[ "${pattern}" == /* || "${pattern}" == */../* || "${pattern}" == ../* || "${pattern}" == *"/.." ]]; then - echo "${field} must be a relative path inside artifact-directory: ${pattern}" >&2 - exit 1 - fi -} - -resolve_primary_file() { - local pattern="$1" - local -a matches=() - - ensure_relative_pattern "artifact path" "${pattern}" - while IFS= read -r match; do - matches+=("${match}") - done < <(compgen -G "${artifact_directory}/${pattern}" || true) - if [[ "${#matches[@]}" -ne 1 || ! -f "${matches[0]:-}" ]]; then - echo "artifact path must resolve to exactly one file: ${pattern}" >&2 - exit 1 - fi - - local resolved - resolved="$(realpath "${matches[0]}")" - if [[ "${resolved}" != "${artifact_directory}"/* ]]; then - echo "artifact path must resolve inside artifact-directory: ${pattern}" >&2 - exit 1 - fi - printf '%s\n' "${resolved}" -} - -if [[ -z "${artifacts}" ]]; then - conda_descriptors='[]' - wheel_descriptors='[]' - if [[ -n "$(find "${artifact_directory}" -type f \( -name '*.conda' -o -name '*.tar.bz2' \) -print -quit)" ]]; then - conda_descriptors="$("${script_directory}/describe-conda.sh" "${artifact_directory}")" - fi - if [[ -n "$(find "${artifact_directory}" -type f -name '*.whl' -print -quit)" ]]; then - wheel_descriptors="$("${script_directory}/describe-wheels.sh" "${artifact_directory}")" - fi - artifacts="$(jq -cn --argjson conda "${conda_descriptors}" --argjson wheel "${wheel_descriptors}" '$conda + $wheel')" - if [[ "$(jq 'length' <<<"${artifacts}")" -eq 0 ]]; then - echo "artifact-directory contains no detectable Conda or wheel artifacts; explicitly selected artifacts require package_identity_file when their identity cannot be parsed" >&2 - exit 1 - fi -else - prepared_artifacts='[]' - while IFS= read -r descriptor; do - primary_file="$(resolve_primary_file "$(jq -r '.path' <<<"${descriptor}")")" - identity_file="$(jq -r '.package_identity_file // empty' <<<"${descriptor}")" - package='' - - case "${primary_file}" in - *.whl) - if [[ -n "${identity_file}" ]]; then - echo "package_identity_file is not allowed when wheel identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" >&2 - exit 1 - fi - package="$("${script_directory}/describe-wheels.sh" "${artifact_directory}" "${primary_file}" | jq -c '.[0].package')" - ;; - *.conda) - if [[ -n "${identity_file}" ]]; then - echo "package_identity_file is not allowed when Conda identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" >&2 - exit 1 - fi - package="$("${script_directory}/describe-conda.sh" "${artifact_directory}" "${primary_file}" | jq -c '.[0].package')" - ;; - *.tar.bz2) - if conda_descriptor="$("${script_directory}/describe-conda.sh" "${artifact_directory}" "${primary_file}" 2>/dev/null)"; then - if [[ -n "${identity_file}" ]]; then - echo "package_identity_file is not allowed when Conda identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" >&2 - exit 1 - fi - package="$(jq -c '.[0].package' <<<"${conda_descriptor}")" - elif [[ -z "${identity_file}" ]]; then - echo "artifact is not a valid Conda package and requires package_identity_file: $(jq -r '.path' <<<"${descriptor}")" >&2 - exit 1 - fi - ;; - *) - if [[ -z "${identity_file}" ]]; then - echo "artifact identity cannot be extracted; package_identity_file is required: $(jq -r '.path' <<<"${descriptor}")" >&2 - exit 1 - fi - ;; - esac - - if [[ -n "${package}" ]]; then - descriptor="$(jq -c --argjson package "${package}" '. + {package: $package}' <<<"${descriptor}")" - fi - prepared_artifacts="$(jq -cn --argjson current "${prepared_artifacts}" --argjson descriptor "${descriptor}" '$current + [$descriptor]')" - done < <(jq -c '.[]' <<<"${artifacts}") - artifacts="${prepared_artifacts}" -fi - -printf 'artifacts=%s\n' "${artifacts}" >>"${GITHUB_OUTPUT}" diff --git a/tests/release_catalog_prepare_test.sh b/tests/release_catalog_discovery_test.sh similarity index 66% rename from tests/release_catalog_prepare_test.sh rename to tests/release_catalog_discovery_test.sh index f897badc..711a8089 100755 --- a/tests/release_catalog_prepare_test.sh +++ b/tests/release_catalog_discovery_test.sh @@ -7,6 +7,12 @@ repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" temporary_directory="$(mktemp -d)" trap 'rm -rf "${temporary_directory}"' EXIT +RELEASE_CATALOG_KEY="test:discovery" +RELEASE_ENTRIES_NAME="release-catalog-entries.json" +RELEASE_SOURCE_ARTIFACT_NAME="discovery-test" +RELEASE_SOURCE_SHA="0123456789012345678901234567890123456789" +export RELEASE_CATALOG_KEY RELEASE_ENTRIES_NAME RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA + mkdir -p "${temporary_directory}/wheel/libkvikio_cu12-26.8.0.dist-info" printf '%s\n' \ 'Metadata-Version: 2.1' \ @@ -23,15 +29,14 @@ RELEASE_ARTIFACTS='' RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY -"${repository_root}/release-catalog/prepare.sh" +"${repository_root}/release-catalog/materialize.sh" -artifacts="$(sed -n 's/^artifacts=//p' "${GITHUB_OUTPUT}")" jq -e ' - . == [{ + [.entries[] | {path, package}] == [{ path: "libkvikio_cu12-26.8.0-py3-none-manylinux_2_28_x86_64.whl", package: {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0"} }] -' <<<"${artifacts}" >/dev/null +' "${temporary_directory}/release-catalog-entries.json" >/dev/null conda_output_directory="${temporary_directory}/conda" conda_staging_directory="${temporary_directory}/conda-staging" @@ -48,11 +53,10 @@ GITHUB_OUTPUT="${temporary_directory}/conda-output" RELEASE_ARTIFACT_DIRECTORY="${conda_output_directory}" export GITHUB_OUTPUT RELEASE_ARTIFACT_DIRECTORY -"${repository_root}/release-catalog/prepare.sh" +"${repository_root}/release-catalog/materialize.sh" -artifacts="$(sed -n 's/^artifacts=//p' "${GITHUB_OUTPUT}")" jq -e ' - . == [{ + [.entries[] | {path, package}] == [{ path: "noarch/rapids-dask-dependency-26.08.0-py_0.tar.bz2", package: { ecosystem: "conda", @@ -62,33 +66,36 @@ jq -e ' platform: "noarch" } }] -' <<<"${artifacts}" >/dev/null +' "${conda_output_directory}/release-catalog-entries.json" >/dev/null GITHUB_OUTPUT="${temporary_directory}/custom-output" RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" printf '%s\n' bundle >"${temporary_directory}/bundle.tar.gz" +printf '%s\n' sbom >"${temporary_directory}/bundle.spdx.json" jq -n '{ecosystem: "archive", name: "bundle", version: "1.0"}' \ >"${temporary_directory}/release-package-identity.json" RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz", package_identity_file: "release-package-identity.json", sbom: "bundle.spdx.json"}]')" export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY -"${repository_root}/release-catalog/prepare.sh" +"${repository_root}/release-catalog/materialize.sh" -grep -Fx 'artifacts=[{"path":"bundle.tar.gz","package_identity_file":"release-package-identity.json","sbom":"bundle.spdx.json"}]' "${GITHUB_OUTPUT}" +jq -e ' + .entries[0].path == "bundle.tar.gz" + and .entries[0].package == {ecosystem: "archive", name: "bundle", version: "1.0"} + and .entries[0].sbom_kind == "producer-dependency" +' "${temporary_directory}/release-catalog-entries.json" >/dev/null GITHUB_OUTPUT="${temporary_directory}/explicit-wheel-output" RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" +printf '%s\n' signature >"${temporary_directory}/libkvikio.sig" RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl", signature: "libkvikio.sig"}]')" export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY -"${repository_root}/release-catalog/prepare.sh" -artifacts="$(sed -n 's/^artifacts=//p' "${GITHUB_OUTPUT}")" +"${repository_root}/release-catalog/materialize.sh" jq -e ' - . == [{ - path: "libkvikio_cu12-*.whl", - signature: "libkvikio.sig", - package: {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0"} - }] -' <<<"${artifacts}" >/dev/null + .entries[0].path == "libkvikio_cu12-26.8.0-py3-none-manylinux_2_28_x86_64.whl" + and .entries[0].package == {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0"} + and (.entries[0].signature | startswith("release-evidence/")) +' "${temporary_directory}/release-catalog-entries.json" >/dev/null mixed_directory="${temporary_directory}/mixed" mkdir -p "${mixed_directory}" @@ -98,20 +105,19 @@ GITHUB_OUTPUT="${temporary_directory}/mixed-output" RELEASE_ARTIFACTS='' RELEASE_ARTIFACT_DIRECTORY="${mixed_directory}" export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY -"${repository_root}/release-catalog/prepare.sh" -artifacts="$(sed -n 's/^artifacts=//p' "${GITHUB_OUTPUT}")" +"${repository_root}/release-catalog/materialize.sh" jq -e ' - length == 2 - and ([.[].package.ecosystem] | sort) == ["conda", "wheel"] -' <<<"${artifacts}" >/dev/null + (.entries | length == 2) + and ([.entries[].package.ecosystem] | sort) == ["conda", "wheel"] +' "${mixed_directory}/release-catalog-entries.json" >/dev/null empty_directory="${temporary_directory}/empty" mkdir -p "${empty_directory}" GITHUB_OUTPUT="${temporary_directory}/empty-output" RELEASE_ARTIFACT_DIRECTORY="${empty_directory}" export GITHUB_OUTPUT RELEASE_ARTIFACT_DIRECTORY -if "${repository_root}/release-catalog/prepare.sh" 2>"${temporary_directory}/empty-error"; then - echo "prepare.sh unexpectedly accepted an artifact directory without detectable artifacts" >&2 +if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/empty-error"; then + echo "materialize.sh unexpectedly accepted an artifact directory without detectable artifacts" >&2 exit 1 fi grep -Fx 'artifact-directory contains no detectable Conda or wheel artifacts; explicitly selected artifacts require package_identity_file when their identity cannot be parsed' "${temporary_directory}/empty-error" @@ -120,8 +126,8 @@ GITHUB_OUTPUT="${temporary_directory}/unsupported-output" RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz"}]')" export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY -if "${repository_root}/release-catalog/prepare.sh" 2>"${temporary_directory}/unsupported-error"; then - echo "prepare.sh unexpectedly accepted an unsupported artifact without identity" >&2 +if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/unsupported-error"; then + echo "materialize.sh unexpectedly accepted an unsupported artifact without identity" >&2 exit 1 fi grep -Fx 'artifact identity cannot be extracted; package_identity_file is required: bundle.tar.gz' "${temporary_directory}/unsupported-error" @@ -129,8 +135,8 @@ grep -Fx 'artifact identity cannot be extracted; package_identity_file is requir GITHUB_OUTPUT="${temporary_directory}/conflicting-identity-output" RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl", package_identity_file: "release-package-identity.json"}]')" export GITHUB_OUTPUT RELEASE_ARTIFACTS -if "${repository_root}/release-catalog/prepare.sh" 2>"${temporary_directory}/conflicting-identity-error"; then - echo "prepare.sh unexpectedly accepted an identity file for a parseable artifact" >&2 +if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/conflicting-identity-error"; then + echo "materialize.sh unexpectedly accepted an identity file for a parseable artifact" >&2 exit 1 fi grep -Fx 'package_identity_file is not allowed when wheel identity can be extracted: libkvikio_cu12-*.whl' "${temporary_directory}/conflicting-identity-error" diff --git a/tests/release_catalog_descriptors_test.sh b/tests/release_catalog_package_parsing_test.sh similarity index 60% rename from tests/release_catalog_descriptors_test.sh rename to tests/release_catalog_package_parsing_test.sh index 1c608e51..a54a4f53 100755 --- a/tests/release_catalog_descriptors_test.sh +++ b/tests/release_catalog_package_parsing_test.sh @@ -7,6 +7,14 @@ repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" temporary_directory="$(mktemp -d)" trap 'rm -rf "${temporary_directory}"' EXIT +GITHUB_OUTPUT="${temporary_directory}/github-output" +RELEASE_ARTIFACTS='' +RELEASE_CATALOG_KEY="test:package-parsing" +RELEASE_ENTRIES_NAME="release-catalog-entries.json" +RELEASE_SOURCE_ARTIFACT_NAME="package-parsing-test" +RELEASE_SOURCE_SHA="0123456789012345678901234567890123456789" +export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_CATALOG_KEY RELEASE_ENTRIES_NAME RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA + wheel_output_directory="${temporary_directory}/wheels" wheel_staging_directory="${temporary_directory}/wheel-staging" mkdir -p "${wheel_output_directory}" "${wheel_staging_directory}/libkvikio_cu12-26.8.0.dist-info" @@ -20,13 +28,15 @@ printf '%s\n' \ zip -qr "${wheel_output_directory}/libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl" . ) -wheel_descriptors="$("${repository_root}/release-catalog/describe-wheels.sh" "${wheel_output_directory}")" +RELEASE_ARTIFACT_DIRECTORY="${wheel_output_directory}" +export RELEASE_ARTIFACT_DIRECTORY +"${repository_root}/release-catalog/materialize.sh" jq -e ' - . == [{ + [.entries[] | {path, package}] == [{ path: "libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl", package: {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"} }] -' <<<"${wheel_descriptors}" >/dev/null +' "${wheel_output_directory}/release-catalog-entries.json" >/dev/null conda_output_directory="${temporary_directory}/conda" tar_bz2_staging_directory="${temporary_directory}/tar-bz2-staging" @@ -57,10 +67,13 @@ zstd -q -f "${temporary_directory}/info-librmm.tar" -o "${temporary_directory}/i info-librmm.tar.zst ) -conda_descriptors="$("${repository_root}/release-catalog/describe-conda.sh" "${conda_output_directory}")" +GITHUB_OUTPUT="${temporary_directory}/conda-github-output" +RELEASE_ARTIFACT_DIRECTORY="${conda_output_directory}" +export GITHUB_OUTPUT RELEASE_ARTIFACT_DIRECTORY +"${repository_root}/release-catalog/materialize.sh" jq -e ' - length == 2 - and any(.[]; + (.entries | length == 2) + and any(.entries[]; .path == "linux-64/librmm-26.08.00a32-cuda12_260714_2f567060.conda" and .package == { ecosystem: "conda", @@ -70,7 +83,7 @@ jq -e ' platform: "linux-64" } ) - and any(.[]; + and any(.entries[]; .path == "noarch/rapids-dask-dependency-26.08.0-py_0.tar.bz2" and .package == { ecosystem: "conda", @@ -80,4 +93,17 @@ jq -e ' platform: "noarch" } ) -' <<<"${conda_descriptors}" >/dev/null +' "${conda_output_directory}/release-catalog-entries.json" >/dev/null + +invalid_wheel_directory="${temporary_directory}/invalid-wheel" +mkdir -p "${invalid_wheel_directory}" +printf '%s\n' invalid >"${invalid_wheel_directory}/invalid.whl" +GITHUB_OUTPUT="${temporary_directory}/invalid-wheel-output" +RELEASE_ARTIFACT_DIRECTORY="${invalid_wheel_directory}" +export GITHUB_OUTPUT RELEASE_ARTIFACT_DIRECTORY +if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/invalid-wheel-error"; then + echo "materialize.sh unexpectedly accepted an invalid wheel" >&2 + exit 1 +fi +grep -F 'wheel must contain exactly one .dist-info/METADATA file: invalid.whl' \ + "${temporary_directory}/invalid-wheel-error" >/dev/null diff --git a/tests/release_catalog_test.sh b/tests/release_catalog_test.sh index 5278994b..02042c87 100755 --- a/tests/release_catalog_test.sh +++ b/tests/release_catalog_test.sh @@ -97,9 +97,11 @@ jq -e ' generated_directory="${temporary_directory}/generated-bundle" mkdir -p "${generated_directory}/linux-64" -printf '%s\n' conda >"${generated_directory}/linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.conda" +printf '%s\n' conda >"${generated_directory}/linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.bin" +jq -n '{ecosystem: "conda", name: "kvikio", version: "26.08.00a32"}' \ + >"${generated_directory}/kvikio.identity.json" -RELEASE_ARTIFACTS="$(jq -cn '[{path: "linux-64/kvikio-*.conda", package: {ecosystem: "conda", name: "kvikio", version: "26.08.00a32"}}]')" +RELEASE_ARTIFACTS="$(jq -cn '[{path: "linux-64/kvikio-*.bin", package_identity_file: "kvikio.identity.json"}]')" RELEASE_ARTIFACT_DIRECTORY="${generated_directory}" RELEASE_SOURCE_ARTIFACT_NAME="kvikio_conda_python_kvikio_x86_64_abi3_cu12" RELEASE_SOURCE_SHA="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" @@ -115,7 +117,7 @@ jq -e ' .source.artifact == "kvikio_conda_python_kvikio_x86_64_abi3_cu12" and .source.sha == "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" and .entries[0].release_catalog_key == "conda:kvikio" - and .entries[0].path == "linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.conda" + and .entries[0].path == "linux-64/kvikio-26.08.00a32-cuda12_260714_2f567060.bin" and .entries[0].package == {ecosystem: "conda", name: "kvikio", version: "26.08.00a32"} and .entries[0].sbom_kind == "generated-identity" ' "${generated_entries_path}" >/dev/null @@ -131,9 +133,11 @@ jq -e ' ' "${generated_directory}/${generated_provenance_path}" >/dev/null wheel_directory="${temporary_directory}/wheel-bundle" mkdir -p "${wheel_directory}" -printf '%s\n' wheel >"${wheel_directory}/libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl" +printf '%s\n' wheel >"${wheel_directory}/libkvikio_cu12-26.8.0a32.bin" +jq -n '{ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"}' \ + >"${wheel_directory}/libkvikio.identity.json" -RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl", package: {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"}}]')" +RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.bin", package_identity_file: "libkvikio.identity.json"}]')" RELEASE_ARTIFACT_DIRECTORY="${wheel_directory}" RELEASE_SOURCE_ARTIFACT_NAME="kvikio_wheel_cpp_libkvikio_x86_64_cu12" RELEASE_SOURCE_SHA="cccccccccccccccccccccccccccccccccccccccc" @@ -144,7 +148,7 @@ export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME jq -e ' .entries[0].release_catalog_key == "wheel:kvikio" - and .entries[0].path == "libkvikio_cu12-26.8.0a32-py3-none-manylinux_2_28_x86_64.whl" + and .entries[0].path == "libkvikio_cu12-26.8.0a32.bin" and .entries[0].package == {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"} and .entries[0].sbom_kind == "generated-identity" ' "${wheel_directory}/release-catalog-entries.json" >/dev/null From 930136dc1c6b1328f75be83bf9cf88f8fbb36927 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 18 Aug 2026 12:14:20 -0500 Subject: [PATCH 12/36] simplify release catalog action interface --- release-catalog-dispatch/action.yml | 26 ++--------------- release-catalog/README.md | 13 ++++----- release-catalog/materialize.sh | 5 ++-- tests/release_catalog_discovery_test.sh | 28 +++++++----------- tests/release_catalog_package_parsing_test.sh | 11 +++---- tests/release_catalog_test.sh | 29 ++++++++++--------- 6 files changed, 41 insertions(+), 71 deletions(-) diff --git a/release-catalog-dispatch/action.yml b/release-catalog-dispatch/action.yml index 9c2e9257..77758741 100644 --- a/release-catalog-dispatch/action.yml +++ b/release-catalog-dispatch/action.yml @@ -8,20 +8,8 @@ inputs: https://github.com/rapidsai/shared-actions/blob/main/release-catalog/config.schema.json required: true source-artifact-name: - description: Name of the GitHub Actions artifact bundle containing this output. + description: Name of the existing GitHub Actions artifact whose files are being cataloged. required: true - source-sha: - description: Source revision built by the producing job. - required: false - -outputs: - entries-path: - description: Absolute path to the generated release catalog entries. - value: ${{ steps.materialize.outputs.entries-path }} - companion-artifact-name: - description: Name of the uploaded GitHub Actions companion artifact. - value: ${{ steps.companion-name.outputs.name }} - runs: using: composite steps: @@ -51,8 +39,7 @@ runs: IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} RELEASE_CATALOG_CONFIG: ${{ inputs.config }} run: "${IMPLEMENTATION_PATH}/validate-config.sh" - - id: materialize - name: Materialize release catalog entries + - name: Materialize release catalog entries shell: bash env: IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} @@ -60,20 +47,13 @@ runs: RELEASE_ENTRIES_NAME: release-catalog-entries.json RELEASE_ARTIFACT_DIRECTORY: ${{ steps.configuration.outputs.artifact_directory }} RELEASE_SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} - RELEASE_SOURCE_SHA: ${{ inputs.source-sha || github.sha }} RELEASE_CATALOG_KEY: ${{ steps.configuration.outputs.release_catalog_key }} run: "${IMPLEMENTATION_PATH}/materialize.sh" - - id: companion-name - name: Set companion artifact name - shell: bash - env: - SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} - run: echo "name=release-catalog-${SOURCE_ARTIFACT_NAME}" >>"${GITHUB_OUTPUT}" - name: Upload release catalog companion uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: if-no-files-found: error - name: ${{ steps.companion-name.outputs.name }} + name: release-catalog-${{ inputs.source-artifact-name }} path: | ${{ steps.configuration.outputs.artifact_directory }}/release-catalog-entries.json ${{ steps.configuration.outputs.artifact_directory }}/release-evidence/** diff --git a/release-catalog/README.md b/release-catalog/README.md index 8e26c08e..e0d2d576 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -61,8 +61,8 @@ unambiguously. ## Source revision -`source-sha` must identify the repository revision actually checked out and -built. RAPIDS shared workflows pass `${{ env.RAPIDS_SHA }}`: +The action requires `RAPIDS_SHA` in the job environment. It must identify the +repository revision actually checked out and built: - Conda build workflows set `RAPIDS_SHA` to `git rev-parse HEAD` immediately after checkout. @@ -70,9 +70,10 @@ built. RAPIDS shared workflows pass `${{ env.RAPIDS_SHA }}`: when supplied and otherwise sets `RAPIDS_SHA` to `git rev-parse HEAD`. This distinction matters when a reusable workflow checks out a repository or -revision different from the workflow event. Direct callers should likewise -resolve the checked-out commit instead of assuming `${{ github.sha }}` names -the built source. +revision different from the workflow event. Direct callers must likewise set +`RAPIDS_SHA` to the checked-out commit instead of assuming `${{ github.sha }}` +names the built source. Variables written to `GITHUB_ENV` are available to the +action when it runs as a subsequent job step. ## Standard package example @@ -86,7 +87,6 @@ the built source. "artifact_directory": ${{ toJSON(steps.package-name.outputs.WHEEL_OUTPUT_DIR) }} } source-artifact-name: ${{ steps.package-name.outputs.RAPIDS_PACKAGE_NAME }} - source-sha: ${{ env.RAPIDS_SHA }} ``` ## Custom package example @@ -105,7 +105,6 @@ the built source. }] } source-artifact-name: cuvs-java - source-sha: ${{ env.RAPIDS_SHA }} ``` Before the action runs, the producer creates diff --git a/release-catalog/materialize.sh b/release-catalog/materialize.sh index 1a353a15..62500806 100755 --- a/release-catalog/materialize.sh +++ b/release-catalog/materialize.sh @@ -17,8 +17,8 @@ require_nonempty "RELEASE_ARTIFACT_DIRECTORY" "${RELEASE_ARTIFACT_DIRECTORY:-}" require_nonempty "RELEASE_ENTRIES_NAME" "${RELEASE_ENTRIES_NAME:-}" require_nonempty "RELEASE_SOURCE_ARTIFACT_NAME" "${RELEASE_SOURCE_ARTIFACT_NAME:-}" -source_sha="${RELEASE_SOURCE_SHA:-${GITHUB_SHA:-}}" -require_nonempty "RELEASE_SOURCE_SHA or GITHUB_SHA" "${source_sha}" +source_sha="${RAPIDS_SHA:-}" +require_nonempty "RAPIDS_SHA" "${source_sha}" require_plain_filename() { local label="$1" @@ -447,4 +447,3 @@ jq -n -S \ }, entries: $entries }' >"${entries_path}" -echo "entries-path=${entries_path}" >>"${GITHUB_OUTPUT}" diff --git a/tests/release_catalog_discovery_test.sh b/tests/release_catalog_discovery_test.sh index 711a8089..ed24007e 100755 --- a/tests/release_catalog_discovery_test.sh +++ b/tests/release_catalog_discovery_test.sh @@ -10,8 +10,8 @@ trap 'rm -rf "${temporary_directory}"' EXIT RELEASE_CATALOG_KEY="test:discovery" RELEASE_ENTRIES_NAME="release-catalog-entries.json" RELEASE_SOURCE_ARTIFACT_NAME="discovery-test" -RELEASE_SOURCE_SHA="0123456789012345678901234567890123456789" -export RELEASE_CATALOG_KEY RELEASE_ENTRIES_NAME RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA +RAPIDS_SHA="0123456789012345678901234567890123456789" +export RAPIDS_SHA RELEASE_CATALOG_KEY RELEASE_ENTRIES_NAME RELEASE_SOURCE_ARTIFACT_NAME mkdir -p "${temporary_directory}/wheel/libkvikio_cu12-26.8.0.dist-info" printf '%s\n' \ @@ -24,10 +24,9 @@ printf '%s\n' \ zip -qr "${temporary_directory}/libkvikio_cu12-26.8.0-py3-none-manylinux_2_28_x86_64.whl" . ) -GITHUB_OUTPUT="${temporary_directory}/wheel-output" RELEASE_ARTIFACTS='' RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY "${repository_root}/release-catalog/materialize.sh" @@ -49,9 +48,8 @@ tar -cjf \ -C "${conda_staging_directory}" \ info/index.json -GITHUB_OUTPUT="${temporary_directory}/conda-output" RELEASE_ARTIFACT_DIRECTORY="${conda_output_directory}" -export GITHUB_OUTPUT RELEASE_ARTIFACT_DIRECTORY +export RELEASE_ARTIFACT_DIRECTORY "${repository_root}/release-catalog/materialize.sh" @@ -68,14 +66,13 @@ jq -e ' }] ' "${conda_output_directory}/release-catalog-entries.json" >/dev/null -GITHUB_OUTPUT="${temporary_directory}/custom-output" RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" printf '%s\n' bundle >"${temporary_directory}/bundle.tar.gz" printf '%s\n' sbom >"${temporary_directory}/bundle.spdx.json" jq -n '{ecosystem: "archive", name: "bundle", version: "1.0"}' \ >"${temporary_directory}/release-package-identity.json" RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz", package_identity_file: "release-package-identity.json", sbom: "bundle.spdx.json"}]')" -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY "${repository_root}/release-catalog/materialize.sh" @@ -85,11 +82,10 @@ jq -e ' and .entries[0].sbom_kind == "producer-dependency" ' "${temporary_directory}/release-catalog-entries.json" >/dev/null -GITHUB_OUTPUT="${temporary_directory}/explicit-wheel-output" RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" printf '%s\n' signature >"${temporary_directory}/libkvikio.sig" RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl", signature: "libkvikio.sig"}]')" -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY "${repository_root}/release-catalog/materialize.sh" jq -e ' .entries[0].path == "libkvikio_cu12-26.8.0-py3-none-manylinux_2_28_x86_64.whl" @@ -101,10 +97,9 @@ mixed_directory="${temporary_directory}/mixed" mkdir -p "${mixed_directory}" cp "${temporary_directory}/libkvikio_cu12-26.8.0-py3-none-manylinux_2_28_x86_64.whl" "${mixed_directory}/" cp "${conda_output_directory}/noarch/rapids-dask-dependency-26.08.0-py_0.tar.bz2" "${mixed_directory}/" -GITHUB_OUTPUT="${temporary_directory}/mixed-output" RELEASE_ARTIFACTS='' RELEASE_ARTIFACT_DIRECTORY="${mixed_directory}" -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY "${repository_root}/release-catalog/materialize.sh" jq -e ' (.entries | length == 2) @@ -113,28 +108,25 @@ jq -e ' empty_directory="${temporary_directory}/empty" mkdir -p "${empty_directory}" -GITHUB_OUTPUT="${temporary_directory}/empty-output" RELEASE_ARTIFACT_DIRECTORY="${empty_directory}" -export GITHUB_OUTPUT RELEASE_ARTIFACT_DIRECTORY +export RELEASE_ARTIFACT_DIRECTORY if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/empty-error"; then echo "materialize.sh unexpectedly accepted an artifact directory without detectable artifacts" >&2 exit 1 fi grep -Fx 'artifact-directory contains no detectable Conda or wheel artifacts; explicitly selected artifacts require package_identity_file when their identity cannot be parsed' "${temporary_directory}/empty-error" -GITHUB_OUTPUT="${temporary_directory}/unsupported-output" RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz"}]')" -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/unsupported-error"; then echo "materialize.sh unexpectedly accepted an unsupported artifact without identity" >&2 exit 1 fi grep -Fx 'artifact identity cannot be extracted; package_identity_file is required: bundle.tar.gz' "${temporary_directory}/unsupported-error" -GITHUB_OUTPUT="${temporary_directory}/conflicting-identity-output" RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl", package_identity_file: "release-package-identity.json"}]')" -export GITHUB_OUTPUT RELEASE_ARTIFACTS +export RELEASE_ARTIFACTS if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/conflicting-identity-error"; then echo "materialize.sh unexpectedly accepted an identity file for a parseable artifact" >&2 exit 1 diff --git a/tests/release_catalog_package_parsing_test.sh b/tests/release_catalog_package_parsing_test.sh index a54a4f53..3b037c4d 100755 --- a/tests/release_catalog_package_parsing_test.sh +++ b/tests/release_catalog_package_parsing_test.sh @@ -7,13 +7,12 @@ repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" temporary_directory="$(mktemp -d)" trap 'rm -rf "${temporary_directory}"' EXIT -GITHUB_OUTPUT="${temporary_directory}/github-output" RELEASE_ARTIFACTS='' RELEASE_CATALOG_KEY="test:package-parsing" RELEASE_ENTRIES_NAME="release-catalog-entries.json" RELEASE_SOURCE_ARTIFACT_NAME="package-parsing-test" -RELEASE_SOURCE_SHA="0123456789012345678901234567890123456789" -export GITHUB_OUTPUT RELEASE_ARTIFACTS RELEASE_CATALOG_KEY RELEASE_ENTRIES_NAME RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA +RAPIDS_SHA="0123456789012345678901234567890123456789" +export RAPIDS_SHA RELEASE_ARTIFACTS RELEASE_CATALOG_KEY RELEASE_ENTRIES_NAME RELEASE_SOURCE_ARTIFACT_NAME wheel_output_directory="${temporary_directory}/wheels" wheel_staging_directory="${temporary_directory}/wheel-staging" @@ -67,9 +66,8 @@ zstd -q -f "${temporary_directory}/info-librmm.tar" -o "${temporary_directory}/i info-librmm.tar.zst ) -GITHUB_OUTPUT="${temporary_directory}/conda-github-output" RELEASE_ARTIFACT_DIRECTORY="${conda_output_directory}" -export GITHUB_OUTPUT RELEASE_ARTIFACT_DIRECTORY +export RELEASE_ARTIFACT_DIRECTORY "${repository_root}/release-catalog/materialize.sh" jq -e ' (.entries | length == 2) @@ -98,9 +96,8 @@ jq -e ' invalid_wheel_directory="${temporary_directory}/invalid-wheel" mkdir -p "${invalid_wheel_directory}" printf '%s\n' invalid >"${invalid_wheel_directory}/invalid.whl" -GITHUB_OUTPUT="${temporary_directory}/invalid-wheel-output" RELEASE_ARTIFACT_DIRECTORY="${invalid_wheel_directory}" -export GITHUB_OUTPUT RELEASE_ARTIFACT_DIRECTORY +export RELEASE_ARTIFACT_DIRECTORY if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/invalid-wheel-error"; then echo "materialize.sh unexpectedly accepted an invalid wheel" >&2 exit 1 diff --git a/tests/release_catalog_test.sh b/tests/release_catalog_test.sh index 02042c87..1383f32d 100755 --- a/tests/release_catalog_test.sh +++ b/tests/release_catalog_test.sh @@ -17,22 +17,20 @@ jq -n \ '{ecosystem: "maven", name: "ai.rapids:cuvs-java", version: "26.08.0"}' \ >"${bundle_directory}/cuvs-java-package-identity.json" -GITHUB_OUTPUT="${temporary_directory}/github-output" GITHUB_REPOSITORY="rapidsai/cuvs" GITHUB_RUN_ATTEMPT="1" GITHUB_RUN_ID="1234" GITHUB_SHA="0123456789012345678901234567890123456789" GITHUB_WORKFLOW_REF="rapidsai/cuvs/.github/workflows/build.yaml@refs/heads/release/26.08" -export GITHUB_OUTPUT RELEASE_ARTIFACTS="$(jq -cn '[{path: "cuvs-java-*.jar", package_identity_file: "cuvs-java-package-identity.json", sbom: "cuvs-java-*.spdx.json", provenance: "cuvs-java-*.provenance.jsonl", signature: "cuvs-java-*.jar.asc"}]')" RELEASE_ENTRIES_NAME="release-catalog-entries.json" RELEASE_ARTIFACT_DIRECTORY="${bundle_directory}" RELEASE_SOURCE_ARTIFACT_NAME="cuvs-java-cuda12.9.1" -RELEASE_SOURCE_SHA="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +RAPIDS_SHA="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" RELEASE_CATALOG_KEY="maven:cuvs-java" export GITHUB_REPOSITORY GITHUB_RUN_ATTEMPT GITHUB_RUN_ID GITHUB_SHA GITHUB_WORKFLOW_REF export RELEASE_ARTIFACTS RELEASE_ENTRIES_NAME RELEASE_ARTIFACT_DIRECTORY -export RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY +export RAPIDS_SHA RELEASE_SOURCE_ARTIFACT_NAME RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" @@ -51,7 +49,6 @@ jq -e ' and .entries[0].package.name == "ai.rapids:cuvs-java" and .entries[0].sbom_kind == "producer-dependency" ' "${entries_path}" >/dev/null -grep -Fx "entries-path=${entries_path}" "${GITHUB_OUTPUT}" supplied_sbom_path="$(jq -r '.entries[0].sbom' "${entries_path}")" supplied_provenance_path="$(jq -r '.entries[0].provenance' "${entries_path}")" @@ -85,9 +82,9 @@ RELEASE_ARTIFACTS="$(jq -cn '[ ]')" RELEASE_ARTIFACT_DIRECTORY="${multiple_identity_directory}" RELEASE_SOURCE_ARTIFACT_NAME="multiple-identities" -RELEASE_SOURCE_SHA="eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" +RAPIDS_SHA="eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" RELEASE_CATALOG_KEY="archive:multiple" -export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY +export RAPIDS_SHA RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" jq -e ' (.entries | length == 2) @@ -104,9 +101,9 @@ jq -n '{ecosystem: "conda", name: "kvikio", version: "26.08.00a32"}' \ RELEASE_ARTIFACTS="$(jq -cn '[{path: "linux-64/kvikio-*.bin", package_identity_file: "kvikio.identity.json"}]')" RELEASE_ARTIFACT_DIRECTORY="${generated_directory}" RELEASE_SOURCE_ARTIFACT_NAME="kvikio_conda_python_kvikio_x86_64_abi3_cu12" -RELEASE_SOURCE_SHA="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +RAPIDS_SHA="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" RELEASE_CATALOG_KEY="conda:kvikio" -export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY +export RAPIDS_SHA RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" @@ -140,9 +137,9 @@ jq -n '{ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"}' \ RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.bin", package_identity_file: "libkvikio.identity.json"}]')" RELEASE_ARTIFACT_DIRECTORY="${wheel_directory}" RELEASE_SOURCE_ARTIFACT_NAME="kvikio_wheel_cpp_libkvikio_x86_64_cu12" -RELEASE_SOURCE_SHA="cccccccccccccccccccccccccccccccccccccccc" +RAPIDS_SHA="cccccccccccccccccccccccccccccccccccccccc" RELEASE_CATALOG_KEY="wheel:kvikio" -export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY +export RAPIDS_SHA RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" @@ -163,12 +160,18 @@ jq -n \ RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz", package_identity_file: "bundle-package-identity.json"}]')" RELEASE_ARTIFACT_DIRECTORY="${missing_version_directory}" RELEASE_SOURCE_ARTIFACT_NAME="bundle-archive" -RELEASE_SOURCE_SHA="dddddddddddddddddddddddddddddddddddddddd" +RAPIDS_SHA="dddddddddddddddddddddddddddddddddddddddd" RELEASE_CATALOG_KEY="archive:bundle" -export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY +export RAPIDS_SHA RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_CATALOG_KEY if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/missing-version-error"; then echo "materialize.sh unexpectedly accepted a custom artifact without a package version" >&2 exit 1 fi grep -Fx 'package identity for bundle.tar.gz must contain non-empty ecosystem, name, and version strings and only optional build or platform strings' "${temporary_directory}/missing-version-error" + +if (unset RAPIDS_SHA; "${repository_root}/release-catalog/materialize.sh") 2>"${temporary_directory}/missing-rapids-sha-error"; then + echo "materialize.sh unexpectedly accepted a missing RAPIDS_SHA" >&2 + exit 1 +fi +grep -Fx 'RAPIDS_SHA must be a non-empty string' "${temporary_directory}/missing-rapids-sha-error" From 81713a77d3b1fab056f744628dac3a9448c52410 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 18 Aug 2026 13:29:37 -0500 Subject: [PATCH 13/36] defer producer-supplied release evidence --- README.md | 2 +- release-catalog/README.md | 39 +++++-------- release-catalog/config.schema.json | 22 +------ release-catalog/materialize.sh | 57 ++++--------------- release-catalog/validate-config.sh | 4 +- .../package-identity-file.json | 3 +- tests/release_catalog_config_test.sh | 5 +- tests/release_catalog_discovery_test.sh | 10 ++-- tests/release_catalog_test.sh | 19 +------ 9 files changed, 39 insertions(+), 122 deletions(-) diff --git a/README.md b/README.md index 88248366..40f29a86 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ directory and uploads a companion artifact named See the [release catalog documentation](release-catalog/README.md) for configuration, examples, source-revision handling, companion contents, and -evidence semantics. +generated identity evidence semantics. The dispatch wrapper honors `SHARED_ACTIONS_REPO` and `SHARED_ACTIONS_REF`. When neither is set, it checks out the same repository and ref used to invoke diff --git a/release-catalog/README.md b/release-catalog/README.md index e0d2d576..f17bb2ef 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -3,8 +3,8 @@ `release-catalog-dispatch` records the exact files produced by a package build and uploads a companion GitHub Actions artifact named `release-catalog-`. Release tooling uses the -companion to associate primary artifacts with their build identity and -available evidence without reconstructing the build later. +companion to associate primary artifacts with their build identity and generated +identity evidence without reconstructing the build later. The companion contains: @@ -31,8 +31,7 @@ When `artifacts` is omitted, the action discovers every Conda and wheel output in `artifact_directory` and extracts identity from each package independently. When `artifacts` is supplied, each descriptor is resolved independently: supported Conda and wheel files are parsed, while any other artifact requires -its own `package_identity_file`. Evidence paths also belong to the individual -artifact descriptor. +its own `package_identity_file`. `release_catalog_key` identifies the release catalog entry that owns these artifacts. Every file from every matrix variant in the same publishable artifact @@ -56,8 +55,7 @@ keys: for example, `cudf` and `dask-cudf` Conda packages remain part of The action validates the configuration before inspecting build outputs. It then verifies properties that depend on produced files, including the package -identity contents and whether primary-artifact and evidence paths resolve -unambiguously. +identity contents and whether primary-artifact paths resolve unambiguously. ## Source revision @@ -120,25 +118,14 @@ relative to `artifact_directory`. For example: } ``` -## Evidence semantics +## Generated identity evidence -A descriptor-selected producer SBOM is classified as `producer-dependency`. -It is evidence supplied by the producer and may contain a dependency inventory. +For every artifact, the action generates an SPDX artifact-identity envelope +classified as `generated-identity` and a build-context provenance statement. +They record the artifact SHA-256, package identity, source revision, and workflow +context, but contain no dependency or source-license inventory. They must not be +reported as producer-supplied dependency coverage. -When no SBOM is selected, the action generates an SPDX artifact-identity -envelope classified as `generated-identity`. It records package identity and -the primary artifact SHA-256, but contains no dependency or source-license -inventory. It must not be reported as producer-supplied dependency coverage. - -Producer-supplied SBOM, provenance, and signature sidecars are copied under -`release-evidence/` so the companion remains independently consumable. - -Concrete producer-supplied evidence examples include: - -- an official [SPDX 2.3 dependency SBOM](https://github.com/spdx/spdx-examples/blob/2181917ef6ff74de89252ee785583c27a38d6199/presentations/OSS-NA-2023/SPDXVersion2.3/03-SBOMwDependency.json); -- an official [SLSA provenance v1 statement](https://github.com/slsa-framework/github-actions-buildtypes/blob/5f855ef0106dad3ee0e0f1046dc31b3b65152956/workflow/v1/example.json); -- a Maven Central [detached ASCII-armored signature](https://repo1.maven.org/maven2/org/apache/commons/commons-lang3/3.17.0/commons-lang3-3.17.0.jar.asc). - -These examples illustrate the expected purpose of the files, not required -serialization formats. The action copies producer-supplied evidence as opaque -sidecars and does not validate their contents or require these formats. +Producer-supplied SBOM, provenance, and signature inputs are intentionally +deferred until a future revision defines a generic, unambiguous way to associate +them with detected artifacts. diff --git a/release-catalog/config.schema.json b/release-catalog/config.schema.json index 0f099e1b..7df259e0 100644 --- a/release-catalog/config.schema.json +++ b/release-catalog/config.schema.json @@ -16,14 +16,14 @@ "pattern": "^[^\\r\\n]+$" }, "artifact_directory": { - "description": "Base directory for artifact and evidence paths. Defaults to the job working directory.", + "description": "Base directory for artifact paths. Defaults to the job working directory.", "type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$", "default": "." }, "artifacts": { - "description": "Optional explicit primary release files and their identity or evidence sidecars. When omitted, Conda and wheel artifacts are discovered in artifact_directory.", + "description": "Optional explicit primary release files. When omitted, Conda and wheel artifacts are discovered in artifact_directory.", "type": "array", "minItems": 1, "items": { @@ -49,24 +49,6 @@ "type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$" - }, - "sbom": { - "description": "Dependency SBOM path or glob. Example file: https://github.com/spdx/spdx-examples/blob/2181917ef6ff74de89252ee785583c27a38d6199/presentations/OSS-NA-2023/SPDXVersion2.3/03-SBOMwDependency.json", - "type": "string", - "minLength": 1, - "pattern": "^[^\\r\\n]+$" - }, - "provenance": { - "description": "Provenance path or glob. Example file: https://github.com/slsa-framework/github-actions-buildtypes/blob/5f855ef0106dad3ee0e0f1046dc31b3b65152956/workflow/v1/example.json", - "type": "string", - "minLength": 1, - "pattern": "^[^\\r\\n]+$" - }, - "signature": { - "description": "Signature path or glob. Example file: https://repo1.maven.org/maven2/org/apache/commons/commons-lang3/3.17.0/commons-lang3-3.17.0.jar.asc", - "type": "string", - "minLength": 1, - "pattern": "^[^\\r\\n]+$" } } } diff --git a/release-catalog/materialize.sh b/release-catalog/materialize.sh index 62500806..1bde11ee 100755 --- a/release-catalog/materialize.sh +++ b/release-catalog/materialize.sh @@ -260,20 +260,6 @@ generated_evidence_path() { printf 'release-evidence/%s.%s.%s.json\n' "${artifact_label}" "${artifact_digest}" "${kind}" } -copy_supplied_evidence() { - local primary_path="$1" - local supplied_path="$2" - local kind="$3" - local artifact_digest - artifact_digest="$(sha256sum "${artifact_directory}/${primary_path}" | awk '{print $1}')" - local artifact_label="${primary_path//\//_}" - local destination - destination="release-evidence/${artifact_label}.${artifact_digest}/${kind}-$(basename "${supplied_path}")" - mkdir -p "$(dirname "${artifact_directory}/${destination}")" - cp "${artifact_directory}/${supplied_path}" "${artifact_directory}/${destination}" - printf '%s\n' "${destination}" -} - write_generated_sbom() { local primary_path="$1" local package="$2" @@ -311,7 +297,7 @@ write_generated_sbom() { relationshipType: "DESCRIBES", relatedSpdxElement: "SPDXRef-Artifact" }], - comment: "Artifact-identity SBOM envelope. A producer-supplied dependency SBOM may replace this record." + comment: "Artifact-identity SBOM envelope. It does not contain a dependency inventory." }' >"${artifact_directory}/${destination}" } @@ -356,22 +342,16 @@ shopt -s globstar nullglob while IFS= read -r descriptor; do if ! jq -e ' type == "object" - and (keys - ["path", "package_identity_file", "sbom", "provenance", "signature", "package"] | length == 0) + and (keys - ["path", "package_identity_file", "package"] | length == 0) and (.path | type == "string" and length > 0) and ((.package_identity_file // "") | type == "string") - and ((.sbom // "") | type == "string") - and ((.provenance // "") | type == "string") - and ((.signature // "") | type == "string") and ([has("package"), has("package_identity_file")] | map(select(.)) | length == 1) ' <<<"${descriptor}" >/dev/null; then - echo "release artifact descriptor must contain path, exactly one package identity source, and optional evidence paths: ${descriptor}" >&2 + echo "release artifact descriptor must contain path and exactly one package identity source: ${descriptor}" >&2 exit 1 fi primary_path="$(resolve_one_file path "$(jq -r '.path' <<<"${descriptor}")")" - sbom_pattern="$(jq -r '.sbom // empty' <<<"${descriptor}")" - provenance_pattern="$(jq -r '.provenance // empty' <<<"${descriptor}")" - signature_pattern="$(jq -r '.signature // empty' <<<"${descriptor}")" package_identity_file="$(jq -r '.package_identity_file // empty' <<<"${descriptor}")" if [[ -n "${package_identity_file}" ]]; then package_identity_path="$(resolve_one_file package_identity_file "${package_identity_file}")" @@ -387,35 +367,20 @@ while IFS= read -r descriptor; do exit 1 fi - if [[ -n "${sbom_pattern}" ]]; then - supplied_sbom_path="$(resolve_one_file sbom "${sbom_pattern}")" - sbom_path="$(copy_supplied_evidence "${primary_path}" "${supplied_sbom_path}" "sbom")" - sbom_kind="producer-dependency" - else - sbom_path="$(generated_evidence_path "${primary_path}" "spdx")" - write_generated_sbom "${primary_path}" "${package}" "${sbom_path}" - sbom_kind="generated-identity" - fi - if [[ -n "${provenance_pattern}" ]]; then - supplied_provenance_path="$(resolve_one_file provenance "${provenance_pattern}")" - provenance_path="$(copy_supplied_evidence "${primary_path}" "${supplied_provenance_path}" "provenance")" - else - provenance_path="$(generated_evidence_path "${primary_path}" "provenance")" - write_generated_provenance "${primary_path}" "${package}" "${provenance_path}" - fi + sbom_path="$(generated_evidence_path "${primary_path}" "spdx")" + write_generated_sbom "${primary_path}" "${package}" "${sbom_path}" + provenance_path="$(generated_evidence_path "${primary_path}" "provenance")" + write_generated_provenance "${primary_path}" "${package}" "${provenance_path}" + + # TODO: Accept producer-supplied SBOM, provenance, and signature evidence after + # the release catalog defines a generic artifact-to-evidence association. entry="$(jq -cn \ --arg release_catalog_key "${RELEASE_CATALOG_KEY}" \ --arg path "${primary_path}" \ --arg sbom "${sbom_path}" \ - --arg sbom_kind "${sbom_kind}" \ --arg provenance "${provenance_path}" \ --argjson package "${package}" \ - '{release_catalog_key: $release_catalog_key, path: $path, sbom: $sbom, sbom_kind: $sbom_kind, provenance: $provenance, package: $package}')" - if [[ -n "${signature_pattern}" ]]; then - supplied_signature_path="$(resolve_one_file signature "${signature_pattern}")" - signature_path="$(copy_supplied_evidence "${primary_path}" "${supplied_signature_path}" "signature")" - entry="$(jq -c --arg signature "${signature_path}" '. + {signature: $signature}' <<<"${entry}")" - fi + '{release_catalog_key: $release_catalog_key, path: $path, sbom: $sbom, sbom_kind: "generated-identity", provenance: $provenance, package: $package}')" jq --argjson entry "${entry}" '.entries += [$entry]' "${temporary_manifest}" >"${temporary_manifest}.next" mv "${temporary_manifest}.next" "${temporary_manifest}" diff --git a/release-catalog/validate-config.sh b/release-catalog/validate-config.sh index 6a5496ec..5ed529e9 100755 --- a/release-catalog/validate-config.sh +++ b/release-catalog/validate-config.sh @@ -50,14 +50,14 @@ validation_errors="$(jq -r ' | if ($artifact | type) != "object" then "artifacts[\($index)] must be an object" else - ($artifact | keys - ["path", "package_identity_file", "sbom", "provenance", "signature"]) as $artifact_unknown + ($artifact | keys - ["path", "package_identity_file"]) as $artifact_unknown | if ($artifact_unknown | length) > 0 then "artifacts[\($index)] has unknown field(s): " + ($artifact_unknown | join(", ")) else empty end, if ($artifact.path | single_line_string) then empty else "artifacts[\($index)].path must be a non-empty, single-line string" end, ($artifact | to_entries[] - | select(.key == "package_identity_file" or .key == "sbom" or .key == "provenance" or .key == "signature") + | select(.key == "package_identity_file") | select((.value | single_line_string) | not) | "artifacts[\($index)].\(.key) must be a non-empty, single-line string") end diff --git a/tests/release-catalog-config/package-identity-file.json b/tests/release-catalog-config/package-identity-file.json index 99047495..c3b55a05 100644 --- a/tests/release-catalog-config/package-identity-file.json +++ b/tests/release-catalog-config/package-identity-file.json @@ -4,8 +4,7 @@ "artifacts": [ { "path": "cuvs-java-*-x86_64-cuda*.jar", - "package_identity_file": "cuvs-java.release-package-identity.json", - "sbom": "cuvs-java.spdx.json" + "package_identity_file": "cuvs-java.release-package-identity.json" } ] } diff --git a/tests/release_catalog_config_test.sh b/tests/release_catalog_config_test.sh index 8bfcce06..cf615f90 100755 --- a/tests/release_catalog_config_test.sh +++ b/tests/release_catalog_config_test.sh @@ -32,7 +32,7 @@ RELEASE_CATALOG_CONFIG="$(<"${repository_root}/tests/release-catalog-config/pack grep -Fx 'release_catalog_key=maven:cuvs-java' "${valid_output}" grep -Fx 'artifact_directory=java/cuvs-java/target' "${valid_output}" -grep -Fx 'artifacts=[{"path":"cuvs-java-*-x86_64-cuda*.jar","package_identity_file":"cuvs-java.release-package-identity.json","sbom":"cuvs-java.spdx.json"}]' "${valid_output}" +grep -Fx 'artifacts=[{"path":"cuvs-java-*-x86_64-cuda*.jar","package_identity_file":"cuvs-java.release-package-identity.json"}]' "${valid_output}" assert_invalid \ malformed-json \ @@ -77,10 +77,9 @@ assert_invalid \ assert_invalid \ malformed-artifact \ '{"release_catalog_key":"archive:smoke","artifacts":[{"file":"smoke.tar.gz","package_identity_file":false,"sbom":false}]}' \ - 'artifacts[0] has unknown field(s): file' + 'artifacts[0] has unknown field(s): file, sbom' grep -F 'artifacts[0].path must be a non-empty, single-line string' "${temporary_directory}/malformed-artifact.error" >/dev/null grep -F 'artifacts[0].package_identity_file must be a non-empty, single-line string' "${temporary_directory}/malformed-artifact.error" >/dev/null -grep -F 'artifacts[0].sbom must be a non-empty, single-line string' "${temporary_directory}/malformed-artifact.error" >/dev/null default_output="${temporary_directory}/default.output" RELEASE_CATALOG_CONFIG='{"release_catalog_key":"conda:smoke"}' \ diff --git a/tests/release_catalog_discovery_test.sh b/tests/release_catalog_discovery_test.sh index ed24007e..fce7a1e6 100755 --- a/tests/release_catalog_discovery_test.sh +++ b/tests/release_catalog_discovery_test.sh @@ -68,10 +68,9 @@ jq -e ' RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" printf '%s\n' bundle >"${temporary_directory}/bundle.tar.gz" -printf '%s\n' sbom >"${temporary_directory}/bundle.spdx.json" jq -n '{ecosystem: "archive", name: "bundle", version: "1.0"}' \ >"${temporary_directory}/release-package-identity.json" -RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz", package_identity_file: "release-package-identity.json", sbom: "bundle.spdx.json"}]')" +RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz", package_identity_file: "release-package-identity.json"}]')" export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY "${repository_root}/release-catalog/materialize.sh" @@ -79,18 +78,17 @@ export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY jq -e ' .entries[0].path == "bundle.tar.gz" and .entries[0].package == {ecosystem: "archive", name: "bundle", version: "1.0"} - and .entries[0].sbom_kind == "producer-dependency" + and .entries[0].sbom_kind == "generated-identity" ' "${temporary_directory}/release-catalog-entries.json" >/dev/null RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" -printf '%s\n' signature >"${temporary_directory}/libkvikio.sig" -RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl", signature: "libkvikio.sig"}]')" +RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl"}]')" export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY "${repository_root}/release-catalog/materialize.sh" jq -e ' .entries[0].path == "libkvikio_cu12-26.8.0-py3-none-manylinux_2_28_x86_64.whl" and .entries[0].package == {ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0"} - and (.entries[0].signature | startswith("release-evidence/")) + and .entries[0].sbom_kind == "generated-identity" ' "${temporary_directory}/release-catalog-entries.json" >/dev/null mixed_directory="${temporary_directory}/mixed" diff --git a/tests/release_catalog_test.sh b/tests/release_catalog_test.sh index 1383f32d..86ce8c89 100755 --- a/tests/release_catalog_test.sh +++ b/tests/release_catalog_test.sh @@ -10,9 +10,6 @@ trap 'rm -rf "${temporary_directory}"' EXIT bundle_directory="${temporary_directory}/bundle" mkdir -p "${bundle_directory}" printf '%s\n' jar >"${bundle_directory}/cuvs-java-26.08.0.jar" -printf '%s\n' sbom >"${bundle_directory}/cuvs-java-26.08.0.spdx.json" -printf '%s\n' provenance >"${bundle_directory}/cuvs-java-26.08.0.provenance.jsonl" -printf '%s\n' signature >"${bundle_directory}/cuvs-java-26.08.0.jar.asc" jq -n \ '{ecosystem: "maven", name: "ai.rapids:cuvs-java", version: "26.08.0"}' \ >"${bundle_directory}/cuvs-java-package-identity.json" @@ -22,7 +19,7 @@ GITHUB_RUN_ATTEMPT="1" GITHUB_RUN_ID="1234" GITHUB_SHA="0123456789012345678901234567890123456789" GITHUB_WORKFLOW_REF="rapidsai/cuvs/.github/workflows/build.yaml@refs/heads/release/26.08" -RELEASE_ARTIFACTS="$(jq -cn '[{path: "cuvs-java-*.jar", package_identity_file: "cuvs-java-package-identity.json", sbom: "cuvs-java-*.spdx.json", provenance: "cuvs-java-*.provenance.jsonl", signature: "cuvs-java-*.jar.asc"}]')" +RELEASE_ARTIFACTS="$(jq -cn '[{path: "cuvs-java-*.jar", package_identity_file: "cuvs-java-package-identity.json"}]')" RELEASE_ENTRIES_NAME="release-catalog-entries.json" RELEASE_ARTIFACT_DIRECTORY="${bundle_directory}" RELEASE_SOURCE_ARTIFACT_NAME="cuvs-java-cuda12.9.1" @@ -47,26 +44,16 @@ jq -e ' and .entries[0].release_catalog_key == "maven:cuvs-java" and .entries[0].path == "cuvs-java-26.08.0.jar" and .entries[0].package.name == "ai.rapids:cuvs-java" - and .entries[0].sbom_kind == "producer-dependency" + and .entries[0].sbom_kind == "generated-identity" ' "${entries_path}" >/dev/null -supplied_sbom_path="$(jq -r '.entries[0].sbom' "${entries_path}")" -supplied_provenance_path="$(jq -r '.entries[0].provenance' "${entries_path}")" -supplied_signature_path="$(jq -r '.entries[0].signature' "${entries_path}")" -[[ "${supplied_sbom_path}" == release-evidence/*/sbom-* ]] -[[ "${supplied_provenance_path}" == release-evidence/*/provenance-* ]] -[[ "${supplied_signature_path}" == release-evidence/*/signature-* ]] -grep -Fx sbom "${canonical_bundle_directory}/${supplied_sbom_path}" -grep -Fx provenance "${canonical_bundle_directory}/${supplied_provenance_path}" -grep -Fx signature "${canonical_bundle_directory}/${supplied_signature_path}" - isolated_companion_directory="${temporary_directory}/isolated-companion" mkdir -p "${isolated_companion_directory}" cp "${entries_path}" "${isolated_companion_directory}/" cp -R "${canonical_bundle_directory}/release-evidence" "${isolated_companion_directory}/" while IFS= read -r evidence_path; do test -f "${isolated_companion_directory}/${evidence_path}" -done < <(jq -r '.entries[] | .sbom, .provenance, (.signature // empty)' "${isolated_companion_directory}/release-catalog-entries.json") +done < <(jq -r '.entries[] | .sbom, .provenance' "${isolated_companion_directory}/release-catalog-entries.json") multiple_identity_directory="${temporary_directory}/multiple-identities" mkdir -p "${multiple_identity_directory}" From c916c33d6e9172973450e5657e595c833f6447e3 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 18 Aug 2026 14:54:00 -0500 Subject: [PATCH 14/36] manual README edits --- release-catalog/README.md | 124 +++++++++++++++++------------ release-catalog/config.schema.json | 2 +- 2 files changed, 76 insertions(+), 50 deletions(-) diff --git a/release-catalog/README.md b/release-catalog/README.md index f17bb2ef..fb2cdc4d 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -4,7 +4,11 @@ build and uploads a companion GitHub Actions artifact named `release-catalog-`. Release tooling uses the companion to associate primary artifacts with their build identity and generated -identity evidence without reconstructing the build later. +identity evidence without reconstructing the build later. This evidence supports: + +- [assembling releases that can be tested and verified before formal tagging](https://github.com/rapidsai/release-scripts/issues/102) +- using build-time metadata to triage CVE scan records more quickly (internal + GitLab project, `nspect-manager`) The companion contains: @@ -22,58 +26,48 @@ contains the release catalog entries produced by that job. The release platform validates and aggregates entry arrays from selected builds into the release catalog; there is no separate metadata document to keep synchronized. +Each matrix job produces its own companion artifact. The release platform merges +their entries into one composite release catalog. + ## Configuration -Every producer passes one `config` JSON object. Its canonical schema and field -documentation are in [`config.schema.json`](config.schema.json). +Every build job that calls this action passes one `config` JSON object. Its +canonical schema and field documentation are in +[`config.schema.json`](config.schema.json). + +The action validates the configuration before inspecting build outputs. It then +verifies properties that depend on produced files, including the package +identity contents and whether primary-artifact paths resolve unambiguously. -When `artifacts` is omitted, the action discovers every Conda and wheel output -in `artifact_directory` and extracts identity from each package independently. -When `artifacts` is supplied, each descriptor is resolved independently: -supported Conda and wheel files are parsed, while any other artifact requires -its own `package_identity_file`. +### `release_catalog_key` -`release_catalog_key` identifies the release catalog entry that owns these -artifacts. Every file from every matrix variant in the same publishable artifact -set uses the same key, and multiple files are aggregated. Standard RAPIDS Conda -and wheel workflows construct it as `:`, such as -`conda:cudf`; custom producers select an existing catalog key, such as -`maven:cuvs-java`. Do not generate a UUID or a per-build value. +The `release_catalog_key` is used to group artifacts in the catalog. Every file +from every matrix variant in the same publishable artifact set uses the same +key. The release platform aggregates multiple `release-catalog-entries.json` +files and coalesces entries with the same `release_catalog_key`. Standard RAPIDS +conda and wheel workflows construct the key as `:`, +such as `conda:cudf`. Custom producers select a key, such as `maven:cuvs-java`, +that represents their release policy. -Use a distinct `release_catalog_key` only when the release catalog intentionally gives the -outputs different release policy. Examples include: +Use a distinct `release_catalog_key` only when the outputs intentionally have +different release policies. Reasons include differences in: -* different versioning -* validation requirements -* dependency ordering -* publication destinations -* promotion strategy. +- versioning schemes +- validation requirements +- dependency ordering +- publication destinations +- promotion strategy Multiple package names from one repository do not by themselves justify separate keys: for example, `cudf` and `dask-cudf` Conda packages remain part of `conda:cudf` when they share one release policy. -The action validates the configuration before inspecting build outputs. It then -verifies properties that depend on produced files, including the package -identity contents and whether primary-artifact paths resolve unambiguously. - -## Source revision - -The action requires `RAPIDS_SHA` in the job environment. It must identify the -repository revision actually checked out and built: +### `artifacts` -- Conda build workflows set `RAPIDS_SHA` to `git rev-parse HEAD` immediately - after checkout. -- Wheel and custom workflows use `rapids-github-info`; it uses `inputs.sha` - when supplied and otherwise sets `RAPIDS_SHA` to `git rev-parse HEAD`. - -This distinction matters when a reusable workflow checks out a repository or -revision different from the workflow event. Direct callers must likewise set -`RAPIDS_SHA` to the checked-out commit instead of assuming `${{ github.sha }}` -names the built source. Variables written to `GITHUB_ENV` are available to the -action when it runs as a subsequent job step. - -## Standard package example +This is a list of objects, where each object corresponds to exactly one artifact +(filename). Standard Conda and wheel workflows do not explicitly specify +`artifacts`. Instead, the action discovers every Conda and wheel output in +`artifact_directory` and creates corresponding artifact entries. ```yaml - name: Create wheel release catalog companion @@ -87,7 +81,8 @@ action when it runs as a subsequent job step. source-artifact-name: ${{ steps.package-name.outputs.RAPIDS_PACKAGE_NAME }} ``` -## Custom package example +For artifacts other than conda and wheels, path and package_identity_file must +be specified: ```yaml - name: Create custom release catalog companion @@ -105,10 +100,21 @@ action when it runs as a subsequent job step. source-artifact-name: cuvs-java ``` -Before the action runs, the producer creates -`java/cuvs-java/target/cuvs-java.release-package-identity.json`. The -artifact descriptor's `package_identity_file` value is the path to that file -relative to `artifact_directory`. For example: +Wildcards are allowed to allow for variance in filenames, but each wildcard must +resolve to only one file. In other words, each list object corresponds to +exactly one artifact, and any ambiguity is an error. + +#### Package identity file + +Package identity is high-level information about an artifact outside of its +filename. It requires `ecosystem`, `name`, and `version` and may include `build` +and `platform`. The action implicitly parses this information from conda and +wheel artifacts, but other artifact formats require the producer to provide it +explicitly. + +The workflow that calls release-catalog must create each package identity file before this +action runs. An artifact descriptor's `package_identity_file` is the path to +that file relative to `artifact_directory`. For example: ```json { @@ -118,6 +124,25 @@ relative to `artifact_directory`. For example: } ``` +Multiple artifact descriptors may reference the same identity file when those +artifacts have the same package identity. + +## Prerequisite state + +The action requires `RAPIDS_SHA` in the job environment. It must identify the +repository revision actually checked out and built: + +- Conda build workflows set `RAPIDS_SHA` to `git rev-parse HEAD` immediately + after checkout. +- Wheel and custom workflows use `rapids-github-info`; it uses `inputs.sha` + when supplied and otherwise sets `RAPIDS_SHA` to `git rev-parse HEAD`. + +This distinction matters when a reusable workflow checks out a repository or +revision different from the workflow event. Direct callers must likewise set +`RAPIDS_SHA` to the checked-out commit instead of assuming `${{ github.sha }}` +names the built source. Variables written to `GITHUB_ENV` are available to the +action when it runs as a subsequent job step. + ## Generated identity evidence For every artifact, the action generates an SPDX artifact-identity envelope @@ -126,6 +151,7 @@ They record the artifact SHA-256, package identity, source revision, and workflo context, but contain no dependency or source-license inventory. They must not be reported as producer-supplied dependency coverage. -Producer-supplied SBOM, provenance, and signature inputs are intentionally -deferred until a future revision defines a generic, unambiguous way to associate -them with detected artifacts. +Producer-supplied dependency SBOMs and richer build provenance, such as build +dependencies and compiler flags, would make package contents easier to +understand without downloading them. Associating that input-side evidence with +artifacts is left for future work. diff --git a/release-catalog/config.schema.json b/release-catalog/config.schema.json index 7df259e0..11165ee5 100644 --- a/release-catalog/config.schema.json +++ b/release-catalog/config.schema.json @@ -45,7 +45,7 @@ "pattern": "^[^\\r\\n]+$" }, "package_identity_file": { - "description": "Path, relative to artifact_directory, to producer-created package identity JSON for this artifact. Required only when identity cannot be extracted from the artifact. The file must contain ecosystem, name, and version and may contain build and platform.", + "description": "Path, relative to artifact_directory, to producer-created package identity JSON for this artifact. Required only when identity cannot be extracted from the artifact. See https://github.com/rapidsai/shared-actions/blob/main/release-catalog/README.md#package-identity-file", "type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$" From f596863141aca042647553d6323b44599a69be17 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 18 Aug 2026 14:58:27 -0500 Subject: [PATCH 15/36] fix release catalog CI smoke test --- .github/workflows/pr.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 2a578473..de53b856 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -23,29 +23,33 @@ jobs: - name: Test release catalog materialization run: | ./tests/release_catalog_config_test.sh - ./tests/release_catalog_descriptors_test.sh - ./tests/release_catalog_prepare_test.sh + ./tests/release_catalog_discovery_test.sh + ./tests/release_catalog_package_parsing_test.sh ./tests/release_catalog_test.sh - name: Prepare release catalog dispatch smoke test run: | mkdir -p release-catalog-smoke printf '%s\n' smoke >release-catalog-smoke/package.tar.gz + jq -n \ + '{ecosystem: "archive", name: "smoke", version: "1.0"}' \ + >release-catalog-smoke/package-identity.json - name: Run release catalog dispatch smoke test uses: ./release-catalog-dispatch env: + RAPIDS_SHA: ${{ github.sha }} SHARED_ACTIONS_REPO: ${{ github.event.pull_request.head.repo.full_name }} SHARED_ACTIONS_REF: ${{ github.event.pull_request.head.sha }} with: config: >- { - "artifact_type": "custom", "release_catalog_key": "archive:smoke", - "output_directory": "release-catalog-smoke", - "package": {"ecosystem": "archive", "name": "smoke", "version": "1.0"}, - "artifacts": [{"path": "package.tar.gz"}] + "artifact_directory": "release-catalog-smoke", + "artifacts": [{ + "path": "package.tar.gz", + "package_identity_file": "package-identity.json" + }] } source-artifact-name: release-catalog-dispatch-smoke - source-sha: ${{ github.event.pull_request.head.sha }} - name: Verify release catalog dispatch smoke test run: | jq -e ' From 609f2d2fdafd0e1c247dc78c08fb91ba686aaf3d Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 18 Aug 2026 15:06:10 -0500 Subject: [PATCH 16/36] more readme edits --- release-catalog/README.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/release-catalog/README.md b/release-catalog/README.md index fb2cdc4d..c37275df 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -81,7 +81,7 @@ This is a list of objects, where each object corresponds to exactly one artifact source-artifact-name: ${{ steps.package-name.outputs.RAPIDS_PACKAGE_NAME }} ``` -For artifacts other than conda and wheels, path and package_identity_file must +For artifacts other than conda and wheels, `path` and `package_identity_file` must be specified: ```yaml @@ -110,11 +110,10 @@ Package identity is high-level information about an artifact outside of its filename. It requires `ecosystem`, `name`, and `version` and may include `build` and `platform`. The action implicitly parses this information from conda and wheel artifacts, but other artifact formats require the producer to provide it -explicitly. - -The workflow that calls release-catalog must create each package identity file before this -action runs. An artifact descriptor's `package_identity_file` is the path to -that file relative to `artifact_directory`. For example: +explicitly using the `package_identity_file` parameter. Any referenced package +identity file must be created before this action runs. An artifact descriptor's +`package_identity_file` is the path to that file relative to +`artifact_directory`. Example contents of a package_identity_file: ```json { From 91ddd9d2fbcbb80e02c04f59842d6914dbdd164c Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 21 Aug 2026 13:50:27 -0500 Subject: [PATCH 17/36] feat: stage release catalog bundles in S3 --- .github/workflows/pr.yml | 1 + release-catalog-dispatch/action.yml | 38 ++++++++---- release-catalog/upload-s3.sh | 91 +++++++++++++++++++++++++++++ 3 files changed, 120 insertions(+), 10 deletions(-) create mode 100644 release-catalog/upload-s3.sh diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index de53b856..31212bef 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -50,6 +50,7 @@ jobs: }] } source-artifact-name: release-catalog-dispatch-smoke + candidate-train-sha256: ${{ github.sha }} - name: Verify release catalog dispatch smoke test run: | jq -e ' diff --git a/release-catalog-dispatch/action.yml b/release-catalog-dispatch/action.yml index 77758741..0df32784 100644 --- a/release-catalog-dispatch/action.yml +++ b/release-catalog-dispatch/action.yml @@ -1,5 +1,5 @@ name: Dispatch release catalog -description: Create a release catalog companion from the selected shared-actions revision. +description: Write a release-candidate catalog bundle directly to the private S3 store. inputs: config: @@ -8,8 +8,23 @@ inputs: https://github.com/rapidsai/shared-actions/blob/main/release-catalog/config.schema.json required: true source-artifact-name: - description: Name of the existing GitHub Actions artifact whose files are being cataloged. + description: Stable name of this producing artifact bundle. required: true + candidate-train-sha256: + description: Canonical SHA-256 of the immutable release-train JSON. + required: true + candidate-bucket: + description: Private S3 bucket that holds candidate-build bytes. + required: false + default: rapids-release-candidates + candidate-prefix: + description: Root prefix for train-scoped candidate-build bytes. + required: false + default: candidate-builds + upload-to-s3: + description: Whether to upload the generated bundle to the candidate store. + required: false + default: "false" runs: using: composite steps: @@ -49,11 +64,14 @@ runs: RELEASE_SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} RELEASE_CATALOG_KEY: ${{ steps.configuration.outputs.release_catalog_key }} run: "${IMPLEMENTATION_PATH}/materialize.sh" - - name: Upload release catalog companion - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - if-no-files-found: error - name: release-catalog-${{ inputs.source-artifact-name }} - path: | - ${{ steps.configuration.outputs.artifact_directory }}/release-catalog-entries.json - ${{ steps.configuration.outputs.artifact_directory }}/release-evidence/** + - name: Write release-candidate bundle to S3 + if: ${{ inputs.upload-to-s3 == 'true' }} + shell: bash + env: + IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} + RELEASE_ARTIFACT_DIRECTORY: ${{ steps.configuration.outputs.artifact_directory }} + RELEASE_CANDIDATE_BUCKET: ${{ inputs.candidate-bucket }} + RELEASE_CANDIDATE_PREFIX: ${{ inputs.candidate-prefix }} + RELEASE_CANDIDATE_TRAIN_SHA256: ${{ inputs.candidate-train-sha256 }} + RELEASE_SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} + run: "${IMPLEMENTATION_PATH}/upload-s3.sh" diff --git a/release-catalog/upload-s3.sh b/release-catalog/upload-s3.sh new file mode 100644 index 00000000..0e8c92db --- /dev/null +++ b/release-catalog/upload-s3.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +# Upload only the files declared by the generated catalog document. GitHub +# Actions retains logs and this receipt, but never a second release-artifact +# copy. Conditional S3 writes make reruns safe: an existing object is accepted +# only when its checksum exactly matches the byte about to be uploaded. +set -euo pipefail + +require_value() { + local name="$1" + local value="$2" + if [[ -z "${value}" || "${value}" == *$'\n'* || "${value}" == *$'\r'* ]]; then + echo "${name} must be a non-empty single-line string" >&2 + exit 1 + fi +} + +safe_prefix() { + local prefix="$1" + [[ "${prefix}" != /* && "${prefix}" != */ && "${prefix}" != *'//' && "${prefix}" != *'..'* ]] +} + +for value in RELEASE_ARTIFACT_DIRECTORY RELEASE_CANDIDATE_BUCKET RELEASE_CANDIDATE_PREFIX RELEASE_CANDIDATE_TRAIN_SHA256 RELEASE_SOURCE_ARTIFACT_NAME GITHUB_REPOSITORY GITHUB_RUN_ID; do + require_value "${value}" "${!value:-}" +done +if [[ ! "${RELEASE_CANDIDATE_TRAIN_SHA256}" =~ ^[[:xdigit:]]{64}$ ]]; then + echo "RELEASE_CANDIDATE_TRAIN_SHA256 must be a SHA-256 hex digest" >&2 + exit 1 +fi +if [[ "${RELEASE_SOURCE_ARTIFACT_NAME}" == */* || "${RELEASE_SOURCE_ARTIFACT_NAME}" == *".."* ]]; then + echo "RELEASE_SOURCE_ARTIFACT_NAME must be a plain bundle name" >&2 + exit 1 +fi +if ! safe_prefix "${RELEASE_CANDIDATE_PREFIX}"; then + echo "RELEASE_CANDIDATE_PREFIX must be a safe, relative S3 prefix" >&2 + exit 1 +fi + +bundle_root="$(realpath "${RELEASE_ARTIFACT_DIRECTORY}")" +entries_path="${bundle_root}/release-catalog-entries.json" +if [[ ! -f "${entries_path}" ]]; then + echo "release catalog entries are missing: ${entries_path}" >&2 + exit 1 +fi + +# Catalog paths are relative to the build bundle. Reject traversal before an +# S3 key is constructed, even though materialize.sh already validates them. +safe_relative_path() { + local path="$1" + [[ -n "${path}" && "${path}" != /* && "${path}" != ../* && "${path}" != */../* && "${path}" != *'/..' ]] +} + +base_key="${RELEASE_CANDIDATE_PREFIX}/${RELEASE_CANDIDATE_TRAIN_SHA256}/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID}/${RELEASE_SOURCE_ARTIFACT_NAME}" +mapfile -t bundle_paths < <(jq -r '["release-catalog-entries.json"] + ([.entries[] | .path, .sbom, .provenance, .signature?] | unique) | .[]' "${entries_path}") +head_metadata="$(mktemp)" +trap 'rm -f "${head_metadata}"' EXIT + +upload_one() { + local relative_path="$1" + local local_path="${bundle_root}/${relative_path}" + local object_key="${base_key}/${relative_path}" + local checksum + checksum="$(openssl dgst -sha256 -binary "${local_path}" | base64)" + + if aws s3api head-object --bucket "${RELEASE_CANDIDATE_BUCKET}" --key "${object_key}" --checksum-mode ENABLED >"${head_metadata}" 2>/dev/null; then + if [[ "$(jq -r '.ChecksumSHA256 // empty' "${head_metadata}")" != "${checksum}" ]]; then + echo "existing candidate object has different bytes: ${object_key}" >&2 + exit 1 + fi + return + fi + + if ! aws s3api put-object --bucket "${RELEASE_CANDIDATE_BUCKET}" --key "${object_key}" --body "${local_path}" --checksum-algorithm SHA256 --checksum-sha256 "${checksum}" --if-none-match '*' --tagging 'release-candidate-status=candidate' >/dev/null 2>&1; then + aws s3api head-object --bucket "${RELEASE_CANDIDATE_BUCKET}" --key "${object_key}" --checksum-mode ENABLED >"${head_metadata}" + if [[ "$(jq -r '.ChecksumSHA256 // empty' "${head_metadata}")" != "${checksum}" ]]; then + echo "candidate object could not be written immutably: ${object_key}" >&2 + exit 1 + fi + fi +} + +for relative_path in "${bundle_paths[@]}"; do + if ! safe_relative_path "${relative_path}" || [[ ! -f "${bundle_root}/${relative_path}" ]]; then + echo "catalog declares a missing or unsafe candidate file: ${relative_path}" >&2 + exit 1 + fi + upload_one "${relative_path}" +done + +printf 'Release candidate bundle: s3://%s/%s\n' "${RELEASE_CANDIDATE_BUCKET}" "${base_key}" >>"${GITHUB_STEP_SUMMARY}" From 94423626faa949e76b5bcaf8351e27bfd0c0e230 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 21 Aug 2026 14:55:31 -0500 Subject: [PATCH 18/36] fix: make candidate upload script executable --- release-catalog/upload-s3.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 release-catalog/upload-s3.sh diff --git a/release-catalog/upload-s3.sh b/release-catalog/upload-s3.sh old mode 100644 new mode 100755 From 503e3fb79b970740b2c3c28406a6b3988d286523 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 21 Aug 2026 16:52:11 -0500 Subject: [PATCH 19/36] fix: ignore absent release signatures --- release-catalog/upload-s3.sh | 5 ++++- tests/release_catalog_test.sh | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/release-catalog/upload-s3.sh b/release-catalog/upload-s3.sh index 0e8c92db..a9a9556a 100755 --- a/release-catalog/upload-s3.sh +++ b/release-catalog/upload-s3.sh @@ -52,7 +52,10 @@ safe_relative_path() { } base_key="${RELEASE_CANDIDATE_PREFIX}/${RELEASE_CANDIDATE_TRAIN_SHA256}/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID}/${RELEASE_SOURCE_ARTIFACT_NAME}" -mapfile -t bundle_paths < <(jq -r '["release-catalog-entries.json"] + ([.entries[] | .path, .sbom, .provenance, .signature?] | unique) | .[]' "${entries_path}") +# `signature` is optional, so select only declared string paths. Without the +# filter, jq renders a missing optional value as the literal text `null` and +# the uploader attempts to find a file with that name. +mapfile -t bundle_paths < <(jq -r '["release-catalog-entries.json"] + ([.entries[] | .path, .sbom, .provenance, .signature? | strings] | unique) | .[]' "${entries_path}") head_metadata="$(mktemp)" trap 'rm -f "${head_metadata}"' EXIT diff --git a/tests/release_catalog_test.sh b/tests/release_catalog_test.sh index 86ce8c89..3a019b51 100755 --- a/tests/release_catalog_test.sh +++ b/tests/release_catalog_test.sh @@ -47,6 +47,12 @@ jq -e ' and .entries[0].sbom_kind == "generated-identity" ' "${entries_path}" >/dev/null +# Optional signatures must not cause the S3 uploader to request a literal +# `null` file. This mirrors its declared-file selection without requiring AWS. +mapfile -t upload_paths < <(jq -r '["release-catalog-entries.json"] + ([.entries[] | .path, .sbom, .provenance, .signature? | strings] | unique) | .[]' "${entries_path}") +test "${upload_paths[*]}" = "release-catalog-entries.json cuvs-java-26.08.0.jar release-evidence/cuvs-java-26.08.0.jar.sbom.spdx.json release-evidence/cuvs-java-26.08.0.jar.provenance.json" +! printf '%s\n' "${upload_paths[@]}" | grep -Fx null + isolated_companion_directory="${temporary_directory}/isolated-companion" mkdir -p "${isolated_companion_directory}" cp "${entries_path}" "${isolated_companion_directory}/" From b36cc4fc952e3109d3a70438fce01240b1eb5c65 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 31 Aug 2026 12:38:43 -0500 Subject: [PATCH 20/36] Address release catalog review feedback --- .github/workflows/pr.yml | 16 +- README.md | 28 --- ci/run_release_catalog_tests.sh | 11 ++ release-catalog/README.md | 173 ++++++++---------- release-catalog/config.schema.json | 2 +- release-catalog/examples/README.md | 17 ++ .../cuvs-java/release-catalog-entries.json | 29 +++ ...1715b14a3f7c349cba300cb2b9.provenance.json | 41 +++++ ...cd1715b14a3f7c349cba300cb2b9.sbom.cdx.json | 30 +++ ...d1715b14a3f7c349cba300cb2b9.sbom.spdx.json | 39 ++++ release-catalog/materialize.sh | 141 ++++++++++++-- release-catalog/upload-s3.sh | 2 +- release-catalog/validate-config.sh | 5 + tests/release_catalog_discovery_test.sh | 5 + tests/release_catalog_package_parsing_test.sh | 5 + tests/release_catalog_test.sh | 102 ++++++++++- 16 files changed, 484 insertions(+), 162 deletions(-) create mode 100755 ci/run_release_catalog_tests.sh create mode 100644 release-catalog/examples/README.md create mode 100644 release-catalog/examples/cuvs-java/release-catalog-entries.json create mode 100644 release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.provenance.json create mode 100644 release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.cdx.json create mode 100644 release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.spdx.json diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 31212bef..ba2ac20c 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -20,12 +20,18 @@ jobs: fetch-depth: 1 persist-credentials: false - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 + + test-release-catalog-actions: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false - name: Test release catalog materialization - run: | - ./tests/release_catalog_config_test.sh - ./tests/release_catalog_discovery_test.sh - ./tests/release_catalog_package_parsing_test.sh - ./tests/release_catalog_test.sh + run: ./ci/run_release_catalog_tests.sh - name: Prepare release catalog dispatch smoke test run: | mkdir -p release-catalog-smoke diff --git a/README.md b/README.md index 40f29a86..995a28c8 100644 --- a/README.md +++ b/README.md @@ -9,34 +9,6 @@ A dispatch action is one that: * clones the shared-actions repository (repo/ref changeable using env vars) * runs (dispatches to) another action within the clone, using a relative path -## Release catalog companions - -`release-catalog-dispatch` validates a producer's local build artifact -directory and uploads a companion artifact named -`release-catalog-`. - -See the [release catalog documentation](release-catalog/README.md) -for configuration, examples, source-revision handling, companion contents, and -generated identity evidence semantics. - -The dispatch wrapper honors `SHARED_ACTIONS_REPO` and `SHARED_ACTIONS_REF`. -When neither is set, it checks out the same repository and ref used to invoke -the wrapper, which allows a feature-branch wrapper to dispatch to its matching -implementation during canary testing. - -There can be more complicated arrangements of more actions, but the idea is to -have the local clone of the shared-actions repository be the first step of an action. - -Actions that refer to each other assume that they have been checked out to the -./shared-actions folder. This *should* be the root of the GitHub Actions workspace. -This assumption is what allow code reuse between actions. - -Actions that use this pattern should include "dispatch" in their folder name, so -that they can be readily distinguished from any actions that are either -standalone or otherwise implementations that assume that the ./shared-actions -folder is already cloned, so that they can use relative paths to reference other -actions and files. - ## Example dispatch action ```yaml diff --git a/ci/run_release_catalog_tests.sh b/ci/run_release_catalog_tests.sh new file mode 100755 index 00000000..e598d2ba --- /dev/null +++ b/ci/run_release_catalog_tests.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +set -euo pipefail + +repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +"${repository_root}/tests/release_catalog_config_test.sh" +"${repository_root}/tests/release_catalog_discovery_test.sh" +"${repository_root}/tests/release_catalog_package_parsing_test.sh" +"${repository_root}/tests/release_catalog_test.sh" diff --git a/release-catalog/README.md b/release-catalog/README.md index c37275df..0b7da2e9 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -1,73 +1,85 @@ -# Release catalog companions +# Release catalog companion archives -`release-catalog-dispatch` records the exact files produced by a package -build and uploads a companion GitHub Actions artifact named -`release-catalog-`. Release tooling uses the -companion to associate primary artifacts with their build identity and generated -identity evidence without reconstructing the build later. This evidence supports: +This action creates a companion archive alongside our binary artifacts (conda, +wheel or otherwise) that contains an inventory of the binary artifacts that were +produced by that job. That inventory is used by the RAPIDS release system to +assemble release candidates. "Release candidates" here refers to a set of +artifacts that can be deployed to create a release, rather than a release +candidate version of one artifact. -- [assembling releases that can be tested and verified before formal tagging](https://github.com/rapidsai/release-scripts/issues/102) -- using build-time metadata to triage CVE scan records more quickly (internal - GitLab project, `nspect-manager`) +## Companion contents -The companion contains: +Each build job creates one companion archive with this shape: ```text . ├── release-catalog-entries.json └── release-evidence - ├── .provenance.json - └── .spdx.json + ├── ..provenance.json + ├── ..sbom.cdx.json + ├── ..sbom.spdx.json + ├── ..provenance.json + ├── ..sbom.cdx.json + ├── ..sbom.spdx.json + └── ... ``` -`release-catalog-entries.json` is one atomic job-level envelope. Its `source` -object records the source artifact and build context, while its `entries` array -contains the release catalog entries produced by that job. The release platform -validates and aggregates entry arrays from selected builds into the release -catalog; there is no separate metadata document to keep synchronized. - -Each matrix job produces its own companion artifact. The release platform merges -their entries into one composite release catalog. - -## Configuration - -Every build job that calls this action passes one `config` JSON object. Its -canonical schema and field documentation are in -[`config.schema.json`](config.schema.json). - -The action validates the configuration before inspecting build outputs. It then -verifies properties that depend on produced files, including the package -identity contents and whether primary-artifact paths resolve unambiguously. +`release-catalog-entries.json` records the artifacts produced by the job: +`.conda`, `.whl`, or otherwise, as well as the checksums of those files. + +The release-evidence folder carries additional metadata that may be useful to +consumers of our packages: provenance and software bill of materials (SBOM). +Provenance describes how a package was built and by whom. An SBOM lists +components that make up the package. They complement each other: an SBOM without +provenance may describe the wrong or an untrusted artifact, while provenance +without an SBOM cannot efficiently answer what vulnerable components are inside. +For each artifact in its `entries` array, the action creates one provenance file +and equivalent identity SBOMs in SPDX and CycloneDX formats. The SHA-256 in each +evidence filename is the digest of that specific artifact's contents. The +catalog entry identifies both SBOM paths under `sboms`, allowing consumers to +select their preferred format without inferring it from the filename. + +The provenance.json file follows the +[SLSA](https://slsa.dev/spec/v1.2/attestation-model) standard format. + +The `sbom.spdx.json` file uses [SPDX +2.3](https://spdx.github.io/spdx-spec/v2.3/), while `sbom.cdx.json` uses +[CycloneDX 1.6](https://cyclonedx.org/docs/1.6/json/). These generated documents +identify the artifact and its digest; they do not contain a dependency +inventory. + +Provenance remains separate because it is an independently verifiable statement +about how and where the artifact was built. Although both SBOM standards can +carry some build metadata, embedding provenance in the SBOMs would not replace +the in-toto/SLSA statement and would couple evidence with different consumers +and lifecycles. + +## Action Configuration + +Every caller passes one `config` JSON object. The canonical schema and field +documentation are in [`config.schema.json`](config.schema.json). This is +validated at runtime with the [validate-config.sh](validate-config.sh) script. ### `release_catalog_key` -The `release_catalog_key` is used to group artifacts in the catalog. Every file -from every matrix variant in the same publishable artifact set uses the same -key. The release platform aggregates multiple `release-catalog-entries.json` -files and coalesces entries with the same `release_catalog_key`. Standard RAPIDS -conda and wheel workflows construct the key as `:`, -such as `conda:cudf`. Custom producers select a key, such as `maven:cuvs-java`, -that represents their release policy. +This is a grouping label that represents a common "release policy." Artifacts +that share release_catalog_key are versioned, validated, ordered, published, and +promoted together. The key does not need to be unique per artifact, nor per +matrix variant. For example, `cudf` and `dask-cudf` Conda packages can both use +`conda:cudf` because they share the same version, validation, inter-package +order, publishing destination, and are ultimately published together. -Use a distinct `release_catalog_key` only when the outputs intentionally have -different release policies. Reasons include differences in: +Standard RAPIDS Conda and wheel workflows use `:`, +such as `conda:cudf`, because those workflows apply a repository-level release +policy. A custom producer should use `:`, such as +`maven:cuvs-java`, or simply `maven:cuvs`. Use a separate key only when the +outputs intentionally have a separate release workflows that must be followed. -- versioning schemes -- validation requirements -- dependency ordering -- publication destinations -- promotion strategy +### `artifact_directory` and `artifacts` -Multiple package names from one repository do not by themselves justify separate -keys: for example, `cudf` and `dask-cudf` Conda packages remain part of -`conda:cudf` when they share one release policy. - -### `artifacts` - -This is a list of objects, where each object corresponds to exactly one artifact -(filename). Standard Conda and wheel workflows do not explicitly specify -`artifacts`. Instead, the action discovers every Conda and wheel output in -`artifact_directory` and creates corresponding artifact entries. +`artifact_directory` is the base directory containing the primary artifacts. +For standard Conda and wheel jobs, omit `artifacts`; the action discovers all +Conda packages and wheels below that directory and parses metadata from them. ```yaml - name: Create wheel release catalog companion @@ -81,8 +93,9 @@ This is a list of objects, where each object corresponds to exactly one artifact source-artifact-name: ${{ steps.package-name.outputs.RAPIDS_PACKAGE_NAME }} ``` -For artifacts other than conda and wheels, `path` and `package_identity_file` must -be specified: +Other formats require an explicit `artifacts` list. Each dictionary in the list +corresponds to one artifact file. Wildcards are allowed, but a wildcard that +resolves to zero files or multiple files results in an error. ```yaml - name: Create custom release catalog companion @@ -100,20 +113,11 @@ be specified: source-artifact-name: cuvs-java ``` -Wildcards are allowed to allow for variance in filenames, but each wildcard must -resolve to only one file. In other words, each list object corresponds to -exactly one artifact, and any ambiguity is an error. - #### Package identity file -Package identity is high-level information about an artifact outside of its -filename. It requires `ecosystem`, `name`, and `version` and may include `build` -and `platform`. The action implicitly parses this information from conda and -wheel artifacts, but other artifact formats require the producer to provide it -explicitly using the `package_identity_file` parameter. Any referenced package -identity file must be created before this action runs. An artifact descriptor's -`package_identity_file` is the path to that file relative to -`artifact_directory`. Example contents of a package_identity_file: +package_identity_file is a JSON file containing at least `ecosystem`, `name`, +and `version`; `build` and `platform` are optional. This path is relative to +`artifact_directory`. ```json { @@ -123,34 +127,5 @@ identity file must be created before this action runs. An artifact descriptor's } ``` -Multiple artifact descriptors may reference the same identity file when those -artifacts have the same package identity. - -## Prerequisite state - -The action requires `RAPIDS_SHA` in the job environment. It must identify the -repository revision actually checked out and built: - -- Conda build workflows set `RAPIDS_SHA` to `git rev-parse HEAD` immediately - after checkout. -- Wheel and custom workflows use `rapids-github-info`; it uses `inputs.sha` - when supplied and otherwise sets `RAPIDS_SHA` to `git rev-parse HEAD`. - -This distinction matters when a reusable workflow checks out a repository or -revision different from the workflow event. Direct callers must likewise set -`RAPIDS_SHA` to the checked-out commit instead of assuming `${{ github.sha }}` -names the built source. Variables written to `GITHUB_ENV` are available to the -action when it runs as a subsequent job step. - -## Generated identity evidence - -For every artifact, the action generates an SPDX artifact-identity envelope -classified as `generated-identity` and a build-context provenance statement. -They record the artifact SHA-256, package identity, source revision, and workflow -context, but contain no dependency or source-license inventory. They must not be -reported as producer-supplied dependency coverage. - -Producer-supplied dependency SBOMs and richer build provenance, such as build -dependencies and compiler flags, would make package contents easier to -understand without downloading them. Associating that input-side evidence with -artifacts is left for future work. +Multiple artifacts in the list may reference the same identity file when the +files have the same package identity. diff --git a/release-catalog/config.schema.json b/release-catalog/config.schema.json index 11165ee5..09f1a5b9 100644 --- a/release-catalog/config.schema.json +++ b/release-catalog/config.schema.json @@ -10,7 +10,7 @@ ], "properties": { "release_catalog_key": { - "description": "Release-catalog entry name that owns these artifacts. Default for conda & wheels is :, such as conda:cudf. Custom jobs specify their own, such as maven:cuvs-java.", + "description": "Caller-supplied stable label for artifacts that share one release policy. This is not a UUID, is not generated by the action, and need not be unique per package. Standard workflows use :, such as conda:cudf; custom jobs use :, such as maven:cuvs-java. See https://github.com/rapidsai/shared-actions/blob/main/release-catalog/README.md#release_catalog_key", "type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$" diff --git a/release-catalog/examples/README.md b/release-catalog/examples/README.md new file mode 100644 index 00000000..0edcda07 --- /dev/null +++ b/release-catalog/examples/README.md @@ -0,0 +1,17 @@ +# Example companion + +[`cuvs-java`](cuvs-java) is a complete, representative companion for one cuVS +Java build job. It was derived from the test fixture in +`tests/release_catalog_test.sh`; its source SHA and GitHub Actions run ID are +deliberately non-production example values. + +The primary JAR is not part of the companion and is therefore not checked in. +Its example contents are `jar` followed by a newline, whose SHA-256 digest is +`fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9`. +That digest links the catalog entry, SPDX and CycloneDX documents, provenance +statement, and the separately uploaded primary artifact. + +The four JSON files are about 4 KiB in total. Companion size grows +linearly with the number of primary artifacts. This version does not copy +producer dependency SBOMs into the companion; those could be substantially +larger and require a future explicit catalog association. diff --git a/release-catalog/examples/cuvs-java/release-catalog-entries.json b/release-catalog/examples/cuvs-java/release-catalog-entries.json new file mode 100644 index 00000000..b446b7a7 --- /dev/null +++ b/release-catalog/examples/cuvs-java/release-catalog-entries.json @@ -0,0 +1,29 @@ +{ + "entries": [ + { + "package": { + "ecosystem": "maven", + "name": "ai.rapids:cuvs-java", + "version": "26.08.0" + }, + "path": "cuvs-java-26.08.0.jar", + "provenance": "release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.provenance.json", + "release_catalog_key": "maven:cuvs-java", + "sbom_kind": "generated-identity", + "sboms": { + "cyclonedx": "release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.cdx.json", + "spdx": "release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.spdx.json" + } + } + ], + "producer": "shared-workflows", + "schema_version": 1, + "source": { + "artifact": "cuvs-java-cuda12.9.1", + "repository": "NVIDIA/cuvs", + "run_attempt": "1", + "run_id": "1234", + "sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "workflow_ref": "NVIDIA/cuvs/.github/workflows/build.yaml@refs/heads/release/26.08" + } +} diff --git a/release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.provenance.json b/release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.provenance.json new file mode 100644 index 00000000..8b7189b8 --- /dev/null +++ b/release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.provenance.json @@ -0,0 +1,41 @@ +{ + "_type": "https://in-toto.io/Statement/v1", + "predicate": { + "buildDefinition": { + "buildType": "https://rapids.ai/release-platform/catalog-record/v1", + "externalParameters": { + "package": { + "ecosystem": "maven", + "name": "ai.rapids:cuvs-java", + "version": "26.08.0" + }, + "release_catalog_key": "maven:cuvs-java" + }, + "resolvedDependencies": [ + { + "digest": { + "gitCommit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + "uri": "git+https://github.com/NVIDIA/cuvs@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + ] + }, + "runDetails": { + "builder": { + "id": "https://github.com/NVIDIA/cuvs/.github/workflows/build.yaml@refs/heads/release/26.08" + }, + "metadata": { + "invocationId": "https://github.com/NVIDIA/cuvs/actions/runs/1234/attempts/1" + } + } + }, + "predicateType": "https://slsa.dev/provenance/v1", + "subject": [ + { + "digest": { + "sha256": "fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9" + }, + "name": "cuvs-java-26.08.0.jar" + } + ] +} diff --git a/release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.cdx.json b/release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.cdx.json new file mode 100644 index 00000000..617e270c --- /dev/null +++ b/release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.cdx.json @@ -0,0 +1,30 @@ +{ + "bomFormat": "CycloneDX", + "metadata": { + "component": { + "bom-ref": "urn:sha256:fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9", + "hashes": [ + { + "alg": "SHA-256", + "content": "fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9" + } + ], + "name": "ai.rapids:cuvs-java", + "properties": [ + { + "name": "rapids:artifact:path", + "value": "cuvs-java-26.08.0.jar" + }, + { + "name": "rapids:package:ecosystem", + "value": "maven" + } + ], + "type": "file", + "version": "26.08.0" + }, + "timestamp": "2026-08-21T00:00:00Z" + }, + "specVersion": "1.6", + "version": 1 +} diff --git a/release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.spdx.json b/release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.spdx.json new file mode 100644 index 00000000..dc1ea043 --- /dev/null +++ b/release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.spdx.json @@ -0,0 +1,39 @@ +{ + "SPDXID": "SPDXRef-DOCUMENT", + "comment": "Artifact-identity SBOM envelope. It does not contain a dependency inventory.", + "creationInfo": { + "created": "2026-08-21T00:00:00Z", + "creators": [ + "Tool: rapidsai/shared-workflows release catalog" + ] + }, + "dataLicense": "CC0-1.0", + "documentDescribes": [ + "SPDXRef-Artifact" + ], + "documentNamespace": "https://rapids.ai/release-platform/spdx/fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9", + "name": "RAPIDS release artifact cuvs-java-26.08.0.jar", + "packages": [ + { + "SPDXID": "SPDXRef-Artifact", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9" + } + ], + "downloadLocation": "NOASSERTION", + "filesAnalyzed": false, + "name": "ai.rapids:cuvs-java", + "versionInfo": "26.08.0" + } + ], + "relationships": [ + { + "relatedSpdxElement": "SPDXRef-Artifact", + "relationshipType": "DESCRIBES", + "spdxElementId": "SPDXRef-DOCUMENT" + } + ], + "spdxVersion": "SPDX-2.3" +} diff --git a/release-catalog/materialize.sh b/release-catalog/materialize.sh index 1bde11ee..17304ce4 100755 --- a/release-catalog/materialize.sh +++ b/release-catalog/materialize.sh @@ -3,6 +3,19 @@ set -euo pipefail +# Convert one completed build directory into a release catalog companion. +# +# Inputs come from release-catalog-dispatch/action.yml. RELEASE_ARTIFACTS is +# either empty (discover Conda packages and wheels) or a validated JSON array +# selecting custom artifacts. GitHub Actions supplies the GITHUB_* build +# context, while RAPIDS_SHA identifies the commit actually checked out. +# +# The script validates all context before touching outputs, resolves every +# artifact to exactly one file inside RELEASE_ARTIFACT_DIRECTORY, reads or +# extracts its package identity, generates identity-only SPDX and CycloneDX +# SBOMs plus provenance evidence, and writes RELEASE_ENTRIES_NAME. It never +# modifies primary files. + require_nonempty() { local name="$1" local value="$2" @@ -12,13 +25,54 @@ require_nonempty() { fi } -require_nonempty "RELEASE_CATALOG_KEY" "${RELEASE_CATALOG_KEY:-}" -require_nonempty "RELEASE_ARTIFACT_DIRECTORY" "${RELEASE_ARTIFACT_DIRECTORY:-}" -require_nonempty "RELEASE_ENTRIES_NAME" "${RELEASE_ENTRIES_NAME:-}" -require_nonempty "RELEASE_SOURCE_ARTIFACT_NAME" "${RELEASE_SOURCE_ARTIFACT_NAME:-}" +require_single_line() { + local name="$1" + local value="$2" + require_nonempty "${name}" "${value}" + if [[ "${value}" == *$'\n'* || "${value}" == *$'\r'* ]]; then + echo "${name} must be a single-line string" >&2 + exit 1 + fi +} + +require_positive_integer() { + local name="$1" + local value="$2" + if [[ ! "${value}" =~ ^[1-9][0-9]*$ ]]; then + echo "${name} must be a positive integer" >&2 + exit 1 + fi +} + +# Fail before materialization when provenance would otherwise contain empty or +# malformed source fields. These values are guaranteed in GitHub Actions; a +# local caller must set explicit test values. +require_single_line "RELEASE_CATALOG_KEY" "${RELEASE_CATALOG_KEY:-}" +require_single_line "RELEASE_ARTIFACT_DIRECTORY" "${RELEASE_ARTIFACT_DIRECTORY:-}" +require_single_line "RELEASE_ENTRIES_NAME" "${RELEASE_ENTRIES_NAME:-}" +require_single_line "RELEASE_SOURCE_ARTIFACT_NAME" "${RELEASE_SOURCE_ARTIFACT_NAME:-}" source_sha="${RAPIDS_SHA:-}" -require_nonempty "RAPIDS_SHA" "${source_sha}" +require_single_line "RAPIDS_SHA" "${source_sha}" +if [[ ! "${source_sha}" =~ ^[[:xdigit:]]{40}$ && ! "${source_sha}" =~ ^[[:xdigit:]]{64}$ ]]; then + echo "RAPIDS_SHA must be a 40- or 64-character hexadecimal Git object ID" >&2 + exit 1 +fi + +github_repository="${GITHUB_REPOSITORY:-}" +github_run_attempt="${GITHUB_RUN_ATTEMPT:-}" +github_run_id="${GITHUB_RUN_ID:-}" +github_workflow_ref="${GITHUB_WORKFLOW_REF:-}" +require_single_line "GITHUB_REPOSITORY" "${github_repository}" +require_single_line "GITHUB_RUN_ATTEMPT" "${github_run_attempt}" +require_single_line "GITHUB_RUN_ID" "${github_run_id}" +require_single_line "GITHUB_WORKFLOW_REF" "${github_workflow_ref}" +if [[ ! "${github_repository}" =~ ^[^/[:space:]]+/[^/[:space:]]+$ ]]; then + echo "GITHUB_REPOSITORY must have owner/repository form" >&2 + exit 1 +fi +require_positive_integer "GITHUB_RUN_ATTEMPT" "${github_run_attempt}" +require_positive_integer "GITHUB_RUN_ID" "${github_run_id}" require_plain_filename() { local label="$1" @@ -47,6 +101,8 @@ ensure_relative_pattern() { artifact_directory="$(realpath "${RELEASE_ARTIFACT_DIRECTORY}")" +# Artifact paths may contain globs for matrix-dependent filenames, but each +# descriptor must resolve to one regular file below the configured directory. resolve_one_file() { local field="$1" local pattern="$2" @@ -70,6 +126,9 @@ resolve_one_file() { printf '%s\n' "${resolved#"${artifact_directory}/"}" } +# Normalize supported package formats into the common package identity stored +# in each catalog entry. Wheels expose Core Metadata; Conda packages expose +# info/index.json. Custom formats provide the same fields in a sidecar file. describe_wheel_package() { local wheel_path="$1" local relative_path="${wheel_path#"${artifact_directory}/"}" @@ -232,6 +291,8 @@ prepare_artifacts() { printf '%s\n' "${prepared_artifacts}" } +# From here onward, every descriptor has an exact path and either extracted +# package identity or a producer-created package identity file. artifacts="$(prepare_artifacts)" entries_path="${artifact_directory}/${RELEASE_ENTRIES_NAME}" temporary_manifest="$(mktemp "${artifact_directory}/.release-catalog.XXXXXX")" @@ -260,6 +321,8 @@ generated_evidence_path() { printf 'release-evidence/%s.%s.%s.json\n' "${artifact_label}" "${artifact_digest}" "${kind}" } +# This SPDX document proves which named package and digest the entry refers to. +# It deliberately sets filesAnalyzed=false and contains no dependency inventory. write_generated_sbom() { local primary_path="$1" local package="$2" @@ -301,6 +364,40 @@ write_generated_sbom() { }' >"${artifact_directory}/${destination}" } +# This CycloneDX document describes the same artifact identity as the SPDX +# document. It intentionally has no component inventory or dependency graph. +write_generated_cyclonedx_sbom() { + local primary_path="$1" + local package="$2" + local destination="$3" + local artifact_digest + artifact_digest="$(sha256sum "${artifact_directory}/${primary_path}" | awk '{print $1}')" + mkdir -p "$(dirname "${artifact_directory}/${destination}")" + jq -n -S \ + --arg artifact_digest "${artifact_digest}" \ + --arg artifact_path "${primary_path}" \ + --argjson package "${package}" \ + '{ + bomFormat: "CycloneDX", + specVersion: "1.6", + version: 1, + metadata: { + timestamp: (now | strftime("%Y-%m-%dT%H:%M:%SZ")), + component: { + type: "file", + "bom-ref": ("urn:sha256:" + $artifact_digest), + name: $package.name, + version: $package.version, + hashes: [{alg: "SHA-256", content: $artifact_digest}], + properties: [ + {name: "rapids:artifact:path", value: $artifact_path}, + {name: "rapids:package:ecosystem", value: $package.ecosystem} + ] + } + } + }' >"${artifact_directory}/${destination}" +} + write_generated_provenance() { local primary_path="$1" local package="$2" @@ -311,11 +408,11 @@ write_generated_provenance() { jq -n -S \ --arg artifact_digest "${artifact_digest}" \ --arg artifact_path "${primary_path}" \ - --arg repository "${GITHUB_REPOSITORY:-}" \ - --arg run_attempt "${GITHUB_RUN_ATTEMPT:-}" \ - --arg run_id "${GITHUB_RUN_ID:-}" \ + --arg repository "${github_repository}" \ + --arg run_attempt "${github_run_attempt}" \ + --arg run_id "${github_run_id}" \ --arg source_sha "${source_sha}" \ - --arg workflow_ref "${GITHUB_WORKFLOW_REF:-}" \ + --arg workflow_ref "${github_workflow_ref}" \ --argjson package "${package}" \ '{ _type: "https://in-toto.io/Statement/v1", @@ -339,6 +436,9 @@ write_generated_provenance() { } shopt -s globstar nullglob +# Create one catalog entry, two identity-only SBOM documents, and one provenance +# document per primary artifact. Producer-supplied dependency evidence is +# intentionally not accepted by this version of the schema. while IFS= read -r descriptor; do if ! jq -e ' type == "object" @@ -367,20 +467,21 @@ while IFS= read -r descriptor; do exit 1 fi - sbom_path="$(generated_evidence_path "${primary_path}" "spdx")" - write_generated_sbom "${primary_path}" "${package}" "${sbom_path}" + spdx_sbom_path="$(generated_evidence_path "${primary_path}" "sbom.spdx")" + write_generated_sbom "${primary_path}" "${package}" "${spdx_sbom_path}" + cyclonedx_sbom_path="$(generated_evidence_path "${primary_path}" "sbom.cdx")" + write_generated_cyclonedx_sbom "${primary_path}" "${package}" "${cyclonedx_sbom_path}" provenance_path="$(generated_evidence_path "${primary_path}" "provenance")" write_generated_provenance "${primary_path}" "${package}" "${provenance_path}" - # TODO: Accept producer-supplied SBOM, provenance, and signature evidence after - # the release catalog defines a generic artifact-to-evidence association. entry="$(jq -cn \ --arg release_catalog_key "${RELEASE_CATALOG_KEY}" \ --arg path "${primary_path}" \ - --arg sbom "${sbom_path}" \ + --arg spdx_sbom "${spdx_sbom_path}" \ + --arg cyclonedx_sbom "${cyclonedx_sbom_path}" \ --arg provenance "${provenance_path}" \ --argjson package "${package}" \ - '{release_catalog_key: $release_catalog_key, path: $path, sbom: $sbom, sbom_kind: "generated-identity", provenance: $provenance, package: $package}')" + '{release_catalog_key: $release_catalog_key, path: $path, sboms: {spdx: $spdx_sbom, cyclonedx: $cyclonedx_sbom}, sbom_kind: "generated-identity", provenance: $provenance, package: $package}')" jq --argjson entry "${entry}" '.entries += [$entry]' "${temporary_manifest}" >"${temporary_manifest}.next" mv "${temporary_manifest}.next" "${temporary_manifest}" @@ -391,13 +492,15 @@ if ! jq -e '.entries as $items | ($items | map([.release_catalog_key, .path] | j exit 1 fi +# Store job-level source context once around the entry array. A downstream +# assembler can retain that association while combining selected companions. jq -n -S \ --arg artifact_name "${RELEASE_SOURCE_ARTIFACT_NAME}" \ - --arg repository "${GITHUB_REPOSITORY:-}" \ - --arg run_attempt "${GITHUB_RUN_ATTEMPT:-}" \ - --arg run_id "${GITHUB_RUN_ID:-}" \ + --arg repository "${github_repository}" \ + --arg run_attempt "${github_run_attempt}" \ + --arg run_id "${github_run_id}" \ --arg sha "${source_sha}" \ - --arg workflow_ref "${GITHUB_WORKFLOW_REF:-}" \ + --arg workflow_ref "${github_workflow_ref}" \ --argjson entries "$(jq -c '.entries' "${temporary_manifest}")" \ '{ schema_version: 1, diff --git a/release-catalog/upload-s3.sh b/release-catalog/upload-s3.sh index a9a9556a..4f5cb390 100755 --- a/release-catalog/upload-s3.sh +++ b/release-catalog/upload-s3.sh @@ -55,7 +55,7 @@ base_key="${RELEASE_CANDIDATE_PREFIX}/${RELEASE_CANDIDATE_TRAIN_SHA256}/${GITHUB # `signature` is optional, so select only declared string paths. Without the # filter, jq renders a missing optional value as the literal text `null` and # the uploader attempts to find a file with that name. -mapfile -t bundle_paths < <(jq -r '["release-catalog-entries.json"] + ([.entries[] | .path, .sbom, .provenance, .signature? | strings] | unique) | .[]' "${entries_path}") +mapfile -t bundle_paths < <(jq -r '["release-catalog-entries.json"] + ([.entries[] | .path, (.sboms[]), .provenance, .signature? | strings] | unique) | .[]' "${entries_path}") head_metadata="$(mktemp)" trap 'rm -f "${head_metadata}"' EXIT diff --git a/release-catalog/validate-config.sh b/release-catalog/validate-config.sh index 5ed529e9..560b774b 100755 --- a/release-catalog/validate-config.sh +++ b/release-catalog/validate-config.sh @@ -3,6 +3,11 @@ set -euo pipefail +# Validate the public JSON input before materialize.sh reads any build output. +# Structural errors are emitted as GitHub Actions annotations so callers see +# every invalid field in one run. Validated values are written to GITHUB_OUTPUT +# for the dispatch action's later materialization step. + emit_error() { local message="$1" printf '::error title=Invalid release catalog configuration::%s\n' "${message}" >&2 diff --git a/tests/release_catalog_discovery_test.sh b/tests/release_catalog_discovery_test.sh index fce7a1e6..59c71d58 100755 --- a/tests/release_catalog_discovery_test.sh +++ b/tests/release_catalog_discovery_test.sh @@ -7,10 +7,15 @@ repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" temporary_directory="$(mktemp -d)" trap 'rm -rf "${temporary_directory}"' EXIT +GITHUB_REPOSITORY="rapidsai/shared-actions" +GITHUB_RUN_ATTEMPT="1" +GITHUB_RUN_ID="1234" +GITHUB_WORKFLOW_REF="rapidsai/shared-actions/.github/workflows/pr.yml@refs/pull/136/merge" RELEASE_CATALOG_KEY="test:discovery" RELEASE_ENTRIES_NAME="release-catalog-entries.json" RELEASE_SOURCE_ARTIFACT_NAME="discovery-test" RAPIDS_SHA="0123456789012345678901234567890123456789" +export GITHUB_REPOSITORY GITHUB_RUN_ATTEMPT GITHUB_RUN_ID GITHUB_WORKFLOW_REF export RAPIDS_SHA RELEASE_CATALOG_KEY RELEASE_ENTRIES_NAME RELEASE_SOURCE_ARTIFACT_NAME mkdir -p "${temporary_directory}/wheel/libkvikio_cu12-26.8.0.dist-info" diff --git a/tests/release_catalog_package_parsing_test.sh b/tests/release_catalog_package_parsing_test.sh index 3b037c4d..86284f7e 100755 --- a/tests/release_catalog_package_parsing_test.sh +++ b/tests/release_catalog_package_parsing_test.sh @@ -7,11 +7,16 @@ repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" temporary_directory="$(mktemp -d)" trap 'rm -rf "${temporary_directory}"' EXIT +GITHUB_REPOSITORY="rapidsai/shared-actions" +GITHUB_RUN_ATTEMPT="1" +GITHUB_RUN_ID="1234" +GITHUB_WORKFLOW_REF="rapidsai/shared-actions/.github/workflows/pr.yml@refs/pull/136/merge" RELEASE_ARTIFACTS='' RELEASE_CATALOG_KEY="test:package-parsing" RELEASE_ENTRIES_NAME="release-catalog-entries.json" RELEASE_SOURCE_ARTIFACT_NAME="package-parsing-test" RAPIDS_SHA="0123456789012345678901234567890123456789" +export GITHUB_REPOSITORY GITHUB_RUN_ATTEMPT GITHUB_RUN_ID GITHUB_WORKFLOW_REF export RAPIDS_SHA RELEASE_ARTIFACTS RELEASE_CATALOG_KEY RELEASE_ENTRIES_NAME RELEASE_SOURCE_ARTIFACT_NAME wheel_output_directory="${temporary_directory}/wheels" diff --git a/tests/release_catalog_test.sh b/tests/release_catalog_test.sh index 3a019b51..dcb6b498 100755 --- a/tests/release_catalog_test.sh +++ b/tests/release_catalog_test.sh @@ -14,11 +14,11 @@ jq -n \ '{ecosystem: "maven", name: "ai.rapids:cuvs-java", version: "26.08.0"}' \ >"${bundle_directory}/cuvs-java-package-identity.json" -GITHUB_REPOSITORY="rapidsai/cuvs" +GITHUB_REPOSITORY="NVIDIA/cuvs" GITHUB_RUN_ATTEMPT="1" GITHUB_RUN_ID="1234" GITHUB_SHA="0123456789012345678901234567890123456789" -GITHUB_WORKFLOW_REF="rapidsai/cuvs/.github/workflows/build.yaml@refs/heads/release/26.08" +GITHUB_WORKFLOW_REF="NVIDIA/cuvs/.github/workflows/build.yaml@refs/heads/release/26.08" RELEASE_ARTIFACTS="$(jq -cn '[{path: "cuvs-java-*.jar", package_identity_file: "cuvs-java-package-identity.json"}]')" RELEASE_ENTRIES_NAME="release-catalog-entries.json" RELEASE_ARTIFACT_DIRECTORY="${bundle_directory}" @@ -38,20 +38,47 @@ jq -e ' .schema_version == 1 and .producer == "shared-workflows" and .source.artifact == "cuvs-java-cuda12.9.1" - and .source.repository == "rapidsai/cuvs" + and .source.repository == "NVIDIA/cuvs" and .source.sha == "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" and (.entries | length == 1) and .entries[0].release_catalog_key == "maven:cuvs-java" and .entries[0].path == "cuvs-java-26.08.0.jar" and .entries[0].package.name == "ai.rapids:cuvs-java" and .entries[0].sbom_kind == "generated-identity" + and (.entries[0].sboms | keys) == ["cyclonedx", "spdx"] ' "${entries_path}" >/dev/null # Optional signatures must not cause the S3 uploader to request a literal # `null` file. This mirrors its declared-file selection without requiring AWS. -mapfile -t upload_paths < <(jq -r '["release-catalog-entries.json"] + ([.entries[] | .path, .sbom, .provenance, .signature? | strings] | unique) | .[]' "${entries_path}") -test "${upload_paths[*]}" = "release-catalog-entries.json cuvs-java-26.08.0.jar release-evidence/cuvs-java-26.08.0.jar.sbom.spdx.json release-evidence/cuvs-java-26.08.0.jar.provenance.json" -! printf '%s\n' "${upload_paths[@]}" | grep -Fx null +mapfile -t upload_paths < <(jq -r '["release-catalog-entries.json"] + ([.entries[] | .path, (.sboms[]), .provenance, .signature? | strings] | unique) | .[]' "${entries_path}") +test "${upload_paths[*]}" = "release-catalog-entries.json cuvs-java-26.08.0.jar release-evidence/cuvs-java-26.08.0.jar.sbom.cdx.json release-evidence/cuvs-java-26.08.0.jar.sbom.spdx.json release-evidence/cuvs-java-26.08.0.jar.provenance.json" +if printf '%s\n' "${upload_paths[@]}" | grep -Fx null; then + echo "optional signature emitted a literal null upload path" >&2 + exit 1 +fi + +# Keep the checked-in example synchronized with the exact output owned by this +# action. Only the generated SBOM timestamps are normalized. +example_directory="${repository_root}/release-catalog/examples/cuvs-java" +diff -u "${example_directory}/release-catalog-entries.json" "${entries_path}" +example_spdx_sbom_path="$(jq -r '.entries[0].sboms.spdx' "${entries_path}")" +example_cyclonedx_sbom_path="$(jq -r '.entries[0].sboms.cyclonedx' "${entries_path}")" +example_provenance_path="$(jq -r '.entries[0].provenance' "${entries_path}")" +jq -S '.creationInfo.created = "2026-08-21T00:00:00Z"' \ + "${canonical_bundle_directory}/${example_spdx_sbom_path}" \ + >"${temporary_directory}/normalized-example.spdx.json" +diff -u \ + "${example_directory}/${example_spdx_sbom_path}" \ + "${temporary_directory}/normalized-example.spdx.json" +jq -S '.metadata.timestamp = "2026-08-21T00:00:00Z"' \ + "${canonical_bundle_directory}/${example_cyclonedx_sbom_path}" \ + >"${temporary_directory}/normalized-example.cdx.json" +diff -u \ + "${example_directory}/${example_cyclonedx_sbom_path}" \ + "${temporary_directory}/normalized-example.cdx.json" +diff -u \ + "${example_directory}/${example_provenance_path}" \ + "${canonical_bundle_directory}/${example_provenance_path}" isolated_companion_directory="${temporary_directory}/isolated-companion" mkdir -p "${isolated_companion_directory}" @@ -59,7 +86,7 @@ cp "${entries_path}" "${isolated_companion_directory}/" cp -R "${canonical_bundle_directory}/release-evidence" "${isolated_companion_directory}/" while IFS= read -r evidence_path; do test -f "${isolated_companion_directory}/${evidence_path}" -done < <(jq -r '.entries[] | .sbom, .provenance' "${isolated_companion_directory}/release-catalog-entries.json") +done < <(jq -r '.entries[] | .sboms[], .provenance' "${isolated_companion_directory}/release-catalog-entries.json") multiple_identity_directory="${temporary_directory}/multiple-identities" mkdir -p "${multiple_identity_directory}" @@ -84,6 +111,7 @@ jq -e ' and .entries[0].package == {ecosystem: "archive", name: "first", version: "1.0"} and .entries[1].package == {ecosystem: "maven", name: "example:second", version: "2.0"} ' "${multiple_identity_directory}/release-catalog-entries.json" >/dev/null +test "$(find "${multiple_identity_directory}/release-evidence" -type f | wc -l | tr -d ' ')" = "6" generated_directory="${temporary_directory}/generated-bundle" mkdir -p "${generated_directory}/linux-64" @@ -101,7 +129,8 @@ export RAPIDS_SHA RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_AR "${repository_root}/release-catalog/materialize.sh" generated_entries_path="${generated_directory}/release-catalog-entries.json" -generated_sbom_path="$(jq -r '.entries[0].sbom' "${generated_entries_path}")" +generated_spdx_sbom_path="$(jq -r '.entries[0].sboms.spdx' "${generated_entries_path}")" +generated_cyclonedx_sbom_path="$(jq -r '.entries[0].sboms.cyclonedx' "${generated_entries_path}")" generated_provenance_path="$(jq -r '.entries[0].provenance' "${generated_entries_path}")" jq -e ' .source.artifact == "kvikio_conda_python_kvikio_x86_64_abi3_cu12" @@ -115,7 +144,15 @@ jq -e ' .spdxVersion == "SPDX-2.3" and .packages[0].name == "kvikio" and .packages[0].versionInfo == "26.08.00a32" -' "${generated_directory}/${generated_sbom_path}" >/dev/null +' "${generated_directory}/${generated_spdx_sbom_path}" >/dev/null +jq -e ' + .bomFormat == "CycloneDX" + and .specVersion == "1.6" + and .metadata.component.type == "file" + and .metadata.component.name == "kvikio" + and .metadata.component.version == "26.08.00a32" + and .metadata.component.hashes[0].alg == "SHA-256" +' "${generated_directory}/${generated_cyclonedx_sbom_path}" >/dev/null jq -e ' .predicateType == "https://slsa.dev/provenance/v1" and .predicate.buildDefinition.externalParameters.release_catalog_key == "conda:kvikio" @@ -168,3 +205,50 @@ if (unset RAPIDS_SHA; "${repository_root}/release-catalog/materialize.sh") 2>"${ exit 1 fi grep -Fx 'RAPIDS_SHA must be a non-empty string' "${temporary_directory}/missing-rapids-sha-error" + +if RAPIDS_SHA="not-a-git-object-id" \ + "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/invalid-rapids-sha-error"; then + echo "materialize.sh unexpectedly accepted an invalid RAPIDS_SHA" >&2 + exit 1 +fi +grep -Fx 'RAPIDS_SHA must be a 40- or 64-character hexadecimal Git object ID' \ + "${temporary_directory}/invalid-rapids-sha-error" + +assert_missing_build_context() { + local variable_name="$1" + local error_file="${temporary_directory}/missing-${variable_name}.error" + if (unset "${variable_name}"; "${repository_root}/release-catalog/materialize.sh") 2>"${error_file}"; then + echo "materialize.sh unexpectedly accepted a missing ${variable_name}" >&2 + exit 1 + fi + grep -Fx "${variable_name} must be a non-empty string" "${error_file}" +} + +assert_missing_build_context GITHUB_REPOSITORY +assert_missing_build_context GITHUB_RUN_ATTEMPT +assert_missing_build_context GITHUB_RUN_ID +assert_missing_build_context GITHUB_WORKFLOW_REF + +if GITHUB_REPOSITORY="cuvs" \ + "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/invalid-repository.error"; then + echo "materialize.sh unexpectedly accepted an invalid GITHUB_REPOSITORY" >&2 + exit 1 +fi +grep -Fx 'GITHUB_REPOSITORY must have owner/repository form' \ + "${temporary_directory}/invalid-repository.error" + +if GITHUB_RUN_ATTEMPT="0" \ + "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/invalid-run-attempt.error"; then + echo "materialize.sh unexpectedly accepted an invalid GITHUB_RUN_ATTEMPT" >&2 + exit 1 +fi +grep -Fx 'GITHUB_RUN_ATTEMPT must be a positive integer' \ + "${temporary_directory}/invalid-run-attempt.error" + +if GITHUB_RUN_ID="not-an-integer" \ + "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/invalid-run-id.error"; then + echo "materialize.sh unexpectedly accepted an invalid GITHUB_RUN_ID" >&2 + exit 1 +fi +grep -Fx 'GITHUB_RUN_ID must be a positive integer' \ + "${temporary_directory}/invalid-run-id.error" From 4134d2e819a4314a3c5f0b1aeb0d456b2fd55aa5 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 1 Sep 2026 15:35:39 -0500 Subject: [PATCH 21/36] improve purpose and consistency of verbiage --- .github/workflows/pr.yml | 1 - README.md | 23 +++ release-catalog-dispatch/action.yml | 24 ++- release-catalog/README.md | 163 ++++++++++++------ ...d1715b14a3f7c349cba300cb2b9.sbom.spdx.json | 2 +- release-catalog/materialize.sh | 7 +- release-catalog/upload-s3.sh | 18 +- 7 files changed, 164 insertions(+), 74 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index ba2ac20c..efdb6c74 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -56,7 +56,6 @@ jobs: }] } source-artifact-name: release-catalog-dispatch-smoke - candidate-train-sha256: ${{ github.sha }} - name: Verify release catalog dispatch smoke test run: | jq -e ' diff --git a/README.md b/README.md index 995a28c8..b3d2ebd7 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,29 @@ especially the telemetry actions, use a pattern that we refer to as "dispatch ac The general idea of a dispatch action is to make it easier to depend on other actions at a specific revision, and also to simplify using files beyond a given action .yml file. +## Release catalog companions + +The release catalog action creates a companion archive for each build job. See +the [release catalog documentation](release-catalog/README.md) for its +configuration and archive contract. + +The checked-in [cuVS Java companion example](release-catalog/examples/cuvs-java) +shows the complete output for one primary JAR: + +- [`release-catalog-entries.json`](release-catalog/examples/cuvs-java/release-catalog-entries.json) + associates the artifact with its package identity and evidence. +- [SLSA/in-toto provenance](release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.provenance.json) + identifies the source, workflow, invocation, and exact artifact digest. +- The SBOM is provided in both [SPDX + 2.3](release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.spdx.json) + and [CycloneDX + 1.6](release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.cdx.json) + formats so consumers can choose the serialization their tooling supports. + +The [example companion notes](release-catalog/examples/README.md) explain how +the catalog, both SBOMs, provenance, and separately uploaded primary artifact +are bound together by the artifact SHA-256. + A dispatch action is one that: * clones the shared-actions repository (repo/ref changeable using env vars) * runs (dispatches to) another action within the clone, using a relative path diff --git a/release-catalog-dispatch/action.yml b/release-catalog-dispatch/action.yml index 0df32784..c39b2411 100644 --- a/release-catalog-dispatch/action.yml +++ b/release-catalog-dispatch/action.yml @@ -1,5 +1,8 @@ name: Dispatch release catalog -description: Write a release-candidate catalog bundle directly to the private S3 store. +description: >- + Create a release catalog companion (artifact inventory, provenance, and SBOMs) + for the artifacts a build job produced, and optionally upload it to S3. + See release-catalog/README.md. inputs: config: @@ -8,21 +11,26 @@ inputs: https://github.com/rapidsai/shared-actions/blob/main/release-catalog/config.schema.json required: true source-artifact-name: - description: Stable name of this producing artifact bundle. + description: >- + Stable name for this job's output, such as the GitHub Actions artifact + name. Used as the last path component in S3. required: true candidate-train-sha256: - description: Canonical SHA-256 of the immutable release-train JSON. - required: true + description: >- + SHA-256 of the release train JSON this build belongs to. Required when + upload-to-s3 is true; ignored otherwise. + required: false + default: "" candidate-bucket: - description: Private S3 bucket that holds candidate-build bytes. + description: S3 bucket that receives the companion when upload-to-s3 is true. required: false default: rapids-release-candidates candidate-prefix: - description: Root prefix for train-scoped candidate-build bytes. + description: Root prefix inside candidate-bucket. required: false default: candidate-builds upload-to-s3: - description: Whether to upload the generated bundle to the candidate store. + description: Whether to upload the companion to S3. When false, the companion is only written locally. required: false default: "false" runs: @@ -64,7 +72,7 @@ runs: RELEASE_SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} RELEASE_CATALOG_KEY: ${{ steps.configuration.outputs.release_catalog_key }} run: "${IMPLEMENTATION_PATH}/materialize.sh" - - name: Write release-candidate bundle to S3 + - name: Upload release catalog companion to S3 if: ${{ inputs.upload-to-s3 == 'true' }} shell: bash env: diff --git a/release-catalog/README.md b/release-catalog/README.md index 0b7da2e9..65e81384 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -1,15 +1,52 @@ -# Release catalog companion archives - -This action creates a companion archive alongside our binary artifacts (conda, -wheel or otherwise) that contains an inventory of the binary artifacts that were -produced by that job. That inventory is used by the RAPIDS release system to -assemble release candidates. "Release candidates" here refers to a set of -artifacts that can be deployed to create a release, rather than a release -candidate version of one artifact. +# Release catalog companions + +## Why this exists + +RAPIDS builds hundreds of binary artifacts (conda packages, wheels, JARs, and +others) across many repositories and CI jobs. Two things are hard to do with +those artifacts today: + +- **Assemble and test a release before tagging it.** We want to pick a + consistent set of artifacts, test them together as a candidate, and only + then publish. That requires a single inventory of what each job built. +- **Keep build-time evidence.** Facts such as which commit, workflow, and run + produced a wheel cannot be reliably reconstructed after the fact. They have + to be captured by the job that built the artifact. + +This action solves both by having each build job write a small **companion** +next to its artifacts: an inventory of the files it produced, plus provenance +and SBOM documents for each one. The companion never modifies the artifacts +themselves. + +## Lifecycle + +1. A build job produces its artifacts as usual. +2. This action runs as a later step in that job and writes a companion into + the artifact directory. +3. The companion is uploaded to a private S3 bucket (`upload-to-s3: true`). +4. The RAPIDS release platform reads the companions from every job in a + release train, merges them into the **release catalog**, and uses the + catalog to assemble and test a **release candidate**. + +"Release candidate" here means a set of artifacts that can be deployed to +create a release, not a release-candidate version of one artifact. + +## Glossary + +| Term | Meaning | +| ---- | ------- | +| **Artifact** | A file a build job produces and that we may publish: a `.conda` file, a wheel, a JAR, and so on. | +| **Companion** | The set of files this action writes next to a job's artifacts: `release-catalog-entries.json` plus a `release-evidence/` directory. One companion per build job. | +| **Producer** | The build job (and the workflow it runs in) that created the artifacts. | +| **Release catalog** | The merged inventory of all companions for a release, maintained by the release platform. | +| **Release train** | One planned release of RAPIDS, identified by an immutable JSON document. Its SHA-256 is used to group all companions that belong to that release in S3. | +| **Release candidate** | A set of artifacts selected from the release catalog to be tested and, if it passes, published. | +| **Provenance** | A [SLSA](https://slsa.dev/spec/v1.2/attestation-model) statement describing how, where, and from what source an artifact was built. | +| **SBOM** | Software bill of materials. The SBOMs this action generates identify the artifact and its digest only; they do not list its dependencies. | ## Companion contents -Each build job creates one companion archive with this shape: +Each build job creates one companion with this shape: ```text . @@ -24,56 +61,67 @@ Each build job creates one companion archive with this shape: └── ... ``` -`release-catalog-entries.json` records the artifacts produced by the job: -`.conda`, `.whl`, or otherwise, as well as the checksums of those files. - -The release-evidence folder carries additional metadata that may be useful to -consumers of our packages: provenance and software bill of materials (SBOM). -Provenance describes how a package was built and by whom. An SBOM lists -components that make up the package. They complement each other: an SBOM without -provenance may describe the wrong or an untrusted artifact, while provenance -without an SBOM cannot efficiently answer what vulnerable components are inside. -For each artifact in its `entries` array, the action creates one provenance file -and equivalent identity SBOMs in SPDX and CycloneDX formats. The SHA-256 in each -evidence filename is the digest of that specific artifact's contents. The -catalog entry identifies both SBOM paths under `sboms`, allowing consumers to -select their preferred format without inferring it from the filename. - -The provenance.json file follows the -[SLSA](https://slsa.dev/spec/v1.2/attestation-model) standard format. - -The `sbom.spdx.json` file uses [SPDX -2.3](https://spdx.github.io/spdx-spec/v2.3/), while `sbom.cdx.json` uses -[CycloneDX 1.6](https://cyclonedx.org/docs/1.6/json/). These generated documents -identify the artifact and its digest; they do not contain a dependency -inventory. - -Provenance remains separate because it is an independently verifiable statement -about how and where the artifact was built. Although both SBOM standards can -carry some build metadata, embedding provenance in the SBOMs would not replace -the in-toto/SLSA statement and would couple evidence with different consumers -and lifecycles. - -## Action Configuration - -Every caller passes one `config` JSON object. The canonical schema and field -documentation are in [`config.schema.json`](config.schema.json). This is -validated at runtime with the [validate-config.sh](validate-config.sh) script. +`release-catalog-entries.json` records the artifacts produced by the job +(`.conda`, `.whl`, or otherwise), the checksums of those files, the package +identity of each one, and the source context of the job (repository, commit, +workflow, run). See [`examples/cuvs-java`](examples/cuvs-java) for a complete +companion. + +For each artifact in its `entries` array, the action creates one provenance +file and two equivalent SBOMs, one in [SPDX +2.3](https://spdx.github.io/spdx-spec/v2.3/) and one in [CycloneDX +1.6](https://cyclonedx.org/docs/1.6/json/). The SHA-256 in each evidence +filename is the digest of that artifact's contents, so evidence files can +never be confused across artifacts. The catalog entry lists both SBOM paths +under `sboms`, allowing consumers to select their preferred format without +inferring it from the filename. + +Each entry also carries `sbom_kind: "generated-identity"`. This marks the +SBOMs as identity-only documents generated by this action. A future version +may accept producer-supplied dependency SBOMs, which would use a different +`sbom_kind`. + +## Action inputs + +The action is used via `rapidsai/shared-actions/release-catalog-dispatch`. + +| Input | Required | Description | +| ----- | -------- | ----------- | +| `config` | yes | JSON object selecting the artifacts and their release catalog key. Schema: [`config.schema.json`](config.schema.json). Details below. | +| `source-artifact-name` | yes | Stable name for this job's output, for example the GitHub Actions artifact name. Used as the last path component in S3. | +| `upload-to-s3` | no | `"true"` to upload the companion to S3. Defaults to `"false"`, which only writes the companion locally. | +| `candidate-train-sha256` | when uploading | SHA-256 of the release train JSON this build belongs to. All companions for one release share it. | +| `candidate-bucket` | no | S3 bucket for companions. Defaults to `rapids-release-candidates`. | +| `candidate-prefix` | no | Root prefix inside the bucket. Defaults to `candidate-builds`. | + +When uploading, the companion lands at +`s3:////////`. +Uploads are conditional: an existing object is accepted only if its bytes +match, so reruns are safe. + +## `config` contents + +The canonical schema and field documentation are in +[`config.schema.json`](config.schema.json). It is validated at runtime by +[validate-config.sh](validate-config.sh). ### `release_catalog_key` This is a grouping label that represents a common "release policy." Artifacts -that share release_catalog_key are versioned, validated, ordered, published, and -promoted together. The key does not need to be unique per artifact, nor per -matrix variant. For example, `cudf` and `dask-cudf` Conda packages can both use -`conda:cudf` because they share the same version, validation, inter-package -order, publishing destination, and are ultimately published together. +that share a `release_catalog_key` are versioned, validated, ordered, published, +and promoted together. For example, the `cudf` and `dask-cudf` Conda packages +both use `conda:cudf` because they share the same version, validation, +inter-package order, and publishing destination, and are ultimately published +together. + +The key does not need to be unique per artifact or per matrix variant. Use a +separate key only when the outputs intentionally follow a separate release +workflow. Standard RAPIDS Conda and wheel workflows use `:`, such as `conda:cudf`, because those workflows apply a repository-level release policy. A custom producer should use `:`, such as -`maven:cuvs-java`, or simply `maven:cuvs`. Use a separate key only when the -outputs intentionally have a separate release workflows that must be followed. +`maven:cuvs-java`, or simply `maven:cuvs`. ### `artifact_directory` and `artifacts` @@ -115,7 +163,7 @@ resolves to zero files or multiple files results in an error. #### Package identity file -package_identity_file is a JSON file containing at least `ecosystem`, `name`, +`package_identity_file` is a JSON file containing at least `ecosystem`, `name`, and `version`; `build` and `platform` are optional. This path is relative to `artifact_directory`. @@ -129,3 +177,14 @@ and `version`; `build` and `platform` are optional. This path is relative to Multiple artifacts in the list may reference the same identity file when the files have the same package identity. + +## Design notes + +Provenance and SBOMs are kept as separate documents. Provenance describes how a +package was built and by whom; an SBOM lists what makes up the package. They +complement each other: an SBOM without provenance may describe the wrong or an +untrusted artifact, while provenance without an SBOM cannot efficiently answer +what vulnerable components are inside. Although both SBOM standards can carry +some build metadata, embedding provenance in the SBOMs would not replace the +in-toto/SLSA statement and would couple evidence with different consumers and +lifecycles. diff --git a/release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.spdx.json b/release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.spdx.json index dc1ea043..c0584944 100644 --- a/release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.spdx.json +++ b/release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.spdx.json @@ -1,6 +1,6 @@ { "SPDXID": "SPDXRef-DOCUMENT", - "comment": "Artifact-identity SBOM envelope. It does not contain a dependency inventory.", + "comment": "Artifact-identity SBOM. It identifies only this artifact and its digest, not its dependencies.", "creationInfo": { "created": "2026-08-21T00:00:00Z", "creators": [ diff --git a/release-catalog/materialize.sh b/release-catalog/materialize.sh index 17304ce4..d6b5b25c 100755 --- a/release-catalog/materialize.sh +++ b/release-catalog/materialize.sh @@ -360,7 +360,7 @@ write_generated_sbom() { relationshipType: "DESCRIBES", relatedSpdxElement: "SPDXRef-Artifact" }], - comment: "Artifact-identity SBOM envelope. It does not contain a dependency inventory." + comment: "Artifact-identity SBOM. It identifies only this artifact and its digest, not its dependencies." }' >"${artifact_directory}/${destination}" } @@ -492,8 +492,9 @@ if ! jq -e '.entries as $items | ($items | map([.release_catalog_key, .path] | j exit 1 fi -# Store job-level source context once around the entry array. A downstream -# assembler can retain that association while combining selected companions. +# Store the job's source context (repository, commit, workflow, run) once at +# the top level rather than on every entry. The release platform keeps this +# association when it merges companions from many jobs into the catalog. jq -n -S \ --arg artifact_name "${RELEASE_SOURCE_ARTIFACT_NAME}" \ --arg repository "${github_repository}" \ diff --git a/release-catalog/upload-s3.sh b/release-catalog/upload-s3.sh index 4f5cb390..9ba4f2ad 100755 --- a/release-catalog/upload-s3.sh +++ b/release-catalog/upload-s3.sh @@ -29,7 +29,7 @@ if [[ ! "${RELEASE_CANDIDATE_TRAIN_SHA256}" =~ ^[[:xdigit:]]{64}$ ]]; then exit 1 fi if [[ "${RELEASE_SOURCE_ARTIFACT_NAME}" == */* || "${RELEASE_SOURCE_ARTIFACT_NAME}" == *".."* ]]; then - echo "RELEASE_SOURCE_ARTIFACT_NAME must be a plain bundle name" >&2 + echo "RELEASE_SOURCE_ARTIFACT_NAME must be a plain name without path separators" >&2 exit 1 fi if ! safe_prefix "${RELEASE_CANDIDATE_PREFIX}"; then @@ -37,14 +37,14 @@ if ! safe_prefix "${RELEASE_CANDIDATE_PREFIX}"; then exit 1 fi -bundle_root="$(realpath "${RELEASE_ARTIFACT_DIRECTORY}")" -entries_path="${bundle_root}/release-catalog-entries.json" +companion_root="$(realpath "${RELEASE_ARTIFACT_DIRECTORY}")" +entries_path="${companion_root}/release-catalog-entries.json" if [[ ! -f "${entries_path}" ]]; then echo "release catalog entries are missing: ${entries_path}" >&2 exit 1 fi -# Catalog paths are relative to the build bundle. Reject traversal before an +# Catalog paths are relative to the companion root. Reject traversal before an # S3 key is constructed, even though materialize.sh already validates them. safe_relative_path() { local path="$1" @@ -55,13 +55,13 @@ base_key="${RELEASE_CANDIDATE_PREFIX}/${RELEASE_CANDIDATE_TRAIN_SHA256}/${GITHUB # `signature` is optional, so select only declared string paths. Without the # filter, jq renders a missing optional value as the literal text `null` and # the uploader attempts to find a file with that name. -mapfile -t bundle_paths < <(jq -r '["release-catalog-entries.json"] + ([.entries[] | .path, (.sboms[]), .provenance, .signature? | strings] | unique) | .[]' "${entries_path}") +mapfile -t companion_paths < <(jq -r '["release-catalog-entries.json"] + ([.entries[] | .path, (.sboms[]), .provenance, .signature? | strings] | unique) | .[]' "${entries_path}") head_metadata="$(mktemp)" trap 'rm -f "${head_metadata}"' EXIT upload_one() { local relative_path="$1" - local local_path="${bundle_root}/${relative_path}" + local local_path="${companion_root}/${relative_path}" local object_key="${base_key}/${relative_path}" local checksum checksum="$(openssl dgst -sha256 -binary "${local_path}" | base64)" @@ -83,12 +83,12 @@ upload_one() { fi } -for relative_path in "${bundle_paths[@]}"; do - if ! safe_relative_path "${relative_path}" || [[ ! -f "${bundle_root}/${relative_path}" ]]; then +for relative_path in "${companion_paths[@]}"; do + if ! safe_relative_path "${relative_path}" || [[ ! -f "${companion_root}/${relative_path}" ]]; then echo "catalog declares a missing or unsafe candidate file: ${relative_path}" >&2 exit 1 fi upload_one "${relative_path}" done -printf 'Release candidate bundle: s3://%s/%s\n' "${RELEASE_CANDIDATE_BUCKET}" "${base_key}" >>"${GITHUB_STEP_SUMMARY}" +printf 'Release catalog companion: s3://%s/%s\n' "${RELEASE_CANDIDATE_BUCKET}" "${base_key}" >>"${GITHUB_STEP_SUMMARY}" From 5902edafb4a7227e4b0edfd705f282a6c7ad7af5 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 1 Sep 2026 15:47:15 -0500 Subject: [PATCH 22/36] Name ADI Build Operations as catalog producer --- .github/workflows/pr.yml | 2 +- release-catalog/README.md | 5 +++++ .../examples/cuvs-java/release-catalog-entries.json | 2 +- release-catalog/materialize.sh | 2 +- tests/release_catalog_test.sh | 2 +- 5 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index efdb6c74..b3b329ba 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -59,7 +59,7 @@ jobs: - name: Verify release catalog dispatch smoke test run: | jq -e ' - .producer == "shared-workflows" + .producer == "NVIDIA ADI Build/Operations" and .source.artifact == "release-catalog-dispatch-smoke" and .entries[0].release_catalog_key == "archive:smoke" and .entries[0].path == "package.tar.gz" diff --git a/release-catalog/README.md b/release-catalog/README.md index 65e81384..cc4ce230 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -67,6 +67,11 @@ identity of each one, and the source context of the job (repository, commit, workflow, run). See [`examples/cuvs-java`](examples/cuvs-java) for a complete companion. +The top-level `producer` identifies the organization responsible for producing +and maintaining the companion: `NVIDIA ADI Build/Operations`. Implementation +details such as the reusable workflow remain in the source context and evidence +rather than serving as the producer identity. + For each artifact in its `entries` array, the action creates one provenance file and two equivalent SBOMs, one in [SPDX 2.3](https://spdx.github.io/spdx-spec/v2.3/) and one in [CycloneDX diff --git a/release-catalog/examples/cuvs-java/release-catalog-entries.json b/release-catalog/examples/cuvs-java/release-catalog-entries.json index b446b7a7..34e2bf97 100644 --- a/release-catalog/examples/cuvs-java/release-catalog-entries.json +++ b/release-catalog/examples/cuvs-java/release-catalog-entries.json @@ -16,7 +16,7 @@ } } ], - "producer": "shared-workflows", + "producer": "NVIDIA ADI Build/Operations", "schema_version": 1, "source": { "artifact": "cuvs-java-cuda12.9.1", diff --git a/release-catalog/materialize.sh b/release-catalog/materialize.sh index d6b5b25c..99ee80f3 100755 --- a/release-catalog/materialize.sh +++ b/release-catalog/materialize.sh @@ -505,7 +505,7 @@ jq -n -S \ --argjson entries "$(jq -c '.entries' "${temporary_manifest}")" \ '{ schema_version: 1, - producer: "shared-workflows", + producer: "NVIDIA ADI Build/Operations", source: { artifact: $artifact_name, repository: $repository, diff --git a/tests/release_catalog_test.sh b/tests/release_catalog_test.sh index dcb6b498..665ed8bf 100755 --- a/tests/release_catalog_test.sh +++ b/tests/release_catalog_test.sh @@ -36,7 +36,7 @@ entries_path="${canonical_bundle_directory}/release-catalog-entries.json" test ! -e "${canonical_bundle_directory}/release-catalog-metadata.json" jq -e ' .schema_version == 1 - and .producer == "shared-workflows" + and .producer == "NVIDIA ADI Build/Operations" and .source.artifact == "cuvs-java-cuda12.9.1" and .source.repository == "NVIDIA/cuvs" and .source.sha == "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" From d1795ce463654b4397505f63c143e77a1fe151d3 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 1 Sep 2026 16:36:41 -0500 Subject: [PATCH 23/36] Validate release catalog input and output schemas --- .pre-commit-config.yaml | 11 ++- ci/test_release_catalog_schemas.py | 51 +++++++++++++ release-catalog/README.md | 5 ++ release-catalog/config.schema.json | 11 ++- release-catalog/entries.schema.json | 74 +++++++++++++++++++ release-catalog/validate-config.sh | 12 ++- .../invalid/absolute-artifact-path.json | 4 + .../invalid/absolute-identity-path.json | 4 + .../invalid/parent-artifact-path.json | 4 + .../invalid/unknown-field.json | 4 + .../{ => valid}/package-identity-file.json | 0 tests/release_catalog_config_test.sh | 26 ++++++- 12 files changed, 195 insertions(+), 11 deletions(-) create mode 100644 ci/test_release_catalog_schemas.py create mode 100644 release-catalog/entries.schema.json create mode 100644 tests/release-catalog-config/invalid/absolute-artifact-path.json create mode 100644 tests/release-catalog-config/invalid/absolute-identity-path.json create mode 100644 tests/release-catalog-config/invalid/parent-artifact-path.json create mode 100644 tests/release-catalog-config/invalid/unknown-field.json rename tests/release-catalog-config/{ => valid}/package-identity-file.json (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6cf849f3..de5d229e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -65,7 +65,7 @@ repos: - id: check-jsonschema name: validate release catalog config schema fixtures args: [--schemafile, release-catalog/config.schema.json] - files: ^tests/release-catalog-config/.*\.json$ + files: ^tests/release-catalog-config/valid/.*\.json$ - repo: local hooks: - id: release-catalog-config @@ -73,4 +73,11 @@ repos: entry: ./tests/release_catalog_config_test.sh language: system pass_filenames: false - files: ^(release-catalog/(config\.schema\.json|validate-config\.sh)|tests/release_catalog_config_test\.sh)$ + files: ^(release-catalog/(config\.schema\.json|validate-config\.sh)|tests/release-catalog-config/.*\.json|tests/release_catalog_config_test\.sh)$ + - id: release-catalog-schemas + name: validate release catalog schema contracts + entry: python ci/test_release_catalog_schemas.py + language: python + additional_dependencies: [jsonschema==4.25.1] + pass_filenames: false + files: ^(release-catalog/(config|entries)\.schema\.json|release-catalog/examples/cuvs-java/release-catalog-entries\.json|tests/release-catalog-config/.*\.json|ci/test_release_catalog_schemas\.py)$ diff --git a/ci/test_release_catalog_schemas.py b/ci/test_release_catalog_schemas.py new file mode 100644 index 00000000..2b597106 --- /dev/null +++ b/ci/test_release_catalog_schemas.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +"""Verify valid and invalid release-catalog contracts against JSON Schema.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from jsonschema import Draft202012Validator + +ROOT = Path(__file__).parents[1] + + +def _load(path: Path) -> dict[str, object]: + value = json.loads(path.read_text()) + if not isinstance(value, dict): + raise ValueError(f"schema fixture must be a JSON object: {path}") + return value + + +def _validator(path: Path) -> Draft202012Validator: + schema = _load(path) + Draft202012Validator.check_schema(schema) + return Draft202012Validator(schema) + + +def main() -> None: + config = _validator(ROOT / "release-catalog/config.schema.json") + fixtures = ROOT / "tests/release-catalog-config" + for path in sorted((fixtures / "valid").glob("*.json")): + errors = list(config.iter_errors(_load(path))) + if errors: + raise ValueError(f"valid fixture rejected: {path}: {errors[0].message}") + for path in sorted((fixtures / "invalid").glob("*.json")): + if not list(config.iter_errors(_load(path))): + raise ValueError(f"invalid fixture accepted: {path}") + + entries = _validator(ROOT / "release-catalog/entries.schema.json") + example = _load(ROOT / "release-catalog/examples/cuvs-java/release-catalog-entries.json") + errors = list(entries.iter_errors(example)) + if errors: + raise ValueError(f"generated entries example rejected: {errors[0].message}") + wrong_producer = {**example, "producer": "shared-workflows"} + if not list(entries.iter_errors(wrong_producer)): + raise ValueError("entries schema accepted an implementation name as producer") + + +if __name__ == "__main__": + main() diff --git a/release-catalog/README.md b/release-catalog/README.md index cc4ce230..5899773c 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -67,6 +67,11 @@ identity of each one, and the source context of the job (repository, commit, workflow, run). See [`examples/cuvs-java`](examples/cuvs-java) for a complete companion. +Generated entry documents conform to +[`entries.schema.json`](entries.schema.json). The schema fixes the versioned +output contract for producer identity, source context, package identity, and +evidence paths. + The top-level `producer` identifies the organization responsible for producing and maintaining the companion: `NVIDIA ADI Build/Operations`. Implementation details such as the reusable workflow remain in the source context and evidence diff --git a/release-catalog/config.schema.json b/release-catalog/config.schema.json index 09f1a5b9..380deec0 100644 --- a/release-catalog/config.schema.json +++ b/release-catalog/config.schema.json @@ -40,15 +40,22 @@ ], "properties": { "path": { + "description": "Artifact path or glob, relative to artifact_directory. Absolute paths and parent-directory traversal are forbidden.", "type": "string", "minLength": 1, - "pattern": "^[^\\r\\n]+$" + "pattern": "^[^\\r\\n]+$", + "not": { + "pattern": "(^/)|((^|/)\\.\\.(/|$))" + } }, "package_identity_file": { "description": "Path, relative to artifact_directory, to producer-created package identity JSON for this artifact. Required only when identity cannot be extracted from the artifact. See https://github.com/rapidsai/shared-actions/blob/main/release-catalog/README.md#package-identity-file", "type": "string", "minLength": 1, - "pattern": "^[^\\r\\n]+$" + "pattern": "^[^\\r\\n]+$", + "not": { + "pattern": "(^/)|((^|/)\\.\\.(/|$))" + } } } } diff --git a/release-catalog/entries.schema.json b/release-catalog/entries.schema.json new file mode 100644 index 00000000..7de80d80 --- /dev/null +++ b/release-catalog/entries.schema.json @@ -0,0 +1,74 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://rapids.ai/schemas/release-catalog-entries-v1.json", + "title": "Release catalog companion entries", + "description": "Generated inventory and evidence references for one build job.", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "producer", "source", "entries"], + "properties": { + "schema_version": {"const": 1}, + "producer": {"const": "NVIDIA ADI Build/Operations"}, + "source": {"$ref": "#/$defs/source"}, + "entries": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/entry"} + } + }, + "$defs": { + "relative_path": { + "type": "string", + "minLength": 1, + "pattern": "^[^\\r\\n]+$", + "not": {"pattern": "(^/)|((^|/)\\.\\.(/|$))"} + }, + "source": { + "type": "object", + "additionalProperties": false, + "required": ["artifact", "repository", "sha", "workflow_ref", "run_id", "run_attempt"], + "properties": { + "artifact": {"type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$"}, + "repository": {"type": "string", "pattern": "^[^/\\r\\n]+/[^/\\r\\n]+$"}, + "sha": {"type": "string", "pattern": "^(?:[0-9a-fA-F]{40}|[0-9a-fA-F]{64})$"}, + "workflow_ref": {"type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$"}, + "run_id": {"type": "string", "pattern": "^[1-9][0-9]*$"}, + "run_attempt": {"type": "string", "pattern": "^[1-9][0-9]*$"} + } + }, + "package": { + "type": "object", + "additionalProperties": false, + "required": ["ecosystem", "name", "version"], + "properties": { + "ecosystem": {"type": "string", "minLength": 1}, + "name": {"type": "string", "minLength": 1}, + "version": {"type": "string", "minLength": 1}, + "build": {"type": "string", "minLength": 1}, + "platform": {"type": "string", "minLength": 1} + } + }, + "entry": { + "type": "object", + "additionalProperties": false, + "required": ["release_catalog_key", "path", "package", "sboms", "sbom_kind", "provenance"], + "properties": { + "release_catalog_key": {"type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$"}, + "path": {"$ref": "#/$defs/relative_path"}, + "package": {"$ref": "#/$defs/package"}, + "sboms": { + "type": "object", + "additionalProperties": false, + "required": ["spdx", "cyclonedx"], + "properties": { + "spdx": {"$ref": "#/$defs/relative_path"}, + "cyclonedx": {"$ref": "#/$defs/relative_path"} + } + }, + "sbom_kind": {"const": "generated-identity"}, + "provenance": {"$ref": "#/$defs/relative_path"}, + "signature": {"$ref": "#/$defs/relative_path"} + } + } + } +} diff --git a/release-catalog/validate-config.sh b/release-catalog/validate-config.sh index 560b774b..f37795b7 100755 --- a/release-catalog/validate-config.sh +++ b/release-catalog/validate-config.sh @@ -28,6 +28,10 @@ fi validation_errors="$(jq -r ' def single_line_string: type == "string" and length > 0 and (test("[\\r\\n]") | not); + def relative_path: + single_line_string + and (startswith("/") | not) + and (split("/") | index("..") | not); if type != "object" then ["release catalog configuration must be a JSON object"] else @@ -59,12 +63,12 @@ validation_errors="$(jq -r ' | if ($artifact_unknown | length) > 0 then "artifacts[\($index)] has unknown field(s): " + ($artifact_unknown | join(", ")) else empty end, - if ($artifact.path | single_line_string) then empty - else "artifacts[\($index)].path must be a non-empty, single-line string" end, + if ($artifact.path | relative_path) then empty + else "artifacts[\($index)].path must be a non-empty relative path without parent traversal" end, ($artifact | to_entries[] | select(.key == "package_identity_file") - | select((.value | single_line_string) | not) - | "artifacts[\($index)].\(.key) must be a non-empty, single-line string") + | select((.value | relative_path) | not) + | "artifacts[\($index)].\(.key) must be a non-empty relative path without parent traversal") end ] else [] end diff --git a/tests/release-catalog-config/invalid/absolute-artifact-path.json b/tests/release-catalog-config/invalid/absolute-artifact-path.json new file mode 100644 index 00000000..0a9fbd0b --- /dev/null +++ b/tests/release-catalog-config/invalid/absolute-artifact-path.json @@ -0,0 +1,4 @@ +{ + "release_catalog_key": "archive:smoke", + "artifacts": [{"path": "/tmp/smoke.tar.gz", "package_identity_file": "package.json"}] +} diff --git a/tests/release-catalog-config/invalid/absolute-identity-path.json b/tests/release-catalog-config/invalid/absolute-identity-path.json new file mode 100644 index 00000000..b4f8a39d --- /dev/null +++ b/tests/release-catalog-config/invalid/absolute-identity-path.json @@ -0,0 +1,4 @@ +{ + "release_catalog_key": "archive:smoke", + "artifacts": [{"path": "smoke.tar.gz", "package_identity_file": "/tmp/package.json"}] +} diff --git a/tests/release-catalog-config/invalid/parent-artifact-path.json b/tests/release-catalog-config/invalid/parent-artifact-path.json new file mode 100644 index 00000000..1d7bc648 --- /dev/null +++ b/tests/release-catalog-config/invalid/parent-artifact-path.json @@ -0,0 +1,4 @@ +{ + "release_catalog_key": "archive:smoke", + "artifacts": [{"path": "../smoke.tar.gz", "package_identity_file": "package.json"}] +} diff --git a/tests/release-catalog-config/invalid/unknown-field.json b/tests/release-catalog-config/invalid/unknown-field.json new file mode 100644 index 00000000..316cb46d --- /dev/null +++ b/tests/release-catalog-config/invalid/unknown-field.json @@ -0,0 +1,4 @@ +{ + "release_catalog_key": "archive:smoke", + "unexpected": true +} diff --git a/tests/release-catalog-config/package-identity-file.json b/tests/release-catalog-config/valid/package-identity-file.json similarity index 100% rename from tests/release-catalog-config/package-identity-file.json rename to tests/release-catalog-config/valid/package-identity-file.json diff --git a/tests/release_catalog_config_test.sh b/tests/release_catalog_config_test.sh index cf615f90..34842386 100755 --- a/tests/release_catalog_config_test.sh +++ b/tests/release_catalog_config_test.sh @@ -27,7 +27,7 @@ assert_invalid() { } valid_output="${temporary_directory}/valid.output" -RELEASE_CATALOG_CONFIG="$(<"${repository_root}/tests/release-catalog-config/package-identity-file.json")" \ +RELEASE_CATALOG_CONFIG="$(<"${repository_root}/tests/release-catalog-config/valid/package-identity-file.json")" \ GITHUB_OUTPUT="${valid_output}" "${validator}" grep -Fx 'release_catalog_key=maven:cuvs-java' "${valid_output}" @@ -78,8 +78,8 @@ assert_invalid \ malformed-artifact \ '{"release_catalog_key":"archive:smoke","artifacts":[{"file":"smoke.tar.gz","package_identity_file":false,"sbom":false}]}' \ 'artifacts[0] has unknown field(s): file, sbom' -grep -F 'artifacts[0].path must be a non-empty, single-line string' "${temporary_directory}/malformed-artifact.error" >/dev/null -grep -F 'artifacts[0].package_identity_file must be a non-empty, single-line string' "${temporary_directory}/malformed-artifact.error" >/dev/null +grep -F 'artifacts[0].path must be a non-empty relative path without parent traversal' "${temporary_directory}/malformed-artifact.error" >/dev/null +grep -F 'artifacts[0].package_identity_file must be a non-empty relative path without parent traversal' "${temporary_directory}/malformed-artifact.error" >/dev/null default_output="${temporary_directory}/default.output" RELEASE_CATALOG_CONFIG='{"release_catalog_key":"conda:smoke"}' \ @@ -97,3 +97,23 @@ RELEASE_CATALOG_CONFIG='{ grep -Fx 'release_catalog_key=wheel:kvikio' "${standard_output}" grep -Fx 'artifact_directory=dist' "${standard_output}" grep -Fx 'artifacts=' "${standard_output}" + +for invalid_fixture in "${repository_root}"/tests/release-catalog-config/invalid/*.json; do + fixture_name="$(basename "${invalid_fixture}" .json)" + case "${fixture_name}" in + absolute-artifact-path | parent-artifact-path) + expected="artifacts[0].path must be a non-empty relative path without parent traversal" + ;; + absolute-identity-path) + expected="artifacts[0].package_identity_file must be a non-empty relative path without parent traversal" + ;; + unknown-field) + expected="unknown field(s): unexpected" + ;; + *) + echo "missing expected runtime error for ${fixture_name}" >&2 + exit 1 + ;; + esac + assert_invalid "fixture-${fixture_name}" "$(<"${invalid_fixture}")" "${expected}" +done From c3ed113818b8d8608ee2a1f2396dd9c8057ec77e Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 1 Sep 2026 16:45:43 -0500 Subject: [PATCH 24/36] Standardize release catalog SBOMs on CycloneDX --- README.md | 10 ++- release-catalog-dispatch/action.yml | 2 +- release-catalog/README.md | 23 +++---- release-catalog/entries.schema.json | 12 +--- release-catalog/examples/README.md | 4 +- .../cuvs-java/release-catalog-entries.json | 7 +- ...d1715b14a3f7c349cba300cb2b9.sbom.spdx.json | 39 ----------- release-catalog/materialize.sh | 64 +++---------------- release-catalog/upload-s3.sh | 2 +- tests/release_catalog_test.sh | 28 +++----- 10 files changed, 38 insertions(+), 153 deletions(-) delete mode 100644 release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.spdx.json diff --git a/README.md b/README.md index b3d2ebd7..7368e574 100644 --- a/README.md +++ b/README.md @@ -18,14 +18,12 @@ shows the complete output for one primary JAR: associates the artifact with its package identity and evidence. - [SLSA/in-toto provenance](release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.provenance.json) identifies the source, workflow, invocation, and exact artifact digest. -- The SBOM is provided in both [SPDX - 2.3](release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.spdx.json) - and [CycloneDX - 1.6](release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.cdx.json) - formats so consumers can choose the serialization their tooling supports. +- The [CycloneDX 1.6 + SBOM](release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.cdx.json) + identifies the artifact and binds it to its digest. The [example companion notes](release-catalog/examples/README.md) explain how -the catalog, both SBOMs, provenance, and separately uploaded primary artifact +the catalog, CycloneDX SBOM, provenance, and separately uploaded primary artifact are bound together by the artifact SHA-256. A dispatch action is one that: diff --git a/release-catalog-dispatch/action.yml b/release-catalog-dispatch/action.yml index c39b2411..aa395af6 100644 --- a/release-catalog-dispatch/action.yml +++ b/release-catalog-dispatch/action.yml @@ -1,6 +1,6 @@ name: Dispatch release catalog description: >- - Create a release catalog companion (artifact inventory, provenance, and SBOMs) + Create a release catalog companion (artifact inventory, provenance, and CycloneDX SBOM) for the artifacts a build job produced, and optionally upload it to S3. See release-catalog/README.md. diff --git a/release-catalog/README.md b/release-catalog/README.md index 5899773c..ae31ff3c 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -42,7 +42,7 @@ create a release, not a release-candidate version of one artifact. | **Release train** | One planned release of RAPIDS, identified by an immutable JSON document. Its SHA-256 is used to group all companions that belong to that release in S3. | | **Release candidate** | A set of artifacts selected from the release catalog to be tested and, if it passes, published. | | **Provenance** | A [SLSA](https://slsa.dev/spec/v1.2/attestation-model) statement describing how, where, and from what source an artifact was built. | -| **SBOM** | Software bill of materials. The SBOMs this action generates identify the artifact and its digest only; they do not list its dependencies. | +| **SBOM** | Software bill of materials. The CycloneDX SBOM this action generates identifies the artifact and its digest only; it does not list its dependencies. | ## Companion contents @@ -54,10 +54,8 @@ Each build job creates one companion with this shape: └── release-evidence ├── ..provenance.json ├── ..sbom.cdx.json - ├── ..sbom.spdx.json ├── ..provenance.json ├── ..sbom.cdx.json - ├── ..sbom.spdx.json └── ... ``` @@ -78,17 +76,14 @@ details such as the reusable workflow remain in the source context and evidence rather than serving as the producer identity. For each artifact in its `entries` array, the action creates one provenance -file and two equivalent SBOMs, one in [SPDX -2.3](https://spdx.github.io/spdx-spec/v2.3/) and one in [CycloneDX -1.6](https://cyclonedx.org/docs/1.6/json/). The SHA-256 in each evidence +file and one [CycloneDX 1.6](https://cyclonedx.org/docs/1.6/json/) SBOM. The SHA-256 in each evidence filename is the digest of that artifact's contents, so evidence files can -never be confused across artifacts. The catalog entry lists both SBOM paths -under `sboms`, allowing consumers to select their preferred format without -inferring it from the filename. +never be confused across artifacts. The catalog entry lists the SBOM path +under `sbom`. Each entry also carries `sbom_kind: "generated-identity"`. This marks the -SBOMs as identity-only documents generated by this action. A future version -may accept producer-supplied dependency SBOMs, which would use a different +SBOM as an identity-only document generated by this action. A future version +may accept a producer-supplied dependency SBOM, which would use a different `sbom_kind`. ## Action inputs @@ -190,11 +185,11 @@ files have the same package identity. ## Design notes -Provenance and SBOMs are kept as separate documents. Provenance describes how a +Provenance and the SBOM are kept as separate documents. Provenance describes how a package was built and by whom; an SBOM lists what makes up the package. They complement each other: an SBOM without provenance may describe the wrong or an untrusted artifact, while provenance without an SBOM cannot efficiently answer -what vulnerable components are inside. Although both SBOM standards can carry -some build metadata, embedding provenance in the SBOMs would not replace the +what vulnerable components are inside. Although CycloneDX can carry some build +metadata, embedding provenance in the SBOM would not replace the in-toto/SLSA statement and would couple evidence with different consumers and lifecycles. diff --git a/release-catalog/entries.schema.json b/release-catalog/entries.schema.json index 7de80d80..3f192f98 100644 --- a/release-catalog/entries.schema.json +++ b/release-catalog/entries.schema.json @@ -51,20 +51,12 @@ "entry": { "type": "object", "additionalProperties": false, - "required": ["release_catalog_key", "path", "package", "sboms", "sbom_kind", "provenance"], + "required": ["release_catalog_key", "path", "package", "sbom", "sbom_kind", "provenance"], "properties": { "release_catalog_key": {"type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$"}, "path": {"$ref": "#/$defs/relative_path"}, "package": {"$ref": "#/$defs/package"}, - "sboms": { - "type": "object", - "additionalProperties": false, - "required": ["spdx", "cyclonedx"], - "properties": { - "spdx": {"$ref": "#/$defs/relative_path"}, - "cyclonedx": {"$ref": "#/$defs/relative_path"} - } - }, + "sbom": {"$ref": "#/$defs/relative_path"}, "sbom_kind": {"const": "generated-identity"}, "provenance": {"$ref": "#/$defs/relative_path"}, "signature": {"$ref": "#/$defs/relative_path"} diff --git a/release-catalog/examples/README.md b/release-catalog/examples/README.md index 0edcda07..7460913e 100644 --- a/release-catalog/examples/README.md +++ b/release-catalog/examples/README.md @@ -8,10 +8,10 @@ deliberately non-production example values. The primary JAR is not part of the companion and is therefore not checked in. Its example contents are `jar` followed by a newline, whose SHA-256 digest is `fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9`. -That digest links the catalog entry, SPDX and CycloneDX documents, provenance +That digest links the catalog entry, CycloneDX document, provenance statement, and the separately uploaded primary artifact. -The four JSON files are about 4 KiB in total. Companion size grows +The three JSON files are about 3 KiB in total. Companion size grows linearly with the number of primary artifacts. This version does not copy producer dependency SBOMs into the companion; those could be substantially larger and require a future explicit catalog association. diff --git a/release-catalog/examples/cuvs-java/release-catalog-entries.json b/release-catalog/examples/cuvs-java/release-catalog-entries.json index 34e2bf97..ab411ce4 100644 --- a/release-catalog/examples/cuvs-java/release-catalog-entries.json +++ b/release-catalog/examples/cuvs-java/release-catalog-entries.json @@ -9,11 +9,8 @@ "path": "cuvs-java-26.08.0.jar", "provenance": "release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.provenance.json", "release_catalog_key": "maven:cuvs-java", - "sbom_kind": "generated-identity", - "sboms": { - "cyclonedx": "release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.cdx.json", - "spdx": "release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.spdx.json" - } + "sbom": "release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.cdx.json", + "sbom_kind": "generated-identity" } ], "producer": "NVIDIA ADI Build/Operations", diff --git a/release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.spdx.json b/release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.spdx.json deleted file mode 100644 index c0584944..00000000 --- a/release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.spdx.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "SPDXID": "SPDXRef-DOCUMENT", - "comment": "Artifact-identity SBOM. It identifies only this artifact and its digest, not its dependencies.", - "creationInfo": { - "created": "2026-08-21T00:00:00Z", - "creators": [ - "Tool: rapidsai/shared-workflows release catalog" - ] - }, - "dataLicense": "CC0-1.0", - "documentDescribes": [ - "SPDXRef-Artifact" - ], - "documentNamespace": "https://rapids.ai/release-platform/spdx/fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9", - "name": "RAPIDS release artifact cuvs-java-26.08.0.jar", - "packages": [ - { - "SPDXID": "SPDXRef-Artifact", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9" - } - ], - "downloadLocation": "NOASSERTION", - "filesAnalyzed": false, - "name": "ai.rapids:cuvs-java", - "versionInfo": "26.08.0" - } - ], - "relationships": [ - { - "relatedSpdxElement": "SPDXRef-Artifact", - "relationshipType": "DESCRIBES", - "spdxElementId": "SPDXRef-DOCUMENT" - } - ], - "spdxVersion": "SPDX-2.3" -} diff --git a/release-catalog/materialize.sh b/release-catalog/materialize.sh index 99ee80f3..df2736d8 100755 --- a/release-catalog/materialize.sh +++ b/release-catalog/materialize.sh @@ -12,8 +12,8 @@ set -euo pipefail # # The script validates all context before touching outputs, resolves every # artifact to exactly one file inside RELEASE_ARTIFACT_DIRECTORY, reads or -# extracts its package identity, generates identity-only SPDX and CycloneDX -# SBOMs plus provenance evidence, and writes RELEASE_ENTRIES_NAME. It never +# extracts its package identity, generates an identity-only CycloneDX SBOM plus +# provenance evidence, and writes RELEASE_ENTRIES_NAME. It never # modifies primary files. require_nonempty() { @@ -321,51 +321,8 @@ generated_evidence_path() { printf 'release-evidence/%s.%s.%s.json\n' "${artifact_label}" "${artifact_digest}" "${kind}" } -# This SPDX document proves which named package and digest the entry refers to. -# It deliberately sets filesAnalyzed=false and contains no dependency inventory. -write_generated_sbom() { - local primary_path="$1" - local package="$2" - local destination="$3" - local artifact_digest - artifact_digest="$(sha256sum "${artifact_directory}/${primary_path}" | awk '{print $1}')" - mkdir -p "$(dirname "${artifact_directory}/${destination}")" - jq -n -S \ - --arg artifact_digest "${artifact_digest}" \ - --arg artifact_path "${primary_path}" \ - --argjson package "${package}" \ - '{ - spdxVersion: "SPDX-2.3", - dataLicense: "CC0-1.0", - SPDXID: "SPDXRef-DOCUMENT", - name: ("RAPIDS release artifact " + $artifact_path), - # This is a unique identifier, not a retrieval URL; SPDX requires an - # absolute URI for the document namespace but does not require it to resolve. - documentNamespace: ("https://rapids.ai/release-platform/spdx/" + $artifact_digest), - creationInfo: { - creators: ["Tool: rapidsai/shared-workflows release catalog"], - created: (now | strftime("%Y-%m-%dT%H:%M:%SZ")) - }, - documentDescribes: ["SPDXRef-Artifact"], - packages: [{ - SPDXID: "SPDXRef-Artifact", - name: $package.name, - versionInfo: $package.version, - downloadLocation: "NOASSERTION", - filesAnalyzed: false, - checksums: [{algorithm: "SHA256", checksumValue: $artifact_digest}] - }], - relationships: [{ - spdxElementId: "SPDXRef-DOCUMENT", - relationshipType: "DESCRIBES", - relatedSpdxElement: "SPDXRef-Artifact" - }], - comment: "Artifact-identity SBOM. It identifies only this artifact and its digest, not its dependencies." - }' >"${artifact_directory}/${destination}" -} - -# This CycloneDX document describes the same artifact identity as the SPDX -# document. It intentionally has no component inventory or dependency graph. +# This CycloneDX document identifies the artifact and intentionally has no +# component inventory or dependency graph. write_generated_cyclonedx_sbom() { local primary_path="$1" local package="$2" @@ -436,7 +393,7 @@ write_generated_provenance() { } shopt -s globstar nullglob -# Create one catalog entry, two identity-only SBOM documents, and one provenance +# Create one catalog entry, one identity-only SBOM document, and one provenance # document per primary artifact. Producer-supplied dependency evidence is # intentionally not accepted by this version of the schema. while IFS= read -r descriptor; do @@ -467,21 +424,18 @@ while IFS= read -r descriptor; do exit 1 fi - spdx_sbom_path="$(generated_evidence_path "${primary_path}" "sbom.spdx")" - write_generated_sbom "${primary_path}" "${package}" "${spdx_sbom_path}" - cyclonedx_sbom_path="$(generated_evidence_path "${primary_path}" "sbom.cdx")" - write_generated_cyclonedx_sbom "${primary_path}" "${package}" "${cyclonedx_sbom_path}" + sbom_path="$(generated_evidence_path "${primary_path}" "sbom.cdx")" + write_generated_cyclonedx_sbom "${primary_path}" "${package}" "${sbom_path}" provenance_path="$(generated_evidence_path "${primary_path}" "provenance")" write_generated_provenance "${primary_path}" "${package}" "${provenance_path}" entry="$(jq -cn \ --arg release_catalog_key "${RELEASE_CATALOG_KEY}" \ --arg path "${primary_path}" \ - --arg spdx_sbom "${spdx_sbom_path}" \ - --arg cyclonedx_sbom "${cyclonedx_sbom_path}" \ + --arg sbom "${sbom_path}" \ --arg provenance "${provenance_path}" \ --argjson package "${package}" \ - '{release_catalog_key: $release_catalog_key, path: $path, sboms: {spdx: $spdx_sbom, cyclonedx: $cyclonedx_sbom}, sbom_kind: "generated-identity", provenance: $provenance, package: $package}')" + '{release_catalog_key: $release_catalog_key, path: $path, sbom: $sbom, sbom_kind: "generated-identity", provenance: $provenance, package: $package}')" jq --argjson entry "${entry}" '.entries += [$entry]' "${temporary_manifest}" >"${temporary_manifest}.next" mv "${temporary_manifest}.next" "${temporary_manifest}" diff --git a/release-catalog/upload-s3.sh b/release-catalog/upload-s3.sh index 9ba4f2ad..1f4815e4 100755 --- a/release-catalog/upload-s3.sh +++ b/release-catalog/upload-s3.sh @@ -55,7 +55,7 @@ base_key="${RELEASE_CANDIDATE_PREFIX}/${RELEASE_CANDIDATE_TRAIN_SHA256}/${GITHUB # `signature` is optional, so select only declared string paths. Without the # filter, jq renders a missing optional value as the literal text `null` and # the uploader attempts to find a file with that name. -mapfile -t companion_paths < <(jq -r '["release-catalog-entries.json"] + ([.entries[] | .path, (.sboms[]), .provenance, .signature? | strings] | unique) | .[]' "${entries_path}") +mapfile -t companion_paths < <(jq -r '["release-catalog-entries.json"] + ([.entries[] | .path, .sbom, .provenance, .signature? | strings] | unique) | .[]' "${entries_path}") head_metadata="$(mktemp)" trap 'rm -f "${head_metadata}"' EXIT diff --git a/tests/release_catalog_test.sh b/tests/release_catalog_test.sh index 665ed8bf..8ce50cf5 100755 --- a/tests/release_catalog_test.sh +++ b/tests/release_catalog_test.sh @@ -45,13 +45,14 @@ jq -e ' and .entries[0].path == "cuvs-java-26.08.0.jar" and .entries[0].package.name == "ai.rapids:cuvs-java" and .entries[0].sbom_kind == "generated-identity" - and (.entries[0].sboms | keys) == ["cyclonedx", "spdx"] + and (.entries[0].sbom | endswith(".sbom.cdx.json")) ' "${entries_path}" >/dev/null # Optional signatures must not cause the S3 uploader to request a literal # `null` file. This mirrors its declared-file selection without requiring AWS. -mapfile -t upload_paths < <(jq -r '["release-catalog-entries.json"] + ([.entries[] | .path, (.sboms[]), .provenance, .signature? | strings] | unique) | .[]' "${entries_path}") -test "${upload_paths[*]}" = "release-catalog-entries.json cuvs-java-26.08.0.jar release-evidence/cuvs-java-26.08.0.jar.sbom.cdx.json release-evidence/cuvs-java-26.08.0.jar.sbom.spdx.json release-evidence/cuvs-java-26.08.0.jar.provenance.json" +mapfile -t upload_paths < <(jq -r '["release-catalog-entries.json"] + ([.entries[] | .path, .sbom, .provenance, .signature? | strings] | unique) | .[]' "${entries_path}") +artifact_sha256="$(sha256sum "${canonical_bundle_directory}/cuvs-java-26.08.0.jar" | awk '{print $1}')" +test "${upload_paths[*]}" = "release-catalog-entries.json cuvs-java-26.08.0.jar release-evidence/cuvs-java-26.08.0.jar.${artifact_sha256}.provenance.json release-evidence/cuvs-java-26.08.0.jar.${artifact_sha256}.sbom.cdx.json" if printf '%s\n' "${upload_paths[@]}" | grep -Fx null; then echo "optional signature emitted a literal null upload path" >&2 exit 1 @@ -61,15 +62,8 @@ fi # action. Only the generated SBOM timestamps are normalized. example_directory="${repository_root}/release-catalog/examples/cuvs-java" diff -u "${example_directory}/release-catalog-entries.json" "${entries_path}" -example_spdx_sbom_path="$(jq -r '.entries[0].sboms.spdx' "${entries_path}")" -example_cyclonedx_sbom_path="$(jq -r '.entries[0].sboms.cyclonedx' "${entries_path}")" +example_cyclonedx_sbom_path="$(jq -r '.entries[0].sbom' "${entries_path}")" example_provenance_path="$(jq -r '.entries[0].provenance' "${entries_path}")" -jq -S '.creationInfo.created = "2026-08-21T00:00:00Z"' \ - "${canonical_bundle_directory}/${example_spdx_sbom_path}" \ - >"${temporary_directory}/normalized-example.spdx.json" -diff -u \ - "${example_directory}/${example_spdx_sbom_path}" \ - "${temporary_directory}/normalized-example.spdx.json" jq -S '.metadata.timestamp = "2026-08-21T00:00:00Z"' \ "${canonical_bundle_directory}/${example_cyclonedx_sbom_path}" \ >"${temporary_directory}/normalized-example.cdx.json" @@ -86,7 +80,7 @@ cp "${entries_path}" "${isolated_companion_directory}/" cp -R "${canonical_bundle_directory}/release-evidence" "${isolated_companion_directory}/" while IFS= read -r evidence_path; do test -f "${isolated_companion_directory}/${evidence_path}" -done < <(jq -r '.entries[] | .sboms[], .provenance' "${isolated_companion_directory}/release-catalog-entries.json") +done < <(jq -r '.entries[] | .sbom, .provenance' "${isolated_companion_directory}/release-catalog-entries.json") multiple_identity_directory="${temporary_directory}/multiple-identities" mkdir -p "${multiple_identity_directory}" @@ -111,7 +105,7 @@ jq -e ' and .entries[0].package == {ecosystem: "archive", name: "first", version: "1.0"} and .entries[1].package == {ecosystem: "maven", name: "example:second", version: "2.0"} ' "${multiple_identity_directory}/release-catalog-entries.json" >/dev/null -test "$(find "${multiple_identity_directory}/release-evidence" -type f | wc -l | tr -d ' ')" = "6" +test "$(find "${multiple_identity_directory}/release-evidence" -type f | wc -l | tr -d ' ')" = "4" generated_directory="${temporary_directory}/generated-bundle" mkdir -p "${generated_directory}/linux-64" @@ -129,8 +123,7 @@ export RAPIDS_SHA RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_AR "${repository_root}/release-catalog/materialize.sh" generated_entries_path="${generated_directory}/release-catalog-entries.json" -generated_spdx_sbom_path="$(jq -r '.entries[0].sboms.spdx' "${generated_entries_path}")" -generated_cyclonedx_sbom_path="$(jq -r '.entries[0].sboms.cyclonedx' "${generated_entries_path}")" +generated_cyclonedx_sbom_path="$(jq -r '.entries[0].sbom' "${generated_entries_path}")" generated_provenance_path="$(jq -r '.entries[0].provenance' "${generated_entries_path}")" jq -e ' .source.artifact == "kvikio_conda_python_kvikio_x86_64_abi3_cu12" @@ -140,11 +133,6 @@ jq -e ' and .entries[0].package == {ecosystem: "conda", name: "kvikio", version: "26.08.00a32"} and .entries[0].sbom_kind == "generated-identity" ' "${generated_entries_path}" >/dev/null -jq -e ' - .spdxVersion == "SPDX-2.3" - and .packages[0].name == "kvikio" - and .packages[0].versionInfo == "26.08.00a32" -' "${generated_directory}/${generated_spdx_sbom_path}" >/dev/null jq -e ' .bomFormat == "CycloneDX" and .specVersion == "1.6" From f8b0e07d858cf6e7a130b5c7685421f7536acd74 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 1 Sep 2026 16:52:48 -0500 Subject: [PATCH 25/36] polish/simplifying --- release-catalog/config.schema.json | 6 +++--- release-catalog/examples/README.md | 5 ----- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/release-catalog/config.schema.json b/release-catalog/config.schema.json index 380deec0..d94cc600 100644 --- a/release-catalog/config.schema.json +++ b/release-catalog/config.schema.json @@ -10,7 +10,7 @@ ], "properties": { "release_catalog_key": { - "description": "Caller-supplied stable label for artifacts that share one release policy. This is not a UUID, is not generated by the action, and need not be unique per package. Standard workflows use :, such as conda:cudf; custom jobs use :, such as maven:cuvs-java. See https://github.com/rapidsai/shared-actions/blob/main/release-catalog/README.md#release_catalog_key", + "description": "Label for artifact group. Artifacts within a group share one release policy. This is not a UUID, is not generated by the action, and need not be unique per package. Standard workflows use :, such as conda:cudf; custom jobs use :, such as maven:cuvs-java. See https://github.com/rapidsai/shared-actions/blob/main/release-catalog/README.md#release_catalog_key", "type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$" @@ -23,7 +23,7 @@ "default": "." }, "artifacts": { - "description": "Optional explicit primary release files. When omitted, Conda and wheel artifacts are discovered in artifact_directory.", + "description": "Optional explicit artifact list. When omitted, Conda and wheel artifacts are discovered in artifact_directory.", "type": "array", "minItems": 1, "items": { @@ -49,7 +49,7 @@ } }, "package_identity_file": { - "description": "Path, relative to artifact_directory, to producer-created package identity JSON for this artifact. Required only when identity cannot be extracted from the artifact. See https://github.com/rapidsai/shared-actions/blob/main/release-catalog/README.md#package-identity-file", + "description": "Path, relative to artifact_directory, to package identity JSON file for this artifact. Required only when identity cannot be extracted from the artifact. See https://github.com/rapidsai/shared-actions/blob/main/release-catalog/README.md#package-identity-file", "type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$", diff --git a/release-catalog/examples/README.md b/release-catalog/examples/README.md index 7460913e..e4eb5a8b 100644 --- a/release-catalog/examples/README.md +++ b/release-catalog/examples/README.md @@ -10,8 +10,3 @@ Its example contents are `jar` followed by a newline, whose SHA-256 digest is `fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9`. That digest links the catalog entry, CycloneDX document, provenance statement, and the separately uploaded primary artifact. - -The three JSON files are about 3 KiB in total. Companion size grows -linearly with the number of primary artifacts. This version does not copy -producer dependency SBOMs into the companion; those could be substantially -larger and require a future explicit catalog association. From 5a606a743c2100be360ad8a8b4b5b39a6af2b0eb Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 1 Sep 2026 19:55:07 -0500 Subject: [PATCH 26/36] massage readme and materialize.sh script --- release-catalog/README.md | 22 +++++++++------------- release-catalog/materialize.sh | 6 +++--- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/release-catalog/README.md b/release-catalog/README.md index ae31ff3c..99545ae2 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -11,7 +11,8 @@ those artifacts today: then publish. That requires a single inventory of what each job built. - **Keep build-time evidence.** Facts such as which commit, workflow, and run produced a wheel cannot be reliably reconstructed after the fact. They have - to be captured by the job that built the artifact. + to be captured by the job that built the artifact. This captured metadata + can make security scans more accurate. This action solves both by having each build job write a small **companion** next to its artifacts: an inventory of the files it produced, plus provenance @@ -35,14 +36,14 @@ create a release, not a release-candidate version of one artifact. | Term | Meaning | | ---- | ------- | -| **Artifact** | A file a build job produces and that we may publish: a `.conda` file, a wheel, a JAR, and so on. | +| **Artifact** | A file produced by a build job: a `.conda` file, a wheel, a JAR, and so on. | | **Companion** | The set of files this action writes next to a job's artifacts: `release-catalog-entries.json` plus a `release-evidence/` directory. One companion per build job. | | **Producer** | The build job (and the workflow it runs in) that created the artifacts. | | **Release catalog** | The merged inventory of all companions for a release, maintained by the release platform. | | **Release train** | One planned release of RAPIDS, identified by an immutable JSON document. Its SHA-256 is used to group all companions that belong to that release in S3. | | **Release candidate** | A set of artifacts selected from the release catalog to be tested and, if it passes, published. | | **Provenance** | A [SLSA](https://slsa.dev/spec/v1.2/attestation-model) statement describing how, where, and from what source an artifact was built. | -| **SBOM** | Software bill of materials. The CycloneDX SBOM this action generates identifies the artifact and its digest only; it does not list its dependencies. | +| **SBOM** | Software bill of materials, which lists components by ecosystem, name and version. The [CycloneDX](https://cyclonedx.org/docs/1.6/json/) SBOM this action generates identifies the artifact and its digest only; it does not list its dependencies. Adding dependencies here will be a future project. | ## Companion contents @@ -70,11 +71,6 @@ Generated entry documents conform to output contract for producer identity, source context, package identity, and evidence paths. -The top-level `producer` identifies the organization responsible for producing -and maintaining the companion: `NVIDIA ADI Build/Operations`. Implementation -details such as the reusable workflow remain in the source context and evidence -rather than serving as the producer identity. - For each artifact in its `entries` array, the action creates one provenance file and one [CycloneDX 1.6](https://cyclonedx.org/docs/1.6/json/) SBOM. The SHA-256 in each evidence filename is the digest of that artifact's contents, so evidence files can @@ -83,7 +79,7 @@ under `sbom`. Each entry also carries `sbom_kind: "generated-identity"`. This marks the SBOM as an identity-only document generated by this action. A future version -may accept a producer-supplied dependency SBOM, which would use a different +will accept a producer-supplied dependency SBOM, which would use a different `sbom_kind`. ## Action inputs @@ -113,7 +109,7 @@ The canonical schema and field documentation are in ### `release_catalog_key` This is a grouping label that represents a common "release policy." Artifacts -that share a `release_catalog_key` are versioned, validated, ordered, published, +that share a common `release_catalog_key` are versioned, validated, ordered, published, and promoted together. For example, the `cudf` and `dask-cudf` Conda packages both use `conda:cudf` because they share the same version, validation, inter-package order, and publishing destination, and are ultimately published @@ -168,9 +164,9 @@ resolves to zero files or multiple files results in an error. #### Package identity file -`package_identity_file` is a JSON file containing at least `ecosystem`, `name`, -and `version`; `build` and `platform` are optional. This path is relative to -`artifact_directory`. +`package_identity_file` is a path to a JSON file that contains at least +`ecosystem`, `name`, and `version` keys; `build` and `platform` are optional. +This path is relative to `artifact_directory`. ```json { diff --git a/release-catalog/materialize.sh b/release-catalog/materialize.sh index df2736d8..8be64e6f 100755 --- a/release-catalog/materialize.sh +++ b/release-catalog/materialize.sh @@ -3,7 +3,7 @@ set -euo pipefail -# Convert one completed build directory into a release catalog companion. +# Record artifact metadata from completed build directory into a release catalog companion. # # Inputs come from release-catalog-dispatch/action.yml. RELEASE_ARTIFACTS is # either empty (discover Conda packages and wheels) or a validated JSON array @@ -127,8 +127,7 @@ resolve_one_file() { } # Normalize supported package formats into the common package identity stored -# in each catalog entry. Wheels expose Core Metadata; Conda packages expose -# info/index.json. Custom formats provide the same fields in a sidecar file. +# in each catalog entry. Wheels expose Core Metadata; describe_wheel_package() { local wheel_path="$1" local relative_path="${wheel_path#"${artifact_directory}/"}" @@ -155,6 +154,7 @@ describe_wheel_package() { '{ecosystem: "wheel", name: $name, version: $version}' } +# Conda packages expose info/index.json. describe_conda_package() { local package_path="$1" local relative_path="${package_path#"${artifact_directory}/"}" From 53f1f55264170e6ccaa8eac441bb76accf35a03c Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 1 Sep 2026 20:57:00 -0500 Subject: [PATCH 27/36] improve consistency of 'producer' meaning --- .github/workflows/pr.yml | 2 +- ci/test_release_catalog_schemas.py | 4 ++-- release-catalog/README.md | 22 +++++++++++-------- release-catalog/config.schema.json | 2 +- release-catalog/entries.schema.json | 8 +++++-- release-catalog/examples/README.md | 3 +++ .../cuvs-java/release-catalog-entries.json | 5 +++-- ...cd1715b14a3f7c349cba300cb2b9.sbom.cdx.json | 11 +++++++++- release-catalog/materialize.sh | 22 ++++++++++++++----- tests/release_catalog_test.sh | 3 ++- 10 files changed, 58 insertions(+), 24 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index b3b329ba..7a87b287 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -59,7 +59,7 @@ jobs: - name: Verify release catalog dispatch smoke test run: | jq -e ' - .producer == "NVIDIA ADI Build/Operations" + .producer == "rapidsai/shared-actions/release-catalog" and .source.artifact == "release-catalog-dispatch-smoke" and .entries[0].release_catalog_key == "archive:smoke" and .entries[0].path == "package.tar.gz" diff --git a/ci/test_release_catalog_schemas.py b/ci/test_release_catalog_schemas.py index 2b597106..a03695c5 100644 --- a/ci/test_release_catalog_schemas.py +++ b/ci/test_release_catalog_schemas.py @@ -42,9 +42,9 @@ def main() -> None: errors = list(entries.iter_errors(example)) if errors: raise ValueError(f"generated entries example rejected: {errors[0].message}") - wrong_producer = {**example, "producer": "shared-workflows"} + wrong_producer = {**example, "producer": "some-other-tool"} if not list(entries.iter_errors(wrong_producer)): - raise ValueError("entries schema accepted an implementation name as producer") + raise ValueError("entries schema accepted an unknown producer") if __name__ == "__main__": diff --git a/release-catalog/README.md b/release-catalog/README.md index 99545ae2..8243e2ee 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -38,12 +38,12 @@ create a release, not a release-candidate version of one artifact. | ---- | ------- | | **Artifact** | A file produced by a build job: a `.conda` file, a wheel, a JAR, and so on. | | **Companion** | The set of files this action writes next to a job's artifacts: `release-catalog-entries.json` plus a `release-evidence/` directory. One companion per build job. | -| **Producer** | The build job (and the workflow it runs in) that created the artifacts. | +| **Producer** | The tool that wrote the companion: this action, recorded as `rapidsai/shared-actions/release-catalog` in the `producer` field. The build job that created the artifacts is identified by the `source` block instead. | | **Release catalog** | The merged inventory of all companions for a release, maintained by the release platform. | -| **Release train** | One planned release of RAPIDS, identified by an immutable JSON document. Its SHA-256 is used to group all companions that belong to that release in S3. | +| **Release train** | One planned release of RAPIDS, identified by an immutable JSON document that the [release platform](https://github.com/rapidsai/release-scripts/issues/102) maintains. Its SHA-256 is used to group all companions that belong to that release in S3. | | **Release candidate** | A set of artifacts selected from the release catalog to be tested and, if it passes, published. | | **Provenance** | A [SLSA](https://slsa.dev/spec/v1.2/attestation-model) statement describing how, where, and from what source an artifact was built. | -| **SBOM** | Software bill of materials, which lists components by ecosystem, name and version. The [CycloneDX](https://cyclonedx.org/docs/1.6/json/) SBOM this action generates identifies the artifact and its digest only; it does not list its dependencies. Adding dependencies here will be a future project. | +| **SBOM** | Software bill of materials. In general an SBOM lists a package's components by ecosystem, name, and version. The [CycloneDX](https://cyclonedx.org/docs/1.6/json/) SBOM this action generates lists only the artifact itself and its digest; listing dependencies is a future addition. | ## Companion contents @@ -61,9 +61,9 @@ Each build job creates one companion with this shape: ``` `release-catalog-entries.json` records the artifacts produced by the job -(`.conda`, `.whl`, or otherwise), the checksums of those files, the package -identity of each one, and the source context of the job (repository, commit, -workflow, run). See [`examples/cuvs-java`](examples/cuvs-java) for a complete +(`.conda`, `.whl`, or otherwise), the SHA-256 of each file, the package +identity of each one, the paths to its evidence files, and the source context +of the job (repository, commit, workflow, run). See [`examples/cuvs-java`](examples/cuvs-java) for a complete companion. Generated entry documents conform to @@ -79,8 +79,12 @@ under `sbom`. Each entry also carries `sbom_kind: "generated-identity"`. This marks the SBOM as an identity-only document generated by this action. A future version -will accept a producer-supplied dependency SBOM, which would use a different -`sbom_kind`. +will accept a caller-supplied dependency SBOM; that will be a new +`sbom_kind` value and a new `schema_version`, so existing consumers can tell +the two apart. + +Each entry may also carry an optional `signature` path. It is reserved for a +future signing step; this version never populates it. ## Action inputs @@ -121,7 +125,7 @@ workflow. Standard RAPIDS Conda and wheel workflows use `:`, such as `conda:cudf`, because those workflows apply a repository-level release -policy. A custom producer should use `:`, such as +policy. A custom job should use `:`, such as `maven:cuvs-java`, or simply `maven:cuvs`. ### `artifact_directory` and `artifacts` diff --git a/release-catalog/config.schema.json b/release-catalog/config.schema.json index d94cc600..80ff13c7 100644 --- a/release-catalog/config.schema.json +++ b/release-catalog/config.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://rapids.ai/schemas/release-catalog-config-v1.json", "title": "Release catalog configuration", - "description": "Producer configuration for one release catalog companion.", + "description": "Caller configuration for one release catalog companion.", "type": "object", "additionalProperties": false, "required": [ diff --git a/release-catalog/entries.schema.json b/release-catalog/entries.schema.json index 3f192f98..16192a0a 100644 --- a/release-catalog/entries.schema.json +++ b/release-catalog/entries.schema.json @@ -8,7 +8,10 @@ "required": ["schema_version", "producer", "source", "entries"], "properties": { "schema_version": {"const": 1}, - "producer": {"const": "NVIDIA ADI Build/Operations"}, + "producer": { + "description": "The tool that generated this document. Matches metadata.tools in the CycloneDX SBOM.", + "const": "rapidsai/shared-actions/release-catalog" + }, "source": {"$ref": "#/$defs/source"}, "entries": { "type": "array", @@ -51,10 +54,11 @@ "entry": { "type": "object", "additionalProperties": false, - "required": ["release_catalog_key", "path", "package", "sbom", "sbom_kind", "provenance"], + "required": ["release_catalog_key", "path", "sha256", "package", "sbom", "sbom_kind", "provenance"], "properties": { "release_catalog_key": {"type": "string", "minLength": 1, "pattern": "^[^\\r\\n]+$"}, "path": {"$ref": "#/$defs/relative_path"}, + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, "package": {"$ref": "#/$defs/package"}, "sbom": {"$ref": "#/$defs/relative_path"}, "sbom_kind": {"const": "generated-identity"}, diff --git a/release-catalog/examples/README.md b/release-catalog/examples/README.md index e4eb5a8b..fa34d5eb 100644 --- a/release-catalog/examples/README.md +++ b/release-catalog/examples/README.md @@ -10,3 +10,6 @@ Its example contents are `jar` followed by a newline, whose SHA-256 digest is `fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9`. That digest links the catalog entry, CycloneDX document, provenance statement, and the separately uploaded primary artifact. + +The three JSON files are about 3 KiB in total. Companion size grows linearly +with the number of artifacts in the job. diff --git a/release-catalog/examples/cuvs-java/release-catalog-entries.json b/release-catalog/examples/cuvs-java/release-catalog-entries.json index ab411ce4..93ca43cc 100644 --- a/release-catalog/examples/cuvs-java/release-catalog-entries.json +++ b/release-catalog/examples/cuvs-java/release-catalog-entries.json @@ -10,10 +10,11 @@ "provenance": "release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.provenance.json", "release_catalog_key": "maven:cuvs-java", "sbom": "release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.cdx.json", - "sbom_kind": "generated-identity" + "sbom_kind": "generated-identity", + "sha256": "fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9" } ], - "producer": "NVIDIA ADI Build/Operations", + "producer": "rapidsai/shared-actions/release-catalog", "schema_version": 1, "source": { "artifact": "cuvs-java-cuda12.9.1", diff --git a/release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.cdx.json b/release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.cdx.json index 617e270c..8b237a4e 100644 --- a/release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.cdx.json +++ b/release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.cdx.json @@ -23,7 +23,16 @@ "type": "file", "version": "26.08.0" }, - "timestamp": "2026-08-21T00:00:00Z" + "timestamp": "2026-08-21T00:00:00Z", + "tools": { + "components": [ + { + "group": "rapidsai", + "name": "shared-actions/release-catalog", + "type": "application" + } + ] + } }, "specVersion": "1.6", "version": 1 diff --git a/release-catalog/materialize.sh b/release-catalog/materialize.sh index 8be64e6f..2f032202 100755 --- a/release-catalog/materialize.sh +++ b/release-catalog/materialize.sh @@ -127,7 +127,10 @@ resolve_one_file() { } # Normalize supported package formats into the common package identity stored -# in each catalog entry. Wheels expose Core Metadata; +# in each catalog entry. Custom formats (JARs, tarballs) supply the same fields +# via package_identity_file instead. + +# Wheels expose Core Metadata in .dist-info/METADATA. describe_wheel_package() { local wheel_path="$1" local relative_path="${wheel_path#"${artifact_directory}/"}" @@ -292,7 +295,7 @@ prepare_artifacts() { } # From here onward, every descriptor has an exact path and either extracted -# package identity or a producer-created package identity file. +# package identity or a caller-created package identity file. artifacts="$(prepare_artifacts)" entries_path="${artifact_directory}/${RELEASE_ENTRIES_NAME}" temporary_manifest="$(mktemp "${artifact_directory}/.release-catalog.XXXXXX")" @@ -340,6 +343,13 @@ write_generated_cyclonedx_sbom() { version: 1, metadata: { timestamp: (now | strftime("%Y-%m-%dT%H:%M:%SZ")), + tools: { + components: [{ + type: "application", + group: "rapidsai", + name: "shared-actions/release-catalog" + }] + }, component: { type: "file", "bom-ref": ("urn:sha256:" + $artifact_digest), @@ -394,7 +404,7 @@ write_generated_provenance() { shopt -s globstar nullglob # Create one catalog entry, one identity-only SBOM document, and one provenance -# document per primary artifact. Producer-supplied dependency evidence is +# document per primary artifact. Caller-supplied dependency evidence is # intentionally not accepted by this version of the schema. while IFS= read -r descriptor; do if ! jq -e ' @@ -424,6 +434,7 @@ while IFS= read -r descriptor; do exit 1 fi + artifact_sha256="$(sha256sum "${artifact_directory}/${primary_path}" | awk '{print $1}')" sbom_path="$(generated_evidence_path "${primary_path}" "sbom.cdx")" write_generated_cyclonedx_sbom "${primary_path}" "${package}" "${sbom_path}" provenance_path="$(generated_evidence_path "${primary_path}" "provenance")" @@ -432,10 +443,11 @@ while IFS= read -r descriptor; do entry="$(jq -cn \ --arg release_catalog_key "${RELEASE_CATALOG_KEY}" \ --arg path "${primary_path}" \ + --arg sha256 "${artifact_sha256}" \ --arg sbom "${sbom_path}" \ --arg provenance "${provenance_path}" \ --argjson package "${package}" \ - '{release_catalog_key: $release_catalog_key, path: $path, sbom: $sbom, sbom_kind: "generated-identity", provenance: $provenance, package: $package}')" + '{release_catalog_key: $release_catalog_key, path: $path, sha256: $sha256, sbom: $sbom, sbom_kind: "generated-identity", provenance: $provenance, package: $package}')" jq --argjson entry "${entry}" '.entries += [$entry]' "${temporary_manifest}" >"${temporary_manifest}.next" mv "${temporary_manifest}.next" "${temporary_manifest}" @@ -459,7 +471,7 @@ jq -n -S \ --argjson entries "$(jq -c '.entries' "${temporary_manifest}")" \ '{ schema_version: 1, - producer: "NVIDIA ADI Build/Operations", + producer: "rapidsai/shared-actions/release-catalog", source: { artifact: $artifact_name, repository: $repository, diff --git a/tests/release_catalog_test.sh b/tests/release_catalog_test.sh index 8ce50cf5..2014d0d7 100755 --- a/tests/release_catalog_test.sh +++ b/tests/release_catalog_test.sh @@ -36,13 +36,14 @@ entries_path="${canonical_bundle_directory}/release-catalog-entries.json" test ! -e "${canonical_bundle_directory}/release-catalog-metadata.json" jq -e ' .schema_version == 1 - and .producer == "NVIDIA ADI Build/Operations" + and .producer == "rapidsai/shared-actions/release-catalog" and .source.artifact == "cuvs-java-cuda12.9.1" and .source.repository == "NVIDIA/cuvs" and .source.sha == "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" and (.entries | length == 1) and .entries[0].release_catalog_key == "maven:cuvs-java" and .entries[0].path == "cuvs-java-26.08.0.jar" + and .entries[0].sha256 == "fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9" and .entries[0].package.name == "ai.rapids:cuvs-java" and .entries[0].sbom_kind == "generated-identity" and (.entries[0].sbom | endswith(".sbom.cdx.json")) From 580c07cc18bc0d78662ad31fe07b6cba740bf3c8 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 1 Sep 2026 21:57:55 -0500 Subject: [PATCH 28/36] more language clarification --- README.md | 9 +++-- release-catalog/README.md | 60 ++++++++++++++++++------------ release-catalog/examples/README.md | 5 ++- 3 files changed, 44 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 7368e574..8f1a4825 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,10 @@ at a specific revision, and also to simplify using files beyond a given action . ## Release catalog companions -The release catalog action creates a companion archive for each build job. See -the [release catalog documentation](release-catalog/README.md) for its -configuration and archive contract. +The release catalog action creates a companion (an artifact inventory plus +provenance and SBOM files) for each build job. See the [release catalog +documentation](release-catalog/README.md) for its configuration and output +contract. The checked-in [cuVS Java companion example](release-catalog/examples/cuvs-java) shows the complete output for one primary JAR: @@ -23,7 +24,7 @@ shows the complete output for one primary JAR: identifies the artifact and binds it to its digest. The [example companion notes](release-catalog/examples/README.md) explain how -the catalog, CycloneDX SBOM, provenance, and separately uploaded primary artifact +the catalog, CycloneDX SBOM, provenance, and the artifact uploaded alongside them are bound together by the artifact SHA-256. A dispatch action is one that: diff --git a/release-catalog/README.md b/release-catalog/README.md index 8243e2ee..997d7aa0 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -24,7 +24,8 @@ themselves. 1. A build job produces its artifacts as usual. 2. This action runs as a later step in that job and writes a companion into the artifact directory. -3. The companion is uploaded to a private S3 bucket (`upload-to-s3: true`). +3. The artifacts and their companion are uploaded to a private S3 bucket + (`upload-to-s3: true`). 4. The RAPIDS release platform reads the companions from every job in a release train, merges them into the **release catalog**, and uses the catalog to assemble and test a **release candidate**. @@ -36,7 +37,7 @@ create a release, not a release-candidate version of one artifact. | Term | Meaning | | ---- | ------- | -| **Artifact** | A file produced by a build job: a `.conda` file, a wheel, a JAR, and so on. | +| **Artifact** | A file produced by a build job that we may publish: a `.conda` file, a wheel, a JAR, and so on. Sometimes called the *primary* artifact to distinguish it from its evidence files. | | **Companion** | The set of files this action writes next to a job's artifacts: `release-catalog-entries.json` plus a `release-evidence/` directory. One companion per build job. | | **Producer** | The tool that wrote the companion: this action, recorded as `rapidsai/shared-actions/release-catalog` in the `producer` field. The build job that created the artifacts is identified by the `source` block instead. | | **Release catalog** | The merged inventory of all companions for a release, maintained by the release platform. | @@ -60,11 +61,15 @@ Each build job creates one companion with this shape: └── ... ``` +Evidence filenames are derived from the artifact's path relative to +`artifact_directory`, with `/` replaced by `_`, so `linux-64/librmm-1.0.conda` +produces `release-evidence/linux-64_librmm-1.0.conda..provenance.json`. + `release-catalog-entries.json` records the artifacts produced by the job (`.conda`, `.whl`, or otherwise), the SHA-256 of each file, the package identity of each one, the paths to its evidence files, and the source context -of the job (repository, commit, workflow, run). See [`examples/cuvs-java`](examples/cuvs-java) for a complete -companion. +of the job (repository, commit, workflow, run). See +[`examples/cuvs-java`](examples/cuvs-java) for a complete companion. Generated entry documents conform to [`entries.schema.json`](entries.schema.json). The schema fixes the versioned @@ -78,10 +83,12 @@ never be confused across artifacts. The catalog entry lists the SBOM path under `sbom`. Each entry also carries `sbom_kind: "generated-identity"`. This marks the -SBOM as an identity-only document generated by this action. A future version -will accept a caller-supplied dependency SBOM; that will be a new -`sbom_kind` value and a new `schema_version`, so existing consumers can tell -the two apart. +SBOM as an identity-only document generated by this action. On its own, each +identity SBOM binds a name, version, and digest in a format scanners already +consume. Merged across all jobs, they inventory every package a release ships. +They do not yet describe what is inside each package; that is what the future +dependency SBOM will add. That will be a new `sbom_kind` value and a new +`schema_version`, so existing consumers can tell the two apart. Each entry may also carry an optional `signature` path. It is reserved for a future signing step; this version never populates it. @@ -90,16 +97,21 @@ future signing step; this version never populates it. The action is used via `rapidsai/shared-actions/release-catalog-dispatch`. +The action also requires `RAPIDS_SHA` in the job environment, set to the +commit that was checked out and built. Standard RAPIDS build workflows set it; +custom workflows must export it before this step runs. + | Input | Required | Description | | ----- | -------- | ----------- | | `config` | yes | JSON object selecting the artifacts and their release catalog key. Schema: [`config.schema.json`](config.schema.json). Details below. | | `source-artifact-name` | yes | Stable name for this job's output, for example the GitHub Actions artifact name. Used as the last path component in S3. | -| `upload-to-s3` | no | `"true"` to upload the companion to S3. Defaults to `"false"`, which only writes the companion locally. | +| `upload-to-s3` | no | `"true"` to upload the artifacts and companion to S3. Defaults to `"false"`, which only writes the companion locally. | | `candidate-train-sha256` | when uploading | SHA-256 of the release train JSON this build belongs to. All companions for one release share it. | | `candidate-bucket` | no | S3 bucket for companions. Defaults to `rapids-release-candidates`. | | `candidate-prefix` | no | Root prefix inside the bucket. Defaults to `candidate-builds`. | -When uploading, the companion lands at +When uploading, every file the companion lists (the artifacts, the evidence +files, and `release-catalog-entries.json` itself) lands under `s3:////////`. Uploads are conditional: an existing object is accepted only if its bytes match, so reruns are safe. @@ -113,11 +125,11 @@ The canonical schema and field documentation are in ### `release_catalog_key` This is a grouping label that represents a common "release policy." Artifacts -that share a common `release_catalog_key` are versioned, validated, ordered, published, -and promoted together. For example, the `cudf` and `dask-cudf` Conda packages -both use `conda:cudf` because they share the same version, validation, -inter-package order, and publishing destination, and are ultimately published -together. +that share a `release_catalog_key` are versioned, validated, ordered, +published, and promoted together. For example, the `cudf` and `dask-cudf` +Conda packages both use `conda:cudf` because they share the same version, +validation, inter-package order, and publishing destination, and are +ultimately published together. The key does not need to be unique per artifact or per matrix variant. Use a separate key only when the outputs intentionally follow a separate release @@ -130,7 +142,7 @@ policy. A custom job should use `:`, such as ### `artifact_directory` and `artifacts` -`artifact_directory` is the base directory containing the primary artifacts. +`artifact_directory` is the base directory containing the artifacts. For standard Conda and wheel jobs, omit `artifacts`; the action discovers all Conda packages and wheels below that directory and parses metadata from them. @@ -185,11 +197,11 @@ files have the same package identity. ## Design notes -Provenance and the SBOM are kept as separate documents. Provenance describes how a -package was built and by whom; an SBOM lists what makes up the package. They -complement each other: an SBOM without provenance may describe the wrong or an -untrusted artifact, while provenance without an SBOM cannot efficiently answer -what vulnerable components are inside. Although CycloneDX can carry some build -metadata, embedding provenance in the SBOM would not replace the -in-toto/SLSA statement and would couple evidence with different consumers and -lifecycles. +Provenance and the SBOM are kept as separate documents. Provenance describes +how a package was built and by whom; an SBOM identifies the package and, once +dependency SBOMs are added, what makes it up. They complement each other: an +SBOM without provenance may describe the wrong or an untrusted artifact, while +provenance without an SBOM cannot be matched to a package by name and version. +Although CycloneDX can carry some build metadata, embedding provenance in the +SBOM would not replace the in-toto/SLSA statement and would couple evidence +with different consumers and lifecycles. diff --git a/release-catalog/examples/README.md b/release-catalog/examples/README.md index fa34d5eb..acc8132f 100644 --- a/release-catalog/examples/README.md +++ b/release-catalog/examples/README.md @@ -5,11 +5,12 @@ Java build job. It was derived from the test fixture in `tests/release_catalog_test.sh`; its source SHA and GitHub Actions run ID are deliberately non-production example values. -The primary JAR is not part of the companion and is therefore not checked in. +The JAR itself is an artifact, not part of the companion, and is therefore not +checked in. Its example contents are `jar` followed by a newline, whose SHA-256 digest is `fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9`. That digest links the catalog entry, CycloneDX document, provenance -statement, and the separately uploaded primary artifact. +statement, and the artifact, which is uploaded to S3 alongside the companion. The three JSON files are about 3 KiB in total. Companion size grows linearly with the number of artifacts in the job. From 02bead3d7405cf86294831c39597007b7926f0b7 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 2 Sep 2026 08:25:01 -0500 Subject: [PATCH 29/36] remove release catalog entry from root README --- README.md | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/README.md b/README.md index 8f1a4825..995a28c8 100644 --- a/README.md +++ b/README.md @@ -5,28 +5,6 @@ especially the telemetry actions, use a pattern that we refer to as "dispatch ac The general idea of a dispatch action is to make it easier to depend on other actions at a specific revision, and also to simplify using files beyond a given action .yml file. -## Release catalog companions - -The release catalog action creates a companion (an artifact inventory plus -provenance and SBOM files) for each build job. See the [release catalog -documentation](release-catalog/README.md) for its configuration and output -contract. - -The checked-in [cuVS Java companion example](release-catalog/examples/cuvs-java) -shows the complete output for one primary JAR: - -- [`release-catalog-entries.json`](release-catalog/examples/cuvs-java/release-catalog-entries.json) - associates the artifact with its package identity and evidence. -- [SLSA/in-toto provenance](release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.provenance.json) - identifies the source, workflow, invocation, and exact artifact digest. -- The [CycloneDX 1.6 - SBOM](release-catalog/examples/cuvs-java/release-evidence/cuvs-java-26.08.0.jar.fb8ce05502991565de98e3e21d9ab98151c1cd1715b14a3f7c349cba300cb2b9.sbom.cdx.json) - identifies the artifact and binds it to its digest. - -The [example companion notes](release-catalog/examples/README.md) explain how -the catalog, CycloneDX SBOM, provenance, and the artifact uploaded alongside them -are bound together by the artifact SHA-256. - A dispatch action is one that: * clones the shared-actions repository (repo/ref changeable using env vars) * runs (dispatches to) another action within the clone, using a relative path From 5e43f29a9c948850df7923a68b659f0d32b8f701 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 2 Sep 2026 08:45:39 -0500 Subject: [PATCH 30/36] Consolidate the release catalog action --- .github/workflows/pr.yml | 6 ++--- README.md | 27 +++++++++++++++++-- release-catalog/README.md | 12 +++++---- .../action.yml | 24 ++++++++--------- release-catalog/materialize.sh | 2 +- release-catalog/upload-s3.sh | 8 ++++-- 6 files changed, 53 insertions(+), 26 deletions(-) rename {release-catalog-dispatch => release-catalog}/action.yml (83%) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 7a87b287..cc80089f 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -40,7 +40,7 @@ jobs: '{ecosystem: "archive", name: "smoke", version: "1.0"}' \ >release-catalog-smoke/package-identity.json - name: Run release catalog dispatch smoke test - uses: ./release-catalog-dispatch + uses: ./release-catalog env: RAPIDS_SHA: ${{ github.sha }} SHARED_ACTIONS_REPO: ${{ github.event.pull_request.head.repo.full_name }} @@ -55,12 +55,12 @@ jobs: "package_identity_file": "package-identity.json" }] } - source-artifact-name: release-catalog-dispatch-smoke + source-artifact-name: release-catalog-smoke - name: Verify release catalog dispatch smoke test run: | jq -e ' .producer == "rapidsai/shared-actions/release-catalog" - and .source.artifact == "release-catalog-dispatch-smoke" + and .source.artifact == "release-catalog-smoke" and .entries[0].release_catalog_key == "archive:smoke" and .entries[0].path == "package.tar.gz" and .entries[0].sbom_kind == "generated-identity" diff --git a/README.md b/README.md index 995a28c8..6ead1668 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,28 @@ A dispatch action is one that: * clones the shared-actions repository (repo/ref changeable using env vars) * runs (dispatches to) another action within the clone, using a relative path +The checkout is the important part of the pattern. It gives every subsequent +step a stable `./shared-actions` path and lets a caller test changes by setting +`SHARED_ACTIONS_REPO` and `SHARED_ACTIONS_REF` without changing the caller's +`uses:` line. When those variables are absent, dispatch actions normally use +`rapidsai/shared-actions` at `main`. + +The public action and its implementation may use either of two layouts: + +* A small dispatch action can invoke a separate implementation action under the + checked-out `./shared-actions` directory. This is useful when several public + actions share one implementation. +* A self-dispatching action can keep its `action.yml`, scripts, schemas, and + documentation in one folder. After checkout, its shell steps invoke the files + in the corresponding `./shared-actions/` folder. This is simpler + when the implementation has no independent callers. + +In both layouts, actions must use files from the checked-out `./shared-actions` +tree rather than from the revision that initially loaded the public +`action.yml`. Otherwise `SHARED_ACTIONS_REPO` and `SHARED_ACTIONS_REF` would +select only the wrapper while the implementation continued to come from a +different revision. + ## Example dispatch action ```yaml @@ -24,9 +46,10 @@ runs: - name: Clone shared-actions repo uses: actions/checkout@v4 with: - repository: ${{ env.SHARED_ACTIONS_REPO }} - ref: ${{ env.SHARED_ACTIONS_REF }} + repository: ${{ env.SHARED_ACTIONS_REPO || 'rapidsai/shared-actions' }} + ref: ${{ env.SHARED_ACTIONS_REF || 'main' }} path: ./shared-actions + persist-credentials: false - name: Run local implementation action uses: ./shared-actions/impls/example-action ``` diff --git a/release-catalog/README.md b/release-catalog/README.md index 997d7aa0..c43c6008 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -95,7 +95,7 @@ future signing step; this version never populates it. ## Action inputs -The action is used via `rapidsai/shared-actions/release-catalog-dispatch`. +The action is used via `rapidsai/shared-actions/release-catalog`. The action also requires `RAPIDS_SHA` in the job environment, set to the commit that was checked out and built. Standard RAPIDS build workflows set it; @@ -108,11 +108,13 @@ custom workflows must export it before this step runs. | `upload-to-s3` | no | `"true"` to upload the artifacts and companion to S3. Defaults to `"false"`, which only writes the companion locally. | | `candidate-train-sha256` | when uploading | SHA-256 of the release train JSON this build belongs to. All companions for one release share it. | | `candidate-bucket` | no | S3 bucket for companions. Defaults to `rapids-release-candidates`. | -| `candidate-prefix` | no | Root prefix inside the bucket. Defaults to `candidate-builds`. | +| `candidate-prefix` | no | Optional root prefix inside the bucket. Defaults to empty, placing each train directly below the bucket root. | When uploading, every file the companion lists (the artifacts, the evidence files, and `release-catalog-entries.json` itself) lands under -`s3:////////`. +`s3:///////`. +When `candidate-prefix` is set explicitly, it is inserted between the bucket +name and the train SHA-256. Uploads are conditional: an existing object is accepted only if its bytes match, so reruns are safe. @@ -148,7 +150,7 @@ Conda packages and wheels below that directory and parses metadata from them. ```yaml - name: Create wheel release catalog companion - uses: rapidsai/shared-actions/release-catalog-dispatch@main + uses: rapidsai/shared-actions/release-catalog@main with: config: >- { @@ -164,7 +166,7 @@ resolves to zero files or multiple files results in an error. ```yaml - name: Create custom release catalog companion - uses: rapidsai/shared-actions/release-catalog-dispatch@main + uses: rapidsai/shared-actions/release-catalog@main with: config: >- { diff --git a/release-catalog-dispatch/action.yml b/release-catalog/action.yml similarity index 83% rename from release-catalog-dispatch/action.yml rename to release-catalog/action.yml index aa395af6..87cf12ea 100644 --- a/release-catalog-dispatch/action.yml +++ b/release-catalog/action.yml @@ -1,4 +1,4 @@ -name: Dispatch release catalog +name: Release catalog description: >- Create a release catalog companion (artifact inventory, provenance, and CycloneDX SBOM) for the artifacts a build job produced, and optionally upload it to S3. @@ -26,9 +26,9 @@ inputs: required: false default: rapids-release-candidates candidate-prefix: - description: Root prefix inside candidate-bucket. + description: Optional root prefix inside candidate-bucket. Empty writes below the bucket root. required: false - default: candidate-builds + default: "" upload-to-s3: description: Whether to upload the companion to S3. When false, the companion is only written locally. required: false @@ -36,20 +36,18 @@ inputs: runs: using: composite steps: + - name: Clone shared-actions repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ env.SHARED_ACTIONS_REPO || 'rapidsai/shared-actions' }} + ref: ${{ env.SHARED_ACTIONS_REF || 'main' }} + path: ./shared-actions + persist-credentials: false - id: implementation name: Resolve release catalog implementation shell: bash - env: - ACTION_PATH: ${{ github.action_path }} - HOST_WORKSPACE: ${{ github.workspace }} run: | - host_root="$(dirname "$(dirname "${HOST_WORKSPACE}")")" - container_root="$(dirname "$(dirname "${PWD}")")" - action_path="${ACTION_PATH}" - if [[ ! -d "${action_path}" && "${action_path}" == "${host_root}/"* ]]; then - action_path="${container_root}/${action_path#"${host_root}/"}" - fi - implementation_path="$(dirname "${action_path}")/release-catalog" + implementation_path="${PWD}/shared-actions/release-catalog" if [[ ! -d "${implementation_path}" ]]; then echo "release-catalog implementation not found at ${implementation_path}" >&2 exit 1 diff --git a/release-catalog/materialize.sh b/release-catalog/materialize.sh index 2f032202..da78c3cc 100755 --- a/release-catalog/materialize.sh +++ b/release-catalog/materialize.sh @@ -5,7 +5,7 @@ set -euo pipefail # Record artifact metadata from completed build directory into a release catalog companion. # -# Inputs come from release-catalog-dispatch/action.yml. RELEASE_ARTIFACTS is +# Inputs come from release-catalog/action.yml. RELEASE_ARTIFACTS is # either empty (discover Conda packages and wheels) or a validated JSON array # selecting custom artifacts. GitHub Actions supplies the GITHUB_* build # context, while RAPIDS_SHA identifies the commit actually checked out. diff --git a/release-catalog/upload-s3.sh b/release-catalog/upload-s3.sh index 1f4815e4..88800676 100755 --- a/release-catalog/upload-s3.sh +++ b/release-catalog/upload-s3.sh @@ -21,9 +21,10 @@ safe_prefix() { [[ "${prefix}" != /* && "${prefix}" != */ && "${prefix}" != *'//' && "${prefix}" != *'..'* ]] } -for value in RELEASE_ARTIFACT_DIRECTORY RELEASE_CANDIDATE_BUCKET RELEASE_CANDIDATE_PREFIX RELEASE_CANDIDATE_TRAIN_SHA256 RELEASE_SOURCE_ARTIFACT_NAME GITHUB_REPOSITORY GITHUB_RUN_ID; do +for value in RELEASE_ARTIFACT_DIRECTORY RELEASE_CANDIDATE_BUCKET RELEASE_CANDIDATE_TRAIN_SHA256 RELEASE_SOURCE_ARTIFACT_NAME GITHUB_REPOSITORY GITHUB_RUN_ID; do require_value "${value}" "${!value:-}" done +RELEASE_CANDIDATE_PREFIX="${RELEASE_CANDIDATE_PREFIX:-}" if [[ ! "${RELEASE_CANDIDATE_TRAIN_SHA256}" =~ ^[[:xdigit:]]{64}$ ]]; then echo "RELEASE_CANDIDATE_TRAIN_SHA256 must be a SHA-256 hex digest" >&2 exit 1 @@ -51,7 +52,10 @@ safe_relative_path() { [[ -n "${path}" && "${path}" != /* && "${path}" != ../* && "${path}" != */../* && "${path}" != *'/..' ]] } -base_key="${RELEASE_CANDIDATE_PREFIX}/${RELEASE_CANDIDATE_TRAIN_SHA256}/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID}/${RELEASE_SOURCE_ARTIFACT_NAME}" +base_key="${RELEASE_CANDIDATE_TRAIN_SHA256}/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID}/${RELEASE_SOURCE_ARTIFACT_NAME}" +if [[ -n "${RELEASE_CANDIDATE_PREFIX}" ]]; then + base_key="${RELEASE_CANDIDATE_PREFIX}/${base_key}" +fi # `signature` is optional, so select only declared string paths. Without the # filter, jq renders a missing optional value as the literal text `null` and # the uploader attempts to find a file with that name. From 6bf65a3e42eb2c3673736a280f5791b585b7116d Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 2 Sep 2026 08:58:15 -0500 Subject: [PATCH 31/36] Extract Maven identity from JAR metadata --- release-catalog/README.md | 15 +++-- release-catalog/config.schema.json | 2 +- release-catalog/entries.schema.json | 2 + release-catalog/materialize.sh | 65 +++++++++++++++++-- tests/release_catalog_discovery_test.sh | 2 +- tests/release_catalog_package_parsing_test.sh | 46 +++++++++++++ 6 files changed, 121 insertions(+), 11 deletions(-) diff --git a/release-catalog/README.md b/release-catalog/README.md index c43c6008..39330322 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -145,8 +145,9 @@ policy. A custom job should use `:`, such as ### `artifact_directory` and `artifacts` `artifact_directory` is the base directory containing the artifacts. -For standard Conda and wheel jobs, omit `artifacts`; the action discovers all -Conda packages and wheels below that directory and parses metadata from them. +For standard Conda, wheel, and Maven JAR jobs, omit `artifacts`; the action +discovers supported artifacts below that directory and parses package identity +from their embedded metadata. ```yaml - name: Create wheel release catalog companion @@ -173,13 +174,19 @@ resolves to zero files or multiple files results in an error. "release_catalog_key": "maven:cuvs-java", "artifact_directory": "java/cuvs-java/target", "artifacts": [{ - "path": "cuvs-java-*-x86_64-cuda*.jar", - "package_identity_file": "cuvs-java.release-package-identity.json" + "path": "cuvs-java-*-x86_64-cuda*.jar" }] } source-artifact-name: cuvs-java ``` +For a JAR, the action requires exactly one +`META-INF/maven///pom.properties` file and reads its +`groupId`, `artifactId`, and `version`. The directory coordinates must match +the property values. JARs without this metadata, or shaded JARs containing +multiple Maven descriptors, require `package_identity_file` so the action does +not guess which package the artifact represents. + #### Package identity file `package_identity_file` is a path to a JSON file that contains at least diff --git a/release-catalog/config.schema.json b/release-catalog/config.schema.json index 80ff13c7..197cac0c 100644 --- a/release-catalog/config.schema.json +++ b/release-catalog/config.schema.json @@ -23,7 +23,7 @@ "default": "." }, "artifacts": { - "description": "Optional explicit artifact list. When omitted, Conda and wheel artifacts are discovered in artifact_directory.", + "description": "Optional explicit artifact list. When omitted, Conda packages, wheels, and Maven JARs are discovered in artifact_directory.", "type": "array", "minItems": 1, "items": { diff --git a/release-catalog/entries.schema.json b/release-catalog/entries.schema.json index 16192a0a..938414d9 100644 --- a/release-catalog/entries.schema.json +++ b/release-catalog/entries.schema.json @@ -27,6 +27,7 @@ "not": {"pattern": "(^/)|((^|/)\\.\\.(/|$))"} }, "source": { + "description": "GitHub Actions context for the job that produced this companion. It binds every entry to the built source revision and the workflow run that created it.", "type": "object", "additionalProperties": false, "required": ["artifact", "repository", "sha", "workflow_ref", "run_id", "run_attempt"], @@ -40,6 +41,7 @@ } }, "package": { + "description": "Package identity used by release tooling and publication destinations. It is extracted from supported artifacts or supplied explicitly when the artifact format cannot provide it.", "type": "object", "additionalProperties": false, "required": ["ecosystem", "name", "version"], diff --git a/release-catalog/materialize.sh b/release-catalog/materialize.sh index da78c3cc..5e0ed891 100755 --- a/release-catalog/materialize.sh +++ b/release-catalog/materialize.sh @@ -6,7 +6,7 @@ set -euo pipefail # Record artifact metadata from completed build directory into a release catalog companion. # # Inputs come from release-catalog/action.yml. RELEASE_ARTIFACTS is -# either empty (discover Conda packages and wheels) or a validated JSON array +# either empty (discover Conda packages, wheels, and Maven JARs) or a validated JSON array # selecting custom artifacts. GitHub Actions supplies the GITHUB_* build # context, while RAPIDS_SHA identifies the commit actually checked out. # @@ -127,8 +127,8 @@ resolve_one_file() { } # Normalize supported package formats into the common package identity stored -# in each catalog entry. Custom formats (JARs, tarballs) supply the same fields -# via package_identity_file instead. +# in each catalog entry. Other formats supply the same fields via +# package_identity_file instead. # Wheels expose Core Metadata in .dist-info/METADATA. describe_wheel_package() { @@ -157,6 +157,45 @@ describe_wheel_package() { '{ecosystem: "wheel", name: $name, version: $version}' } +# Maven-built JARs normally expose their coordinates in exactly one +# META-INF/maven///pom.properties file. Shaded JARs may +# contain several descriptors, so ambiguity is an error rather than a guess. +describe_maven_jar_package() { + local jar_path="$1" + local relative_path="${jar_path#"${artifact_directory}/"}" + local -a metadata_members=() + local metadata_member + while IFS= read -r metadata_member; do + metadata_members+=("${metadata_member}") + done < <(unzip -Z1 "${jar_path}" | awk '$0 ~ "^META-INF/maven/[^/]+/[^/]+/pom[.]properties$"') + if [[ "${#metadata_members[@]}" -ne 1 ]]; then + echo "Maven JAR must contain exactly one META-INF/maven///pom.properties file: ${relative_path}" >&2 + return 1 + fi + + local metadata coordinates_path coordinate_group_id coordinate_artifact_id + local group_id artifact_id package_version + metadata="$(unzip -p "${jar_path}" "${metadata_members[0]}")" + coordinates_path="${metadata_members[0]#META-INF/maven/}" + coordinates_path="${coordinates_path%/pom.properties}" + coordinate_group_id="${coordinates_path%%/*}" + coordinate_artifact_id="${coordinates_path#*/}" + group_id="$(awk -F= '$1 == "groupId" {sub(/^[^=]*=/, ""); sub(/\r$/, ""); print; exit}' <<<"${metadata}")" + artifact_id="$(awk -F= '$1 == "artifactId" {sub(/^[^=]*=/, ""); sub(/\r$/, ""); print; exit}' <<<"${metadata}")" + package_version="$(awk -F= '$1 == "version" {sub(/^[^=]*=/, ""); sub(/\r$/, ""); print; exit}' <<<"${metadata}")" + if [[ -z "${group_id}" || -z "${artifact_id}" || -z "${package_version}" ]]; then + echo "Maven pom.properties must contain non-empty groupId, artifactId, and version fields: ${relative_path}" >&2 + return 1 + fi + if [[ "${group_id}" != "${coordinate_group_id}" || "${artifact_id}" != "${coordinate_artifact_id}" ]]; then + echo "Maven pom.properties coordinates do not match its META-INF path: ${relative_path}" >&2 + return 1 + fi + + jq -cn --arg name "${group_id}:${artifact_id}" --arg version "${package_version}" \ + '{ecosystem: "maven", name: $name, version: $version}' +} + # Conda packages expose info/index.json. describe_conda_package() { local package_path="$1" @@ -207,7 +246,7 @@ prepare_artifacts() { local -a detected_files=() while IFS= read -r primary_file; do detected_files+=("${primary_file}") - done < <(find "${artifact_directory}" -type f \( -name '*.conda' -o -name '*.tar.bz2' -o -name '*.whl' \) -print | sort) + done < <(find "${artifact_directory}" -type f \( -name '*.conda' -o -name '*.tar.bz2' -o -name '*.whl' -o -name '*.jar' \) -print | sort) for primary_file in "${detected_files[@]}"; do primary_path="${primary_file#"${artifact_directory}/"}" @@ -217,6 +256,11 @@ prepare_artifacts() { return 1 fi ;; + *.jar) + if ! package="$(describe_maven_jar_package "${primary_file}")"; then + return 1 + fi + ;; *) if ! package="$(describe_conda_package "${primary_file}")"; then return 1 @@ -260,6 +304,17 @@ prepare_artifacts() { return 1 fi ;; + *.jar) + if package="$(describe_maven_jar_package "${primary_file}" 2>/dev/null)"; then + if [[ -n "${identity_file}" ]]; then + echo "package_identity_file is not allowed when Maven JAR identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" >&2 + exit 1 + fi + elif [[ -z "${identity_file}" ]]; then + echo "JAR does not contain one unambiguous Maven package identity and requires package_identity_file: $(jq -r '.path' <<<"${descriptor}")" >&2 + exit 1 + fi + ;; *.tar.bz2) if conda_package="$(describe_conda_package "${primary_file}" 2>/dev/null)"; then if [[ -n "${identity_file}" ]]; then @@ -288,7 +343,7 @@ prepare_artifacts() { fi if [[ "$(jq 'length' <<<"${prepared_artifacts}")" -eq 0 ]]; then - echo "artifact-directory contains no detectable Conda or wheel artifacts; explicitly selected artifacts require package_identity_file when their identity cannot be parsed" >&2 + echo "artifact-directory contains no detectable Conda, wheel, or Maven JAR artifacts; explicitly selected artifacts require package_identity_file when their identity cannot be parsed" >&2 exit 1 fi printf '%s\n' "${prepared_artifacts}" diff --git a/tests/release_catalog_discovery_test.sh b/tests/release_catalog_discovery_test.sh index 59c71d58..2e31dde0 100755 --- a/tests/release_catalog_discovery_test.sh +++ b/tests/release_catalog_discovery_test.sh @@ -117,7 +117,7 @@ if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory} echo "materialize.sh unexpectedly accepted an artifact directory without detectable artifacts" >&2 exit 1 fi -grep -Fx 'artifact-directory contains no detectable Conda or wheel artifacts; explicitly selected artifacts require package_identity_file when their identity cannot be parsed' "${temporary_directory}/empty-error" +grep -Fx 'artifact-directory contains no detectable Conda, wheel, or Maven JAR artifacts; explicitly selected artifacts require package_identity_file when their identity cannot be parsed' "${temporary_directory}/empty-error" RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz"}]')" diff --git a/tests/release_catalog_package_parsing_test.sh b/tests/release_catalog_package_parsing_test.sh index 86284f7e..8ad87969 100755 --- a/tests/release_catalog_package_parsing_test.sh +++ b/tests/release_catalog_package_parsing_test.sh @@ -42,6 +42,52 @@ jq -e ' }] ' "${wheel_output_directory}/release-catalog-entries.json" >/dev/null +jar_output_directory="${temporary_directory}/jars" +jar_staging_directory="${temporary_directory}/jar-staging" +mkdir -p "${jar_output_directory}" "${jar_staging_directory}/META-INF/maven/ai.rapids/cuvs-java" +printf '%s\n' \ + 'artifactId=cuvs-java' \ + 'groupId=ai.rapids' \ + 'version=26.08.0' \ + >"${jar_staging_directory}/META-INF/maven/ai.rapids/cuvs-java/pom.properties" +( + cd "${jar_staging_directory}" + zip -qr "${jar_output_directory}/cuvs-java-26.08.0.jar" . +) + +RELEASE_ARTIFACT_DIRECTORY="${jar_output_directory}" +export RELEASE_ARTIFACT_DIRECTORY +"${repository_root}/release-catalog/materialize.sh" +jq -e ' + [.entries[] | {path, package}] == [{ + path: "cuvs-java-26.08.0.jar", + package: {ecosystem: "maven", name: "ai.rapids:cuvs-java", version: "26.08.0"} + }] +' "${jar_output_directory}/release-catalog-entries.json" >/dev/null + +ambiguous_jar_directory="${temporary_directory}/ambiguous-jar" +ambiguous_jar_staging_directory="${temporary_directory}/ambiguous-jar-staging" +mkdir -p \ + "${ambiguous_jar_directory}" \ + "${ambiguous_jar_staging_directory}/META-INF/maven/example/first" \ + "${ambiguous_jar_staging_directory}/META-INF/maven/example/second" +printf '%s\n' 'groupId=example' 'artifactId=first' 'version=1.0' \ + >"${ambiguous_jar_staging_directory}/META-INF/maven/example/first/pom.properties" +printf '%s\n' 'groupId=example' 'artifactId=second' 'version=1.0' \ + >"${ambiguous_jar_staging_directory}/META-INF/maven/example/second/pom.properties" +( + cd "${ambiguous_jar_staging_directory}" + zip -qr "${ambiguous_jar_directory}/shaded.jar" . +) +RELEASE_ARTIFACT_DIRECTORY="${ambiguous_jar_directory}" +export RELEASE_ARTIFACT_DIRECTORY +if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/ambiguous-jar-error"; then + echo "materialize.sh unexpectedly accepted a JAR with ambiguous Maven identity" >&2 + exit 1 +fi +grep -F 'Maven JAR must contain exactly one META-INF/maven///pom.properties file: shaded.jar' \ + "${temporary_directory}/ambiguous-jar-error" >/dev/null + conda_output_directory="${temporary_directory}/conda" tar_bz2_staging_directory="${temporary_directory}/tar-bz2-staging" conda_staging_directory="${temporary_directory}/conda-staging" From 6401705df24363722aaef4307f3303fff3036582 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 2 Sep 2026 09:02:59 -0500 Subject: [PATCH 32/36] Clarify future dependency SBOM support --- release-catalog/materialize.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/release-catalog/materialize.sh b/release-catalog/materialize.sh index 5e0ed891..9a4b58e4 100755 --- a/release-catalog/materialize.sh +++ b/release-catalog/materialize.sh @@ -379,8 +379,9 @@ generated_evidence_path() { printf 'release-evidence/%s.%s.%s.json\n' "${artifact_label}" "${artifact_digest}" "${kind}" } -# This CycloneDX document identifies the artifact and intentionally has no -# component inventory or dependency graph. +# This initial CycloneDX implementation identifies the artifact only. Component +# inventory and the dependency graph are intentionally deferred until a future +# implementation can accept evidence captured by the producer during the build. write_generated_cyclonedx_sbom() { local primary_path="$1" local package="$2" From 29f75dfc370ce05ecd5226538ca3fb47f263bc0c Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 2 Sep 2026 09:24:07 -0500 Subject: [PATCH 33/36] Make release catalog inputs explicit --- .github/workflows/pr.yml | 2 +- release-catalog/README.md | 14 +++--- release-catalog/action.yml | 7 ++- release-catalog/config.schema.json | 13 +++--- release-catalog/entries.schema.json | 3 +- release-catalog/materialize.sh | 25 +++-------- release-catalog/upload-s3.sh | 7 ++- release-catalog/validate-config.sh | 6 +-- tests/release_catalog_config_test.sh | 20 ++++++--- tests/release_catalog_discovery_test.sh | 5 +-- tests/release_catalog_package_parsing_test.sh | 5 +-- tests/release_catalog_test.sh | 44 ++++++++----------- 12 files changed, 72 insertions(+), 79 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index cc80089f..55513ec1 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -42,7 +42,6 @@ jobs: - name: Run release catalog dispatch smoke test uses: ./release-catalog env: - RAPIDS_SHA: ${{ github.sha }} SHARED_ACTIONS_REPO: ${{ github.event.pull_request.head.repo.full_name }} SHARED_ACTIONS_REF: ${{ github.event.pull_request.head.sha }} with: @@ -56,6 +55,7 @@ jobs: }] } source-artifact-name: release-catalog-smoke + source-sha: ${{ github.event.pull_request.head.sha }} - name: Verify release catalog dispatch smoke test run: | jq -e ' diff --git a/release-catalog/README.md b/release-catalog/README.md index 39330322..4b6ec67d 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -90,20 +90,14 @@ They do not yet describe what is inside each package; that is what the future dependency SBOM will add. That will be a new `sbom_kind` value and a new `schema_version`, so existing consumers can tell the two apart. -Each entry may also carry an optional `signature` path. It is reserved for a -future signing step; this version never populates it. - ## Action inputs The action is used via `rapidsai/shared-actions/release-catalog`. -The action also requires `RAPIDS_SHA` in the job environment, set to the -commit that was checked out and built. Standard RAPIDS build workflows set it; -custom workflows must export it before this step runs. - | Input | Required | Description | | ----- | -------- | ----------- | | `config` | yes | JSON object selecting the artifacts and their release catalog key. Schema: [`config.schema.json`](config.schema.json). Details below. | +| `source-sha` | yes | Full 40- or 64-character Git object ID of the commit that was checked out and built. Standard RAPIDS workflows pass `${{ env.RAPIDS_SHA }}` from `rapids-github-info`. | | `source-artifact-name` | yes | Stable name for this job's output, for example the GitHub Actions artifact name. Used as the last path component in S3. | | `upload-to-s3` | no | `"true"` to upload the artifacts and companion to S3. Defaults to `"false"`, which only writes the companion locally. | | `candidate-train-sha256` | when uploading | SHA-256 of the release train JSON this build belongs to. All companions for one release share it. | @@ -144,7 +138,9 @@ policy. A custom job should use `:`, such as ### `artifact_directory` and `artifacts` -`artifact_directory` is the base directory containing the artifacts. +`artifact_directory` is the required base directory containing the artifacts, +relative to the job working directory. The action does not scan the working +directory by default. For standard Conda, wheel, and Maven JAR jobs, omit `artifacts`; the action discovers supported artifacts below that directory and parses package identity from their embedded metadata. @@ -159,6 +155,7 @@ from their embedded metadata. "artifact_directory": ${{ toJSON(steps.package-name.outputs.WHEEL_OUTPUT_DIR) }} } source-artifact-name: ${{ steps.package-name.outputs.RAPIDS_PACKAGE_NAME }} + source-sha: ${{ env.RAPIDS_SHA }} ``` Other formats require an explicit `artifacts` list. Each dictionary in the list @@ -178,6 +175,7 @@ resolves to zero files or multiple files results in an error. }] } source-artifact-name: cuvs-java + source-sha: ${{ env.RAPIDS_SHA }} ``` For a JAR, the action requires exactly one diff --git a/release-catalog/action.yml b/release-catalog/action.yml index 87cf12ea..493fc3d7 100644 --- a/release-catalog/action.yml +++ b/release-catalog/action.yml @@ -10,6 +10,11 @@ inputs: JSON string with configuration for this action. Schema and field documentation: https://github.com/rapidsai/shared-actions/blob/main/release-catalog/config.schema.json required: true + source-sha: + description: >- + Full 40- or 64-character Git object ID of the commit that was checked out + and built. Standard RAPIDS workflows pass RAPIDS_SHA from rapids-github-info. + required: true source-artifact-name: description: >- Stable name for this job's output, such as the GitHub Actions artifact @@ -65,9 +70,9 @@ runs: env: IMPLEMENTATION_PATH: ${{ steps.implementation.outputs.path }} RELEASE_ARTIFACTS: ${{ steps.configuration.outputs.artifacts }} - RELEASE_ENTRIES_NAME: release-catalog-entries.json RELEASE_ARTIFACT_DIRECTORY: ${{ steps.configuration.outputs.artifact_directory }} RELEASE_SOURCE_ARTIFACT_NAME: ${{ inputs.source-artifact-name }} + RELEASE_SOURCE_SHA: ${{ inputs.source-sha }} RELEASE_CATALOG_KEY: ${{ steps.configuration.outputs.release_catalog_key }} run: "${IMPLEMENTATION_PATH}/materialize.sh" - name: Upload release catalog companion to S3 diff --git a/release-catalog/config.schema.json b/release-catalog/config.schema.json index 197cac0c..2e2c4a7f 100644 --- a/release-catalog/config.schema.json +++ b/release-catalog/config.schema.json @@ -6,21 +6,24 @@ "type": "object", "additionalProperties": false, "required": [ - "release_catalog_key" + "release_catalog_key", + "artifact_directory" ], "properties": { "release_catalog_key": { "description": "Label for artifact group. Artifacts within a group share one release policy. This is not a UUID, is not generated by the action, and need not be unique per package. Standard workflows use :, such as conda:cudf; custom jobs use :, such as maven:cuvs-java. See https://github.com/rapidsai/shared-actions/blob/main/release-catalog/README.md#release_catalog_key", "type": "string", "minLength": 1, - "pattern": "^[^\\r\\n]+$" + "pattern": "^[^\\r\\n]+$", + "not": { + "pattern": "(^/)|((^|/)\\.\\.(/|$))" + } }, "artifact_directory": { - "description": "Base directory for artifact paths. Defaults to the job working directory.", + "description": "Base directory for artifact paths, relative to the job working directory.", "type": "string", "minLength": 1, - "pattern": "^[^\\r\\n]+$", - "default": "." + "pattern": "^[^\\r\\n]+$" }, "artifacts": { "description": "Optional explicit artifact list. When omitted, Conda packages, wheels, and Maven JARs are discovered in artifact_directory.", diff --git a/release-catalog/entries.schema.json b/release-catalog/entries.schema.json index 938414d9..c51669d9 100644 --- a/release-catalog/entries.schema.json +++ b/release-catalog/entries.schema.json @@ -64,8 +64,7 @@ "package": {"$ref": "#/$defs/package"}, "sbom": {"$ref": "#/$defs/relative_path"}, "sbom_kind": {"const": "generated-identity"}, - "provenance": {"$ref": "#/$defs/relative_path"}, - "signature": {"$ref": "#/$defs/relative_path"} + "provenance": {"$ref": "#/$defs/relative_path"} } } } diff --git a/release-catalog/materialize.sh b/release-catalog/materialize.sh index 9a4b58e4..9b1ef669 100755 --- a/release-catalog/materialize.sh +++ b/release-catalog/materialize.sh @@ -8,12 +8,12 @@ set -euo pipefail # Inputs come from release-catalog/action.yml. RELEASE_ARTIFACTS is # either empty (discover Conda packages, wheels, and Maven JARs) or a validated JSON array # selecting custom artifacts. GitHub Actions supplies the GITHUB_* build -# context, while RAPIDS_SHA identifies the commit actually checked out. +# context, while RELEASE_SOURCE_SHA identifies the commit actually checked out. # # The script validates all context before touching outputs, resolves every # artifact to exactly one file inside RELEASE_ARTIFACT_DIRECTORY, reads or # extracts its package identity, generates an identity-only CycloneDX SBOM plus -# provenance evidence, and writes RELEASE_ENTRIES_NAME. It never +# provenance evidence, and writes release-catalog-entries.json. It never # modifies primary files. require_nonempty() { @@ -49,13 +49,12 @@ require_positive_integer() { # local caller must set explicit test values. require_single_line "RELEASE_CATALOG_KEY" "${RELEASE_CATALOG_KEY:-}" require_single_line "RELEASE_ARTIFACT_DIRECTORY" "${RELEASE_ARTIFACT_DIRECTORY:-}" -require_single_line "RELEASE_ENTRIES_NAME" "${RELEASE_ENTRIES_NAME:-}" require_single_line "RELEASE_SOURCE_ARTIFACT_NAME" "${RELEASE_SOURCE_ARTIFACT_NAME:-}" -source_sha="${RAPIDS_SHA:-}" -require_single_line "RAPIDS_SHA" "${source_sha}" +source_sha="${RELEASE_SOURCE_SHA:-}" +require_single_line "RELEASE_SOURCE_SHA" "${source_sha}" if [[ ! "${source_sha}" =~ ^[[:xdigit:]]{40}$ && ! "${source_sha}" =~ ^[[:xdigit:]]{64}$ ]]; then - echo "RAPIDS_SHA must be a 40- or 64-character hexadecimal Git object ID" >&2 + echo "RELEASE_SOURCE_SHA must be a 40- or 64-character hexadecimal Git object ID" >&2 exit 1 fi @@ -74,17 +73,6 @@ fi require_positive_integer "GITHUB_RUN_ATTEMPT" "${github_run_attempt}" require_positive_integer "GITHUB_RUN_ID" "${github_run_id}" -require_plain_filename() { - local label="$1" - local filename="$2" - if [[ "${filename}" == */* || "${filename}" == .* || "${filename}" == *".."* ]]; then - echo "${label} must be a plain filename" >&2 - exit 1 - fi -} - -require_plain_filename "entries-name" "${RELEASE_ENTRIES_NAME}" - if [[ ! -d "${RELEASE_ARTIFACT_DIRECTORY}" ]]; then echo "artifact-directory does not exist or is not a directory: ${RELEASE_ARTIFACT_DIRECTORY}" >&2 exit 1 @@ -352,7 +340,8 @@ prepare_artifacts() { # From here onward, every descriptor has an exact path and either extracted # package identity or a caller-created package identity file. artifacts="$(prepare_artifacts)" -entries_path="${artifact_directory}/${RELEASE_ENTRIES_NAME}" +# Fixed name: upload-s3.sh, the schema, and consumers all depend on it. +entries_path="${artifact_directory}/release-catalog-entries.json" temporary_manifest="$(mktemp "${artifact_directory}/.release-catalog.XXXXXX")" trap 'rm -f "${temporary_manifest}"' EXIT diff --git a/release-catalog/upload-s3.sh b/release-catalog/upload-s3.sh index 88800676..23c0dad3 100755 --- a/release-catalog/upload-s3.sh +++ b/release-catalog/upload-s3.sh @@ -56,10 +56,9 @@ base_key="${RELEASE_CANDIDATE_TRAIN_SHA256}/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID if [[ -n "${RELEASE_CANDIDATE_PREFIX}" ]]; then base_key="${RELEASE_CANDIDATE_PREFIX}/${base_key}" fi -# `signature` is optional, so select only declared string paths. Without the -# filter, jq renders a missing optional value as the literal text `null` and -# the uploader attempts to find a file with that name. -mapfile -t companion_paths < <(jq -r '["release-catalog-entries.json"] + ([.entries[] | .path, .sbom, .provenance, .signature? | strings] | unique) | .[]' "${entries_path}") +# Upload exactly the files the companion declares: the entries document, each +# artifact, and each artifact's evidence files. +mapfile -t companion_paths < <(jq -r '["release-catalog-entries.json"] + ([.entries[] | .path, .sbom, .provenance] | unique) | .[]' "${entries_path}") head_metadata="$(mktemp)" trap 'rm -f "${head_metadata}"' EXIT diff --git a/release-catalog/validate-config.sh b/release-catalog/validate-config.sh index f37795b7..43b7ac31 100755 --- a/release-catalog/validate-config.sh +++ b/release-catalog/validate-config.sh @@ -42,8 +42,8 @@ validation_errors="$(jq -r ' else empty end, if (.release_catalog_key | single_line_string) then empty else "release_catalog_key must be a non-empty, single-line string" end, - if (has("artifact_directory") | not) or (.artifact_directory | single_line_string) then empty - else "artifact_directory must be a non-empty, single-line string when supplied" end + if (.artifact_directory | relative_path) then empty + else "artifact_directory must be a non-empty relative path without parent traversal" end ] + if has("artifacts") then [ @@ -90,6 +90,6 @@ fi { printf 'release_catalog_key=%s\n' "$(jq -r '.release_catalog_key' <<<"${compact_config}")" - printf 'artifact_directory=%s\n' "$(jq -r '.artifact_directory // "."' <<<"${compact_config}")" + printf 'artifact_directory=%s\n' "$(jq -r '.artifact_directory' <<<"${compact_config}")" printf 'artifacts=%s\n' "$(jq -c '.artifacts // empty' <<<"${compact_config}")" } >>"${GITHUB_OUTPUT}" diff --git a/tests/release_catalog_config_test.sh b/tests/release_catalog_config_test.sh index 34842386..2226feb7 100755 --- a/tests/release_catalog_config_test.sh +++ b/tests/release_catalog_config_test.sh @@ -81,12 +81,20 @@ assert_invalid \ grep -F 'artifacts[0].path must be a non-empty relative path without parent traversal' "${temporary_directory}/malformed-artifact.error" >/dev/null grep -F 'artifacts[0].package_identity_file must be a non-empty relative path without parent traversal' "${temporary_directory}/malformed-artifact.error" >/dev/null -default_output="${temporary_directory}/default.output" -RELEASE_CATALOG_CONFIG='{"release_catalog_key":"conda:smoke"}' \ - GITHUB_OUTPUT="${default_output}" "${validator}" -grep -Fx 'release_catalog_key=conda:smoke' "${default_output}" -grep -Fx 'artifact_directory=.' "${default_output}" -grep -Fx 'artifacts=' "${default_output}" +assert_invalid \ + missing-artifact-directory \ + '{"release_catalog_key":"conda:smoke"}' \ + 'artifact_directory must be a non-empty relative path without parent traversal' + +assert_invalid \ + absolute-artifact-directory \ + '{"release_catalog_key":"conda:smoke","artifact_directory":"/tmp/artifacts"}' \ + 'artifact_directory must be a non-empty relative path without parent traversal' + +assert_invalid \ + parent-artifact-directory \ + '{"release_catalog_key":"conda:smoke","artifact_directory":"../artifacts"}' \ + 'artifact_directory must be a non-empty relative path without parent traversal' standard_output="${temporary_directory}/standard.output" RELEASE_CATALOG_CONFIG='{ diff --git a/tests/release_catalog_discovery_test.sh b/tests/release_catalog_discovery_test.sh index 2e31dde0..d287572f 100755 --- a/tests/release_catalog_discovery_test.sh +++ b/tests/release_catalog_discovery_test.sh @@ -12,11 +12,10 @@ GITHUB_RUN_ATTEMPT="1" GITHUB_RUN_ID="1234" GITHUB_WORKFLOW_REF="rapidsai/shared-actions/.github/workflows/pr.yml@refs/pull/136/merge" RELEASE_CATALOG_KEY="test:discovery" -RELEASE_ENTRIES_NAME="release-catalog-entries.json" RELEASE_SOURCE_ARTIFACT_NAME="discovery-test" -RAPIDS_SHA="0123456789012345678901234567890123456789" +RELEASE_SOURCE_SHA="0123456789012345678901234567890123456789" export GITHUB_REPOSITORY GITHUB_RUN_ATTEMPT GITHUB_RUN_ID GITHUB_WORKFLOW_REF -export RAPIDS_SHA RELEASE_CATALOG_KEY RELEASE_ENTRIES_NAME RELEASE_SOURCE_ARTIFACT_NAME +export RELEASE_SOURCE_SHA RELEASE_CATALOG_KEY RELEASE_SOURCE_ARTIFACT_NAME mkdir -p "${temporary_directory}/wheel/libkvikio_cu12-26.8.0.dist-info" printf '%s\n' \ diff --git a/tests/release_catalog_package_parsing_test.sh b/tests/release_catalog_package_parsing_test.sh index 8ad87969..726f55f1 100755 --- a/tests/release_catalog_package_parsing_test.sh +++ b/tests/release_catalog_package_parsing_test.sh @@ -13,11 +13,10 @@ GITHUB_RUN_ID="1234" GITHUB_WORKFLOW_REF="rapidsai/shared-actions/.github/workflows/pr.yml@refs/pull/136/merge" RELEASE_ARTIFACTS='' RELEASE_CATALOG_KEY="test:package-parsing" -RELEASE_ENTRIES_NAME="release-catalog-entries.json" RELEASE_SOURCE_ARTIFACT_NAME="package-parsing-test" -RAPIDS_SHA="0123456789012345678901234567890123456789" +RELEASE_SOURCE_SHA="0123456789012345678901234567890123456789" export GITHUB_REPOSITORY GITHUB_RUN_ATTEMPT GITHUB_RUN_ID GITHUB_WORKFLOW_REF -export RAPIDS_SHA RELEASE_ARTIFACTS RELEASE_CATALOG_KEY RELEASE_ENTRIES_NAME RELEASE_SOURCE_ARTIFACT_NAME +export RELEASE_SOURCE_SHA RELEASE_ARTIFACTS RELEASE_CATALOG_KEY RELEASE_SOURCE_ARTIFACT_NAME wheel_output_directory="${temporary_directory}/wheels" wheel_staging_directory="${temporary_directory}/wheel-staging" diff --git a/tests/release_catalog_test.sh b/tests/release_catalog_test.sh index 2014d0d7..9c617547 100755 --- a/tests/release_catalog_test.sh +++ b/tests/release_catalog_test.sh @@ -20,14 +20,13 @@ GITHUB_RUN_ID="1234" GITHUB_SHA="0123456789012345678901234567890123456789" GITHUB_WORKFLOW_REF="NVIDIA/cuvs/.github/workflows/build.yaml@refs/heads/release/26.08" RELEASE_ARTIFACTS="$(jq -cn '[{path: "cuvs-java-*.jar", package_identity_file: "cuvs-java-package-identity.json"}]')" -RELEASE_ENTRIES_NAME="release-catalog-entries.json" RELEASE_ARTIFACT_DIRECTORY="${bundle_directory}" RELEASE_SOURCE_ARTIFACT_NAME="cuvs-java-cuda12.9.1" -RAPIDS_SHA="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +RELEASE_SOURCE_SHA="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" RELEASE_CATALOG_KEY="maven:cuvs-java" export GITHUB_REPOSITORY GITHUB_RUN_ATTEMPT GITHUB_RUN_ID GITHUB_SHA GITHUB_WORKFLOW_REF -export RELEASE_ARTIFACTS RELEASE_ENTRIES_NAME RELEASE_ARTIFACT_DIRECTORY -export RAPIDS_SHA RELEASE_SOURCE_ARTIFACT_NAME RELEASE_CATALOG_KEY +export RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY +export RELEASE_SOURCE_SHA RELEASE_SOURCE_ARTIFACT_NAME RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" @@ -49,15 +48,10 @@ jq -e ' and (.entries[0].sbom | endswith(".sbom.cdx.json")) ' "${entries_path}" >/dev/null -# Optional signatures must not cause the S3 uploader to request a literal -# `null` file. This mirrors its declared-file selection without requiring AWS. -mapfile -t upload_paths < <(jq -r '["release-catalog-entries.json"] + ([.entries[] | .path, .sbom, .provenance, .signature? | strings] | unique) | .[]' "${entries_path}") +# Mirror the S3 uploader's declared-file selection without requiring AWS. +mapfile -t upload_paths < <(jq -r '["release-catalog-entries.json"] + ([.entries[] | .path, .sbom, .provenance] | unique) | .[]' "${entries_path}") artifact_sha256="$(sha256sum "${canonical_bundle_directory}/cuvs-java-26.08.0.jar" | awk '{print $1}')" test "${upload_paths[*]}" = "release-catalog-entries.json cuvs-java-26.08.0.jar release-evidence/cuvs-java-26.08.0.jar.${artifact_sha256}.provenance.json release-evidence/cuvs-java-26.08.0.jar.${artifact_sha256}.sbom.cdx.json" -if printf '%s\n' "${upload_paths[@]}" | grep -Fx null; then - echo "optional signature emitted a literal null upload path" >&2 - exit 1 -fi # Keep the checked-in example synchronized with the exact output owned by this # action. Only the generated SBOM timestamps are normalized. @@ -97,9 +91,9 @@ RELEASE_ARTIFACTS="$(jq -cn '[ ]')" RELEASE_ARTIFACT_DIRECTORY="${multiple_identity_directory}" RELEASE_SOURCE_ARTIFACT_NAME="multiple-identities" -RAPIDS_SHA="eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" +RELEASE_SOURCE_SHA="eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" RELEASE_CATALOG_KEY="archive:multiple" -export RAPIDS_SHA RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_CATALOG_KEY +export RELEASE_SOURCE_SHA RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" jq -e ' (.entries | length == 2) @@ -117,9 +111,9 @@ jq -n '{ecosystem: "conda", name: "kvikio", version: "26.08.00a32"}' \ RELEASE_ARTIFACTS="$(jq -cn '[{path: "linux-64/kvikio-*.bin", package_identity_file: "kvikio.identity.json"}]')" RELEASE_ARTIFACT_DIRECTORY="${generated_directory}" RELEASE_SOURCE_ARTIFACT_NAME="kvikio_conda_python_kvikio_x86_64_abi3_cu12" -RAPIDS_SHA="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +RELEASE_SOURCE_SHA="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" RELEASE_CATALOG_KEY="conda:kvikio" -export RAPIDS_SHA RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_CATALOG_KEY +export RELEASE_SOURCE_SHA RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" @@ -156,9 +150,9 @@ jq -n '{ecosystem: "wheel", name: "libkvikio-cu12", version: "26.8.0a32"}' \ RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.bin", package_identity_file: "libkvikio.identity.json"}]')" RELEASE_ARTIFACT_DIRECTORY="${wheel_directory}" RELEASE_SOURCE_ARTIFACT_NAME="kvikio_wheel_cpp_libkvikio_x86_64_cu12" -RAPIDS_SHA="cccccccccccccccccccccccccccccccccccccccc" +RELEASE_SOURCE_SHA="cccccccccccccccccccccccccccccccccccccccc" RELEASE_CATALOG_KEY="wheel:kvikio" -export RAPIDS_SHA RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_CATALOG_KEY +export RELEASE_SOURCE_SHA RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_CATALOG_KEY "${repository_root}/release-catalog/materialize.sh" @@ -179,9 +173,9 @@ jq -n \ RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz", package_identity_file: "bundle-package-identity.json"}]')" RELEASE_ARTIFACT_DIRECTORY="${missing_version_directory}" RELEASE_SOURCE_ARTIFACT_NAME="bundle-archive" -RAPIDS_SHA="dddddddddddddddddddddddddddddddddddddddd" +RELEASE_SOURCE_SHA="dddddddddddddddddddddddddddddddddddddddd" RELEASE_CATALOG_KEY="archive:bundle" -export RAPIDS_SHA RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_CATALOG_KEY +export RELEASE_SOURCE_SHA RELEASE_ARTIFACTS RELEASE_ARTIFACT_DIRECTORY RELEASE_SOURCE_ARTIFACT_NAME RELEASE_CATALOG_KEY if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/missing-version-error"; then echo "materialize.sh unexpectedly accepted a custom artifact without a package version" >&2 @@ -189,18 +183,18 @@ if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory} fi grep -Fx 'package identity for bundle.tar.gz must contain non-empty ecosystem, name, and version strings and only optional build or platform strings' "${temporary_directory}/missing-version-error" -if (unset RAPIDS_SHA; "${repository_root}/release-catalog/materialize.sh") 2>"${temporary_directory}/missing-rapids-sha-error"; then - echo "materialize.sh unexpectedly accepted a missing RAPIDS_SHA" >&2 +if (unset RELEASE_SOURCE_SHA; "${repository_root}/release-catalog/materialize.sh") 2>"${temporary_directory}/missing-source-sha-error"; then + echo "materialize.sh unexpectedly accepted a missing RELEASE_SOURCE_SHA" >&2 exit 1 fi -grep -Fx 'RAPIDS_SHA must be a non-empty string' "${temporary_directory}/missing-rapids-sha-error" +grep -Fx 'RELEASE_SOURCE_SHA must be a non-empty string' "${temporary_directory}/missing-source-sha-error" -if RAPIDS_SHA="not-a-git-object-id" \ +if RELEASE_SOURCE_SHA="not-a-git-object-id" \ "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/invalid-rapids-sha-error"; then - echo "materialize.sh unexpectedly accepted an invalid RAPIDS_SHA" >&2 + echo "materialize.sh unexpectedly accepted an invalid RELEASE_SOURCE_SHA" >&2 exit 1 fi -grep -Fx 'RAPIDS_SHA must be a 40- or 64-character hexadecimal Git object ID' \ +grep -Fx 'RELEASE_SOURCE_SHA must be a 40- or 64-character hexadecimal Git object ID' \ "${temporary_directory}/invalid-rapids-sha-error" assert_missing_build_context() { From 5008d0689cd66ec2f4911b12cdb731ea54b8acec Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 2 Sep 2026 09:48:04 -0500 Subject: [PATCH 34/36] Standardize release catalog error reporting --- ci/run_release_catalog_tests.sh | 1 + release-catalog/README.md | 4 ++ release-catalog/error.sh | 20 ++++++++ release-catalog/materialize.sh | 67 +++++++++++++------------ release-catalog/upload-s3.sh | 27 +++++----- release-catalog/validate-config.sh | 18 +++---- tests/release_catalog_discovery_test.sh | 6 +-- tests/release_catalog_error_test.sh | 21 ++++++++ tests/release_catalog_test.sh | 14 +++--- 9 files changed, 116 insertions(+), 62 deletions(-) create mode 100644 release-catalog/error.sh create mode 100755 tests/release_catalog_error_test.sh diff --git a/ci/run_release_catalog_tests.sh b/ci/run_release_catalog_tests.sh index e598d2ba..5df0f660 100755 --- a/ci/run_release_catalog_tests.sh +++ b/ci/run_release_catalog_tests.sh @@ -7,5 +7,6 @@ repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" "${repository_root}/tests/release_catalog_config_test.sh" "${repository_root}/tests/release_catalog_discovery_test.sh" +"${repository_root}/tests/release_catalog_error_test.sh" "${repository_root}/tests/release_catalog_package_parsing_test.sh" "${repository_root}/tests/release_catalog_test.sh" diff --git a/release-catalog/README.md b/release-catalog/README.md index 4b6ec67d..d3e4f5ec 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -94,6 +94,10 @@ dependency SBOM will add. That will be a new `sbom_kind` value and a new The action is used via `rapidsai/shared-actions/release-catalog`. +Configuration, materialization, and upload failures appear as GitHub Actions +error annotations. When the scripts run outside GitHub Actions, the same errors +are written to stderr with the responsible script name. + | Input | Required | Description | | ----- | -------- | ----------- | | `config` | yes | JSON object selecting the artifacts and their release catalog key. Schema: [`config.schema.json`](config.schema.json). Details below. | diff --git a/release-catalog/error.sh b/release-catalog/error.sh new file mode 100644 index 00000000..d62d0434 --- /dev/null +++ b/release-catalog/error.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +# Shared error rendering for release-catalog scripts. GitHub Actions receives a +# workflow-command annotation; local callers receive a conventional stderr line. + +release_catalog_error() { + local message="$1" + local script_name="${RELEASE_CATALOG_SCRIPT_NAME:-release-catalog}" + local title="${RELEASE_CATALOG_ERROR_TITLE:-Release catalog error}" + + if [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then + message="${message//'%'/'%25'}" + message="${message//$'\r'/'%0D'}" + message="${message//$'\n'/'%0A'}" + printf '::error title=%s::%s\n' "${title}" "${message}" >&2 + else + printf '[%s] Error: %s\n' "${script_name}" "${message}" >&2 + fi +} diff --git a/release-catalog/materialize.sh b/release-catalog/materialize.sh index 9b1ef669..7eb0baad 100755 --- a/release-catalog/materialize.sh +++ b/release-catalog/materialize.sh @@ -3,6 +3,11 @@ set -euo pipefail +export RELEASE_CATALOG_SCRIPT_NAME="materialize.sh" +export RELEASE_CATALOG_ERROR_TITLE="Release catalog materialization failed" +# shellcheck source=release-catalog/error.sh +source "$(dirname "${BASH_SOURCE[0]}")/error.sh" + # Record artifact metadata from completed build directory into a release catalog companion. # # Inputs come from release-catalog/action.yml. RELEASE_ARTIFACTS is @@ -20,7 +25,7 @@ require_nonempty() { local name="$1" local value="$2" if [[ -z "${value}" ]]; then - echo "${name} must be a non-empty string" >&2 + release_catalog_error "${name} must be a non-empty string" exit 1 fi } @@ -30,7 +35,7 @@ require_single_line() { local value="$2" require_nonempty "${name}" "${value}" if [[ "${value}" == *$'\n'* || "${value}" == *$'\r'* ]]; then - echo "${name} must be a single-line string" >&2 + release_catalog_error "${name} must be a single-line string" exit 1 fi } @@ -39,7 +44,7 @@ require_positive_integer() { local name="$1" local value="$2" if [[ ! "${value}" =~ ^[1-9][0-9]*$ ]]; then - echo "${name} must be a positive integer" >&2 + release_catalog_error "${name} must be a positive integer" exit 1 fi } @@ -54,7 +59,7 @@ require_single_line "RELEASE_SOURCE_ARTIFACT_NAME" "${RELEASE_SOURCE_ARTIFACT_NA source_sha="${RELEASE_SOURCE_SHA:-}" require_single_line "RELEASE_SOURCE_SHA" "${source_sha}" if [[ ! "${source_sha}" =~ ^[[:xdigit:]]{40}$ && ! "${source_sha}" =~ ^[[:xdigit:]]{64}$ ]]; then - echo "RELEASE_SOURCE_SHA must be a 40- or 64-character hexadecimal Git object ID" >&2 + release_catalog_error "RELEASE_SOURCE_SHA must be a 40- or 64-character hexadecimal Git object ID" exit 1 fi @@ -67,14 +72,14 @@ require_single_line "GITHUB_RUN_ATTEMPT" "${github_run_attempt}" require_single_line "GITHUB_RUN_ID" "${github_run_id}" require_single_line "GITHUB_WORKFLOW_REF" "${github_workflow_ref}" if [[ ! "${github_repository}" =~ ^[^/[:space:]]+/[^/[:space:]]+$ ]]; then - echo "GITHUB_REPOSITORY must have owner/repository form" >&2 + release_catalog_error "GITHUB_REPOSITORY must have owner/repository form" exit 1 fi require_positive_integer "GITHUB_RUN_ATTEMPT" "${github_run_attempt}" require_positive_integer "GITHUB_RUN_ID" "${github_run_id}" if [[ ! -d "${RELEASE_ARTIFACT_DIRECTORY}" ]]; then - echo "artifact-directory does not exist or is not a directory: ${RELEASE_ARTIFACT_DIRECTORY}" >&2 + release_catalog_error "artifact-directory does not exist or is not a directory: ${RELEASE_ARTIFACT_DIRECTORY}" exit 1 fi @@ -82,7 +87,7 @@ ensure_relative_pattern() { local field="$1" local pattern="$2" if [[ "${pattern}" == /* || "${pattern}" == */../* || "${pattern}" == ../* || "${pattern}" == *"/.." ]]; then - echo "${field} must be a relative path inside artifact-directory: ${pattern}" >&2 + release_catalog_error "${field} must be a relative path inside artifact-directory: ${pattern}" exit 1 fi } @@ -101,14 +106,14 @@ resolve_one_file() { matches+=("${match}") done < <(compgen -G "${artifact_directory}/${pattern}" || true) if [[ "${#matches[@]}" -ne 1 || ! -f "${matches[0]:-}" ]]; then - echo "${field} pattern must resolve to exactly one file: ${pattern}" >&2 + release_catalog_error "${field} pattern must resolve to exactly one file: ${pattern}" exit 1 fi local resolved resolved="$(realpath "${matches[0]}")" if [[ "${resolved}" != "${artifact_directory}"/* ]]; then - echo "${field} must resolve inside artifact-directory: ${pattern}" >&2 + release_catalog_error "${field} must resolve inside artifact-directory: ${pattern}" exit 1 fi printf '%s\n' "${resolved#"${artifact_directory}/"}" @@ -128,7 +133,7 @@ describe_wheel_package() { metadata_members+=("${metadata_member}") done < <(unzip -Z1 "${wheel_path}" | awk '/\.dist-info\/METADATA$/') if [[ "${#metadata_members[@]}" -ne 1 ]]; then - echo "wheel must contain exactly one .dist-info/METADATA file: ${relative_path}" >&2 + release_catalog_error "wheel must contain exactly one .dist-info/METADATA file: ${relative_path}" exit 1 fi @@ -137,7 +142,7 @@ describe_wheel_package() { package_name="$(awk 'tolower($0) ~ /^name:[[:space:]]*/ {sub(/^[^:]*:[[:space:]]*/, ""); sub(/\r$/, ""); print; exit}' <<<"${metadata}")" package_version="$(awk 'tolower($0) ~ /^version:[[:space:]]*/ {sub(/^[^:]*:[[:space:]]*/, ""); sub(/\r$/, ""); print; exit}' <<<"${metadata}")" if [[ -z "${package_name}" || -z "${package_version}" ]]; then - echo "wheel metadata must contain non-empty Name and Version fields: ${relative_path}" >&2 + release_catalog_error "wheel metadata must contain non-empty Name and Version fields: ${relative_path}" exit 1 fi @@ -157,7 +162,7 @@ describe_maven_jar_package() { metadata_members+=("${metadata_member}") done < <(unzip -Z1 "${jar_path}" | awk '$0 ~ "^META-INF/maven/[^/]+/[^/]+/pom[.]properties$"') if [[ "${#metadata_members[@]}" -ne 1 ]]; then - echo "Maven JAR must contain exactly one META-INF/maven///pom.properties file: ${relative_path}" >&2 + release_catalog_error "Maven JAR must contain exactly one META-INF/maven///pom.properties file: ${relative_path}" return 1 fi @@ -172,11 +177,11 @@ describe_maven_jar_package() { artifact_id="$(awk -F= '$1 == "artifactId" {sub(/^[^=]*=/, ""); sub(/\r$/, ""); print; exit}' <<<"${metadata}")" package_version="$(awk -F= '$1 == "version" {sub(/^[^=]*=/, ""); sub(/\r$/, ""); print; exit}' <<<"${metadata}")" if [[ -z "${group_id}" || -z "${artifact_id}" || -z "${package_version}" ]]; then - echo "Maven pom.properties must contain non-empty groupId, artifactId, and version fields: ${relative_path}" >&2 + release_catalog_error "Maven pom.properties must contain non-empty groupId, artifactId, and version fields: ${relative_path}" return 1 fi if [[ "${group_id}" != "${coordinate_group_id}" || "${artifact_id}" != "${coordinate_artifact_id}" ]]; then - echo "Maven pom.properties coordinates do not match its META-INF path: ${relative_path}" >&2 + release_catalog_error "Maven pom.properties coordinates do not match its META-INF path: ${relative_path}" return 1 fi @@ -197,7 +202,7 @@ describe_conda_package() { info_members+=("${info_member}") done < <(unzip -Z1 "${package_path}" | awk '/^info-.*\.tar\.zst$/') if [[ "${#info_members[@]}" -ne 1 ]]; then - echo ".conda package must contain exactly one info-*.tar.zst member: ${relative_path}" >&2 + release_catalog_error ".conda package must contain exactly one info-*.tar.zst member: ${relative_path}" exit 1 fi index_json="$(unzip -p "${package_path}" "${info_members[0]}" | zstd -dc | tar -xOf - info/index.json)" @@ -206,7 +211,7 @@ describe_conda_package() { index_json="$(tar -xOjf "${package_path}" info/index.json)" ;; *) - echo "unsupported Conda package extension: ${relative_path}" >&2 + release_catalog_error "unsupported Conda package extension: ${relative_path}" exit 1 ;; esac @@ -218,7 +223,7 @@ describe_conda_package() { and (.build | type == "string" and length > 0) and (.subdir | type == "string" and length > 0) ' <<<"${index_json}" >/dev/null; then - echo "Conda info/index.json must contain exact name, version, build, and subdir fields: ${relative_path}" >&2 + release_catalog_error "Conda info/index.json must contain exact name, version, build, and subdir fields: ${relative_path}" exit 1 fi @@ -260,12 +265,12 @@ prepare_artifacts() { done else if ! jq -e 'type == "array" and length > 0' <<<"${configured_artifacts}" >/dev/null; then - echo "release-artifacts must be a non-empty JSON array when supplied" >&2 + release_catalog_error "release-artifacts must be a non-empty JSON array when supplied" exit 1 fi while IFS= read -r descriptor; do if jq -e 'has("package")' <<<"${descriptor}" >/dev/null; then - echo "package identity must not be supplied inline: ${descriptor}" >&2 + release_catalog_error "package identity must not be supplied inline: ${descriptor}" return 1 fi primary_path="$(resolve_one_file path "$(jq -r '.path' <<<"${descriptor}")")" @@ -276,7 +281,7 @@ prepare_artifacts() { case "${primary_file}" in *.whl) if [[ -n "${identity_file}" ]]; then - echo "package_identity_file is not allowed when wheel identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" >&2 + release_catalog_error "package_identity_file is not allowed when wheel identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" exit 1 fi if ! package="$(describe_wheel_package "${primary_file}")"; then @@ -285,7 +290,7 @@ prepare_artifacts() { ;; *.conda) if [[ -n "${identity_file}" ]]; then - echo "package_identity_file is not allowed when Conda identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" >&2 + release_catalog_error "package_identity_file is not allowed when Conda identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" exit 1 fi if ! package="$(describe_conda_package "${primary_file}")"; then @@ -295,29 +300,29 @@ prepare_artifacts() { *.jar) if package="$(describe_maven_jar_package "${primary_file}" 2>/dev/null)"; then if [[ -n "${identity_file}" ]]; then - echo "package_identity_file is not allowed when Maven JAR identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" >&2 + release_catalog_error "package_identity_file is not allowed when Maven JAR identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" exit 1 fi elif [[ -z "${identity_file}" ]]; then - echo "JAR does not contain one unambiguous Maven package identity and requires package_identity_file: $(jq -r '.path' <<<"${descriptor}")" >&2 + release_catalog_error "JAR does not contain one unambiguous Maven package identity and requires package_identity_file: $(jq -r '.path' <<<"${descriptor}")" exit 1 fi ;; *.tar.bz2) if conda_package="$(describe_conda_package "${primary_file}" 2>/dev/null)"; then if [[ -n "${identity_file}" ]]; then - echo "package_identity_file is not allowed when Conda identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" >&2 + release_catalog_error "package_identity_file is not allowed when Conda identity can be extracted: $(jq -r '.path' <<<"${descriptor}")" exit 1 fi package="${conda_package}" elif [[ -z "${identity_file}" ]]; then - echo "artifact is not a valid Conda package and requires package_identity_file: $(jq -r '.path' <<<"${descriptor}")" >&2 + release_catalog_error "artifact is not a valid Conda package and requires package_identity_file: $(jq -r '.path' <<<"${descriptor}")" exit 1 fi ;; *) if [[ -z "${identity_file}" ]]; then - echo "artifact identity cannot be extracted; package_identity_file is required: $(jq -r '.path' <<<"${descriptor}")" >&2 + release_catalog_error "artifact identity cannot be extracted; package_identity_file is required: $(jq -r '.path' <<<"${descriptor}")" exit 1 fi ;; @@ -331,7 +336,7 @@ prepare_artifacts() { fi if [[ "$(jq 'length' <<<"${prepared_artifacts}")" -eq 0 ]]; then - echo "artifact-directory contains no detectable Conda, wheel, or Maven JAR artifacts; explicitly selected artifacts require package_identity_file when their identity cannot be parsed" >&2 + release_catalog_error "artifact-directory contains no detectable Conda, wheel, or Maven JAR artifacts; explicitly selected artifacts require package_identity_file when their identity cannot be parsed" exit 1 fi printf '%s\n' "${prepared_artifacts}" @@ -459,7 +464,7 @@ while IFS= read -r descriptor; do and ((.package_identity_file // "") | type == "string") and ([has("package"), has("package_identity_file")] | map(select(.)) | length == 1) ' <<<"${descriptor}" >/dev/null; then - echo "release artifact descriptor must contain path and exactly one package identity source: ${descriptor}" >&2 + release_catalog_error "release artifact descriptor must contain path and exactly one package identity source: ${descriptor}" exit 1 fi @@ -468,14 +473,14 @@ while IFS= read -r descriptor; do if [[ -n "${package_identity_file}" ]]; then package_identity_path="$(resolve_one_file package_identity_file "${package_identity_file}")" if ! package="$(jq -ce . "${artifact_directory}/${package_identity_path}" 2>/dev/null)"; then - echo "package_identity_file must contain valid JSON: ${package_identity_file}" >&2 + release_catalog_error "package_identity_file must contain valid JSON: ${package_identity_file}" exit 1 fi else package="$(jq -c '.package' <<<"${descriptor}")" fi if ! validate_package_identity <<<"${package}"; then - echo "package identity for ${primary_path} must contain non-empty ecosystem, name, and version strings and only optional build or platform strings" >&2 + release_catalog_error "package identity for ${primary_path} must contain non-empty ecosystem, name, and version strings and only optional build or platform strings" exit 1 fi @@ -499,7 +504,7 @@ while IFS= read -r descriptor; do done < <(jq -c '.[]' <<<"${artifacts}") if ! jq -e '.entries as $items | ($items | map([.release_catalog_key, .path] | join("\u0000")) | unique | length) == ($items | length)' "${temporary_manifest}" >/dev/null; then - echo "release-artifacts contains duplicate release_catalog_key/path entries" >&2 + release_catalog_error "release-artifacts contains duplicate release_catalog_key/path entries" exit 1 fi diff --git a/release-catalog/upload-s3.sh b/release-catalog/upload-s3.sh index 23c0dad3..503eb2c9 100755 --- a/release-catalog/upload-s3.sh +++ b/release-catalog/upload-s3.sh @@ -1,17 +1,20 @@ #!/usr/bin/env bash # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# Upload only the files declared by the generated catalog document. GitHub -# Actions retains logs and this receipt, but never a second release-artifact -# copy. Conditional S3 writes make reruns safe: an existing object is accepted -# only when its checksum exactly matches the byte about to be uploaded. +# Upload the files declared by the generated catalog document. Uploading +# existing filenames with different content will raise an error. set -euo pipefail +export RELEASE_CATALOG_SCRIPT_NAME="upload-s3.sh" +export RELEASE_CATALOG_ERROR_TITLE="Release catalog upload failed" +# shellcheck source=release-catalog/error.sh +source "$(dirname "${BASH_SOURCE[0]}")/error.sh" + require_value() { local name="$1" local value="$2" if [[ -z "${value}" || "${value}" == *$'\n'* || "${value}" == *$'\r'* ]]; then - echo "${name} must be a non-empty single-line string" >&2 + release_catalog_error "${name} must be a non-empty single-line string" exit 1 fi } @@ -26,22 +29,22 @@ for value in RELEASE_ARTIFACT_DIRECTORY RELEASE_CANDIDATE_BUCKET RELEASE_CANDIDA done RELEASE_CANDIDATE_PREFIX="${RELEASE_CANDIDATE_PREFIX:-}" if [[ ! "${RELEASE_CANDIDATE_TRAIN_SHA256}" =~ ^[[:xdigit:]]{64}$ ]]; then - echo "RELEASE_CANDIDATE_TRAIN_SHA256 must be a SHA-256 hex digest" >&2 + release_catalog_error "RELEASE_CANDIDATE_TRAIN_SHA256 must be a SHA-256 hex digest" exit 1 fi if [[ "${RELEASE_SOURCE_ARTIFACT_NAME}" == */* || "${RELEASE_SOURCE_ARTIFACT_NAME}" == *".."* ]]; then - echo "RELEASE_SOURCE_ARTIFACT_NAME must be a plain name without path separators" >&2 + release_catalog_error "RELEASE_SOURCE_ARTIFACT_NAME must be a plain name without path separators" exit 1 fi if ! safe_prefix "${RELEASE_CANDIDATE_PREFIX}"; then - echo "RELEASE_CANDIDATE_PREFIX must be a safe, relative S3 prefix" >&2 + release_catalog_error "RELEASE_CANDIDATE_PREFIX must be a safe, relative S3 prefix" exit 1 fi companion_root="$(realpath "${RELEASE_ARTIFACT_DIRECTORY}")" entries_path="${companion_root}/release-catalog-entries.json" if [[ ! -f "${entries_path}" ]]; then - echo "release catalog entries are missing: ${entries_path}" >&2 + release_catalog_error "release catalog entries are missing: ${entries_path}" exit 1 fi @@ -71,7 +74,7 @@ upload_one() { if aws s3api head-object --bucket "${RELEASE_CANDIDATE_BUCKET}" --key "${object_key}" --checksum-mode ENABLED >"${head_metadata}" 2>/dev/null; then if [[ "$(jq -r '.ChecksumSHA256 // empty' "${head_metadata}")" != "${checksum}" ]]; then - echo "existing candidate object has different bytes: ${object_key}" >&2 + release_catalog_error "Filename already exists remotely and has different content from current file: ${object_key}" exit 1 fi return @@ -80,7 +83,7 @@ upload_one() { if ! aws s3api put-object --bucket "${RELEASE_CANDIDATE_BUCKET}" --key "${object_key}" --body "${local_path}" --checksum-algorithm SHA256 --checksum-sha256 "${checksum}" --if-none-match '*' --tagging 'release-candidate-status=candidate' >/dev/null 2>&1; then aws s3api head-object --bucket "${RELEASE_CANDIDATE_BUCKET}" --key "${object_key}" --checksum-mode ENABLED >"${head_metadata}" if [[ "$(jq -r '.ChecksumSHA256 // empty' "${head_metadata}")" != "${checksum}" ]]; then - echo "candidate object could not be written immutably: ${object_key}" >&2 + release_catalog_error "candidate object could not be written immutably: ${object_key}" exit 1 fi fi @@ -88,7 +91,7 @@ upload_one() { for relative_path in "${companion_paths[@]}"; do if ! safe_relative_path "${relative_path}" || [[ ! -f "${companion_root}/${relative_path}" ]]; then - echo "catalog declares a missing or unsafe candidate file: ${relative_path}" >&2 + release_catalog_error "catalog declares a missing or unsafe candidate file: ${relative_path}" exit 1 fi upload_one "${relative_path}" diff --git a/release-catalog/validate-config.sh b/release-catalog/validate-config.sh index 43b7ac31..56545dca 100755 --- a/release-catalog/validate-config.sh +++ b/release-catalog/validate-config.sh @@ -3,25 +3,25 @@ set -euo pipefail +export RELEASE_CATALOG_SCRIPT_NAME="validate-config.sh" +export RELEASE_CATALOG_ERROR_TITLE="Invalid release catalog configuration" +# shellcheck source=release-catalog/error.sh +source "$(dirname "${BASH_SOURCE[0]}")/error.sh" + # Validate the public JSON input before materialize.sh reads any build output. # Structural errors are emitted as GitHub Actions annotations so callers see # every invalid field in one run. Validated values are written to GITHUB_OUTPUT # for the dispatch action's later materialization step. -emit_error() { - local message="$1" - printf '::error title=Invalid release catalog configuration::%s\n' "${message}" >&2 -} - config="${RELEASE_CATALOG_CONFIG:-}" if [[ -z "${config}" ]]; then - emit_error "release catalog configuration must be a non-empty JSON object" + release_catalog_error "release catalog configuration must be a non-empty JSON object" exit 1 fi if ! compact_config="$(jq -ce . <<<"${config}" 2>/dev/null)"; then parse_error="$(jq -ce . <<<"${config}" 2>&1 || true)" - emit_error "release catalog configuration must be valid JSON: ${parse_error}" + release_catalog_error "release catalog configuration must be valid JSON: ${parse_error}" exit 1 fi @@ -78,13 +78,13 @@ validation_errors="$(jq -r ' if [[ -n "${validation_errors}" ]]; then while IFS= read -r validation_error; do - emit_error "${validation_error}" + release_catalog_error "${validation_error}" done <<<"${validation_errors}" exit 1 fi if [[ -z "${GITHUB_OUTPUT:-}" ]]; then - emit_error "GITHUB_OUTPUT is required" + release_catalog_error "GITHUB_OUTPUT is required" exit 1 fi diff --git a/tests/release_catalog_discovery_test.sh b/tests/release_catalog_discovery_test.sh index d287572f..a4b07e51 100755 --- a/tests/release_catalog_discovery_test.sh +++ b/tests/release_catalog_discovery_test.sh @@ -116,7 +116,7 @@ if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory} echo "materialize.sh unexpectedly accepted an artifact directory without detectable artifacts" >&2 exit 1 fi -grep -Fx 'artifact-directory contains no detectable Conda, wheel, or Maven JAR artifacts; explicitly selected artifacts require package_identity_file when their identity cannot be parsed' "${temporary_directory}/empty-error" +grep -Fx '[materialize.sh] Error: artifact-directory contains no detectable Conda, wheel, or Maven JAR artifacts; explicitly selected artifacts require package_identity_file when their identity cannot be parsed' "${temporary_directory}/empty-error" RELEASE_ARTIFACT_DIRECTORY="${temporary_directory}" RELEASE_ARTIFACTS="$(jq -cn '[{path: "bundle.tar.gz"}]')" @@ -125,7 +125,7 @@ if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory} echo "materialize.sh unexpectedly accepted an unsupported artifact without identity" >&2 exit 1 fi -grep -Fx 'artifact identity cannot be extracted; package_identity_file is required: bundle.tar.gz' "${temporary_directory}/unsupported-error" +grep -Fx '[materialize.sh] Error: artifact identity cannot be extracted; package_identity_file is required: bundle.tar.gz' "${temporary_directory}/unsupported-error" RELEASE_ARTIFACTS="$(jq -cn '[{path: "libkvikio_cu12-*.whl", package_identity_file: "release-package-identity.json"}]')" export RELEASE_ARTIFACTS @@ -133,4 +133,4 @@ if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory} echo "materialize.sh unexpectedly accepted an identity file for a parseable artifact" >&2 exit 1 fi -grep -Fx 'package_identity_file is not allowed when wheel identity can be extracted: libkvikio_cu12-*.whl' "${temporary_directory}/conflicting-identity-error" +grep -Fx '[materialize.sh] Error: package_identity_file is not allowed when wheel identity can be extracted: libkvikio_cu12-*.whl' "${temporary_directory}/conflicting-identity-error" diff --git a/tests/release_catalog_error_test.sh b/tests/release_catalog_error_test.sh new file mode 100755 index 00000000..df48b9f6 --- /dev/null +++ b/tests/release_catalog_error_test.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +set -euo pipefail + +repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +temporary_directory="$(mktemp -d)" +trap 'rm -rf "${temporary_directory}"' EXIT + +# shellcheck source=release-catalog/error.sh +source "${repository_root}/release-catalog/error.sh" + +RELEASE_CATALOG_SCRIPT_NAME="materialize.sh" +release_catalog_error "invalid artifact" 2>"${temporary_directory}/local-error" +grep -Fx '[materialize.sh] Error: invalid artifact' "${temporary_directory}/local-error" + +GITHUB_ACTIONS="true" +RELEASE_CATALOG_ERROR_TITLE="Release catalog upload failed" +release_catalog_error $'invalid 100%\r\nartifact' 2>"${temporary_directory}/github-error" +grep -Fx '::error title=Release catalog upload failed::invalid 100%25%0D%0Aartifact' \ + "${temporary_directory}/github-error" diff --git a/tests/release_catalog_test.sh b/tests/release_catalog_test.sh index 9c617547..7c92474b 100755 --- a/tests/release_catalog_test.sh +++ b/tests/release_catalog_test.sh @@ -181,20 +181,20 @@ if "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory} echo "materialize.sh unexpectedly accepted a custom artifact without a package version" >&2 exit 1 fi -grep -Fx 'package identity for bundle.tar.gz must contain non-empty ecosystem, name, and version strings and only optional build or platform strings' "${temporary_directory}/missing-version-error" +grep -Fx '[materialize.sh] Error: package identity for bundle.tar.gz must contain non-empty ecosystem, name, and version strings and only optional build or platform strings' "${temporary_directory}/missing-version-error" if (unset RELEASE_SOURCE_SHA; "${repository_root}/release-catalog/materialize.sh") 2>"${temporary_directory}/missing-source-sha-error"; then echo "materialize.sh unexpectedly accepted a missing RELEASE_SOURCE_SHA" >&2 exit 1 fi -grep -Fx 'RELEASE_SOURCE_SHA must be a non-empty string' "${temporary_directory}/missing-source-sha-error" +grep -Fx '[materialize.sh] Error: RELEASE_SOURCE_SHA must be a non-empty string' "${temporary_directory}/missing-source-sha-error" if RELEASE_SOURCE_SHA="not-a-git-object-id" \ "${repository_root}/release-catalog/materialize.sh" 2>"${temporary_directory}/invalid-rapids-sha-error"; then echo "materialize.sh unexpectedly accepted an invalid RELEASE_SOURCE_SHA" >&2 exit 1 fi -grep -Fx 'RELEASE_SOURCE_SHA must be a 40- or 64-character hexadecimal Git object ID' \ +grep -Fx '[materialize.sh] Error: RELEASE_SOURCE_SHA must be a 40- or 64-character hexadecimal Git object ID' \ "${temporary_directory}/invalid-rapids-sha-error" assert_missing_build_context() { @@ -204,7 +204,7 @@ assert_missing_build_context() { echo "materialize.sh unexpectedly accepted a missing ${variable_name}" >&2 exit 1 fi - grep -Fx "${variable_name} must be a non-empty string" "${error_file}" + grep -Fx "[materialize.sh] Error: ${variable_name} must be a non-empty string" "${error_file}" } assert_missing_build_context GITHUB_REPOSITORY @@ -217,7 +217,7 @@ if GITHUB_REPOSITORY="cuvs" \ echo "materialize.sh unexpectedly accepted an invalid GITHUB_REPOSITORY" >&2 exit 1 fi -grep -Fx 'GITHUB_REPOSITORY must have owner/repository form' \ +grep -Fx '[materialize.sh] Error: GITHUB_REPOSITORY must have owner/repository form' \ "${temporary_directory}/invalid-repository.error" if GITHUB_RUN_ATTEMPT="0" \ @@ -225,7 +225,7 @@ if GITHUB_RUN_ATTEMPT="0" \ echo "materialize.sh unexpectedly accepted an invalid GITHUB_RUN_ATTEMPT" >&2 exit 1 fi -grep -Fx 'GITHUB_RUN_ATTEMPT must be a positive integer' \ +grep -Fx '[materialize.sh] Error: GITHUB_RUN_ATTEMPT must be a positive integer' \ "${temporary_directory}/invalid-run-attempt.error" if GITHUB_RUN_ID="not-an-integer" \ @@ -233,5 +233,5 @@ if GITHUB_RUN_ID="not-an-integer" \ echo "materialize.sh unexpectedly accepted an invalid GITHUB_RUN_ID" >&2 exit 1 fi -grep -Fx 'GITHUB_RUN_ID must be a positive integer' \ +grep -Fx '[materialize.sh] Error: GITHUB_RUN_ID must be a positive integer' \ "${temporary_directory}/invalid-run-id.error" From 2bd36f0db6c8c553a14e8886ea62ce7b908c6a2c Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 2 Sep 2026 09:57:49 -0500 Subject: [PATCH 35/36] note that a companion is a folder and add platform link --- release-catalog/README.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/release-catalog/README.md b/release-catalog/README.md index d3e4f5ec..16a5ffa4 100644 --- a/release-catalog/README.md +++ b/release-catalog/README.md @@ -15,9 +15,9 @@ those artifacts today: can make security scans more accurate. This action solves both by having each build job write a small **companion** -next to its artifacts: an inventory of the files it produced, plus provenance -and SBOM documents for each one. The companion never modifies the artifacts -themselves. +next to its artifacts: a folder with an inventory of the files that the job +produced, plus provenance and SBOM documents for each one. The companion never +modifies the artifacts themselves. ## Lifecycle @@ -26,9 +26,10 @@ themselves. the artifact directory. 3. The artifacts and their companion are uploaded to a private S3 bucket (`upload-to-s3: true`). -4. The RAPIDS release platform reads the companions from every job in a - release train, merges them into the **release catalog**, and uses the - catalog to assemble and test a **release candidate**. +4. The [RAPIDS release platform (in + development)](https://github.com/rapidsai/release-scripts/pull/113) reads the + companions from every job in a release train, merges them into the **release + catalog**, and uses the catalog to assemble and test a **release candidate**. "Release candidate" here means a set of artifacts that can be deployed to create a release, not a release-candidate version of one artifact. From 421d41edf46289e251aff12550bc927934b12f86 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 2 Sep 2026 10:38:07 -0500 Subject: [PATCH 36/36] Make release catalog error tests CI-safe --- tests/release_catalog_discovery_test.sh | 4 ++++ tests/release_catalog_error_test.sh | 13 +++++++------ tests/release_catalog_test.sh | 4 ++++ 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/tests/release_catalog_discovery_test.sh b/tests/release_catalog_discovery_test.sh index a4b07e51..72728211 100755 --- a/tests/release_catalog_discovery_test.sh +++ b/tests/release_catalog_discovery_test.sh @@ -3,6 +3,10 @@ set -euo pipefail +# Error output assertions below exercise the local rendering path. The +# GitHub-specific rendering path is covered by release_catalog_error_test.sh. +export GITHUB_ACTIONS="" + repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" temporary_directory="$(mktemp -d)" trap 'rm -rf "${temporary_directory}"' EXIT diff --git a/tests/release_catalog_error_test.sh b/tests/release_catalog_error_test.sh index df48b9f6..1539f5b6 100755 --- a/tests/release_catalog_error_test.sh +++ b/tests/release_catalog_error_test.sh @@ -7,15 +7,16 @@ repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" temporary_directory="$(mktemp -d)" trap 'rm -rf "${temporary_directory}"' EXIT -# shellcheck source=release-catalog/error.sh +# shellcheck source=/dev/null source "${repository_root}/release-catalog/error.sh" -RELEASE_CATALOG_SCRIPT_NAME="materialize.sh" +export RELEASE_CATALOG_SCRIPT_NAME="materialize.sh" +export GITHUB_ACTIONS="" release_catalog_error "invalid artifact" 2>"${temporary_directory}/local-error" -grep -Fx '[materialize.sh] Error: invalid artifact' "${temporary_directory}/local-error" +grep -Fxq '[materialize.sh] Error: invalid artifact' "${temporary_directory}/local-error" -GITHUB_ACTIONS="true" -RELEASE_CATALOG_ERROR_TITLE="Release catalog upload failed" +export GITHUB_ACTIONS="true" +export RELEASE_CATALOG_ERROR_TITLE="Release catalog upload failed" release_catalog_error $'invalid 100%\r\nartifact' 2>"${temporary_directory}/github-error" -grep -Fx '::error title=Release catalog upload failed::invalid 100%25%0D%0Aartifact' \ +grep -Fxq '::error title=Release catalog upload failed::invalid 100%25%0D%0Aartifact' \ "${temporary_directory}/github-error" diff --git a/tests/release_catalog_test.sh b/tests/release_catalog_test.sh index 7c92474b..f77d2faa 100755 --- a/tests/release_catalog_test.sh +++ b/tests/release_catalog_test.sh @@ -3,6 +3,10 @@ set -euo pipefail +# Error output assertions below exercise the local rendering path. The +# GitHub-specific rendering path is covered by release_catalog_error_test.sh. +export GITHUB_ACTIONS="" + repository_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" temporary_directory="$(mktemp -d)" trap 'rm -rf "${temporary_directory}"' EXIT