diff --git a/.github/workflows/release_candidate.yaml b/.github/workflows/release_candidate.yaml new file mode 100644 index 00000000..aaba9ec9 --- /dev/null +++ b/.github/workflows/release_candidate.yaml @@ -0,0 +1,135 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Release Candidate + +on: + pull_request: + paths: + - ".github/workflows/release_candidate.yaml" + - ".github/.rat-excludes" + - "CMakeLists.txt" + - "LICENSE" + - "NOTICE" + - "docs/source/conf.py" + - "docs/source/_static/versions.json" + - "scripts/releasing/**" + push: + tags: + - "v*-rc*" + +concurrency: + group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + archive: + name: Create and audit source archive + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - name: Checkout source + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Run release tool tests + run: python3 -m unittest discover -s scripts/releasing/tests -v + + - name: Resolve release version + shell: bash + run: | + version=$( + awk '$1 == "VERSION" && $2 ~ /^[0-9]+\.[0-9]+\.[0-9]+$/ { + print $2 + exit + }' CMakeLists.txt + ) + if [[ "${GITHUB_REF_TYPE}" == "tag" ]]; then + tag_version=${GITHUB_REF_NAME#v} + tag_version=${tag_version%-rc*} + [[ "${tag_version}" == "${version}" ]] + fi + scripts/releasing/bump_version.py --check "${version}" + echo "RELEASE_VERSION=${version}" >> "${GITHUB_ENV}" + + - name: Create source archive + run: | + scripts/releasing/create_source_release.sh \ + --version "${RELEASE_VERSION}" \ + --git-ref HEAD \ + --output-dir release/ci + + - name: Audit source archive + run: | + scripts/releasing/verify_release_candidate.sh \ + --allow-unsigned \ + --skip-build \ + "release/ci/apache-paimon-cpp-${RELEASE_VERSION}-src.tgz" + + - name: Upload source archive + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: source-archive + path: release/ci/ + if-no-files-found: error + + verify: + name: Verify source archive (${{ matrix.compiler }}) + if: github.ref_type == 'tag' + needs: archive + runs-on: ubuntu-24.04 + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: + compiler: + - gcc-14 + - clang + include: + - compiler: gcc-14 + cc: gcc-14 + cxx: g++-14 + - compiler: clang + cc: clang + cxx: clang++ + steps: + - name: Checkout source + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + persist-credentials: false + + - name: Download source archive + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: source-archive + path: release/ci + + - name: Verify, build, test, and install + shell: bash + env: + CC: ${{ matrix.cc }} + CXX: ${{ matrix.cxx }} + run: | + artifact=$(find release/ci -name 'apache-paimon-cpp-*-src.tgz' -print -quit) + scripts/releasing/verify_release_candidate.sh \ + --allow-unsigned \ + --skip-rat \ + "${artifact}" diff --git a/ci/scripts/build_paimon.sh b/ci/scripts/build_paimon.sh index 16e6fe04..ecc83a75 100755 --- a/ci/scripts/build_paimon.sh +++ b/ci/scripts/build_paimon.sh @@ -21,8 +21,19 @@ source_dir=${1} enable_sanitizer=${2:-false} check_clang_tidy=${3:-false} build_type=${4:-Debug} +install_smoke=${5:-false} build_dir="${source_dir}/build" +if [[ -n "${PAIMON_BUILD_JOBS:-}" ]]; then + build_jobs="${PAIMON_BUILD_JOBS}" +elif command -v nproc >/dev/null 2>&1; then + build_jobs=$(nproc) +elif command -v sysctl >/dev/null 2>&1; then + build_jobs=$(sysctl -n hw.ncpu) +else + build_jobs=4 +fi + # Display ccache status if available if command -v ccache &> /dev/null; then echo "=== ccache found: $(ccache --version | head -1) ===" @@ -57,13 +68,40 @@ if [[ "${enable_sanitizer}" == "true" ]]; then fi cmake "${CMAKE_ARGS[@]}" "${source_dir}" -cmake --build . -- -j "$(nproc)" -ctest --output-on-failure -j "$(nproc)" +cmake --build . -- -j "${build_jobs}" +ctest --output-on-failure -j "${build_jobs}" if [[ "${check_clang_tidy}" == "true" ]]; then cmake --build . --target check-clang-tidy fi +if [[ "${install_smoke}" == "true" ]]; then + install_dir="${source_dir}/install-test" + smoke_build_dir="${source_dir}/build-install-smoke" + rm -rf "${install_dir}" "${smoke_build_dir}" + + cmake --install . --prefix "${install_dir}" + cmake -G Ninja \ + -S "${source_dir}/scripts/releasing/install_smoke" \ + -B "${smoke_build_dir}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH="${install_dir};${build_dir}/arrow_ep-install" + cmake --build "${smoke_build_dir}" -- -j "${build_jobs}" + + runtime_library_path="${install_dir}/lib:${install_dir}/lib64" + runtime_library_path+=":${build_dir}/arrow_ep-install/lib" + runtime_library_path+=":${build_dir}/arrow_ep-install/lib64" + if [[ "$(uname -s)" == "Darwin" ]]; then + cmake -E env \ + "DYLD_LIBRARY_PATH=${runtime_library_path}" \ + "${smoke_build_dir}/paimon_install_smoke" + else + cmake -E env \ + "LD_LIBRARY_PATH=${runtime_library_path}" \ + "${smoke_build_dir}/paimon_install_smoke" + fi +fi + # Print ccache statistics after build if command -v ccache &> /dev/null; then echo "=== ccache statistics after build ===" @@ -73,3 +111,6 @@ fi popd rm -rf "${build_dir}" +if [[ "${install_smoke}" == "true" ]]; then + rm -rf "${install_dir}" "${smoke_build_dir}" +fi diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 385b482d..05bdee19 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -37,7 +37,8 @@ if(TARGET Arrow::arrow_shared) elseif(TARGET Arrow::arrow_static) set(PAIMON_EXAMPLE_ARROW_TARGET Arrow::arrow_static) else() - message(FATAL_ERROR "Neither Arrow::arrow_shared nor Arrow::arrow_static is available") + message(FATAL_ERROR "Neither Arrow::arrow_shared nor Arrow::arrow_static is available" + ) endif() add_executable(read_write_demo read_write_demo.cpp) diff --git a/scripts/releasing/README.md b/scripts/releasing/README.md index bf9b7ddf..d7757a9e 100644 --- a/scripts/releasing/README.md +++ b/scripts/releasing/README.md @@ -20,29 +20,91 @@ # Apache Paimon C++ release scripts These scripts create and verify the source artifact voted on by the Apache Paimon -PMC. They do not publish artifacts, create tags, or move files between Apache -distribution repositories. +PMC, stage a release candidate in the ASF distribution repository, and publish +an approved candidate. Run release operations from a clean checkout of +`apache/paimon-cpp`, not from a fork. -## Create a release candidate +The source archive is the official Apache release. Git tags, GitHub Releases, +and binary packages are supplementary. + +## Prerequisites + +Before starting a release: + +- obtain an ASF code-signing key, publish it through the ASF account system, + and make sure it is present in + [Paimon KEYS](https://downloads.apache.org/paimon/KEYS); +- install `git`, `gpg`, `svn`, `gh`, `python3`, `curl` or `wget`, Java, CMake, + Ninja, and the toolchain needed by `ci/scripts/build_paimon.sh` (Java is + required by Apache RAT); +- authenticate `gh` with access to read GitHub Actions runs in + `apache/paimon-cpp`; +- make sure the Apache Git remote points directly to + `apache/paimon-cpp`; +- prepare and merge a release-preparation PR that updates the release notes and + all version metadata, and passes the normal and release-candidate workflows. -Create and push a signed RC tag before creating the source artifact: +For example, update all version locations and review the diff: ```bash -git tag -s release-0.2.3-rc1 -m "Apache Paimon C++ 0.2.3 RC1" -git push upstream release-0.2.3-rc1 +scripts/releasing/bump_version.py 0.2.2 0.2.3 +scripts/releasing/bump_version.py --check 0.2.3 ``` -Create the source artifact, SHA-512 checksum, and detached OpenPGP signature: +### Signing key setup and security + +Complete signing-key setup well before creating the first release candidate: + +- new signing keys must use RSA with at least 2048 bits; ASF recommends 4096 + bits for new keys; +- publish the public key to the global public keyserver network; +- append the public key to `dist/release/paimon/KEYS`. Never remove historical + keys because they are required to verify archived releases; +- wait until the updated key is visible from + `https://downloads.apache.org/paimon/KEYS` before creating an RC; and +- never store the private key or create release signatures on ASF machines. + Sign only on a secure machine controlled by the release manager. + +By default, updating `dist/release/paimon/KEYS` requires PMC membership. A +non-PMC release manager should ask a PMC member to add the key. + +See the ASF +[release-signing](https://infra.apache.org/release-signing.html) and +[release-distribution](https://infra.apache.org/release-distribution.html) +policies for the complete requirements. + +The release scripts use `vVERSION-rcRC` for release-candidate tags and +`vVERSION` for the final release tag. For example, the first 0.2.3 candidate is +`v0.2.3-rc1`. + +## Create a release candidate + +Start from the exact clean commit approved for the candidate. Before publishing, +the wrapper fetches the release branch and requires `HEAD` to be contained in +its current history. It then creates and verifies a signed RC tag, creates the +source archive and its checksum/signature, performs the full source-release +verification, pushes the tag, waits for the tag-triggered release-candidate +workflow to succeed, and imports the artifacts into ASF `dist/dev`: ```bash -scripts/releasing/create_source_release.sh \ +scripts/releasing/release_rc.sh \ --version 0.2.3 \ - --git-ref release-0.2.3-rc1 \ - --output-dir release/0.2.3-rc1 \ - --signing-key ASF_GPG_KEY_ID + --rc 1 \ + --signing-key ASF_GPG_KEY_ID \ + --remote upstream ``` -The output files are: +The release branch defaults to `main`; use `--release-branch NAME` for a +maintenance release from another Apache branch. + +Use `--prepare-only` to create and verify artifacts without pushing the tag or +uploading to ASF infrastructure. This local-only mode does not require `HEAD` +to match the remote release branch. Use `--dry-run` to print identifiers +without making changes. A resumed run reuses an existing local tag or complete +artifact set only after validating it. A prepare-only run does not print a vote +email and must not be used to start a vote. + +The candidate directory contains: ```text apache-paimon-cpp-0.2.3-src.tgz @@ -50,32 +112,114 @@ apache-paimon-cpp-0.2.3-src.tgz.asc apache-paimon-cpp-0.2.3-src.tgz.sha512 ``` -Existing files are never overwritten. A changed candidate must use a new RC -directory and a new vote. +The wrapper prints a vote-email template. Send it to `dev@paimon.apache.org`. +Keep the vote open for at least 72 hours. An Apache release vote requires at +least three binding `+1` votes and more binding `+1` than binding `-1` votes. +If the vote has not met these requirements after 72 hours, do not publish the +release; either extend the vote or close it as unsuccessful. + +Before casting a binding `+1`, a PMC member must download the signed source +artifact onto hardware they control, verify its signature and ASF policy +compliance, compile it as provided, and test it on their platform. CI results +do not replace this voter responsibility. + +After closing the vote, send a result email as a reply to the vote thread. Use +the subject +`[RESULT][VOTE][C++] Release Apache Paimon C++ VERSION RCNUMBER`, state whether +the vote passed, list binding and non-binding votes and voters separately, and +include the archived vote-thread link. + +If a candidate changes for any reason, cancel or close its vote, fix the +release-preparation branch, increment the RC number, create a new signed tag +and artifacts, and start a new vote. Never overwrite an existing candidate. +After its vote is closed, a failed or superseded candidate may be removed from +`dist/dev`. ## Verify a release candidate -Download the source artifact, its `.asc` and `.sha512` files, and the Paimon -`KEYS` file. Import `KEYS` into a temporary or dedicated GPG keyring, then run: +Voters can download and verify an ASF-staged candidate in one command: ```bash -RAT_JAR=/path/to/apache-rat-0.16.1.jar \ - scripts/releasing/verify_release_candidate.sh \ +scripts/releasing/verify_release_candidate.sh --version 0.2.3 --rc 1 +``` + +To verify files that were downloaded separately, use an explicitly downloaded +KEYS file so signature verification runs in an isolated GPG home: + +```bash +scripts/releasing/verify_release_candidate.sh \ + --keys-file /path/to/paimon-KEYS \ apache-paimon-cpp-0.2.3-src.tgz ``` The verifier checks: - the SHA-512 checksum and detached OpenPGP signature; -- archive paths and the single `paimon-cpp-0.2.3/` root directory; +- archive path safety, portable filename collisions, file types, permissions, + and the single `paimon-cpp-0.2.3/` root directory; - required `LICENSE`, `NOTICE`, build, and documentation files; - the CMake and documentation versions; -- absence of common compiled artifact types; +- absence of compiled artifacts by filename and file magic; - Apache RAT results; -- a release build and the test suite from the extracted source archive. +- a release build and the test suite from the extracted source archive; and +- installation plus compilation and execution of an external CMake consumer. + +Pass `--git-ref v0.2.3-rc1` when the Git repository is available to regenerate +the archive from the signed tag and compare it byte-for-byte. + +`--allow-unsigned`, `--skip-rat`, `--skip-build`, and `--skip-install` exist for +CI or local development of the release process. They are not a substitute for +the corresponding checks when voting. The release-candidate workflow creates +an unsigned archive for deterministic CI validation; official artifacts must +always be signed by the release manager. + +## Publish an approved release + +After closing the vote and confirming that it passed, publish the exact +approved candidate: -The `--allow-unsigned` and `--skip-rat` options are only for local development -of the release process. A voter may use `--skip-build` when the repository's -Linux CI build command is not suitable for their platform, but must then build -and test the extracted source distribution separately before casting a binding -vote. +```bash +scripts/releasing/publish_release.sh \ + --version 0.2.3 \ + --rc 1 \ + --signing-key ASF_GPG_KEY_ID \ + --remote upstream \ + --confirm-vote-passed +``` + +By default, only PMC members can publish to `dist/release`. A non-PMC release +manager must ask a PMC member to perform this step unless Infra has configured +the project, following project consensus, to allow all committers to publish. + +The script verifies the signed RC tag, creates a signed final tag pointing to +the same commit, moves the candidate from ASF `dist/dev` to `dist/release`, and +creates a GitHub Release containing byte-identical copies of the ASF source +artifacts. It then prints the remaining ASF reporting, old-release cleanup, +documentation, mirror-propagation, and announcement steps. + +Download-page and release-note PRs may be prepared before publication, but do +not merge them while they point users at an unapproved RC. After publication, +verify the artifacts on `downloads.apache.org`, then wait at least 24 hours +before merging public download/documentation changes and sending the release +announcement. This follows the Paimon project convention, which is stricter +than ASF's general one-hour minimum. + +## Individual tools + +- `bump_version.py`: consistently check or update CMake and documentation + version metadata. +- `create_source_release.sh`: deterministically create an archive, SHA-512 + checksum, and optional detached signature from an immutable Git ref. +- `validate_source_archive.py`: reject unsafe or non-portable tar members and + compiled files. +- `verify_release_candidate.sh`: perform voter-facing integrity, license, + build, test, and install checks. +- `release_rc.sh`: orchestrate release-candidate tagging, verification, and ASF + staging. +- `publish_release.sh`: publish an approved candidate. + +Run the release-tool regression tests with: + +```bash +python3 -m unittest discover -s scripts/releasing/tests -v +``` diff --git a/scripts/releasing/bump_version.py b/scripts/releasing/bump_version.py new file mode 100755 index 00000000..f2c6df80 --- /dev/null +++ b/scripts/releasing/bump_version.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Check or update the Apache Paimon C++ project and documentation version.""" + +import argparse +import json +import re +import sys +from pathlib import Path +from typing import Dict, List + + +VERSION_PATTERN = re.compile(r"^\d+\.\d+\.\d+$") + + +class VersionError(RuntimeError): + """Version metadata is missing or inconsistent.""" + + +def require_version(value: str) -> None: + if VERSION_PATTERN.fullmatch(value) is None: + raise VersionError(f"invalid version {value!r}; expected MAJOR.MINOR.PATCH") + + +def replace_once(text: str, pattern: str, replacement: str, description: str) -> str: + result, count = re.subn(pattern, replacement, text, flags=re.MULTILINE) + if count != 1: + raise VersionError(f"expected one {description}, found {count}") + return result + + +def load_versions(path: Path) -> List[Dict[str, object]]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise VersionError(f"cannot read {path}: {error}") from error + if not isinstance(value, list) or not all(isinstance(item, dict) for item in value): + raise VersionError(f"{path} must contain a JSON array of objects") + return value + + +def checked_files(root: Path, expected: str) -> Dict[Path, str]: + cmake_path = root / "CMakeLists.txt" + docs_path = root / "docs/source/conf.py" + versions_path = root / "docs/source/_static/versions.json" + + cmake = cmake_path.read_text(encoding="utf-8") + docs = docs_path.read_text(encoding="utf-8") + versions = load_versions(versions_path) + + cmake_matches = re.findall( + r"^[ \t]*VERSION[ \t]+(\d+\.\d+\.\d+)[ \t]*$", cmake, re.MULTILINE + ) + docs_matches = re.findall( + r'^version = "(\d+\.\d+\.\d+)"$', docs, re.MULTILINE + ) + json_matches = [ + item + for item in versions + if item.get("name") == expected and item.get("version") == expected + ] + + if cmake_matches != [expected]: + raise VersionError(f"CMake version is {cmake_matches}, expected [{expected!r}]") + if docs_matches != [expected]: + raise VersionError( + f"documentation version is {docs_matches}, expected [{expected!r}]" + ) + if len(json_matches) != 1: + raise VersionError( + "versions.json must contain exactly one entry whose name and " + f"version are both {expected}" + ) + return { + cmake_path: cmake, + docs_path: docs, + versions_path: json.dumps(versions, indent=4) + "\n", + } + + +def updated_files(root: Path, current: str, new: str) -> Dict[Path, str]: + files = checked_files(root, current) + cmake_path = root / "CMakeLists.txt" + docs_path = root / "docs/source/conf.py" + versions_path = root / "docs/source/_static/versions.json" + + files[cmake_path] = replace_once( + files[cmake_path], + rf"^([ \t]*VERSION[ \t]+){re.escape(current)}([ \t]*)$", + rf"\g<1>{new}\g<2>", + f"CMake VERSION {current}", + ) + files[docs_path] = replace_once( + files[docs_path], + rf'^version = "{re.escape(current)}"$', + f'version = "{new}"', + f"documentation version {current}", + ) + + versions = load_versions(versions_path) + matches = [item for item in versions if item.get("version") == current] + if len(matches) != 1: + raise VersionError( + f"versions.json contains {len(matches)} entries for {current}" + ) + matches[0]["name"] = new + matches[0]["version"] = new + files[versions_path] = json.dumps(versions, indent=4) + "\n" + return files + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("current_version", nargs="?") + parser.add_argument("new_version", nargs="?") + parser.add_argument("--check", metavar="VERSION") + parser.add_argument("--dry-run", action="store_true") + parser.add_argument( + "--root", type=Path, default=Path(__file__).resolve().parents[2] + ) + args = parser.parse_args() + + try: + if args.check: + if args.current_version or args.new_version: + raise VersionError("--check cannot be combined with version arguments") + require_version(args.check) + checked_files(args.root.resolve(), args.check) + print(f"Release version metadata is consistent: {args.check}") + return 0 + + if not args.current_version or not args.new_version: + raise VersionError("CURRENT_VERSION and NEW_VERSION are required") + require_version(args.current_version) + require_version(args.new_version) + if args.current_version == args.new_version: + raise VersionError("current and new versions must differ") + + files = updated_files( + args.root.resolve(), args.current_version, args.new_version + ) + for path, content in files.items(): + if args.dry_run: + print(f"Would update {path}") + else: + path.write_text(content, encoding="utf-8") + print(f"Updated {path}") + return 0 + except (OSError, VersionError) as error: + print(f"Version update failed: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/releasing/create_source_release.sh b/scripts/releasing/create_source_release.sh index fb0345a7..f65ae2b2 100755 --- a/scripts/releasing/create_source_release.sh +++ b/scripts/releasing/create_source_release.sh @@ -115,8 +115,7 @@ done CMAKE_VERSION=$( git -C "${SOURCE_ROOT}" show "${GIT_REF}:CMakeLists.txt" | - sed -n 's/^[[:space:]]*VERSION[[:space:]]\+\([0-9][0-9.]*\).*$/\1/p' | - head -n 1 + awk '$1 == "VERSION" && $2 ~ /^[0-9]+\.[0-9]+\.[0-9]+$/ { print $2; exit }' ) [[ "${CMAKE_VERSION}" == "${RELEASE_VERSION}" ]] || fail "CMake version ${CMAKE_VERSION:-} does not match ${RELEASE_VERSION}" diff --git a/scripts/releasing/install_smoke/CMakeLists.txt b/scripts/releasing/install_smoke/CMakeLists.txt new file mode 100644 index 00000000..7665e304 --- /dev/null +++ b/scripts/releasing/install_smoke/CMakeLists.txt @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.16) +project(paimon_install_smoke LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +find_package(Paimon CONFIG REQUIRED) + +add_executable(paimon_install_smoke main.cpp) +target_link_libraries(paimon_install_smoke PRIVATE Paimon::paimon_shared) diff --git a/scripts/releasing/install_smoke/main.cpp b/scripts/releasing/install_smoke/main.cpp new file mode 100644 index 00000000..914adfce --- /dev/null +++ b/scripts/releasing/install_smoke/main.cpp @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "paimon/status.h" + +int main() { + const paimon::Status status = paimon::Status::Invalid("install smoke test"); + return !status.ok() && status.ToString().find("install smoke test") != std::string::npos ? 0 + : 1; +} diff --git a/scripts/releasing/publish_release.sh b/scripts/releasing/publish_release.sh new file mode 100755 index 00000000..c3081a8f --- /dev/null +++ b/scripts/releasing/publish_release.sh @@ -0,0 +1,328 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +SOURCE_ROOT=$(cd "${SCRIPT_DIR}/../.." && pwd) + +VERSION="" +RC="" +SIGNING_KEY="" +REMOTE="origin" +DIST_DEV_BASE_URL="https://dist.apache.org/repos/dist/dev/paimon" +DIST_RELEASE_BASE_URL="https://dist.apache.org/repos/dist/release/paimon" +CONFIRM_VOTE_PASSED=false +SKIP_GITHUB_RELEASE=false +DRY_RUN=false + +usage() { + cat <<'EOF' +Publish an approved Apache Paimon C++ release candidate. + +Usage: + publish_release.sh --version VERSION --rc RC --signing-key KEY_ID \ + --confirm-vote-passed [options] + +Required: + --version VERSION Approved release version + --rc RC Approved release candidate number + --signing-key KEY_ID OpenPGP key used to sign the final Git tag + --confirm-vote-passed Explicitly confirm that the PMC vote passed + +Options: + --remote REMOTE Apache Git remote (default: origin) + --dist-dev-base URL ASF dist/dev project URL + --dist-release-base URL ASF dist/release project URL + --skip-github-release Do not create the GitHub Release + --dry-run Print the planned publication identifiers and exit + -h, --help Show this help + +The script is resumable if the final tag, SVN move, or GitHub Release already +completed and still points to the approved RC commit. +EOF +} + +fail() { + echo "Error: $*" >&2 + exit 1 +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || fail "$1 is required" +} + +validate_release_directory() { + local directory=$1 + local -a entries + local entry + local name + + shopt -s dotglob nullglob + entries=("${directory}"/*) + shopt -u dotglob nullglob + [[ ${#entries[@]} -eq 3 ]] || + fail "${directory} must contain exactly the archive, signature, and checksum" + for entry in "${entries[@]}"; do + [[ -f "${entry}" ]] || fail "release contains a non-file entry: ${entry}" + name=$(basename "${entry}") + case "${name}" in + "${ARTIFACT_NAME}" | \ + "${ARTIFACT_NAME}.asc" | \ + "${ARTIFACT_NAME}.sha512") + ;; + *) + fail "release contains an unexpected file: ${name}" + ;; + esac + done +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --version) + [[ $# -ge 2 ]] || fail "--version requires a value" + VERSION=$2 + shift 2 + ;; + --rc) + [[ $# -ge 2 ]] || fail "--rc requires a value" + RC=$2 + shift 2 + ;; + --signing-key) + [[ $# -ge 2 ]] || fail "--signing-key requires a value" + SIGNING_KEY=$2 + shift 2 + ;; + --remote) + [[ $# -ge 2 ]] || fail "--remote requires a value" + REMOTE=$2 + shift 2 + ;; + --dist-dev-base) + [[ $# -ge 2 ]] || fail "--dist-dev-base requires a value" + DIST_DEV_BASE_URL=${2%/} + shift 2 + ;; + --dist-release-base) + [[ $# -ge 2 ]] || fail "--dist-release-base requires a value" + DIST_RELEASE_BASE_URL=${2%/} + shift 2 + ;; + --confirm-vote-passed) + CONFIRM_VOTE_PASSED=true + shift + ;; + --skip-github-release) + SKIP_GITHUB_RELEASE=true + shift + ;; + --dry-run) + DRY_RUN=true + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + fail "unknown argument: $1" + ;; + esac +done + +[[ "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || + fail "--version must use MAJOR.MINOR.PATCH format" +[[ "${RC}" =~ ^[0-9]+$ ]] || fail "--rc must be a non-negative integer" +[[ -n "${SIGNING_KEY}" ]] || fail "--signing-key is required" +[[ "${CONFIRM_VOTE_PASSED}" == true ]] || + fail "--confirm-vote-passed is required" + +RC_TAG="v${VERSION}-rc${RC}" +RELEASE_TAG="v${VERSION}" +RC_ID="paimon-cpp-${VERSION}-rc${RC}" +RELEASE_ID="paimon-cpp-${VERSION}" +ARTIFACT_NAME="apache-paimon-cpp-${VERSION}-src.tgz" +RC_URL="${DIST_DEV_BASE_URL}/${RC_ID}" +RELEASE_URL="${DIST_RELEASE_BASE_URL}/${RELEASE_ID}" + +if [[ "${DRY_RUN}" == true ]]; then + cat </dev/null) || + fail "Git remote does not exist: ${REMOTE}" +case "${REMOTE_URL}" in + git@github.com:apache/paimon-cpp.git | \ + https://github.com/apache/paimon-cpp | \ + https://github.com/apache/paimon-cpp.git | \ + ssh://git@github.com/apache/paimon-cpp.git) + ;; + *) + fail "${REMOTE} must point to apache/paimon-cpp, found ${REMOTE_URL}" + ;; +esac + +git rev-parse --verify "${RC_TAG}^{tag}" >/dev/null 2>&1 || + fail "signed RC tag is missing: ${RC_TAG}" +git verify-tag "${RC_TAG}" +RC_COMMIT=$(git rev-parse "${RC_TAG}^{commit}") + +RC_PRESENT=false +RELEASE_PRESENT=false +if svn info "${RC_URL}" >/dev/null 2>&1; then + RC_PRESENT=true +fi +if svn info "${RELEASE_URL}" >/dev/null 2>&1; then + RELEASE_PRESENT=true +fi +if [[ "${RC_PRESENT}" == true && "${RELEASE_PRESENT}" == true ]]; then + fail "both RC and final release directories exist; resolve SVN state manually" +fi +if [[ "${RC_PRESENT}" == false && "${RELEASE_PRESENT}" == false ]]; then + fail "approved artifacts are missing from both dist/dev and dist/release" +fi + +TEMP_DIR=$(mktemp -d) +trap 'rm -rf "${TEMP_DIR}"' EXIT +RELEASE_FILES_DIR="${TEMP_DIR}/${RELEASE_ID}" +if [[ "${RC_PRESENT}" == true ]]; then + svn export --quiet "${RC_URL}" "${RELEASE_FILES_DIR}" +else + svn export --quiet "${RELEASE_URL}" "${RELEASE_FILES_DIR}" +fi +validate_release_directory "${RELEASE_FILES_DIR}" +for suffix in "" ".asc" ".sha512"; do + [[ -f "${RELEASE_FILES_DIR}/${ARTIFACT_NAME}${suffix}" ]] || + fail "approved release is missing ${ARTIFACT_NAME}${suffix}" +done + +"${SCRIPT_DIR}/verify_release_candidate.sh" \ + --keys-url "https://downloads.apache.org/paimon/KEYS" \ + --git-ref "${RC_TAG}" \ + --skip-build \ + "${RELEASE_FILES_DIR}/${ARTIFACT_NAME}" + +if git rev-parse --verify "${RELEASE_TAG}^{tag}" >/dev/null 2>&1; then + RELEASE_COMMIT=$(git rev-parse "${RELEASE_TAG}^{commit}") + [[ "${RELEASE_COMMIT}" == "${RC_COMMIT}" ]] || + fail "${RELEASE_TAG} does not point to the approved RC commit" + git verify-tag "${RELEASE_TAG}" + echo "Reusing verified final tag ${RELEASE_TAG}." +else + git tag -s -u "${SIGNING_KEY}" \ + -m "Release Apache Paimon C++ ${VERSION}" \ + "${RELEASE_TAG}" "${RC_COMMIT}" + git verify-tag "${RELEASE_TAG}" +fi +git push "${REMOTE}" "${RELEASE_TAG}" + +if [[ "${RELEASE_PRESENT}" == true ]]; then + echo "Release artifacts are already present at ${RELEASE_URL}." +else + svn mv "${RC_URL}" "${RELEASE_URL}" \ + -m "Release Apache Paimon C++ ${VERSION}" +fi + +if [[ "${SKIP_GITHUB_RELEASE}" == false ]]; then + if gh release view "${RELEASE_TAG}" --repo apache/paimon-cpp >/dev/null 2>&1; then + echo "Checking existing GitHub Release ${RELEASE_TAG}." + EXISTING_ASSETS_DIR="${TEMP_DIR}/existing-assets" + mkdir "${EXISTING_ASSETS_DIR}" + for suffix in "" ".asc" ".sha512"; do + ASSET_NAME="${ARTIFACT_NAME}${suffix}" + if gh release view "${RELEASE_TAG}" \ + --repo apache/paimon-cpp \ + --json assets \ + --jq '.assets[].name' | + grep -Fxq "${ASSET_NAME}"; then + gh release download "${RELEASE_TAG}" \ + --repo apache/paimon-cpp \ + --dir "${EXISTING_ASSETS_DIR}" \ + --pattern "${ASSET_NAME}" + cmp "${EXISTING_ASSETS_DIR}/${ASSET_NAME}" \ + "${RELEASE_FILES_DIR}/${ASSET_NAME}" >/dev/null || + fail "GitHub Release asset differs from ASF release: ${ASSET_NAME}" + else + gh release upload "${RELEASE_TAG}" \ + --repo apache/paimon-cpp \ + "${RELEASE_FILES_DIR}/${ASSET_NAME}" + fi + done + else + gh release create "${RELEASE_TAG}" \ + --repo apache/paimon-cpp \ + --verify-tag \ + --generate-notes \ + --title "Apache Paimon C++ ${VERSION}" \ + "${RELEASE_FILES_DIR}/${ARTIFACT_NAME}" \ + "${RELEASE_FILES_DIR}/${ARTIFACT_NAME}.asc" \ + "${RELEASE_FILES_DIR}/${ARTIFACT_NAME}.sha512" + fi +fi + +cat <&2 + exit 1 +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || fail "$1 is required" +} + +validate_release_branch() { + local remote_branch_ref="refs/remotes/${REMOTE}/${RELEASE_BRANCH}" + local release_branch_commit + + git check-ref-format "refs/heads/${RELEASE_BRANCH}" >/dev/null 2>&1 || + fail "invalid release branch name: ${RELEASE_BRANCH}" + + echo "Fetching ${REMOTE}/${RELEASE_BRANCH} before publishing the RC." + git fetch --no-tags "${REMOTE}" \ + "refs/heads/${RELEASE_BRANCH}:${remote_branch_ref}" + release_branch_commit=$(git rev-parse --verify "${remote_branch_ref}^{commit}") + git merge-base --is-ancestor "${HEAD_COMMIT}" "${release_branch_commit}" || + fail "HEAD ${HEAD_COMMIT} is not contained in ${REMOTE}/${RELEASE_BRANCH} (${release_branch_commit})" +} + +wait_for_release_candidate_workflow() { + local deadline=$((SECONDS + WORKFLOW_DISCOVERY_TIMEOUT_SECONDS)) + local run_id="" + + echo "Waiting for the ${RC_TAG} Release Candidate workflow to start." + while [[ -z "${run_id}" ]]; do + if ! run_id=$( + gh run list \ + --repo apache/paimon-cpp \ + --workflow release_candidate.yaml \ + --branch "${RC_TAG}" \ + --commit "${HEAD_COMMIT}" \ + --event push \ + --limit 1 \ + --json databaseId \ + --jq '.[0].databaseId // empty' + ); then + fail "unable to query the Release Candidate workflow for ${RC_TAG}" + fi + if [[ -n "${run_id}" ]]; then + break + fi + if ((SECONDS >= deadline)); then + fail "timed out waiting for the Release Candidate workflow for ${RC_TAG}" + fi + sleep 10 + done + + echo "Waiting for Release Candidate workflow run ${run_id} to succeed." + gh run watch "${run_id}" \ + --repo apache/paimon-cpp \ + --compact \ + --exit-status \ + --interval 30 || + fail "Release Candidate workflow run ${run_id} failed" +} + +validate_artifact_directory() { + local -a entries + local entry + local name + + shopt -s dotglob nullglob + entries=("${OUTPUT_DIR}"/*) + shopt -u dotglob nullglob + [[ ${#entries[@]} -eq 3 ]] || + fail "${OUTPUT_DIR} must contain exactly the archive, signature, and checksum" + for entry in "${entries[@]}"; do + [[ -f "${entry}" ]] || + fail "release candidate contains a non-file entry: ${entry}" + name=$(basename "${entry}") + case "${name}" in + "${ARTIFACT_NAME}" | \ + "${ARTIFACT_NAME}.asc" | \ + "${ARTIFACT_NAME}.sha512") + ;; + *) + fail "release candidate contains an unexpected file: ${name}" + ;; + esac + done +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --version) + [[ $# -ge 2 ]] || fail "--version requires a value" + VERSION=$2 + shift 2 + ;; + --rc) + [[ $# -ge 2 ]] || fail "--rc requires a value" + RC=$2 + shift 2 + ;; + --signing-key) + [[ $# -ge 2 ]] || fail "--signing-key requires a value" + SIGNING_KEY=$2 + shift 2 + ;; + --remote) + [[ $# -ge 2 ]] || fail "--remote requires a value" + REMOTE=$2 + shift 2 + ;; + --release-branch) + [[ $# -ge 2 ]] || fail "--release-branch requires a value" + RELEASE_BRANCH=$2 + shift 2 + ;; + --output-dir) + [[ $# -ge 2 ]] || fail "--output-dir requires a value" + OUTPUT_DIR=$2 + shift 2 + ;; + --dist-dev-base) + [[ $# -ge 2 ]] || fail "--dist-dev-base requires a value" + DIST_DEV_BASE_URL=${2%/} + shift 2 + ;; + --prepare-only) + PREPARE_ONLY=true + shift + ;; + --dry-run) + DRY_RUN=true + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + fail "unknown argument: $1" + ;; + esac +done + +[[ "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || + fail "--version must use MAJOR.MINOR.PATCH format" +[[ "${RC}" =~ ^[0-9]+$ ]] || fail "--rc must be a non-negative integer" +[[ -n "${SIGNING_KEY}" ]] || fail "--signing-key is required" + +RC_TAG="v${VERSION}-rc${RC}" +RC_ID="paimon-cpp-${VERSION}-rc${RC}" +ARTIFACT_NAME="apache-paimon-cpp-${VERSION}-src.tgz" +RC_URL="${DIST_DEV_BASE_URL}/${RC_ID}" +OUTPUT_DIR=${OUTPUT_DIR:-"${SOURCE_ROOT}/release/${VERSION}-rc${RC}"} + +if [[ "${DRY_RUN}" == true ]]; then + cat </dev/null) || + fail "Git remote does not exist: ${REMOTE}" +case "${REMOTE_URL}" in + git@github.com:apache/paimon-cpp.git | \ + https://github.com/apache/paimon-cpp | \ + https://github.com/apache/paimon-cpp.git | \ + ssh://git@github.com/apache/paimon-cpp.git) + ;; + *) + fail "${REMOTE} must point to apache/paimon-cpp, found ${REMOTE_URL}" + ;; +esac + +"${SCRIPT_DIR}/bump_version.py" --check "${VERSION}" + +HEAD_COMMIT=$(git rev-parse HEAD) +if [[ "${PREPARE_ONLY}" == false ]]; then + validate_release_branch +fi + +if git rev-parse --verify "${RC_TAG}^{tag}" >/dev/null 2>&1; then + TAG_COMMIT=$(git rev-parse "${RC_TAG}^{commit}") + [[ "${TAG_COMMIT}" == "${HEAD_COMMIT}" ]] || + fail "${RC_TAG} points to ${TAG_COMMIT}, expected HEAD ${HEAD_COMMIT}" + git verify-tag "${RC_TAG}" + echo "Reusing verified local tag ${RC_TAG}." +else + git tag -s -u "${SIGNING_KEY}" -m "Apache Paimon C++ ${VERSION} RC${RC}" \ + "${RC_TAG}" + git verify-tag "${RC_TAG}" +fi + +ARTIFACT="${OUTPUT_DIR}/${ARTIFACT_NAME}" +if [[ -e "${ARTIFACT}" || -e "${ARTIFACT}.asc" || -e "${ARTIFACT}.sha512" ]]; then + [[ -f "${ARTIFACT}" && -f "${ARTIFACT}.asc" && -f "${ARTIFACT}.sha512" ]] || + fail "artifact directory contains an incomplete release candidate" + echo "Reusing existing artifacts in ${OUTPUT_DIR}." +else + "${SCRIPT_DIR}/create_source_release.sh" \ + --version "${VERSION}" \ + --git-ref "${RC_TAG}" \ + --output-dir "${OUTPUT_DIR}" \ + --signing-key "${SIGNING_KEY}" +fi + +"${SCRIPT_DIR}/verify_release_candidate.sh" \ + --git-ref "${RC_TAG}" \ + --keys-url "https://downloads.apache.org/paimon/KEYS" \ + "${ARTIFACT}" + +validate_artifact_directory + +if [[ "${PREPARE_ONLY}" == true ]]; then + cat </dev/null 2>&1; then + fail "release candidate already exists in ASF dist/dev: ${RC_URL}" +fi +git push "${REMOTE}" "${RC_TAG}" +wait_for_release_candidate_workflow +svn import "${OUTPUT_DIR}" "${RC_URL}" \ + -m "Add Apache Paimon C++ ${VERSION} RC${RC}" + +cat < subprocess.CompletedProcess: + result = subprocess.run( + [sys.executable, str(tool), *args], + universal_newlines=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + self.assertEqual( + result.returncode, + expected_returncode, + msg=f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}", + ) + return result + + def create_archive( + self, + path: Path, + *, + extra_members: Optional[List[Tuple[tarfile.TarInfo, bytes]]] = None, + ) -> None: + with tarfile.open(path, mode="w:gz") as archive: + root = tarfile.TarInfo("paimon-cpp-1.2.3/") + root.type = tarfile.DIRTYPE + root.mode = 0o755 + archive.addfile(root) + + license_info = tarfile.TarInfo("paimon-cpp-1.2.3/LICENSE") + license_info.size = len(b"Apache License\n") + license_info.mode = 0o644 + archive.addfile(license_info, io.BytesIO(b"Apache License\n")) + + for member, content in extra_members or []: + member.size = len(content) if member.isfile() else 0 + archive.addfile(member, io.BytesIO(content) if member.isfile() else None) + + def create_verifier_archive(self, directory: Path) -> Path: + artifact = directory / "apache-paimon-cpp-1.2.3-src.tgz" + files = { + "LICENSE": b"Apache License\n", + "NOTICE": b"Apache Paimon\n", + "CMakeLists.txt": ( + b"project(paimon\n" + b" VERSION 1.2.3\n" + b' DESCRIPTION "Paimon C++ Project")\n' + ), + "docs/source/conf.py": b'version = "1.2.3"\n', + "docs/source/_static/versions.json": ( + b'[{"name": "1.2.3", "version": "1.2.3", ' + b'"url": "https://paimon.apache.org/docs/cpp/"}]\n' + ), + ".github/.rat-excludes": b"", + "scripts/releasing/create_source_release.sh": b"#!/usr/bin/env bash\n", + } + with tarfile.open(artifact, mode="w:gz") as archive: + root = tarfile.TarInfo("paimon-cpp-1.2.3/") + root.type = tarfile.DIRTYPE + root.mode = 0o755 + archive.addfile(root) + for name, content in files.items(): + member = tarfile.TarInfo(f"paimon-cpp-1.2.3/{name}") + member.size = len(content) + member.mode = 0o755 if name.endswith(".sh") else 0o644 + archive.addfile(member, io.BytesIO(content)) + + digest = hashlib.sha512(artifact.read_bytes()).hexdigest() + artifact.with_suffix(artifact.suffix + ".sha512").write_text( + f"{digest} {artifact.name}\n", encoding="utf-8" + ) + return artifact + + def run_verifier( + self, artifact: Path, *, expected_returncode: int = 0 + ) -> subprocess.CompletedProcess: + result = subprocess.run( + [ + "bash", + str(RELEASE_VERIFIER), + "--allow-unsigned", + "--skip-rat", + "--skip-build", + str(artifact), + ], + universal_newlines=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + self.assertEqual( + result.returncode, + expected_returncode, + msg=f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}", + ) + return result + + def test_archive_validator_accepts_regular_archive(self) -> None: + with tempfile.TemporaryDirectory() as temp: + artifact = Path(temp) / "valid.tgz" + self.create_archive(artifact) + self.run_tool( + ARCHIVE_VALIDATOR, + "--expected-root", + "paimon-cpp-1.2.3", + str(artifact), + ) + + def test_archive_validator_rejects_path_traversal(self) -> None: + with tempfile.TemporaryDirectory() as temp: + artifact = Path(temp) / "traversal.tgz" + member = tarfile.TarInfo("../outside") + member.mode = 0o644 + self.create_archive(artifact, extra_members=[(member, b"bad")]) + self.run_tool( + ARCHIVE_VALIDATOR, + "--expected-root", + "paimon-cpp-1.2.3", + str(artifact), + expected_returncode=1, + ) + + def test_archive_validator_rejects_symlink(self) -> None: + with tempfile.TemporaryDirectory() as temp: + artifact = Path(temp) / "symlink.tgz" + member = tarfile.TarInfo("paimon-cpp-1.2.3/link") + member.type = tarfile.SYMTYPE + member.linkname = "../../outside" + member.mode = 0o777 + self.create_archive(artifact, extra_members=[(member, b"")]) + self.run_tool( + ARCHIVE_VALIDATOR, + "--expected-root", + "paimon-cpp-1.2.3", + str(artifact), + expected_returncode=1, + ) + + def test_archive_validator_rejects_compiled_magic(self) -> None: + with tempfile.TemporaryDirectory() as temp: + artifact = Path(temp) / "binary.tgz" + member = tarfile.TarInfo("paimon-cpp-1.2.3/generated") + member.mode = 0o755 + self.create_archive( + artifact, extra_members=[(member, b"\x7fELFcompiled")] + ) + self.run_tool( + ARCHIVE_VALIDATOR, + "--expected-root", + "paimon-cpp-1.2.3", + str(artifact), + expected_returncode=1, + ) + + def test_archive_validator_rejects_portable_collision(self) -> None: + with tempfile.TemporaryDirectory() as temp: + artifact = Path(temp) / "collision.tgz" + first = tarfile.TarInfo("paimon-cpp-1.2.3/README") + first.mode = 0o644 + second = tarfile.TarInfo("paimon-cpp-1.2.3/readme") + second.mode = 0o644 + self.create_archive( + artifact, + extra_members=[(first, b"one"), (second, b"two")], + ) + self.run_tool( + ARCHIVE_VALIDATOR, + "--expected-root", + "paimon-cpp-1.2.3", + str(artifact), + expected_returncode=1, + ) + + def test_verifier_accepts_valid_checksum_and_archive(self) -> None: + with tempfile.TemporaryDirectory() as temp: + artifact = self.create_verifier_archive(Path(temp)) + result = self.run_verifier(artifact) + self.assertIn("Release candidate verification completed", result.stdout) + + def test_verifier_rejects_checksum_mismatch(self) -> None: + with tempfile.TemporaryDirectory() as temp: + artifact = self.create_verifier_archive(Path(temp)) + checksum = artifact.with_suffix(artifact.suffix + ".sha512") + checksum.write_text(f"{'0' * 128} {artifact.name}\n", encoding="utf-8") + result = self.run_verifier(artifact, expected_returncode=1) + self.assertIn("SHA-512 checksum does not match", result.stderr) + + def test_verifier_rejects_multiple_checksum_lines(self) -> None: + with tempfile.TemporaryDirectory() as temp: + artifact = self.create_verifier_archive(Path(temp)) + checksum = artifact.with_suffix(artifact.suffix + ".sha512") + checksum.write_text( + checksum.read_text(encoding="utf-8") + + f"{'0' * 128} attacker-controlled-file\n", + encoding="utf-8", + ) + result = self.run_verifier(artifact, expected_returncode=1) + self.assertIn( + "checksum file must contain exactly one non-empty line", + result.stderr, + ) + + def create_version_tree(self, root: Path) -> None: + (root / "docs/source/_static").mkdir(parents=True) + (root / "CMakeLists.txt").write_text( + "project(paimon\n VERSION 1.2.3\n" + ' DESCRIPTION "Paimon C++ Project")\n', + encoding="utf-8", + ) + (root / "docs/source/conf.py").write_text( + 'version = "1.2.3"\n', encoding="utf-8" + ) + (root / "docs/source/_static/versions.json").write_text( + json.dumps( + [ + { + "name": "1.2.3", + "version": "1.2.3", + "url": "https://paimon.apache.org/docs/cpp/", + } + ], + indent=4, + ) + + "\n", + encoding="utf-8", + ) + + def test_version_tool_checks_and_updates_all_metadata(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + self.create_version_tree(root) + self.run_tool(VERSION_TOOL, "--root", str(root), "--check", "1.2.3") + self.run_tool(VERSION_TOOL, "--root", str(root), "1.2.3", "1.2.4") + self.run_tool(VERSION_TOOL, "--root", str(root), "--check", "1.2.4") + self.assertIn("VERSION 1.2.4", (root / "CMakeLists.txt").read_text()) + self.assertIn( + 'version = "1.2.4"', (root / "docs/source/conf.py").read_text() + ) + + def test_version_tool_rejects_inconsistent_metadata(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + self.create_version_tree(root) + (root / "docs/source/conf.py").write_text( + 'version = "9.9.9"\n', encoding="utf-8" + ) + self.run_tool( + VERSION_TOOL, + "--root", + str(root), + "--check", + "1.2.3", + expected_returncode=1, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/releasing/validate_source_archive.py b/scripts/releasing/validate_source_archive.py new file mode 100755 index 00000000..a6464535 --- /dev/null +++ b/scripts/releasing/validate_source_archive.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Validate the portable and security-sensitive layout of a source archive.""" + +import argparse +import stat +import sys +import tarfile +import unicodedata +from pathlib import PurePosixPath +from typing import Optional, Set + + +COMPILED_MAGICS = { + b"\x00asm": "WebAssembly module", + b"\x7fELF": "ELF binary", + b"BC\xc0\xde": "LLVM bitcode", + b"\xca\xfe\xba\xbe": "Java class or Mach-O universal binary", + b"\xce\xfa\xed\xfe": "Mach-O binary", + b"\xcf\xfa\xed\xfe": "Mach-O binary", + b"\xfe\xed\xfa\xce": "Mach-O binary", + b"\xfe\xed\xfa\xcf": "Mach-O binary", +} + + +class ValidationError(RuntimeError): + """The archive does not satisfy release safety requirements.""" + + +def require(condition: bool, message: str) -> None: + if not condition: + raise ValidationError(message) + + +def normalized_name(member: tarfile.TarInfo) -> str: + value = member.name + require(bool(value), "archive contains an empty path") + require("\\" not in value, f"archive path contains a backslash: {value!r}") + candidate = value[:-1] if member.isdir() and value.endswith("/") else value + path = PurePosixPath(candidate) + require(not path.is_absolute(), f"archive contains an absolute path: {value!r}") + require(".." not in path.parts, f"archive contains path traversal: {value!r}") + require(path.as_posix() == candidate, f"archive path is not canonical: {value!r}") + return path.as_posix() + + +def portable_key(name: str) -> str: + return unicodedata.normalize("NFC", name).casefold() + + +def compiled_description( + archive: tarfile.TarFile, member: tarfile.TarInfo +) -> Optional[str]: + extracted = archive.extractfile(member) + if extracted is None: + raise ValidationError(f"cannot read archive member: {member.name}") + header = extracted.read(4096) + for magic, description in COMPILED_MAGICS.items(): + if header.startswith(magic): + return description + if header.startswith(b"!\n"): + return "Unix archive" + if header.startswith(b"MZ") and len(header) >= 64: + pe_offset = int.from_bytes(header[60:64], "little") + if pe_offset + 4 <= len(header) and header[pe_offset : pe_offset + 4] == b"PE\0\0": + return "Windows PE binary" + return None + + +def validate(artifact: str, expected_root: str) -> None: + expected_root = expected_root.rstrip("/") + require(bool(expected_root), "expected root must not be empty") + require("/" not in expected_root, "expected root must be one path component") + + try: + archive = tarfile.open(artifact, mode="r:gz") + except (OSError, tarfile.TarError) as error: + raise ValidationError(f"cannot open source archive: {error}") from error + + with archive: + members = archive.getmembers() + require(bool(members), "source archive is empty") + names: Set[str] = set() + portable_names: Set[str] = set() + found_root = False + + for member in members: + require( + member.isfile() or member.isdir(), + f"archive contains a link or special member: {member.name!r}", + ) + name = normalized_name(member) + parts = PurePosixPath(name).parts + require( + bool(parts) and parts[0] == expected_root, + f"archive entry is outside {expected_root}/: {member.name!r}", + ) + if name == expected_root and member.isdir(): + found_root = True + + require(name not in names, f"archive contains duplicate path: {name!r}") + names.add(name) + key = portable_key(name) + require( + key not in portable_names, + f"archive contains a portable path collision: {name!r}", + ) + portable_names.add(key) + + require( + member.mode & (stat.S_IWGRP | stat.S_IWOTH) == 0, + f"archive contains a group- or world-writable member: {name!r}", + ) + require( + member.mode & (stat.S_ISUID | stat.S_ISGID | stat.S_ISVTX) == 0, + f"archive contains a member with special permission bits: {name!r}", + ) + + if member.isfile(): + description = compiled_description(archive, member) + require( + description is None, + f"source archive contains a compiled file ({description}): {name}", + ) + + require(found_root, f"archive root directory is missing: {expected_root}/") + print(f"Validated {len(members)} archive members under {expected_root}/") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("artifact", help="source .tar.gz or .tgz archive") + parser.add_argument("--expected-root", required=True) + args = parser.parse_args() + try: + validate(args.artifact, args.expected_root) + except ValidationError as error: + print(f"Archive validation failed: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/releasing/verify_release_candidate.sh b/scripts/releasing/verify_release_candidate.sh index cf6660d6..dee7d812 100755 --- a/scripts/releasing/verify_release_candidate.sh +++ b/scripts/releasing/verify_release_candidate.sh @@ -10,20 +10,31 @@ # # http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. set -euo pipefail +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +SOURCE_ROOT=$(cd "${SCRIPT_DIR}/../.." && pwd) + ARTIFACT="" +REQUESTED_VERSION="" +RC="" +DIST_DEV_BASE_URL="https://dist.apache.org/repos/dist/dev/paimon" +KEYS_URL="" +KEYS_FILE="" +GIT_REF="" RAT_JAR=${RAT_JAR:-} +RAT_VERSION="0.16.1" ALLOW_UNSIGNED=false SKIP_RAT=false SKIP_BUILD=false +SKIP_INSTALL=false +JOBS=${PAIMON_BUILD_JOBS:-} usage() { cat <<'EOF' @@ -31,22 +42,31 @@ Verify an Apache Paimon C++ source release candidate. Usage: verify_release_candidate.sh [options] ARTIFACT + verify_release_candidate.sh --version VERSION --rc RC [options] + +Download options: + --version VERSION Release version to download + --rc RC Release candidate number to download + --dist-dev-base URL ASF dist/dev project URL + +Trust and reproducibility: + --keys-url URL Download KEYS and verify in an isolated GPG home + --keys-file FILE Import this KEYS file into an isolated GPG home + --git-ref REF Regenerate the archive from REF and compare bytes + +Verification options: + --rat-jar FILE Apache RAT executable jar (or set RAT_JAR) + --rat-version VERSION RAT version to download (default: 0.16.1) + --jobs JOBS Parallel build and test jobs + --allow-unsigned Allow a missing .asc file for local/CI preparation + --skip-rat Skip Apache RAT for local preparation only + --skip-build Skip the release build, tests, and install smoke test + --skip-install Skip only the install and consumer smoke test + -h, --help Show this help -Options: - --rat-jar FILE Apache RAT executable jar (or set RAT_JAR) - --allow-unsigned Allow a missing .asc file for local preparation only - --skip-rat Skip Apache RAT for local preparation only - --skip-build Skip the release build and test suite - -h, --help Show this help - -By default the script requires: - ARTIFACT.sha512 - ARTIFACT.asc - an Apache RAT jar - -It verifies the checksum and signature, inspects and extracts the archive, -checks release metadata, runs Apache RAT, then builds and tests from the -extracted source distribution. +Download mode defaults to https://downloads.apache.org/paimon/KEYS. For a +local artifact, pass --keys-url or --keys-file to avoid trusting the user's +default GPG keyring. EOF } @@ -66,13 +86,66 @@ calculate_sha512() { fi } +download_file() { + local url=$1 + local destination=$2 + if command -v curl >/dev/null 2>&1; then + curl --fail --location --show-error --silent \ + --output "${destination}" "${url}" + elif command -v wget >/dev/null 2>&1; then + wget --quiet --output-document="${destination}" "${url}" + else + fail "curl or wget is required to download release files" + fi +} + while [[ $# -gt 0 ]]; do case "$1" in + --version) + [[ $# -ge 2 ]] || fail "--version requires a value" + REQUESTED_VERSION=$2 + shift 2 + ;; + --rc) + [[ $# -ge 2 ]] || fail "--rc requires a value" + RC=$2 + shift 2 + ;; + --dist-dev-base) + [[ $# -ge 2 ]] || fail "--dist-dev-base requires a value" + DIST_DEV_BASE_URL=${2%/} + shift 2 + ;; + --keys-url) + [[ $# -ge 2 ]] || fail "--keys-url requires a value" + KEYS_URL=$2 + shift 2 + ;; + --keys-file) + [[ $# -ge 2 ]] || fail "--keys-file requires a value" + KEYS_FILE=$2 + shift 2 + ;; + --git-ref) + [[ $# -ge 2 ]] || fail "--git-ref requires a value" + GIT_REF=$2 + shift 2 + ;; --rat-jar) [[ $# -ge 2 ]] || fail "--rat-jar requires a value" RAT_JAR=$2 shift 2 ;; + --rat-version) + [[ $# -ge 2 ]] || fail "--rat-version requires a value" + RAT_VERSION=$2 + shift 2 + ;; + --jobs) + [[ $# -ge 2 ]] || fail "--jobs requires a value" + JOBS=$2 + shift 2 + ;; --allow-unsigned) ALLOW_UNSIGNED=true shift @@ -85,6 +158,10 @@ while [[ $# -gt 0 ]]; do SKIP_BUILD=true shift ;; + --skip-install) + SKIP_INSTALL=true + shift + ;; -h|--help) usage exit 0 @@ -100,9 +177,39 @@ while [[ $# -gt 0 ]]; do esac done -[[ -n "${ARTIFACT}" ]] || fail "an artifact is required" -[[ -f "${ARTIFACT}" ]] || fail "artifact does not exist: ${ARTIFACT}" +[[ -z "${KEYS_URL}" || -z "${KEYS_FILE}" ]] || + fail "--keys-url and --keys-file are mutually exclusive" +[[ -z "${REQUESTED_VERSION}" || + "${REQUESTED_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || + fail "--version must use MAJOR.MINOR.PATCH format" +[[ -z "${RC}" || "${RC}" =~ ^[0-9]+$ ]] || + fail "--rc must be a non-negative integer" +[[ -z "${JOBS}" || "${JOBS}" =~ ^[1-9][0-9]*$ ]] || + fail "--jobs must be a positive integer" +TEMP_DIR=$(mktemp -d) +trap 'rm -rf "${TEMP_DIR}"' EXIT + +if [[ -z "${ARTIFACT}" ]]; then + [[ -n "${REQUESTED_VERSION}" && -n "${RC}" ]] || + fail "ARTIFACT or both --version and --rc are required" + ARTIFACT_NAME="apache-paimon-cpp-${REQUESTED_VERSION}-src.tgz" + RC_URL="${DIST_DEV_BASE_URL}/paimon-cpp-${REQUESTED_VERSION}-rc${RC}" + DOWNLOAD_DIR="${TEMP_DIR}/download" + mkdir -p "${DOWNLOAD_DIR}" + echo "Downloading Apache Paimon C++ ${REQUESTED_VERSION} RC${RC}..." + download_file "${RC_URL}/${ARTIFACT_NAME}" "${DOWNLOAD_DIR}/${ARTIFACT_NAME}" + download_file "${RC_URL}/${ARTIFACT_NAME}.sha512" \ + "${DOWNLOAD_DIR}/${ARTIFACT_NAME}.sha512" + if [[ "${ALLOW_UNSIGNED}" == false ]]; then + download_file "${RC_URL}/${ARTIFACT_NAME}.asc" \ + "${DOWNLOAD_DIR}/${ARTIFACT_NAME}.asc" + fi + ARTIFACT="${DOWNLOAD_DIR}/${ARTIFACT_NAME}" + KEYS_URL=${KEYS_URL:-"https://downloads.apache.org/paimon/KEYS"} +fi + +[[ -f "${ARTIFACT}" ]] || fail "artifact does not exist: ${ARTIFACT}" ARTIFACT_DIR=$(cd "$(dirname "${ARTIFACT}")" && pwd) ARTIFACT_NAME=$(basename "${ARTIFACT}") ARTIFACT="${ARTIFACT_DIR}/${ARTIFACT_NAME}" @@ -112,18 +219,26 @@ if [[ "${ARTIFACT_NAME}" =~ ^apache-paimon-cpp-([0-9]+\.[0-9]+\.[0-9]+)-src\.tgz else fail "unexpected artifact name: ${ARTIFACT_NAME}" fi +[[ -z "${REQUESTED_VERSION}" || "${REQUESTED_VERSION}" == "${RELEASE_VERSION}" ]] || + fail "artifact version ${RELEASE_VERSION} does not match ${REQUESTED_VERSION}" CHECKSUM_FILE="${ARTIFACT}.sha512" SIGNATURE_FILE="${ARTIFACT}.asc" ARCHIVE_ROOT="paimon-cpp-${RELEASE_VERSION}" [[ -f "${CHECKSUM_FILE}" ]] || fail "missing checksum: ${CHECKSUM_FILE}" -EXPECTED_SHA512=$(awk 'NR == 1 {print $1}' "${CHECKSUM_FILE}") -CHECKSUM_ARTIFACT=$(awk 'NR == 1 {print $2}' "${CHECKSUM_FILE}") +CHECKSUM_LINE_COUNT=$(awk 'NF { count++ } END { print count + 0 }' "${CHECKSUM_FILE}") +[[ "${CHECKSUM_LINE_COUNT}" == "1" ]] || + fail "checksum file must contain exactly one non-empty line" +CHECKSUM_FIELD_COUNT=$(awk 'NF { print NF; exit }' "${CHECKSUM_FILE}") +[[ "${CHECKSUM_FIELD_COUNT}" == "2" ]] || + fail "checksum line must contain a digest and a filename" +EXPECTED_SHA512=$(awk 'NF { print $1; exit }' "${CHECKSUM_FILE}") +CHECKSUM_ARTIFACT=$(awk 'NF { print $2; exit }' "${CHECKSUM_FILE}") [[ "${EXPECTED_SHA512}" =~ ^[0-9a-fA-F]{128}$ ]] || fail "invalid SHA-512 file: ${CHECKSUM_FILE}" [[ "${CHECKSUM_ARTIFACT}" == "${ARTIFACT_NAME}" ]] || - fail "checksum file names ${CHECKSUM_ARTIFACT:-}, expected ${ARTIFACT_NAME}" + fail "checksum names ${CHECKSUM_ARTIFACT:-}, expected ${ARTIFACT_NAME}" ACTUAL_SHA512=$(calculate_sha512 "${ARTIFACT}") EXPECTED_SHA512=$(printf '%s' "${EXPECTED_SHA512}" | tr '[:upper:]' '[:lower:]') ACTUAL_SHA512=$(printf '%s' "${ACTUAL_SHA512}" | tr '[:upper:]' '[:lower:]') @@ -133,74 +248,90 @@ echo "SHA-512 checksum: valid" if [[ -f "${SIGNATURE_FILE}" ]]; then command -v gpg >/dev/null 2>&1 || fail "gpg is required to verify the signature" - gpg --verify "${SIGNATURE_FILE}" "${ARTIFACT}" - echo "OpenPGP signature: valid" + if [[ -n "${KEYS_URL}" || -n "${KEYS_FILE}" ]]; then + GNUPG_HOME="${TEMP_DIR}/gnupg" + mkdir -m 700 "${GNUPG_HOME}" + if [[ -n "${KEYS_URL}" ]]; then + KEYS_FILE="${TEMP_DIR}/KEYS" + download_file "${KEYS_URL}" "${KEYS_FILE}" + else + KEYS_FILE=$(cd "$(dirname "${KEYS_FILE}")" && pwd)/$(basename "${KEYS_FILE}") + fi + [[ -f "${KEYS_FILE}" ]] || fail "KEYS file does not exist: ${KEYS_FILE}" + gpg --batch --homedir "${GNUPG_HOME}" --import "${KEYS_FILE}" >/dev/null + gpg --batch --homedir "${GNUPG_HOME}" \ + --verify "${SIGNATURE_FILE}" "${ARTIFACT}" + echo "OpenPGP signature: valid against ${KEYS_FILE}" + else + gpg --verify "${SIGNATURE_FILE}" "${ARTIFACT}" + echo "OpenPGP signature: valid against the default GPG keyring" + fi elif [[ "${ALLOW_UNSIGNED}" == true ]]; then - echo "OpenPGP signature: skipped for local preparation" + echo "OpenPGP signature: skipped for local/CI preparation" else fail "missing signature: ${SIGNATURE_FILE}" fi -TEMP_DIR=$(mktemp -d) -trap 'rm -rf "${TEMP_DIR}"' EXIT -CONTENTS_FILE="${TEMP_DIR}/archive-contents.txt" - -tar -tzf "${ARTIFACT}" >"${CONTENTS_FILE}" -[[ -s "${CONTENTS_FILE}" ]] || fail "source archive is empty" - -while IFS= read -r entry; do - [[ "${entry}" != /* ]] || fail "archive contains an absolute path: ${entry}" - [[ "${entry}" != ".." && "${entry}" != ../* && "${entry}" != */.. && - "${entry}" != *"/../"* ]] || - fail "archive contains path traversal: ${entry}" - [[ "${entry}" == "${ARCHIVE_ROOT}" || "${entry}" == "${ARCHIVE_ROOT}/"* ]] || - fail "archive entry is outside ${ARCHIVE_ROOT}: ${entry}" -done <"${CONTENTS_FILE}" +command -v python3 >/dev/null 2>&1 || fail "python3 is required" +python3 "${SCRIPT_DIR}/validate_source_archive.py" \ + --expected-root "${ARCHIVE_ROOT}" "${ARTIFACT}" tar -xzf "${ARTIFACT}" -C "${TEMP_DIR}" SOURCE_DIR="${TEMP_DIR}/${ARCHIVE_ROOT}" [[ -d "${SOURCE_DIR}" ]] || fail "archive root is missing: ${ARCHIVE_ROOT}" -for required_file in LICENSE NOTICE CMakeLists.txt docs/source/conf.py; do +for required_file in \ + LICENSE \ + NOTICE \ + CMakeLists.txt \ + docs/source/conf.py \ + docs/source/_static/versions.json \ + .github/.rat-excludes \ + scripts/releasing/create_source_release.sh; do [[ -f "${SOURCE_DIR}/${required_file}" ]] || fail "required release file is missing: ${required_file}" done -CMAKE_VERSION=$( - sed -n 's/^[[:space:]]*VERSION[[:space:]]\+\([0-9][0-9.]*\).*$/\1/p' \ - "${SOURCE_DIR}/CMakeLists.txt" | - head -n 1 -) -[[ "${CMAKE_VERSION}" == "${RELEASE_VERSION}" ]] || - fail "CMake version ${CMAKE_VERSION:-} does not match ${RELEASE_VERSION}" - -DOCS_VERSION=$( - sed -n 's/^version = "\([^"]*\)"$/\1/p' "${SOURCE_DIR}/docs/source/conf.py" | - head -n 1 -) -[[ "${DOCS_VERSION}" == "${RELEASE_VERSION}" ]] || - fail "documentation version ${DOCS_VERSION:-} does not match ${RELEASE_VERSION}" +python3 "${SCRIPT_DIR}/bump_version.py" \ + --root "${SOURCE_DIR}" \ + --check "${RELEASE_VERSION}" UNEXPECTED_BINARIES=$( find "${SOURCE_DIR}" -type f \ - \( -name '*.a' -o -name '*.class' -o -name '*.dll' -o -name '*.dylib' \ - -o -name '*.exe' -o -name '*.jar' -o -name '*.lib' -o -name '*.o' \ - -o -name '*.pdb' -o -name '*.pyc' -o -name '*.so' \) \ + \( -name '*.a' -o -name '*.bc' -o -name '*.class' -o -name '*.dll' \ + -o -name '*.dylib' -o -name '*.exe' -o -name '*.jar' -o -name '*.la' \ + -o -name '*.lib' -o -name '*.lo' -o -name '*.o' -o -name '*.obj' \ + -o -name '*.pdb' -o -name '*.pyc' -o -name '*.so' -o -name '*.wasm' \) \ -print ) [[ -z "${UNEXPECTED_BINARIES}" ]] || fail "source archive contains unexpected compiled files:${UNEXPECTED_BINARIES}" - -UNSAFE_PERMISSIONS=$( - find "${SOURCE_DIR}" -type f \( -perm -0020 -o -perm -0002 \) -print -) -[[ -z "${UNSAFE_PERMISSIONS}" ]] || - fail "source archive contains group- or world-writable files:${UNSAFE_PERMISSIONS}" - echo "Archive layout and release metadata: valid" +if [[ -n "${GIT_REF}" ]]; then + git -C "${SOURCE_ROOT}" rev-parse --verify "${GIT_REF}^{commit}" >/dev/null 2>&1 || + fail "Git ref does not resolve to a commit: ${GIT_REF}" + if git -C "${SOURCE_ROOT}" rev-parse --verify "${GIT_REF}^{tag}" \ + >/dev/null 2>&1; then + git -C "${SOURCE_ROOT}" verify-tag "${GIT_REF}" + fi + REPRO_DIR="${TEMP_DIR}/reproduced" + "${SCRIPT_DIR}/create_source_release.sh" \ + --version "${RELEASE_VERSION}" \ + --git-ref "${GIT_REF}" \ + --output-dir "${REPRO_DIR}" + cmp "${ARTIFACT}" "${REPRO_DIR}/${ARTIFACT_NAME}" >/dev/null || + fail "artifact bytes differ from a fresh archive of ${GIT_REF}" + echo "Git ref reproducibility: valid (${GIT_REF})" +fi + if [[ "${SKIP_RAT}" == false ]]; then - [[ -n "${RAT_JAR}" ]] || fail "--rat-jar or RAT_JAR is required" + if [[ -z "${RAT_JAR}" ]]; then + RAT_JAR="${TEMP_DIR}/apache-rat-${RAT_VERSION}.jar" + download_file \ + "https://repo.maven.apache.org/maven2/org/apache/rat/apache-rat/${RAT_VERSION}/apache-rat-${RAT_VERSION}.jar" \ + "${RAT_JAR}" + fi [[ -f "${RAT_JAR}" ]] || fail "Apache RAT jar does not exist: ${RAT_JAR}" RAT_REPORT="${TEMP_DIR}/rat-report.txt" @@ -218,10 +349,21 @@ else fi if [[ "${SKIP_BUILD}" == false ]]; then - "${SOURCE_DIR}/ci/scripts/build_paimon.sh" "${SOURCE_DIR}" false false Release + INSTALL_SMOKE=true + if [[ "${SKIP_INSTALL}" == true ]]; then + INSTALL_SMOKE=false + fi + PAIMON_BUILD_JOBS="${JOBS}" \ + "${SOURCE_DIR}/ci/scripts/build_paimon.sh" \ + "${SOURCE_DIR}" false false Release "${INSTALL_SMOKE}" echo "Release build and tests: valid" + if [[ "${INSTALL_SMOKE}" == true ]]; then + echo "Install and consumer smoke test: valid" + else + echo "Install and consumer smoke test: skipped" + fi else - echo "Release build and tests: skipped" + echo "Release build, tests, and install smoke test: skipped" fi echo "Release candidate verification completed successfully."