diff --git a/.github/workflows/check-style.yml b/.github/workflows/check-style.yml index 108a2de903310c..b265fe35cbfc51 100644 --- a/.github/workflows/check-style.yml +++ b/.github/workflows/check-style.yml @@ -20,7 +20,7 @@ jobs: jobname: ClangFormat runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: fetch-depth: 0 diff --git a/.github/workflows/check-whitespace.yml b/.github/workflows/check-whitespace.yml index ea6f49f742108e..3379f89a814abd 100644 --- a/.github/workflows/check-whitespace.yml +++ b/.github/workflows/check-whitespace.yml @@ -19,7 +19,7 @@ jobs: check-whitespace: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: fetch-depth: 0 diff --git a/.github/workflows/codex-pgo-training.sh b/.github/workflows/codex-pgo-training.sh new file mode 100644 index 00000000000000..070d8b54cae486 --- /dev/null +++ b/.github/workflows/codex-pgo-training.sh @@ -0,0 +1,60 @@ +#!/bin/sh + +# Keep this workload short and biased toward the local Git operations Codex +# invokes frequently. The full Git test suite is too slow for each release +# target and would weight test-harness paths more heavily than status, diff, +# clone, fetch, and repository maintenance. + +set -eu + +git_bin="$PWD/bin-wrappers/git" +training_dir=$(mktemp -d "${TMPDIR:-/tmp}/codex-git-pgo.XXXXXX") +repo="$training_dir/repo" +clone="$training_dir/clone" + +cleanup () { + rm -rf "$training_dir" +} +trap cleanup EXIT HUP INT TERM + +mkdir -p "$training_dir/home" +export HOME="$training_dir/home" +export GIT_CONFIG_NOSYSTEM=1 +export GIT_TERMINAL_PROMPT=0 + +"$git_bin" clone --quiet --no-local "$PWD" "$repo" +"$git_bin" -C "$repo" config user.name "Codex Git PGO" +"$git_bin" -C "$repo" config user.email "codex-git-pgo@openai.com" + +i=0 +while test "$i" -lt 256 +do + dir="$repo/training/$((i % 16))" + mkdir -p "$dir" + printf '%s\n' "$i" >"$dir/file-$i" + i=$((i + 1)) +done + +"$git_bin" -C "$repo" status --porcelain=v2 --branch >/dev/null +"$git_bin" -C "$repo" status --porcelain=v2 --branch --untracked-files=all >/dev/null +"$git_bin" -C "$repo" ls-files --others --exclude-standard >/dev/null +"$git_bin" -C "$repo" add training +"$git_bin" -C "$repo" diff --cached --stat >/dev/null +"$git_bin" -C "$repo" commit --quiet -m "add training files" + +printf 'changed\n' >>"$repo/training/0/file-0" +rm "$repo/training/1/file-1" +mkdir -p "$repo/untracked" +printf 'new\n' >"$repo/untracked/file" + +"$git_bin" -C "$repo" status --porcelain=v2 --branch >/dev/null +"$git_bin" -C "$repo" diff --stat >/dev/null +"$git_bin" -C "$repo" diff --name-status >/dev/null +"$git_bin" -C "$repo" ls-files --stage >/dev/null +"$git_bin" -C "$repo" log --oneline --decorate -20 >/dev/null +"$git_bin" -C "$repo" rev-list --objects --all >/dev/null +"$git_bin" -C "$repo" for-each-ref --format='%(refname) %(objectname)' >/dev/null +"$git_bin" -C "$repo" repack -ad +"$git_bin" clone --quiet --no-local "$repo" "$clone" +"$git_bin" -C "$clone" status --porcelain=v2 --branch >/dev/null +"$git_bin" -C "$clone" fetch --quiet "$repo" diff --git a/.github/workflows/codex-release.yml b/.github/workflows/codex-release.yml new file mode 100644 index 00000000000000..a08b76c5062dce --- /dev/null +++ b/.github/workflows/codex-release.yml @@ -0,0 +1,651 @@ +name: Codex Git release + +on: + workflow_call: + inputs: + source_sha: + description: Full source commit in the caller repository; build only. + required: true + type: string + version: + required: true + type: string + upstream_tag: + required: true + type: string + recipe_sha: + description: Full commit used to call this workflow. + required: true + type: string + push: + branches: + - codex + - codex-unstable + +permissions: + contents: read + +concurrency: + group: codex-git-release-${{ github.repository }}-${{ inputs.source_sha || github.sha }} + cancel-in-progress: false + +jobs: + publication: + name: Verify controller publication + runs-on: ubuntu-24.04 + if: inputs.source_sha == '' && github.event.deleted == false + outputs: + published: ${{ steps.verify.outputs.published }} + steps: + - name: Check the published controller output + id: verify + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + case "$GITHUB_REF" in + refs/heads/codex) + output_key=codex.output-tip + ;; + refs/heads/codex-unstable) + output_key=codex-unstable.output-tip + ;; + *) + printf 'unexpected release ref: %s\n' "$GITHUB_REF" >&2 + exit 1 + ;; + esac + + meta=$(gh api \ + "repos/$GITHUB_REPOSITORY/git/ref/heads/meta" \ + --jq '.object.sha') + recorded=$(gh api \ + "repos/$GITHUB_REPOSITORY/contents/codex.config?ref=$meta" \ + -H 'Accept: application/vnd.github.raw+json' | + git config --no-includes --file /dev/stdin \ + --get "$output_key") + + if test "$GITHUB_SHA" = "$recorded" + then + printf 'published=true\n' >>"$GITHUB_OUTPUT" + printf 'Releasing controller-published commit %s.\n' "$GITHUB_SHA" + else + printf 'published=false\n' >>"$GITHUB_OUTPUT" + printf 'Skipping non-controller publication %s.\n' "$GITHUB_SHA" + fi + + version: + name: Determine version + needs: publication + if: ${{ !cancelled() && (inputs.source_sha != '' || needs.publication.outputs.published == 'true') }} + runs-on: ubuntu-24.04 + outputs: + describe: ${{ steps.version.outputs.describe }} + upstream_tag: ${{ inputs.upstream_tag || steps.version.outputs.upstream_tag }} + version: ${{ inputs.version || steps.version.outputs.version }} + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + if: inputs.source_sha == '' + with: + fetch-depth: 0 + persist-credentials: false + + - name: Derive OpenAI version from git describe + if: inputs.source_sha == '' + id: version + shell: bash + run: | + set -euo pipefail + describe="$(git describe \ + --match 'v[0-9]*' \ + --exclude 'v*-openai.*' \ + --long \ + --always \ + --abbrev=12 \ + "$GITHUB_SHA")" + + if [[ "$describe" =~ ^(.+)-([0-9]+)-g([0-9a-f]+)$ ]] + then + upstream_tag="${BASH_REMATCH[1]}" + version="$upstream_tag-openai.${BASH_REMATCH[2]}.g${BASH_REMATCH[3]}" + else + upstream_tag= + version="openai-$describe" + fi + if test "$GITHUB_REF_NAME" = codex-unstable + then version="$version.codex-unstable" + fi + git check-ref-format "refs/tags/$version" + printf 'describe=%s\n' "$describe" | tee -a "$GITHUB_OUTPUT" + printf 'upstream_tag=%s\n' "$upstream_tag" | tee -a "$GITHUB_OUTPUT" + printf 'version=%s\n' "$version" | tee -a "$GITHUB_OUTPUT" + + - name: Validate requested build inputs + if: inputs.source_sha != '' + env: + SOURCE_SHA: ${{ inputs.source_sha }} + RECIPE_SHA: ${{ inputs.recipe_sha }} + VERSION: ${{ inputs.version }} + run: | + [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ && "$RECIPE_SHA" =~ ^[0-9a-f]{40}$ ]] + [[ "$VERSION" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] + + build: + if: ${{ !cancelled() && needs.version.result == 'success' }} + name: ${{ matrix.name }} + needs: version + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + env: + SOURCE_SHA: ${{ inputs.source_sha || github.sha }} + RECIPE_SHA: ${{ inputs.recipe_sha || github.sha }} + strategy: + fail-fast: false + matrix: + include: + - name: macOS arm64 + os: macos-15 + target_platform: macOS + asset_platform: macOS + arch: arm64 + binary: /tmp/build/git/bin/git + file_pattern: Mach-O 64-bit executable arm64 + has_gcm: false + lto: thin + profile_format: LLVM + llvm_profdata: xcrun llvm-profdata + - name: macOS x64 + os: macos-15-intel + target_platform: macOS + asset_platform: macOS + arch: x64 + binary: /tmp/build/git/bin/git + file_pattern: Mach-O 64-bit executable x86_64 + has_gcm: false + lto: thin + profile_format: LLVM + llvm_profdata: xcrun llvm-profdata + # Keep arm64 builds native so release smoke tests can execute them. + - name: Linux arm64 + os: ubuntu-22.04-arm + target_platform: ubuntu + asset_platform: ubuntu + arch: arm64 + binary: /tmp/build/git/bin/git + file_pattern: ELF 64-bit.*ARM aarch64 + has_gcm: false + lto: auto + profile_format: GCC + - name: Linux x64 + os: ubuntu-22.04 + target_platform: ubuntu + asset_platform: ubuntu + arch: x64 + binary: /tmp/build/git/bin/git + file_pattern: ELF 64-bit.*x86-64 + has_gcm: false + lto: auto + profile_format: GCC + - name: Windows arm64 + os: windows-11-arm + target_platform: win32 + asset_platform: windows + arch: arm64 + binary: /tmp/build/git/clangarm64/bin/git.exe + file_pattern: PE32\+.*ARM64 + has_gcm: true + lto: thin + profile_format: LLVM + llvm_profdata: llvm-profdata + sdk_arch: aarch64 + sdk_flavor: full + mingw_dir: clangarm64 + mingit_arch: arm64 + mingit_filename: MinGit-2.55.0.2-arm64.zip + mingit_url: https://github.com/git-for-windows/git/releases/download/v2.55.0.windows.2/MinGit-2.55.0.2-arm64.zip + mingit_sha256: 0b2b81fdce284efd174cbb51b886ccea2fd271679c4b5c21f07d9e03bae51413 + - name: Windows x64 + os: windows-2025 + target_platform: win32 + asset_platform: windows + arch: x64 + binary: /tmp/build/git/mingw64/bin/git.exe + file_pattern: PE32\+.*x86-64 + has_gcm: true + lto: auto + profile_format: GCC + sdk_arch: x86_64 + sdk_flavor: full + mingw_dir: mingw64 + mingit_arch: amd64 + mingit_filename: MinGit-2.55.0.2-64-bit.zip + mingit_url: https://github.com/git-for-windows/git/releases/download/v2.55.0.windows.2/MinGit-2.55.0.2-64-bit.zip + mingit_sha256: e3ea2944cea4b3fabcd69c7c1669ef69b1b66c05ac7806d81224d0abad2dec31 + + steps: + - name: Check out the manifest helper from the recipe + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: openai/git + ref: ${{ inputs.recipe_sha || github.sha }} + path: release-tooling + sparse-checkout: .github/workflows/git-release-manifest.sh + sparse-checkout-cone-mode: false + persist-credentials: false + + # Keep the packaging contract, dependency pins, and platform build logic + # aligned with the artifacts already consumed by Codex and GitHub Desktop. + - name: Check out Dugite Native + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: dreynaud-oai/dugite-native + ref: b6f4473557acb85433fdf9deffe0854a34fd9cc5 + path: dugite-native + fetch-depth: 0 + persist-credentials: false + + - name: Check out this Git revision + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + ref: ${{ inputs.source_sha || github.sha }} + path: dugite-native/git + fetch-depth: 1 + persist-credentials: false + + - name: Give the source an immutable package version + shell: bash + working-directory: dugite-native/git + env: + VERSION: ${{ needs.version.outputs.version }} + run: | + git \ + -c 'user.name=github-actions[bot]' \ + -c 'user.email=41898282+github-actions[bot]@users.noreply.github.com' \ + tag -a "$VERSION" -m "$VERSION" + + - name: Install OpenAI build configuration + shell: bash + working-directory: dugite-native/git + run: cp config.mak.openai config.mak + + # Match Dugite Native's compatibility choice for its macOS x64 build. + - name: Select Xcode 16.4 + if: matrix.target_platform == 'macOS' && matrix.arch == 'x64' + run: | + sudo xcode-select -s /Applications/Xcode_16.4.app/Contents/Developer/ + sudo rm -rf /Library/Developer/CommandLineTools + + - name: Install Linux build dependencies + if: matrix.target_platform == 'ubuntu' + run: | + sudo apt-get update + sudo apt-get install -y \ + autoconf \ + automake \ + build-essential \ + ca-certificates \ + curl \ + gettext \ + jq \ + lsb-release \ + pkg-config + + - name: Install Linux x64 build dependencies + if: matrix.target_platform == 'ubuntu' && matrix.arch == 'x64' + run: | + sudo apt-get install -y \ + libcurl4-gnutls-dev \ + libexpat1-dev \ + libssl-dev \ + zlib1g-dev + + - name: Install Linux arm64 build dependencies + if: matrix.target_platform == 'ubuntu' && matrix.arch == 'arm64' + run: | + sudo apt-get install -y \ + binutils-aarch64-linux-gnu \ + gcc-aarch64-linux-gnu \ + libcurl4-gnutls-dev \ + libexpat1-dev \ + libssl-dev \ + zlib1g-dev + + # Dugite Native currently pins MinGit 2.53. Keep its build script and + # dependency schema, but match the runtime to the Git series we compile. + - name: Select the matching MinGit runtime + if: matrix.target_platform == 'win32' + shell: bash + working-directory: dugite-native + env: + MINGIT_ARCH: ${{ matrix.mingit_arch }} + MINGIT_FILENAME: ${{ matrix.mingit_filename }} + MINGIT_SHA256: ${{ matrix.mingit_sha256 }} + MINGIT_URL: ${{ matrix.mingit_url }} + MINGIT_VERSION: v2.55.0 + SOURCE_UPSTREAM_TAG: ${{ needs.version.outputs.upstream_tag }} + run: | + set -euo pipefail + test "$SOURCE_UPSTREAM_TAG" = "$MINGIT_VERSION" + updated="$(mktemp)" + jq \ + --arg arch "$MINGIT_ARCH" \ + --arg checksum "$MINGIT_SHA256" \ + --arg filename "$MINGIT_FILENAME" \ + --arg url "$MINGIT_URL" \ + --arg version "$MINGIT_VERSION" \ + '.git.version = $version | + (.git.files[] | + select(.platform == "windows" and .arch == $arch)) |= + (.filename = $filename | + .url = $url | + .checksum = $checksum)' \ + dependencies.json >"$updated" + mv "$updated" dependencies.json + + # Codex does not configure or invoke GCM on macOS or Linux. The + # self-contained .NET payload accounts for most of those bundles, while + # Windows MinGit configures credential.helper=manager and must retain it. + - name: Omit unused GCM from POSIX bundles + if: matrix.target_platform != 'win32' + shell: bash + working-directory: dugite-native + run: | + set -euo pipefail + updated="$(mktemp)" + jq '."git-credential-manager".files = []' \ + dependencies.json >"$updated" + mv "$updated" dependencies.json + + # Build an instrumented Git, run a representative local workload, then + # rebuild with its profile. Keep Git's optional Rust library disabled + # until its Makefile can direct Cargo at these targets. + - name: Build the Dugite Native distribution + shell: bash + working-directory: dugite-native + env: + NO_RUST: 1 + OPENAI_LLVM_PROFDATA: ${{ matrix.llvm_profdata }} + OPENAI_LTO: ${{ matrix.lto }} + OPENAI_PROFILE: BUILD + OPENAI_PROFILE_FORMAT: ${{ matrix.profile_format }} + TARGET_PLATFORM: ${{ matrix.target_platform }} + TARGET_ARCH: ${{ matrix.arch }} + run: | + set -euo pipefail + script/build.sh + + - name: Set up Git for Windows SDK + if: matrix.target_platform == 'win32' && matrix.arch != 'x64' + uses: git-for-windows/setup-git-for-windows-sdk@335917db02da4280d3d5e87915d7b86196677f9f # v2 + with: + architecture: ${{ matrix.sdk_arch }} + flavor: ${{ matrix.sdk_flavor }} + cache: false + + # Keep the MINGW64 toolchain paired with our pinned MinGit runtime. + # The SDK's main branch has moved to UCRT64. + - name: Check out the pinned Windows x64 SDK + if: matrix.target_platform == 'win32' && matrix.arch == 'x64' + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: git-for-windows/git-sdk-64 + ref: 7ca76c34e6aa2edcba062582a5338be2b50fe769 + path: windows-sdk + persist-credentials: false + + - name: Select the pinned Windows x64 SDK + if: matrix.target_platform == 'win32' && matrix.arch == 'x64' + shell: bash + run: | + set -euo pipefail + for directory in usr/bin/core_perl usr/bin mingw64/bin + do + cygpath -w "$GITHUB_WORKSPACE/windows-sdk/$directory" >>"$GITHUB_PATH" + done + printf 'MSYSTEM=MINGW64\nLC_CTYPE=C.UTF-8\n' >>"$GITHUB_ENV" + + # Dugite Native compiles its Git submodule on macOS and Linux. On + # Windows it starts from MinGit, so replace MinGit's Git programs with + # the build from this repository while retaining the portable runtime. + # MinGit omits dashed builtin aliases; installing them as copies would + # add hundreds of redundant MiB to the archive. + - name: Install this Git build into the Windows distribution + if: matrix.target_platform == 'win32' + shell: bash + working-directory: dugite-native/git + env: + MINGW_DIR: ${{ matrix.mingw_dir }} + OPENAI_LLVM_PROFDATA: ${{ matrix.llvm_profdata }} + OPENAI_LTO: ${{ matrix.lto }} + OPENAI_PROFILE_FORMAT: ${{ matrix.profile_format }} + run: | + set -euo pipefail + + make_args=( + "prefix=/$MINGW_DIR" + NO_PERL=YesPlease + NO_RUST=YesPlease + NO_TCLTK=YesPlease + NO_GETTEXT=YesPlease + NO_INSTALL_HARDLINKS=YesPlease + NO_CROSS_DIRECTORY_HARDLINKS=YesPlease + SKIP_DASHED_BUILT_INS=YesPlease + ) + jobs="${NUMBER_OF_PROCESSORS:-2}" + make -j"$jobs" "${make_args[@]}" OPENAI_PROFILE=BUILD all + make "${make_args[@]}" OPENAI_PROFILE=USE DESTDIR=/tmp/build/git strip install + + - name: Verify distribution layout and provenance + shell: bash + env: + TARGET_PLATFORM: ${{ matrix.target_platform }} + MINGW_DIR: ${{ matrix.mingw_dir }} + GIT_BINARY: ${{ matrix.binary }} + FILE_PATTERN: ${{ matrix.file_pattern }} + HAS_GCM: ${{ matrix.has_gcm }} + LTO: ${{ matrix.lto }} + PROFILE_FORMAT: ${{ matrix.profile_format }} + run: | + set -euo pipefail + if test "$TARGET_PLATFORM" = win32 + then + test -f /tmp/build/git/cmd/git.exe + test -f "/tmp/build/git/$MINGW_DIR/libexec/git-core/git-lfs.exe" + test -d "/tmp/build/git/$MINGW_DIR/share/git-core/templates" + test ! -e "/tmp/build/git/$MINGW_DIR/libexec/git-core/git-add.exe" + if test "$HAS_GCM" = true + then + test -f "/tmp/build/git/$MINGW_DIR/bin/git-credential-manager.exe" + fi + else + test -x /tmp/build/git/libexec/git-core/git-lfs + test -d /tmp/build/git/share/git-core/templates + if test "$HAS_GCM" = true + then + test -x /tmp/build/git/libexec/git-core/git-credential-manager + else + test ! -e /tmp/build/git/libexec/git-core/git-credential-manager + fi + fi + test -f /tmp/build/git/etc/gitconfig + + file "$GIT_BINARY" | tee /tmp/git-file-type + grep -E "$FILE_PATTERN" /tmp/git-file-type + strings "$GIT_BINARY" | grep -F "$SOURCE_SHA" + grep -F -- "-flto=$LTO" dugite-native/git/GIT-CFLAGS + if test "$PROFILE_FORMAT" = LLVM + then + grep -F -- "-fprofile-instr-use=" dugite-native/git/GIT-CFLAGS + else + grep -F -- "-fprofile-use=" dugite-native/git/GIT-CFLAGS + fi + + - name: Smoke-test the native distribution + shell: bash + env: + TARGET_PLATFORM: ${{ matrix.target_platform }} + MINGW_DIR: ${{ matrix.mingw_dir }} + HAS_GCM: ${{ matrix.has_gcm }} + run: | + set -euo pipefail + smoke=/tmp/codex-git-smoke + mkdir -p "$smoke/home" + + if test "$TARGET_PLATFORM" = win32 + then + git_binary=/tmp/build/git/cmd/git.exe + git_env=( + "PATH=/tmp/build/git/cmd:/tmp/build/git/$MINGW_DIR/bin:/tmp/build/git/usr/bin:$PATH" + ) + else + git_binary=/tmp/build/git/bin/git + git_env=( + GIT_CONFIG_SYSTEM=/tmp/build/git/etc/gitconfig + GIT_EXEC_PATH=/tmp/build/git/libexec/git-core + GIT_TEMPLATE_DIR=/tmp/build/git/share/git-core/templates + ) + if test "$TARGET_PLATFORM" = ubuntu + then + git_env+=( + GIT_SSL_CAINFO=/tmp/build/git/ssl/cacert.pem + PREFIX=/tmp/build/git + ) + fi + fi + git_env+=("HOME=$smoke/home" GIT_TERMINAL_PROMPT=0) + + build_options="$(env "${git_env[@]}" "$git_binary" --version --build-options)" + printf '%s\n' "$build_options" + grep -F "built from commit: $SOURCE_SHA" <<<"$build_options" + env "${git_env[@]}" "$git_binary" lfs version + if test "$HAS_GCM" = true + then + env "${git_env[@]}" "$git_binary" credential-manager --version + fi + + env "${git_env[@]}" "$git_binary" init --quiet "$smoke/repo" + echo test >"$smoke/repo/file" + env "${git_env[@]}" "$git_binary" -C "$smoke/repo" add file + env "${git_env[@]}" "$git_binary" -C "$smoke/repo" \ + -c user.name='Codex Git CI' \ + -c user.email='codex-git-ci@openai.com' \ + commit --quiet -m initial + test -z "$(env "${git_env[@]}" "$git_binary" -C "$smoke/repo" status --porcelain)" + + - name: Package with Dugite Native + shell: bash + working-directory: dugite-native + env: + ASSET_PLATFORM: ${{ matrix.asset_platform }} + TARGET_PLATFORM: ${{ matrix.target_platform }} + TARGET_ARCH: ${{ matrix.arch }} + VERSION: ${{ needs.version.outputs.version }} + run: | + set -euo pipefail + script/package.sh + + for extension in tar.gz lzma + do + matches=( + output/dugite-native-"$VERSION"-*-"$ASSET_PLATFORM"-"$TARGET_ARCH.$extension" + ) + test "${#matches[@]}" -eq 1 + test -f "${matches[0]}" + + destination="output/git-$VERSION-$ASSET_PLATFORM-$TARGET_ARCH.$extension" + mv "${matches[0]}" "$destination" + mv "${matches[0]}.sha256" "$destination.sha256" + done + + bash ../release-tooling/.github/workflows/git-release-manifest.sh \ + record output "$ASSET_PLATFORM-$TARGET_ARCH" + + - name: Upload release assets + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: git-${{ needs.version.outputs.version }}-${{ matrix.asset_platform }}-${{ matrix.arch }} + path: dugite-native/output/git-* + if-no-files-found: error + retention-days: 7 + + # The arm64 SDK puts its target Git first on PATH, but action cleanup + # runs on the x64 host and therefore needs the runner's native Git. + - name: Restore native Git for action cleanup + if: always() && matrix.target_platform == 'win32' + shell: pwsh + run: | + "C:\Program Files\Git\cmd" | + Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + + release: + if: inputs.source_sha == '' + name: Publish GitHub prerelease + needs: + - version + - build + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: write + steps: + - name: Check out the manifest helper + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + ref: ${{ github.sha }} + sparse-checkout: .github/workflows/git-release-manifest.sh + sparse-checkout-cone-mode: false + persist-credentials: false + + - name: Download release assets + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + pattern: git-${{ needs.version.outputs.version }}-* + path: artifacts + merge-multiple: true + + - name: Combine the verified native receipts + env: + SOURCE_SHA: ${{ github.sha }} + RECIPE_SHA: ${{ github.sha }} + VERSION: ${{ needs.version.outputs.version }} + UPSTREAM_TAG: ${{ needs.version.outputs.upstream_tag }} + CHANNEL: ${{ github.ref_name }} + VISIBILITY: public + run: bash .github/workflows/git-release-manifest.sh collect artifacts >manifest.json + + - name: Publish immutable prerelease + env: + GH_TOKEN: ${{ github.token }} + SOURCE_DESCRIPTION: ${{ needs.version.outputs.describe }} + SOURCE_REF: ${{ github.ref }} + VERSION: ${{ needs.version.outputs.version }} + run: | + set -euo pipefail + assets=(manifest.json) + for file in artifacts/git-* + do + case "$file" in *.build.json) continue ;; esac + assets+=("$file") + done + release_notes=$( + printf '%s\n' \ + "source_ref=$SOURCE_REF" \ + "source_sha=$GITHUB_SHA" \ + "" \ + "OpenAI Git release artifacts for $SOURCE_DESCRIPTION, built from $GITHUB_SHA for Codex." + ) + + if gh release view "$VERSION" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1 + then + gh release upload "$VERSION" "${assets[@]}" \ + --repo "$GITHUB_REPOSITORY" \ + --clobber + else + gh release create "$VERSION" "${assets[@]}" \ + --repo "$GITHUB_REPOSITORY" \ + --target "$GITHUB_SHA" \ + --title "$VERSION" \ + --notes "$release_notes" \ + --prerelease + fi diff --git a/.github/workflows/codex.yml b/.github/workflows/codex.yml new file mode 100644 index 00000000000000..650cfd55baef14 --- /dev/null +++ b/.github/workflows/codex.yml @@ -0,0 +1,243 @@ +name: Refresh codex + +on: + schedule: + - cron: '*/5 * * * *' + workflow_dispatch: + inputs: + operation: + description: Refresh, scan, remove, or reorder a pinned topic + type: choice + options: + - refresh + - scan + - remove + - reorder + default: refresh + lane: + description: codex or codex-unstable for a plan operation + required: false + type: string + topic: + description: Exact topic branch for a plan operation + required: false + type: string + after: + description: Existing topic or root for reorder + required: false + type: string + plan_branch: + description: Optional codex-plan/* branch name + required: false + type: string + pull_request_target: + branches: + - meta + types: + - opened + - reopened + - synchronize + - ready_for_review + +permissions: + actions: read + contents: read + pull-requests: read + +jobs: + refresh: + if: >- + github.event_name == 'workflow_dispatch' && + github.ref == 'refs/heads/codex' && + inputs.operation == 'refresh' + uses: openai/git/.github/workflows/codex.yml@meta + topic_plan_scan: + name: Find one approved topic plan + if: >- + github.event_name == 'schedule' || + (github.event_name == 'workflow_dispatch' && + github.ref == 'refs/heads/codex' && + inputs.operation == 'scan') + runs-on: ubuntu-24.04 + permissions: + contents: read + pull-requests: read + concurrency: + group: codex-topic-plan-scan + cancel-in-progress: false + outputs: + lane: ${{ steps.reviewed.outputs.lane }} + topic: ${{ steps.reviewed.outputs.topic }} + source_tip: ${{ steps.reviewed.outputs.source_tip }} + review_pr: ${{ steps.reviewed.outputs.review_pr }} + env: + GH_TOKEN: ${{ github.token }} + steps: + - name: Pin trusted meta + id: meta + run: | + set -euo pipefail + test "$GITHUB_REPOSITORY" = openai/git + test "$GITHUB_REF" = refs/heads/codex + sha=$(gh api "repos/$GITHUB_REPOSITORY/git/ref/heads/meta" \ + --jq .object.sha) + case "$sha" in + ''|*[!0-9a-f]*) exit 1 ;; + esac + test "${#sha}" = 40 + printf 'sha=%s\n' "$sha" >>"$GITHUB_OUTPUT" + + - name: Check out trusted meta + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + repository: ${{ github.repository }} + ref: ${{ steps.meta.outputs.sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Find one exact approved topic PR + id: reviewed + env: + META_SHA: ${{ steps.meta.outputs.sha }} + run: | + set -euo pipefail + + die () { + printf '%s\n' "$*" >&2 + exit 1 + } + + test "$GITHUB_REPOSITORY" = openai/git + test "$GITHUB_REF" = refs/heads/codex || + die "topic scan must run from the trusted default branch" + test "$(git rev-parse HEAD)" = "$META_SHA" || + die "trusted checkout does not match pinned meta" + gh auth setup-git + mkdir -p "$RUNNER_TEMP/codex-plan-scan" + for lane in codex codex-unstable + do + case "$lane" in + codex) plan=codex.plan ;; + codex-unstable) plan=codex-unstable.plan ;; + esac + test -f "$plan" || + die "trusted meta has no $plan" + gh pr list --repo "$GITHUB_REPOSITORY" --state open \ + --base "$lane" --limit 1000 \ + --json number,isDraft,headRefName,headRefOid,headRepository,reviewDecision | + jq -r --arg lane "$lane" ' + .[] | + select(.isDraft | not) | + select(.reviewDecision == "APPROVED") | + select(.headRepository.nameWithOwner == "openai/git") | + [$lane, .headRefName, .headRefOid, + (.number | tostring)] | @tsv + ' + done | sort -k4,4n >"$RUNNER_TEMP/codex-plan-scan/candidates" + + while IFS=$'\t' read -r lane topic source_tip review_pr + do + test -n "$review_pr" || continue + case "$review_pr" in + *[!0-9]*) die "approved topic PR has invalid number '$review_pr'" ;; + esac + case "$source_tip" in + *[!0-9a-f]*|'') die "approved topic PR has invalid source SHA" ;; + esac + test "${#source_tip}" = 40 || + die "approved topic PR has invalid source SHA" + git check-ref-format "refs/heads/$topic" >/dev/null 2>&1 || + die "approved topic PR has invalid branch '$topic'" + case "$topic" in + ??/codex/*) ;; + *) continue ;; + esac + suffix=${topic#??/codex/} + case "$suffix" in + ''|*/*|*-wip|*-stale) continue ;; + esac + case "$lane" in + codex) + case "$topic" in + *-unstable) continue ;; + esac + plan=codex.plan + ;; + codex-unstable) + case "$topic" in + *-unstable) ;; + *) continue ;; + esac + plan=codex-unstable.plan + ;; + *) die "approved topic PR has invalid lane '$lane'" ;; + esac + pinned=$(git config --no-includes \ + --file "$plan" \ + --get "branch.$topic.source-tip" || :) + test "$pinned" = "$source_tip" && continue + short=$(printf '%.12s' "$source_tip") + slug=${topic##*/} + plan_branch=codex-plan/$lane-$slug-$short + pending=$(gh pr list --repo "$GITHUB_REPOSITORY" \ + --state open --base meta --head "$plan_branch" \ + --json number --jq '.[0].number // empty') || + die "could not inspect pending Codex plan PR" + test -n "$pending" && continue + if ! sh .github/workflows/codex-branch.sh propose-plan \ + --remote origin --lane "$lane" --topic "$topic" \ + --action auto --source-tip "$source_tip" \ + --review-pr "$review_pr" --expected-meta "$META_SHA" \ + --no-push >/dev/null + then + printf 'skipping approved topic PR #%s: preflight failed\n' \ + "$review_pr" >&2 + continue + fi + { + printf 'lane=%s\n' "$lane" + printf 'topic=%s\n' "$topic" + printf 'source_tip=%s\n' "$source_tip" + printf 'review_pr=%s\n' "$review_pr" + } >>"$GITHUB_OUTPUT" + exit 0 + done <"$RUNNER_TEMP/codex-plan-scan/candidates" + topic_plan_propose: + name: Propose reviewed topic plan + needs: topic_plan_scan + if: needs.topic_plan_scan.outputs.review_pr != '' + permissions: + contents: read + pull-requests: read + uses: openai/git/.github/workflows/codex-plan-propose.yml@meta + with: + lane: ${{ needs.topic_plan_scan.outputs.lane }} + topic: ${{ needs.topic_plan_scan.outputs.topic }} + action: auto + source_tip: ${{ needs.topic_plan_scan.outputs.source_tip }} + review_pr: ${{ needs.topic_plan_scan.outputs.review_pr }} + policy_plan_propose: + name: Propose explicit plan policy + if: >- + github.event_name == 'workflow_dispatch' && + github.ref == 'refs/heads/codex' && + (inputs.operation == 'remove' || inputs.operation == 'reorder') + permissions: + contents: read + pull-requests: read + uses: openai/git/.github/workflows/codex-plan-propose.yml@meta + with: + lane: ${{ inputs.lane }} + topic: ${{ inputs.topic }} + action: ${{ inputs.operation }} + after: ${{ inputs.after }} + plan_branch: ${{ inputs.plan_branch }} + plan_admission: + name: Codex plan admission + if: >- + github.event_name == 'pull_request_target' && + github.event.pull_request.base.ref == 'meta' + permissions: + contents: read + pull-requests: write + uses: openai/git/.github/workflows/codex-plan-admission.yml@meta diff --git a/.github/workflows/git-release-manifest.sh b/.github/workflows/git-release-manifest.sh new file mode 100755 index 00000000000000..9ad9ac649f250c --- /dev/null +++ b/.github/workflows/git-release-manifest.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Record native artifacts, then combine only matching receipts into a manifest. +set -euo pipefail +test "$#" -ge 2 +mode=$1 directory=$2 +scratch=$(mktemp -d) +trap 'rm -rf "$scratch"' EXIT +[[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ && "$RECIPE_SHA" =~ ^[0-9a-f]{40}$ ]] +[[ "$VERSION" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] +hash () { + if command -v sha256sum >/dev/null + then sha256sum "$1" + else shasum -a 256 "$1" + fi | awk '{print $1}' +} +record () { + local target=$1 archive file sum bytes limit=67108864 + case "$target" in + windows-arm64|windows-x64) limit=134217728 ;; + macOS-arm64|macOS-x64|ubuntu-arm64|ubuntu-x64) ;; + *) return 1 ;; + esac + : >"$scratch/assets" + for extension in tar.gz lzma + do + archive=$directory/git-$VERSION-$target.$extension + for file in "$archive" "$archive.sha256" + do + test -f "$file" && test ! -L "$file" + sum=$(hash "$file"); bytes=$(wc -c <"$file") + test "$bytes" -gt 0 + jq -n --arg name "${file##*/}" --arg hash "$sum" --argjson size "$bytes" \ + '{name:$name,sha256:$hash,size:$size}' >>"$scratch/assets" + done + test "$(tr -d '\r\n' <"$archive.sha256")" = "$(hash "$archive")" + done + test "$(wc -c <"$directory/git-$VERSION-$target.tar.gz")" -le "$limit" + jq -S -a -s --arg repository "$GITHUB_REPOSITORY" --arg version "$VERSION" \ + --arg source "$SOURCE_SHA" --arg recipe "$RECIPE_SHA" --arg run "$GITHUB_RUN_ID" \ + --arg target "$target" '{repository:$repository,source_sha:$source,version:$version, + recipe:{repository:"openai/git",sha:$recipe},run_id:$run,target:$target,assets:.}' "$scratch/assets" +} +case "$mode" in +record) record "${3:?target required}" >"$directory/git-$VERSION-$3.build.json" ;; +collect) + for target in macOS-arm64 macOS-x64 ubuntu-arm64 ubuntu-x64 windows-arm64 windows-x64 + do + receipt=$directory/git-$VERSION-$target.build.json + test -f "$receipt" && test ! -L "$receipt" + record "$target" >"$scratch/$target.json" + jq -S -a . "$receipt" >"$scratch/receipt" + cmp "$scratch/$target.json" "$scratch/receipt" + done + shopt -s nullglob dotglob + files=("$directory"/*) + test "${#files[@]}" -eq 30 + jq -S -a -s --arg channel "$CHANNEL" --arg tag "$UPSTREAM_TAG" --arg visibility "$VISIBILITY" ' + .[0] as $first | {schema_version:1,repository:$first.repository,visibility:$visibility, + channel:$channel,version:$first.version,source_sha:$first.source_sha,upstream_tag:$tag, + recipe:$first.recipe,targets:(map({key:.target,value:.assets}) | from_entries), + builds:(map({key:.target,value:.}) | from_entries)}' "$scratch/"*.json + ;; +*) echo 'usage: git-release-manifest.sh record | collect ' >&2; exit 129 ;; +esac diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 205325eb33b064..44337a67422e01 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -63,7 +63,7 @@ jobs: echo "skip_concurrent=$skip_concurrent" >>$GITHUB_OUTPUT - name: skip if the commit or tree was already tested id: skip-if-redundant - uses: actions/github-script@v9 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 if: steps.check-ref.outputs.enabled == 'yes' with: github-token: ${{secrets.GITHUB_TOKEN}} @@ -112,8 +112,10 @@ jobs: group: windows-build-${{ github.ref }} cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - - uses: actions/checkout@v6 - - uses: git-for-windows/setup-git-for-windows-sdk@v2 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - uses: git-for-windows/setup-git-for-windows-sdk@aed153d1258c311b680b81584a72a1a4807401f0 # v2.2.1 + with: + architecture: mingw64 - name: build shell: bash env: @@ -123,7 +125,7 @@ jobs: - name: zip up tracked files run: git archive -o artifacts/tracked.tar.gz HEAD - name: upload tracked files and build artifacts - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: windows-artifacts path: artifacts @@ -140,14 +142,16 @@ jobs: cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - name: download tracked files and build artifacts - uses: actions/download-artifact@v8 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: windows-artifacts path: ${{github.workspace}} - name: extract tracked files and build artifacts shell: bash run: tar xf artifacts.tar.gz && tar xf tracked.tar.gz - - uses: git-for-windows/setup-git-for-windows-sdk@v2 + - uses: git-for-windows/setup-git-for-windows-sdk@aed153d1258c311b680b81584a72a1a4807401f0 # v2.2.1 + with: + architecture: mingw64 - name: test shell: bash run: . /etc/profile && ci/run-test-slice.sh $((${{matrix.nr}} + 1)) 10 @@ -157,7 +161,7 @@ jobs: run: ci/print-test-failures.sh - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: failed-tests-windows-${{ matrix.nr }} path: ${{env.FAILED_TEST_ARTIFACTS}} @@ -173,20 +177,22 @@ jobs: group: vs-build-${{ github.ref }} cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - - uses: actions/checkout@v6 - - uses: git-for-windows/setup-git-for-windows-sdk@v2 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - uses: git-for-windows/setup-git-for-windows-sdk@aed153d1258c311b680b81584a72a1a4807401f0 # v2.2.1 + with: + architecture: mingw64 - name: initialize vcpkg - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: repository: 'microsoft/vcpkg' path: 'compat/vcbuild/vcpkg' - name: download vcpkg artifacts - uses: git-for-windows/get-azure-pipelines-artifact@v0 + uses: git-for-windows/get-azure-pipelines-artifact@2e424b98e1251b725d6064c5f1a9f854a33e5b88 # v0 with: repository: git/git definitionId: 9 - name: add msbuild to PATH - uses: microsoft/setup-msbuild@v3 + uses: microsoft/setup-msbuild@30375c66a4eea26614e0d39710365f22f8b0af57 # v3 - name: copy dlls to root shell: cmd run: compat\vcbuild\vcpkg_copy_dlls.bat release @@ -208,7 +214,7 @@ jobs: - name: zip up tracked files run: git archive -o artifacts/tracked.tar.gz HEAD - name: upload tracked files and build artifacts - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: vs-artifacts path: artifacts @@ -224,9 +230,11 @@ jobs: group: vs-test-${{ matrix.nr }}-${{ github.ref }} cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - - uses: git-for-windows/setup-git-for-windows-sdk@v2 + - uses: git-for-windows/setup-git-for-windows-sdk@aed153d1258c311b680b81584a72a1a4807401f0 # v2.2.1 + with: + architecture: mingw64 - name: download tracked files and build artifacts - uses: actions/download-artifact@v8 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: vs-artifacts path: ${{github.workspace}} @@ -244,7 +252,7 @@ jobs: run: ci/print-test-failures.sh - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: failed-tests-windows-vs-${{ matrix.nr }} path: ${{env.FAILED_TEST_ARTIFACTS}} @@ -258,8 +266,8 @@ jobs: group: windows-meson-build-${{ github.ref }} cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 - name: Set up dependencies shell: pwsh run: pip install meson ninja @@ -270,7 +278,7 @@ jobs: shell: pwsh run: meson compile -C build - name: Upload build artifacts - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: windows-meson-artifacts path: build @@ -286,13 +294,13 @@ jobs: group: windows-meson-test-${{ matrix.nr }}-${{ github.ref }} cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 - name: Set up dependencies shell: pwsh run: pip install meson ninja - name: Download build artifacts - uses: actions/download-artifact@v8 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: windows-meson-artifacts path: build @@ -305,7 +313,7 @@ jobs: run: ci/print-test-failures.sh - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: failed-tests-windows-meson-${{ matrix.nr }} path: ${{env.FAILED_TEST_ARTIFACTS}} @@ -341,7 +349,7 @@ jobs: TEST_OUTPUT_DIRECTORY: ${{github.workspace}}/t runs-on: ${{matrix.vector.pool}} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - run: ci/install-dependencies.sh - run: ci/run-build-and-tests.sh - name: print test failures @@ -349,7 +357,7 @@ jobs: run: ci/print-test-failures.sh - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: failed-tests-${{matrix.vector.jobname}} path: ${{env.FAILED_TEST_ARTIFACTS}} @@ -362,7 +370,7 @@ jobs: CI_JOB_IMAGE: ubuntu-latest runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - run: ci/install-dependencies.sh - run: ci/run-build-and-minimal-fuzzers.sh dockerized: @@ -411,9 +419,9 @@ jobs: # A RHEL 8 compatible distro. Supported until 2029-05-31. - jobname: almalinux-8 image: almalinux:8 - # Supported until 2026-08-31. - - jobname: debian-11 - image: debian:11 + # Supported until 2028-06-30. + - jobname: debian-12 + image: debian:12 env: jobname: ${{matrix.vector.jobname}} CC: ${{matrix.vector.cc}} @@ -441,7 +449,7 @@ jobs: else apt-get -q update && apt-get -q -y install git fi - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - run: ci/install-dependencies.sh - run: useradd builder --create-home - run: chown -R builder . @@ -451,7 +459,7 @@ jobs: run: sudo --preserve-env --set-home --user=builder ci/print-test-failures.sh - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: failed-tests-${{matrix.vector.jobname}} path: ${{env.FAILED_TEST_ARTIFACTS}} @@ -466,7 +474,7 @@ jobs: group: static-analysis-${{ github.ref }} cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - run: ci/install-dependencies.sh - run: ci/run-static-analysis.sh - run: ci/check-directional-formatting.bash @@ -482,7 +490,7 @@ jobs: group: rust-analysis-${{ github.ref }} cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - run: ci/install-dependencies.sh - run: ci/run-rust-checks.sh sparse: @@ -496,7 +504,7 @@ jobs: group: sparse-${{ github.ref }} cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - name: Install other dependencies run: ci/install-dependencies.sh - run: make sparse @@ -512,6 +520,6 @@ jobs: CI_JOB_IMAGE: ubuntu-latest runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - run: ci/install-dependencies.sh - run: ci/test-documentation.sh diff --git a/Documentation/config/http.adoc b/Documentation/config/http.adoc index 792a71b41350d4..d5af5a3114705f 100644 --- a/Documentation/config/http.adoc +++ b/Documentation/config/http.adoc @@ -421,6 +421,16 @@ if the URL is `https://user@example.com/foo/bar` a config key match of All URLs are normalized before attempting any matching (the password part, if embedded in the URL, is always ignored for matching purposes) so that equivalent URLs that are simply spelled differently will match properly. -Environment variable settings always override any matches. The URLs that are -matched against are those given directly to Git commands. This means any URLs -visited as a result of a redirection do not participate in matching. +Environment variable settings always override any matches. The URLs matched +against are normally those given directly to Git commands. ++ +During repository discovery, Git matches each redirect destination before +contacting it to select credentials, proactive authentication, additional +headers, and TLS authentication and verification settings. Proxy selection and +other HTTP settings continue to use the original URL. Changing +`http.sslBackend` during a redirect is not supported. ++ +Credential helpers are selected using the destination's `credential.*` +settings, including `credential.useHttpPath`. If the credential identity and +configuration are unchanged, Git retains the existing credentials and helper +state. diff --git a/Documentation/git-multi-pack-index.adoc b/Documentation/git-multi-pack-index.adoc index c6d23aeeb9a037..80bd66339bbc5a 100644 --- a/Documentation/git-multi-pack-index.adoc +++ b/Documentation/git-multi-pack-index.adoc @@ -125,6 +125,10 @@ compact:: as the base for the compacted result, instead of using the immediate parent of ``. The special value `none` indicates that the result should have no base. + When updating a chain, changing the base requires `` + to be the tip. Layers above `` cannot retain their + bitmap positions when the base changes. Use + `--no-write-chain-file` to write the layer separately. -- + Note that the compact command requires writing a version-2 midx that diff --git a/Documentation/gitprotocol-capabilities.adoc b/Documentation/gitprotocol-capabilities.adoc index 37f7051459f74d..9c73dea96ba110 100644 --- a/Documentation/gitprotocol-capabilities.adoc +++ b/Documentation/gitprotocol-capabilities.adoc @@ -35,8 +35,8 @@ and server advertised. As a consequence of these rules, server MUST NOT advertise capabilities it does not understand. The 'atomic', 'report-status', 'report-status-v2', 'delete-refs', -'no-ref-delta', 'quiet', and 'push-cert' capabilities are sent and -recognized by the receive-pack (push to server) process. +'explicit-haves', 'no-ref-delta', 'quiet', and 'push-cert' capabilities +are sent and recognized by the receive-pack (push to server) process. The 'ofs-delta' and 'side-band-64k' capabilities are sent and recognized by both upload-pack and receive-pack protocols. The 'agent' and 'session-id' @@ -189,6 +189,33 @@ Protocol v2 `fetch` uses the same name for the corresponding upload-pack request. There, the promise covers both the inline pack and any packs named by a `packfile-uris` section. +explicit-haves +-------------- + +The receive-pack server can advertise 'explicit-haves' to indicate that +the object IDs of its named refs are not necessarily pack prerequisites. +When a client requests this capability, it MUST NOT treat an object as a +pack prerequisite merely because the server advertised a named ref that +points to it. + +Objects named by ".have" pseudo-refs and common objects explicitly +acknowledged by push negotiation remain valid pack prerequisites. This +capability does not otherwise change how the client uses advertised refs, +including for reference matching and update preconditions. + +A client that supports 'explicit-haves' SHOULD request it whenever the +server advertises it. A server that relies on this behavior MUST verify +that the client requested the capability before accepting the pack. + +For smart HTTP, such a client SHOULD also send the bare 'explicit-haves' +Extra Parameter in its initial receive-pack discovery request, before +the server advertises refs. This lets a server choose which refs it can +advertise to that client. The Extra Parameter does not activate the +capability: the server must still advertise it, and the client must +still request it in the update command. A server that ignores the Extra +Parameter and does not advertise the capability retains the usual pack +prerequisite behavior. + agent ----- diff --git a/Documentation/gitprotocol-pack.adoc b/Documentation/gitprotocol-pack.adoc index 633deecf2d3422..f2b9caac702ea4 100644 --- a/Documentation/gitprotocol-pack.adoc +++ b/Documentation/gitprotocol-pack.adoc @@ -73,9 +73,14 @@ Parameters", and are supported by the Git, SSH, and HTTP protocols. Each Extra Parameter takes the form of `=` or ``. Servers that receive any such Extra Parameters MUST ignore all -unrecognized keys. Currently, the only Extra Parameter recognized is -"version" with a value of '1' or '2'. See linkgit:gitprotocol-v2[5] for more -information on protocol version 2. +unrecognized keys. The "version" Extra Parameter takes a value of '1' or +'2'. See linkgit:gitprotocol-v2[5] for more information on protocol +version 2. + +For smart HTTP receive-pack discovery, a client that supports the +'explicit-haves' capability can send the bare 'explicit-haves' Extra +Parameter before the server advertises refs. See linkgit:gitprotocol-capabilities[5] +for how this parameter relates to the receive-pack capability of the same name. Git Transport ------------- @@ -518,7 +523,7 @@ fetching protocol. Each reference obj-id and name on the server is sent in packet-line format to the client, followed by a flush-pkt. The only real difference is that the capability listing is different - the only possible values are 'report-status', 'report-status-v2', 'delete-refs', -'ofs-delta', 'atomic' and 'push-options'. +'ofs-delta', 'atomic', 'push-options' and 'explicit-haves'. Reference Update Request and Packfile Transfer ---------------------------------------------- @@ -580,6 +585,13 @@ After that the packfile that should contain all the objects that the server will need to complete the new references will be sent. +When 'explicit-haves' is in effect, the client MUST NOT infer pack +prerequisites from the object IDs of named refs in the reference +advertisement. Only ".have" pseudo-refs and common objects explicitly +acknowledged by push negotiation may be used as pack prerequisites. The +advertised object ID of a named ref is still used as the old-id in the +corresponding update command. + ---- packfile = "PACK" 28*(OCTET) ---- diff --git a/Documentation/technical/packfile-uri.adoc b/Documentation/technical/packfile-uri.adoc index 3a65a92dc43793..42b3937f251609 100644 --- a/Documentation/technical/packfile-uri.adoc +++ b/Documentation/technical/packfile-uri.adoc @@ -70,6 +70,20 @@ fail without retrying. Interrupted pack transfers are not retried. HTTP 429 uses `http.maxRetries`, `http.retryAfter`, and `http.maxRetryTime` (disabled by default). +Each `http-fetch` process emits Trace2 `data` events in category `http-fetch` +with key `pack/received_bytes`. This is a cumulative count of PACK response +payload received during this invocation, starting at zero. It excludes an +existing partial file, HTTP error bodies, headers, and local indexing reads. +Bytes count when received, even if writing them to disk fails. Retries do not +reset the count. Quiet and serial downloads emit the same measurements. + +Snapshots are emitted during transfer no more often than twice per second. +When the HTTP operation returns, including on failure, a final snapshot precedes +`pack/received_bytes_complete=1`. Completion describes the measurement, not +download success. If a process is killed before that marker, the last snapshot +is only a lower bound. Consumers should take the last count per process, not +sum its snapshots. These events contain no URL or credentials. + When the client downloads the given URIs, it should store them with "keep" files, just like it does with the packfile in the `packfile` section. These additional "keep" files can only be removed after the refs have been updated - diff --git a/builtin/cat-file.c b/builtin/cat-file.c index 8870a210ec2e94..b1f8efb30e671c 100644 --- a/builtin/cat-file.c +++ b/builtin/cat-file.c @@ -543,6 +543,15 @@ static void batch_object_write(const char *obj_name, if (opt->objects_filter.choice == LOFC_BLOB_LIMIT) data->info.sizep = &data->size; + /* + * With "--batch-all-objects --unordered" we read straight + * from the pack the walk handed us. If a concurrent repack + * removed it, fall back to a normal lookup; is_pack_valid() + * pins the pack's fd, so a passing check stays valid even if + * the pack is unlinked right after. + */ + if (pack && !is_pack_valid(pack)) + pack = NULL; if (pack) ret = packed_object_info(NULL, pack, offset, &data->info); else @@ -1008,10 +1017,27 @@ static int batch_one_object_oi(const struct object_id *oid, return payload->callback(oid, NULL, 0, payload->payload); } -static void batch_each_object(struct batch_options *opt, - for_each_object_fn callback, - unsigned flags, - void *_payload) +/* + * Open every pack index up front so the enumeration's object set is + * fixed: an index mmap survives unlink() of the .idx and pack fd + * pressure (close_one_pack() closes only the pack fd). This narrows + * the race with concurrent repacks, like f6b262581a88 (fsck: snapshot + * default refs before object walk, 2026-01-09). + */ +static int snapshot_pack_indexes(void) +{ + struct packed_git *p; + + repo_for_each_pack(the_repository, p) + if (open_pack_index(p)) + return -1; + return 0; +} + +static int batch_each_object(struct batch_options *opt, + for_each_object_fn callback, + unsigned flags, + void *_payload) { struct for_each_object_payload payload = { .callback = callback, @@ -1026,8 +1052,8 @@ static void batch_each_object(struct batch_options *opt, .filter = &opt->objects_filter, }; - odb_for_each_object_ext(the_repository->objects, &oi, - batch_one_object_oi, &payload, &opts); + return odb_for_each_object_ext(the_repository->objects, &oi, + batch_one_object_oi, &payload, &opts); } static int batch_objects(struct batch_options *opt) @@ -1075,6 +1101,11 @@ static int batch_objects(struct batch_options *opt) disable_replace_refs(); + if (snapshot_pack_indexes()) { + strbuf_release(&output); + return error(_("unable to enumerate all objects")); + } + cb.opt = opt; cb.expand = &data; cb.scratch = &output; @@ -1084,20 +1115,22 @@ static int batch_objects(struct batch_options *opt) cb.seen = &seen; - batch_each_object(opt, batch_unordered_object, - ODB_FOR_EACH_OBJECT_PACK_ORDER, &cb); + retval = batch_each_object(opt, batch_unordered_object, + ODB_FOR_EACH_OBJECT_PACK_ORDER, &cb); oidset_clear(&seen); } else { struct oid_array sa = OID_ARRAY_INIT; - batch_each_object(opt, collect_object, 0, &sa); + retval = batch_each_object(opt, collect_object, 0, &sa); oid_array_for_each_unique(&sa, batch_object_cb, &cb); oid_array_clear(&sa); } strbuf_release(&output); + if (retval) + return error(_("unable to enumerate all objects")); return 0; } diff --git a/builtin/multi-pack-index.c b/builtin/multi-pack-index.c index 6e73c85cde324f..fc4d67a6d896e2 100644 --- a/builtin/multi-pack-index.c +++ b/builtin/multi-pack-index.c @@ -6,6 +6,7 @@ #include "gettext.h" #include "parse-options.h" #include "midx.h" +#include "lockfile.h" #include "strbuf.h" #include "trace2.h" #include "odb.h" @@ -101,6 +102,9 @@ static struct option common_opts[] = { parse_object_dir), OPT_BIT(0, "progress", &opts.flags, N_("force progress reporting"), MIDX_PROGRESS), + OPT_BIT_F(0, "write-lock-held", &opts.flags, + N_("the caller holds the MIDX writer lock"), + MIDX_WRITE_LOCK_HELD, PARSE_OPT_HIDDEN | PARSE_OPT_NONEG), OPT_END(), }; @@ -225,7 +229,8 @@ static int cmd_multi_pack_index_write(int argc, const char **argv, } ret = write_midx_file(source->packed, opts.preferred_pack, - opts.refs_snapshot, opts.flags); + opts.refs_snapshot, opts.incremental_base, + opts.flags); free(opts.refs_snapshot); return ret; @@ -239,6 +244,7 @@ static int cmd_multi_pack_index_compact(int argc, const char **argv, struct multi_pack_index *from_midx = NULL; struct multi_pack_index *to_midx = NULL; struct odb_source_files *source; + struct lock_file write_lock = LOCK_INIT; int ret; struct option *options; @@ -283,6 +289,11 @@ static int cmd_multi_pack_index_compact(int argc, const char **argv, FREE_AND_NULL(options); + if (!(opts.flags & MIDX_WRITE_LOCK_HELD)) { + hold_midx_write_lock(source->packed, &write_lock); + opts.flags |= MIDX_WRITE_LOCK_HELD; + } + m = get_multi_pack_index(source->packed); for (cur = m; cur && !(from_midx && to_midx); cur = cur->base_midx) { @@ -309,6 +320,7 @@ static int cmd_multi_pack_index_compact(int argc, const char **argv, ret = write_midx_file_compact(source->packed, from_midx, to_midx, opts.incremental_base, opts.flags); + rollback_lock_file(&write_lock); return ret; } diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 5405a2a7fcfe11..76039633f421da 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -1351,6 +1351,7 @@ static void write_pack_file(void) do { unsigned char hash[GIT_MAX_RAWSZ]; char *pack_tmp_name = NULL; + off_t pack_bytes; if (pack_to_stdout) { /* @@ -1393,7 +1394,7 @@ static void write_pack_file(void) display_progress(progress_state, written); } - bytes_written += hashfile_total(f) + + pack_bytes = hashfile_total(f) + the_repository->hash_algo->rawsz; if (pack_to_stdout) { /* @@ -1425,6 +1426,7 @@ static void write_pack_file(void) write_bitmap_index = 0; } } + bytes_written += pack_bytes; if (!pack_to_stdout) { struct stat st; @@ -1792,7 +1794,7 @@ static int want_object_in_pack_mtime(const struct object_id *oid, struct multi_pack_index *m = get_multi_pack_index(files->packed); struct pack_entry e; - if (m && fill_midx_entry(m, oid, &e, NULL)) { + if (m && midx_fill_entry(m, oid, &e, NULL) == MIDX_FILL_HIT) { want = want_object_in_pack_one(e.p, oid, exclude, found_pack, found_offset, found_mtime); if (want != -1) return want; diff --git a/builtin/receive-pack.c b/builtin/receive-pack.c index c75b3ae7ff39d3..867e4630560edd 100644 --- a/builtin/receive-pack.c +++ b/builtin/receive-pack.c @@ -61,6 +61,7 @@ static int receive_fsck_objects = -1; static int transfer_fsck_objects = -1; static struct strbuf fsck_msg_types = STRBUF_INIT; static int advertise_atomic_push = 1; +static int advertise_explicit_haves; static int advertise_no_ref_delta; static int advertise_push_options; static int advertise_sid; @@ -272,6 +273,8 @@ static void show_ref(const char *path, const struct object_id *oid) strbuf_addstr(&cap, " atomic"); if (prefer_ofs_delta) strbuf_addstr(&cap, " ofs-delta"); + if (advertise_explicit_haves) + strbuf_addstr(&cap, " explicit-haves"); if (advertise_no_ref_delta) strbuf_addstr(&cap, " no-ref-delta"); if (push_cert_nonce) @@ -2511,6 +2514,8 @@ int cmd_receive_pack(int argc, OPT_HIDDEN_BOOL(0, "http-backend-info-refs", &advertise_refs, NULL), OPT_ALIAS(0, "advertise-refs", "http-backend-info-refs"), OPT_HIDDEN_BOOL(0, "reject-thin-pack-for-testing", &reject_thin, NULL), + OPT_HIDDEN_BOOL(0, "advertise-explicit-haves-for-testing", + &advertise_explicit_haves, NULL), OPT_HIDDEN_BOOL(0, "advertise-no-ref-delta-for-testing", &advertise_no_ref_delta, NULL), OPT_END() diff --git a/builtin/repack.c b/builtin/repack.c index c4360382c1fce2..d06dabad3ac1c4 100644 --- a/builtin/repack.c +++ b/builtin/repack.c @@ -9,6 +9,7 @@ #include "server-info.h" #include "string-list.h" #include "midx.h" +#include "lockfile.h" #include "packfile.h" #include "prune-packed.h" #include "promisor-remote.h" @@ -167,6 +168,7 @@ int cmd_repack(int argc, struct oidset drop_oids = OIDSET_INIT; struct pack_geometry geometry = { 0 }; struct tempfile *refs_snapshot = NULL; + struct lock_file midx_write_lock = LOCK_INIT; int i, ret; int show_progress; @@ -277,6 +279,9 @@ int cmd_repack(int argc, argc = parse_options(argc, argv, prefix, builtin_repack_options, git_repack_usage, 0); + hold_midx_write_lock(odb_source_files_downcast(repo->objects->sources)->packed, + &midx_write_lock); + po_args.window = xstrdup_or_null(opt_window); po_args.window_memory = xstrdup_or_null(opt_window_memory); po_args.depth = xstrdup_or_null(opt_depth); @@ -771,14 +776,15 @@ int cmd_repack(int argc, if (git_env_bool(GIT_TEST_MULTI_PACK_INDEX, 0)) { struct odb_source_files *files = odb_source_files_downcast(existing.source); - unsigned flags = 0; + unsigned flags = MIDX_WRITE_LOCK_HELD; if (git_env_bool(GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL, 0)) flags |= MIDX_WRITE_INCREMENTAL; - write_midx_file(files->packed, NULL, NULL, flags); + write_midx_file(files->packed, NULL, NULL, NULL, flags); } cleanup: + rollback_lock_file(&midx_write_lock); string_list_clear(&keep_pack_list, 0); string_list_clear(&names, 1); oidset_clear(&drop_oids); diff --git a/config.mak.openai b/config.mak.openai new file mode 100644 index 00000000000000..0ae133c8bb2ecf --- /dev/null +++ b/config.mak.openai @@ -0,0 +1,74 @@ +# OpenAI release build settings. +# +# The Codex release workflow copies this file to config.mak before +# building. Keep release-only optimizations here instead of in the +# upstream Makefile. + +OPENAI_PROFILE_DIR := $(CURDIR) +OPENAI_PROFILE_RAW := $(OPENAI_PROFILE_DIR)/default_%m_%p.profraw +OPENAI_PROFILE_DATA := $(OPENAI_PROFILE_DIR)/default.profdata +OPENAI_PROFILE_TRAINING ?= .github/workflows/codex-pgo-training.sh +OPENAI_LLVM_PROFDATA ?= llvm-profdata + +ifdef OPENAI_LTO +CFLAGS_APPEND += -flto=$(OPENAI_LTO) +LDFLAGS_APPEND += -flto=$(OPENAI_LTO) +endif + +ifeq ("$(OPENAI_PROFILE_FORMAT)","LLVM") +ifeq ("$(OPENAI_PROFILE)","GEN") + BASIC_CFLAGS += -fprofile-instr-generate=$(OPENAI_PROFILE_RAW) + BASIC_CFLAGS += -DNO_NORETURN=1 + export CCACHE_DISABLE = t + V = 1 +else +ifneq ("$(OPENAI_PROFILE)","") + BASIC_CFLAGS += -fprofile-instr-use=$(OPENAI_PROFILE_DATA) + BASIC_CFLAGS += -Wno-profile-instr-unprofiled -DNO_NORETURN=1 + export CCACHE_DISABLE = t + V = 1 +endif +endif +else +ifeq ("$(OPENAI_PROFILE)","GEN") + BASIC_CFLAGS += -fprofile-generate=$(OPENAI_PROFILE_DIR) + BASIC_CFLAGS += -DNO_NORETURN=1 + EXTLIBS += -lgcov + export CCACHE_DISABLE = t + V = 1 +else +ifneq ("$(OPENAI_PROFILE)","") + BASIC_CFLAGS += -fprofile-use=$(OPENAI_PROFILE_DIR) + BASIC_CFLAGS += -fprofile-correction -DNO_NORETURN=1 + export CCACHE_DISABLE = t + V = 1 +endif +endif +endif + +# Every C object already waits for Git's forced GIT-CFLAGS target. Gate that +# target so "make strip install" finishes PGO before its normal prerequisites +# can start compiling with profile-use flags. +ifeq "$(OPENAI_PROFILE)" "BUILD" +GIT-CFLAGS: openai-profile +endif + +openai-profile: profile-clean openai-profile-clean + $(MAKE) OPENAI_PROFILE=GEN all + $(SHELL_PATH) $(OPENAI_PROFILE_TRAINING) + $(MAKE) OPENAI_PROFILE= openai-profile-merge + $(MAKE) OPENAI_PROFILE=USE all + +openai-profile-merge: +ifeq ("$(OPENAI_PROFILE_FORMAT)","LLVM") + $(OPENAI_LLVM_PROFDATA) merge \ + -output=$(OPENAI_PROFILE_DATA) \ + $(OPENAI_PROFILE_DIR)/*.profraw +endif + +clean: openai-profile-clean + +openai-profile-clean: + $(RM) $(OPENAI_PROFILE_DIR)/*.profraw $(OPENAI_PROFILE_DATA) + +.PHONY: openai-profile openai-profile-merge openai-profile-clean diff --git a/credential.c b/credential.c index 2594c0c4229ba0..5611c6fca882cd 100644 --- a/credential.c +++ b/credential.c @@ -207,6 +207,44 @@ static void credential_apply_config(struct repository *r, struct credential *c) } } +int credential_update_url(struct repository *r, struct credential *c, + const char *old_url, const char *new_url) +{ + struct credential old = CREDENTIAL_INIT; + struct credential new = CREDENTIAL_INIT; + char *old_normalized = url_normalize(old_url, NULL); + char *new_normalized = url_normalize(new_url, NULL); + int changed; + size_t i; + + if (!old_normalized || !new_normalized || + credential_from_url_gently(&old, old_normalized, 1) || + credential_from_url_gently(&new, new_normalized, 1)) + die(_("unable to parse credential URL after redirection")); + + /* Helpers may have changed c's identity, so compare the URL contexts. */ + credential_apply_config(r, &old); + credential_apply_config(r, &new); + changed = !credential_match(&old, &new, 1) || + !credential_match(&new, &old, 1) || + old.use_http_path != new.use_http_path || + old.username_from_proto != new.username_from_proto || + old.sanitize_prompt != new.sanitize_prompt || + old.protect_protocol != new.protect_protocol || + old.helpers.nr != new.helpers.nr; + for (i = 0; !changed && i < old.helpers.nr; i++) + changed = strcmp(old.helpers.items[i].string, + new.helpers.items[i].string); + + if (changed) + SWAP(*c, new); + credential_clear(&old); + credential_clear(&new); + free(old_normalized); + free(new_normalized); + return !!changed; +} + static void credential_describe(struct credential *c, struct strbuf *out) { if (!c->protocol) diff --git a/credential.h b/credential.h index c78b72d110eaac..591e81ea297d70 100644 --- a/credential.h +++ b/credential.h @@ -305,6 +305,17 @@ void credential_write(const struct credential *, FILE *, void credential_from_url(struct credential *, const char *url); int credential_from_url_gently(struct credential *, const char *url, int quiet); +/* + * Replace a credential with the configured context for new_url only if its + * identity or credential configuration differs from old_url. No helpers are run. + * Return 1 when replaced, or 0 when credentials and helper state are retained. + * old_url must be the URL that established the current credential context, + * rather than a URL reconstructed from helper-supplied credential fields. + * The caller must preserve any response headers needed after replacement. + */ +int credential_update_url(struct repository *, struct credential *, + const char *old_url, const char *new_url); + int credential_match(const struct credential *want, const struct credential *have, int match_password); diff --git a/http-fetch.c b/http-fetch.c index 920a1ff1b81f6d..96f986cd74fc50 100644 --- a/http-fetch.c +++ b/http-fetch.c @@ -56,8 +56,15 @@ static int fetch_using_walker(const char *raw_url, int get_verbosely, struct packfile_progress { uint64_t bytes; uint64_t last_update; + int report_progress; }; +static void trace_packfile_progress(struct packfile_progress *progress) +{ + trace2_data_intmax("http-fetch", the_repository, + "pack/received_bytes", progress->bytes); +} + static void report_packfile_progress(struct packfile_progress *progress) { printf("bytes %"PRIu64"\n", progress->bytes); @@ -71,8 +78,13 @@ static void update_packfile_progress(void *data, size_t bytes) struct packfile_progress *progress = data; progress->bytes += bytes; - if (getnanotime() - progress->last_update >= 500000000) - report_packfile_progress(progress); + if (getnanotime() - progress->last_update >= 500000000) { + trace_packfile_progress(progress); + if (progress->report_progress) + report_packfile_progress(progress); + else + progress->last_update = getnanotime(); + } } static void fetch_single_packfile(struct object_id *packfile_hash, @@ -81,9 +93,11 @@ static void fetch_single_packfile(struct object_id *packfile_hash, int report_progress) { struct http_pack_request *preq; - struct packfile_progress progress = { 0 }; + struct packfile_progress progress = { .report_progress = report_progress }; int ret; + progress.last_update = getnanotime(); + trace_packfile_progress(&progress); http_init(NULL, url, 0); preq = new_direct_http_pack_request(packfile_hash->hash, xstrdup(url)); @@ -91,13 +105,14 @@ static void fetch_single_packfile(struct object_id *packfile_hash, die("couldn't create http pack request"); preq->index_pack_args = index_pack_args; preq->preserve_index_pack_stdout = 1; - if (report_progress) { - progress.last_update = getnanotime(); - preq->progress = update_packfile_progress; - preq->progress_data = &progress; - } + preq->progress = update_packfile_progress; + preq->progress_data = &progress; ret = run_http_pack_request(preq); + /* Preserve partial transfers before ordinary HTTP failures exit. */ + trace_packfile_progress(&progress); + trace2_data_intmax("http-fetch", the_repository, + "pack/received_bytes_complete", 1); if (ret != HTTP_START_FAILED) { if (ret != HTTP_OK) { struct url_info url; diff --git a/http.c b/http.c index 5f0cd56a3544a2..18b22d8ffeeb29 100644 --- a/http.c +++ b/http.c @@ -124,6 +124,8 @@ static char *curl_cookie_file; static int curl_save_cookies; struct credential http_auth = CREDENTIAL_INIT; static enum proactive_auth http_proactive_auth; +/* The caller's fallback when no proactive authentication method is selected. */ +static int http_proactive_auth_from_caller; static char *user_agent; static int curl_empty_auth = -1; @@ -1180,8 +1182,7 @@ static CURL *get_curl_handle(void) if (http_proactive_auth != PROACTIVE_AUTH_NONE) init_curl_http_auth(result); - if (getenv("GIT_SSL_VERSION")) - ssl_version = getenv("GIT_SSL_VERSION"); + var_override(&ssl_version, getenv("GIT_SSL_VERSION")); if (ssl_version && *ssl_version) { int i; for (i = 0; i < ARRAY_SIZE(sslversions); i++) { @@ -1196,8 +1197,7 @@ static CURL *get_curl_handle(void) ssl_version); } - if (getenv("GIT_SSL_CIPHER_LIST")) - ssl_cipherlist = getenv("GIT_SSL_CIPHER_LIST"); + var_override(&ssl_cipherlist, getenv("GIT_SSL_CIPHER_LIST")); if (ssl_cipherlist != NULL && *ssl_cipherlist) curl_easy_setopt(result, CURLOPT_SSL_CIPHER_LIST, ssl_cipherlist); @@ -1308,13 +1308,16 @@ static CURL *get_curl_handle(void) } else if (curl_http_proxy) { struct strbuf proxy = STRBUF_INIT; - if (strstr(curl_http_proxy, "://")) - credential_from_url(&proxy_auth, curl_http_proxy); - else { - struct strbuf url = STRBUF_INIT; - strbuf_addf(&url, "http://%s", curl_http_proxy); - credential_from_url(&proxy_auth, url.buf); - strbuf_release(&url); + /* Rebuilding a destination handle does not change its proxy. */ + if (!proxy_auth.host) { + if (strstr(curl_http_proxy, "://")) + credential_from_url(&proxy_auth, curl_http_proxy); + else { + struct strbuf url = STRBUF_INIT; + strbuf_addf(&url, "http://%s", curl_http_proxy); + credential_from_url(&proxy_auth, url.buf); + strbuf_release(&url); + } } if (set_curl_proxy_type(result, proxy_auth.protocol) < 0) @@ -1398,11 +1401,154 @@ static void set_long_from_env(long *var, const char *envname) } } +static int equal_nullable_strings(const char *a, const char *b) +{ + return a && b ? !strcmp(a, b) : a == b; +} + +static int http_redirect_options(const char *var, const char *value, + const struct config_context *ctx, void *data) +{ + static const char * const names[] = { + "http.proactiveauth", "http.emptyauth", "http.extraheader", + "http.sslverify", "http.sslcert", "http.sslcerttype", + "http.sslkey", "http.sslkeytype", "http.sslcapath", + "http.sslcainfo", "http.sslcertpasswordprotected", + "http.sslversion", "http.sslcipherlist", "http.pinnedpubkey", + "http.schannelcheckrevoke", "http.schannelusesslcainfo", + "http.delegation" + }; + size_t i; + + if (!strcmp(var, "http.sslbackend")) + return git_config_string(data, var, value); + for (i = 0; i < ARRAY_SIZE(names); i++) + if (!strcmp(var, names[i])) + return http_options(var, value, ctx, NULL); + return 0; +} + +/* Preserve cookies without carrying a connection or TLS session to a new context. */ +static CURL *http_redirect_handle(CURL *old, CURL *new) +{ + struct curl_slist *cookies = NULL, *cookie; + + if (curl_easy_getinfo(old, CURLINFO_COOKIELIST, &cookies) != CURLE_OK) + die(_("unable to preserve HTTP cookies after a redirect")); + for (cookie = cookies; cookie; cookie = cookie->next) + if (curl_easy_setopt(new, CURLOPT_COOKIELIST, cookie->data) != CURLE_OK) + die(_("unable to restore HTTP cookies after a redirect")); + curl_slist_free_all(cookies); + curl_easy_cleanup(old); + return new; +} + +/* Discovery is serial; no request may retain the previous destination's settings. */ +static void http_redirect_config(const char *url) +{ + struct urlmatch_config config = URLMATCH_CONFIG_INIT; + struct active_request_slot *slot; + char *normalized_url, *backend = NULL; + char *old_cert = xstrdup_or_null(ssl_cert); + char *old_key = xstrdup_or_null(ssl_key); + char *old_cert_type = xstrdup_or_null(ssl_cert_type); + char *old_key_type = xstrdup_or_null(ssl_key_type); + int old_password_required = ssl_cert_password_required; + CURL *new_default; + + if (active_requests) + BUG("HTTP redirect configuration changed during an active request"); + for (slot = active_queue_head; slot; slot = slot->next) + if (slot->in_use) + BUG("HTTP redirect configuration changed with a reserved slot"); + + FREE_AND_NULL(ssl_cert); + FREE_AND_NULL(ssl_cert_type); + FREE_AND_NULL(ssl_key); + FREE_AND_NULL(ssl_key_type); + FREE_AND_NULL(ssl_capath); + FREE_AND_NULL(ssl_cainfo); + FREE_AND_NULL(ssl_version); + FREE_AND_NULL(ssl_cipherlist); + FREE_AND_NULL(ssl_pinnedkey); +#ifdef CURLGSSAPI_DELEGATION_FLAG + FREE_AND_NULL(curl_deleg); +#endif + curl_ssl_verify = 1; + ssl_cert_password_required = 0; + http_schannel_check_revoke = 1; + http_schannel_use_ssl_cainfo = 0; + http_proactive_auth = PROACTIVE_AUTH_NONE; + curl_empty_auth = -1; + string_list_clear(&extra_http_headers, 0); + + config.section = "http"; + config.collect_fn = http_redirect_options; + config.cb = &backend; + normalized_url = url_normalize(url, &config.url); + if (!normalized_url) + die(_("invalid HTTP redirect URL")); + repo_config(the_repository, urlmatch_config_entry, &config); + free(normalized_url); + string_list_clear(&config.vars, 1); + if (!equal_nullable_strings(backend, http_ssl_backend)) + die(_("cannot change the SSL backend after an HTTP redirect")); + free(backend); + if (http_proactive_auth_from_caller && + http_proactive_auth == PROACTIVE_AUTH_NONE) + http_proactive_auth = PROACTIVE_AUTH_IF_CREDENTIALS; + + if (getenv("GIT_SSL_NO_VERIFY")) + curl_ssl_verify = 0; + set_from_env(&ssl_cert, "GIT_SSL_CERT"); + set_from_env(&ssl_cert_type, "GIT_SSL_CERT_TYPE"); + set_from_env(&ssl_key, "GIT_SSL_KEY"); + set_from_env(&ssl_key_type, "GIT_SSL_KEY_TYPE"); + set_from_env(&ssl_capath, "GIT_SSL_CAPATH"); + set_from_env(&ssl_cainfo, "GIT_SSL_CAINFO"); + if (getenv("GIT_SSL_CERT_PASSWORD_PROTECTED") && + starts_with(url, "https://")) + ssl_cert_password_required = 1; + if (!equal_nullable_strings(old_cert, ssl_cert) || + !equal_nullable_strings(old_key, ssl_key) || + !equal_nullable_strings(old_cert_type, ssl_cert_type) || + !equal_nullable_strings(old_key_type, ssl_key_type) || + old_password_required != ssl_cert_password_required) + credential_clear(&cert_auth); + free(old_cert); + free(old_key); + free(old_cert_type); + free(old_key_type); + + /* The multi handle owns connections shared by the easy handles. */ + curl_multi_cleanup(curlm); + curlm = curl_multi_init(); + if (!curlm) + die("curl_multi_init failed"); + new_default = get_curl_handle(); + for (slot = active_queue_head; slot; slot = slot->next) { + CURL *new; + + if (!slot->curl) + continue; + new = curl_easy_duphandle(new_default); + if (!new) + die("curl_easy_duphandle failed"); + slot->curl = http_redirect_handle(slot->curl, new); + } + curl_default = http_redirect_handle(curl_default, new_default); + curl_slist_free_all(pragma_header); + pragma_header = curl_slist_append(http_copy_default_headers(), + "Pragma: no-cache"); +} + void http_init(struct remote *remote, const char *url, int proactive_auth) { char *normalized_url; struct urlmatch_config config = URLMATCH_CONFIG_INIT; + http_proactive_auth_from_caller = proactive_auth; + config.section = "http"; config.key = NULL; config.collect_fn = http_options; @@ -1668,7 +1814,11 @@ struct active_request_slot *get_active_slot(void) curl_easy_setopt(slot->curl, CURLOPT_IPRESOLVE, git_curl_ipresolve); curl_easy_setopt(slot->curl, CURLOPT_HTTPAUTH, http_auth_methods); - if (http_auth.password || http_auth.credential || curl_empty_auth_enabled()) + /* Credentials from a previous request must not survive a context change. */ + curl_easy_setopt(slot->curl, CURLOPT_USERNAME, NULL); + curl_easy_setopt(slot->curl, CURLOPT_PASSWORD, NULL); + if (http_auth.password || http_auth.credential || curl_empty_auth_enabled() || + always_auth_proactively()) init_curl_http_auth(slot->curl); return slot; @@ -2235,9 +2385,13 @@ static void http_opt_request_remainder(CURL *curl, off_t pos) #define HTTP_REQUEST_STRBUF 0 #define HTTP_REQUEST_FILE 1 +/* An unfollowed discovery redirect, handled before configuring its destination. */ +#define HTTP_REDIRECT 8 + static int http_request(const char *url, void *result, int target, - struct http_get_options *options) + struct http_get_options *options, + struct strbuf *redirect) { struct active_request_slot *slot; struct slot_results results = { .retry_after = -1 }; @@ -2279,6 +2433,8 @@ static int http_request(const char *url, if (options->initial_request && http_follow_config == HTTP_FOLLOW_INITIAL) curl_easy_setopt(slot->curl, CURLOPT_FOLLOWLOCATION, 1L); + if (redirect) + curl_easy_setopt(slot->curl, CURLOPT_FOLLOWLOCATION, 0L); headers = curl_slist_append(headers, buf.buf); @@ -2297,6 +2453,16 @@ static int http_request(const char *url, curl_easy_setopt(slot->curl, CURLOPT_FAILONERROR, 0L); ret = run_one_slot(slot, &results); + if (redirect && + (results.http_code == 301 || results.http_code == 302 || + results.http_code == 303 || results.http_code == 307 || + results.http_code == 308) && + (results.curl_result == CURLE_OK || + results.curl_result == CURLE_HTTP_RETURNED_ERROR)) { + curlinfo_strbuf(slot->curl, CURLINFO_REDIRECT_URL, redirect); + if (redirect->len) + ret = HTTP_REDIRECT; + } options->retry_after = results.retry_after; @@ -2411,6 +2577,114 @@ static long handle_rate_limit_retry(long slot_retry_after) return slot_retry_after; } +/* Discard response bodies before following a redirect or retrying a request. */ +static int http_request_rewind(void *result, int target) +{ + if (!result) + return 0; + switch (target) { + case HTTP_REQUEST_STRBUF: + strbuf_reset(result); + return 0; + case HTTP_REQUEST_FILE: { + FILE *f = result; + if (fflush(f)) + return error_errno("unable to flush a file"); + rewind(f); + if (ftruncate(fileno(f), 0) < 0) + return error_errno("unable to truncate a file"); + return 0; + } + default: + BUG("Unknown http_request target"); + } +} + +/* + * Discovery redirects must select credentials before, not after, the next + * request. In particular, curl cannot enforce credential.useHttpPath or select + * URL-specific helpers, headers and client certificates while following one. + */ +static int http_request_with_redirects(const char *url, void *result, int target, + struct http_get_options *options) +{ + struct strbuf current = STRBUF_INIT; + struct strbuf redirect = STRBUF_INIT; + struct strbuf auth_url = STRBUF_INIT; + const char *tail; + char *asked = xstrdup(url); + int ret, redirects = 0; + int follow = options->base_url && options->effective_url && + (http_follow_config == HTTP_FOLLOW_ALWAYS || + (options->initial_request && http_follow_config == HTTP_FOLLOW_INITIAL)); + + if (!follow) { + ret = http_request(asked, result, target, options, NULL); + goto out; + } + if (!skip_prefix(asked, options->base_url->buf, &tail)) + BUG("discovery URL does not start with its repository URL"); + strbuf_addstr(¤t, asked); + strbuf_addbuf(&auth_url, options->base_url); + for (;;) { + struct url_info info; + char *normalized, *protocol; + size_t len; + + strbuf_reset(&redirect); + ret = http_request(current.buf, result, target, options, &redirect); + if (ret != HTTP_REDIRECT) + break; + if (++redirects > 20) { + xsnprintf(curl_errorstr, sizeof(curl_errorstr), + "Maximum (20) redirects followed"); + ret = HTTP_ERROR; + break; + } + normalized = url_normalize(redirect.buf, &info); + if (!normalized) + die(_("invalid HTTP redirect URL")); + protocol = xmemdupz(normalized, info.scheme_len); + if ((strcmp(protocol, "http") && strcmp(protocol, "https") && + strcmp(protocol, "ftp") && strcmp(protocol, "ftps")) || + !is_transport_allowed(protocol, 0)) + die(_("transport '%s' not allowed in HTTP redirect"), protocol); + free(protocol); + free(normalized); + + /* Preserve the repository-level credential path when possible. */ + len = redirect.len; + strip_suffix_mem(redirect.buf, &len, tail); + { + char *next_auth_url = xmemdupz(redirect.buf, len); + if (credential_update_url(the_repository, &http_auth, + auth_url.buf, next_auth_url)) { + http_auth_methods = CURLAUTH_ANY; + http_auth_methods_restricted = 0; + empty_auth_try_negotiate = 0; + } + strvec_clear(&http_auth.wwwauth_headers); + http_redirect_config(next_auth_url); + strbuf_reset(&auth_url); + strbuf_addstr(&auth_url, next_auth_url); + free(next_auth_url); + } + if (http_request_rewind(result, target)) { + ret = HTTP_START_FAILED; + break; + } + strbuf_swap(¤t, &redirect); + } + if (ret == HTTP_OK || ret == HTTP_REAUTH || ret == HTTP_RATE_LIMITED) + update_url_from_redirect(options->base_url, asked, options->effective_url); +out: + free(asked); + strbuf_release(¤t); + strbuf_release(&redirect); + strbuf_release(&auth_url); + return ret; +} + static int http_request_recoverable(const char *url, void *result, int target, struct http_get_options *options) @@ -2426,7 +2700,7 @@ static int http_request_recoverable(const char *url, if (always_auth_proactively()) credential_fill(the_repository, &http_auth, 1); - ret = http_request(url, result, target, options); + ret = http_request_with_redirects(url, result, target, options); if (ret != HTTP_OK && ret != HTTP_REAUTH && ret != HTTP_RATE_LIMITED) return ret; @@ -2435,41 +2709,14 @@ static int http_request_recoverable(const char *url, if (ret == HTTP_RATE_LIMITED && !http_max_retries) return HTTP_ERROR; - if (options->effective_url && options->base_url) { - if (update_url_from_redirect(options->base_url, - url, options->effective_url)) { - credential_from_url(&http_auth, options->base_url->buf); - url = options->effective_url->buf; - } - } + if (options->effective_url && options->base_url) + url = options->effective_url->buf; while ((ret == HTTP_REAUTH && --i) || (ret == HTTP_RATE_LIMITED && --rate_limit_retries)) { long retry_delay = -1; - /* - * The previous request may have put cruft into our output stream; we - * should clear it out before making our next request. - */ - switch (target) { - case HTTP_REQUEST_STRBUF: - strbuf_reset(result); - break; - case HTTP_REQUEST_FILE: { - FILE *f = result; - if (fflush(f)) { - error_errno("unable to flush a file"); - return HTTP_START_FAILED; - } - rewind(f); - if (ftruncate(fileno(f), 0) < 0) { - error_errno("unable to truncate a file"); - return HTTP_START_FAILED; - } - break; - } - default: - BUG("Unknown http_request target"); - } + if (http_request_rewind(result, target)) + return HTTP_START_FAILED; if (ret == HTTP_RATE_LIMITED) { retry_delay = handle_rate_limit_retry(options->retry_after); if (retry_delay < 0) @@ -2478,7 +2725,9 @@ static int http_request_recoverable(const char *url, http_reauth_prepare(1); } - ret = http_request(url, result, target, options); + ret = http_request_with_redirects(url, result, target, options); + if (options->effective_url && options->base_url) + url = options->effective_url->buf; } if (ret == HTTP_RATE_LIMITED) { trace2_data_string("http", the_repository, @@ -2787,17 +3036,17 @@ static size_t fwrite_http_pack(char *ptr, size_t size, size_t nmemb, void *data) { struct http_pack_request *preq = data; long code; - size_t written; + size_t bytes = st_mult(size, nmemb); if (curl_easy_getinfo(preq->slot->curl, CURLINFO_HTTP_CODE, &code) != CURLE_OK) return 0; /* Error bodies must not overwrite a partial pack shared with another fetch. */ if (code >= 300) - return size * nmemb; - written = fwrite(ptr, 1, st_mult(size, nmemb), preq->packfile); + return bytes; + /* Count received PACK payload even if the local write fails. */ if (preq->progress) - preq->progress(preq->progress_data, written); - return written; + preq->progress(preq->progress_data, bytes); + return fwrite(ptr, 1, bytes, preq->packfile); } int run_http_pack_request(struct http_pack_request *preq) diff --git a/http.h b/http.h index d3c33adec9b3de..0f6144f7f7fd3c 100644 --- a/http.h +++ b/http.h @@ -212,6 +212,7 @@ int http_get_info_packs(const char *base_url, const char *http_get_accept_language_header(void); struct http_pack_request { + /* Newly received PACK payload, excluding HTTP error bodies and resume offsets. */ void (*progress)(void *data, size_t bytes); void *progress_data; char *url; diff --git a/midx-write.c b/midx-write.c index 85371022543836..7ba9335e1a88d4 100644 --- a/midx-write.c +++ b/midx-write.c @@ -115,6 +115,7 @@ struct write_midx_context { struct multi_pack_index *compact_from; struct multi_pack_index *compact_to; int compact; + int compact_reuse; struct string_list *to_include; @@ -133,28 +134,41 @@ static uint32_t midx_pack_perm(struct write_midx_context *ctx, static int should_include_pack(const struct write_midx_context *ctx, const char *file_name) { + struct multi_pack_index *m = ctx->m; /* - * Note that at most one of ctx->m and ctx->to_include are set, - * so we are testing midx_contains_pack() and - * string_list_has_string() independently (guarded by the - * appropriate NULL checks). + * When writing incrementally, ctx->m may contain layers above + * the selected base MIDX, which must be included in the new + * layer. + */ + if (ctx->incremental) + m = ctx->base_midx; + + /* + * Note that it is OK for both ctx->base_midx and + * ctx->to_include to be non-NULL, but at most one of ctx->m + * and ctx->to_include may be non-NULL. * - * We could support passing to_include while reusing an existing - * MIDX, but don't currently since the reuse process drags - * forward all packs from an existing MIDX (without checking - * whether or not they appear in the to_include list). + * When ctx->m is NULL we are writing a new MIDX without reusing + * any packs from the previous layer(s). In that case, we care + * that both: * - * If we added support for that, these next two conditional - * should be performed independently (likely checking - * to_include before the existing MIDX). + * - the new layer's base MIDX (ctx->base_midx) does not + * already contain the pack we are considering, or the new + * layer has no base (including --incremental --base=none) + * + * - the pack appears in ctx->to_include, or ctx->to_include + * is NULL, meaning that we can include any pack provided + * the above condition is met. + * + * When ctx->m is non-NULL for a non-incremental write, we reuse + * its packs without checking ctx->to_include. We could support + * filtering those packs too, but currently don't. Incremental + * writes instead exclude only packs covered by ctx->base_midx. */ - if (ctx->m && midx_contains_pack(ctx->m, file_name)) + if (ctx->to_include && + !string_list_has_string(ctx->to_include, file_name)) return 0; - else if (ctx->base_midx && midx_contains_pack(ctx->base_midx, - file_name)) - return 0; - else if (ctx->to_include && - !string_list_has_string(ctx->to_include, file_name)) + if (m && midx_contains_pack(m, file_name)) return 0; return 1; } @@ -372,6 +386,19 @@ static void midx_fanout_add_compact(struct midx_fanout *fanout, ASSERT(ctx->compact); + if (!ctx->compact_reuse) { + for (uint32_t i = 0; i < ctx->nr; i++) { + size_t start = fanout->nr; + + midx_fanout_add_pack_fanout(fanout, ctx->info, i, + i == ctx->preferred_pack_idx, + cur_fanout); + for (size_t j = start; j < fanout->nr; j++) + fanout->entries[j].pack_int_id = ctx->info[i].orig_pack_int_id; + } + return; + } + while (m && m != ctx->compact_from->base_midx) { midx_fanout_add_midx_fanout_1(fanout, m, cur_fanout, NO_PREFERRED_PACK); @@ -432,7 +459,8 @@ static void compute_sorted_entries(struct write_midx_context *ctx, if (cur_object && oideq(&fanout.entries[cur_object - 1].oid, &fanout.entries[cur_object].oid)) continue; - if (ctx->incremental && ctx->base_midx && + /* Reused entries already exclude objects in their base. */ + if (ctx->incremental && !ctx->compact_reuse && ctx->base_midx && midx_has_oid(ctx->base_midx, &fanout.entries[cur_object].oid)) continue; @@ -675,7 +703,7 @@ static uint32_t *midx_pack_order(struct write_midx_context *ctx) struct pack_midx_entry *e = &ctx->entries[i]; data[i].nr = i; data[i].pack = midx_pack_perm(ctx, e->pack_int_id); - if (!e->preferred || ctx->compact) + if (!e->preferred || ctx->compact_reuse) data[i].pack |= (1U << 31); data[i].offset = e->offset; } @@ -1021,6 +1049,8 @@ static int fill_packs_from_midx_range(struct write_midx_context *ctx, ctx->compact_to); ALLOC_GROW(ctx->info, packs_nr, ctx->alloc); + /* Layers are visited newest first, leaving holes until the end. */ + memset(ctx->info, 0, st_mult(packs_nr, sizeof(*ctx->info))); while (m != ctx->compact_from->base_midx) { uint32_t pack_int_id, preferred_pack_id; @@ -1037,7 +1067,7 @@ static int fill_packs_from_midx_range(struct write_midx_context *ctx, if (fill_pack_from_midx(&ctx->info[pack_int_id++], m, preferred_pack_id) < 0) - return -1; + goto error; for (i = m->num_packs_in_base; i < m->num_packs_in_base + m->num_packs; i++) { @@ -1046,7 +1076,7 @@ static int fill_packs_from_midx_range(struct write_midx_context *ctx, if (fill_pack_from_midx(&ctx->info[pack_int_id++], m, i) < 0) - return -1; + goto error; } ctx->nr += m->num_packs; @@ -1055,7 +1085,22 @@ static int fill_packs_from_midx_range(struct write_midx_context *ctx, ASSERT(ctx->nr == packs_nr); + if (!ctx->compact_reuse) { + for (size_t i = 0; i < ctx->nr; i++) { + if (open_pack_index(ctx->info[i].p)) { + error(_("failed to open pack-index '%s'"), + ctx->info[i].pack_name); + goto error; + } + } + } + return 0; + +error: + /* Include initialized slots beyond the completed layers in cleanup. */ + ctx->nr = packs_nr; + return -1; } static struct { @@ -1067,6 +1112,44 @@ static struct { {MIDX_EXT_REV, MIDX_EXT_REV}, }; +static int link_midx_file(const char *from, const char *to) +{ + struct stat from_stat, to_stat; + int saved_errno; + + if (!link(from, to) || errno == ENOENT) + return 0; + saved_errno = errno; + + /* An interrupted promotion may have linked this file already. */ + if (saved_errno == EEXIST && + !lstat(from, &from_stat) && !lstat(to, &to_stat) && + from_stat.st_ino && + from_stat.st_dev == to_stat.st_dev && + from_stat.st_ino == to_stat.st_ino) + return 0; + + errno = saved_errno; + return error_errno(_("unable to link '%s' to '%s'"), from, to); +} + +static int check_midx_link(struct multi_pack_index *m, const char *path) +{ + const struct git_hash_algo *algop = m->source->base.odb->repo->hash_algo; + unsigned char hash[GIT_MAX_RAWSZ]; + int fd, ret = 0; + + fd = open(path, O_RDONLY); + if (fd < 0) + return error_errno(_("unable to open promoted multi-pack-index '%s'"), path); + if (lseek(fd, -(off_t)algop->rawsz, SEEK_END) < 0 || + read_in_full(fd, hash, algop->rawsz) != (ssize_t)algop->rawsz || + hashcmp(hash, midx_get_checksum_hash(m), algop)) + ret = error(_("multi-pack-index changed during promotion")); + close(fd); + return ret; +} + static int link_midx_to_chain(struct multi_pack_index *m) { struct strbuf from = STRBUF_INIT; @@ -1091,11 +1174,11 @@ static int link_midx_to_chain(struct multi_pack_index *m) get_split_midx_filename_ext(m->source, &to, hash, midx_exts[i].split); - if (link(from.buf, to.buf) < 0 && errno != ENOENT) { - ret = error_errno(_("unable to link '%s' to '%s'"), - from.buf, to.buf); + ret = link_midx_file(from.buf, to.buf); + if (!ret && !midx_exts[i].non_split) + ret = check_midx_link(m, to.buf); + if (ret) goto done; - } strbuf_reset(&from); strbuf_reset(&to); @@ -1126,11 +1209,8 @@ static void clear_midx_files(struct odb_source_packed *source, for (i = 0; i < ARRAY_SIZE(exts); i++) { clear_incremental_midx_files_ext(source, exts[i], hashes); - if (hashes) { - for (size_t j = 0; j < hashes->nr; j++) - clear_midx_files_ext(source, exts[i], - hashes->v[j]); - } + clear_midx_files_ext(source, exts[i], + incremental ? NULL : hashes->v[0]); } if (incremental) @@ -1150,6 +1230,14 @@ static bool midx_needs_update(struct multi_pack_index *midx, struct write_midx_c struct strbuf buf = STRBUF_INIT; bool needed = true; + /* + * Incremental writes either add a new pack or bail out later when + * there is nothing to add. Compaction always requires an update. + * Neither can reuse the existing MIDX unchanged. + */ + if (ctx->incremental || ctx->compact) + return true; + /* * Ensure that we have a valid checksum before consulting the * existing MIDX in order to determine if we can avoid an @@ -1170,17 +1258,6 @@ static bool midx_needs_update(struct multi_pack_index *midx, struct write_midx_c if (midx->version != ctx->version) goto out; - /* - * Ignore incremental updates for now. The assumption is that any - * incremental update would be either empty (in which case we will bail - * out later) or it would actually cover at least one new pack. - */ - if (ctx->incremental) - goto out; - - if (ctx->compact) - goto out; /* Compaction always requires an update. */ - /* * Otherwise, we need to verify that the packs covered by the existing * MIDX match the packs that we already have. The logic to do so is way @@ -1259,6 +1336,7 @@ static int write_midx_internal(struct write_midx_opts *opts) uint32_t start_pack; struct hashfile *f = NULL; struct lock_file lk = LOCK_INIT; + struct lock_file write_lock = LOCK_INIT; struct tempfile *incr; struct write_midx_context ctx = { .preferred_pack_idx = NO_PREFERRED_PACK, @@ -1273,6 +1351,9 @@ static int write_midx_internal(struct write_midx_opts *opts) trace2_region_enter("midx", "write_midx_internal", r); + if (!(opts->flags & MIDX_WRITE_LOCK_HELD)) + hold_midx_write_lock(opts->source, &write_lock); + ctx.repo = r; ctx.source = opts->source; @@ -1357,6 +1438,21 @@ static int write_midx_internal(struct write_midx_opts *opts) } } + if (ctx.compact) { + /* A standalone MIDX must index every object in its packs. */ + if (!ctx.incremental) + ctx.base_midx = NULL; + ctx.compact_reuse = ctx.base_midx == ctx.compact_from->base_midx; + + /* Retained layers depend on their original bitmap positions. */ + if (!ctx.compact_reuse && ctx.incremental && + !(opts->flags & MIDX_WRITE_NO_CHAIN) && + ctx.compact_to != ctx.m) { + error(_("cannot change the base when retaining layers above the compacted range")); + goto cleanup; + } + } + ctx.nr = 0; ctx.alloc = ctx.m ? ctx.m->num_packs + ctx.m->num_packs_in_base : 16; ctx.info = NULL; @@ -1393,14 +1489,15 @@ static int write_midx_internal(struct write_midx_opts *opts) else if (opts->flags & (MIDX_WRITE_REV_INDEX | MIDX_WRITE_BITMAP)) bitmap_order |= 1; - fill_packs_from_midx_range(&ctx, bitmap_order); + if (fill_packs_from_midx_range(&ctx, bitmap_order) < 0) + goto cleanup; } else { ctx.to_include = opts->packs_to_include; for_each_file_in_pack_dir(opts->source->base.path, add_pack_to_midx, &ctx); } stop_progress(&ctx.progress); - if (!opts->packs_to_drop) { + if (!opts->packs_to_drop && !ctx.incremental && !ctx.compact) { /* * If there is no MIDX then either it doesn't exist, or we're * doing a geometric repack. Try to load it from the source to @@ -1617,9 +1714,8 @@ static int write_midx_internal(struct write_midx_opts *opts) } if (!ctx.entries_nr) { - if (opts->flags & MIDX_WRITE_BITMAP) - warning(_("refusing to write multi-pack .bitmap without any objects")); - opts->flags &= ~(MIDX_WRITE_REV_INDEX | MIDX_WRITE_BITMAP); + result = 0; + goto cleanup; } if (ctx.incremental) { @@ -1760,6 +1856,7 @@ static int write_midx_internal(struct write_midx_opts *opts) if (rename_tempfile(&incr, final_midx_name.buf) < 0) { error_errno(_("unable to rename new multi-pack-index layer")); + strbuf_release(&final_midx_name); goto cleanup; } @@ -1827,6 +1924,8 @@ static int write_midx_internal(struct write_midx_opts *opts) result = 0; cleanup: + rollback_lock_file(&lk); + rollback_lock_file(&write_lock); for (size_t i = 0; i < ctx.nr; i++) { if (ctx.info[i].p) { close_pack(ctx.info[i].p); @@ -1851,12 +1950,14 @@ static int write_midx_internal(struct write_midx_opts *opts) int write_midx_file(struct odb_source_packed *source, const char *preferred_pack_name, const char *refs_snapshot, + const char *incremental_base, unsigned flags) { struct write_midx_opts opts = { .source = source, .preferred_pack_name = preferred_pack_name, .refs_snapshot = refs_snapshot, + .incremental_base = incremental_base, .flags = flags, }; @@ -1903,11 +2004,14 @@ int expire_midx_packs(struct odb_source_packed *source, unsigned flags) { uint32_t i, *count, result = 0; struct string_list packs_to_drop = STRING_LIST_INIT_DUP; - struct multi_pack_index *m = get_multi_pack_index(source); + struct multi_pack_index *m; + struct lock_file write_lock = LOCK_INIT; struct progress *progress = NULL; + hold_midx_write_lock(source, &write_lock); + m = get_multi_pack_index(source); if (!m) - return 0; + goto cleanup; if (m->base_midx) die(_("cannot expire packs from an incremental multi-pack-index")); @@ -1959,11 +2063,13 @@ int expire_midx_packs(struct odb_source_packed *source, unsigned flags) struct write_midx_opts opts = { .source = source, .packs_to_drop = &packs_to_drop, - .flags = flags & MIDX_PROGRESS, + .flags = (flags & MIDX_PROGRESS) | MIDX_WRITE_LOCK_HELD, }; result = write_midx_internal(&opts); } +cleanup: + rollback_lock_file(&write_lock); string_list_clear(&packs_to_drop, 0); return result; @@ -2091,13 +2197,14 @@ int midx_repack(struct odb_source_packed *source, size_t batch_size, unsigned fl struct repository *r = source->base.odb->repo; int result = 0; uint32_t i, packs_to_repack = 0; - unsigned char *include_pack; + unsigned char *include_pack = NULL; struct child_process cmd = CHILD_PROCESS_INIT; FILE *cmd_in; - struct multi_pack_index *m = get_multi_pack_index(source); + struct multi_pack_index *m; + struct lock_file write_lock = LOCK_INIT; struct write_midx_opts opts = { .source = source, - .flags = flags, + .flags = flags | MIDX_WRITE_LOCK_HELD, }; /* @@ -2108,8 +2215,10 @@ int midx_repack(struct odb_source_packed *source, size_t batch_size, unsigned fl int delta_base_offset = 1; int use_delta_islands = 0; + hold_midx_write_lock(source, &write_lock); + m = get_multi_pack_index(source); if (!m) - return 0; + goto cleanup; if (m->base_midx) die(_("cannot repack an incremental multi-pack-index")); @@ -2176,6 +2285,7 @@ int midx_repack(struct odb_source_packed *source, size_t batch_size, unsigned fl result = write_midx_internal(&opts); cleanup: + rollback_lock_file(&write_lock); free(include_pack); return result; } diff --git a/midx.c b/midx.c index 37f082dbdd5558..3324cc0acf9882 100644 --- a/midx.c +++ b/midx.c @@ -2,6 +2,8 @@ #include "git-compat-util.h" #include "config.h" +#include "lockfile.h" +#include "path.h" #include "dir.h" #include "hex.h" #include "packfile.h" @@ -322,8 +324,14 @@ static struct multi_pack_index *load_midx_chain_fd_st(struct odb_source_packed * struct multi_pack_index *m; struct object_id layer; - if (strbuf_getline_lf(&buf, fp) == EOF) + if (strbuf_getline_lf(&buf, fp) == EOF || ferror(fp)) { + if (ferror(fp)) + warning_errno(_("unable to read multi-pack-index chain")); + else + warning(_("unexpected end of multi-pack-index chain")); + valid = 0; break; + } if (get_oid_hex_algop(buf.buf, &layer, hash_algo)) { warning(_("invalid multi-pack-index chain: line '%s' " @@ -339,6 +347,11 @@ static struct multi_pack_index *load_midx_chain_fd_st(struct odb_source_packed * get_split_midx_filename_ext(source, &buf, layer.hash, MIDX_EXT_MIDX); m = load_multi_pack_index_one(source, buf.buf); + if (m && hashcmp(layer.hash, midx_get_checksum_hash(m), hash_algo)) { + warning(_("multi-pack-index checksum does not match chain")); + close_midx(m); + m = NULL; + } if (m) { if (add_midx_to_chain(m, midx_chain)) { @@ -361,7 +374,8 @@ static struct multi_pack_index *load_midx_chain_fd_st(struct odb_source_packed * return midx_chain; } -static struct multi_pack_index *load_multi_pack_index_chain(struct odb_source_packed *source) +static struct multi_pack_index *load_multi_pack_index_chain(struct odb_source_packed *source, + int *incomplete_chain) { struct strbuf chain_file = STRBUF_INIT; struct stat st; @@ -371,31 +385,47 @@ static struct multi_pack_index *load_multi_pack_index_chain(struct odb_source_pa get_midx_chain_filename(source, &chain_file); if (open_multi_pack_index_chain(source->base.odb->repo->hash_algo, chain_file.buf, &fd, &st)) { - int incomplete; /* ownership of fd is taken over by load function */ - m = load_midx_chain_fd_st(source, fd, &st, &incomplete); + m = load_midx_chain_fd_st(source, fd, &st, incomplete_chain); + + } else if (errno != ENOENT) { + *incomplete_chain = 1; } strbuf_release(&chain_file); return m; } -struct multi_pack_index *load_multi_pack_index(struct odb_source_packed *source) +static struct multi_pack_index *load_multi_pack_index_with_status(struct odb_source_packed *source, + int *incomplete_chain) { struct strbuf midx_name = STRBUF_INIT; struct multi_pack_index *m; get_midx_filename(source, &midx_name); - m = load_multi_pack_index_one(source, midx_name.buf); - if (!m) - m = load_multi_pack_index_chain(source); + for (int attempt = 0; ; attempt++) { + *incomplete_chain = 0; + m = load_multi_pack_index_one(source, midx_name.buf); + if (!m) + m = load_multi_pack_index_chain(source, incomplete_chain); + if (!*incomplete_chain || attempt) + break; + /* A writer may have retired layers after we opened its old chain. */ + close_midx(m); + } strbuf_release(&midx_name); return m; } +struct multi_pack_index *load_multi_pack_index(struct odb_source_packed *source) +{ + int incomplete_chain; + return load_multi_pack_index_with_status(source, &incomplete_chain); +} + void close_midx(struct multi_pack_index *m) { uint32_t i; @@ -484,6 +514,14 @@ int prepare_midx_pack(struct multi_pack_index *m, return 0; } +void clear_midx_pack_errors(struct multi_pack_index *m) +{ + for (; m; m = m->base_midx) + for (uint32_t i = 0; i < m->num_packs; i++) + if (m->packs[i] == MIDX_PACK_ERROR) + m->packs[i] = NULL; +} + struct packed_git *nth_midxed_pack(struct multi_pack_index *m, uint32_t pack_int_id) { @@ -589,23 +627,23 @@ uint32_t nth_midxed_pack_int_id(struct multi_pack_index *m, uint32_t pos) (off_t)pos * MIDX_CHUNK_OFFSET_WIDTH); } -int fill_midx_entry(struct multi_pack_index *m, - const struct object_id *oid, - struct pack_entry *e, - struct packed_git **bad_pack) +enum midx_fill_result midx_fill_entry(struct multi_pack_index *m, + const struct object_id *oid, + struct pack_entry *e, + struct packed_git **bad_pack) { uint32_t pos; uint32_t pack_int_id; struct packed_git *p; if (!bsearch_midx(oid, m, &pos)) - return 0; + return MIDX_FILL_MISS; midx_for_object(&m, pos); pack_int_id = nth_midxed_pack_int_id(m, pos); if (prepare_midx_pack(m, pack_int_id)) - return 0; + return MIDX_FILL_OWNER_UNAVAILABLE; p = m->packs[pack_int_id - m->num_packs_in_base]; /* @@ -616,19 +654,19 @@ int fill_midx_entry(struct multi_pack_index *m, * loaded! */ if (!is_pack_valid(p)) - return 0; + return MIDX_FILL_OWNER_UNAVAILABLE; if (oidset_size(&p->bad_objects) && oidset_contains(&p->bad_objects, oid)) { if (bad_pack && !*bad_pack) *bad_pack = p; - return 0; + return MIDX_FILL_OWNER_UNAVAILABLE; } e->offset = nth_midxed_offset(m, pos); e->p = p; - return 1; + return MIDX_FILL_HIT; } /* Match "foo.idx" against either "foo.pack" _or_ "foo.idx". */ @@ -827,6 +865,38 @@ void clear_incremental_midx_files_ext(struct odb_source_packed *source, const ch strset_clear(&data.keep); } +void hold_midx_write_lock(struct odb_source_packed *source, + struct lock_file *lock) +{ + struct repository *r = source->base.odb->repo; + struct strbuf path = STRBUF_INIT; + + strbuf_addf(&path, "%s/pack/multi-pack-index-write", source->base.path); + if (safe_create_leading_directories(r, path.buf)) + die_errno(_("unable to create leading directories of %s"), path.buf); + repo_hold_lock_file_for_update(r, lock, path.buf, LOCK_DIE_ON_ERROR); + strbuf_release(&path); + + if (source->midx) { + struct multi_pack_index *cached = source->midx; + struct multi_pack_index *current, *m; + int incomplete; + + current = m = load_multi_pack_index_with_status(source, &incomplete); + while (cached && m && cached->has_chain == m->has_chain && + !hashcmp(midx_get_checksum_hash(cached), + midx_get_checksum_hash(m), r->hash_algo)) { + cached = cached->base_midx; + m = m->base_midx; + } + if (incomplete || cached || m) + die(_("multi-pack-index changed while acquiring writer lock")); + close_midx(current); + } + /* Pick up packs added before we acquired the lock. */ + source->initialized = false; +} + void clear_midx_file(struct repository *r) { struct odb_source_files *files; @@ -928,12 +998,17 @@ int verify_midx_file(struct odb_source_packed *source, unsigned flags) struct pair_pos_vs_id *pairs = NULL; uint32_t i; struct progress *progress = NULL; - struct multi_pack_index *m = load_multi_pack_index(source); + int incomplete_chain; + struct multi_pack_index *m = load_multi_pack_index_with_status(source, + &incomplete_chain); struct multi_pack_index *curr; verify_midx_error = 0; + if (incomplete_chain) + midx_report(_("one or more multi-pack-index chain files could not be loaded")); + if (!m) { - int result = 0; + int result = verify_midx_error; struct stat sb; struct strbuf filename = STRBUF_INIT; @@ -947,8 +1022,9 @@ int verify_midx_file(struct odb_source_packed *source, unsigned flags) return result; } - if (!midx_checksum_valid(m)) - midx_report(_("incorrect checksum")); + for (curr = m; curr; curr = curr->base_midx) + if (!midx_checksum_valid(curr)) + midx_report(_("incorrect checksum")); if (flags & MIDX_PROGRESS) progress = start_delayed_progress(r, @@ -962,26 +1038,26 @@ int verify_midx_file(struct odb_source_packed *source, unsigned flags) } stop_progress(&progress); - if (m->num_objects == 0) { - midx_report(_("the midx contains no oid")); - /* - * Remaining tests assume that we have objects, so we can - * return here. - */ - goto cleanup; - } + for (curr = m; curr; curr = curr->base_midx) { + if (curr->num_objects == 0) { + midx_report(_("the midx contains no oid")); + /* + * Remaining tests assume that we have objects, so we can + * return here. + */ + goto cleanup; + } - if (flags & MIDX_PROGRESS) - progress = start_sparse_progress(r, - _("Verifying OID order in multi-pack-index"), - m->num_objects - 1); + if (flags & MIDX_PROGRESS) + progress = start_sparse_progress(r, + _("Verifying OID order in multi-pack-index"), + curr->num_objects - 1); - for (curr = m; curr; curr = curr->base_midx) { - for (i = 0; i < m->num_objects - 1; i++) { + for (i = 0; i < curr->num_objects - 1; i++) { struct object_id oid1, oid2; - nth_midxed_object_oid(&oid1, m, m->num_objects_in_base + i); - nth_midxed_object_oid(&oid2, m, m->num_objects_in_base + i + 1); + nth_midxed_object_oid(&oid1, curr, curr->num_objects_in_base + i); + nth_midxed_object_oid(&oid2, curr, curr->num_objects_in_base + i + 1); if (oidcmp(&oid1, &oid2) >= 0) midx_report(_("oid lookup out of order: oid[%d] = %s >= %s = oid[%d]"), @@ -989,8 +1065,8 @@ int verify_midx_file(struct odb_source_packed *source, unsigned flags) midx_display_sparse_progress(progress, i + 1); } + stop_progress(&progress); } - stop_progress(&progress); /* * Create an array mapping each object to its packfile id. Sort it @@ -1007,15 +1083,16 @@ int verify_midx_file(struct odb_source_packed *source, unsigned flags) if (flags & MIDX_PROGRESS) progress = start_sparse_progress(r, _("Sorting objects by packfile"), - m->num_objects); + m->num_objects + m->num_objects_in_base); display_progress(progress, 0); /* TODO: Measure QSORT() progress */ - QSORT(pairs, m->num_objects, compare_pair_pos_vs_id); + QSORT(pairs, m->num_objects + m->num_objects_in_base, + compare_pair_pos_vs_id); stop_progress(&progress); if (flags & MIDX_PROGRESS) progress = start_sparse_progress(r, _("Verifying object offsets"), - m->num_objects); + m->num_objects + m->num_objects_in_base); for (i = 0; i < m->num_objects + m->num_objects_in_base; i++) { struct object_id oid; struct pack_entry e; @@ -1032,7 +1109,7 @@ int verify_midx_file(struct odb_source_packed *source, unsigned flags) nth_midxed_object_oid(&oid, m, pairs[i].pos); - if (!fill_midx_entry(m, &oid, &e, NULL)) { + if (midx_fill_entry(m, &oid, &e, NULL) != MIDX_FILL_HIT) { midx_report(_("failed to load pack entry for oid[%d] = %s"), pairs[i].pos, oid_to_hex(&oid)); continue; diff --git a/midx.h b/midx.h index 1f2f2d53214da5..eed55858e4d538 100644 --- a/midx.h +++ b/midx.h @@ -10,6 +10,7 @@ struct bitmapped_pack; struct git_hash_algo; struct odb_source; struct strvec; +struct lock_file; #define MIDX_SIGNATURE 0x4d494458 /* "MIDX" */ #define MIDX_VERSION_V1 1 @@ -85,6 +86,8 @@ struct multi_pack_index { #define MIDX_WRITE_INCREMENTAL (1 << 5) #define MIDX_WRITE_COMPACT (1 << 6) #define MIDX_WRITE_NO_CHAIN (1 << 7) +/* Internal: the caller holds the object store's MIDX writer lock. */ +#define MIDX_WRITE_LOCK_HELD (1 << 8) #define MIDX_EXT_REV "rev" #define MIDX_EXT_BITMAP "bitmap" @@ -103,6 +106,7 @@ void get_split_midx_filename_ext(struct odb_source_packed *source, struct strbuf struct multi_pack_index *get_multi_pack_index(struct odb_source_packed *source); struct multi_pack_index *load_multi_pack_index(struct odb_source_packed *source); int prepare_midx_pack(struct multi_pack_index *m, uint32_t pack_int_id); +void clear_midx_pack_errors(struct multi_pack_index *m); struct packed_git *nth_midxed_pack(struct multi_pack_index *m, uint32_t pack_int_id); int nth_bitmapped_pack(struct multi_pack_index *m, @@ -117,8 +121,25 @@ uint32_t nth_midxed_pack_int_id(struct multi_pack_index *m, uint32_t pos); struct object_id *nth_midxed_object_oid(struct object_id *oid, struct multi_pack_index *m, uint32_t n); -int fill_midx_entry(struct multi_pack_index *m, const struct object_id *oid, - struct pack_entry *e, struct packed_git **bad_pack); +/* + * Result of looking an object up in a multi-pack-index. MIDX_FILL_HIT means + * "e was filled in"; the two miss variants distinguish an object the midx does + * not know about (MIDX_FILL_MISS) from one it does know about but whose owning + * pack is unavailable or whose selected copy is corrupt + * (MIDX_FILL_OWNER_UNAVAILABLE). A known-bad object also sets *bad_pack, if + * provided, to the owning pack so the caller can tell "corrupt" apart from + * "absent". + */ +enum midx_fill_result { + MIDX_FILL_MISS = 0, + MIDX_FILL_HIT, + MIDX_FILL_OWNER_UNAVAILABLE, +}; + +enum midx_fill_result midx_fill_entry(struct multi_pack_index *m, + const struct object_id *oid, + struct pack_entry *e, + struct packed_git **bad_pack); int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name); int midx_layer_contains_pack(struct multi_pack_index *m, @@ -132,7 +153,7 @@ int prepare_multi_pack_index_one(struct odb_source_packed *source); */ int write_midx_file(struct odb_source_packed *source, const char *preferred_pack_name, const char *refs_snapshot, - unsigned flags); + const char *incremental_base, unsigned flags); int write_midx_file_only(struct odb_source_packed *source, struct string_list *packs_to_include, const char *preferred_pack_name, @@ -144,6 +165,9 @@ int write_midx_file_compact(struct odb_source_packed *source, struct multi_pack_index *to, const char *incremental_base, unsigned flags); +void hold_midx_write_lock(struct odb_source_packed *source, + struct lock_file *lock); +/* The caller must hold the MIDX writer lock while clearing files. */ void clear_midx_file(struct repository *r); void clear_incremental_midx_files(struct repository *r, const struct strvec *keep_hashes); diff --git a/odb/source-files.c b/odb/source-files.c index 9b4b67cc976113..9d31c7c1e6d81b 100644 --- a/odb/source-files.c +++ b/odb/source-files.c @@ -576,7 +576,7 @@ bool odb_source_files_optimize_required(struct odb_source *source, * When we'd merge at least two packs with one another we always * perform the repack. */ - if (geometry.split) { + if (geometry.split || geometry.promisor_split) { ret = true; goto out; } @@ -710,7 +710,8 @@ int odb_source_files_optimize(struct odb_source *source, pack_geometry_init(&geometry, &existing_packs, &po_args); pack_geometry_split(&geometry); - if (geometry.split < geometry.pack_nr) { + if (geometry.split < geometry.pack_nr || + geometry.promisor_split < geometry.promisor_pack_nr) { strvec_pushf(&repack_cmd.args, "--geometric=%d", geometry.split_factor); } else { diff --git a/odb/source-packed.c b/odb/source-packed.c index 1d90e714e62ed2..3ffe87ccd6502c 100644 --- a/odb/source-packed.c +++ b/odb/source-packed.c @@ -17,13 +17,18 @@ static int find_pack_entry(struct odb_source_packed *store, const struct object_id *oid, struct pack_entry *e, + enum object_info_flags flags, struct packed_git **bad_pack) { struct packfile_list_entry *l; + enum midx_fill_result midx_result = MIDX_FILL_MISS; odb_source_prepare(&store->base, 0); - if (store->midx && fill_midx_entry(store->midx, oid, e, bad_pack)) - return 1; + if (store->midx) { + midx_result = midx_fill_entry(store->midx, oid, e, bad_pack); + if (midx_result == MIDX_FILL_HIT) + return 1; + } for (l = store->packs.head; l; l = l->next) { struct packed_git *p = l->pack; @@ -35,6 +40,33 @@ static int find_pack_entry(struct odb_source_packed *store, } } + /* + * A stale MIDX may name a vanished owning pack, or its selected copy + * may be corrupt, even though another covered pack has a usable copy. + * The regular fallback above skips MIDX-covered packs, and repreparing + * the on-disk pack set does not reload the borrowed, cached MIDX, so + * scan its packs directly for the survivor. + * + * Do this only on the second read, by which point repreparing packs has + * already had a chance to find an object merely relocated into a new, + * uncovered pack; only a genuine hidden duplicate reaches here. + */ + if (midx_result == MIDX_FILL_OWNER_UNAVAILABLE && + (flags & OBJECT_INFO_SECOND_READ)) { + struct multi_pack_index *m = store->midx; + uint32_t i; + + for (i = 0; i < m->num_packs + m->num_packs_in_base; i++) { + struct packed_git *p; + + if (prepare_midx_pack(m, i)) + continue; + p = nth_midxed_pack(m, i); + if (p && packfile_fill_entry(p, oid, e, bad_pack)) + return 1; + } + } + return 0; } @@ -57,7 +89,8 @@ static enum odb_read_status odb_source_packed_read_object_info(struct odb_source if (flags & OBJECT_INFO_SECOND_READ) odb_source_prepare(source, ODB_PREPARE_FLUSH_CACHES); - if (!find_pack_entry(packed, oid, &e, &bad_pack)) { +retry: + if (!find_pack_entry(packed, oid, &e, flags, &bad_pack)) { /* * The lookup may have failed because the object is known to be * corrupt in one of the packfiles. Report the object as @@ -85,7 +118,7 @@ static enum odb_read_status odb_source_packed_read_object_info(struct odb_source if (ret < 0) { bad_pack = e.p; mark_bad_packed_object(e.p, oid); - goto out; + goto retry; } ret = 0; @@ -105,7 +138,7 @@ static int odb_source_packed_read_object_stream(struct odb_stream **out, struct odb_source_packed *packed = odb_source_packed_downcast(source); struct pack_entry e; - if (!find_pack_entry(packed, oid, &e, NULL)) + if (!find_pack_entry(packed, oid, &e, 0, NULL)) return -1; return packfile_read_object_stream(out, oid, e.p, e.offset); @@ -310,7 +343,8 @@ static int odb_source_packed_for_each_prefixed_object( store->skip_mru_updates = true; - m = get_multi_pack_index(store); + /* A MIDX owner need not be the copy that satisfies the pack flags. */ + m = opts->flags ? NULL : get_multi_pack_index(store); if (m) { ret = for_each_prefixed_object_in_midx(store, m, opts, data); if (ret) @@ -318,7 +352,7 @@ static int odb_source_packed_for_each_prefixed_object( } for (e = packfile_store_get_packs(store); e; e = e->next) { - if (e->pack->multi_pack_index) + if (m && e->pack->multi_pack_index) continue; if (should_exclude_pack(e->pack, opts->flags)) continue; @@ -611,7 +645,7 @@ static int odb_source_packed_freshen_object(struct odb_source *source, timesp = × } - if (!find_pack_entry(packed, oid, &e, NULL)) + if (!find_pack_entry(packed, oid, &e, 0, NULL)) return 0; if (e.p->is_cruft) return 0; @@ -711,6 +745,7 @@ static void report_pack_garbage(struct string_list *list) struct prepare_pack_data { struct odb_source_packed *source; struct string_list *garbage; + struct strset midx_packs; }; static void prepare_pack(const char *full_name, size_t full_name_len, @@ -720,8 +755,7 @@ static void prepare_pack(const char *full_name, size_t full_name_len, size_t base_len = full_name_len; if (strip_suffix_mem(full_name, &base_len, ".idx") && - !(data->source->midx && - midx_contains_pack(data->source->midx, file_name))) { + !strset_contains(&data->midx_packs, file_name)) { char *trimmed_path = xstrndup(full_name, full_name_len); packfile_store_load_pack(data->source, trimmed_path, data->source->base.local); @@ -755,12 +789,19 @@ static void prepare_packed_git_one(struct odb_source_packed *source) struct prepare_pack_data data = { .source = source, .garbage = &garbage, + .midx_packs = STRSET_INIT, }; + struct multi_pack_index *m; + + for (m = source->midx; m; m = m->base_midx) + for (uint32_t i = 0; i < m->num_packs; i++) + strset_add(&data.midx_packs, m->pack_names[i]); for_each_file_in_pack_dir(source->base.path, prepare_pack, &data); report_pack_garbage(data.garbage); string_list_clear(data.garbage, 0); + strset_clear(&data.midx_packs); } DEFINE_LIST_SORT(static, sort_packs, struct packfile_list_entry, next); @@ -797,8 +838,10 @@ static void odb_source_packed_prepare(struct odb_source *source, { struct odb_source_packed *packed = odb_source_packed_downcast(source); - if (flags & ODB_PREPARE_FLUSH_CACHES) + if (flags & ODB_PREPARE_FLUSH_CACHES) { packed->initialized = false; + clear_midx_pack_errors(packed->midx); + } if (packed->initialized) return; diff --git a/pack-bitmap.c b/pack-bitmap.c index f0b3161590f146..928dd90751624b 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -513,10 +513,9 @@ static int open_midx_bitmap(struct bitmap_index *bitmap_git, goto cleanup; } - for (i = 0; i < bitmap_git->midx->num_packs + bitmap_git->midx->num_packs_in_base; i++) { - if (prepare_midx_pack(bitmap_git->midx, i)) { - warning(_("could not open pack %s"), - bitmap_git->midx->pack_names[i]); + for (i = 0; i < midx->num_packs; i++) { + if (prepare_midx_pack(midx, midx->num_packs_in_base + i)) { + warning(_("could not open pack %s"), midx->pack_names[i]); goto cleanup; } } @@ -1944,7 +1943,7 @@ static void filter_bitmap_blob_limit(struct bitmap_index *bitmap_git, } for (i = 0; i < eindex->count; i++) { - size_t pos = st_add(i, bitmap_num_objects(bitmap_git)); + size_t pos = st_add(i, bitmap_num_objects_total(bitmap_git)); if (eindex->objects[i]->type == OBJ_BLOB && bitmap_get(to_filter, pos) && !bitmap_get(tips, pos) && @@ -2092,7 +2091,7 @@ int for_each_bitmapped_object(struct bitmap_index *bitmap_git, goto out; } - objects_nr = bitmap_num_objects(bitmap_git); + objects_nr = bitmap_num_objects_total(bitmap_git); full_word_count = objects_nr / BITS_IN_EWORD; /* We start from the all-1 bitmap and then filter down from there. */ diff --git a/remote-curl.c b/remote-curl.c index 2c35dd52400f83..82b96e8a850510 100644 --- a/remote-curl.c +++ b/remote-curl.c @@ -501,7 +501,16 @@ static struct discovery *discover_refs(const char *service, int for_push) version = protocol_v0; /* Add the extra Git-Protocol header */ - if (get_protocol_http_header(version, &protocol_header)) + get_protocol_http_header(version, &protocol_header); + if (maybe_smart && !strcmp(service, "git-receive-pack")) { + if (protocol_header.len) + strbuf_addch(&protocol_header, ':'); + else + strbuf_addstr(&protocol_header, GIT_PROTOCOL_HEADER ": "); + /* The server may need this before deciding which refs to advertise. */ + strbuf_addstr(&protocol_header, "explicit-haves"); + } + if (protocol_header.len) string_list_append(&extra_headers, protocol_header.buf); memset(&http_options, 0, sizeof(http_options)); @@ -999,9 +1008,10 @@ static int post_rpc(struct rpc_state *rpc, int stateless_connect, int flush_rece } else if (gzip_body) { /* * If we are looping to retry authentication, then the previous - * run will have set up the headers and gzip buffer already, - * and we just need to send it. + * run will have set up the gzip buffer already. The headers, + * however, are rebuilt for each request. */ + headers = curl_slist_append(headers, "Content-Encoding: gzip"); curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, gzip_body); curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE_LARGE, cast_size_t_to_curl_off_t(gzip_size)); diff --git a/repack-midx.c b/repack-midx.c index 64c7f8d0f42372..95ac62bdd0cc6c 100644 --- a/repack-midx.c +++ b/repack-midx.c @@ -285,7 +285,8 @@ static void repack_prepare_midx_command(struct child_process *cmd, { cmd->git_cmd = 1; - strvec_pushl(&cmd->args, "multi-pack-index", subcommand, NULL); + strvec_pushl(&cmd->args, "multi-pack-index", subcommand, + "--write-lock-held", NULL); if (opts->show_progress) strvec_push(&cmd->args, "--progress"); @@ -467,6 +468,8 @@ static int midx_compaction_step_exec_write(struct midx_compaction_step *step, } ret = repack_fill_midx_stdin_packs(&cmd, &step->u.write, &hash); + if (ret || !hash.nr) + goto out; if (hash.nr != 1) { ret = error(_("expected exactly one line during MIDX write, " "got: %"PRIuMAX), @@ -514,6 +517,8 @@ static int midx_compaction_step_exec_compact(struct midx_compaction_step *step, } ret = finish_command(&cmd); + if (!ret && !step->csum) + ret = error(_("missing MIDX output during compaction")); out: if (out) @@ -771,7 +776,9 @@ static int repack_make_midx_compaction_plan(struct repack_write_midx_opts *opts, break; } - if (midx_preferred_pack(m, &preferred_pack_idx) < 0) { + preferred_pack_idx = m->num_packs_in_base; + if (opts->write_bitmaps && + midx_preferred_pack(m, &preferred_pack_idx) < 0) { ret = error(_("could not find preferred pack for MIDX " "%s"), midx_get_checksum_hex(m)); goto out; @@ -943,7 +950,7 @@ static int write_midx_incremental(struct repack_write_midx_opts *opts) struct odb_source_files *files = odb_source_files_downcast(opts->existing->source); struct midx_compaction_step *steps = NULL; struct strbuf lock_name = STRBUF_INIT; - struct lock_file lf; + struct lock_file lf = LOCK_INIT; struct strvec keep_hashes = STRVEC_INIT; size_t steps_nr = 0; size_t i; @@ -978,7 +985,7 @@ static int write_midx_incremental(struct repack_write_midx_opts *opts) if (i + 1 < steps_nr) base = xstrdup(midx_compaction_step_base(&steps[i + 1])); - if (midx_compaction_step_exec(step, opts, base) < 0) { + if (midx_compaction_step_exec(step, opts, base)) { ret = error(_("unable to execute compaction step %"PRIuMAX), (uintmax_t)i); free(base); @@ -991,6 +998,9 @@ static int write_midx_incremental(struct repack_write_midx_opts *opts) i = steps_nr; while (i--) { struct midx_compaction_step *step = &steps[i]; + /* A write containing only base objects produces no layer. */ + if (!step->csum && step->type == MIDX_COMPACTION_STEP_WRITE) + continue; if (!step->csum) BUG("missing result for compaction step %"PRIuMAX, (uintmax_t)i); @@ -998,11 +1008,15 @@ static int write_midx_incremental(struct repack_write_midx_opts *opts) strvec_push(&keep_hashes, step->csum); } - commit_lock_file(&lf); + if (commit_lock_file(&lf)) { + ret = error_errno(_("could not write multi-pack-index chain")); + goto done; + } clear_incremental_midx_files(opts->existing->repo, &keep_hashes); done: + rollback_lock_file(&lf); strvec_clear(&keep_hashes); strbuf_release(&lock_name); for (i = 0; i < steps_nr; i++) diff --git a/send-pack.c b/send-pack.c index 5a8bdee69af5a3..ae661e7e7401b3 100644 --- a/send-pack.c +++ b/send-pack.c @@ -55,12 +55,16 @@ static void append_negative_object(struct repository *r, oid_array_append(haves, oid); } +static int check_to_send_update(const struct ref *ref, + const struct send_pack_args *args); + /* * Make a pack stream and spit it out into file descriptor fd */ static int pack_objects(struct repository *r, int fd, struct ref *refs, struct oid_array *advertised, struct oid_array *negotiated, + int use_explicit_haves, struct send_pack_args *args) { struct odb_generate_pack_options opts = ODB_GENERATE_PACK_OPTIONS_INIT; @@ -90,9 +94,14 @@ static int pack_objects(struct repository *r, append_negative_object(r, &opts.haves, &negotiated->oid[i]); while (refs) { - if (!is_null_oid(&refs->old_oid)) + /* + * Named refs still describe the remote state for update checks, + * but are not pack prerequisites under explicit-haves. + */ + if (!use_explicit_haves && !is_null_oid(&refs->old_oid)) append_negative_object(r, &opts.haves, &refs->old_oid); - if (!is_null_oid(&refs->new_oid)) + if (check_to_send_update(refs, args) == 0 && + !is_null_oid(&refs->new_oid)) oid_array_append(&opts.wants, &refs->new_oid); refs = refs->next; } @@ -500,6 +509,7 @@ int send_pack(struct repository *r, int use_push_options = 0; int push_options_supported = 0; int object_format_supported = 0; + int use_explicit_haves = 0; unsigned cmds_sent = 0; int ret; struct async demux; @@ -540,6 +550,8 @@ int send_pack(struct repository *r, args->use_ofs_delta = 1; if (server_supports("no-ref-delta")) args->no_ref_delta = 1; + if (server_supports("explicit-haves")) + use_explicit_haves = 1; if (server_supports("side-band-64k")) use_sideband = 1; if (server_supports("quiet")) @@ -596,6 +608,8 @@ int send_pack(struct repository *r, strbuf_addstr(&cap_buf, " atomic"); if (use_push_options) strbuf_addstr(&cap_buf, " push-options"); + if (use_explicit_haves) + strbuf_addstr(&cap_buf, " explicit-haves"); if (object_format_supported) strbuf_addf(&cap_buf, " object-format=%s", r->hash_algo->name); if (agent_supported) @@ -713,7 +727,8 @@ int send_pack(struct repository *r, PACKET_READ_DIE_ON_ERR_PACKET); if (need_pack_data && cmds_sent) { - if (pack_objects(r, out, remote_refs, extra_have, &commons, args) < 0) { + if (pack_objects(r, out, remote_refs, extra_have, &commons, + use_explicit_haves, args) < 0) { if (args->stateless_rpc) close(out); if (git_connection_is_socket(conn)) diff --git a/t/helper/test-read-midx.c b/t/helper/test-read-midx.c index 27a05da957afc2..9c5e30876102eb 100644 --- a/t/helper/test-read-midx.c +++ b/t/helper/test-read-midx.c @@ -82,7 +82,7 @@ static int read_midx_file(const char *object_dir, const char *checksum, for (i = 0; i < m->num_objects; i++) { nth_midxed_object_oid(&oid, m, i + m->num_objects_in_base); - fill_midx_entry(m, &oid, &e, NULL); + midx_fill_entry(m, &oid, &e, NULL); printf("%s %"PRIu64"\t%s\n", oid_to_hex(&oid), e.offset, e.p->pack_name); diff --git a/t/lib-httpd/nph-custom-auth.sh b/t/lib-httpd/nph-custom-auth.sh index d408d2caadb63d..65ba8204a0f221 100644 --- a/t/lib-httpd/nph-custom-auth.sh +++ b/t/lib-httpd/nph-custom-auth.sh @@ -3,6 +3,31 @@ VALID_CREDS_FILE=custom-auth.valid CHALLENGE_FILE=custom-auth.challenge +# Optional routes select separate credentials for a host and PATH_INFO prefix. +# Each line contains a prefix and a basename for .valid and .challenge files. +if test -f custom-auth.routes +then + while read -r prefix config + do + case "$HTTP_HOST$PATH_INFO" in + "$prefix"*) + VALID_CREDS_FILE=$config.valid + CHALLENGE_FILE=$config.challenge + break + ;; + esac + done >custom-auth.requests +fi + # # If $VALID_CREDS_FILE exists in $HTTPD_ROOT_PATH, consider each line as a valid # credential for the current request. Each line in the file is considered a diff --git a/t/lib-midx.sh b/t/lib-midx.sh index e38c609604c301..b522dbdb0f431a 100644 --- a/t/lib-midx.sh +++ b/t/lib-midx.sh @@ -34,3 +34,9 @@ compare_results_with_midx () { midx_git_two_modes "cat-file --batch-all-objects --batch-check --unordered" sorted ' } + +nth_line() { + local n="$1" + shift + awk "NR==$n" "$@" +} diff --git a/t/meson.build b/t/meson.build index 7f53cca7d1f891..2169ca82fbda93 100644 --- a/t/meson.build +++ b/t/meson.build @@ -1131,6 +1131,7 @@ integration_tests = [ 't9902-completion.sh', 't9903-bash-prompt.sh', 't9904-url-parse.sh', + 't9907-git-release-manifest.sh', ] benchmarks = [ diff --git a/t/t0212-trace2-event.sh b/t/t0212-trace2-event.sh index f5358a1dd44991..12330d6504de2f 100755 --- a/t/t0212-trace2-event.sh +++ b/t/t0212-trace2-event.sh @@ -362,4 +362,56 @@ test_expect_success 'unsafe URLs are redacted by default in def_param events' ' test_grep ! user:pwd trace.event ' +test_expect_success 'signature query parameters are redacted' ' + test_when_finished "rm trace.event" && + for name in sig Signature X-Amz-Signature %73ig + do + url="HtTpS://example.com/?$name=secret&signature-extra=public" && + expect="HtTpS://example.com/?$name=&signature-extra=public" && + : >trace.event && + GIT_TRACE2_EVENT="$(pwd)/trace.event" \ + test-tool trace2 303redact_def_param url "$url" && + test_grep -F "\"value\":\"$expect\"" trace.event || return 1 + done +' + +test_expect_success 'error targets redact URLs without changing stderr' ' + test_when_finished "rm trace.normal trace.perf trace.event stderr" && + url="https://example.com/?sig=secret." && + message="unable to access $SQ$url$SQ" && + expect="unable to access ${SQ}https://example.com/?sig=$SQ" && + GIT_TRACE2="$(pwd)/trace.normal" \ + GIT_TRACE2_PERF="$(pwd)/trace.perf" \ + GIT_TRACE2_EVENT="$(pwd)/trace.event" \ + test-tool trace2 003error "$message" 2>stderr && + test_grep -F "$expect" trace.normal && + test_grep -F "$expect" trace.perf && + test_grep -F "$expect" trace.event && + test_grep -F "\"fmt\":\"%s\"" trace.event && + test_grep -F "$message" stderr +' + +test_expect_success 'error redaction finds adjacent URLs' ' + test_when_finished "rm trace.event stderr" && + message="https://example.com/#fragment,https://example.com/?sig=secret" && + GIT_TRACE2_EVENT="$(pwd)/trace.event" \ + test-tool trace2 003error "$message" 2>stderr && + test_grep "\"event\":\"error\".*sig=" trace.event +' + +test_expect_success 'URL signature redaction can be disabled' ' + test_when_finished "rm trace.event stderr" && + GIT_TRACE2_REDACT=0 GIT_TRACE2_EVENT="$(pwd)/trace.event" \ + test-tool trace2 003error "https://example.com/?sig=secret" 2>stderr && + test_grep "\"event\":\"start\".*sig=secret" trace.event && + test_grep "\"event\":\"error\".*sig=secret" trace.event +' + +test_expect_success 'alias events redact URL signatures' ' + test_when_finished "rm trace.event" && + test_config alias.redact "rev-parse --sq-quote https://example.com/?sig=secret" && + GIT_TRACE2_EVENT="$(pwd)/trace.event" git redact >/dev/null && + test_grep "\"event\":\"alias\".*sig=" trace.event +' + test_done diff --git a/t/t5313-pack-bounds-checks.sh b/t/t5313-pack-bounds-checks.sh index 5be01260d7760f..3edcc0936a1a47 100755 --- a/t/t5313-pack-bounds-checks.sh +++ b/t/t5313-pack-bounds-checks.sh @@ -100,9 +100,12 @@ test_expect_success 'matched bogus object count' ' clear_base && # Unlike above, we should notice early that the .idx is totally - # bogus, and not even enumerate its contents. - git cat-file --batch-all-objects --batch-check >actual && + # bogus, report failure, and not enumerate its contents. + test_must_fail git cat-file --batch-all-objects \ + --batch-check >actual 2>err && test_must_be_empty actual && + test_grep "non-monotonic index" err && + test_grep "unable to enumerate all objects" err && # But as before, we can do the same object-access checks. test_must_fail git cat-file blob $object && diff --git a/t/t5319-multi-pack-index.sh b/t/t5319-multi-pack-index.sh index 68143cb5b76952..2377c118103504 100755 --- a/t/t5319-multi-pack-index.sh +++ b/t/t5319-multi-pack-index.sh @@ -54,6 +54,41 @@ test_expect_success "don't write midx with no packs" ' test_path_is_missing pack/multi-pack-index ' +test_expect_success 'skip non-incremental MIDX with no objects' ' + git init --bare empty.git && + ( + cd empty.git && + git pack-objects objects/pack/pack files.expect && + + for bitmap in "" --bitmap + do + git multi-pack-index write $bitmap >out 2>&1 && + test_must_be_empty out && + test_path_is_missing objects/pack/multi-pack-index && + ls objects/pack >files.actual && + test_cmp files.expect files.actual || return 1 + done && + + git multi-pack-index write --incremental --bitmap && + test_dir_is_empty objects/pack/multi-pack-index.d && + + echo blob | git hash-object -w --stdin >in && + git pack-objects objects/pack/pack in && + git pack-objects objects/pack/pack dup && + git add dup && + git commit -m dup && + dup_oid=$(git rev-parse HEAD:dup) && + + # Roll every object, including dup, into a single big pack. + git repack -adq && + + # Build a second, "moderate" pack that also contains dup, so dup + # now lives in two packs that the midx will cover. + moderate=$(echo "$dup_oid" | + git pack-objects --quiet $objdir/pack/pack) && + + # Attribute dup to the moderate pack in the midx. + git multi-pack-index write \ + --preferred-pack="pack-$moderate.idx" && + + # Simulate a concurrent "git repack" retiring the moderate pack: + # its files disappear, but the now-stale midx still names it as + # the owner of dup. A valid copy of dup survives in the big pack. + rm -f $objdir/pack/pack-$moderate.* && + + # The midx routes the lookup to the deleted pack, and the regular + # pack fallback skips midx-covered packs, so without recovery dup + # would appear missing even though it is physically present. + echo blob >expect && + git cat-file -t "$dup_oid" >actual && + test_cmp expect actual + ) +' + +test_expect_success PIPE 'refresh retries a restored MIDX pack' ' + test_when_finished "rm -fr repo" && + git init repo && + ( + cd repo && + victim=$(echo victim | git hash-object -w --stdin) && + pack=$(echo "$victim" | git pack-objects $objdir/pack/pack) && + git multi-pack-index write --incremental && + prime=$(echo prime | git hash-object -w --stdin) && + echo "$prime" | git pack-objects $objdir/pack/pack && + git prune-packed && + git multi-pack-index write --incremental && + + mkfifo in out && + (git cat-file --batch-check="%(objecttype)" out &) && + exec 9>in && + exec 8&9 && + read response <&8 && + test "$response" = blob && + + mv $objdir/pack/pack-$pack.pack saved.pack && + echo "$victim" >&9 && + read response <&8 && + test "$response" = "$victim missing" && + mv saved.pack $objdir/pack/pack-$pack.pack && + echo "$victim" >&9 && + read response <&8 && + test "$response" = blob && + exec 9>&- && + exec 8<&- + ) +' + +test_expect_success 'lookup recovers from a corrupt MIDX-selected copy' ' + test_when_finished "rm -fr repo" && + git init repo && + ( + cd repo && + test_commit one && + blob=$(git rev-parse HEAD:one.t) && + git repack -ad && + pack=$(echo "$blob" | git pack-objects $objdir/pack/pack) && + git multi-pack-index write --preferred-pack="pack-$pack.idx" && + chmod u+w $objdir/pack/pack-$pack.pack && + # The four-byte blob has a one-byte pack header. + corrupt_data $objdir/pack/pack-$pack.pack 13 && + git cat-file blob "$blob" >actual && + test_cmp one.t actual + ) +' + +test_expect_success 'lookup tries all copies after corrupt packed objects' ' + test_when_finished "rm -fr repo" && + git init repo && + ( + cd repo && + test_commit one && + blob=$(git rev-parse HEAD:one.t) && + pack1=$(echo "$blob" | git pack-objects $objdir/pack/pack) && + git multi-pack-index write --incremental && + test_commit two && + { + echo "$blob" && + git rev-parse HEAD:two.t + } >oids && + pack2=$(git pack-objects $objdir/pack/pack index && + offset=$(grep " $blob " index | cut -d" " -f1) && + chmod u+w $objdir/pack/pack-$pack.pack && + corrupt_data $objdir/pack/pack-$pack.pack $((offset + 1)) || + return 1 + done && + git cat-file blob "$blob" >actual && + test_cmp one.t actual + ) +' + test_done diff --git a/t/t5326-multi-pack-bitmaps.sh b/t/t5326-multi-pack-bitmaps.sh index 86beab1dae491e..490008d1d76ce5 100755 --- a/t/t5326-multi-pack-bitmaps.sh +++ b/t/t5326-multi-pack-bitmaps.sh @@ -305,7 +305,7 @@ test_midx_bitmap_cases () { ) ' - test_expect_success 'no .bitmap is written without any objects' ' + test_expect_success 'no MIDX or .bitmap is written without any objects' ' rm -fr repo && git init repo && test_when_finished "rm -fr repo" && @@ -318,13 +318,14 @@ test_midx_bitmap_cases () { pack-$empty.idx EOF + ls $objdir/pack >files.expect && git multi-pack-index write --bitmap --stdin-packs \ - err && + out 2>&1 && - test_grep "bitmap without any objects" err && - - test_path_is_file $midx && - test_path_is_missing $midx-$(midx_checksum $objdir).bitmap + test_must_be_empty out && + test_path_is_missing $midx && + ls $objdir/pack >files.actual && + test_cmp files.expect files.actual ) ' diff --git a/t/t5331-pack-objects-stdin.sh b/t/t5331-pack-objects-stdin.sh index c74b5861af322f..2a983e28ac43e7 100755 --- a/t/t5331-pack-objects-stdin.sh +++ b/t/t5331-pack-objects-stdin.sh @@ -368,7 +368,8 @@ test_expect_success '--stdin-packs does not perform backfill fetch' ' git -C remote config set --local uploadpack.allowfilter 1 && git -C remote config set --local uploadpack.allowanysha1inwant 1 && - git clone --filter=tree:0 "file://$(pwd)/remote" client && + git -c maintenance.auto=false clone --filter=tree:0 \ + "file://$(pwd)/remote" client && ( cd client && ls .git/objects/pack/*.promisor | sed "s|.*/||; s/\.promisor$/.pack/" >packs && diff --git a/t/t5334-incremental-multi-pack-index.sh b/t/t5334-incremental-multi-pack-index.sh index f0b82b5f65cf40..0f2fc3d9801da6 100755 --- a/t/t5334-incremental-multi-pack-index.sh +++ b/t/t5334-incremental-multi-pack-index.sh @@ -4,6 +4,7 @@ test_description='incremental multi-pack-index' . ./test-lib.sh . "$TEST_DIRECTORY"/lib-midx.sh +. "$TEST_DIRECTORY"/lib-chunk.sh GIT_TEST_MULTI_PACK_INDEX=0 export GIT_TEST_MULTI_PACK_INDEX @@ -80,6 +81,68 @@ test_expect_success 'midx verify with multiple layers' ' git multi-pack-index verify ' + +test_expect_success 'verify checksum of base MIDX' ' + midx="$midxdir/multi-pack-index-$(sed -n 1p "$midx_chain").midx" && + cp "$midx" midx.bak && + test_when_finished "mv midx.bak \"$midx\"" && + chmod u+w "$midx" && + echo extra >>"$midx" && + test_must_fail git multi-pack-index verify 2>err && + test_grep "multi-pack-index checksum does not match chain" err +' + +test_expect_success PERL_TEST_HELPERS 'verify OID order of base MIDX' ' + midx="$midxdir/multi-pack-index-$(sed -n 1p "$midx_chain").midx" && + cp "$midx" midx.bak && + test_when_finished "mv midx.bak \"$midx\"" && + corrupt_chunk_file "$midx" OIDL "$(test_oid rawsz)" "$(test_oid zero)" && + test_must_fail git multi-pack-index verify 2>err && + test_grep "oid lookup out of order" err +' + +test_expect_success SHA1 'reject empty base MIDX layer' ' + cp "$midx_chain" chain.bak && + test_when_finished "mv chain.bak \"$midx_chain\"" && + cp "$TEST_DIRECTORY"/t5319/no-objects.midx $packdir/multi-pack-index && + test_when_finished "rm -f $packdir/multi-pack-index" && + empty=$(midx_checksum "$objdir") && + mv $packdir/multi-pack-index "$midxdir/multi-pack-index-$empty.midx" && + test_when_finished "rm -f \"$midxdir/multi-pack-index-$empty.midx\"" && + { + echo "$empty" && + cat chain.bak + } >"$midx_chain" && + test_must_fail git multi-pack-index verify 2>err && + test_grep "the midx contains no oid" err +' + +for missing in 1 2 +do + test_expect_success "verify missing MIDX layer $missing" ' + midx="$midxdir/multi-pack-index-$(sed -n "${missing}p" "$midx_chain").midx" && + mv "$midx" missing.midx && + test_when_finished "mv missing.midx \"$midx\"" && + test_must_fail git multi-pack-index verify 2>err && + test_grep "one or more multi-pack-index chain files could not be loaded" err && + git cat-file -e 1.1 && + git cat-file -e 2.2 + ' +done + + +test_expect_success 'verify premature end of MIDX chain' ' + cp "$midx_chain" chain.bak && + test_when_finished "mv chain.bak \"$midx_chain\"" && + tr "\n" " " "$midx_chain" && + echo >>"$midx_chain" && + test_must_fail git multi-pack-index verify 2>err && + test_grep "unexpected end of multi-pack-index chain" err && + test_grep "one or more multi-pack-index chain files could not be loaded" err && + git cat-file -e 1.1 && + git cat-file -e 2.2 +' + test_expect_success 'read bitmap from second MIDX layer' ' git rev-list --test-bitmap 2.2 ' @@ -88,6 +151,45 @@ test_expect_success 'read earlier bitmap from second MIDX layer' ' git rev-list --test-bitmap 1.2 ' +test_expect_success 'bitmap falls back when a pack in the tip layer is missing' ' + git init missing-pack && + ( + cd missing-pack && + for i in 1 2 3 + do + echo "$i" | git hash-object -w --stdin >oid && + git pack-objects "$packdir/pack" oid && + pack=$(git pack-objects "$packdir/pack" actual 2>err && + test_cmp oid actual && + test_grep "could not open pack pack-$pack.idx" err + ) +' + +test_expect_success 'blob:limit filters loose objects above an incremental bitmap' ' + blob=$(echo loose | git hash-object -w --stdin) && + git rev-list --use-bitmap-index --objects --no-object-names \ + --filter=blob:limit=1 --filter-provided-objects \ + 2.2:2.2.t "$blob" >actual && + test_must_be_empty actual +' + +test_expect_success 'filtered enumeration covers the whole MIDX chain' ' + git -c core.multiPackIndex=false cat-file --batch-all-objects \ + --filter=object:type=tree --batch-check="%(objectname)" >expect && + git cat-file --batch-all-objects \ + --filter=object:type=tree --batch-check="%(objectname)" >actual && + test_cmp expect actual +' + test_expect_success 'show object from first pack' ' git cat-file -p 1.1 ' @@ -128,7 +230,12 @@ test_expect_success 'write MIDX layer with --base=none and --no-write-chain-file --no-write-chain-file --base=none)" && test_cmp "$midx_chain.bak" "$midx_chain" && - test_path_is_file "$midxdir/multi-pack-index-$layer.midx" + test_path_is_file "$midxdir/multi-pack-index-$layer.midx" && + + echo "$layer" >"$midx_chain" && + test-tool read-midx --show-objects "$objdir" "$layer" >midx.objects && + test_grep "^$(git rev-parse 2.2) " midx.objects && + cp "$midx_chain.bak" "$midx_chain" ' test_expect_success 'write MIDX layer with --base= and --no-write-chain-file' ' @@ -136,11 +243,42 @@ test_expect_success 'write MIDX layer with --base= and --no-write-chain-fi git repack -d && cp "$midx_chain" "$midx_chain.bak" && + base="$(nth_line 1 "$midx_chain")" && layer="$(git multi-pack-index write --bitmap --incremental \ - --no-write-chain-file --base="$(nth_line 1 "$midx_chain")")" && + --no-write-chain-file --base="$base")" && test_cmp "$midx_chain.bak" "$midx_chain" && - test_path_is_file "$midxdir/multi-pack-index-$layer.midx" + test_path_is_file "$midxdir/multi-pack-index-$layer.midx" && + + { + echo "$base" && + echo "$layer" + } >"$midx_chain" && + test-tool read-midx --show-objects "$objdir" "$layer" >midx.objects && + test_grep "^$(git rev-parse 2.2) " midx.objects && + cp "$midx_chain.bak" "$midx_chain" +' + +test_expect_success 'write MIDX layer with --stdin-packs and a custom base' ' + base="$(nth_line 1 "$midx_chain")" && + test-tool read-midx "$objdir" "$base" >base.midx && + test-tool read-midx "$objdir" >tip.midx && + sed -n "/^pack-.*\\.idx$/p" base.midx >base.packs && + sed -n "/^pack-.*\\.idx$/p" tip.midx >tip.packs && + test_line_count = 2 tip.packs && + sed -n 1p tip.packs >expect && + cat base.packs expect >packs && + layer="$(git multi-pack-index write --incremental --stdin-packs \ + --no-write-chain-file --base="$base" "$midx_chain" && + test-tool read-midx "$objdir" >layer.midx && + sed -n "/^pack-.*\\.idx$/p" layer.midx >actual && + test_cmp expect actual ' for reuse in false single multi @@ -179,6 +317,47 @@ test_expect_success 'relink existing MIDX layer' ' ' + +test_expect_success POSIXPERM 'retry interrupted monolithic MIDX promotion' ' + git init interrupted-promotion && + ( + cd interrupted-promotion && + test_commit base && + git repack -ad && + GIT_TEST_MIDX_WRITE_REV=1 git multi-pack-index write --bitmap && + hash=$(midx_checksum $objdir) && + mkdir "$midxdir" && + ln "$packdir/multi-pack-index" "$midxdir/multi-pack-index-$hash.midx" && + ln "$packdir/multi-pack-index-$hash.bitmap" "$midxdir/multi-pack-index-$hash.bitmap" && + ln "$packdir/multi-pack-index-$hash.rev" "$midxdir/multi-pack-index-$hash.rev" && + test_commit other && + git repack -d && + git multi-pack-index write --incremental --bitmap && + test_line_count = 2 "$midx_chain" && + git multi-pack-index verify + ) +' + +test_expect_success 'do not replace an unrelated MIDX during promotion' ' + git init unrelated-promotion && + ( + cd unrelated-promotion && + test_commit base && + git repack -ad && + git multi-pack-index write && + hash=$(midx_checksum $objdir) && + mkdir "$midxdir" && + echo unrelated >expect && + cp expect "$midxdir/multi-pack-index-$hash.midx" && + test_commit other && + git repack -d && + test_must_fail git multi-pack-index write --incremental 2>err && + test_grep "unable to link" err && + test_cmp expect "$midxdir/multi-pack-index-$hash.midx" && + test_path_is_missing "$midx_chain" + ) +' + test_expect_success 'non-incremental write with existing incremental chain' ' git init non-incremental-write-with-existing && test_when_finished "rm -fr non-incremental-write-with-existing" && @@ -195,4 +374,130 @@ test_expect_success 'non-incremental write with existing incremental chain' ' ) ' +test_expect_success 'skip initial MIDX layer with no objects' ' + git init empty && + ( + cd empty && + git config maintenance.auto false && + git pack-objects $packdir/pack out 2>&1 && + test_must_be_empty out && + test_dir_is_empty "$midxdir" || return 1 + done && + + write_midx_layer && + test_line_count = 1 "$midx_chain" && + git multi-pack-index verify + ) +' + +test_expect_success 'skip MIDX layer with empty pack' ' + git init empty-pack && + ( + cd empty-pack && + git config maintenance.auto false && + write_midx_layer && + + git pack-objects $packdir/pack files.expect && + + for bitmap in --bitmap --no-bitmap + do + git multi-pack-index write --incremental "$bitmap" >out 2>&1 && + test_must_be_empty out && + test_cmp chain.expect "$midx_chain" && + ls "$packdir" "$midxdir" >files.actual && + test_cmp files.expect files.actual || return 1 + done && + + write_midx_layer && + test_line_count = 2 "$midx_chain" && + git multi-pack-index verify && + git rev-list --test-bitmap 2.2 + ) +' + +test_expect_success 'skip MIDX layer with duplicate pack' ' + git init duplicate-pack && + ( + cd duplicate-pack && + git config maintenance.auto false && + write_midx_layer && + + git rev-parse HEAD^{tree} >in && + git pack-objects $packdir/pack files.expect && + + for bitmap in --bitmap --no-bitmap + do + git multi-pack-index write --incremental "$bitmap" >out 2>&1 && + test_must_be_empty out && + test_cmp chain.expect "$midx_chain" && + ls "$packdir" "$midxdir" >files.actual && + test_cmp files.expect files.actual || return 1 + done && + + write_midx_layer && + test_line_count = 2 "$midx_chain" && + git multi-pack-index verify && + git rev-list --test-bitmap 2.2 + ) +' + +test_expect_success 'MIDX maintenance shares a writer lock' ' + git init writer-lock && + ( + cd writer-lock && + test_commit base && + git repack -d && + git multi-pack-index write && + cp "$packdir/multi-pack-index" before && + touch "$packdir/multi-pack-index-write.lock" && + for cmd in write expire repack + do + test_must_fail git multi-pack-index $cmd 2>err && + test_grep "multi-pack-index-write.lock" err || return 1 + done && + test_must_fail git multi-pack-index write --incremental \ + --no-write-chain-file 2>err && + test_grep "multi-pack-index-write.lock" err && + test_must_fail git multi-pack-index compact \ + "$(test_oid zero)" "$(test_oid zero)" 2>err && + test_grep "multi-pack-index-write.lock" err && + test_must_fail git repack -ad 2>err && + test_grep "multi-pack-index-write.lock" err && + test_cmp before "$packdir/multi-pack-index" && + rm "$packdir/multi-pack-index-write.lock" && + git repack -d --write-midx=incremental && + test_path_is_missing "$packdir/multi-pack-index-write.lock" && + git multi-pack-index verify + ) +' + +test_expect_success 'layer checksum must match the chain filename' ' + git init misnamed-layer && + ( + cd misnamed-layer && + test_commit base && + git repack -d && + git multi-pack-index write --incremental && + layer=$(cat "$midx_chain") && + zero=$(test_oid zero) && + mv "$midxdir/multi-pack-index-$layer.midx" \ + "$midxdir/multi-pack-index-$zero.midx" && + echo "$zero" >"$midx_chain" && + test_must_fail git multi-pack-index verify 2>err && + test_grep "checksum does not match chain" err && + mv "$midxdir/multi-pack-index-$zero.midx" \ + "$midxdir/multi-pack-index-$layer.midx" && + echo "$layer" >"$midx_chain" && + git multi-pack-index verify + ) +' + test_done diff --git a/t/t5335-compact-multi-pack-index.sh b/t/t5335-compact-multi-pack-index.sh index 0f75ef2ccc96b3..39ef68b6c688c8 100755 --- a/t/t5335-compact-multi-pack-index.sh +++ b/t/t5335-compact-multi-pack-index.sh @@ -3,6 +3,7 @@ test_description='multi-pack-index compaction' . ./test-lib.sh +. "$TEST_DIRECTORY"/lib-midx.sh GIT_TEST_MULTI_PACK_INDEX=0 GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP=0 @@ -13,12 +14,6 @@ packdir=$objdir/pack midxdir=$packdir/multi-pack-index.d midx_chain=$midxdir/multi-pack-index-chain -nth_line() { - local n="$1" - shift - awk "NR==$n" "$@" -} - write_packs () { for c in "$@" do @@ -114,6 +109,27 @@ test_expect_success 'setup for bogus MIDX compaction scenarios' ' ) ' + +for missing in A C +do + test_expect_success "MIDX compaction with missing pack $missing" ' + pack=$(echo midx-compact-bogus/$packdir/pack-$missing-*.pack) && + mv "$pack" missing.pack && + test_when_finished "test ! -f missing.pack || mv missing.pack \"$pack\"" && + ( + cd midx-compact-bogus && + cp "$midx_chain" chain.expect && + test_must_fail git multi-pack-index compact --incremental \ + "$(nth_line 1 "$midx_chain")" \ + "$(nth_line 3 "$midx_chain")" 2>err && + test_grep "could not load pack" err && + test_cmp chain.expect "$midx_chain" + ) && + mv missing.pack "$pack" && + git -C midx-compact-bogus multi-pack-index verify + ' +done + test_expect_success 'MIDX compaction with missing endpoints' ' ( cd midx-compact-bogus && @@ -386,6 +402,88 @@ test_expect_success 'MIDX compaction with --base=none' ' ) ' +test_expect_success 'setup compaction with duplicate objects' ' + git init compact-duplicates && + ( + cd compact-duplicates && + git config maintenance.auto false && + write_packs A B && + test_commit C && + git pack-objects --all $packdir/pack-C && + git prune-packed && + git multi-pack-index write --incremental --bitmap && + test_commit D && + { + git rev-list --objects --no-object-names B && + git rev-list --objects --no-object-names D ^C + } >objects && + git pack-objects $packdir/pack-D expect && + git rev-list --objects --no-object-names --all >objects && + sort objects >objects.expect && + rm $packdir/pack-B-* && + git multi-pack-index compact --bitmap "$@" \ + "$(nth_line 3 "$midx_chain")" \ + "$(nth_line 4 "$midx_chain")" && + git cat-file commit "$oid" >actual && + test_cmp expect actual && + git multi-pack-index verify && + git rev-list --test-bitmap D && + GIT_TRACE2_EVENT="$PWD/trace" \ + git -c pack.allowPackReuse=multi pack-objects \ + --all --use-bitmap-index --delta-base-offset --stdout \ + >reuse.pack && + test_trace2_data pack-objects packs-reused "[23]" objects && + cut -d" " -f2 objects | sort >objects.actual && + test_cmp objects.expect objects.actual && + if test "$mode" != standalone + then + test_midx_layer_object_uniqueness + fi + ) + ' +done + +test_expect_success 'reject changing compaction base below retained layers' ' + ( + cd compact-duplicates && + cp "$midx_chain" chain.before && + test_must_fail git multi-pack-index compact --incremental \ + --base=none \ + "$(nth_line 2 "$midx_chain")" \ + "$(nth_line 3 "$midx_chain")" 2>err && + test_grep "cannot change the base when retaining layers" err && + test_cmp chain.before "$midx_chain" && + git multi-pack-index compact --incremental \ + --no-write-chain-file --base=none \ + "$(nth_line 2 "$midx_chain")" \ + "$(nth_line 3 "$midx_chain")" >layer && + test_line_count = 1 layer && + test_cmp chain.before "$midx_chain" + ) +' + test_expect_success 'MIDX compaction with bogus --base checksum' ' git init midx-compact-bogus-base && ( diff --git a/t/t5410-receive-pack.sh b/t/t5410-receive-pack.sh index 09d6bfd2a10f5a..3c1a5d0a21f4af 100755 --- a/t/t5410-receive-pack.sh +++ b/t/t5410-receive-pack.sh @@ -97,4 +97,61 @@ test_expect_success TEE_DOES_NOT_HANG \ test_must_fail git -C remote.git rev-list $(git -C repo rev-parse HEAD) ' +test_expect_success 'explicit-haves does not infer haves from named refs' ' + test_when_finished "rm -rf explicit-haves-*" && + + git init explicit-haves-src && + test_commit -C explicit-haves-src B && + git init --bare explicit-haves-base.git && + git -C explicit-haves-src push ../explicit-haves-base.git \ + B:refs/heads/main && + test_commit -C explicit-haves-src F1 && + test_commit -C explicit-haves-src F2 && + git -C explicit-haves-src reset --hard F1 && + test_commit -C explicit-haves-src F3 && + B=$(git -C explicit-haves-src rev-parse B) && + F1=$(git -C explicit-haves-src rev-parse F1) && + F2=$(git -C explicit-haves-src rev-parse F2) && + git clone --bare --shared explicit-haves-base.git \ + explicit-haves-remote.git && + git -C explicit-haves-remote.git update-ref -d refs/heads/main && + old_pack=$(printf "F2\n^B\n" | \ + git -C explicit-haves-src pack-objects --revs \ + ../explicit-haves-remote.git/objects/pack/old) && + git -C explicit-haves-remote.git update-ref \ + refs/heads/feature "$F2" && + git -C explicit-haves-remote.git update-ref \ + refs/heads/unchanged "$F2" && + git -C explicit-haves-remote.git config receive.unpackLimit 0 && + git -C explicit-haves-remote.git config maintenance.auto false && + + # The advertised ref is still used to reject a stale lease. + test_must_fail git -C explicit-haves-src push --no-thin \ + --force-with-lease=refs/heads/feature:$F1 \ + --receive-pack="git receive-pack --advertise-explicit-haves-for-testing" \ + ../explicit-haves-remote.git F3:refs/heads/feature \ + 2>explicit-haves-stale.err && + test_grep "stale info" explicit-haves-stale.err && + + # With the capability, only B is negative and only F3 is positive. + # The new pack is complete after replacing the old F2 pack. + rcvpck="unset GIT_TRACE_PACKET GIT_TRACE2_EVENT; git receive-pack --advertise-explicit-haves-for-testing" && + GIT_TRACE_PACKET="$(pwd)/explicit-haves.trace" \ + GIT_TRACE2_EVENT="$(pwd)/explicit-haves.event" \ + git -C explicit-haves-src push --no-thin \ + --force-with-lease=refs/heads/feature:$F2 \ + --receive-pack="$rcvpck" \ + ../explicit-haves-remote.git F3:refs/heads/feature \ + F2:refs/heads/unchanged && + test_grep "push< $B \\.have" explicit-haves.trace && + test_grep "push> .* explicit-haves" explicit-haves.trace && + test_grep "write_pack_file/wrote.*\"value\":\"6\"" \ + explicit-haves.event && + git -C explicit-haves-remote.git update-ref -d \ + refs/heads/unchanged && + rm explicit-haves-remote.git/objects/pack/old-"$old_pack".pack \ + explicit-haves-remote.git/objects/pack/old-"$old_pack".idx && + git -C explicit-haves-remote.git fsck --full +' + test_done diff --git a/t/t5541-http-push-smart.sh b/t/t5541-http-push-smart.sh index 8c348330277907..5adac6ed0b8b73 100755 --- a/t/t5541-http-push-smart.sh +++ b/t/t5541-http-push-smart.sh @@ -58,6 +58,19 @@ test_expect_success 'push to remote repository (standard)' ' GIT_TRACE_CURL=true git push -v -v 2>err && test_grep ! "Expect: 100-continue" err && test_grep "POST git-receive-pack ([0-9]* bytes)" err && + cat >expect-discovery <<-\EOF && + request + explicit-haves + response + EOF + sed -n \ + -e "/=> Send header: GET .*service=git-receive-pack /s/.*/request/p" \ + -e "/=> Send header: Git-Protocol: explicit-haves$/s/.*/explicit-haves/p" \ + -e "/<= Recv header: HTTP\/1.1 200/{s/.*/response/;p;q;}" \ + err >actual-discovery && + test_cmp expect-discovery actual-discovery && + grep "=> Send header: Git-Protocol:.*explicit-haves" err >sent-explicit-haves && + test_line_count = 1 sent-explicit-haves && (cd "$HTTPD_DOCUMENT_ROOT_PATH"/test_repo.git && test $HEAD = $(git rev-parse --verify HEAD)) ' @@ -71,6 +84,30 @@ test_expect_success 'used receive-pack service' ' check_access_log exp ' +test_expect_success 'receive-pack discovery retains explicit-haves across an HTTP redirect' ' + test_when_finished "git --git-dir=\"$HTTPD_DOCUMENT_ROOT_PATH/test_repo.git\" update-ref -d refs/heads/discovery-redirect" && + >"$HTTPD_ROOT_PATH"/access.log && + GIT_TRACE_CURL=true git push "$HTTPD_URL/smart-redir-perm/test_repo.git" \ + HEAD:refs/heads/discovery-redirect 2>err && + grep "=> Send header: Git-Protocol:.*explicit-haves" err >sent-explicit-haves && + test_line_count = 2 sent-explicit-haves && + cat >exp <<-\EOF && + GET /smart-redir-perm/test_repo.git/info/refs?service=git-receive-pack HTTP/1.1 301 + GET /smart/test_repo.git/info/refs?service=git-receive-pack HTTP/1.1 200 + POST /smart/test_repo.git/git-receive-pack HTTP/1.1 200 + EOF + check_access_log exp +' + +test_expect_success 'receive-pack discovery combines version and explicit-haves' ' + test_when_finished "git --git-dir=\"$HTTPD_DOCUMENT_ROOT_PATH/test_repo.git\" update-ref -d refs/heads/discovery-v1" && + GIT_TRACE_CURL=true git -c protocol.version=1 push origin \ + HEAD:refs/heads/discovery-v1 2>err && + grep "=> Send header: Git-Protocol:.*explicit-haves" err >sent-explicit-haves && + test_line_count = 1 sent-explicit-haves && + test_grep "Git-Protocol: version=1:explicit-haves$" sent-explicit-haves +' + test_expect_success 'push to remote repository (standard) with sending Accept-Language' ' cat >exp <<-\EOF && => Send header: Accept-Language: ko-KR, *;q=0.9 diff --git a/t/t5551-http-fetch-smart.sh b/t/t5551-http-fetch-smart.sh index 805bec025c589f..6784ab86c43487 100755 --- a/t/t5551-http-fetch-smart.sh +++ b/t/t5551-http-fetch-smart.sh @@ -17,6 +17,19 @@ fi test "$HTTP_PROTO" = "HTTP/2" && enable_http2 start_httpd +# The cURL version which Debian 12 ships (v7.88.1) can fail to retry +# authentication after an early HTTP/2 response. This bug was introduced +# in cURL v7.88.0 (8c762f5998 (http2: minor buffer and error path fixes, +# 2023-02-08)) and fixed in v8.3.0 (https://github.com/curl/curl/pull/11756). +test_lazy_prereq HAVE_CURL_HTTP2_BUG " + test_have_prereq HTTP2 && + build_option libcurl | + awk -F. ' + (\$1 == 7 && \$2 >= 88) || (\$1 == 8 && \$2 < 3) { broken = 1 } + END { exit !broken } + ' +" + test_expect_success HTTP2 'enable client-side http/2' ' git config --global http.version HTTP/2 ' @@ -224,7 +237,8 @@ test_expect_success 'clone from auth-only-for-push repository' ' test_cmp expect actual ' -test_expect_success 'clone from auth-only-for-objects repository' ' +test_expect_success !HAVE_CURL_HTTP2_BUG \ + 'clone from auth-only-for-objects repository' ' echo two >expect && set_askpass user@host pass@host && git clone --bare "$HTTPD_URL/auth-fetch/smart/repo.git" half-auth && @@ -233,7 +247,22 @@ test_expect_success 'clone from auth-only-for-objects repository' ' test_cmp expect actual ' -test_expect_success 'no-op half-auth fetch does not require a password' ' +test_expect_success !HAVE_CURL_HTTP2_BUG \ + 'http auth retries gzip-encoded requests' ' + server_option=$(test-tool genzeros 1024 | tr "\000" x) && + set_askpass user@host pass@host && + GIT_TRACE_CURL="$TRASH_DIRECTORY/trace" \ + GIT_TEST_PROTOCOL_VERSION=2 \ + git -c protocol.version=2 ls-remote \ + --server-option="$server_option" \ + "$HTTPD_URL/auth-fetch/smart/repo.git" >/dev/null && + expect_askpass both user%40host && + grep -i "=> Send header: content-encoding: gzip" trace >gzip.headers && + test_line_count = 2 gzip.headers +' + +test_expect_success !HAVE_CURL_HTTP2_BUG \ + 'no-op half-auth fetch does not require a password' ' set_askpass wrong && # NEEDSWORK: When using HTTP(S), protocol v0 supports a "half-auth" diff --git a/t/t5563-simple-http-auth.sh b/t/t5563-simple-http-auth.sh index 99a73b54a24a4f..270110cad39772 100755 --- a/t/t5563-simple-http-auth.sh +++ b/t/t5563-simple-http-auth.sh @@ -54,7 +54,11 @@ expect_credential_query () { per_test_cleanup () { rm -f *.cred && rm -f "$HTTPD_ROOT_PATH"/custom-auth.valid \ - "$HTTPD_ROOT_PATH"/custom-auth.challenge + "$HTTPD_ROOT_PATH"/custom-auth.challenge \ + "$HTTPD_ROOT_PATH"/custom-auth.routes \ + "$HTTPD_ROOT_PATH"/custom-auth.requests \ + "$HTTPD_ROOT_PATH"/custom-auth.destination.valid \ + "$HTTPD_ROOT_PATH"/custom-auth.destination.challenge } test_expect_success 'setup repository' ' @@ -907,4 +911,250 @@ test_expect_success SPNEGO 'http.emptyAuth=false skips Negotiate' ' test_line_count = 1 actual_401s ' +for auth in challenged preauthenticated redirected +do +test_expect_success "push follows discovery redirects ($auth)" ' + test_when_finished per_test_cleanup && + test_config -C "$HTTPD_DOCUMENT_ROOT_PATH/repo.git" http.receivepack true && + set_credential_reply get <<-EOF && + capability[]=authtype + authtype=Bearer + credential=redirect-token + EOF + cat >"$HTTPD_ROOT_PATH/custom-auth.valid" <<-EOF && + id=1 creds=Bearer redirect-token + EOF + cat >"$HTTPD_ROOT_PATH/custom-auth.challenge" <<-EOF && + id=1 status=302 response=Location: $HTTPD_URL/smart/repo.git/info/refs?service=git-receive-pack + id=default response=WWW-Authenticate: Bearer realm="example.com" + EOF + test_config_global credential.helper test-helper && + url="$HTTPD_URL/custom_auth/repo.git" && + >expect && + if test "$auth" = redirected + then + url="$HTTPD_URL/redir-to/custom_auth/repo.git" && + cat >>expect <<-EOF + GET /redir-to/custom_auth/repo.git/info/refs?service=git-receive-pack HTTP/1.1 302 + GET /really-redir-to?path=custom_auth/repo.git/info/refs&qs=service=git-receive-pack HTTP/1.1 302 + EOF + fi && + if test "$auth" = preauthenticated + then + test_config_global http.extraHeader "Authorization: Bearer redirect-token" + else + echo "GET /custom_auth/repo.git/info/refs?service=git-receive-pack HTTP/1.1 200 -" >>expect + fi && + cat >>expect <<-EOF && + GET /custom_auth/repo.git/info/refs?service=git-receive-pack HTTP/1.1 200 - + GET /smart/repo.git/info/refs?service=git-receive-pack HTTP/1.1 200 + POST /smart/repo.git/git-receive-pack HTTP/1.1 200 + EOF + >"$HTTPD_ROOT_PATH/access.log" && + GIT_TRACE_CURL="$TRASH_DIRECTORY/redirect-$auth.trace" \ + git -c http.followRedirects=initial push \ + "$url" HEAD:refs/heads/redirect-$auth && + # Apache does not record the status written by the NPH CGI. + test_grep "Recv header: HTTP/1.1 302" "redirect-$auth.trace" && + if test "$auth" = preauthenticated + then + test_grep ! "Recv header: HTTP/1.1 401" "redirect-$auth.trace" + else + test_grep "Recv header: HTTP/1.1 401" "redirect-$auth.trace" + fi && + check_access_log expect && + git rev-parse HEAD >expect-oid && + git -C "$HTTPD_DOCUMENT_ROOT_PATH/repo.git" \ + rev-parse refs/heads/redirect-$auth >actual-oid && + test_cmp expect-oid actual-oid +' +done + +for extra_header in none cookie +do +test_expect_success "select credentials after cross-host retry redirect ($extra_header)" ' + test_when_finished per_test_cleanup && + test_config -C "$HTTPD_DOCUMENT_ROOT_PATH/repo.git" http.receivepack true && + write_script "$TRASH_DIRECTORY/bin/git-credential-redirect-helper" <<-\EOF && + test "$1" = get || exit 0 + host= + while IFS= read -r line + do + case "$line" in host=*) host=${line#host=} ;; esac + done + case "$host" in + "$ORIGIN_HOST") echo username=alice ;; + "$DESTINATION_HOST") echo username=bob ;; + *) exit 1 ;; + esac + echo password=secret-passwd + EOF + cat >"$HTTPD_ROOT_PATH/custom-auth.valid" <<-EOF && + id=1 creds=Basic YWxpY2U6c2VjcmV0LXBhc3N3ZA== + id=2 creds=Basic Ym9iOnNlY3JldC1wYXNzd2Q= + EOF + cat >"$HTTPD_ROOT_PATH/custom-auth.challenge" <<-EOF && + id=1 status=302 response=Location: $HTTPD_PROTO://localhost:$LIB_HTTPD_PORT/custom_auth/repo.git/info/refs?service=git-receive-pack + id=2 status=200 + id=default response=WWW-Authenticate: Basic realm="example.com" + EOF + test_config_global credential.helper redirect-helper && + if test "$extra_header" = cookie + then + test_config_global "http.$HTTPD_URL/.extraHeader" "Cookie: origin-secret" + fi && + >"$HTTPD_ROOT_PATH/access.log" && + ORIGIN_HOST=$HTTPD_DEST DESTINATION_HOST=localhost:$LIB_HTTPD_PORT \ + git -c http.followRedirects=initial push \ + "$HTTPD_URL/custom_auth/repo.git" \ + HEAD:refs/heads/cross-host-$extra_header 2>err && + cat >expect <<-EOF && + GET /custom_auth/repo.git/info/refs?service=git-receive-pack HTTP/1.1 200 - + GET /custom_auth/repo.git/info/refs?service=git-receive-pack HTTP/1.1 200 - + GET /custom_auth/repo.git/info/refs?service=git-receive-pack HTTP/1.1 200 - + GET /custom_auth/repo.git/info/refs?service=git-receive-pack HTTP/1.1 200 - + POST /custom_auth/repo.git/git-receive-pack HTTP/1.1 200 - + EOF + check_access_log expect && + git rev-parse HEAD >expect-oid && + git -C "$HTTPD_DOCUMENT_ROOT_PATH/repo.git" \ + rev-parse refs/heads/cross-host-$extra_header >actual-oid && + test_cmp expect-oid actual-oid +' +done + +setup_redirect_credentials () { + test_when_finished per_test_cleanup && + test_when_finished "unset REDIRECT_HELPER_LOG" && + REDIRECT_HELPER_LOG="$TRASH_DIRECTORY/redirect-helper.calls" && + export REDIRECT_HELPER_LOG && + >"$REDIRECT_HELPER_LOG" && + >"$HTTPD_ROOT_PATH/custom-auth.requests" && + origin_url="$HTTPD_URL/custom_auth/repo.git" && + destination_url="$HTTPD_PROTO://$1/custom_auth/$2" && + if ! test -d "$HTTPD_DOCUMENT_ROOT_PATH/other.git" + then + git clone --bare "$HTTPD_DOCUMENT_ROOT_PATH/repo.git" \ + "$HTTPD_DOCUMENT_ROOT_PATH/other.git" + fi && + test_config -C "$HTTPD_DOCUMENT_ROOT_PATH/$2" http.receivepack true && + test_config_global credential.helper "" && + write_script "$TRASH_DIRECTORY/bin/git-credential-redirect-trace" <<-\EOF && + provider=$1 + operation=$2 + path= + while IFS= read -r line + do + case "$line" in path=*) path=${line#path=} ;; esac + done + if test "$provider" = by-path + then + case "$path" in + custom_auth/other.git) provider=destination ;; + *) provider=origin ;; + esac + fi + echo "$provider $operation" >>"$REDIRECT_HELPER_LOG" + if test "$operation" = get + then + echo "capability[]=authtype" + echo "authtype=Bearer" + echo "credential=$provider-token" + fi + EOF + cat >"$HTTPD_ROOT_PATH/custom-auth.valid" <<-EOF && + id=1 creds=Bearer origin-token + EOF + cat >"$HTTPD_ROOT_PATH/custom-auth.challenge" <<-EOF && + id=1 status=302 response=Location: $destination_url/info/refs?service=git-receive-pack + id=default response=WWW-Authenticate: Bearer realm="origin" + EOF + cat >"$HTTPD_ROOT_PATH/custom-auth.destination.valid" <<-EOF && + id=1 creds=Bearer $3-token + EOF + cat >"$HTTPD_ROOT_PATH/custom-auth.destination.challenge" <<-EOF && + id=1 status=200 + id=default response=WWW-Authenticate: Bearer realm="destination" + EOF + echo "$1/$2 custom-auth.destination" >"$HTTPD_ROOT_PATH/custom-auth.routes" +} + +test_expect_success 'same credential identity retains Bearer auth after redirect' ' + setup_redirect_credentials "$HTTPD_DEST" other.git origin && + test_config_global credential.helper "redirect-trace origin" && + git push "$origin_url" HEAD:refs/heads/same-identity && + cat >expect <<-EOF && + GET $HTTPD_DEST /repo.git/info/refs||| + GET $HTTPD_DEST /repo.git/info/refs|Bearer origin-token|| + GET $HTTPD_DEST /other.git/info/refs|Bearer origin-token|| + POST $HTTPD_DEST /other.git/git-receive-pack|Bearer origin-token|| + EOF + test_cmp expect "$HTTPD_ROOT_PATH/custom-auth.requests" && + grep " get$" "$REDIRECT_HELPER_LOG" >actual && + echo "origin get" >expect && + test_cmp expect actual && + test_grep ! " erase$" "$REDIRECT_HELPER_LOG" +' + +test_expect_success 'destination proactive auth applies to the first redirected GET' ' + setup_redirect_credentials "localhost:$LIB_HTTPD_PORT" repo.git destination && + test_config_global "credential.$origin_url.helper" "redirect-trace origin" && + test_config_global "credential.$destination_url.helper" "redirect-trace destination" && + test_config_global "http.$origin_url.proactiveAuth" none && + test_config_global "http.$destination_url.proactiveAuth" auto && + git push "$origin_url" HEAD:refs/heads/destination-proactive && + cat >expect <<-EOF && + GET $HTTPD_DEST /repo.git/info/refs||| + GET $HTTPD_DEST /repo.git/info/refs|Bearer origin-token|| + GET localhost:$LIB_HTTPD_PORT /repo.git/info/refs|Bearer destination-token|| + POST localhost:$LIB_HTTPD_PORT /repo.git/git-receive-pack|Bearer destination-token|| + EOF + test_cmp expect "$HTTPD_ROOT_PATH/custom-auth.requests" && + test_grep ! " erase$" "$REDIRECT_HELPER_LOG" +' + +for scope in useHttpPath helper +do +test_expect_success "redirect selects path-scoped credentials ($scope)" ' + setup_redirect_credentials "$HTTPD_DEST" other.git destination && + if test "$scope" = useHttpPath + then + test_config_global credential.useHttpPath true && + test_config_global credential.helper "redirect-trace by-path" + else + test_config_global credential.useHttpPath false && + test_config_global "credential.$origin_url.helper" "redirect-trace origin" && + test_config_global "credential.$destination_url.helper" "redirect-trace destination" + fi && + git push "$origin_url" HEAD:refs/heads/path-$scope && + cat >expect <<-EOF && + GET $HTTPD_DEST /repo.git/info/refs||| + GET $HTTPD_DEST /repo.git/info/refs|Bearer origin-token|| + GET $HTTPD_DEST /other.git/info/refs||| + GET $HTTPD_DEST /other.git/info/refs|Bearer destination-token|| + POST $HTTPD_DEST /other.git/git-receive-pack|Bearer destination-token|| + EOF + test_cmp expect "$HTTPD_ROOT_PATH/custom-auth.requests" && + test_grep ! " erase$" "$REDIRECT_HELPER_LOG" +' +done + +test_expect_success 'redirect selects destination headers before its first GET' ' + setup_redirect_credentials "localhost:$LIB_HTTPD_PORT" repo.git destination && + test_config_global "credential.$origin_url.helper" "redirect-trace origin" && + test_config_global "credential.$destination_url.helper" "redirect-trace destination" && + test_config_global "http.$origin_url.extraHeader" "X-Origin-Secret: origin-only" && + test_config_global "http.$destination_url.extraHeader" "X-Destination-Secret: destination-only" && + git config --global --add "http.$origin_url.extraHeader" "Authorization: Bearer origin-token" && + git config --global --add "http.$destination_url.extraHeader" "Authorization: Bearer destination-token" && + git push "$origin_url" HEAD:refs/heads/destination-headers && + cat >expect <<-EOF && + GET $HTTPD_DEST /repo.git/info/refs|Bearer origin-token|origin-only| + GET localhost:$LIB_HTTPD_PORT /repo.git/info/refs|Bearer destination-token||destination-only + POST localhost:$LIB_HTTPD_PORT /repo.git/git-receive-pack|Bearer destination-token||destination-only + EOF + test_cmp expect "$HTTPD_ROOT_PATH/custom-auth.requests" && + test_grep ! " erase$" "$REDIRECT_HELPER_LOG" +' + test_done diff --git a/t/t5616-partial-clone.sh b/t/t5616-partial-clone.sh index 7cc6f58a31fe90..c65a569f434806 100755 --- a/t/t5616-partial-clone.sh +++ b/t/t5616-partial-clone.sh @@ -121,6 +121,34 @@ test_expect_success 'partial fetch does not spawn rev-list connectivity check' ' test_subcommand_flex git rev-list --objects --stdin oid && + pack=$(git pack-objects "$packdir/pack" packs && + git multi-pack-index write --incremental --stdin-packs pack-bytes-counts && + tail -n 1 pack-bytes-counts >pack-bytes-last && + test_grep "\"value\":\"$1\"" pack-bytes-last && + test_grep "\"category\":\"http-fetch\".*\"key\":\"pack/received_bytes_complete\".*\"value\":\"1\"" pack-bytes.trace +} + test_expect_success 'resumed packfile download survives two transient errors' ' setup_pack_retry 503 502 200 && cp pack-prefix "pack-retry-client/.git/objects/pack/pack-$pack_hash.pack.temp" && - fetch_pack && + >pack-bytes.trace && + GIT_TRACE2_EVENT="$TRASH_DIRECTORY/pack-bytes.trace" fetch_pack && + size=$(wc -c <"$pack") && + check_pack_received_bytes "$((size - 12))" && printf "GET|bytes=12-|\nGET|bytes=12-|\nGET|bytes=12-|\n" >expect && test_cmp expect "$HTTPD_ROOT_PATH/pack-retry.requests" && test_cmp "$pack" "pack-retry-client/.git/objects/pack/pack-$pack_hash.pack" @@ -1452,7 +1462,9 @@ do test_config_global http.maxRetries 3 && partial="pack-retry-client/.git/objects/pack/pack-$pack_hash.pack.temp" && cp pack-prefix "$partial" && - fetch_pack test_must_fail && + >pack-bytes.trace && + GIT_TRACE2_EVENT="$TRASH_DIRECTORY/pack-bytes.trace" fetch_pack test_must_fail && + check_pack_received_bytes 0 && test_cmp pack-prefix "$partial" && printf "GET|bytes=12-|\nGET|bytes=12-|\nGET|bytes=12-|\n" >expect && test_cmp expect "$HTTPD_ROOT_PATH/pack-retry.requests" @@ -1470,7 +1482,9 @@ done test_expect_success 'an interrupted response is not retried or truncated' ' setup_pack_retry truncated 200 && - fetch_pack test_must_fail && + >pack-bytes.trace && + GIT_TRACE2_EVENT="$TRASH_DIRECTORY/pack-bytes.trace" fetch_pack test_must_fail && + check_pack_received_bytes 12 && test_line_count = 1 "$HTTPD_ROOT_PATH/pack-retry.requests" && test_cmp pack-prefix "pack-retry-client/.git/objects/pack/pack-$pack_hash.pack.temp" ' @@ -1525,11 +1539,16 @@ test_expect_success 'parallel packfile URI fetch completes after HTTP 503 and 42 test_expect_success 'parallel URI progress respects quiet and no-progress' ' for option in --quiet --no-progress do + >pack-bytes.trace && + GIT_TRACE2_EVENT="$TRASH_DIRECTORY/pack-bytes.trace" \ GIT_TEST_SIDEBAND_ALL=1 \ git -c protocol.version=2 -c fetch.uriprotocols=http \ -c fetch.packfileUriJobs=2 clone "$option" \ "$HTTPD_URL/smart/http_parent" "http_child-$option" \ 2>no-ref-progress && + grep "\"category\":\"http-fetch\".*\"key\":\"pack/received_bytes_complete\".*\"value\":\"1\"" \ + pack-bytes.trace >pack-bytes-complete && + test_line_count = 3 pack-bytes-complete && test_grep ! -E "Fetching packs|Receiving objects|^bytes " no-ref-progress || return 1 done diff --git a/t/t5812-proto-disable-http.sh b/t/t5812-proto-disable-http.sh index 96187efaa82ac6..a062ccf2609f1b 100755 --- a/t/t5812-proto-disable-http.sh +++ b/t/t5812-proto-disable-http.sh @@ -21,7 +21,7 @@ test_expect_success 'http(s) transport respects GIT_ALLOW_PROTOCOL' ' test_must_fail env GIT_ALLOW_PROTOCOL=http:https \ GIT_SMART_HTTP=0 \ git clone "$HTTPD_URL/ftp-redir/repo.git" 2>stderr && - test_grep -E "(ftp.*disabled|your curl version is too old)" stderr + test_grep -E "(ftp.*(disabled|not allowed)|your curl version is too old)" stderr ' test_expect_success 'curl limits redirects' ' diff --git a/t/t7705-repack-incremental-midx.sh b/t/t7705-repack-incremental-midx.sh index 25a8c40e8ee5cc..dd0afdf7706363 100755 --- a/t/t7705-repack-incremental-midx.sh +++ b/t/t7705-repack-incremental-midx.sh @@ -522,4 +522,104 @@ test_expect_success 'repack rejects invalid midxNewLayerThreshold' ' ) ' +test_expect_success 'failed MIDX write does not publish its checksum' ' + git init failed-midx-write && + ( + cd failed-midx-write && + test_commit_bulk 3 && + git multi-pack-index write --incremental && + cp "$midx_chain" before && + test_commit extra && + git repack -d && + layer=$(git multi-pack-index write --incremental --no-write-chain-file) && + rm "$midxdir/multi-pack-index-$layer.midx" && + mkdir "$midxdir/multi-pack-index-$layer.midx" && + ls "$packdir"/pack-*.pack >packs.before && + + test_must_fail git -c repack.midxNewLayerThreshold=100 \ + repack --geometric=2 -d --write-midx=incremental 2>err && + test_grep "unable to execute compaction step" err && + test_cmp before "$midx_chain" && + ls "$packdir"/pack-*.pack >packs.after && + test_cmp packs.before packs.after && + rmdir "$midxdir/multi-pack-index-$layer.midx" && + git multi-pack-index verify + ) +' + +test_lazy_prereq ULIMIT_FSIZE ' + (ulimit -f 1 && trap "" XFSZ) +' + +with_small_file_limit () ( + ulimit -f 1 && + trap "" XFSZ && + "$@" +) + +test_expect_success ULIMIT_FSIZE 'failed chain publication leaves the old chain intact' ' + git init failed-chain-write && + ( + cd failed-chain-write && + for i in $(test_seq 1 26) + do + echo "$i" | git hash-object -w --stdin >oid && + git pack-objects "$packdir/pack" packs.before && + test_expect_code 255 with_small_file_limit \ + git repack --write-midx=incremental >/dev/null 2>err && + test_grep "could not write multi-pack-index chain" err && + test_cmp before "$midx_chain" && + ls "$packdir"/pack-*.pack >packs.after && + test_cmp packs.before packs.after && + test_path_is_missing "$midx_chain.lock" && + git multi-pack-index verify + ) +' + +test_expect_success 'repacking duplicates does not append an empty layer' ' + git init duplicate-packs && + ( + cd duplicate-packs && + test_commit_bulk 10 && + git multi-pack-index write --incremental && + cp "$midx_chain" before && + for rev in HEAD HEAD^{tree} + do + git rev-parse "$rev" >oid && + git pack-objects "$packdir/pack" oid && + git pack-objects "$packdir/pack" /dev/null && + git prune-packed && + + ls .git/objects/pack/*.promisor | sort >promisors.before && + GIT_TRACE2_EVENT="$(pwd)/trace2.txt" \ + git maintenance run --quiet --task=geometric-repack && + ls .git/objects/pack/*.promisor | sort >promisors.after && + + test_subcommand git repack -d -l -q --geometric=2 \ + --write-midx expect && + comm -23 promisors.before promisors.after >actual && + test_cmp expect actual + ) +' + test_geometric_repack_needed () { NEEDED="$1" GEOMETRIC_CONFIG="$2" && @@ -748,6 +790,29 @@ test_expect_success 'geometric repacking with --auto' ' ) ' +test_expect_success 'geometric repacking with --auto handles promisor packs' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + git config set maintenance.auto false && + git remote add promisor garbage && + git config set remote.promisor.promisor true && + + for n in $(test_seq 6) + do + test_commit $n || return 1 + done && + + pack_promisor 1 >/dev/null && + pack_promisor 1..2 >/dev/null && + pack_promisor 2..6 >/dev/null && + git prune-packed && + + test_geometric_repack_needed true auto=9000 + ) +' + test_expect_success 'geometric repacking honors configured split factor' ' test_when_finished "rm -rf repo" && git init repo && diff --git a/t/t9907-git-release-manifest.sh b/t/t9907-git-release-manifest.sh new file mode 100755 index 00000000000000..6a297c71346991 --- /dev/null +++ b/t/t9907-git-release-manifest.sh @@ -0,0 +1,99 @@ +#!/bin/sh + +test_description='native Git release receipts and manifest' +. ./test-lib.sh +if ! command -v jq >/dev/null || ! command -v bash >/dev/null +then + skip_all='jq and bash are required' + test_done +fi +manifest_script=${RELEASE_MANIFEST_SCRIPT:-$TEST_DIRECTORY/../.github/workflows/git-release-manifest.sh} +GITHUB_REPOSITORY=example/distribution GITHUB_RUN_ID=42 +SOURCE_SHA=1111111111111111111111111111111111111111 +RECIPE_SHA=2222222222222222222222222222222222222222 +VERSION=v2.55.0-example CHANNEL=example UPSTREAM_TAG=v2.55.0 VISIBILITY=public +export GITHUB_REPOSITORY GITHUB_RUN_ID SOURCE_SHA RECIPE_SHA VERSION CHANNEL UPSTREAM_TAG VISIBILITY +run_manifest () { "${RELEASE_BASH:-bash}" "$manifest_script" "$@"; } +hash () { + if command -v sha256sum >/dev/null; then sha256sum "$1"; else shasum -a 256 "$1"; fi | awk '{print $1}' +} +restore () { rm -rf artifacts && cp -R original artifacts; } +reject () { if run_manifest collect artifacts >actual; then return 1; fi; } + +test_expect_success 'record all six native builds' ' + mkdir original && + for target in macOS-arm64 macOS-x64 ubuntu-arm64 ubuntu-x64 windows-arm64 windows-x64 + do + for extension in tar.gz lzma + do + file=original/git-$VERSION-$target.$extension && + printf "%s\n" "$file" >"$file" && hash "$file" >"$file.sha256" || return 1 + done && + run_manifest record original "$target" || return 1 + done +' + +test_expect_success 'manifest identifies the source, recipe and all 24 assets' ' + restore && run_manifest collect artifacts >actual && + jq -e --arg source "$SOURCE_SHA" --arg recipe "$RECIPE_SHA" ".source_sha==\$source and .recipe.sha==\$recipe and (.targets|length)==6 and ([.targets[][]]|length)==24 and all(.builds[]; .run_id==\"42\")" actual +' + +test_expect_success 'Windows receipt line endings preserve the recorded provenance' ' + restore && + for receipt in artifacts/*-windows-*.build.json + do + append_cr <"$receipt" >crlf && mv crlf "$receipt" || return 1 + done && + run_manifest collect artifacts >actual && + jq -e "(.targets|length)==6 and all(.builds[]; .run_id==\"42\")" actual +' + +test_expect_success 'incomplete targets and mismatching sidecars are rejected' ' + restore && rm artifacts/git-$VERSION-macOS-arm64.lzma && reject && + restore && echo changed >artifacts/git-$VERSION-macOS-arm64.lzma && reject +' + +test_expect_success 'matching sidecars cannot disguise a changed native artifact' ' + restore && file=artifacts/git-$VERSION-macOS-arm64.lzma && + echo changed >"$file" && hash "$file" >"$file.sha256" && reject +' + +test_expect_success 'receipts must match the requested source, recipe and original run' ' + for edit in ".source_sha=\"wrong\"" ".recipe.sha=\"wrong\"" ".run_id=\"wrong\"" + do + restore && receipt=artifacts/git-$VERSION-macOS-arm64.build.json && + jq "$edit" "$receipt" >changed && mv changed "$receipt" && reject || return 1 + done +' + +test_expect_success SYMLINKS 'symlinks are rejected' ' + restore && file=artifacts/git-$VERSION-macOS-arm64.tar.gz && + mv "$file" archive && ln -s "$TRASH_DIRECTORY/archive" "$file" && reject +' + +test_expect_success 'unexpected files are rejected' ' + restore && touch artifacts/.unexpected && reject +' + +test_expect_success 'recording rejects empty archives and unknown targets' ' + restore && file=artifacts/git-$VERSION-macOS-arm64.tar.gz && + : >"$file" && hash "$file" >"$file.sha256" && + if run_manifest record artifacts macOS-arm64; then return 1; fi && + if run_manifest record artifacts unknown; then return 1; fi +' + +test_expect_success 'recording enforces the existing native package size limits' ' + restore && file=artifacts/git-$VERSION-macOS-arm64.tar.gz && + dd if=/dev/zero of="$file" bs=1048576 count=1 seek=64 2>/dev/null && + hash "$file" >"$file.sha256" && + if run_manifest record artifacts macOS-arm64; then return 1; fi && + mv "$file" artifacts/git-$VERSION-windows-arm64.tar.gz && + mv "$file.sha256" artifacts/git-$VERSION-windows-arm64.tar.gz.sha256 && + run_manifest record artifacts windows-arm64 && + file=artifacts/git-$VERSION-windows-arm64.tar.gz && + dd if=/dev/zero of="$file" bs=1048576 count=1 seek=128 2>/dev/null && + hash "$file" >"$file.sha256" && + if run_manifest record artifacts windows-arm64; then return 1; fi +' + +test_done diff --git a/trace2.c b/trace2.c index c23c0a227b7032..07337768ecb5c6 100644 --- a/trace2.c +++ b/trace2.c @@ -17,6 +17,7 @@ #include "trace2/tr2_tgt.h" #include "trace2/tr2_tls.h" #include "trace2/tr2_tmr.h" +#include "url.h" static int trace2_enabled; static int trace2_redact = 1; @@ -249,32 +250,134 @@ int trace2_is_enabled(void) return trace2_enabled; } +static int is_signature_parameter(const char *parameter, int is_format) +{ + struct strbuf unescaped = STRBUF_INIT; + char *name, *p; + int ret; + + /* A literal percent sign is doubled in a printf format string. */ + if (is_format) { + while (*parameter && *parameter != '=') { + if (parameter[0] == '%' && parameter[1] == '%') + parameter++; + strbuf_addch(&unescaped, *parameter++); + } + parameter = unescaped.buf; + } + + name = url_decode_parameter_name(¶meter); + for (p = name; *p; p++) + *p = tolower(*p); + ret = !strcmp(name, "sig") || !strcmp(name, "signature") || + ends_with(name, "-signature"); + + free(name); + strbuf_release(&unescaped); + return ret; +} + /* - * Redacts an argument, i.e. ensures that no password in - * https://user:password@host/-style URLs is logged. + * Redact passwords and signature query parameters in HTTP(S) URLs. * * Returns the original if nothing needed to be redacted. * Returns a pointer that needs to be `free()`d otherwise. */ -static const char *redact_arg(const char *arg) +static const char *redact_url(const char *arg, int is_format) { const char *p, *colon; + const char *unredacted = arg; + struct strbuf buf = STRBUF_INIT; size_t at; if (!trace2_redact || - (!skip_prefix(arg, "https://", &p) && - !skip_prefix(arg, "http://", &p))) + (!skip_iprefix(arg, "https://", &p) && + !skip_iprefix(arg, "http://", &p))) return arg; - at = strcspn(p, "@/"); - if (p[at] != '@') - return arg; + /* Only an '@' in the authority can terminate user information. */ + at = strcspn(p, "@/?#"); + if (p[at] == '@') { + colon = memchr(p, ':', at); + if (colon) { + strbuf_add(&buf, arg, colon + 1 - arg); + strbuf_addstr(&buf, ""); + unredacted = p + at; + } + } + + /* A '?' following a fragment marker does not start a query. */ + p += strcspn(p, "?#"); + if (*p == '?') { + p++; + while (*p && *p != '#') { + const char *end = p + strcspn(p, "&#"); + const char *equals = memchr(p, '=', end - p); + + if (equals && equals + 1 < end && + is_signature_parameter(p, is_format)) { + strbuf_add(&buf, unredacted, + equals + 1 - unredacted); + strbuf_addstr(&buf, ""); + unredacted = end; + } + p = *end == '&' ? end + 1 : end; + } + } - colon = memchr(p, ':', at); - if (!colon) + if (!buf.len) return arg; + strbuf_addstr(&buf, unredacted); + return strbuf_detach(&buf, NULL); +} - return xstrfmt("%.*s:%s", (int)(colon - arg), arg, p + at); +static const char *redact_arg(const char *arg) +{ + return redact_url(arg, 0); +} + +void tr2_redact_error(struct strbuf *buf, int is_format) +{ + size_t pos = 0; + + if (!trace2_redact) + return; + + while (pos < buf->len) { + const char *p = strcasestr(buf->buf + pos, "http"); + const char *end, *scheme_end; + const char *redacted; + char *url; + size_t start; + + if (!p) + break; + start = p - buf->buf; + pos = start + 4; + if (!skip_iprefix(p, "http://", &scheme_end) && + !skip_iprefix(p, "https://", &scheme_end)) + continue; + + /* + * Whitespace, double quotes and angle brackets delimit URLs. + * Other punctuation can belong to a signature, so preserve a + * single quote only at the end of a single-quoted token. + */ + end = p + strcspn(p, " \t\r\n\v\f\"<>"); + if (start && p[-1] == '\'' && end[-1] == '\'') + end--; + + /* Also examine any further URLs in the unredacted text. */ + pos = scheme_end - buf->buf; + url = xmemdupz(p, end - p); + redacted = redact_url(url, is_format); + if (redacted != url) { + strbuf_splice(buf, start, end - p, + redacted, strlen(redacted)); + free((char *)redacted); + } + free(url); + } } /* @@ -457,14 +560,19 @@ void trace2_cmd_alias_fl(const char *file, int line, const char *alias, const char **argv) { struct tr2_tgt *tgt_j; + const char **redacted; int j; if (!trace2_enabled) return; + redacted = redact_argv(argv); + for_each_wanted_builtin (j, tgt_j) if (tgt_j->pfn_alias_fl) - tgt_j->pfn_alias_fl(file, line, alias, argv); + tgt_j->pfn_alias_fl(file, line, alias, redacted); + + free_redacted_argv(redacted, argv); } void trace2_cmd_list_config_fl(const char *file, int line) diff --git a/trace2/tr2_tgt.h b/trace2/tr2_tgt.h index 1f626cffea0fc0..bff9a847f7f3c3 100644 --- a/trace2/tr2_tgt.h +++ b/trace2/tr2_tgt.h @@ -3,6 +3,7 @@ struct child_process; struct repository; +struct strbuf; struct json_writer; struct tr2_timer_metadata; struct tr2_timer; @@ -157,4 +158,10 @@ extern struct tr2_tgt tr2_tgt_event; extern struct tr2_tgt tr2_tgt_normal; extern struct tr2_tgt tr2_tgt_perf; +/* + * Redact URLs in an error payload before writing it. Set is_format when + * redacting a printf format string, where literal '%' characters are doubled. + */ +void tr2_redact_error(struct strbuf *buf, int is_format); + #endif /* TR2_TGT_H */ diff --git a/trace2/tr2_tgt_event.c b/trace2/tr2_tgt_event.c index 5a0381791f7eb4..b401045cfd5c45 100644 --- a/trace2/tr2_tgt_event.c +++ b/trace2/tr2_tgt_event.c @@ -238,15 +238,31 @@ static void fn_error_va_fl(const char *file, int line, const char *fmt, jw_object_begin(&jw, 0); event_fmt_prepare(event_name, file, line, NULL, &jw); - maybe_add_string_va(&jw, "msg", fmt, ap); + if (fmt && *fmt) { + struct strbuf message = STRBUF_INIT; + va_list copy_ap; + + va_copy(copy_ap, ap); + strbuf_vaddf(&message, fmt, copy_ap); + va_end(copy_ap); + tr2_redact_error(&message, 0); + jw_object_string(&jw, "msg", message.buf); + strbuf_release(&message); + } /* * Also emit the format string as a field in case * post-processors want to aggregate common error * messages by type without argument fields (such * as pathnames or branch names) cluttering it up. */ - if (fmt && *fmt) - jw_object_string(&jw, "fmt", fmt); + if (fmt && *fmt) { + struct strbuf format = STRBUF_INIT; + + strbuf_addstr(&format, fmt); + tr2_redact_error(&format, 1); + jw_object_string(&jw, "fmt", format.buf); + strbuf_release(&format); + } jw_end(&jw); tr2_dst_write_line(&tr2dst_event, &jw.json); diff --git a/trace2/tr2_tgt_normal.c b/trace2/tr2_tgt_normal.c index 924736ab36093b..f63b2ee93a4267 100644 --- a/trace2/tr2_tgt_normal.c +++ b/trace2/tr2_tgt_normal.c @@ -152,6 +152,7 @@ static void fn_error_va_fl(const char *file, int line, const char *fmt, strbuf_addch(&buf_payload, ' '); maybe_append_string_va(&buf_payload, fmt, ap); } + tr2_redact_error(&buf_payload, 0); normal_io_write_fl(file, line, &buf_payload); strbuf_release(&buf_payload); } diff --git a/trace2/tr2_tgt_perf.c b/trace2/tr2_tgt_perf.c index 4eb9289f950505..60b3af148a880d 100644 --- a/trace2/tr2_tgt_perf.c +++ b/trace2/tr2_tgt_perf.c @@ -241,6 +241,7 @@ static void fn_error_va_fl(const char *file, int line, const char *fmt, struct strbuf buf_payload = STRBUF_INIT; maybe_append_string_va(&buf_payload, fmt, ap); + tr2_redact_error(&buf_payload, 0); perf_io_write_fl(file, line, event_name, NULL, NULL, NULL, NULL, &buf_payload);