diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e322ae2..542f473 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -49,7 +49,9 @@ jobs: - uses: actions/setup-dotnet@v4 with: - dotnet-version: '8.0.x' + dotnet-version: | + 8.0.x + 10.0.x - name: Build and pack run: sh ./build/build_runtime_macos_packages.sh @@ -80,7 +82,9 @@ jobs: - uses: actions/setup-dotnet@v4 with: - dotnet-version: '8.0.x' + dotnet-version: | + 8.0.x + 10.0.x - name: Collect runtime packages into a local feed uses: actions/download-artifact@v4 @@ -115,7 +119,9 @@ jobs: - uses: actions/setup-dotnet@v4 with: - dotnet-version: '8.0.x' + dotnet-version: | + 8.0.x + 10.0.x - name: Collect runtime packages into a local feed uses: actions/download-artifact@v4 @@ -152,7 +158,9 @@ jobs: steps: - uses: actions/setup-dotnet@v4 with: - dotnet-version: '8.0.x' + dotnet-version: | + 8.0.x + 10.0.x - uses: actions/download-artifact@v4 with: diff --git a/.gitignore b/.gitignore index 86c063c..cedb7b8 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,6 @@ runtime.*.liblouis/runtimes/ # Rider / ReSharper per-user settings *.DotSettings.user + +# Worktrees created by spawned Claude Code sessions +.claude/worktrees/ diff --git a/Dockerfile b/Dockerfile index 916d491..828c6fb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,15 +20,28 @@ FROM --platform=linux/amd64 mcr.microsoft.com/dotnet/sdk:8.0-jammy AS base LABEL org.opencontainers.image.source=https://github.com/Notalib/LibLouis.NET/ -RUN apt-get update && \ - apt-get upgrade -y && \ - apt-get install -y --no-install-recommends \ - build-essential \ - ca-certificates \ - curl \ - m4 \ - xz-utils \ - && rm -rf /var/lib/apt/lists/* +# Retried, because a single apt-get run is a coin flip against archive.ubuntu.com: the index and +# the pool are not updated atomically, so a package version can be listed after it has been removed +# and the fetch 404s. That is what it did. Each attempt refreshes the index first, since a newer +# index is usually what resolves it. The explicit ok check matters: without it a loop that never +# succeeds still falls through and the layer builds with nothing installed. +RUN set -eu; \ + ok=0; \ + for attempt in 1 2 3; do \ + if apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + curl \ + m4 \ + xz-utils \ + llvm; then \ + ok=1; break; \ + fi; \ + echo "apt attempt $attempt failed, retrying" >&2; \ + sleep 10; \ + done; \ + [ "$ok" = 1 ] || exit 1; \ + rm -rf /var/lib/apt/lists/* ENV PACKAGE_OUTPUT_DIR=/packages WORKDIR /source @@ -37,18 +50,26 @@ WORKDIR /source # The five targets Ubuntu has cross compilers for. FROM base AS gcc-targets -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - gcc-i686-linux-gnu \ - gcc-aarch64-linux-gnu \ - gcc-mingw-w64-i686 \ - gcc-mingw-w64-x86-64 \ - # The cross gcc packages only Recommend their target libc, so with - # --no-install-recommends they install a compiler that cannot link. Name them explicitly - # rather than dropping the flag, so the requirement is visible. - libc6-dev-i386-cross \ - libc6-dev-arm64-cross \ - && rm -rf /var/lib/apt/lists/* +# The cross gcc packages only Recommend their target libc, so with --no-install-recommends they +# install a compiler that cannot link. Named explicitly rather than dropping the flag, so the +# requirement is visible. Retried for the same reason as the base stage. +RUN set -eu; \ + ok=0; \ + for attempt in 1 2 3; do \ + if apt-get update && apt-get install -y --no-install-recommends \ + gcc-i686-linux-gnu \ + gcc-aarch64-linux-gnu \ + gcc-mingw-w64-i686 \ + gcc-mingw-w64-x86-64 \ + libc6-dev-i386-cross \ + libc6-dev-arm64-cross; then \ + ok=1; break; \ + fi; \ + echo "apt attempt $attempt failed, retrying" >&2; \ + sleep 10; \ + done; \ + [ "$ok" = 1 ] || exit 1; \ + rm -rf /var/lib/apt/lists/* COPY . /source RUN sh ./build/build_runtime_packages.sh gcc diff --git a/LibLouis.NET.Test/LibLouis.NET.Test.csproj b/LibLouis.NET.Test/LibLouis.NET.Test.csproj index 8dee374..bf607e2 100644 --- a/LibLouis.NET.Test/LibLouis.NET.Test.csproj +++ b/LibLouis.NET.Test/LibLouis.NET.Test.csproj @@ -1,7 +1,7 @@  - net8.0 + net8.0;net10.0 enable false true @@ -12,8 +12,9 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - - + + + all diff --git a/LibLouis.NET/LibLouis.NET.csproj b/LibLouis.NET/LibLouis.NET.csproj index 96e70b3..c75c311 100644 --- a/LibLouis.NET/LibLouis.NET.csproj +++ b/LibLouis.NET/LibLouis.NET.csproj @@ -1,7 +1,7 @@  - net8.0 + net8.0;net10.0 enable True Nota @@ -20,8 +20,12 @@ - - + + + + + + + diff --git a/PACKAGING.md b/PACKAGING.md index a1ae783..6089b8d 100644 --- a/PACKAGING.md +++ b/PACKAGING.md @@ -134,6 +134,41 @@ Worth re-checking after a toolchain or upstream bump. The binaries should import llvm-readobj --coff-imports liblouis.dll | grep Name: ``` +### Post-build verification + +Every native binary is inspected before it is packed, by `build/verify_native_binary.sh`, called +from `pack_runtime_package`. A binary that fails never becomes a package. + +| Check | Why | +| --- | --- | +| Architecture matches the RID | A mis-targeted binary restores fine and never loads. | +| Every P/Invoke symbol is exported | Otherwise `EntryPointNotFoundException` at first use. | +| No dependency outside an allowlist | Catches the `libgcc_s_dw2-1.dll` class of bug. | +| Max `GLIBC_` version within the floor | The floor decides which distributions can consume the packages, and it is a property of the build image. | + +The expected symbols are read out of the `EntryPoint` attributes in `LibLouis.NET/NativeMethod.cs` +rather than listed in the script, so the check cannot drift away from what the wrapper actually +imports. + +The glibc floor is `MAX_GLIBC` in the script, currently 2.34, which covers RHEL 9, Debian 12 and +Ubuntu 22.04 and later. `linux-x86` sits exactly on it. Raising it drops support for older +distributions, so it should be a deliberate decision rather than a side effect of bumping the base +image. + +Run it by hand against an extracted package to audit a published one: + +```bash +sh build/verify_native_binary.sh win-x64 runtimes/win-x64/native/liblouis.dll +``` + +It needs tools that can read the format being checked. The container has them. On macOS, +`brew install llvm` covers all three formats, since llvm-readelf, llvm-nm and llvm-readobj read ELF, +PE and Mach-O alike; the script finds them under Homebrew's keg-only prefix. Without that, only the +macOS RIDs can be checked locally, because BSD nm has no -D and cannot read ELF. + +`SKIP_NATIVE_VERIFICATION=1` bypasses it, which is only reasonable when deliberately building +something the checks were not written for. + ### Parallel make The Linux and macOS targets build with `make -j`. The Windows targets deliberately do not. diff --git a/build/common.sh b/build/common.sh index 7d397b2..ed63a49 100755 --- a/build/common.sh +++ b/build/common.sh @@ -88,8 +88,18 @@ stage_tables() { # Pack a per-RID runtime package. The native binary must already be staged into # runtime..liblouis/runtimes//native/, which RuntimePackage.props verifies. +# +# The binary is inspected before packing rather than after, so a bad build never becomes a package +# at all. Set SKIP_NATIVE_VERIFICATION=1 to bypass, which is only reasonable when deliberately +# building something the checks are not written for. pack_runtime_package() { rid=$1 + + if [ -z "${SKIP_NATIVE_VERIFICATION:-}" ]; then + binary=$(find "$REPO_ROOT/runtime.$rid.liblouis/runtimes/$rid/native" -type f | head -n 1) + sh "$REPO_ROOT/build/verify_native_binary.sh" "$rid" "$binary" + fi + mkdir -p "$PACKAGE_OUTPUT_DIR" dotnet pack "$REPO_ROOT/runtime.$rid.liblouis/runtime.$rid.liblouis.csproj" \ --configuration Release \ diff --git a/build/verify_native_binary.sh b/build/verify_native_binary.sh new file mode 100755 index 0000000..e2b4592 --- /dev/null +++ b/build/verify_native_binary.sh @@ -0,0 +1,221 @@ +#!/bin/sh +# Verifies one built native liblouis binary before it is packed. +# +# verify_native_binary.sh +# +# common.sh calls this from pack_runtime_package, so every runtime package is checked on the +# machine that built it. Run it by hand against an extracted package to audit a published one. +# +# The checks exist because each of them corresponds to something that actually went wrong, or that +# would only surface on an end user's machine: +# +# architecture a mis-targeted binary produces a package that restores fine and never loads. +# exports the expected symbols come from the EntryPoint attributes in NativeMethod.cs +# rather than a list kept here, so the check cannot drift away from the wrapper. +# dependencies win-x86 once imported libgcc_s_dw2-1.dll, a mingw runtime DLL not shipped in the +# package. It works on a developer machine that has mingw and fails everywhere +# else. +# glibc the floor decides which distributions can consume the Linux packages. It is a +# property of the build image, so it can rise silently when that image is bumped. +# +# Exits non-zero listing every failure, rather than stopping at the first. + +set -eu + +if [ $# -ne 2 ]; then + echo "usage: $0 " >&2 + exit 2 +fi + +rid=$1 +binary=$2 + +REPO_ROOT=$(cd "$(dirname "$0")/.." && pwd) +NATIVE_METHODS="$REPO_ROOT/LibLouis.NET/NativeMethod.cs" + +# Highest glibc symbol version the Linux binaries may require. Raising this drops support for +# distributions older than the new value, so it is a deliberate decision, not a build detail. +# 2.34 covers RHEL 9 (2.34), Debian 12 (2.36) and Ubuntu 22.04 (2.35) and later. +MAX_GLIBC=2.34 + +failures=0 +fail() { + echo " FAIL $*" >&2 + failures=$((failures + 1)) +} +ok() { + echo " ok $*" +} + +if [ ! -f "$binary" ]; then + echo "verify_native_binary: '$binary' does not exist" >&2 + exit 1 +fi + +# Finds the first of several interchangeable tools. Takes them in preference order, because the +# right one depends on the platform: reading ELF on macOS needs llvm-nm, since BSD nm has no -D. +# +# llvm ships its tools with a version suffix on Debian and Ubuntu, unsuffixed in llvm-mingw, and +# under a keg-only prefix from Homebrew. +find_tool() { + for name in "$@"; do + if command -v "$name" >/dev/null 2>&1; then + command -v "$name" + return 0 + fi + done + + for name in "$@"; do + for candidate in $(ls \ + /usr/bin/"$name"-* \ + /usr/lib/llvm-*/bin/"$name" \ + /opt/homebrew/opt/llvm/bin/"$name" \ + /opt/homebrew/opt/binutils/bin/"$name" \ + /usr/local/opt/llvm/bin/"$name" \ + /usr/local/opt/binutils/bin/"$name" 2>/dev/null | sort -Vr); do + if [ -x "$candidate" ]; then + echo "$candidate" + return 0 + fi + done + done + + return 1 +} + +# Every EntryPoint the managed wrapper P/Invokes. If the wrapper gains a function and the native +# library does not export it, that is a runtime EntryPointNotFoundException, so catch it here. +# Resolved once, here in the main shell. Doing this inside a function called through command +# substitution would run it in a subshell, where exit cannot stop the script: the error would print +# and every symbol check would still pass against an empty list. +if [ ! -f "$NATIVE_METHODS" ]; then + echo "verify_native_binary: cannot read $NATIVE_METHODS" >&2 + exit 1 +fi + +EXPECTED_SYMBOLS=$(grep -o 'EntryPoint = "[^"]*"' "$NATIVE_METHODS" | sed 's/.*"\(.*\)"/\1/' | sort -u) + +if [ -z "$EXPECTED_SYMBOLS" ]; then + echo "verify_native_binary: found no EntryPoint attributes in $NATIVE_METHODS" >&2 + exit 1 +fi + +EXPECTED_SYMBOL_COUNT=$(printf '%s\n' "$EXPECTED_SYMBOLS" | wc -l | tr -d ' ') + +echo "verifying $rid: $binary" + +case "$rid" in + linux-*) + readelf=$(find_tool llvm-readelf readelf) || { echo "readelf not found" >&2; exit 1; } + + case "$rid" in + linux-x86) want_class=ELF32; want_machine="Intel 80386" ;; + linux-x64) want_class=ELF64; want_machine="X86-64" ;; + linux-arm64) want_class=ELF64; want_machine="AArch64" ;; + *) echo "unknown rid $rid" >&2; exit 2 ;; + esac + + header=$("$readelf" -h "$binary") + if echo "$header" | grep -q "$want_class" && echo "$header" | grep -q "$want_machine"; then + ok "architecture $want_class $want_machine" + else + fail "architecture: expected $want_class $want_machine, got $(echo "$header" | grep Machine:)" + fi + + missing="" + nm=$(find_tool llvm-nm nm) || { echo "nm not found" >&2; exit 1; } + exports=$("$nm" -D --defined-only "$binary" | awk '$2 == "T" { print $3 }') + for symbol in $EXPECTED_SYMBOLS; do + echo "$exports" | grep -qx "$symbol" || missing="$missing $symbol" + done + [ -z "$missing" ] && ok "all $EXPECTED_SYMBOL_COUNT P/Invoke symbols exported" \ + || fail "not exported:$missing" + + # ld-linux is the loader, not a library that has to be shipped. + unexpected=$("$readelf" -d "$binary" | sed -n 's/.*NEEDED.*\[\(.*\)\].*/\1/p' \ + | grep -vE '^(libc\.so\.6|libm\.so\.6|libdl\.so\.2|libpthread\.so\.0|ld-linux.*)$' || true) + [ -z "$unexpected" ] && ok "no dependencies outside the allowlist" \ + || fail "unexpected dependencies: $(echo "$unexpected" | tr '\n' ' ')" + + required=$("$readelf" -V "$binary" 2>/dev/null | grep -o 'GLIBC_[0-9.]*' | sed 's/GLIBC_//' \ + | sort -uV | tail -1) + if [ -z "$required" ]; then + ok "no versioned glibc references" + elif [ "$(printf '%s\n%s\n' "$required" "$MAX_GLIBC" | sort -V | tail -1)" = "$MAX_GLIBC" ]; then + ok "requires at most GLIBC_$required (floor is $MAX_GLIBC)" + else + fail "requires GLIBC_$required, above the declared floor of $MAX_GLIBC. Either the build image changed or new code pulled in a newer symbol; raising MAX_GLIBC drops support for older distributions." + fi + ;; + + win-*) + # binutils cannot read aarch64 PE, so all three Windows targets go through llvm. + readobj=$(find_tool llvm-readobj) || { echo "llvm-readobj not found" >&2; exit 1; } + + case "$rid" in + win-x86) want_machine=IMAGE_FILE_MACHINE_I386 ;; + win-x64) want_machine=IMAGE_FILE_MACHINE_AMD64 ;; + win-arm64) want_machine=IMAGE_FILE_MACHINE_ARM64 ;; + *) echo "unknown rid $rid" >&2; exit 2 ;; + esac + + if "$readobj" --file-headers "$binary" 2>/dev/null | grep -q "$want_machine"; then + ok "architecture $want_machine" + else + fail "architecture: expected $want_machine" + fi + + missing="" + exports=$("$readobj" --coff-exports "$binary" 2>/dev/null | sed -n 's/.*Name: \(.*\)/\1/p') + for symbol in $EXPECTED_SYMBOLS; do + echo "$exports" | grep -qx "$symbol" || missing="$missing $symbol" + done + [ -z "$missing" ] && ok "all $EXPECTED_SYMBOL_COUNT P/Invoke symbols exported" \ + || fail "not exported:$missing" + + # Anything outside this list has to ship with the package, and nothing else does. + unexpected=$("$readobj" --coff-imports "$binary" 2>/dev/null | sed -n 's/.*Name: \(.*\)/\1/p' \ + | sort -u | grep -viE '^(KERNEL32\.dll|msvcrt\.dll|USER32\.dll|ADVAPI32\.dll|api-ms-win-.*\.dll)$' || true) + [ -z "$unexpected" ] && ok "no dependencies outside the allowlist" \ + || fail "unexpected dependencies: $(echo "$unexpected" | tr '\n' ' ')" + ;; + + osx-*) + case "$rid" in + osx-x64) want_arch=x86_64 ;; + osx-arm64) want_arch=arm64 ;; + *) echo "unknown rid $rid" >&2; exit 2 ;; + esac + + if file -b "$binary" | grep -q "$want_arch"; then + ok "architecture $want_arch" + else + fail "architecture: expected $want_arch, got $(file -b "$binary")" + fi + + missing="" + # Mach-O prefixes symbols with an underscore. + exports=$(nm -gU "$binary" | awk '$2 == "T" { print $3 }' | sed 's/^_//') + for symbol in $EXPECTED_SYMBOLS; do + echo "$exports" | grep -qx "$symbol" || missing="$missing $symbol" + done + [ -z "$missing" ] && ok "all $EXPECTED_SYMBOL_COUNT P/Invoke symbols exported" \ + || fail "not exported:$missing" + + # The first otool -L line is the library's own install name, not a dependency. + unexpected=$(otool -L "$binary" | tail -n +3 | sed 's/^[[:space:]]*//; s/ (.*//' \ + | grep -vE '^(/usr/lib/libSystem\.B\.dylib|/usr/lib/libc\+\+\.1\.dylib)$' || true) + [ -z "$unexpected" ] && ok "no dependencies outside the allowlist" \ + || fail "unexpected dependencies: $(echo "$unexpected" | tr '\n' ' ')" + ;; + + *) + echo "verify_native_binary: unknown runtime identifier '$rid'" >&2 + exit 2 + ;; +esac + +if [ "$failures" -ne 0 ]; then + echo "verify_native_binary: $rid failed $failures check(s)" >&2 + exit 1 +fi