From e828f03b2a079600d1a44ef9ce129a909d9d64f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcos=20M=C3=A9ndez?= Date: Tue, 22 Sep 2026 18:59:03 +0000 Subject: [PATCH 1/4] Honour SOURCE_DATE_EPOCH when set With SOURCE_DATE_EPOCH exported, two builds of the same product currently still differ: the packers read the clock. Three changes, all no-ops when the variable is unset, so existing builds are untouched: - add SOURCE_DATE_EPOCH to CONF_VARS_BUILTIN, so it is exported to fab-chroot and fab-install (dpkg-deb, mkinitramfs and friends read it) - pass --set_all_file_dates to xorriso: it honours the variable for its own metadata, but not for the mtimes of the files it stores - pass --id to isohybrid, which otherwise writes a random MBR id mksquashfs needs nothing: it reads the variable itself, and fails with "SOURCE_DATE_EPOCH and command line options can't be used at the same time to set timestamp(s)" if given equivalent options as well. Measured on TKLDev 19.0 (Debian 13, mksquashfs 4.6.1, xorriso 1.5.6), building core from the same root.patched, rm -rf build/cdroot between runs: SOURCE_DATE_EPOCH=1700000000 squashfs 8c339c02.. iso 6fe6fdca.. SOURCE_DATE_EPOCH=1700000000 squashfs 8c339c02.. iso 6fe6fdca.. SOURCE_DATE_EPOCH=1800000000 squashfs b888b670.. iso 153097f3.. Without it, consecutive builds differ, as before. --- share/product.mk | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) mode change 100755 => 100644 share/product.mk diff --git a/share/product.mk b/share/product.mk old mode 100755 new mode 100644 index 700c54b..c89c5d3 --- a/share/product.mk +++ b/share/product.mk @@ -23,7 +23,7 @@ endif COMMON_PATCHES := turnkey.d $(COMMON_PATCHES) -CONF_VARS_BUILTIN ?= FAB_ARCH HOST_ARCH FAB_HTTP_PROXY AMD64 ARM64 RELEASE DISTRO CODENAME DEBIAN UBUNTU KERNEL DEBUG CHROOT_ONLY DI_LIVE_DEBUG +CONF_VARS_BUILTIN ?= FAB_ARCH HOST_ARCH FAB_HTTP_PROXY AMD64 ARM64 RELEASE DISTRO CODENAME DEBIAN UBUNTU KERNEL DEBUG CHROOT_ONLY DI_LIVE_DEBUG SOURCE_DATE_EPOCH define filter-undefined-vars $(foreach var,$1,$(if $($(var)), $(var))) @@ -53,6 +53,18 @@ CDROOT ?= generic MKSQUASHFS ?= /usr/bin/mksquashfs MKSQUASHFS_OPTS ?= -no-sparse +# https://reproducible-builds.org/docs/source-date-epoch/ - when set, pin +# every timestamp the image would otherwise take from the clock. mksquashfs +# and xorriso read the variable themselves (mksquashfs even refuses to take +# both it and equivalent options), but xorriso only fixes its own metadata, +# not the mtimes of the files it stores, and isohybrid writes a random MBR +# id. Exporting the variable (CONF_VARS_BUILTIN above) also reaches dpkg-deb +# and mkinitramfs inside the chroot. Unset, nothing here applies. +ifneq ($(SOURCE_DATE_EPOCH),) +XORRISO_DATE_OPTS := --set_all_file_dates @$(SOURCE_DATE_EPOCH) +ISOHYBRID_OPTS := --id $(shell echo $$(( $(SOURCE_DATE_EPOCH) & 0xffffffff ))) +endif + # if the CDROOT is a relative path, prefix CDROOTS_PATH # we set _CDROOT with eval to improve the readability of $(value _CDROOT) # in help target @@ -454,7 +466,7 @@ endef define run-genisoimage xorriso -as mkisofs \ -o $O/product.iso -r -J \ - -V ${ISOLABEL} \ + -V ${ISOLABEL} $(XORRISO_DATE_OPTS) \ -b isolinux/isolinux.bin \ -c isolinux/boot.cat \ -no-emul-boot \ @@ -466,7 +478,7 @@ endef define run-genisoimage-uefi xorriso -as mkisofs \ -o $O/product.iso -r -J \ - -V ${ISOLABEL} \ + -V ${ISOLABEL} $(XORRISO_DATE_OPTS) \ -b isolinux/isolinux.bin \ -c isolinux/boot.cat \ -isohybrid-mbr /usr/lib/ISOLINUX/isohdpfx.bin \ @@ -481,7 +493,7 @@ define run-genisoimage-uefi endef define run-isohybrid - isohybrid $O/product.iso + isohybrid $(ISOHYBRID_OPTS) $O/product.iso endef # target: product.iso From af79ca7db3d4bcef2bf3398332fd278b7e528add Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcos=20M=C3=A9ndez?= Date: Fri, 25 Sep 2026 17:59:59 +0000 Subject: [PATCH 2/4] Add a make-level test for SOURCE_DATE_EPOCH tests/source-date-epoch.sh dry-runs (make -n) product.iso for a minimal fake product against share/product.mk and checks, with the variable unset and set: --set_all_file_dates on both xorriso recipes, the isohybrid --id (decimal, epoch & 0xffffffff, wrapping at 2^32), SOURCE_DATE_EPOCH in FAB_CHROOT_ENV and in fab-plan-resolve -D, and mksquashfs options unchanged. TAP output, 19 checks; 7 fail against the previous product.mk. --- tests/source-date-epoch.sh | 143 +++++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100755 tests/source-date-epoch.sh diff --git a/tests/source-date-epoch.sh b/tests/source-date-epoch.sh new file mode 100755 index 0000000..d56a1b4 --- /dev/null +++ b/tests/source-date-epoch.sh @@ -0,0 +1,143 @@ +#!/bin/bash +# make-level test for SOURCE_DATE_EPOCH in share/product.mk. +# +# Builds a minimal fake product under a scratch FAB_PATH and dry-runs +# (make -n) the product.iso recipe against the product.mk of this checkout, +# once with SOURCE_DATE_EPOCH unset and once with it set. Output is TAP; the +# exit status is the number of failed checks. +# +# product.mk includes /tmp/.build_env, written by share/load_env; the test +# runs load_env from the fake product, so that file is overwritten for the +# duration of the run and restored afterwards. Nothing else outside the +# scratch directory is touched and no fab tool is executed. +# +# tests/source-date-epoch.sh + +set -uo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +share="$(cd "$here/.." && pwd)/share" +work="$(mktemp -d)" + +restore_build_env() { + if [[ -e "$work/build_env.saved" ]]; then + cp -p "$work/build_env.saved" /tmp/.build_env + else + rm -f /tmp/.build_env + fi +} +cleanup() { + restore_build_env + rm -rf "$work" +} +trap cleanup EXIT +[[ ! -e /tmp/.build_env ]] || cp -p /tmp/.build_env "$work/build_env.saved" + +# --- fake product ----------------------------------------------------------- + +export FAB_PATH="$work/fab" +export FAB_SHARE_PATH="$share" +export RELEASE=debian/trixie +export FAB_ARCH=amd64 +export NO_PROXY=true +unset SOURCE_DATE_EPOCH BOOTSTRAP O ISOLABEL CONF_VARS CHROOT_ONLY + +product="$work/fakeprod" +mkdir -p "$FAB_PATH/bootstraps/trixie-amd64" "$FAB_PATH/cdroots/generic" \ + "$product/plan" +: > "$product/plan/main" +printf 'include $(FAB_SHARE_PATH)/product.mk\n' > "$product/Makefile" + +# dry_run TARGET [VAR=VALUE ...]: stdout and stderr of make -n in the product +dry_run() { + ( + cd "$product" || exit 1 + "$share/load_env" 2>/dev/null + make -n "$@" 2>&1 + ) +} + +# --- TAP helpers ------------------------------------------------------------ + +count=0 +failed=0 + +ok() { + count=$((count + 1)) + echo "ok $count - $1" +} + +not_ok() { + count=$((count + 1)) + failed=$((failed + 1)) + echo "not ok $count - $1" + printf '%s\n' "$2" | sed 's/^/# /' +} + +# check DESC TEXT REGEX: a line of TEXT matches REGEX +check() { + if grep -qE -e "$3" <<<"$2"; then + ok "$1" + else + not_ok "$1" "no line matches: $3" + fi +} + +# check_not DESC TEXT REGEX: no line of TEXT matches REGEX +check_not() { + if grep -qE -e "$3" <<<"$2"; then + not_ok "$1" "$(grep -E -e "$3" <<<"$2")" + else + ok "$1" + fi +} + +mksquashfs_line='^/usr/bin/mksquashfs build/root.patched build/cdroot/live/10root.squashfs -no-sparse$' +plan_resolve_epoch="fab-plan-resolve .* -D 'SOURCE_DATE_EPOCH=" + +# --- SOURCE_DATE_EPOCH unset -------------------------------------------------- + +out="$(dry_run product.iso)" +check_not "unset: xorriso gets no --set_all_file_dates" "$out" '--set_all_file_dates' +check "unset: xorriso volume id is followed by nothing" "$out" '^xorriso -as mkisofs -o build/product.iso -r -J -V fakeprod +-b isolinux/isolinux.bin ' +check_not "unset: isohybrid gets no --id" "$out" '--id' +check "unset: isohybrid gets the iso only" "$out" '^isohybrid +build/product.iso$' +check "unset: mksquashfs options are unchanged" "$out" "$mksquashfs_line" +check_not "unset: fab-plan-resolve gets no -D SOURCE_DATE_EPOCH" "$out" "$plan_resolve_epoch" + +debug="$(dry_run debug V=FAB_CHROOT_ENV)" +check "unset: FAB_CHROOT_ENV holds the other builtins" "$debug" 'product.mk:[0-9]+: FAB_CHROOT_ENV = FAB_ARCH:' +check_not "unset: FAB_CHROOT_ENV has no SOURCE_DATE_EPOCH" "$debug" 'SOURCE_DATE_EPOCH' + +out="$(SOURCE_DATE_EPOCH= dry_run product.iso)" +check "empty: isohybrid gets the iso only" "$out" '^isohybrid +build/product.iso$' +check_not "empty: xorriso gets no --set_all_file_dates" "$out" '--set_all_file_dates' + +# --- SOURCE_DATE_EPOCH set ---------------------------------------------------- + +export SOURCE_DATE_EPOCH=1700000000 + +out="$(dry_run product.iso)" +check "set: xorriso gets --set_all_file_dates @epoch" "$out" \ + '^xorriso -as mkisofs -o build/product.iso -r -J -V fakeprod --set_all_file_dates @1700000000 -b isolinux/isolinux.bin ' +check "set: isohybrid gets --id epoch" "$out" '^isohybrid --id 1700000000 build/product.iso$' +check "set: mksquashfs options are unchanged" "$out" "$mksquashfs_line" +check "set: fab-plan-resolve gets -D SOURCE_DATE_EPOCH" "$out" "${plan_resolve_epoch}1700000000'" + +debug="$(dry_run debug V=FAB_CHROOT_ENV)" +check "set: FAB_CHROOT_ENV ends with SOURCE_DATE_EPOCH" "$debug" \ + 'product.mk:[0-9]+: FAB_CHROOT_ENV = FAB_ARCH:.*:SOURCE_DATE_EPOCH$' + +out="$(dry_run product.iso 'product.iso/body=$(run-genisoimage-uefi)')" +check "set: the UEFI xorriso recipe is the one used" "$out" '^xorriso .* -isohybrid-mbr /usr/lib/ISOLINUX/isohdpfx.bin .* -e efi.img ' +check "set: the UEFI xorriso recipe gets --set_all_file_dates @epoch" "$out" \ + '^xorriso -as mkisofs -o build/product.iso -r -J -V fakeprod --set_all_file_dates @1700000000 -b isolinux/isolinux.bin ' + +# the MBR id is 32 bits wide: decimal value of epoch & 0xffffffff +out="$(SOURCE_DATE_EPOCH=5000000000 dry_run product.iso)" +check "set: isohybrid id is epoch & 0xffffffff" "$out" '^isohybrid --id 705032704 build/product.iso$' +out="$(SOURCE_DATE_EPOCH=4294967296 dry_run product.iso)" +check "set: isohybrid id wraps to 0 at 2^32" "$out" '^isohybrid --id 0 build/product.iso$' + +echo "1..$count" +exit "$failed" From bf252884c658d753faca54af74a61c41ca372f0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcos=20M=C3=A9ndez?= Date: Fri, 25 Sep 2026 18:00:00 +0000 Subject: [PATCH 3/4] Add tests/coverage.sh for the SOURCE_DATE_EPOCH checks (threshold 95) make has no line coverage tool, so the number is the share of passing checks in tests/source-date-epoch.sh, one per effect of the touched code. 19 of 19 on a TKLDev 19 host. --- tests/coverage.sh | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100755 tests/coverage.sh diff --git a/tests/coverage.sh b/tests/coverage.sh new file mode 100755 index 0000000..325c8ec --- /dev/null +++ b/tests/coverage.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Coverage of the SOURCE_DATE_EPOCH code in share/product.mk. +# +# make has no line coverage tool, so the number here is the share of the +# checks in tests/source-date-epoch.sh that pass: one check per effect of +# the touched code (both arms of the ifneq, both xorriso recipes, isohybrid, +# FAB_CHROOT_ENV, fab-plan-resolve and the untouched mksquashfs options). +# Exits 1 when that share is below the threshold (default 95). +# +# tests/coverage.sh [THRESHOLD] +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +threshold="${1:-95}" + +tap="$("$here/source-date-epoch.sh" || true)" +printf '%s\n' "$tap" +total="$(grep -cE '^(not )?ok [0-9]+ ' <<<"$tap")" +passed="$(grep -cE '^ok [0-9]+ ' <<<"$tap")" +percent="$(awk -v p="$passed" -v t="$total" 'BEGIN { printf "%.2f", (t ? 100 * p / t : 0) }')" + +echo "product.mk SOURCE_DATE_EPOCH: $percent percent ($passed of $total checks) covered, threshold $threshold" +if ! awk -v p="$percent" -v t="$threshold" 'BEGIN { exit !(p + 0 >= t + 0) }'; then + echo "coverage below threshold" >&2 + exit 1 +fi From d8882cd837f26372336452690da9f817c10ab150 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcos=20M=C3=A9ndez?= Date: Sat, 26 Sep 2026 09:10:27 +0000 Subject: [PATCH 4/4] tests/coverage.sh: read COVERAGE_THRESHOLD when no argument is given The reusable workflow test-shell.yml exports the caller's threshold as COVERAGE_THRESHOLD and runs the script without arguments, so the script enforced its own default (95) whatever the workflow said. The positional argument still wins, the environment variable comes next, 95 remains the fallback. --- tests/coverage.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/coverage.sh b/tests/coverage.sh index 325c8ec..9c62427 100755 --- a/tests/coverage.sh +++ b/tests/coverage.sh @@ -7,11 +7,11 @@ # FAB_CHROOT_ENV, fab-plan-resolve and the untouched mksquashfs options). # Exits 1 when that share is below the threshold (default 95). # -# tests/coverage.sh [THRESHOLD] +# tests/coverage.sh [THRESHOLD] (or COVERAGE_THRESHOLD in the environment) set -euo pipefail here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -threshold="${1:-95}" +threshold="${1:-${COVERAGE_THRESHOLD:-95}}" tap="$("$here/source-date-epoch.sh" || true)" printf '%s\n' "$tap"