Honour SOURCE_DATE_EPOCH in product.mk for reproducible squashfs and ISO images

share/product.mk

@@ -23,7 +23,7 @@ endif
 
 COMMON_PATCHES := turnkey.d $(COMMON_PATCHES)
 
-CONF_VARS_BUILTIN ?= FAB_ARCH HOST_ARCH FAB_HTTP_PROXY AMD64 ARM64 RELEASE DISTRO CODENAME DEBIAN UBUNTU KERNEL DEBUG CHROOT_ONLY DI_LIVE_DEBUG
+CONF_VARS_BUILTIN ?= FAB_ARCH HOST_ARCH FAB_HTTP_PROXY AMD64 ARM64 RELEASE DISTRO CODENAME DEBIAN UBUNTU KERNEL DEBUG CHROOT_ONLY DI_LIVE_DEBUG SOURCE_DATE_EPOCH
 
 define filter-undefined-vars
 	$(foreach var,$1,$(if $($(var)), $(var)))
@@ -53,6 +53,18 @@ CDROOT ?= generic
 MKSQUASHFS ?= /usr/bin/mksquashfs
 MKSQUASHFS_OPTS ?= -no-sparse
 
+# https://reproducible-builds.org/docs/source-date-epoch/ - when set, pin
+# every timestamp the image would otherwise take from the clock. mksquashfs
+# and xorriso read the variable themselves (mksquashfs even refuses to take
+# both it and equivalent options), but xorriso only fixes its own metadata,
+# not the mtimes of the files it stores, and isohybrid writes a random MBR
+# id. Exporting the variable (CONF_VARS_BUILTIN above) also reaches dpkg-deb
+# and mkinitramfs inside the chroot. Unset, nothing here applies.
+ifneq ($(SOURCE_DATE_EPOCH),)
+XORRISO_DATE_OPTS := --set_all_file_dates @$(SOURCE_DATE_EPOCH)
+ISOHYBRID_OPTS := --id $(shell echo $$(( $(SOURCE_DATE_EPOCH) & 0xffffffff )))
+endif
+
 # if the CDROOT is a relative path, prefix CDROOTS_PATH
 # we set _CDROOT with eval to improve the readability of $(value _CDROOT) 
 # in help target
@@ -454,7 +466,7 @@ endef
 define run-genisoimage
     xorriso -as mkisofs \
         -o $O/product.iso -r -J \
-        -V ${ISOLABEL} \
+        -V ${ISOLABEL} $(XORRISO_DATE_OPTS) \
         -b isolinux/isolinux.bin \
         -c isolinux/boot.cat \
         -no-emul-boot \
@@ -466,7 +478,7 @@ endef
 define run-genisoimage-uefi
 	xorriso -as mkisofs \
 		-o $O/product.iso -r -J \
-		-V ${ISOLABEL} \
+		-V ${ISOLABEL} $(XORRISO_DATE_OPTS) \
 		-b isolinux/isolinux.bin \
 		-c isolinux/boot.cat \
 		-isohybrid-mbr /usr/lib/ISOLINUX/isohdpfx.bin \
@@ -481,7 +493,7 @@ define run-genisoimage-uefi
 endef
 
 define run-isohybrid
-	isohybrid $O/product.iso
+	isohybrid $(ISOHYBRID_OPTS) $O/product.iso
 endef
 
 # target: product.iso

tests/coverage.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Coverage of the SOURCE_DATE_EPOCH code in share/product.mk.
+#
+# make has no line coverage tool, so the number here is the share of the
+# checks in tests/source-date-epoch.sh that pass: one check per effect of
+# the touched code (both arms of the ifneq, both xorriso recipes, isohybrid,
+# FAB_CHROOT_ENV, fab-plan-resolve and the untouched mksquashfs options).
+# Exits 1 when that share is below the threshold (default 95).
+#
+#   tests/coverage.sh [THRESHOLD]     (or COVERAGE_THRESHOLD in the environment)
+set -euo pipefail
+
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+threshold="${1:-${COVERAGE_THRESHOLD:-95}}"
+
+tap="$("$here/source-date-epoch.sh" || true)"
+printf '%s\n' "$tap"
+total="$(grep -cE '^(not )?ok [0-9]+ ' <<<"$tap")"
+passed="$(grep -cE '^ok [0-9]+ ' <<<"$tap")"
+percent="$(awk -v p="$passed" -v t="$total" 'BEGIN { printf "%.2f", (t ? 100 * p / t : 0) }')"
+
+echo "product.mk SOURCE_DATE_EPOCH: $percent percent ($passed of $total checks) covered, threshold $threshold"
+if ! awk -v p="$percent" -v t="$threshold" 'BEGIN { exit !(p + 0 >= t + 0) }'; then
+    echo "coverage below threshold" >&2
+    exit 1
+fi

tests/source-date-epoch.sh

@@ -0,0 +1,143 @@
+#!/bin/bash
+# make-level test for SOURCE_DATE_EPOCH in share/product.mk.
+#
+# Builds a minimal fake product under a scratch FAB_PATH and dry-runs
+# (make -n) the product.iso recipe against the product.mk of this checkout,
+# once with SOURCE_DATE_EPOCH unset and once with it set. Output is TAP; the
+# exit status is the number of failed checks.
+#
+# product.mk includes /tmp/.build_env, written by share/load_env; the test
+# runs load_env from the fake product, so that file is overwritten for the
+# duration of the run and restored afterwards. Nothing else outside the
+# scratch directory is touched and no fab tool is executed.
+#
+#   tests/source-date-epoch.sh
+
+set -uo pipefail
+
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+share="$(cd "$here/.." && pwd)/share"
+work="$(mktemp -d)"
+
+restore_build_env() {
+    if [[ -e "$work/build_env.saved" ]]; then
+        cp -p "$work/build_env.saved" /tmp/.build_env
+    else
+        rm -f /tmp/.build_env
+    fi
+}
+cleanup() {
+    restore_build_env
+    rm -rf "$work"
+}
+trap cleanup EXIT
+[[ ! -e /tmp/.build_env ]] || cp -p /tmp/.build_env "$work/build_env.saved"
+
+# --- fake product -----------------------------------------------------------
+
+export FAB_PATH="$work/fab"
+export FAB_SHARE_PATH="$share"
+export RELEASE=debian/trixie
+export FAB_ARCH=amd64
+export NO_PROXY=true
+unset SOURCE_DATE_EPOCH BOOTSTRAP O ISOLABEL CONF_VARS CHROOT_ONLY
+
+product="$work/fakeprod"
+mkdir -p "$FAB_PATH/bootstraps/trixie-amd64" "$FAB_PATH/cdroots/generic" \
+    "$product/plan"
+: > "$product/plan/main"
+printf 'include $(FAB_SHARE_PATH)/product.mk\n' > "$product/Makefile"
+
+# dry_run TARGET [VAR=VALUE ...]: stdout and stderr of make -n in the product
+dry_run() {
+    (
+        cd "$product" || exit 1
+        "$share/load_env" 2>/dev/null
+        make -n "$@" 2>&1
+    )
+}
+
+# --- TAP helpers ------------------------------------------------------------
+
+count=0
+failed=0
+
+ok() {
+    count=$((count + 1))
+    echo "ok $count - $1"
+}
+
+not_ok() {
+    count=$((count + 1))
+    failed=$((failed + 1))
+    echo "not ok $count - $1"
+    printf '%s\n' "$2" | sed 's/^/# /'
+}
+
+# check DESC TEXT REGEX: a line of TEXT matches REGEX
+check() {
+    if grep -qE -e "$3" <<<"$2"; then
+        ok "$1"
+    else
+        not_ok "$1" "no line matches: $3"
+    fi
+}
+
+# check_not DESC TEXT REGEX: no line of TEXT matches REGEX
+check_not() {
+    if grep -qE -e "$3" <<<"$2"; then
+        not_ok "$1" "$(grep -E -e "$3" <<<"$2")"
+    else
+        ok "$1"
+    fi
+}
+
+mksquashfs_line='^/usr/bin/mksquashfs build/root.patched build/cdroot/live/10root.squashfs -no-sparse$'
+plan_resolve_epoch="fab-plan-resolve .* -D 'SOURCE_DATE_EPOCH="
+
+# --- SOURCE_DATE_EPOCH unset --------------------------------------------------
+
+out="$(dry_run product.iso)"
+check_not "unset: xorriso gets no --set_all_file_dates" "$out" '--set_all_file_dates'
+check "unset: xorriso volume id is followed by nothing" "$out" '^xorriso -as mkisofs -o build/product.iso -r -J -V fakeprod +-b isolinux/isolinux.bin '
+check_not "unset: isohybrid gets no --id" "$out" '--id'
+check "unset: isohybrid gets the iso only" "$out" '^isohybrid +build/product.iso$'
+check "unset: mksquashfs options are unchanged" "$out" "$mksquashfs_line"
+check_not "unset: fab-plan-resolve gets no -D SOURCE_DATE_EPOCH" "$out" "$plan_resolve_epoch"
+
+debug="$(dry_run debug V=FAB_CHROOT_ENV)"
+check "unset: FAB_CHROOT_ENV holds the other builtins" "$debug" 'product.mk:[0-9]+: FAB_CHROOT_ENV = FAB_ARCH:'
+check_not "unset: FAB_CHROOT_ENV has no SOURCE_DATE_EPOCH" "$debug" 'SOURCE_DATE_EPOCH'
+
+out="$(SOURCE_DATE_EPOCH= dry_run product.iso)"
+check "empty: isohybrid gets the iso only" "$out" '^isohybrid +build/product.iso$'
+check_not "empty: xorriso gets no --set_all_file_dates" "$out" '--set_all_file_dates'
+
+# --- SOURCE_DATE_EPOCH set ----------------------------------------------------
+
+export SOURCE_DATE_EPOCH=1700000000
+
+out="$(dry_run product.iso)"
+check "set: xorriso gets --set_all_file_dates @epoch" "$out" \
+    '^xorriso -as mkisofs -o build/product.iso -r -J -V fakeprod --set_all_file_dates @1700000000 -b isolinux/isolinux.bin '
+check "set: isohybrid gets --id epoch" "$out" '^isohybrid --id 1700000000 build/product.iso$'
+check "set: mksquashfs options are unchanged" "$out" "$mksquashfs_line"
+check "set: fab-plan-resolve gets -D SOURCE_DATE_EPOCH" "$out" "${plan_resolve_epoch}1700000000'"
+
+debug="$(dry_run debug V=FAB_CHROOT_ENV)"
+check "set: FAB_CHROOT_ENV ends with SOURCE_DATE_EPOCH" "$debug" \
+    'product.mk:[0-9]+: FAB_CHROOT_ENV = FAB_ARCH:.*:SOURCE_DATE_EPOCH$'
+
+out="$(dry_run product.iso 'product.iso/body=$(run-genisoimage-uefi)')"
+check "set: the UEFI xorriso recipe is the one used" "$out" '^xorriso .* -isohybrid-mbr /usr/lib/ISOLINUX/isohdpfx.bin .* -e efi.img '
+check "set: the UEFI xorriso recipe gets --set_all_file_dates @epoch" "$out" \
+    '^xorriso -as mkisofs -o build/product.iso -r -J -V fakeprod --set_all_file_dates @1700000000 -b isolinux/isolinux.bin '
+
+# the MBR id is 32 bits wide: decimal value of epoch & 0xffffffff
+out="$(SOURCE_DATE_EPOCH=5000000000 dry_run product.iso)"
+check "set: isohybrid id is epoch & 0xffffffff" "$out" '^isohybrid --id 705032704 build/product.iso$'
+out="$(SOURCE_DATE_EPOCH=4294967296 dry_run product.iso)"
+check "set: isohybrid id wraps to 0 at 2^32" "$out" '^isohybrid --id 0 build/product.iso$'
+
+echo "1..$count"
+exit "$failed"