From 888699796e2ff6a0e437e00e7ec9fd1701709311 Mon Sep 17 00:00:00 2001 From: Mohammed Sazeed Date: Mon, 2 Nov 2026 06:13:56 +0000 Subject: [PATCH] [Nexthop] fboss-image: root filesystem headroom guard before platform_manager On a boot with / full, rpm and dnf cannot open the rpmdb (rpm's sqlite backend runs a write transaction on open and dies with SIGBUS below ~128 KB free), so platform_manager, and everything ordered after it, fails. Nothing in the image frees space before the stack starts, and btrfs has no root-reserved blocks, so root daemons get no headroom once the disk fills. Add fboss_disk_guard.service, a oneshot ordered before platform_manager.service (which Wants= and After= it). When free space on / is below MIN_FREE_KB (100 MB) it logs an FBOSS_ALERT and frees, in order, the journal (vacuum to 16 MB), systemd coredumps and rotated logs under /var/log, then deletes a preallocated ballast file (/var/lib/fboss/.ballast, sized to the threshold). It re-polls df after each step because btrfs releases unlinked extents asynchronously, and stops at the first step that recovers. On a healthy boot with at least 10x the threshold free it recreates the ballast. It always exits 0 and never writes to disk itself. Co-Authored-By: Claude Opus 5.5 --- .../templates/centos-09.0/config.sh | 1 + .../systemd/system/fboss_disk_guard.service | 11 +++++ .../systemd/system/platform_manager.service | 2 + .../usr/local/bin/fboss_disk_guard.sh | 42 +++++++++++++++++++ 4 files changed, 56 insertions(+) create mode 100644 fboss-image/image_builder/templates/centos-09.0/root_files/usr/lib/systemd/system/fboss_disk_guard.service create mode 100755 fboss-image/image_builder/templates/centos-09.0/root_files/usr/local/bin/fboss_disk_guard.sh diff --git a/fboss-image/image_builder/templates/centos-09.0/config.sh b/fboss-image/image_builder/templates/centos-09.0/config.sh index 4586d21037612..727fc835c2786 100755 --- a/fboss-image/image_builder/templates/centos-09.0/config.sh +++ b/fboss-image/image_builder/templates/centos-09.0/config.sh @@ -546,6 +546,7 @@ systemctl enable sai-device-nodes.service || echo "WARNING: sai-device-nodes.service not present; SAI kmods will not be loaded at boot" systemctl enable fboss_init.service systemctl enable local_rpm_repo.service +systemctl enable fboss_disk_guard.service systemctl enable platform_manager.service systemctl enable data_corral_service.service systemctl enable fan_service.service diff --git a/fboss-image/image_builder/templates/centos-09.0/root_files/usr/lib/systemd/system/fboss_disk_guard.service b/fboss-image/image_builder/templates/centos-09.0/root_files/usr/lib/systemd/system/fboss_disk_guard.service new file mode 100644 index 0000000000000..9198a5fbfb08c --- /dev/null +++ b/fboss-image/image_builder/templates/centos-09.0/root_files/usr/lib/systemd/system/fboss_disk_guard.service @@ -0,0 +1,11 @@ +[Unit] +Description=FBOSS root filesystem headroom guard +Before=platform_manager.service + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/local/bin/fboss_disk_guard.sh + +[Install] +WantedBy=multi-user.target diff --git a/fboss-image/image_builder/templates/centos-09.0/root_files/usr/lib/systemd/system/platform_manager.service b/fboss-image/image_builder/templates/centos-09.0/root_files/usr/lib/systemd/system/platform_manager.service index 9552dea39abd3..7294f78e0d2df 100644 --- a/fboss-image/image_builder/templates/centos-09.0/root_files/usr/lib/systemd/system/platform_manager.service +++ b/fboss-image/image_builder/templates/centos-09.0/root_files/usr/lib/systemd/system/platform_manager.service @@ -10,6 +10,8 @@ Before=fruid.service After=rc-local.service Before=fboss_init.service Wants=local_rpm_repo.service +Wants=fboss_disk_guard.service +After=fboss_disk_guard.service StartLimitIntervalSec=600 StartLimitBurst=5 diff --git a/fboss-image/image_builder/templates/centos-09.0/root_files/usr/local/bin/fboss_disk_guard.sh b/fboss-image/image_builder/templates/centos-09.0/root_files/usr/local/bin/fboss_disk_guard.sh new file mode 100755 index 0000000000000..25cf08b97e298 --- /dev/null +++ b/fboss-image/image_builder/templates/centos-09.0/root_files/usr/local/bin/fboss_disk_guard.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Ensure root-fs headroom before platform_manager starts (rpmdb dies below ~128KB). +set -f +MIN_FREE_KB=${MIN_FREE_KB:-102400} +BALLAST=/var/lib/fboss/.ballast +BALLAST_MB=${BALLAST_MB:-$((MIN_FREE_KB / 1024))} # ballast >= threshold so rm always recovers + +free_kb() { df --output=avail -k / | tail -1 | tr -d ' '; } +log() { echo "$*" >&2; } +alert() { log "FBOSS_ALERT $1 "; } + +start=$(free_kb) +if [ "$start" -ge "$MIN_FREE_KB" ]; then + if [ ! -s "$BALLAST" ] && [ "$start" -ge $((MIN_FREE_KB * 10)) ]; then + mkdir -p "$(dirname "$BALLAST")" && fallocate -l "${BALLAST_MB}M" "$BALLAST" && + log "ballast ${BALLAST_MB}MB created at $BALLAST" + fi + exit 0 +fi + +alert "Root filesystem low before platform_manager start: ${start}KB free" low_disk_space +steps=( + "journalctl --vacuum-size=16M" + "rm -f /var/lib/systemd/coredump/*" + "find /var/log -xdev -type f ( -name *.gz -o -name *.[0-9] -o -name *.old ) -delete" + "rm -f $BALLAST" +) +for s in "${steps[@]}"; do + $s >/dev/null 2>&1 + sync + for _ in 1 2 3 4 5 6 7 8 9 10; do # btrfs frees unlinked extents asynchronously + now=$(free_kb) + [ "$now" -ge "$MIN_FREE_KB" ] && break + sleep 1 + done + if [ "$now" -ge "$MIN_FREE_KB" ]; then + alert "Recovered ${start}KB -> ${now}KB via: $s" low_disk_space_recovered + exit 0 + fi +done +alert "Still low after cleanup: $(free_kb)KB free" low_disk_space_unrecovered +exit 0