diff --git a/fboss-image/image_builder/templates/centos-09.0/config.sh b/fboss-image/image_builder/templates/centos-09.0/config.sh index 4586d21037612..727fc835c2786 100755 --- a/fboss-image/image_builder/templates/centos-09.0/config.sh +++ b/fboss-image/image_builder/templates/centos-09.0/config.sh @@ -546,6 +546,7 @@ systemctl enable sai-device-nodes.service || echo "WARNING: sai-device-nodes.service not present; SAI kmods will not be loaded at boot" systemctl enable fboss_init.service systemctl enable local_rpm_repo.service +systemctl enable fboss_disk_guard.service systemctl enable platform_manager.service systemctl enable data_corral_service.service systemctl enable fan_service.service diff --git a/fboss-image/image_builder/templates/centos-09.0/root_files/usr/lib/systemd/system/fboss_disk_guard.service b/fboss-image/image_builder/templates/centos-09.0/root_files/usr/lib/systemd/system/fboss_disk_guard.service new file mode 100644 index 0000000000000..9198a5fbfb08c --- /dev/null +++ b/fboss-image/image_builder/templates/centos-09.0/root_files/usr/lib/systemd/system/fboss_disk_guard.service @@ -0,0 +1,11 @@ +[Unit] +Description=FBOSS root filesystem headroom guard +Before=platform_manager.service + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/local/bin/fboss_disk_guard.sh + +[Install] +WantedBy=multi-user.target diff --git a/fboss-image/image_builder/templates/centos-09.0/root_files/usr/lib/systemd/system/platform_manager.service b/fboss-image/image_builder/templates/centos-09.0/root_files/usr/lib/systemd/system/platform_manager.service index 9552dea39abd3..7294f78e0d2df 100644 --- a/fboss-image/image_builder/templates/centos-09.0/root_files/usr/lib/systemd/system/platform_manager.service +++ b/fboss-image/image_builder/templates/centos-09.0/root_files/usr/lib/systemd/system/platform_manager.service @@ -10,6 +10,8 @@ Before=fruid.service After=rc-local.service Before=fboss_init.service Wants=local_rpm_repo.service +Wants=fboss_disk_guard.service +After=fboss_disk_guard.service StartLimitIntervalSec=600 StartLimitBurst=5 diff --git a/fboss-image/image_builder/templates/centos-09.0/root_files/usr/local/bin/fboss_disk_guard.sh b/fboss-image/image_builder/templates/centos-09.0/root_files/usr/local/bin/fboss_disk_guard.sh new file mode 100755 index 0000000000000..25cf08b97e298 --- /dev/null +++ b/fboss-image/image_builder/templates/centos-09.0/root_files/usr/local/bin/fboss_disk_guard.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Ensure root-fs headroom before platform_manager starts (rpmdb dies below ~128KB). +set -f +MIN_FREE_KB=${MIN_FREE_KB:-102400} +BALLAST=/var/lib/fboss/.ballast +BALLAST_MB=${BALLAST_MB:-$((MIN_FREE_KB / 1024))} # ballast >= threshold so rm always recovers + +free_kb() { df --output=avail -k / | tail -1 | tr -d ' '; } +log() { echo "$*" >&2; } +alert() { log "FBOSS_ALERT $1 "; } + +start=$(free_kb) +if [ "$start" -ge "$MIN_FREE_KB" ]; then + if [ ! -s "$BALLAST" ] && [ "$start" -ge $((MIN_FREE_KB * 10)) ]; then + mkdir -p "$(dirname "$BALLAST")" && fallocate -l "${BALLAST_MB}M" "$BALLAST" && + log "ballast ${BALLAST_MB}MB created at $BALLAST" + fi + exit 0 +fi + +alert "Root filesystem low before platform_manager start: ${start}KB free" low_disk_space +steps=( + "journalctl --vacuum-size=16M" + "rm -f /var/lib/systemd/coredump/*" + "find /var/log -xdev -type f ( -name *.gz -o -name *.[0-9] -o -name *.old ) -delete" + "rm -f $BALLAST" +) +for s in "${steps[@]}"; do + $s >/dev/null 2>&1 + sync + for _ in 1 2 3 4 5 6 7 8 9 10; do # btrfs frees unlinked extents asynchronously + now=$(free_kb) + [ "$now" -ge "$MIN_FREE_KB" ] && break + sleep 1 + done + if [ "$now" -ge "$MIN_FREE_KB" ]; then + alert "Recovered ${start}KB -> ${now}KB via: $s" low_disk_space_recovered + exit 0 + fi +done +alert "Still low after cleanup: $(free_kb)KB free" low_disk_space_unrecovered +exit 0