confd: reset to fail-secure if loading startup-config hangs

A plugin callback that never returns while loading startup-config hangs
the boot, and the unit never reaches failure-config.

Supervise the bootstrap event pump with watchdogd.  If it misses its
deadline, the supervisor script marks the boot as failed and watchdogd
resets the unit, so the next boot goes straight to failure-config.  The
fail-secure boot itself is not supervised, a hang there would be a reset
loop.

Issue #1637

Signed-off-by: Joachim Wiberg <[email protected]>

Author: troglobit

board/common/rootfs/etc/watchdogd.conf

@@ -62,6 +62,11 @@ device /dev/watchdog {
 #    priority = 98
 #    script = "/path/to/supervisor-script.sh"
 #}
+supervisor {
+    enabled  = true
+    priority = 98
+    script   = "/usr/libexec/infix/supervisor"
+}
 
 ### Reset reason #######################################################
 # The following section controls if/how the reset reason & reset counter

board/common/rootfs/usr/libexec/infix/supervisor

@@ -0,0 +1,14 @@
+#!/bin/sh
+# Called by watchdogd when a supervised process misses its deadline:
+#
+#    $0 supervisor CODE PID LABEL
+#
+# A hung confd bootstrap goes to fail-secure on next boot, issue #1637.
+# Exit non-zero so watchdogd saves the reset reason and resets.
+
+if [ "$4" = "confd-bootstrap" ]; then
+    touch /mnt/aux/startup-config.failed
+    sync
+fi
+
+exit 1

doc/ChangeLog.md

@@ -116,7 +116,9 @@ All notable changes to the project are documented in this file.
   the WebUI software page shows when the other partition has a
   different version
 - A startup-config that fails to load now resets the unit, and the next
-  boot goes straight to failure-config from a clean state, issue #1637
+  boot goes straight to failure-config from a clean state, issue #1637.
+  The same applies if loading startup-config hangs, the system watchdog
+  then resets the unit
 - Fix #1619: Raspberry Pi kernel panic when configure Wi-Fi
 - WebUI: "Save" in the interface editor and "OK" in Add Interface
   did nothing for Wi-Fi and WireGuard interfaces. The inline "+ New"

doc/boot.md

@@ -190,6 +190,11 @@ the next boot starts from a clean state and goes straight to *Fail
 Secure Mode*.  The mark is cleared on that boot, so a reboot after it
 tries `startup-config` again.
 
+The same happens if loading `startup-config` hangs, e.g., a service that
+never responds.  The system watchdog then resets the unit, by default
+after 70 seconds, and the next boot goes straight to *Fail Secure
+Mode*.
+
 > [!TIP]
 > Please see the [Branding & Releases](branding.md) document for how to
 > provide per-product `failure-config`, or `factory-config` to suit your

package/confd/Config.in

@@ -6,6 +6,7 @@ config BR2_PACKAGE_CONFD
 	select BR2_PACKAGE_SYSREPO
 	select BR2_PACKAGE_LIBSRX
 	select BR2_PACKAGE_SUPPORT
+	select BR2_PACKAGE_WATCHDOGD
 	help
 	  A plugin to sysrepo that provides the core YANG models used to
 	  manage an Infix based system.  Configuration can be done using

package/confd/confd.mk

@@ -10,7 +10,7 @@ CONFD_SITE = $(BR2_EXTERNAL_INFIX_PATH)/src/confd
 CONFD_LICENSE = BSD-3-Clause
 CONFD_LICENSE_FILES = LICENSE
 CONFD_REDISTRIBUTE = NO
-CONFD_DEPENDENCIES = host-sysrepo sysrepo rousette netopeer2 jansson libite sysrepo libsrx libglib2 libev sysklogd
+CONFD_DEPENDENCIES = host-sysrepo sysrepo rousette netopeer2 jansson libite sysrepo libsrx libglib2 libev sysklogd watchdogd
 CONFD_AUTORECONF = YES
 CONFD_CONF_OPTS += --disable-silent-rules --with-crypt=$(BR2_PACKAGE_CONFD_DEFAULT_CRYPT)
 CONFD_SYSREPO_SHM_PREFIX = sr_buildroot$(subst /,_,$(CONFIG_DIR))_confd

src/confd/configure.ac

@@ -120,6 +120,7 @@ PKG_CHECK_MODULES([libyang], [libyang >= 4.2.2])
 PKG_CHECK_MODULES([libsrx],  [libsrx >= 1.0.0])
 PKG_CHECK_MODULES([libsyslog], [libsyslog >= 2.7.0])
 PKG_CHECK_MODULES([libcrypto], [libcrypto])
+PKG_CHECK_MODULES([libwdog], [libwdog >= 4.0])
 
 AC_CHECK_HEADER([ev.h],
     [saved_LIBS="$LIBS"

src/confd/src/Makefile.am

@@ -6,8 +6,8 @@ plugin_LTLIBRARIES      = confd-plugin.la
 
 sbin_PROGRAMS           = confd
 
-confd_CFLAGS            = $(sysrepo_CFLAGS) $(libyang_CFLAGS) $(jansson_CFLAGS) $(libite_CFLAGS) $(libsrx_CFLAGS)
-confd_LDADD             = $(sysrepo_LIBS)   $(libyang_LIBS)   $(jansson_LIBS)   $(libite_LIBS)   $(libsrx_LIBS) $(EV_LIBS) -ldl
+confd_CFLAGS            = $(sysrepo_CFLAGS) $(libyang_CFLAGS) $(jansson_CFLAGS) $(libite_CFLAGS) $(libsrx_CFLAGS) $(libwdog_CFLAGS)
+confd_LDADD             = $(sysrepo_LIBS)   $(libyang_LIBS)   $(jansson_LIBS)   $(libite_LIBS)   $(libsrx_LIBS) $(libwdog_LIBS) $(EV_LIBS) -ldl
 confd_SOURCES           = main.c
 
 confd_plugin_la_LDFLAGS = -module -avoid-version -shared

src/confd/src/main.c

@@ -35,6 +35,7 @@
 #include <srx/common.h>
 #include <sysrepo.h>
 #include <sysrepo/version.h>
+#include <wdog/wdog.h>
 
 /* Maximum number of sysrepo event pipe file descriptors across all plugins */
 #define MAX_EVENT_FDS 64
@@ -192,11 +193,15 @@ static void pump_sigterm(int sig)
 	pump_running = 0;
 }
 
-static void event_pump(struct plugin *plugins, int plugin_count)
+/* With a deadline, a hung callback resets us to fail-secure, issue #1637 */
+static void event_pump(struct plugin *plugins, int plugin_count, uint32_t deadline_ms)
 {
 	sr_subscription_ctx_t *subs[MAX_EVENT_FDS];
 	struct pollfd fds[MAX_EVENT_FDS];
+	time_t now, last = 0;
+	unsigned int ack;
 	int nfds = 0;
+	int id = -1;
 
 	for (int i = 0; i < plugin_count; i++) {
 		struct plugin *p = &plugins[i];
@@ -215,14 +220,30 @@ static void event_pump(struct plugin *plugins, int plugin_count)
 
 	signal(SIGTERM, pump_sigterm);
 
+	if (deadline_ms) {
+		id = wdog_subscribe("confd-bootstrap", deadline_ms, &ack);
+		if (id < 0)
+			WARN("Not supervised by watchdogd: %s", strerror(-id));
+	}
+
 	while (pump_running) {
 		if (poll(fds, nfds, 100) > 0) {
 			for (int i = 0; i < nfds; i++)
 				if (fds[i].revents & POLLIN)
 					sr_subscription_process_events(subs[i], NULL, NULL);
 		}
+
+		/* Each kick is a round trip to watchdogd, once a second is enough */
+		now = time(NULL);
+		if (id >= 0 && now != last) {
+			last = now;
+			wdog_kick2(id, &ack);
+		}
 	}
 
+	if (id >= 0)
+		wdog_unsubscribe(id, ack);
+
 	_exit(0);
 }
 
@@ -719,7 +740,7 @@ int main(int argc, char **argv)
 	uint32_t timeout_s = 60;
 	int plugin_count = 0;
 	int fatal_fail = 0;
-	uint32_t timeout_ms;
+	uint32_t timeout_ms, deadline_ms;
 	int status;
 
 	struct option options[] = {
@@ -912,13 +933,15 @@ int main(int argc, char **argv)
 		/* Phase 9: Fork event pump process for bootstrap.
 		 * With SR_SUBSCR_NO_THREAD, sr_replace_config() blocks waiting
 		 * for callbacks.  The pump process processes those events. */
+		/* Past the sysrepo timeout, not on a fail-secure boot (reset loop) */
+		deadline_ms = fexist(FAILED_PATH) ? 0 : timeout_ms + 10000;
 		pump_pid = fork();
 		if (pump_pid < 0) {
 			ERRNO("Failed to fork event pump");
 			goto cleanup;
 		}
 		if (pump_pid == 0)
-			event_pump(plugins, plugin_count);
+			event_pump(plugins, plugin_count, deadline_ms);
 
 		/* Phase 10: Load startup config -- plugins are now subscribed, so
 		 * sr_replace_config() will trigger their change callbacks.