CN9130 Watchdog

board/aarch32/linux_defconfig

@@ -394,6 +394,7 @@ CONFIG_BOOTPARAM_HARDLOCKUP_PANIC=y
 CONFIG_BOOTPARAM_HUNG_TASK_PANIC=y
 CONFIG_WQ_WATCHDOG=y
 CONFIG_WQ_CPU_INTENSIVE_REPORT=y
+CONFIG_TEST_LOCKUP=m
 # CONFIG_SCHED_DEBUG is not set
 # CONFIG_RCU_TRACE is not set
 CONFIG_FUNCTION_TRACER=y

board/aarch64/linux_defconfig

@@ -407,6 +407,7 @@ CONFIG_WATCHDOG=y
 CONFIG_WATCHDOG_SYSFS=y
 CONFIG_SOFT_WATCHDOG=y
 CONFIG_GPIO_WATCHDOG=y
+CONFIG_ARM_SBSA_WATCHDOG=y
 CONFIG_ARMADA_37XX_WATCHDOG=y
 CONFIG_I6300ESB_WDT=y
 CONFIG_MFD_MAX77620=y
@@ -564,6 +565,7 @@ CONFIG_BOOTPARAM_HARDLOCKUP_PANIC=y
 CONFIG_BOOTPARAM_HUNG_TASK_PANIC=y
 CONFIG_WQ_WATCHDOG=y
 CONFIG_WQ_CPU_INTENSIVE_REPORT=y
+CONFIG_TEST_LOCKUP=m
 # CONFIG_SCHED_DEBUG is not set
 # CONFIG_RCU_TRACE is not set
 CONFIG_FUNCTION_TRACER=y

board/aarch64/styx-dcp-sc-28p/dts/styx/dcp-sc-28p-a.dts

@@ -59,3 +59,9 @@
 		XSWP(a, "e13", 13, &sfp0);
 	};
 };
+
+&cp0_spi1 {
+	spi-flash@0 {
+		broken-flash-reset;
+	};
+};

board/riscv64/linux_defconfig

@@ -481,6 +481,7 @@ CONFIG_BOOTPARAM_HARDLOCKUP_PANIC=y
 CONFIG_BOOTPARAM_HUNG_TASK_PANIC=y
 CONFIG_WQ_WATCHDOG=y
 CONFIG_WQ_CPU_INTENSIVE_REPORT=y
+CONFIG_TEST_LOCKUP=m
 # CONFIG_SCHED_DEBUG is not set
 CONFIG_STACKTRACE=y
 CONFIG_RCU_CPU_STALL_TIMEOUT=60

board/x86_64/linux_defconfig

@@ -270,5 +270,6 @@ CONFIG_BOOTPARAM_HARDLOCKUP_PANIC=y
 CONFIG_BOOTPARAM_HUNG_TASK_PANIC=y
 CONFIG_WQ_WATCHDOG=y
 CONFIG_WQ_CPU_INTENSIVE_REPORT=y
+CONFIG_TEST_LOCKUP=m
 CONFIG_FUNCTION_TRACER=y
 CONFIG_UNWINDER_FRAME_POINTER=y

test/case/hardware/Readme.adoc

@@ -5,7 +5,10 @@ Tests verifying hardware monitoring and management:
 
  - USB device detection and enumeration
  - Multiple USB port management and device handling
+ - Watchdog reset capability
 
 include::usb/Readme.adoc[]
 
 include::usb_two_ports/Readme.adoc[]
+
+include::watchdog/Readme.adoc[]

test/case/hardware/all.yaml

@@ -4,3 +4,6 @@
 
 - name: USB configuration with two USB ports
   case: usb_two_ports/test.py
+
+- name: Watchdog reset on system lockup
+  case: watchdog/test.py

test/case/hardware/watchdog/Readme.adoc

@@ -0,0 +1 @@
+test.adoc

test/case/hardware/watchdog/test.adoc

@@ -0,0 +1,28 @@
+=== Watchdog reset on system lockup
+
+ifdef::topdoc[:imagesdir: {topdoc}../../test/case/hardware/watchdog]
+
+==== Description
+
+Verify that a system's watchdog trips and successfully reboots the
+system back to a working state if a lockup occurs.
+
+This is tested by using the Linux kernel's `test_lockup` module to
+inject a hard lockup (i.e., blocking servicing of all interrupts) on
+all CPU cores that lasts for twice as long as the watchdog's reported
+timeout.
+
+==== Topology
+
+image::topology.svg[Watchdog reset on system lockup topology, align=center, scaledwidth=75%]
+
+==== Sequence
+
+. Set up topology and attach to target DUT
+. Verify the presence of a watchdog device
+. Verify the presence of the test_lockup module
+. Trigger a hard lockup on all CPU cores
+. Wait for the watchdog to trip
+. Verify that the system reboots
+
+

test/case/hardware/watchdog/test.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+"""Watchdog reset on system lockup
+
+Verify that a system's watchdog trips and successfully reboots the
+system back to a working state if a lockup occurs.
+
+This is tested by using the Linux kernel's `test_lockup` module to
+inject a hard lockup (i.e., blocking servicing of all interrupts) on
+all CPU cores that lasts for twice as long as the watchdog's reported
+timeout.
+
+"""
+import base64
+import infamy
+import json
+import subprocess
+import time
+
+with infamy.Test() as test:
+    with test.step("Set up topology and attach to target DUT"):
+        env = infamy.Env()
+        target = env.attach("target", "mgmt")
+        tgtssh = env.attach("target", "mgmt", "ssh")
+
+    with test.step("Verify the presence of a watchdog device"):
+        wctl = tgtssh.run(["watchdogctl"], stdout=subprocess.PIPE)
+        conf = json.loads(wctl.stdout)
+
+        dogs = [ dog for dog in conf.get("device", []) if dog.get("name", "") == "/dev/watchdog" ]
+        if len(dogs) < 1:
+            test.fail("No watchdog device available")
+        else:
+            dog = dogs[0]
+
+    print(f"Found {dog['name']} ({dog['identity']}), timeout:{dog['timeout']}s")
+
+    with test.step("Verify the presence of the test_lockup module"):
+        if tgtssh.run(["modprobe", "-q", "-n", "test_lockup"]).returncode != 0:
+            test.fail("test_lockup module is not available")
+
+    with test.step("Trigger a hard lockup on all CPU cores"):
+        tgtssh.runsh(f"""
+        lockup()
+        {{
+            # Give the SSH session some time to properly shut down
+            sleep 3
+
+            sudo modprobe test_lockup \
+                disable_irq=1 \
+                all_cpus=1 \
+                time_secs={dog['timeout'] * 2}
+        }}
+
+        lockup </dev/null &>/dev/null &
+        """)
+
+    with test.step("Wait for the watchdog to trip"):
+        time.sleep(dog["timeout"])
+
+    with test.step("Verify that the system reboots"):
+        infamy.util.wait_boot(target, env)
+
+    test.succeed()

test/case/hardware/watchdog/topology.dot

@@ -0,0 +1,23 @@
+graph "1x1" {
+	layout="neato";
+	overlap="false";
+	esep="+80";
+
+        node [shape=record, fontname="DejaVu Sans Mono, Book"];
+	edge [color="cornflowerblue", penwidth="2", fontname="DejaVu Serif, Book"];
+
+	host [
+	    label="host | { <mgmt> mgmt }",
+	    pos="0,12!",
+	    requires="controller",
+	];
+
+        target [
+	    label="{ <mgmt> mgmt } | target",
+	    pos="10,12!",
+
+	    requires="infix watchdog",
+	];
+
+	host:mgmt -- target:mgmt [requires="mgmt", color="lightgray"]
+}

test/infamy/ssh.py

@@ -1,7 +1,7 @@
 import subprocess
 
 from dataclasses import dataclass
-from . import env
+from . import env, netutil, util
 
 
 @dataclass
@@ -15,6 +15,9 @@ class Location:
 import subprocess
 import os
 
+def ssh_syn(addr, port=22):
+    return netutil.tcp_port_is_open(addr, port)
+
 def fetch_file(remote_user, remote_address, remote_file, local_file, key_file, check=False, remove=False):
     """
     Fetches a file over SSH using scp and the provided private key.
@@ -62,9 +65,12 @@ def fetch_file(remote_user, remote_address, remote_file, local_file, key_file, c
             print(f"Error removing fetched file {local_file}: {e}")
 
 class Device(object):
-    def __init__(self, name: str, location: Location):
+    def __init__(self, name: str, location: Location, wait: bool=True):
         self.name = name
         self.location = location
+        if wait:
+            util.until(lambda: ssh_syn(location.host, location.port))
+
     def __str__(self):
         nm = f"{self.name}"
         if env.ENV.ltop:

test/virt/dual/topology.dot.in

@@ -24,7 +24,7 @@ graph "dual" {
         dut1 [
 	    label="{ <e1> e1 | <e2> e2 | <e3> e3 } | dut1 | { <e4> e4 | <e5> e5 | <e6> e6 }",
 	    pos="10,18!",
-	    provides="infix",
+	    provides="infix watchdog",
 	    expected_boot="primary",
 	    qn_console=9001,
 	    qn_mem="384M",
@@ -33,7 +33,7 @@ graph "dual" {
         dut2 [
 	    label="{ <e1> e1 | <e2> e2 | <e3> e3 } | dut2 | { <e4> e4 | <e5> e5 | <e6> e6 }",
 	    pos="10,12!",
-	    provides="infix",
+	    provides="infix watchdog",
 	    expected_boot="primary",
 	    qn_console=9002,
 	    qn_mem="384M",

test/virt/quad/topology.dot.in

@@ -23,7 +23,7 @@ graph "quad" {
         dut1 [
 	    label="{ <e1> e1 | <e2> e2 | <e3> e3 | <e4> e4 } | dut1 | { <e5> e5 | <e6> e6 | <e7> e7 | <e8> e8}",
 	    pos="10,30!",
-	    provides="infix",
+	    provides="infix watchdog",
 	    expected_boot="primary",
 	    qn_console=9001,
 	    qn_mem="384M",
@@ -32,7 +32,7 @@ graph "quad" {
         dut2 [
 	    label="{ <e1> e1 | <e2> e2 | <e3> e3 | <e4> e4 } | dut2 | { <e5> e5 | <e6> e6  | <e7> e7 | <e8> e8}",
 	    pos="0,20!",
-	    provides="infix",
+	    provides="infix watchdog",
 	    expected_boot="primary",
 	    qn_console=9002,
 	    qn_mem="384M",
@@ -41,7 +41,7 @@ graph "quad" {
 	dut3 [
 	    label="{ <e1> e1 | <e2> e2 | <e3> e3 | <e4> e4 } | dut3 | { <e5> e5 | <e6> e6  | <e7> e7 | <e8> e8}",
 	    pos="0,10!",
-	    provides="infix",
+	    provides="infix watchdog",
 	    expected_boot="primary",
 	    qn_console=9003,
 	    qn_mem="384M",
@@ -51,7 +51,7 @@ graph "quad" {
 	dut4 [
 	    label="{ <e1> e1 | <e2> e2 | <e3> e3 | <e4> e4 } | dut4 | { <e5> e5 | <e6> e6  | <e7> e7 | <e8> e8}",
 	    pos="10,0!",
-	    provides="infix",
+	    provides="infix watchdog",
 	    expected_boot="primary",
 	    qn_console=9004,
 	    qn_mem="384M",