fix(web): surface heartbeat failures with backoff and warning state

apps/web/src/components/DeviceInfoPanel.tsx

@@ -77,6 +77,8 @@ export const DeviceInfoPanel = ({
       case "configuring":
       case "disconnecting":
         return "bg-amber-500";
+      case "warning":
+        return "bg-amber-500";
       case "error":
         return "bg-red-500";
       default:

apps/web/src/core/connections/heartbeat.test.ts

@@ -0,0 +1,162 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { useDeviceStore } from "@core/stores/deviceStore";
+import type { MeshDevice } from "@meshtastic/sdk";
+import { startMaintenanceHeartbeat, stopHeartbeat } from "./heartbeat.ts";
+
+function mockMeshDevice(
+  fails: number,
+): MeshDevice & { callCount: () => number } {
+  let count = 0;
+  const md = {
+    heartbeat: vi.fn(() => {
+      count += 1;
+      if (count <= fails) return Promise.reject(new Error(`fail ${count}`));
+      return Promise.resolve(0);
+    }),
+  } as unknown as MeshDevice & { callCount: () => number };
+  (md as any).callCount = () => count;
+  return md as any;
+}
+
+describe("heartbeat", () => {
+  beforeEach(() => {
+    vi.useFakeTimers();
+    // Reset store
+    const store = useDeviceStore.getState();
+    // Ensure a connection exists
+    if (!store.savedConnections.find((c) => c.id === 999)) {
+      store.addSavedConnection({
+        id: 999,
+        type: "http",
+        name: "test",
+        url: "http://192.168.1.3",
+        status: "configured",
+        createdAt: Date.now(),
+      } as any);
+    } else {
+      store.updateSavedConnection(999, {
+        status: "configured",
+        error: undefined,
+      });
+    }
+    stopHeartbeat(999);
+  });
+
+  it("flips to warning after 3 consecutive failures", async () => {
+    const md = mockMeshDevice(10); // always fail
+    startMaintenanceHeartbeat(999, md);
+
+    // One interval (5min) fails -> schedules retry (~1s), retry fails -> retry (~2s), retry fails -> warning
+    await vi.advanceTimersByTimeAsync(5 * 60 * 1000);
+    await vi.advanceTimersByTimeAsync(0);
+    await vi.advanceTimersByTimeAsync(5000);
+    await vi.advanceTimersByTimeAsync(0);
+    await vi.advanceTimersByTimeAsync(5000);
+    await vi.advanceTimersByTimeAsync(0);
+
+    const conn = useDeviceStore
+      .getState()
+      .savedConnections.find((c) => c.id === 999);
+    expect(conn?.status).toBe("warning");
+    expect(conn?.error).toMatch(/Heartbeat failed/);
+
+    stopHeartbeat(999);
+    vi.useRealTimers();
+  });
+
+  it("recovers from warning on success", async () => {
+    const md = mockMeshDevice(3); // fail 3, then succeed
+    startMaintenanceHeartbeat(999, md);
+
+    await vi.advanceTimersByTimeAsync(5 * 60 * 1000);
+    await vi.advanceTimersByTimeAsync(0);
+    await vi.advanceTimersByTimeAsync(5000);
+    await vi.advanceTimersByTimeAsync(0);
+    await vi.advanceTimersByTimeAsync(5000);
+    await vi.advanceTimersByTimeAsync(0);
+
+    let conn = useDeviceStore
+      .getState()
+      .savedConnections.find((c) => c.id === 999);
+    expect(conn?.status).toBe("warning");
+
+    // Next interval should succeed and clear warning
+    await vi.advanceTimersByTimeAsync(5 * 60 * 1000);
+    await vi.advanceTimersByTimeAsync(0);
+
+    conn = useDeviceStore.getState().savedConnections.find((c) => c.id === 999);
+    expect(conn?.status).toBe("configured");
+    expect(conn?.error).toBeUndefined();
+
+    stopHeartbeat(999);
+    vi.useRealTimers();
+  });
+
+  it("clears warning when a restarted session succeeds", async () => {
+    const failing = mockMeshDevice(10);
+    startMaintenanceHeartbeat(999, failing);
+
+    await vi.advanceTimersByTimeAsync(5 * 60 * 1000);
+    await vi.advanceTimersByTimeAsync(0);
+    await vi.advanceTimersByTimeAsync(5000);
+    await vi.advanceTimersByTimeAsync(0);
+    await vi.advanceTimersByTimeAsync(5000);
+    await vi.advanceTimersByTimeAsync(0);
+
+    expect(
+      useDeviceStore.getState().savedConnections.find((c) => c.id === 999)
+        ?.status,
+    ).toBe("warning");
+
+    stopHeartbeat(999);
+    const recovered = mockMeshDevice(0);
+    startMaintenanceHeartbeat(999, recovered);
+
+    await vi.advanceTimersByTimeAsync(5 * 60 * 1000);
+    await vi.advanceTimersByTimeAsync(0);
+
+    const conn = useDeviceStore
+      .getState()
+      .savedConnections.find((c) => c.id === 999);
+    expect(conn?.status).toBe("configured");
+    expect(conn?.error).toBeUndefined();
+
+    stopHeartbeat(999);
+    vi.useRealTimers();
+  });
+
+  it("ignores late failure from stopped session after restart", async () => {
+    let rejectA!: (e: Error) => void;
+    const pendingA = new Promise<number>((_, rej) => {
+      rejectA = rej;
+    });
+    const mdA = { heartbeat: vi.fn(() => pendingA) } as unknown as MeshDevice;
+
+    startMaintenanceHeartbeat(999, mdA);
+    await vi.advanceTimersByTimeAsync(5 * 60 * 1000);
+    await vi.advanceTimersByTimeAsync(0);
+    stopHeartbeat(999);
+    const mdB = mockMeshDevice(0);
+    useDeviceStore
+      .getState()
+      .updateSavedConnection(999, { status: "configured", error: undefined });
+    startMaintenanceHeartbeat(999, mdB);
+
+    rejectA(new Error("stale fail"));
+    await vi.advanceTimersByTimeAsync(0);
+
+    let conn = useDeviceStore
+      .getState()
+      .savedConnections.find((c) => c.id === 999);
+    expect(conn?.status).toBe("configured");
+    expect(mdB.heartbeat).not.toHaveBeenCalled();
+
+    await vi.advanceTimersByTimeAsync(5 * 60 * 1000);
+    await vi.advanceTimersByTimeAsync(0);
+    conn = useDeviceStore.getState().savedConnections.find((c) => c.id === 999);
+    expect(conn?.status).toBe("configured");
+
+    stopHeartbeat(999);
+    vi.useRealTimers();
+  });
+});

apps/web/src/core/connections/heartbeat.ts

@@ -1,20 +1,105 @@
 import type { ConnectionId } from "@core/stores/deviceStore/types";
+import { useDeviceStore } from "@core/stores/deviceStore";
 import type { MeshDevice } from "@meshtastic/sdk";
 
 const HEARTBEAT_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes (post-config)
 const CONFIG_HEARTBEAT_INTERVAL_MS = 5_000; // 5s (during initial config)
 
+const MAX_CONSECUTIVE_FAILURES = 3;
+const BASE_BACKOFF_MS = 1_000;
+const MAX_BACKOFF_MS = 30_000;
+
 const heartbeats = new Map<ConnectionId, ReturnType<typeof setInterval>>();
+const failures = new Map<ConnectionId, number>();
+const retryTimers = new Map<ConnectionId, ReturnType<typeof setTimeout>>();
+const meshDevices = new Map<ConnectionId, MeshDevice>();
+const generations = new Map<ConnectionId, number>();
+
+function backoffDelay(attempt: number): number {
+  // Full jitter: random(0, min(cap, base * 2^(attempt-1))) — AWS best practice
+  const capped = Math.min(MAX_BACKOFF_MS, BASE_BACKOFF_MS * 2 ** (attempt - 1));
+  return Math.random() * capped;
+}
 
 /**
  * Stops + clears any active heartbeat for the connection. Safe to call when
  * no heartbeat is running.
  */
 export function stopHeartbeat(id: ConnectionId): void {
   const h = heartbeats.get(id);
-  if (!h) return;
-  clearInterval(h);
-  heartbeats.delete(id);
+  if (h) {
+    clearInterval(h);
+    heartbeats.delete(id);
+  }
+  const rt = retryTimers.get(id);
+  if (rt) {
+    clearTimeout(rt);
+    retryTimers.delete(id);
+  }
+  failures.delete(id);
+  meshDevices.delete(id);
+  // Bump generation so any in-flight heartbeat that settles late is ignored
+  generations.set(id, (generations.get(id) ?? 0) + 1);
+}
+
+function handleHeartbeatResult(
+  id: ConnectionId,
+  gen: number,
+  success: boolean,
+  error?: unknown,
+  expectedStatus: "configuring" | "configured" = "configured",
+): void {
+  // Ignore stale callbacks from a previous start/stop generation
+  if ((generations.get(id) ?? 0) !== gen) return;
+
+  if (success) {
+    failures.set(id, 0);
+    // Clear warning whenever the current connection recovers, including after
+    // a restart that reset the in-memory failure counter.
+    const conn = useDeviceStore
+      .getState()
+      .savedConnections.find((c) => c.id === id);
+    if (conn?.status === "warning") {
+      useDeviceStore.getState().updateSavedConnection(id, {
+        status: expectedStatus,
+        error: undefined,
+      });
+    }
+    return;
+  }
+
+  const count = (failures.get(id) ?? 0) + 1;
+  failures.set(id, count);
+  console.warn(
+    `[heartbeat] ${expectedStatus} heartbeat failed (${count}/${MAX_CONSECUTIVE_FAILURES}):`,
+    error,
+  );
+
+  if (count >= MAX_CONSECUTIVE_FAILURES) {
+    useDeviceStore.getState().updateSavedConnection(id, {
+      status: "warning",
+      error: `Heartbeat failed ${count} times — device may be unreachable`,
+    });
+    return;
+  }
+
+  // Schedule a one-off retry with backoff, not waiting for next interval
+  const delay = backoffDelay(count);
+  const existing = retryTimers.get(id);
+  if (existing) clearTimeout(existing);
+  const md = meshDevices.get(id);
+  if (!md) return;
+  const timer = setTimeout(() => {
+    retryTimers.delete(id);
+    // Re-check generation before firing the retry
+    if ((generations.get(id) ?? 0) !== gen) return;
+    md.heartbeat()
+      .then(() =>
+        handleHeartbeatResult(id, gen, true, undefined, expectedStatus),
+      )
+      .catch((e) => handleHeartbeatResult(id, gen, false, e, expectedStatus));
+  }, delay);
+  retryTimers.set(id, timer);
 }
 
 /**
@@ -26,10 +111,19 @@ export function startConfigHeartbeat(
   meshDevice: MeshDevice,
 ): void {
   stopHeartbeat(id);
+  failures.set(id, 0);
+  meshDevices.set(id, meshDevice);
+  const gen = (generations.get(id) ?? 0) + 1;
+  generations.set(id, gen);
   const intervalId = setInterval(() => {
-    meshDevice.heartbeat().catch((error) => {
-      console.warn("[heartbeat] config heartbeat failed:", error);
-    });
+    meshDevice
+      .heartbeat()
+      .then(() =>
+        handleHeartbeatResult(id, gen, true, undefined, "configuring"),
+      )
+      .catch((error) =>
+        handleHeartbeatResult(id, gen, false, error, "configuring"),
+      );
   }, CONFIG_HEARTBEAT_INTERVAL_MS);
   heartbeats.set(id, intervalId);
 }
@@ -42,10 +136,17 @@ export function startMaintenanceHeartbeat(
   meshDevice: MeshDevice,
 ): void {
   stopHeartbeat(id);
+  failures.set(id, 0);
+  meshDevices.set(id, meshDevice);
+  const gen = (generations.get(id) ?? 0) + 1;
+  generations.set(id, gen);
   const intervalId = setInterval(() => {
-    meshDevice.heartbeat().catch((error) => {
-      console.warn("[heartbeat] maintenance heartbeat failed:", error);
-    });
+    meshDevice
+      .heartbeat()
+      .then(() => handleHeartbeatResult(id, gen, true, undefined, "configured"))
+      .catch((error) =>
+        handleHeartbeatResult(id, gen, false, error, "configured"),
+      );
   }, HEARTBEAT_INTERVAL_MS);
   heartbeats.set(id, intervalId);
 }

apps/web/src/core/stores/deviceStore/types.ts

@@ -43,7 +43,8 @@ export type ConnectionStatus =
   | "configuring"
   | "configured"
   | "online"
-  | "error";
+  | "error"
+  | "warning";
 
 export type Connection = {
   id: ConnectionId;