diff --git a/apps/web/src/components/DeviceInfoPanel.tsx b/apps/web/src/components/DeviceInfoPanel.tsx index ca517981b..5d148bdd7 100644 --- a/apps/web/src/components/DeviceInfoPanel.tsx +++ b/apps/web/src/components/DeviceInfoPanel.tsx @@ -77,6 +77,8 @@ export const DeviceInfoPanel = ({ case "configuring": case "disconnecting": return "bg-amber-500"; + case "warning": + return "bg-amber-500"; case "error": return "bg-red-500"; default: diff --git a/apps/web/src/core/connections/heartbeat.test.ts b/apps/web/src/core/connections/heartbeat.test.ts new file mode 100644 index 000000000..a6a0edb3c --- /dev/null +++ b/apps/web/src/core/connections/heartbeat.test.ts @@ -0,0 +1,162 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { useDeviceStore } from "@core/stores/deviceStore"; +import type { MeshDevice } from "@meshtastic/sdk"; +import { startMaintenanceHeartbeat, stopHeartbeat } from "./heartbeat.ts"; + +function mockMeshDevice( + fails: number, +): MeshDevice & { callCount: () => number } { + let count = 0; + const md = { + heartbeat: vi.fn(() => { + count += 1; + if (count <= fails) return Promise.reject(new Error(`fail ${count}`)); + return Promise.resolve(0); + }), + } as unknown as MeshDevice & { callCount: () => number }; + (md as any).callCount = () => count; + return md as any; +} + +describe("heartbeat", () => { + beforeEach(() => { + vi.useFakeTimers(); + // Reset store + const store = useDeviceStore.getState(); + // Ensure a connection exists + if (!store.savedConnections.find((c) => c.id === 999)) { + store.addSavedConnection({ + id: 999, + type: "http", + name: "test", + url: "http://192.168.1.3", + status: "configured", + createdAt: Date.now(), + } as any); + } else { + store.updateSavedConnection(999, { + status: "configured", + error: undefined, + }); + } + stopHeartbeat(999); + }); + + it("flips to warning after 3 consecutive failures", async () => { + const md = mockMeshDevice(10); // always fail + startMaintenanceHeartbeat(999, md); + + // One interval (5min) fails -> schedules retry (~1s), retry fails -> retry (~2s), retry fails -> warning + await vi.advanceTimersByTimeAsync(5 * 60 * 1000); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(5000); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(5000); + await vi.advanceTimersByTimeAsync(0); + + const conn = useDeviceStore + .getState() + .savedConnections.find((c) => c.id === 999); + expect(conn?.status).toBe("warning"); + expect(conn?.error).toMatch(/Heartbeat failed/); + + stopHeartbeat(999); + vi.useRealTimers(); + }); + + it("recovers from warning on success", async () => { + const md = mockMeshDevice(3); // fail 3, then succeed + startMaintenanceHeartbeat(999, md); + + await vi.advanceTimersByTimeAsync(5 * 60 * 1000); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(5000); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(5000); + await vi.advanceTimersByTimeAsync(0); + + let conn = useDeviceStore + .getState() + .savedConnections.find((c) => c.id === 999); + expect(conn?.status).toBe("warning"); + + // Next interval should succeed and clear warning + await vi.advanceTimersByTimeAsync(5 * 60 * 1000); + await vi.advanceTimersByTimeAsync(0); + + conn = useDeviceStore.getState().savedConnections.find((c) => c.id === 999); + expect(conn?.status).toBe("configured"); + expect(conn?.error).toBeUndefined(); + + stopHeartbeat(999); + vi.useRealTimers(); + }); + + it("clears warning when a restarted session succeeds", async () => { + const failing = mockMeshDevice(10); + startMaintenanceHeartbeat(999, failing); + + await vi.advanceTimersByTimeAsync(5 * 60 * 1000); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(5000); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(5000); + await vi.advanceTimersByTimeAsync(0); + + expect( + useDeviceStore.getState().savedConnections.find((c) => c.id === 999) + ?.status, + ).toBe("warning"); + + stopHeartbeat(999); + const recovered = mockMeshDevice(0); + startMaintenanceHeartbeat(999, recovered); + + await vi.advanceTimersByTimeAsync(5 * 60 * 1000); + await vi.advanceTimersByTimeAsync(0); + + const conn = useDeviceStore + .getState() + .savedConnections.find((c) => c.id === 999); + expect(conn?.status).toBe("configured"); + expect(conn?.error).toBeUndefined(); + + stopHeartbeat(999); + vi.useRealTimers(); + }); + + it("ignores late failure from stopped session after restart", async () => { + let rejectA!: (e: Error) => void; + const pendingA = new Promise((_, rej) => { + rejectA = rej; + }); + const mdA = { heartbeat: vi.fn(() => pendingA) } as unknown as MeshDevice; + + startMaintenanceHeartbeat(999, mdA); + await vi.advanceTimersByTimeAsync(5 * 60 * 1000); + await vi.advanceTimersByTimeAsync(0); + stopHeartbeat(999); + const mdB = mockMeshDevice(0); + useDeviceStore + .getState() + .updateSavedConnection(999, { status: "configured", error: undefined }); + startMaintenanceHeartbeat(999, mdB); + + rejectA(new Error("stale fail")); + await vi.advanceTimersByTimeAsync(0); + + let conn = useDeviceStore + .getState() + .savedConnections.find((c) => c.id === 999); + expect(conn?.status).toBe("configured"); + expect(mdB.heartbeat).not.toHaveBeenCalled(); + + await vi.advanceTimersByTimeAsync(5 * 60 * 1000); + await vi.advanceTimersByTimeAsync(0); + conn = useDeviceStore.getState().savedConnections.find((c) => c.id === 999); + expect(conn?.status).toBe("configured"); + + stopHeartbeat(999); + vi.useRealTimers(); + }); +}); diff --git a/apps/web/src/core/connections/heartbeat.ts b/apps/web/src/core/connections/heartbeat.ts index becaac343..e074ec1f2 100644 --- a/apps/web/src/core/connections/heartbeat.ts +++ b/apps/web/src/core/connections/heartbeat.ts @@ -1,10 +1,25 @@ import type { ConnectionId } from "@core/stores/deviceStore/types"; +import { useDeviceStore } from "@core/stores/deviceStore"; import type { MeshDevice } from "@meshtastic/sdk"; const HEARTBEAT_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes (post-config) const CONFIG_HEARTBEAT_INTERVAL_MS = 5_000; // 5s (during initial config) +const MAX_CONSECUTIVE_FAILURES = 3; +const BASE_BACKOFF_MS = 1_000; +const MAX_BACKOFF_MS = 30_000; + const heartbeats = new Map>(); +const failures = new Map(); +const retryTimers = new Map>(); +const meshDevices = new Map(); +const generations = new Map(); + +function backoffDelay(attempt: number): number { + // Full jitter: random(0, min(cap, base * 2^(attempt-1))) — AWS best practice + const capped = Math.min(MAX_BACKOFF_MS, BASE_BACKOFF_MS * 2 ** (attempt - 1)); + return Math.random() * capped; +} /** * Stops + clears any active heartbeat for the connection. Safe to call when @@ -12,9 +27,79 @@ const heartbeats = new Map>(); */ export function stopHeartbeat(id: ConnectionId): void { const h = heartbeats.get(id); - if (!h) return; - clearInterval(h); - heartbeats.delete(id); + if (h) { + clearInterval(h); + heartbeats.delete(id); + } + const rt = retryTimers.get(id); + if (rt) { + clearTimeout(rt); + retryTimers.delete(id); + } + failures.delete(id); + meshDevices.delete(id); + // Bump generation so any in-flight heartbeat that settles late is ignored + generations.set(id, (generations.get(id) ?? 0) + 1); +} + +function handleHeartbeatResult( + id: ConnectionId, + gen: number, + success: boolean, + error?: unknown, + expectedStatus: "configuring" | "configured" = "configured", +): void { + // Ignore stale callbacks from a previous start/stop generation + if ((generations.get(id) ?? 0) !== gen) return; + + if (success) { + failures.set(id, 0); + // Clear warning whenever the current connection recovers, including after + // a restart that reset the in-memory failure counter. + const conn = useDeviceStore + .getState() + .savedConnections.find((c) => c.id === id); + if (conn?.status === "warning") { + useDeviceStore.getState().updateSavedConnection(id, { + status: expectedStatus, + error: undefined, + }); + } + return; + } + + const count = (failures.get(id) ?? 0) + 1; + failures.set(id, count); + console.warn( + `[heartbeat] ${expectedStatus} heartbeat failed (${count}/${MAX_CONSECUTIVE_FAILURES}):`, + error, + ); + + if (count >= MAX_CONSECUTIVE_FAILURES) { + useDeviceStore.getState().updateSavedConnection(id, { + status: "warning", + error: `Heartbeat failed ${count} times — device may be unreachable`, + }); + return; + } + + // Schedule a one-off retry with backoff, not waiting for next interval + const delay = backoffDelay(count); + const existing = retryTimers.get(id); + if (existing) clearTimeout(existing); + const md = meshDevices.get(id); + if (!md) return; + const timer = setTimeout(() => { + retryTimers.delete(id); + // Re-check generation before firing the retry + if ((generations.get(id) ?? 0) !== gen) return; + md.heartbeat() + .then(() => + handleHeartbeatResult(id, gen, true, undefined, expectedStatus), + ) + .catch((e) => handleHeartbeatResult(id, gen, false, e, expectedStatus)); + }, delay); + retryTimers.set(id, timer); } /** @@ -26,10 +111,19 @@ export function startConfigHeartbeat( meshDevice: MeshDevice, ): void { stopHeartbeat(id); + failures.set(id, 0); + meshDevices.set(id, meshDevice); + const gen = (generations.get(id) ?? 0) + 1; + generations.set(id, gen); const intervalId = setInterval(() => { - meshDevice.heartbeat().catch((error) => { - console.warn("[heartbeat] config heartbeat failed:", error); - }); + meshDevice + .heartbeat() + .then(() => + handleHeartbeatResult(id, gen, true, undefined, "configuring"), + ) + .catch((error) => + handleHeartbeatResult(id, gen, false, error, "configuring"), + ); }, CONFIG_HEARTBEAT_INTERVAL_MS); heartbeats.set(id, intervalId); } @@ -42,10 +136,17 @@ export function startMaintenanceHeartbeat( meshDevice: MeshDevice, ): void { stopHeartbeat(id); + failures.set(id, 0); + meshDevices.set(id, meshDevice); + const gen = (generations.get(id) ?? 0) + 1; + generations.set(id, gen); const intervalId = setInterval(() => { - meshDevice.heartbeat().catch((error) => { - console.warn("[heartbeat] maintenance heartbeat failed:", error); - }); + meshDevice + .heartbeat() + .then(() => handleHeartbeatResult(id, gen, true, undefined, "configured")) + .catch((error) => + handleHeartbeatResult(id, gen, false, error, "configured"), + ); }, HEARTBEAT_INTERVAL_MS); heartbeats.set(id, intervalId); } diff --git a/apps/web/src/core/stores/deviceStore/types.ts b/apps/web/src/core/stores/deviceStore/types.ts index d965b0e8a..e255d9501 100644 --- a/apps/web/src/core/stores/deviceStore/types.ts +++ b/apps/web/src/core/stores/deviceStore/types.ts @@ -43,7 +43,8 @@ export type ConnectionStatus = | "configuring" | "configured" | "online" - | "error"; + | "error" + | "warning"; export type Connection = { id: ConnectionId;