From eb757136b94cff3f1761a5d134bd4a1b638d4b79 Mon Sep 17 00:00:00 2001 From: ProjectZero02 <331545278+ProjectZero02@users.noreply.github.com> Date: Wed, 28 Oct 2026 06:10:16 +0000 Subject: [PATCH 1/2] Plan phase 3: hang detection and events Co-Authored-By: Claude Opus 5.5 --- docs/plans/recovery-layer.md | 111 ++++++++++++++++++++++++++++++++--- 1 file changed, 103 insertions(+), 8 deletions(-) diff --git a/docs/plans/recovery-layer.md b/docs/plans/recovery-layer.md index e9365f2..cfa9caa 100644 --- a/docs/plans/recovery-layer.md +++ b/docs/plans/recovery-layer.md @@ -138,14 +138,109 @@ Test: ## Phase 3: hang detection and events -Build: the hub watches each machine with a KVM. A machine is **hung** when, for N minutes (default -3): its helper misses heartbeats, `tailscale ping` fails, and the KVM reads the power LED on. -`off` = LED off; `on` = helper answers; `unknown` = not enough evidence. Extra evidence item: -"TCP accepts but nothing answers". State with evidence on the API; changes published as events -over SSE and to webhooks (HMAC-signed). - -Test: detector table tests with a fake clock and fake helper/ping/KVM; SSE and webhook delivery and -retry; signature check. +Today the hub asks each machine's helper and reads its KVM's power light every 2 minutes: `on` when +the helper answers, `off` when a trusted light reads off, `unknown` otherwise. Phase 3 tells a hung +machine apart from one that is booting, asleep, installing updates or just missing its helper, says +why, and tells whoever listens. + +What can look like a hang and isn't (each one has its own rule below): +- **Booting** or restarting: helper silent and light on for a few minutes. +- **Installing Windows updates** during a restart: light on, network down, helper stopped, for up + to an hour. +- **Asleep**: on most boards the light goes off or blinks, but on a Modern Standby PC (and some + boards in S3) it stays on while the network sleeps. +- **The helper stopped** (crashed, uninstalled, updating) while the PC is fine: the network still + answers. + +Build: +1. **Watcher.** The 2-minute loop becomes a 30-second watcher for every machine with a KVM here. Each + round collects evidence in parallel (8 s at most): + - `helper`: answers `hello` (two tries), or silent. + - `tcp`: the helper's port refuses (the OS is up and answered, the helper isn't running), accepts + but never answers (what a hung Windows does: the kernel still accepts), or no reply. + - `ping`: `tailscale ping --icmp -c 2 --timeout 3s --until-direct=false `. ICMP + goes through WireGuard to the machine's own network stack, so it needs more of the machine + alive than a disco or TSMP ping (both answered by tailscaled alone). *Checked:* a Windows 11 PC + with its default firewall answers it. Replies, no reply, or not available (the hub isn't on + Tailscale, or the machine has no Tailscale address). + - `led`: three readings 0.5 s apart: on, off, blinking (mixed), or unreadable; and whether the + light is trusted (phase 2's wiring). + - `tailnet`: whether Tailscale's control plane lists the node online. Shown, never decides: a + hung PC stays "online" there for minutes. + - The helper's last word: its last `hello` (Windows updates pending, a shutdown or restart + counting down) and its notes (next item). + Wiring is learned from the same readings as before (no more often: at most every 2 minutes). +2. **Notes from a helper to its hubs.** When its machine is about to sleep, shut down or restart, and + when it wakes, a helper tells every helper it holds a peer key for, best effort (1.5 s each): + `POST /v2/machines/{name}/note {"event": "sleep|shutdown|restart|resume", "updates": true}`. + Peer keys only, and only for the caller's own machine. Windows: the service accepts power events + (`PBT_APMSUSPEND`, `PBT_APMRESUMEAUTOMATIC`) and pre-shutdown, and reports whether updates are + pending. Linux: shutdown and restart. Not relied on: Windows may not send the suspend event on + Modern Standby PCs, and two seconds may not be enough, so item 3 covers sleep without it. +3. **Sleep facts.** A Windows helper's `hello` gains + `"sleep": {"s3": true, "modernStandby": false, "afterMinutes": 30}` from `GetPwrCapabilities` + and the active power plan's sleep timeout on AC (`PowerReadACValueIndex`, 0 = never); no + `powercfg` text, which is translated. The hub keeps the last one it saw. The first `sleep` note + teaches the hub what the light does while that machine sleeps (read 30 s after the note): stored + as `wiring.ledAsleep` = off, blinking or on. +4. **Detector**, a pure function of the evidence history (table-tested): + - `on`: the helper answers. + - `off`: a trusted light reads off. Blinking, or off after a `sleep` note: `off` with + `asleep: true`. + - `hung`: for `hungAfter` (default 3 minutes; owner setting, 2 to 30), every round saw the helper + silent, the port not refusing, ping not replying (or not available), and a trusted light + steadily on; and none of these hold: + - booting: 10 minutes after the hub woke, restarted, reset or forced off the machine, or after + its light came on; + - a shutdown or restart was coming (a note, or a countdown in its last `hello`): 10 minutes, + or 90 minutes when Windows updates were pending (they install with the light on and the + network down); + - it may be asleep with the light on: a `sleep` note since it last answered, or it can sleep + on its own (`afterMinutes` > 0) and its light while asleep is on or not learned yet. + - Anything that contradicts a hang (helper answers, ping replies, port refuses, light off or + blinking) starts the count again. An unreadable light neither counts nor resets. + - `unknown`: everything else, with a `reason`: `no_helper`, `led_not_trusted`, `booting`, + `updating`, `may_be_asleep`, `helper_stopped` (silent while the network answers), `checking` + (silent for less than `hungAfter`). + A machine without a helper, or without a trusted light, is never `hung`. + `GET /v2/machines` and each machine gain `since` (when the state began), `reason`, and + `evidence` (the last round: each item with its value and time). +5. **Events.** Each state change is an event: `{"id", "at", "machine", "state", "from", "reason", + "evidence"}`, plus `recovered` when a hung machine answers again. The last 500 are kept in + memory; `hung` and `recovered` also go to the activity log. + - `GET /v2/machines/events` (status scope): Server-Sent Events. Starts with a `snapshot` of every + machine; with `Last-Event-ID`, replays what came after it instead (a snapshot when that's no + longer held). A comment every 25 s keeps proxies open. The server's 150 s write timeout is + lifted for this stream only, with a 10 s deadline per write: a client that stops reading is + dropped. At most 16 streams. + - **Webhooks** (owner devices): `GET/POST /v2/webhooks`, `DELETE /v2/webhooks/{id}`, + `POST /v2/webhooks/{id}/test`. The URL is https, or http to a Tailscale or loopback address + (WireGuard already encrypts those). Event filter (default `hung`, `recovered`). The secret is + shown once when it's created and sealed at rest like KVM passwords. Each delivery is the event + as JSON with `X-PowerDeck-Event`, `X-PowerDeck-Delivery` (the event id), + `X-PowerDeck-Timestamp` and `X-PowerDeck-Signature: sha256=`. Any 2xx is delivered; otherwise retried after 10 s, 1 min and + 5 min, then dropped and logged. 10 s timeout, redirects not followed. The list shows each + hook's last result. + - `docs/protocol.md` gains "Machine states and events", with how to check a signature. +6. **App:** a hung machine shows as hung on its device page, with the evidence and since when. The + recovery card comes in phase 4. + +Test: +- Detector table tests with a fake clock: hung after `hungAfter` and not before; each + contradiction starts the count again; an unreadable light holds it; boot grace; shutdown and + restart notes; 90 minutes with updates; Modern Standby and a light that stays on in sleep; + blinking; no helper or untrusted light is never hung; recovered. +- Watcher end to end against the fake NanoKVM, a fake helper, a fake pinger, and real TCP + listeners: one that accepts and never answers, one that's closed. +- Notes: accepted from the machine's own peer key only. +- Sleep facts on the Windows CI runner (values logged); the power-event handler's mapping in a + unit test (a CI runner can't be put to sleep: the hardware checklist covers a real one). +- SSE: snapshot first, events in order, `Last-Event-ID` replay, heartbeat, stream limit, a client + that stops reading is dropped. +- Webhooks: signature checked by an independent implementation, retries on a fake clock, URL + rules, the secret never in a reply and sealed at rest, owner devices only. +- Android: Robolectric test for the hung state and its evidence on the device page. ## Phase 4: recovery with confirmation From 48362fe1be657d8504defd048dac4b96e2dd74bf Mon Sep 17 00:00:00 2001 From: ProjectZero02 <331545278+ProjectZero02@users.noreply.github.com> Date: Wed, 28 Oct 2026 06:42:49 +0000 Subject: [PATCH 2/2] Phase 3: hang detection and events The hub watches every machine with a KVM every 30 s (helper status, its port, an ICMP ping over Tailscale, three power-light readings) and tells a hung machine from one that's booting, shutting down, installing updates, asleep or missing its helper. Hung needs 3 minutes (2 to 30, owner setting) of silence with a wired light steadily on and nothing to excuse it. - Helpers tell their hubs before sleep, shutdown and restart and after a wake (POST /v2/note): Windows through PowerRegisterSuspendResumeNotification and the service's pre-shutdown notice, Linux when systemd is stopping. - Windows helpers report their sleep facts (S3, Modern Standby, idle timeout on AC); the hub learns what the light does while asleep. - State changes are events: GET /v2/machines/events (SSE with replay and heartbeats), and owner webhooks signed with HMAC-SHA256, with retries. - GET /v2/machines gains hung, reason, since and evidence. - App: a frozen computer says so on its page, with what the hub saw, and the phone notifies about frozen and recovered machines. - docs/protocol.md: Machine states and events. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/build.yml | 4 +- CHANGELOG.md | 11 + .../net/odysseylabs/powerdeck/data/Model.kt | 34 +- .../odysseylabs/powerdeck/data/PowerDeck.kt | 3 +- .../odysseylabs/powerdeck/ui/Components.kt | 33 +- .../odysseylabs/powerdeck/ui/DeviceScreen.kt | 35 ++ .../powerdeck/work/EventsWorker.kt | 6 + .../odysseylabs/powerdeck/ScreenshotTest.kt | 21 ++ .../net/odysseylabs/powerdeck/UiRulesTest.kt | 45 +++ docs/plans/recovery-layer.md | 33 +- docs/protocol.md | 149 +++++++- helper/cmd/powerdeck/main.go | 3 +- helper/cmd/powerdeck/main_linux.go | 12 + helper/cmd/powerdeck/main_windows.go | 23 +- helper/internal/app/app.go | 82 +++- helper/internal/fleet/fleet.go | 26 +- helper/internal/fleet/fleet_test.go | 35 ++ helper/internal/machines/detect.go | 207 ++++++++++ helper/internal/machines/detect_test.go | 166 ++++++++ helper/internal/machines/events.go | 118 ++++++ helper/internal/machines/machines.go | 77 ++-- helper/internal/machines/machines_test.go | 16 +- helper/internal/machines/watch.go | 303 +++++++++++++++ helper/internal/machines/watch_test.go | 307 +++++++++++++++ helper/internal/nanokvm/fakekvm/fakekvm.go | 34 +- helper/internal/server/server.go | 45 ++- helper/internal/server/server_test.go | 5 +- helper/internal/server/watch.go | 277 ++++++++++++++ helper/internal/server/watch_test.go | 356 ++++++++++++++++++ helper/internal/sleepinfo/sleepinfo.go | 27 ++ helper/internal/sleepinfo/sleepinfo_other.go | 9 + .../internal/sleepinfo/sleepinfo_windows.go | 75 ++++ .../sleepinfo/sleepinfo_windows_test.go | 40 ++ helper/internal/store/store.go | 96 ++++- helper/internal/tailscale/tailscale.go | 35 +- helper/internal/tailscale/tailscale_test.go | 2 +- helper/internal/webhook/webhook.go | 228 +++++++++++ helper/internal/webhook/webhook_test.go | 204 ++++++++++ helper/internal/winsvc/suspend_windows.go | 76 ++++ .../internal/winsvc/suspend_windows_test.go | 20 + helper/internal/winsvc/winsvc_windows.go | 26 +- 41 files changed, 3225 insertions(+), 79 deletions(-) create mode 100644 helper/internal/machines/detect.go create mode 100644 helper/internal/machines/detect_test.go create mode 100644 helper/internal/machines/events.go create mode 100644 helper/internal/machines/watch.go create mode 100644 helper/internal/machines/watch_test.go create mode 100644 helper/internal/server/watch.go create mode 100644 helper/internal/server/watch_test.go create mode 100644 helper/internal/sleepinfo/sleepinfo.go create mode 100644 helper/internal/sleepinfo/sleepinfo_other.go create mode 100644 helper/internal/sleepinfo/sleepinfo_windows.go create mode 100644 helper/internal/sleepinfo/sleepinfo_windows_test.go create mode 100644 helper/internal/webhook/webhook.go create mode 100644 helper/internal/webhook/webhook_test.go create mode 100644 helper/internal/winsvc/suspend_windows.go create mode 100644 helper/internal/winsvc/suspend_windows_test.go diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0fdf6f0..d852c47 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -152,8 +152,8 @@ jobs: - uses: actions/checkout@v7 - uses: actions/setup-go@v7 with: { go-version-file: helper/go.mod, cache-dependency-path: helper/go.sum } - - name: Unit tests for Windows-only code (DPAPI sealing, NanoKVM client, window access, PowerShell) - run: go test -count=1 -v -run . ./internal/secret/ ./internal/nanokvm/... ./internal/winsvc/ ./internal/setup/ + - name: Unit tests for Windows-only code (DPAPI sealing, NanoKVM client, window access, PowerShell, sleep settings, suspend notices) + run: go test -count=1 -v -run . ./internal/secret/ ./internal/nanokvm/... ./internal/winsvc/ ./internal/sleepinfo/ ./internal/setup/ - name: Build PowerDeck.exe run: | go install github.com/tc-hib/go-winres@v0.3.3 diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e0b71c..3e08478 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## 2.3.0 + +The hub now notices when a computer freezes, and says why it thinks so. + +**Update the helper on the hub (the always-on computer that drives your KVMs), the helpers on your computers, and the phone app.** A computer's helper before 2.3 still works, but can't tell the hub it's about to sleep or restart, so the hub is more careful before calling it frozen. + +- **Frozen or just busy?** Every 30 seconds the hub checks each computer with a KVM: does its PowerDeck answer, does it answer a ping, does its PowerDeck port turn connections away or swallow them, and is its power light on, off or blinking. A computer is called frozen only when, for 3 minutes in a row, nothing answers and the light stays on, and it isn't starting up (10 minutes after it's turned on, restarted or reset), shutting down or restarting (10 minutes; 90 when Windows is installing updates), or possibly asleep. A computer without the helper, or whose power light isn't known to be connected, is never called frozen. +- **Sleep is learned.** A Windows helper tells the hub how the PC sleeps (Modern Standby or classic, and after how long), and tells it just before it sleeps, shuts down or restarts, and when it wakes. The first time a computer sleeps, the hub learns what its power light does then (off, blinking or on). +- **App:** a frozen computer's page says **Frozen since** with the time, and shows what the hub saw. **Reset** is still one tap away. A computer that's starting up, installing updates or may be asleep says so instead of "On". +- **Events for assistants:** `GET /v2/machines/events` streams every computer's state as it changes (Server-Sent Events, with replay after a reconnect), and owners can add **webhooks** that receive `hung` and `recovered` events, signed with HMAC-SHA256 and retried. Machines in `GET /v2/machines` gain `hung`, a `reason`, `since` and the `evidence`. `hungAfterMinutes` (2 to 30) changes the 3 minutes. Frozen and recovered computers are also in the hub's activity log, and the phone notifies about both (it checks about every 15 minutes). See [docs/protocol.md](docs/protocol.md#machine-states-and-events). + ## 2.2.3 - **PC window setup shows once.** The first-run setup was only marked done by **Done** on its last step, so closing the window part-way (the main screen has the same QR codes) started it over on every open. It's now marked done as soon as it's shown. diff --git a/android/app/src/main/java/net/odysseylabs/powerdeck/data/Model.kt b/android/app/src/main/java/net/odysseylabs/powerdeck/data/Model.kt index 0a71e4d..b9e4312 100644 --- a/android/app/src/main/java/net/odysseylabs/powerdeck/data/Model.kt +++ b/android/app/src/main/java/net/odysseylabs/powerdeck/data/Model.kt @@ -96,18 +96,42 @@ data class KvmWiring(val led: String, val power: String, val reset: String) { } } -/** A machine as a hub sees it (GET /v2/machines). [led] null: not read. */ -data class HubMachine(val name: String, val state: String, val led: Boolean?, val ledError: String?, val kvm: String?, val wiring: KvmWiring?, val actions: List) { +/** + * A machine as a hub sees it (GET /v2/machines). [led] null: not read. From 2.3 the hub's watch adds + * [state] "hung", a [reason] for "unknown", [since] and its last look ([evidence]). + */ +data class HubMachine( + val name: String, val state: String, val led: Boolean?, val ledError: String?, val kvm: String?, val wiring: KvmWiring?, val actions: List, + val reason: String? = null, val asleep: Boolean = false, val since: Instant? = null, val evidence: HubEvidence? = null, +) { + val hung get() = state == HUNG + companion object { + const val HUNG = "hung" fun from(o: JSONObject): HubMachine { val k = o.optJSONObject("kvm") return HubMachine(o.optString("name"), o.optString("state", "unknown"), if (o.has("led") && !o.isNull("led")) o.optBoolean("led") else null, o.optStringOrNull("ledError"), k?.optStringOrNull("name"), k?.optJSONObject("wiring")?.let(KvmWiring::from), - o.optJSONArray("actions")?.strings() ?: emptyList()) + o.optJSONArray("actions")?.strings() ?: emptyList(), + o.optStringOrNull("reason"), o.optBoolean("asleep"), o.instant("since"), o.optJSONObject("evidence")?.let(HubEvidence::from)) } } } +/** + * The hub's last look at a machine: its helper ("answers", "silent"), the helper's port ("refuses", + * "accepts", "no_reply"), a ping ("replies", "no_reply", "unavailable") and the power light ("on", + * "off", "blinking", "unreadable"). Null: not looked at (no helper). + */ +data class HubEvidence(val at: Instant?, val helper: String?, val port: String?, val ping: String?, val led: String?, val ledTrusted: Boolean) { + companion object { + fun from(o: JSONObject) = HubEvidence(o.instant("at"), o.optStringOrNull("helper"), o.optStringOrNull("port"), o.optStringOrNull("ping"), + o.optStringOrNull("led"), o.optBoolean("ledTrusted")) + } +} + +private fun JSONObject.instant(k: String) = optStringOrNull(k)?.let { runCatching { Instant.parse(it) }.getOrNull() } + /** What a hub did for an action: its message, and each step with the power light read. */ data class HubResult(val ok: Boolean, val code: String?, val message: String, val steps: List) { companion object { @@ -239,7 +263,6 @@ data class HelperUpdate( val autoAt: Instant? = null, val pausedUntil: Instant? = null, ) { companion object { - private fun JSONObject.instant(k: String) = optStringOrNull(k)?.let { runCatching { Instant.parse(it) }.getOrNull() } fun from(o: JSONObject) = HelperUpdate(o.optBoolean("available"), o.optStringOrNull("latest")?.ifEmpty { null }, o.optBoolean("installing"), o.optBoolean("auto", true), o.optStringOrNull("error")?.ifEmpty { null }, o.instant("autoAt"), o.instant("pausedUntil")) @@ -275,7 +298,8 @@ data class Schedule(val id: String, val action: String, val target: String?, val /** Live state of a device, as shown on its card. */ sealed interface DeviceState { data object Checking : DeviceState - data class On(val status: HelperStatus?) : DeviceState // status null when only a KVM LED says so + /** [status] null when only a KVM LED says so; [hub] is then what the hub that drives the KVM makes of it. */ + data class On(val status: HelperStatus?, val hub: HubMachine? = null) : DeviceState data object Off : DeviceState data class Problem(val message: String) : DeviceState } diff --git a/android/app/src/main/java/net/odysseylabs/powerdeck/data/PowerDeck.kt b/android/app/src/main/java/net/odysseylabs/powerdeck/data/PowerDeck.kt index 432f346..5abdcb6 100644 --- a/android/app/src/main/java/net/odysseylabs/powerdeck/data/PowerDeck.kt +++ b/android/app/src/main/java/net/odysseylabs/powerdeck/data/PowerDeck.kt @@ -367,7 +367,8 @@ class PowerDeck( when (ledOn(d)) { true -> { if (!d.hasHelper) seeLed(d) // a helper device is confirmed in helperState, with the helper agreeing - DeviceState.On(null) + // The hub watches it: frozen, booting, asleep… (read with the light, so no extra call) + DeviceState.On(null, if (it.hub != null) runCatching { hubMachine(d) }.getOrNull() else null) } false -> if (it.ledSeen || d.hasHelper) DeviceState.Off else DeviceState.Problem("Turn it on by hand once so PowerDeck can check its power light") diff --git a/android/app/src/main/java/net/odysseylabs/powerdeck/ui/Components.kt b/android/app/src/main/java/net/odysseylabs/powerdeck/ui/Components.kt index 51fba06..9f6e72f 100644 --- a/android/app/src/main/java/net/odysseylabs/powerdeck/ui/Components.kt +++ b/android/app/src/main/java/net/odysseylabs/powerdeck/ui/Components.kt @@ -62,8 +62,12 @@ import kotlinx.coroutines.delay import kotlinx.coroutines.launch import net.odysseylabs.powerdeck.data.Device import net.odysseylabs.powerdeck.data.DeviceState +import net.odysseylabs.powerdeck.data.HubMachine import java.time.Duration import java.time.Instant +import java.time.ZoneId +import java.time.format.DateTimeFormatter +import java.time.format.FormatStyle /** "Hold for 3 s to activate": how long Shut down and Restart have to be held. */ const val HOLD_MILLIS = 3000 @@ -110,7 +114,7 @@ fun describe(state: DeviceState?, now: Long): Pair = when (state) val left = Duration.between(Instant.ofEpochMilli(now), p.at).seconds.coerceAtLeast(0) "${if (p.action == "restart") "Restarting" else "Shutting down"} in ${left / 60}:${"%02d".format(left % 60)}" to Tone.Busy } - s == null -> "On" to Tone.On + s == null -> hubNote(state.hub, now) ?: ("On" to Tone.On) s.presence?.active == true -> "On · in use${s.presence.user?.let { " by $it" } ?: ""}" to Tone.On else -> "On · up ${uptime(s.uptimeSeconds)}" to Tone.On } @@ -128,6 +132,7 @@ fun note(state: DeviceState?, now: Long): Pair? = when (state) { val p = s?.pending when { p != null -> describe(state, now) + s == null -> hubNote(state.hub, now) s?.presence?.active == true -> "In use${s.presence.user?.let { " by $it" } ?: ""}" to Tone.On else -> null } @@ -135,6 +140,32 @@ fun note(state: DeviceState?, now: Long): Pair? = when (state) { else -> null } +/** + * What the hub that drives a machine's KVM makes of it while its helper is silent and its light is + * on, or null when it has nothing to add (see docs/protocol.md, Machine states and events). + */ +fun hubNote(m: HubMachine?, now: Long): Pair? = when { + m == null -> null + m.hung -> (m.since?.let { "Frozen since " + clockTime(it, now) } ?: "Frozen") to Tone.Warn + else -> when (m.reason) { + "booting" -> "Starting up" to Tone.Busy + "restarting" -> "Restarting" to Tone.Busy + "shutting_down" -> "Shutting down" to Tone.Busy + "updating" -> "Installing updates" to Tone.Busy + "may_be_asleep" -> "May be asleep" to Tone.Unknown + "helper_stopped" -> "Running, but PowerDeck on it isn't answering" to Tone.Warn + "checking" -> "Not answering" to Tone.Unknown + else -> null + } +} + +private val shortTime = DateTimeFormatter.ofLocalizedTime(FormatStyle.SHORT).withZone(ZoneId.systemDefault()) +private val shortDateTime = DateTimeFormatter.ofLocalizedDateTime(FormatStyle.SHORT).withZone(ZoneId.systemDefault()) + +/** A time of day, with the date when it isn't within the last 20 hours. */ +fun clockTime(at: Instant, now: Long): String = + if (Duration.between(at, Instant.ofEpochMilli(now)).toHours() < 20) shortTime.format(at) else shortDateTime.format(at) + @Composable fun toneColor(tone: Tone): Color = when (tone) { Tone.On -> OnGreen diff --git a/android/app/src/main/java/net/odysseylabs/powerdeck/ui/DeviceScreen.kt b/android/app/src/main/java/net/odysseylabs/powerdeck/ui/DeviceScreen.kt index 0ace22d..d8a8392 100644 --- a/android/app/src/main/java/net/odysseylabs/powerdeck/ui/DeviceScreen.kt +++ b/android/app/src/main/java/net/odysseylabs/powerdeck/ui/DeviceScreen.kt @@ -75,6 +75,7 @@ import net.odysseylabs.powerdeck.data.Device import net.odysseylabs.powerdeck.data.DeviceState import net.odysseylabs.powerdeck.data.Event import net.odysseylabs.powerdeck.data.HelperStatus +import net.odysseylabs.powerdeck.data.HubMachine import net.odysseylabs.powerdeck.data.KeyInfo import net.odysseylabs.powerdeck.data.Peer import net.odysseylabs.powerdeck.data.KvmWiring @@ -145,6 +146,35 @@ private fun Row2(title: String, sub: String? = null, icon: (@Composable () -> Un ) } +/** The hub's last look at [name]: each signal in words, so the owner can see why it's called frozen. */ +@Composable +private fun HubEvidenceSection(name: String, hubName: String, m: HubMachine) { + val e = m.evidence ?: return + Section("What $hubName sees") { + e.helper?.let { + Row2("PowerDeck on $name", if (it == "answers") "Answers" else "Not answering") + } + val network = listOfNotNull( + when (e.ping) { "replies" -> "Answers ping"; "no_reply" -> "No reply to ping"; else -> null }, + when (e.port) { + "refuses" -> "turns PowerDeck's connection away (it's up, PowerDeck isn't running)" + "accepts" -> "takes PowerDeck's connection but says nothing" + "no_reply" -> "nothing on PowerDeck's port" + else -> null + }, + ) + if (network.isNotEmpty()) Row2("Network", network.joinToString("; ").replaceFirstChar(Char::uppercase)) + Row2("Power light", when (e.led) { + "on" -> "On" + "off" -> "Off" + "blinking" -> "Blinking (asleep)" + else -> "Can't be read" + } + if (!e.ledTrusted) " · not known to be connected yet" else "") + // The card above already says what it makes of it. + e.at?.let { at -> Row2("Last checked " + clockTime(at, System.currentTimeMillis()), if (m.hung) "A reset brings it back; unsaved work is lost" else null) } + } +} + @OptIn(ExperimentalMaterial3Api::class, ExperimentalLayoutApi::class) @Composable fun DeviceScreen(d: Device, state: UiState, detail: Detail, a: DeviceActions, snackbar: SnackbarHostState) { @@ -238,6 +268,11 @@ fun DeviceScreen(d: Device, state: UiState, detail: Detail, a: DeviceActions, sn } } + // ---- What the hub sees, while the helper is silent and the light is on (frozen, booting…) + (st as? DeviceState.On)?.hub?.takeIf { status == null && it.evidence != null && (it.hung || it.reason != null) }?.let { m -> + HubEvidenceSection(d.name, hub?.name ?: "The hub", m) + } + // ---- PowerDeck helper update status?.update?.takeIf { it.available || it.installing }?.let { u -> Section("Update") { diff --git a/android/app/src/main/java/net/odysseylabs/powerdeck/work/EventsWorker.kt b/android/app/src/main/java/net/odysseylabs/powerdeck/work/EventsWorker.kt index be366ab..3c9fc8d 100644 --- a/android/app/src/main/java/net/odysseylabs/powerdeck/work/EventsWorker.kt +++ b/android/app/src/main/java/net/odysseylabs/powerdeck/work/EventsWorker.kt @@ -88,6 +88,7 @@ class EventsWorker(ctx: Context, params: WorkerParameters) : CoroutineWorker(ctx } private fun notable(e: Event, d: Device, helpers: Set): Boolean { + if (hubWatch(e)) return true if (d.me != null && (e.who == d.me || e.who.startsWith("${d.me} via "))) return false val verb = e.action.substringBefore(' ').lowercase() val target = e.action.substringAfter(' ', "").lowercase() @@ -97,6 +98,7 @@ class EventsWorker(ctx: Context, params: WorkerParameters) : CoroutineWorker(ctx } private fun describe(e: Event, d: Device): String { + if (hubWatch(e)) return e.action + "." // already a sentence: "Workstation looks frozen: …" val verb = e.action.substringBefore(' ') val target = e.action.substringAfter(' ', "").ifBlank { d.name } val what = when (verb) { @@ -113,6 +115,10 @@ class EventsWorker(ctx: Context, params: WorkerParameters) : CoroutineWorker(ctx } companion object { + /** A hub (2.3+) saw a machine with a KVM freeze, or answer again after that. */ + internal fun hubWatch(e: Event) = + e.result == "detected" && e.action.contains(" looks frozen") || e.result == "recovered" && e.action.endsWith(" answers again") + private const val NAME = "events" private const val CHANNEL = "activity" private const val UPDATES = "updates" diff --git a/android/app/src/test/java/net/odysseylabs/powerdeck/ScreenshotTest.kt b/android/app/src/test/java/net/odysseylabs/powerdeck/ScreenshotTest.kt index 1f5753d..55875ba 100644 --- a/android/app/src/test/java/net/odysseylabs/powerdeck/ScreenshotTest.kt +++ b/android/app/src/test/java/net/odysseylabs/powerdeck/ScreenshotTest.kt @@ -52,6 +52,8 @@ import net.odysseylabs.powerdeck.data.DeviceState import net.odysseylabs.powerdeck.data.Event import net.odysseylabs.powerdeck.data.HelperStatus import net.odysseylabs.powerdeck.data.HelperUpdate +import net.odysseylabs.powerdeck.data.HubEvidence +import net.odysseylabs.powerdeck.data.HubMachine import net.odysseylabs.powerdeck.data.KvmLink import net.odysseylabs.powerdeck.data.KvmWiring import net.odysseylabs.powerdeck.data.Peer @@ -269,6 +271,25 @@ class ScreenshotTest { compose.onRoot().captureRoboImage("build/outputs/roborazzi/device-frozen.png") } + /** The hub calls it hung: the device page says so, with what the hub saw, and still offers Reset. */ + @Test fun deviceHungOnHub() { + val ws = Device("ws", "Workstation", "100.101.1.11", pin = "w", os = "windows", mac = "02:00:5e:00:00:01", + kvm = KvmLink("nanokvm", "kvm-2.tail1234.ts.net", "kvm-2", "admin", ledSeen = true, hub = "pve", machine = "workstation", + wiring = KvmWiring("wired", "wired", "unknown"))) + val seen = java.time.Instant.now().minusSeconds(20) + val m = HubMachine("workstation", "hung", true, null, "kvm-2", ws.kvm!!.wiring, listOf("reset", "force-off"), + since = seen.minusSeconds(200), evidence = HubEvidence(seen, "silent", "accepts", "no_reply", "on", true)) + val state = all.copy(devices = all.devices + ws, states = all.states + ("ws" to DeviceState.On(null, m))) + shot("device-hung") { DeviceScreen(ws, state, Detail(events = emptyList()), DeviceActions(), SnackbarHostState()) } + compose.onNodeWithText("Frozen since", substring = true).assertIsDisplayed() + compose.onNodeWithText("A reset brings it back; unsaved work is lost").performScrollTo().assertIsDisplayed() + compose.onNodeWithText("Reset").assertIsDisplayed() + compose.onNodeWithText("What pve sees").performScrollTo().assertIsDisplayed() + compose.onNodeWithText("No reply to ping; takes PowerDeck's connection but says nothing").assertExists() + compose.onNodeWithText("Not answering").assertExists() + compose.onRoot().captureRoboImage("build/outputs/roborazzi/device-hung.png") + } + @Test fun settingsHub() { shot("settings-hub") { SettingsScreen(Settings(), SettingsActions(), hubs = listOf(pc, pve), showHub = true) } compose.onNodeWithText("Automatic: pve").assertExists() diff --git a/android/app/src/test/java/net/odysseylabs/powerdeck/UiRulesTest.kt b/android/app/src/test/java/net/odysseylabs/powerdeck/UiRulesTest.kt index 1591861..be24dce 100644 --- a/android/app/src/test/java/net/odysseylabs/powerdeck/UiRulesTest.kt +++ b/android/app/src/test/java/net/odysseylabs/powerdeck/UiRulesTest.kt @@ -2,24 +2,69 @@ package net.odysseylabs.powerdeck import net.odysseylabs.powerdeck.data.Device import net.odysseylabs.powerdeck.data.DeviceState +import net.odysseylabs.powerdeck.data.Event import net.odysseylabs.powerdeck.data.HelperStatus +import net.odysseylabs.powerdeck.data.HubMachine import com.google.zxing.BarcodeFormat import com.google.zxing.qrcode.QRCodeReader import com.google.zxing.qrcode.QRCodeWriter +import net.odysseylabs.powerdeck.ui.Tone import net.odysseylabs.powerdeck.ui.agentKeysFor import net.odysseylabs.powerdeck.ui.decodeLuma +import net.odysseylabs.powerdeck.ui.describe +import net.odysseylabs.powerdeck.ui.hubNote import net.odysseylabs.powerdeck.ui.uninstallBlocker +import net.odysseylabs.powerdeck.work.EventsWorker import org.json.JSONObject import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse import org.junit.Assert.assertNotNull import org.junit.Assert.assertNull import org.junit.Assert.assertTrue import org.junit.Test +import java.time.Instant class UiRulesTest { private fun status(json: String) = HelperStatus.from(JSONObject(json)) private val pc = Device("pc", "My PC", "100.64.0.1", pin = "x", os = "windows") + @Test fun parsesTheHubsVerdict() { + val m = HubMachine.from(JSONObject("""{"name":"workstation","state":"hung","led":true,"since":"2026-10-28T08:00:30Z", + "evidence":{"at":"2026-10-28T08:03:30Z","helper":"silent","port":"accepts","ping":"no_reply","led":"on","ledTrusted":true,"tailnetOnline":true}, + "kvm":{"name":"kvm-2","wiring":{"led":"wired","power":"wired","reset":"unknown"}},"actions":["reset","force-off"]}""")) + assertTrue(m.hung) + assertEquals(Instant.parse("2026-10-28T08:00:30Z"), m.since) + val e = m.evidence!! + assertEquals(listOf("silent", "accepts", "no_reply", "on"), listOf(e.helper, e.port, e.ping, e.led)) + assertTrue(e.ledTrusted) + assertEquals(Instant.parse("2026-10-28T08:03:30Z"), e.at) + + val now = Instant.parse("2026-10-28T08:05:00Z").toEpochMilli() + val (text, tone) = describe(DeviceState.On(null, m), now) + assertTrue(text, text.startsWith("Frozen since ")) + assertEquals(Tone.Warn, tone) + assertEquals("On" to Tone.On, describe(DeviceState.On(null), now)) + + // A hub before 2.3 has no watch: nothing to add. + val old = HubMachine.from(JSONObject("""{"name":"workstation","state":"unknown","led":true}""")) + assertNull(old.reason) + assertNull(old.since) + assertNull(old.evidence) + assertNull(hubNote(old, now)) + assertEquals("Installing updates" to Tone.Busy, hubNote(old.copy(reason = "updating"), now)) + assertEquals("May be asleep" to Tone.Unknown, hubNote(old.copy(reason = "may_be_asleep"), now)) + assertNull(hubNote(old.copy(reason = "led_not_trusted"), now)) + } + + /** The hub's log lines about a frozen machine are notified, whoever wrote them. */ + @Test fun frozenMachinesAreNotified() { + val at = Instant.parse("2026-10-28T08:03:30Z") + assertTrue(EventsWorker.hubWatch(Event(at, "pve", "Workstation looks frozen: nothing answers, power light on", "detected"))) + assertTrue(EventsWorker.hubWatch(Event(at, "pve", "Workstation answers again", "recovered"))) + assertFalse(EventsWorker.hubWatch(Event(at, "Pixel 9 Pro", "shutdown Workstation", "ok"))) + assertFalse(EventsWorker.hubWatch(Event(at, "Atlas", "Workstation looks frozen", "ok"))) + } + @Test fun parsesAgentsAndCaps() { val s = status("""{"name":"pc","caps":["shutdown","restart","uninstall"],"agents":{"allowed":false,"keys":[{"name":"Atlas","devices":["My PC"]}]}}""") assertEquals(listOf("shutdown", "restart", "uninstall"), s.caps) diff --git a/docs/plans/recovery-layer.md b/docs/plans/recovery-layer.md index cfa9caa..874e3ce 100644 --- a/docs/plans/recovery-layer.md +++ b/docs/plans/recovery-layer.md @@ -155,7 +155,7 @@ What can look like a hang and isn't (each one has its own rule below): Build: 1. **Watcher.** The 2-minute loop becomes a 30-second watcher for every machine with a KVM here. Each round collects evidence in parallel (8 s at most): - - `helper`: answers `hello` (two tries), or silent. + - `helper`: answers its status (two tries), or silent. - `tcp`: the helper's port refuses (the OS is up and answered, the helper isn't running), accepts but never answers (what a hung Windows does: the kernel still accepts), or no reply. - `ping`: `tailscale ping --icmp -c 2 --timeout 3s --until-direct=false `. ICMP @@ -167,17 +167,21 @@ Build: light is trusted (phase 2's wiring). - `tailnet`: whether Tailscale's control plane lists the node online. Shown, never decides: a hung PC stays "online" there for minutes. - - The helper's last word: its last `hello` (Windows updates pending, a shutdown or restart + - The helper's last word: its last status (Windows updates pending, a shutdown or restart counting down) and its notes (next item). - Wiring is learned from the same readings as before (no more often: at most every 2 minutes). + Wiring is learned from the same readings: a light seen on while the machine runs is wired; seen + off while its helper answers counts towards "not wired" at most once every 30 s. 2. **Notes from a helper to its hubs.** When its machine is about to sleep, shut down or restart, and - when it wakes, a helper tells every helper it holds a peer key for, best effort (1.5 s each): - `POST /v2/machines/{name}/note {"event": "sleep|shutdown|restart|resume", "updates": true}`. - Peer keys only, and only for the caller's own machine. Windows: the service accepts power events - (`PBT_APMSUSPEND`, `PBT_APMRESUMEAUTOMATIC`) and pre-shutdown, and reports whether updates are - pending. Linux: shutdown and restart. Not relied on: Windows may not send the suspend event on - Modern Standby PCs, and two seconds may not be enough, so item 3 covers sleep without it. -3. **Sleep facts.** A Windows helper's `hello` gains + when it wakes, a helper tells every helper it holds a peer key for, all at once, best effort + (1.5 s): `POST /v2/note {"event": "sleep|shutdown|restart|resume", "updates": true}`. Peer keys + only; the hub works out the machine from the caller's address, so a helper speaks only for its + own. Windows: suspend and resume through `PowerRegisterSuspendResumeNotification` (a callback + Windows waits for, unlike a service's power events, which are answered as soon as they're + queued), and the service's pre-shutdown notice (Windows waits for that too); a shutdown is sent + as `restart` when PowerDeck's own restart is counting down, with whether updates are pending. + Linux: `shutdown` when systemd is stopping the whole system. Not relied on: Windows may not send + the suspend event on Modern Standby PCs, so item 3 covers sleep without it. +3. **Sleep facts.** A Windows helper's status gains `"sleep": {"s3": true, "modernStandby": false, "afterMinutes": 30}` from `GetPwrCapabilities` and the active power plan's sleep timeout on AC (`PowerReadACValueIndex`, 0 = never); no `powercfg` text, which is translated. The hub keeps the last one it saw. The first `sleep` note @@ -197,11 +201,14 @@ Build: network down); - it may be asleep with the light on: a `sleep` note since it last answered, or it can sleep on its own (`afterMinutes` > 0) and its light while asleep is on or not learned yet. + (These carry the reasons `booting`, `shutting_down`, `restarting`, `updating` and + `may_be_asleep`.) - Anything that contradicts a hang (helper answers, ping replies, port refuses, light off or blinking) starts the count again. An unreadable light neither counts nor resets. - - `unknown`: everything else, with a `reason`: `no_helper`, `led_not_trusted`, `booting`, - `updating`, `may_be_asleep`, `helper_stopped` (silent while the network answers), `checking` - (silent for less than `hungAfter`). + - `unknown`: everything else, with a `reason`: `no_helper`, `led_not_trusted`, + `led_unreadable`, `booting`, `shutting_down`, `restarting`, `updating`, `may_be_asleep`, + `helper_stopped` (silent while the network answers), `checking` (silent for less than + `hungAfter`). A machine without a helper, or without a trusted light, is never `hung`. `GET /v2/machines` and each machine gain `since` (when the state began), `reason`, and `evidence` (the last round: each item with its value and time). diff --git a/docs/protocol.md b/docs/protocol.md index b13f273..b7e5551 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -133,7 +133,7 @@ All bodies are JSON. Errors: `{"error": "", "message": ""}`. | DELETE | `/v2/keys/{id}` | owner | → `204` | | GET | `/v2/update` | owner | → `{current, latest, available, checkedAt, error, installing, auto, autoAt?, pausedUntil?}`; `?check=1` asks GitHub first. `autoAt`: when an available release installs on its own | | POST | `/v2/update` | owner | Installs the latest signed release now → `202 {…, installing: true}`, or `200` when up to date; `409 pending` during a shutdown countdown, `502 update_check` when GitHub can't be reached. The helper restarts a few seconds later; `installing` clears itself after 10 minutes if it doesn't, with `error` saying so | -| POST | `/v2/settings` | owner | `{autoUpdate?, agents?, pauseUpdates?}` → `204`. `autoUpdate` (default on for Windows, off for new Linux installs): check every 6 hours, install a release once it has been public for 72 hours and nothing is counting down (see [Updates](#updates)). `pauseUpdates`: `true` holds automatic installs for 7 days, `false` resumes. `agents` (default on): `false` refuses `wake`, `shutdown` and `restart` from automation keys for this device, here and through any hub (`403 agents_off`); status and cancel still work | +| POST | `/v2/settings` | owner | `{autoUpdate?, agents?, pauseUpdates?, hungAfterMinutes?}` → `204`. `autoUpdate` (default on for Windows, off for new Linux installs): check every 6 hours, install a release once it has been public for 72 hours and nothing is counting down (see [Updates](#updates)). `pauseUpdates`: `true` holds automatic installs for 7 days, `false` resumes. `agents` (default on): `false` refuses `wake`, `shutdown` and `restart` from automation keys for this device, here and through any hub (`403 agents_off`); status and cancel still work. `hungAfterMinutes` (on a hub; 2 to 30, `0` for the default, 3): how long a machine must look hung before it's called [`hung`](#machine-states-and-events) | | POST | `/v2/uninstall` | owner | `{confirm: }` → `202`, then the helper uninstalls itself and deletes its keys and settings. `409 pending` during a countdown. Offered when `caps` has `uninstall` | | GET | `/v2/pairing` | owner | → `{link, code, expires}`: the add link and the current pairing code (to let someone on another account in) | | POST | `/v2/pair` | pairing code | `{code, deviceName}` → `{owner:true, status}` (same account: now an owner device) or `{key, status}` (another account) | @@ -145,6 +145,12 @@ All bodies are JSON. Errors: `{"error": "", "message": ""}`. | DELETE | `/v2/kvms/{name}` | owner | → `204` | | GET | `/v2/machines` | `status` | → `{machines:[Machine]}`: the devices the caller may use, with live state; `?kvm=1` only those with a KVM here | | POST | `/v2/machines/{name}/{action}` | per action | `{force?, stayOff?, press?}` → `200 Result`, or the error with the Result's fields. `action`: `wake`, `shutdown`, `restart`, `reset`, `force-off` | +| GET | `/v2/machines/events` | `status` | Server-Sent Events: each machine's state as it changes, see [Machine states and events](#machine-states-and-events) (2.3 and later) | +| POST | `/v2/note` | peer key | `{event, updates?}` → `204`: a machine's own helper telling its hub it's about to `sleep`, `shutdown` or `restart`, or did `resume` | +| GET | `/v2/webhooks` | owner | → `{webhooks:[Webhook]}` (without secrets) | +| POST | `/v2/webhooks` | owner | `{url, events?}` → `201 {webhook}` with its `secret`, shown this once. `400 bad_url`, `400 bad_event`, `409 too_many` (10 at most) | +| DELETE | `/v2/webhooks/{id}` | owner | → `204` | +| POST | `/v2/webhooks/{id}/test` | owner | Sends a made-up `hung` event for a machine called `test` → `{ok, result}` | | POST | `/mcp` | owner or automation key | [MCP](https://modelcontextprotocol.io) endpoint for AI agents | `delaySeconds` defaults to 60 (0–600). Without `force`, shutdown/restart is refused with @@ -164,13 +170,15 @@ before retrying with `force: true`. Callers without the `force` scope can't over "guests": {"running": 3, "names": ["public", "private", "agent"]}, "network": {"mac": "02:00:5e:00:00:01", "lan": ["192.168.1.0/24"], "wired": true}, "caps": ["shutdown", "restart", "wake", "kvm", "uninstall"], + "sleep": {"s3": true, "modernStandby": false, "afterMinutes": 30}, + "hub": {"hungAfterMinutes": 3}, "checks": [Check], "agents": {"allowed": true, "keys": [{"name": "Atlas", "devices": ["Workstation", "nas"]}]}, "you": {"kind": "owner", "name": "Pixel 9 Pro", "scopes": ["status", "wake", "power", "force", "admin"]} } ``` -`guests` appears on Proxmox hosts. The `kvm` cap means this helper can drive NanoKVMs (`/v2/kvms`, `/v2/machines`; 2.2 and later). `agents` and the `uninstall` cap are shown to owners only; `agents.keys` lists the automation keys this helper holds and the devices each may use. `updatesPending: true` (Windows) means "Update and shut down" is available. `presence` is omitted when the helper can't tell. +`guests` appears on Proxmox hosts. The `kvm` cap means this helper can drive NanoKVMs (`/v2/kvms`, `/v2/machines`; 2.2 and later). `agents` and the `uninstall` cap are shown to owners only; `agents.keys` lists the automation keys this helper holds and the devices each may use. `updatesPending: true` (Windows) means "Update and shut down" is available. `presence` is omitted when the helper can't tell. `sleep` (Windows, 2.3 and later): whether the PC can sleep the classic way (S3) or uses Modern Standby, and after how many idle minutes on AC power it sleeps on its own (0: never); a hub uses it to tell sleep from a hang. `hub` is shown to owners on a helper that drives KVMs. ### Check @@ -267,8 +275,10 @@ and nothing is pressed on a guess: | Power switch | A press or hold changes the LED | A press or hold doesn't (`power_not_wired`, `no_effect`) | | Reset switch | The machine's helper answers again within 5 minutes of a reset | It doesn't: the next reset is refused (`reset_not_wired`) unless forced | -The hub reads every KVM's LED every 2 minutes, so the LED is usually learned without anyone pressing -anything. `PATCH {recheck: true}` (the app's **Check again**) forgets what was learned. +The hub reads every KVM's LED every 30 seconds (see [Machine states and events](#machine-states-and-events)), +so the LED is usually learned without anyone pressing anything. `wiring.ledAsleep` is what the light +does while the machine sleeps (`off`, `blinking` or `on`), read 30 s after its helper said it was +going to sleep. `PATCH {recheck: true}` (the app's **Check again**) forgets what was learned. ### Machine @@ -278,7 +288,8 @@ anything. `PATCH {recheck: true}` (the app's **Check again**) forgets what was l "state": "unknown", "actions": ["wake", "shutdown", "restart", "reset", "force-off"]} ``` -`state`: `on` (its helper answers), `off` (a wired LED reads off) or `unknown`. `led` is the power +`state`: `on`, `off`, `hung` or `unknown`, with `reason`, `asleep`, `since` and `evidence` from the +hub's watch (see [Machine states and events](#machine-states-and-events)). `led` is the power light right now (absent without a KVM, `ledError` when it can't be read). `actions` is what the hub can do for it now; `reset` and `force-off` appear only while a wired LED reads on. @@ -316,6 +327,131 @@ Result (also in the error body on failure, with `error` and `message` added): `kvm_forbidden` (the account lacks the power permission), `kvm_cert_changed`, `kvm_unreachable`, `kvm_failed`. Every action is in the hub's event log with who asked and the outcome. +## Machine states and events + +From 2.3 a hub looks at every machine with a KVM every 30 seconds and tells a hung machine apart from +one that's booting, asleep, installing updates, or just missing its helper. Each look gathers, at +once and in 8 s at most: + +| Evidence | Values | What it says | +|---|---|---| +| `helper` | `answers`, `silent` | The machine's PowerDeck helper answered its status (two tries, 1.5 s apart) | +| `port` | `refuses`, `accepts`, `no_reply` | A TCP connection to the helper's port. `refuses`: the OS is up and answered. `accepts`: the kernel took it (a hung Windows still does) | +| `ping` | `replies`, `no_reply`, `unavailable` | `tailscale ping --icmp` to the machine: answered by its own network stack, not by Tailscale alone. `unavailable` when the hub can't ping | +| `led` | `on`, `off`, `blinking`, `unreadable` | Three readings of the power light 0.5 s apart; on and off in one look is `blinking` | +| `ledTrusted` | `true`, `false` | The light is [wired](#wiring-is-learned-never-assumed) | +| `tailnetOnline` | `true`, `false` | Tailscale's control plane lists the machine online. Shown, never decides: a hung PC stays online there for minutes | + +What the machine's helper said also counts: its last status (a shutdown or restart counting down, +Windows updates pending, its [`sleep`](#status) facts) and its notes. A helper tells every hub it +has a peer key for, just before its machine sleeps, shuts down or restarts, and when it wakes +(`POST /v2/note`, best effort, 1.5 s). The hub works out which machine from the caller's address, +so a helper only speaks for its own. + +**States:** + +- `on`: the helper answers. +- `off`: a wired light reads off. `asleep: true` when it blinks, or after a `sleep` note. +- `hung`: for `hungAfter` (3 minutes; the owner's `hungAfterMinutes` setting, 2 to 30) every look + saw the helper silent, the port not refusing, ping not replying (or unavailable), and a wired + light steadily on; and nothing excused it (below). A machine without a helper or without a wired + light is never `hung`. +- `unknown`, with a `reason`: + +| `reason` | When | +|---|---| +| `no_helper` | No PowerDeck helper on the machine: nothing can say it's fine | +| `led_not_trusted` | The light hasn't been seen on yet | +| `led_unreadable` | The KVM didn't answer this look | +| `checking` | Silent, light on, for less than `hungAfter` | +| `helper_stopped` | Silent while ping replies or the port refuses: the machine runs, its helper doesn't | +| `booting` | Within 10 minutes of the hub waking, restarting, resetting or forcing it off, or of its light coming on | +| `shutting_down`, `restarting` | Within 10 minutes of a shutdown or restart note or countdown | +| `updating` | The same, with Windows updates pending: 90 minutes (they install with the light on and the network down) | +| `may_be_asleep` | After a `sleep` note, or the machine sleeps on its own after `afterMinutes` and its light while asleep is `on` or not learned yet | + +Anything that contradicts a hang (the helper answers, ping replies, the port refuses, the light is +off or blinking) starts the count again. An unreadable light neither counts nor resets it, and a +gap of more than `hungAfter` between looks (the hub was off) starts it again too. + +`since` is when the hub first saw the current state; `evidence` is its last look, with `at`. + +### Event stream + +`GET /v2/machines/events` (status scope; the machines with a KVM on this hub that the caller may +use, as in `GET /v2/machines?kvm=1`) is a +[Server-Sent Events](https://html.spec.whatwg.org/multipage/server-sent-events.html) stream: + +``` +id: 1761638400000123 +event: snapshot +data: {"machines":[Machine, …]} + +id: 1761638430000456 +event: machine +data: {"id":"1761638430000456","at":"2026-10-28T08:00:30Z","event":"hung","machine":"Workstation","state":"hung","from":"unknown","evidence":{"at":"…","helper":"silent","port":"accepts","ping":"no_reply","led":"on","ledTrusted":true}} + +: still here +``` + +Each `machine` event is a state change: `event` is the new state, or `recovered` when a `hung` +machine answers again (`state` is then `on`). `reason` and `asleep` are included when set. IDs +grow, also across restarts of the hub. Reconnect with `Last-Event-ID` (or `?lastEventId=` for +clients that can't set headers) to get what came after it, from the last 500 the hub holds; an ID +it no longer holds gets a fresh snapshot. A comment every 25 s keeps proxies from closing the +stream. A client that stops reading for 10 s is dropped, and so is one that falls 64 events behind +(it reconnects and replays). At most 16 streams per hub (`503 busy`). + +`hung` and `recovered` also go to the hub's event log (`GET /v2/events`), which the phone +[notifies](#notifications) about. + +### Webhooks + +For an assistant that would rather be called than hold a stream open. An owner device adds one: + +```json +POST /v2/webhooks +{"url": "https://assistant.example.com/powerdeck", "events": ["hung", "recovered"]} + +201 {"webhook": {"id": "c0ffee…", "url": "…", "events": ["hung", "recovered"], + "createdBy": "Pixel 9 Pro", "createdAt": "…", "secret": "pdwh_…"}} +``` + +`events` defaults to `hung` and `recovered` (any of `on`, `off`, `hung`, `unknown`, `recovered`). +The URL must be https, or plain http to a Tailscale or loopback address or a `*.ts.net` name +(WireGuard already encrypts those); no user names in it. The secret is shown this once and stored +sealed, like KVM passwords. The list shows each webhook's `lastAt` and `lastResult` (`ok`, +`HTTP 500`, a network error). + +Each delivery is a POST of the [event](#event-stream) as JSON, with: + +``` +X-PowerDeck-Event: hung +X-PowerDeck-Delivery: 1761638430000456 (the event's id; the same on every retry) +X-PowerDeck-Timestamp: 1761638431 (Unix seconds of this attempt) +X-PowerDeck-Signature: sha256= +``` + +Any 2xx is delivered. Otherwise it's tried again after 10 s, 1 minute and 5 minutes, then dropped +and logged. 10 s per try; redirects aren't followed. A receiver checks the signature over the raw +body, refuses a timestamp more than 5 minutes off (so an old delivery can't be replayed), and can +use the delivery ID to ignore repeats: + +```python +import hashlib, hmac, time + +def verify(secret: str, headers, body: bytes) -> bool: + ts = headers["X-PowerDeck-Timestamp"] + if abs(time.time() - int(ts)) > 300: + return False + mac = hmac.new(secret.encode(), ts.encode() + b"." + body, hashlib.sha256).hexdigest() + return hmac.compare_digest("sha256=" + mac, headers["X-PowerDeck-Signature"]) +``` + +To check an implementation: secret `pdwh_test`, timestamp `1700000000` and body `{"id":"1"}` sign as +`sha256=8fc0a23c6a794d26f2a6c83ac48c588dfbbe64deefd997f2bb8cc5b0ac0c0914` +(`printf '1700000000.{"id":"1"}' | openssl dgst -sha256 -hmac pdwh_test`). + ## Schedules ```json @@ -333,7 +469,8 @@ is logged. Each run is an event in the log. The phone checks each device's `/v2/events` in the background (Android allows about every 15 minutes) and notifies about things the user didn't do from this phone: an agent or schedule acting, -a device going offline unexpectedly. Actions started on the phone show live progress instead. +a device going offline unexpectedly, and (from a 2.3 hub) a machine that +[looks frozen](#machine-states-and-events) or answers again. Actions started on the phone show live progress instead. ## Agents (MCP) diff --git a/helper/cmd/powerdeck/main.go b/helper/cmd/powerdeck/main.go index 74a3a65..fef0bbe 100644 --- a/helper/cmd/powerdeck/main.go +++ b/helper/cmd/powerdeck/main.go @@ -105,7 +105,8 @@ func runForeground(args []string) error { if *service && ui == "" { ui = "-" // headless } - return app.Run(ctx, app.Options{DataDir: *data, APIAddr: *apiAddr, MCPAddr: *mcpAddr, UIAddr: ui, System: sys, Log: log, Installed: *service && !*dry}) + return app.Run(ctx, app.Options{DataDir: *data, APIAddr: *apiAddr, MCPAddr: *mcpAddr, UIAddr: ui, System: sys, Log: log, Installed: *service && !*dry, + Stopping: systemStopping}) } func status() error { diff --git a/helper/cmd/powerdeck/main_linux.go b/helper/cmd/powerdeck/main_linux.go index 1850f8b..8edbe04 100644 --- a/helper/cmd/powerdeck/main_linux.go +++ b/helper/cmd/powerdeck/main_linux.go @@ -3,9 +3,12 @@ package main import ( + "context" "fmt" "log/slog" "os" + "os/exec" + "strings" "time" "github.com/projectzero02/powerdeck/helper/internal/cli" @@ -19,6 +22,15 @@ func newSystem(*slog.Logger) power.System { return &power.Linux{} } func platformCommand(string, []string, *error) bool { return false } +// systemStopping reports whether systemd is shutting the system down (rather than stopping or +// restarting just this service), so the helper can tell its hubs first. +func systemStopping() bool { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + out, _ := exec.CommandContext(ctx, "systemctl", "is-system-running").Output() + return strings.TrimSpace(string(out)) == "stopping" +} + // install sets up the systemd service, then shows the checks and the QR code that adds this // device to the app. func install(args []string) error { diff --git a/helper/cmd/powerdeck/main_windows.go b/helper/cmd/powerdeck/main_windows.go index cba9f7a..be4fa03 100644 --- a/helper/cmd/powerdeck/main_windows.go +++ b/helper/cmd/powerdeck/main_windows.go @@ -8,6 +8,7 @@ import ( "log/slog" "os" "path/filepath" + "time" "golang.org/x/sys/windows" @@ -39,7 +40,25 @@ func platformCommand(cmd string, _ []string, err *error) bool { return true } +// systemStopping: on Windows the service hears of shutdowns itself (see runService). +var systemStopping func() bool + func runService() error { + notes := make(chan app.Notice) + // notify hands what Windows is about to do to the running helper, which tells the hubs; it + // waits for that, briefly, as Windows goes on without it. + notify := func(event string) { + n := app.Notice{Event: event, Done: make(chan struct{})} + select { + case notes <- n: + case <-time.After(200 * time.Millisecond): + return // not running + } + select { + case <-n.Done: + case <-time.After(2 * time.Second): + } + } return winsvc.Run(setup.ServiceName, func(ctx context.Context) error { log, closer, err := applog.Open(setup.DataDir(), nil) if err != nil { @@ -48,12 +67,12 @@ func runService() error { defer closer.Close() go companionLoop(ctx, log) err = app.Run(ctx, app.Options{DataDir: setup.DataDir(), System: newSystem(log), Reports: reports, Log: log, Installed: true, - AllowLocal: func(remote string) bool { return winsvc.LocalCaller(remote, app.UIPort) }}) + Notes: notes, AllowLocal: func(remote string) bool { return winsvc.LocalCaller(remote, app.UIPort) }}) if err != nil { log.Error("PowerDeck exited", "err", err) } return err - }) + }, notify) } func install([]string) error { diff --git a/helper/internal/app/app.go b/helper/internal/app/app.go index 0d29449..fe8f2e1 100644 --- a/helper/internal/app/app.go +++ b/helper/internal/app/app.go @@ -12,6 +12,7 @@ import ( "log/slog" "net" "net/http" + "net/netip" "os" "path/filepath" "runtime" @@ -34,11 +35,13 @@ import ( "github.com/projectzero02/powerdeck/helper/internal/secret" "github.com/projectzero02/powerdeck/helper/internal/server" "github.com/projectzero02/powerdeck/helper/internal/setup" + "github.com/projectzero02/powerdeck/helper/internal/sleepinfo" "github.com/projectzero02/powerdeck/helper/internal/store" "github.com/projectzero02/powerdeck/helper/internal/tailscale" "github.com/projectzero02/powerdeck/helper/internal/tlscert" "github.com/projectzero02/powerdeck/helper/internal/update" "github.com/projectzero02/powerdeck/helper/internal/version" + "github.com/projectzero02/powerdeck/helper/internal/webhook" "github.com/projectzero02/powerdeck/helper/internal/wol" ) @@ -67,6 +70,17 @@ type Options struct { AllowLocal func(remote string) bool // Installed: running as the installed service, so the phone may uninstall it. Installed bool + // Notes carries what the OS is about to do (sleep, shutdown) or did (resume), from the Windows + // service; each is told to the hubs, then Done is closed. + Notes <-chan Notice + // Stopping reports, as the helper stops, whether the whole system is shutting down (Linux). + Stopping func() bool +} + +// Notice is something the OS is about to do, to tell the hubs before it happens. +type Notice struct { + Event string // sleep, shutdown, resume + Done chan struct{} } func uninstall(o Options) func() error { @@ -144,11 +158,15 @@ func Run(ctx context.Context, o Options) error { } // Every helper can drive NanoKVMs; the app links them to one (the hub). - hub := &machines.Hub{Store: st, Box: secret.Default(), Fleet: fl, Self: name, Log: log, Now: now} + hub := &machines.Hub{Store: st, Box: secret.Default(), Fleet: fl, Self: name, Log: log, Now: now, + Events: machines.NewBus(now()), Ping: tailnetPing(ts), Online: tailnetOnline(ts), + HungAfter: func() time.Duration { return time.Duration(st.Settings().HungAfterMinutes) * time.Minute }} + hooks := &webhook.Sender{Store: st, Box: hub.Box, Bus: hub.Events, Log: log, Now: now} api := server.New(server.Deps{ Auth: &auth.Authenticator{TS: ts, Store: st, Now: now, LocalToken: localToken}, - Store: st, Pairing: pm, Power: pc, Fleet: fl, Machines: hub, Name: name, OS: runtime.GOOS, + Store: st, Pairing: pm, Power: pc, Fleet: fl, Machines: hub, Webhooks: hooks, Sleep: sleepinfo.Cached(), + Name: name, OS: runtime.GOOS, SelfNode: func(ctx context.Context) string { if s, err := ts.Status(ctx); err == nil { return s.Self.ID @@ -240,7 +258,8 @@ func Run(ctx context.Context, o Options) error { go func(l listener) { errc <- l.serve(l.ln) }(l) } go fl.Loop(ctx, 5*time.Minute, nil) - go hub.Loop(ctx, 2*time.Minute) // learns the wiring from the power light + go hub.Watch(ctx, 30*time.Second) // hang detection, and the wiring learned from the power light + go hooks.Run(ctx) go schedule.Loop(ctx, st.Schedules, api.RunSchedule, now) go updateLoop(ctx, upd, st, pc, log) go func() { // drivers on some Linux systems drop Wake-on-LAN after link changes @@ -254,9 +273,35 @@ func Run(ctx context.Context, o Options) error { } }() + // Tell the hubs before sleep or shutdown, so they don't take it for a hang. + tell := func(event string) { + updates := false + if event == "shutdown" { + if p := pc.Pending(); p != nil && p.Action == power.Restart { + event = "restart" + } + updates = pc.UpdatesPending() + } + fl.Tell(context.WithoutCancel(ctx), event, updates) + } + go func() { + for { + select { + case <-ctx.Done(): + return + case n := <-o.Notes: + tell(n.Event) + close(n.Done) + } + } + }() + var runErr error select { case <-ctx.Done(): + if o.Stopping != nil && o.Stopping() { + tell("shutdown") + } case runErr = <-errc: } shutCtx, done := context.WithTimeout(context.Background(), 5*time.Second) @@ -271,6 +316,37 @@ func Run(ctx context.Context, o Options) error { return runErr } +// tailnetPing is the hub's ping of a machine: through Tailscale, to its own network stack. +func tailnetPing(ts *tailscale.Client) func(ctx context.Context, addr string) string { + return func(ctx context.Context, addr string) string { + ip, err := netip.ParseAddr(addr) + if err != nil { + return machines.Unavailable + } + ok, err := ts.Ping(ctx, ip) + switch { + case err != nil: + return machines.Unavailable + case ok: + return machines.Replies + } + return machines.NoReply + } +} + +func tailnetOnline(ts *tailscale.Client) func(ctx context.Context, addr string) *bool { + return func(ctx context.Context, addr string) *bool { + ip, err := netip.ParseAddr(addr) + if err != nil { + return nil + } + if online, known := ts.Online(ctx, ip); known { + return &online + } + return nil + } +} + // updateLoop checks for updates shortly after start and every 6 hours, and installs them when // automatic updates are on, the release has been out for update.Staging, updates aren't paused and // nothing is scheduled. diff --git a/helper/internal/fleet/fleet.go b/helper/internal/fleet/fleet.go index cdaf51f..838697d 100644 --- a/helper/internal/fleet/fleet.go +++ b/helper/internal/fleet/fleet.go @@ -104,12 +104,13 @@ func (f *Fleet) remember(ctx context.Context, p discovery.Peer) { Agents *struct { Allowed bool `json:"allowed"` } `json:"agents"` + Sleep *store.Sleep `json:"sleep"` } cctx, cancel := context.WithTimeout(ctx, 8*time.Second) defer cancel() err := f.Call(cctx, rec, "GET", "/v2/status", nil, &st, "") if err == nil { - rec.MAC, rec.LAN = st.Network.MAC, st.Network.LAN + rec.MAC, rec.LAN, rec.Sleep = st.Network.MAC, st.Network.LAN, st.Sleep rec.NoAgents = st.Agents != nil && !st.Agents.Allowed } else { f.Log.Info("peer status unavailable", "peer", p.Name, "err", err) @@ -210,6 +211,29 @@ func (f *Fleet) Forward(ctx context.Context, name, method, path string, body, ou return err } +// Tell tells every helper this one holds a peer key for what its machine is about to do (sleep, +// shutdown, restart) or just did (resume), so a hub watching it doesn't take it for a hang. Best +// effort: all at once, 1.5 s at most, as Windows gives a sleeping service about 2 s. +func (f *Fleet) Tell(ctx context.Context, event string, updates bool) { + ctx, cancel := context.WithTimeout(ctx, 1500*time.Millisecond) + defer cancel() + var wg sync.WaitGroup + for _, p := range f.Store.Peers() { + if p.Key == "" { + continue + } + wg.Add(1) + go func(p store.Peer) { + defer wg.Done() + err := f.Call(ctx, p, "POST", "/v2/note", map[string]any{"event": event, "updates": updates}, nil, "") + if err != nil { + f.Log.Info("couldn't tell a peer", "peer", p.Name, "event", event, "err", err) + } + }(p) + } + wg.Wait() +} + // keyRefused forgets a peer key the target no longer accepts (it was removed there, or the target // was reset). Sending it again would only get this helper throttled; without it the app sees the // pair as unlinked and creates a new key. diff --git a/helper/internal/fleet/fleet_test.go b/helper/internal/fleet/fleet_test.go index 573527f..41ddb46 100644 --- a/helper/internal/fleet/fleet_test.go +++ b/helper/internal/fleet/fleet_test.go @@ -7,6 +7,7 @@ import ( "io" "log/slog" "net/netip" + "sync" "testing" "time" @@ -132,3 +133,37 @@ func TestRefusedKeyIsForgotten(t *testing.T) { t.Errorf("key dropped on a 409: %+v", p) } } + +// Tell reaches every helper it holds a key for, at once, and gives up on a slow one in 1.5 s. +func TestTell(t *testing.T) { + st, _ := store.Open(t.TempDir()) + for _, p := range []store.Peer{{Name: "hub", Key: "pdk_hub"}, {Name: "nas", Key: "pdk_nas"}, {Name: "laptop"}} { + if err := st.PutPeer(p); err != nil { + t.Fatal(err) + } + } + var mu sync.Mutex + got := map[string]string{} + dns := "" + f := New(Deps{TS: ts{&dns}, Store: st, Log: slog.New(slog.NewTextHandler(io.Discard, nil)), Now: time.Now, + Call: func(ctx context.Context, p store.Peer, method, path string, in, _ any, _ string) error { + b, _ := json.Marshal(in) + mu.Lock() + got[p.Name] = method + " " + path + " " + string(b) + mu.Unlock() + if p.Name == "nas" { + <-ctx.Done() // hangs + return ctx.Err() + } + return nil + }}) + start := time.Now() + f.Tell(context.Background(), "restart", true) + if d := time.Since(start); d < 1400*time.Millisecond || d > 3*time.Second { + t.Errorf("took %v", d) + } + want := `POST /v2/note {"event":"restart","updates":true}` + if len(got) != 2 || got["hub"] != want || got["nas"] != want { + t.Errorf("told %v", got) + } +} diff --git a/helper/internal/machines/detect.go b/helper/internal/machines/detect.go new file mode 100644 index 0000000..8ca1f79 --- /dev/null +++ b/helper/internal/machines/detect.go @@ -0,0 +1,207 @@ +package machines + +import ( + "time" + + "github.com/projectzero02/powerdeck/helper/internal/store" +) + +// What a look at a machine can see. +const ( + Answers = "answers" // helper: it replied + Silent = "silent" // helper: no reply; ping: no reply + Refuses = "refuses" // port: the OS turned the connection away, so it's running + Accepts = "accepts" // port: connected, but the helper behind it said nothing + NoReply = "no_reply" // port, ping: nothing came back + Replies = "replies" // ping: the machine's network stack answered + Unavailable = "unavailable" // ping: the hub can't ping it (no Tailscale here or there) + On = "on" // led + Off = "off" // led + Blinking = "blinking" // led: on and off within one look (asleep, on many boards) + Unreadable = "unreadable" // led: the KVM didn't say +) + +// Machine states. +const ( + StateOn = "on" + StateOff = "off" + StateHung = "hung" + StateUnknown = "unknown" +) + +// Why a machine that isn't answering isn't called hung (Machine.Reason). +const ( + ReasonNoHelper = "no_helper" // nothing on it can say it's fine + ReasonLEDNotTrusted = "led_not_trusted" // the light's wiring isn't known yet + ReasonLEDUnreadable = "led_unreadable" + ReasonHelperStopped = "helper_stopped" // the network answers: the machine runs, its helper doesn't + ReasonBooting = "booting" + ReasonShuttingDown = "shutting_down" + ReasonRestarting = "restarting" + ReasonUpdating = "updating" // installing Windows updates during a restart + ReasonMayBeAsleep = "may_be_asleep" + ReasonChecking = "checking" // silent for less than HungAfter +) + +// Evidence is what one look at a machine saw. +type Evidence struct { + At time.Time `json:"at"` + Helper string `json:"helper,omitempty"` // answers, silent; absent without a helper + Port string `json:"port,omitempty"` // the helper's port: refuses, accepts, no_reply + Ping string `json:"ping,omitempty"` // replies, no_reply, unavailable + LED string `json:"led,omitempty"` // on, off, blinking, unreadable + LEDTrusted bool `json:"ledTrusted"` + LEDError string `json:"ledError,omitempty"` + // Tailnet is whether Tailscale's control plane lists the machine online. It never decides: a + // hung PC stays "online" there for minutes. + Tailnet *bool `json:"tailnetOnline,omitempty"` +} + +// Word is what the machine's helper last said about itself. +type Word struct { + At time.Time // when it last answered + PendingAction string // a shutdown or restart counting down + PendingAt time.Time // when it's due + UpdatesPending bool // Windows updates install at the next restart or shutdown + Sleep *store.Sleep // whether it sleeps on its own +} + +// Note is something a machine's helper told the hub just before it happened. +type Note struct { + At time.Time `json:"at"` + Event string `json:"event"` // sleep, shutdown, restart, resume + Updates bool `json:"updates,omitempty"` // Windows updates will install +} + +// Note events. +var NoteEvents = []string{"sleep", "shutdown", "restart", "resume"} + +// Rules are the detector's timings, and what it knows about the machine's light. +type Rules struct { + HungAfter time.Duration // how long every look must agree (3 min) + Boot time.Duration // grace after the machine was started, or a shutdown or restart began (10 min) + Updating time.Duration // the same with Windows updates to install (90 min) + LEDAsleep string // what the light does while the machine sleeps; "" = not seen yet +} + +// DefaultRules are the timings unless the owner changes HungAfter. +var DefaultRules = Rules{HungAfter: 3 * time.Minute, Boot: 10 * time.Minute, Updating: 90 * time.Minute} + +// Track is what the detector remembers about one machine between looks. +type Track struct { + State string + Reason string + Since time.Time // when State began + Asleep bool // off because it sleeps + Evidence Evidence // the last look + // Suspect is the first look of the current run in which the machine looked hung. + Suspect time.Time + LastLED string // the last reading that said something (on, off or blinking) + LEDOnAt time.Time // when the light last came on + Acted time.Time // when the hub last started, restarted, reset or forced it off + Word Word + Note *Note // the newest note since the helper last answered +} + +// Step takes one look into account and reports whether the state changed (from what). +// +// A machine is hung when, for HungAfter, every look saw its helper silent, its network silent (no +// ping reply, the helper's port not refusing) and a trusted power light steadily on, and nothing +// explains it: starting, shutting down, restarting, installing updates or maybe asleep. +func (t *Track) Step(e Evidence, r Rules) (from string, changed bool) { + now := e.At + if e.LED == On && (t.LastLED == Off || t.LastLED == Blinking) { + t.LEDOnAt = now + } + if e.LED == On || e.LED == Off || e.LED == Blinking { + t.LastLED = e.LED + } + if e.Helper == Answers { + t.Note = nil // whatever it announced is over + } + if !t.Suspect.IsZero() && !t.Evidence.At.IsZero() && now.Sub(t.Evidence.At) > r.HungAfter { + t.Suspect = time.Time{} // the hub looked away too long (it was off, or restarting) to count on + } + state, reason, asleep := t.decide(e, r) + t.Evidence = e + from = t.State + t.Reason, t.Asleep = reason, asleep + if state == t.State { + return from, false + } + t.State, t.Since = state, now + return from, true +} + +func (t *Track) decide(e Evidence, r Rules) (state, reason string, asleep bool) { + now := e.At + switch { + case e.Helper == Answers: + t.Suspect = time.Time{} + return StateOn, "", false + case e.LEDTrusted && (e.LED == Off || e.LED == Blinking): + t.Suspect = time.Time{} + return StateOff, "", e.LED == Blinking || t.Note != nil && t.Note.Event == "sleep" + case e.Helper == "": + t.Suspect = time.Time{} + return StateUnknown, ReasonNoHelper, false + case !e.LEDTrusted: + t.Suspect = time.Time{} + return StateUnknown, ReasonLEDNotTrusted, false + case e.Ping == Replies || e.Port == Refuses: + t.Suspect = time.Time{} + return StateUnknown, ReasonHelperStopped, false + case e.LED != On: // unreadable: it neither counts nor clears + if t.State == StateHung { + return StateHung, "", false + } + return StateUnknown, ReasonLEDUnreadable, false + } + // Helper and network silent, light steadily on. + if why := t.excuse(now, r); why != "" { + t.Suspect = time.Time{} + return StateUnknown, why, false + } + if t.Suspect.IsZero() { + t.Suspect = now + } + if now.Sub(t.Suspect) >= r.HungAfter { + return StateHung, "", false + } + return StateUnknown, ReasonChecking, false +} + +// excuse is a harmless reason for a silent machine with its light on, or "". +func (t *Track) excuse(now time.Time, r Rules) string { + within := func(since time.Time, d time.Duration) bool { return !since.IsZero() && now.Sub(since) < d } + if within(t.Acted, r.Boot) || within(t.LEDOnAt, r.Boot) { + return ReasonBooting + } + // A shutdown or restart it announced, or that was counting down when it last answered. + action, at, updates := "", time.Time{}, t.Word.UpdatesPending + if n := t.Note; n != nil && (n.Event == "shutdown" || n.Event == "restart") { + action, at, updates = n.Event, n.At, updates || n.Updates + } + if t.Word.PendingAction != "" && t.Word.PendingAt.After(at) { + action, at = t.Word.PendingAction, t.Word.PendingAt + } + if action != "" { + switch { + case updates && within(at, r.Updating): + return ReasonUpdating + case within(at, r.Boot) || at.After(now): + if action == "restart" { + return ReasonRestarting + } + return ReasonShuttingDown + } + } + if t.Note != nil && t.Note.Event == "sleep" { + return ReasonMayBeAsleep + } + // It sleeps on its own, and its light may stay on while it does. + if s := t.Word.Sleep; s != nil && s.AfterMinutes > 0 && r.LEDAsleep != Off && r.LEDAsleep != Blinking { + return ReasonMayBeAsleep + } + return "" +} diff --git a/helper/internal/machines/detect_test.go b/helper/internal/machines/detect_test.go new file mode 100644 index 0000000..9e838f0 --- /dev/null +++ b/helper/internal/machines/detect_test.go @@ -0,0 +1,166 @@ +package machines + +import ( + "fmt" + "strings" + "testing" + "time" + + "github.com/projectzero02/powerdeck/helper/internal/store" +) + +var t0 = time.Date(2026, 10, 28, 3, 0, 0, 0, time.UTC) + +// Looks, as short codes: h = helper answers, x = everything silent with the light on (a hang's +// look), o = light off, b = blinking, u = light unreadable, p = ping replies, r = port refuses, +// n = no helper at all, t = light not trusted. +func look(c byte, at time.Time) Evidence { + e := Evidence{At: at, Helper: Silent, Port: Accepts, Ping: NoReply, LED: On, LEDTrusted: true} + switch c { + case 'h': + e.Helper, e.Port, e.Ping = Answers, Accepts, Replies + case 'o': + e.LED, e.Port = Off, NoReply + case 'b': + e.LED, e.Port = Blinking, NoReply + case 'u': + e.LED = Unreadable + case 'p': + e.Ping = Replies + case 'r': + e.Port = Refuses + case 'n': + e.Helper = "" + case 't': + e.LEDTrusted = false + } + return e +} + +// feed feeds looks 30 s apart and returns each resulting state (with its reason when unknown). +func feed(tr *Track, r Rules, looks string, start time.Time) []string { + var out []string + for i := 0; i < len(looks); i++ { + tr.Step(look(looks[i], start.Add(time.Duration(i)*30*time.Second)), r) + s := tr.State + if tr.Reason != "" { + s += "/" + tr.Reason + } + if tr.Asleep { + s += "/asleep" + } + out = append(out, s) + } + return out +} + +func last(s []string) string { return s[len(s)-1] } + +func TestDetector(t *testing.T) { + r := DefaultRules + for _, c := range []struct { + name string + setup func(*Track) + looks string + want string + }{ + {"answers", nil, "h", "on"}, + {"silent 2.5 min is not yet hung", nil, "hxxxxx", "unknown/checking"}, + {"silent 3 min is hung", nil, "hxxxxxxx", "hung"}, + {"off", nil, "ho", "off"}, + {"blinking is asleep", nil, "hb", "off/asleep"}, + {"ping replies: the helper stopped", nil, "hxxxxxxpp", "unknown/helper_stopped"}, + {"port refuses: the helper stopped", nil, "hxxxxxxxr", "unknown/helper_stopped"}, + {"a contradiction starts the count again", nil, "hxxxxxpxxxxxx", "unknown/checking"}, + {"... and it ends 3 min after", nil, "hxxxxxpxxxxxxx", "hung"}, + {"light off, then on again: booting", nil, "hxxxxoxxxxxx", "unknown/booting"}, + {"unreadable light neither counts nor resets", nil, "hxxuuxxxx", "hung"}, + {"unreadable alone isn't hung", nil, "huuuuuuuuu", "unknown/led_unreadable"}, + {"hung stays hung while unreadable", nil, "hxxxxxxxuu", "hung"}, + {"no helper is never hung", nil, "nnnnnnnnnnnn", "unknown/no_helper"}, + {"untrusted light is never hung", nil, "tttttttttttt", "unknown/led_not_trusted"}, + {"answers again", nil, "hxxxxxxxh", "on"}, + {"light came on: booting for 10 min", nil, "o" + strings.Repeat("x", 20), "unknown/booting"}, + {"... then counted from 10 min", nil, "o" + strings.Repeat("x", 26), "unknown/checking"}, + {"... hung 3 min after that", nil, "o" + strings.Repeat("x", 27), "hung"}, + {"hub acted 1 min ago", func(tr *Track) { tr.Acted = t0 }, "xxxxxxxxxx", "unknown/booting"}, + {"restart note", func(tr *Track) { tr.Note = &Note{At: t0, Event: "restart"} }, "xxxxxxxxxx", "unknown/restarting"}, + {"shutdown note, light still on after 10 min", func(tr *Track) { tr.Note = &Note{At: t0, Event: "shutdown"} }, + strings.Repeat("x", 27), "hung"}, + {"restart with updates", func(tr *Track) { tr.Note = &Note{At: t0, Event: "restart", Updates: true} }, + strings.Repeat("x", 120), "unknown/updating"}, + {"updates pending, countdown seen", func(tr *Track) { + tr.Word = Word{At: t0, PendingAction: "restart", PendingAt: t0.Add(time.Minute), UpdatesPending: true} + }, strings.Repeat("x", 120), "unknown/updating"}, + {"updates pending but no restart: a hang", func(tr *Track) { tr.Word = Word{At: t0, UpdatesPending: true} }, + "xxxxxxx", "hung"}, + {"countdown due later", func(tr *Track) { + tr.Word = Word{At: t0, PendingAction: "shutdown", PendingAt: t0.Add(5 * time.Minute)} + }, "xxxxxxxxxxxxxxxxxxxx", "unknown/shutting_down"}, + {"sleep note", func(tr *Track) { tr.Note = &Note{At: t0, Event: "sleep"} }, "xxxxxxxxxx", "unknown/may_be_asleep"}, + {"sleep note, then off", func(tr *Track) { tr.Note = &Note{At: t0, Event: "sleep"} }, "xo", "off/asleep"}, + {"sleeps on its own, light in sleep unknown", func(tr *Track) { + tr.Word.Sleep = &store.Sleep{ModernStandby: true, AfterMinutes: 30} + }, "xxxxxxxxxx", "unknown/may_be_asleep"}, + {"never sleeps on its own", func(tr *Track) { tr.Word.Sleep = &store.Sleep{S3: true} }, "xxxxxxx", "hung"}, + } { + t.Run(c.name, func(t *testing.T) { + tr := &Track{} + if c.setup != nil { + c.setup(tr) + } + got := feed(tr, r, c.looks, t0) + if last(got) != c.want { + t.Errorf("%s → %v, want %s at the end", c.looks, got, c.want) + } + }) + } +} + +func TestDetectorLightInSleep(t *testing.T) { + sleeps := Word{Sleep: &store.Sleep{S3: true, AfterMinutes: 30}} + for led, want := range map[string]string{"": "unknown/may_be_asleep", On: "unknown/may_be_asleep", Off: "hung", Blinking: "hung"} { + r := DefaultRules + r.LEDAsleep = led + tr := &Track{Word: sleeps} + if got := last(feed(tr, r, "xxxxxxx", t0)); got != want { + t.Errorf("light in sleep %q: %s, want %s", led, got, want) + } + } +} + +func TestDetectorHungAfter(t *testing.T) { + r := DefaultRules + r.HungAfter = 10 * time.Minute + got := feed(&Track{}, r, strings.Repeat("x", 21), t0) + if got[19] != "unknown/checking" || got[20] != "hung" { + t.Errorf("%v", got) + } +} + +func TestDetectorGapStartsAgain(t *testing.T) { + tr := &Track{} + feed(tr, DefaultRules, "xxxx", t0) // 1.5 min silent + // The hub was down for an hour: one look after it isn't 3 minutes of evidence. + tr.Step(look('x', t0.Add(time.Hour)), DefaultRules) + if tr.State != StateUnknown || tr.Reason != ReasonChecking { + t.Errorf("after a gap: %s/%s", tr.State, tr.Reason) + } +} + +func TestDetectorSinceAndChanges(t *testing.T) { + tr := &Track{} + var changes []string + for i, c := range "hhxxxxxxxxh" { + at := t0.Add(time.Duration(i) * 30 * time.Second) + if from, ok := tr.Step(look(byte(c), at), DefaultRules); ok { + changes = append(changes, fmt.Sprintf("%s→%s@%d", from, tr.State, i)) + } + } + if got := strings.Join(changes, " "); got != "→on@0 on→unknown@2 unknown→hung@8 hung→on@10" { + t.Errorf("changes %s", got) + } + if !tr.Since.Equal(t0.Add(5 * time.Minute)) { + t.Errorf("since %v", tr.Since) + } +} diff --git a/helper/internal/machines/events.go b/helper/internal/machines/events.go new file mode 100644 index 0000000..007044a --- /dev/null +++ b/helper/internal/machines/events.go @@ -0,0 +1,118 @@ +package machines + +import ( + "strconv" + "sync" + "time" +) + +// Event is a machine's state changing, as the hub saw it. +type Event struct { + ID string `json:"id"` + At time.Time `json:"at"` + Event string `json:"event"` // the new state, or "recovered" (hung, then answering) + Machine string `json:"machine"` + State string `json:"state"` + From string `json:"from"` + Reason string `json:"reason,omitempty"` + Asleep bool `json:"asleep,omitempty"` + Evidence Evidence `json:"evidence"` +} + +// EventNames are what an event can be (webhook filters use them). +var EventNames = []string{StateOn, StateOff, StateHung, StateUnknown, "recovered"} + +// maxHeld is how many events the bus keeps for replay. +const maxHeld = 500 + +// Bus keeps the last events for replay and hands new ones to subscribers. IDs are numbers that +// keep growing across restarts (they start from the clock), so an ID from before a restart is +// simply too old to replay. +type Bus struct { + mu sync.Mutex + next int64 + held []Event + subs map[chan Event]bool +} + +// NewBus starts event IDs from now. +func NewBus(now time.Time) *Bus { + return &Bus{next: now.UnixMicro(), subs: map[chan Event]bool{}} +} + +// Publish gives e an ID, keeps it and sends it to every subscriber. A subscriber that isn't +// keeping up is cut off (its channel closed); it can come back and replay from its last ID. +func (b *Bus) Publish(e Event) Event { + b.mu.Lock() + defer b.mu.Unlock() + e.ID = strconv.FormatInt(b.next, 10) + b.next++ + b.held = append(b.held, e) + if len(b.held) > maxHeld { + b.held = append([]Event(nil), b.held[len(b.held)-maxHeld:]...) + } + for ch := range b.subs { + select { + case ch <- e: + default: + delete(b.subs, ch) + close(ch) + } + } + return e +} + +// After returns the events after id, oldest first. ok is false when id isn't one this bus can +// replay from (unknown, or older than what it still holds). +func (b *Bus) After(id string) (events []Event, ok bool) { + n, err := strconv.ParseInt(id, 10, 64) + b.mu.Lock() + defer b.mu.Unlock() + first := b.next + if len(b.held) > 0 { + first, _ = strconv.ParseInt(b.held[0].ID, 10, 64) + } + if err != nil || n < first-1 || n >= b.next { + return nil, false + } + for _, e := range b.held { + if v, _ := strconv.ParseInt(e.ID, 10, 64); v > n { + events = append(events, e) + } + } + return events, true +} + +// Recent returns up to n of the newest events, oldest first. +func (b *Bus) Recent(n int) []Event { + b.mu.Lock() + defer b.mu.Unlock() + if n > len(b.held) { + n = len(b.held) + } + return append([]Event(nil), b.held[len(b.held)-n:]...) +} + +// Subscribe returns a channel of new events and a function to stop. The channel is closed when the +// subscriber falls more than buf events behind. +func (b *Bus) Subscribe(buf int) (<-chan Event, func()) { + ch := make(chan Event, buf) + b.mu.Lock() + b.subs[ch] = true + b.mu.Unlock() + return ch, func() { + b.mu.Lock() + defer b.mu.Unlock() + if b.subs[ch] { + delete(b.subs, ch) + close(ch) + } + } +} + +// Subscribers counts current subscribers. +func (b *Bus) Subscribers() int { + b.mu.Lock() + defer b.mu.Unlock() + return len(b.subs) +} diff --git a/helper/internal/machines/machines.go b/helper/internal/machines/machines.go index db2db45..2afa085 100644 --- a/helper/internal/machines/machines.go +++ b/helper/internal/machines/machines.go @@ -63,10 +63,21 @@ type Hub struct { OffWait time.Duration // between a forced off and the power press that follows (5 s) ResetVerify time.Duration // how long a reset machine may take to answer again (5 min) VerifyEvery time.Duration // between checks while verifying a reset (10 s) + LEDGap time.Duration // between the three light readings of one look (0.5 s) + AskGap time.Duration // between the two tries at a silent helper in one look (1.5 s) + AsleepAfter time.Duration // after a sleep note, when to read the light asleep (30 s) + + // Watching (Watch). Ping is the hub's ping of a machine's address (replies, no_reply, + // unavailable; nil: unavailable); Online is whether Tailscale lists it online (nil: unknown). + Ping func(ctx context.Context, addr string) string + Online func(ctx context.Context, addr string) *bool + HungAfter func() time.Duration // the owner's setting (0: the default) + Events *Bus // state changes; nil: not published mu sync.Mutex busy map[string]bool clients map[string]cached + tracks map[string]*Track wg sync.WaitGroup } @@ -427,9 +438,15 @@ type Machine struct { HelperAnswers *bool `json:"helperAnswers,omitempty"` LED *bool `json:"led,omitempty"` // the power light now; absent without a KVM or reading LEDError string `json:"ledError,omitempty"` - // State: on (helper answers), off (wired LED reads off), unknown (not enough evidence). + // State: on (helper answers), off (wired LED reads off), hung (see Track.Step), unknown (not + // enough evidence, Reason says why). State string `json:"state"` + Reason string `json:"reason,omitempty"` + Asleep bool `json:"asleep,omitempty"` Actions []string `json:"actions"` // what the hub can do for it now + // Since is when the hub's watch first saw the current state; Evidence is its last look. + Since *time.Time `json:"since,omitempty"` + Evidence *Evidence `json:"evidence,omitempty"` } // HasKVM reports whether a machine has a KVM here. @@ -529,11 +546,32 @@ func (h *Hub) look(ctx context.Context, m *Machine) { trusted := m.KVM != nil && m.KVM.Wiring.LED == store.Wired switch { case m.Helper && answers: - m.State = "on" + m.State = StateOn case trusted && led != nil && !*led: - m.State = "off" + m.State = StateOff default: - m.State = "unknown" // phase 3 tells "hung" from "booting" here + m.State, m.Reason = StateUnknown, ReasonNoHelper + if m.Helper { + m.Reason = ReasonLEDNotTrusted + if trusted { + m.Reason = ReasonChecking + } + } + } + // The watch has the history: hung, or why not. What was just seen decides on and off. + if tr, ok := h.Tracked(m.Name); ok && m.KVM != nil { + if m.State == StateUnknown { + m.State, m.Reason = tr.State, tr.Reason + if m.State == StateOn || m.State == StateOff { // it changed since the last look + m.State, m.Reason = StateUnknown, ReasonChecking + } + } + if m.State == tr.State { + since := tr.Since + m.Since, m.Asleep = &since, tr.Asleep + } + ev := tr.Evidence + m.Evidence = &ev } m.Actions = []string{"wake"} if m.Helper { @@ -587,24 +625,6 @@ func (h *Hub) observe(name string, helperUp, led bool) { }) } -// Loop reads every KVM's power light every interval, so the wiring is learned without anyone -// pressing anything (phase 3 builds hang detection on the same readings). -func (h *Hub) Loop(ctx context.Context, interval time.Duration) { - t := time.NewTicker(interval) - defer t.Stop() - for { - if len(h.Store.KVMs()) > 0 { - _ = h.list(ctx, true) - } - select { - case <-ctx.Done(): - h.wg.Wait() - return - case <-t.C: - } - } -} - // ---- actions ---- // Step is one thing an action did, for the reply (and the audit log in phase 4). @@ -762,18 +782,23 @@ func (h *Hub) Act(ctx context.Context, name, action string, o Options) Result { c = unusable{e} // the helper can still wake, shut down or restart it } r.kvm = c + var out Result switch action { case "wake": - return r.wake(hasHelper) + out = r.wake(hasHelper) case "shutdown", "restart": - return r.powerOff(hasHelper, action, o) + out = r.powerOff(hasHelper, action, o) case "reset": - return r.reset(hasHelper, o) + out = r.reset(hasHelper, o) case "force-off": - return r.forceOff(hasHelper, o) + out = r.forceOff(hasHelper, o) default: return r.refuse(http.StatusNotFound, "not_found", "Unknown action %s.", action) } + if out.OK && action != "shutdown" && out.Code == "" { + h.acted(k.Machine) // it's starting: give it time to boot before calling it hung + } + return out } func (r *run) wake(hasHelper bool) Result { diff --git a/helper/internal/machines/machines_test.go b/helper/internal/machines/machines_test.go index e02060d..229f03e 100644 --- a/helper/internal/machines/machines_test.go +++ b/helper/internal/machines/machines_test.go @@ -30,7 +30,8 @@ type fakeFleet struct { forwards []string fwdErr error wakeErr error - silent bool // the helper never answers (e.g. not installed yet) + silent bool // the helper never answers (e.g. not installed yet) + status string // its GET /v2/status reply (default {}) } func (f *fakeFleet) Peer(name string) (store.Peer, error) { @@ -58,14 +59,23 @@ func (f *fakeFleet) Forward(_ context.Context, name, method, path string, body, f.mu.Lock() b, _ := json.Marshal(body) f.forwards = append(f.forwards, method+" "+path+" "+string(b)+" "+who) - err := f.fwdErr + err, silent := f.fwdErr, f.silent f.mu.Unlock() if err != nil { return err } - if !f.kvm.Up() { + if !f.kvm.Up() || silent && method == "GET" { return fleet.ErrOffline } + if method == "GET" && path == "/v2/status" { + f.mu.Lock() + st := f.status + f.mu.Unlock() + if st == "" { + st = "{}" + } + return json.Unmarshal([]byte(st), out) + } return json.Unmarshal([]byte(`{"pending":{"action":"shutdown"}}`), out) } diff --git a/helper/internal/machines/watch.go b/helper/internal/machines/watch.go new file mode 100644 index 0000000..5071a6d --- /dev/null +++ b/helper/internal/machines/watch.go @@ -0,0 +1,303 @@ +package machines + +import ( + "context" + "errors" + "fmt" + "net" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "github.com/projectzero02/powerdeck/helper/internal/fleet" + "github.com/projectzero02/powerdeck/helper/internal/peer" + "github.com/projectzero02/powerdeck/helper/internal/store" +) + +// Watch looks at every machine with a KVM here every interval (30 s): it tells hung from booting, +// asleep or updating, publishes state changes on Events, and learns the wiring from the light. +func (h *Hub) Watch(ctx context.Context, interval time.Duration) { + t := time.NewTicker(interval) + defer t.Stop() + for { + h.WatchOnce(ctx) + select { + case <-ctx.Done(): + h.wg.Wait() + return + case <-t.C: + } + } +} + +// WatchOnce takes one look at every machine with a KVM here. +func (h *Hub) WatchOnce(ctx context.Context) { + var wg sync.WaitGroup + for _, k := range h.Store.KVMs() { + wg.Add(1) + go func(k store.KVM) { + defer wg.Done() + h.watch(ctx, k) + }(k) + } + wg.Wait() +} + +// lookTime bounds one look: the helper twice, three light readings, a ping and a connection. +const lookTime = 8 * time.Second + +func (h *Hub) watch(ctx context.Context, k store.KVM) { + ctx, cancel := context.WithTimeout(ctx, lookTime) + defer cancel() + p, err := h.Fleet.Peer(k.Machine) + hasHelper := err == nil + e := Evidence{At: h.now().UTC()} + var ( + word *Word + answers bool + led string + ledErr error + wg sync.WaitGroup + ) + if hasHelper { + wg.Add(4) + go func() { defer wg.Done(); answers, word = h.ask(ctx, k.Machine) }() + go func() { defer wg.Done(); e.Port = h.port(ctx, p.Address, p.Port) }() + go func() { defer wg.Done(); e.Ping = h.ping(ctx, p.Address) }() + go func() { defer wg.Done(); e.Tailnet = h.online(ctx, p.Address) }() + } + led, ledErr = h.sampleLED(ctx, k) + wg.Wait() + if hasHelper { + e.Helper = map[bool]string{true: Answers, false: Silent}[answers] + } + e.LED = led + if ledErr != nil { + e.LEDError = ledErr.Error() + } + if led == On || led == Off { + h.observe(k.Name, answers, led == On) + } + if fresh, ok := h.Store.KVM(k.Name); ok { + k = fresh + } + e.LEDTrusted = k.Wiring.LED == store.Wired + + h.mu.Lock() + tr := h.track(k.Machine) + if word != nil { + tr.Word = *word + } + if tr.Word.Sleep == nil && hasHelper { + tr.Word.Sleep = p.Sleep // what it said before the hub last started + } + from, changed := tr.Step(e, h.rules(k)) + now := *tr + h.mu.Unlock() + if changed && from != "" { + h.publish(k.Machine, from, now) + } +} + +// track is the detector's memory for a machine (h.mu held). +func (h *Hub) track(machine string) *Track { + if h.tracks == nil { + h.tracks = map[string]*Track{} + } + key := strings.ToLower(machine) + tr := h.tracks[key] + if tr == nil { + tr = &Track{} + h.tracks[key] = tr + } + return tr +} + +// Tracked is the detector's view of a machine, if it has looked at it. +func (h *Hub) Tracked(machine string) (Track, bool) { + h.mu.Lock() + defer h.mu.Unlock() + tr, ok := h.tracks[strings.ToLower(machine)] + if !ok || tr.State == "" { + return Track{}, false + } + return *tr, true +} + +func (h *Hub) rules(k store.KVM) Rules { + r := DefaultRules + if h.HungAfter != nil { + if d := h.HungAfter(); d > 0 { + r.HungAfter = d + } + } + r.LEDAsleep = k.Wiring.LEDAsleep + return r +} + +func (h *Hub) publish(machine, from string, tr Track) { + ev := Event{At: tr.Since, Event: tr.State, Machine: machine, State: tr.State, From: from, Reason: tr.Reason, + Asleep: tr.Asleep, Evidence: tr.Evidence} + if from == StateHung && tr.State == StateOn { + ev.Event = "recovered" + } + if h.Events != nil { + ev = h.Events.Publish(ev) + } + switch ev.Event { + case StateHung: + h.log().Warn("machine is hung", "machine", machine, "evidence", fmt.Sprintf("%+v", tr.Evidence)) + h.Store.AddEvent(store.Event{At: ev.At, Who: h.Self, Action: machine + " looks frozen: nothing answers, power light on", Result: "detected"}) + case "recovered": + h.log().Info("machine answers again", "machine", machine) + h.Store.AddEvent(store.Event{At: ev.At, Who: h.Self, Action: machine + " answers again", Result: "recovered"}) + default: + h.log().Info("machine state", "machine", machine, "state", tr.State, "from", from, "reason", tr.Reason) + } +} + +// acted gives a machine the hub just started, restarted, reset or forced off time to boot. +func (h *Hub) acted(machine string) { + h.mu.Lock() + defer h.mu.Unlock() + h.track(machine).Acted = h.now().UTC() +} + +// ask gets the machine's helper's status (twice before calling it silent). answers is true when +// the helper replied at all; word is set when it replied with its status. +func (h *Hub) ask(ctx context.Context, machine string) (answers bool, word *Word) { + for i := 0; i < 2; i++ { + var st struct { + Pending *struct { + Action string `json:"action"` + At time.Time `json:"at"` + } `json:"pending"` + UpdatesPending bool `json:"updatesPending"` + Sleep *store.Sleep `json:"sleep"` + } + err := h.Fleet.Forward(ctx, machine, "GET", "/v2/status", nil, &st, "") + var pe *peer.Error + switch { + case err == nil: + w := &Word{At: h.now().UTC(), UpdatesPending: st.UpdatesPending, Sleep: st.Sleep} + if st.Pending != nil { + w.PendingAction, w.PendingAt = st.Pending.Action, st.Pending.At + } + return true, w + case errors.As(err, &pe): + return true, nil // it replied (without a key for it: not linked yet) + case !errors.Is(err, fleet.ErrOffline): + return false, nil + } + if i == 0 { + select { + case <-ctx.Done(): + return false, nil + case <-time.After(dur(h.AskGap, 1500*time.Millisecond)): + } + } + } + return false, nil +} + +// port connects to the helper's port: refused means the machine's OS is up and answered. +func (h *Hub) port(ctx context.Context, addr string, port int) string { + if port == 0 { + port = peer.Port + } + ctx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + var d net.Dialer + c, err := d.DialContext(ctx, "tcp", net.JoinHostPort(addr, strconv.Itoa(port))) + if err == nil { + c.Close() + return Accepts + } + if errors.Is(err, syscall.ECONNREFUSED) || strings.Contains(err.Error(), "refused") { + return Refuses + } + return NoReply +} + +func (h *Hub) ping(ctx context.Context, addr string) string { + if h.Ping == nil { + return Unavailable + } + return h.Ping(ctx, addr) +} + +func (h *Hub) online(ctx context.Context, addr string) *bool { + if h.Online == nil { + return nil + } + return h.Online(ctx, addr) +} + +// sampleLED reads the light three times, half a second apart: on and off within one look is +// blinking, which many boards do while asleep. +func (h *Hub) sampleLED(ctx context.Context, k store.KVM) (string, error) { + var on, off int + var last error + for i := 0; i < 3; i++ { + if i > 0 { + select { + case <-ctx.Done(): + case <-time.After(dur(h.LEDGap, 500*time.Millisecond)): + } + } + led, err := h.readLED(ctx, k) + switch { + case err != nil: + last = err + case *led: + on++ + default: + off++ + } + } + switch { + case on > 0 && off > 0: + return Blinking, nil + case on > 0: + return On, nil + case off > 0: + return Off, nil + } + return Unreadable, last +} + +// Note records what a machine's helper said is about to happen. After a sleep note, the hub +// learns what the machine's light does while it sleeps. +func (h *Hub) Note(machine string, n Note) { + n.At = h.now().UTC() + h.mu.Lock() + h.track(machine).Note = &n + h.mu.Unlock() + h.log().Info("machine says", "machine", machine, "event", n.Event, "updates", n.Updates) + k, ok := h.Store.KVMFor(machine) + if n.Event != "sleep" || !ok { + return + } + h.wg.Add(1) + go func() { + defer h.wg.Done() + time.Sleep(dur(h.AsleepAfter, 30*time.Second)) + ctx, cancel := context.WithTimeout(context.Background(), lookTime) + defer cancel() + if h.Fleet.Answers(ctx, machine) { + return // it didn't sleep after all + } + led, _ := h.sampleLED(ctx, k) + if led == Unreadable { + return + } + _ = h.Store.UpdateKVM(k.Name, func(k *store.KVM) { + if k.Wiring.LEDAsleep != led { + h.log().Info("learned the power light while asleep", "kvm", k.Name, "led", led) + k.Wiring.LEDAsleep = led + } + }) + }() +} diff --git a/helper/internal/machines/watch_test.go b/helper/internal/machines/watch_test.go new file mode 100644 index 0000000..1cf5bde --- /dev/null +++ b/helper/internal/machines/watch_test.go @@ -0,0 +1,307 @@ +package machines + +import ( + "context" + "net" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/projectzero02/powerdeck/helper/internal/nanokvm/fakekvm" + "github.com/projectzero02/powerdeck/helper/internal/store" +) + +// watchEnv is a hub watching gaming-pc: a fake KVM, a fake helper, a fake ping that answers while +// the OS is up (or always, with pingUp), and the helper's port on a real loopback listener that +// accepts and never answers (what a hung Windows does). +type watchEnv struct { + *env + mu sync.Mutex + clock time.Time + pingUp bool + events []Event +} + +func newWatchEnv(t *testing.T, o fakekvm.Options) *watchEnv { + t.Helper() + w := &watchEnv{env: newEnv(t, o), clock: t0} + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { ln.Close() }) + go func() { // accept, say nothing + var held []net.Conn + for { + c, err := ln.Accept() + if err != nil { + for _, c := range held { + c.Close() + } + return + } + held = append(held, c) + } + }() + w.setPort(ln.Addr().(*net.TCPAddr).Port) + w.hub.Now = func() time.Time { w.mu.Lock(); defer w.mu.Unlock(); return w.clock } + w.hub.LEDGap, w.hub.AskGap, w.hub.AsleepAfter = time.Millisecond, time.Millisecond, time.Millisecond + w.hub.Ping = func(context.Context, string) string { + w.mu.Lock() + up := w.pingUp + w.mu.Unlock() + if up || w.kvm.Up() { + return Replies + } + return NoReply + } + w.hub.Events = NewBus(t0) + ch, stop := w.hub.Events.Subscribe(100) + t.Cleanup(stop) + go func() { + for e := range ch { + w.mu.Lock() + w.events = append(w.events, e) + w.mu.Unlock() + } + }() + return w +} + +func (w *watchEnv) setPort(port int) { + p, _ := w.st.Peer("gaming-pc") + p.Address, p.Port = "127.0.0.1", port + _ = w.st.PutPeer(p) + w.fleet.peers["gaming-pc"] = p +} + +// rounds takes n looks, 30 s apart, and returns the state after the last. +func (w *watchEnv) rounds(n int) Track { + for i := 0; i < n; i++ { + w.hub.WatchOnce(context.Background()) + w.mu.Lock() + w.clock = w.clock.Add(30 * time.Second) + w.mu.Unlock() + } + tr, _ := w.hub.Tracked("gaming-pc") + return tr +} + +func (w *watchEnv) eventNames() string { + time.Sleep(20 * time.Millisecond) // let the subscriber catch up + w.mu.Lock() + defer w.mu.Unlock() + var out []string + for _, e := range w.events { + out = append(out, e.Event) + } + return strings.Join(out, ",") +} + +func TestWatchFindsAHangAndTheRecovery(t *testing.T) { + w := newWatchEnv(t, fakekvm.Options{On: true}) + w.link("gaming-pc") + if tr := w.rounds(1); tr.State != StateOn { + t.Fatalf("on: %+v", tr) + } + if w := w.wiring(); w.LED != store.Wired { + t.Fatalf("the light was seen on while the helper answered: %+v", w) + } + w.kvm.Hang() + tr := w.rounds(6) + if tr.State != StateUnknown || tr.Reason != ReasonChecking { + t.Fatalf("2.5 min silent: %+v", tr) + } + tr = w.rounds(1) + if tr.State != StateHung { + t.Fatalf("3 min silent: %+v", tr) + } + e := tr.Evidence + if e.Helper != Silent || e.Port != Accepts || e.Ping != NoReply || e.LED != On || !e.LEDTrusted { + t.Errorf("evidence %+v", e) + } + m, _ := w.hub.Get(context.Background(), "gaming-pc") + if m.State != StateHung || m.Since == nil || !m.Since.Equal(t0.Add(3*time.Minute+30*time.Second)) || m.Evidence == nil { + t.Errorf("machine %+v", m) + } + if got := w.eventNames(); got != "unknown,hung" { + t.Errorf("events %s", got) + } + found := false + for _, ev := range w.st.Events() { + found = found || ev.Action == "gaming-pc looks frozen: nothing answers, power light on" && ev.Result == "detected" + } + if !found { + t.Errorf("not in the activity log: %+v", w.st.Events()) + } + + // Reset it: booting, then answering again. + if r := w.act("reset", Options{}); !r.OK { + t.Fatalf("reset %+v", r) + } + time.Sleep(60 * time.Millisecond) // the fake boots in 30 ms + if tr := w.rounds(1); tr.State != StateOn { + t.Fatalf("after reset: %+v", tr) + } + if got := w.eventNames(); got != "unknown,hung,recovered" { + t.Errorf("events %s", got) + } +} + +func TestWatchBootGraceAfterAWake(t *testing.T) { + w := newWatchEnv(t, fakekvm.Options{On: true, BootDelay: time.Hour}) + w.link("gaming-pc") + w.rounds(1) + w.kvm.SetOn(false) + if tr := w.rounds(1); tr.State != StateOff { + t.Fatalf("off: %+v", tr) + } + if r := w.act("wake", Options{}); !r.OK { + t.Fatalf("wake %+v", r) + } + // The fake takes an hour to boot: silent, light on. Booting for 10 minutes, then counted. + if tr := w.rounds(20); tr.State != StateUnknown || tr.Reason != ReasonBooting { + t.Fatalf("after 10 min: %+v", tr) + } + if tr := w.rounds(8); tr.State != StateHung { + t.Fatalf("after 14 min: %+v", tr) + } +} + +func TestWatchHelperStopped(t *testing.T) { + w := newWatchEnv(t, fakekvm.Options{On: true}) + w.link("gaming-pc") + w.rounds(1) + w.fleet.mu.Lock() + w.fleet.silent = true // the helper crashed; the machine is fine + w.fleet.mu.Unlock() + if tr := w.rounds(10); tr.State != StateUnknown || tr.Reason != ReasonHelperStopped || tr.Evidence.Ping != Replies { + t.Fatalf("%+v", tr) + } + + // Without a ping (no Tailscale), the port refusing says the same. + w.hub.Ping = nil + ln, _ := net.Listen("tcp", "127.0.0.1:0") + port := ln.Addr().(*net.TCPAddr).Port + ln.Close() + w.setPort(port) + if tr := w.rounds(10); tr.State != StateUnknown || tr.Reason != ReasonHelperStopped || + tr.Evidence.Port != Refuses || tr.Evidence.Ping != Unavailable { + t.Fatalf("%+v", tr) + } +} + +func TestWatchSleep(t *testing.T) { + for _, c := range []struct { + led string + state string + }{{"off", "off/asleep"}, {"blinking", "off/asleep"}, {"on", "unknown/may_be_asleep"}} { + t.Run(c.led, func(t *testing.T) { + w := newWatchEnv(t, fakekvm.Options{On: true, SleepLED: c.led}) + w.link("gaming-pc") + w.rounds(1) + w.hub.Note("gaming-pc", Note{Event: "sleep"}) + w.kvm.Sleep() + w.hub.Wait() // the light asleep is learned + if got := w.wiring().LEDAsleep; got != c.led { + t.Errorf("learned the light asleep as %q", got) + } + tr := w.rounds(20) + got := tr.State + "/" + tr.Reason + if tr.Asleep { + got = tr.State + "/asleep" + } + if got != c.state { + t.Errorf("asleep 10 min: %s, want %s", got, c.state) + } + }) + } +} + +func TestWatchModernStandby(t *testing.T) { + w := newWatchEnv(t, fakekvm.Options{On: true, SleepLED: "on"}) + w.fleet.status = `{"sleep":{"s3":false,"modernStandby":true,"afterMinutes":30}}` + w.link("gaming-pc") + w.rounds(1) + w.kvm.Sleep() // no note: Windows didn't send one + if tr := w.rounds(20); tr.State != StateUnknown || tr.Reason != ReasonMayBeAsleep { + t.Fatalf("%+v", tr) + } + // It wakes up and goes back to work: answering again. + if r := w.act("wake", Options{}); !r.OK { + t.Fatalf("%+v", r) + } +} + +func TestWatchUpdatesAtRestart(t *testing.T) { + w := newWatchEnv(t, fakekvm.Options{On: true}) + w.fleet.status = `{"updatesPending":true,"pending":{"action":"restart","at":"` + t0.Add(time.Minute).Format(time.RFC3339) + `"}}` + w.link("gaming-pc") + w.rounds(1) + w.kvm.Hang() // "Working on updates": light on, nothing answers + if tr := w.rounds(60); tr.State != StateUnknown || tr.Reason != ReasonUpdating { + t.Fatalf("30 min: %+v", tr) + } + if got := w.eventNames(); got != "unknown" { + t.Errorf("events %s", got) + } +} + +func TestWatchMachineWithoutHelper(t *testing.T) { + w := newWatchEnv(t, fakekvm.Options{On: true}) + _, _, err := w.hub.Link(context.Background(), LinkRequest{Name: "kvm", Address: w.kvm.URL, Password: "admin", + Machine: "nas", MAC: "02:00:5e:00:00:09"}) + if err != nil { + t.Fatal(err) + } + if tr, _ := w.hub.Tracked("nas"); tr.State != "" { + t.Fatal(tr) + } + w.kvm.Hang() + w.rounds(20) + if tr, _ := w.hub.Tracked("nas"); tr.State != StateUnknown || tr.Reason != ReasonNoHelper { + t.Fatalf("%+v", tr) + } +} + +func TestBusReplay(t *testing.T) { + b := NewBus(t0) + var ids []string + for i := 0; i < maxHeld+10; i++ { + ids = append(ids, b.Publish(Event{Machine: strconv.Itoa(i)}).ID) + } + if got, ok := b.After(ids[len(ids)-3]); !ok || len(got) != 2 || got[1].ID != ids[len(ids)-1] { + t.Errorf("after the third last: %v %v", ok, got) + } + if got, ok := b.After(ids[len(ids)-1]); !ok || len(got) != 0 { + t.Errorf("after the last: %v %v", ok, got) + } + if _, ok := b.After(ids[0]); ok { + t.Error("replayed from an ID no longer held") + } + if _, ok := b.After("12"); ok { + t.Error("replayed from an ID before this bus") + } + if _, ok := b.After("x"); ok { + t.Error("replayed from garbage") + } + if b2 := NewBus(t0.Add(time.Hour)); b2.Publish(Event{}).ID <= ids[len(ids)-1] { + t.Error("IDs after a restart don't grow") + } + + // A subscriber that stops reading is cut off. + ch, stop := b.Subscribe(2) + defer stop() + for i := 0; i < 3; i++ { + b.Publish(Event{}) + } + n := 0 + for range ch { + n++ + } + if n != 2 || b.Subscribers() != 0 { + t.Errorf("got %d before the cut, %d subscribers", n, b.Subscribers()) + } +} diff --git a/helper/internal/nanokvm/fakekvm/fakekvm.go b/helper/internal/nanokvm/fakekvm/fakekvm.go index 214118a..d9052be 100644 --- a/helper/internal/nanokvm/fakekvm/fakekvm.go +++ b/helper/internal/nanokvm/fakekvm/fakekvm.go @@ -45,6 +45,9 @@ type Options struct { BootDelay time.Duration // default 30 s (scaled) ShutdownDelay time.Duration // default 20 s (scaled) TimeScale float64 // default 1 + // SleepLED is what the power light does while the computer sleeps: "off" (default), + // "blinking" (on and off on alternate readings) or "on" (Modern Standby on many boards). + SleepLED string } // Press is one button press the KVM received. @@ -64,6 +67,8 @@ type Fake struct { mu sync.Mutex on bool hung bool + asleep bool + blink bool // the next reading while asleep and blinking up bool // the OS is up (the helper would answer) gen int // cancels pending boot/shutdown timers reboots int @@ -154,7 +159,18 @@ func (f *Fake) SetOn(on bool) { f.mu.Lock() defer f.mu.Unlock() f.gen++ - f.on, f.up, f.hung = on, on, false + f.on, f.up, f.hung, f.asleep = on, on, false, false +} + +// Sleep puts the computer to sleep: nothing answers, the light does Options.SleepLED, and a short +// power press wakes it. +func (f *Fake) Sleep() { + f.mu.Lock() + defer f.mu.Unlock() + if f.on && !f.hung { + f.gen++ + f.asleep, f.up = true, false + } } // Hang freezes the OS: the power light stays on, nothing answers, short presses are ignored. @@ -221,16 +237,18 @@ func (f *Fake) pressed(button string, d time.Duration) { if f.opt.NoResetSwitch || !f.on { return } - f.hung = false + f.hung, f.asleep = false, false f.reboots++ f.bootLocked() case f.opt.NoPowerSwitch: case d >= 4*time.Second: // ATX: holding power cuts it f.gen++ - f.on, f.up, f.hung = false, false, false + f.on, f.up, f.hung, f.asleep = false, false, false, false case !f.on: f.on = true f.bootLocked() + case f.asleep: + f.asleep, f.up = false, true case f.hung: // A frozen OS doesn't handle the power-button event. default: @@ -349,6 +367,16 @@ func (f *Fake) getGPIO(w http.ResponseWriter, _ *http.Request) { } f.mu.Lock() pwr := f.on && !f.opt.NoLED + if f.asleep && pwr { + switch f.opt.SleepLED { + case "on": + case "blinking": + f.blink = !f.blink + pwr = f.blink + default: + pwr = false + } + } f.mu.Unlock() send(w, 0, "success", map[string]bool{"pwr": pwr, "hdd": false}) } diff --git a/helper/internal/server/server.go b/helper/internal/server/server.go index 6582277..d5ba97d 100644 --- a/helper/internal/server/server.go +++ b/helper/internal/server/server.go @@ -10,12 +10,14 @@ import ( "context" "encoding/json" "errors" + "fmt" "log/slog" "net" "net/http" "net/netip" "strings" "sync" + "sync/atomic" "time" "unicode" @@ -32,6 +34,7 @@ import ( "github.com/projectzero02/powerdeck/helper/internal/tailscale" "github.com/projectzero02/powerdeck/helper/internal/update" "github.com/projectzero02/powerdeck/helper/internal/version" + "github.com/projectzero02/powerdeck/helper/internal/webhook" ) // Deps are the server's collaborators. @@ -41,7 +44,8 @@ type Deps struct { Pairing *pairing.Manager Power *power.Controller Fleet *fleet.Fleet - Machines *machines.Hub // KVMs this helper drives (nil: none) + Machines *machines.Hub // KVMs this helper drives (nil: none) + Webhooks *webhook.Sender // posts machine events (nil: not here) Name string OS string SelfNode func(ctx context.Context) string // this device's tailnet node ID @@ -51,6 +55,8 @@ type Deps struct { Guests func(ctx context.Context) *guests.Info Wake func(mac string, lans []string) ([]string, error) Update Updater // nil: updates aren't managed here (tests, development builds) + // Sleep reports whether this machine sleeps on its own (nil, or returning nil: not known here). + Sleep func() *store.Sleep // Uninstall removes PowerDeck from this device in the background (nil: not supported here). Uninstall func() error // PairLink returns this device's add link (QR payload, with the code) and the current pairing code. @@ -73,6 +79,8 @@ type server struct { limiter *limiter quota *quota + streams atomic.Int32 // open machine event streams + checksMu sync.Mutex checks []check.Result // cached: on Windows they run PowerShell, and the app polls every few seconds checksAt time.Time @@ -133,6 +141,12 @@ func New(d Deps) *API { mux.HandleFunc("PATCH /v2/kvms/{name}", s.with(store.ScopeAdmin, s.ownersOnly(s.patchKVM))) mux.HandleFunc("DELETE /v2/kvms/{name}", s.with(store.ScopeAdmin, s.ownersOnly(s.removeKVM))) mux.HandleFunc("GET /v2/machines", s.with(store.ScopeStatus, s.listMachines)) + mux.HandleFunc("GET /v2/machines/events", s.with(store.ScopeStatus, s.machineEvents)) + mux.HandleFunc("POST /v2/note", s.with(store.ScopeStatus, s.note)) + mux.HandleFunc("GET /v2/webhooks", s.with(store.ScopeAdmin, s.ownersOnly(s.listWebhooks))) + mux.HandleFunc("POST /v2/webhooks", s.with(store.ScopeAdmin, s.ownersOnly(s.addWebhook))) + mux.HandleFunc("DELETE /v2/webhooks/{id}", s.with(store.ScopeAdmin, s.ownersOnly(s.removeWebhook))) + mux.HandleFunc("POST /v2/webhooks/{id}/test", s.with(store.ScopeAdmin, s.ownersOnly(s.testWebhook))) mux.HandleFunc("POST /v2/machines/{name}/{action}", s.with(store.ScopeStatus, s.machineAction)) mux.HandleFunc("POST /mcp", s.with(store.ScopeStatus, s.ownersAndAgents(s.mcp))) mux.HandleFunc("GET /mcp", func(w http.ResponseWriter, _ *http.Request) { @@ -312,6 +326,14 @@ func (s *server) logEvent(who, action, result string) { s.Log.Info("event", "who", who, "action", action, "result", result) } +// hungAfter is the hang setting in minutes, with the default applied. +func hungAfter(st store.Settings) int { + if st.HungAfterMinutes > 0 { + return st.HungAfterMinutes + } + return int(machines.DefaultRules.HungAfter / time.Minute) +} + func (s *server) hello(w http.ResponseWriter, _ *http.Request) { writeJSON(w, http.StatusOK, map[string]any{ "app": "PowerDeck", "apiVersion": version.APIVersion, "version": version.Version, "name": s.Name, "os": s.OS, @@ -506,6 +528,14 @@ func (s *server) statusBody(ctx context.Context, c auth.Caller) map[string]any { } if s.Machines != nil { body["caps"] = append(body["caps"].([]string), "kvm") // drives NanoKVMs (/v2/kvms, /v2/machines) + if c.Kind == auth.Owner { + body["hub"] = map[string]any{"hungAfterMinutes": hungAfter(s.Store.Settings())} + } + } + if sl := s.Sleep; sl != nil { + if f := sl(); f != nil { + body["sleep"] = f // so a hub can tell sleep from a hang + } } if g := s.Guests(ctx); g != nil { body["guests"] = g @@ -622,11 +652,24 @@ func (s *server) putSettings(w http.ResponseWriter, r *http.Request, c auth.Call AutoUpdate *bool `json:"autoUpdate"` Agents *bool `json:"agents"` PauseUpdates *bool `json:"pauseUpdates"` // true: hold automatic installs for 7 days; false: resume + // HungAfterMinutes: how long a machine with a KVM here must look hung (2 to 30; 0: default 3). + HungAfterMinutes *int `json:"hungAfterMinutes"` } if !decode(r, &req) { fail(w, http.StatusBadRequest, "bad_request", "Malformed settings.") return } + if n := req.HungAfterMinutes; n != nil { + if *n != 0 && (*n < 2 || *n > 30) { + fail(w, http.StatusBadRequest, "bad_request", "hungAfterMinutes is 2 to 30 (0 for the default, 3).") + return + } + if err := s.Store.UpdateSettings(func(st *store.Settings) { st.HungAfterMinutes = *n }); err != nil { + fail(w, http.StatusInternalServerError, "store", err.Error()) + return + } + s.logEvent(c.Name, fmt.Sprintf("set hung after to %d minutes", hungAfter(s.Store.Settings())), "ok") + } if req.Agents != nil && *req.Agents == s.Store.Settings().NoAgents { if err := s.Store.UpdateSettings(func(st *store.Settings) { st.NoAgents = !*req.Agents }); err != nil { fail(w, http.StatusInternalServerError, "store", err.Error()) diff --git a/helper/internal/server/server_test.go b/helper/internal/server/server_test.go index c032673..d8b45ea 100644 --- a/helper/internal/server/server_test.go +++ b/helper/internal/server/server_test.go @@ -41,6 +41,7 @@ const ( selfAddr = "100.101.1.10:51234" // this device itself laptop = "100.101.1.41:51234" // same Tailscale user, not paired as an owner device hubAddr = "100.101.1.12:51234" // another helper of the owner, untagged (e.g. a server) + wsAddr = "100.101.1.11:51234" // workstation, the machine with a KVM in the hub tests ) type fakeTS struct{} @@ -57,7 +58,7 @@ func (fakeTS) Status(context.Context) (*tailscale.Status, error) { func (fakeTS) WhoIs(_ context.Context, ip netip.Addr) (tailscale.Identity, error) { switch ip.String() { case "100.101.1.40": - return tailscale.Identity{NodeID: "nPHONE", NodeName: "oppo", Hostname: "Pixel 9 Pro", UserID: 1}, nil + return tailscale.Identity{NodeID: "nPHONE", NodeName: "pixel-9-pro", Hostname: "Pixel 9 Pro", UserID: 1}, nil case "100.101.1.60": return tailscale.Identity{NodeID: "nFRIEND", NodeName: "friend", Hostname: "Friend's phone", UserID: 7}, nil case "100.101.1.50": @@ -66,6 +67,8 @@ func (fakeTS) WhoIs(_ context.Context, ip netip.Addr) (tailscale.Identity, error return tailscale.Identity{NodeID: "nTWIN", NodeName: "twin", UserID: 99, Tagged: true}, nil case "100.101.1.41": return tailscale.Identity{NodeID: "nLAPTOP", NodeName: "laptop", Hostname: "Laptop", UserID: 1}, nil + case "100.101.1.11": + return tailscale.Identity{NodeID: "nWS", NodeName: "workstation", Hostname: "Workstation", UserID: 1}, nil case "100.101.1.12": return tailscale.Identity{NodeID: "nHUB", NodeName: "hub", Hostname: "hub", UserID: 1}, nil case "100.101.1.10": diff --git a/helper/internal/server/watch.go b/helper/internal/server/watch.go new file mode 100644 index 0000000..070e883 --- /dev/null +++ b/helper/internal/server/watch.go @@ -0,0 +1,277 @@ +package server + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "slices" + "strconv" + "strings" + "time" + + "github.com/projectzero02/powerdeck/helper/internal/auth" + "github.com/projectzero02/powerdeck/helper/internal/machines" + "github.com/projectzero02/powerdeck/helper/internal/store" + "github.com/projectzero02/powerdeck/helper/internal/webhook" +) + +// ---- machine events (the hub's watch) ---- + +// maxStreams bounds open event streams. +const maxStreams = 16 + +// heartbeat keeps proxies from closing a quiet stream. +var heartbeat = 25 * time.Second + +// mayWatch reports whether the caller may see a machine (the same rule as GET /v2/machines). +func (s *server) mayWatch(c auth.Caller, machine string) bool { + names := []string{machine} + if p, err := s.Fleet.Peer(machine); err == nil { + names = peerNames(p) + } + return mayUseAny(c, names) +} + +// machineEvents streams machine state changes as Server-Sent Events: a snapshot first (or, with +// Last-Event-ID, what came after that event), then each change, with a comment every 25 s. +func (s *server) machineEvents(w http.ResponseWriter, r *http.Request, c auth.Caller) { + if s.noHub(w) { + return + } + bus := s.Machines.Events + if bus == nil { + fail(w, http.StatusNotFound, "not_found", "This helper doesn't watch machines.") + return + } + if s.streams.Add(1) > maxStreams { + s.streams.Add(-1) + fail(w, http.StatusServiceUnavailable, "busy", "Too many event streams are open. Try again later.") + return + } + defer s.streams.Add(-1) + rc := http.NewResponseController(w) // each write sets its own deadline: the server's would end the stream + ch, stop := bus.Subscribe(64) // before the replay, so nothing falls in between + defer stop() + + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-store") + w.Header().Set("X-Accel-Buffering", "no") + w.WriteHeader(http.StatusOK) + var last int64 + send := func(name, id string, v any) error { + b, err := json.Marshal(v) + if err != nil { + return err + } + _ = rc.SetWriteDeadline(time.Now().Add(10 * time.Second)) // a client that stops reading is dropped + if id != "" { + fmt.Fprintf(w, "id: %s\n", id) + } + if _, err := fmt.Fprintf(w, "event: %s\ndata: %s\n\n", name, b); err != nil { + return err + } + return rc.Flush() + } + event := func(e machines.Event) error { + n, _ := strconv.ParseInt(e.ID, 10, 64) + if n <= last { + return nil + } + last = n + if !s.mayWatch(c, e.Machine) { + return nil + } + return send("machine", e.ID, e) + } + + from := r.Header.Get("Last-Event-ID") + if from == "" { + from = r.URL.Query().Get("lastEventId") // for clients that can't set headers + } + replay, ok := bus.After(from) + if from == "" || !ok { + newest := "" + if recent := bus.Recent(1); len(recent) > 0 { + newest = recent[0].ID + last, _ = strconv.ParseInt(newest, 10, 64) + } + ms := []machines.Machine{} + for _, m := range s.Machines.KVMMachines(r.Context()) { + if s.mayWatch(c, m.Name) { + ms = append(ms, m) + } + } + if send("snapshot", newest, map[string]any{"machines": ms}) != nil { + return + } + } + for _, e := range replay { + if event(e) != nil { + return + } + } + tick := time.NewTicker(heartbeat) + defer tick.Stop() + for { + select { + case <-r.Context().Done(): + return + case <-tick.C: + _ = rc.SetWriteDeadline(time.Now().Add(10 * time.Second)) + if _, err := fmt.Fprint(w, ": still here\n\n"); err != nil || rc.Flush() != nil { + return + } + case e, open := <-ch: + if !open || event(e) != nil { + return // fell behind (it reconnects and replays) or went away + } + } + } +} + +// note takes a machine's helper's word that it's about to sleep, shut down or restart, or has +// woken: only from a peer key, about the machine it calls from. +func (s *server) note(w http.ResponseWriter, r *http.Request, c auth.Caller) { + if s.noHub(w) { + return + } + if !c.IsPeer() { + fail(w, http.StatusForbidden, "scope", "Only a machine's own PowerDeck helper can say what it's about to do.") + return + } + var n machines.Note + if !decode(r, &n) || !slices.Contains(machines.NoteEvents, n.Event) { + fail(w, http.StatusBadRequest, "bad_request", "Send {\"event\": \"sleep|shutdown|restart|resume\"}.") + return + } + ip := remoteAddr(r).String() + for _, p := range s.Store.Peers() { + if p.Address == ip && s.Machines.HasKVM(p.Name) { + s.Machines.Note(p.Name, n) + w.WriteHeader(http.StatusNoContent) + return + } + } + w.WriteHeader(http.StatusNoContent) // no KVM here for it: nothing to watch +} + +// ---- webhooks ---- + +type webhookView struct { + ID string `json:"id"` + URL string `json:"url"` + Events []string `json:"events"` + CreatedBy string `json:"createdBy,omitempty"` + CreatedAt time.Time `json:"createdAt"` + LastAt time.Time `json:"lastAt,omitzero"` + LastResult string `json:"lastResult,omitempty"` + Secret string `json:"secret,omitempty"` // only when it's created +} + +func viewWebhook(h store.Webhook) webhookView { + return webhookView{ID: h.ID, URL: h.URL, Events: h.Events, CreatedBy: h.CreatedBy, CreatedAt: h.CreatedAt, + LastAt: h.LastAt, LastResult: h.LastResult} +} + +func (s *server) listWebhooks(w http.ResponseWriter, _ *http.Request, _ auth.Caller) { + if s.noHub(w) { + return + } + out := []webhookView{} + for _, h := range s.Store.Webhooks() { + out = append(out, viewWebhook(h)) + } + writeJSON(w, http.StatusOK, map[string]any{"webhooks": out}) +} + +func (s *server) addWebhook(w http.ResponseWriter, r *http.Request, c auth.Caller) { + if s.noHub(w) { + return + } + var req struct { + URL string `json:"url"` + Events []string `json:"events"` + } + if !decode(r, &req) { + fail(w, http.StatusBadRequest, "bad_request", "Malformed request.") + return + } + if err := webhook.CheckURL(req.URL); err != nil { + fail(w, http.StatusBadRequest, "bad_url", err.Error()) + return + } + if len(req.Events) == 0 { + req.Events = webhook.DefaultEvents + } + for _, e := range req.Events { + if !slices.Contains(machines.EventNames, e) { + fail(w, http.StatusBadRequest, "bad_event", "Events are "+strings.Join(machines.EventNames, ", ")+".") + return + } + } + secret := store.NewWebhookSecret() + sealed, err := s.Machines.Box.Seal(secret) + if err != nil { + fail(w, http.StatusInternalServerError, "seal", err.Error()) + return + } + h := store.Webhook{ID: store.NewID(), URL: req.URL, Events: slices.Compact(slices.Sorted(slices.Values(req.Events))), + Secret: sealed, CreatedBy: c.Name, CreatedAt: s.Now().UTC()} + if err := s.Store.AddWebhook(h); err != nil { + if errors.Is(err, store.ErrTooManyWebhooks) { + fail(w, http.StatusConflict, "too_many", fmt.Sprintf("There are %d webhooks already. Remove one first.", store.MaxWebhooks)) + return + } + fail(w, http.StatusInternalServerError, "store", err.Error()) + return + } + s.logEvent(c.Name, "add webhook "+clean(h.URL, 80), "ok") + v := viewWebhook(h) + v.Secret = secret // shown once + writeJSON(w, http.StatusCreated, map[string]any{"webhook": v}) +} + +func (s *server) removeWebhook(w http.ResponseWriter, r *http.Request, c auth.Caller) { + if s.noHub(w) { + return + } + if err := s.Store.RemoveWebhook(r.PathValue("id")); err != nil { + fail(w, http.StatusNotFound, "not_found", "No such webhook.") + return + } + s.logEvent(c.Name, "remove webhook", "ok") + w.WriteHeader(http.StatusNoContent) +} + +// testWebhook sends one made-up "hung" event (machine "test") now and reports what came back. +func (s *server) testWebhook(w http.ResponseWriter, r *http.Request, _ auth.Caller) { + if s.noHub(w) { + return + } + if s.Webhooks == nil { + fail(w, http.StatusNotFound, "not_found", "Webhooks aren't sent from here.") + return + } + i := slices.IndexFunc(s.Store.Webhooks(), func(h store.Webhook) bool { return h.ID == r.PathValue("id") }) + if i < 0 { + fail(w, http.StatusNotFound, "not_found", "No such webhook.") + return + } + h := s.Store.Webhooks()[i] + now := s.Now().UTC() + e := machines.Event{ID: "test-" + strconv.FormatInt(now.Unix(), 10), At: now, Event: machines.StateHung, Machine: "test", + State: machines.StateHung, From: machines.StateUnknown, + Evidence: machines.Evidence{At: now, Helper: machines.Silent, Port: machines.Accepts, Ping: machines.NoReply, + LED: machines.On, LEDTrusted: true}} + ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second) + defer cancel() + err := s.Webhooks.Send(ctx, h, e) + result := "ok" + if err != nil { + result = err.Error() + } + s.Store.WebhookDelivered(h.ID, now, result) + writeJSON(w, http.StatusOK, map[string]any{"ok": err == nil, "result": result}) +} diff --git a/helper/internal/server/watch_test.go b/helper/internal/server/watch_test.go new file mode 100644 index 0000000..afc67be --- /dev/null +++ b/helper/internal/server/watch_test.go @@ -0,0 +1,356 @@ +package server + +import ( + "bufio" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/projectzero02/powerdeck/helper/internal/machines" + "github.com/projectzero02/powerdeck/helper/internal/nanokvm/fakekvm" + "github.com/projectzero02/powerdeck/helper/internal/store" + "github.com/projectzero02/powerdeck/helper/internal/webhook" +) + +func (e *env) peerKey(name string) string { + e.t.Helper() + code, out := e.do(req{method: "POST", path: "/v2/keys", from: phone, body: map[string]any{"name": name, "kind": "peer"}}) + if code != 200 { + e.t.Fatalf("peer key: %d %v", code, out) + } + return out["key"].(string) +} + +// Only a machine's own helper says what it's about to do, and only about itself. +func TestNotes(t *testing.T) { + e := newEnv(t) + k := e.withHub(fakekvm.Options{On: true}) + e.linkKVM(k) + h := e.api.s.Machines + h.AsleepAfter = time.Millisecond + k.Sleep() + sleep := map[string]any{"event": "sleep"} + learned := func() string { + h.Wait() + kv, _ := e.st.KVMFor("workstation") + return kv.Wiring.LEDAsleep + } + + for _, r := range []req{ + {from: phone}, + {from: atlas, key: e.agentKey([]string{"status", "power"}, "workstation")}, + {from: hubAddr, key: e.peerKey("hub")}, // a peer, but not workstation: nothing happens + } { + r.method, r.path, r.body = "POST", "/v2/note", sleep + code, _ := e.do(r) + if r.from == hubAddr && code != http.StatusNoContent || r.from != hubAddr && code != http.StatusForbidden { + t.Errorf("note from %s: %d", r.from, code) + } + } + if got := learned(); got != "" { + t.Fatalf("a note from another machine counted: %q", got) + } + + pk := e.peerKey("workstation") + if code, _ := e.do(req{method: "POST", path: "/v2/note", from: wsAddr, key: pk, body: map[string]any{"event": "nap"}}); code != http.StatusBadRequest { + t.Errorf("unknown event: %d", code) + } + if code, _ := e.do(req{method: "POST", path: "/v2/note", from: wsAddr, key: pk, body: sleep}); code != http.StatusNoContent { + t.Errorf("note: %d", code) + } + if got := learned(); got != "off" { + t.Errorf("light while asleep: %q", got) + } +} + +// sseClient reads one event stream. +type sseClient struct { + t *testing.T + resp *http.Response + r *bufio.Reader +} + +type sse struct{ id, event, data, comment string } + +func (e *env) stream(srv *httptest.Server, from, key, lastID string) (*sseClient, int) { + e.t.Helper() + r, _ := http.NewRequest("GET", srv.URL+"/v2/machines/events", nil) + r.Header.Set("X-Test-From", from) + if key != "" { + r.Header.Set("Authorization", "Bearer "+key) + } + if lastID != "" { + r.Header.Set("Last-Event-ID", lastID) + } + resp, err := srv.Client().Do(r) + if err != nil { + e.t.Fatal(err) + } + if resp.StatusCode != 200 { + resp.Body.Close() + return nil, resp.StatusCode + } + c := &sseClient{t: e.t, resp: resp, r: bufio.NewReader(resp.Body)} + e.t.Cleanup(func() { resp.Body.Close() }) + return c, 200 +} + +// next reads the next event or comment, failing after 2 s. +func (c *sseClient) next() sse { + c.t.Helper() + got := make(chan sse, 1) + go func() { + var m sse + for { + line, err := c.r.ReadString('\n') + if err != nil { + got <- sse{event: "EOF " + err.Error()} + return + } + line = strings.TrimRight(line, "\n") + switch { + case line == "": + got <- m + return + case strings.HasPrefix(line, ":"): + m.comment = strings.TrimSpace(line[1:]) + default: + f, v, _ := strings.Cut(line, ": ") + switch f { + case "id": + m.id = v + case "event": + m.event = v + case "data": + m.data = v + } + } + } + }() + select { + case m := <-got: + return m + case <-time.After(2 * time.Second): + c.t.Fatal("no event in 2 s") + return sse{} + } +} + +// nextEvent skips heartbeats. +func (c *sseClient) nextEvent() sse { + c.t.Helper() + for { + if m := c.next(); m.comment == "" { + return m + } + } +} + +func TestMachineEvents(t *testing.T) { + e := newEnv(t) + k := e.withHub(fakekvm.Options{On: true}) + e.linkKVM(k) + bus := machines.NewBus(time.Now()) + e.api.s.Machines.Events = bus + old := heartbeat + heartbeat = 100 * time.Millisecond + defer func() { heartbeat = old }() + + // A real server with short timeouts: the stream must outlive them. + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + r.RemoteAddr = r.Header.Get("X-Test-From") + e.api.ServeHTTP(w, r) + })) + srv.Config.ReadTimeout, srv.Config.WriteTimeout = 150*time.Millisecond, 150*time.Millisecond + srv.Start() + defer func() { srv.CloseClientConnections(); srv.Close() }() // streams don't end by themselves + + c, _ := e.stream(srv, phone, "", "") + snap := c.nextEvent() + if snap.event != "snapshot" || !strings.Contains(snap.data, `"name":"workstation"`) { + t.Fatalf("snapshot %+v", snap) + } + first := bus.Publish(machines.Event{Event: "hung", Machine: "workstation", State: "hung", From: "unknown"}) + if m := c.nextEvent(); m.event != "machine" || m.id != first.ID || !strings.Contains(m.data, `"state":"hung"`) { + t.Fatalf("event %+v", m) + } + time.Sleep(300 * time.Millisecond) // past the timeouts + if m := c.next(); m.comment != "still here" { + t.Errorf("heartbeat %+v", m) + } + second := bus.Publish(machines.Event{Event: "recovered", Machine: "workstation", State: "on", From: "hung"}) + if m := c.nextEvent(); m.id != second.ID { + t.Fatalf("after the timeouts: %+v", m) + } + + // Reconnecting with the last ID seen replays what came after it, without a snapshot. + c2, _ := e.stream(srv, phone, "", first.ID) + if m := c2.nextEvent(); m.event != "machine" || m.id != second.ID { + t.Errorf("replay %+v", m) + } + // An ID no longer held: a fresh snapshot. + c3, _ := e.stream(srv, phone, "", "12") + if m := c3.nextEvent(); m.event != "snapshot" || m.id != second.ID { + t.Errorf("unknown ID: %+v", m) + } + + // A key for atlas sees atlas only. + key := e.agentKey([]string{"status"}, "atlas") + c4, _ := e.stream(srv, atlas, key, "") + if m := c4.nextEvent(); m.event != "snapshot" || strings.Contains(m.data, "workstation") { + t.Errorf("atlas's snapshot: %+v", m) + } + bus.Publish(machines.Event{Event: "hung", Machine: "workstation", State: "hung"}) + third := bus.Publish(machines.Event{Event: "on", Machine: "atlas", State: "on"}) + if m := c4.nextEvent(); m.id != third.ID { + t.Errorf("atlas's key got %+v", m) + } + + // No more than maxStreams at once. + e.api.s.streams.Add(maxStreams) + if _, code := e.stream(srv, phone, "", ""); code != http.StatusServiceUnavailable { + t.Errorf("over the limit: %d", code) + } + e.api.s.streams.Add(-maxStreams) +} + +func TestWebhookRoutes(t *testing.T) { + e := newEnv(t) + if code, _ := e.do(req{method: "GET", path: "/v2/webhooks", from: phone}); code != http.StatusNotFound { + t.Errorf("no hub: %d", code) + } + k := e.withHub(fakekvm.Options{On: true}) + e.linkKVM(k) + e.api.s.Webhooks = &webhook.Sender{Store: e.st, Box: e.api.s.Machines.Box, Bus: machines.NewBus(time.Now())} + + var mu sync.Mutex + var got []*http.Request + var bodies [][]byte + rcv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + mu.Lock() + got, bodies = append(got, r), append(bodies, b) + mu.Unlock() + w.WriteHeader(http.StatusNoContent) + })) + defer rcv.Close() + body := map[string]any{"url": rcv.URL + "/powerdeck"} + + // Owners only. + agent := e.agentKey([]string{"status", "wake", "power"}) + for _, r := range []req{ + {from: atlas, key: agent}, + {from: hubAddr, key: e.peerKey("hub")}, + {from: friend, key: pairedKey(t, e)}, + } { + r.method, r.path, r.body = "POST", "/v2/webhooks", body + if code, _ := e.do(r); code != http.StatusForbidden { + t.Errorf("webhook added from %s: %d", r.from, code) + } + } + for _, c := range []struct { + body map[string]any + code string + }{ + {map[string]any{"url": "http://example.com/hook"}, "bad_url"}, + {map[string]any{"url": "https://example.com/hook", "events": []string{"exploded"}}, "bad_event"}, + } { + if code, out := e.do(req{method: "POST", path: "/v2/webhooks", from: phone, body: c.body}); code != http.StatusBadRequest || out["error"] != c.code { + t.Errorf("%v: %d %v", c.body, code, out) + } + } + + code, out := e.do(req{method: "POST", path: "/v2/webhooks", from: phone, body: body}) + if code != http.StatusCreated { + t.Fatalf("add: %d %v", code, out) + } + h := out["webhook"].(map[string]any) + sec, _ := h["secret"].(string) + if !strings.HasPrefix(sec, "pdwh_") || h["events"].([]any)[0] != "hung" { + t.Fatalf("created %v", h) + } + id := h["id"].(string) + if raw := e.raw(req{method: "GET", path: "/v2/webhooks", from: phone}); strings.Contains(raw, sec) || !strings.Contains(raw, id) { + t.Errorf("list: %s", raw) + } + if strings.Contains(e.raw(req{method: "GET", path: "/v2/status", from: phone}), sec) { + t.Error("secret in the status") + } + if !e.hasEvent("add webhook "+rcv.URL+"/powerdeck", "ok") { + t.Error("not in the activity log") + } + + // The test delivery is signed with the secret shown once. + code, out = e.do(req{method: "POST", path: "/v2/webhooks/" + id + "/test", from: phone}) + if code != 200 || out["ok"] != true { + t.Fatalf("test: %d %v", code, out) + } + mu.Lock() + r, b := got[0], bodies[0] + mu.Unlock() + if r.Header.Get("X-PowerDeck-Signature") != webhook.Sign(sec, r.Header.Get("X-PowerDeck-Timestamp"), b) || + r.Header.Get("X-PowerDeck-Event") != "hung" { + t.Errorf("test delivery %v %s", r.Header, b) + } + var ev machines.Event + if json.Unmarshal(b, &ev) != nil || ev.Machine != "test" { + t.Errorf("test event %s", b) + } + + if code, _ := e.do(req{method: "DELETE", path: "/v2/webhooks/" + id, from: phone}); code != http.StatusNoContent { + t.Errorf("remove: %d", code) + } + if code, _ := e.do(req{method: "DELETE", path: "/v2/webhooks/" + id, from: phone}); code != http.StatusNotFound { + t.Errorf("remove again: %d", code) + } + + for i := 0; i < store.MaxWebhooks; i++ { + if code, _ := e.do(req{method: "POST", path: "/v2/webhooks", from: phone, body: body}); code != http.StatusCreated { + t.Fatalf("add %d: %d", i, code) + } + } + if code, out := e.do(req{method: "POST", path: "/v2/webhooks", from: phone, body: body}); code != http.StatusConflict { + t.Errorf("one too many: %d %v", code, out) + } +} + +func TestHungAfterSetting(t *testing.T) { + e := newEnv(t) + k := e.withHub(fakekvm.Options{On: true}) + e.linkKVM(k) + hub := func(from, key string) any { + _, st := e.do(req{method: "GET", path: "/v2/status", from: from, key: key}) + return st["hub"] + } + if h := hub(phone, ""); h.(map[string]any)["hungAfterMinutes"] != 3.0 { + t.Errorf("default: %v", h) + } + for _, n := range []int{1, 31, -1} { + if code, _ := e.do(req{method: "POST", path: "/v2/settings", from: phone, body: map[string]any{"hungAfterMinutes": n}}); code != http.StatusBadRequest { + t.Errorf("%d minutes: %d", n, code) + } + } + if code, _ := e.do(req{method: "POST", path: "/v2/settings", from: phone, body: map[string]any{"hungAfterMinutes": 5}}); code/100 != 2 { + t.Errorf("5 minutes: %d", code) + } + if h := hub(phone, ""); h.(map[string]any)["hungAfterMinutes"] != 5.0 { + t.Errorf("after setting: %v", h) + } + if e.st.Settings().HungAfterMinutes != 5 { + t.Errorf("stored %+v", e.st.Settings()) + } + if h := hub(atlas, e.agentKey([]string{"status"})); h != nil { + t.Errorf("an agent sees the hub settings: %v", h) + } + if code, _ := e.do(req{method: "POST", path: "/v2/settings", from: phone, body: map[string]any{"hungAfterMinutes": 0}}); code/100 != 2 { + t.Errorf("back to the default: %d", code) + } + if h := hub(phone, ""); h.(map[string]any)["hungAfterMinutes"] != 3.0 { + t.Errorf("default again: %v", h) + } +} diff --git a/helper/internal/sleepinfo/sleepinfo.go b/helper/internal/sleepinfo/sleepinfo.go new file mode 100644 index 0000000..4e3f3ac --- /dev/null +++ b/helper/internal/sleepinfo/sleepinfo.go @@ -0,0 +1,27 @@ +// Package sleepinfo tells a hub whether this machine sleeps on its own, and what kind of sleep it +// has, so a sleeping machine isn't taken for a hung one. +package sleepinfo + +import ( + "sync" + "time" + + "github.com/projectzero02/powerdeck/helper/internal/store" +) + +// Cached returns Read, read again at most every 5 minutes (the owner may change the power plan). +func Cached() func() *store.Sleep { + var ( + mu sync.Mutex + last *store.Sleep + at time.Time + ) + return func() *store.Sleep { + mu.Lock() + defer mu.Unlock() + if at.IsZero() || time.Since(at) > 5*time.Minute { + last, at = Read(), time.Now() + } + return last + } +} diff --git a/helper/internal/sleepinfo/sleepinfo_other.go b/helper/internal/sleepinfo/sleepinfo_other.go new file mode 100644 index 0000000..e114099 --- /dev/null +++ b/helper/internal/sleepinfo/sleepinfo_other.go @@ -0,0 +1,9 @@ +//go:build !windows + +package sleepinfo + +import "github.com/projectzero02/powerdeck/helper/internal/store" + +// Read is nil here: on Linux the desktop decides when to sleep, out of a service's sight. A hub +// treats an unknown machine as one that doesn't sleep on its own; a sleep note still counts. +func Read() *store.Sleep { return nil } diff --git a/helper/internal/sleepinfo/sleepinfo_windows.go b/helper/internal/sleepinfo/sleepinfo_windows.go new file mode 100644 index 0000000..3a1eb79 --- /dev/null +++ b/helper/internal/sleepinfo/sleepinfo_windows.go @@ -0,0 +1,75 @@ +//go:build windows + +package sleepinfo + +import ( + "unsafe" + + "golang.org/x/sys/windows" + + "github.com/projectzero02/powerdeck/helper/internal/store" +) + +var ( + powrprof = windows.NewLazySystemDLL("powrprof.dll") + getPwrCapabilities = powrprof.NewProc("GetPwrCapabilities") + powerGetActiveScheme = powrprof.NewProc("PowerGetActiveScheme") + powerReadACValueIndex = powrprof.NewProc("PowerReadACValueIndex") + + // SUB_SLEEP and STANDBYIDLE, as powercfg calls them. + sleepSubgroup = windows.GUID{Data1: 0x238c9fa8, Data2: 0x0aad, Data3: 0x41ed, Data4: [8]byte{0x83, 0xf4, 0x97, 0xbe, 0x24, 0x2c, 0x8f, 0x20}} + standbyTimeout = windows.GUID{Data1: 0x29f6c1db, Data2: 0x86da, Data3: 0x48c5, Data4: [8]byte{0x9f, 0xdb, 0xf2, 0xb6, 0x7b, 0x1f, 0x44, 0xda}} +) + +// powerCapabilities is SYSTEM_POWER_CAPABILITIES from winnt.h (76 bytes). +type powerCapabilities struct { + PowerButtonPresent, SleepButtonPresent, LidPresent byte + SystemS1, SystemS2, SystemS3, SystemS4, SystemS5 byte + HiberFilePresent, FullWake, VideoDimPresent, ApmPresent byte + UpsPresent, ThermalControl, ProcessorThrottle byte + ProcessorMinThrottle, ProcessorMaxThrottle byte + FastSystemS4, Hiberboot, WakeAlarmPresent, AoAc byte + DiskSpinDown, HiberFileType, AoAcConnectivitySupported byte + _ [6]byte + SystemBatteriesPresent, BatteriesAreShortTerm byte + BatteryScale [3]struct{ Granularity, Capacity uint32 } + AcOnLineWake, SoftLidWake, RtcWake int32 + MinDeviceWakeState, DefaultLowLatencyWake int32 +} + +func capabilities() (powerCapabilities, bool) { + var c powerCapabilities + ok, _, _ := getPwrCapabilities.Call(uintptr(unsafe.Pointer(&c))) + return c, ok&0xff != 0 // BOOLEAN +} + +// sleepAfter is the active power plan's "sleep after" on mains power, in seconds (0 = never). +func sleepAfter() (uint32, bool) { + var scheme *windows.GUID + if r, _, _ := powerGetActiveScheme.Call(0, uintptr(unsafe.Pointer(&scheme))); r != 0 || scheme == nil { + return 0, false + } + defer windows.LocalFree(windows.Handle(unsafe.Pointer(scheme))) + var secs uint32 + r, _, _ := powerReadACValueIndex.Call(0, uintptr(unsafe.Pointer(scheme)), uintptr(unsafe.Pointer(&sleepSubgroup)), + uintptr(unsafe.Pointer(&standbyTimeout)), uintptr(unsafe.Pointer(&secs))) + return secs, r == 0 +} + +// Read is the machine's sleep facts, or nil when Windows won't say. +func Read() *store.Sleep { + c, ok := capabilities() + if !ok { + return nil + } + s := &store.Sleep{S3: c.SystemS3 != 0, ModernStandby: c.AoAc != 0} + if !s.S3 && !s.ModernStandby { + return s // it can't sleep: "never" whatever the plan says + } + secs, ok := sleepAfter() + if !ok { + return nil + } + s.AfterMinutes = int((secs + 59) / 60) + return s +} diff --git a/helper/internal/sleepinfo/sleepinfo_windows_test.go b/helper/internal/sleepinfo/sleepinfo_windows_test.go new file mode 100644 index 0000000..3a92efb --- /dev/null +++ b/helper/internal/sleepinfo/sleepinfo_windows_test.go @@ -0,0 +1,40 @@ +//go:build windows + +package sleepinfo + +import ( + "testing" + "unsafe" +) + +func TestCapabilitiesLayout(t *testing.T) { + var c powerCapabilities + if n := unsafe.Sizeof(c); n != 76 { + t.Fatalf("SYSTEM_POWER_CAPABILITIES is 76 bytes, this is %d", n) + } + if off := unsafe.Offsetof(c.AoAc); off != 20 { + t.Errorf("AoAc at %d, want 20", off) + } + if off := unsafe.Offsetof(c.BatteryScale); off != 32 { + t.Errorf("BatteryScale at %d, want 32", off) + } +} + +// What this Windows says about sleep (logged: a CI runner is a VM, which usually can't sleep). +func TestRead(t *testing.T) { + c, ok := capabilities() + if !ok { + t.Fatal("GetPwrCapabilities failed") + } + secs, ok := sleepAfter() + t.Logf("S1-S5 %d%d%d%d%d, AoAc %d, hibernate file %d; sleep after %d s on AC (read: %v)", + c.SystemS1, c.SystemS2, c.SystemS3, c.SystemS4, c.SystemS5, c.AoAc, c.HiberFilePresent, secs, ok) + s := Read() + if s == nil { + t.Fatal("no sleep facts") + } + t.Logf("reported: %+v", *s) + if !s.S3 && !s.ModernStandby && s.AfterMinutes != 0 { + t.Errorf("can't sleep, yet sleeps after %d minutes", s.AfterMinutes) + } +} diff --git a/helper/internal/store/store.go b/helper/internal/store/store.go index e23d0e6..0269c2f 100644 --- a/helper/internal/store/store.go +++ b/helper/internal/store/store.go @@ -111,6 +111,7 @@ type Peer struct { MAC string `json:"mac,omitempty"` LAN []string `json:"lan,omitempty"` NoAgents bool `json:"noAgents,omitempty"` // it turns agents away (last we heard) + Sleep *Sleep `json:"sleep,omitempty"` // whether it sleeps on its own (last we heard) LastSeen time.Time `json:"lastSeen"` // Key is the peer key this helper presents to it (set up by the owner's phone). Key string `json:"key,omitempty"` @@ -136,6 +137,9 @@ type Settings struct { Tray bool `json:"tray,omitempty"` // SetupDone records that the PC window has shown its first-run setup (once is enough). SetupDone bool `json:"setupDone,omitempty"` + // HungAfterMinutes is how long a machine with a KVM here must look hung before it's called hung + // (0: the default, 3). + HungAfterMinutes int `json:"hungAfterMinutes,omitempty"` // UpdatesPausedUntil holds automatic updates back until then. UpdatesPausedUntil time.Time `json:"updatesPausedUntil,omitzero"` } @@ -149,6 +153,7 @@ type state struct { Owners []Owner `json:"owners,omitempty"` OwnerMode string `json:"ownerMode,omitempty"` KVMs []KVM `json:"kvms,omitempty"` + Webhooks []Webhook `json:"webhooks,omitempty"` } // fresh is a new install's state. @@ -270,6 +275,12 @@ func random(n int) string { return base64.RawURLEncoding.EncodeToString(b) } +// NewID is a short random ID. +func NewID() string { return random(9) } + +// NewWebhookSecret is a random secret for signing webhook deliveries. +func NewWebhookSecret() string { return "pdwh_" + random(32) } + // NewKey describes a key to issue. type NewKey struct { Name string @@ -388,6 +399,9 @@ func (s *Store) PutPeer(p Peer) error { if p.LAN == nil { p.LAN = old.LAN } + if p.Sleep == nil { + p.Sleep = old.Sleep + } } p.LastSeen = p.LastSeen.UTC() s.st.Peers[p.Name] = p @@ -439,8 +453,11 @@ type Wiring struct { Power string `json:"power"` // the power switch Reset string `json:"reset"` // the reset switch // LEDOffWhileUp counts readings in a row with the LED off while the machine's helper answered. - LEDOffWhileUp int `json:"ledOffWhileUp,omitempty"` - CheckedAt time.Time `json:"checkedAt,omitzero"` + LEDOffWhileUp int `json:"ledOffWhileUp,omitempty"` + // LEDAsleep is what the light did while the machine slept (on, off, blinking), seen after its + // helper said it was going to sleep; empty until then. + LEDAsleep string `json:"ledAsleep,omitempty"` + CheckedAt time.Time `json:"checkedAt,omitzero"` } // Wiring states. @@ -543,6 +560,74 @@ func (s *Store) RemoveKVM(name string) error { return ErrNotFound } +// Webhook is an address the hub posts machine events to. +type Webhook struct { + ID string `json:"id"` + URL string `json:"url"` + Events []string `json:"events"` + Secret string `json:"secret"` // sealed (secret.Box): signs each delivery + CreatedBy string `json:"createdBy,omitempty"` + CreatedAt time.Time `json:"createdAt"` + // The last delivery: when, and "200" or what went wrong. + LastAt time.Time `json:"lastAt,omitzero"` + LastResult string `json:"lastResult,omitempty"` +} + +// MaxWebhooks bounds the webhook list. +const MaxWebhooks = 10 + +// ErrTooManyWebhooks is returned when the webhook list is full. +var ErrTooManyWebhooks = errors.New("too many webhooks") + +// Webhooks lists the webhooks. +func (s *Store) Webhooks() []Webhook { + s.mu.Lock() + defer s.mu.Unlock() + out := make([]Webhook, len(s.st.Webhooks)) + for i, h := range s.st.Webhooks { + h.Events = slices.Clone(h.Events) + out[i] = h + } + return out +} + +// AddWebhook stores a new webhook. +func (s *Store) AddWebhook(h Webhook) error { + s.mu.Lock() + defer s.mu.Unlock() + if len(s.st.Webhooks) >= MaxWebhooks { + return ErrTooManyWebhooks + } + s.st.Webhooks = append(s.st.Webhooks, h) + return s.saveLocked() +} + +// RemoveWebhook deletes a webhook by ID. +func (s *Store) RemoveWebhook(id string) error { + s.mu.Lock() + defer s.mu.Unlock() + for i, h := range s.st.Webhooks { + if h.ID == id { + s.st.Webhooks = slices.Delete(s.st.Webhooks, i, i+1) + return s.saveLocked() + } + } + return ErrNotFound +} + +// WebhookDelivered records a delivery's outcome. +func (s *Store) WebhookDelivered(id string, at time.Time, result string) { + s.mu.Lock() + defer s.mu.Unlock() + for i := range s.st.Webhooks { + if s.st.Webhooks[i].ID == id { + s.st.Webhooks[i].LastAt, s.st.Webhooks[i].LastResult = at.UTC(), result + _ = s.saveLocked() + return + } + } +} + const maxSchedules = 20 // ErrTooManySchedules is returned when the schedule list is full. @@ -600,6 +685,13 @@ func (s *Store) UpdateSettings(f func(*Settings)) error { return s.saveLocked() } +// Sleep is whether a machine can sleep and goes to sleep on its own, as its helper reports it. +type Sleep struct { + S3 bool `json:"s3"` // classic sleep: the light usually goes off or blinks + ModernStandby bool `json:"modernStandby"` // S0 low power idle: the light may stay on + AfterMinutes int `json:"afterMinutes"` // idle minutes before it sleeps on mains power; 0 = never +} + // SetPeerKey stores the key this helper presents to a remembered peer ("" removes it). func (s *Store) SetPeerKey(name, key string) error { s.mu.Lock() diff --git a/helper/internal/tailscale/tailscale.go b/helper/internal/tailscale/tailscale.go index 25de9ca..dd5a474 100644 --- a/helper/internal/tailscale/tailscale.go +++ b/helper/internal/tailscale/tailscale.go @@ -93,7 +93,7 @@ func ParseStatus(b []byte) (*Status, error) { // Identity is who a tailnet caller is, from `tailscale whois`. type Identity struct { NodeID string - NodeName string // short MagicDNS name, e.g. "oppo-find-x9-ultra" + NodeName string // short MagicDNS name, e.g. "pixel-9-pro" Hostname string // the device's own name, e.g. "Pixel 9 Pro" UserID int64 Login string @@ -222,6 +222,39 @@ func (c *Client) WhoIs(ctx context.Context, ip netip.Addr) (Identity, error) { return id, err } +// Ping pings a peer's Tailscale address with ICMP through WireGuard, which the peer's own network +// stack answers (a disco or TSMP ping is answered by tailscaled alone, so it says less about the +// machine). ErrUnavailable: Tailscale isn't running here, or the address isn't a peer's. +func (c *Client) Ping(ctx context.Context, ip netip.Addr) (bool, error) { + ctx, cancel := context.WithTimeout(ctx, 8*time.Second) + defer cancel() + out, err := c.run(ctx, "ping", "--icmp", "--c", "2", "--timeout", "3s", "--until-direct=false", ip.String()) + if strings.Contains(string(out), "pong from") { + return true, nil + } + var ee *exec.ExitError + if errors.As(err, &ee) { + if s := string(ee.Stderr); strings.Contains(s, "no reply") || strings.Contains(s, "timed out") { + return false, nil + } + } + return false, ErrUnavailable +} + +// Online reports whether the control plane lists the peer with this address as online. +func (c *Client) Online(ctx context.Context, ip netip.Addr) (online, known bool) { + st, err := c.Status(ctx) + if err != nil { + return false, false + } + for _, p := range st.Peers { + if p.IPv4 == ip { + return p.Online, true + } + } + return false, false +} + // Unattended reports Windows' "Run unattended" preference (ForceDaemon). func (c *Client) Unattended(ctx context.Context) (bool, error) { out, err := c.run(ctx, "debug", "prefs") diff --git a/helper/internal/tailscale/tailscale_test.go b/helper/internal/tailscale/tailscale_test.go index ca38954..7bd55ff 100644 --- a/helper/internal/tailscale/tailscale_test.go +++ b/helper/internal/tailscale/tailscale_test.go @@ -39,7 +39,7 @@ func TestParseStatus(t *testing.T) { } } -const whoisJSON = `{"Node": {"StableID": "nPHONE", "Name": "oppo-find-x9-ultra.example-tailnet.ts.net.", "ComputedName": "oppo-find-x9-ultra", +const whoisJSON = `{"Node": {"StableID": "nPHONE", "Name": "pixel-9-pro.example-tailnet.ts.net.", "ComputedName": "pixel-9-pro", "Hostinfo": {"Hostname": "Pixel 9 Pro"}, "User": 1}, "UserProfile": {"ID": 1, "LoginName": "owner@example.com"}, "CapMap": null}` diff --git a/helper/internal/webhook/webhook.go b/helper/internal/webhook/webhook.go new file mode 100644 index 0000000..499b816 --- /dev/null +++ b/helper/internal/webhook/webhook.go @@ -0,0 +1,228 @@ +// Package webhook posts the hub's machine events to the owner's webhooks, signed so the receiver +// can tell they came from this hub, with retries. +// +// Each delivery is the event as JSON, with these headers: +// +// X-PowerDeck-Event: hung +// X-PowerDeck-Delivery: +// X-PowerDeck-Timestamp: +// X-PowerDeck-Signature: sha256= +// +// A receiver checks the signature, and that the timestamp is recent (a few minutes) so an old +// delivery can't be replayed. +package webhook + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/netip" + "net/url" + "slices" + "strconv" + "strings" + "sync" + "time" + + "github.com/projectzero02/powerdeck/helper/internal/machines" + "github.com/projectzero02/powerdeck/helper/internal/secret" + "github.com/projectzero02/powerdeck/helper/internal/store" + "github.com/projectzero02/powerdeck/helper/internal/version" +) + +// DefaultEvents are what a webhook gets unless it asks for others. +var DefaultEvents = []string{machines.StateHung, "recovered"} + +// Retries are the waits before each new try after a failed delivery. +var Retries = []time.Duration{10 * time.Second, time.Minute, 5 * time.Minute} + +// Sign is the signature header's value for a delivery. +func Sign(secret, timestamp string, body []byte) string { + m := hmac.New(sha256.New, []byte(secret)) + m.Write([]byte(timestamp)) + m.Write([]byte(".")) + m.Write(body) + return "sha256=" + hex.EncodeToString(m.Sum(nil)) +} + +var tailnet4 = netip.MustParsePrefix("100.64.0.0/10") +var tailnet6 = netip.MustParsePrefix("fd7a:115c:a1e0::/48") + +// CheckURL accepts https anywhere, and plain http only to a Tailscale or loopback address or a +// MagicDNS name (WireGuard encrypts those already). +func CheckURL(raw string) error { + u, err := url.Parse(raw) + if err != nil || u.Host == "" || u.User != nil || len(raw) > 2048 { + return errors.New("that isn't a web address") + } + switch u.Scheme { + case "https": + return nil + case "http": + host := u.Hostname() + if a, err := netip.ParseAddr(host); err == nil { + if a.IsLoopback() || tailnet4.Contains(a) || tailnet6.Contains(a) { + return nil + } + } else if host == "localhost" || strings.HasSuffix(host, ".ts.net") { + return nil + } + return errors.New("plain http is only allowed to a Tailscale or loopback address; use https") + } + return errors.New("the address must start with https://") +} + +// Sender delivers events to every webhook that wants them. +type Sender struct { + Store *store.Store + Box secret.Box + Bus *machines.Bus + Log *slog.Logger + Now func() time.Time + Client *http.Client // default: 10 s timeout, redirects not followed + Retries []time.Duration // default: Retries + + wg sync.WaitGroup +} + +func (s *Sender) now() time.Time { + if s.Now != nil { + return s.Now() + } + return time.Now() +} + +func (s *Sender) client() *http.Client { + if s.Client != nil { + return s.Client + } + return defaultClient +} + +var defaultClient = &http.Client{ + Timeout: 10 * time.Second, + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{Timeout: 5 * time.Second}).DialContext, + }, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, +} + +// Run delivers events until ctx ends, then waits for deliveries in progress (their retries stop). +func (s *Sender) Run(ctx context.Context) { + defer s.wg.Wait() + for { + ch, stop := s.Bus.Subscribe(64) + for open := true; open; { + select { + case <-ctx.Done(): + stop() + return + case e, ok := <-ch: + if !ok { + open = false // fell behind: events were missed + s.log().Warn("webhook deliveries fell behind; some events weren't sent") + break + } + s.dispatch(ctx, e) + } + } + } +} + +func (s *Sender) log() *slog.Logger { + if s.Log != nil { + return s.Log + } + return slog.Default() +} + +func (s *Sender) dispatch(ctx context.Context, e machines.Event) { + for _, h := range s.Store.Webhooks() { + if !slices.Contains(h.Events, e.Event) { + continue + } + s.wg.Add(1) + go func(h store.Webhook) { + defer s.wg.Done() + s.deliver(ctx, h, e) + }(h) + } +} + +// deliver tries once, then after each retry wait, until a 2xx. +func (s *Sender) deliver(ctx context.Context, h store.Webhook, e machines.Event) { + waits := s.Retries + if waits == nil { + waits = Retries + } + for i := 0; ; i++ { + err := s.Send(ctx, h, e) + result := "ok" + if err != nil { + result = err.Error() + } + s.Store.WebhookDelivered(h.ID, s.now(), result) + if err == nil { + return + } + if i == len(waits) { + s.log().Warn("webhook delivery failed; giving up", "webhook", h.ID, "event", e.ID, "err", err) + return + } + select { + case <-ctx.Done(): + return + case <-time.After(waits[i]): + } + } +} + +// Send makes one delivery of e to h. +func (s *Sender) Send(ctx context.Context, h store.Webhook, e machines.Event) error { + sec, err := s.Box.Open(h.Secret) + if err != nil { + return fmt.Errorf("the webhook's secret can't be opened: %w", err) + } + body, err := json.Marshal(e) + if err != nil { + return err + } + ts := strconv.FormatInt(s.now().Unix(), 10) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, h.URL, bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", "PowerDeck/"+version.Version) + req.Header.Set("X-PowerDeck-Event", e.Event) + req.Header.Set("X-PowerDeck-Delivery", e.ID) + req.Header.Set("X-PowerDeck-Timestamp", ts) + req.Header.Set("X-PowerDeck-Signature", Sign(sec, ts, body)) + resp, err := s.client().Do(req) + if err != nil { + var ue *url.Error + if errors.As(err, &ue) { + err = ue.Err // the URL is in the list already + } + return err + } + defer resp.Body.Close() + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10)) + if resp.StatusCode/100 != 2 { + return fmt.Errorf("HTTP %d", resp.StatusCode) + } + return nil +} + +// Wait waits for deliveries in progress (tests). +func (s *Sender) Wait() { s.wg.Wait() } diff --git a/helper/internal/webhook/webhook_test.go b/helper/internal/webhook/webhook_test.go new file mode 100644 index 0000000..c766e62 --- /dev/null +++ b/helper/internal/webhook/webhook_test.go @@ -0,0 +1,204 @@ +package webhook + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/projectzero02/powerdeck/helper/internal/machines" + "github.com/projectzero02/powerdeck/helper/internal/secret" + "github.com/projectzero02/powerdeck/helper/internal/store" +) + +// The same signature from Python's hmac and from openssl: +// +// printf '1700000000.{"id":"1"}' | openssl dgst -sha256 -hmac pdwh_test +func TestSignKnownAnswer(t *testing.T) { + got := Sign("pdwh_test", "1700000000", []byte(`{"id":"1"}`)) + if got != "sha256=8fc0a23c6a794d26f2a6c83ac48c588dfbbe64deefd997f2bb8cc5b0ac0c0914" { + t.Errorf("got %s", got) + } +} + +func TestCheckURL(t *testing.T) { + for u, ok := range map[string]bool{ + "https://example.com/hook": true, + "https://192.168.1.5/hook": true, + "http://100.101.1.20:8080/hook": true, + "http://127.0.0.1:9000/": true, + "http://localhost:9000/": true, + "http://assistant.example.ts.net/x": true, + "http://[fd7a:115c:a1e0::5]:80/x": true, + "http://example.com/hook": false, + "http://192.168.1.5/hook": false, + "ftp://example.com/": false, + "https://user:pw@example.com/": false, + "example.com/hook": false, + "https:///nohost": false, + "http://100.128.0.1/ (outside CGNAT)": false, + } { + if err := CheckURL(u); (err == nil) != ok { + t.Errorf("%s: %v, want ok=%v", u, err, ok) + } + } +} + +type receiver struct { + mu sync.Mutex + got []*http.Request + body [][]byte + fails int // answer 500 this many times first +} + +func (r *receiver) ServeHTTP(w http.ResponseWriter, req *http.Request) { + b, _ := io.ReadAll(req.Body) + r.mu.Lock() + defer r.mu.Unlock() + r.got, r.body = append(r.got, req), append(r.body, b) + if r.fails > 0 { + r.fails-- + w.WriteHeader(http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (r *receiver) count() int { r.mu.Lock(); defer r.mu.Unlock(); return len(r.got) } + +func setup(t *testing.T, events ...string) (*Sender, *receiver, *httptest.Server, string) { + t.Helper() + st, err := store.Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + box := &secret.KeyFile{Path: filepath.Join(t.TempDir(), "secret.key")} + rc := &receiver{} + srv := httptest.NewServer(rc) + t.Cleanup(srv.Close) + sec := store.NewWebhookSecret() + sealed, err := box.Seal(sec) + if err != nil { + t.Fatal(err) + } + if events == nil { + events = DefaultEvents + } + if err := st.AddWebhook(store.Webhook{ID: "w1", URL: srv.URL, Events: events, Secret: sealed}); err != nil { + t.Fatal(err) + } + s := &Sender{Store: st, Box: box, Bus: machines.NewBus(time.Now()), Retries: []time.Duration{time.Millisecond, time.Millisecond}} + return s, rc, srv, sec +} + +func run(t *testing.T, s *Sender) func() { + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { s.Run(ctx); close(done) }() + for s.Bus.Subscribers() == 0 { + time.Sleep(time.Millisecond) + } + return func() { cancel(); <-done } +} + +func waitFor(t *testing.T, f func() bool) { + t.Helper() + for i := 0; i < 500 && !f(); i++ { + time.Sleep(2 * time.Millisecond) + } + if !f() { + t.Fatal("timed out") + } +} + +func TestDeliverySignedAndFiltered(t *testing.T) { + s, rc, _, sec := setup(t) + stop := run(t, s) + s.Bus.Publish(machines.Event{Event: "unknown", Machine: "gaming-pc", State: "unknown"}) // not asked for + hung := s.Bus.Publish(machines.Event{Event: "hung", Machine: "gaming-pc", State: "hung", From: "unknown", + Evidence: machines.Evidence{Helper: machines.Silent, LED: machines.On, LEDTrusted: true}}) + waitFor(t, func() bool { return rc.count() == 1 }) + stop() + req, body := rc.got[0], rc.body[0] + ts := req.Header.Get("X-PowerDeck-Timestamp") + if req.Header.Get("X-PowerDeck-Event") != "hung" || req.Header.Get("X-PowerDeck-Delivery") != hung.ID || + req.Header.Get("Content-Type") != "application/json" { + t.Errorf("headers %v", req.Header) + } + if req.Header.Get("X-PowerDeck-Signature") != Sign(sec, ts, body) { + t.Errorf("signature %s", req.Header.Get("X-PowerDeck-Signature")) + } + if Sign("pdwh_other", ts, body) == req.Header.Get("X-PowerDeck-Signature") { + t.Error("another secret signs the same") + } + var got machines.Event + if err := json.Unmarshal(body, &got); err != nil || got.Machine != "gaming-pc" || got.Evidence.LED != "on" || got.ID != hung.ID { + t.Errorf("body %s (%v)", body, err) + } + if h := s.Store.Webhooks()[0]; h.LastResult != "ok" || h.LastAt.IsZero() { + t.Errorf("last delivery %+v", h) + } +} + +func TestRetriesThenGivesUp(t *testing.T) { + s, rc, _, _ := setup(t) + rc.fails = 2 + stop := run(t, s) + s.Bus.Publish(machines.Event{Event: "recovered", Machine: "gaming-pc"}) + waitFor(t, func() bool { return rc.count() == 3 }) + s.Wait() + if h := s.Store.Webhooks()[0]; h.LastResult != "ok" { + t.Errorf("after two failures: %+v", h) + } + ids := map[string]bool{} + for _, r := range rc.got { + ids[r.Header.Get("X-PowerDeck-Delivery")] = true + } + if len(ids) != 1 { + t.Errorf("retries carry the same delivery ID: %v", ids) + } + + rc.mu.Lock() + rc.fails = 10 + rc.mu.Unlock() + s.Bus.Publish(machines.Event{Event: "hung", Machine: "gaming-pc"}) + waitFor(t, func() bool { return rc.count() == 6 }) + s.Wait() + stop() + if h := s.Store.Webhooks()[0]; h.LastResult != "HTTP 500" { + t.Errorf("gave up: %+v", h) + } + if rc.count() != 6 { + t.Errorf("tries %d, want 3 (one and two retries)", rc.count()) + } +} + +func TestRedirectNotFollowed(t *testing.T) { + s, rc, srv, _ := setup(t) + target := httptest.NewServer(rc) + defer target.Close() + redirect := httptest.NewServer(http.RedirectHandler(target.URL, http.StatusTemporaryRedirect)) + defer redirect.Close() + h := s.Store.Webhooks()[0] + h.URL = redirect.URL + err := s.Send(context.Background(), h, machines.Event{Event: "hung"}) + if err == nil || !strings.Contains(err.Error(), "307") || rc.count() != 0 { + t.Errorf("err %v, deliveries %d", err, rc.count()) + } + _ = srv +} + +func TestSecretTampered(t *testing.T) { + s, rc, _, _ := setup(t) + h := s.Store.Webhooks()[0] + h.Secret = h.Secret[:len(h.Secret)-4] + "AAAA" + if err := s.Send(context.Background(), h, machines.Event{Event: "hung"}); err == nil || rc.count() != 0 { + t.Errorf("err %v, deliveries %d", err, rc.count()) + } +} diff --git a/helper/internal/winsvc/suspend_windows.go b/helper/internal/winsvc/suspend_windows.go new file mode 100644 index 0000000..b084a00 --- /dev/null +++ b/helper/internal/winsvc/suspend_windows.go @@ -0,0 +1,76 @@ +//go:build windows + +package winsvc + +import ( + "sync" + "unsafe" + + "golang.org/x/sys/windows" +) + +var ( + powrprof = windows.NewLazySystemDLL("powrprof.dll") + procRegister = powrprof.NewProc("PowerRegisterSuspendResumeNotification") + procUnregister = powrprof.NewProc("PowerUnregisterSuspendResumeNotification") + suspendCallback uintptr + suspendOnce sync.Once + suspendMu sync.Mutex + suspendNotify func(event string) + suspendSubscribe deviceNotifySubscribeParameters // Windows holds on to it: not on the Go heap +) + +// DEVICE_NOTIFY_SUBSCRIBE_PARAMETERS +type deviceNotifySubscribeParameters struct { + callback uintptr + context uintptr +} + +const ( + deviceNotifyCallback = 2 // DEVICE_NOTIFY_CALLBACK + pbtAPMSuspend = 0x4 // PBT_APMSUSPEND: about to sleep or hibernate + pbtAPMResumeAutomatic = 0x12 // PBT_APMRESUMEAUTOMATIC: woke, for any reason +) + +// powerEvent names what a suspend/resume notification means to the hub ("" for the rest). +func powerEvent(typ uintptr) string { + switch typ { + case pbtAPMSuspend: + return "sleep" + case pbtAPMResumeAutomatic: + return "resume" + } + return "" +} + +// watchSuspend calls notify("sleep") just before the PC sleeps and notify("resume") when it wakes. +// Unlike a service's power events (answered as soon as they're queued), Windows waits for this +// callback before it goes on to sleep, so the hub hears of it while the network is still up. +func watchSuspend(notify func(event string)) (stop func(), err error) { + suspendOnce.Do(func() { + suspendCallback = windows.NewCallback(func(_, typ, _ uintptr) uintptr { + suspendMu.Lock() + f := suspendNotify + suspendMu.Unlock() + if e := powerEvent(typ); e != "" && f != nil { + f(e) + } + return 0 + }) + }) + suspendMu.Lock() + suspendNotify = notify + suspendMu.Unlock() + suspendSubscribe = deviceNotifySubscribeParameters{callback: suspendCallback} + var h uintptr + r, _, _ := procRegister.Call(deviceNotifyCallback, uintptr(unsafe.Pointer(&suspendSubscribe)), uintptr(unsafe.Pointer(&h))) + if r != 0 { + return nil, windows.Errno(r) + } + return func() { + procUnregister.Call(h) + suspendMu.Lock() + suspendNotify = nil + suspendMu.Unlock() + }, nil +} diff --git a/helper/internal/winsvc/suspend_windows_test.go b/helper/internal/winsvc/suspend_windows_test.go new file mode 100644 index 0000000..546cba4 --- /dev/null +++ b/helper/internal/winsvc/suspend_windows_test.go @@ -0,0 +1,20 @@ +package winsvc + +import "testing" + +func TestPowerEvent(t *testing.T) { + for typ, want := range map[uintptr]string{4: "sleep", 0x12: "resume", 7: "", 0xA: "", 0x8013: ""} { + if got := powerEvent(typ); got != want { + t.Errorf("%#x: %q, want %q", typ, got, want) + } + } +} + +// Registering for suspend and resume works for an ordinary process too (CI isn't a service). +func TestWatchSuspendRegisters(t *testing.T) { + stop, err := watchSuspend(func(string) {}) + if err != nil { + t.Fatal(err) + } + stop() +} diff --git a/helper/internal/winsvc/winsvc_windows.go b/helper/internal/winsvc/winsvc_windows.go index c21c78a..46631c5 100644 --- a/helper/internal/winsvc/winsvc_windows.go +++ b/helper/internal/winsvc/winsvc_windows.go @@ -10,7 +10,8 @@ import ( ) type handler struct { - run func(ctx context.Context) error + run func(ctx context.Context) error + notify func(event string) } func (h *handler) Execute(_ []string, req <-chan svc.ChangeRequest, status chan<- svc.Status) (bool, uint32) { @@ -19,15 +20,26 @@ func (h *handler) Execute(_ []string, req <-chan svc.ChangeRequest, status chan< defer cancel() done := make(chan error, 1) go func() { done <- h.run(ctx) }() - status <- svc.Status{State: svc.Running, Accepts: svc.AcceptStop | svc.AcceptShutdown} + accepts := svc.AcceptStop | svc.AcceptShutdown + if h.notify != nil { + // Pre-shutdown comes while the network is still up, and Windows waits for it. + accepts |= svc.AcceptPreShutdown + if stop, err := watchSuspend(h.notify); err == nil { + defer stop() + } + } + status <- svc.Status{State: svc.Running, Accepts: accepts} for { select { case c := <-req: switch c.Cmd { case svc.Interrogate: status <- c.CurrentStatus - case svc.Stop, svc.Shutdown: + case svc.PreShutdown, svc.Stop, svc.Shutdown: status <- svc.Status{State: svc.StopPending} + if c.Cmd == svc.PreShutdown { + h.notify("shutdown") + } cancel() <-done return false, 0 @@ -41,7 +53,9 @@ func (h *handler) Execute(_ []string, req <-chan svc.ChangeRequest, status chan< } } -// Run blocks, serving the SCM until the service is stopped. -func Run(name string, run func(ctx context.Context) error) error { - return svc.Run(name, &handler{run: run}) +// Run blocks, serving the SCM until the service is stopped. notify, if set, is told "sleep" just +// before the PC sleeps, "resume" when it wakes and "shutdown" before it shuts down or restarts; it +// should return within a second or two, as Windows goes on without it. +func Run(name string, run func(ctx context.Context) error, notify func(event string)) error { + return svc.Run(name, &handler{run: run, notify: notify}) }