From e5282ccb02025bd18fe20aa42e283da7c88dc09a Mon Sep 17 00:00:00 2001 From: ProjectZero02 <331545278+ProjectZero02@users.noreply.github.com> Date: Wed, 28 Oct 2026 07:05:25 +0000 Subject: [PATCH 1/2] Plan phase 4: recovery with confirmation Co-Authored-By: Claude Opus 5.5 --- docs/plans/recovery-layer.md | 114 ++++++++++++++++++++++++++++++++--- 1 file changed, 104 insertions(+), 10 deletions(-) diff --git a/docs/plans/recovery-layer.md b/docs/plans/recovery-layer.md index 874e3ce..a0bcc3b 100644 --- a/docs/plans/recovery-layer.md +++ b/docs/plans/recovery-layer.md @@ -251,17 +251,111 @@ Test: ## Phase 4: recovery with confirmation -Build: reset and force off as **propose → confirm → execute → verify**. -- Anyone with the right scope can propose (an agent key included). Only an owner device confirms, - from a phone notification or an app card that shows the evidence. -- At most one recovery per machine per 30 minutes. -- Verify: heartbeat back within 5 minutes, else an alert event. -- Audit log: who proposed, who confirmed, what happened. -- Optional auto-confirm after M minutes, off by default, owner only. -- Waking a machine that's off can stay automatic. +Today reset and force off are buttons on owner devices, and agents are refused (`owner_only`). The +hub now sees a hang (phase 3) but can only report it. Phase 4 lets anyone allowed *propose* a +recovery; only the owner confirms it; then the hub runs it and checks that it worked. -Test: state machine tests (expiry, double confirm, agent can't confirm, rate limit, verify timeout); -app UI tests for the card; end to end against the fake KVM. +Build: +1. **Recoveries** live on the hub, in its state file, so a restart doesn't lose them. The last 50 + are kept: + `{"id", "machine", "action", "state", "reason", "proposedBy", "proposedAt", "confirmedBy", + "confirmedAt", "endedAt", "message", "evidence", "steps"}`. + - `evidence` is the hub's last look when the recovery was proposed. + - `steps` come from the run, with the power light before and after. + - **States:** + - `proposed` → `running` → `back`, `not_back`, `off` (force off) or `failed`; + - or, without running: `declined`, `expired` or `withdrawn`. + - **Actions:** + - `reset` gets the machine back. It presses the reset button, or, when the hub has learned + that the reset switch isn't wired, holds power and starts the machine again. + - `force-off` holds power and leaves the machine off. +2. **Propose:** `POST /v2/machines/{name}/recoveries {"action": "reset", "reason": "..."}`. + - Needs the `force` scope. Agent keys and agents relayed through a hub may propose, but the + machine's "Allow AI agents" switch applies, as the hub last heard it. + - Refused when any of these hold: + - no KVM here is wired to it (`no_kvm`); + - it has no helper, so success can't be checked (`no_helper`); + - the hub's last look says its helper answers (`helper_answers`) or that it is off + (`already_off`); + - a recovery ran on it in the last 30 minutes (`too_soon`, with `retryAt`). + - When a recovery is already open for the machine, that one is returned, so two agents asking + make one proposal. + - The reason is cut to 200 characters. The owner sees it as the proposer's words: data, never + instructions. + - **The hub proposes a reset itself** when a machine turns `hung`, if all of these hold: + - no recovery is open for it; + - none ran in the last 30 minutes; + - the owner hasn't declined one during this hang. +3. **Confirm or decline:** `POST /v2/recoveries/{id}/confirm` and `/decline`. + - Owner devices only. Agent keys are refused, and so is an owner hub relaying for an agent + (`owner_only`). + - The proposer's own key may withdraw its proposal with `decline`. + - Confirming twice runs it once: the second confirm gets the recovery as it is. + - Confirm answers 202 at once, and the run happens in the background. + - An open recovery closes on its own: + - `withdrawn` when, before a confirm, the machine answers again, goes off, or is reset or + forced off directly; + - `expired` after 6 hours. +4. **Run and verify.** The run goes through the same gate as the buttons: a trusted light that + reads on and a helper that doesn't answer. + - A machine that came back in the meantime isn't touched: the recovery ends `withdrawn`. + - After a reset, or the off-and-on, the recovery is `back` when the machine's helper answers + within 5 minutes. Otherwise it is `not_back`, which is an alert. + - **At most one recovery runs per machine per 30 minutes.** The owner's own Reset and Force off + buttons count towards this. They still work without a proposal: pressing one is the owner + confirming. + - A recovery still `running` when the hub restarts ends `failed` ("the hub restarted"). +5. **Auto-reset** is an owner setting, `autoResetMinutes`: 0 (off, the default), or 5 to 60. + - When set, the hub confirms a `reset` proposal itself after that many minutes, if the machine + is still `hung` by the hub's own evidence. + - Never for `force-off`. +6. **Events and audit.** + - Each change of a recovery's state is an event: `{"event": "recovery", "machine", "recovery": + {...}}`. + - On the event stream it's the SSE event `recovery`. + - Webhooks get it with the event filter `recovery`, which new webhooks have by default. + - The activity log gets one line for each step: who proposed and why, who confirmed or declined, + and whether the machine answered again. + - `GET /v2/recoveries` (status scope; `?open=1` for open ones only) and + `GET /v2/recoveries/{id}`. Both show only machines the caller may see. +7. **App:** + - **Device page:** while a recovery is open, a card shows: + - "pve suggests a reset", or "Atlas asks to reset"; + - what pve sees, and the proposer's reason; + - **Reset** and **Not now** buttons. + + While it runs: "Resetting…, waiting up to 5 minutes for it to answer". After: "Back at 14:23", + or "Didn't come back", with Force off. + - **Notification:** "gaming-pc looks frozen. pve can reset it." + - Its **Reset** action works only on an unlocked phone (`setAuthenticationRequired`). + - Tapping it opens the device page. + - "Didn't come back" is notified too. + - For a machine with a reset proposal, this replaces the plain "looks frozen" notification. + - **Settings → KVM hub:** "Reset a frozen machine on its own": Off, or after 10, 30 or 60 + minutes. + +Test: +- State machine tests with a fake clock: + - every propose refusal, and an open recovery returned instead of a second one; + - confirm by owner devices only: agent key, owner hub relaying for an agent, and the proposer's + key are all refused; + - a double confirm runs once; + - decline, and withdrawal when the machine answers or goes off; + - expiry at 6 hours; + - the 30-minute limit, counting the direct buttons; + - auto-reset after M minutes only while hung, and never for force off; + - a confirm after the machine came back presses nothing. +- End to end against the fake NanoKVM and a fake helper: + - hang → hub proposal → confirm → reset → `back`; + - a reset that doesn't bring it back → `not_back` event and log line; + - reset switch not wired → off and on. +- A hub restart during a run ends it `failed`. +- Server: routes, scopes, recovery events on SSE and webhooks. +- Android: + - card states (Robolectric screenshot); + - parsing recoveries; + - the notification's Reset action needs an unlocked phone and confirms; + - no duplicate "looks frozen". ## Phase 5: assistant API, MCP, docs, hardware checklist From 5fd3a4690cc9a4c4fe825450629d8cd23fe94973 Mon Sep 17 00:00:00 2001 From: ProjectZero02 <331545278+ProjectZero02@users.noreply.github.com> Date: Wed, 28 Oct 2026 07:29:52 +0000 Subject: [PATCH 2/2] Phase 4: recovery with confirmation Anyone allowed proposes a reset or force off; only the owner confirms it; the hub runs it and checks the machine came back. - Hub: recoveries in the state file (proposed, running, back, not_back, off, done, failed, declined, withdrawn, expired). The hub proposes a reset when a machine turns hung, once per hang. At most one runs per machine per 30 minutes, the owner's direct buttons included. Proposals are withdrawn when the machine answers or goes off and expire after 6 hours. A run cut short by a restart ends failed. - Optional auto-reset (autoResetMinutes, off by default): confirms the hub's reset proposal after N minutes while the machine is still hung. Never for force off. - API: POST /v2/machines/{name}/recoveries (power scope; agents subject to the machine's Allow AI agents switch), GET /v2/recoveries[/{id}], confirm (owner devices only) and decline (owner, or the proposer's key). recovery events on SSE and webhooks; Machine.recovery. - App: recovery card on the device page (Reset / Not now, running, back, didn't come back with Force off); notification with Reset and Not now that need an unlocked phone; the hub's Frozen machines setting. - docs/protocol.md: Recoveries section. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 22 + .../powerdeck/data/HelperClient.kt | 15 + .../net/odysseylabs/powerdeck/data/Model.kt | 43 +- .../odysseylabs/powerdeck/data/PowerDeck.kt | 27 + .../odysseylabs/powerdeck/ui/Components.kt | 39 ++ .../odysseylabs/powerdeck/ui/DeviceScreen.kt | 75 ++- .../odysseylabs/powerdeck/ui/MainActivity.kt | 2 + .../odysseylabs/powerdeck/ui/MainViewModel.kt | 13 + .../powerdeck/ui/QuickActionActivity.kt | 19 +- .../powerdeck/work/EventsWorker.kt | 125 ++++- .../odysseylabs/powerdeck/ScreenshotTest.kt | 72 +++ .../net/odysseylabs/powerdeck/UiRulesTest.kt | 74 +++ docs/plans/recovery-layer.md | 20 +- docs/protocol.md | 136 ++++- helper/internal/app/app.go | 3 +- helper/internal/machines/events.go | 16 +- helper/internal/machines/machines.go | 90 +++- helper/internal/machines/recovery.go | 487 ++++++++++++++++++ helper/internal/machines/recovery_test.go | 366 +++++++++++++ helper/internal/machines/watch.go | 1 + helper/internal/machines/watch_test.go | 22 +- helper/internal/server/machines.go | 2 +- helper/internal/server/recovery.go | 150 ++++++ helper/internal/server/recovery_test.go | 164 ++++++ helper/internal/server/relay.go | 12 +- helper/internal/server/server.go | 29 +- helper/internal/server/watch.go | 12 +- helper/internal/store/store.go | 19 + helper/internal/webhook/webhook.go | 2 +- helper/internal/webhook/webhook_test.go | 15 + 30 files changed, 1993 insertions(+), 79 deletions(-) create mode 100644 helper/internal/machines/recovery.go create mode 100644 helper/internal/machines/recovery_test.go create mode 100644 helper/internal/server/recovery.go create mode 100644 helper/internal/server/recovery_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e08478..2287e89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## 2.4.0 + +A frozen computer can now be reset with one tap from a notification, and an assistant can ask for a reset that you confirm. + +**Update the helper on the hub and the phone app.** The computers' own helpers don't need 2.4. + +- **The hub suggests a reset.** When a computer is called frozen, the hub proposes a reset. Its page shows a card: what the hub saw, and **Reset** or **Not now**. The phone also notifies you, with **Reset** and **Not now** buttons that work only once the phone is unlocked. While the reset runs the card says so. After it, the card says the computer is back, or that it didn't answer within 5 minutes, with **Force off**. You're notified either way. +- **Assistants propose, you confirm.** An AI agent can't reset or force off a computer itself. With the `power` scope it can now ask (`POST /v2/machines/{name}/recoveries`, with its reason), and you confirm on the phone. Its reason is shown quoted, as its words. The computer's **Allow AI agents** switch still applies. Two agents asking make one request. +- **Safety rules on the hub:** + - At most one reset or force off per computer every 30 minutes. Your own Reset and Force off buttons count towards this, but are never held back. + - A request is dropped when the computer answers again or goes off, and expires after 6 hours. + - The reset presses nothing if the computer came back in the meantime. + - A reset that's still running when the hub restarts is marked failed. + - Every step is in the hub's activity log, with who did it. +- **Reset on its own (optional, off by default).** On the hub's page, **Frozen machines → Reset a frozen machine on its own**: after 10, 30 or 60 minutes without an answer from you, the hub resets a computer that is still frozen. It never forces one off on its own. +- **API:** + - `GET /v2/recoveries`, plus confirm and decline for each one. + - `recovery` events on the event stream and to webhooks (new webhooks get them by default). + - Machines gain a `recovery` field. + - The `autoResetMinutes` setting. + - See [docs/protocol.md](docs/protocol.md#recoveries). + ## 2.3.0 The hub now notices when a computer freezes, and says why it thinks so. diff --git a/android/app/src/main/java/net/odysseylabs/powerdeck/data/HelperClient.kt b/android/app/src/main/java/net/odysseylabs/powerdeck/data/HelperClient.kt index 96dfe55..1ff3e38 100644 --- a/android/app/src/main/java/net/odysseylabs/powerdeck/data/HelperClient.kt +++ b/android/app/src/main/java/net/odysseylabs/powerdeck/data/HelperClient.kt @@ -264,6 +264,21 @@ class HelperClient( HubResult.from(call("POST", "/v2/machines/${path(machine)}/$action", JSONObject().put("force", force).put("stayOff", stayOff).put("press", press), timeoutSeconds = 150)) // the hub bounds an action to 2 minutes + // ---- Recoveries (a hub, 2.4+): proposed by an agent or the hub, confirmed here ---- + + /** Runs a proposed reset or force off. The hub answers at once; the machine's [HubMachine.recovery] tells how it went. */ + suspend fun confirmRecovery(id: String): HubRecovery = HubRecovery.from(call("POST", "/v2/recoveries/${path(id)}/confirm").getJSONObject("recovery")) + + suspend fun declineRecovery(id: String): HubRecovery = HubRecovery.from(call("POST", "/v2/recoveries/${path(id)}/decline").getJSONObject("recovery")) + + suspend fun recoveries(open: Boolean = true): List = + call("GET", "/v2/recoveries" + if (open) "?open=1" else "").optJSONArray("recoveries").list(HubRecovery::from) + + /** Resets a machine that stays frozen this many minutes without asking (10 to 60); 0 turns it off. */ + suspend fun setAutoReset(minutes: Int) { + call("POST", "/v2/settings", JSONObject().put("autoResetMinutes", minutes)) + } + private fun path(s: String) = URLEncoder.encode(s, "UTF-8").replace("+", "%20") private fun enc(s: String) = URLEncoder.encode(s, "UTF-8") diff --git a/android/app/src/main/java/net/odysseylabs/powerdeck/data/Model.kt b/android/app/src/main/java/net/odysseylabs/powerdeck/data/Model.kt index b9e4312..ae1573b 100644 --- a/android/app/src/main/java/net/odysseylabs/powerdeck/data/Model.kt +++ b/android/app/src/main/java/net/odysseylabs/powerdeck/data/Model.kt @@ -98,11 +98,13 @@ data class KvmWiring(val led: String, val power: String, val reset: String) { /** * A machine as a hub sees it (GET /v2/machines). [led] null: not read. From 2.3 the hub's watch adds - * [state] "hung", a [reason] for "unknown", [since] and its last look ([evidence]). + * [state] "hung", a [reason] for "unknown", [since] and its last look ([evidence]); from 2.4 its open + * [recovery], or the last one for 30 minutes after it ended. */ data class HubMachine( val name: String, val state: String, val led: Boolean?, val ledError: String?, val kvm: String?, val wiring: KvmWiring?, val actions: List, val reason: String? = null, val asleep: Boolean = false, val since: Instant? = null, val evidence: HubEvidence? = null, + val recovery: HubRecovery? = null, ) { val hung get() = state == HUNG @@ -113,11 +115,45 @@ data class HubMachine( return HubMachine(o.optString("name"), o.optString("state", "unknown"), if (o.has("led") && !o.isNull("led")) o.optBoolean("led") else null, o.optStringOrNull("ledError"), k?.optStringOrNull("name"), k?.optJSONObject("wiring")?.let(KvmWiring::from), o.optJSONArray("actions")?.strings() ?: emptyList(), - o.optStringOrNull("reason"), o.optBoolean("asleep"), o.instant("since"), o.optJSONObject("evidence")?.let(HubEvidence::from)) + o.optStringOrNull("reason"), o.optBoolean("asleep"), o.instant("since"), o.optJSONObject("evidence")?.let(HubEvidence::from), + o.optJSONObject("recovery")?.let(HubRecovery::from)) } } } +/** + * A reset or force off on a hub (2.4+): someone proposes it, the owner confirms, the hub runs it and + * checks the machine comes back. [reason] is the proposer's words, shown as they are. [direct]: the + * owner's own button, never proposed. + */ +data class HubRecovery( + val id: String, val machine: String, val action: String, val state: String, val reason: String?, + val proposedBy: String, val agent: Boolean, val proposedAt: Instant?, val confirmedBy: String?, val confirmedAt: Instant?, + val endedAt: Instant?, val message: String?, val direct: Boolean, +) { + val proposed get() = state == PROPOSED + val running get() = state == RUNNING + val open get() = proposed || running + val forceOff get() = action == FORCE_OFF + + companion object { + const val PROPOSED = "proposed" + const val RUNNING = "running" + const val BACK = "back" + const val NOT_BACK = "not_back" + const val OFF = "off" + const val DONE = "done" + const val FAILED = "failed" + const val DECLINED = "declined" + const val EXPIRED = "expired" + const val WITHDRAWN = "withdrawn" + const val FORCE_OFF = "force-off" + fun from(o: JSONObject) = HubRecovery(o.optString("id"), o.optString("machine"), o.optString("action", "reset"), o.optString("state"), + o.optStringOrNull("reason"), o.optString("proposedBy"), o.optBoolean("agent"), o.instant("proposedAt"), o.optStringOrNull("confirmedBy"), + o.instant("confirmedAt"), o.instant("endedAt"), o.optStringOrNull("message"), o.optBoolean("direct")) + } +} + /** * The hub's last look at a machine: its helper ("answers", "silent"), the helper's port ("refuses", * "accepts", "no_reply"), a ping ("replies", "no_reply", "unavailable") and the power light ("on", @@ -209,6 +245,8 @@ data class HelperStatus( val ownersLegacy: Boolean = false, /** The helpers this one holds a peer key for (owners only; null from helpers before 2.1). */ val peerKeys: List? = null, + /** A hub's settings (2.4+, owners only): automatic resets after this many minutes of a hang, 0 off. */ + val autoResetMinutes: Int? = null, ) { companion object { fun pendingFrom(p: JSONObject?) = p?.let { Pending(it.optString("action"), Instant.parse(it.optString("at")), it.optString("by")) } @@ -237,6 +275,7 @@ data class HelperStatus( caps = o.optJSONArray("caps")?.strings() ?: emptyList(), ownersLegacy = o.optJSONObject("owners")?.optBoolean("legacy") ?: false, peerKeys = o.optJSONArray("peerKeys")?.strings(), + autoResetMinutes = o.optJSONObject("hub")?.let { if (it.has("autoResetMinutes")) it.optInt("autoResetMinutes") else null }, ) } } diff --git a/android/app/src/main/java/net/odysseylabs/powerdeck/data/PowerDeck.kt b/android/app/src/main/java/net/odysseylabs/powerdeck/data/PowerDeck.kt index 5abdcb6..42225ec 100644 --- a/android/app/src/main/java/net/odysseylabs/powerdeck/data/PowerDeck.kt +++ b/android/app/src/main/java/net/odysseylabs/powerdeck/data/PowerDeck.kt @@ -640,6 +640,33 @@ class PowerDeck( else r.outcome() } + /** + * Answers a reset or force off that [d]'s hub or an agent proposed: [confirm] has the hub run it + * (it answers at once, then checks [d] comes back); otherwise it's declined. + */ + suspend fun answerRecovery(d: Device, id: String, confirm: Boolean): Outcome = oneAtATime(d.id) { + guarded { + val hub = d.kvm?.hub?.let { store.get(it) } ?: return@guarded Outcome(false, "${d.name} has no hub any more.") + val c = client(hub) ?: return@guarded Outcome(false, "${hub.name} has no helper.") + try { + val r = if (confirm) c.confirmRecovery(id) else c.declineRecovery(id) + val what = if (r.forceOff) "force ${d.name} off" else "reset ${d.name}" + when (r.state) { + HubRecovery.RUNNING -> Outcome(true, if (r.forceOff) "Forcing ${d.name} off." else "Resetting ${d.name}. ${hub.name} checks it answers within 5 minutes.") + HubRecovery.DECLINED -> Outcome(true, "OK, ${hub.name} won't $what.") + HubRecovery.EXPIRED -> Outcome(false, "That request expired. Nothing was pressed.") + HubRecovery.WITHDRAWN -> Outcome(true, r.message ?: "It isn't needed any more. Nothing was pressed.") + HubRecovery.FAILED, HubRecovery.NOT_BACK -> Outcome(false, r.message ?: "It didn't work.") + else -> Outcome(true, r.message ?: "Done.") + } + } catch (e: HelperException.Api) { + if (e.code == "not_found") Outcome(false, "${hub.name} no longer has that request.") else throw e + } finally { + hubLock.withLock { hubCache.remove(hub.id) } + } + } + } + /** Helpers that can send this device's wake signal on a schedule (always-on, on its LAN). */ suspend fun wakeHosts(d: Device): List = store.all().filter { it.id != d.id && it.hasHelper && (d.lan.isEmpty() || Lan.overlaps(it.lan, d.lan)) } diff --git a/android/app/src/main/java/net/odysseylabs/powerdeck/ui/Components.kt b/android/app/src/main/java/net/odysseylabs/powerdeck/ui/Components.kt index 9f6e72f..1f9f31c 100644 --- a/android/app/src/main/java/net/odysseylabs/powerdeck/ui/Components.kt +++ b/android/app/src/main/java/net/odysseylabs/powerdeck/ui/Components.kt @@ -63,6 +63,7 @@ import kotlinx.coroutines.launch import net.odysseylabs.powerdeck.data.Device import net.odysseylabs.powerdeck.data.DeviceState import net.odysseylabs.powerdeck.data.HubMachine +import net.odysseylabs.powerdeck.data.HubRecovery import java.time.Duration import java.time.Instant import java.time.ZoneId @@ -146,6 +147,7 @@ fun note(state: DeviceState?, now: Long): Pair? = when (state) { */ fun hubNote(m: HubMachine?, now: Long): Pair? = when { m == null -> null + m.recovery?.running == true -> (if (m.recovery.forceOff) "Forcing it off" else "Resetting") to Tone.Busy m.hung -> (m.since?.let { "Frozen since " + clockTime(it, now) } ?: "Frozen") to Tone.Warn else -> when (m.reason) { "booting" -> "Starting up" to Tone.Busy @@ -159,6 +161,43 @@ fun hubNote(m: HubMachine?, now: Long): Pair? = when { } } +/** What a recovery card says. */ +data class RecoveryText(val title: String, val detail: String?, val tone: Tone) + +/** + * The card for a reset or force off on [device], run by [hub] (which calls itself [hubSelf]), or null + * when there's nothing to say: declined, expired, or not needed any more. Someone else's reason is + * their words, so it's quoted as theirs. + */ +fun recoveryText(r: HubRecovery, device: String, hub: String, now: Long, hubSelf: String = hub): RecoveryText? { + val reset = !r.forceOff + val at = r.endedAt?.let { " at " + clockTime(it, now) } ?: "" + return when (r.state) { + HubRecovery.PROPOSED -> { + val byHub = r.proposedBy.equals(hubSelf, ignoreCase = true) || r.proposedBy.equals(hub, ignoreCase = true) + val who = if (byHub) hub else r.proposedBy + RecoveryText( + if (reset) "$who suggests a reset" else "$who asks to force $device off", + listOfNotNull( + r.reason?.let { if (byHub) it else "“$it”" }, + if (reset) "A reset is like pressing its reset button: unsaved work is lost. $hub then checks it answers within 5 minutes." + else "Like pulling the plug: unsaved work is lost, and it stays off.", + ).joinToString("\n"), + Tone.Warn, + ) + } + HubRecovery.RUNNING -> if (reset) RecoveryText("Resetting $device…", "$hub waits up to 5 minutes for it to answer.", Tone.Busy) + else RecoveryText("Forcing $device off…", "Holding its power button for 12 seconds.", Tone.Busy) + HubRecovery.BACK -> RecoveryText("$device is back", "It answered$at, after the ${if (reset) "reset" else "force off"}.", Tone.On) + HubRecovery.NOT_BACK -> RecoveryText("$device didn't come back", + "It didn't answer within 5 minutes of the reset. Force it off and on, or check it in person.", Tone.Warn) + HubRecovery.OFF -> RecoveryText("$device is off", "Forced off$at.", Tone.Off) + HubRecovery.DONE -> RecoveryText(if (reset) "Reset $device" else "Forced $device off", r.message, Tone.On) + HubRecovery.FAILED -> RecoveryText(if (reset) "The reset didn't work" else "Couldn't force $device off", r.message, Tone.Warn) + else -> null + } +} + private val shortTime = DateTimeFormatter.ofLocalizedTime(FormatStyle.SHORT).withZone(ZoneId.systemDefault()) private val shortDateTime = DateTimeFormatter.ofLocalizedDateTime(FormatStyle.SHORT).withZone(ZoneId.systemDefault()) diff --git a/android/app/src/main/java/net/odysseylabs/powerdeck/ui/DeviceScreen.kt b/android/app/src/main/java/net/odysseylabs/powerdeck/ui/DeviceScreen.kt index d8a8392..0959c92 100644 --- a/android/app/src/main/java/net/odysseylabs/powerdeck/ui/DeviceScreen.kt +++ b/android/app/src/main/java/net/odysseylabs/powerdeck/ui/DeviceScreen.kt @@ -30,6 +30,7 @@ import androidx.compose.material3.AlertDialog import androidx.compose.material3.Button import androidx.compose.material3.ButtonDefaults import androidx.compose.material3.Card +import androidx.compose.material3.CircularProgressIndicator import androidx.compose.material3.CardDefaults import androidx.compose.material3.DropdownMenu import androidx.compose.material3.DropdownMenuItem @@ -76,6 +77,7 @@ import net.odysseylabs.powerdeck.data.DeviceState import net.odysseylabs.powerdeck.data.Event import net.odysseylabs.powerdeck.data.HelperStatus import net.odysseylabs.powerdeck.data.HubMachine +import net.odysseylabs.powerdeck.data.HubRecovery import net.odysseylabs.powerdeck.data.KeyInfo import net.odysseylabs.powerdeck.data.Peer import net.odysseylabs.powerdeck.data.KvmWiring @@ -114,6 +116,8 @@ data class DeviceActions( val updateHelper: () -> Unit = {}, val pauseUpdates: (Boolean) -> Unit = {}, val removeOwner: (OwnerInfo) -> Unit = {}, + val answerRecovery: (HubRecovery, confirm: Boolean) -> Unit = { _, _ -> }, + val setAutoReset: (minutes: Int) -> Unit = {}, ) private val DAYS = listOf("mon", "tue", "wed", "thu", "fri", "sat", "sun") @@ -175,6 +179,41 @@ private fun HubEvidenceSection(name: String, hubName: String, m: HubMachine) { } } +/** A reset or force off on [d]'s hub: proposed (Reset or Not now), running, or how it ended. */ +@Composable +private fun RecoveryCard(r: HubRecovery, t: RecoveryText, busy: Boolean, answer: (Boolean) -> Unit, forceOff: () -> Unit) { + val warn = t.tone == Tone.Warn + Card(shape = RoundedCornerShape(24.dp), modifier = Modifier.fillMaxWidth(), colors = CardDefaults.cardColors( + containerColor = if (warn) MaterialTheme.colorScheme.errorContainer else MaterialTheme.colorScheme.surfaceContainer, + contentColor = if (warn) MaterialTheme.colorScheme.onErrorContainer else MaterialTheme.colorScheme.onSurface)) { + Column(Modifier.padding(16.dp)) { + Row(verticalAlignment = Alignment.CenterVertically) { + if (r.running) CircularProgressIndicator(Modifier.size(18.dp), strokeWidth = 2.dp) + else Icon(if (warn) Icons.Rounded.WarningAmber else Icons.Rounded.CheckCircle, null, Modifier.size(20.dp)) + Spacer(Modifier.width(12.dp)) + Text(t.title, style = MaterialTheme.typography.titleMedium) + } + t.detail?.let { Text(it, style = MaterialTheme.typography.bodyMedium, modifier = Modifier.padding(top = 8.dp)) } + if (r.proposed) Row(Modifier.padding(top = 16.dp), horizontalArrangement = Arrangement.spacedBy(8.dp)) { + OutlinedButton(onClick = { answer(false) }, enabled = !busy, modifier = Modifier.weight(1f).height(48.dp), shape = RoundedCornerShape(16.dp)) { + Text("Not now") + } + Button(onClick = { answer(true) }, enabled = !busy, modifier = Modifier.weight(1f).height(48.dp), shape = RoundedCornerShape(16.dp), + colors = ButtonDefaults.buttonColors(containerColor = MaterialTheme.colorScheme.error, contentColor = MaterialTheme.colorScheme.onError)) { + Icon(if (r.forceOff) Icons.Rounded.PowerSettingsNew else Icons.Rounded.RestartAlt, null, Modifier.size(18.dp)) + Spacer(Modifier.width(6.dp)) + Text(if (r.forceOff) "Force off" else "Reset") + } + } + if (r.state == HubRecovery.NOT_BACK) OutlinedButton(onClick = forceOff, enabled = !busy, modifier = Modifier.padding(top = 16.dp).height(48.dp), + shape = RoundedCornerShape(16.dp)) { Text("Force off") } + } + } +} + +/** The minutes a hub may wait before it resets a frozen machine on its own; 0 is off (it asks). */ +private val AUTO_RESET = listOf(0, 10, 30, 60) + @OptIn(ExperimentalMaterial3Api::class, ExperimentalLayoutApi::class) @Composable fun DeviceScreen(d: Device, state: UiState, detail: Detail, a: DeviceActions, snackbar: SnackbarHostState) { @@ -190,6 +229,7 @@ fun DeviceScreen(d: Device, state: UiState, detail: Detail, a: DeviceActions, sn var confirmForce by remember { mutableStateOf(false) } var confirmReset by remember { mutableStateOf(false) } val hub = d.kvm?.hub?.let { id -> state.devices.firstOrNull { it.id == id } } + val recovery = (st as? DeviceState.On)?.hub?.recovery var renaming by remember { mutableStateOf(false) } var scheduling by remember { mutableStateOf(false) } var linkKvm by remember { mutableStateOf(null) } @@ -245,11 +285,14 @@ fun DeviceScreen(d: Device, state: UiState, detail: Detail, a: DeviceActions, sn if (d.hasHelper || d.kvm != null) HoldButton("Shut down", Icons.Rounded.PowerSettingsNew, true, !busy, Modifier.weight(1f), hold = hold, onHint = { a.hint("Hold to shut down ${d.name}") }) { a.shutdown(false) } // The light is on but the helper is silent: it may be frozen, which only a reset fixes. - if (d.hasHelper && status == null && hub != null) FilledTonalButton(onClick = { confirmReset = true }, enabled = !busy, - modifier = Modifier.weight(1f).height(48.dp), shape = RoundedCornerShape(16.dp)) { - Icon(Icons.Rounded.RestartAlt, null, Modifier.size(18.dp)) - Spacer(Modifier.width(6.dp)) - Text("Reset") + // A proposed or running recovery has its own card (and Reset button) below. + if (d.hasHelper && status == null && hub != null) { + if (recovery?.open != true) FilledTonalButton(onClick = { confirmReset = true }, enabled = !busy, + modifier = Modifier.weight(1f).height(48.dp), shape = RoundedCornerShape(16.dp)) { + Icon(Icons.Rounded.RestartAlt, null, Modifier.size(18.dp)) + Spacer(Modifier.width(6.dp)) + Text("Reset") + } } else if (d.hasHelper) HoldButton("Restart", Icons.Rounded.RestartAlt, false, !busy, Modifier.weight(1f), hold = hold, tonal = true, onHint = { a.hint("Hold to restart ${d.name}") }, onConfirm = a.restart) @@ -268,6 +311,13 @@ fun DeviceScreen(d: Device, state: UiState, detail: Detail, a: DeviceActions, sn } } + // ---- A reset or force off someone proposed, or one running (the hub checks it comes back) + recovery?.let { r -> + recoveryText(r, d.name, hub?.name ?: "The hub", now, hub?.fleetName ?: "")?.let { t -> + RecoveryCard(r, t, busy, answer = { a.answerRecovery(r, it) }, forceOff = { confirmForce = true }) + } + } + // ---- What the hub sees, while the helper is silent and the light is on (frozen, booting…) (st as? DeviceState.On)?.hub?.takeIf { status == null && it.evidence != null && (it.hung || it.reason != null) }?.let { m -> HubEvidenceSection(d.name, hub?.name ?: "The hub", m) @@ -390,6 +440,21 @@ fun DeviceScreen(d: Device, state: UiState, detail: Detail, a: DeviceActions, sn } } + // ---- A hub's frozen machines: it asks first, or resets on its own after a while (owners only) + val auto = status?.autoResetMinutes + if (auto != null && state.devices.any { it.kvm?.hub == d.id }) Section("Frozen machines") { + Row2("Reset a frozen machine on its own", when (auto) { + 0 -> "${d.name} asks you first, here and in a notification" + else -> "After $auto minutes without an answer from you, ${d.name} presses its reset button. At most once every 30 minutes" + }) + FlowRow(Modifier.padding(horizontal = 16.dp, vertical = 4.dp), horizontalArrangement = Arrangement.spacedBy(8.dp)) { + (AUTO_RESET + auto).distinct().sorted().forEach { m -> + FilterChip(selected = auto == m, onClick = { if (auto != m) a.setAutoReset(m) }, enabled = !busy, + label = { Text(if (m == 0) "Off" else "After $m min") }) + } + } + } + // ---- Agents: one switch once an agent can reach this device, a Set up button until then if (d.hasHelper) Section("AI agents") { val keys = agentKeysFor(d, state.states) diff --git a/android/app/src/main/java/net/odysseylabs/powerdeck/ui/MainActivity.kt b/android/app/src/main/java/net/odysseylabs/powerdeck/ui/MainActivity.kt index 45940da..149986b 100644 --- a/android/app/src/main/java/net/odysseylabs/powerdeck/ui/MainActivity.kt +++ b/android/app/src/main/java/net/odysseylabs/powerdeck/ui/MainActivity.kt @@ -208,6 +208,8 @@ private fun App(vm: MainViewModel, activity: MainActivity) { updateHelper = { vm.updateHelper(d) }, pauseUpdates = { vm.pauseUpdates(d, it) }, removeOwner = { vm.removeOwner(d, it) }, + answerRecovery = { r, yes -> vm.answerRecovery(d, r, yes) }, + setAutoReset = { vm.setAutoReset(d, it) }, ), snackbar, ) diff --git a/android/app/src/main/java/net/odysseylabs/powerdeck/ui/MainViewModel.kt b/android/app/src/main/java/net/odysseylabs/powerdeck/ui/MainViewModel.kt index f1a61b9..fca44c7 100644 --- a/android/app/src/main/java/net/odysseylabs/powerdeck/ui/MainViewModel.kt +++ b/android/app/src/main/java/net/odysseylabs/powerdeck/ui/MainViewModel.kt @@ -20,6 +20,7 @@ import net.odysseylabs.powerdeck.data.AddLink import net.odysseylabs.powerdeck.data.AddResult import net.odysseylabs.powerdeck.data.Check import net.odysseylabs.powerdeck.data.Device +import net.odysseylabs.powerdeck.data.HubRecovery import net.odysseylabs.powerdeck.data.DeviceState import net.odysseylabs.powerdeck.data.Event import net.odysseylabs.powerdeck.data.KeyInfo @@ -192,6 +193,8 @@ class MainViewModel(app: Application) : AndroidViewModel(app) { fun forceOff(d: Device) = act(d) { deck.forceOff(d).also { o -> ask(d, o) } } fun reset(d: Device) = act(d) { deck.reset(d).also { o -> ask(d, o) } } fun recheckKvm(d: Device) = act(d) { deck.recheckKvm(d) } + /** Confirms or declines a reset or force off that [d]'s hub or an agent proposed. */ + fun answerRecovery(d: Device, r: HubRecovery, confirm: Boolean) = act(d) { deck.answerRecovery(d, r.id, confirm) } private fun ask(d: Device, o: Outcome) { o.ask?.let { a -> _state.update { it.copy(force = ForcePrompt(d, false, a)) } } @@ -456,6 +459,16 @@ class MainViewModel(app: Application) : AndroidViewModel(app) { } } + /** Has hub [d] reset a machine that stays frozen [minutes] without asking; 0 turns it off. */ + fun setAutoReset(d: Device, minutes: Int) { + viewModelScope.launch { + deck.withHelper(d) { setAutoReset(minutes) } + .onSuccess { _messages.send(if (minutes == 0) "${d.name} asks you before each reset." else "${d.name} resets a machine that stays frozen $minutes minutes.") } + .onFailure { _messages.send(it.message ?: "Couldn't change it.") } + refreshNow() + } + } + fun setTile(slot: Int, d: Device?) = viewModelScope.launch { deck.store.setTile(slot, d?.id) Shortcuts.refreshAll(getApplication()) diff --git a/android/app/src/main/java/net/odysseylabs/powerdeck/ui/QuickActionActivity.kt b/android/app/src/main/java/net/odysseylabs/powerdeck/ui/QuickActionActivity.kt index f57bad4..a345d04 100644 --- a/android/app/src/main/java/net/odysseylabs/powerdeck/ui/QuickActionActivity.kt +++ b/android/app/src/main/java/net/odysseylabs/powerdeck/ui/QuickActionActivity.kt @@ -3,6 +3,7 @@ package net.odysseylabs.powerdeck.ui import android.content.Context import android.content.Intent import android.os.Bundle +import androidx.core.app.NotificationManagerCompat import androidx.activity.ComponentActivity import androidx.activity.compose.setContent import androidx.compose.foundation.layout.Arrangement @@ -47,22 +48,33 @@ import androidx.compose.runtime.collectAsState /** * The small dialog the widget, tiles and shortcuts open: checks the device, then turns it on straight - * away, or asks before shutting it down. + * away, or asks before shutting it down. A notification's Reset or Not now opens it to answer the + * hub's proposal, then it says how that went. */ class QuickActionActivity : ComponentActivity() { companion object { private const val EXTRA_DEVICE = "device" + private const val EXTRA_RECOVERY = "recovery" + private const val EXTRA_CONFIRM = "confirm" + private const val EXTRA_NOTIFICATION = "notification" fun intent(ctx: Context, deviceId: String): Intent = Intent(ctx, QuickActionActivity::class.java) .setAction("net.odysseylabs.powerdeck.QUICK") .putExtra(EXTRA_DEVICE, deviceId) .addFlags(Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_CLEAR_TASK) + + /** Confirms or declines recovery [recoveryId] of [deviceId], and clears [notification]. */ + fun answer(ctx: Context, deviceId: String, recoveryId: String, confirm: Boolean, notification: Int): Intent = + intent(ctx, deviceId).setAction("net.odysseylabs.powerdeck.RECOVERY") + .putExtra(EXTRA_RECOVERY, recoveryId).putExtra(EXTRA_CONFIRM, confirm).putExtra(EXTRA_NOTIFICATION, notification) } override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) val id = intent.getStringExtra(EXTRA_DEVICE) ?: return finish() - setContent { PowerDeckTheme { QuickAction(id, deck, ::finish) } } + val answer = intent.getStringExtra(EXTRA_RECOVERY)?.let { it to intent.getBooleanExtra(EXTRA_CONFIRM, false) } + if (answer != null) NotificationManagerCompat.from(this).cancel(intent.getIntExtra(EXTRA_NOTIFICATION, 0)) + setContent { PowerDeckTheme { QuickAction(id, deck, ::finish, answer) } } } } @@ -76,7 +88,7 @@ private sealed interface QuickStep { } @Composable -private fun QuickAction(id: String, deck: PowerDeck, close: () -> Unit) { +private fun QuickAction(id: String, deck: PowerDeck, close: () -> Unit, answer: Pair? = null) { val scope = rememberCoroutineScope() val ctx = LocalContext.current var step by remember { mutableStateOf(QuickStep.Checking) } @@ -107,6 +119,7 @@ private fun QuickAction(id: String, deck: PowerDeck, close: () -> Unit) { step = QuickStep.Done(Outcome(false, "Tailscale isn't connected. Open Tailscale and connect.")) return@LaunchedEffect } + if (answer != null) return@LaunchedEffect perform({ deck.answerRecovery(d, answer.first, answer.second) }) when (val s = deck.state(d)) { is DeviceState.On -> step = QuickStep.AskOff(d.hasHelper, s.status?.pending) DeviceState.Off -> step = QuickStep.AskOn diff --git a/android/app/src/main/java/net/odysseylabs/powerdeck/work/EventsWorker.kt b/android/app/src/main/java/net/odysseylabs/powerdeck/work/EventsWorker.kt index 3c9fc8d..eec82a1 100644 --- a/android/app/src/main/java/net/odysseylabs/powerdeck/work/EventsWorker.kt +++ b/android/app/src/main/java/net/odysseylabs/powerdeck/work/EventsWorker.kt @@ -21,11 +21,15 @@ import net.odysseylabs.powerdeck.R import net.odysseylabs.powerdeck.data.Device import net.odysseylabs.powerdeck.data.DeviceState import net.odysseylabs.powerdeck.data.Event +import net.odysseylabs.powerdeck.data.HubRecovery import net.odysseylabs.powerdeck.data.Power import net.odysseylabs.powerdeck.data.Tailscale import net.odysseylabs.powerdeck.data.Updater import net.odysseylabs.powerdeck.deck import net.odysseylabs.powerdeck.ui.MainActivity +import net.odysseylabs.powerdeck.ui.QuickActionActivity +import net.odysseylabs.powerdeck.ui.RecoveryText +import net.odysseylabs.powerdeck.ui.recoveryText import net.odysseylabs.powerdeck.ui.Shortcuts import java.time.Instant import java.util.concurrent.TimeUnit @@ -54,7 +58,7 @@ class EventsWorker(ctx: Context, params: WorkerParameters) : CoroutineWorker(ctx val seen = deck.store.seenEvents(d.id)?.let { runCatching { Instant.parse(it) }.getOrNull() } val fresh = events.filter { seen == null || it.at.isAfter(seen) }.sortedBy { it.at } // On the first run only remember where the log is; don't replay history. - if (seen != null) fresh.filter { notable(it, d, helperNames) }.takeLast(3).forEach { notify(ctx, d, describe(it, d)) } + if (seen != null) tell(ctx, d, all, quiet(fresh.filter { notable(it, d, helperNames) }).takeLast(3)) events.maxOfOrNull { it.at }?.let { deck.store.setSeenEvents(d.id, it.toString()) } } else if (before == Power.On && st == DeviceState.Off) { notify(ctx, d, "${d.name} went offline.") @@ -64,6 +68,24 @@ class EventsWorker(ctx: Context, params: WorkerParameters) : CoroutineWorker(ctx return Result.success() } + /** + * Notifies [events] from [d]'s log. A reset proposed for a machine of this hub comes with Reset and + * Not now, which work only on an unlocked phone; tapping opens that machine's page. + */ + private suspend fun tell(ctx: Context, d: Device, all: List, events: List) { + val open = if (events.any { it.result == PROPOSED }) ctx.deck.withHelper(d) { recoveries() }.getOrNull().orEmpty() else emptyList() + for (e in events) { + val m = recoveryMachine(e) + val target = m?.let { name -> all.firstOrNull { it.kvm?.hub == d.id && (it.kvm.machine ?: it.fleetName).equals(name, ignoreCase = true) } } + val r = if (e.result == PROPOSED) open.firstOrNull { it.proposed && it.machine.equals(m, ignoreCase = true) } else null + val text = if (r != null && target != null) recoveryText(r, target.name, d.name, System.currentTimeMillis(), d.fleetName) else null + when { + text != null -> notifyRecovery(ctx, target!!, r!!, text) + else -> notify(ctx, target ?: d, describe(e, d), urgent = e.result == NOT_BACK) + } + } + } + /** Once a day: tell you about a new version of this app (Android installs it only after you tap). */ private suspend fun appUpdate(ctx: Context) { val store = ctx.deck.store @@ -89,7 +111,9 @@ class EventsWorker(ctx: Context, params: WorkerParameters) : CoroutineWorker(ctx private fun notable(e: Event, d: Device, helpers: Set): Boolean { if (hubWatch(e)) return true - if (d.me != null && (e.who == d.me || e.who.startsWith("${d.me} via "))) return false + val mine = d.me != null && (e.who == d.me || e.who.startsWith("${d.me} via ")) + if (recoveryMachine(e) != null) return !(mine && e.result == "confirmed") + if (mine) return false val verb = e.action.substringBefore(' ').lowercase() val target = e.action.substringAfter(' ', "").lowercase() // A hub relaying to another device: that device's own log reports it. @@ -99,6 +123,17 @@ class EventsWorker(ctx: Context, params: WorkerParameters) : CoroutineWorker(ctx private fun describe(e: Event, d: Device): String { if (hubWatch(e)) return e.action + "." // already a sentence: "Workstation looks frozen: …" + recoveryMachine(e)?.let { m -> + val what = if (e.action.contains("force-off") || e.action.contains("force off")) "force $m off" else "reset $m" + return when { + e.result == PROPOSED -> "${e.who} asks to $what." + e.result == "confirmed" && e.who == AUTO -> "${d.name} is going to $what on its own: it stayed frozen." + e.result == "confirmed" -> "${e.who} confirmed: ${d.name} is going to $what." + e.result.startsWith("failed") -> "${d.name} couldn't $what: ${e.result.substringAfter(": ")}" + e.result == NOT_BACK -> e.action + ". Force it off and on, or check it in person." + else -> e.action + "." // "Workstation answers again after the reset", "Workstation is off" + } + } val verb = e.action.substringBefore(' ') val target = e.action.substringAfter(' ', "").ifBlank { d.name } val what = when (verb) { @@ -119,8 +154,41 @@ class EventsWorker(ctx: Context, params: WorkerParameters) : CoroutineWorker(ctx internal fun hubWatch(e: Event) = e.result == "detected" && e.action.contains(" looks frozen") || e.result == "recovered" && e.action.endsWith(" answers again") + private const val PROPOSED = "proposed" + private const val NOT_BACK = "not_back" + private const val AUTO = "PowerDeck (automatic)" + private val proposal = Regex("""^(?:propose|confirm) (?:reset|force-off) ([^\s:]+)""") + private val ran = Regex("""^(?:reset|force-off) ([^\s:]+)$""") + + /** + * The machine a hub's (2.4+) recovery line is about: proposed, confirmed, back, not back, off + * or failed; null for any other line (see docs/protocol.md, Recoveries). + */ + internal fun recoveryMachine(e: Event): String? = when (e.result) { + PROPOSED, "confirmed" -> proposal.find(e.action)?.groupValues?.get(1) + "back" -> e.action.substringBefore(" answers again after the ", "").ifEmpty { null } + NOT_BACK -> e.action.substringBefore(" didn't answer within ", "").ifEmpty { null } + "off" -> e.action.removeSuffix(" is off").takeIf { it != e.action } + else -> if (e.result.startsWith("failed")) ran.find(e.action)?.groupValues?.get(1) else null + } + + /** + * Drops what a recovery line already says: "looks frozen" when a reset is proposed for that + * machine, and "answers again" when it's back after one. + */ + internal fun quiet(events: List): List { + fun machines(result: String) = events.filter { it.result == result }.mapNotNull(::recoveryMachine).map(String::lowercase).toSet() + val proposed = machines(PROPOSED) + val back = machines("back") + return events.filterNot { e -> + e.result == "detected" && e.action.substringBefore(" looks frozen").lowercase() in proposed || + e.result == "recovered" && e.action.removeSuffix(" answers again").lowercase() in back + } + } + private const val NAME = "events" private const val CHANNEL = "activity" + private const val URGENT = "recovery" private const val UPDATES = "updates" fun schedule(ctx: Context) { @@ -132,23 +200,56 @@ class EventsWorker(ctx: Context, params: WorkerParameters) : CoroutineWorker(ctx fun cancel(ctx: Context) = WorkManager.getInstance(ctx).cancelUniqueWork(NAME) - private fun notify(ctx: Context, d: Device, text: String) { - if (Build.VERSION.SDK_INT >= 33 && ctx.checkSelfPermission(Manifest.permission.POST_NOTIFICATIONS) != PackageManager.PERMISSION_GRANTED) return + private fun allowed(ctx: Context) = + Build.VERSION.SDK_INT < 33 || ctx.checkSelfPermission(Manifest.permission.POST_NOTIFICATIONS) == PackageManager.PERMISSION_GRANTED + + /** [urgent]: a reset or a machine that needs you (its own channel, which alerts). */ + private fun channel(ctx: Context, urgent: Boolean): String { val nm = ctx.getSystemService(NotificationManager::class.java) - nm.createNotificationChannel(NotificationChannel(CHANNEL, "Device activity", NotificationManager.IMPORTANCE_DEFAULT)) - val open = PendingIntent.getActivity( - ctx, d.id.hashCode(), - Intent(ctx, MainActivity::class.java).putExtra(MainActivity.EXTRA_DEVICE, d.id).addFlags(Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_CLEAR_TOP), - PendingIntent.FLAG_IMMUTABLE or PendingIntent.FLAG_UPDATE_CURRENT, - ) - val n = NotificationCompat.Builder(ctx, CHANNEL) + if (urgent) nm.createNotificationChannel(NotificationChannel(URGENT, "Frozen machines", NotificationManager.IMPORTANCE_HIGH)) + else nm.createNotificationChannel(NotificationChannel(CHANNEL, "Device activity", NotificationManager.IMPORTANCE_DEFAULT)) + return if (urgent) URGENT else CHANNEL + } + + private fun openDevice(ctx: Context, d: Device): PendingIntent = PendingIntent.getActivity( + ctx, d.id.hashCode(), + Intent(ctx, MainActivity::class.java).putExtra(MainActivity.EXTRA_DEVICE, d.id).addFlags(Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_CLEAR_TOP), + PendingIntent.FLAG_IMMUTABLE or PendingIntent.FLAG_UPDATE_CURRENT, + ) + + private fun notify(ctx: Context, d: Device, text: String, urgent: Boolean = false) { + if (!allowed(ctx)) return + val n = NotificationCompat.Builder(ctx, channel(ctx, urgent)) .setSmallIcon(R.drawable.ic_power) .setContentTitle(d.name) .setContentText(text) - .setContentIntent(open) + .setStyle(NotificationCompat.BigTextStyle().bigText(text)) + .setContentIntent(openDevice(ctx, d)) .setAutoCancel(true) .build() NotificationManagerCompat.from(ctx).notify(("${d.id}$text").hashCode(), n) } + + /** A proposed reset or force off of [d], with Reset and Not now (unlocked phone only). */ + private fun notifyRecovery(ctx: Context, d: Device, r: HubRecovery, t: RecoveryText) { + if (!allowed(ctx)) return + val id = "recovery ${r.id}".hashCode() + fun answer(yes: Boolean, label: String) = NotificationCompat.Action.Builder( + 0, label, + PendingIntent.getActivity(ctx, id + if (yes) 1 else 2, QuickActionActivity.answer(ctx, d.id, r.id, yes, id), + PendingIntent.FLAG_IMMUTABLE or PendingIntent.FLAG_UPDATE_CURRENT), + ).setAuthenticationRequired(true).build() + val n = NotificationCompat.Builder(ctx, channel(ctx, urgent = true)) + .setSmallIcon(R.drawable.ic_power) + .setContentTitle(t.title) + .setContentText(t.detail?.substringBefore('\n')) + .setStyle(NotificationCompat.BigTextStyle().bigText(t.detail)) + .setContentIntent(openDevice(ctx, d)) + .addAction(answer(true, if (r.forceOff) "Force off" else "Reset")) + .addAction(answer(false, "Not now")) + .setAutoCancel(true) + .build() + NotificationManagerCompat.from(ctx).notify(id, n) + } } } diff --git a/android/app/src/test/java/net/odysseylabs/powerdeck/ScreenshotTest.kt b/android/app/src/test/java/net/odysseylabs/powerdeck/ScreenshotTest.kt index 55875ba..c3f7b7c 100644 --- a/android/app/src/test/java/net/odysseylabs/powerdeck/ScreenshotTest.kt +++ b/android/app/src/test/java/net/odysseylabs/powerdeck/ScreenshotTest.kt @@ -54,6 +54,7 @@ import net.odysseylabs.powerdeck.data.HelperStatus import net.odysseylabs.powerdeck.data.HelperUpdate import net.odysseylabs.powerdeck.data.HubEvidence import net.odysseylabs.powerdeck.data.HubMachine +import net.odysseylabs.powerdeck.data.HubRecovery import net.odysseylabs.powerdeck.data.KvmLink import net.odysseylabs.powerdeck.data.KvmWiring import net.odysseylabs.powerdeck.data.Peer @@ -67,6 +68,8 @@ import android.Manifest import androidx.compose.runtime.CompositionLocalProvider import androidx.compose.ui.platform.LocalInspectionMode import androidx.compose.ui.test.onNodeWithContentDescription +import androidx.compose.ui.test.assertCountEquals +import androidx.compose.ui.test.onAllNodesWithText import androidx.compose.ui.test.performClick import androidx.compose.ui.test.performScrollTo import androidx.compose.ui.test.assertIsDisplayed @@ -92,6 +95,7 @@ import net.odysseylabs.powerdeck.ui.PowerDeckTheme import net.odysseylabs.powerdeck.ui.SettingsActions import net.odysseylabs.powerdeck.ui.SettingsScreen import net.odysseylabs.powerdeck.ui.UiState +import org.junit.Assert.assertEquals import org.junit.Rule import org.junit.Test import org.junit.runner.RunWith @@ -290,6 +294,74 @@ class ScreenshotTest { compose.onRoot().captureRoboImage("build/outputs/roborazzi/device-hung.png") } + private val ws = Device("ws", "Workstation", "100.101.1.11", pin = "w", os = "windows", mac = "02:00:5e:00:00:01", + kvm = KvmLink("nanokvm", "kvm-2.tail1234.ts.net", "kvm-2", "admin", ledSeen = true, hub = "pve", machine = "workstation", + wiring = KvmWiring("wired", "wired", "unknown"))) + + private fun hungWith(r: HubRecovery): UiState { + val seen = Instant.now().minusSeconds(20) + val m = HubMachine("workstation", "hung", true, null, "kvm-2", ws.kvm!!.wiring, listOf("reset", "force-off"), + since = seen.minusSeconds(200), evidence = HubEvidence(seen, "silent", "accepts", "no_reply", "on", true), recovery = r) + return all.copy(devices = all.devices + ws, states = all.states + ("ws" to DeviceState.On(null, m))) + } + + private val proposal = HubRecovery("r1", "workstation", "reset", HubRecovery.PROPOSED, "Nothing answers and its power light is on.", "pve", false, + Instant.now().minusSeconds(10), null, null, null, null, false) + + /** The hub proposes a reset: the card asks, with Reset and Not now; the plain Reset button gives way to it. */ + @Test fun deviceRecoveryProposed() { + var answered: Boolean? = null + shot("device-recovery") { + DeviceScreen(ws, hungWith(proposal), Detail(events = emptyList()), DeviceActions(answerRecovery = { _, yes -> answered = yes }), SnackbarHostState()) + } + compose.onNodeWithText("pve suggests a reset").assertIsDisplayed() + compose.onNodeWithText("Not now").assertIsDisplayed() + compose.onAllNodesWithText("Reset").assertCountEquals(1) + compose.onNodeWithText("Restart").assertDoesNotExist() // its helper can't answer + compose.onRoot().captureRoboImage("build/outputs/roborazzi/device-recovery.png") + compose.onNodeWithText("Reset").performClick() + assertEquals(true, answered) + } + + @Test fun deviceRecoveryRunning() { + val r = proposal.copy(state = HubRecovery.RUNNING, confirmedBy = "Pixel", confirmedAt = Instant.now()) + shot("device-recovery-running") { DeviceScreen(ws, hungWith(r), Detail(events = emptyList()), DeviceActions(), SnackbarHostState()) } + compose.onNodeWithText("Resetting Workstation…").assertIsDisplayed() + compose.onNodeWithText("Resetting").assertIsDisplayed() // the state line + compose.onNodeWithText("Reset").assertDoesNotExist() + compose.onNodeWithText("Restart").assertDoesNotExist() + } + + @Test fun deviceRecoveryNotBack() { + val r = proposal.copy(state = HubRecovery.NOT_BACK, endedAt = Instant.now()) + shot("device-recovery-not-back") { DeviceScreen(ws, hungWith(r), Detail(events = emptyList()), DeviceActions(), SnackbarHostState()) } + compose.onNodeWithText("Workstation didn't come back").assertIsDisplayed() + compose.onNodeWithText("Force off").assertIsDisplayed() + } + + /** The hub's own page: whether it resets a frozen machine on its own. */ + @Test fun hubAutoReset() { + val hub = status("pve", "linux", guests = 3).copy(autoResetMinutes = 0) + val state = all.copy(devices = all.devices + ws, states = all.states + ("pve" to DeviceState.On(hub))) + var set: Int? = null + shot("hub-auto-reset") { DeviceScreen(pve, state, Detail(events = emptyList()), DeviceActions(setAutoReset = { set = it }), SnackbarHostState()) } + compose.onNodeWithText("Reset a frozen machine on its own").performScrollTo().assertIsDisplayed() + compose.onNodeWithText("pve asks you first, here and in a notification").assertExists() + compose.onRoot().captureRoboImage("build/outputs/roborazzi/hub-auto-reset.png") + compose.onNodeWithText("After 10 min").performScrollTo().performClick() + assertEquals(10, set) + } + + /** Not on a helper that drives no KVM. */ + @Test fun noAutoResetWithoutMachines() { + val hub = status("pve", "linux", guests = 3).copy(autoResetMinutes = 0) + compose.setContent { + PowerDeckTheme(dynamic = false) { DeviceScreen(pve, all.copy(states = all.states + ("pve" to DeviceState.On(hub))), Detail(events = emptyList()), DeviceActions(), SnackbarHostState()) } + } + compose.onNodeWithText("AI agents").performScrollTo().assertIsDisplayed() + compose.onNodeWithText("Reset a frozen machine on its own").assertDoesNotExist() + } + @Test fun settingsHub() { shot("settings-hub") { SettingsScreen(Settings(), SettingsActions(), hubs = listOf(pc, pve), showHub = true) } compose.onNodeWithText("Automatic: pve").assertExists() diff --git a/android/app/src/test/java/net/odysseylabs/powerdeck/UiRulesTest.kt b/android/app/src/test/java/net/odysseylabs/powerdeck/UiRulesTest.kt index be24dce..6e2b78a 100644 --- a/android/app/src/test/java/net/odysseylabs/powerdeck/UiRulesTest.kt +++ b/android/app/src/test/java/net/odysseylabs/powerdeck/UiRulesTest.kt @@ -5,6 +5,7 @@ import net.odysseylabs.powerdeck.data.DeviceState import net.odysseylabs.powerdeck.data.Event import net.odysseylabs.powerdeck.data.HelperStatus import net.odysseylabs.powerdeck.data.HubMachine +import net.odysseylabs.powerdeck.data.HubRecovery import com.google.zxing.BarcodeFormat import com.google.zxing.qrcode.QRCodeReader import com.google.zxing.qrcode.QRCodeWriter @@ -13,6 +14,7 @@ import net.odysseylabs.powerdeck.ui.agentKeysFor import net.odysseylabs.powerdeck.ui.decodeLuma import net.odysseylabs.powerdeck.ui.describe import net.odysseylabs.powerdeck.ui.hubNote +import net.odysseylabs.powerdeck.ui.recoveryText import net.odysseylabs.powerdeck.ui.uninstallBlocker import net.odysseylabs.powerdeck.work.EventsWorker import org.json.JSONObject @@ -65,6 +67,78 @@ class UiRulesTest { assertFalse(EventsWorker.hubWatch(Event(at, "Atlas", "Workstation looks frozen", "ok"))) } + @Test fun parsesARecovery() { + val m = HubMachine.from(JSONObject("""{"name":"workstation","state":"hung","led":true,"kvm":{"name":"kvm-2"}, + "recovery":{"id":"r1","machine":"workstation","action":"reset","state":"proposed","reason":"Atlas can't reach it", + "proposedBy":"Atlas","agent":true,"proposedAt":"2026-10-28T08:04:00Z"}}""")) + val r = m.recovery!! + assertEquals(listOf("r1", "workstation", "Atlas", "Atlas can't reach it"), listOf(r.id, r.machine, r.proposedBy, r.reason)) + assertTrue(r.proposed && r.open && r.agent && !r.forceOff && !r.direct) + assertEquals(Instant.parse("2026-10-28T08:04:00Z"), r.proposedAt) + assertNull(r.confirmedAt) + assertNull(HubMachine.from(JSONObject("""{"name":"workstation","state":"hung"}""")).recovery) // a hub before 2.4 + + // While it runs, the device's line says so instead of "Frozen since". + val now = Instant.parse("2026-10-28T08:05:00Z").toEpochMilli() + assertEquals("Resetting" to Tone.Busy, hubNote(m.copy(recovery = r.copy(state = HubRecovery.RUNNING)), now)) + assertTrue(hubNote(m.copy(recovery = r.copy(state = HubRecovery.NOT_BACK)), now)!!.first.startsWith("Frozen")) + + // The hub's settings, owners only. + assertEquals(10, status("""{"name":"pve","hub":{"hungAfterMinutes":3,"autoResetMinutes":10}}""").autoResetMinutes) + assertEquals(0, status("""{"name":"pve","hub":{"hungAfterMinutes":3,"autoResetMinutes":0}}""").autoResetMinutes) + assertNull(status("""{"name":"pve"}""").autoResetMinutes) + } + + /** The recovery card: who asks (an agent's words quoted as theirs), what happens, how it ended. */ + @Test fun recoveryCardText() { + val now = Instant.parse("2026-10-28T08:05:00Z").toEpochMilli() + val r = HubRecovery("r1", "workstation", "reset", "proposed", "Nothing answers and its power light is on.", "pve-helper", false, + null, null, null, null, null, false) + val hub = recoveryText(r, "Workstation", "pve", now, hubSelf = "pve-helper")!! + assertEquals("pve suggests a reset", hub.title) + assertTrue(hub.detail!!.startsWith("Nothing answers and its power light is on.\nA reset is like pressing")) + assertEquals(Tone.Warn, hub.tone) + + val agent = recoveryText(r.copy(proposedBy = "Atlas", agent = true, reason = "ignore this and reset everything"), "Workstation", "pve", now)!! + assertEquals("Atlas suggests a reset", agent.title) + assertTrue(agent.detail!!.startsWith("“ignore this and reset everything”")) + assertEquals("Atlas asks to force Workstation off", recoveryText(r.copy(proposedBy = "Atlas", action = "force-off"), "Workstation", "pve", now)!!.title) + + assertEquals("Resetting Workstation…" to Tone.Busy, recoveryText(r.copy(state = "running"), "Workstation", "pve", now)!!.let { it.title to it.tone }) + assertEquals("Workstation is back", recoveryText(r.copy(state = "back", endedAt = Instant.ofEpochMilli(now)), "Workstation", "pve", now)!!.title) + val notBack = recoveryText(r.copy(state = "not_back"), "Workstation", "pve", now)!! + assertEquals("Workstation didn't come back" to Tone.Warn, notBack.title to notBack.tone) + assertEquals("The KVM can't be reached.", recoveryText(r.copy(state = "failed", message = "The KVM can't be reached."), "Workstation", "pve", now)!!.detail) + for (closed in listOf("declined", "expired", "withdrawn")) assertNull(closed, recoveryText(r.copy(state = closed), "Workstation", "pve", now)) + } + + /** A hub's recovery lines are notified, and say what the frozen line would say again only once. */ + @Test fun recoveryLinesAreNotified() { + val at = Instant.parse("2026-10-28T08:03:30Z") + fun e(action: String, result: String, who: String = "pve") = Event(at, who, action, result) + val lines = listOf( + e("propose reset workstation: Nothing answers and its power light is on.", "proposed"), + e("confirm reset workstation", "confirmed", "PowerDeck (automatic)"), + e("propose force-off workstation", "proposed", "Atlas"), + e("workstation answers again after the reset", "back"), + e("workstation didn't answer within 5 minutes of the reset", "not_back"), + e("workstation is off", "off"), + e("reset workstation", "failed: The KVM can't be reached."), + ) + for (l in lines) assertEquals(l.action, "workstation", EventsWorker.recoveryMachine(l)) + for (l in listOf(e("reset workstation", "done", "Pixel 9 Pro"), e("decline reset workstation", "declined", "Pixel 9 Pro"), + e("reset workstation", "withdrawn: workstation answers again."), e("workstation answers again", "recovered"))) { + assertNull(l.toString(), EventsWorker.recoveryMachine(l)) + } + + val frozen = e("workstation looks frozen: nothing answers, power light on", "detected") + val again = e("workstation answers again", "recovered") + val other = e("NAS looks frozen: nothing answers, power light on", "detected") + assertEquals(listOf(lines[0], other), EventsWorker.quiet(listOf(frozen, lines[0], other))) + assertEquals(listOf(lines[3]), EventsWorker.quiet(listOf(again, lines[3]))) + assertEquals(listOf(frozen, again), EventsWorker.quiet(listOf(frozen, again))) // nothing proposed (too soon): both stay + } + @Test fun parsesAgentsAndCaps() { val s = status("""{"name":"pc","caps":["shutdown","restart","uninstall"],"agents":{"allowed":false,"keys":[{"name":"Atlas","devices":["My PC"]}]}}""") assertEquals(listOf("shutdown", "restart", "uninstall"), s.caps) diff --git a/docs/plans/recovery-layer.md b/docs/plans/recovery-layer.md index a0bcc3b..98f8325 100644 --- a/docs/plans/recovery-layer.md +++ b/docs/plans/recovery-layer.md @@ -258,20 +258,22 @@ recovery; only the owner confirms it; then the hub runs it and checks that it wo Build: 1. **Recoveries** live on the hub, in its state file, so a restart doesn't lose them. The last 50 are kept: - `{"id", "machine", "action", "state", "reason", "proposedBy", "proposedAt", "confirmedBy", - "confirmedAt", "endedAt", "message", "evidence", "steps"}`. + `{"id", "machine", "action", "state", "reason", "proposedBy", "agent", "proposedAt", + "confirmedBy", "confirmedAt", "endedAt", "message", "evidence", "steps", "direct"}`. - `evidence` is the hub's last look when the recovery was proposed. - `steps` come from the run, with the power light before and after. - **States:** - - `proposed` → `running` → `back`, `not_back`, `off` (force off) or `failed`; + - `proposed` → `running` → `back`, `not_back`, `off` (force off), `done` (an owner's forced + button, pressed while the helper answered: coming back can't be checked) or `failed`; - or, without running: `declined`, `expired` or `withdrawn`. - **Actions:** - `reset` gets the machine back. It presses the reset button, or, when the hub has learned that the reset switch isn't wired, holds power and starts the machine again. - `force-off` holds power and leaves the machine off. 2. **Propose:** `POST /v2/machines/{name}/recoveries {"action": "reset", "reason": "..."}`. - - Needs the `force` scope. Agent keys and agents relayed through a hub may propose, but the - machine's "Allow AI agents" switch applies, as the hub last heard it. + - Needs the `power` scope, not `force`: agent keys can't hold `force`, which also overrides + "someone is using it", and the owner confirms anyway. Agent keys and agents relayed through a + hub may propose, but the machine's "Allow AI agents" switch applies, as the hub last heard it. - Refused when any of these hold: - no KVM here is wired to it (`no_kvm`); - it has no helper, so success can't be checked (`no_helper`); @@ -303,7 +305,8 @@ Build: within 5 minutes. Otherwise it is `not_back`, which is an alert. - **At most one recovery runs per machine per 30 minutes.** The owner's own Reset and Force off buttons count towards this. They still work without a proposal: pressing one is the owner - confirming. + confirming. Each press is recorded as a recovery (`direct: true`), kept only if something was + pressed. - A recovery still `running` when the hub restarts ends `failed` ("the hub restarted"). 5. **Auto-reset** is an owner setting, `autoResetMinutes`: 0 (off, the default), or 5 to 60. - When set, the hub confirms a `reset` proposal itself after that many minutes, if the machine @@ -331,8 +334,9 @@ Build: - Tapping it opens the device page. - "Didn't come back" is notified too. - For a machine with a reset proposal, this replaces the plain "looks frozen" notification. - - **Settings → KVM hub:** "Reset a frozen machine on its own": Off, or after 10, 30 or 60 - minutes. + - **The hub's own page → Frozen machines:** "Reset a frozen machine on its own": Off, or after + 10, 30 or 60 minutes. (It's the hub's setting, so it sits with the hub, shown once it drives a + KVM; Settings → KVM hub only picks which helper is the hub.) Test: - State machine tests with a fake clock: diff --git a/docs/protocol.md b/docs/protocol.md index b7e5551..538a17a 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -133,7 +133,7 @@ All bodies are JSON. Errors: `{"error": "", "message": ""}`. | DELETE | `/v2/keys/{id}` | owner | → `204` | | GET | `/v2/update` | owner | → `{current, latest, available, checkedAt, error, installing, auto, autoAt?, pausedUntil?}`; `?check=1` asks GitHub first. `autoAt`: when an available release installs on its own | | POST | `/v2/update` | owner | Installs the latest signed release now → `202 {…, installing: true}`, or `200` when up to date; `409 pending` during a shutdown countdown, `502 update_check` when GitHub can't be reached. The helper restarts a few seconds later; `installing` clears itself after 10 minutes if it doesn't, with `error` saying so | -| POST | `/v2/settings` | owner | `{autoUpdate?, agents?, pauseUpdates?, hungAfterMinutes?}` → `204`. `autoUpdate` (default on for Windows, off for new Linux installs): check every 6 hours, install a release once it has been public for 72 hours and nothing is counting down (see [Updates](#updates)). `pauseUpdates`: `true` holds automatic installs for 7 days, `false` resumes. `agents` (default on): `false` refuses `wake`, `shutdown` and `restart` from automation keys for this device, here and through any hub (`403 agents_off`); status and cancel still work. `hungAfterMinutes` (on a hub; 2 to 30, `0` for the default, 3): how long a machine must look hung before it's called [`hung`](#machine-states-and-events) | +| POST | `/v2/settings` | owner | `{autoUpdate?, agents?, pauseUpdates?, hungAfterMinutes?}` → `204`. `autoUpdate` (default on for Windows, off for new Linux installs): check every 6 hours, install a release once it has been public for 72 hours and nothing is counting down (see [Updates](#updates)). `pauseUpdates`: `true` holds automatic installs for 7 days, `false` resumes. `agents` (default on): `false` refuses `wake`, `shutdown` and `restart` from automation keys for this device, here and through any hub (`403 agents_off`); status and cancel still work. `hungAfterMinutes` (on a hub; 2 to 30, `0` for the default, 3): how long a machine must look hung before it's called [`hung`](#machine-states-and-events). `autoResetMinutes` (on a hub, 2.4 and later; 5 to 60, `0` off, the default): confirm the hub's own reset proposal after that many minutes, see [Auto-reset](#auto-reset); refused for a hub relaying for an agent (`403 owner_only`) | | POST | `/v2/uninstall` | owner | `{confirm: }` → `202`, then the helper uninstalls itself and deletes its keys and settings. `409 pending` during a countdown. Offered when `caps` has `uninstall` | | GET | `/v2/pairing` | owner | → `{link, code, expires}`: the add link and the current pairing code (to let someone on another account in) | | POST | `/v2/pair` | pairing code | `{code, deviceName}` → `{owner:true, status}` (same account: now an owner device) or `{key, status}` (another account) | @@ -145,6 +145,11 @@ All bodies are JSON. Errors: `{"error": "", "message": ""}`. | DELETE | `/v2/kvms/{name}` | owner | → `204` | | GET | `/v2/machines` | `status` | → `{machines:[Machine]}`: the devices the caller may use, with live state; `?kvm=1` only those with a KVM here | | POST | `/v2/machines/{name}/{action}` | per action | `{force?, stayOff?, press?}` → `200 Result`, or the error with the Result's fields. `action`: `wake`, `shutdown`, `restart`, `reset`, `force-off` | +| POST | `/v2/machines/{name}/recoveries` | `power` | `{action?, reason?}` → `201 {recovery}`, or `200 {recovery}` when one is open for the machine already. Proposes a reset (the default) or `force-off` for the owner to confirm, see [Recoveries](#recoveries) (2.4 and later) | +| GET | `/v2/recoveries` | `status` | → `{recoveries:[Recovery]}`, newest first; `?open=1` only those `proposed` or `running` | +| GET | `/v2/recoveries/{id}` | `status` | → `{recovery}` | +| POST | `/v2/recoveries/{id}/confirm` | owner device | → `202 {recovery}` (now `running`), or `200 {recovery}` as it is when it isn't `proposed` any more | +| POST | `/v2/recoveries/{id}/decline` | owner device, or the key that proposed it | → `200 {recovery}`: `declined`, or `withdrawn` by its proposer | | GET | `/v2/machines/events` | `status` | Server-Sent Events: each machine's state as it changes, see [Machine states and events](#machine-states-and-events) (2.3 and later) | | POST | `/v2/note` | peer key | `{event, updates?}` → `204`: a machine's own helper telling its hub it's about to `sleep`, `shutdown` or `restart`, or did `resume` | | GET | `/v2/webhooks` | owner | → `{webhooks:[Webhook]}` (without secrets) | @@ -171,7 +176,7 @@ before retrying with `force: true`. Callers without the `force` scope can't over "network": {"mac": "02:00:5e:00:00:01", "lan": ["192.168.1.0/24"], "wired": true}, "caps": ["shutdown", "restart", "wake", "kvm", "uninstall"], "sleep": {"s3": true, "modernStandby": false, "afterMinutes": 30}, - "hub": {"hungAfterMinutes": 3}, + "hub": {"hungAfterMinutes": 3, "autoResetMinutes": 0}, "checks": [Check], "agents": {"allowed": true, "keys": [{"name": "Atlas", "devices": ["Workstation", "nas"]}]}, "you": {"kind": "owner", "name": "Pixel 9 Pro", "scopes": ["status", "wake", "power", "force", "admin"]} @@ -289,7 +294,8 @@ going to sleep. `PATCH {recheck: true}` (the app's **Check again**) forgets what ``` `state`: `on`, `off`, `hung` or `unknown`, with `reason`, `asleep`, `since` and `evidence` from the -hub's watch (see [Machine states and events](#machine-states-and-events)). `led` is the power +hub's watch (see [Machine states and events](#machine-states-and-events)). `recovery` (2.4 and +later) is the machine's open [recovery](#recoveries), or the last one for 30 minutes after it ended. `led` is the power light right now (absent without a KVM, `ledError` when it can't be read). `actions` is what the hub can do for it now; `reset` and `force-off` appear only while a wired LED reads on. @@ -308,8 +314,9 @@ One action per machine at a time (`409 busy`), at most 2 minutes. An action runs the caller hangs up, so a machine held off is also started again. A KVM whose stored password can't be opened (`kvm_credentials`: link it again) still lets the machine's helper wake, shut down or restart it. Nothing is learned about the wiring from a reading that failed. **`reset` and `force-off` are for owner devices -only** (`403 owner_only`), never an automation key or an agent relayed through a hub; agents get them -with the owner's confirmation in a later release. +only** (`403 owner_only`), never an automation key or an agent relayed through a hub; agents +[propose](#recoveries) one instead, and the owner confirms it. Each press of either button is also +recorded as a recovery (`direct: true`), so it counts towards the 30-minute limit. Result (also in the error body on failure, with `error` and `message` added): @@ -405,6 +412,15 @@ stream. A client that stops reading for 10 s is dropped, and so is one that fall `hung` and `recovered` also go to the hub's event log (`GET /v2/events`), which the phone [notifies](#notifications) about. +From 2.4 the stream also carries `recovery` events, one each time a [recovery](#recoveries) changes +state, and the snapshot lists the open ones (`{"machines": [...], "recoveries": [Recovery, …]}`): + +``` +id: 1761638430000789 +event: recovery +data: {"id":"1761638430000789","at":"…","event":"recovery","machine":"Workstation","recovery":{"id":"…","state":"proposed",…}} +``` + ### Webhooks For an assistant that would rather be called than hold a stream open. An owner device adds one: @@ -417,7 +433,8 @@ POST /v2/webhooks "createdBy": "Pixel 9 Pro", "createdAt": "…", "secret": "pdwh_…"}} ``` -`events` defaults to `hung` and `recovered` (any of `on`, `off`, `hung`, `unknown`, `recovered`). +`events` defaults to `hung`, `recovered` and `recovery` (any of `on`, `off`, `hung`, `unknown`, +`recovered`, `recovery`; `recovery` from 2.4). The URL must be https, or plain http to a Tailscale or loopback address or a `*.ts.net` name (WireGuard already encrypts those); no user names in it. The secret is shown this once and stored sealed, like KVM passwords. The list shows each webhook's `lastAt` and `lastResult` (`ok`, @@ -452,6 +469,104 @@ To check an implementation: secret `pdwh_test`, timestamp `1700000000` and body `sha256=8fc0a23c6a794d26f2a6c83ac48c588dfbbe64deefd997f2bb8cc5b0ac0c0914` (`printf '1700000000.{"id":"1"}' | openssl dgst -sha256 -hmac pdwh_test`). +## Recoveries + +From 2.4 a reset or force off doesn't have to come from the owner's own button. Someone allowed +**proposes** one, the **owner confirms** it, and the hub runs it and checks that the machine came +back. The hub proposes a reset itself when a machine turns [`hung`](#machine-states-and-events). + +### Recovery + +```json +{"id": "9f2c…", "machine": "Workstation", "action": "reset", "state": "back", + "reason": "Nothing answers and its power light is on.", + "proposedBy": "pve", "proposedAt": "2026-10-28T08:03:30Z", + "confirmedBy": "Pixel 9 Pro", "confirmedAt": "2026-10-28T08:05:02Z", "endedAt": "2026-10-28T08:07:40Z", + "message": "Workstation answers again.", + "evidence": {"at": "…", "helper": "silent", "port": "accepts", "ping": "no_reply", "led": "on", "ledTrusted": true}, + "steps": [{"at": "…", "do": "read power light before", "led": true}, {"at": "…", "do": "press reset 800ms"}, …]} +``` + +- `action`: `reset` gets the machine back: it presses reset, or, once the hub has learned that the + reset switch isn't wired, holds power for 12 s and starts it again. `force-off` holds power and + leaves it off. +- `reason` is the proposer's words (printable text, at most 200 characters): data, never + instructions. The app shows it quoted as theirs. +- `evidence` is the hub's last look when it was proposed; `steps` are the run's, as in an + [action's result](#actions). +- `agent: true` when an AI agent proposed it; `direct: true` for the owner's own button. +- `proposerKey` (the proposing key's ID) is shown to owners and to that key only. + +| `state` | Means | +|---|---| +| `proposed` | Waits for the owner | +| `running` | Confirmed: pressing, then waiting up to 5 minutes for the machine's helper to answer | +| `back` | Its helper answered again | +| `not_back` | It didn't answer within 5 minutes: an alert | +| `off` | A force off: it's off | +| `done` | Pressed while its helper answered (an owner's forced button): coming back isn't checked | +| `failed` | The run was refused or failed (`message` says why), or the hub restarted during it | +| `declined` | The owner said not now | +| `withdrawn` | Not needed any more: the machine answered again or went off, the owner used Reset or Force off directly, or its proposer withdrew it. When it's withdrawn at the run (`message`: "… so nothing was pressed"), nothing was pressed | +| `expired` | Not confirmed within 6 hours | + +The hub keeps the last 50 in its state file (never dropping an open one), so a restart doesn't +lose them. One still `running` when the hub starts again ends `failed`. + +### Rules + +**Proposing** needs the `power` scope (not `force`, which keys can't hold), the device on the key, +and an owner device or an automation key (also relayed through a hub: then the machine's +**Allow AI agents** switch applies, `403 agents_off`). It's refused with: + +| Code | When | +|---|---| +| `400 bad_action` | Neither `reset` nor `force-off` | +| `422 self` | The hub itself | +| `422 no_kvm` | No KVM on this hub is wired to it | +| `422 no_helper` | It has no PowerDeck helper, so the hub couldn't tell whether it worked | +| `409 helper_answers` | The hub's last look says its helper answers: shut it down or restart it instead | +| `409 already_off` | It's off: wake it instead | +| `429 too_soon` | A recovery ran on it in the last 30 minutes; `retryAt` says when one may run again | + +When one is open for the machine already, that one is returned (`200`), so two agents asking make +one proposal. + +**Confirming** is for owner devices acting for themselves (`403 owner_only` for an automation key, +another account's key, or an owner hub relaying for an agent). Confirming twice runs it once. +**Declining** is for owner devices too, and the key that proposed it may withdraw its own. + +**Running** goes through the same checks as the [buttons](#actions): a wired power light that reads +on, and a helper that doesn't answer. A machine that came back in the meantime isn't touched. + +**At most one recovery runs per machine per 30 minutes.** The owner's own Reset and Force off +buttons count towards this but are never held back by it: pressing one is the owner confirming. + +**The hub proposes a reset** when a machine turns `hung`, once per hang, if nothing is open for it +and none ran in the last 30 minutes. A declined or expired one isn't proposed again until the +machine has left `hung`. Each look also withdraws an open proposal when the machine answers or is +off, and expires one after 6 hours. + +### Auto-reset + +`autoResetMinutes` (owner setting, `POST /v2/settings`; app: the hub's page → **Frozen machines**): +`0`, the default, asks every time. Set to 5 to 60, the hub confirms a `reset` proposal itself that +many minutes after it was made, if the machine is still `hung` by its own evidence at that moment. +`confirmedBy` is then `PowerDeck (automatic)`. Never for `force-off`, and never for a machine that's +booting, updating, asleep or otherwise not `hung`. + +### Log, events and webhooks + +The hub's event log gets one line per step, with who: `propose reset Workstation: ` +(`proposed`), `confirm reset Workstation` (`confirmed`), `decline …` (`declined`), `withdraw …` +(`withdrawn`), `Workstation answers again after the reset` (`back`), `Workstation didn't answer +within 5 minutes of the reset` (`not_back`), `Workstation is off` (`off`), and `reset Workstation` +with `failed: …`, `withdrawn: …` or `expired: …`. For the owner's direct buttons the button's own +line (`reset Workstation`, `done`) is followed by `back` or `not_back`. + +Every change is also an event `{"event": "recovery", "machine", "recovery": Recovery}` on the +[event stream](#event-stream) and to [webhooks](#webhooks) that list `recovery`. + ## Schedules ```json @@ -472,6 +587,11 @@ minutes) and notifies about things the user didn't do from this phone: an agent a device going offline unexpectedly, and (from a 2.3 hub) a machine that [looks frozen](#machine-states-and-events) or answers again. Actions started on the phone show live progress instead. +From a 2.4 hub, a [proposed recovery](#recoveries) comes with **Reset** (or **Force off**) and +**Not now**, which work only on an unlocked phone; it replaces the plain "looks frozen" +notification for that machine. A recovery that's confirmed on its own or by someone else, comes +back, doesn't come back (on the alerting **Frozen machines** channel) or fails is notified too. + ## Agents (MCP) AI agents and scripts talk to **one always-on helper** (a "hub", e.g. a home server). The hub @@ -510,7 +630,9 @@ Rules the helper enforces for automation keys, whatever the agent asks: - Never the device the agent itself runs on (the helper on that device refuses, since the caller shares its IP's node), unless the owner ticks "allow" for that key. - Every action is in the device's event log with the key's name, so the phone shows "Atlas shut down Workstation". -- No `reset` or `force-off` (`403 owner_only`), even relayed through an owner device. Where the hub +- No `reset` or `force-off` (`403 owner_only`), even relayed through an owner device. From 2.4 an + agent [proposes](#recoveries) one (`POST /v2/machines/{name}/recoveries`, `power` scope) and the + owner confirms it on the phone. Where the hub drives a machine's KVM, `power_on` may press its power button (only while a wired LED reads off), and `power_off` goes to the machine's helper; pressing power to shut it down is owner-only. diff --git a/helper/internal/app/app.go b/helper/internal/app/app.go index fe8f2e1..853a012 100644 --- a/helper/internal/app/app.go +++ b/helper/internal/app/app.go @@ -160,7 +160,8 @@ func Run(ctx context.Context, o Options) error { // Every helper can drive NanoKVMs; the app links them to one (the hub). hub := &machines.Hub{Store: st, Box: secret.Default(), Fleet: fl, Self: name, Log: log, Now: now, Events: machines.NewBus(now()), Ping: tailnetPing(ts), Online: tailnetOnline(ts), - HungAfter: func() time.Duration { return time.Duration(st.Settings().HungAfterMinutes) * time.Minute }} + HungAfter: func() time.Duration { return time.Duration(st.Settings().HungAfterMinutes) * time.Minute }, + AutoReset: func() time.Duration { return time.Duration(st.Settings().AutoResetMinutes) * time.Minute }} hooks := &webhook.Sender{Store: st, Box: hub.Box, Bus: hub.Events, Log: log, Now: now} api := server.New(server.Deps{ diff --git a/helper/internal/machines/events.go b/helper/internal/machines/events.go index 007044a..86a4ac0 100644 --- a/helper/internal/machines/events.go +++ b/helper/internal/machines/events.go @@ -6,21 +6,25 @@ import ( "time" ) -// Event is a machine's state changing, as the hub saw it. +// Event is a machine's state changing, as the hub saw it, or a recovery's. type Event struct { ID string `json:"id"` At time.Time `json:"at"` - Event string `json:"event"` // the new state, or "recovered" (hung, then answering) + Event string `json:"event"` // the new state, "recovered" (hung, then answering) or "recovery" Machine string `json:"machine"` - State string `json:"state"` - From string `json:"from"` + State string `json:"state,omitempty"` + From string `json:"from,omitempty"` Reason string `json:"reason,omitempty"` Asleep bool `json:"asleep,omitempty"` - Evidence Evidence `json:"evidence"` + Evidence Evidence `json:"evidence,omitzero"` + Recovery *Recovery `json:"recovery,omitempty"` // a recovery event: the recovery as it is now } +// EventRecovery is a recovery's state changing. +const EventRecovery = "recovery" + // EventNames are what an event can be (webhook filters use them). -var EventNames = []string{StateOn, StateOff, StateHung, StateUnknown, "recovered"} +var EventNames = []string{StateOn, StateOff, StateHung, StateUnknown, "recovered", EventRecovery} // maxHeld is how many events the bus keeps for replay. const maxHeld = 500 diff --git a/helper/internal/machines/machines.go b/helper/internal/machines/machines.go index 2afa085..8df0def 100644 --- a/helper/internal/machines/machines.go +++ b/helper/internal/machines/machines.go @@ -72,13 +72,19 @@ type Hub struct { Ping func(ctx context.Context, addr string) string Online func(ctx context.Context, addr string) *bool HungAfter func() time.Duration // the owner's setting (0: the default) - Events *Bus // state changes; nil: not published + AutoReset func() time.Duration // the owner's setting: confirm the hub's own reset after this (0: never) + Events *Bus // state changes and recoveries; nil: not published mu sync.Mutex busy map[string]bool clients map[string]cached tracks map[string]*Track wg sync.WaitGroup + + rmu sync.Mutex // recoveries (recovery.go); never held while taking mu + recs []Recovery + recsLoaded bool + asked map[string]bool // machines the hub proposed a reset for during this hang } type cached struct { @@ -91,12 +97,13 @@ type Error struct { Status int Code string Message string + Extra map[string]any // more fields for the reply } func (e *Error) Error() string { return e.Message } func fail(status int, code, format string, a ...any) *Error { - return &Error{status, code, fmt.Sprintf(format, a...)} + return &Error{Status: status, Code: code, Message: fmt.Sprintf(format, a...)} } func dur(d, def time.Duration) time.Duration { @@ -447,6 +454,8 @@ type Machine struct { // Since is when the hub's watch first saw the current state; Evidence is its last look. Since *time.Time `json:"since,omitempty"` Evidence *Evidence `json:"evidence,omitempty"` + // Recovery is its open recovery, or the last one if it ended in the last 30 minutes. + Recovery *Recovery `json:"recovery,omitempty"` } // HasKVM reports whether a machine has a KVM here. @@ -500,6 +509,11 @@ func (h *Hub) list(ctx context.Context, kvmOnly bool) []Machine { }(&out[i]) } wg.Wait() + for i := range out { + if out[i].KVM != nil { + out[i].Recovery = h.latest(out[i].Name) + } + } return out } @@ -645,6 +659,8 @@ type Result struct { Steps []Step `json:"steps"` Helper map[string]any `json:"helper,omitempty"` // the machine's helper's reply (shutdown, restart) Status int `json:"-"` // HTTP status when not OK + + verifying bool // Options.Back will be called } // Options change how an action runs. @@ -654,6 +670,10 @@ type Options struct { StayOff bool // force off without powering on again Press bool // shutdown: press the power button (the OS shuts down cleanly) if the helper doesn't answer Agent bool // on behalf of an AI agent (relayed to the machine's helper) + // Back, when set, is called once the hub knows whether the machine answered again after a reset + // or an off-and-on (within ResetVerify). Not called when that can't be checked: no helper, or it + // answered all along. + Back func(back bool) } type run struct { @@ -742,8 +762,16 @@ var Actions = []string{"wake", "shutdown", "restart", "reset", "force-off"} const actionTime = 2 * time.Minute // Act runs one action on a machine with a KVM here. One action per machine at a time. The caller -// checks permissions (internal/server's act). +// checks permissions (internal/server's act). A reset or force off is recorded as a recovery that +// its caller confirmed. func (h *Hub) Act(ctx context.Context, name, action string, o Options) Result { + if action == "reset" || action == "force-off" { + return h.direct(ctx, name, action, o) + } + return h.act(ctx, name, action, o) +} + +func (h *Hub) act(ctx context.Context, name, action string, o Options) Result { ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), actionTime) defer cancel() res := &Result{Machine: name, Action: action, Steps: []Step{}} @@ -983,36 +1011,48 @@ func (r *run) reset(hasHelper bool, o Options) Result { } r.led("after") // stays on through a reset on most boards: the result shows when the helper is back if hasHelper && !answered { // a machine that answered all along can't show whether the reset worked - r.h.verifyReset(r.k.Name, m) + r.verifyBack(true, o.Back) return r.done(true, "", "Pressed reset on %s. It should answer again in a few minutes.", m) } return r.done(true, "", "Pressed reset on %s.", m) } -// verifyReset waits in the background for the machine's helper and records whether the reset -// switch works. -func (h *Hub) verifyReset(kvm, machine string) { +// verifyBack waits in the background for the machine's helper to answer again, then calls back (if +// set). After a reset it also records whether the reset switch works. +func (r *run) verifyBack(reset bool, back func(bool)) { + h, kvm, machine := r.h, r.k.Name, r.k.Machine + if !reset && back == nil { + return // nothing to learn and nobody waiting + } + r.res.verifying = back != nil h.wg.Add(1) go func() { defer h.wg.Done() - deadline := h.now().Add(dur(h.ResetVerify, 5*time.Minute)) - for h.now().Before(deadline) { - time.Sleep(dur(h.VerifyEvery, 10*time.Second)) + up := false + every := dur(h.VerifyEvery, 10*time.Second) + for i := time.Duration(0); !up && i < dur(h.ResetVerify, 5*time.Minute)/every; i++ { + time.Sleep(every) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - up := h.Fleet.Answers(ctx, machine) + up = h.Fleet.Answers(ctx, machine) cancel() - if up { - _ = h.Store.UpdateKVM(kvm, func(k *store.KVM) { k.Wiring.Reset = store.Wired }) - h.log().Info("reset worked: the machine answers again", "machine", machine) - return - } } - _ = h.Store.UpdateKVM(kvm, func(k *store.KVM) { - if k.Wiring.Reset != store.Wired { - k.Wiring.Reset = store.NotWired - } - }) - h.log().Warn("the machine didn't answer within the time after a reset", "machine", machine) + if up { + h.log().Info("the machine answers again", "machine", machine) + } else { + h.log().Warn("the machine didn't answer in time", "machine", machine) + } + if reset { + _ = h.Store.UpdateKVM(kvm, func(k *store.KVM) { + if up { + k.Wiring.Reset = store.Wired + } else if k.Wiring.Reset != store.Wired { + k.Wiring.Reset = store.NotWired + } + }) + } + if back != nil { + back(up) + } }() } @@ -1028,7 +1068,8 @@ func (h *Hub) Wait() { h.wg.Wait() } func (r *run) forceOff(hasHelper bool, o Options) Result { m := r.k.Machine - if _, res := r.gate(hasHelper, o, "forcing it off"); res != nil { + answered, res := r.gate(hasHelper, o, "forcing it off") + if res != nil { return *res } if err := r.press(nanokvm.Power, 12*time.Second); err != nil { @@ -1056,5 +1097,8 @@ func (r *run) forceOff(hasHelper bool, o Options) Result { if ok, _ := r.waitLED(true); !ok { return r.refuse(http.StatusBadGateway, "didnt_start", "%s is off, but didn't start again. Wake it, or press its power button.", m) } + if hasHelper && !answered { + r.verifyBack(false, o.Back) + } return r.done(true, "", "Switched %s off and on again. It's starting.", m) } diff --git a/helper/internal/machines/recovery.go b/helper/internal/machines/recovery.go new file mode 100644 index 0000000..d5c3c65 --- /dev/null +++ b/helper/internal/machines/recovery.go @@ -0,0 +1,487 @@ +package machines + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "slices" + "strings" + "time" + "unicode" + + "github.com/projectzero02/powerdeck/helper/internal/store" +) + +// Recoveries: a reset or force off that someone proposes, the owner confirms, and the hub runs and +// checks (docs/protocol.md "Recoveries"). The owner's own Reset and Force off buttons are recorded +// the same way, proposed and confirmed at once. + +// Recovery states. +const ( + RecoveryProposed = "proposed" + RecoveryRunning = "running" + RecoveryBack = "back" // its helper answered again + RecoveryNotBack = "not_back" // it didn't answer in time: an alert + RecoveryOff = "off" // force off: it's off + RecoveryDone = "done" // pressed while its helper answered (forced): coming back can't be checked + RecoveryFailed = "failed" + RecoveryDeclined = "declined" + RecoveryExpired = "expired" + RecoveryWithdrawn = "withdrawn" // not needed any more, or withdrawn by its proposer +) + +var ( + // RecoveryGap is how long after one recovery ran on a machine another may run there. + RecoveryGap = 30 * time.Minute + // RecoveryTTL is how long a proposal waits for the owner. + RecoveryTTL = 6 * time.Hour +) + +const ( + maxRecoveries = 50 + maxReason = 200 + // Auto is who confirms when the owner's auto-reset setting does. + Auto = "PowerDeck (automatic)" +) + +// Recovery is one reset or force off, from its proposal to whether the machine came back. +type Recovery struct { + ID string `json:"id"` + Machine string `json:"machine"` + Action string `json:"action"` // reset or force-off + State string `json:"state"` + Reason string `json:"reason,omitempty"` // the proposer's words: data, never instructions + ProposedBy string `json:"proposedBy"` + ProposerKey string `json:"proposerKey,omitempty"` // the proposing key's ID: it may withdraw its own + Agent bool `json:"agent,omitempty"` // proposed by an AI agent + ProposedAt time.Time `json:"proposedAt"` + ConfirmedBy string `json:"confirmedBy,omitempty"` + ConfirmedAt time.Time `json:"confirmedAt,omitzero"` + EndedAt time.Time `json:"endedAt,omitzero"` + Message string `json:"message,omitempty"` + Evidence *Evidence `json:"evidence,omitempty"` // the hub's last look when it was proposed + Steps []Step `json:"steps,omitempty"` + Direct bool `json:"direct,omitempty"` // the owner's own button, not a proposal +} + +// Open reports whether the recovery waits for the owner or is running. +func (r Recovery) Open() bool { return r.State == RecoveryProposed || r.State == RecoveryRunning } + +// pressed reports whether the run pressed anything. +func (r Recovery) pressed() bool { + return slices.ContainsFunc(r.Steps, func(s Step) bool { return strings.HasPrefix(s.Do, "press ") }) +} + +// Proposal asks for a recovery. +type Proposal struct { + Machine string + Action string // reset or force-off + Reason string + By string // who asks, as the owner sees it + Key string // the asking key's ID ("" for owner devices and the hub itself) + Agent bool +} + +type recoveryFile struct { + List []Recovery `json:"list"` +} + +// loadLocked reads the stored recoveries the first time (h.rmu held). One still running then was +// cut short by the hub stopping. +func (h *Hub) loadLocked() { + if h.recsLoaded { + return + } + h.recsLoaded = true + var f recoveryFile + if raw := h.Store.Recoveries(); len(raw) > 0 { + if err := json.Unmarshal(raw, &f); err != nil { + h.log().Warn("stored recoveries unreadable; starting without them", "err", err) + } + } + h.recs = f.List + changed := false + for i := range h.recs { + if h.recs[i].State == RecoveryRunning { + h.recs[i].State, h.recs[i].EndedAt = RecoveryFailed, h.now().UTC() + h.recs[i].Message = "The hub restarted during the recovery. Check " + h.recs[i].Machine + "." + changed = true + } + } + if changed { + h.saveLocked() + } +} + +// saveLocked stores the recoveries, keeping open ones and the newest of the rest. +func (h *Hub) saveLocked() { + for len(h.recs) > maxRecoveries { + i := slices.IndexFunc(h.recs, func(r Recovery) bool { return !r.Open() }) + if i < 0 { + break + } + h.recs = slices.Delete(h.recs, i, i+1) + } + b, err := json.Marshal(recoveryFile{List: h.recs}) + if err == nil { + err = h.Store.PutRecoveries(b) + } + if err != nil { + h.log().Warn("can't store recoveries", "err", err) + } +} + +func (h *Hub) findLocked(id string) int { + return slices.IndexFunc(h.recs, func(r Recovery) bool { return r.ID == id }) +} + +// openLocked is the index of the machine's open recovery, or -1. +func (h *Hub) openLocked(machine string) int { + return slices.IndexFunc(h.recs, func(r Recovery) bool { return r.Open() && strings.EqualFold(r.Machine, machine) }) +} + +// retryAtLocked is when another recovery may run on the machine (zero: now). +func (h *Hub) retryAtLocked(machine string) time.Time { + for _, r := range slices.Backward(h.recs) { + if strings.EqualFold(r.Machine, machine) && !r.ConfirmedAt.IsZero() && (r.pressed() || r.State == RecoveryRunning) { + if at := r.ConfirmedAt.Add(RecoveryGap); h.now().Before(at) { + return at.UTC() + } + break + } + } + return time.Time{} +} + +// changedLocked stores a recovery's new state, publishes it and writes the activity log line. +func (h *Hub) changedLocked(i int, who, action, result string) { + r := h.recs[i] + h.saveLocked() + if h.Events != nil { + h.Events.Publish(Event{At: h.now().UTC(), Event: EventRecovery, Machine: r.Machine, Recovery: &r}) + } + if action != "" { + h.Store.AddEvent(store.Event{At: h.now(), Who: who, Action: action, Result: result}) + } +} + +// endLocked closes recovery i with state and message. +func (h *Hub) endLocked(i int, state, message string) { + r := &h.recs[i] + r.State, r.Message, r.EndedAt = state, message, h.now().UTC() + what := r.Action + " " + r.Machine + if r.Direct && state != RecoveryBack && state != RecoveryNotBack { + h.changedLocked(i, "", "", "") // the button's own log line says what happened + return + } + switch state { + case RecoveryBack: + h.changedLocked(i, h.Self, r.Machine+" answers again after the "+verb(r.Action), state) + case RecoveryNotBack: + h.changedLocked(i, h.Self, message, state) + case RecoveryOff: + h.changedLocked(i, h.Self, r.Machine+" is off", state) + case RecoveryDone: + h.changedLocked(i, "", "", "") + default: // failed, withdrawn, expired (declined logs who declined) + h.changedLocked(i, h.Self, what, state+": "+message) + } +} + +func verb(action string) string { + if action == "force-off" { + return "force off" + } + return action +} + +// cleanReason keeps printable text, at most maxReason characters. +func cleanReason(s string) string { + s = strings.Map(func(r rune) rune { + if unicode.IsPrint(r) { + return r + } + return ' ' + }, strings.TrimSpace(s)) + if r := []rune(s); len(r) > maxReason { + s = string(r[:maxReason-1]) + "…" + } + return s +} + +// Recoveries lists recoveries, newest first; with open, only those waiting or running. +func (h *Hub) Recoveries(open bool) []Recovery { + h.rmu.Lock() + defer h.rmu.Unlock() + h.loadLocked() + out := []Recovery{} + for _, r := range slices.Backward(h.recs) { + if !open || r.Open() { + out = append(out, r) + } + } + return out +} + +// Recovery is one recovery by ID. +func (h *Hub) Recovery(id string) (Recovery, bool) { + h.rmu.Lock() + defer h.rmu.Unlock() + h.loadLocked() + if i := h.findLocked(id); i >= 0 { + return h.recs[i], true + } + return Recovery{}, false +} + +// latest is the machine's open recovery, or the last one if it ended in the last RecoveryGap. +func (h *Hub) latest(machine string) *Recovery { + h.rmu.Lock() + defer h.rmu.Unlock() + h.loadLocked() + for _, r := range slices.Backward(h.recs) { + if strings.EqualFold(r.Machine, machine) { + if r.Open() || h.now().Sub(r.EndedAt) < RecoveryGap { + return &r + } + return nil + } + } + return nil +} + +// Propose records a recovery for the owner to confirm. When one is open for the machine already, +// that one is returned instead (created false). +func (h *Hub) Propose(p Proposal) (r Recovery, created bool, err error) { + if p.Action != "reset" && p.Action != "force-off" { + return Recovery{}, false, fail(http.StatusBadRequest, "bad_action", "A recovery is a reset or a force-off.") + } + k, ok := h.Store.KVMFor(p.Machine) + if !ok { + return Recovery{}, false, fail(http.StatusUnprocessableEntity, "no_kvm", "No KVM on %s is wired to %s.", h.Self, p.Machine) + } + if _, err := h.Fleet.Peer(k.Machine); err != nil { + return Recovery{}, false, fail(http.StatusUnprocessableEntity, "no_helper", + "%s has no PowerDeck helper, so %s can't tell whether a recovery worked. Its owner can use the Reset button in the app.", k.Machine, h.Self) + } + tr, _ := h.Tracked(k.Machine) + h.rmu.Lock() + defer h.rmu.Unlock() + h.loadLocked() + if i := h.openLocked(k.Machine); i >= 0 { + return h.recs[i], false, nil + } + switch tr.State { + case StateOn: + return Recovery{}, false, fail(http.StatusConflict, "helper_answers", "%s is answering, so it isn't frozen. Shut it down or restart it instead.", k.Machine) + case StateOff: + return Recovery{}, false, fail(http.StatusConflict, "already_off", "%s is off. Wake it instead.", k.Machine) + } + if at := h.retryAtLocked(k.Machine); !at.IsZero() { + e := fail(http.StatusTooManyRequests, "too_soon", "%s was reset or forced off less than %d minutes ago. Try again after %s.", + k.Machine, int(RecoveryGap/time.Minute), at.Format(time.RFC3339)) + e.Extra = map[string]any{"retryAt": at} + return Recovery{}, false, e + } + return h.proposeLocked(k.Machine, p, tr), true, nil +} + +func (h *Hub) proposeLocked(machine string, p Proposal, tr Track) Recovery { + r := Recovery{ID: store.NewID(), Machine: machine, Action: p.Action, State: RecoveryProposed, Reason: cleanReason(p.Reason), + ProposedBy: p.By, ProposerKey: p.Key, Agent: p.Agent, ProposedAt: h.now().UTC()} + if tr.State != "" { + e := tr.Evidence + r.Evidence = &e + } + h.recs = append(h.recs, r) + what := "propose " + r.Action + " " + machine + if r.Reason != "" { + what += ": " + string([]rune(r.Reason)[:min(len([]rune(r.Reason)), 80)]) + } + h.changedLocked(len(h.recs)-1, p.By, what, RecoveryProposed) + return r +} + +// Confirm runs a proposed recovery; the caller checked that it's the owner. Confirming one that's +// running or over returns it as it is. +func (h *Hub) Confirm(id, by string) (Recovery, error) { + h.rmu.Lock() + defer h.rmu.Unlock() + h.loadLocked() + i := h.findLocked(id) + if i < 0 { + return Recovery{}, fail(http.StatusNotFound, "not_found", "No such recovery.") + } + if h.recs[i].State == RecoveryProposed { + h.confirmLocked(i, by) + } + return h.recs[i], nil +} + +func (h *Hub) confirmLocked(i int, by string) { + r := &h.recs[i] + r.State, r.ConfirmedBy, r.ConfirmedAt = RecoveryRunning, by, h.now().UTC() + h.changedLocked(i, by, "confirm "+r.Action+" "+r.Machine, "confirmed") + h.wg.Add(1) + go h.runRecovery(*r) +} + +// Decline closes a proposed recovery without running it: the owner declines, or (own) its proposer +// withdraws it. One that's running or over is returned as it is. +func (h *Hub) Decline(id, by string, own bool) (Recovery, error) { + h.rmu.Lock() + defer h.rmu.Unlock() + h.loadLocked() + i := h.findLocked(id) + if i < 0 { + return Recovery{}, fail(http.StatusNotFound, "not_found", "No such recovery.") + } + r := &h.recs[i] + if r.State != RecoveryProposed { + return *r, nil + } + r.EndedAt = h.now().UTC() + if own { + r.State, r.Message = RecoveryWithdrawn, "Withdrawn by "+by+"." + h.changedLocked(i, by, "withdraw "+r.Action+" "+r.Machine, RecoveryWithdrawn) + } else { + r.State, r.Message = RecoveryDeclined, "Declined by "+by+"." + h.changedLocked(i, by, "decline "+r.Action+" "+r.Machine, RecoveryDeclined) + } + return *r, nil +} + +// runRecovery runs a confirmed recovery through the same gate as the owner's buttons, then waits +// for the machine to answer again. +func (h *Hub) runRecovery(r Recovery) { + defer h.wg.Done() + action, o := r.Action, Options{Who: r.ConfirmedBy, StayOff: r.Action == "force-off"} + if k, ok := h.Store.KVMFor(r.Machine); ok && action == "reset" && k.Wiring.Reset == store.NotWired { + action = "force-off" // the reset switch isn't wired: off and on again does the same + } + back := make(chan bool, 1) + o.Back = func(up bool) { back <- up } + res := h.act(context.Background(), r.Machine, action, o) + h.ran(r.ID, res, back) +} + +// ran records what a run did and, when it waits for the machine, finishes once it knows. +func (h *Hub) ran(id string, res Result, back <-chan bool) { + h.rmu.Lock() + defer h.rmu.Unlock() + i := h.findLocked(id) + if i < 0 { + return + } + r := &h.recs[i] + r.Steps = res.Steps + m := r.Machine + switch { + case !res.OK && (res.Code == "helper_answers" || res.Code == "already_off"): + why := m + " answers again" + if res.Code == "already_off" { + why = m + "'s power light is off" + } + h.endLocked(i, RecoveryWithdrawn, why+", so nothing was pressed.") + case !res.OK: + h.endLocked(i, RecoveryFailed, res.Message) + case res.verifying: + r.Message = res.Message + h.changedLocked(i, "", "", "") + h.wg.Add(1) + go func() { + defer h.wg.Done() + up := <-back + h.rmu.Lock() + defer h.rmu.Unlock() + if i := h.findLocked(id); i >= 0 && h.recs[i].State == RecoveryRunning { + if up { + h.endLocked(i, RecoveryBack, m+" answers again.") + } else { + h.endLocked(i, RecoveryNotBack, fmt.Sprintf("%s didn't answer within %d minutes of the %s", + m, int(dur(h.ResetVerify, 5*time.Minute)/time.Minute), verb(h.recs[i].Action))) + } + } + }() + case r.Action == "force-off": + h.endLocked(i, RecoveryOff, res.Message) + default: + h.endLocked(i, RecoveryDone, res.Message) + } +} + +// direct runs the owner's own Reset or Force off button: a recovery proposed and confirmed at once, +// with no wait and no limit (the owner is there). It's kept only if something was pressed, and it +// closes any proposal for the machine. +func (h *Hub) direct(ctx context.Context, name, action string, o Options) Result { + h.rmu.Lock() + h.loadLocked() + now := h.now().UTC() + id := store.NewID() + h.recs = append(h.recs, Recovery{ID: id, Machine: name, Action: action, State: RecoveryRunning, ProposedBy: o.Who, + ProposedAt: now, ConfirmedBy: o.Who, ConfirmedAt: now, Direct: true}) + h.rmu.Unlock() + + back := make(chan bool, 1) + o.Back = func(up bool) { back <- up } + res := h.act(ctx, name, action, o) + h.rmu.Lock() + if i := h.findLocked(id); i >= 0 { + if !slices.ContainsFunc(res.Steps, func(s Step) bool { return strings.HasPrefix(s.Do, "press ") }) { + h.recs = slices.Delete(h.recs, i, i+1) // refused before any press: the reply says why + h.rmu.Unlock() + return res + } + h.recs[i].Machine = res.Machine + if j := h.openLocked(res.Machine); j >= 0 && j != i && h.recs[j].State == RecoveryProposed { + h.endLocked(j, RecoveryWithdrawn, o.Who+" used "+verb(action)+" directly.") + } + } + h.rmu.Unlock() + h.ran(id, res, back) + return res +} + +// recoveryRound runs after each look at a machine: it closes proposals that aren't needed any more +// or waited too long, confirms one when the owner's auto-reset setting says so, and proposes a reset +// of a machine that just turned hung. +func (h *Hub) recoveryRound(machine string, tr Track) { + h.rmu.Lock() + defer h.rmu.Unlock() + h.loadLocked() + key := strings.ToLower(machine) + if tr.State != StateHung { + delete(h.asked, key) + } + now := h.now() + if i := h.openLocked(machine); i >= 0 { + r := h.recs[i] + if r.State != RecoveryProposed { + return + } + auto := time.Duration(0) + if h.AutoReset != nil { + auto = h.AutoReset() + } + switch { + case tr.State == StateOn: + h.endLocked(i, RecoveryWithdrawn, machine+" answers again.") + case tr.State == StateOff: + h.endLocked(i, RecoveryWithdrawn, machine+" is off.") + case now.Sub(r.ProposedAt) >= RecoveryTTL: + h.endLocked(i, RecoveryExpired, fmt.Sprintf("Not confirmed within %d hours.", int(RecoveryTTL/time.Hour))) + case auto > 0 && r.Action == "reset" && tr.State == StateHung && now.Sub(r.ProposedAt) >= auto: + h.confirmLocked(i, Auto) + } + return + } + if tr.State != StateHung || h.asked[key] || !h.retryAtLocked(machine).IsZero() { + return + } + if h.asked == nil { + h.asked = map[string]bool{} + } + h.asked[key] = true // once per hang: a declined or expired one isn't asked again + h.proposeLocked(machine, Proposal{Action: "reset", Reason: "Nothing answers and its power light is on.", By: h.Self}, tr) +} diff --git a/helper/internal/machines/recovery_test.go b/helper/internal/machines/recovery_test.go new file mode 100644 index 0000000..497919e --- /dev/null +++ b/helper/internal/machines/recovery_test.go @@ -0,0 +1,366 @@ +package machines + +import ( + "context" + "errors" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/projectzero02/powerdeck/helper/internal/nanokvm/fakekvm" + "github.com/projectzero02/powerdeck/helper/internal/store" +) + +// hung is a hub watching gaming-pc, frozen long enough to be called hung (and so proposed). +func hung(t *testing.T, o fakekvm.Options) *watchEnv { + t.Helper() + o.On = true + w := newWatchEnv(t, o) + w.link("gaming-pc") + w.rounds(1) + w.kvm.Hang() + if tr := w.rounds(7); tr.State != StateHung { + t.Fatalf("not hung: %+v", tr) + } + return w +} + +func (w *watchEnv) open() Recovery { + w.t.Helper() + open := w.hub.Recoveries(true) + if len(open) != 1 { + w.t.Fatalf("open recoveries: %+v", open) + } + return open[0] +} + +func (w *watchEnv) logged(action, result string) bool { + for _, e := range w.st.Events() { + if strings.HasPrefix(e.Action, action) && e.Result == result { + return true + } + } + return false +} + +func code(err error) string { + var e *Error + if errors.As(err, &e) { + return e.Code + } + return "" +} + +func TestHubProposesAResetAndTheOwnerConfirms(t *testing.T) { + w := hung(t, fakekvm.Options{}) + r := w.open() + if r.Action != "reset" || r.State != RecoveryProposed || r.ProposedBy != "atlas" || r.Evidence == nil || r.Evidence.LED != On { + t.Fatalf("%+v", r) + } + if !w.logged("propose reset gaming-pc: Nothing answers", RecoveryProposed) { + t.Errorf("log %+v", w.st.Events()) + } + if m, _ := w.hub.Get(context.Background(), "gaming-pc"); m.Recovery == nil || m.Recovery.ID != r.ID { + t.Errorf("machine view %+v", m.Recovery) + } + if presses(w.kvm) != "" { + t.Fatal("pressed before the owner confirmed") + } + + got, err := w.hub.Confirm(r.ID, "Pixel 9 Pro") + if err != nil || got.State != RecoveryRunning || got.ConfirmedBy != "Pixel 9 Pro" { + t.Fatalf("%+v %v", got, err) + } + w.hub.Wait() + got, _ = w.hub.Recovery(r.ID) + if got.State != RecoveryBack || presses(w.kvm) != "reset 800ms" || len(got.Steps) == 0 || got.EndedAt.IsZero() { + t.Fatalf("%+v %s", got, presses(w.kvm)) + } + if !w.logged("confirm reset gaming-pc", "confirmed") || !w.logged("gaming-pc answers again after the reset", RecoveryBack) { + t.Errorf("log %+v", w.st.Events()) + } + if s := w.recoveryStates(); s != "proposed,running,running,back" { + t.Errorf("events %s", s) + } + // Answering again: no new proposal. + if w.rounds(1); len(w.hub.Recoveries(true)) != 0 { + t.Errorf("open %+v", w.hub.Recoveries(true)) + } +} + +func TestProposeRefusals(t *testing.T) { + w := newWatchEnv(t, fakekvm.Options{On: true}) + w.link("gaming-pc") + p := Proposal{Machine: "gaming-pc", Action: "reset", By: "Atlas agent", Key: "k1", Agent: true} + if _, _, err := w.hub.Propose(Proposal{Machine: "gaming-pc", Action: "wake"}); code(err) != "bad_action" { + t.Errorf("wake: %v", err) + } + if _, _, err := w.hub.Propose(Proposal{Machine: "nas", Action: "reset"}); code(err) != "no_kvm" { + t.Errorf("no KVM: %v", err) + } + w.rounds(1) + if _, _, err := w.hub.Propose(p); code(err) != "helper_answers" { + t.Errorf("answering: %v", err) + } + w.kvm.SetOn(false) + w.rounds(1) + if _, _, err := w.hub.Propose(p); code(err) != "already_off" { + t.Errorf("off: %v", err) + } + + // Silent but not yet hung: an agent may know better; the owner decides. + w.kvm.SetOn(true) + time.Sleep(60 * time.Millisecond) + w.rounds(1) + w.kvm.Hang() + w.rounds(2) + p.Reason = "Can't reach it.\x1b[31m " + strings.Repeat("x", 300) + r, created, err := w.hub.Propose(p) + if err != nil || !created || r.ProposerKey != "k1" || !r.Agent || len([]rune(r.Reason)) != maxReason || strings.ContainsRune(r.Reason, '\x1b') { + t.Fatalf("%+v %v %v", r, created, err) + } + if again, created, _ := w.hub.Propose(Proposal{Machine: "GAMING-PC", Action: "force-off", By: "Pixel 9 Pro"}); created || again.ID != r.ID { + t.Errorf("a second proposal: %+v %v", again, created) + } + // The hub's own proposal at the hang is the same one. + if w.rounds(6); len(w.hub.Recoveries(false)) != 1 { + t.Errorf("%+v", w.hub.Recoveries(false)) + } +} + +func TestProposeWithoutHelper(t *testing.T) { + w := newWatchEnv(t, fakekvm.Options{On: true}) + if _, _, err := w.hub.Link(context.Background(), LinkRequest{Name: "kvm", Address: w.kvm.URL, Password: "admin", Machine: "nas"}); err != nil { + t.Fatal(err) + } + if _, _, err := w.hub.Propose(Proposal{Machine: "nas", Action: "reset"}); code(err) != "no_helper" { + t.Errorf("%v", err) + } +} + +func TestDeclineAndWithdraw(t *testing.T) { + w := hung(t, fakekvm.Options{}) + r := w.open() + if got, _ := w.hub.Decline(r.ID, "Pixel 9 Pro", false); got.State != RecoveryDeclined { + t.Fatalf("%+v", got) + } + if got, _ := w.hub.Confirm(r.ID, "Pixel 9 Pro"); got.State != RecoveryDeclined || presses(w.kvm) != "" { + t.Errorf("confirmed after declining: %+v", got) + } + if !w.logged("decline reset gaming-pc", RecoveryDeclined) { + t.Errorf("log %+v", w.st.Events()) + } + // Not asked again during this hang. + if w.rounds(10); len(w.hub.Recoveries(true)) != 0 { + t.Errorf("asked again: %+v", w.hub.Recoveries(true)) + } + + // An agent withdraws its own proposal. + r, _, err := w.hub.Propose(Proposal{Machine: "gaming-pc", Action: "force-off", By: "Atlas agent", Key: "k1", Agent: true}) + if err != nil { + t.Fatal(err) + } + if got, _ := w.hub.Decline(r.ID, "Atlas agent", true); got.State != RecoveryWithdrawn { + t.Errorf("%+v", got) + } + + // Moot once it answers again. + r, _, _ = w.hub.Propose(Proposal{Machine: "gaming-pc", Action: "reset", By: "Atlas agent"}) + w.kvm.SetOn(false) + w.kvm.SetOn(true) + time.Sleep(60 * time.Millisecond) + w.rounds(1) + if got, _ := w.hub.Recovery(r.ID); got.State != RecoveryWithdrawn || got.Message != "gaming-pc answers again." { + t.Errorf("%+v", got) + } + + // A new hang is asked about again. + w.kvm.Hang() + if w.rounds(7); len(w.hub.Recoveries(true)) != 1 { + t.Errorf("not asked at the next hang: %+v", w.hub.Recoveries(true)) + } +} + +func TestProposalExpires(t *testing.T) { + w := hung(t, fakekvm.Options{}) + r := w.open() + w.mu.Lock() + w.clock = w.clock.Add(RecoveryTTL) + w.mu.Unlock() + w.rounds(1) + if got, _ := w.hub.Recovery(r.ID); got.State != RecoveryExpired { + t.Errorf("%+v", got) + } +} + +func TestConfirmAfterItCameBackPressesNothing(t *testing.T) { + w := hung(t, fakekvm.Options{}) + r := w.open() + w.kvm.SetOn(false) + w.kvm.SetOn(true) + time.Sleep(60 * time.Millisecond) // back, but the hub hasn't looked yet + w.hub.Confirm(r.ID, "Pixel 9 Pro") + w.hub.Wait() + if got, _ := w.hub.Recovery(r.ID); got.State != RecoveryWithdrawn || presses(w.kvm) != "" { + t.Errorf("%+v %s", got, presses(w.kvm)) + } + // Nothing was pressed, so another may run. + w.kvm.Hang() + if _, _, err := w.hub.Propose(Proposal{Machine: "gaming-pc", Action: "reset"}); err != nil { + t.Errorf("%v", err) + } +} + +func TestConfirmTwiceRunsOnce(t *testing.T) { + w := hung(t, fakekvm.Options{}) + r := w.open() + var wg sync.WaitGroup + for range 5 { + wg.Add(1) + go func() { defer wg.Done(); w.hub.Confirm(r.ID, "Pixel 9 Pro") }() + } + wg.Wait() + w.hub.Wait() + confirms := 0 + for _, e := range w.st.Events() { + if e.Action == "confirm reset gaming-pc" { + confirms++ + } + } + if got, _ := w.hub.Recovery(r.ID); presses(w.kvm) != "reset 800ms" || confirms != 1 || got.State != RecoveryBack { + t.Errorf("presses %s, %d confirms, %+v", presses(w.kvm), confirms, got) + } +} + +func TestNotBackThenOffAndOn(t *testing.T) { + w := hung(t, fakekvm.Options{NoResetSwitch: true}) + w.hub.ResetVerify = 100 * time.Millisecond + r := w.open() + w.hub.Confirm(r.ID, "Pixel 9 Pro") + w.hub.Wait() + got, _ := w.hub.Recovery(r.ID) + if got.State != RecoveryNotBack || !w.logged("gaming-pc didn't answer within", RecoveryNotBack) { + t.Fatalf("%+v %+v", got, w.st.Events()) + } + if _, _, err := w.hub.Propose(Proposal{Machine: "gaming-pc", Action: "reset"}); code(err) != "too_soon" { + t.Errorf("within 30 min: %v", err) + } else { + var e *Error + errors.As(err, &e) + if at, _ := e.Extra["retryAt"].(time.Time); !at.Equal(got.ConfirmedAt.Add(RecoveryGap)) { + t.Errorf("retryAt %v", e.Extra) + } + } + // Still frozen after the boot grace and the 30 minutes: asked again, and this time it's off + // and on, since the reset switch did nothing. + w.rounds(61) + r = w.open() + w.hub.Confirm(r.ID, "Pixel 9 Pro") + w.hub.Wait() + if got, _ := w.hub.Recovery(r.ID); got.State != RecoveryBack || presses(w.kvm) != "reset 800ms, power 12s, power 800ms" { + t.Errorf("%+v %s", got, presses(w.kvm)) + } +} + +func TestAutoReset(t *testing.T) { + w := hung(t, fakekvm.Options{}) + w.hub.AutoReset = func() time.Duration { return 10 * time.Minute } + r := w.open() + w.rounds(19) // 9.5 minutes after the proposal + if got, _ := w.hub.Recovery(r.ID); got.State != RecoveryProposed { + t.Fatalf("early: %+v", got) + } + w.rounds(2) + w.hub.Wait() + if got, _ := w.hub.Recovery(r.ID); got.State != RecoveryBack || got.ConfirmedBy != Auto || presses(w.kvm) != "reset 800ms" { + t.Errorf("%+v %s", got, presses(w.kvm)) + } +} + +func TestAutoResetOnlyResetsHungMachines(t *testing.T) { + w := newWatchEnv(t, fakekvm.Options{On: true}) + w.hub.AutoReset = func() time.Duration { return 5 * time.Minute } + w.link("gaming-pc") + w.rounds(1) + w.kvm.Hang() + w.rounds(2) + r, _, err := w.hub.Propose(Proposal{Machine: "gaming-pc", Action: "force-off", By: "Atlas agent"}) + if err != nil { + t.Fatal(err) + } + if tr := w.rounds(30); tr.State != StateHung { + t.Fatalf("%+v", tr) + } + if got, _ := w.hub.Recovery(r.ID); got.State != RecoveryProposed || presses(w.kvm) != "" { + t.Errorf("force off: %+v %s", got, presses(w.kvm)) + } + + // Installing updates at a restart: not hung, so an agent's reset waits for the owner. + w2 := newWatchEnv(t, fakekvm.Options{On: true}) + w2.hub.AutoReset = func() time.Duration { return 5 * time.Minute } + w2.fleet.status = `{"updatesPending":true,"pending":{"action":"restart","at":"` + t0.Add(time.Minute).Format(time.RFC3339) + `"}}` + w2.link("gaming-pc") + w2.rounds(1) + w2.kvm.Hang() + w2.rounds(2) + r, _, _ = w2.hub.Propose(Proposal{Machine: "gaming-pc", Action: "reset", By: "Atlas agent"}) + if tr := w2.rounds(30); tr.Reason != ReasonUpdating { + t.Fatalf("%+v", tr) + } + if got, _ := w2.hub.Recovery(r.ID); got.State != RecoveryProposed || presses(w2.kvm) != "" { + t.Errorf("updating: %+v %s", got, presses(w2.kvm)) + } +} + +func TestOwnerButtonsAreRecoveries(t *testing.T) { + w := hung(t, fakekvm.Options{}) + proposal := w.open() + if r := w.act("force-off", Options{Who: "Pixel 9 Pro", StayOff: true}); !r.OK { + t.Fatalf("%+v", r) + } + w.hub.Wait() + if got, _ := w.hub.Recovery(proposal.ID); got.State != RecoveryWithdrawn || got.Message != "Pixel 9 Pro used force off directly." { + t.Errorf("proposal %+v", got) + } + all := w.hub.Recoveries(false) + if len(all) != 2 || !all[0].Direct || all[0].State != RecoveryOff || all[0].ConfirmedBy != "Pixel 9 Pro" { + t.Fatalf("%+v", all) + } + // It counts towards the limit, but doesn't stop the owner's buttons. + w.kvm.SetOn(true) + time.Sleep(60 * time.Millisecond) + w.kvm.Hang() + if _, _, err := w.hub.Propose(Proposal{Machine: "gaming-pc", Action: "reset"}); code(err) != "too_soon" { + t.Errorf("%v", err) + } + if r := w.act("reset", Options{Who: "Pixel 9 Pro"}); !r.OK { + t.Errorf("%+v", r) + } + // Refused before any press: not kept. + w.hub.Wait() + before := len(w.hub.Recoveries(false)) + if r := w.act("reset", Options{Who: "Pixel 9 Pro"}); r.Code != "helper_answers" || len(w.hub.Recoveries(false)) != before { + t.Errorf("%+v %d", r, len(w.hub.Recoveries(false))) + } +} + +func TestRecoveryCutShortByARestart(t *testing.T) { + w := hung(t, fakekvm.Options{}) + r := w.open() + // Running when the hub stopped: as stored. + w.hub.rmu.Lock() + w.hub.recs[0].State = RecoveryRunning + w.hub.saveLocked() + w.hub.rmu.Unlock() + + st, err := store.Open(filepath.Dir(w.st.Path())) + if err != nil { + t.Fatal(err) + } + h2 := &Hub{Store: st, Fleet: w.fleet, Self: "atlas", Now: w.hub.Now} + if got, _ := h2.Recovery(r.ID); got.State != RecoveryFailed || !strings.Contains(got.Message, "restarted") { + t.Errorf("%+v", got) + } +} diff --git a/helper/internal/machines/watch.go b/helper/internal/machines/watch.go index 5071a6d..f729562 100644 --- a/helper/internal/machines/watch.go +++ b/helper/internal/machines/watch.go @@ -99,6 +99,7 @@ func (h *Hub) watch(ctx context.Context, k store.KVM) { if changed && from != "" { h.publish(k.Machine, from, now) } + h.recoveryRound(k.Machine, now) } // track is the detector's memory for a machine (h.mu held). diff --git a/helper/internal/machines/watch_test.go b/helper/internal/machines/watch_test.go index 1cf5bde..e1ac1b1 100644 --- a/helper/internal/machines/watch_test.go +++ b/helper/internal/machines/watch_test.go @@ -95,7 +95,23 @@ func (w *watchEnv) eventNames() string { defer w.mu.Unlock() var out []string for _, e := range w.events { - out = append(out, e.Event) + if e.Event != EventRecovery { + out = append(out, e.Event) + } + } + return strings.Join(out, ",") +} + +// recoveryStates lists the recovery events' states, in order. +func (w *watchEnv) recoveryStates() string { + time.Sleep(20 * time.Millisecond) + w.mu.Lock() + defer w.mu.Unlock() + var out []string + for _, e := range w.events { + if e.Event == EventRecovery { + out = append(out, e.Recovery.State) + } } return strings.Join(out, ",") } @@ -148,6 +164,10 @@ func TestWatchFindsAHangAndTheRecovery(t *testing.T) { if got := w.eventNames(); got != "unknown,hung,recovered" { t.Errorf("events %s", got) } + // The hub proposed a reset when it saw the hang; the owner's own button made it moot. + if got := w.recoveryStates(); got != "proposed,withdrawn,running,back" { + t.Errorf("recoveries %s", got) + } } func TestWatchBootGraceAfterAWake(t *testing.T) { diff --git a/helper/internal/server/machines.go b/helper/internal/server/machines.go index 3106eb6..a0ffa2d 100644 --- a/helper/internal/server/machines.go +++ b/helper/internal/server/machines.go @@ -24,7 +24,7 @@ func (s *server) noHub(w http.ResponseWriter) bool { func hubError(w http.ResponseWriter, err error) { var e *machines.Error if errors.As(err, &e) { - fail(w, e.Status, e.Code, e.Message) + failErr(w, &apiError{Status: e.Status, Code: e.Code, Message: e.Message, Extra: e.Extra}) return } fail(w, http.StatusInternalServerError, "internal", err.Error()) diff --git a/helper/internal/server/recovery.go b/helper/internal/server/recovery.go new file mode 100644 index 0000000..36e3074 --- /dev/null +++ b/helper/internal/server/recovery.go @@ -0,0 +1,150 @@ +package server + +import ( + "net/http" + + "github.com/projectzero02/powerdeck/helper/internal/auth" + "github.com/projectzero02/powerdeck/helper/internal/fleet" + "github.com/projectzero02/powerdeck/helper/internal/machines" + "github.com/projectzero02/powerdeck/helper/internal/store" +) + +// ---- recoveries (the hub): proposed by anyone allowed, confirmed by the owner ---- + +// viewRecovery hides which key proposed it from everyone but the owner and that key. +func viewRecovery(c auth.Caller, r machines.Recovery) machines.Recovery { + if c.Kind != auth.Owner && (c.Key == nil || c.Key.ID != r.ProposerKey) { + r.ProposerKey = "" + } + return r +} + +// ownerHere is an owner device acting for itself: not an agent, not a hub relaying for one. +func ownerHere(c auth.Caller) bool { return c.Kind == auth.Owner && !c.ForAgent() } + +func (s *server) proposeRecovery(w http.ResponseWriter, r *http.Request, c auth.Caller) { + if s.noHub(w) { + return + } + var req struct { + Action string `json:"action"` // reset (the default) or force-off + Reason string `json:"reason"` + } + if !decode(r, &req) { + fail(w, http.StatusBadRequest, "bad_request", "Send {\"action\": \"reset|force-off\", \"reason\": \"...\"}.") + return + } + if req.Action == "" { + req.Action = "reset" + } + if !c.Has(store.ScopePower) { // not force: agent keys can't hold it, and the owner confirms anyway + fail(w, http.StatusForbidden, "scope", "This key isn't allowed to propose a reset or force off (it needs the power scope).") + return + } + device := r.PathValue("name") + if fleet.IsSelf(device, s.Name) { + fail(w, http.StatusUnprocessableEntity, "self", s.Name+" can't reset itself.") + return + } + names := []string{device} + if p, err := s.Fleet.Peer(device); err == nil { + device, names = p.Name, peerNames(p) + } + if !mayUseAny(c, append(names, device)) || c.Kind != auth.Owner && !c.IsAutomation() { + fail(w, http.StatusForbidden, "device", "This key can't control "+device+".") + return + } + if c.ForAgent() { + if e := s.agentsAllowed(r.Context(), device, false); e != nil { + failErr(w, e) + return + } + } + p := machines.Proposal{Machine: device, Action: req.Action, Reason: req.Reason, By: c.Name, Agent: c.ForAgent()} + if c.Key != nil { + p.Key = c.Key.ID + } + rec, created, err := s.Machines.Propose(p) + if err != nil { + hubError(w, err) + return + } + status := http.StatusOK // one was open already + if created { + status = http.StatusCreated + } + writeJSON(w, status, map[string]any{"recovery": viewRecovery(c, rec)}) +} + +func (s *server) listRecoveries(w http.ResponseWriter, r *http.Request, c auth.Caller) { + if s.noHub(w) { + return + } + out := []machines.Recovery{} + for _, rec := range s.Machines.Recoveries(r.URL.Query().Get("open") == "1") { + if s.mayWatch(c, rec.Machine) { + out = append(out, viewRecovery(c, rec)) + } + } + writeJSON(w, http.StatusOK, map[string]any{"recoveries": out}) +} + +// recovery finds the recovery in the path, if the caller may see it. +func (s *server) recovery(w http.ResponseWriter, r *http.Request, c auth.Caller) (machines.Recovery, bool) { + if s.noHub(w) { + return machines.Recovery{}, false + } + rec, ok := s.Machines.Recovery(r.PathValue("id")) + if !ok || !s.mayWatch(c, rec.Machine) { + fail(w, http.StatusNotFound, "not_found", "No such recovery.") + return machines.Recovery{}, false + } + return rec, true +} + +func (s *server) getRecovery(w http.ResponseWriter, r *http.Request, c auth.Caller) { + if rec, ok := s.recovery(w, r, c); ok { + writeJSON(w, http.StatusOK, map[string]any{"recovery": viewRecovery(c, rec)}) + } +} + +// confirmRecovery runs a proposed recovery: owner devices only. It answers at once (202); the +// recovery's events and GET /v2/recoveries/{id} tell how it went. +func (s *server) confirmRecovery(w http.ResponseWriter, r *http.Request, c auth.Caller) { + if _, ok := s.recovery(w, r, c); !ok { + return + } + if !ownerHere(c) { + fail(w, http.StatusForbidden, "owner_only", "Only the owner's devices can confirm a reset or force off.") + return + } + rec, err := s.Machines.Confirm(r.PathValue("id"), c.Name) + if err != nil { + hubError(w, err) + return + } + status := http.StatusOK // running or over already: as it is + if rec.State == machines.RecoveryRunning && rec.ConfirmedBy == c.Name { + status = http.StatusAccepted + } + writeJSON(w, status, map[string]any{"recovery": viewRecovery(c, rec)}) +} + +// declineRecovery closes a proposed recovery: the owner declines it, or its proposer withdraws it. +func (s *server) declineRecovery(w http.ResponseWriter, r *http.Request, c auth.Caller) { + rec, ok := s.recovery(w, r, c) + if !ok { + return + } + own := c.Key != nil && rec.ProposerKey != "" && c.Key.ID == rec.ProposerKey + if !own && !ownerHere(c) { + fail(w, http.StatusForbidden, "owner_only", "Only the owner's devices, or the key that proposed it, can decline it.") + return + } + rec, err := s.Machines.Decline(rec.ID, c.Name, own) + if err != nil { + hubError(w, err) + return + } + writeJSON(w, http.StatusOK, map[string]any{"recovery": viewRecovery(c, rec)}) +} diff --git a/helper/internal/server/recovery_test.go b/helper/internal/server/recovery_test.go new file mode 100644 index 0000000..16a8b5c --- /dev/null +++ b/helper/internal/server/recovery_test.go @@ -0,0 +1,164 @@ +package server + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/projectzero02/powerdeck/helper/internal/machines" + "github.com/projectzero02/powerdeck/helper/internal/nanokvm/fakekvm" + "github.com/projectzero02/powerdeck/helper/internal/peer" +) + +func presses(k *fakekvm.Fake) string { + var out []string + for _, p := range k.Presses() { + out = append(out, p.Button+" "+p.Duration.String()) + } + return strings.Join(out, ", ") +} + +func recoveryOf(out map[string]any) map[string]any { + r, _ := out["recovery"].(map[string]any) + return r +} + +// Agents propose; only the owner's own devices confirm. +func TestRecoveryRoutes(t *testing.T) { + e := newEnv(t) + k := e.withHub(fakekvm.Options{On: true}) + e.linkKVM(k) + k.Hang() + agent := e.agentKey([]string{"status", "power"}, "workstation") + other := e.agentKey([]string{"status", "power"}) + noPower := e.agentKey([]string{"status", "wake"}) + path := "/v2/machines/workstation/recoveries" + + if code, out := e.do(req{method: "POST", path: "/v2/devices/workstation/reset", from: atlas, key: agent}); code != http.StatusForbidden || + out["error"] != "owner_only" || !strings.Contains(out["message"].(string), path) { + t.Errorf("agent's own reset: %d %v", code, out) + } + if code, out := e.do(req{method: "POST", path: path, from: atlas, key: noPower}); code != http.StatusForbidden || out["error"] != "scope" { + t.Errorf("without power: %d %v", code, out) + } + if code, out := e.do(req{method: "POST", path: "/v2/machines/Gaming-PC/recoveries", from: phone}); code != http.StatusUnprocessableEntity { + t.Errorf("the hub itself: %d %v", code, out) + } + + code, out := e.do(req{method: "POST", path: path, from: atlas, key: agent, body: map[string]any{"reason": "Atlas can't reach it"}}) + r := recoveryOf(out) + if code != http.StatusCreated || r["state"] != "proposed" || r["action"] != "reset" || r["proposedBy"] != "Atlas" || + r["agent"] != true || r["reason"] != "Atlas can't reach it" || r["proposerKey"] == nil { + t.Fatalf("propose: %d %v", code, out) + } + id := r["id"].(string) + if code, out := e.do(req{method: "POST", path: path, from: atlas, key: other, body: map[string]any{"action": "force-off"}}); code != 200 || recoveryOf(out)["id"] != id { + t.Errorf("second proposal: %d %v", code, out) + } + _, out = e.do(req{method: "GET", path: "/v2/recoveries?open=1", from: atlas, key: other}) + if list := out["recoveries"].([]any); len(list) != 1 || list[0].(map[string]any)["proposerKey"] != nil { + t.Errorf("another key sees: %v", out) + } + + // Confirming: not an agent key, not the owner's hub relaying for one, not another key. + for _, rq := range []req{ + {from: atlas, key: agent}, + {from: phone, header: map[string]string{peer.AgentHeader: "1"}}, + {from: friend, key: pairedKey(t, e)}, + } { + rq.method, rq.path = "POST", "/v2/recoveries/"+id+"/confirm" + if code, _ := e.do(rq); code != http.StatusForbidden && code != http.StatusNotFound { + t.Errorf("confirmed from %s: %d", rq.from, code) + } + } + if code, _ := e.do(req{method: "POST", path: "/v2/recoveries/" + id + "/decline", from: atlas, key: other}); code != http.StatusForbidden { + t.Errorf("another key declined: %d", code) + } + if presses(k) != "" { + t.Fatalf("pressed: %s", presses(k)) + } + + // The proposer withdraws its own; then the owner confirms a new one. + if code, out := e.do(req{method: "POST", path: "/v2/recoveries/" + id + "/decline", from: atlas, key: agent}); code != 200 || recoveryOf(out)["state"] != "withdrawn" { + t.Errorf("withdraw: %d %v", code, out) + } + _, out = e.do(req{method: "POST", path: path, from: atlas, key: agent}) + id = recoveryOf(out)["id"].(string) + if code, out := e.do(req{method: "POST", path: "/v2/recoveries/" + id + "/confirm", from: phone}); code != http.StatusAccepted || recoveryOf(out)["state"] != "running" { + t.Fatalf("confirm: %d %v", code, out) + } + e.api.s.Machines.Wait() + if _, out := e.do(req{method: "GET", path: "/v2/recoveries/" + id, from: atlas, key: agent}); recoveryOf(out)["state"] != "back" || presses(k) != "reset 800ms" { + t.Errorf("after: %v %s", out, presses(k)) + } + if !e.hasEvent("confirm reset workstation", "confirmed") || !e.hasEvent("workstation answers again after the reset", "back") { + t.Errorf("log %+v", e.st.Events()) + } + + // Within 30 minutes: not again. + k.Hang() + if code, out := e.do(req{method: "POST", path: path, from: atlas, key: agent}); code != http.StatusTooManyRequests || out["error"] != "too_soon" || out["retryAt"] == nil { + t.Errorf("too soon: %d %v", code, out) + } + + // The machine's own "Allow AI agents" switch. + p, _ := e.st.Peer("workstation") + p.NoAgents = true + _ = e.st.PutPeer(p) + if code, out := e.do(req{method: "POST", path: path, from: atlas, key: agent}); code != http.StatusForbidden || out["error"] != "agents_off" { + t.Errorf("agents off: %d %v", code, out) + } +} + +func TestAutoResetSetting(t *testing.T) { + e := newEnv(t) + e.withHub(fakekvm.Options{On: true}) + for n, want := range map[int]int{3: 400, 61: 400, 10: 204, 0: 204} { + if code, _ := e.do(req{method: "POST", path: "/v2/settings", from: phone, body: map[string]any{"autoResetMinutes": n}}); code != want { + t.Errorf("%d minutes: %d", n, code) + } + } + if code, _ := e.do(req{method: "POST", path: "/v2/settings", from: phone, header: map[string]string{peer.AgentHeader: "1"}, + body: map[string]any{"autoResetMinutes": 10}}); code != http.StatusForbidden { + t.Errorf("for an agent: %d", code) + } + e.do(req{method: "POST", path: "/v2/settings", from: phone, body: map[string]any{"autoResetMinutes": 15}}) + _, st := e.do(req{method: "GET", path: "/v2/status", from: phone}) + if hub, _ := st["hub"].(map[string]any); hub["autoResetMinutes"] != 15.0 || e.st.Settings().AutoResetMinutes != 15 { + t.Errorf("status %v", st["hub"]) + } + if !e.hasEvent("set automatic resets after 15 minutes", "ok") { + t.Errorf("log %+v", e.st.Events()) + } +} + +// Recovery events go out as their own SSE event, and open ones are in the snapshot. +func TestRecoveryEvents(t *testing.T) { + e := newEnv(t) + k := e.withHub(fakekvm.Options{On: true}) + e.linkKVM(k) + k.Hang() + e.api.s.Machines.Events = machines.NewBus(time.Now()) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + r.RemoteAddr = r.Header.Get("X-Test-From") + e.api.ServeHTTP(w, r) + })) + defer func() { srv.CloseClientConnections(); srv.Close() }() + + c, _ := e.stream(srv, phone, "", "") + if m := c.nextEvent(); m.event != "snapshot" || !strings.Contains(m.data, `"recoveries":[]`) { + t.Fatalf("snapshot %+v", m) + } + agent := e.agentKey([]string{"status", "power"}, "workstation") + e.do(req{method: "POST", path: "/v2/machines/workstation/recoveries", from: atlas, key: agent}) + if m := c.nextEvent(); m.event != "recovery" || !strings.Contains(m.data, `"event":"recovery"`) || !strings.Contains(m.data, `"state":"proposed"`) || + strings.Contains(m.data, `"evidence"`) && !strings.Contains(m.data, `"recovery":{`) { + t.Errorf("event %+v", m) + } + c2, _ := e.stream(srv, phone, "", "") + if m := c2.nextEvent(); !strings.Contains(m.data, `"state":"proposed"`) { + t.Errorf("snapshot with an open recovery: %+v", m) + } +} diff --git a/helper/internal/server/relay.go b/helper/internal/server/relay.go index 5fc1b4f..7c29301 100644 --- a/helper/internal/server/relay.go +++ b/helper/internal/server/relay.go @@ -24,7 +24,7 @@ type discoveryPeer = discovery.Peer var actionScope = map[string]string{ "status": store.ScopeStatus, "wake": store.ScopeWake, "shutdown": store.ScopePower, "restart": store.ScopePower, "cancel": store.ScopePower, - // Through a KVM on this helper. Owner devices only until recovery needs a confirmation (phase 4). + // Through a KVM on this helper. Owner devices only; anyone else proposes a recovery. "reset": store.ScopeForce, "force-off": store.ScopeForce, } @@ -49,6 +49,11 @@ func (s *server) actWith(ctx context.Context, c auth.Caller, device, action stri if !ok { return nil, &apiError{Status: http.StatusNotFound, Code: "not_found", Message: "Unknown action " + action + "."} } + recovery := action == "reset" || action == "force-off" + if recovery && (c.Kind != auth.Owner || c.ForAgent()) { // an agent relayed by a hub is still an agent + return nil, &apiError{Status: http.StatusForbidden, Code: "owner_only", + Message: "Only the owner's devices can reset a machine or force it off. Propose it instead (POST /v2/machines/" + device + "/recoveries): the owner confirms."} + } if !c.Has(scope) { return nil, &apiError{Status: http.StatusForbidden, Code: "scope", Message: "This key isn't allowed to " + action + "."} } @@ -77,11 +82,6 @@ func (s *server) actWith(ctx context.Context, c auth.Caller, device, action stri return nil, e } } - recovery := action == "reset" || action == "force-off" - if recovery && (c.Kind != auth.Owner || c.ForAgent()) { // an agent relayed by a hub is still an agent - return nil, &apiError{Status: http.StatusForbidden, Code: "owner_only", - Message: "Only the owner's devices can reset a machine or force it off."} - } if c.ForAgent() { ctx = peer.ForAgent(ctx) // the device checks its own switch too } diff --git a/helper/internal/server/server.go b/helper/internal/server/server.go index d5ba97d..c8912fc 100644 --- a/helper/internal/server/server.go +++ b/helper/internal/server/server.go @@ -147,6 +147,11 @@ func New(d Deps) *API { mux.HandleFunc("POST /v2/webhooks", s.with(store.ScopeAdmin, s.ownersOnly(s.addWebhook))) mux.HandleFunc("DELETE /v2/webhooks/{id}", s.with(store.ScopeAdmin, s.ownersOnly(s.removeWebhook))) mux.HandleFunc("POST /v2/webhooks/{id}/test", s.with(store.ScopeAdmin, s.ownersOnly(s.testWebhook))) + mux.HandleFunc("POST /v2/machines/{name}/recoveries", s.with(store.ScopeStatus, s.proposeRecovery)) + mux.HandleFunc("GET /v2/recoveries", s.with(store.ScopeStatus, s.listRecoveries)) + mux.HandleFunc("GET /v2/recoveries/{id}", s.with(store.ScopeStatus, s.getRecovery)) + mux.HandleFunc("POST /v2/recoveries/{id}/confirm", s.with(store.ScopeStatus, s.confirmRecovery)) + mux.HandleFunc("POST /v2/recoveries/{id}/decline", s.with(store.ScopeStatus, s.declineRecovery)) mux.HandleFunc("POST /v2/machines/{name}/{action}", s.with(store.ScopeStatus, s.machineAction)) mux.HandleFunc("POST /mcp", s.with(store.ScopeStatus, s.ownersAndAgents(s.mcp))) mux.HandleFunc("GET /mcp", func(w http.ResponseWriter, _ *http.Request) { @@ -529,7 +534,8 @@ func (s *server) statusBody(ctx context.Context, c auth.Caller) map[string]any { if s.Machines != nil { body["caps"] = append(body["caps"].([]string), "kvm") // drives NanoKVMs (/v2/kvms, /v2/machines) if c.Kind == auth.Owner { - body["hub"] = map[string]any{"hungAfterMinutes": hungAfter(s.Store.Settings())} + st := s.Store.Settings() + body["hub"] = map[string]any{"hungAfterMinutes": hungAfter(st), "autoResetMinutes": st.AutoResetMinutes} } } if sl := s.Sleep; sl != nil { @@ -654,6 +660,8 @@ func (s *server) putSettings(w http.ResponseWriter, r *http.Request, c auth.Call PauseUpdates *bool `json:"pauseUpdates"` // true: hold automatic installs for 7 days; false: resume // HungAfterMinutes: how long a machine with a KVM here must look hung (2 to 30; 0: default 3). HungAfterMinutes *int `json:"hungAfterMinutes"` + // AutoResetMinutes: confirm the hub's own reset of a hung machine after this (5 to 60; 0: never). + AutoResetMinutes *int `json:"autoResetMinutes"` } if !decode(r, &req) { fail(w, http.StatusBadRequest, "bad_request", "Malformed settings.") @@ -670,6 +678,25 @@ func (s *server) putSettings(w http.ResponseWriter, r *http.Request, c auth.Call } s.logEvent(c.Name, fmt.Sprintf("set hung after to %d minutes", hungAfter(s.Store.Settings())), "ok") } + if n := req.AutoResetMinutes; n != nil { + if c.ForAgent() { + fail(w, http.StatusForbidden, "owner_only", "Only the owner's devices can turn automatic resets on or off.") + return + } + if *n != 0 && (*n < 5 || *n > 60) { + fail(w, http.StatusBadRequest, "bad_request", "autoResetMinutes is 5 to 60 (0 turns automatic resets off).") + return + } + if err := s.Store.UpdateSettings(func(st *store.Settings) { st.AutoResetMinutes = *n }); err != nil { + fail(w, http.StatusInternalServerError, "store", err.Error()) + return + } + what := "turned automatic resets off" + if *n > 0 { + what = fmt.Sprintf("set automatic resets after %d minutes", *n) + } + s.logEvent(c.Name, what, "ok") + } if req.Agents != nil && *req.Agents == s.Store.Settings().NoAgents { if err := s.Store.UpdateSettings(func(st *store.Settings) { st.NoAgents = !*req.Agents }); err != nil { fail(w, http.StatusInternalServerError, "store", err.Error()) diff --git a/helper/internal/server/watch.go b/helper/internal/server/watch.go index 070e883..73bdfbf 100644 --- a/helper/internal/server/watch.go +++ b/helper/internal/server/watch.go @@ -83,6 +83,9 @@ func (s *server) machineEvents(w http.ResponseWriter, r *http.Request, c auth.Ca if !s.mayWatch(c, e.Machine) { return nil } + if e.Event == machines.EventRecovery { + return send("recovery", e.ID, e) + } return send("machine", e.ID, e) } @@ -97,13 +100,18 @@ func (s *server) machineEvents(w http.ResponseWriter, r *http.Request, c auth.Ca newest = recent[0].ID last, _ = strconv.ParseInt(newest, 10, 64) } - ms := []machines.Machine{} + ms, recs := []machines.Machine{}, []machines.Recovery{} for _, m := range s.Machines.KVMMachines(r.Context()) { if s.mayWatch(c, m.Name) { ms = append(ms, m) } } - if send("snapshot", newest, map[string]any{"machines": ms}) != nil { + for _, rec := range s.Machines.Recoveries(true) { + if s.mayWatch(c, rec.Machine) { + recs = append(recs, viewRecovery(c, rec)) + } + } + if send("snapshot", newest, map[string]any{"machines": ms, "recoveries": recs}) != nil { return } } diff --git a/helper/internal/store/store.go b/helper/internal/store/store.go index 0269c2f..e239977 100644 --- a/helper/internal/store/store.go +++ b/helper/internal/store/store.go @@ -140,6 +140,8 @@ type Settings struct { // HungAfterMinutes is how long a machine with a KVM here must look hung before it's called hung // (0: the default, 3). HungAfterMinutes int `json:"hungAfterMinutes,omitempty"` + // AutoResetMinutes lets the hub confirm its own reset of a hung machine after that long (0: never). + AutoResetMinutes int `json:"autoResetMinutes,omitempty"` // UpdatesPausedUntil holds automatic updates back until then. UpdatesPausedUntil time.Time `json:"updatesPausedUntil,omitzero"` } @@ -154,6 +156,8 @@ type state struct { OwnerMode string `json:"ownerMode,omitempty"` KVMs []KVM `json:"kvms,omitempty"` Webhooks []Webhook `json:"webhooks,omitempty"` + // Recoveries are the hub's, kept as it writes them (internal/machines). + Recoveries json.RawMessage `json:"recoveries,omitempty"` } // fresh is a new install's state. @@ -628,6 +632,21 @@ func (s *Store) WebhookDelivered(id string, at time.Time, result string) { } } +// Recoveries returns what the hub stored with PutRecoveries. +func (s *Store) Recoveries() json.RawMessage { + s.mu.Lock() + defer s.mu.Unlock() + return slices.Clone(s.st.Recoveries) +} + +// PutRecoveries stores the hub's recoveries. +func (s *Store) PutRecoveries(b json.RawMessage) error { + s.mu.Lock() + defer s.mu.Unlock() + s.st.Recoveries = slices.Clone(b) + return s.saveLocked() +} + const maxSchedules = 20 // ErrTooManySchedules is returned when the schedule list is full. diff --git a/helper/internal/webhook/webhook.go b/helper/internal/webhook/webhook.go index 499b816..c65ba39 100644 --- a/helper/internal/webhook/webhook.go +++ b/helper/internal/webhook/webhook.go @@ -40,7 +40,7 @@ import ( ) // DefaultEvents are what a webhook gets unless it asks for others. -var DefaultEvents = []string{machines.StateHung, "recovered"} +var DefaultEvents = []string{machines.StateHung, "recovered", machines.EventRecovery} // Retries are the waits before each new try after a failed delivery. var Retries = []time.Duration{10 * time.Second, time.Minute, 5 * time.Minute} diff --git a/helper/internal/webhook/webhook_test.go b/helper/internal/webhook/webhook_test.go index c766e62..7ec268b 100644 --- a/helper/internal/webhook/webhook_test.go +++ b/helper/internal/webhook/webhook_test.go @@ -146,6 +146,21 @@ func TestDeliverySignedAndFiltered(t *testing.T) { } } +// New webhooks get recovery events too: "didn't come back" is the alert an assistant needs. +func TestRecoveryEventsByDefault(t *testing.T) { + s, rc, _, _ := setup(t) + stop := run(t, s) + s.Bus.Publish(machines.Event{Event: machines.EventRecovery, Machine: "gaming-pc", + Recovery: &machines.Recovery{ID: "r1", Machine: "gaming-pc", Action: "reset", State: machines.RecoveryNotBack}}) + waitFor(t, func() bool { return rc.count() == 1 }) + stop() + var got machines.Event + if err := json.Unmarshal(rc.body[0], &got); err != nil || rc.got[0].Header.Get("X-PowerDeck-Event") != "recovery" || + got.Recovery == nil || got.Recovery.State != "not_back" || strings.Contains(string(rc.body[0]), `"evidence"`) { + t.Errorf("%s %v", rc.body[0], err) + } +} + func TestRetriesThenGivesUp(t *testing.T) { s, rc, _, _ := setup(t) rc.fails = 2