Phase 4: recovery with confirmation

CHANGELOG.md

@@ -1,5 +1,27 @@
 # Changelog
 
+## 2.4.0
+
+A frozen computer can now be reset with one tap from a notification, and an assistant can ask for a reset that you confirm.
+
+**Update the helper on the hub and the phone app.** The computers' own helpers don't need 2.4.
+
+- **The hub suggests a reset.** When a computer is called frozen, the hub proposes a reset. Its page shows a card: what the hub saw, and **Reset** or **Not now**. The phone also notifies you, with **Reset** and **Not now** buttons that work only once the phone is unlocked. While the reset runs the card says so. After it, the card says the computer is back, or that it didn't answer within 5 minutes, with **Force off**. You're notified either way.
+- **Assistants propose, you confirm.** An AI agent can't reset or force off a computer itself. With the `power` scope it can now ask (`POST /v2/machines/{name}/recoveries`, with its reason), and you confirm on the phone. Its reason is shown quoted, as its words. The computer's **Allow AI agents** switch still applies. Two agents asking make one request.
+- **Safety rules on the hub:**
+  - At most one reset or force off per computer every 30 minutes. Your own Reset and Force off buttons count towards this, but are never held back.
+  - A request is dropped when the computer answers again or goes off, and expires after 6 hours.
+  - The reset presses nothing if the computer came back in the meantime.
+  - A reset that's still running when the hub restarts is marked failed.
+  - Every step is in the hub's activity log, with who did it.
+- **Reset on its own (optional, off by default).** On the hub's page, **Frozen machines → Reset a frozen machine on its own**: after 10, 30 or 60 minutes without an answer from you, the hub resets a computer that is still frozen. It never forces one off on its own.
+- **API:**
+  - `GET /v2/recoveries`, plus confirm and decline for each one.
+  - `recovery` events on the event stream and to webhooks (new webhooks get them by default).
+  - Machines gain a `recovery` field.
+  - The `autoResetMinutes` setting.
+  - See [docs/protocol.md](docs/protocol.md#recoveries).
+
 ## 2.3.0
 
 The hub now notices when a computer freezes, and says why it thinks so.

android/app/src/main/java/net/odysseylabs/powerdeck/data/HelperClient.kt

@@ -264,6 +264,21 @@ class HelperClient(
         HubResult.from(call("POST", "/v2/machines/${path(machine)}/$action",
             JSONObject().put("force", force).put("stayOff", stayOff).put("press", press), timeoutSeconds = 150)) // the hub bounds an action to 2 minutes
 
+    // ---- Recoveries (a hub, 2.4+): proposed by an agent or the hub, confirmed here ----
+
+    /** Runs a proposed reset or force off. The hub answers at once; the machine's [HubMachine.recovery] tells how it went. */
+    suspend fun confirmRecovery(id: String): HubRecovery = HubRecovery.from(call("POST", "/v2/recoveries/${path(id)}/confirm").getJSONObject("recovery"))
+
+    suspend fun declineRecovery(id: String): HubRecovery = HubRecovery.from(call("POST", "/v2/recoveries/${path(id)}/decline").getJSONObject("recovery"))
+
+    suspend fun recoveries(open: Boolean = true): List<HubRecovery> =
+        call("GET", "/v2/recoveries" + if (open) "?open=1" else "").optJSONArray("recoveries").list(HubRecovery::from)
+
+    /** Resets a machine that stays frozen this many minutes without asking (10 to 60); 0 turns it off. */
+    suspend fun setAutoReset(minutes: Int) {
+        call("POST", "/v2/settings", JSONObject().put("autoResetMinutes", minutes))
+    }
+
     private fun path(s: String) = URLEncoder.encode(s, "UTF-8").replace("+", "%20")
 
     private fun enc(s: String) = URLEncoder.encode(s, "UTF-8")

android/app/src/main/java/net/odysseylabs/powerdeck/data/Model.kt

@@ -98,11 +98,13 @@ data class KvmWiring(val led: String, val power: String, val reset: String) {
 
 /**
  * A machine as a hub sees it (GET /v2/machines). [led] null: not read. From 2.3 the hub's watch adds
- * [state] "hung", a [reason] for "unknown", [since] and its last look ([evidence]).
+ * [state] "hung", a [reason] for "unknown", [since] and its last look ([evidence]); from 2.4 its open
+ * [recovery], or the last one for 30 minutes after it ended.
  */
 data class HubMachine(
     val name: String, val state: String, val led: Boolean?, val ledError: String?, val kvm: String?, val wiring: KvmWiring?, val actions: List<String>,
     val reason: String? = null, val asleep: Boolean = false, val since: Instant? = null, val evidence: HubEvidence? = null,
+    val recovery: HubRecovery? = null,
 ) {
     val hung get() = state == HUNG
 
@@ -113,11 +115,45 @@ data class HubMachine(
             return HubMachine(o.optString("name"), o.optString("state", "unknown"), if (o.has("led") && !o.isNull("led")) o.optBoolean("led") else null,
                 o.optStringOrNull("ledError"), k?.optStringOrNull("name"), k?.optJSONObject("wiring")?.let(KvmWiring::from),
                 o.optJSONArray("actions")?.strings() ?: emptyList(),
-                o.optStringOrNull("reason"), o.optBoolean("asleep"), o.instant("since"), o.optJSONObject("evidence")?.let(HubEvidence::from))
+                o.optStringOrNull("reason"), o.optBoolean("asleep"), o.instant("since"), o.optJSONObject("evidence")?.let(HubEvidence::from),
+                o.optJSONObject("recovery")?.let(HubRecovery::from))
         }
     }
 }
 
+/**
+ * A reset or force off on a hub (2.4+): someone proposes it, the owner confirms, the hub runs it and
+ * checks the machine comes back. [reason] is the proposer's words, shown as they are. [direct]: the
+ * owner's own button, never proposed.
+ */
+data class HubRecovery(
+    val id: String, val machine: String, val action: String, val state: String, val reason: String?,
+    val proposedBy: String, val agent: Boolean, val proposedAt: Instant?, val confirmedBy: String?, val confirmedAt: Instant?,
+    val endedAt: Instant?, val message: String?, val direct: Boolean,
+) {
+    val proposed get() = state == PROPOSED
+    val running get() = state == RUNNING
+    val open get() = proposed || running
+    val forceOff get() = action == FORCE_OFF
+
+    companion object {
+        const val PROPOSED = "proposed"
+        const val RUNNING = "running"
+        const val BACK = "back"
+        const val NOT_BACK = "not_back"
+        const val OFF = "off"
+        const val DONE = "done"
+        const val FAILED = "failed"
+        const val DECLINED = "declined"
+        const val EXPIRED = "expired"
+        const val WITHDRAWN = "withdrawn"
+        const val FORCE_OFF = "force-off"
+        fun from(o: JSONObject) = HubRecovery(o.optString("id"), o.optString("machine"), o.optString("action", "reset"), o.optString("state"),
+            o.optStringOrNull("reason"), o.optString("proposedBy"), o.optBoolean("agent"), o.instant("proposedAt"), o.optStringOrNull("confirmedBy"),
+            o.instant("confirmedAt"), o.instant("endedAt"), o.optStringOrNull("message"), o.optBoolean("direct"))
+    }
+}
+
 /**
  * The hub's last look at a machine: its helper ("answers", "silent"), the helper's port ("refuses",
  * "accepts", "no_reply"), a ping ("replies", "no_reply", "unavailable") and the power light ("on",
@@ -209,6 +245,8 @@ data class HelperStatus(
     val ownersLegacy: Boolean = false,
     /** The helpers this one holds a peer key for (owners only; null from helpers before 2.1). */
     val peerKeys: List<String>? = null,
+    /** A hub's settings (2.4+, owners only): automatic resets after this many minutes of a hang, 0 off. */
+    val autoResetMinutes: Int? = null,
 ) {
     companion object {
         fun pendingFrom(p: JSONObject?) = p?.let { Pending(it.optString("action"), Instant.parse(it.optString("at")), it.optString("by")) }
@@ -237,6 +275,7 @@ data class HelperStatus(
                 caps = o.optJSONArray("caps")?.strings() ?: emptyList(),
                 ownersLegacy = o.optJSONObject("owners")?.optBoolean("legacy") ?: false,
                 peerKeys = o.optJSONArray("peerKeys")?.strings(),
+                autoResetMinutes = o.optJSONObject("hub")?.let { if (it.has("autoResetMinutes")) it.optInt("autoResetMinutes") else null },
             )
         }
     }

android/app/src/main/java/net/odysseylabs/powerdeck/data/PowerDeck.kt

@@ -640,6 +640,33 @@ class PowerDeck(
         else r.outcome()
     }
 
+    /**
+     * Answers a reset or force off that [d]'s hub or an agent proposed: [confirm] has the hub run it
+     * (it answers at once, then checks [d] comes back); otherwise it's declined.
+     */
+    suspend fun answerRecovery(d: Device, id: String, confirm: Boolean): Outcome = oneAtATime(d.id) {
+        guarded {
+            val hub = d.kvm?.hub?.let { store.get(it) } ?: return@guarded Outcome(false, "${d.name} has no hub any more.")
+            val c = client(hub) ?: return@guarded Outcome(false, "${hub.name} has no helper.")
+            try {
+                val r = if (confirm) c.confirmRecovery(id) else c.declineRecovery(id)
+                val what = if (r.forceOff) "force ${d.name} off" else "reset ${d.name}"
+                when (r.state) {
+                    HubRecovery.RUNNING -> Outcome(true, if (r.forceOff) "Forcing ${d.name} off." else "Resetting ${d.name}. ${hub.name} checks it answers within 5 minutes.")
+                    HubRecovery.DECLINED -> Outcome(true, "OK, ${hub.name} won't $what.")
+                    HubRecovery.EXPIRED -> Outcome(false, "That request expired. Nothing was pressed.")
+                    HubRecovery.WITHDRAWN -> Outcome(true, r.message ?: "It isn't needed any more. Nothing was pressed.")
+                    HubRecovery.FAILED, HubRecovery.NOT_BACK -> Outcome(false, r.message ?: "It didn't work.")
+                    else -> Outcome(true, r.message ?: "Done.")
+                }
+            } catch (e: HelperException.Api) {
+                if (e.code == "not_found") Outcome(false, "${hub.name} no longer has that request.") else throw e
+            } finally {
+                hubLock.withLock { hubCache.remove(hub.id) }
+            }
+        }
+    }
+
     /** Helpers that can send this device's wake signal on a schedule (always-on, on its LAN). */
     suspend fun wakeHosts(d: Device): List<Device> =
         store.all().filter { it.id != d.id && it.hasHelper && (d.lan.isEmpty() || Lan.overlaps(it.lan, d.lan)) }

android/app/src/main/java/net/odysseylabs/powerdeck/ui/Components.kt

@@ -63,6 +63,7 @@ import kotlinx.coroutines.launch
 import net.odysseylabs.powerdeck.data.Device
 import net.odysseylabs.powerdeck.data.DeviceState
 import net.odysseylabs.powerdeck.data.HubMachine
+import net.odysseylabs.powerdeck.data.HubRecovery
 import java.time.Duration
 import java.time.Instant
 import java.time.ZoneId
@@ -146,6 +147,7 @@ fun note(state: DeviceState?, now: Long): Pair<String, Tone>? = when (state) {
  */
 fun hubNote(m: HubMachine?, now: Long): Pair<String, Tone>? = when {
     m == null -> null
+    m.recovery?.running == true -> (if (m.recovery.forceOff) "Forcing it off" else "Resetting") to Tone.Busy
     m.hung -> (m.since?.let { "Frozen since " + clockTime(it, now) } ?: "Frozen") to Tone.Warn
     else -> when (m.reason) {
         "booting" -> "Starting up" to Tone.Busy
@@ -159,6 +161,43 @@ fun hubNote(m: HubMachine?, now: Long): Pair<String, Tone>? = when {
     }
 }
 
+/** What a recovery card says. */
+data class RecoveryText(val title: String, val detail: String?, val tone: Tone)
+
+/**
+ * The card for a reset or force off on [device], run by [hub] (which calls itself [hubSelf]), or null
+ * when there's nothing to say: declined, expired, or not needed any more. Someone else's reason is
+ * their words, so it's quoted as theirs.
+ */
+fun recoveryText(r: HubRecovery, device: String, hub: String, now: Long, hubSelf: String = hub): RecoveryText? {
+    val reset = !r.forceOff
+    val at = r.endedAt?.let { " at " + clockTime(it, now) } ?: ""
+    return when (r.state) {
+        HubRecovery.PROPOSED -> {
+            val byHub = r.proposedBy.equals(hubSelf, ignoreCase = true) || r.proposedBy.equals(hub, ignoreCase = true)
+            val who = if (byHub) hub else r.proposedBy
+            RecoveryText(
+                if (reset) "$who suggests a reset" else "$who asks to force $device off",
+                listOfNotNull(
+                    r.reason?.let { if (byHub) it else "“$it”" },
+                    if (reset) "A reset is like pressing its reset button: unsaved work is lost. $hub then checks it answers within 5 minutes."
+                    else "Like pulling the plug: unsaved work is lost, and it stays off.",
+                ).joinToString("\n"),
+                Tone.Warn,
+            )
+        }
+        HubRecovery.RUNNING -> if (reset) RecoveryText("Resetting $device…", "$hub waits up to 5 minutes for it to answer.", Tone.Busy)
+            else RecoveryText("Forcing $device off…", "Holding its power button for 12 seconds.", Tone.Busy)
+        HubRecovery.BACK -> RecoveryText("$device is back", "It answered$at, after the ${if (reset) "reset" else "force off"}.", Tone.On)
+        HubRecovery.NOT_BACK -> RecoveryText("$device didn't come back",
+            "It didn't answer within 5 minutes of the reset. Force it off and on, or check it in person.", Tone.Warn)
+        HubRecovery.OFF -> RecoveryText("$device is off", "Forced off$at.", Tone.Off)
+        HubRecovery.DONE -> RecoveryText(if (reset) "Reset $device" else "Forced $device off", r.message, Tone.On)
+        HubRecovery.FAILED -> RecoveryText(if (reset) "The reset didn't work" else "Couldn't force $device off", r.message, Tone.Warn)
+        else -> null
+    }
+}
+
 private val shortTime = DateTimeFormatter.ofLocalizedTime(FormatStyle.SHORT).withZone(ZoneId.systemDefault())
 private val shortDateTime = DateTimeFormatter.ofLocalizedDateTime(FormatStyle.SHORT).withZone(ZoneId.systemDefault())