fix(drivers): preempt blocked commands for autonomous defaults

.changeset/preempt-blocked-driver-defaults.md

@@ -0,0 +1,5 @@
+---
+"ftw": patch
+---
+
+Core now cancels blocked Lua work, read-only HTTP, or sleep before it runs the driver's autonomous default. Mutating HTTP stays ordered until the host transport returns, and a dedicated default queue keeps the safety request ahead of stale control commands without calling one driver in parallel.

go/internal/drivers/lua.go

@@ -609,6 +609,13 @@ func luaReturnError(name string, ret lua.LValue) error {
 
 // ---- host.* API exposed to Lua ----
 
+func luaCallContext(L *lua.LState) context.Context {
+	if ctx := L.Context(); ctx != nil {
+		return ctx
+	}
+	return context.Background()
+}
+
 func registerHost(L *lua.LState, env *HostEnv) {
 	host := L.NewTable()
 
@@ -720,7 +727,12 @@ func registerHost(L *lua.LState, env *HostEnv) {
 			return 1
 		}
 		if ms > 0 {
-			time.Sleep(time.Duration(ms) * time.Millisecond)
+			timer := time.NewTimer(time.Duration(ms) * time.Millisecond)
+			defer timer.Stop()
+			select {
+			case <-timer.C:
+			case <-luaCallContext(L).Done():
+			}
 		}
 		return 0
 	}))
@@ -1280,7 +1292,7 @@ func registerHost(L *lua.LState, env *HostEnv) {
 			L.Push(lua.LString("http: " + reason))
 			return 2
 		}
-		req, err := net_http.NewRequest("GET", url, nil)
+		req, err := net_http.NewRequestWithContext(luaCallContext(L), "GET", url, nil)
 		if err != nil {
 			L.Push(lua.LNil)
 			L.Push(lua.LString(err.Error()))
@@ -1332,6 +1344,10 @@ func registerHost(L *lua.LState, env *HostEnv) {
 			return 2
 		}
 		payload := L.CheckString(2)
+		// Do not cancel a mutating request when the Lua command context ends.
+		// Once the device may have received the write, a following default must
+		// stay behind this request in the registry actor. The host client's
+		// 15-second timeout still bounds transport failure.
 		req, err := net_http.NewRequest("POST", url, strings.NewReader(payload))
 		if err != nil {
 			L.Push(lua.LNil)
@@ -1393,6 +1409,8 @@ func registerHost(L *lua.LState, env *HostEnv) {
 			return 2
 		}
 		payload := L.CheckString(2)
+		// Keep the same ordering rule as POST: the registry actor must not send
+		// a default while this older mutating request can still finish normally.
 		req, err := net_http.NewRequest("PATCH", url, strings.NewReader(payload))
 		if err != nil {
 			L.Push(lua.LNil)

go/internal/drivers/registry.go

@@ -263,9 +263,10 @@ type runningDriver struct {
 	evPauseCycleID  uint64
 	evPausePending  bool
 	// Poll loop coordination
-	cmdCh chan driverCmd
-	stop  chan bool
-	done  chan struct{}
+	cmdCh     chan driverCmd
+	defaultCh chan driverCmd
+	stop      chan bool
+	done      chan struct{}
 }
 
 func (rd *runningDriver) controlStatus() DriverControlStatus {
@@ -657,6 +658,7 @@ func (r *Registry) add(ctx context.Context, cfg config.Driver, startupDefault bo
 		lifecycleCtx:    lifecycleCtx,
 		lifecycleCancel: lifecycleCancel,
 		cmdCh:           make(chan driverCmd, 8),
+		defaultCh:       make(chan driverCmd, 1),
 		stop:            make(chan bool, 1),
 		done:            make(chan struct{}),
 	}
@@ -820,7 +822,45 @@ func (r *Registry) runLoop(rd *runningDriver) {
 		}
 		return commandOutcome
 	}
+	handleDefault := func(cmd driverCmd) {
+		invalidateCommandSequence()
+		cmdCtx := cmd.ctx
+		if cmdCtx == nil {
+			cmdCtx = ctx
+		}
+		// Once accepted, a safety default must survive its caller timing out
+		// behind an older command. A fresh bounded attempt keeps that request
+		// durable without letting the actor block forever.
+		var cancel context.CancelFunc
+		if cmdCtx.Err() != nil {
+			cmdCtx, cancel = context.WithTimeout(context.Background(), defaultRecoveryTimeout)
+		}
+		err := r.defaultDriver(cmdCtx, rd, "host_request")
+		if cancel != nil {
+			cancel()
+		}
+		if err == nil {
+			clearLease()
+			rd.markDefaultConfirmed()
+			clearRecoveryTimer()
+			r.clearRecoveryRequired(rd.cfg.Name, rd)
+		} else {
+			scheduleRecovery()
+		}
+		if cmd.result != nil {
+			cmd.result <- err
+		}
+	}
 	for {
+		// Service an accepted autonomous default before normal control. The
+		// blocked status still rejects a normal command if both queues become
+		// ready between this check and the main select.
+		select {
+		case cmd := <-rd.defaultCh:
+			handleDefault(cmd)
+			continue
+		default:
+		}
 		select {
 		case skipDefault := <-rd.stop:
 			if !skipDefault {
@@ -856,6 +896,8 @@ func (r *Registry) runLoop(rd *runningDriver) {
 				_ = rd.env.TCP.Close()
 			}
 			return
+		case cmd := <-rd.defaultCh:
+			handleDefault(cmd)
 		case cmd := <-rd.cmdCh:
 			var err error
 			reportOutcome := cmd.outcome != nil
@@ -925,6 +967,15 @@ func (r *Registry) runLoop(rd *runningDriver) {
 					invalidateCommandSequence()
 				}
 				commandCtx, finishCommand := rd.beginCommand(cmdCtx)
+				// SendDefault may close the control window after the earlier queue
+				// check but before this actor installs activeCancel. Recheck once the
+				// cancel hook exists; after this point a racing default cancels the
+				// context passed to the runtime.
+				if rd.controlIsBlocked() {
+					finishCommand()
+					err = ErrControlBlocked
+					break
+				}
 				if rd.policy != nil && rd.policy.IsControlV2() {
 					var result DriverCommandResultV1
 					var leaseExpiresAt time.Time
@@ -961,17 +1012,6 @@ func (r *Registry) runLoop(rd *runningDriver) {
 					rd.evPauseRevision = rd.commandRevision
 					rd.evPauseCycleID = cmd.cycleID
 				}
-			case "default":
-				invalidateCommandSequence()
-				err = r.defaultDriver(cmdCtx, rd, "host_request")
-				if err == nil {
-					clearLease()
-					rd.markDefaultConfirmed()
-					clearRecoveryTimer()
-					r.clearRecoveryRequired(rd.cfg.Name, rd)
-				} else {
-					scheduleRecovery()
-				}
 			}
 			if reportOutcome {
 				cmd.outcome(err)
@@ -1279,12 +1319,10 @@ func (r *Registry) sendWithGeneration(ctx context.Context, name string, payload
 	}
 }
 
-// SendDefault sends the default/watchdog command to a driver. Symmetric
-// with Send: both the channel-push and the result-wait honour ctx. A
-// driver whose cmdCh is full (because its goroutine is slow / stuck mid
-// I/O) would otherwise block the caller forever; the watchdog-fallback
-// path runs on every dispatch tick, so an unblocked send into a wedged
-// driver deadlocks the entire control loop.
+// SendDefault sends the default/watchdog command to a driver. Defaults use a
+// dedicated one-slot queue so stale normal commands cannot prevent the
+// autonomous path from being accepted. Once accepted, the generation stays
+// blocked until the default succeeds or the recovery timer retries it.
 func (r *Registry) SendDefault(ctx context.Context, name string) error {
 	if ctx == nil {
 		ctx = context.Background()
@@ -1295,11 +1333,26 @@ func (r *Registry) SendDefault(ctx context.Context, name string) error {
 	if !ok {
 		return fmt.Errorf("driver %q not found", name)
 	}
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	// Close the control window before enqueueing. Canceling the active Lua call
+	// lets a context-aware host operation return to this same actor before the
+	// queued default runs; the registry never calls one driver in parallel.
+	rd.markDefaultRecoveryPending()
+	rd.cancelActiveCommand()
 	resCh := make(chan error, 1)
+	cmd := driverCmd{kind: "default", ctx: ctx, result: resCh}
 	select {
-	case rd.cmdCh <- driverCmd{kind: "default", ctx: ctx, result: resCh}:
-	case <-ctx.Done():
-		return ctx.Err()
+	case rd.defaultCh <- cmd:
+	default:
+		// Another accepted default already supplies the durable safety request.
+		// Wait only for room or this caller's deadline.
+		select {
+		case rd.defaultCh <- cmd:
+		case <-ctx.Done():
+			return ctx.Err()
+		}
 	}
 	select {
 	case err := <-resCh:

go/internal/drivers/registry_command_deadline_test.go

@@ -54,12 +54,13 @@ func TestSendReturnsAtDeadlineWhileDriverIsWedged(t *testing.T) {
 		release: make(chan struct{}),
 	}
 	rd := &runningDriver{
-		driver: rt,
-		env:    rt.env,
-		cfg:    config.Driver{Name: "d1"},
-		cmdCh:  make(chan driverCmd, 1),
-		stop:   make(chan bool, 1),
-		done:   make(chan struct{}),
+		driver:    rt,
+		env:       rt.env,
+		cfg:       config.Driver{Name: "d1"},
+		cmdCh:     make(chan driverCmd, 1),
+		defaultCh: make(chan driverCmd, 1),
+		stop:      make(chan bool, 1),
+		done:      make(chan struct{}),
 	}
 	r.rec["d1"] = rd
 	go r.runLoop(rd)

go/internal/drivers/registry_ev_command_owner_test.go

@@ -96,6 +96,7 @@ func newEVCommandOwnerRegistry(t *testing.T, blocked bool) (*Registry, *runningD
 		lifecycleCtx:    lifecycleCtx,
 		lifecycleCancel: lifecycleCancel,
 		cmdCh:           make(chan driverCmd, 8),
+		defaultCh:       make(chan driverCmd, 1),
 		stop:            make(chan bool, 1),
 		done:            make(chan struct{}),
 	}
@@ -128,26 +129,29 @@ func TestDefaultBoundaryInvalidatesEarlierEVPause(t *testing.T) {
 	defaultDone := make(chan error, 1)
 	go func() { defaultDone <- r.SendDefault(ctx, "charger") }()
 	deadline := time.Now().Add(2 * time.Second)
-	for len(rd.cmdCh) == 0 && time.Now().Before(deadline) {
+	for len(rd.defaultCh) == 0 && time.Now().Before(deadline) {
 		time.Sleep(time.Millisecond)
 	}
-	if len(rd.cmdCh) == 0 {
+	if len(rd.defaultCh) == 0 {
 		close(runtime.pauseRelease)
 		t.Fatal("default was not queued behind the parked pause")
 	}
 	close(runtime.pauseRelease)
-	for name, done := range map[string]<-chan error{
-		"pause":   pauseDone,
-		"default": defaultDone,
-	} {
-		select {
-		case err := <-done:
-			if err != nil {
-				t.Fatalf("%s: %v", name, err)
-			}
-		case <-time.After(2 * time.Second):
-			t.Fatalf("%s did not finish", name)
+	select {
+	case err := <-pauseDone:
+		if !errors.Is(err, ErrCommandMayHaveRun) || !errors.Is(err, context.Canceled) {
+			t.Fatalf("pause = %v, want command-may-have-run plus canceled", err)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("pause did not finish")
+	}
+	select {
+	case err := <-defaultDone:
+		if err != nil {
+			t.Fatalf("default: %v", err)
 		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("default did not finish")
 	}
 	// Prove the actor's default revision, not the outer health check, owns
 	// the rejection: telemetry may recover before this stale continuation.

go/internal/drivers/registry_restart_test.go

@@ -130,12 +130,13 @@ func TestSendDefaultPassesCallerContextToRuntime(t *testing.T) {
 		entered: make(chan struct{}),
 	}
 	rd := &runningDriver{
-		driver: rt,
-		env:    rt.env,
-		cfg:    config.Driver{Name: "d1"},
-		cmdCh:  make(chan driverCmd, 1),
-		stop:   make(chan bool, 1),
-		done:   make(chan struct{}),
+		driver:    rt,
+		env:       rt.env,
+		cfg:       config.Driver{Name: "d1"},
+		cmdCh:     make(chan driverCmd, 1),
+		defaultCh: make(chan driverCmd, 1),
+		stop:      make(chan bool, 1),
+		done:      make(chan struct{}),
 	}
 	r.rec["d1"] = rd
 	go r.runLoop(rd)
@@ -171,6 +172,7 @@ func TestRegistryCancelAfterCommandStartedRestoresDefault(t *testing.T) {
 		lifecycleCtx:    lifecycleCtx,
 		lifecycleCancel: lifecycleCancel,
 		cmdCh:           make(chan driverCmd, 1),
+		defaultCh:       make(chan driverCmd, 1),
 		stop:            make(chan bool, 1),
 		done:            make(chan struct{}),
 	}