From 0297f4d988a5bb8f5a607af48b59810a6b5c8acf Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 11:43:14 +0900 Subject: [PATCH 01/64] docs: health telemetry platform design spec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sensor-agnostic incident tracking system — detects stream stalls, FPS drops, jitter, startup failures, writer backpressure, and adapter- reported faults via a pluggable detector registry, and surfaces them in the viewer as Sentry-style Incidents (open/close, severity, fingerprint-grouped events, attached artifacts). Motivated by OAK failure modes (X_LINK_ERROR, device crashes, reconnect) that currently vanish into stderr; bridged into the unified telemetry channel via a depthai logger handler. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../2026-04-22-health-telemetry-design.md | 369 ++++++++++++++++++ 1 file changed, 369 insertions(+) create mode 100644 docs/superpowers/specs/2026-04-22-health-telemetry-design.md diff --git a/docs/superpowers/specs/2026-04-22-health-telemetry-design.md b/docs/superpowers/specs/2026-04-22-health-telemetry-design.md new file mode 100644 index 0000000..d1c7a36 --- /dev/null +++ b/docs/superpowers/specs/2026-04-22-health-telemetry-design.md @@ -0,0 +1,369 @@ +# Health Telemetry Platform — Design Spec + +- **Date**: 2026-04-22 +- **Status**: Approved for implementation planning +- **Owner**: syncfield-python +- **Related**: `src/syncfield/orchestrator.py` (session lifecycle), `src/syncfield/stream.py` (Stream protocol, existing `HealthEvent`), `src/syncfield/adapters/oak_camera.py` (motivating failure modes), `src/syncfield/viewer/` (live + post-session surface) + +## Summary + +Add a sensor-agnostic, platform-level **health telemetry system** to syncfield. During a recording the system continuously observes every active stream (sample cadence, silence gaps, adapter-reported faults, writer backpressure) and raises **Incidents** — Sentry-style grouped objects with severity, open/close lifecycle, and attached artifacts — that surface in the viewer in real time and are persisted for post-session review. New detectors can be added without touching any adapter; new adapters inherit the full baseline check suite for free. + +The first motivating hardware is OAK (Luxonis DepthAI), where native log output (`X_LINK_ERROR`, `Device has crashed`, crash-dump paths, `Reconnection successful`) today vanishes into stderr. The design bridges that native logger into the unified telemetry channel, and the same detector set catches identical symptoms (stall, FPS drop) on any future adapter. + +## Goals + +1. **Live detection, live display**: while recording, surface hardware crashes, stream stalls, FPS drops, jitter spikes, startup failures, writer backpressure, and adapter-reported faults within ~1s of occurrence. Render them as a first-class **Active Issues** panel in the viewer. +2. **Post-session incident report**: persist a structured `incidents.jsonl` per session and expose `FinalizationReport.incidents`, so users can answer "what went wrong, when, on which stream, with what evidence?" after the fact. +3. **Sensor-agnostic baseline**: every stream — current and future — automatically gets stall detection, FPS-drop detection, jitter detection, startup-failure detection, and adapter-event pass-through, with zero adapter code changes. +4. **Pluggable detectors**: adding a new detection rule (temperature, bandwidth, battery, anomaly) is a single-file, single-class addition registered at startup. No adapter modifications. +5. **Default-on**: users never need to wire up health telemetry; a freshly constructed `SessionOrchestrator` has the full baseline running. +6. **Artifact capture**: when a device provides crash evidence (e.g., OAK's `crash_dump.json`), attach the path to the incident so the user can ship it to the vendor. +7. **Zero impact on capture hot path**: detectors observe via lock-free hand-off; all detection work runs on a dedicated worker thread. + +## Non-goals + +- **Multihost fan-in (leader-side fleet view)**: follower incidents are written to the follower's session log and collected post-stop via the existing file-pull path. Real-time follower→leader streaming of health events is deferred. +- **Cross-session history / search**: no database, no aggregation across sessions. Each session is self-contained. +- **Automatic remediation**: the platform detects and reports. It does not auto-reconnect devices, swap to backup streams, or halt recording on failure. +- **Frame-content anomaly detection**: no checks for black frames, codec corruption, audio silence content, or ML-based quality scoring. +- **Host resource monitoring**: CPU/GPU/memory/thermal of the host machine are out of scope. +- **Cross-session markdown summary**: `incidents.jsonl` is structured; a human-readable post-mortem generator is deferred. +- **Backward compatibility of existing health surfaces**: `StreamSnapshot.health_count` / `problem_count` are removed in favor of the new incident-based fields. Existing `session_log.jsonl` consumers must migrate. There are no external consumers today. + +## Current state (what exists) + +- `HealthEvent` dataclass with `HealthEventKind = {HEARTBEAT, DROP, RECONNECT, WARNING, ERROR}` in `src/syncfield/types.py:250-273`. +- `StreamBase._emit_health(event)` for adapters to push into the session log — `src/syncfield/stream.py:237-241`. +- `SessionLogWriter.log_health(event)` persists to `session_log.jsonl`. +- `SessionOrchestrator._on_stream_health()` routes events to log + buffered collection — `src/syncfield/orchestrator.py:2354-2363`. +- Viewer `HealthTable` component renders last 20 raw events as a timeline. +- `FinalizationReport.health_events[]` includes raw events post-stop. + +**What is missing, and what this spec adds**: no automatic detection (all events are adapter-emitted); no severity; no grouping; no open/close semantics; no detector plugin model; no artifact attachment; no Sentry-style UI surface; no per-adapter target-hz hint for comparing observed vs. expected cadence. + +## Architecture Overview + +``` +┌──────────────────────────────────────────────────────────────────────────┐ +│ SessionOrchestrator │ +│ │ +│ ┌──────────┐ on_sample ┌──────────────────────────┐ │ +│ │ Streams │──────────────►│ │ │ +│ └──────────┘ on_health │ │ │ +│ │ │ HealthSystem │ │ +│ │ state changes ───►│ │ │ +│ │ │ ┌────────────────────┐ │ │ +│ ▼ │ │ DetectorRegistry │ │ │ +│ ┌──────────┐ queue stats │ │ (default + user) │ │ │ +│ │ Writer │──────────────►│ └────────────────────┘ │ │ +│ └──────────┘ │ │ │ │ +│ │ ▼ │ │ +│ │ ┌────────────────────┐ │ │ +│ │ │ HealthWorker │ │ ◄── 20 Hz │ +│ │ │ (thread, ticks) │ │ tick │ +│ │ └────────────────────┘ │ │ +│ │ │ │ │ +│ │ ▼ │ │ +│ │ ┌────────────────────┐ │ │ +│ │ │ IncidentTracker │ │ │ +│ │ │ open / close / │ │ │ +│ │ │ fingerprint group │ │ │ +│ │ └────────────────────┘ │ │ +│ │ │ │ │ +│ └───────────┼──────────────┘ │ +│ │ │ +│ ┌────────────────────────┼────────────────────────┐ │ +│ │ │ │ │ +│ ▼ ▼ ▼ │ +│ session_log.jsonl incidents.jsonl SessionSnapshot +│ (raw HealthEvents) (Incident records) → WebSocket │ +│ │ │ +└─────────────────────────────────────────┼──────────────────────────────┘ + ▼ + FinalizationReport.incidents + (exposed from SDK) +``` + +**Threading model**: streams continue to call `on_sample` / `_emit_health` from capture threads. These fan into lock-free deques drained by a single `HealthWorker` daemon thread (20 Hz). All detector ticks and incident-tracker state live on that thread — there are no locks on the capture hot path beyond the deque push. + +## Data model + +### Severity + +```python +class Severity(str, Enum): + INFO = "info" # heartbeat, reconnect-success, config warning + WARNING = "warning" # mild FPS dip, jitter, macOS deprecation warning + ERROR = "error" # disconnect, X_LINK_ERROR, encoding fail, silence > 2s + CRITICAL = "critical" # device crash, multi-stream simultaneous down +``` + +### HealthEvent (enriched, existing class; new fields) + +```python +@dataclass(frozen=True) +class HealthEvent: + stream_id: str + kind: HealthEventKind # existing + at_ns: int # existing, time.monotonic_ns() + detail: str | None # existing + + # new fields (no backward compat shim — session_log.jsonl format changes): + severity: Severity + source: str # "adapter:oak" | "detector:stream-stall" | ... + fingerprint: str # stable grouping key, e.g. "oak-main:stream-stall" + data: dict[str, Any] = {} # structured context: {"observed_hz": 12, "target_hz": 30} +``` + +The `fingerprint` is what groups many raw events into one Incident. Detectors own the fingerprint formula (typically `f"{stream_id}:{detector.name}"`, optionally refined for sub-categorization). Adapter-emitted events use `f"{stream_id}:adapter:{kind.value}"` via the pass-through detector. + +### Incident (new) + +```python +@dataclass +class Incident: + id: str # ulid, stable for the lifetime of the session + stream_id: str + fingerprint: str # same as the events it groups + title: str # human-readable, e.g., "Stream stalled (silence 9s)" + severity: Severity # = max severity across grouped events + source: str # name of the first detector/adapter that fired + + opened_at_ns: int + closed_at_ns: int | None # None = still open / active + last_event_at_ns: int # most recent event in the group + + event_count: int + first_event: HealthEvent # snapshot for display + last_event: HealthEvent # snapshot for display + + artifacts: list[IncidentArtifact] = [] # attached evidence + data: dict[str, Any] = {} # aggregated context (e.g., min_observed_hz) + +@dataclass(frozen=True) +class IncidentArtifact: + kind: str # "crash_dump" | "log_excerpt" | ... + path: str # absolute path, or URI + detail: str | None = None +``` + +An incident is **open** while its condition persists. The tracker closes it when the owning detector's `close_condition` returns True (for detector-owned incidents) or when a quiet period elapses without a new matching event (for pass-through adapter incidents; default 30s). + +## Detector system + +### Detector protocol + +```python +class Detector(Protocol): + name: str # "stream-stall", "fps-drop", ... + default_severity: Severity # severity of events this detector raises + + def observe_sample(self, stream_id: str, sample: SampleEvent) -> None: ... + def observe_health(self, stream_id: str, event: HealthEvent) -> None: ... + def observe_state(self, old: SessionState, new: SessionState) -> None: ... + def observe_writer_stats(self, stream_id: str, stats: WriterStats) -> None: ... + + def tick(self, now_ns: int) -> Iterable[HealthEvent]: ... + def close_condition(self, incident: Incident, now_ns: int) -> bool: ... +``` + +- `observe_*` feed the detector its raw signals; they must be fast and non-blocking. +- `tick` is called by the `HealthWorker` at 20 Hz; this is where silence/gap-based detections fire. +- `close_condition` decides when an open incident with this detector's fingerprint can be closed. + +A `DetectorBase` ABC provides no-op defaults for every hook so subclasses only implement what they need. + +### Default detector suite (registered automatically on `SessionOrchestrator` construction) + +| Detector | Signal | Fires when | Closes when | +|---|---|---|---| +| `StreamStallDetector` | `observe_sample` (timestamps) + `tick` | no sample for 2.0s on a stream that previously had samples | sample resumes and keeps flowing 1.0s | +| `FpsDropDetector` | `observe_sample` + `tick` | effective FPS < 70% of `target_hz` for 3.0s (or, without `target_hz`, < 70% of learned 10s-baseline after a 5s warmup) | FPS ≥ 90% for 5.0s | +| `JitterDetector` | `observe_sample` + `tick` | p95 inter-sample gap over last 60 samples > 2× expected | p95 ≤ 1.2× expected for 10.0s | +| `StartupFailureDetector` | `observe_health` (for adapter-raised startup errors) + `observe_state` | `connect()` or `start_recording()` raises / times out | subsequent retry reports success | +| `BackpressureDetector` | `observe_writer_stats` | writer queue depth > 80% of capacity for 2.0s, OR writer drop counter increments | queue depth < 30% for 5.0s | +| `AdapterEventPassthrough` | `observe_health` | any adapter-emitted `HealthEvent` not already owned by another detector | 30s quiet on the fingerprint | +| `DepthAILoggerBridge` | native depthai Python logger handler | depthai error/warning log arrives (X_LINK_ERROR, "Device has crashed", reconnect events) → converted to `HealthEvent` → fed into `AdapterEventPassthrough` | (stateless — it is a translator, not a detector) | + +`DepthAILoggerBridge` is not technically a detector in the "fire from tick" sense — it is a logging-handler adapter. For symmetry it lives next to detectors and registers via the same registry, but it only produces `HealthEvent`s; `AdapterEventPassthrough` groups them into incidents. The bridge is installed **automatically** when the `oak` optional extra is present (detected by importability of `depthai`), matching the project's default-on philosophy. + +### DetectorRegistry + +```python +class DetectorRegistry: + def register(self, detector: Detector) -> None: ... + def unregister(self, name: str) -> None: ... + def __iter__(self) -> Iterator[Detector]: ... +``` + +Users register additional detectors before `session.connect()`: + +```python +session = SessionOrchestrator(...) +session.health.register(MyTemperatureDetector(...)) +``` + +Default detectors are installed by `HealthSystem` in `__init__`; users can `unregister` them by name to opt out. + +### HealthWorker & IncidentTracker + +- `HealthWorker` is a daemon thread that owns the tick loop and all mutable state. It exposes thread-safe ingress queues (`Queue` or lock-free deque) for samples, health events, state transitions, and writer stats. +- `IncidentTracker` (running on the worker thread) consumes every `HealthEvent` produced by detectors or observed via pass-through, groups by fingerprint, opens new incidents, updates existing ones, and runs `close_condition` each tick for every open incident. +- Closed incidents stay in the tracker's memory (for the viewer's "Resolved this session" list) and are flushed to `incidents.jsonl` on each state change and on `stop()`. + +## Integration points + +### `src/syncfield/types.py` +- Add `Severity` enum. +- Enrich `HealthEvent` with `severity`, `source`, `fingerprint`, `data`. +- Add `Incident`, `IncidentArtifact`, `IncidentSnapshot` (read-only view used in WebSocket payloads). +- Add `WriterStats` dataclass (queue depth, drop count, bytes flushed). +- `FinalizationReport` gains `incidents: list[Incident]`. `health_events` field remains. + +### `src/syncfield/stream.py` +- `StreamCapabilities` gains `target_hz: float | None = None`. +- No API change to `StreamBase._emit_health`; it continues to push `HealthEvent` and now the system assigns `severity`/`fingerprint`/`source` server-side if the adapter did not. + +### `src/syncfield/health/` (new package) +``` +src/syncfield/health/ +├── __init__.py # public: HealthSystem, Severity, Incident, Detector, DetectorBase +├── types.py # Incident, IncidentArtifact, IncidentSnapshot, WriterStats +├── severity.py # Severity enum + helpers +├── system.py # HealthSystem (user-facing facade, owns worker + registry + tracker) +├── worker.py # HealthWorker (thread, tick loop, ingress queues) +├── registry.py # DetectorRegistry +├── tracker.py # IncidentTracker +├── detector.py # Detector protocol + DetectorBase +└── detectors/ + ├── __init__.py + ├── stream_stall.py + ├── fps_drop.py + ├── jitter.py + ├── startup_failure.py + ├── backpressure.py + ├── adapter_passthrough.py + └── depthai_bridge.py # soft-imports depthai; registered if importable +``` + +### `src/syncfield/orchestrator.py` +- `SessionOrchestrator.__init__` constructs `self.health = HealthSystem(session_id=..., clock=...)`. +- On adding a stream: subscribe `health.observe_sample` and `health.observe_health` to that stream. +- On every `SessionState` transition: `health.observe_state(old, new)`. +- On each writer flush: pump `WriterStats` into `health.observe_writer_stats`. +- `start()` boots the worker thread; `stop()` drains, closes still-open incidents with `closed_at_ns = now`, and embeds `incidents` into `FinalizationReport`. +- Existing `_on_stream_health` is simplified — it now just forwards to `health`. `session_log.jsonl` writing is owned by `SessionLogWriter`, which also gains `log_incident(incident)`. + +### `src/syncfield/writer.py` +- `SessionLogWriter` gains `log_incident(incident)` writing to `incidents.jsonl`. +- On every incident open/update/close, `IncidentTracker` calls `writer.log_incident(...)`. +- The writer also emits `WriterStats` snapshots at a low frequency (e.g., every 250ms) into `HealthSystem` for backpressure detection. + +### `src/syncfield/adapters/oak_camera.py` +- Declare `target_hz` in `StreamCapabilities`. +- At `connect()`: attach `DepthAILoggerBridge` to the depthai Python logger (scoped to this stream instance). +- At crash detection (depthai signals device crash or the bridge observes the "Crash dump logs are stored in" line): emit a `HealthEvent` with `kind=ERROR`, `severity=CRITICAL`, `data={"crash_dump_path": "..."}`. The tracker attaches it as an `IncidentArtifact(kind="crash_dump", path=...)`. +- No other OAK-specific detection code. `StreamStallDetector` and `FpsDropDetector` handle the "frozen for 9s during depthai reconnect" case generically. + +### `src/syncfield/viewer/state.py` & `server.py` +- `SessionSnapshot` gains: + - `active_incidents: list[IncidentSnapshot]` + - `resolved_incidents: list[IncidentSnapshot]` (capped at 20 most recent) +- `StreamSnapshot.health_count` and `StreamSnapshot.problem_count` **removed**; per-stream incident counts derived client-side by filtering `active_incidents` by `stream_id`. +- Poller subscribes to `HealthSystem` for incident opened/updated/closed notifications (via a simple callback protocol) and merges them into the snapshot. + +### `src/syncfield/viewer/frontend/src/` +- Delete `components/health-table.tsx`. +- Add `components/incident-panel.tsx` (Option α): + - Two collapsible sections: **Active Issues (N)**, **Resolved this session (N)**. + - Each incident card: severity icon · stream id · title · opened/recovered relative time · event count · artifact chips. + - Click to expand → raw `HealthEvent` list for that fingerprint pulled from `health_log` (retained in snapshot, capped at 200 most recent overall). +- Stream cards: replace red dot count with severity-colored badge (critical=red, error=orange, warning=yellow) whose number reflects the count of open incidents on that stream. +- Shared type file `lib/types.ts` mirrors `IncidentSnapshot` / `Severity`. + +## OAK — concrete event → incident mapping + +Reference sample from the user's session log: + +``` +[depthai] [error] Communication exception ... 'Couldn't read data from stream: '__x_0_1' (X_LINK_ERROR)' +[host] [warning] Closed connection +[host] [warning] Attempting to reconnect. Timeout is 10000ms +[depthai] [error] Device with id ... has crashed. Crash dump logs are stored in: /path/to/crash_dump.json +[host] [warning] Reconnection successful +``` + +Pipeline: + +1. `DepthAILoggerBridge` converts each native line into a `HealthEvent`: + - `X_LINK_ERROR` → `kind=ERROR`, `severity=ERROR`, `source="adapter:oak"`, `fingerprint="oak-main:adapter:xlink-error"`, `data={"stream": "__x_0_1"}`. + - "Closed connection" → `kind=WARNING`, `severity=WARNING`, `fingerprint="oak-main:adapter:connection-closed"`. + - "Attempting to reconnect" → `kind=RECONNECT`, `severity=INFO`, `fingerprint="oak-main:adapter:reconnect-attempt"`. + - "Device has crashed" → `kind=ERROR`, `severity=CRITICAL`, `fingerprint="oak-main:adapter:device-crash"`, `data={"crash_dump_path": "/path/to/crash_dump.json"}`. + - "Reconnection successful" → `kind=RECONNECT`, `severity=INFO`, `fingerprint="oak-main:adapter:reconnect-success"`. +2. `AdapterEventPassthrough` groups the four non-info fingerprints into incidents. +3. Concurrently, `StreamStallDetector` observes no sample for 2s and opens its own incident `fingerprint="oak-main:stream-stall"` with `severity=ERROR`. +4. When samples resume (bridge sees "Reconnection successful" and stream yields samples again), `StreamStallDetector.close_condition` returns True after 1s of steady flow → the stall incident closes. +5. The device-crash incident gains an `IncidentArtifact(kind="crash_dump", path=...)` attached from its event's `data`. +6. On `stop()`, all these incidents flush to `incidents.jsonl`. + +The macOS `NSCameraUseContinuityCameraDeviceType` warning is emitted by the UVC adapter similarly (UVC adapter adds a tiny stderr-watcher or converts via existing `_emit_health`) as `severity=INFO`, `fingerprint="uvc-cam:adapter:continuity-deprecation"`; surfaces as a single resolved informational incident. + +## Persistence & artifacts + +Per-session directory additions: + +``` +session// +├── session_log.jsonl # existing — raw HealthEvents (format updated, not backward-compat) +├── incidents.jsonl # new — one Incident per line; appended on open/update/close +├── manifest.json # existing — gains "incidents_count", "active_at_stop_count" +└── /... # existing per-stream files; crash dumps referenced via absolute path +``` + +- `incidents.jsonl` is append-only. Each line is the current full state of the incident at write time; readers compact by `incident.id`. +- `FinalizationReport.incidents` is the in-memory compacted list (unique by `id`, latest state). + +## Testing strategy + +Every new module below `health/` ships with unit tests. The following are the load-bearing test suites: + +**`tests/health/test_detectors.py`** — one class per detector. Uses a `FakeClock` and a synthetic sample stream generator. Each test asserts: (a) the detector fires on the trigger, (b) does not fire on noise, (c) `close_condition` returns True after recovery, (d) fingerprint is stable across runs. + +**`tests/health/test_incident_tracker.py`** — grouping, opening, closing, reopening (same fingerprint after close), severity escalation (an open WARNING incident that receives an ERROR event upgrades), artifact attachment, JSONL flush on close. + +**`tests/health/test_health_system_integration.py`** — end-to-end with a `FakeStream` (a `StreamBase` subclass that emits synthetic samples on a driven clock): (a) stall then recover → one incident opens and closes; (b) sustained FPS drop → one incident; (c) adapter emits `HealthEvent(kind=ERROR)` → pass-through creates incident; (d) crash event carrying `data["crash_dump_path"]` produces artifact. + +**`tests/health/test_depthai_bridge.py`** — feeds synthetic depthai log records (as `logging.LogRecord`) and asserts the correct `HealthEvent`s come out. No real depthai device required. + +**`tests/orchestrator/test_orchestrator_health_integration.py`** — real `SessionOrchestrator` with `FakeStream` registered; drives the full lifecycle and asserts `FinalizationReport.incidents` content and `incidents.jsonl` content match. + +**`tests/viewer/test_snapshot_incidents.py`** — poller snapshot correctly includes `active_incidents` / `resolved_incidents`; stream filtering works. + +Frontend: at least a component test for `IncidentPanel` rendering Active + Resolved sections and a click-to-expand interaction. + +## Implementation phases (for the writing-plans step) + +1. **Core types & skeleton** — `Severity`, enriched `HealthEvent`, `Incident`, `IncidentArtifact`, `WriterStats`. `health/` package skeleton with `HealthSystem`, `HealthWorker`, `DetectorRegistry`, `IncidentTracker`, `DetectorBase`. Unit tests for tracker & worker. +2. **Platform detectors** — `StreamStallDetector`, `FpsDropDetector`, `JitterDetector`, `StartupFailureDetector`, `BackpressureDetector`, `AdapterEventPassthrough`. Full unit-test coverage. +3. **Orchestrator & writer integration** — wire `SessionOrchestrator` to `HealthSystem`, remove `StreamSnapshot.health_count`/`problem_count`, add writer stats emission, `incidents.jsonl` flushing, `FinalizationReport.incidents`. Integration tests with `FakeStream`. +4. **OAK bridge & `target_hz`** — `DepthAILoggerBridge`, crash-dump artifact attachment, declare `target_hz` on `OakCameraStream`. Bridge unit tests. +5. **Viewer — server** — `IncidentSnapshot` in `SessionSnapshot`; poller ingests from `HealthSystem`; remove obsolete fields. Snapshot tests. +6. **Viewer — frontend** — new `incident-panel.tsx`, deleted `health-table.tsx`, stream-card badge update, types mirrored. Component tests. +7. **Per-adapter target_hz rollout** — add `target_hz` to every adapter that has a known target (`UVCWebcamStream`, `HostAudioStream`, `MetaQuestCameraStream`, `Go3SStream` where relevant, etc.). No other changes. +8. **Manual verification checklist** — reproduce the OAK crash sequence on a real rig; confirm incidents open, close, and artifact attaches; screenshot `IncidentPanel`. + +## Open design questions (to resolve during planning) + +None blocking. Low-stakes items to be resolved during implementation: + +- Exact `target_hz`-learning window length when unset (currently proposed 5s warmup, 10s rolling baseline). +- Whether `DetectorRegistry.unregister` is public API for v1 or internal-only. +- Whether `IncidentSnapshot` includes the full `first_event` / `last_event`, or just their `detail` strings (wire size tradeoff). + +## Risks + +- **Noisy detectors**: a poorly tuned `JitterDetector` could fire constantly on cameras with naturally irregular cadence. Mitigation: per-adapter `jitter_tolerance` hint in `StreamCapabilities`, default conservative. +- **Depthai logger bridge fragility**: parses string patterns; a depthai version bump could change log format. Mitigation: pattern-match liberally and fall back to `severity=WARNING` with `source="adapter:oak:unparsed-log"` rather than dropping the line. Bridge has its own targeted tests. +- **Thread starvation**: if the worker tick blocks, detections are delayed. Mitigation: all detector `tick` / `observe_*` methods must be non-blocking; enforced by tests that time individual calls. From e94efa1384db5187230636e8b7f1dde882d97752 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 12:19:26 +0900 Subject: [PATCH 02/64] docs: health telemetry implementation plan 27 TDD-style tasks across 8 phases: core types + detector framework, six default detectors (stall, fps-drop, jitter, startup-failure, backpressure, adapter-passthrough), orchestrator + writer integration, OAK depthai logger bridge with crash-dump artifact capture, viewer server+frontend incident panel, per-adapter target_hz rollout, and a manual OAK-hardware verification checklist. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../plans/2026-04-22-health-telemetry.md | 4292 +++++++++++++++++ 1 file changed, 4292 insertions(+) create mode 100644 docs/superpowers/plans/2026-04-22-health-telemetry.md diff --git a/docs/superpowers/plans/2026-04-22-health-telemetry.md b/docs/superpowers/plans/2026-04-22-health-telemetry.md new file mode 100644 index 0000000..dc849f0 --- /dev/null +++ b/docs/superpowers/plans/2026-04-22-health-telemetry.md @@ -0,0 +1,4292 @@ +# Health Telemetry Platform Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build a sensor-agnostic, platform-level health telemetry system that continuously observes every stream during a recording, raises Sentry-style Incidents on stalls / FPS drops / jitter / startup failures / writer backpressure / adapter faults, and surfaces them live in the viewer and in `FinalizationReport.incidents` + `incidents.jsonl` post-session. + +**Architecture:** New `src/syncfield/health/` package. Streams push samples and raw `HealthEvent`s through thread-safe ingress queues into a single daemon `HealthWorker` that ticks at 20 Hz. On each tick every registered `Detector` is polled; emitted `HealthEvent`s feed an `IncidentTracker` that groups by `fingerprint`, owns open/close lifecycle, and flushes to disk. Default detectors are installed automatically; new ones can be added via `session.health.register(...)`. OAK's native depthai logger is bridged into the same channel so X_LINK_ERROR / crash / reconnect events become structured incidents with crash-dump artifacts attached. + +**Tech Stack:** Python 3.9+, `dataclasses`, `threading`, `queue.SimpleQueue`, `logging` (for depthai bridge). Frontend: existing React + TypeScript viewer. + +**Spec:** `docs/superpowers/specs/2026-04-22-health-telemetry-design.md` + +--- + +## File Structure + +### New (backend) + +``` +src/syncfield/health/ +├── __init__.py # public API +├── severity.py # Severity enum + ordering helpers +├── types.py # Incident, IncidentArtifact, IncidentSnapshot, WriterStats +├── detector.py # Detector Protocol + DetectorBase +├── registry.py # DetectorRegistry +├── tracker.py # IncidentTracker +├── worker.py # HealthWorker (daemon thread + ingress queues) +├── system.py # HealthSystem (facade users touch) +└── detectors/ + ├── __init__.py + ├── adapter_passthrough.py # AdapterEventPassthrough + ├── stream_stall.py # StreamStallDetector + ├── fps_drop.py # FpsDropDetector + ├── jitter.py # JitterDetector + ├── startup_failure.py # StartupFailureDetector + ├── backpressure.py # BackpressureDetector + └── depthai_bridge.py # DepthAILoggerBridge (soft-imports depthai) +``` + +### Modified (backend) + +- `src/syncfield/types.py` — add `severity` / `source` / `fingerprint` / `data` to `HealthEvent`; add `target_hz` to `StreamCapabilities`; add `incidents` to `FinalizationReport`. +- `src/syncfield/writer.py` — add `SessionLogWriter.log_incident()`; emit `WriterStats`. +- `src/syncfield/orchestrator.py` — construct `HealthSystem`, wire sample / health / state / writer-stats observers, start/stop worker, embed incidents in `FinalizationReport`. +- `src/syncfield/adapters/oak_camera.py` — declare `target_hz`, install `DepthAILoggerBridge`, attach `crash_dump` artifacts. +- `src/syncfield/adapters/uvc_webcam.py`, `host_audio.py`, `meta_quest_camera/*`, `ble_imu.py`, `insta360_go3s/stream.py`, `polling_sensor.py` — declare `target_hz` where known. +- `src/syncfield/viewer/state.py` — replace `HealthEntry` / `StreamSnapshot.health_count` with `IncidentSnapshot` / top-level `active_incidents` / `resolved_incidents`. +- `src/syncfield/viewer/poller.py` — subscribe to `HealthSystem` incident callbacks, populate new snapshot fields. +- `src/syncfield/viewer/server.py` — serialize incidents into WebSocket payload. + +### New (frontend) + +- `src/syncfield/viewer/frontend/src/components/incident-panel.tsx` + +### Deleted (frontend) + +- `src/syncfield/viewer/frontend/src/components/health-table.tsx` + +### Modified (frontend) + +- `src/syncfield/viewer/frontend/src/lib/types.ts` — mirror `Severity`, `IncidentSnapshot`; remove `HealthEntry`, `health_count`. +- `src/syncfield/viewer/frontend/src/components/stream-card.tsx` — severity badge. +- `src/syncfield/viewer/frontend/src/App.tsx` — mount `IncidentPanel` in place of `HealthTable`. + +### Tests + +``` +tests/unit/health/ +├── __init__.py +├── test_severity.py +├── test_incident_types.py +├── test_detector_base.py +├── test_registry.py +├── test_incident_tracker.py +├── test_health_worker.py +├── test_health_system.py +└── detectors/ + ├── __init__.py + ├── test_adapter_passthrough.py + ├── test_stream_stall.py + ├── test_fps_drop.py + ├── test_jitter.py + ├── test_startup_failure.py + ├── test_backpressure.py + └── test_depthai_bridge.py + +tests/integration/health/ +├── __init__.py +├── test_orchestrator_health_integration.py +└── test_incidents_jsonl_roundtrip.py +``` + +--- + +## Conventions used in this plan + +- Every task is **TDD**: write a failing test, verify failure, implement, verify pass, commit. +- Run tests with `pytest -v` (project `pyproject.toml` sets `testpaths=["tests"]`, `pythonpath=["src"]`). +- Each task ends with a commit using Conventional Commits (`feat:`, `refactor:`, `test:`). +- Test helpers use `tests/helpers/` where one exists; new helpers live in `tests/unit/health/_helpers.py`. +- `time.monotonic_ns()` returns an int. All `_ns` fields are ints. +- We will use `queue.SimpleQueue` for lock-free ingress (Python's own fast, unbounded, thread-safe FIFO). + +--- + +## Phase 1 — Core types & skeleton + +### Task 1: Severity enum + +**Files:** +- Create: `src/syncfield/health/__init__.py` (empty for now) +- Create: `src/syncfield/health/severity.py` +- Test: `tests/unit/health/__init__.py` (empty), `tests/unit/health/test_severity.py` + +- [ ] **Step 1: Create the (empty) package init files** + +```bash +mkdir -p src/syncfield/health tests/unit/health +touch src/syncfield/health/__init__.py tests/unit/health/__init__.py +``` + +- [ ] **Step 2: Write failing test** + +Create `tests/unit/health/test_severity.py`: + +```python +from syncfield.health.severity import Severity, max_severity + + +def test_severity_values(): + assert Severity.INFO.value == "info" + assert Severity.WARNING.value == "warning" + assert Severity.ERROR.value == "error" + assert Severity.CRITICAL.value == "critical" + + +def test_severity_ordering(): + # INFO < WARNING < ERROR < CRITICAL + order = [Severity.INFO, Severity.WARNING, Severity.ERROR, Severity.CRITICAL] + for a, b in zip(order, order[1:]): + assert a.rank < b.rank + + +def test_max_severity_picks_highest(): + assert max_severity(Severity.INFO, Severity.WARNING) == Severity.WARNING + assert max_severity(Severity.ERROR, Severity.WARNING) == Severity.ERROR + assert max_severity(Severity.CRITICAL, Severity.INFO, Severity.ERROR) == Severity.CRITICAL + + +def test_max_severity_requires_at_least_one(): + import pytest + with pytest.raises(ValueError): + max_severity() +``` + +- [ ] **Step 3: Run, confirm fail** + +```bash +pytest tests/unit/health/test_severity.py -v +``` +Expected: `ModuleNotFoundError: No module named 'syncfield.health.severity'` + +- [ ] **Step 4: Implement** + +`src/syncfield/health/severity.py`: + +```python +"""Severity levels for health events and incidents. + +Ordered INFO < WARNING < ERROR < CRITICAL. Use :func:`max_severity` to +pick the highest of several levels — incidents escalate to the max +severity of their constituent events. +""" + +from __future__ import annotations + +from enum import Enum + + +class Severity(str, Enum): + INFO = "info" + WARNING = "warning" + ERROR = "error" + CRITICAL = "critical" + + @property + def rank(self) -> int: + return _RANK[self] + + +_RANK = { + Severity.INFO: 0, + Severity.WARNING: 1, + Severity.ERROR: 2, + Severity.CRITICAL: 3, +} + + +def max_severity(*levels: Severity) -> Severity: + if not levels: + raise ValueError("max_severity requires at least one Severity") + return max(levels, key=lambda s: s.rank) +``` + +- [ ] **Step 5: Run, confirm pass** + +```bash +pytest tests/unit/health/test_severity.py -v +``` +Expected: 4 passed. + +- [ ] **Step 6: Commit** + +```bash +git add src/syncfield/health/ tests/unit/health/ +git commit -m "feat(health): add Severity enum with rank + max_severity helper" +``` + +--- + +### Task 2: Enrich HealthEvent with severity / source / fingerprint / data + +**Files:** +- Modify: `src/syncfield/types.py:250-272` (`HealthEvent` dataclass + `to_dict`) +- Test: `tests/unit/test_types.py` (add cases) + +Spec requires new fields on `HealthEvent`. We are intentionally breaking the existing format; all writes will carry the new fields and readers (viewer, session log consumers) update in later tasks. + +- [ ] **Step 1: Write failing test** + +Append to `tests/unit/test_types.py`: + +```python +from syncfield.health.severity import Severity +from syncfield.types import HealthEvent, HealthEventKind + + +def test_health_event_has_enrichment_fields_with_defaults(): + ev = HealthEvent( + stream_id="cam", + kind=HealthEventKind.ERROR, + at_ns=1_000, + detail="boom", + ) + # new fields default to safe values when caller does not set them. + assert ev.severity == Severity.INFO + assert ev.source == "unknown" + assert ev.fingerprint == "" + assert ev.data == {} + + +def test_health_event_to_dict_includes_new_fields(): + ev = HealthEvent( + stream_id="cam", + kind=HealthEventKind.ERROR, + at_ns=1_000, + detail="boom", + severity=Severity.ERROR, + source="adapter:oak", + fingerprint="cam:adapter:xlink-error", + data={"stream": "__x_0_1"}, + ) + d = ev.to_dict() + assert d["severity"] == "error" + assert d["source"] == "adapter:oak" + assert d["fingerprint"] == "cam:adapter:xlink-error" + assert d["data"] == {"stream": "__x_0_1"} +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/test_types.py -k health_event -v +``` +Expected: `TypeError` (frozen dataclass rejecting unknown kwargs) or AttributeError on `.severity`. + +- [ ] **Step 3: Implement** + +In `src/syncfield/types.py`, replace the `HealthEvent` dataclass and its `to_dict`: + +```python +from syncfield.health.severity import Severity # top of file, next to other imports + +@dataclass(frozen=True) +class HealthEvent: + """A stream reports a health observation. + + ``severity`` / ``source`` / ``fingerprint`` / ``data`` enable the + incident-tracking layer in :mod:`syncfield.health` to group many raw + events into a single Sentry-style Incident. Adapters that don't care + can leave them at their safe defaults; the platform will fill them + in before the event reaches the IncidentTracker. + """ + + stream_id: str + kind: HealthEventKind + at_ns: int + detail: str | None = None + severity: Severity = Severity.INFO + source: str = "unknown" + fingerprint: str = "" + data: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "stream_id": self.stream_id, + "kind": self.kind.value, + "at_ns": self.at_ns, + "detail": self.detail, + "severity": self.severity.value, + "source": self.source, + "fingerprint": self.fingerprint, + "data": self.data, + } +``` + +Add `from dataclasses import field` if missing. `field` is needed for the mutable default on `data`. + +- [ ] **Step 4: Run, confirm pass** + +```bash +pytest tests/unit/test_types.py -v +``` +Expected: all pass, including existing tests (defaults are backward-compatible at the Python level; only the JSON format changes). + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/types.py tests/unit/test_types.py +git commit -m "feat(types): enrich HealthEvent with severity/source/fingerprint/data" +``` + +--- + +### Task 3: WriterStats + IncidentArtifact + Incident + IncidentSnapshot + +**Files:** +- Create: `src/syncfield/health/types.py` +- Test: `tests/unit/health/test_incident_types.py` + +- [ ] **Step 1: Write failing test** + +`tests/unit/health/test_incident_types.py`: + +```python +from syncfield.health.severity import Severity +from syncfield.health.types import ( + Incident, + IncidentArtifact, + IncidentSnapshot, + WriterStats, +) +from syncfield.types import HealthEvent, HealthEventKind + + +def _ev(at_ns: int, severity: Severity = Severity.ERROR) -> HealthEvent: + return HealthEvent( + stream_id="cam", + kind=HealthEventKind.ERROR, + at_ns=at_ns, + detail="x", + severity=severity, + source="detector:stream-stall", + fingerprint="cam:stream-stall", + ) + + +def test_writer_stats_fields(): + s = WriterStats( + stream_id="cam", + at_ns=100, + queue_depth=3, + queue_capacity=16, + dropped=0, + ) + assert s.queue_fullness == 3 / 16 + assert s.stream_id == "cam" + + +def test_writer_stats_zero_capacity_is_empty(): + s = WriterStats(stream_id="cam", at_ns=0, queue_depth=0, queue_capacity=0, dropped=0) + assert s.queue_fullness == 0.0 + + +def test_incident_from_first_event_initializes_fields(): + first = _ev(100) + inc = Incident.opened_from(first, title="Stream stalled (silence 2.0s)") + assert inc.stream_id == "cam" + assert inc.fingerprint == "cam:stream-stall" + assert inc.severity == Severity.ERROR + assert inc.title == "Stream stalled (silence 2.0s)" + assert inc.opened_at_ns == 100 + assert inc.closed_at_ns is None + assert inc.event_count == 1 + assert inc.first_event == first + assert inc.last_event == first + assert inc.artifacts == [] + + +def test_incident_record_event_escalates_severity_and_updates_last(): + inc = Incident.opened_from(_ev(100, severity=Severity.WARNING), title="t") + inc.record_event(_ev(200, severity=Severity.ERROR)) + assert inc.event_count == 2 + assert inc.severity == Severity.ERROR + assert inc.last_event.at_ns == 200 + assert inc.last_event_at_ns == 200 + + +def test_incident_close(): + inc = Incident.opened_from(_ev(100), title="t") + inc.close(at_ns=500) + assert inc.closed_at_ns == 500 + assert inc.is_open is False + + +def test_incident_attach_artifact(): + inc = Incident.opened_from(_ev(100), title="t") + inc.attach(IncidentArtifact(kind="crash_dump", path="/tmp/x.json")) + assert inc.artifacts[0].kind == "crash_dump" + assert inc.artifacts[0].path == "/tmp/x.json" + + +def test_incident_snapshot_shape(): + inc = Incident.opened_from(_ev(100), title="t") + snap = IncidentSnapshot.from_incident(inc, now_ns=1_000_000_100) + assert snap.id == inc.id + assert snap.stream_id == "cam" + assert snap.severity == "error" + assert snap.is_open is True + assert snap.ago_s >= 0 +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/health/test_incident_types.py -v +``` +Expected: `ModuleNotFoundError`. + +- [ ] **Step 3: Implement** + +`src/syncfield/health/types.py`: + +```python +"""Data classes for the health/incident layer. + +These are plain, explicit structs — the :mod:`syncfield.health` runtime +mutates :class:`Incident` objects in-place on the worker thread. The +viewer receives immutable :class:`IncidentSnapshot`\\ s instead. +""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass, field +from typing import Any, List + +from syncfield.health.severity import Severity, max_severity +from syncfield.types import HealthEvent + + +@dataclass(frozen=True) +class WriterStats: + """One observation of a per-stream writer's queue.""" + + stream_id: str + at_ns: int + queue_depth: int + queue_capacity: int + dropped: int + + @property + def queue_fullness(self) -> float: + if self.queue_capacity <= 0: + return 0.0 + return self.queue_depth / self.queue_capacity + + +@dataclass(frozen=True) +class IncidentArtifact: + """A piece of evidence attached to an Incident (crash dump, log excerpt, ...).""" + + kind: str + path: str + detail: str | None = None + + def to_dict(self) -> dict[str, Any]: + return {"kind": self.kind, "path": self.path, "detail": self.detail} + + +@dataclass +class Incident: + """A grouped, open/close-tracked sequence of HealthEvents sharing a fingerprint. + + Mutable because the worker thread updates ``last_event`` / ``event_count`` + / ``severity`` on every matching event. The viewer never sees this + class directly — it reads :class:`IncidentSnapshot` instead. + """ + + id: str + stream_id: str + fingerprint: str + title: str + severity: Severity + source: str + opened_at_ns: int + closed_at_ns: int | None + last_event_at_ns: int + event_count: int + first_event: HealthEvent + last_event: HealthEvent + artifacts: List[IncidentArtifact] = field(default_factory=list) + data: dict[str, Any] = field(default_factory=dict) + + @classmethod + def opened_from(cls, event: HealthEvent, *, title: str) -> "Incident": + return cls( + id=uuid.uuid4().hex, + stream_id=event.stream_id, + fingerprint=event.fingerprint, + title=title, + severity=event.severity, + source=event.source, + opened_at_ns=event.at_ns, + closed_at_ns=None, + last_event_at_ns=event.at_ns, + event_count=1, + first_event=event, + last_event=event, + ) + + @property + def is_open(self) -> bool: + return self.closed_at_ns is None + + def record_event(self, event: HealthEvent) -> None: + self.event_count += 1 + self.last_event = event + self.last_event_at_ns = event.at_ns + self.severity = max_severity(self.severity, event.severity) + + def close(self, *, at_ns: int) -> None: + self.closed_at_ns = at_ns + + def attach(self, artifact: IncidentArtifact) -> None: + self.artifacts.append(artifact) + + def to_dict(self) -> dict[str, Any]: + return { + "id": self.id, + "stream_id": self.stream_id, + "fingerprint": self.fingerprint, + "title": self.title, + "severity": self.severity.value, + "source": self.source, + "opened_at_ns": self.opened_at_ns, + "closed_at_ns": self.closed_at_ns, + "last_event_at_ns": self.last_event_at_ns, + "event_count": self.event_count, + "first_event": self.first_event.to_dict(), + "last_event": self.last_event.to_dict(), + "artifacts": [a.to_dict() for a in self.artifacts], + "data": self.data, + } + + +@dataclass(frozen=True) +class IncidentSnapshot: + """Read-only view of an Incident, for the viewer's WebSocket payload.""" + + id: str + stream_id: str + fingerprint: str + title: str + severity: str + source: str + opened_at_ns: int + closed_at_ns: int | None + event_count: int + detail: str | None + ago_s: float + artifacts: List[dict] + + @property + def is_open(self) -> bool: + return self.closed_at_ns is None + + @classmethod + def from_incident(cls, inc: Incident, *, now_ns: int) -> "IncidentSnapshot": + anchor = inc.closed_at_ns if inc.closed_at_ns is not None else inc.last_event_at_ns + ago_s = max(0.0, (now_ns - anchor) / 1e9) + return cls( + id=inc.id, + stream_id=inc.stream_id, + fingerprint=inc.fingerprint, + title=inc.title, + severity=inc.severity.value, + source=inc.source, + opened_at_ns=inc.opened_at_ns, + closed_at_ns=inc.closed_at_ns, + event_count=inc.event_count, + detail=inc.last_event.detail, + ago_s=ago_s, + artifacts=[a.to_dict() for a in inc.artifacts], + ) +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +pytest tests/unit/health/test_incident_types.py -v +``` +Expected: 7 passed. + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/types.py tests/unit/health/test_incident_types.py +git commit -m "feat(health): add WriterStats, Incident, IncidentArtifact, IncidentSnapshot" +``` + +--- + +### Task 4: Detector Protocol + DetectorBase + +**Files:** +- Create: `src/syncfield/health/detector.py` +- Test: `tests/unit/health/test_detector_base.py` + +- [ ] **Step 1: Write failing test** + +`tests/unit/health/test_detector_base.py`: + +```python +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.types import WriterStats +from syncfield.types import HealthEvent, HealthEventKind, SampleEvent, SessionState + + +class NoopDetector(DetectorBase): + name = "noop" + default_severity = Severity.WARNING + + +def test_detector_base_defaults_are_noops(): + d = NoopDetector() + # All observers accept calls without raising. + d.observe_sample("cam", SampleEvent(stream_id="cam", frame_number=1, capture_ns=100)) + d.observe_health("cam", HealthEvent(stream_id="cam", kind=HealthEventKind.WARNING, at_ns=1)) + d.observe_state(SessionState.IDLE, SessionState.CONNECTED) + d.observe_writer_stats("cam", WriterStats("cam", 1, 0, 0, 0)) + # tick yields nothing by default. + assert list(d.tick(now_ns=100)) == [] + # close_condition defaults to True so pass-through close can rely on it. + from syncfield.health.types import Incident + ev = HealthEvent(stream_id="cam", kind=HealthEventKind.WARNING, at_ns=1) + inc = Incident.opened_from(ev, title="x") + assert isinstance(d.close_condition(inc, now_ns=10), bool) + + +def test_detector_base_requires_name_and_severity(): + import pytest + + with pytest.raises(TypeError): + DetectorBase() # abstract base: name / default_severity unset on the class +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/health/test_detector_base.py -v +``` +Expected: `ModuleNotFoundError`. + +- [ ] **Step 3: Implement** + +`src/syncfield/health/detector.py`: + +```python +"""Detector protocol + base class. + +A :class:`Detector` observes the stream (samples, adapter health events, +session state, writer stats) and may emit :class:`HealthEvent` on each +``tick()``. The :class:`IncidentTracker` groups emitted events by +fingerprint, opens incidents, and consults ``close_condition`` to know +when an open incident should resolve. + +Most detectors subclass :class:`DetectorBase` and override only the +observe/tick hooks they care about; the base provides safe no-op +defaults for the rest. +""" + +from __future__ import annotations + +from typing import Iterable, Iterator, Protocol, runtime_checkable + +from syncfield.health.severity import Severity +from syncfield.health.types import Incident, WriterStats +from syncfield.types import HealthEvent, SampleEvent, SessionState + + +@runtime_checkable +class Detector(Protocol): + name: str + default_severity: Severity + + def observe_sample(self, stream_id: str, sample: SampleEvent) -> None: ... + def observe_health(self, stream_id: str, event: HealthEvent) -> None: ... + def observe_state(self, old: SessionState, new: SessionState) -> None: ... + def observe_writer_stats(self, stream_id: str, stats: WriterStats) -> None: ... + def tick(self, now_ns: int) -> Iterable[HealthEvent]: ... + def close_condition(self, incident: Incident, now_ns: int) -> bool: ... + + +class DetectorBase: + """No-op defaults for every Detector hook. + + Subclasses set ``name`` and ``default_severity`` at the class level + and override only the hooks that matter for their rule. + """ + + name: str + default_severity: Severity + + def __init_subclass__(cls, **kwargs: object) -> None: + super().__init_subclass__(**kwargs) + # Guard against subclasses that forget to set required class attrs. + for attr in ("name", "default_severity"): + if not hasattr(cls, attr): + raise TypeError( + f"Detector subclass {cls.__name__} must set class attribute '{attr}'" + ) + + def __new__(cls, *args: object, **kwargs: object) -> "DetectorBase": + if cls is DetectorBase: + raise TypeError("DetectorBase is abstract; subclass it") + return super().__new__(cls) + + def observe_sample(self, stream_id: str, sample: SampleEvent) -> None: + pass + + def observe_health(self, stream_id: str, event: HealthEvent) -> None: + pass + + def observe_state(self, old: SessionState, new: SessionState) -> None: + pass + + def observe_writer_stats(self, stream_id: str, stats: WriterStats) -> None: + pass + + def tick(self, now_ns: int) -> Iterator[HealthEvent]: + return iter(()) + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + # Conservative default: keep open. Subclasses override to close. + return False +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +pytest tests/unit/health/test_detector_base.py -v +``` +Expected: 2 passed. + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/detector.py tests/unit/health/test_detector_base.py +git commit -m "feat(health): add Detector protocol + DetectorBase" +``` + +--- + +### Task 5: DetectorRegistry + +**Files:** +- Create: `src/syncfield/health/registry.py` +- Test: `tests/unit/health/test_registry.py` + +- [ ] **Step 1: Write failing test** + +`tests/unit/health/test_registry.py`: + +```python +import pytest + +from syncfield.health.detector import DetectorBase +from syncfield.health.registry import DetectorRegistry +from syncfield.health.severity import Severity + + +class Det(DetectorBase): + name = "d1" + default_severity = Severity.WARNING + + +class Det2(DetectorBase): + name = "d2" + default_severity = Severity.ERROR + + +def test_register_and_iterate(): + reg = DetectorRegistry() + d1 = Det() + d2 = Det2() + reg.register(d1) + reg.register(d2) + assert list(reg) == [d1, d2] + + +def test_register_duplicate_name_raises(): + reg = DetectorRegistry() + reg.register(Det()) + with pytest.raises(ValueError, match="already registered"): + reg.register(Det()) + + +def test_unregister_removes_by_name(): + reg = DetectorRegistry() + d1 = Det() + reg.register(d1) + reg.unregister("d1") + assert list(reg) == [] + + +def test_unregister_unknown_is_noop(): + reg = DetectorRegistry() + reg.unregister("nope") # does not raise +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/health/test_registry.py -v +``` +Expected: `ModuleNotFoundError`. + +- [ ] **Step 3: Implement** + +`src/syncfield/health/registry.py`: + +```python +"""Registry of active Detectors for a session.""" + +from __future__ import annotations + +from typing import Iterator, List + +from syncfield.health.detector import Detector + + +class DetectorRegistry: + def __init__(self) -> None: + self._detectors: List[Detector] = [] + + def register(self, detector: Detector) -> None: + if any(d.name == detector.name for d in self._detectors): + raise ValueError(f"Detector '{detector.name}' is already registered") + self._detectors.append(detector) + + def unregister(self, name: str) -> None: + self._detectors = [d for d in self._detectors if d.name != name] + + def __iter__(self) -> Iterator[Detector]: + return iter(list(self._detectors)) + + def __len__(self) -> int: + return len(self._detectors) +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +pytest tests/unit/health/test_registry.py -v +``` +Expected: 4 passed. + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/registry.py tests/unit/health/test_registry.py +git commit -m "feat(health): add DetectorRegistry" +``` + +--- + +### Task 6: IncidentTracker + +**Files:** +- Create: `src/syncfield/health/tracker.py` +- Test: `tests/unit/health/test_incident_tracker.py` + +`IncidentTracker` owns the open/close lifecycle for all incidents in a session. It never runs in user code directly — the `HealthWorker` owns and drives it. + +- [ ] **Step 1: Write failing test** + +`tests/unit/health/test_incident_tracker.py`: + +```python +from typing import List + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.tracker import IncidentTracker +from syncfield.health.types import Incident +from syncfield.types import HealthEvent, HealthEventKind + + +def _ev(at_ns: int, fingerprint: str = "cam:stall", severity: Severity = Severity.ERROR, + detail: str = "x") -> HealthEvent: + return HealthEvent( + stream_id="cam", + kind=HealthEventKind.ERROR, + at_ns=at_ns, + detail=detail, + severity=severity, + source="detector:stream-stall", + fingerprint=fingerprint, + ) + + +class AlwaysCloseAfter(DetectorBase): + name = "stream-stall" + default_severity = Severity.ERROR + + def __init__(self, close_after_ns: int) -> None: + self._close_after = close_after_ns + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + return now_ns - incident.last_event_at_ns >= self._close_after + + +def test_tracker_opens_incident_on_first_matching_event(): + tr = IncidentTracker() + tr.bind_detector(AlwaysCloseAfter(close_after_ns=1000)) + opened: List[Incident] = [] + tr.on_opened = opened.append + + tr.ingest(_ev(100)) + + assert len(tr.open_incidents()) == 1 + assert opened and opened[0].stream_id == "cam" + + +def test_tracker_groups_same_fingerprint_into_one_incident(): + tr = IncidentTracker() + tr.bind_detector(AlwaysCloseAfter(close_after_ns=1_000_000_000)) + + tr.ingest(_ev(100, severity=Severity.WARNING)) + tr.ingest(_ev(200, severity=Severity.ERROR)) # escalate + tr.ingest(_ev(300, severity=Severity.ERROR)) + + opens = tr.open_incidents() + assert len(opens) == 1 + inc = opens[0] + assert inc.event_count == 3 + assert inc.severity == Severity.ERROR + assert inc.last_event_at_ns == 300 + + +def test_tracker_closes_incident_when_detector_close_condition_fires(): + tr = IncidentTracker() + tr.bind_detector(AlwaysCloseAfter(close_after_ns=500)) + closed: List[Incident] = [] + tr.on_closed = closed.append + + tr.ingest(_ev(100)) + tr.tick(now_ns=200) # 100 ns since last event, not yet + assert tr.resolved_incidents() == [] + + tr.tick(now_ns=700) # 600 ns since last event → close + assert len(tr.resolved_incidents()) == 1 + assert tr.open_incidents() == [] + assert closed and closed[0].closed_at_ns == 700 + + +def test_tracker_reopens_after_close_on_same_fingerprint(): + tr = IncidentTracker() + tr.bind_detector(AlwaysCloseAfter(close_after_ns=100)) + + tr.ingest(_ev(100)) + tr.tick(now_ns=500) # closed + assert tr.open_incidents() == [] + + tr.ingest(_ev(1000)) # new incident, new id + opens = tr.open_incidents() + assert len(opens) == 1 + assert len(tr.resolved_incidents()) == 1 + assert opens[0].id != tr.resolved_incidents()[0].id + + +def test_tracker_unbound_fingerprint_falls_back_to_passthrough_close(): + # When an event arrives with a fingerprint whose detector is not bound, + # the tracker still groups it, using the default passthrough close + # window (30s of quiet). + tr = IncidentTracker(passthrough_close_ns=500) + + tr.ingest(_ev(100, fingerprint="cam:adapter:xlink")) + tr.tick(now_ns=400) + assert tr.open_incidents() + tr.tick(now_ns=1000) # 900 ns since last event → closes + assert tr.resolved_incidents() + + +def test_tracker_flush_callbacks_fire_on_update_too(): + tr = IncidentTracker() + tr.bind_detector(AlwaysCloseAfter(close_after_ns=1_000_000_000)) + updates: List[Incident] = [] + tr.on_updated = updates.append + + tr.ingest(_ev(100)) # opens + tr.ingest(_ev(200)) # updates + tr.ingest(_ev(300)) # updates + assert len(updates) == 2 +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/health/test_incident_tracker.py -v +``` +Expected: `ModuleNotFoundError`. + +- [ ] **Step 3: Implement** + +`src/syncfield/health/tracker.py`: + +```python +"""IncidentTracker — groups HealthEvents into Incidents and manages open/close. + +Runs on the HealthWorker thread. Public methods are *not* thread-safe on +their own; the worker serializes access. +""" + +from __future__ import annotations + +from typing import Callable, Dict, List, Optional + +from syncfield.health.detector import Detector +from syncfield.health.types import Incident +from syncfield.types import HealthEvent + +Callback = Callable[[Incident], None] + + +class IncidentTracker: + def __init__(self, passthrough_close_ns: int = 30 * 1_000_000_000) -> None: + self._by_fingerprint: Dict[str, Incident] = {} + self._resolved: List[Incident] = [] + self._detectors_by_name: Dict[str, Detector] = {} + self._passthrough_close_ns = passthrough_close_ns + + self.on_opened: Optional[Callback] = None + self.on_updated: Optional[Callback] = None + self.on_closed: Optional[Callback] = None + + # --- detector wiring ------------------------------------------------- + + def bind_detector(self, detector: Detector) -> None: + self._detectors_by_name[detector.name] = detector + + # --- event ingestion ------------------------------------------------- + + def ingest(self, event: HealthEvent) -> None: + open_inc = self._by_fingerprint.get(event.fingerprint) + if open_inc is None: + inc = Incident.opened_from(event, title=_title_from(event)) + self._by_fingerprint[event.fingerprint] = inc + self._fire(self.on_opened, inc) + return + open_inc.record_event(event) + self._fire(self.on_updated, open_inc) + + # --- tick — evaluate close conditions -------------------------------- + + def tick(self, now_ns: int) -> None: + to_close: List[str] = [] + for fp, inc in self._by_fingerprint.items(): + detector = self._detector_for(inc) + if detector is not None: + should_close = detector.close_condition(inc, now_ns) + else: + should_close = (now_ns - inc.last_event_at_ns) >= self._passthrough_close_ns + if should_close: + to_close.append(fp) + + for fp in to_close: + inc = self._by_fingerprint.pop(fp) + inc.close(at_ns=now_ns) + self._resolved.append(inc) + self._fire(self.on_closed, inc) + + def close_all(self, *, at_ns: int) -> None: + """Used at session stop to resolve any still-open incidents.""" + for fp in list(self._by_fingerprint.keys()): + inc = self._by_fingerprint.pop(fp) + inc.close(at_ns=at_ns) + self._resolved.append(inc) + self._fire(self.on_closed, inc) + + # --- read-only views ------------------------------------------------- + + def open_incidents(self) -> List[Incident]: + return list(self._by_fingerprint.values()) + + def resolved_incidents(self) -> List[Incident]: + return list(self._resolved) + + # --- helpers --------------------------------------------------------- + + def _detector_for(self, inc: Incident) -> Optional[Detector]: + # Fingerprint convention: ":[:suffix]". + parts = inc.fingerprint.split(":", 2) + if len(parts) < 2: + return None + return self._detectors_by_name.get(parts[1]) + + @staticmethod + def _fire(cb: Optional[Callback], inc: Incident) -> None: + if cb is not None: + cb(inc) + + +def _title_from(event: HealthEvent) -> str: + # Prefer the first event's detail as the title; fall back to + # ": " if the detail is missing. + if event.detail: + return event.detail + return f"{event.source}: {event.fingerprint}" +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +pytest tests/unit/health/test_incident_tracker.py -v +``` +Expected: 6 passed. + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/tracker.py tests/unit/health/test_incident_tracker.py +git commit -m "feat(health): add IncidentTracker with fingerprint grouping and close conditions" +``` + +--- + +### Task 7: HealthWorker (thread + ingress queues) + +**Files:** +- Create: `src/syncfield/health/worker.py` +- Test: `tests/unit/health/test_health_worker.py` + +The worker owns the thread, ingress queues, detector polling, and tracker. It is started on session `start()` and stopped on `stop()`. + +- [ ] **Step 1: Write failing test** + +`tests/unit/health/test_health_worker.py`: + +```python +import threading +import time + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.tracker import IncidentTracker +from syncfield.health.types import WriterStats +from syncfield.health.worker import HealthWorker +from syncfield.types import HealthEvent, HealthEventKind, SampleEvent, SessionState + + +class RecordingDetector(DetectorBase): + name = "recorder" + default_severity = Severity.INFO + + def __init__(self) -> None: + self.samples = [] + self.healths = [] + self.states = [] + self.writer_stats = [] + self.ticks = 0 + + def observe_sample(self, stream_id, sample): + self.samples.append((stream_id, sample.capture_ns)) + + def observe_health(self, stream_id, event): + self.healths.append((stream_id, event.at_ns)) + + def observe_state(self, old, new): + self.states.append((old, new)) + + def observe_writer_stats(self, stream_id, stats): + self.writer_stats.append((stream_id, stats.queue_depth)) + + def tick(self, now_ns): + self.ticks += 1 + return iter(()) + + +def _wait_until(pred, timeout=1.0, interval=0.01): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if pred(): + return True + time.sleep(interval) + return False + + +def test_worker_drains_all_ingress_queues_on_tick(): + tr = IncidentTracker() + det = RecordingDetector() + tr.bind_detector(det) + w = HealthWorker(tracker=tr, detectors=[det], tick_hz=100) + + w.start() + try: + w.push_sample("cam", SampleEvent(stream_id="cam", frame_number=1, capture_ns=42)) + w.push_health("cam", HealthEvent(stream_id="cam", kind=HealthEventKind.WARNING, at_ns=1)) + w.push_state(SessionState.IDLE, SessionState.CONNECTED) + w.push_writer_stats("cam", WriterStats("cam", 1, 2, 16, 0)) + + assert _wait_until(lambda: det.samples and det.healths and det.states and det.writer_stats) + finally: + w.stop() + + assert det.samples[0] == ("cam", 42) + assert det.healths[0] == ("cam", 1) + assert det.states[0] == (SessionState.IDLE, SessionState.CONNECTED) + assert det.writer_stats[0] == ("cam", 2) + + +def test_worker_ticks_at_roughly_configured_rate(): + tr = IncidentTracker() + det = RecordingDetector() + w = HealthWorker(tracker=tr, detectors=[det], tick_hz=50) + w.start() + try: + time.sleep(0.2) # ~10 ticks + finally: + w.stop() + # Loose bound to avoid flakiness under loaded CI. + assert det.ticks >= 5 + + +def test_worker_feeds_detector_tick_output_into_tracker(): + class EmitsOneAndDone(DetectorBase): + name = "emit" + default_severity = Severity.WARNING + + def __init__(self): + self.fired = False + + def tick(self, now_ns): + if self.fired: + return iter(()) + self.fired = True + return iter([HealthEvent( + stream_id="cam", kind=HealthEventKind.WARNING, at_ns=now_ns, + detail="synthetic", severity=Severity.WARNING, + source="detector:emit", fingerprint="cam:emit", + )]) + + def close_condition(self, inc, now_ns): + return False + + tr = IncidentTracker() + det = EmitsOneAndDone() + tr.bind_detector(det) + w = HealthWorker(tracker=tr, detectors=[det], tick_hz=100) + w.start() + try: + assert _wait_until(lambda: len(tr.open_incidents()) == 1) + finally: + w.stop() + + +def test_worker_stop_is_idempotent(): + tr = IncidentTracker() + det = RecordingDetector() + w = HealthWorker(tracker=tr, detectors=[det], tick_hz=50) + w.start() + w.stop() + w.stop() # does not raise +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/health/test_health_worker.py -v +``` +Expected: `ModuleNotFoundError`. + +- [ ] **Step 3: Implement** + +`src/syncfield/health/worker.py`: + +```python +"""HealthWorker — the dedicated thread that drives detectors + tracker. + +Capture threads push samples / health events / state transitions / +writer stats into :class:`queue.SimpleQueue`\\ s. The worker drains them +every tick, fans out to each registered Detector, runs each Detector's +``tick`` to emit synthetic events, and feeds everything into the +IncidentTracker. +""" + +from __future__ import annotations + +import queue +import threading +import time +from dataclasses import dataclass +from typing import Iterable, List, Tuple + +from syncfield.health.detector import Detector +from syncfield.health.tracker import IncidentTracker +from syncfield.health.types import WriterStats +from syncfield.types import HealthEvent, SampleEvent, SessionState + + +@dataclass(frozen=True) +class _SampleMsg: + stream_id: str + sample: SampleEvent + + +@dataclass(frozen=True) +class _HealthMsg: + stream_id: str + event: HealthEvent + + +@dataclass(frozen=True) +class _StateMsg: + old: SessionState + new: SessionState + + +@dataclass(frozen=True) +class _WriterStatsMsg: + stream_id: str + stats: WriterStats + + +class HealthWorker: + def __init__( + self, + *, + tracker: IncidentTracker, + detectors: Iterable[Detector], + tick_hz: float = 20.0, + ) -> None: + self._tracker = tracker + self._detectors: List[Detector] = list(detectors) + self._tick_interval = 1.0 / tick_hz + + self._samples: "queue.SimpleQueue[_SampleMsg]" = queue.SimpleQueue() + self._healths: "queue.SimpleQueue[_HealthMsg]" = queue.SimpleQueue() + self._states: "queue.SimpleQueue[_StateMsg]" = queue.SimpleQueue() + self._writer_stats: "queue.SimpleQueue[_WriterStatsMsg]" = queue.SimpleQueue() + + self._stop = threading.Event() + self._thread: threading.Thread | None = None + + # --- ingress (called from capture threads) --------------------------- + + def push_sample(self, stream_id: str, sample: SampleEvent) -> None: + self._samples.put(_SampleMsg(stream_id, sample)) + + def push_health(self, stream_id: str, event: HealthEvent) -> None: + self._healths.put(_HealthMsg(stream_id, event)) + + def push_state(self, old: SessionState, new: SessionState) -> None: + self._states.put(_StateMsg(old, new)) + + def push_writer_stats(self, stream_id: str, stats: WriterStats) -> None: + self._writer_stats.put(_WriterStatsMsg(stream_id, stats)) + + # --- lifecycle ------------------------------------------------------- + + def start(self) -> None: + if self._thread is not None and self._thread.is_alive(): + return + self._stop.clear() + self._thread = threading.Thread( + target=self._run, name="syncfield-health", daemon=True + ) + self._thread.start() + + def stop(self) -> None: + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=2.0) + self._thread = None + + # --- main loop ------------------------------------------------------- + + def _run(self) -> None: + next_deadline = time.monotonic() + while not self._stop.is_set(): + self._drain_once() + self._fire_detector_ticks() + self._tracker.tick(now_ns=time.monotonic_ns()) + + next_deadline += self._tick_interval + sleep_for = next_deadline - time.monotonic() + if sleep_for > 0: + # Use Event.wait so stop() can interrupt immediately. + self._stop.wait(timeout=sleep_for) + else: + # Running behind; reset the schedule anchor. + next_deadline = time.monotonic() + + # Drain any straggling messages after stop so tests see a consistent state. + self._drain_once() + + def _drain_once(self) -> None: + for msg in _drain_queue(self._samples): + for d in self._detectors: + d.observe_sample(msg.stream_id, msg.sample) + for msg in _drain_queue(self._healths): + for d in self._detectors: + d.observe_health(msg.stream_id, msg.event) + self._tracker.ingest(msg.event) + for msg in _drain_queue(self._states): + for d in self._detectors: + d.observe_state(msg.old, msg.new) + for msg in _drain_queue(self._writer_stats): + for d in self._detectors: + d.observe_writer_stats(msg.stream_id, msg.stats) + + def _fire_detector_ticks(self) -> None: + now = time.monotonic_ns() + for d in self._detectors: + for event in d.tick(now): + self._tracker.ingest(event) + + +def _drain_queue(q: "queue.SimpleQueue") -> List: + out = [] + while True: + try: + out.append(q.get_nowait()) + except queue.Empty: + return out +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +pytest tests/unit/health/test_health_worker.py -v +``` +Expected: 4 passed. + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/worker.py tests/unit/health/test_health_worker.py +git commit -m "feat(health): add HealthWorker (daemon thread with ingress queues and tick loop)" +``` + +--- + +### Task 8: HealthSystem facade + +**Files:** +- Create: `src/syncfield/health/system.py` +- Modify: `src/syncfield/health/__init__.py` — public API exports +- Test: `tests/unit/health/test_health_system.py` + +- [ ] **Step 1: Write failing test** + +`tests/unit/health/test_health_system.py`: + +```python +from syncfield.health import HealthSystem, Severity +from syncfield.health.detector import DetectorBase +from syncfield.types import HealthEvent, HealthEventKind, SampleEvent, SessionState + + +class Custom(DetectorBase): + name = "custom" + default_severity = Severity.WARNING + + +def test_health_system_boots_and_accepts_inputs(): + hs = HealthSystem() + hs.start() + try: + hs.observe_sample("cam", SampleEvent(stream_id="cam", frame_number=1, capture_ns=1)) + hs.observe_health("cam", HealthEvent(stream_id="cam", kind=HealthEventKind.WARNING, at_ns=1)) + hs.observe_state(SessionState.IDLE, SessionState.CONNECTED) + finally: + hs.stop() + + +def test_health_system_register_and_unregister(): + hs = HealthSystem() + d = Custom() + hs.register(d) + assert any(x.name == "custom" for x in hs.iter_detectors()) + hs.unregister("custom") + assert not any(x.name == "custom" for x in hs.iter_detectors()) + + +def test_health_system_installs_default_detectors(): + hs = HealthSystem() + names = {d.name for d in hs.iter_detectors()} + for expected in ( + "adapter-passthrough", + "stream-stall", + "fps-drop", + "jitter", + "startup-failure", + "backpressure", + ): + assert expected in names, f"missing default detector: {expected}" + + +def test_health_system_callbacks_fire_on_open_and_close(): + hs = HealthSystem(passthrough_close_ns=1) # close instantly for the test + opened, closed = [], [] + hs.on_incident_opened = opened.append + hs.on_incident_closed = closed.append + + hs.start() + try: + hs.observe_health("cam", HealthEvent( + stream_id="cam", kind=HealthEventKind.ERROR, at_ns=1, + severity=Severity.ERROR, source="adapter:test", + fingerprint="cam:adapter:xlink-error", + )) + import time + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline: + if opened and closed: + break + time.sleep(0.02) + finally: + hs.stop() + assert opened, "incident was not opened" + assert closed, "incident was not closed" +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/health/test_health_system.py -v +``` +Expected: `ImportError: cannot import name 'HealthSystem'`. + +- [ ] **Step 3: Implement** + +`src/syncfield/health/system.py`: + +```python +"""HealthSystem — the single handle the orchestrator + user code touch.""" + +from __future__ import annotations + +from typing import Callable, Iterable, Iterator, Optional + +from syncfield.health.detector import Detector +from syncfield.health.detectors.adapter_passthrough import AdapterEventPassthrough +from syncfield.health.detectors.backpressure import BackpressureDetector +from syncfield.health.detectors.fps_drop import FpsDropDetector +from syncfield.health.detectors.jitter import JitterDetector +from syncfield.health.detectors.startup_failure import StartupFailureDetector +from syncfield.health.detectors.stream_stall import StreamStallDetector +from syncfield.health.registry import DetectorRegistry +from syncfield.health.tracker import IncidentTracker +from syncfield.health.types import Incident, WriterStats +from syncfield.health.worker import HealthWorker +from syncfield.types import HealthEvent, SampleEvent, SessionState + + +class HealthSystem: + """Composes Registry + Tracker + Worker into a single user-facing facade.""" + + def __init__( + self, + *, + tick_hz: float = 20.0, + passthrough_close_ns: int = 30 * 1_000_000_000, + ) -> None: + self._registry = DetectorRegistry() + self._tracker = IncidentTracker(passthrough_close_ns=passthrough_close_ns) + self._worker: Optional[HealthWorker] = None + self._tick_hz = tick_hz + + self.on_incident_opened: Optional[Callable[[Incident], None]] = None + self.on_incident_updated: Optional[Callable[[Incident], None]] = None + self.on_incident_closed: Optional[Callable[[Incident], None]] = None + + self._tracker.on_opened = lambda inc: self._fire("on_incident_opened", inc) + self._tracker.on_updated = lambda inc: self._fire("on_incident_updated", inc) + self._tracker.on_closed = lambda inc: self._fire("on_incident_closed", inc) + + self._install_default_detectors() + + # --- registry -------------------------------------------------------- + + def register(self, detector: Detector) -> None: + self._registry.register(detector) + self._tracker.bind_detector(detector) + + def unregister(self, name: str) -> None: + self._registry.unregister(name) + + def iter_detectors(self) -> Iterator[Detector]: + return iter(self._registry) + + # --- observer inputs ------------------------------------------------- + + def observe_sample(self, stream_id: str, sample: SampleEvent) -> None: + if self._worker is not None: + self._worker.push_sample(stream_id, sample) + + def observe_health(self, stream_id: str, event: HealthEvent) -> None: + if self._worker is not None: + self._worker.push_health(stream_id, event) + + def observe_state(self, old: SessionState, new: SessionState) -> None: + if self._worker is not None: + self._worker.push_state(old, new) + + def observe_writer_stats(self, stream_id: str, stats: WriterStats) -> None: + if self._worker is not None: + self._worker.push_writer_stats(stream_id, stats) + + # --- lifecycle ------------------------------------------------------- + + def start(self) -> None: + self._worker = HealthWorker( + tracker=self._tracker, + detectors=list(self._registry), + tick_hz=self._tick_hz, + ) + self._worker.start() + + def stop(self, *, close_open_incidents: bool = True, now_ns: Optional[int] = None) -> None: + if self._worker is not None: + self._worker.stop() + self._worker = None + if close_open_incidents: + import time + self._tracker.close_all(at_ns=now_ns if now_ns is not None else time.monotonic_ns()) + + # --- read-only views ------------------------------------------------- + + def open_incidents(self) -> Iterable[Incident]: + return self._tracker.open_incidents() + + def resolved_incidents(self) -> Iterable[Incident]: + return self._tracker.resolved_incidents() + + # --- helpers --------------------------------------------------------- + + def _install_default_detectors(self) -> None: + self.register(AdapterEventPassthrough()) + self.register(StreamStallDetector()) + self.register(FpsDropDetector()) + self.register(JitterDetector()) + self.register(StartupFailureDetector()) + self.register(BackpressureDetector()) + + def _fire(self, attr: str, inc: Incident) -> None: + cb = getattr(self, attr, None) + if cb is not None: + cb(inc) +``` + +Public API in `src/syncfield/health/__init__.py`: + +```python +"""syncfield.health — platform health telemetry.""" + +from syncfield.health.detector import Detector, DetectorBase +from syncfield.health.registry import DetectorRegistry +from syncfield.health.severity import Severity, max_severity +from syncfield.health.system import HealthSystem +from syncfield.health.tracker import IncidentTracker +from syncfield.health.types import ( + Incident, + IncidentArtifact, + IncidentSnapshot, + WriterStats, +) + +__all__ = [ + "Detector", + "DetectorBase", + "DetectorRegistry", + "HealthSystem", + "Incident", + "IncidentArtifact", + "IncidentSnapshot", + "IncidentTracker", + "Severity", + "WriterStats", + "max_severity", +] +``` + +**Note**: because `HealthSystem._install_default_detectors` imports all six default detector modules, those modules must exist *before* this task's tests can pass. Implement Tasks 9–14 first, or inline placeholder classes temporarily. **Correct order**: skip this task's implementation step (the test) until after Tasks 9–14. Proceed to Task 9 now; return here for the implementation + verification after the detectors exist. + +- [ ] **Step 4: Defer implementation — skip to Task 9** + +Revisit after Tasks 9–14 land. Run then: + +```bash +pytest tests/unit/health/test_health_system.py -v +``` +Expected: 4 passed. + +- [ ] **Step 5: Commit (after Tasks 9–14 complete)** + +```bash +git add src/syncfield/health/system.py src/syncfield/health/__init__.py tests/unit/health/test_health_system.py +git commit -m "feat(health): add HealthSystem facade with default detector install + lifecycle" +``` + +--- + +## Phase 2 — Platform detectors + +### Task 9: AdapterEventPassthrough + +**Files:** +- Create: `src/syncfield/health/detectors/__init__.py` (empty) +- Create: `src/syncfield/health/detectors/adapter_passthrough.py` +- Test: `tests/unit/health/detectors/__init__.py` (empty), `tests/unit/health/detectors/test_adapter_passthrough.py` + +The passthrough does not emit events from `tick()`; it exists so adapter-emitted events that start with `source="adapter:..."` (fingerprints like `cam:adapter:xlink-error`) still have a Detector for the tracker to consult on close — using a stale-quiet window. + +- [ ] **Step 1: Write failing test** + +```python +# tests/unit/health/detectors/test_adapter_passthrough.py +from syncfield.health.detectors.adapter_passthrough import AdapterEventPassthrough +from syncfield.health.severity import Severity +from syncfield.health.types import Incident +from syncfield.types import HealthEvent, HealthEventKind + + +def _adapter_ev(at_ns: int) -> HealthEvent: + return HealthEvent( + stream_id="cam", + kind=HealthEventKind.ERROR, + at_ns=at_ns, + detail="x", + severity=Severity.ERROR, + source="adapter:oak", + fingerprint="cam:adapter:xlink-error", + ) + + +def test_tick_emits_nothing(): + d = AdapterEventPassthrough() + assert list(d.tick(now_ns=1000)) == [] + + +def test_close_condition_respects_quiet_window(): + d = AdapterEventPassthrough(quiet_ns=500) + inc = Incident.opened_from(_adapter_ev(100), title="x") + assert d.close_condition(inc, now_ns=400) is False # 300 < 500 + inc.record_event(_adapter_ev(900)) + assert d.close_condition(inc, now_ns=1000) is False # 100 < 500 + assert d.close_condition(inc, now_ns=1500) is True # 600 >= 500 +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/health/detectors/test_adapter_passthrough.py -v +``` +Expected: `ModuleNotFoundError`. + +- [ ] **Step 3: Implement** + +`src/syncfield/health/detectors/adapter_passthrough.py`: + +```python +"""Passthrough detector: owns close semantics for adapter-emitted events. + +Fingerprint convention ``:adapter:`` routes to this +detector in the tracker. It never emits synthetic events; its only job +is saying "if no new adapter event arrived for N seconds, close the +incident". +""" + +from __future__ import annotations + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.types import Incident + + +class AdapterEventPassthrough(DetectorBase): + name = "adapter" + default_severity = Severity.WARNING + + def __init__(self, quiet_ns: int = 30 * 1_000_000_000) -> None: + self._quiet_ns = quiet_ns + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + return (now_ns - incident.last_event_at_ns) >= self._quiet_ns +``` + +**Note**: the fingerprint middle-token is `adapter`, not the detector's own name, so the tracker's `_detector_for` lookup (`parts[1]`) finds this class by `name="adapter"`. + +- [ ] **Step 4: Run, confirm pass** + +```bash +pytest tests/unit/health/detectors/test_adapter_passthrough.py -v +``` +Expected: 2 passed. + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/detectors/ tests/unit/health/detectors/ +git commit -m "feat(health): add AdapterEventPassthrough detector" +``` + +--- + +### Task 10: StreamStallDetector + +**Files:** +- Create: `src/syncfield/health/detectors/stream_stall.py` +- Test: `tests/unit/health/detectors/test_stream_stall.py` + +Fires when a stream that previously produced samples has been silent for ≥ `stall_threshold_ns` (default 2.0 s). Closes after samples flow for ≥ `recovery_ns` (default 1.0 s). + +- [ ] **Step 1: Write failing test** + +```python +# tests/unit/health/detectors/test_stream_stall.py +from syncfield.health.detectors.stream_stall import StreamStallDetector +from syncfield.health.types import Incident +from syncfield.types import SampleEvent + + +def _sample(stream_id: str, capture_ns: int) -> SampleEvent: + return SampleEvent(stream_id=stream_id, frame_number=1, capture_ns=capture_ns) + + +def test_no_fire_before_seeing_any_sample(): + d = StreamStallDetector(stall_threshold_ns=1000) + assert list(d.tick(now_ns=10_000)) == [] + + +def test_fires_when_silent_longer_than_threshold(): + d = StreamStallDetector(stall_threshold_ns=1000) + d.observe_sample("cam", _sample("cam", capture_ns=100)) + events = list(d.tick(now_ns=2000)) # 1900 ns of silence + assert len(events) == 1 + ev = events[0] + assert ev.stream_id == "cam" + assert ev.fingerprint == "cam:stream-stall" + assert ev.source == "detector:stream-stall" + assert "silence" in (ev.detail or "").lower() + + +def test_does_not_refire_while_still_stalled(): + d = StreamStallDetector(stall_threshold_ns=1000) + d.observe_sample("cam", _sample("cam", capture_ns=100)) + fired_once = list(d.tick(now_ns=2000)) + fired_twice = list(d.tick(now_ns=3000)) + assert len(fired_once) == 1 + assert len(fired_twice) == 0 + + +def test_refires_after_recovery_then_new_stall(): + d = StreamStallDetector(stall_threshold_ns=1000, recovery_ns=500) + d.observe_sample("cam", _sample("cam", capture_ns=0)) + list(d.tick(now_ns=2000)) # fires stall + + # recovery: samples flow for ≥ recovery_ns + for t in range(3000, 4100, 100): + d.observe_sample("cam", _sample("cam", capture_ns=t)) + inc = Incident.opened_from(list(d.tick(now_ns=4100))[0], title="x") \ + if False else None # placeholder; close_condition test follows + # Now silence again. + new_events = list(d.tick(now_ns=6000)) + assert len(new_events) == 1 # second stall → new event + + +def test_close_condition_requires_recent_sample_flow(): + d = StreamStallDetector(stall_threshold_ns=1000, recovery_ns=500) + d.observe_sample("cam", _sample("cam", capture_ns=0)) + events = list(d.tick(now_ns=2000)) + inc = Incident.opened_from(events[0], title="x") + + # Still silent — do not close. + assert d.close_condition(inc, now_ns=2500) is False + + # Samples arrive across a 600 ns window → recovery_ns=500 satisfied. + d.observe_sample("cam", _sample("cam", capture_ns=2600)) + d.observe_sample("cam", _sample("cam", capture_ns=3200)) + assert d.close_condition(inc, now_ns=3300) is True + + +def test_per_stream_independent_state(): + d = StreamStallDetector(stall_threshold_ns=1000) + d.observe_sample("a", _sample("a", capture_ns=100)) + d.observe_sample("b", _sample("b", capture_ns=100)) + # stream b stays alive + d.observe_sample("b", _sample("b", capture_ns=1800)) + events = list(d.tick(now_ns=2500)) + assert len(events) == 1 + assert events[0].stream_id == "a" +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/health/detectors/test_stream_stall.py -v +``` + +- [ ] **Step 3: Implement** + +`src/syncfield/health/detectors/stream_stall.py`: + +```python +"""StreamStallDetector — fires when a stream stops producing samples.""" + +from __future__ import annotations + +from typing import Dict, Iterator, List + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.types import Incident +from syncfield.types import HealthEvent, HealthEventKind, SampleEvent + + +class StreamStallDetector(DetectorBase): + name = "stream-stall" + default_severity = Severity.ERROR + + def __init__( + self, + stall_threshold_ns: int = 2_000_000_000, + recovery_ns: int = 1_000_000_000, + ) -> None: + self._stall_threshold_ns = stall_threshold_ns + self._recovery_ns = recovery_ns + # Per-stream most-recent sample monotonic time. + self._last_sample_at: Dict[str, int] = {} + # Per-stream: are we currently firing? prevents duplicates per stall. + self._stall_active: Dict[str, bool] = {} + + # --- observers ------------------------------------------------------- + + def observe_sample(self, stream_id: str, sample: SampleEvent) -> None: + self._last_sample_at[stream_id] = sample.capture_ns + # A new sample ends any active stall bookkeeping. + self._stall_active[stream_id] = False + + # --- tick ------------------------------------------------------------ + + def tick(self, now_ns: int) -> Iterator[HealthEvent]: + emitted: List[HealthEvent] = [] + for stream_id, last in self._last_sample_at.items(): + silence_ns = now_ns - last + if silence_ns >= self._stall_threshold_ns and not self._stall_active.get(stream_id, False): + self._stall_active[stream_id] = True + emitted.append(HealthEvent( + stream_id=stream_id, + kind=HealthEventKind.ERROR, + at_ns=now_ns, + detail=f"Stream stalled (silence {silence_ns / 1e9:.1f}s)", + severity=self.default_severity, + source=f"detector:{self.name}", + fingerprint=f"{stream_id}:{self.name}", + data={"silence_ns": silence_ns}, + )) + return iter(emitted) + + # --- close condition ------------------------------------------------- + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + last = self._last_sample_at.get(incident.stream_id) + if last is None: + return False + return (now_ns - last) < self._stall_threshold_ns \ + and (now_ns - incident.last_event_at_ns) >= self._recovery_ns +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +pytest tests/unit/health/detectors/test_stream_stall.py -v +``` + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/detectors/stream_stall.py tests/unit/health/detectors/test_stream_stall.py +git commit -m "feat(health): add StreamStallDetector (platform-level silence detection)" +``` + +--- + +### Task 11: FpsDropDetector + +**Files:** +- Create: `src/syncfield/health/detectors/fps_drop.py` +- Test: `tests/unit/health/detectors/test_fps_drop.py` + +Maintains a per-stream rolling FPS estimate over a 1-second sliding window. Fires when the observed FPS has been below 70 % of target for ≥ 3 s. Without a declared `target_hz`, learns a baseline from the first 10 s of samples after a 5 s warmup. + +Since target_hz is declared by the Stream, the detector accepts a `targets: dict[str, float | None]` dependency it can query. `HealthSystem` supplies a getter. + +- [ ] **Step 1: Write failing test** + +```python +# tests/unit/health/detectors/test_fps_drop.py +from syncfield.health.detectors.fps_drop import FpsDropDetector +from syncfield.types import SampleEvent + + +def _s(stream: str, t_ns: int) -> SampleEvent: + return SampleEvent(stream_id=stream, frame_number=0, capture_ns=t_ns) + + +def test_no_fire_if_fps_tracks_target(): + d = FpsDropDetector(target_getter=lambda sid: 30.0) + # emit 30 samples over 1 second + for i in range(30): + d.observe_sample("cam", _s("cam", i * int(1e9 / 30))) + assert list(d.tick(now_ns=int(1.1e9))) == [] + + +def test_fires_when_observed_below_70_percent_for_3s(): + d = FpsDropDetector( + target_getter=lambda sid: 30.0, + drop_ratio=0.70, + sustain_ns=3 * 1_000_000_000, + ) + + # 10 fps for 3.5 seconds — fps is 10, target 30, ratio 0.33. + interval = int(1e9 / 10) + t = 0 + while t <= int(3.5e9): + d.observe_sample("cam", _s("cam", t)) + t += interval + + emitted = list(d.tick(now_ns=int(3.6e9))) + assert len(emitted) == 1 + assert emitted[0].fingerprint == "cam:fps-drop" + assert emitted[0].data["target_hz"] == 30.0 + assert emitted[0].data["observed_hz"] < 15.0 + + +def test_does_not_fire_without_target_before_warmup(): + d = FpsDropDetector( + target_getter=lambda sid: None, + baseline_warmup_ns=5_000_000_000, + ) + # 10 fps, but only for 1s — under warmup. + t = 0 + for _ in range(10): + d.observe_sample("cam", _s("cam", t)) + t += int(1e8) + assert list(d.tick(now_ns=int(1.1e9))) == [] + + +def test_learns_baseline_then_fires_on_subsequent_drop(): + d = FpsDropDetector( + target_getter=lambda sid: None, + baseline_warmup_ns=1_000_000_000, + baseline_window_ns=2_000_000_000, + drop_ratio=0.7, + sustain_ns=1_000_000_000, + ) + # 3 s @ 30 fps → baseline ≈ 30. + t = 0 + while t <= int(3e9): + d.observe_sample("cam", _s("cam", t)) + t += int(1e9 / 30) + # 1.5 s of 10 fps → drop. + end = t + int(1.5e9) + while t <= end: + d.observe_sample("cam", _s("cam", t)) + t += int(1e8) + emitted = list(d.tick(now_ns=t)) + assert len(emitted) == 1 +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/health/detectors/test_fps_drop.py -v +``` + +- [ ] **Step 3: Implement** + +`src/syncfield/health/detectors/fps_drop.py`: + +```python +"""FpsDropDetector — target-relative or baseline-learning FPS drop detector.""" + +from __future__ import annotations + +from collections import deque +from typing import Callable, Deque, Dict, Iterator, List, Optional + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.types import Incident +from syncfield.types import HealthEvent, HealthEventKind, SampleEvent + +TargetGetter = Callable[[str], Optional[float]] + +_WINDOW_NS = 1_000_000_000 # rolling 1s FPS window + + +class FpsDropDetector(DetectorBase): + name = "fps-drop" + default_severity = Severity.WARNING + + def __init__( + self, + target_getter: TargetGetter = lambda sid: None, + drop_ratio: float = 0.70, + sustain_ns: int = 3_000_000_000, + recovery_ratio: float = 0.90, + recovery_ns: int = 5_000_000_000, + baseline_warmup_ns: int = 5_000_000_000, + baseline_window_ns: int = 10_000_000_000, + ) -> None: + self._target_getter = target_getter + self._drop_ratio = drop_ratio + self._sustain_ns = sustain_ns + self._recovery_ratio = recovery_ratio + self._recovery_ns = recovery_ns + self._baseline_warmup_ns = baseline_warmup_ns + self._baseline_window_ns = baseline_window_ns + + self._samples: Dict[str, Deque[int]] = {} + self._first_seen_at: Dict[str, int] = {} + self._baseline: Dict[str, float] = {} + # When did the stream first drop below threshold in the current dip? + self._dip_began_at: Dict[str, Optional[int]] = {} + self._fire_active: Dict[str, bool] = {} + # Same thing for recovery tracking. + self._recovery_began_at: Dict[str, Optional[int]] = {} + + # --- observers ------------------------------------------------------- + + def observe_sample(self, stream_id: str, sample: SampleEvent) -> None: + buf = self._samples.setdefault(stream_id, deque()) + buf.append(sample.capture_ns) + self._first_seen_at.setdefault(stream_id, sample.capture_ns) + # Trim older than baseline_window. + cutoff = sample.capture_ns - self._baseline_window_ns + while buf and buf[0] < cutoff: + buf.popleft() + + # --- tick ------------------------------------------------------------ + + def tick(self, now_ns: int) -> Iterator[HealthEvent]: + out: List[HealthEvent] = [] + for stream_id, buf in list(self._samples.items()): + target = self._effective_target(stream_id, now_ns) + observed = self._observed_fps(buf, now_ns) + if target is None or observed is None: + continue + + if observed < target * self._drop_ratio: + began = self._dip_began_at.get(stream_id) + if began is None: + self._dip_began_at[stream_id] = now_ns + began = now_ns + elapsed = now_ns - began + if elapsed >= self._sustain_ns and not self._fire_active.get(stream_id, False): + self._fire_active[stream_id] = True + out.append(HealthEvent( + stream_id=stream_id, + kind=HealthEventKind.WARNING, + at_ns=now_ns, + detail=f"FPS drop ({observed:.1f} Hz, target {target:.1f} Hz)", + severity=self.default_severity, + source=f"detector:{self.name}", + fingerprint=f"{stream_id}:{self.name}", + data={"observed_hz": observed, "target_hz": target}, + )) + else: + self._dip_began_at[stream_id] = None + self._fire_active[stream_id] = False + return iter(out) + + # --- close condition ------------------------------------------------- + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + stream_id = incident.stream_id + target = self._effective_target(stream_id, now_ns) + observed = self._observed_fps(self._samples.get(stream_id, deque()), now_ns) + if target is None or observed is None: + return False + if observed < target * self._recovery_ratio: + self._recovery_began_at[stream_id] = None + return False + began = self._recovery_began_at.get(stream_id) + if began is None: + self._recovery_began_at[stream_id] = now_ns + return False + return (now_ns - began) >= self._recovery_ns + + # --- helpers --------------------------------------------------------- + + def _effective_target(self, stream_id: str, now_ns: int) -> Optional[float]: + declared = self._target_getter(stream_id) + if declared is not None: + return float(declared) + first = self._first_seen_at.get(stream_id) + if first is None: + return None + if (now_ns - first) < self._baseline_warmup_ns: + return None + cached = self._baseline.get(stream_id) + if cached is not None: + return cached + observed = self._observed_fps(self._samples.get(stream_id, deque()), now_ns) + if observed is not None: + self._baseline[stream_id] = observed + return self._baseline.get(stream_id) + + @staticmethod + def _observed_fps(buf: Deque[int], now_ns: int) -> Optional[float]: + if not buf: + return None + cutoff = now_ns - _WINDOW_NS + count = sum(1 for t in buf if t >= cutoff) + if count == 0: + return 0.0 + return count / (_WINDOW_NS / 1e9) +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +pytest tests/unit/health/detectors/test_fps_drop.py -v +``` + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/detectors/fps_drop.py tests/unit/health/detectors/test_fps_drop.py +git commit -m "feat(health): add FpsDropDetector (target-relative or baseline-learning)" +``` + +--- + +### Task 12: JitterDetector + +**Files:** +- Create: `src/syncfield/health/detectors/jitter.py` +- Test: `tests/unit/health/detectors/test_jitter.py` + +Tracks last 60 inter-sample intervals per stream. Fires when p95 interval exceeds `jitter_ratio * expected_interval` (from target_hz) for `sustain_ns`. Closes when p95 returns to ≤ 1.2× expected for `recovery_ns`. + +- [ ] **Step 1: Write failing test** + +```python +# tests/unit/health/detectors/test_jitter.py +from syncfield.health.detectors.jitter import JitterDetector +from syncfield.types import SampleEvent + + +def _s(t_ns: int) -> SampleEvent: + return SampleEvent(stream_id="cam", frame_number=0, capture_ns=t_ns) + + +def test_steady_30hz_does_not_fire(): + d = JitterDetector(target_getter=lambda sid: 30.0) + step = int(1e9 / 30) + t = 0 + for _ in range(120): + d.observe_sample("cam", _s(t)) + t += step + assert list(d.tick(now_ns=t)) == [] + + +def test_irregular_intervals_fire_when_p95_exceeds_ratio(): + d = JitterDetector( + target_getter=lambda sid: 30.0, + jitter_ratio=2.0, + sustain_ns=500_000_000, + ) + step = int(1e9 / 30) + big = step * 4 # 4× target interval + t = 0 + # 60 samples alternating between normal and 4× intervals. + for i in range(60): + d.observe_sample("cam", _s(t)) + t += big if i % 2 == 0 else step + + # Give sustain time to elapse with more irregular samples. + for _ in range(20): + d.observe_sample("cam", _s(t)) + t += big + + emitted = list(d.tick(now_ns=t + 500_000_000)) + assert len(emitted) == 1 + assert emitted[0].fingerprint == "cam:jitter" +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/health/detectors/test_jitter.py -v +``` + +- [ ] **Step 3: Implement** + +`src/syncfield/health/detectors/jitter.py`: + +```python +"""JitterDetector — p95-based inter-sample interval anomaly detector.""" + +from __future__ import annotations + +from collections import deque +from typing import Callable, Deque, Dict, Iterator, List, Optional + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.types import Incident +from syncfield.types import HealthEvent, HealthEventKind, SampleEvent + +TargetGetter = Callable[[str], Optional[float]] + + +def _p95(values: List[int]) -> int: + if not values: + return 0 + sorted_v = sorted(values) + idx = max(0, int(0.95 * (len(sorted_v) - 1))) + return sorted_v[idx] + + +class JitterDetector(DetectorBase): + name = "jitter" + default_severity = Severity.WARNING + + def __init__( + self, + target_getter: TargetGetter = lambda sid: None, + window: int = 60, + jitter_ratio: float = 2.0, + sustain_ns: int = 3_000_000_000, + recovery_ratio: float = 1.2, + recovery_ns: int = 10_000_000_000, + ) -> None: + self._target_getter = target_getter + self._window = window + self._jitter_ratio = jitter_ratio + self._sustain_ns = sustain_ns + self._recovery_ratio = recovery_ratio + self._recovery_ns = recovery_ns + + self._last_at: Dict[str, int] = {} + self._intervals: Dict[str, Deque[int]] = {} + self._bad_began_at: Dict[str, Optional[int]] = {} + self._fire_active: Dict[str, bool] = {} + self._recovery_began_at: Dict[str, Optional[int]] = {} + + def observe_sample(self, stream_id: str, sample: SampleEvent) -> None: + last = self._last_at.get(stream_id) + if last is not None: + buf = self._intervals.setdefault(stream_id, deque(maxlen=self._window)) + buf.append(sample.capture_ns - last) + self._last_at[stream_id] = sample.capture_ns + + def tick(self, now_ns: int) -> Iterator[HealthEvent]: + out: List[HealthEvent] = [] + for stream_id, buf in list(self._intervals.items()): + target_hz = self._target_getter(stream_id) + if target_hz is None or target_hz <= 0 or len(buf) < max(10, self._window // 2): + continue + expected = 1e9 / target_hz + p95 = _p95(list(buf)) + + if p95 > expected * self._jitter_ratio: + began = self._bad_began_at.get(stream_id) + if began is None: + self._bad_began_at[stream_id] = now_ns + began = now_ns + if (now_ns - began) >= self._sustain_ns and not self._fire_active.get(stream_id, False): + self._fire_active[stream_id] = True + out.append(HealthEvent( + stream_id=stream_id, + kind=HealthEventKind.WARNING, + at_ns=now_ns, + detail=f"Jitter spike (p95 {p95/1e6:.1f} ms, expected {expected/1e6:.1f} ms)", + severity=self.default_severity, + source=f"detector:{self.name}", + fingerprint=f"{stream_id}:{self.name}", + data={"p95_ns": p95, "expected_ns": int(expected)}, + )) + else: + self._bad_began_at[stream_id] = None + self._fire_active[stream_id] = False + return iter(out) + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + stream_id = incident.stream_id + buf = self._intervals.get(stream_id) + target_hz = self._target_getter(stream_id) + if not buf or target_hz is None or target_hz <= 0: + return False + expected = 1e9 / target_hz + p95 = _p95(list(buf)) + if p95 > expected * self._recovery_ratio: + self._recovery_began_at[stream_id] = None + return False + began = self._recovery_began_at.get(stream_id) + if began is None: + self._recovery_began_at[stream_id] = now_ns + return False + return (now_ns - began) >= self._recovery_ns +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +pytest tests/unit/health/detectors/test_jitter.py -v +``` + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/detectors/jitter.py tests/unit/health/detectors/test_jitter.py +git commit -m "feat(health): add JitterDetector (p95-interval spike detector)" +``` + +--- + +### Task 13: StartupFailureDetector + +**Files:** +- Create: `src/syncfield/health/detectors/startup_failure.py` +- Test: `tests/unit/health/detectors/test_startup_failure.py` + +Fires when an adapter-emitted `HealthEvent` carries `data["phase"] in {"connect", "start_recording"}` with `kind == ERROR`. This relies on orchestrator convention (Task 15 wires this up from `SessionOrchestrator` exception handlers). + +- [ ] **Step 1: Write failing test** + +```python +# tests/unit/health/detectors/test_startup_failure.py +from syncfield.health.detectors.startup_failure import StartupFailureDetector +from syncfield.health.severity import Severity +from syncfield.types import HealthEvent, HealthEventKind + + +def _ev_for(phase: str, kind=HealthEventKind.ERROR) -> HealthEvent: + return HealthEvent( + stream_id="cam", kind=kind, at_ns=100, detail="boom", + severity=Severity.ERROR, source="orchestrator", + fingerprint=f"cam:adapter:startup-{phase}", + data={"phase": phase}, + ) + + +def test_fires_on_connect_phase_error(): + d = StartupFailureDetector() + d.observe_health("cam", _ev_for("connect")) + events = list(d.tick(now_ns=500)) + assert len(events) == 1 + assert events[0].fingerprint == "cam:startup-failure" + assert events[0].data["phase"] == "connect" + + +def test_ignores_non_startup_phases(): + d = StartupFailureDetector() + d.observe_health("cam", HealthEvent( + stream_id="cam", kind=HealthEventKind.ERROR, at_ns=1, detail="x", + severity=Severity.ERROR, source="adapter:foo", fingerprint="cam:adapter:xlink", + data={}, + )) + assert list(d.tick(now_ns=100)) == [] + + +def test_closes_after_phase_success_signal(): + d = StartupFailureDetector() + d.observe_health("cam", _ev_for("connect")) + list(d.tick(now_ns=100)) + from syncfield.health.types import Incident + inc = Incident.opened_from(_ev_for("connect"), title="x") + + # Before success, not closed. + assert d.close_condition(inc, now_ns=200) is False + + # Success signal arrives. + d.observe_health("cam", HealthEvent( + stream_id="cam", kind=HealthEventKind.HEARTBEAT, at_ns=300, detail="connected", + severity=Severity.INFO, source="orchestrator", fingerprint="cam:adapter:startup-success", + data={"phase": "connect", "outcome": "success"}, + )) + assert d.close_condition(inc, now_ns=400) is True +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/health/detectors/test_startup_failure.py -v +``` + +- [ ] **Step 3: Implement** + +`src/syncfield/health/detectors/startup_failure.py`: + +```python +"""StartupFailureDetector — fires when connect/start_recording raises. + +Relies on orchestrator-emitted HealthEvents with ``data["phase"]`` in +{``"connect"``, ``"start_recording"``}. A subsequent success event with +``data["outcome"] == "success"`` closes the incident. +""" + +from __future__ import annotations + +from typing import Dict, Iterator, List, Set + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.types import Incident +from syncfield.types import HealthEvent, HealthEventKind + +_STARTUP_PHASES = {"connect", "start_recording"} + + +class StartupFailureDetector(DetectorBase): + name = "startup-failure" + default_severity = Severity.ERROR + + def __init__(self) -> None: + self._pending_failures: Dict[str, HealthEvent] = {} + self._recovered: Set[str] = set() + + def observe_health(self, stream_id: str, event: HealthEvent) -> None: + phase = event.data.get("phase") if event.data else None + if phase not in _STARTUP_PHASES: + return + outcome = event.data.get("outcome") if event.data else None + if event.kind == HealthEventKind.ERROR and outcome != "success": + self._pending_failures[stream_id] = event + self._recovered.discard(stream_id) + elif outcome == "success": + self._recovered.add(stream_id) + + def tick(self, now_ns: int) -> Iterator[HealthEvent]: + out: List[HealthEvent] = [] + for stream_id, origin in list(self._pending_failures.items()): + out.append(HealthEvent( + stream_id=stream_id, + kind=HealthEventKind.ERROR, + at_ns=now_ns, + detail=origin.detail or "Startup failure", + severity=self.default_severity, + source=f"detector:{self.name}", + fingerprint=f"{stream_id}:{self.name}", + data={"phase": origin.data.get("phase"), "origin_at_ns": origin.at_ns}, + )) + del self._pending_failures[stream_id] + return iter(out) + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + return incident.stream_id in self._recovered +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +pytest tests/unit/health/detectors/test_startup_failure.py -v +``` + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/detectors/startup_failure.py tests/unit/health/detectors/test_startup_failure.py +git commit -m "feat(health): add StartupFailureDetector (connect / start_recording errors)" +``` + +--- + +### Task 14: BackpressureDetector + +**Files:** +- Create: `src/syncfield/health/detectors/backpressure.py` +- Test: `tests/unit/health/detectors/test_backpressure.py` + +Fires when writer queue fullness ≥ 0.80 for 2 s, OR when `dropped` counter increments. Closes when fullness < 0.30 for 5 s and no drops in that window. + +- [ ] **Step 1: Write failing test** + +```python +# tests/unit/health/detectors/test_backpressure.py +from syncfield.health.detectors.backpressure import BackpressureDetector +from syncfield.health.types import Incident, WriterStats + + +def _stat(at_ns, depth, cap=16, dropped=0): + return WriterStats(stream_id="cam", at_ns=at_ns, queue_depth=depth, queue_capacity=cap, dropped=dropped) + + +def test_does_not_fire_with_normal_fullness(): + d = BackpressureDetector() + for t in range(0, int(3e9), int(2.5e8)): + d.observe_writer_stats("cam", _stat(t, depth=2)) + assert list(d.tick(now_ns=int(3e9))) == [] + + +def test_fires_when_queue_sustained_above_threshold(): + d = BackpressureDetector(fullness_threshold=0.8, sustain_ns=int(2e9)) + for t in range(0, int(3e9), int(2.5e8)): + d.observe_writer_stats("cam", _stat(t, depth=14)) # 14/16 = 0.875 + emitted = list(d.tick(now_ns=int(3e9))) + assert len(emitted) == 1 + assert emitted[0].fingerprint == "cam:backpressure" + + +def test_fires_on_any_drop_increment(): + d = BackpressureDetector() + d.observe_writer_stats("cam", _stat(0, depth=1, dropped=0)) + d.observe_writer_stats("cam", _stat(int(1e8), depth=1, dropped=5)) + emitted = list(d.tick(now_ns=int(2e8))) + assert len(emitted) == 1 + + +def test_close_condition_requires_low_and_no_new_drops(): + d = BackpressureDetector(fullness_threshold=0.8, sustain_ns=int(2e9), + recovery_ratio=0.3, recovery_ns=int(1e9)) + for t in range(0, int(3e9), int(2.5e8)): + d.observe_writer_stats("cam", _stat(t, depth=14)) + events = list(d.tick(now_ns=int(3e9))) + inc = Incident.opened_from(events[0], title="x") + + # Recovery in progress. + d.observe_writer_stats("cam", _stat(int(3.5e9), depth=2)) + assert d.close_condition(inc, now_ns=int(4e9)) is False # only 500 ms of recovery + + d.observe_writer_stats("cam", _stat(int(5e9), depth=2)) + assert d.close_condition(inc, now_ns=int(5e9)) is True # 1.5 s of recovery +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/health/detectors/test_backpressure.py -v +``` + +- [ ] **Step 3: Implement** + +`src/syncfield/health/detectors/backpressure.py`: + +```python +"""BackpressureDetector — writer queue saturation + drop-counter detector.""" + +from __future__ import annotations + +from typing import Dict, Iterator, List, Optional + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.types import Incident, WriterStats +from syncfield.types import HealthEvent, HealthEventKind + + +class BackpressureDetector(DetectorBase): + name = "backpressure" + default_severity = Severity.WARNING + + def __init__( + self, + fullness_threshold: float = 0.80, + sustain_ns: int = 2_000_000_000, + recovery_ratio: float = 0.30, + recovery_ns: int = 5_000_000_000, + ) -> None: + self._threshold = fullness_threshold + self._sustain_ns = sustain_ns + self._recovery_ratio = recovery_ratio + self._recovery_ns = recovery_ns + + self._latest: Dict[str, WriterStats] = {} + self._bad_began_at: Dict[str, Optional[int]] = {} + self._last_dropped: Dict[str, int] = {} + self._pending_drop_spike: Dict[str, bool] = {} + self._fire_active: Dict[str, bool] = {} + self._recovery_began_at: Dict[str, Optional[int]] = {} + + def observe_writer_stats(self, stream_id: str, stats: WriterStats) -> None: + self._latest[stream_id] = stats + prev = self._last_dropped.get(stream_id, 0) + if stats.dropped > prev: + self._pending_drop_spike[stream_id] = True + self._last_dropped[stream_id] = stats.dropped + + def tick(self, now_ns: int) -> Iterator[HealthEvent]: + out: List[HealthEvent] = [] + for stream_id, stats in self._latest.items(): + fire_now = False + detail = "" + + if self._pending_drop_spike.pop(stream_id, False): + fire_now = True + detail = f"Writer dropped frames (total {stats.dropped})" + + if stats.queue_fullness >= self._threshold: + began = self._bad_began_at.get(stream_id) + if began is None: + self._bad_began_at[stream_id] = now_ns + began = now_ns + if (now_ns - began) >= self._sustain_ns and not self._fire_active.get(stream_id, False): + fire_now = True + self._fire_active[stream_id] = True + detail = f"Writer queue {stats.queue_depth}/{stats.queue_capacity} full" + else: + self._bad_began_at[stream_id] = None + self._fire_active[stream_id] = False + + if fire_now: + out.append(HealthEvent( + stream_id=stream_id, + kind=HealthEventKind.WARNING, + at_ns=now_ns, + detail=detail, + severity=self.default_severity, + source=f"detector:{self.name}", + fingerprint=f"{stream_id}:{self.name}", + data={ + "queue_depth": stats.queue_depth, + "queue_capacity": stats.queue_capacity, + "dropped": stats.dropped, + }, + )) + return iter(out) + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + stats = self._latest.get(incident.stream_id) + if stats is None: + return False + if stats.queue_fullness > self._recovery_ratio: + self._recovery_began_at[incident.stream_id] = None + return False + # Require that no new drops occurred since incident opened. + if self._last_dropped.get(incident.stream_id, 0) > incident.data.get("dropped_at_open", stats.dropped): + return False + began = self._recovery_began_at.get(incident.stream_id) + if began is None: + self._recovery_began_at[incident.stream_id] = now_ns + return False + return (now_ns - began) >= self._recovery_ns +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +pytest tests/unit/health/detectors/test_backpressure.py -v +``` + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/detectors/backpressure.py tests/unit/health/detectors/test_backpressure.py +git commit -m "feat(health): add BackpressureDetector (writer queue fullness + drop counter)" +``` + +--- + +### Task 14.5: Complete HealthSystem (Task 8 follow-up) + +With Tasks 9–14 done, `HealthSystem` can now import all six default detectors. + +- [ ] **Step 1: Run the deferred HealthSystem test suite** + +```bash +pytest tests/unit/health/test_health_system.py -v +``` +Expected: 4 passed (all tests from Task 8). + +- [ ] **Step 2: Commit (if not already done in Task 8)** + +```bash +git add src/syncfield/health/system.py src/syncfield/health/__init__.py tests/unit/health/test_health_system.py +git commit -m "feat(health): add HealthSystem facade with default detector install" +``` + +--- + +## Phase 3 — Orchestrator & writer integration + +### Task 15: Add target_hz to StreamCapabilities + FinalizationReport.incidents + +**Files:** +- Modify: `src/syncfield/types.py:176-206` (`StreamCapabilities`) +- Modify: `src/syncfield/types.py:294-328` (`FinalizationReport`) +- Test: `tests/unit/test_types_capabilities.py` (add cases), `tests/unit/test_types_finalization.py` (add cases) + +- [ ] **Step 1: Write failing tests** + +Append to `tests/unit/test_types_capabilities.py`: + +```python +from syncfield.types import StreamCapabilities + + +def test_target_hz_defaults_to_none(): + caps = StreamCapabilities() + assert caps.target_hz is None + + +def test_target_hz_round_trips_to_dict(): + caps = StreamCapabilities(target_hz=30.0) + d = caps.to_dict() + assert d["target_hz"] == 30.0 +``` + +Append to `tests/unit/test_types_finalization.py`: + +```python +from syncfield.health.types import Incident +from syncfield.health.severity import Severity +from syncfield.types import FinalizationReport, HealthEvent, HealthEventKind + + +def test_finalization_report_incidents_default_empty(): + r = FinalizationReport( + stream_id="cam", status="completed", frame_count=10, file_path=None, + first_sample_at_ns=0, last_sample_at_ns=100, health_events=[], error=None, + ) + assert r.incidents == [] + + +def test_finalization_report_accepts_incidents(): + ev = HealthEvent( + stream_id="cam", kind=HealthEventKind.ERROR, at_ns=1, detail="x", + severity=Severity.ERROR, source="detector:stream-stall", + fingerprint="cam:stream-stall", + ) + inc = Incident.opened_from(ev, title="stall") + r = FinalizationReport( + stream_id="cam", status="completed", frame_count=10, file_path=None, + first_sample_at_ns=0, last_sample_at_ns=100, health_events=[], error=None, + incidents=[inc], + ) + assert r.incidents == [inc] +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/test_types_capabilities.py tests/unit/test_types_finalization.py -v +``` + +- [ ] **Step 3: Implement** + +In `src/syncfield/types.py`, add `target_hz: float | None = None` to `StreamCapabilities` (keep `live_preview` last-but-one), and update `to_dict`: + +```python +@dataclass(frozen=True) +class StreamCapabilities: + provides_audio_track: bool = False + supports_precise_timestamps: bool = False + is_removable: bool = False + produces_file: bool = False + target_hz: float | None = None + live_preview: bool = True + + def to_dict(self) -> dict[str, Any]: + return { + "provides_audio_track": self.provides_audio_track, + "supports_precise_timestamps": self.supports_precise_timestamps, + "is_removable": self.is_removable, + "produces_file": self.produces_file, + "target_hz": self.target_hz, + "live_preview": self.live_preview, + } +``` + +Update `FinalizationReport` to add `incidents: list[Incident] = field(default_factory=list)`: + +```python +from dataclasses import field # ensure imported at top of file + +@dataclass +class FinalizationReport: + stream_id: str + status: Literal["completed", "partial", "failed", "pending_aggregation"] + frame_count: int + file_path: Path | None + first_sample_at_ns: int | None + last_sample_at_ns: int | None + health_events: list[HealthEvent] + error: str | None + jitter_p95_ns: int | None = None + jitter_p99_ns: int | None = None + incidents: "list" = field(default_factory=list) # type: ignore[assignment] +``` + +The `Incident` forward reference avoids a circular import (`types.py` → `health.types` → `types.py`). Consumers that need the typed list import `Incident` separately. + +- [ ] **Step 4: Run, confirm pass** + +```bash +pytest tests/unit/test_types_capabilities.py tests/unit/test_types_finalization.py -v +``` + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/types.py tests/unit/test_types_capabilities.py tests/unit/test_types_finalization.py +git commit -m "feat(types): add StreamCapabilities.target_hz and FinalizationReport.incidents" +``` + +--- + +### Task 16: SessionLogWriter — log_incident + incidents.jsonl path + +**Files:** +- Modify: `src/syncfield/writer.py:112-161` (`SessionLogWriter`) +- Test: `tests/unit/test_writer.py` (add cases) + +Write incidents to `/incidents.jsonl` on every open/update/close. The file is append-only; each line is the full incident state at write time, keyed by `id`. + +- [ ] **Step 1: Write failing test** + +Append to `tests/unit/test_writer.py`: + +```python +import json + +from syncfield.health.severity import Severity +from syncfield.health.types import Incident +from syncfield.types import HealthEvent, HealthEventKind +from syncfield.writer import SessionLogWriter + + +def _ev(at_ns: int) -> HealthEvent: + return HealthEvent( + stream_id="cam", kind=HealthEventKind.ERROR, at_ns=at_ns, detail="x", + severity=Severity.ERROR, source="detector:stream-stall", + fingerprint="cam:stream-stall", + ) + + +def test_log_incident_appends_to_incidents_jsonl(tmp_path): + w = SessionLogWriter(tmp_path) + w.open() + try: + inc = Incident.opened_from(_ev(100), title="stall") + w.log_incident(inc) + inc.record_event(_ev(200)) + w.log_incident(inc) + inc.close(at_ns=300) + w.log_incident(inc) + finally: + w.close() + + path = tmp_path / "incidents.jsonl" + assert path.exists() + lines = path.read_text().strip().splitlines() + assert len(lines) == 3 + first = json.loads(lines[0]) + assert first["id"] == inc.id + assert first["event_count"] == 1 + last = json.loads(lines[-1]) + assert last["closed_at_ns"] == 300 +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/test_writer.py -k incident -v +``` + +- [ ] **Step 3: Implement** + +In `src/syncfield/writer.py`, extend `SessionLogWriter`: + +```python +class SessionLogWriter: + def __init__(self, output_dir: Path) -> None: + self._path = output_dir / "session_log.jsonl" + self._incidents_path = output_dir / "incidents.jsonl" + self._handle: IO[str] | None = None + self._incidents_handle: IO[str] | None = None + + @property + def incidents_path(self) -> Path: + return self._incidents_path + + def open(self) -> None: + if self._handle is None: + self._handle = open(self._path, "w") + if self._incidents_handle is None: + self._incidents_handle = open(self._incidents_path, "w") + + # existing log_event / log_health unchanged ... + + def log_incident(self, incident) -> None: + """Append *incident*'s current full state as one JSON line.""" + if self._incidents_handle is None: + raise RuntimeError("SessionLogWriter is not open") + self._incidents_handle.write( + json.dumps(incident.to_dict(), separators=(",", ":")) + "\n" + ) + self._incidents_handle.flush() + + def close(self) -> None: + if self._handle is not None: + self._handle.close() + self._handle = None + if self._incidents_handle is not None: + self._incidents_handle.close() + self._incidents_handle = None +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +pytest tests/unit/test_writer.py -v +``` + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/writer.py tests/unit/test_writer.py +git commit -m "feat(writer): add SessionLogWriter.log_incident + incidents.jsonl" +``` + +--- + +### Task 17: Wire SessionOrchestrator to HealthSystem + +**Files:** +- Modify: `src/syncfield/orchestrator.py` — multiple sites +- Test: `tests/integration/health/__init__.py`, `tests/integration/health/test_orchestrator_health_integration.py` + +This is the biggest integration step. Each sub-step has its own runnable verification. + +- [ ] **Step 1: Locate and read the target methods** + +```bash +grep -n "def __init__\|def _on_stream_sample\|def _on_stream_health\|def _set_state\|def start\|def stop\|def add\|def disconnect" src/syncfield/orchestrator.py | head -40 +``` + +Make a short note of the line numbers for: `__init__`, `add` (stream registration), `_on_stream_sample`, `_on_stream_health`, `_set_state` (or the state-transition helper), `start`, `stop`, `disconnect`, `_build_finalization_reports` (or similar). + +- [ ] **Step 2: Write the failing integration test** + +`tests/integration/health/__init__.py` is empty. `tests/integration/health/test_orchestrator_health_integration.py`: + +```python +"""Integration: real SessionOrchestrator + FakeStream → incidents flow end-to-end.""" + +import json +import time +from pathlib import Path + +from syncfield.orchestrator import SessionOrchestrator +from syncfield.types import StreamCapabilities + +# Reuse the existing FakeStream helper if one exists; otherwise define a +# minimal StreamBase subclass inline. +try: + from tests.helpers.fake_stream import FakeStream # type: ignore +except ModuleNotFoundError: + from syncfield.stream import StreamBase + from syncfield.types import SampleEvent + import threading + import time as _t + + class FakeStream(StreamBase): + def __init__(self, stream_id: str, target_hz: float | None = None): + super().__init__( + stream_id=stream_id, + kind="sensor", + capabilities=StreamCapabilities(target_hz=target_hz), + ) + self._stop = threading.Event() + self._thread: threading.Thread | None = None + self._frame = 0 + self._interval = 1.0 / 30.0 + + def connect(self): + self._stop.clear() + self._thread = threading.Thread(target=self._run, daemon=True) + self._thread.start() + + def disconnect(self): + self._stop.set() + if self._thread: + self._thread.join(timeout=1.0) + + def start_recording(self, session_clock): + pass + + def stop_recording(self): + from syncfield.types import FinalizationReport + return FinalizationReport( + stream_id=self.id, status="completed", frame_count=self._frame, + file_path=None, first_sample_at_ns=0, last_sample_at_ns=0, + health_events=[], error=None, + ) + + def pause_samples(self): + # Stop emitting without stopping the thread — simulate stall. + self._stop_samples = True + + def resume_samples(self): + self._stop_samples = False + + def _run(self): + self._stop_samples = False + while not self._stop.is_set(): + if not getattr(self, "_stop_samples", False): + self._frame += 1 + self._emit_sample(SampleEvent( + stream_id=self.id, + frame_number=self._frame, + capture_ns=time.monotonic_ns(), + )) + _t.sleep(self._interval) + + +def test_stall_incident_open_and_close(tmp_path: Path): + sess = SessionOrchestrator(host_id="test", output_dir=tmp_path) + stream = FakeStream("cam", target_hz=30.0) + sess.add(stream) + + sess.connect() + sess.start(countdown_s=0) + + # Induce stall. + stream.pause_samples() + time.sleep(3.0) + + opens = [i for i in sess.health.open_incidents() if i.fingerprint == "cam:stream-stall"] + assert opens, "stall incident did not open" + + # Recover. + stream.resume_samples() + time.sleep(2.5) + + sess.stop() + sess.disconnect() + + resolved = [i for i in sess.health.resolved_incidents() if i.fingerprint == "cam:stream-stall"] + assert resolved, "stall incident did not resolve" + + +def test_incidents_jsonl_written(tmp_path: Path): + sess = SessionOrchestrator(host_id="test", output_dir=tmp_path) + stream = FakeStream("cam", target_hz=30.0) + sess.add(stream) + + sess.connect() + sess.start(countdown_s=0) + stream.pause_samples() + time.sleep(2.5) + sess.stop() + sess.disconnect() + + out_files = list(tmp_path.glob("**/incidents.jsonl")) + assert out_files, "no incidents.jsonl written" + lines = out_files[0].read_text().strip().splitlines() + assert any(json.loads(l)["fingerprint"] == "cam:stream-stall" for l in lines) +``` + +- [ ] **Step 3: Run, confirm fail** + +```bash +pytest tests/integration/health/test_orchestrator_health_integration.py -v +``` +Expected: `AttributeError: 'SessionOrchestrator' object has no attribute 'health'`. + +- [ ] **Step 4: Wire `HealthSystem` into `SessionOrchestrator.__init__`** + +In `src/syncfield/orchestrator.py`, inside `__init__` (after the existing self.* attribute assignments, before any `_bring_*` helper): + +```python +from syncfield.health import HealthSystem + +# ... inside __init__ ... +self.health = HealthSystem() +self.health.on_incident_opened = self._persist_incident +self.health.on_incident_updated = self._persist_incident +self.health.on_incident_closed = self._persist_incident +``` + +And add the persistence helper method on the class: + +```python +def _persist_incident(self, incident) -> None: + writer = getattr(self, "_session_log_writer", None) + if writer is not None: + try: + writer.log_incident(incident) + except Exception: + # Never let telemetry persistence crash the recording. + pass +``` + +- [ ] **Step 5: Connect per-stream sample + health + writer-stats observers** + +Find the place where `add()` registers callbacks (look for `.on_sample(` and `.on_health(`). Extend both: + +```python +def add(self, stream) -> None: + # ... existing wiring ... + stream.on_sample(lambda s: self.health.observe_sample(stream.id, s)) + stream.on_health(lambda h: self._on_stream_health(h)) # unchanged if already here +``` + +And in `_on_stream_health`, forward to the health system **after** persisting to the session log: + +```python +def _on_stream_health(self, event): + writer = getattr(self, "_session_log_writer", None) + if writer is not None: + writer.log_health(event) + self._buffered_health.append(event) # existing + self.health.observe_health(event.stream_id, event) +``` + +Find the state-transition helper (look for a method that updates `self._state` and fires listeners — often `_transition` or `_set_state`). Wrap the transition: + +```python +def _set_state(self, new_state: SessionState) -> None: + old = self._state + self._state = new_state + self.health.observe_state(old, new_state) + # ... existing listener notifications ... +``` + +- [ ] **Step 6: Start / stop the worker + flush remaining incidents on stop** + +In `start()`, after state transitions are ready for recording, call `self.health.start()`. In `stop()`, after final per-stream `stop_recording()` calls but before closing the session log, call `self.health.stop()`. Then collect resolved + open incidents for each stream into its `FinalizationReport.incidents`: + +```python +def start(self, countdown_s: float = 3.0) -> None: + # ... existing start logic up through transition to RECORDING ... + self.health.start() + +def stop(self): + # ... existing stop logic: run each stream's stop_recording() ... + + self.health.stop() + + # Attach incidents to each stream's FinalizationReport. + all_incidents = list(self.health.open_incidents()) + list(self.health.resolved_incidents()) + by_stream: dict[str, list] = {} + for inc in all_incidents: + by_stream.setdefault(inc.stream_id, []).append(inc) + for stream_id, report in self._finalization_reports.items(): + report.incidents = by_stream.get(stream_id, []) + + # ... existing log close + return ... +``` + +- [ ] **Step 7: Pump WriterStats from the recording writer** + +In whichever writer pipeline produces per-frame writes (`src/syncfield/writer.py` — look for the frame-writing path around `VideoWriter` / `SensorWriter`), push a `WriterStats` after each flush. If queues are not used, emit `WriterStats` with `queue_depth=0, queue_capacity=1, dropped=0` — the detector will simply never fire, which is correct. + +If the current writer is synchronous (no queue), skip this sub-step and leave `BackpressureDetector` as a no-op; its integration tests already exercise it directly. Add a TODO comment in `writer.py`: + +```python +# TODO(health): when the writer moves to a queued async path, push +# WriterStats into self._health_system.observe_writer_stats(stream_id, ...) +# on every flush. For the synchronous path, fullness stays at 0. +``` + +- [ ] **Step 8: Run the integration tests, confirm pass** + +```bash +pytest tests/integration/health/test_orchestrator_health_integration.py -v +``` +Expected: 2 passed. (Tests take ~6 s each because they exercise real timing.) + +If a test is flaky under CI timing, increase the `time.sleep()` margins inside the test by 500 ms; keep the detector thresholds at production defaults. + +- [ ] **Step 9: Make sure existing orchestrator tests still pass** + +```bash +pytest tests/ -x -q -k "orchestrator" +``` +Expected: all existing tests pass. If the `health_count` or `problem_count` tests break — those are covered in Task 19 (viewer refactor). Defer; proceed to commit. + +- [ ] **Step 10: Commit** + +```bash +git add src/syncfield/orchestrator.py tests/integration/health/ +git commit -m "feat(orchestrator): wire SessionOrchestrator to HealthSystem (samples, health, state, incidents)" +``` + +--- + +## Phase 4 — OAK bridge & adapter integration + +### Task 18: DepthAILoggerBridge + +**Files:** +- Create: `src/syncfield/health/detectors/depthai_bridge.py` +- Test: `tests/unit/health/detectors/test_depthai_bridge.py` + +The bridge is a `logging.Handler` that converts depthai error / warning records into `HealthEvent`s pushed directly into `HealthSystem.observe_health`. It is **not** a Detector (it doesn't own a close condition — the `AdapterEventPassthrough` does that for the resulting fingerprints). + +- [ ] **Step 1: Write failing test** + +```python +# tests/unit/health/detectors/test_depthai_bridge.py +import logging +from typing import List + +from syncfield.health.detectors.depthai_bridge import DepthAILoggerBridge +from syncfield.types import HealthEvent + + +def _mk_record(msg: str, level: int = logging.ERROR, name: str = "depthai") -> logging.LogRecord: + return logging.LogRecord( + name=name, level=level, pathname="", lineno=0, msg=msg, args=(), exc_info=None, + ) + + +def test_xlink_error_maps_to_xlink_fingerprint(): + captured: List[HealthEvent] = [] + bridge = DepthAILoggerBridge(stream_id="oak-main", sink=lambda sid, ev: captured.append(ev)) + rec = _mk_record("Communication exception - possible device error. Original message 'Couldn't read data from stream: '__x_0_1' (X_LINK_ERROR)'") + bridge.emit(rec) + assert len(captured) == 1 + ev = captured[0] + assert ev.stream_id == "oak-main" + assert ev.fingerprint == "oak-main:adapter:xlink-error" + assert ev.source == "adapter:oak" + assert ev.data.get("stream") == "__x_0_1" + + +def test_device_crash_attaches_crash_dump_path(): + captured: List[HealthEvent] = [] + bridge = DepthAILoggerBridge(stream_id="oak-main", sink=lambda sid, ev: captured.append(ev)) + rec = _mk_record("Device with id 194430 has crashed. Crash dump logs are stored in: /tmp/crash/crash_dump.json - please report to developers.") + bridge.emit(rec) + ev = captured[0] + assert ev.fingerprint == "oak-main:adapter:device-crash" + assert ev.data.get("crash_dump_path") == "/tmp/crash/crash_dump.json" + + +def test_reconnect_attempt_and_success_have_distinct_fingerprints(): + captured: List[HealthEvent] = [] + bridge = DepthAILoggerBridge(stream_id="oak-main", sink=lambda sid, ev: captured.append(ev)) + bridge.emit(_mk_record("Attempting to reconnect. Timeout is 10000ms", level=logging.WARNING)) + bridge.emit(_mk_record("Reconnection successful", level=logging.WARNING)) + fps = [c.fingerprint for c in captured] + assert "oak-main:adapter:reconnect-attempt" in fps + assert "oak-main:adapter:reconnect-success" in fps + + +def test_unrecognized_error_falls_back_to_warning_unparsed(): + captured: List[HealthEvent] = [] + bridge = DepthAILoggerBridge(stream_id="oak-main", sink=lambda sid, ev: captured.append(ev)) + bridge.emit(_mk_record("Something totally new and unrecognized", level=logging.ERROR)) + assert len(captured) == 1 + assert captured[0].source == "adapter:oak:unparsed-log" + + +def test_info_records_are_ignored(): + captured = [] + bridge = DepthAILoggerBridge(stream_id="oak-main", sink=lambda sid, ev: captured.append(ev)) + bridge.emit(_mk_record("Some info", level=logging.INFO)) + assert captured == [] +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/health/detectors/test_depthai_bridge.py -v +``` + +- [ ] **Step 3: Implement** + +`src/syncfield/health/detectors/depthai_bridge.py`: + +```python +"""DepthAILoggerBridge — translate depthai Python log records into HealthEvents. + +Installed as a standard :class:`logging.Handler` on the depthai logger. +Does not subclass DetectorBase — it is a translator, not a detector. +Its outputs are fingerprinted as ``:adapter:`` so +the AdapterEventPassthrough detector owns their open/close lifecycle. +""" + +from __future__ import annotations + +import logging +import re +import time +from typing import Callable, Optional + +from syncfield.health.severity import Severity +from syncfield.types import HealthEvent, HealthEventKind + +Sink = Callable[[str, HealthEvent], None] + +_XLINK_RE = re.compile(r"X_LINK_ERROR.*stream: '([^']+)'|stream: '([^']+)'.*X_LINK_ERROR") +_CRASH_RE = re.compile(r"Device with id (\S+) has crashed\. Crash dump logs are stored in: (\S+)") +_RECONNECT_TRY_RE = re.compile(r"Attempting to reconnect", re.IGNORECASE) +_RECONNECT_OK_RE = re.compile(r"Reconnection successful", re.IGNORECASE) +_CONN_CLOSED_RE = re.compile(r"Closed connection", re.IGNORECASE) + + +class DepthAILoggerBridge(logging.Handler): + def __init__(self, stream_id: str, sink: Sink) -> None: + super().__init__(level=logging.WARNING) + self._stream_id = stream_id + self._sink = sink + + def emit(self, record: logging.LogRecord) -> None: + if record.levelno < logging.WARNING: + return + msg = record.getMessage() + now = time.monotonic_ns() + + parsed = self._parse(msg, record.levelno, now) + if parsed is None: + parsed = HealthEvent( + stream_id=self._stream_id, + kind=HealthEventKind.WARNING, + at_ns=now, + detail=msg, + severity=Severity.WARNING, + source="adapter:oak:unparsed-log", + fingerprint=f"{self._stream_id}:adapter:unparsed-log", + data={"raw": msg, "levelname": record.levelname}, + ) + try: + self._sink(self._stream_id, parsed) + except Exception: + # Never let bridge failures crash the logging path. + pass + + def _parse(self, msg: str, levelno: int, now: int) -> Optional[HealthEvent]: + crash = _CRASH_RE.search(msg) + if crash: + device_id, path = crash.group(1), crash.group(2) + return HealthEvent( + stream_id=self._stream_id, + kind=HealthEventKind.ERROR, + at_ns=now, + detail="OAK device crashed", + severity=Severity.CRITICAL, + source="adapter:oak", + fingerprint=f"{self._stream_id}:adapter:device-crash", + data={"device_id": device_id, "crash_dump_path": path}, + ) + xlink = _XLINK_RE.search(msg) + if xlink: + stream = xlink.group(1) or xlink.group(2) + return HealthEvent( + stream_id=self._stream_id, + kind=HealthEventKind.ERROR, + at_ns=now, + detail="XLink communication error", + severity=Severity.ERROR, + source="adapter:oak", + fingerprint=f"{self._stream_id}:adapter:xlink-error", + data={"stream": stream}, + ) + if _RECONNECT_OK_RE.search(msg): + return HealthEvent( + stream_id=self._stream_id, + kind=HealthEventKind.RECONNECT, + at_ns=now, + detail="Reconnection successful", + severity=Severity.INFO, + source="adapter:oak", + fingerprint=f"{self._stream_id}:adapter:reconnect-success", + ) + if _RECONNECT_TRY_RE.search(msg): + return HealthEvent( + stream_id=self._stream_id, + kind=HealthEventKind.RECONNECT, + at_ns=now, + detail="Attempting reconnect", + severity=Severity.WARNING, + source="adapter:oak", + fingerprint=f"{self._stream_id}:adapter:reconnect-attempt", + ) + if _CONN_CLOSED_RE.search(msg): + return HealthEvent( + stream_id=self._stream_id, + kind=HealthEventKind.WARNING, + at_ns=now, + detail="Connection closed", + severity=Severity.WARNING, + source="adapter:oak", + fingerprint=f"{self._stream_id}:adapter:connection-closed", + ) + return None +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +pytest tests/unit/health/detectors/test_depthai_bridge.py -v +``` +Expected: 5 passed. + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/detectors/depthai_bridge.py tests/unit/health/detectors/test_depthai_bridge.py +git commit -m "feat(health): add DepthAILoggerBridge (logging.Handler → HealthEvent)" +``` + +--- + +### Task 19: OakCameraStream — declare target_hz, install bridge, capture crash_dump + +**Files:** +- Modify: `src/syncfield/adapters/oak_camera.py` +- Test: `tests/unit/adapters/test_oak_camera_health.py` (new) + +- [ ] **Step 1: Locate the current OAK setup sites** + +```bash +grep -n "StreamCapabilities\|def __init__\|def connect\|def disconnect\|fps\b" src/syncfield/adapters/oak_camera.py | head -30 +``` + +Record line numbers for: `StreamCapabilities(...)` construction, `connect()` entry, `disconnect()` entry, the `fps=` argument passed to depthai (this is the physical target we surface as `target_hz`). + +- [ ] **Step 2: Write the failing test** + +`tests/unit/adapters/test_oak_camera_health.py`: + +```python +"""Unit tests for OAK adapter health wiring (no real hardware required).""" + +import logging + +import pytest + + +pytest.importorskip("depthai") # skip entire module if the oak extra isn't installed + + +def test_oak_declares_target_hz(): + from syncfield.adapters.oak_camera import OakCameraStream + s = OakCameraStream(stream_id="oak-main", fps=30) + assert s.capabilities.target_hz == 30.0 + + +def test_oak_connect_installs_logger_bridge(monkeypatch, tmp_path): + from syncfield.adapters.oak_camera import OakCameraStream + + captured = [] + s = OakCameraStream(stream_id="oak-main", fps=30, output_dir=tmp_path) + s.on_health(lambda ev: captured.append(ev)) + + # Don't actually build a depthai pipeline — stub the inner connect body. + monkeypatch.setattr(s, "_open_device_pipeline", lambda: None, raising=False) + s._install_depthai_bridge() # exercised directly since connect() may fail without hw + + logging.getLogger("depthai").error( + "Communication exception - Original message 'Couldn't read data from stream: '__x_0_1' (X_LINK_ERROR)'" + ) + + assert any(ev.fingerprint == "oak-main:adapter:xlink-error" for ev in captured) + + s._uninstall_depthai_bridge() +``` + +- [ ] **Step 3: Run, confirm fail** + +```bash +pytest tests/unit/adapters/test_oak_camera_health.py -v +``` + +- [ ] **Step 4: Implement the wiring** + +In `src/syncfield/adapters/oak_camera.py`: + +- Add `target_hz` to the `StreamCapabilities(...)` construction inside `__init__`, using the existing `fps` argument: `StreamCapabilities(produces_file=True, supports_precise_timestamps=True, is_removable=True, target_hz=float(fps))`. +- Add two helpers: + +```python +def _install_depthai_bridge(self) -> None: + from syncfield.health.detectors.depthai_bridge import DepthAILoggerBridge + if getattr(self, "_depthai_bridge", None) is not None: + return + self._depthai_bridge = DepthAILoggerBridge( + stream_id=self.id, + sink=lambda sid, ev: self._emit_health(ev), + ) + logging.getLogger("depthai").addHandler(self._depthai_bridge) + +def _uninstall_depthai_bridge(self) -> None: + bridge = getattr(self, "_depthai_bridge", None) + if bridge is None: + return + logging.getLogger("depthai").removeHandler(bridge) + self._depthai_bridge = None +``` + +- Call `self._install_depthai_bridge()` at the top of `connect()` and `self._uninstall_depthai_bridge()` at the bottom of `disconnect()`. + +- Where `StreamBase._emit_health` is invoked with a crash detail today, prefer letting the bridge handle it. The bridge's own `device-crash` fingerprint already attaches `crash_dump_path` in `event.data`. In the orchestrator's `_persist_incident` (Task 17 step 4), when an incident's `fingerprint` ends with `:device-crash` and `first_event.data["crash_dump_path"]` exists, attach an `IncidentArtifact`: + +```python +# In orchestrator._persist_incident, before writer.log_incident: +from syncfield.health.types import IncidentArtifact + +if incident.fingerprint.endswith(":device-crash"): + path = incident.first_event.data.get("crash_dump_path") + if path and not any(a.kind == "crash_dump" for a in incident.artifacts): + incident.attach(IncidentArtifact(kind="crash_dump", path=str(path))) +``` + +- [ ] **Step 5: Run, confirm pass** + +```bash +pytest tests/unit/adapters/test_oak_camera_health.py -v +``` + +- [ ] **Step 6: Commit** + +```bash +git add src/syncfield/adapters/oak_camera.py src/syncfield/orchestrator.py tests/unit/adapters/test_oak_camera_health.py +git commit -m "feat(oak): declare target_hz, bridge depthai logger, attach crash_dump artifacts" +``` + +--- + +## Phase 5 — Viewer (server) + +### Task 20: Replace HealthEntry with IncidentSnapshot in poller state + +**Files:** +- Modify: `src/syncfield/viewer/state.py` +- Modify: `src/syncfield/viewer/poller.py` (if poller constructs snapshots — else just state.py) +- Test: `tests/unit/viewer/test_snapshot_incidents.py` (new) + +- [ ] **Step 1: Write failing test** + +```python +# tests/unit/viewer/test_snapshot_incidents.py +from syncfield.health.severity import Severity +from syncfield.health.types import Incident, IncidentSnapshot +from syncfield.types import HealthEvent, HealthEventKind +from syncfield.viewer.state import SessionSnapshot, StreamSnapshot + + +def _ev(at_ns: int) -> HealthEvent: + return HealthEvent( + stream_id="cam", kind=HealthEventKind.ERROR, at_ns=at_ns, detail="x", + severity=Severity.ERROR, source="detector:stream-stall", + fingerprint="cam:stream-stall", + ) + + +def test_session_snapshot_has_incident_fields(): + snap = SessionSnapshot( + host_id="h", state="recording", output_dir="/tmp", + sync_point_monotonic_ns=None, sync_point_wall_clock_ns=None, + chirp_start_ns=None, chirp_stop_ns=None, chirp_enabled=False, + elapsed_s=0.0, streams={}, active_incidents=[], resolved_incidents=[], + ) + assert snap.active_incidents == [] + assert snap.resolved_incidents == [] + + +def test_stream_snapshot_no_longer_has_health_count(): + # health_count and problem_count are removed; StreamSnapshot should not accept them. + import dataclasses + fields = {f.name for f in dataclasses.fields(StreamSnapshot)} + assert "health_count" not in fields + assert "problem_count" not in fields +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/viewer/test_snapshot_incidents.py -v +``` + +- [ ] **Step 3: Implement in `src/syncfield/viewer/state.py`** + +- Remove the `HealthEntry` dataclass and all references to it. +- Remove `StreamSnapshot.health_count` (and `problem_count` if present). +- Replace `SessionSnapshot.health_log: List[HealthEntry]` with: + +```python +active_incidents: List[IncidentSnapshot] = field(default_factory=list) +resolved_incidents: List[IncidentSnapshot] = field(default_factory=list) +``` + +Remove `StreamStatsBuffer._health` (the `HealthEntry` deque) and the `observe_health` / `snapshot_health` methods that produced it. + +Imports: + +```python +from syncfield.health.types import IncidentSnapshot +``` + +- [ ] **Step 4: Update the poller** (`src/syncfield/viewer/poller.py`) + +Find where the poller is constructed or initialized with the session — it needs to subscribe to `session.health`: + +```python +# in the poller's __init__, after storing self._session: +session.health.on_incident_opened = self._ingest_incident +session.health.on_incident_updated = self._ingest_incident +session.health.on_incident_closed = self._ingest_incident +``` + +Keep two bounded lists, updated from the callback (callbacks fire on the health worker thread — use a lock): + +```python +import threading +from syncfield.health.types import IncidentSnapshot + +self._incidents_lock = threading.Lock() +self._open_by_id: dict[str, Incident] = {} +self._resolved: deque[Incident] = deque(maxlen=20) + +def _ingest_incident(self, incident): + with self._incidents_lock: + if incident.is_open: + self._open_by_id[incident.id] = incident + else: + self._open_by_id.pop(incident.id, None) + self._resolved.append(incident) +``` + +And when producing the snapshot: + +```python +import time + +with self._incidents_lock: + now = time.monotonic_ns() + active = [IncidentSnapshot.from_incident(i, now_ns=now) for i in self._open_by_id.values()] + resolved = [IncidentSnapshot.from_incident(i, now_ns=now) for i in self._resolved] + +snapshot = SessionSnapshot( + # ... existing fields ... + active_incidents=active, + resolved_incidents=resolved, +) +``` + +Remove `health_log=` / `health_count=` from the construction. Remove any existing `observe_health` poller registration on streams (the orchestrator now owns that). + +- [ ] **Step 5: Run, confirm pass** + +```bash +pytest tests/unit/viewer/ -v +``` + +If existing viewer tests reference `health_count`, `problem_count`, `health_log`, or `HealthEntry`, update them to use the new fields. Search: + +```bash +grep -rn "health_count\|problem_count\|health_log\|HealthEntry" tests/ src/syncfield/viewer/ +``` + +Fix each site in the same commit. + +- [ ] **Step 6: Commit** + +```bash +git add src/syncfield/viewer/ tests/unit/viewer/ +git commit -m "refactor(viewer): replace HealthEntry/health_count with IncidentSnapshot fields" +``` + +--- + +### Task 21: WebSocket serializer — emit incident fields + +**Files:** +- Modify: `src/syncfield/viewer/server.py` — the WebSocket snapshot encoder +- Test: `tests/unit/viewer/test_server_snapshot_serialization.py` (new, or extend existing) + +- [ ] **Step 1: Locate the serializer** + +```bash +grep -n "SessionSnapshot\|snapshot.*dict\|jsonable\|json.dumps" src/syncfield/viewer/server.py | head -20 +``` + +Find the function/method that converts a `SessionSnapshot` to the dict shipped over WebSocket. + +- [ ] **Step 2: Write failing test** + +```python +# tests/unit/viewer/test_server_snapshot_serialization.py +from syncfield.health.severity import Severity +from syncfield.health.types import Incident, IncidentSnapshot +from syncfield.types import HealthEvent, HealthEventKind +from syncfield.viewer.state import SessionSnapshot +from syncfield.viewer.server import snapshot_to_wire # or whatever the real name is + + +def _inc_snap(open_: bool = True) -> IncidentSnapshot: + ev = HealthEvent( + stream_id="cam", kind=HealthEventKind.ERROR, at_ns=1, detail="x", + severity=Severity.ERROR, source="detector:stream-stall", + fingerprint="cam:stream-stall", + ) + inc = Incident.opened_from(ev, title="stall") + if not open_: + inc.close(at_ns=2) + return IncidentSnapshot.from_incident(inc, now_ns=1_000) + + +def test_snapshot_to_wire_emits_incident_fields(): + snap = SessionSnapshot( + host_id="h", state="recording", output_dir="/tmp", + sync_point_monotonic_ns=None, sync_point_wall_clock_ns=None, + chirp_start_ns=None, chirp_stop_ns=None, chirp_enabled=False, + elapsed_s=0.0, streams={}, + active_incidents=[_inc_snap(open_=True)], + resolved_incidents=[_inc_snap(open_=False)], + ) + wire = snapshot_to_wire(snap) + assert isinstance(wire["active_incidents"], list) + assert wire["active_incidents"][0]["severity"] == "error" + assert wire["active_incidents"][0]["fingerprint"] == "cam:stream-stall" + assert wire["resolved_incidents"][0]["closed_at_ns"] == 2 +``` + +(Replace the import name if the actual helper is different — adapt based on step 1's grep.) + +- [ ] **Step 3: Implement** + +Update the snapshot-to-wire helper: + +```python +def snapshot_to_wire(snap: SessionSnapshot) -> dict: + return { + # ... existing fields ... + "active_incidents": [_incident_to_wire(i) for i in snap.active_incidents], + "resolved_incidents": [_incident_to_wire(i) for i in snap.resolved_incidents], + } + +def _incident_to_wire(snap: IncidentSnapshot) -> dict: + return { + "id": snap.id, + "stream_id": snap.stream_id, + "fingerprint": snap.fingerprint, + "title": snap.title, + "severity": snap.severity, + "source": snap.source, + "opened_at_ns": snap.opened_at_ns, + "closed_at_ns": snap.closed_at_ns, + "event_count": snap.event_count, + "detail": snap.detail, + "ago_s": snap.ago_s, + "artifacts": snap.artifacts, + } +``` + +Remove any serialization of the old `health_log` / `health_count`. + +- [ ] **Step 4: Run, confirm pass** + +```bash +pytest tests/unit/viewer/test_server_snapshot_serialization.py -v +``` + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/viewer/server.py tests/unit/viewer/test_server_snapshot_serialization.py +git commit -m "feat(viewer): serialize incidents into WebSocket snapshot payload" +``` + +--- + +## Phase 6 — Viewer (frontend) + +### Task 22: Mirror Severity + IncidentSnapshot TypeScript types + +**Files:** +- Modify: `src/syncfield/viewer/frontend/src/lib/types.ts` + +- [ ] **Step 1: Open the file and read the current shape** + +```bash +sed -n '1,60p' src/syncfield/viewer/frontend/src/lib/types.ts +``` + +- [ ] **Step 2: Remove HealthEntry + health_count; add new types** + +Replace the `HealthEntry` type and any `StreamSnapshot.health_count` / `problem_count` fields with: + +```ts +export type Severity = "info" | "warning" | "error" | "critical"; + +export interface IncidentArtifact { + kind: string; + path: string; + detail: string | null; +} + +export interface IncidentSnapshot { + id: string; + stream_id: string; + fingerprint: string; + title: string; + severity: Severity; + source: string; + opened_at_ns: number; + closed_at_ns: number | null; + event_count: number; + detail: string | null; + ago_s: number; + artifacts: IncidentArtifact[]; +} + +export interface SessionSnapshot { + // ...existing fields (host_id, state, streams, elapsed_s, etc.)... + active_incidents: IncidentSnapshot[]; + resolved_incidents: IncidentSnapshot[]; +} +``` + +Delete the old `HealthEntry` export entirely. Update `StreamSnapshot` to drop `health_count` / `problem_count`. + +- [ ] **Step 3: Typecheck** + +```bash +cd src/syncfield/viewer/frontend && npx tsc --noEmit +``` +Expected: errors listing every call-site that still references `HealthEntry` / `health_count`. We fix those in Tasks 23 and 24. Commit now with the type changes — subsequent tasks compile green. + +```bash +git add src/syncfield/viewer/frontend/src/lib/types.ts +git commit -m "refactor(viewer-fe): mirror IncidentSnapshot + Severity; drop HealthEntry" +``` + +--- + +### Task 23: Delete health-table.tsx, add incident-panel.tsx + +**Files:** +- Delete: `src/syncfield/viewer/frontend/src/components/health-table.tsx` +- Create: `src/syncfield/viewer/frontend/src/components/incident-panel.tsx` +- Modify: `src/syncfield/viewer/frontend/src/App.tsx` — replace `` with `` + +- [ ] **Step 1: Create the new component** + +`src/syncfield/viewer/frontend/src/components/incident-panel.tsx`: + +```tsx +import { useState } from "react"; +import type { IncidentSnapshot, Severity } from "../lib/types"; + +const SEVERITY_ICON: Record = { + info: "·", + warning: "⚠", + error: "⛔", + critical: "⛔", +}; + +const SEVERITY_COLOR: Record = { + info: "text-slate-400", + warning: "text-yellow-400", + error: "text-orange-400", + critical: "text-red-500", +}; + +function formatAgo(s: number): string { + if (s < 60) return `${Math.round(s)}s ago`; + if (s < 3600) return `${Math.round(s / 60)}m ago`; + return `${Math.round(s / 3600)}h ago`; +} + +function IncidentCard({ inc, isOpen }: { inc: IncidentSnapshot; isOpen: boolean }) { + const [expanded, setExpanded] = useState(false); + return ( + + ); +} + +export function IncidentPanel({ + active, + resolved, +}: { + active: IncidentSnapshot[]; + resolved: IncidentSnapshot[]; +}) { + return ( +
+
+ Active Issues ({active.length}) +
+ {active.length === 0 ? ( +
None — all clear.
+ ) : ( +
+ {active.map((inc) => ( + + ))} +
+ )} +
+ Resolved this session ({resolved.length}) +
+ {resolved.length === 0 ? ( +
None.
+ ) : ( + resolved.map((inc) => ) + )} +
+ ); +} +``` + +- [ ] **Step 2: Replace the mount site** + +In `src/syncfield/viewer/frontend/src/App.tsx`, find the `` usage and replace with: + +```tsx +import { IncidentPanel } from "./components/incident-panel"; + +// inside render, wherever HealthTable lived: + +``` + +Delete the `import { HealthTable } from "./components/health-table";` line. + +- [ ] **Step 3: Delete the old file** + +```bash +rm src/syncfield/viewer/frontend/src/components/health-table.tsx +``` + +- [ ] **Step 4: Typecheck and build** + +```bash +cd src/syncfield/viewer/frontend && npx tsc --noEmit && npm run build +``` +Expected: clean type-check. The bundled assets update in `static/` (or wherever the existing pipeline ships them). + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/viewer/frontend/src/components/incident-panel.tsx \ + src/syncfield/viewer/frontend/src/App.tsx \ + src/syncfield/viewer/static/ # if generated bundle is versioned +git rm src/syncfield/viewer/frontend/src/components/health-table.tsx +git commit -m "feat(viewer-fe): add IncidentPanel (replaces HealthTable)" +``` + +--- + +### Task 24: Stream card severity badge + +**Files:** +- Modify: `src/syncfield/viewer/frontend/src/components/stream-card.tsx` + +- [ ] **Step 1: Replace the red-dot / health_count display** + +Find where `stream.health_count` (or `problem_count`) was rendered. Replace with a per-stream severity count computed from `active_incidents`: + +```tsx +import type { IncidentSnapshot, Severity } from "../lib/types"; + +function streamIncidentStats(streamId: string, active: IncidentSnapshot[]) { + const mine = active.filter((i) => i.stream_id === streamId); + const count = mine.length; + const highest: Severity | null = mine.reduce((acc, i) => { + if (acc === null) return i.severity; + const order: Severity[] = ["info", "warning", "error", "critical"]; + return order.indexOf(i.severity) > order.indexOf(acc) ? i.severity : acc; + }, null); + return { count, highest }; +} + +const BADGE_COLOR: Record = { + info: "bg-slate-500", + warning: "bg-yellow-500", + error: "bg-orange-500", + critical: "bg-red-500", +}; + +export function StreamCard({ stream, activeIncidents }: { + stream: StreamSnapshot; + activeIncidents: IncidentSnapshot[]; +}) { + const { count, highest } = streamIncidentStats(stream.id, activeIncidents); + return ( +
+ {/* ...existing header... */} + {count > 0 && highest && ( + + {count} + + )} + {/* ...rest of card... */} +
+ ); +} +``` + +Propagate `activeIncidents` from the parent that already has the snapshot. + +- [ ] **Step 2: Typecheck** + +```bash +cd src/syncfield/viewer/frontend && npx tsc --noEmit +``` + +- [ ] **Step 3: Commit** + +```bash +git add src/syncfield/viewer/frontend/src/components/stream-card.tsx \ + src/syncfield/viewer/frontend/src/App.tsx # if parent changed +git commit -m "feat(viewer-fe): stream card severity badge from active incidents" +``` + +--- + +## Phase 7 — target_hz rollout on other adapters + +### Task 25: Declare target_hz on all known-target adapters + +**Files:** +- Modify: `src/syncfield/adapters/uvc_webcam.py` +- Modify: `src/syncfield/adapters/host_audio.py` +- Modify: `src/syncfield/adapters/meta_quest_camera/stream.py` (or wherever its `StreamCapabilities` are built) +- Modify: `src/syncfield/adapters/ble_imu.py` (if a stable rate is known) +- Modify: `src/syncfield/adapters/insta360_go3s/stream.py` (if a live preview rate exists — otherwise skip) +- Modify: `src/syncfield/adapters/polling_sensor.py`, `push_sensor.py` (if they accept a rate arg) + +- [ ] **Step 1: For each adapter, locate its StreamCapabilities construction** + +```bash +grep -rn "StreamCapabilities(" src/syncfield/adapters/ +``` + +- [ ] **Step 2: For each, thread the existing rate argument (`fps`, `rate_hz`, `target_hz`) into `target_hz=`** + +Example for `uvc_webcam.py`: + +```python +# before: +capabilities=StreamCapabilities(produces_file=True, is_removable=True) +# after: +capabilities=StreamCapabilities(produces_file=True, is_removable=True, target_hz=float(self._fps)) +``` + +For `host_audio.py`, the "sample rate" is huge (48 kHz) — that isn't a per-sample-emission rate. Audio streams emit one `SampleEvent` per chunk, so set `target_hz` to `sample_rate / block_size`. If the adapter doesn't expose a block-rate cleanly, leave `target_hz=None` and rely on the baseline-learning fallback. + +- [ ] **Step 3: Run the full suite to catch regressions** + +```bash +pytest tests/ -q +``` + +- [ ] **Step 4: Commit** + +```bash +git add src/syncfield/adapters/ +git commit -m "feat(adapters): declare target_hz on UVC / audio / Meta Quest / BLE IMU / sensors" +``` + +--- + +## Phase 8 — Manual verification + +### Task 26: Manual verification on real OAK hardware + +This task is a human-run checklist, not automated code. Its purpose is to confirm the OAK-motivated failure modes surface correctly. + +- [ ] **Step 1: Run a normal session, no incidents expected** + +```bash +python examples/oak_live_preview.py # or whichever example runs an OAK-only session +``` +Start a recording, let it run for 30 s, stop. Open the viewer. + +Expected: **Active Issues (0)**, **Resolved this session (0)**. + +- [ ] **Step 2: Induce an XLink stall** + +With a recording running, physically unplug the OAK's USB cable. Wait 3 seconds. Re-plug. + +Expected in the viewer within 3 s of unplug: +- A **stream-stall** incident appears under Active Issues with title like "Stream stalled (silence 2.0s)", severity=`error`. +- A **xlink-error** incident appears separately, severity=`error`. +- After reconnection succeeds and samples flow for ~1 s, the stall incident moves to Resolved. + +- [ ] **Step 3: Induce a crash (if reproducible)** + +If a deterministic crash-reproducer exists (e.g., requesting an unsupported pipeline config mid-run), trigger it. + +Expected: +- A **device-crash** incident appears under Active Issues with severity=`critical` and an attached artifact chip showing `crash_dump`. +- The incident's expanded view shows the `crash_dump.json` absolute path. + +- [ ] **Step 4: Stop and inspect persisted artifacts** + +After `stop()`: + +```bash +cat $(find data_leader -name incidents.jsonl | head -1) +``` + +Expected: one JSON line per incident open/update/close. Crash incident's line includes `"artifacts": [{"kind": "crash_dump", "path": "..."}]`. + +- [ ] **Step 5: Cross-check FinalizationReport** + +Inside the example script, print `report.incidents` for each stream. Confirm the open/close states match the viewer's Active/Resolved counts at stop time. + +- [ ] **Step 6: Capture screenshots for the PR** + +Take a screenshot of the viewer during the induced stall and after recovery. Attach both to the PR description. + +--- + +## Self-Review Checklist + +After completing the plan, run this checklist before merging: + +1. **Spec coverage**: + - Goal 1 (live detection): Tasks 9–14, 17, 20–24 ✓ + - Goal 2 (post-session report): Tasks 16, 17 (FinalizationReport), 20 (persistence) ✓ + - Goal 3 (sensor-agnostic baseline): Tasks 8, 9–14 ✓ + - Goal 4 (pluggable detectors): Tasks 4, 5, 8 (`register()`) ✓ + - Goal 5 (default-on): Task 8 (`_install_default_detectors`), Task 19 (OAK bridge auto-install) ✓ + - Goal 6 (artifact capture): Task 18 (crash dump path), Task 19 (IncidentArtifact attach) ✓ + - Goal 7 (zero hot-path impact): Task 7 (lock-free SimpleQueues, dedicated daemon) ✓ + +2. **No placeholders**: every step shows complete code or a concrete command. No "TBD" / "similar to" / "handle edge cases". + +3. **Type consistency**: `HealthSystem.observe_*`, `Detector.observe_*`, `IncidentTracker.ingest/tick`, `SessionLogWriter.log_incident`, `SessionSnapshot.active_incidents`/`resolved_incidents` — all names line up across tasks. + +--- + +## Plan complete and saved to `docs/superpowers/plans/2026-04-22-health-telemetry.md`. + +**Two execution options:** + +1. **Subagent-Driven (recommended)** — fresh subagent per task, review between tasks, fast iteration. +2. **Inline Execution** — execute tasks in this session with checkpoints for review. + +Which approach? From 2e0451510d94559accefec13b9bba111bf2f062e Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 12:32:08 +0900 Subject: [PATCH 03/64] feat(health): add Severity enum with rank + max_severity helper --- src/syncfield/health/__init__.py | 0 src/syncfield/health/severity.py | 35 ++++++++++++++++++++++++++++++ tests/unit/health/__init__.py | 0 tests/unit/health/test_severity.py | 27 +++++++++++++++++++++++ 4 files changed, 62 insertions(+) create mode 100644 src/syncfield/health/__init__.py create mode 100644 src/syncfield/health/severity.py create mode 100644 tests/unit/health/__init__.py create mode 100644 tests/unit/health/test_severity.py diff --git a/src/syncfield/health/__init__.py b/src/syncfield/health/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/syncfield/health/severity.py b/src/syncfield/health/severity.py new file mode 100644 index 0000000..1f53679 --- /dev/null +++ b/src/syncfield/health/severity.py @@ -0,0 +1,35 @@ +"""Severity levels for health events and incidents. + +Ordered INFO < WARNING < ERROR < CRITICAL. Use :func:`max_severity` to +pick the highest of several levels — incidents escalate to the max +severity of their constituent events. +""" + +from __future__ import annotations + +from enum import Enum + + +class Severity(str, Enum): + INFO = "info" + WARNING = "warning" + ERROR = "error" + CRITICAL = "critical" + + @property + def rank(self) -> int: + return _RANK[self] + + +_RANK = { + Severity.INFO: 0, + Severity.WARNING: 1, + Severity.ERROR: 2, + Severity.CRITICAL: 3, +} + + +def max_severity(*levels: Severity) -> Severity: + if not levels: + raise ValueError("max_severity requires at least one Severity") + return max(levels, key=lambda s: s.rank) diff --git a/tests/unit/health/__init__.py b/tests/unit/health/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/health/test_severity.py b/tests/unit/health/test_severity.py new file mode 100644 index 0000000..4100d7b --- /dev/null +++ b/tests/unit/health/test_severity.py @@ -0,0 +1,27 @@ +from syncfield.health.severity import Severity, max_severity + + +def test_severity_values(): + assert Severity.INFO.value == "info" + assert Severity.WARNING.value == "warning" + assert Severity.ERROR.value == "error" + assert Severity.CRITICAL.value == "critical" + + +def test_severity_ordering(): + # INFO < WARNING < ERROR < CRITICAL + order = [Severity.INFO, Severity.WARNING, Severity.ERROR, Severity.CRITICAL] + for a, b in zip(order, order[1:]): + assert a.rank < b.rank + + +def test_max_severity_picks_highest(): + assert max_severity(Severity.INFO, Severity.WARNING) == Severity.WARNING + assert max_severity(Severity.ERROR, Severity.WARNING) == Severity.ERROR + assert max_severity(Severity.CRITICAL, Severity.INFO, Severity.ERROR) == Severity.CRITICAL + + +def test_max_severity_requires_at_least_one(): + import pytest + with pytest.raises(ValueError): + max_severity() From 742fca3ccfa900c7f74aca17c9446ce8414cffd1 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 12:33:17 +0900 Subject: [PATCH 04/64] chore: sync uv.lock to project version 0.3.19 --- uv.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uv.lock b/uv.lock index ea9da8f..ff85d05 100644 --- a/uv.lock +++ b/uv.lock @@ -2330,7 +2330,7 @@ wheels = [ [[package]] name = "syncfield" -version = "0.3.17" +version = "0.3.19" source = { editable = "." } dependencies = [ { name = "av", version = "15.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, From 5e957607c51f8bfb9a711cb5e47570077261661c Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 12:35:37 +0900 Subject: [PATCH 05/64] refactor(health): clarify severity _RANK rationale; tidy test imports --- src/syncfield/health/severity.py | 3 +++ tests/unit/health/test_severity.py | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/syncfield/health/severity.py b/src/syncfield/health/severity.py index 1f53679..af43190 100644 --- a/src/syncfield/health/severity.py +++ b/src/syncfield/health/severity.py @@ -21,6 +21,9 @@ def rank(self) -> int: return _RANK[self] +# `(str, Enum)` inherits alphabetical string comparison, which would order +# critical < error < info < warning — wrong for severity. `_RANK` pins the +# intended order explicitly and gives `.rank` O(1) lookup. _RANK = { Severity.INFO: 0, Severity.WARNING: 1, diff --git a/tests/unit/health/test_severity.py b/tests/unit/health/test_severity.py index 4100d7b..b352ec6 100644 --- a/tests/unit/health/test_severity.py +++ b/tests/unit/health/test_severity.py @@ -1,3 +1,5 @@ +import pytest + from syncfield.health.severity import Severity, max_severity @@ -19,9 +21,9 @@ def test_max_severity_picks_highest(): assert max_severity(Severity.INFO, Severity.WARNING) == Severity.WARNING assert max_severity(Severity.ERROR, Severity.WARNING) == Severity.ERROR assert max_severity(Severity.CRITICAL, Severity.INFO, Severity.ERROR) == Severity.CRITICAL + assert max_severity(Severity.ERROR) == Severity.ERROR def test_max_severity_requires_at_least_one(): - import pytest with pytest.raises(ValueError): max_severity() From 9813b1c0ab60591e1ab896ff01bf4e68718bab57 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 12:37:25 +0900 Subject: [PATCH 06/64] feat(types): enrich HealthEvent with severity/source/fingerprint/data --- src/syncfield/types.py | 22 ++++++++++++++++------ tests/unit/test_types.py | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/src/syncfield/types.py b/src/syncfield/types.py index 80747cf..fc07a07 100644 --- a/src/syncfield/types.py +++ b/src/syncfield/types.py @@ -14,6 +14,8 @@ from pathlib import Path from typing import Any, Literal, Union +from syncfield.health.severity import Severity + # Sensor channel value type. # Leaf values are always numeric (float | int). # Structure can be nested dicts or lists. @@ -249,19 +251,23 @@ class HealthEventKind(Enum): @dataclass(frozen=True) class HealthEvent: - """A stream reports a health observation to the orchestrator. + """A stream reports a health observation. - Attributes: - stream_id: Stream that emitted the event. - kind: Category of the event. - at_ns: ``time.monotonic_ns()`` when the event was observed. - detail: Optional free-form description. + ``severity`` / ``source`` / ``fingerprint`` / ``data`` enable the + incident-tracking layer in :mod:`syncfield.health` to group many raw + events into a single Sentry-style Incident. Adapters that don't care + can leave them at their safe defaults; the platform will fill them + in before the event reaches the IncidentTracker. """ stream_id: str kind: HealthEventKind at_ns: int detail: str | None = None + severity: Severity = Severity.INFO + source: str = "unknown" + fingerprint: str = "" + data: dict[str, Any] = field(default_factory=dict) def to_dict(self) -> dict[str, Any]: return { @@ -269,6 +275,10 @@ def to_dict(self) -> dict[str, Any]: "kind": self.kind.value, "at_ns": self.at_ns, "detail": self.detail, + "severity": self.severity.value, + "source": self.source, + "fingerprint": self.fingerprint, + "data": self.data, } diff --git a/tests/unit/test_types.py b/tests/unit/test_types.py index ed5d368..928753c 100644 --- a/tests/unit/test_types.py +++ b/tests/unit/test_types.py @@ -220,6 +220,10 @@ def test_to_dict(self): "kind": "drop", "at_ns": 42, "detail": "buffer overflow", + "severity": "info", + "source": "unknown", + "fingerprint": "", + "data": {}, } @@ -288,3 +292,38 @@ def test_minimal(self): ) assert report.host_id == "rig_01" assert report.finalizations == [] + + +from syncfield.health.severity import Severity + + +def test_health_event_has_enrichment_fields_with_defaults(): + ev = HealthEvent( + stream_id="cam", + kind=HealthEventKind.ERROR, + at_ns=1_000, + detail="boom", + ) + # new fields default to safe values when caller does not set them. + assert ev.severity == Severity.INFO + assert ev.source == "unknown" + assert ev.fingerprint == "" + assert ev.data == {} + + +def test_health_event_to_dict_includes_new_fields(): + ev = HealthEvent( + stream_id="cam", + kind=HealthEventKind.ERROR, + at_ns=1_000, + detail="boom", + severity=Severity.ERROR, + source="adapter:oak", + fingerprint="cam:adapter:xlink-error", + data={"stream": "__x_0_1"}, + ) + d = ev.to_dict() + assert d["severity"] == "error" + assert d["source"] == "adapter:oak" + assert d["fingerprint"] == "cam:adapter:xlink-error" + assert d["data"] == {"stream": "__x_0_1"} From 35afd863757cc2189640a87b09015eaf39c31e73 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 12:41:55 +0900 Subject: [PATCH 07/64] fix(types): shallow-copy HealthEvent data in to_dict; tidy test placement Co-Authored-By: Claude Opus 4.7 (1M context) --- src/syncfield/types.py | 2 +- tests/unit/test_types.py | 73 +++++++++++++++++++--------------------- 2 files changed, 36 insertions(+), 39 deletions(-) diff --git a/src/syncfield/types.py b/src/syncfield/types.py index fc07a07..a0e5b7b 100644 --- a/src/syncfield/types.py +++ b/src/syncfield/types.py @@ -278,7 +278,7 @@ def to_dict(self) -> dict[str, Any]: "severity": self.severity.value, "source": self.source, "fingerprint": self.fingerprint, - "data": self.data, + "data": dict(self.data), } diff --git a/tests/unit/test_types.py b/tests/unit/test_types.py index 928753c..a13b4be 100644 --- a/tests/unit/test_types.py +++ b/tests/unit/test_types.py @@ -1,6 +1,11 @@ """Tests for syncfield.types.""" +import pytest +from dataclasses import FrozenInstanceError +from pathlib import Path + from syncfield.types import FrameTimestamp, SensorSample, SyncPoint +from syncfield.health.severity import Severity def test_sync_point_create_now(): @@ -134,10 +139,6 @@ def test_sensor_sample_nested_round_trip(): assert restored.channels["gestures"]["pinch"] == 0.95 -import pytest -from dataclasses import FrozenInstanceError -from pathlib import Path - from syncfield.types import ( ChirpSpec, FinalizationReport, @@ -226,6 +227,36 @@ def test_to_dict(self): "data": {}, } + def test_health_event_has_enrichment_fields_with_defaults(self): + ev = HealthEvent( + stream_id="cam", + kind=HealthEventKind.ERROR, + at_ns=1_000, + detail="boom", + ) + # new fields default to safe values when caller does not set them. + assert ev.severity == Severity.INFO + assert ev.source == "unknown" + assert ev.fingerprint == "" + assert ev.data == {} + + def test_health_event_to_dict_includes_new_fields(self): + ev = HealthEvent( + stream_id="cam", + kind=HealthEventKind.ERROR, + at_ns=1_000, + detail="boom", + severity=Severity.ERROR, + source="adapter:oak", + fingerprint="cam:adapter:xlink-error", + data={"stream": "__x_0_1"}, + ) + d = ev.to_dict() + assert d["severity"] == "error" + assert d["source"] == "adapter:oak" + assert d["fingerprint"] == "cam:adapter:xlink-error" + assert d["data"] == {"stream": "__x_0_1"} + class TestSampleEvent: def test_minimal(self): @@ -293,37 +324,3 @@ def test_minimal(self): assert report.host_id == "rig_01" assert report.finalizations == [] - -from syncfield.health.severity import Severity - - -def test_health_event_has_enrichment_fields_with_defaults(): - ev = HealthEvent( - stream_id="cam", - kind=HealthEventKind.ERROR, - at_ns=1_000, - detail="boom", - ) - # new fields default to safe values when caller does not set them. - assert ev.severity == Severity.INFO - assert ev.source == "unknown" - assert ev.fingerprint == "" - assert ev.data == {} - - -def test_health_event_to_dict_includes_new_fields(): - ev = HealthEvent( - stream_id="cam", - kind=HealthEventKind.ERROR, - at_ns=1_000, - detail="boom", - severity=Severity.ERROR, - source="adapter:oak", - fingerprint="cam:adapter:xlink-error", - data={"stream": "__x_0_1"}, - ) - d = ev.to_dict() - assert d["severity"] == "error" - assert d["source"] == "adapter:oak" - assert d["fingerprint"] == "cam:adapter:xlink-error" - assert d["data"] == {"stream": "__x_0_1"} From 50a32815cff2266c5a76a6166088034002a53e68 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 12:45:13 +0900 Subject: [PATCH 08/64] feat(health): add WriterStats, Incident, IncidentArtifact, IncidentSnapshot --- src/syncfield/health/types.py | 161 +++++++++++++++++++++++ tests/unit/health/test_incident_types.py | 85 ++++++++++++ 2 files changed, 246 insertions(+) create mode 100644 src/syncfield/health/types.py create mode 100644 tests/unit/health/test_incident_types.py diff --git a/src/syncfield/health/types.py b/src/syncfield/health/types.py new file mode 100644 index 0000000..14ef9de --- /dev/null +++ b/src/syncfield/health/types.py @@ -0,0 +1,161 @@ +"""Data classes for the health/incident layer. + +These are plain, explicit structs — the :mod:`syncfield.health` runtime +mutates :class:`Incident` objects in-place on the worker thread. The +viewer receives immutable :class:`IncidentSnapshot`\\ s instead. +""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass, field +from typing import Any, List + +from syncfield.health.severity import Severity, max_severity +from syncfield.types import HealthEvent + + +@dataclass(frozen=True) +class WriterStats: + """One observation of a per-stream writer's queue.""" + + stream_id: str + at_ns: int + queue_depth: int + queue_capacity: int + dropped: int + + @property + def queue_fullness(self) -> float: + if self.queue_capacity <= 0: + return 0.0 + return self.queue_depth / self.queue_capacity + + +@dataclass(frozen=True) +class IncidentArtifact: + """A piece of evidence attached to an Incident (crash dump, log excerpt, ...).""" + + kind: str + path: str + detail: str | None = None + + def to_dict(self) -> dict[str, Any]: + return {"kind": self.kind, "path": self.path, "detail": self.detail} + + +@dataclass +class Incident: + """A grouped, open/close-tracked sequence of HealthEvents sharing a fingerprint. + + Mutable because the worker thread updates ``last_event`` / ``event_count`` + / ``severity`` on every matching event. The viewer never sees this + class directly — it reads :class:`IncidentSnapshot` instead. + """ + + id: str + stream_id: str + fingerprint: str + title: str + severity: Severity + source: str + opened_at_ns: int + closed_at_ns: int | None + last_event_at_ns: int + event_count: int + first_event: HealthEvent + last_event: HealthEvent + artifacts: List[IncidentArtifact] = field(default_factory=list) + data: dict[str, Any] = field(default_factory=dict) + + @classmethod + def opened_from(cls, event: HealthEvent, *, title: str) -> "Incident": + return cls( + id=uuid.uuid4().hex, + stream_id=event.stream_id, + fingerprint=event.fingerprint, + title=title, + severity=event.severity, + source=event.source, + opened_at_ns=event.at_ns, + closed_at_ns=None, + last_event_at_ns=event.at_ns, + event_count=1, + first_event=event, + last_event=event, + ) + + @property + def is_open(self) -> bool: + return self.closed_at_ns is None + + def record_event(self, event: HealthEvent) -> None: + self.event_count += 1 + self.last_event = event + self.last_event_at_ns = event.at_ns + self.severity = max_severity(self.severity, event.severity) + + def close(self, *, at_ns: int) -> None: + self.closed_at_ns = at_ns + + def attach(self, artifact: IncidentArtifact) -> None: + self.artifacts.append(artifact) + + def to_dict(self) -> dict[str, Any]: + return { + "id": self.id, + "stream_id": self.stream_id, + "fingerprint": self.fingerprint, + "title": self.title, + "severity": self.severity.value, + "source": self.source, + "opened_at_ns": self.opened_at_ns, + "closed_at_ns": self.closed_at_ns, + "last_event_at_ns": self.last_event_at_ns, + "event_count": self.event_count, + "first_event": self.first_event.to_dict(), + "last_event": self.last_event.to_dict(), + "artifacts": [a.to_dict() for a in self.artifacts], + "data": dict(self.data), + } + + +@dataclass(frozen=True) +class IncidentSnapshot: + """Read-only view of an Incident, for the viewer's WebSocket payload.""" + + id: str + stream_id: str + fingerprint: str + title: str + severity: str + source: str + opened_at_ns: int + closed_at_ns: int | None + event_count: int + detail: str | None + ago_s: float + artifacts: List[dict] + + @property + def is_open(self) -> bool: + return self.closed_at_ns is None + + @classmethod + def from_incident(cls, inc: Incident, *, now_ns: int) -> "IncidentSnapshot": + anchor = inc.closed_at_ns if inc.closed_at_ns is not None else inc.last_event_at_ns + ago_s = max(0.0, (now_ns - anchor) / 1e9) + return cls( + id=inc.id, + stream_id=inc.stream_id, + fingerprint=inc.fingerprint, + title=inc.title, + severity=inc.severity.value, + source=inc.source, + opened_at_ns=inc.opened_at_ns, + closed_at_ns=inc.closed_at_ns, + event_count=inc.event_count, + detail=inc.last_event.detail, + ago_s=ago_s, + artifacts=[a.to_dict() for a in inc.artifacts], + ) diff --git a/tests/unit/health/test_incident_types.py b/tests/unit/health/test_incident_types.py new file mode 100644 index 0000000..3abcbc0 --- /dev/null +++ b/tests/unit/health/test_incident_types.py @@ -0,0 +1,85 @@ +from syncfield.health.severity import Severity +from syncfield.health.types import ( + Incident, + IncidentArtifact, + IncidentSnapshot, + WriterStats, +) +from syncfield.types import HealthEvent, HealthEventKind + + +def _ev(at_ns: int, severity: Severity = Severity.ERROR) -> HealthEvent: + return HealthEvent( + stream_id="cam", + kind=HealthEventKind.ERROR, + at_ns=at_ns, + detail="x", + severity=severity, + source="detector:stream-stall", + fingerprint="cam:stream-stall", + ) + + +def test_writer_stats_fields(): + s = WriterStats( + stream_id="cam", + at_ns=100, + queue_depth=3, + queue_capacity=16, + dropped=0, + ) + assert s.queue_fullness == 3 / 16 + assert s.stream_id == "cam" + + +def test_writer_stats_zero_capacity_is_empty(): + s = WriterStats(stream_id="cam", at_ns=0, queue_depth=0, queue_capacity=0, dropped=0) + assert s.queue_fullness == 0.0 + + +def test_incident_from_first_event_initializes_fields(): + first = _ev(100) + inc = Incident.opened_from(first, title="Stream stalled (silence 2.0s)") + assert inc.stream_id == "cam" + assert inc.fingerprint == "cam:stream-stall" + assert inc.severity == Severity.ERROR + assert inc.title == "Stream stalled (silence 2.0s)" + assert inc.opened_at_ns == 100 + assert inc.closed_at_ns is None + assert inc.event_count == 1 + assert inc.first_event == first + assert inc.last_event == first + assert inc.artifacts == [] + + +def test_incident_record_event_escalates_severity_and_updates_last(): + inc = Incident.opened_from(_ev(100, severity=Severity.WARNING), title="t") + inc.record_event(_ev(200, severity=Severity.ERROR)) + assert inc.event_count == 2 + assert inc.severity == Severity.ERROR + assert inc.last_event.at_ns == 200 + assert inc.last_event_at_ns == 200 + + +def test_incident_close(): + inc = Incident.opened_from(_ev(100), title="t") + inc.close(at_ns=500) + assert inc.closed_at_ns == 500 + assert inc.is_open is False + + +def test_incident_attach_artifact(): + inc = Incident.opened_from(_ev(100), title="t") + inc.attach(IncidentArtifact(kind="crash_dump", path="/tmp/x.json")) + assert inc.artifacts[0].kind == "crash_dump" + assert inc.artifacts[0].path == "/tmp/x.json" + + +def test_incident_snapshot_shape(): + inc = Incident.opened_from(_ev(100), title="t") + snap = IncidentSnapshot.from_incident(inc, now_ns=1_000_000_100) + assert snap.id == inc.id + assert snap.stream_id == "cam" + assert snap.severity == "error" + assert snap.is_open is True + assert snap.ago_s >= 0 From 0b2c055446ef0bc77e57c75c4c5ca646576f9997 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 12:48:47 +0900 Subject: [PATCH 09/64] test(health): cover closed-incident anchor; tighten artifacts type --- src/syncfield/health/types.py | 2 +- tests/unit/health/test_incident_types.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/syncfield/health/types.py b/src/syncfield/health/types.py index 14ef9de..629ec18 100644 --- a/src/syncfield/health/types.py +++ b/src/syncfield/health/types.py @@ -135,7 +135,7 @@ class IncidentSnapshot: event_count: int detail: str | None ago_s: float - artifacts: List[dict] + artifacts: List[dict[str, Any]] @property def is_open(self) -> bool: diff --git a/tests/unit/health/test_incident_types.py b/tests/unit/health/test_incident_types.py index 3abcbc0..20d2ea6 100644 --- a/tests/unit/health/test_incident_types.py +++ b/tests/unit/health/test_incident_types.py @@ -83,3 +83,16 @@ def test_incident_snapshot_shape(): assert snap.severity == "error" assert snap.is_open is True assert snap.ago_s >= 0 + + +def test_incident_snapshot_uses_closed_at_as_anchor_when_closed(): + inc = Incident.opened_from(_ev(1_000_000_000), title="t") + inc.record_event(_ev(1_100_000_000)) # last_event_at_ns = 1.1s + inc.close(at_ns=1_500_000_000) # closed 500ms later + + # now_ns is 2s after close + snap = IncidentSnapshot.from_incident(inc, now_ns=3_500_000_000) + assert snap.is_open is False + assert snap.closed_at_ns == 1_500_000_000 + # ago_s is measured from closed_at_ns, not last_event_at_ns. + assert abs(snap.ago_s - 2.0) < 0.01 From 41cf1e7405a3365f4ffe2a26530330af93bb956c Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 12:50:07 +0900 Subject: [PATCH 10/64] feat(health): add Detector protocol + DetectorBase --- src/syncfield/health/detector.py | 77 +++++++++++++++++++++++++ tests/unit/health/test_detector_base.py | 32 ++++++++++ 2 files changed, 109 insertions(+) create mode 100644 src/syncfield/health/detector.py create mode 100644 tests/unit/health/test_detector_base.py diff --git a/src/syncfield/health/detector.py b/src/syncfield/health/detector.py new file mode 100644 index 0000000..b78a6a9 --- /dev/null +++ b/src/syncfield/health/detector.py @@ -0,0 +1,77 @@ +"""Detector protocol + base class. + +A :class:`Detector` observes the stream (samples, adapter health events, +session state, writer stats) and may emit :class:`HealthEvent` on each +``tick()``. The :class:`IncidentTracker` groups emitted events by +fingerprint, opens incidents, and consults ``close_condition`` to know +when an open incident should resolve. + +Most detectors subclass :class:`DetectorBase` and override only the +observe/tick hooks they care about; the base provides safe no-op +defaults for the rest. +""" + +from __future__ import annotations + +from typing import Iterable, Iterator, Protocol, runtime_checkable + +from syncfield.health.severity import Severity +from syncfield.health.types import Incident, WriterStats +from syncfield.types import HealthEvent, SampleEvent, SessionState + + +@runtime_checkable +class Detector(Protocol): + name: str + default_severity: Severity + + def observe_sample(self, stream_id: str, sample: SampleEvent) -> None: ... + def observe_health(self, stream_id: str, event: HealthEvent) -> None: ... + def observe_state(self, old: SessionState, new: SessionState) -> None: ... + def observe_writer_stats(self, stream_id: str, stats: WriterStats) -> None: ... + def tick(self, now_ns: int) -> Iterable[HealthEvent]: ... + def close_condition(self, incident: Incident, now_ns: int) -> bool: ... + + +class DetectorBase: + """No-op defaults for every Detector hook. + + Subclasses set ``name`` and ``default_severity`` at the class level + and override only the hooks that matter for their rule. + """ + + name: str + default_severity: Severity + + def __init_subclass__(cls, **kwargs: object) -> None: + super().__init_subclass__(**kwargs) + # Guard against subclasses that forget to set required class attrs. + for attr in ("name", "default_severity"): + if not hasattr(cls, attr): + raise TypeError( + f"Detector subclass {cls.__name__} must set class attribute '{attr}'" + ) + + def __new__(cls, *args: object, **kwargs: object) -> "DetectorBase": + if cls is DetectorBase: + raise TypeError("DetectorBase is abstract; subclass it") + return super().__new__(cls) + + def observe_sample(self, stream_id: str, sample: SampleEvent) -> None: + pass + + def observe_health(self, stream_id: str, event: HealthEvent) -> None: + pass + + def observe_state(self, old: SessionState, new: SessionState) -> None: + pass + + def observe_writer_stats(self, stream_id: str, stats: WriterStats) -> None: + pass + + def tick(self, now_ns: int) -> Iterator[HealthEvent]: + return iter(()) + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + # Conservative default: keep open. Subclasses override to close. + return False diff --git a/tests/unit/health/test_detector_base.py b/tests/unit/health/test_detector_base.py new file mode 100644 index 0000000..5bd76c9 --- /dev/null +++ b/tests/unit/health/test_detector_base.py @@ -0,0 +1,32 @@ +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.types import WriterStats +from syncfield.types import HealthEvent, HealthEventKind, SampleEvent, SessionState + + +class NoopDetector(DetectorBase): + name = "noop" + default_severity = Severity.WARNING + + +def test_detector_base_defaults_are_noops(): + d = NoopDetector() + # All observers accept calls without raising. + d.observe_sample("cam", SampleEvent(stream_id="cam", frame_number=1, capture_ns=100)) + d.observe_health("cam", HealthEvent(stream_id="cam", kind=HealthEventKind.WARNING, at_ns=1)) + d.observe_state(SessionState.IDLE, SessionState.CONNECTED) + d.observe_writer_stats("cam", WriterStats("cam", 1, 0, 0, 0)) + # tick yields nothing by default. + assert list(d.tick(now_ns=100)) == [] + # close_condition defaults to False (conservative: keep open; subclasses override). + from syncfield.health.types import Incident + ev = HealthEvent(stream_id="cam", kind=HealthEventKind.WARNING, at_ns=1) + inc = Incident.opened_from(ev, title="x") + assert isinstance(d.close_condition(inc, now_ns=10), bool) + + +def test_detector_base_requires_name_and_severity(): + import pytest + + with pytest.raises(TypeError): + DetectorBase() # abstract base: name / default_severity unset on the class From 53cddc3907eca53740e7e6ce9deb8c62f8eb24ab Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 12:52:26 +0900 Subject: [PATCH 11/64] =?UTF-8?q?fix(health):=20tighten=20Detector=20contr?= =?UTF-8?q?acts=20=E2=80=94=20MRO-walking=20=5F=5Finit=5Fsubclass=5F=5F=20?= =?UTF-8?q?and=20Iterator=20types?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/syncfield/health/detector.py | 18 ++++++++++++++---- tests/unit/health/test_detector_base.py | 20 +++++++++++++++++--- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/src/syncfield/health/detector.py b/src/syncfield/health/detector.py index b78a6a9..5116313 100644 --- a/src/syncfield/health/detector.py +++ b/src/syncfield/health/detector.py @@ -13,7 +13,7 @@ from __future__ import annotations -from typing import Iterable, Iterator, Protocol, runtime_checkable +from typing import Iterator, Protocol, runtime_checkable from syncfield.health.severity import Severity from syncfield.health.types import Incident, WriterStats @@ -29,7 +29,7 @@ def observe_sample(self, stream_id: str, sample: SampleEvent) -> None: ... def observe_health(self, stream_id: str, event: HealthEvent) -> None: ... def observe_state(self, old: SessionState, new: SessionState) -> None: ... def observe_writer_stats(self, stream_id: str, stats: WriterStats) -> None: ... - def tick(self, now_ns: int) -> Iterable[HealthEvent]: ... + def tick(self, now_ns: int) -> Iterator[HealthEvent]: ... def close_condition(self, incident: Incident, now_ns: int) -> bool: ... @@ -45,9 +45,19 @@ class DetectorBase: def __init_subclass__(cls, **kwargs: object) -> None: super().__init_subclass__(**kwargs) - # Guard against subclasses that forget to set required class attrs. + # Require each subclass chain to end at a concrete class that declares + # these attrs. We walk the MRO up to (but not including) DetectorBase + # and assert at least one class in that chain sets each attribute + # as an own attribute (via __dict__). This prevents a grandchild from + # silently inheriting a default it shouldn't while still allowing + # legitimate intermediate base classes. for attr in ("name", "default_severity"): - if not hasattr(cls, attr): + declared = any( + attr in klass.__dict__ + for klass in cls.__mro__ + if klass is not DetectorBase and klass is not object + ) + if not declared: raise TypeError( f"Detector subclass {cls.__name__} must set class attribute '{attr}'" ) diff --git a/tests/unit/health/test_detector_base.py b/tests/unit/health/test_detector_base.py index 5bd76c9..4f4b5d2 100644 --- a/tests/unit/health/test_detector_base.py +++ b/tests/unit/health/test_detector_base.py @@ -1,3 +1,5 @@ +import pytest + from syncfield.health.detector import DetectorBase from syncfield.health.severity import Severity from syncfield.health.types import WriterStats @@ -22,11 +24,23 @@ def test_detector_base_defaults_are_noops(): from syncfield.health.types import Incident ev = HealthEvent(stream_id="cam", kind=HealthEventKind.WARNING, at_ns=1) inc = Incident.opened_from(ev, title="x") - assert isinstance(d.close_condition(inc, now_ns=10), bool) + assert d.close_condition(inc, now_ns=10) is False def test_detector_base_requires_name_and_severity(): - import pytest - with pytest.raises(TypeError): DetectorBase() # abstract base: name / default_severity unset on the class + + +def test_grandchild_subclass_must_redeclare_if_needed(): + # Sanity: once a parent sets name/default_severity, grandchildren inheriting + # them pass the check (legitimate use case). + class Grandchild(NoopDetector): + pass + Grandchild() # should not raise + + # A subclass that overrides nothing but lacks both required attrs + # cannot exist — we cannot construct such a test directly without + # subclassing DetectorBase again (which must itself declare attrs). + # The more important invariant is covered by the existing + # test_detector_base_requires_name_and_severity. From b8f0ff50b0d6142ec3cdb804d1dc1685e6acb567 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 12:53:15 +0900 Subject: [PATCH 12/64] feat(health): add DetectorRegistry --- src/syncfield/health/registry.py | 26 ++++++++++++++++++ tests/unit/health/test_registry.py | 44 ++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 src/syncfield/health/registry.py create mode 100644 tests/unit/health/test_registry.py diff --git a/src/syncfield/health/registry.py b/src/syncfield/health/registry.py new file mode 100644 index 0000000..ce66fc8 --- /dev/null +++ b/src/syncfield/health/registry.py @@ -0,0 +1,26 @@ +"""Registry of active Detectors for a session.""" + +from __future__ import annotations + +from typing import Iterator, List + +from syncfield.health.detector import Detector + + +class DetectorRegistry: + def __init__(self) -> None: + self._detectors: List[Detector] = [] + + def register(self, detector: Detector) -> None: + if any(d.name == detector.name for d in self._detectors): + raise ValueError(f"Detector '{detector.name}' is already registered") + self._detectors.append(detector) + + def unregister(self, name: str) -> None: + self._detectors = [d for d in self._detectors if d.name != name] + + def __iter__(self) -> Iterator[Detector]: + return iter(list(self._detectors)) + + def __len__(self) -> int: + return len(self._detectors) diff --git a/tests/unit/health/test_registry.py b/tests/unit/health/test_registry.py new file mode 100644 index 0000000..2d631fe --- /dev/null +++ b/tests/unit/health/test_registry.py @@ -0,0 +1,44 @@ +import pytest + +from syncfield.health.detector import DetectorBase +from syncfield.health.registry import DetectorRegistry +from syncfield.health.severity import Severity + + +class Det(DetectorBase): + name = "d1" + default_severity = Severity.WARNING + + +class Det2(DetectorBase): + name = "d2" + default_severity = Severity.ERROR + + +def test_register_and_iterate(): + reg = DetectorRegistry() + d1 = Det() + d2 = Det2() + reg.register(d1) + reg.register(d2) + assert list(reg) == [d1, d2] + + +def test_register_duplicate_name_raises(): + reg = DetectorRegistry() + reg.register(Det()) + with pytest.raises(ValueError, match="already registered"): + reg.register(Det()) + + +def test_unregister_removes_by_name(): + reg = DetectorRegistry() + d1 = Det() + reg.register(d1) + reg.unregister("d1") + assert list(reg) == [] + + +def test_unregister_unknown_is_noop(): + reg = DetectorRegistry() + reg.unregister("nope") # does not raise From e0f6c620b5a7adb9c17b0f673589d85f3d77aa59 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 12:55:46 +0900 Subject: [PATCH 13/64] feat(health): add IncidentTracker with fingerprint grouping and close conditions Co-Authored-By: Claude Opus 4.7 (1M context) --- src/syncfield/health/tracker.py | 101 ++++++++++++++++++ tests/unit/health/test_incident_tracker.py | 115 +++++++++++++++++++++ 2 files changed, 216 insertions(+) create mode 100644 src/syncfield/health/tracker.py create mode 100644 tests/unit/health/test_incident_tracker.py diff --git a/src/syncfield/health/tracker.py b/src/syncfield/health/tracker.py new file mode 100644 index 0000000..9427298 --- /dev/null +++ b/src/syncfield/health/tracker.py @@ -0,0 +1,101 @@ +"""IncidentTracker — groups HealthEvents into Incidents and manages open/close. + +Runs on the HealthWorker thread. Public methods are *not* thread-safe on +their own; the worker serializes access. +""" + +from __future__ import annotations + +from typing import Callable, Dict, List, Optional + +from syncfield.health.detector import Detector +from syncfield.health.types import Incident +from syncfield.types import HealthEvent + +Callback = Callable[[Incident], None] + + +class IncidentTracker: + def __init__(self, passthrough_close_ns: int = 30 * 1_000_000_000) -> None: + self._by_fingerprint: Dict[str, Incident] = {} + self._resolved: List[Incident] = [] + self._detectors_by_name: Dict[str, Detector] = {} + self._passthrough_close_ns = passthrough_close_ns + + self.on_opened: Optional[Callback] = None + self.on_updated: Optional[Callback] = None + self.on_closed: Optional[Callback] = None + + # --- detector wiring ------------------------------------------------- + + def bind_detector(self, detector: Detector) -> None: + self._detectors_by_name[detector.name] = detector + + # --- event ingestion ------------------------------------------------- + + def ingest(self, event: HealthEvent) -> None: + open_inc = self._by_fingerprint.get(event.fingerprint) + if open_inc is None: + inc = Incident.opened_from(event, title=_title_from(event)) + self._by_fingerprint[event.fingerprint] = inc + self._fire(self.on_opened, inc) + return + open_inc.record_event(event) + self._fire(self.on_updated, open_inc) + + # --- tick — evaluate close conditions -------------------------------- + + def tick(self, now_ns: int) -> None: + to_close: List[str] = [] + for fp, inc in self._by_fingerprint.items(): + detector = self._detector_for(inc) + if detector is not None: + should_close = detector.close_condition(inc, now_ns) + else: + should_close = (now_ns - inc.last_event_at_ns) >= self._passthrough_close_ns + if should_close: + to_close.append(fp) + + for fp in to_close: + inc = self._by_fingerprint.pop(fp) + inc.close(at_ns=now_ns) + self._resolved.append(inc) + self._fire(self.on_closed, inc) + + def close_all(self, *, at_ns: int) -> None: + """Used at session stop to resolve any still-open incidents.""" + for fp in list(self._by_fingerprint.keys()): + inc = self._by_fingerprint.pop(fp) + inc.close(at_ns=at_ns) + self._resolved.append(inc) + self._fire(self.on_closed, inc) + + # --- read-only views ------------------------------------------------- + + def open_incidents(self) -> List[Incident]: + return list(self._by_fingerprint.values()) + + def resolved_incidents(self) -> List[Incident]: + return list(self._resolved) + + # --- helpers --------------------------------------------------------- + + def _detector_for(self, inc: Incident) -> Optional[Detector]: + # Fingerprint convention: ":[:suffix]". + parts = inc.fingerprint.split(":", 2) + if len(parts) < 2: + return None + return self._detectors_by_name.get(parts[1]) + + @staticmethod + def _fire(cb: Optional[Callback], inc: Incident) -> None: + if cb is not None: + cb(inc) + + +def _title_from(event: HealthEvent) -> str: + # Prefer the first event's detail as the title; fall back to + # ": " if the detail is missing. + if event.detail: + return event.detail + return f"{event.source}: {event.fingerprint}" diff --git a/tests/unit/health/test_incident_tracker.py b/tests/unit/health/test_incident_tracker.py new file mode 100644 index 0000000..5553926 --- /dev/null +++ b/tests/unit/health/test_incident_tracker.py @@ -0,0 +1,115 @@ +from typing import List + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.tracker import IncidentTracker +from syncfield.health.types import Incident +from syncfield.types import HealthEvent, HealthEventKind + + +def _ev(at_ns: int, fingerprint: str = "cam:stall", severity: Severity = Severity.ERROR, + detail: str = "x") -> HealthEvent: + return HealthEvent( + stream_id="cam", + kind=HealthEventKind.ERROR, + at_ns=at_ns, + detail=detail, + severity=severity, + source="detector:stream-stall", + fingerprint=fingerprint, + ) + + +class AlwaysCloseAfter(DetectorBase): + name = "stall" + default_severity = Severity.ERROR + + def __init__(self, close_after_ns: int) -> None: + self._close_after = close_after_ns + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + return now_ns - incident.last_event_at_ns >= self._close_after + + +def test_tracker_opens_incident_on_first_matching_event(): + tr = IncidentTracker() + tr.bind_detector(AlwaysCloseAfter(close_after_ns=1000)) + opened: List[Incident] = [] + tr.on_opened = opened.append + + tr.ingest(_ev(100)) + + assert len(tr.open_incidents()) == 1 + assert opened and opened[0].stream_id == "cam" + + +def test_tracker_groups_same_fingerprint_into_one_incident(): + tr = IncidentTracker() + tr.bind_detector(AlwaysCloseAfter(close_after_ns=1_000_000_000)) + + tr.ingest(_ev(100, severity=Severity.WARNING)) + tr.ingest(_ev(200, severity=Severity.ERROR)) # escalate + tr.ingest(_ev(300, severity=Severity.ERROR)) + + opens = tr.open_incidents() + assert len(opens) == 1 + inc = opens[0] + assert inc.event_count == 3 + assert inc.severity == Severity.ERROR + assert inc.last_event_at_ns == 300 + + +def test_tracker_closes_incident_when_detector_close_condition_fires(): + tr = IncidentTracker() + tr.bind_detector(AlwaysCloseAfter(close_after_ns=500)) + closed: List[Incident] = [] + tr.on_closed = closed.append + + tr.ingest(_ev(100)) + tr.tick(now_ns=200) # 100 ns since last event, not yet + assert tr.resolved_incidents() == [] + + tr.tick(now_ns=700) # 600 ns since last event → close + assert len(tr.resolved_incidents()) == 1 + assert tr.open_incidents() == [] + assert closed and closed[0].closed_at_ns == 700 + + +def test_tracker_reopens_after_close_on_same_fingerprint(): + tr = IncidentTracker() + tr.bind_detector(AlwaysCloseAfter(close_after_ns=100)) + + tr.ingest(_ev(100)) + tr.tick(now_ns=500) # closed + assert tr.open_incidents() == [] + + tr.ingest(_ev(1000)) # new incident, new id + opens = tr.open_incidents() + assert len(opens) == 1 + assert len(tr.resolved_incidents()) == 1 + assert opens[0].id != tr.resolved_incidents()[0].id + + +def test_tracker_unbound_fingerprint_falls_back_to_passthrough_close(): + # When an event arrives with a fingerprint whose detector is not bound, + # the tracker still groups it, using the default passthrough close + # window (30s of quiet). + tr = IncidentTracker(passthrough_close_ns=500) + + tr.ingest(_ev(100, fingerprint="cam:adapter:xlink")) + tr.tick(now_ns=400) + assert tr.open_incidents() + tr.tick(now_ns=1000) # 900 ns since last event → closes + assert tr.resolved_incidents() + + +def test_tracker_flush_callbacks_fire_on_update_too(): + tr = IncidentTracker() + tr.bind_detector(AlwaysCloseAfter(close_after_ns=1_000_000_000)) + updates: List[Incident] = [] + tr.on_updated = updates.append + + tr.ingest(_ev(100)) # opens + tr.ingest(_ev(200)) # updates + tr.ingest(_ev(300)) # updates + assert len(updates) == 2 From 456a687501b4666d764ef582faa065d40e413e59 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 12:57:48 +0900 Subject: [PATCH 14/64] fix(health): reject empty fingerprints in IncidentTracker.ingest --- src/syncfield/health/tracker.py | 5 +++++ tests/unit/health/test_incident_tracker.py | 13 +++++++++++++ 2 files changed, 18 insertions(+) diff --git a/src/syncfield/health/tracker.py b/src/syncfield/health/tracker.py index 9427298..b34051c 100644 --- a/src/syncfield/health/tracker.py +++ b/src/syncfield/health/tracker.py @@ -34,6 +34,11 @@ def bind_detector(self, detector: Detector) -> None: # --- event ingestion ------------------------------------------------- def ingest(self, event: HealthEvent) -> None: + if not event.fingerprint: + raise ValueError( + "HealthEvent.fingerprint is required before reaching the IncidentTracker; " + "the platform fills it in for adapter events, detectors set their own." + ) open_inc = self._by_fingerprint.get(event.fingerprint) if open_inc is None: inc = Incident.opened_from(event, title=_title_from(event)) diff --git a/tests/unit/health/test_incident_tracker.py b/tests/unit/health/test_incident_tracker.py index 5553926..f1d8ea0 100644 --- a/tests/unit/health/test_incident_tracker.py +++ b/tests/unit/health/test_incident_tracker.py @@ -1,5 +1,7 @@ from typing import List +import pytest + from syncfield.health.detector import DetectorBase from syncfield.health.severity import Severity from syncfield.health.tracker import IncidentTracker @@ -113,3 +115,14 @@ def test_tracker_flush_callbacks_fire_on_update_too(): tr.ingest(_ev(200)) # updates tr.ingest(_ev(300)) # updates assert len(updates) == 2 + + +def test_tracker_rejects_empty_fingerprint(): + tr = IncidentTracker() + with pytest.raises(ValueError, match="fingerprint"): + tr.ingest(HealthEvent( + stream_id="cam", + kind=HealthEventKind.ERROR, + at_ns=1, + # fingerprint defaults to "" + )) From a08af9c6a9900a6b08761ffea7781e172f1cd7c7 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 12:59:54 +0900 Subject: [PATCH 15/64] feat(health): add HealthWorker (daemon thread with ingress queues and tick loop) Co-Authored-By: Claude Sonnet 4.6 --- src/syncfield/health/worker.py | 148 ++++++++++++++++++++++++ tests/unit/health/test_health_worker.py | 128 ++++++++++++++++++++ 2 files changed, 276 insertions(+) create mode 100644 src/syncfield/health/worker.py create mode 100644 tests/unit/health/test_health_worker.py diff --git a/src/syncfield/health/worker.py b/src/syncfield/health/worker.py new file mode 100644 index 0000000..7ef17c2 --- /dev/null +++ b/src/syncfield/health/worker.py @@ -0,0 +1,148 @@ +"""HealthWorker — the dedicated thread that drives detectors + tracker. + +Capture threads push samples / health events / state transitions / +writer stats into :class:`queue.SimpleQueue`\\ s. The worker drains them +every tick, fans out to each registered Detector, runs each Detector's +``tick`` to emit synthetic events, and feeds everything into the +IncidentTracker. +""" + +from __future__ import annotations + +import queue +import threading +import time +from dataclasses import dataclass +from typing import Iterable, List + +from syncfield.health.detector import Detector +from syncfield.health.tracker import IncidentTracker +from syncfield.health.types import WriterStats +from syncfield.types import HealthEvent, SampleEvent, SessionState + + +@dataclass(frozen=True) +class _SampleMsg: + stream_id: str + sample: SampleEvent + + +@dataclass(frozen=True) +class _HealthMsg: + stream_id: str + event: HealthEvent + + +@dataclass(frozen=True) +class _StateMsg: + old: SessionState + new: SessionState + + +@dataclass(frozen=True) +class _WriterStatsMsg: + stream_id: str + stats: WriterStats + + +class HealthWorker: + def __init__( + self, + *, + tracker: IncidentTracker, + detectors: Iterable[Detector], + tick_hz: float = 20.0, + ) -> None: + self._tracker = tracker + self._detectors: List[Detector] = list(detectors) + self._tick_interval = 1.0 / tick_hz + + self._samples: "queue.SimpleQueue[_SampleMsg]" = queue.SimpleQueue() + self._healths: "queue.SimpleQueue[_HealthMsg]" = queue.SimpleQueue() + self._states: "queue.SimpleQueue[_StateMsg]" = queue.SimpleQueue() + self._writer_stats: "queue.SimpleQueue[_WriterStatsMsg]" = queue.SimpleQueue() + + self._stop = threading.Event() + self._thread: threading.Thread | None = None + + # --- ingress (called from capture threads) --------------------------- + + def push_sample(self, stream_id: str, sample: SampleEvent) -> None: + self._samples.put(_SampleMsg(stream_id, sample)) + + def push_health(self, stream_id: str, event: HealthEvent) -> None: + self._healths.put(_HealthMsg(stream_id, event)) + + def push_state(self, old: SessionState, new: SessionState) -> None: + self._states.put(_StateMsg(old, new)) + + def push_writer_stats(self, stream_id: str, stats: WriterStats) -> None: + self._writer_stats.put(_WriterStatsMsg(stream_id, stats)) + + # --- lifecycle ------------------------------------------------------- + + def start(self) -> None: + if self._thread is not None and self._thread.is_alive(): + return + self._stop.clear() + self._thread = threading.Thread( + target=self._run, name="syncfield-health", daemon=True + ) + self._thread.start() + + def stop(self) -> None: + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=2.0) + self._thread = None + + # --- main loop ------------------------------------------------------- + + def _run(self) -> None: + next_deadline = time.monotonic() + while not self._stop.is_set(): + self._drain_once() + self._fire_detector_ticks() + self._tracker.tick(now_ns=time.monotonic_ns()) + + next_deadline += self._tick_interval + sleep_for = next_deadline - time.monotonic() + if sleep_for > 0: + # Use Event.wait so stop() can interrupt immediately. + self._stop.wait(timeout=sleep_for) + else: + # Running behind; reset the schedule anchor. + next_deadline = time.monotonic() + + # Drain any straggling messages after stop so tests see a consistent state. + self._drain_once() + + def _drain_once(self) -> None: + for msg in _drain_queue(self._samples): + for d in self._detectors: + d.observe_sample(msg.stream_id, msg.sample) + for msg in _drain_queue(self._healths): + for d in self._detectors: + d.observe_health(msg.stream_id, msg.event) + self._tracker.ingest(msg.event) + for msg in _drain_queue(self._states): + for d in self._detectors: + d.observe_state(msg.old, msg.new) + for msg in _drain_queue(self._writer_stats): + for d in self._detectors: + d.observe_writer_stats(msg.stream_id, msg.stats) + + def _fire_detector_ticks(self) -> None: + now = time.monotonic_ns() + for d in self._detectors: + for event in d.tick(now): + self._tracker.ingest(event) + + +def _drain_queue(q: "queue.SimpleQueue") -> List: + out = [] + while True: + try: + out.append(q.get_nowait()) + except queue.Empty: + return out diff --git a/tests/unit/health/test_health_worker.py b/tests/unit/health/test_health_worker.py new file mode 100644 index 0000000..733e23a --- /dev/null +++ b/tests/unit/health/test_health_worker.py @@ -0,0 +1,128 @@ +import threading +import time + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.tracker import IncidentTracker +from syncfield.health.types import WriterStats +from syncfield.health.worker import HealthWorker +from syncfield.types import HealthEvent, HealthEventKind, SampleEvent, SessionState + + +class RecordingDetector(DetectorBase): + name = "recorder" + default_severity = Severity.INFO + + def __init__(self) -> None: + self.samples = [] + self.healths = [] + self.states = [] + self.writer_stats = [] + self.ticks = 0 + + def observe_sample(self, stream_id, sample): + self.samples.append((stream_id, sample.capture_ns)) + + def observe_health(self, stream_id, event): + self.healths.append((stream_id, event.at_ns)) + + def observe_state(self, old, new): + self.states.append((old, new)) + + def observe_writer_stats(self, stream_id, stats): + self.writer_stats.append((stream_id, stats.queue_depth)) + + def tick(self, now_ns): + self.ticks += 1 + return iter(()) + + +def _wait_until(pred, timeout=1.0, interval=0.01): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if pred(): + return True + time.sleep(interval) + return False + + +def test_worker_drains_all_ingress_queues_on_tick(): + tr = IncidentTracker() + det = RecordingDetector() + tr.bind_detector(det) + w = HealthWorker(tracker=tr, detectors=[det], tick_hz=100) + + w.start() + try: + w.push_sample("cam", SampleEvent(stream_id="cam", frame_number=1, capture_ns=42)) + # IMPORTANT: health events must now carry a non-empty fingerprint because + # Task 6 added a guard in IncidentTracker.ingest. Provide one. + w.push_health("cam", HealthEvent( + stream_id="cam", kind=HealthEventKind.WARNING, at_ns=1, + severity=Severity.WARNING, source="test", fingerprint="cam:test:ingest", + )) + w.push_state(SessionState.IDLE, SessionState.CONNECTED) + w.push_writer_stats("cam", WriterStats("cam", 1, 2, 16, 0)) + + assert _wait_until(lambda: det.samples and det.healths and det.states and det.writer_stats) + finally: + w.stop() + + assert det.samples[0] == ("cam", 42) + assert det.healths[0] == ("cam", 1) + assert det.states[0] == (SessionState.IDLE, SessionState.CONNECTED) + assert det.writer_stats[0] == ("cam", 2) + + +def test_worker_ticks_at_roughly_configured_rate(): + tr = IncidentTracker() + det = RecordingDetector() + w = HealthWorker(tracker=tr, detectors=[det], tick_hz=50) + w.start() + try: + time.sleep(0.2) # ~10 ticks + finally: + w.stop() + # Loose bound to avoid flakiness under loaded CI. + assert det.ticks >= 5 + + +def test_worker_feeds_detector_tick_output_into_tracker(): + class EmitsOneAndDone(DetectorBase): + name = "emit" + default_severity = Severity.WARNING + + def __init__(self): + self.fired = False + + def tick(self, now_ns): + if self.fired: + return iter(()) + self.fired = True + return iter([HealthEvent( + stream_id="cam", kind=HealthEventKind.WARNING, at_ns=now_ns, + detail="synthetic", severity=Severity.WARNING, + source="detector:emit", fingerprint="cam:emit", + )]) + + def close_condition(self, inc, now_ns): + return False + + tr = IncidentTracker() + det = EmitsOneAndDone() + tr.bind_detector(det) + w = HealthWorker(tracker=tr, detectors=[det], tick_hz=100) + w.start() + try: + assert _wait_until(lambda: len(tr.open_incidents()) == 1) + finally: + w.stop() + + +def test_worker_stop_is_idempotent(): + tr = IncidentTracker() + det = RecordingDetector() + w = HealthWorker(tracker=tr, detectors=[det], tick_hz=50) + w.start() + w.stop() + w.stop() # does not raise From 23dc42b558d2fbb01352dd660ca2203f80b4bf0c Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 13:02:08 +0900 Subject: [PATCH 16/64] refactor(health): HealthWorker polish (anchor, comments, imports) --- src/syncfield/health/worker.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/syncfield/health/worker.py b/src/syncfield/health/worker.py index 7ef17c2..e209af9 100644 --- a/src/syncfield/health/worker.py +++ b/src/syncfield/health/worker.py @@ -12,8 +12,8 @@ import queue import threading import time +from collections.abc import Iterable from dataclasses import dataclass -from typing import Iterable, List from syncfield.health.detector import Detector from syncfield.health.tracker import IncidentTracker @@ -54,9 +54,13 @@ def __init__( tick_hz: float = 20.0, ) -> None: self._tracker = tracker - self._detectors: List[Detector] = list(detectors) + self._detectors: list[Detector] = list(detectors) self._tick_interval = 1.0 / tick_hz + # SimpleQueue is unbounded. Producer rate is bounded in practice by + # hardware frame rate (tens of Hz) and the worker drains at tick_hz + # (default 20 Hz) plus post-stop. No back-pressure is needed for the + # intended load; if that changes, swap to queue.Queue with maxsize. self._samples: "queue.SimpleQueue[_SampleMsg]" = queue.SimpleQueue() self._healths: "queue.SimpleQueue[_HealthMsg]" = queue.SimpleQueue() self._states: "queue.SimpleQueue[_StateMsg]" = queue.SimpleQueue() @@ -106,15 +110,19 @@ def _run(self) -> None: self._tracker.tick(now_ns=time.monotonic_ns()) next_deadline += self._tick_interval - sleep_for = next_deadline - time.monotonic() + now = time.monotonic() + sleep_for = next_deadline - now if sleep_for > 0: - # Use Event.wait so stop() can interrupt immediately. + # Event.wait lets stop() cut short the sleep. self._stop.wait(timeout=sleep_for) else: - # Running behind; reset the schedule anchor. - next_deadline = time.monotonic() + # Running behind; reset anchor to the clock reading we just took. + next_deadline = now - # Drain any straggling messages after stop so tests see a consistent state. + # Drain any stragglers so post-stop state is consistent. We deliberately + # do NOT call tracker.tick() here — incidents still open at stop time + # are resolved by SessionOrchestrator via IncidentTracker.close_all(), + # not by one last opportunistic close_condition pass. self._drain_once() def _drain_once(self) -> None: @@ -139,7 +147,7 @@ def _fire_detector_ticks(self) -> None: self._tracker.ingest(event) -def _drain_queue(q: "queue.SimpleQueue") -> List: +def _drain_queue(q: "queue.SimpleQueue") -> list: out = [] while True: try: From 5d581f635b6b41d828cfd88d31ea161cdce418b8 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 13:03:28 +0900 Subject: [PATCH 17/64] feat(health): add AdapterEventPassthrough detector --- src/syncfield/health/detectors/__init__.py | 0 .../health/detectors/adapter_passthrough.py | 24 +++++++++++++++ tests/unit/health/detectors/__init__.py | 0 .../detectors/test_adapter_passthrough.py | 30 +++++++++++++++++++ 4 files changed, 54 insertions(+) create mode 100644 src/syncfield/health/detectors/__init__.py create mode 100644 src/syncfield/health/detectors/adapter_passthrough.py create mode 100644 tests/unit/health/detectors/__init__.py create mode 100644 tests/unit/health/detectors/test_adapter_passthrough.py diff --git a/src/syncfield/health/detectors/__init__.py b/src/syncfield/health/detectors/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/syncfield/health/detectors/adapter_passthrough.py b/src/syncfield/health/detectors/adapter_passthrough.py new file mode 100644 index 0000000..34732e8 --- /dev/null +++ b/src/syncfield/health/detectors/adapter_passthrough.py @@ -0,0 +1,24 @@ +"""Passthrough detector: owns close semantics for adapter-emitted events. + +Fingerprint convention ``:adapter:`` routes to this +detector in the tracker. It never emits synthetic events; its only job +is saying "if no new adapter event arrived for N seconds, close the +incident". +""" + +from __future__ import annotations + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.types import Incident + + +class AdapterEventPassthrough(DetectorBase): + name = "adapter" + default_severity = Severity.WARNING + + def __init__(self, quiet_ns: int = 30 * 1_000_000_000) -> None: + self._quiet_ns = quiet_ns + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + return (now_ns - incident.last_event_at_ns) >= self._quiet_ns diff --git a/tests/unit/health/detectors/__init__.py b/tests/unit/health/detectors/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/health/detectors/test_adapter_passthrough.py b/tests/unit/health/detectors/test_adapter_passthrough.py new file mode 100644 index 0000000..2ab615d --- /dev/null +++ b/tests/unit/health/detectors/test_adapter_passthrough.py @@ -0,0 +1,30 @@ +from syncfield.health.detectors.adapter_passthrough import AdapterEventPassthrough +from syncfield.health.severity import Severity +from syncfield.health.types import Incident +from syncfield.types import HealthEvent, HealthEventKind + + +def _adapter_ev(at_ns: int) -> HealthEvent: + return HealthEvent( + stream_id="cam", + kind=HealthEventKind.ERROR, + at_ns=at_ns, + detail="x", + severity=Severity.ERROR, + source="adapter:oak", + fingerprint="cam:adapter:xlink-error", + ) + + +def test_tick_emits_nothing(): + d = AdapterEventPassthrough() + assert list(d.tick(now_ns=1000)) == [] + + +def test_close_condition_respects_quiet_window(): + d = AdapterEventPassthrough(quiet_ns=500) + inc = Incident.opened_from(_adapter_ev(100), title="x") + assert d.close_condition(inc, now_ns=400) is False # 300 < 500 + inc.record_event(_adapter_ev(900)) + assert d.close_condition(inc, now_ns=1000) is False # 100 < 500 + assert d.close_condition(inc, now_ns=1500) is True # 600 >= 500 From 156eec19bf987d084b500bc6824a52d174625e0c Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 13:06:02 +0900 Subject: [PATCH 18/64] feat(health): add StreamStallDetector (platform-level silence detection) Detects when a stream stops producing samples for longer than a configurable threshold. Supports per-stream independent state, deduplication while stalled, and recovery-based incident closure. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../health/detectors/stream_stall.py | 64 +++++++++++++++++ .../health/detectors/test_stream_stall.py | 72 +++++++++++++++++++ 2 files changed, 136 insertions(+) create mode 100644 src/syncfield/health/detectors/stream_stall.py create mode 100644 tests/unit/health/detectors/test_stream_stall.py diff --git a/src/syncfield/health/detectors/stream_stall.py b/src/syncfield/health/detectors/stream_stall.py new file mode 100644 index 0000000..b26df2b --- /dev/null +++ b/src/syncfield/health/detectors/stream_stall.py @@ -0,0 +1,64 @@ +"""StreamStallDetector — fires when a stream stops producing samples.""" + +from __future__ import annotations + +from collections.abc import Iterator +from typing import Dict, List + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.types import Incident +from syncfield.types import HealthEvent, HealthEventKind, SampleEvent + + +class StreamStallDetector(DetectorBase): + name = "stream-stall" + default_severity = Severity.ERROR + + def __init__( + self, + stall_threshold_ns: int = 2_000_000_000, + recovery_ns: int = 1_000_000_000, + ) -> None: + self._stall_threshold_ns = stall_threshold_ns + self._recovery_ns = recovery_ns + # Per-stream most-recent sample monotonic time. + self._last_sample_at: Dict[str, int] = {} + # Per-stream: are we currently firing? prevents duplicates per stall. + self._stall_active: Dict[str, bool] = {} + + # --- observers ------------------------------------------------------- + + def observe_sample(self, stream_id: str, sample: SampleEvent) -> None: + self._last_sample_at[stream_id] = sample.capture_ns + # A new sample ends any active stall bookkeeping. + self._stall_active[stream_id] = False + + # --- tick ------------------------------------------------------------ + + def tick(self, now_ns: int) -> Iterator[HealthEvent]: + emitted: List[HealthEvent] = [] + for stream_id, last in self._last_sample_at.items(): + silence_ns = now_ns - last + if silence_ns >= self._stall_threshold_ns and not self._stall_active.get(stream_id, False): + self._stall_active[stream_id] = True + emitted.append(HealthEvent( + stream_id=stream_id, + kind=HealthEventKind.ERROR, + at_ns=now_ns, + detail=f"Stream stalled (silence {silence_ns / 1e9:.1f}s)", + severity=self.default_severity, + source=f"detector:{self.name}", + fingerprint=f"{stream_id}:{self.name}", + data={"silence_ns": silence_ns}, + )) + return iter(emitted) + + # --- close condition ------------------------------------------------- + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + last = self._last_sample_at.get(incident.stream_id) + if last is None: + return False + return (now_ns - last) < self._stall_threshold_ns \ + and (now_ns - incident.last_event_at_ns) >= self._recovery_ns diff --git a/tests/unit/health/detectors/test_stream_stall.py b/tests/unit/health/detectors/test_stream_stall.py new file mode 100644 index 0000000..06f2cf4 --- /dev/null +++ b/tests/unit/health/detectors/test_stream_stall.py @@ -0,0 +1,72 @@ +from syncfield.health.detectors.stream_stall import StreamStallDetector +from syncfield.health.types import Incident +from syncfield.types import SampleEvent + + +def _sample(stream_id: str, capture_ns: int) -> SampleEvent: + return SampleEvent(stream_id=stream_id, frame_number=1, capture_ns=capture_ns) + + +def test_no_fire_before_seeing_any_sample(): + d = StreamStallDetector(stall_threshold_ns=1000) + assert list(d.tick(now_ns=10_000)) == [] + + +def test_fires_when_silent_longer_than_threshold(): + d = StreamStallDetector(stall_threshold_ns=1000) + d.observe_sample("cam", _sample("cam", capture_ns=100)) + events = list(d.tick(now_ns=2000)) # 1900 ns of silence + assert len(events) == 1 + ev = events[0] + assert ev.stream_id == "cam" + assert ev.fingerprint == "cam:stream-stall" + assert ev.source == "detector:stream-stall" + assert "silence" in (ev.detail or "").lower() + + +def test_does_not_refire_while_still_stalled(): + d = StreamStallDetector(stall_threshold_ns=1000) + d.observe_sample("cam", _sample("cam", capture_ns=100)) + fired_once = list(d.tick(now_ns=2000)) + fired_twice = list(d.tick(now_ns=3000)) + assert len(fired_once) == 1 + assert len(fired_twice) == 0 + + +def test_refires_after_recovery_then_new_stall(): + d = StreamStallDetector(stall_threshold_ns=1000, recovery_ns=500) + d.observe_sample("cam", _sample("cam", capture_ns=0)) + list(d.tick(now_ns=2000)) # fires stall + + # recovery: samples flow for ≥ recovery_ns + for t in range(3000, 4100, 100): + d.observe_sample("cam", _sample("cam", capture_ns=t)) + # Silence again. + new_events = list(d.tick(now_ns=6000)) + assert len(new_events) == 1 # second stall → new event + + +def test_close_condition_requires_recent_sample_flow(): + d = StreamStallDetector(stall_threshold_ns=1000, recovery_ns=500) + d.observe_sample("cam", _sample("cam", capture_ns=0)) + events = list(d.tick(now_ns=2000)) + inc = Incident.opened_from(events[0], title="x") + + # Still silent — do not close. + assert d.close_condition(inc, now_ns=2500) is False + + # Samples arrive across a 600 ns window → recovery_ns=500 satisfied. + d.observe_sample("cam", _sample("cam", capture_ns=2600)) + d.observe_sample("cam", _sample("cam", capture_ns=3200)) + assert d.close_condition(inc, now_ns=3300) is True + + +def test_per_stream_independent_state(): + d = StreamStallDetector(stall_threshold_ns=1000) + d.observe_sample("a", _sample("a", capture_ns=100)) + d.observe_sample("b", _sample("b", capture_ns=100)) + # stream b stays alive + d.observe_sample("b", _sample("b", capture_ns=1800)) + events = list(d.tick(now_ns=2500)) + assert len(events) == 1 + assert events[0].stream_id == "a" From a295bda8bc815877df32626db4bad8e856166d70 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 13:11:28 +0900 Subject: [PATCH 19/64] feat(health): add FpsDropDetector (target-relative or baseline-learning) --- src/syncfield/health/detectors/fps_drop.py | 198 +++++++++++++++++++ tests/unit/health/detectors/test_fps_drop.py | 70 +++++++ 2 files changed, 268 insertions(+) create mode 100644 src/syncfield/health/detectors/fps_drop.py create mode 100644 tests/unit/health/detectors/test_fps_drop.py diff --git a/src/syncfield/health/detectors/fps_drop.py b/src/syncfield/health/detectors/fps_drop.py new file mode 100644 index 0000000..bc8ddb8 --- /dev/null +++ b/src/syncfield/health/detectors/fps_drop.py @@ -0,0 +1,198 @@ +"""FpsDropDetector — target-relative or baseline-learning FPS drop detector.""" + +from __future__ import annotations + +from collections import deque +from collections.abc import Iterator +from typing import Callable, Deque, Dict, List, Optional + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.types import Incident +from syncfield.types import HealthEvent, HealthEventKind, SampleEvent + +TargetGetter = Callable[[str], Optional[float]] + +_WINDOW_NS = 1_000_000_000 # rolling 1s FPS window + + +class FpsDropDetector(DetectorBase): + name = "fps-drop" + default_severity = Severity.WARNING + + def __init__( + self, + target_getter: TargetGetter = lambda sid: None, + drop_ratio: float = 0.70, + sustain_ns: int = 3_000_000_000, + recovery_ratio: float = 0.90, + recovery_ns: int = 5_000_000_000, + baseline_warmup_ns: int = 5_000_000_000, + baseline_window_ns: int = 10_000_000_000, + ) -> None: + self._target_getter = target_getter + self._drop_ratio = drop_ratio + self._sustain_ns = sustain_ns + self._recovery_ratio = recovery_ratio + self._recovery_ns = recovery_ns + self._baseline_warmup_ns = baseline_warmup_ns + self._baseline_window_ns = baseline_window_ns + + self._samples: Dict[str, Deque[int]] = {} + self._first_seen_at: Dict[str, int] = {} + self._baseline: Dict[str, float] = {} + self._baseline_locked: Dict[str, bool] = {} # True once baseline is frozen + # When did the stream first drop below threshold in the current dip? + self._dip_began_at: Dict[str, Optional[int]] = {} + self._fire_active: Dict[str, bool] = {} + # Same thing for recovery tracking. + self._recovery_began_at: Dict[str, Optional[int]] = {} + # Track the most recent observation, for incremental dip tracking + self._last_observed_state: Dict[str, bool] = {} # True if in drop, False if not + + # --- observers ------------------------------------------------------- + + def observe_sample(self, stream_id: str, sample: SampleEvent) -> None: + buf = self._samples.setdefault(stream_id, deque()) + buf.append(sample.capture_ns) + self._first_seen_at.setdefault(stream_id, sample.capture_ns) + # Trim older than baseline_window. + cutoff = sample.capture_ns - self._baseline_window_ns + while buf and buf[0] < cutoff: + buf.popleft() + + # --- tick ------------------------------------------------------------ + + def tick(self, now_ns: int) -> Iterator[HealthEvent]: + out: List[HealthEvent] = [] + for stream_id, buf in list(self._samples.items()): + target = self._effective_target(stream_id, now_ns) + observed = self._observed_fps(buf, now_ns) + if target is None or observed is None: + continue + + if observed < target * self._drop_ratio: + began = self._dip_began_at.get(stream_id) + if began is None: + # Find when the drop started by looking at the buffer + dip_start = self._find_dip_start(buf, target, now_ns) + self._dip_began_at[stream_id] = dip_start + began = dip_start + elapsed = now_ns - began + if elapsed >= self._sustain_ns and not self._fire_active.get(stream_id, False): + self._fire_active[stream_id] = True + out.append(HealthEvent( + stream_id=stream_id, + kind=HealthEventKind.WARNING, + at_ns=now_ns, + detail=f"FPS drop ({observed:.1f} Hz, target {target:.1f} Hz)", + severity=self.default_severity, + source=f"detector:{self.name}", + fingerprint=f"{stream_id}:{self.name}", + data={"observed_hz": observed, "target_hz": target}, + )) + else: + self._dip_began_at[stream_id] = None + self._fire_active[stream_id] = False + return iter(out) + + # --- close condition ------------------------------------------------- + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + stream_id = incident.stream_id + target = self._effective_target(stream_id, now_ns) + observed = self._observed_fps(self._samples.get(stream_id, deque()), now_ns) + if target is None or observed is None: + return False + if observed < target * self._recovery_ratio: + self._recovery_began_at[stream_id] = None + return False + began = self._recovery_began_at.get(stream_id) + if began is None: + self._recovery_began_at[stream_id] = now_ns + return False + return (now_ns - began) >= self._recovery_ns + + # --- helpers --------------------------------------------------------- + + def _find_dip_start(self, buf: Deque[int], target: float, now_ns: int) -> int: + """Find the earliest time when FPS dropped below threshold.""" + if not buf: + return now_ns + # Binary search to find when we first dropped below target + threshold = target * self._drop_ratio + left, right = 0, len(buf) - 1 + result = buf[-1] if buf else now_ns + + while left <= right: + mid = (left + right) // 2 + # Check FPS from buf[mid] onwards + mid_time = buf[mid] + fps_from_mid = self._observed_fps_from(buf, mid_time, now_ns) + + if fps_from_mid < threshold: + result = mid_time + right = mid - 1 + else: + left = mid + 1 + + return result + + def _observed_fps_from(self, buf: Deque[int], start_time: int, now_ns: int) -> Optional[float]: + """Calculate FPS from a specific start time to now.""" + if not buf: + return None + count = sum(1 for t in buf if t >= start_time and t <= now_ns) + if count == 0: + return 0.0 + elapsed_ns = now_ns - start_time + if elapsed_ns < _WINDOW_NS: + # Less than 1 second, use actual window + return count / (elapsed_ns / 1e9) if elapsed_ns > 0 else 0.0 + else: + # More than 1 second, use 1-second rolling window from now + cutoff = now_ns - _WINDOW_NS + count = sum(1 for t in buf if t >= cutoff) + return count / (_WINDOW_NS / 1e9) + + def _effective_target(self, stream_id: str, now_ns: int) -> Optional[float]: + declared = self._target_getter(stream_id) + if declared is not None: + return float(declared) + first = self._first_seen_at.get(stream_id) + if first is None: + return None + if (now_ns - first) < self._baseline_warmup_ns: + return None + cached = self._baseline.get(stream_id) + if cached is not None: + return cached + # Calculate and lock baseline once warmup completes + # Lock it to the FPS observed right after warmup completes + if not self._baseline_locked.get(stream_id, False): + buf = self._samples.get(stream_id, deque()) + if not buf: + return None + # Calculate FPS from immediately after warmup to now + warmup_end = first + self._baseline_warmup_ns + count = sum(1 for t in buf if t >= warmup_end) + if count == 0: + return None + # Use the available time since warmup end + elapsed = min(now_ns - warmup_end, self._baseline_window_ns) + if elapsed <= 0: + return None + observed = count / (elapsed / 1e9) + self._baseline[stream_id] = observed + self._baseline_locked[stream_id] = True + return self._baseline.get(stream_id) + + @staticmethod + def _observed_fps(buf: Deque[int], now_ns: int) -> Optional[float]: + if not buf: + return None + cutoff = now_ns - _WINDOW_NS + count = sum(1 for t in buf if t >= cutoff) + if count == 0: + return 0.0 + return count / (_WINDOW_NS / 1e9) diff --git a/tests/unit/health/detectors/test_fps_drop.py b/tests/unit/health/detectors/test_fps_drop.py new file mode 100644 index 0000000..81baf8e --- /dev/null +++ b/tests/unit/health/detectors/test_fps_drop.py @@ -0,0 +1,70 @@ +from syncfield.health.detectors.fps_drop import FpsDropDetector +from syncfield.types import SampleEvent + + +def _s(stream: str, t_ns: int) -> SampleEvent: + return SampleEvent(stream_id=stream, frame_number=0, capture_ns=t_ns) + + +def test_no_fire_if_fps_tracks_target(): + d = FpsDropDetector(target_getter=lambda sid: 30.0) + # emit 30 samples over 1 second + for i in range(30): + d.observe_sample("cam", _s("cam", i * int(1e9 / 30))) + assert list(d.tick(now_ns=int(1.1e9))) == [] + + +def test_fires_when_observed_below_70_percent_for_3s(): + d = FpsDropDetector( + target_getter=lambda sid: 30.0, + drop_ratio=0.70, + sustain_ns=3 * 1_000_000_000, + ) + + # 10 fps for 3.5 seconds — fps is 10, target 30, ratio 0.33. + interval = int(1e9 / 10) + t = 0 + while t <= int(3.5e9): + d.observe_sample("cam", _s("cam", t)) + t += interval + + emitted = list(d.tick(now_ns=int(3.6e9))) + assert len(emitted) == 1 + assert emitted[0].fingerprint == "cam:fps-drop" + assert emitted[0].data["target_hz"] == 30.0 + assert emitted[0].data["observed_hz"] < 15.0 + + +def test_does_not_fire_without_target_before_warmup(): + d = FpsDropDetector( + target_getter=lambda sid: None, + baseline_warmup_ns=5_000_000_000, + ) + # 10 fps, but only for 1s — under warmup. + t = 0 + for _ in range(10): + d.observe_sample("cam", _s("cam", t)) + t += int(1e8) + assert list(d.tick(now_ns=int(1.1e9))) == [] + + +def test_learns_baseline_then_fires_on_subsequent_drop(): + d = FpsDropDetector( + target_getter=lambda sid: None, + baseline_warmup_ns=1_000_000_000, + baseline_window_ns=2_000_000_000, + drop_ratio=0.7, + sustain_ns=1_000_000_000, + ) + # 3 s @ 30 fps → baseline ≈ 30. + t = 0 + while t <= int(3e9): + d.observe_sample("cam", _s("cam", t)) + t += int(1e9 / 30) + # 1.5 s of 10 fps → drop. + end = t + int(1.5e9) + while t <= end: + d.observe_sample("cam", _s("cam", t)) + t += int(1e8) + emitted = list(d.tick(now_ns=t)) + assert len(emitted) == 1 From d65c8e153c9f4aef9ce848fd230125c495e2c5e1 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 13:14:14 +0900 Subject: [PATCH 20/64] feat(health): add JitterDetector (p95-interval spike detector) Implements p95-based inter-sample interval anomaly detector for monitoring streaming data quality. Detects jitter spikes when 95th percentile interval exceeds target by a configurable ratio, with sustain and recovery phases. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/syncfield/health/detectors/jitter.py | 110 +++++++++++++++++++++ tests/unit/health/detectors/test_jitter.py | 40 ++++++++ 2 files changed, 150 insertions(+) create mode 100644 src/syncfield/health/detectors/jitter.py create mode 100644 tests/unit/health/detectors/test_jitter.py diff --git a/src/syncfield/health/detectors/jitter.py b/src/syncfield/health/detectors/jitter.py new file mode 100644 index 0000000..cbaddb3 --- /dev/null +++ b/src/syncfield/health/detectors/jitter.py @@ -0,0 +1,110 @@ +"""JitterDetector — p95-based inter-sample interval anomaly detector.""" + +from __future__ import annotations + +from collections import deque +from collections.abc import Iterator +from typing import Callable, Deque, Dict, List, Optional + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.types import Incident +from syncfield.types import HealthEvent, HealthEventKind, SampleEvent + +TargetGetter = Callable[[str], Optional[float]] + + +def _p95(values: List[int]) -> int: + if not values: + return 0 + sorted_v = sorted(values) + idx = max(0, int(0.95 * (len(sorted_v) - 1))) + return sorted_v[idx] + + +class JitterDetector(DetectorBase): + name = "jitter" + default_severity = Severity.WARNING + + def __init__( + self, + target_getter: TargetGetter = lambda sid: None, + window: int = 60, + jitter_ratio: float = 2.0, + sustain_ns: int = 3_000_000_000, + recovery_ratio: float = 1.2, + recovery_ns: int = 10_000_000_000, + ) -> None: + self._target_getter = target_getter + self._window = window + self._jitter_ratio = jitter_ratio + self._sustain_ns = sustain_ns + self._recovery_ratio = recovery_ratio + self._recovery_ns = recovery_ns + + self._last_at: Dict[str, int] = {} + self._intervals: Dict[str, Deque[int]] = {} + self._bad_began_at: Dict[str, Optional[int]] = {} + self._fire_active: Dict[str, bool] = {} + self._recovery_began_at: Dict[str, Optional[int]] = {} + + def observe_sample(self, stream_id: str, sample: SampleEvent) -> None: + last = self._last_at.get(stream_id) + if last is not None: + buf = self._intervals.setdefault(stream_id, deque(maxlen=self._window)) + buf.append(sample.capture_ns - last) + self._last_at[stream_id] = sample.capture_ns + + def tick(self, now_ns: int) -> Iterator[HealthEvent]: + out: List[HealthEvent] = [] + for stream_id, buf in list(self._intervals.items()): + target_hz = self._target_getter(stream_id) + if target_hz is None or target_hz <= 0 or len(buf) < max(10, self._window // 2): + continue + expected = 1e9 / target_hz + p95 = _p95(list(buf)) + + if p95 > expected * self._jitter_ratio: + began = self._bad_began_at.get(stream_id) + if began is None: + # Backdate bad_began_at to the start of the current window. + # This is the sum of all intervals in the buffer, which represents + # the elapsed time from the start of this window to now. + window_elapsed = sum(buf) + self._bad_began_at[stream_id] = now_ns - window_elapsed + began = self._bad_began_at[stream_id] + + elapsed = now_ns - began + if elapsed >= self._sustain_ns and not self._fire_active.get(stream_id, False): + self._fire_active[stream_id] = True + out.append(HealthEvent( + stream_id=stream_id, + kind=HealthEventKind.WARNING, + at_ns=now_ns, + detail=f"Jitter spike (p95 {p95/1e6:.1f} ms, expected {expected/1e6:.1f} ms)", + severity=self.default_severity, + source=f"detector:{self.name}", + fingerprint=f"{stream_id}:{self.name}", + data={"p95_ns": p95, "expected_ns": int(expected)}, + )) + else: + self._bad_began_at[stream_id] = None + self._fire_active[stream_id] = False + return iter(out) + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + stream_id = incident.stream_id + buf = self._intervals.get(stream_id) + target_hz = self._target_getter(stream_id) + if not buf or target_hz is None or target_hz <= 0: + return False + expected = 1e9 / target_hz + p95 = _p95(list(buf)) + if p95 > expected * self._recovery_ratio: + self._recovery_began_at[stream_id] = None + return False + began = self._recovery_began_at.get(stream_id) + if began is None: + self._recovery_began_at[stream_id] = now_ns + return False + return (now_ns - began) >= self._recovery_ns diff --git a/tests/unit/health/detectors/test_jitter.py b/tests/unit/health/detectors/test_jitter.py new file mode 100644 index 0000000..2d157dd --- /dev/null +++ b/tests/unit/health/detectors/test_jitter.py @@ -0,0 +1,40 @@ +from syncfield.health.detectors.jitter import JitterDetector +from syncfield.types import SampleEvent + + +def _s(t_ns: int) -> SampleEvent: + return SampleEvent(stream_id="cam", frame_number=0, capture_ns=t_ns) + + +def test_steady_30hz_does_not_fire(): + d = JitterDetector(target_getter=lambda sid: 30.0) + step = int(1e9 / 30) + t = 0 + for _ in range(120): + d.observe_sample("cam", _s(t)) + t += step + assert list(d.tick(now_ns=t)) == [] + + +def test_irregular_intervals_fire_when_p95_exceeds_ratio(): + d = JitterDetector( + target_getter=lambda sid: 30.0, + jitter_ratio=2.0, + sustain_ns=500_000_000, + ) + step = int(1e9 / 30) + big = step * 4 # 4× target interval + t = 0 + # 60 samples alternating between normal and 4× intervals. + for i in range(60): + d.observe_sample("cam", _s(t)) + t += big if i % 2 == 0 else step + + # Give sustain time to elapse with more irregular samples. + for _ in range(20): + d.observe_sample("cam", _s(t)) + t += big + + emitted = list(d.tick(now_ns=t + 500_000_000)) + assert len(emitted) == 1 + assert emitted[0].fingerprint == "cam:jitter" From 463f450cb4746a9941272ccff445f79030ed5be9 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 13:15:51 +0900 Subject: [PATCH 21/64] feat(health): add StartupFailureDetector (connect / start_recording errors) --- .../health/detectors/startup_failure.py | 57 +++++++++++++++++++ .../health/detectors/test_startup_failure.py | 50 ++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 src/syncfield/health/detectors/startup_failure.py create mode 100644 tests/unit/health/detectors/test_startup_failure.py diff --git a/src/syncfield/health/detectors/startup_failure.py b/src/syncfield/health/detectors/startup_failure.py new file mode 100644 index 0000000..23ee8f3 --- /dev/null +++ b/src/syncfield/health/detectors/startup_failure.py @@ -0,0 +1,57 @@ +"""StartupFailureDetector — fires when connect/start_recording raises. + +Relies on orchestrator-emitted HealthEvents with ``data["phase"]`` in +{``"connect"``, ``"start_recording"``}. A subsequent success event with +``data["outcome"] == "success"`` closes the incident. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from typing import Dict, List, Set + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.types import Incident +from syncfield.types import HealthEvent, HealthEventKind + +_STARTUP_PHASES = {"connect", "start_recording"} + + +class StartupFailureDetector(DetectorBase): + name = "startup-failure" + default_severity = Severity.ERROR + + def __init__(self) -> None: + self._pending_failures: Dict[str, HealthEvent] = {} + self._recovered: Set[str] = set() + + def observe_health(self, stream_id: str, event: HealthEvent) -> None: + phase = event.data.get("phase") if event.data else None + if phase not in _STARTUP_PHASES: + return + outcome = event.data.get("outcome") if event.data else None + if event.kind == HealthEventKind.ERROR and outcome != "success": + self._pending_failures[stream_id] = event + self._recovered.discard(stream_id) + elif outcome == "success": + self._recovered.add(stream_id) + + def tick(self, now_ns: int) -> Iterator[HealthEvent]: + out: List[HealthEvent] = [] + for stream_id, origin in list(self._pending_failures.items()): + out.append(HealthEvent( + stream_id=stream_id, + kind=HealthEventKind.ERROR, + at_ns=now_ns, + detail=origin.detail or "Startup failure", + severity=self.default_severity, + source=f"detector:{self.name}", + fingerprint=f"{stream_id}:{self.name}", + data={"phase": origin.data.get("phase"), "origin_at_ns": origin.at_ns}, + )) + del self._pending_failures[stream_id] + return iter(out) + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + return incident.stream_id in self._recovered diff --git a/tests/unit/health/detectors/test_startup_failure.py b/tests/unit/health/detectors/test_startup_failure.py new file mode 100644 index 0000000..b342283 --- /dev/null +++ b/tests/unit/health/detectors/test_startup_failure.py @@ -0,0 +1,50 @@ +from syncfield.health.detectors.startup_failure import StartupFailureDetector +from syncfield.health.severity import Severity +from syncfield.types import HealthEvent, HealthEventKind + + +def _ev_for(phase: str, kind=HealthEventKind.ERROR) -> HealthEvent: + return HealthEvent( + stream_id="cam", kind=kind, at_ns=100, detail="boom", + severity=Severity.ERROR, source="orchestrator", + fingerprint=f"cam:adapter:startup-{phase}", + data={"phase": phase}, + ) + + +def test_fires_on_connect_phase_error(): + d = StartupFailureDetector() + d.observe_health("cam", _ev_for("connect")) + events = list(d.tick(now_ns=500)) + assert len(events) == 1 + assert events[0].fingerprint == "cam:startup-failure" + assert events[0].data["phase"] == "connect" + + +def test_ignores_non_startup_phases(): + d = StartupFailureDetector() + d.observe_health("cam", HealthEvent( + stream_id="cam", kind=HealthEventKind.ERROR, at_ns=1, detail="x", + severity=Severity.ERROR, source="adapter:foo", fingerprint="cam:adapter:xlink", + data={}, + )) + assert list(d.tick(now_ns=100)) == [] + + +def test_closes_after_phase_success_signal(): + d = StartupFailureDetector() + d.observe_health("cam", _ev_for("connect")) + list(d.tick(now_ns=100)) + from syncfield.health.types import Incident + inc = Incident.opened_from(_ev_for("connect"), title="x") + + # Before success, not closed. + assert d.close_condition(inc, now_ns=200) is False + + # Success signal arrives. + d.observe_health("cam", HealthEvent( + stream_id="cam", kind=HealthEventKind.HEARTBEAT, at_ns=300, detail="connected", + severity=Severity.INFO, source="orchestrator", fingerprint="cam:adapter:startup-success", + data={"phase": "connect", "outcome": "success"}, + )) + assert d.close_condition(inc, now_ns=400) is True From 196a6d2b8447a464c21bd48538beb35e8b155e62 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 13:18:18 +0900 Subject: [PATCH 22/64] fix(health): StartupFailureDetector clears pending on success signal --- src/syncfield/health/detectors/startup_failure.py | 3 +++ .../unit/health/detectors/test_startup_failure.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/src/syncfield/health/detectors/startup_failure.py b/src/syncfield/health/detectors/startup_failure.py index 23ee8f3..6f6e144 100644 --- a/src/syncfield/health/detectors/startup_failure.py +++ b/src/syncfield/health/detectors/startup_failure.py @@ -36,6 +36,9 @@ def observe_health(self, stream_id: str, event: HealthEvent) -> None: self._recovered.discard(stream_id) elif outcome == "success": self._recovered.add(stream_id) + # Clear any stale pending failure so the next tick doesn't emit it + # as a spurious post-recovery event. + self._pending_failures.pop(stream_id, None) def tick(self, now_ns: int) -> Iterator[HealthEvent]: out: List[HealthEvent] = [] diff --git a/tests/unit/health/detectors/test_startup_failure.py b/tests/unit/health/detectors/test_startup_failure.py index b342283..2472856 100644 --- a/tests/unit/health/detectors/test_startup_failure.py +++ b/tests/unit/health/detectors/test_startup_failure.py @@ -48,3 +48,18 @@ def test_closes_after_phase_success_signal(): data={"phase": "connect", "outcome": "success"}, )) assert d.close_condition(inc, now_ns=400) is True + + +def test_success_clears_pending_so_no_stale_fire(): + d = StartupFailureDetector() + # Error first, then success. + d.observe_health("cam", _ev_for("connect")) + d.observe_health("cam", HealthEvent( + stream_id="cam", kind=HealthEventKind.HEARTBEAT, at_ns=50, detail="connected", + severity=Severity.INFO, source="orchestrator", + fingerprint="cam:adapter:startup-success", + data={"phase": "connect", "outcome": "success"}, + )) + + # Next tick should emit nothing — success arrived before any tick fired. + assert list(d.tick(now_ns=200)) == [] From 6f81de97b8a0c0ea1a7c75c6531c87301c597cbc Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 13:20:01 +0900 Subject: [PATCH 23/64] feat(health): add BackpressureDetector (writer queue fullness + drop counter) Implements detector for writer queue saturation and frame drops. Fires when queue fullness sustained above threshold, or on any drop increment. Closes when queue recovers to low levels with no new drops for the recovery window. Backdating via _first_bad_observed_at: on first observation above threshold, record that timestamp; clear when fullness drops below. This ensures the sustain window is measured from when the bad state started, not from when we first check it. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../health/detectors/backpressure.py | 105 ++++++++++++++++++ .../health/detectors/test_backpressure.py | 46 ++++++++ 2 files changed, 151 insertions(+) create mode 100644 src/syncfield/health/detectors/backpressure.py create mode 100644 tests/unit/health/detectors/test_backpressure.py diff --git a/src/syncfield/health/detectors/backpressure.py b/src/syncfield/health/detectors/backpressure.py new file mode 100644 index 0000000..d1aef42 --- /dev/null +++ b/src/syncfield/health/detectors/backpressure.py @@ -0,0 +1,105 @@ +"""BackpressureDetector — writer queue saturation + drop-counter detector.""" + +from __future__ import annotations + +from collections.abc import Iterator +from typing import Dict, List, Optional + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.types import Incident, WriterStats +from syncfield.types import HealthEvent, HealthEventKind + + +class BackpressureDetector(DetectorBase): + name = "backpressure" + default_severity = Severity.WARNING + + def __init__( + self, + fullness_threshold: float = 0.80, + sustain_ns: int = 2_000_000_000, + recovery_ratio: float = 0.30, + recovery_ns: int = 5_000_000_000, + ) -> None: + self._threshold = fullness_threshold + self._sustain_ns = sustain_ns + self._recovery_ratio = recovery_ratio + self._recovery_ns = recovery_ns + + self._latest: Dict[str, WriterStats] = {} + self._first_bad_observed_at: Dict[str, Optional[int]] = {} + self._last_dropped: Dict[str, int] = {} + self._pending_drop_spike: Dict[str, bool] = {} + self._fire_active: Dict[str, bool] = {} + self._recovery_began_at: Dict[str, Optional[int]] = {} + + def observe_writer_stats(self, stream_id: str, stats: WriterStats) -> None: + self._latest[stream_id] = stats + prev = self._last_dropped.get(stream_id, 0) + if stats.dropped > prev: + self._pending_drop_spike[stream_id] = True + self._last_dropped[stream_id] = stats.dropped + + # Track when fullness first exceeds threshold + if stats.queue_fullness >= self._threshold: + if self._first_bad_observed_at.get(stream_id) is None: + self._first_bad_observed_at[stream_id] = stats.at_ns + else: + self._first_bad_observed_at[stream_id] = None + + def tick(self, now_ns: int) -> Iterator[HealthEvent]: + out: List[HealthEvent] = [] + for stream_id, stats in self._latest.items(): + fire_now = False + detail = "" + + if self._pending_drop_spike.pop(stream_id, False): + fire_now = True + detail = f"Writer dropped frames (total {stats.dropped})" + + if stats.queue_fullness >= self._threshold: + began = self._first_bad_observed_at.get(stream_id) + if began is not None: + if (now_ns - began) >= self._sustain_ns and not self._fire_active.get(stream_id, False): + fire_now = True + self._fire_active[stream_id] = True + detail = f"Writer queue {stats.queue_depth}/{stats.queue_capacity} full" + else: + self._fire_active[stream_id] = False + + if fire_now: + out.append(HealthEvent( + stream_id=stream_id, + kind=HealthEventKind.WARNING, + at_ns=now_ns, + detail=detail, + severity=self.default_severity, + source=f"detector:{self.name}", + fingerprint=f"{stream_id}:{self.name}", + data={ + "queue_depth": stats.queue_depth, + "queue_capacity": stats.queue_capacity, + "dropped": stats.dropped, + "dropped_at_open": stats.dropped, + }, + )) + return iter(out) + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + stats = self._latest.get(incident.stream_id) + if stats is None: + return False + if stats.queue_fullness > self._recovery_ratio: + self._recovery_began_at[incident.stream_id] = None + return False + # No new drops since incident opened. + opened_dropped = incident.data.get("dropped_at_open") + current_dropped = self._last_dropped.get(incident.stream_id, 0) + if opened_dropped is not None and current_dropped > opened_dropped: + return False + began = self._recovery_began_at.get(incident.stream_id) + if began is None: + self._recovery_began_at[incident.stream_id] = now_ns + return False + return (now_ns - began) >= self._recovery_ns diff --git a/tests/unit/health/detectors/test_backpressure.py b/tests/unit/health/detectors/test_backpressure.py new file mode 100644 index 0000000..0d208e0 --- /dev/null +++ b/tests/unit/health/detectors/test_backpressure.py @@ -0,0 +1,46 @@ +from syncfield.health.detectors.backpressure import BackpressureDetector +from syncfield.health.types import Incident, WriterStats + + +def _stat(at_ns, depth, cap=16, dropped=0): + return WriterStats(stream_id="cam", at_ns=at_ns, queue_depth=depth, queue_capacity=cap, dropped=dropped) + + +def test_does_not_fire_with_normal_fullness(): + d = BackpressureDetector() + for t in range(0, int(3e9), int(2.5e8)): + d.observe_writer_stats("cam", _stat(t, depth=2)) + assert list(d.tick(now_ns=int(3e9))) == [] + + +def test_fires_when_queue_sustained_above_threshold(): + d = BackpressureDetector(fullness_threshold=0.8, sustain_ns=int(2e9)) + for t in range(0, int(3e9), int(2.5e8)): + d.observe_writer_stats("cam", _stat(t, depth=14)) # 14/16 = 0.875 + emitted = list(d.tick(now_ns=int(3e9))) + assert len(emitted) == 1 + assert emitted[0].fingerprint == "cam:backpressure" + + +def test_fires_on_any_drop_increment(): + d = BackpressureDetector() + d.observe_writer_stats("cam", _stat(0, depth=1, dropped=0)) + d.observe_writer_stats("cam", _stat(int(1e8), depth=1, dropped=5)) + emitted = list(d.tick(now_ns=int(2e8))) + assert len(emitted) == 1 + + +def test_close_condition_requires_low_and_no_new_drops(): + d = BackpressureDetector(fullness_threshold=0.8, sustain_ns=int(2e9), + recovery_ratio=0.3, recovery_ns=int(1e9)) + for t in range(0, int(3e9), int(2.5e8)): + d.observe_writer_stats("cam", _stat(t, depth=14)) + events = list(d.tick(now_ns=int(3e9))) + inc = Incident.opened_from(events[0], title="x") + + # Recovery in progress. + d.observe_writer_stats("cam", _stat(int(3.5e9), depth=2)) + assert d.close_condition(inc, now_ns=int(4e9)) is False # only 500 ms of recovery + + d.observe_writer_stats("cam", _stat(int(5e9), depth=2)) + assert d.close_condition(inc, now_ns=int(5e9)) is True # 1.5 s of recovery From 44445ac8e26312dcf8342257395d9639ed9ff3d1 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 13:23:52 +0900 Subject: [PATCH 24/64] feat(health): add HealthSystem facade with default detector install MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ties together Registry + Tracker + Worker into a single user-facing facade (HealthSystem). Installs all six default detectors on construction and exposes lifecycle, observer, incident-view, and callback hooks. Populates syncfield.health.__init__ public API using lazy imports to avoid the existing syncfield.types → syncfield.health.severity circular import. Co-Authored-By: Claude Sonnet 4.6 --- src/syncfield/health/__init__.py | 69 ++++++++++++++ src/syncfield/health/system.py | 114 ++++++++++++++++++++++++ tests/unit/health/test_health_system.py | 71 +++++++++++++++ 3 files changed, 254 insertions(+) create mode 100644 src/syncfield/health/system.py create mode 100644 tests/unit/health/test_health_system.py diff --git a/src/syncfield/health/__init__.py b/src/syncfield/health/__init__.py index e69de29..661d5b3 100644 --- a/src/syncfield/health/__init__.py +++ b/src/syncfield/health/__init__.py @@ -0,0 +1,69 @@ +"""syncfield.health — platform health telemetry.""" + +from __future__ import annotations + +import sys as _sys + +# Guard flag — checked via __dict__ to avoid re-entering __getattr__. +_health_imported: bool = False + + +def __getattr__(name: str) -> object: + # Lazy-load to avoid a circular import: + # syncfield.types → syncfield.health.severity + # → syncfield.health (this __init__) → syncfield.health.detector + # → syncfield.health.types → syncfield.types (partially initialised) + _import_all() + try: + return _sys.modules[__name__].__dict__[name] + except KeyError: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None + + +def _import_all() -> None: + """Populate the module namespace on first access.""" + mod = _sys.modules[__name__] + # Use __dict__ directly to avoid triggering __getattr__ again. + if mod.__dict__.get("_health_imported"): + return + mod.__dict__["_health_imported"] = True + + from syncfield.health.detector import Detector, DetectorBase + from syncfield.health.registry import DetectorRegistry + from syncfield.health.severity import Severity, max_severity + from syncfield.health.system import HealthSystem + from syncfield.health.tracker import IncidentTracker + from syncfield.health.types import ( + Incident, + IncidentArtifact, + IncidentSnapshot, + WriterStats, + ) + + ns = mod.__dict__ + ns["Detector"] = Detector + ns["DetectorBase"] = DetectorBase + ns["DetectorRegistry"] = DetectorRegistry + ns["Severity"] = Severity + ns["max_severity"] = max_severity + ns["HealthSystem"] = HealthSystem + ns["IncidentTracker"] = IncidentTracker + ns["Incident"] = Incident + ns["IncidentArtifact"] = IncidentArtifact + ns["IncidentSnapshot"] = IncidentSnapshot + ns["WriterStats"] = WriterStats + + +__all__ = [ + "Detector", + "DetectorBase", + "DetectorRegistry", + "HealthSystem", + "Incident", + "IncidentArtifact", + "IncidentSnapshot", + "IncidentTracker", + "Severity", + "WriterStats", + "max_severity", +] diff --git a/src/syncfield/health/system.py b/src/syncfield/health/system.py new file mode 100644 index 0000000..e9bcd66 --- /dev/null +++ b/src/syncfield/health/system.py @@ -0,0 +1,114 @@ +"""HealthSystem — the single handle the orchestrator + user code touch.""" + +from __future__ import annotations + +from typing import Callable, Iterable, Iterator, Optional + +from syncfield.health.detector import Detector +from syncfield.health.detectors.adapter_passthrough import AdapterEventPassthrough +from syncfield.health.detectors.backpressure import BackpressureDetector +from syncfield.health.detectors.fps_drop import FpsDropDetector +from syncfield.health.detectors.jitter import JitterDetector +from syncfield.health.detectors.startup_failure import StartupFailureDetector +from syncfield.health.detectors.stream_stall import StreamStallDetector +from syncfield.health.registry import DetectorRegistry +from syncfield.health.tracker import IncidentTracker +from syncfield.health.types import Incident, WriterStats +from syncfield.health.worker import HealthWorker +from syncfield.types import HealthEvent, SampleEvent, SessionState + + +class HealthSystem: + """Composes Registry + Tracker + Worker into a single user-facing facade.""" + + def __init__( + self, + *, + tick_hz: float = 20.0, + passthrough_close_ns: int = 30 * 1_000_000_000, + ) -> None: + self._registry = DetectorRegistry() + self._tracker = IncidentTracker(passthrough_close_ns=passthrough_close_ns) + self._worker: Optional[HealthWorker] = None + self._tick_hz = tick_hz + + self.on_incident_opened: Optional[Callable[[Incident], None]] = None + self.on_incident_updated: Optional[Callable[[Incident], None]] = None + self.on_incident_closed: Optional[Callable[[Incident], None]] = None + + self._tracker.on_opened = lambda inc: self._fire("on_incident_opened", inc) + self._tracker.on_updated = lambda inc: self._fire("on_incident_updated", inc) + self._tracker.on_closed = lambda inc: self._fire("on_incident_closed", inc) + + self._install_default_detectors() + + # --- registry -------------------------------------------------------- + + def register(self, detector: Detector) -> None: + self._registry.register(detector) + self._tracker.bind_detector(detector) + + def unregister(self, name: str) -> None: + self._registry.unregister(name) + + def iter_detectors(self) -> Iterator[Detector]: + return iter(self._registry) + + # --- observer inputs ------------------------------------------------- + + def observe_sample(self, stream_id: str, sample: SampleEvent) -> None: + if self._worker is not None: + self._worker.push_sample(stream_id, sample) + + def observe_health(self, stream_id: str, event: HealthEvent) -> None: + if self._worker is not None: + self._worker.push_health(stream_id, event) + + def observe_state(self, old: SessionState, new: SessionState) -> None: + if self._worker is not None: + self._worker.push_state(old, new) + + def observe_writer_stats(self, stream_id: str, stats: WriterStats) -> None: + if self._worker is not None: + self._worker.push_writer_stats(stream_id, stats) + + # --- lifecycle ------------------------------------------------------- + + def start(self) -> None: + self._worker = HealthWorker( + tracker=self._tracker, + detectors=list(self._registry), + tick_hz=self._tick_hz, + ) + self._worker.start() + + def stop(self, *, close_open_incidents: bool = True, now_ns: Optional[int] = None) -> None: + if self._worker is not None: + self._worker.stop() + self._worker = None + if close_open_incidents: + import time + self._tracker.close_all(at_ns=now_ns if now_ns is not None else time.monotonic_ns()) + + # --- read-only views ------------------------------------------------- + + def open_incidents(self) -> Iterable[Incident]: + return self._tracker.open_incidents() + + def resolved_incidents(self) -> Iterable[Incident]: + return self._tracker.resolved_incidents() + + # --- helpers --------------------------------------------------------- + + def _install_default_detectors(self) -> None: + self.register(AdapterEventPassthrough()) + self.register(StreamStallDetector()) + self.register(FpsDropDetector()) + self.register(JitterDetector()) + self.register(StartupFailureDetector()) + self.register(BackpressureDetector()) + + def _fire(self, attr: str, inc: Incident) -> None: + cb = getattr(self, attr, None) + if cb is not None: + cb(inc) diff --git a/tests/unit/health/test_health_system.py b/tests/unit/health/test_health_system.py new file mode 100644 index 0000000..209b013 --- /dev/null +++ b/tests/unit/health/test_health_system.py @@ -0,0 +1,71 @@ +import time + +from syncfield.health import HealthSystem, Severity +from syncfield.health.detector import DetectorBase +from syncfield.types import HealthEvent, HealthEventKind, SampleEvent, SessionState + + +class Custom(DetectorBase): + name = "custom" + default_severity = Severity.WARNING + + +def test_health_system_boots_and_accepts_inputs(): + hs = HealthSystem() + hs.start() + try: + hs.observe_sample("cam", SampleEvent(stream_id="cam", frame_number=1, capture_ns=1)) + hs.observe_health("cam", HealthEvent( + stream_id="cam", kind=HealthEventKind.WARNING, at_ns=1, + severity=Severity.WARNING, source="test", fingerprint="cam:adapter:test", + )) + hs.observe_state(SessionState.IDLE, SessionState.CONNECTED) + finally: + hs.stop() + + +def test_health_system_register_and_unregister(): + hs = HealthSystem() + d = Custom() + hs.register(d) + assert any(x.name == "custom" for x in hs.iter_detectors()) + hs.unregister("custom") + assert not any(x.name == "custom" for x in hs.iter_detectors()) + + +def test_health_system_installs_default_detectors(): + hs = HealthSystem() + names = {d.name for d in hs.iter_detectors()} + for expected in ( + "adapter", + "stream-stall", + "fps-drop", + "jitter", + "startup-failure", + "backpressure", + ): + assert expected in names, f"missing default detector: {expected}" + + +def test_health_system_callbacks_fire_on_open_and_close(): + hs = HealthSystem(passthrough_close_ns=1) # close instantly for the test + opened, closed = [], [] + hs.on_incident_opened = opened.append + hs.on_incident_closed = closed.append + + hs.start() + try: + hs.observe_health("cam", HealthEvent( + stream_id="cam", kind=HealthEventKind.ERROR, at_ns=1, + severity=Severity.ERROR, source="adapter:test", + fingerprint="cam:adapter:xlink-error", + )) + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline: + if opened and closed: + break + time.sleep(0.02) + finally: + hs.stop() + assert opened, "incident was not opened" + assert closed, "incident was not closed" From ed00154da14548f3aa68176bed22eaab8bb269d3 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 13:26:26 +0900 Subject: [PATCH 25/64] fix(health): HealthSystem start() idempotency + register-after-start warning --- src/syncfield/health/system.py | 11 ++++++++++ tests/unit/health/test_health_system.py | 27 +++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/src/syncfield/health/system.py b/src/syncfield/health/system.py index e9bcd66..cf5e23f 100644 --- a/src/syncfield/health/system.py +++ b/src/syncfield/health/system.py @@ -45,8 +45,17 @@ def __init__( # --- registry -------------------------------------------------------- def register(self, detector: Detector) -> None: + import warnings self._registry.register(detector) self._tracker.bind_detector(detector) + if self._worker is not None: + warnings.warn( + f"Detector '{detector.name}' registered after HealthSystem.start(); " + "it will be bound for close-condition routing but will NOT be ticked " + "until the system is stopped and restarted.", + RuntimeWarning, + stacklevel=2, + ) def unregister(self, name: str) -> None: self._registry.unregister(name) @@ -75,6 +84,8 @@ def observe_writer_stats(self, stream_id: str, stats: WriterStats) -> None: # --- lifecycle ------------------------------------------------------- def start(self) -> None: + if self._worker is not None: + return # already running — idempotent self._worker = HealthWorker( tracker=self._tracker, detectors=list(self._registry), diff --git a/tests/unit/health/test_health_system.py b/tests/unit/health/test_health_system.py index 209b013..df302bb 100644 --- a/tests/unit/health/test_health_system.py +++ b/tests/unit/health/test_health_system.py @@ -69,3 +69,30 @@ def test_health_system_callbacks_fire_on_open_and_close(): hs.stop() assert opened, "incident was not opened" assert closed, "incident was not closed" + + +def test_health_system_double_start_is_idempotent(): + hs = HealthSystem() + hs.start() + first_worker = hs._worker + hs.start() # should not spin up a new thread + second_worker = hs._worker + try: + assert first_worker is second_worker, "start() should be idempotent" + finally: + hs.stop() + + +def test_health_system_register_after_start_warns(): + import warnings + + hs = HealthSystem() + hs.start() + try: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + hs.register(Custom()) + assert any(issubclass(w.category, RuntimeWarning) for w in caught) + assert any("after HealthSystem.start()" in str(w.message) for w in caught) + finally: + hs.stop() From 70a53efe13dcdfcfcd35fadee3e38da6b70e30a0 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 13:30:46 +0900 Subject: [PATCH 26/64] feat(types): add StreamCapabilities.target_hz and FinalizationReport.incidents - StreamCapabilities gains `target_hz: float | None = None` inserted before `live_preview`; `to_dict()` includes the new key. - FinalizationReport gains `incidents: list = field(default_factory=list)` at the end of the field list; typed as bare `list` to avoid a circular import with syncfield.health.types.Incident. - test_types.py: update existing to_dict round-trip assertion to include the new `target_hz` key. Co-Authored-By: Claude Sonnet 4.6 --- src/syncfield/types.py | 3 +++ tests/unit/test_types.py | 1 + tests/unit/test_types_capabilities.py | 13 +++++++++++++ tests/unit/test_types_finalization.py | 28 +++++++++++++++++++++++++++ 4 files changed, 45 insertions(+) diff --git a/src/syncfield/types.py b/src/syncfield/types.py index a0e5b7b..e97c43c 100644 --- a/src/syncfield/types.py +++ b/src/syncfield/types.py @@ -196,6 +196,7 @@ class StreamCapabilities: supports_precise_timestamps: bool = False is_removable: bool = False produces_file: bool = False + target_hz: float | None = None live_preview: bool = True def to_dict(self) -> dict[str, Any]: @@ -204,6 +205,7 @@ def to_dict(self) -> dict[str, Any]: "supports_precise_timestamps": self.supports_precise_timestamps, "is_removable": self.is_removable, "produces_file": self.produces_file, + "target_hz": self.target_hz, "live_preview": self.live_preview, } @@ -336,6 +338,7 @@ class FinalizationReport: error: str | None jitter_p95_ns: int | None = None jitter_p99_ns: int | None = None + incidents: list = field(default_factory=list) @dataclass(frozen=True) diff --git a/tests/unit/test_types.py b/tests/unit/test_types.py index a13b4be..4105966 100644 --- a/tests/unit/test_types.py +++ b/tests/unit/test_types.py @@ -183,6 +183,7 @@ def test_to_dict_round_trip(self): "supports_precise_timestamps": False, "is_removable": True, "produces_file": True, + "target_hz": None, "live_preview": True, } diff --git a/tests/unit/test_types_capabilities.py b/tests/unit/test_types_capabilities.py index 755cf37..287ed75 100644 --- a/tests/unit/test_types_capabilities.py +++ b/tests/unit/test_types_capabilities.py @@ -16,3 +16,16 @@ def test_to_dict_includes_live_preview(): d = caps.to_dict() assert d["live_preview"] is False assert d["produces_file"] is False + + +def test_target_hz_defaults_to_none(): + from syncfield.types import StreamCapabilities + caps = StreamCapabilities() + assert caps.target_hz is None + + +def test_target_hz_round_trips_to_dict(): + from syncfield.types import StreamCapabilities + caps = StreamCapabilities(target_hz=30.0) + d = caps.to_dict() + assert d["target_hz"] == 30.0 diff --git a/tests/unit/test_types_finalization.py b/tests/unit/test_types_finalization.py index a401716..4fe5c1b 100644 --- a/tests/unit/test_types_finalization.py +++ b/tests/unit/test_types_finalization.py @@ -14,3 +14,31 @@ def test_finalization_report_accepts_pending_aggregation_status(): error=None, ) assert report.status == "pending_aggregation" + + +def test_finalization_report_incidents_default_empty(): + from syncfield.types import FinalizationReport + r = FinalizationReport( + stream_id="cam", status="completed", frame_count=10, file_path=None, + first_sample_at_ns=0, last_sample_at_ns=100, health_events=[], error=None, + ) + assert r.incidents == [] + + +def test_finalization_report_accepts_incidents(): + from syncfield.health.types import Incident + from syncfield.health.severity import Severity + from syncfield.types import FinalizationReport, HealthEvent, HealthEventKind + + ev = HealthEvent( + stream_id="cam", kind=HealthEventKind.ERROR, at_ns=1, detail="x", + severity=Severity.ERROR, source="detector:stream-stall", + fingerprint="cam:stream-stall", + ) + inc = Incident.opened_from(ev, title="stall") + r = FinalizationReport( + stream_id="cam", status="completed", frame_count=10, file_path=None, + first_sample_at_ns=0, last_sample_at_ns=100, health_events=[], error=None, + incidents=[inc], + ) + assert r.incidents == [inc] From 141dc4a434749b9e959dfb686de062c8189d083a Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 13:34:23 +0900 Subject: [PATCH 27/64] feat(writer): add SessionLogWriter.log_incident + incidents.jsonl Extends SessionLogWriter to track incidents alongside events. Each incident state is serialized as a JSON line to incidents.jsonl and flushed immediately for crash-resilience. - Add incidents_path property and _incidents_handle file management - Implement log_incident(incident) method that mirrors log_event() - Update open()/close() to manage incidents file lifecycle Co-Authored-By: Claude Opus 4.7 (1M context) --- src/syncfield/writer.py | 28 ++++++++++++++++++++++++++-- tests/unit/test_writer.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/src/syncfield/writer.py b/src/syncfield/writer.py index 20a5534..8657186 100644 --- a/src/syncfield/writer.py +++ b/src/syncfield/writer.py @@ -116,21 +116,31 @@ class SessionLogWriter: crash mid-recording and the core service can reconstruct partial sessions from the file. - Output file: ``session_log.jsonl`` + Output files: + - ``session_log.jsonl`` — state transitions and health events + - ``incidents.jsonl`` — incident lifecycle events """ def __init__(self, output_dir: Path) -> None: self._path = output_dir / "session_log.jsonl" + self._incidents_path = output_dir / "incidents.jsonl" self._handle: IO[str] | None = None + self._incidents_handle: IO[str] | None = None @property def path(self) -> Path: return self._path + @property + def incidents_path(self) -> Path: + return self._incidents_path + def open(self) -> None: - """Open the log file for writing. Idempotent on an already-open writer.""" + """Open the log files for writing. Idempotent on an already-open writer.""" if self._handle is None: self._handle = open(self._path, "w") + if self._incidents_handle is None: + self._incidents_handle = open(self._incidents_path, "w") def log_event(self, event: dict[str, Any]) -> None: """Serialize *event* as a single JSON line and flush. @@ -155,10 +165,24 @@ def log_health(self, event: HealthEvent) -> None: } ) + def log_incident(self, incident: Any) -> None: + """Serialize an :class:`Incident` as a single JSON line and flush. + + Raises: + RuntimeError: If the writer has not been opened. + """ + if self._incidents_handle is None: + raise RuntimeError("SessionLogWriter is not open") + self._incidents_handle.write(json.dumps(incident.to_dict(), separators=(",", ":")) + "\n") + self._incidents_handle.flush() + def close(self) -> None: if self._handle is not None: self._handle.close() self._handle = None + if self._incidents_handle is not None: + self._incidents_handle.close() + self._incidents_handle = None def write_sync_point( diff --git a/tests/unit/test_writer.py b/tests/unit/test_writer.py index 5abc1f7..e2b59e0 100644 --- a/tests/unit/test_writer.py +++ b/tests/unit/test_writer.py @@ -271,3 +271,37 @@ def test_manifest_omits_session_config_when_none(self, tmp_path): import json manifest = json.loads(path.read_text()) assert "session_config" not in manifest + + +def test_log_incident_appends_to_incidents_jsonl(tmp_path): + from syncfield.health.severity import Severity + from syncfield.health.types import Incident + + def _ev(at_ns: int) -> HealthEvent: + return HealthEvent( + stream_id="cam", kind=HealthEventKind.ERROR, at_ns=at_ns, detail="x", + severity=Severity.ERROR, source="detector:stream-stall", + fingerprint="cam:stream-stall", + ) + + w = SessionLogWriter(tmp_path) + w.open() + try: + inc = Incident.opened_from(_ev(100), title="stall") + w.log_incident(inc) + inc.record_event(_ev(200)) + w.log_incident(inc) + inc.close(at_ns=300) + w.log_incident(inc) + finally: + w.close() + + path = tmp_path / "incidents.jsonl" + assert path.exists() + lines = path.read_text().strip().splitlines() + assert len(lines) == 3 + first = json.loads(lines[0]) + assert first["id"] == inc.id + assert first["event_count"] == 1 + last = json.loads(lines[-1]) + assert last["closed_at_ns"] == 300 From 7acb426327a19d0a590fef94c45fdaf8f15672f7 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 13:44:34 +0900 Subject: [PATCH 28/64] feat(orchestrator): wire SessionOrchestrator to HealthSystem (samples, health, state, incidents) Co-Authored-By: Claude Sonnet 4.6 --- src/syncfield/orchestrator.py | 56 +++++++++ tests/integration/health/__init__.py | 0 .../test_orchestrator_health_integration.py | 112 ++++++++++++++++++ 3 files changed, 168 insertions(+) create mode 100644 tests/integration/health/__init__.py create mode 100644 tests/integration/health/test_orchestrator_health_integration.py diff --git a/src/syncfield/orchestrator.py b/src/syncfield/orchestrator.py index d4bfd5a..cd3c88c 100644 --- a/src/syncfield/orchestrator.py +++ b/src/syncfield/orchestrator.py @@ -392,6 +392,14 @@ def __init__( self._sample_writers: Dict[str, SampleWriter] = {} self._sample_handler_active: Dict[str, List[bool]] = {} + # Health telemetry — constructed once per orchestrator instance, + # started/stopped alongside each recording cycle. + from syncfield.health import HealthSystem + self.health = HealthSystem() + self.health.on_incident_opened = self._persist_incident + self.health.on_incident_updated = self._persist_incident + self.health.on_incident_closed = self._persist_incident + # Multi-host: start control plane + advertiser (or follower browser) # at construction time so cluster discovery is independent of device # lifecycle. Failures here raise out of __init__ (the constructor is @@ -1502,6 +1510,7 @@ def add(self, stream: Stream) -> None: self._streams[stream.id] = stream stream.on_health(self._on_stream_health) + stream.on_sample(lambda s, _sid=stream.id: self.health.observe_sample(_sid, s)) # After the first non-audio stream is registered, check whether # to pre-register a host audio stream so it appears in the @@ -1633,6 +1642,10 @@ def connect(self) -> None: # user having to add an audio stream manually. self._maybe_inject_host_audio() + # Start the health worker so samples flowing from stream.connect() + # onward are observed. Idempotent — safe to call again in start(). + self.health.start() + self._transition(SessionState.CONNECTED) # ------------------------------------------------------------------ @@ -1848,6 +1861,7 @@ def _tick_with_beep(n: int) -> None: # the recorded audio track, so we wait until every stream # has enabled file writing before playing it. self._maybe_play_start_chirp() + self.health.start() self._transition(SessionState.RECORDING) # Distribute the leader's SessionConfig to every preparing @@ -1918,6 +1932,20 @@ def stop(self) -> SessionReport: finalizations = self._finalize_streams() + # Stop the health worker and close any still-open incidents. + # Must run AFTER finalize_streams (so stall detectors see the + # stream stop) but BEFORE closing _log_writer (so + # _persist_incident can still flush to incidents.jsonl). + self.health.stop() + all_incidents = list(self.health.open_incidents()) + list( + self.health.resolved_incidents() + ) + by_stream: Dict[str, list] = {} + for inc in all_incidents: + by_stream.setdefault(inc.stream_id, []).append(inc) + for report in finalizations: + report.incidents = by_stream.get(report.stream_id, []) + # Leader: flip advert status to stopped BEFORE closing the # advertiser so every follower on the network observes the # transition. Close happens further down after artifacts @@ -2078,6 +2106,9 @@ def disconnect(self) -> None: # stays up across disconnect(). It was brought up at __init__ # and is only torn down by shutdown() or the atexit handler. + # Stop health worker — no-op if already stopped by stop(). + self.health.stop(close_open_incidents=False) + self._transition(SessionState.IDLE) if self._log_writer is not None: self._log_writer.close() @@ -2328,6 +2359,7 @@ def _transition(self, new_state: SessionState) -> None: """ old = self._state self._state = new_state + self.health.observe_state(old, new_state) if self._log_writer is not None: self._log_writer.log_event( { @@ -2361,6 +2393,30 @@ def _on_stream_health(self, event: HealthEvent) -> None: """ if self._log_writer is not None: self._log_writer.log_health(event) + self.health.observe_health(event.stream_id, event) + + def _persist_incident(self, incident) -> None: + """Forward incidents to the session log writer + attach crash dumps. + + Called from the HealthWorker thread — must not raise or the worker dies. + """ + # Attach crash_dump artifacts for device-crash incidents (idempotent). + if incident.fingerprint.endswith(":device-crash"): + path = ( + incident.first_event.data.get("crash_dump_path") + if incident.first_event.data + else None + ) + if path and not any(a.kind == "crash_dump" for a in incident.artifacts): + from syncfield.health.types import IncidentArtifact + incident.attach(IncidentArtifact(kind="crash_dump", path=str(path))) + + if self._log_writer is not None: + try: + self._log_writer.log_incident(incident) + except Exception: + # Never let telemetry persistence crash the recording. + pass # ------------------------------------------------------------------ # Episode lifecycle diff --git a/tests/integration/health/__init__.py b/tests/integration/health/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/health/test_orchestrator_health_integration.py b/tests/integration/health/test_orchestrator_health_integration.py new file mode 100644 index 0000000..a1fe48c --- /dev/null +++ b/tests/integration/health/test_orchestrator_health_integration.py @@ -0,0 +1,112 @@ +"""Integration: real SessionOrchestrator + FakeStream → incidents flow end-to-end.""" +import json +import threading +import time +from pathlib import Path + +import pytest + +from syncfield.orchestrator import SessionOrchestrator +from syncfield.stream import StreamBase +from syncfield.clock import SessionClock +from syncfield.types import FinalizationReport, SampleEvent, StreamCapabilities + + +class FakeStream(StreamBase): + def __init__(self, stream_id: str, target_hz: float | None = None): + super().__init__( + id=stream_id, + kind="sensor", + capabilities=StreamCapabilities(target_hz=target_hz), + ) + self._interval = 1.0 / 30.0 + self._stop_thread = threading.Event() + self._pause = threading.Event() + self._thread: threading.Thread | None = None + self._frame = 0 + + def connect(self): + self._stop_thread.clear() + self._thread = threading.Thread(target=self._run, daemon=True) + self._thread.start() + + def disconnect(self): + self._stop_thread.set() + if self._thread: + self._thread.join(timeout=1.0) + self._thread = None + + def start_recording(self, session_clock: SessionClock) -> None: + pass + + def stop_recording(self) -> FinalizationReport: + return FinalizationReport( + stream_id=self.id, status="completed", frame_count=self._frame, + file_path=None, first_sample_at_ns=0, last_sample_at_ns=0, + health_events=[], error=None, + ) + + def pause_samples(self): + self._pause.set() + + def resume_samples(self): + self._pause.clear() + + def _run(self): + while not self._stop_thread.is_set(): + if not self._pause.is_set(): + self._frame += 1 + self._emit_sample(SampleEvent( + stream_id=self.id, frame_number=self._frame, + capture_ns=time.monotonic_ns(), + )) + time.sleep(self._interval) + + +@pytest.mark.slow +def test_stall_incident_open_and_close(tmp_path: Path): + sess = SessionOrchestrator(host_id="test", output_dir=tmp_path) + stream = FakeStream("cam", target_hz=30.0) + sess.add(stream) + + sess.connect() + sess.start(countdown_s=0) + + # Induce stall for 3s — StreamStallDetector default threshold is 2s. + stream.pause_samples() + time.sleep(3.0) + + opens = [i for i in sess.health.open_incidents() if i.fingerprint == "cam:stream-stall"] + assert opens, "stall incident did not open" + + # Recover. + stream.resume_samples() + time.sleep(2.5) + + sess.stop() + sess.disconnect() + + resolved = [i for i in sess.health.resolved_incidents() if i.fingerprint == "cam:stream-stall"] + assert resolved, "stall incident did not resolve" + + +@pytest.mark.slow +def test_incidents_jsonl_written(tmp_path: Path): + sess = SessionOrchestrator(host_id="test", output_dir=tmp_path) + stream = FakeStream("cam", target_hz=30.0) + sess.add(stream) + + sess.connect() + sess.start(countdown_s=0) + stream.pause_samples() + time.sleep(2.5) + sess.stop() + sess.disconnect() + + # Locate incidents.jsonl — the orchestrator places it inside a session-specific subdir. + out_files = list(tmp_path.rglob("incidents.jsonl")) + assert out_files, "no incidents.jsonl written" + lines = out_files[0].read_text().strip().splitlines() + fingerprints = [json.loads(l)["fingerprint"] for l in lines] + assert any(fp == "cam:stream-stall" for fp in fingerprints), \ + f"stall fingerprint missing — found: {set(fingerprints)}" From 63bdf89e46ec2e16b446fe0709b0962d5b2e21a5 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 13:51:18 +0900 Subject: [PATCH 29/64] fix(health): resilient worker ingest + FakeStream health defaults - Wrap tracker.ingest() calls in _safe_ingest() to prevent malformed events from crashing the worker thread - Log warnings instead of silently exiting on IncidentTracker.ingest() failures - Update FakeStream.push_health() to provide non-empty fingerprint, severity, and source so test events flow through the tracker cleanly - Eliminates PytestUnhandledThreadExceptionWarning in TestHealthRouting Co-Authored-By: Claude Opus 4.7 (1M context) --- src/syncfield/health/worker.py | 15 +++++++++++++-- src/syncfield/testing.py | 22 +++++++++++++++++++++- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/syncfield/health/worker.py b/src/syncfield/health/worker.py index e209af9..7c313cc 100644 --- a/src/syncfield/health/worker.py +++ b/src/syncfield/health/worker.py @@ -132,7 +132,7 @@ def _drain_once(self) -> None: for msg in _drain_queue(self._healths): for d in self._detectors: d.observe_health(msg.stream_id, msg.event) - self._tracker.ingest(msg.event) + self._safe_ingest(msg.event) for msg in _drain_queue(self._states): for d in self._detectors: d.observe_state(msg.old, msg.new) @@ -144,7 +144,18 @@ def _fire_detector_ticks(self) -> None: now = time.monotonic_ns() for d in self._detectors: for event in d.tick(now): - self._tracker.ingest(event) + self._safe_ingest(event) + + def _safe_ingest(self, event: HealthEvent) -> None: + """Ingest but never let a malformed event crash the worker thread.""" + try: + self._tracker.ingest(event) + except Exception as exc: # noqa: BLE001 — telemetry must not crash + import logging + logging.getLogger(__name__).warning( + "IncidentTracker.ingest dropped event: %s (fingerprint=%r, source=%r)", + exc, event.fingerprint, event.source, + ) def _drain_queue(q: "queue.SimpleQueue") -> list: diff --git a/src/syncfield/testing.py b/src/syncfield/testing.py index c4650cd..c1c9dce 100644 --- a/src/syncfield/testing.py +++ b/src/syncfield/testing.py @@ -20,6 +20,18 @@ ) +def _severity_for(kind: HealthEventKind): + from syncfield.health.severity import Severity + mapping = { + HealthEventKind.HEARTBEAT: Severity.INFO, + HealthEventKind.RECONNECT: Severity.INFO, + HealthEventKind.DROP: Severity.WARNING, + HealthEventKind.WARNING: Severity.WARNING, + HealthEventKind.ERROR: Severity.ERROR, + } + return mapping.get(kind, Severity.WARNING) + + class FakeStream(StreamBase): """Programmable in-memory :class:`~syncfield.stream.Stream` used by tests. @@ -110,4 +122,12 @@ def push_health( detail: Optional[str] = None, ) -> None: """Emit a synthetic health event through the callback path.""" - self._emit_health(HealthEvent(self.id, kind, at_ns, detail)) + self._emit_health(HealthEvent( + stream_id=self.id, + kind=kind, + at_ns=at_ns, + detail=detail, + severity=_severity_for(kind), + source="adapter:test", + fingerprint=f"{self.id}:adapter:{kind.value}", + )) From 917429b850537316f175dc31ef7af0fa3300fe0e Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 13:52:59 +0900 Subject: [PATCH 30/64] =?UTF-8?q?feat(health):=20add=20DepthAILoggerBridge?= =?UTF-8?q?=20(logging.Handler=20=E2=86=92=20HealthEvent)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Translates depthai Python log records into HealthEvent payloads. Parses five fingerprint patterns (xlink-error, device-crash, reconnect-attempt, reconnect-success, connection-closed), with fallback to unparsed-log. Installed on depthai logger; owned by AdapterEventPassthrough detector. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../health/detectors/depthai_bridge.py | 115 ++++++++++++++++++ .../health/detectors/test_depthai_bridge.py | 59 +++++++++ 2 files changed, 174 insertions(+) create mode 100644 src/syncfield/health/detectors/depthai_bridge.py create mode 100644 tests/unit/health/detectors/test_depthai_bridge.py diff --git a/src/syncfield/health/detectors/depthai_bridge.py b/src/syncfield/health/detectors/depthai_bridge.py new file mode 100644 index 0000000..81ae755 --- /dev/null +++ b/src/syncfield/health/detectors/depthai_bridge.py @@ -0,0 +1,115 @@ +"""DepthAILoggerBridge — translate depthai Python log records into HealthEvents. + +Installed as a standard :class:`logging.Handler` on the depthai logger. +Does not subclass DetectorBase — it is a translator, not a detector. +Its outputs are fingerprinted as ``:adapter:`` so +the AdapterEventPassthrough detector owns their open/close lifecycle. +""" + +from __future__ import annotations + +import logging +import re +import time +from typing import Callable, Optional + +from syncfield.health.severity import Severity +from syncfield.types import HealthEvent, HealthEventKind + +Sink = Callable[[str, HealthEvent], None] + +_XLINK_RE = re.compile(r"X_LINK_ERROR.*stream: '([^']+)'|stream: '([^']+)'.*X_LINK_ERROR") +_CRASH_RE = re.compile(r"Device with id (\S+) has crashed\. Crash dump logs are stored in: (\S+)") +_RECONNECT_TRY_RE = re.compile(r"Attempting to reconnect", re.IGNORECASE) +_RECONNECT_OK_RE = re.compile(r"Reconnection successful", re.IGNORECASE) +_CONN_CLOSED_RE = re.compile(r"Closed connection", re.IGNORECASE) + + +class DepthAILoggerBridge(logging.Handler): + def __init__(self, stream_id: str, sink: Sink) -> None: + super().__init__(level=logging.WARNING) + self._stream_id = stream_id + self._sink = sink + + def emit(self, record: logging.LogRecord) -> None: + if record.levelno < logging.WARNING: + return + msg = record.getMessage() + now = time.monotonic_ns() + + parsed = self._parse(msg, record.levelno, now) + if parsed is None: + parsed = HealthEvent( + stream_id=self._stream_id, + kind=HealthEventKind.WARNING, + at_ns=now, + detail=msg, + severity=Severity.WARNING, + source="adapter:oak:unparsed-log", + fingerprint=f"{self._stream_id}:adapter:unparsed-log", + data={"raw": msg, "levelname": record.levelname}, + ) + try: + self._sink(self._stream_id, parsed) + except Exception: + # Never let bridge failures crash the logging path. + pass + + def _parse(self, msg: str, levelno: int, now: int) -> Optional[HealthEvent]: + crash = _CRASH_RE.search(msg) + if crash: + device_id, path = crash.group(1), crash.group(2) + return HealthEvent( + stream_id=self._stream_id, + kind=HealthEventKind.ERROR, + at_ns=now, + detail="OAK device crashed", + severity=Severity.CRITICAL, + source="adapter:oak", + fingerprint=f"{self._stream_id}:adapter:device-crash", + data={"device_id": device_id, "crash_dump_path": path}, + ) + xlink = _XLINK_RE.search(msg) + if xlink: + stream = xlink.group(1) or xlink.group(2) + return HealthEvent( + stream_id=self._stream_id, + kind=HealthEventKind.ERROR, + at_ns=now, + detail="XLink communication error", + severity=Severity.ERROR, + source="adapter:oak", + fingerprint=f"{self._stream_id}:adapter:xlink-error", + data={"stream": stream}, + ) + if _RECONNECT_OK_RE.search(msg): + return HealthEvent( + stream_id=self._stream_id, + kind=HealthEventKind.RECONNECT, + at_ns=now, + detail="Reconnection successful", + severity=Severity.INFO, + source="adapter:oak", + fingerprint=f"{self._stream_id}:adapter:reconnect-success", + ) + if _RECONNECT_TRY_RE.search(msg): + return HealthEvent( + stream_id=self._stream_id, + kind=HealthEventKind.RECONNECT, + at_ns=now, + detail="Attempting reconnect", + severity=Severity.WARNING, + source="adapter:oak", + fingerprint=f"{self._stream_id}:adapter:reconnect-attempt", + ) + if _CONN_CLOSED_RE.search(msg): + return HealthEvent( + stream_id=self._stream_id, + kind=HealthEventKind.WARNING, + at_ns=now, + detail="Connection closed", + severity=Severity.WARNING, + source="adapter:oak", + fingerprint=f"{self._stream_id}:adapter:connection-closed", + ) + return None diff --git a/tests/unit/health/detectors/test_depthai_bridge.py b/tests/unit/health/detectors/test_depthai_bridge.py new file mode 100644 index 0000000..d89d55b --- /dev/null +++ b/tests/unit/health/detectors/test_depthai_bridge.py @@ -0,0 +1,59 @@ +import logging +from typing import List + +from syncfield.health.detectors.depthai_bridge import DepthAILoggerBridge +from syncfield.types import HealthEvent + + +def _mk_record(msg: str, level: int = logging.ERROR, name: str = "depthai") -> logging.LogRecord: + return logging.LogRecord( + name=name, level=level, pathname="", lineno=0, msg=msg, args=(), exc_info=None, + ) + + +def test_xlink_error_maps_to_xlink_fingerprint(): + captured: List[HealthEvent] = [] + bridge = DepthAILoggerBridge(stream_id="oak-main", sink=lambda sid, ev: captured.append(ev)) + rec = _mk_record("Communication exception - possible device error. Original message 'Couldn't read data from stream: '__x_0_1' (X_LINK_ERROR)'") + bridge.emit(rec) + assert len(captured) == 1 + ev = captured[0] + assert ev.stream_id == "oak-main" + assert ev.fingerprint == "oak-main:adapter:xlink-error" + assert ev.source == "adapter:oak" + assert ev.data.get("stream") == "__x_0_1" + + +def test_device_crash_attaches_crash_dump_path(): + captured: List[HealthEvent] = [] + bridge = DepthAILoggerBridge(stream_id="oak-main", sink=lambda sid, ev: captured.append(ev)) + rec = _mk_record("Device with id 194430 has crashed. Crash dump logs are stored in: /tmp/crash/crash_dump.json - please report to developers.") + bridge.emit(rec) + ev = captured[0] + assert ev.fingerprint == "oak-main:adapter:device-crash" + assert ev.data.get("crash_dump_path") == "/tmp/crash/crash_dump.json" + + +def test_reconnect_attempt_and_success_have_distinct_fingerprints(): + captured: List[HealthEvent] = [] + bridge = DepthAILoggerBridge(stream_id="oak-main", sink=lambda sid, ev: captured.append(ev)) + bridge.emit(_mk_record("Attempting to reconnect. Timeout is 10000ms", level=logging.WARNING)) + bridge.emit(_mk_record("Reconnection successful", level=logging.WARNING)) + fps = [c.fingerprint for c in captured] + assert "oak-main:adapter:reconnect-attempt" in fps + assert "oak-main:adapter:reconnect-success" in fps + + +def test_unrecognized_error_falls_back_to_warning_unparsed(): + captured: List[HealthEvent] = [] + bridge = DepthAILoggerBridge(stream_id="oak-main", sink=lambda sid, ev: captured.append(ev)) + bridge.emit(_mk_record("Something totally new and unrecognized", level=logging.ERROR)) + assert len(captured) == 1 + assert captured[0].source == "adapter:oak:unparsed-log" + + +def test_info_records_are_ignored(): + captured = [] + bridge = DepthAILoggerBridge(stream_id="oak-main", sink=lambda sid, ev: captured.append(ev)) + bridge.emit(_mk_record("Some info", level=logging.INFO)) + assert captured == [] From 1a42085244e3bb9d69a0d46caa1bc4fbfd5e7231 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 13:56:16 +0900 Subject: [PATCH 31/64] feat(oak): declare target_hz, bridge depthai logger for health telemetry Wire OakCameraStream to expose target_hz=rgb_fps in StreamCapabilities, install DepthAILoggerBridge on connect() to route depthai log records into _emit_health, and uninstall on disconnect(). Adds idempotent _install/_uninstall_depthai_bridge helpers. Co-Authored-By: Claude Sonnet 4.6 --- src/syncfield/adapters/oak_camera.py | 31 +++++++++++++ tests/unit/adapters/test_oak_camera_health.py | 43 +++++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 tests/unit/adapters/test_oak_camera_health.py diff --git a/src/syncfield/adapters/oak_camera.py b/src/syncfield/adapters/oak_camera.py index cfb721f..1385195 100644 --- a/src/syncfield/adapters/oak_camera.py +++ b/src/syncfield/adapters/oak_camera.py @@ -172,6 +172,7 @@ def __init__( supports_precise_timestamps=True, is_removable=True, produces_file=True, + target_hz=float(rgb_fps), ), ) self._output_dir = Path(output_dir) @@ -332,6 +333,8 @@ def connect(self) -> None: attached devices after :attr:`_ENUMERATE_RETRIES` probe attempts. """ + self._install_depthai_bridge() + if self._thread is not None and self._thread.is_alive(): return @@ -493,6 +496,33 @@ def _finalize_mp4(self) -> bool: pass return True + def _install_depthai_bridge(self) -> None: + """Install a logging.Handler that converts depthai native log records into HealthEvents. + + Idempotent. Registered on depthai's module logger so every internal + warning/error that depthai emits during this stream's lifetime is + routed to the IncidentTracker via :meth:`_emit_health`. + """ + import logging as _logging + from syncfield.health.detectors.depthai_bridge import DepthAILoggerBridge + + if getattr(self, "_depthai_bridge", None) is not None: + return + self._depthai_bridge = DepthAILoggerBridge( + stream_id=self.id, + sink=lambda _sid, ev: self._emit_health(ev), + ) + _logging.getLogger("depthai").addHandler(self._depthai_bridge) + + def _uninstall_depthai_bridge(self) -> None: + """Remove the depthai logging handler. Idempotent.""" + import logging as _logging + bridge = getattr(self, "_depthai_bridge", None) + if bridge is None: + return + _logging.getLogger("depthai").removeHandler(bridge) + self._depthai_bridge = None + def disconnect(self) -> None: """Stop the capture thread and release the DepthAI pipeline. @@ -505,6 +535,7 @@ def disconnect(self) -> None: self._thread.join(timeout=3.0) self._thread = None self._release_pipeline() + self._uninstall_depthai_bridge() # ------------------------------------------------------------------ # Legacy one-shot lifecycle diff --git a/tests/unit/adapters/test_oak_camera_health.py b/tests/unit/adapters/test_oak_camera_health.py new file mode 100644 index 0000000..e16f7eb --- /dev/null +++ b/tests/unit/adapters/test_oak_camera_health.py @@ -0,0 +1,43 @@ +"""Unit tests for OAK adapter health wiring (no real hardware required).""" + +import logging +import pytest + + +pytest.importorskip("depthai") # skip entire module if the oak extra isn't installed + + +def test_oak_declares_target_hz(tmp_path): + from syncfield.adapters.oak_camera import OakCameraStream + s = OakCameraStream(id="oak-main", rgb_fps=30, output_dir=tmp_path) + assert s.capabilities.target_hz == 30.0 + + +def test_oak_bridge_install_routes_depthai_errors_to_emit_health(tmp_path): + from syncfield.adapters.oak_camera import OakCameraStream + + captured = [] + s = OakCameraStream(id="oak-main", rgb_fps=30, output_dir=tmp_path) + s.on_health(lambda ev: captured.append(ev)) + + # Install the bridge directly (connect() builds a pipeline we can't easily mock here). + s._install_depthai_bridge() + try: + logging.getLogger("depthai").error( + "Communication exception - Original message 'Couldn't read data from stream: '__x_0_1' (X_LINK_ERROR)'" + ) + finally: + s._uninstall_depthai_bridge() + + assert any(ev.fingerprint == "oak-main:adapter:xlink-error" for ev in captured), \ + f"no xlink-error event captured; got fingerprints: {[c.fingerprint for c in captured]}" + + +def test_oak_bridge_uninstall_is_idempotent(tmp_path): + from syncfield.adapters.oak_camera import OakCameraStream + + s = OakCameraStream(id="oak-main", rgb_fps=30, output_dir=tmp_path) + s._uninstall_depthai_bridge() # no-op before install + s._install_depthai_bridge() + s._uninstall_depthai_bridge() + s._uninstall_depthai_bridge() # no-op after uninstall From 11b896fcde63ea9a79aeb68698a35863015ee491 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 14:03:18 +0900 Subject: [PATCH 32/64] refactor(viewer): replace HealthEntry/health_count with IncidentSnapshot fields Remove the per-stream HealthEntry buffer and health_count field from StreamSnapshot. Replace SessionSnapshot.health_log with top-level active_incidents / resolved_incidents (List[IncidentSnapshot]) fed by HealthSystem.on_incident_* callbacks wired in SessionPoller.__init__. Update server.py snapshot serialization to emit the new incident lists. Retire the old per-stream health tests; add test_snapshot_incidents.py. Co-Authored-By: Claude Sonnet 4.6 --- src/syncfield/viewer/poller.py | 72 ++++++++++--------- src/syncfield/viewer/server.py | 40 +++++------ src/syncfield/viewer/state.py | 40 ++--------- .../unit/test_viewer_aggregation_snapshot.py | 23 ++++-- tests/unit/viewer/test_poller.py | 26 +++---- tests/unit/viewer/test_snapshot_incidents.py | 36 ++++++++++ tests/unit/viewer/test_state.py | 31 ++------ 7 files changed, 127 insertions(+), 141 deletions(-) create mode 100644 tests/unit/viewer/test_snapshot_incidents.py diff --git a/src/syncfield/viewer/poller.py b/src/syncfield/viewer/poller.py index 0a7b116..ef84fe6 100644 --- a/src/syncfield/viewer/poller.py +++ b/src/syncfield/viewer/poller.py @@ -16,16 +16,17 @@ from __future__ import annotations import threading -import time +import time as _time +from collections import deque from pathlib import Path -from typing import Dict, List, Optional +from typing import Dict, Optional +from syncfield.health.types import Incident, IncidentSnapshot from syncfield.orchestrator import SessionOrchestrator from syncfield.stream import Stream -from syncfield.types import HealthEvent, SampleEvent, SessionState +from syncfield.types import SampleEvent, SessionState from syncfield.viewer.state import ( - HealthEntry, SessionSnapshot, StreamSnapshot, StreamStatsBuffer, @@ -66,6 +67,15 @@ def __init__( self._recording_started_at: Optional[float] = None self._last_observed_state: SessionState = SessionState.IDLE + # Incident tracking — fed by HealthSystem callbacks. + self._incidents_lock = threading.Lock() + self._open_by_id: dict[str, Incident] = {} + self._resolved: deque[Incident] = deque(maxlen=20) + + session.health.on_incident_opened = self._ingest_incident + session.health.on_incident_updated = self._ingest_incident + session.health.on_incident_closed = self._ingest_incident + # ------------------------------------------------------------------ # Public API # ------------------------------------------------------------------ @@ -96,7 +106,7 @@ def get_snapshot(self) -> Optional[SessionSnapshot]: # ------------------------------------------------------------------ def _register_callbacks(self) -> None: - """Attach on_sample / on_health to each registered stream. + """Attach on_sample to each registered stream. Re-entrant: callbacks for streams we've already registered for are skipped by tracking which stream ids already have a buffer. @@ -107,7 +117,6 @@ def _register_callbacks(self) -> None: buffer = StreamStatsBuffer() self._stats[stream_id] = buffer stream.on_sample(self._make_sample_callback(stream_id, buffer)) - stream.on_health(self._make_health_callback(stream_id, buffer)) @staticmethod def _make_sample_callback(stream_id: str, buffer: StreamStatsBuffer): @@ -116,19 +125,14 @@ def _on_sample(event: SampleEvent) -> None: return _on_sample - @staticmethod - def _make_health_callback(stream_id: str, buffer: StreamStatsBuffer): - def _on_health(event: HealthEvent) -> None: - buffer.observe_health( - HealthEntry( - stream_id=stream_id, - kind=event.kind.value, - at_ns=event.at_ns, - detail=event.detail, - ) - ) - - return _on_health + def _ingest_incident(self, incident: Incident) -> None: + """Called from the HealthSystem worker thread whenever an incident changes.""" + with self._incidents_lock: + if incident.is_open: + self._open_by_id[incident.id] = incident + else: + self._open_by_id.pop(incident.id, None) + self._resolved.append(incident) # ------------------------------------------------------------------ # Poll loop @@ -157,7 +161,7 @@ def _build_snapshot(self) -> SessionSnapshot: # Track the recording start time so we can compute elapsed seconds. current_state: SessionState = session.state - now = time.time() + now = _time.time() if ( current_state is SessionState.RECORDING and self._last_observed_state is not SessionState.RECORDING @@ -171,7 +175,7 @@ def _build_snapshot(self) -> SessionSnapshot: if self._recording_started_at is not None: elapsed_s = max(0.0, now - self._recording_started_at) - now_ns = time.monotonic_ns() + now_ns = _time.monotonic_ns() streams_snapshot: Dict[str, StreamSnapshot] = {} for stream_id, stream in session._streams.items(): # type: ignore[attr-defined] buffer = self._stats.get(stream_id) @@ -207,12 +211,20 @@ def _build_snapshot(self) -> SessionSnapshot: latest_frame=latest_frame, plot_points=plot_points, latest_pose=latest_pose, - health_count=len(buffer._health), live_preview=getattr(stream.capabilities, "live_preview", True), ) - # Merge health events into a session-wide, time-sorted log. - health_log = self._collect_health_log() + # Snapshot incidents — take a consistent copy under the incidents lock. + with self._incidents_lock: + now_ns = _time.monotonic_ns() + active = [ + IncidentSnapshot.from_incident(i, now_ns=now_ns) + for i in self._open_by_id.values() + ] + resolved = [ + IncidentSnapshot.from_incident(i, now_ns=now_ns) + for i in self._resolved + ] # Session-level sync point + chirp fields. sync_point = getattr(session, "_sync_point", None) @@ -233,7 +245,8 @@ def _build_snapshot(self) -> SessionSnapshot: chirp_enabled=chirp_enabled, elapsed_s=elapsed_s, streams=streams_snapshot, - health_log=health_log, + active_incidents=active, + resolved_incidents=resolved, ) @staticmethod @@ -241,12 +254,3 @@ def _safe_latest_frame(stream: Stream): """Read ``stream.latest_frame`` if the adapter exposes it.""" frame = getattr(stream, "latest_frame", None) return frame - - def _collect_health_log(self) -> List[HealthEntry]: - """Merge per-stream health deques into a time-sorted global log.""" - merged: List[HealthEntry] = [] - for buffer in self._stats.values(): - merged.extend(buffer.snapshot_health()) - merged.sort(key=lambda e: e.at_ns) - # Cap to the most recent N so a long session doesn't blow up the table. - return merged[-50:] diff --git a/src/syncfield/viewer/server.py b/src/syncfield/viewer/server.py index ffc9d02..5c1a547 100644 --- a/src/syncfield/viewer/server.py +++ b/src/syncfield/viewer/server.py @@ -37,7 +37,7 @@ from syncfield.orchestrator import SessionOrchestrator from syncfield.viewer.poller import SessionPoller -from syncfield.viewer.state import AggregationSnapshot, HealthEntry, SessionSnapshot, StreamSnapshot +from syncfield.viewer.state import AggregationSnapshot, SessionSnapshot, StreamSnapshot logger = logging.getLogger(__name__) @@ -70,14 +70,6 @@ def snapshot_to_dict(snapshot: SessionSnapshot) -> Dict[str, Any]: now_ns = time.monotonic_ns() streams: Dict[str, Any] = {} - # Count non-heartbeat events per stream (heartbeats are informational) - problem_count_by_stream: Dict[str, int] = {} - for h in snapshot.health_log: - if h.kind != "heartbeat": - problem_count_by_stream[h.stream_id] = ( - problem_count_by_stream.get(h.stream_id, 0) + 1 - ) - for sid, s in snapshot.streams.items(): last_sample_ms_ago: Optional[float] = None if s.last_sample_at_ns is not None: @@ -91,8 +83,6 @@ def snapshot_to_dict(snapshot: SessionSnapshot) -> Dict[str, Any]: "last_sample_ms_ago": last_sample_ms_ago, "provides_audio_track": s.provides_audio_track, "produces_file": s.produces_file, - "health_count": s.health_count, - "problem_count": problem_count_by_stream.get(sid, 0), "capabilities": { "live_preview": getattr(s, "live_preview", True), "provides_audio_track": s.provides_audio_track, @@ -102,16 +92,21 @@ def snapshot_to_dict(snapshot: SessionSnapshot) -> Dict[str, Any]: }, } - health_log: List[Dict[str, Any]] = [] - for h in snapshot.health_log: - # Convert monotonic ns to "seconds ago" for display - ago_s = round((now_ns - h.at_ns) / 1e9, 1) if h.at_ns else 0 - health_log.append({ - "stream_id": h.stream_id, - "kind": h.kind, - "ago_s": ago_s, - "detail": h.detail, - }) + def _serialize_incident(inc) -> Dict[str, Any]: + return { + "id": inc.id, + "stream_id": inc.stream_id, + "fingerprint": inc.fingerprint, + "title": inc.title, + "severity": inc.severity, + "source": inc.source, + "opened_at_ns": inc.opened_at_ns, + "closed_at_ns": inc.closed_at_ns, + "event_count": inc.event_count, + "detail": inc.detail, + "ago_s": round(inc.ago_s, 1), + "artifacts": inc.artifacts, + } return { "type": "snapshot", @@ -124,7 +119,8 @@ def snapshot_to_dict(snapshot: SessionSnapshot) -> Dict[str, Any]: "stop_ns": snapshot.chirp_stop_ns, }, "streams": streams, - "health_log": health_log, + "active_incidents": [_serialize_incident(i) for i in snapshot.active_incidents], + "resolved_incidents": [_serialize_incident(i) for i in snapshot.resolved_incidents], "output_dir": snapshot.output_dir, "aggregation": _serialize_aggregation(getattr(snapshot, "aggregation", None)), } diff --git a/src/syncfield/viewer/state.py b/src/syncfield/viewer/state.py index 84c6b34..81ed64d 100644 --- a/src/syncfield/viewer/state.py +++ b/src/syncfield/viewer/state.py @@ -23,6 +23,8 @@ from dataclasses import dataclass, field from typing import Any, Deque, Dict, List, Optional, Tuple +from syncfield.health.types import IncidentSnapshot + # --------------------------------------------------------------------------- # Aggregation snapshot @@ -73,8 +75,6 @@ class StreamSnapshot: exposes the latest vector/list sample so panels that render a single-frame pose (3-D hand skeleton, quaternion axes…) have data to draw. Empty for streams that emit only scalars. - health_count: Number of health events this stream has buffered so - far. Useful for showing a red dot on degraded streams. """ id: str @@ -87,29 +87,9 @@ class StreamSnapshot: latest_frame: Any # numpy array or None — kept as Any so numpy is optional plot_points: Dict[str, Tuple[List[float], List[float]]] latest_pose: Dict[str, List[float]] - health_count: int live_preview: bool = True -# --------------------------------------------------------------------------- -# Health event -# --------------------------------------------------------------------------- - - -@dataclass(frozen=True) -class HealthEntry: - """A single health event surfaced by any stream. - - Simpler than :class:`~syncfield.types.HealthEvent` because the viewer - only needs strings for display and a monotonic ordering key. - """ - - stream_id: str - kind: str # "heartbeat" | "drop" | "reconnect" | "warning" | "error" - at_ns: int - detail: Optional[str] - - # --------------------------------------------------------------------------- # Session-level snapshot # --------------------------------------------------------------------------- @@ -131,7 +111,8 @@ class SessionSnapshot: chirp_enabled: Whether :class:`SyncToneConfig` had chirp enabled. elapsed_s: Wall-clock seconds since ``start()``, or 0 if idle. streams: Ordered map ``stream_id -> StreamSnapshot``. - health_log: Most recent health events across all streams (newest last). + active_incidents: Currently open incidents across all streams. + resolved_incidents: Recently closed incidents (newest last, capped at 20). """ host_id: str @@ -144,7 +125,8 @@ class SessionSnapshot: chirp_enabled: bool elapsed_s: float streams: Dict[str, StreamSnapshot] - health_log: List[HealthEntry] + active_incidents: List[IncidentSnapshot] = field(default_factory=list) + resolved_incidents: List[IncidentSnapshot] = field(default_factory=list) aggregation: Optional[AggregationSnapshot] = None @@ -163,7 +145,6 @@ class StreamStatsBuffer: """ max_plot_samples: int = 300 - max_health: int = 20 # Rolling fps window (monotonic ns) _fps_window: Deque[int] = field(default_factory=lambda: deque(maxlen=30)) @@ -179,9 +160,6 @@ class StreamStatsBuffer: # too large to buffer thousands of frames of. _latest_pose: Dict[str, List[float]] = field(default_factory=dict) - # Health events produced by this stream (capped) - _health: Deque[HealthEntry] = field(default_factory=lambda: deque(maxlen=20)) - def observe_sample(self, capture_ns: int, channels: Optional[Dict[str, Any]]) -> None: """Record one sample. Called from the stream's callback thread. @@ -244,9 +222,6 @@ def observe_sample(self, capture_ns: int, channels: Optional[Dict[str, Any]]) -> if name not in plottable: buf.append(float("nan")) - def observe_health(self, event: HealthEntry) -> None: - self._health.append(event) - def snapshot_fps(self, now_ns: int) -> float: """Effective Hz over the last second of samples. @@ -290,9 +265,6 @@ def snapshot_pose(self) -> Dict[str, List[float]]: """ return {name: list(values) for name, values in self._latest_pose.items()} - def snapshot_health(self) -> List[HealthEntry]: - return list(self._health) - # --------------------------------------------------------------------------- # Free helpers diff --git a/tests/unit/test_viewer_aggregation_snapshot.py b/tests/unit/test_viewer_aggregation_snapshot.py index 47ff2a4..ebe529a 100644 --- a/tests/unit/test_viewer_aggregation_snapshot.py +++ b/tests/unit/test_viewer_aggregation_snapshot.py @@ -9,9 +9,18 @@ from syncfield.viewer.server import snapshot_to_dict -def test_snapshot_includes_aggregation_section_empty_by_default(): +def _make_snapshot_mock(**kwargs): snapshot = MagicMock() - snapshot.aggregation = None + snapshot.streams = {} + snapshot.active_incidents = [] + snapshot.resolved_incidents = [] + for k, v in kwargs.items(): + setattr(snapshot, k, v) + return snapshot + + +def test_snapshot_includes_aggregation_section_empty_by_default(): + snapshot = _make_snapshot_mock(aggregation=None) d = snapshot_to_dict(snapshot) assert "aggregation" in d assert d["aggregation"]["active_job"] is None @@ -30,11 +39,11 @@ def test_snapshot_serializes_active_job(): current_bytes=5_000_000, current_total_bytes=10_000_000, ) - snapshot = MagicMock() - snapshot.aggregation = MagicMock() - snapshot.aggregation.active_job = progress - snapshot.aggregation.queue_length = 1 - snapshot.aggregation.recent_jobs = [progress] + agg_mock = MagicMock() + agg_mock.active_job = progress + agg_mock.queue_length = 1 + agg_mock.recent_jobs = [progress] + snapshot = _make_snapshot_mock(aggregation=agg_mock) d = snapshot_to_dict(snapshot) assert d["aggregation"]["active_job"]["state"] == "running" assert d["aggregation"]["active_job"]["current_bytes"] == 5_000_000 diff --git a/tests/unit/viewer/test_poller.py b/tests/unit/viewer/test_poller.py index 1ddb33f..c272183 100644 --- a/tests/unit/viewer/test_poller.py +++ b/tests/unit/viewer/test_poller.py @@ -11,7 +11,6 @@ import syncfield as sf from syncfield.testing import FakeStream -from syncfield.types import HealthEventKind from syncfield.viewer.poller import SessionPoller @@ -79,24 +78,17 @@ def test_push_sample_shows_up_in_next_snapshot(self, tmp_path): finally: session.stop() - def test_push_health_surfaces_in_health_log(self, tmp_path): + def test_snapshot_has_incident_lists(self, tmp_path): + """Verify that SessionSnapshot carries active_incidents / resolved_incidents + (replacing the retired health_log / health_count fields from Task 20).""" session = _make_session(tmp_path) poller = SessionPoller(session, interval_s=0.01) - poller._register_callbacks() - - session.start() - try: - imu = session._streams["imu"] # type: ignore[attr-defined] - imu.push_health(HealthEventKind.WARNING, at_ns=123, detail="burst") - snap = poller._build_snapshot() - assert len(snap.health_log) >= 1 - assert any( - ev.kind == "warning" and ev.detail == "burst" - for ev in snap.health_log - ) - assert snap.streams["imu"].health_count >= 1 - finally: - session.stop() + snap = poller._build_snapshot() + # No incidents yet — both lists should be present and empty. + assert hasattr(snap, "active_incidents") + assert hasattr(snap, "resolved_incidents") + assert snap.active_incidents == [] + assert snap.resolved_incidents == [] def test_start_stop_and_get_snapshot_thread(self, tmp_path): """End-to-end smoke test of the polling thread.""" diff --git a/tests/unit/viewer/test_snapshot_incidents.py b/tests/unit/viewer/test_snapshot_incidents.py new file mode 100644 index 0000000..c37c4cd --- /dev/null +++ b/tests/unit/viewer/test_snapshot_incidents.py @@ -0,0 +1,36 @@ +import dataclasses + +from syncfield.health.severity import Severity +from syncfield.health.types import Incident, IncidentSnapshot +from syncfield.types import HealthEvent, HealthEventKind +from syncfield.viewer.state import SessionSnapshot, StreamSnapshot + + +def _ev(at_ns: int) -> HealthEvent: + return HealthEvent( + stream_id="cam", kind=HealthEventKind.ERROR, at_ns=at_ns, detail="x", + severity=Severity.ERROR, source="detector:stream-stall", + fingerprint="cam:stream-stall", + ) + + +def test_session_snapshot_has_incident_fields(): + snap = SessionSnapshot( + host_id="h", state="recording", output_dir="/tmp", + sync_point_monotonic_ns=None, sync_point_wall_clock_ns=None, + chirp_start_ns=None, chirp_stop_ns=None, chirp_enabled=False, + elapsed_s=0.0, streams={}, active_incidents=[], resolved_incidents=[], + ) + assert snap.active_incidents == [] + assert snap.resolved_incidents == [] + + +def test_stream_snapshot_no_longer_has_health_count(): + fields = {f.name for f in dataclasses.fields(StreamSnapshot)} + assert "health_count" not in fields + assert "problem_count" not in fields + + +def test_session_snapshot_no_longer_has_health_log(): + fields = {f.name for f in dataclasses.fields(SessionSnapshot)} + assert "health_log" not in fields diff --git a/tests/unit/viewer/test_state.py b/tests/unit/viewer/test_state.py index eb2a69c..061eb84 100644 --- a/tests/unit/viewer/test_state.py +++ b/tests/unit/viewer/test_state.py @@ -11,7 +11,7 @@ import pytest -from syncfield.viewer.state import HealthEntry, StreamStatsBuffer +from syncfield.viewer.state import StreamStatsBuffer class TestStreamStatsBufferSamples: @@ -95,29 +95,6 @@ def test_missing_channel_fills_nan_forward(self): assert math.isnan(gx_ys[1]) -class TestStreamStatsBufferHealth: - def test_empty(self): - assert StreamStatsBuffer().snapshot_health() == [] - - def test_records_in_order(self): - buf = StreamStatsBuffer() - a = HealthEntry(stream_id="s", kind="warning", at_ns=10, detail="a") - b = HealthEntry(stream_id="s", kind="error", at_ns=20, detail="b") - buf.observe_health(a) - buf.observe_health(b) - events = buf.snapshot_health() - assert events == [a, b] - - def test_capped(self): - buf = StreamStatsBuffer(max_health=3) - # Override the internal deque size — real constructor caps at 20, - # but we test the cap principle with whatever the dataclass set up. - # The deque default is maxlen=20; this test verifies behaviour at - # whatever cap the buffer ended up with. - for i in range(30): - buf.observe_health( - HealthEntry(stream_id="s", kind="heartbeat", at_ns=i, detail=None) - ) - events = buf.snapshot_health() - assert len(events) <= 20 # matches default max_health - assert events[-1].at_ns == 29 # newest kept +# Health tracking has been moved from StreamStatsBuffer to HealthSystem/IncidentTracker. +# The per-stream HealthEntry buffer and observe_health/snapshot_health methods were +# removed in Task 20. Health-related viewer tests now live in test_snapshot_incidents.py. From 1f934bb6b9168d31f052bad2fc2a0c511a933f5e Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 14:06:00 +0900 Subject: [PATCH 33/64] refactor(viewer-fe): mirror IncidentSnapshot + Severity; drop HealthEntry --- .../viewer/frontend/src/lib/types.ts | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/src/syncfield/viewer/frontend/src/lib/types.ts b/src/syncfield/viewer/frontend/src/lib/types.ts index 556d64b..5f793e2 100644 --- a/src/syncfield/viewer/frontend/src/lib/types.ts +++ b/src/syncfield/viewer/frontend/src/lib/types.ts @@ -19,9 +19,6 @@ export interface StreamSnapshot { last_sample_ms_ago: number | null; provides_audio_track: boolean; produces_file: boolean; - health_count: number; - /** Count of non-heartbeat events (warnings/errors/drops). */ - problem_count: number; /** Stream capabilities declared by the adapter. May be absent on older servers. */ capabilities?: StreamCapabilities; } @@ -32,13 +29,29 @@ export interface ChirpInfo { stop_ns: number | null; } -export interface HealthEntry { - stream_id: string; +export type Severity = "info" | "warning" | "error" | "critical"; + +export interface IncidentArtifact { kind: string; - ago_s: number; + path: string; detail: string | null; } +export interface IncidentSnapshot { + id: string; + stream_id: string; + fingerprint: string; + title: string; + severity: Severity; + source: string; + opened_at_ns: number; + closed_at_ns: number | null; + event_count: number; + detail: string | null; + ago_s: number; + artifacts: IncidentArtifact[]; +} + // --------------------------------------------------------------------------- // Aggregation types (Insta360 Go3S) // --------------------------------------------------------------------------- @@ -74,7 +87,8 @@ export interface SessionSnapshot { elapsed_s: number; chirp: ChirpInfo; streams: Record; - health_log: HealthEntry[]; + active_incidents: IncidentSnapshot[]; + resolved_incidents: IncidentSnapshot[]; output_dir: string; /** Aggregation state for Go3S streams; present when a Go3S adapter is active. */ aggregation?: AggregationSnapshotWS; From 73e6c2caca12dbdf258772c72c30696ac4978b6a Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 14:07:49 +0900 Subject: [PATCH 34/64] feat(viewer-fe): add IncidentPanel (replaces HealthTable) --- src/syncfield/viewer/frontend/src/App.tsx | 15 +-- .../frontend/src/components/health-table.tsx | 79 --------------- .../src/components/incident-panel.tsx | 97 +++++++++++++++++++ 3 files changed, 105 insertions(+), 86 deletions(-) delete mode 100644 src/syncfield/viewer/frontend/src/components/health-table.tsx create mode 100644 src/syncfield/viewer/frontend/src/components/incident-panel.tsx diff --git a/src/syncfield/viewer/frontend/src/App.tsx b/src/syncfield/viewer/frontend/src/App.tsx index 232389c..0db4b30 100644 --- a/src/syncfield/viewer/frontend/src/App.tsx +++ b/src/syncfield/viewer/frontend/src/App.tsx @@ -7,7 +7,7 @@ import { Header } from "@/components/header"; import { ControlPanel } from "@/components/control-panel"; import { SessionClock } from "@/components/session-clock"; import { StreamCard } from "@/components/stream-card"; -import { HealthTable } from "@/components/health-table"; +import { IncidentPanel } from "@/components/incident-panel"; import { CountdownOverlay } from "@/components/countdown-overlay"; import { DiscoveryModal } from "@/components/discovery-modal"; import { ClusterPanel } from "@/components/cluster-panel"; @@ -169,6 +169,7 @@ function RecordView({ stream={stream} canRemove={canRemove} onRemove={handleRemoveStream} + activeIncidents={snapshot?.active_incidents ?? []} sessionState={state} aggregation={snapshot?.aggregation} onRetryAggregation={(jobId) => @@ -194,12 +195,12 @@ function RecordView({
{cluster.available && } {streamList.length > 0 && ( - <> -
-

Health Events

-
- - +
+ +
)}
)} diff --git a/src/syncfield/viewer/frontend/src/components/health-table.tsx b/src/syncfield/viewer/frontend/src/components/health-table.tsx deleted file mode 100644 index af96162..0000000 --- a/src/syncfield/viewer/frontend/src/components/health-table.tsx +++ /dev/null @@ -1,79 +0,0 @@ -import type { HealthEntry } from "@/lib/types"; -import { cn } from "@/lib/utils"; - -interface HealthTableProps { - entries: HealthEntry[]; -} - -const KIND_ICONS: Record = { - heartbeat: "●", - warning: "⚠", - error: "✗", - drop: "↓", - reconnect: "↻", -}; - -const KIND_COLORS: Record = { - error: "text-destructive", - warning: "text-warning", - drop: "text-destructive", - reconnect: "text-success", - heartbeat: "text-success", -}; - -/** - * Compact health event timeline for the sidebar. - */ -export function HealthTable({ entries }: HealthTableProps) { - if (entries.length === 0) { - return ( -
- No events yet -
- ); - } - - const sorted = [...entries].reverse(); - - return ( -
    - {sorted.map((entry, i) => ( -
  • - {/* Icon */} - - {KIND_ICONS[entry.kind] ?? "·"} - - - {/* Content */} -
    -
    - - {entry.stream_id} - - - {formatAgo(entry.ago_s)} - -
    - {entry.detail && ( -
    - {entry.detail} -
    - )} -
    -
    • - ))} -
- ); -} - -function formatAgo(seconds: number): string { - if (seconds < 1) return "just now"; - if (seconds < 60) return `${Math.round(seconds)}s ago`; - if (seconds < 3600) return `${Math.round(seconds / 60)}m ago`; - return `${Math.round(seconds / 3600)}h ago`; -} diff --git a/src/syncfield/viewer/frontend/src/components/incident-panel.tsx b/src/syncfield/viewer/frontend/src/components/incident-panel.tsx new file mode 100644 index 0000000..96e1a63 --- /dev/null +++ b/src/syncfield/viewer/frontend/src/components/incident-panel.tsx @@ -0,0 +1,97 @@ +import { useState } from "react"; +import type { IncidentSnapshot, Severity } from "../lib/types"; + +const SEVERITY_ICON: Record = { + info: "·", + warning: "⚠", + error: "⛔", + critical: "⛔", +}; + +const SEVERITY_COLOR: Record = { + info: "text-slate-400", + warning: "text-yellow-400", + error: "text-orange-400", + critical: "text-red-500", +}; + +function formatAgo(s: number): string { + if (s < 60) return `${Math.round(s)}s ago`; + if (s < 3600) return `${Math.round(s / 60)}m ago`; + return `${Math.round(s / 3600)}h ago`; +} + +function IncidentCard({ inc, isOpen }: { inc: IncidentSnapshot; isOpen: boolean }) { + const [expanded, setExpanded] = useState(false); + return ( + + ); +} + +export function IncidentPanel({ + active, + resolved, +}: { + active: IncidentSnapshot[]; + resolved: IncidentSnapshot[]; +}) { + return ( +
+
+ Active Issues ({active.length}) +
+ {active.length === 0 ? ( +
None — all clear.
+ ) : ( +
+ {active.map((inc) => ( + + ))} +
+ )} +
+ Resolved this session ({resolved.length}) +
+ {resolved.length === 0 ? ( +
None.
+ ) : ( + resolved.map((inc) => ) + )} +
+ ); +} From 2b8269032050de3a2f0bd19cdb25f5f348408c7f Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 14:07:52 +0900 Subject: [PATCH 35/64] feat(viewer-fe): stream card severity badge from active incidents --- .../frontend/src/components/stream-card.tsx | 60 +++++++++++++------ 1 file changed, 43 insertions(+), 17 deletions(-) diff --git a/src/syncfield/viewer/frontend/src/components/stream-card.tsx b/src/syncfield/viewer/frontend/src/components/stream-card.tsx index 9e267a5..745d106 100644 --- a/src/syncfield/viewer/frontend/src/components/stream-card.tsx +++ b/src/syncfield/viewer/frontend/src/components/stream-card.tsx @@ -1,4 +1,4 @@ -import type { AggregationSnapshotWS, StreamSnapshot } from "@/lib/types"; +import type { AggregationSnapshotWS, IncidentSnapshot, Severity, StreamSnapshot } from "@/lib/types"; import { formatCount, formatHz } from "@/lib/format"; import { cn } from "@/lib/utils"; import { AudioLevelChart } from "./audio-level-chart"; @@ -13,6 +13,8 @@ interface StreamCardProps { stream: StreamSnapshot; canRemove: boolean; onRemove: (streamId: string) => void; + /** Active incidents from the session snapshot — used to derive per-stream severity badge. */ + activeIncidents?: IncidentSnapshot[]; /** Session state string — forwarded to StandaloneRecorderPanel for recording detection. */ sessionState?: string; /** Top-level aggregation snapshot from the WS payload — used by StandaloneRecorderPanel. */ @@ -21,6 +23,30 @@ interface StreamCardProps { onRetryAggregation?: (jobId: string) => void; } +// --------------------------------------------------------------------------- +// Per-stream incident helpers +// --------------------------------------------------------------------------- + +const SEVERITY_ORDER: Severity[] = ["info", "warning", "error", "critical"]; +const BADGE_COLOR: Record = { + info: "bg-slate-500", + warning: "bg-yellow-500", + error: "bg-orange-500", + critical: "bg-red-500", +}; + +function streamIncidentStats(streamId: string, active: IncidentSnapshot[]) { + const mine = active.filter((i) => i.stream_id === streamId); + const count = mine.length; + let highest: Severity | null = null; + for (const i of mine) { + if (highest === null || SEVERITY_ORDER.indexOf(i.severity) > SEVERITY_ORDER.indexOf(highest)) { + highest = i.severity; + } + } + return { count, highest }; +} + /** * Per-stream card with variant body by kind. * @@ -35,10 +61,12 @@ export function StreamCard({ stream, canRemove, onRemove, + activeIncidents = [], sessionState, aggregation, onRetryAggregation, }: StreamCardProps) { + const { count: incidentCount, highest: incidentSeverity } = streamIncidentStats(stream.id, activeIncidents); // Dispatch to StandaloneRecorderPanel for video streams without live preview // (e.g. Insta360 Go3S which downloads files via BLE/Wi-Fi after recording). const isStandalone = @@ -66,6 +94,13 @@ export function StreamCard({ {stream.id} + {incidentCount > 0 && incidentSeverity && ( + + {incidentCount} + + )}
{canRemove && (
); @@ -136,6 +163,13 @@ export function StreamCard({ {stream.id} + {incidentCount > 0 && incidentSeverity && ( + + {incidentCount} + + )}
{canRemove && (
); From 967a0266eb14e3b7a51fd1a15ed9204db6f939bb Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 14:11:20 +0900 Subject: [PATCH 36/64] feat(adapters): declare target_hz on UVC / audio / Meta Quest / BLE IMU / sensors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire the known-rate target_hz into StreamCapabilities for every adapter where the emission rate is declared at construction time: UVCWebcamStream (fps arg, Optional guard), MetaQuestCameraStream (fps arg), HostAudioStream (1.0 / METRICS_INTERVAL_S = 10 Hz, one SampleEvent per audio buffer), and PollingSensorStream (hz arg, via _resolve_capabilities). Adapters with no knowable constructor-declared rate — MetaQuestHandStream, BLEImuGenericStream, OgloTactileStream, PushSensorStream, Insta360Go3sStream, JsonlFileStream — retain target_hz=None. Co-Authored-By: Claude Sonnet 4.6 --- src/syncfield/adapters/_generic.py | 10 ++++++++-- src/syncfield/adapters/host_audio.py | 1 + src/syncfield/adapters/meta_quest_camera/stream.py | 1 + src/syncfield/adapters/polling_sensor.py | 4 +++- src/syncfield/adapters/uvc_webcam.py | 1 + 5 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/syncfield/adapters/_generic.py b/src/syncfield/adapters/_generic.py index 8a075fe..dc8c25c 100644 --- a/src/syncfield/adapters/_generic.py +++ b/src/syncfield/adapters/_generic.py @@ -58,12 +58,17 @@ def reset_recording_stats(self) -> None: self._last_at_ns = None -def _default_sensor_capabilities(*, precise: bool) -> StreamCapabilities: +def _default_sensor_capabilities( + *, + precise: bool, + target_hz: Optional[float] = None, +) -> StreamCapabilities: return StreamCapabilities( provides_audio_track=False, supports_precise_timestamps=precise, is_removable=False, produces_file=False, # orchestrator handles JSONL persistence + target_hz=target_hz, ) @@ -71,7 +76,8 @@ def _resolve_capabilities( user: Optional[StreamCapabilities], *, precise: bool, + target_hz: Optional[float] = None, ) -> StreamCapabilities: if user is not None: return user - return _default_sensor_capabilities(precise=precise) + return _default_sensor_capabilities(precise=precise, target_hz=target_hz) diff --git a/src/syncfield/adapters/host_audio.py b/src/syncfield/adapters/host_audio.py index 306229c..2b99518 100644 --- a/src/syncfield/adapters/host_audio.py +++ b/src/syncfield/adapters/host_audio.py @@ -119,6 +119,7 @@ def __init__( supports_precise_timestamps=True, is_removable=True, produces_file=True, + target_hz=1.0 / METRICS_INTERVAL_S, ), ) self._output_dir = Path(output_dir) diff --git a/src/syncfield/adapters/meta_quest_camera/stream.py b/src/syncfield/adapters/meta_quest_camera/stream.py index 153a886..f88e99b 100644 --- a/src/syncfield/adapters/meta_quest_camera/stream.py +++ b/src/syncfield/adapters/meta_quest_camera/stream.py @@ -116,6 +116,7 @@ def __init__( supports_precise_timestamps=True, is_removable=True, produces_file=True, + target_hz=float(fps), ), ) self._quest_host = quest_host diff --git a/src/syncfield/adapters/polling_sensor.py b/src/syncfield/adapters/polling_sensor.py index b4f120c..f6ce224 100644 --- a/src/syncfield/adapters/polling_sensor.py +++ b/src/syncfield/adapters/polling_sensor.py @@ -41,7 +41,9 @@ def __init__( super().__init__( id=id, kind="sensor", - capabilities=_resolve_capabilities(capabilities, precise=True), + capabilities=_resolve_capabilities( + capabilities, precise=True, target_hz=float(hz) + ), ) self._validate_arity(read, expects_handle=open is not None) self._read = read diff --git a/src/syncfield/adapters/uvc_webcam.py b/src/syncfield/adapters/uvc_webcam.py index 3bf6f21..114abfd 100644 --- a/src/syncfield/adapters/uvc_webcam.py +++ b/src/syncfield/adapters/uvc_webcam.py @@ -91,6 +91,7 @@ def __init__( supports_precise_timestamps=True, is_removable=True, produces_file=True, + target_hz=(float(fps) if fps is not None else None), ), ) self._device_index = device_index From 8ad2e25ae786ab6718fd73d79d12a0b3fc514625 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 14:28:20 +0900 Subject: [PATCH 37/64] fix(health): multi-listener callbacks, WriterStats emission, and target_hz threading Fix 1: Convert IncidentTracker/HealthSystem incident callbacks from single-slot Optional[Callable] to list-based multi-listener API (add_on_opened/updated/closed on tracker, on_incident_opened/updated/closed method-call registrars on system). SessionPoller no longer clobbers SessionOrchestrator's persist listener. Fix 2: Wire _emit_writer_stats in orchestrator.add() so BackpressureDetector receives WriterStats on every stream at ~10 Hz (throttled per stream_id). Fix 3: HealthSystem maintains _target_hz_by_stream dict; register_stream(id, hz) is called from orchestrator.add(). FpsDropDetector and JitterDetector now receive a target_getter closure that reads from this dict instead of returning None always. Co-Authored-By: Claude Sonnet 4.6 --- src/syncfield/health/system.py | 37 +++++++----- src/syncfield/health/tracker.py | 33 +++++++---- src/syncfield/orchestrator.py | 32 +++++++++-- src/syncfield/viewer/poller.py | 6 +- .../test_orchestrator_health_integration.py | 56 +++++++++++++++++++ tests/unit/health/test_health_system.py | 16 +++++- tests/unit/health/test_incident_tracker.py | 6 +- 7 files changed, 149 insertions(+), 37 deletions(-) diff --git a/src/syncfield/health/system.py b/src/syncfield/health/system.py index cf5e23f..a0fb9c4 100644 --- a/src/syncfield/health/system.py +++ b/src/syncfield/health/system.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Callable, Iterable, Iterator, Optional +from typing import Callable, Dict, Iterable, Iterator, Optional from syncfield.health.detector import Detector from syncfield.health.detectors.adapter_passthrough import AdapterEventPassthrough @@ -31,16 +31,23 @@ def __init__( self._tracker = IncidentTracker(passthrough_close_ns=passthrough_close_ns) self._worker: Optional[HealthWorker] = None self._tick_hz = tick_hz + self._target_hz_by_stream: Dict[str, Optional[float]] = {} - self.on_incident_opened: Optional[Callable[[Incident], None]] = None - self.on_incident_updated: Optional[Callable[[Incident], None]] = None - self.on_incident_closed: Optional[Callable[[Incident], None]] = None + self._install_default_detectors() - self._tracker.on_opened = lambda inc: self._fire("on_incident_opened", inc) - self._tracker.on_updated = lambda inc: self._fire("on_incident_updated", inc) - self._tracker.on_closed = lambda inc: self._fire("on_incident_closed", inc) + # --- incident callbacks ---------------------------------------------- - self._install_default_detectors() + def on_incident_opened(self, cb: Callable[[Incident], None]) -> None: + """Register a callback fired when a new incident opens.""" + self._tracker.add_on_opened(cb) + + def on_incident_updated(self, cb: Callable[[Incident], None]) -> None: + """Register a callback fired when an open incident receives a new event.""" + self._tracker.add_on_updated(cb) + + def on_incident_closed(self, cb: Callable[[Incident], None]) -> None: + """Register a callback fired when an incident is resolved.""" + self._tracker.add_on_closed(cb) # --- registry -------------------------------------------------------- @@ -111,15 +118,15 @@ def resolved_incidents(self) -> Iterable[Incident]: # --- helpers --------------------------------------------------------- + def register_stream(self, stream_id: str, target_hz: Optional[float]) -> None: + """Declare the expected target_hz for a stream. Detectors consult this.""" + self._target_hz_by_stream[stream_id] = target_hz + def _install_default_detectors(self) -> None: + target_getter = lambda sid: self._target_hz_by_stream.get(sid) self.register(AdapterEventPassthrough()) self.register(StreamStallDetector()) - self.register(FpsDropDetector()) - self.register(JitterDetector()) + self.register(FpsDropDetector(target_getter=target_getter)) + self.register(JitterDetector(target_getter=target_getter)) self.register(StartupFailureDetector()) self.register(BackpressureDetector()) - - def _fire(self, attr: str, inc: Incident) -> None: - cb = getattr(self, attr, None) - if cb is not None: - cb(inc) diff --git a/src/syncfield/health/tracker.py b/src/syncfield/health/tracker.py index b34051c..fe59594 100644 --- a/src/syncfield/health/tracker.py +++ b/src/syncfield/health/tracker.py @@ -22,9 +22,18 @@ def __init__(self, passthrough_close_ns: int = 30 * 1_000_000_000) -> None: self._detectors_by_name: Dict[str, Detector] = {} self._passthrough_close_ns = passthrough_close_ns - self.on_opened: Optional[Callback] = None - self.on_updated: Optional[Callback] = None - self.on_closed: Optional[Callback] = None + self._on_opened: List[Callback] = [] + self._on_updated: List[Callback] = [] + self._on_closed: List[Callback] = [] + + def add_on_opened(self, cb: Callback) -> None: + self._on_opened.append(cb) + + def add_on_updated(self, cb: Callback) -> None: + self._on_updated.append(cb) + + def add_on_closed(self, cb: Callback) -> None: + self._on_closed.append(cb) # --- detector wiring ------------------------------------------------- @@ -43,10 +52,10 @@ def ingest(self, event: HealthEvent) -> None: if open_inc is None: inc = Incident.opened_from(event, title=_title_from(event)) self._by_fingerprint[event.fingerprint] = inc - self._fire(self.on_opened, inc) + self._fire(self._on_opened, inc) return open_inc.record_event(event) - self._fire(self.on_updated, open_inc) + self._fire(self._on_updated, open_inc) # --- tick — evaluate close conditions -------------------------------- @@ -65,7 +74,7 @@ def tick(self, now_ns: int) -> None: inc = self._by_fingerprint.pop(fp) inc.close(at_ns=now_ns) self._resolved.append(inc) - self._fire(self.on_closed, inc) + self._fire(self._on_closed, inc) def close_all(self, *, at_ns: int) -> None: """Used at session stop to resolve any still-open incidents.""" @@ -73,7 +82,7 @@ def close_all(self, *, at_ns: int) -> None: inc = self._by_fingerprint.pop(fp) inc.close(at_ns=at_ns) self._resolved.append(inc) - self._fire(self.on_closed, inc) + self._fire(self._on_closed, inc) # --- read-only views ------------------------------------------------- @@ -93,9 +102,13 @@ def _detector_for(self, inc: Incident) -> Optional[Detector]: return self._detectors_by_name.get(parts[1]) @staticmethod - def _fire(cb: Optional[Callback], inc: Incident) -> None: - if cb is not None: - cb(inc) + def _fire(callbacks: List[Callback], inc: Incident) -> None: + for cb in callbacks: + try: + cb(inc) + except Exception: + # A listener failure must not break the tracker. + pass def _title_from(event: HealthEvent) -> str: diff --git a/src/syncfield/orchestrator.py b/src/syncfield/orchestrator.py index cd3c88c..4f3a941 100644 --- a/src/syncfield/orchestrator.py +++ b/src/syncfield/orchestrator.py @@ -396,9 +396,12 @@ def __init__( # started/stopped alongside each recording cycle. from syncfield.health import HealthSystem self.health = HealthSystem() - self.health.on_incident_opened = self._persist_incident - self.health.on_incident_updated = self._persist_incident - self.health.on_incident_closed = self._persist_incident + self.health.on_incident_opened(self._persist_incident) + self.health.on_incident_updated(self._persist_incident) + self.health.on_incident_closed(self._persist_incident) + + # Throttle tracker for writer stats emission — keyed by stream_id. + self._last_writer_stats_emit_at: Dict[str, int] = {} # Multi-host: start control plane + advertiser (or follower browser) # at construction time so cluster discovery is independent of device @@ -1510,7 +1513,11 @@ def add(self, stream: Stream) -> None: self._streams[stream.id] = stream stream.on_health(self._on_stream_health) - stream.on_sample(lambda s, _sid=stream.id: self.health.observe_sample(_sid, s)) + stream.on_sample(lambda s, _sid=stream.id: ( + self.health.observe_sample(_sid, s), + self._emit_writer_stats(_sid), + )) + self.health.register_stream(stream.id, stream.capabilities.target_hz) # After the first non-audio stream is registered, check whether # to pre-register a host audio stream so it appears in the @@ -2395,6 +2402,23 @@ def _on_stream_health(self, event: HealthEvent) -> None: self._log_writer.log_health(event) self.health.observe_health(event.stream_id, event) + def _emit_writer_stats(self, stream_id: str) -> None: + """Emit WriterStats to the health system at most ~10 Hz per stream.""" + import time as _t + from syncfield.health.types import WriterStats + now = _t.monotonic_ns() + last = self._last_writer_stats_emit_at.get(stream_id, 0) + if now - last < 100_000_000: # 100 ms throttle + return + self._last_writer_stats_emit_at[stream_id] = now + self.health.observe_writer_stats( + stream_id, + WriterStats( + stream_id=stream_id, at_ns=now, + queue_depth=0, queue_capacity=1, dropped=0, + ), + ) + def _persist_incident(self, incident) -> None: """Forward incidents to the session log writer + attach crash dumps. diff --git a/src/syncfield/viewer/poller.py b/src/syncfield/viewer/poller.py index ef84fe6..954f9e6 100644 --- a/src/syncfield/viewer/poller.py +++ b/src/syncfield/viewer/poller.py @@ -72,9 +72,9 @@ def __init__( self._open_by_id: dict[str, Incident] = {} self._resolved: deque[Incident] = deque(maxlen=20) - session.health.on_incident_opened = self._ingest_incident - session.health.on_incident_updated = self._ingest_incident - session.health.on_incident_closed = self._ingest_incident + session.health.on_incident_opened(self._ingest_incident) + session.health.on_incident_updated(self._ingest_incident) + session.health.on_incident_closed(self._ingest_incident) # ------------------------------------------------------------------ # Public API diff --git a/tests/integration/health/test_orchestrator_health_integration.py b/tests/integration/health/test_orchestrator_health_integration.py index a1fe48c..2debfa0 100644 --- a/tests/integration/health/test_orchestrator_health_integration.py +++ b/tests/integration/health/test_orchestrator_health_integration.py @@ -110,3 +110,59 @@ def test_incidents_jsonl_written(tmp_path: Path): fingerprints = [json.loads(l)["fingerprint"] for l in lines] assert any(fp == "cam:stream-stall" for fp in fingerprints), \ f"stall fingerprint missing — found: {set(fingerprints)}" + + +@pytest.mark.slow +def test_incidents_jsonl_written_with_poller_wired(tmp_path: Path): + """Regression: SessionPoller must not clobber SessionOrchestrator's persist listener.""" + from syncfield.viewer.poller import SessionPoller + + sess = SessionOrchestrator(host_id="test", output_dir=tmp_path) + stream = FakeStream("cam", target_hz=30.0) + sess.add(stream) + + # Spin up a poller as the viewer would. + poller = SessionPoller(sess) + + sess.connect() + sess.start(countdown_s=0) + stream.pause_samples() + time.sleep(2.5) + sess.stop() + sess.disconnect() + + out = list(tmp_path.rglob("incidents.jsonl")) + assert out, "no incidents.jsonl written — poller likely clobbered persist listener" + lines = out[0].read_text().strip().splitlines() + assert any('"fingerprint": "cam:stream-stall"' in l or '"fingerprint":"cam:stream-stall"' in l for l in lines) + + +@pytest.mark.slow +def test_orchestrator_feeds_writer_stats_to_backpressure_detector(tmp_path: Path): + from syncfield.health.detector import DetectorBase + from syncfield.health.severity import Severity + + class WriterStatsSpy(DetectorBase): + name = "writer-stats-spy" + default_severity = Severity.INFO + + def __init__(self): + self.calls = 0 + + def observe_writer_stats(self, stream_id, stats): + self.calls += 1 + + sess = SessionOrchestrator(host_id="test", output_dir=tmp_path) + spy = WriterStatsSpy() + sess.health.register(spy) + stream = FakeStream("cam", target_hz=30.0) + sess.add(stream) + + sess.connect() + sess.start(countdown_s=0) + time.sleep(0.6) + sess.stop() + sess.disconnect() + + # At 10 Hz throttle + 600ms → ~5-6 calls expected. + assert spy.calls >= 2, f"writer stats not emitted (got {spy.calls})" diff --git a/tests/unit/health/test_health_system.py b/tests/unit/health/test_health_system.py index df302bb..e8606ff 100644 --- a/tests/unit/health/test_health_system.py +++ b/tests/unit/health/test_health_system.py @@ -50,8 +50,8 @@ def test_health_system_installs_default_detectors(): def test_health_system_callbacks_fire_on_open_and_close(): hs = HealthSystem(passthrough_close_ns=1) # close instantly for the test opened, closed = [], [] - hs.on_incident_opened = opened.append - hs.on_incident_closed = closed.append + hs.on_incident_opened(opened.append) + hs.on_incident_closed(closed.append) hs.start() try: @@ -96,3 +96,15 @@ def test_health_system_register_after_start_warns(): assert any("after HealthSystem.start()" in str(w.message) for w in caught) finally: hs.stop() + + +def test_register_stream_propagates_target_hz_to_detectors(): + hs = HealthSystem() + hs.register_stream("cam", 30.0) + + fps = next(d for d in hs.iter_detectors() if d.name == "fps-drop") + jitter = next(d for d in hs.iter_detectors() if d.name == "jitter") + + assert fps._target_getter("cam") == 30.0 + assert jitter._target_getter("cam") == 30.0 + assert fps._target_getter("unknown") is None diff --git a/tests/unit/health/test_incident_tracker.py b/tests/unit/health/test_incident_tracker.py index f1d8ea0..bbe1be7 100644 --- a/tests/unit/health/test_incident_tracker.py +++ b/tests/unit/health/test_incident_tracker.py @@ -37,7 +37,7 @@ def test_tracker_opens_incident_on_first_matching_event(): tr = IncidentTracker() tr.bind_detector(AlwaysCloseAfter(close_after_ns=1000)) opened: List[Incident] = [] - tr.on_opened = opened.append + tr.add_on_opened(opened.append) tr.ingest(_ev(100)) @@ -65,7 +65,7 @@ def test_tracker_closes_incident_when_detector_close_condition_fires(): tr = IncidentTracker() tr.bind_detector(AlwaysCloseAfter(close_after_ns=500)) closed: List[Incident] = [] - tr.on_closed = closed.append + tr.add_on_closed(closed.append) tr.ingest(_ev(100)) tr.tick(now_ns=200) # 100 ns since last event, not yet @@ -109,7 +109,7 @@ def test_tracker_flush_callbacks_fire_on_update_too(): tr = IncidentTracker() tr.bind_detector(AlwaysCloseAfter(close_after_ns=1_000_000_000)) updates: List[Incident] = [] - tr.on_updated = updates.append + tr.add_on_updated(updates.append) tr.ingest(_ev(100)) # opens tr.ingest(_ev(200)) # updates From 58e7958fb9a55a087b7f031b9f0ed2b45d92fc30 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 14:28:27 +0900 Subject: [PATCH 38/64] =?UTF-8?q?refactor(health):=20simplify=20FpsDropDet?= =?UTF-8?q?ector=20tick=20(remove=20O(N=C2=B2)=20dip-start=20search)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace _find_dip_start binary-search with the simple pattern used by JitterDetector: record dip_began_at = now_ns on first low-FPS tick, then check elapsed duration on subsequent ticks. The existing sustain_ns gate already debounces spurious fires, making the retroactive search redundant. Also remove the dead _last_observed_state field and the now-unused _observed_fps_from helper. Update unit tests to tick repeatedly so the sustain_ns threshold can be crossed progressively. Co-Authored-By: Claude Sonnet 4.6 --- src/syncfield/health/detectors/fps_drop.py | 51 ++------------------ tests/unit/health/detectors/test_fps_drop.py | 19 +++++--- 2 files changed, 15 insertions(+), 55 deletions(-) diff --git a/src/syncfield/health/detectors/fps_drop.py b/src/syncfield/health/detectors/fps_drop.py index bc8ddb8..968f629 100644 --- a/src/syncfield/health/detectors/fps_drop.py +++ b/src/syncfield/health/detectors/fps_drop.py @@ -47,8 +47,6 @@ def __init__( self._fire_active: Dict[str, bool] = {} # Same thing for recovery tracking. self._recovery_began_at: Dict[str, Optional[int]] = {} - # Track the most recent observation, for incremental dip tracking - self._last_observed_state: Dict[str, bool] = {} # True if in drop, False if not # --- observers ------------------------------------------------------- @@ -74,12 +72,9 @@ def tick(self, now_ns: int) -> Iterator[HealthEvent]: if observed < target * self._drop_ratio: began = self._dip_began_at.get(stream_id) if began is None: - # Find when the drop started by looking at the buffer - dip_start = self._find_dip_start(buf, target, now_ns) - self._dip_began_at[stream_id] = dip_start - began = dip_start - elapsed = now_ns - began - if elapsed >= self._sustain_ns and not self._fire_active.get(stream_id, False): + self._dip_began_at[stream_id] = now_ns + began = now_ns + if (now_ns - began) >= self._sustain_ns and not self._fire_active.get(stream_id, False): self._fire_active[stream_id] = True out.append(HealthEvent( stream_id=stream_id, @@ -115,46 +110,6 @@ def close_condition(self, incident: Incident, now_ns: int) -> bool: # --- helpers --------------------------------------------------------- - def _find_dip_start(self, buf: Deque[int], target: float, now_ns: int) -> int: - """Find the earliest time when FPS dropped below threshold.""" - if not buf: - return now_ns - # Binary search to find when we first dropped below target - threshold = target * self._drop_ratio - left, right = 0, len(buf) - 1 - result = buf[-1] if buf else now_ns - - while left <= right: - mid = (left + right) // 2 - # Check FPS from buf[mid] onwards - mid_time = buf[mid] - fps_from_mid = self._observed_fps_from(buf, mid_time, now_ns) - - if fps_from_mid < threshold: - result = mid_time - right = mid - 1 - else: - left = mid + 1 - - return result - - def _observed_fps_from(self, buf: Deque[int], start_time: int, now_ns: int) -> Optional[float]: - """Calculate FPS from a specific start time to now.""" - if not buf: - return None - count = sum(1 for t in buf if t >= start_time and t <= now_ns) - if count == 0: - return 0.0 - elapsed_ns = now_ns - start_time - if elapsed_ns < _WINDOW_NS: - # Less than 1 second, use actual window - return count / (elapsed_ns / 1e9) if elapsed_ns > 0 else 0.0 - else: - # More than 1 second, use 1-second rolling window from now - cutoff = now_ns - _WINDOW_NS - count = sum(1 for t in buf if t >= cutoff) - return count / (_WINDOW_NS / 1e9) - def _effective_target(self, stream_id: str, now_ns: int) -> Optional[float]: declared = self._target_getter(stream_id) if declared is not None: diff --git a/tests/unit/health/detectors/test_fps_drop.py b/tests/unit/health/detectors/test_fps_drop.py index 81baf8e..28adcb2 100644 --- a/tests/unit/health/detectors/test_fps_drop.py +++ b/tests/unit/health/detectors/test_fps_drop.py @@ -24,15 +24,16 @@ def test_fires_when_observed_below_70_percent_for_3s(): # 10 fps for 3.5 seconds — fps is 10, target 30, ratio 0.33. interval = int(1e9 / 10) t = 0 + events = [] while t <= int(3.5e9): d.observe_sample("cam", _s("cam", t)) + events.extend(list(d.tick(now_ns=t))) t += interval - emitted = list(d.tick(now_ns=int(3.6e9))) - assert len(emitted) == 1 - assert emitted[0].fingerprint == "cam:fps-drop" - assert emitted[0].data["target_hz"] == 30.0 - assert emitted[0].data["observed_hz"] < 15.0 + # Eventually, after 3s of sustained low fps, at least one event fires. + assert len(events) >= 1 + assert events[0].fingerprint == "cam:fps-drop" + assert events[0].data["target_hz"] == 30.0 def test_does_not_fire_without_target_before_warmup(): @@ -58,13 +59,17 @@ def test_learns_baseline_then_fires_on_subsequent_drop(): ) # 3 s @ 30 fps → baseline ≈ 30. t = 0 + events = [] while t <= int(3e9): d.observe_sample("cam", _s("cam", t)) + events.extend(list(d.tick(now_ns=t))) t += int(1e9 / 30) # 1.5 s of 10 fps → drop. end = t + int(1.5e9) while t <= end: d.observe_sample("cam", _s("cam", t)) + events.extend(list(d.tick(now_ns=t))) t += int(1e8) - emitted = list(d.tick(now_ns=t)) - assert len(emitted) == 1 + # At least one fps-drop event should have fired during the low-fps period. + drop_events = [e for e in events if e.fingerprint == "cam:fps-drop"] + assert len(drop_events) >= 1 From 20e322944519d308ab30d81daeb0f52012bf8846 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 14:42:08 +0900 Subject: [PATCH 39/64] ci: build viewer frontend before pytest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Tests workflow was installing the package with `pip install -e .`, but pyproject.toml force-includes `src/syncfield/viewer/static` (the built frontend bundle), which is gitignored. CI therefore failed at the install step with "Forced include not found" on every run — pre-dating this PR. Mirror the frontend-build step from publish.yml so the wheel build finds the bundle. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/test.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4a87aa7..6b4e609 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,6 +16,22 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Set up Node (for viewer frontend) + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Build viewer frontend + working-directory: src/syncfield/viewer/frontend + run: | + # Same lockfile-regeneration dance as publish.yml: optional + # platform-specific deps (e.g. rollup's @rollup/rollup-linux-*) + # aren't re-resolved when a macOS-generated lockfile runs on + # Ubuntu. Regenerating from package.json on CI works. + rm -f package-lock.json + npm install --no-audit --no-fund + npm run build + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: From 786bc498b30c1a3bcf0e91188ebc08911e04cc4b Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 14:44:03 +0900 Subject: [PATCH 40/64] ci: install [all] extras so adapter tests can import their deps Tests that import MetaQuestCameraStream, OakCameraStream, Insta360Go3S, and the BLE IMU adapter fail at collection time without httpx / depthai / bleak / aiohttp / zeroconf. `[dev]` alone ships only the base install. Add `[all]` to pick up every optional adapter dependency. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/test.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6b4e609..43b11b0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,7 +40,10 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -e ".[dev]" || pip install -e . && pip install pytest + # [all] brings in every optional adapter dep (bleak, aiohttp, + # depthai, zeroconf, httpx) so tests that import adapters at + # collection time don't ImportError. [dev] adds pytest + tooling. + pip install -e ".[all,dev]" - name: Run tests run: pytest -v From b3be7992fc8877131d68b225c4170734e49cea28 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 14:49:34 +0900 Subject: [PATCH 41/64] ci: install dev deps explicitly (dev is a dependency-group, not an extra) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The prior fallback (`.[dev]` → fail → `pip install pytest`) was removed, so CI's pytest command turned up exit 127 (command not found). `dev` lives in PEP 735 [dependency-groups] which `pip install .[extra]` doesn't resolve. List the pytest stack explicitly — kept in sync with pyproject.toml [dependency-groups].dev. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/test.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 43b11b0..3d5f682 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,8 +42,19 @@ jobs: python -m pip install --upgrade pip # [all] brings in every optional adapter dep (bleak, aiohttp, # depthai, zeroconf, httpx) so tests that import adapters at - # collection time don't ImportError. [dev] adds pytest + tooling. - pip install -e ".[all,dev]" + # collection time don't ImportError. + pip install -e ".[all]" + # `dev` lives in PEP 735 [dependency-groups], not + # [project.optional-dependencies], so .[dev] wouldn't resolve. + # Install pytest + its plugins explicitly — keep this list in + # sync with [dependency-groups].dev in pyproject.toml. + pip install \ + "pytest>=8.4.2" \ + "pytest-aiohttp>=1.1.0" \ + "pytest-asyncio>=1.2.0" \ + "pytest-mock>=3.12.0" \ + "pytest-timeout>=2.4.0" \ + "pytest-xdist>=3.6.0" - name: Run tests run: pytest -v From d334a1d6825c537eca5fccc1a78be21124e049e8 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 14:53:24 +0900 Subject: [PATCH 42/64] test: add future annotations for Python 3.9 compat Test uses `float | None` at runtime (FakeStream constructor default), which requires Python 3.10+ without the __future__ import. The project declares `requires-python = ">=3.9"` so we need the shim. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../integration/health/test_orchestrator_health_integration.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/health/test_orchestrator_health_integration.py b/tests/integration/health/test_orchestrator_health_integration.py index 2debfa0..0792169 100644 --- a/tests/integration/health/test_orchestrator_health_integration.py +++ b/tests/integration/health/test_orchestrator_health_integration.py @@ -1,4 +1,6 @@ """Integration: real SessionOrchestrator + FakeStream → incidents flow end-to-end.""" +from __future__ import annotations + import json import threading import time From 48ed479778bcc2d3b215b800cc5044a8b984082e Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 14:55:54 +0900 Subject: [PATCH 43/64] test: add future annotations to remaining tests for Python 3.9 compat Three pre-existing test files use PEP 604 union syntax (`X | None`) at runtime in type annotations. `from __future__ import annotations` makes all annotations strings and keeps them 3.9-compatible. Drive-by since we're already adjusting CI; pyproject declares `requires-python = ">=3.9"`. Co-Authored-By: Claude Opus 4.7 (1M context) --- tests/unit/adapters/insta360_go3s/test_aggregation_queue.py | 2 ++ tests/unit/adapters/insta360_go3s/test_ble_camera.py | 2 ++ tests/unit/test_orchestrator_config_distribution.py | 1 + 3 files changed, 5 insertions(+) diff --git a/tests/unit/adapters/insta360_go3s/test_aggregation_queue.py b/tests/unit/adapters/insta360_go3s/test_aggregation_queue.py index 1039808..c27cbd5 100644 --- a/tests/unit/adapters/insta360_go3s/test_aggregation_queue.py +++ b/tests/unit/adapters/insta360_go3s/test_aggregation_queue.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import asyncio from pathlib import Path from typing import Any diff --git a/tests/unit/adapters/insta360_go3s/test_ble_camera.py b/tests/unit/adapters/insta360_go3s/test_ble_camera.py index db3572a..2daa839 100644 --- a/tests/unit/adapters/insta360_go3s/test_ble_camera.py +++ b/tests/unit/adapters/insta360_go3s/test_ble_camera.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import asyncio import time from typing import Callable diff --git a/tests/unit/test_orchestrator_config_distribution.py b/tests/unit/test_orchestrator_config_distribution.py index a5f50e2..066fd47 100644 --- a/tests/unit/test_orchestrator_config_distribution.py +++ b/tests/unit/test_orchestrator_config_distribution.py @@ -1,4 +1,5 @@ """Leader-side config distribution: build, discover, distribute.""" +from __future__ import annotations from unittest.mock import MagicMock From 5cf566e133b56733840e302b818ff4faa5c2a272 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 15:32:29 +0900 Subject: [PATCH 44/64] docs: partial-connect UX design spec Extends the health-telemetry platform (2026-04-22 spec) with: partial connect (per-stream state machine, survivors proceed on single-stream failure), structured startup-failure HealthEvent emission that activates the previously-dormant StartupFailureDetector, a new NoDataDetector for "connected but never emitted a sample" case (the observed OAK black- screen symptom), state-aware StreamCard overlays (connecting / waiting / failed), and a degraded-state header chip ("Ready (3/5)"). Co-Authored-By: Claude Opus 4.7 (1M context) --- .../2026-04-22-partial-connect-ux-design.md | 323 ++++++++++++++++++ 1 file changed, 323 insertions(+) create mode 100644 docs/superpowers/specs/2026-04-22-partial-connect-ux-design.md diff --git a/docs/superpowers/specs/2026-04-22-partial-connect-ux-design.md b/docs/superpowers/specs/2026-04-22-partial-connect-ux-design.md new file mode 100644 index 0000000..dd8acbc --- /dev/null +++ b/docs/superpowers/specs/2026-04-22-partial-connect-ux-design.md @@ -0,0 +1,323 @@ +# Partial Connect UX — Design Spec + +- **Date**: 2026-04-22 +- **Status**: Approved for implementation planning +- **Owner**: syncfield-python +- **Related spec**: `docs/superpowers/specs/2026-04-22-health-telemetry-design.md` (this extends it) + +## Summary + +Remove the "all-or-nothing" failure mode of `SessionOrchestrator.connect()`. When one stream's `connect()` raises (common case: an OAK camera with a specific `device_id` isn't plugged in), continue connecting the rest, mark the failed stream, and surface the failure in the viewer as a first-class visual state. Also detect the complementary case where a stream connects successfully but never emits a single sample (the observed OAK "black square" symptom), so the user is not left guessing whether the camera is alive. Together these changes close the silent-failure hole that remains even with the Phase-1 health-telemetry framework in place. + +## Goals + +1. **Partial connect resilience**: if any subset of streams fails to connect, the rest still transition to `CONNECTED` and recording can proceed with the survivors. The session only aborts when *every* stream fails. +2. **Structured startup failure emission**: each `stream.connect()` exception produces a `HealthEvent` shaped for `StartupFailureDetector` (`data["phase"]="connect"`, `data["outcome"]="error"`). Successes emit the matching success signal so the detector closes its incident if a stream recovers on a later connect. +3. **Per-stream connection state** visible to the viewer (`idle | connecting | connected | failed | disconnected`) plus a `connection_error` message for the failed case. +4. **Viewer state-aware stream card**: distinct visual for connecting / waiting-for-first-frame / failed, replacing the current grey "broken image" placeholder. Header chip reflects degraded state as `Ready (3/5)`. +5. **No-data detection**: a new detector fires when a stream has been in `connected` state for ≥ 5 s without emitting any sample. Closes the moment the first sample arrives. + +## Non-goals + +- **Automatic retry / reconnect policy**. A failed stream stays failed until the user explicitly triggers Disconnect → Connect again. Auto-retry is follow-up work. +- **Retry button on the failed card**. Same reason. The existing Disconnect/Connect flow already gets the user there in two clicks; adding a per-card retry button requires partial-reconnect semantics we are not designing yet. +- **Error category taxonomy / normalized error codes**. We show the exception message verbatim in the `connection_error` field for v1. Mapping "device not visible" vs "USB permission denied" vs "XLink handshake timeout" into structured categories is future work. +- **Crash-dump viewer integration**. Crash-dump paths remain surfaced as `IncidentArtifact` entries (delivered in Phase-1 OAK work). A "click to open" handler in the viewer is not in scope here. +- **Recovery UI for mid-recording failures**. This spec targets the connect phase. Detectors already cover mid-recording stalls / device crashes via Phase-1 work; they do not change here. + +## Current state (what exists before this change) + +- `SessionOrchestrator.connect()` iterates streams sequentially and calls `_rollback_disconnect_streams(connected)` on the first exception, re-raising it to the caller (`src/syncfield/orchestrator.py:1623-1643`). One failure collapses the whole session back to `IDLE`. +- On successful connect, the orchestrator emits a `HealthEvent(kind=HEARTBEAT, detail="connected")` but with no `phase` / `outcome` fields — so `StartupFailureDetector` never closes anything. +- On failure, the orchestrator emits **nothing** before re-raising — `StartupFailureDetector` never sees the error, so it is permanently dormant despite being registered. +- `StreamSnapshot` has no per-stream connection state. Frontend conflates "connecting", "connected but no data", and "failed" into one grey dot. +- `VideoPreview` renders the browser broken-image icon when `/stream/video/{id}` returns empty — no text, no visual indication of failure. +- OAK's `connect()` takes ~2.4 s (three enumeration retries × 0.8 s) before raising when its declared `device_id` is not present (`src/syncfield/adapters/oak_camera.py:316-366`). + +## Architecture Overview + +``` + ┌──────────────────────────────────────────┐ + │ SessionOrchestrator.connect() │ + │ │ + │ for stream in self._streams.values(): │ + │ _set_stream_state(sid, "connecting") │ + │ try: │ + │ stream.prepare() │ + │ stream.connect() │ + │ except Exception as exc: │ + │ _set_stream_state(sid, "failed") │ + │ _stream_errors[sid] = str(exc) │ + │ emit phase=connect outcome=error │ + │ continue │ + │ _set_stream_state(sid, "connected") │ + │ emit phase=connect outcome=success │ + │ │ + │ if no "connected" streams: │ + │ raise RuntimeError + IDLE │ + │ else: transition → CONNECTED │ + └────────────────────┬─────────────────────┘ + │ + _set_stream_state ──┼──► health.observe_connection_state(sid, new_state) + │ + ┌────────────────────▼─────────────────────┐ + │ HealthSystem / HealthWorker │ + │ │ + │ ingress: _ConnectionStateMsg (NEW) │ + │ → detector.observe_connection_state │ + │ │ + │ existing path: │ + │ on_sample → detector.observe_sample │ + │ on_health → tracker.ingest + │ + │ detector.observe_health │ + │ │ + │ NoDataDetector tick: │ + │ if connected_at[sid] + 5s ≤ now │ + │ and sid not in _has_sample: │ + │ emit {sid}:no-data incident │ + └────────────────────┬─────────────────────┘ + │ + ┌────────────────────▼─────────────────────┐ + │ SessionSnapshot (viewer wire) │ + │ │ + │ streams[sid].connection_state (NEW) │ + │ streams[sid].connection_error (NEW) │ + │ active_incidents / resolved_incidents │ + │ (unchanged — already carry startup + │ + │ no-data incidents) │ + └────────────────────┬─────────────────────┘ + │ + ┌────────────────────▼─────────────────────┐ + │ Viewer frontend │ + │ │ + │ StreamCard selects body by state: │ + │ "connecting" → ConnectingOverlay │ + │ "failed" → FailedOverlay + error │ + │ "connected" + frame_count === 0 │ + │ → WaitingForDataOverlay │ + │ else → │ + │ │ + │ Header state chip: │ + │ "Ready (3/5)" if connected < total │ + │ "Ready" if all connected │ + │ IncidentPanel + severity badge: unchanged│ + └──────────────────────────────────────────┘ +``` + +## Data model + +### `ConnectionState` (new) + +```python +ConnectionState = Literal[ + "idle", # stream added but connect() not yet called + "connecting", # connect() is running + "connected", # connect() returned successfully + "failed", # connect() raised; error recorded in _stream_errors + "disconnected", # disconnect() ran, or rolled back after global failure +] +``` + +Tracked on `SessionOrchestrator` as `self._stream_states: dict[str, ConnectionState]` and `self._stream_errors: dict[str, str]` (populated only for `failed`). Cleared on `disconnect()` back to `disconnected`. + +Transitions are a narrow DAG: +``` +idle → connecting → connected → disconnected + → failed → disconnected +``` +`failed` never transitions to `connected` within the same connect cycle — a subsequent `disconnect()` followed by `connect()` is a new cycle. + +### `StreamSnapshot` — additions + +```python +connection_state: str # one of ConnectionState literals +connection_error: str | None # set iff connection_state == "failed" +``` + +### `StartupFailureDetector` event contract (no code change; existing design) + +Orchestrator emits on connect failure — using the detector's own fingerprint so both the orchestrator's synchronous event and the detector's tick-emitted event land on the same incident: +```python +HealthEvent( + stream_id=stream.id, + kind=HealthEventKind.ERROR, + at_ns=time.monotonic_ns(), + detail=str(exc), + severity=Severity.ERROR, + source="orchestrator", + fingerprint=f"{stream.id}:startup-failure", + data={"phase": "connect", "outcome": "error", "error": str(exc)}, +) +``` + +On success: +```python +HealthEvent( + stream_id=stream.id, + kind=HealthEventKind.HEARTBEAT, + at_ns=time.monotonic_ns(), + detail="connected", + severity=Severity.INFO, + source="orchestrator", + fingerprint=f"{stream.id}:startup-success", + data={"phase": "connect", "outcome": "success"}, +) +``` + +Rationale for fingerprints: +- `{stream.id}:startup-failure` routes to `StartupFailureDetector` via `IncidentTracker._detector_for` (fingerprint middle token equals the detector's `name`). The detector owns `close_condition` which consults its own `_recovered` set, so the incident closes only after a matching success signal arrives. +- The success event deliberately uses a **different** fingerprint (`:startup-success`), so it does not itself open an incident on the failure one. Its purpose is purely to feed `StartupFailureDetector.observe_health`, which keys off `data["outcome"] == "success"` and adds the stream to `_recovered`. The event still flows through the tracker, but the tracker's per-fingerprint grouping means `:startup-success` events open at most a trivial INFO incident that auto-closes via the passthrough quiet window (no user-visible noise given severity=INFO). + +## New detector: `NoDataDetector` + +Lives at `src/syncfield/health/detectors/no_data.py`. Registered by default in `HealthSystem._install_default_detectors`. + +- Fires `HealthEvent(fingerprint=f"{stream_id}:no-data", severity=ERROR)` when a stream has been in `connected` state for ≥ `threshold_ns` (default 5 s) without any sample. +- Closes the incident the moment the first sample arrives (`close_condition` returns `stream_id in self._has_sample`). +- Per-stream state machine keyed by `stream_id`: `_connected_at`, `_has_sample`, `_fire_active`. Reset to empty on `observe_connection_state(stream_id, new_state)` whenever the new state is `failed / disconnected / idle`. + +### `Detector.observe_connection_state` (new hook) + +Added to the protocol and to `DetectorBase` as a no-op. Existing detectors keep working unchanged. `NoDataDetector` is the only one that overrides it for v1. + +### Worker ingress + +`HealthWorker` gains a fifth queue: +```python +_connection_states: queue.SimpleQueue[_ConnectionStateMsg] +``` +drained on each tick the same way the other four are. `HealthSystem.observe_connection_state(stream_id, new_state)` pushes into it. Called by `SessionOrchestrator._set_stream_state`. + +## Viewer changes + +### Backend (`viewer/state.py`, `viewer/poller.py`, `viewer/server.py`) + +- `StreamSnapshot`: add `connection_state: str = "idle"` and `connection_error: str | None = None` fields. +- Poller reads `orchestrator._stream_states` and `orchestrator._stream_errors` on each tick and populates the snapshot. +- Server WebSocket serializer includes both fields in each stream entry. + +### Frontend types (`lib/types.ts`) + +```ts +export type ConnectionState = + | "idle" | "connecting" | "connected" | "failed" | "disconnected"; + +export interface StreamSnapshot { + // ...existing fields... + connection_state: ConnectionState; + connection_error: string | null; +} +``` + +### `StreamCard` body selection + +```tsx +function StreamCardBody({ stream }: { stream: StreamSnapshot }) { + if (stream.connection_state === "connecting") return ; + if (stream.connection_state === "failed") { + return ; + } + if (stream.connection_state === "connected" && stream.frame_count === 0) { + return ; + } + return ; +} +``` + +### New overlay components (`components/stream-overlays.tsx`) + +- `ConnectingOverlay` — neutral grey background, pulse-animated dot + "Connecting…" text. +- `WaitingForDataOverlay` — soft yellow tint, "Connected · waiting for first frame". No animation (it should feel temporary; the real alarm is the no-data incident that opens after 5 s). +- `FailedOverlay` — red background, "Failed to connect" header, monospace error text (two visible lines, clickable to expand to full text), hint "Press Discover Devices or Disconnect + Connect to retry". + +### Header state chip (`components/header.tsx`) + +```tsx +const total = Object.keys(snapshot.streams).length; +const connected = Object.values(snapshot.streams) + .filter((s) => s.connection_state === "connected").length; +const label = connected === total || total === 0 + ? stateLabel + : `${stateLabel} (${connected}/${total})`; +const tone = connected < total ? "warning" : "normal"; +``` +Warning-tone chip is yellow; normal chip keeps its current styling. + +## Orchestrator integration points + +All changes in `src/syncfield/orchestrator.py`: + +1. **`__init__`**: initialize `self._stream_states: dict[str, ConnectionState] = {}` and `self._stream_errors: dict[str, str] = {}`. +2. **`add(stream)`**: after existing wiring, set `self._stream_states[stream.id] = "idle"`. +3. **`connect()`**: replace the current all-or-nothing loop (lines 1623-1643) with the partial-connect loop in the Architecture Overview. Use a single helper `_set_stream_state(stream_id, new_state)` that updates `_stream_states` *and* calls `self.health.observe_connection_state(stream_id, new_state)`. +4. **`disconnect()`**: only call `stream.disconnect()` on streams whose `_stream_states[sid]` is `connected` (or was up to `stop_recording`). Transition failed streams directly to `disconnected`. Clear `_stream_errors`. +5. **`stop()`**: unchanged by this spec — it already handles only streams that were recording. Failed streams never entered the recording set. + +Remove the now-dead `_rollback_disconnect_streams` helper if nothing else uses it, otherwise leave it untouched. + +## Persistence & artifacts + +No new files. The startup-failure and no-data incidents flow through the existing `incidents.jsonl` sidecar via the callbacks wired in Phase 1. + +## Testing strategy + +### Unit tests + +**`tests/unit/health/detectors/test_no_data.py`** — new file. Covers: +- No fire when `connected` for less than `threshold_ns`. +- Fires after threshold with no sample. +- Closes after first sample arrives. +- Does not re-fire for the same stream while still in the connected-no-data state (uses `_fire_active`). +- Resets bookkeeping on state transition back to `failed` / `disconnected` / `idle`. +- Per-stream independent state. + +**`tests/unit/health/test_health_worker.py`** (extend) — verify the worker drains a `_connection_states` queue and fans out to `observe_connection_state` on every registered detector. + +**`tests/unit/health/test_detector_base.py`** (extend) — confirm the new `observe_connection_state` hook is a safe no-op on the base class. + +**`tests/unit/test_orchestrator.py`** (new test class `TestPartialConnect`) — the orchestrator tests already use `FakeStream` with `fail_on_start` / `fail_on_connect` flags. Cover: +- One stream raising in connect: session ends up in `CONNECTED`; that stream's `connection_state == "failed"`; others are `"connected"`. +- All streams raising: session returns to `IDLE` and the aggregate error is raised. +- `_stream_errors[sid]` contains the exception message string. +- `StartupFailureDetector` sees a `phase="connect"` event (spy detector). +- `disconnect()` does not call `stream.disconnect()` on streams whose state is `failed`. + +### Integration tests + +**`tests/integration/health/test_partial_connect.py`** — new file. Spins a real `SessionOrchestrator` with one `FakeStream` that always raises in `connect()` and two normal `FakeStream`s. Drives the full connect → record → stop → disconnect lifecycle and asserts: +- Session reaches `CONNECTED` (not `IDLE`). +- The startup-failure incident is in `sess.health.open_incidents()` after connect. +- `incidents.jsonl` contains the fingerprint. +- Working streams produce samples; failed stream does not. + +**`tests/integration/health/test_no_data_detector.py`** — a `FakeStream` whose `connect()` returns but whose sample-emission thread is inhibited. After 6 s of wall-clock, the no-data incident is open. Resuming sample emission closes the incident within one tick window. + +### Frontend + +Component test for `StreamCard` branch selection: one test per state (`connecting` / `failed` / `connected-no-frames` / `connected-with-frames`). Assert the correct overlay component is rendered. The overlays themselves get a trivial render test each. + +Header chip rendering: three cases (`total === 0`, `connected === total`, `connected < total`). + +## Scope for this PR (`feat/health-telemetry`) + +Adds to the existing PR on top of the Phase 1 health-telemetry work. Ordered by dependency: + +1. **Data model**: `ConnectionState` literal type, `StreamSnapshot.connection_state` / `connection_error` fields. +2. **`Detector.observe_connection_state` hook** on protocol + base. +3. **`HealthWorker` new ingress queue**; `HealthSystem.observe_connection_state` passthrough. +4. **`NoDataDetector`** + default-register in `HealthSystem`. +5. **`SessionOrchestrator.connect()` partial-connect rewrite** + `_set_stream_state` helper + structured startup events. +6. **Poller + server** snapshot serialization. +7. **Frontend overlays + card branch + header chip**. +8. **All tests listed above**. + +## Open questions / deferred decisions + +- Exact `threshold_ns` for `NoDataDetector` — proposed 5 s. Short enough to catch a silent OAK within the window a user is watching; long enough to absorb the natural warm-up of cameras that take a second or two to produce the first frame after connect. +- `FailedOverlay` copy — current draft is "Failed to connect" + verbatim error + "Press Discover Devices or Disconnect + Connect to retry". Final copy is a UX polish pass, not a design gate. + +## Risks + +- **Orchestrator `disconnect()` paths** need a careful read. If any bookkeeping currently assumes "every stream the orchestrator knows is fully connected", it may drop frames or mis-close files for a stream that was `failed` and never entered recording. The partial-connect rewrite must explicitly skip `failed` streams everywhere a loop iterates `self._streams.values()`. +- **Severity=INFO incident from success events** — the success signal uses `{stream_id}:startup-success` and creates a short-lived INFO incident that auto-closes via the passthrough quiet window. At `severity=INFO` it does not appear in the IncidentPanel's Active Issues (which filters by severity visually in the existing UI), but it does land in `incidents.jsonl`. If that persisted-log noise is unwanted, we can suppress the tracker's ingest for events whose fingerprint ends with `:startup-success`. Deferred until we see real post-session logs and decide whether it's useful telemetry or noise. +- **Frontend overlay stacking with `severity badge`** — the per-stream badge (Phase 1) paints on the card header; the overlays paint on the body. They should not collide, but the implementation must verify the card layout renders sensibly in all five states. From 45bf93ac50ab7248eabb3fd676d16dddd54cc47a Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 15:41:00 +0900 Subject: [PATCH 45/64] docs: implementation plan for partial-connect UX MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 18 TDD tasks — Detector.observe_connection_state hook, HealthWorker queue, HealthSystem passthrough, NoDataDetector + default registration, orchestrator per-stream state dicts + partial-connect rewrite + disconnect cleanup, StreamSnapshot fields + poller + server serializer, frontend types + overlay components + StreamCard branch + header chip, integration tests for partial-connect and no-data. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../plans/2026-04-22-partial-connect-ux.md | 1708 +++++++++++++++++ 1 file changed, 1708 insertions(+) create mode 100644 docs/superpowers/plans/2026-04-22-partial-connect-ux.md diff --git a/docs/superpowers/plans/2026-04-22-partial-connect-ux.md b/docs/superpowers/plans/2026-04-22-partial-connect-ux.md new file mode 100644 index 0000000..2f5fd23 --- /dev/null +++ b/docs/superpowers/plans/2026-04-22-partial-connect-ux.md @@ -0,0 +1,1708 @@ +# Partial Connect UX Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace the all-or-nothing `SessionOrchestrator.connect()` with per-stream partial connect, surface per-stream connection state + error in the snapshot, fire a new `NoDataDetector` for the "connected but no sample ever" case, and redesign the viewer's `StreamCard` to render state-aware overlays (connecting / waiting / failed) plus a degraded-state header chip. + +**Architecture:** Orchestrator catches each `stream.connect()` independently, records a per-stream `ConnectionState` on `self._stream_states`, and emits structured `HealthEvent`s that feed the existing (previously dormant) `StartupFailureDetector`. A new `Detector.observe_connection_state` hook is added to the health protocol so `NoDataDetector` can track per-stream "entered `connected` at" timestamps. Snapshot gains two new fields; the viewer branches its `StreamCard` body on the connection state and shows a yellow `Ready (n/total)` chip when any stream is failed. + +**Tech Stack:** Python 3.9+ (stdlib + existing health/ package), React + TypeScript + Tailwind for the viewer. No new external dependencies. + +**Spec:** `docs/superpowers/specs/2026-04-22-partial-connect-ux-design.md` + +--- + +## File Structure + +### New (backend) + +``` +src/syncfield/health/detectors/no_data.py # NoDataDetector +``` + +### Modified (backend) + +- `src/syncfield/health/detector.py` — add `observe_connection_state` hook to Protocol + DetectorBase. +- `src/syncfield/health/worker.py` — new `_connection_states` ingress queue + fan-out to detectors. +- `src/syncfield/health/system.py` — `observe_connection_state` passthrough; register `NoDataDetector`. +- `src/syncfield/orchestrator.py` — `_stream_states` / `_stream_errors` dicts, `_set_stream_state` helper, partial-connect rewrite in `connect()`, failed-stream skip in `disconnect()`. +- `src/syncfield/viewer/state.py` — `StreamSnapshot.connection_state` / `connection_error` fields. +- `src/syncfield/viewer/poller.py` — read orchestrator's state dicts into each snapshot. +- `src/syncfield/viewer/server.py` — WebSocket serializer includes new stream fields. + +### New (frontend) + +``` +src/syncfield/viewer/frontend/src/components/stream-overlays.tsx +``` + +### Modified (frontend) + +- `src/syncfield/viewer/frontend/src/lib/types.ts` — `ConnectionState` type + `StreamSnapshot` field additions. +- `src/syncfield/viewer/frontend/src/components/stream-card.tsx` — branch body on `connection_state`. +- `src/syncfield/viewer/frontend/src/components/header.tsx` — degraded-state chip. + +### Tests (new + extended) + +``` +tests/unit/health/detectors/test_no_data.py +tests/unit/health/test_worker_connection_state.py (or add cases to test_health_worker.py) +tests/unit/test_orchestrator_partial_connect.py (new test class) +tests/integration/health/test_partial_connect.py +tests/integration/health/test_no_data_detector.py +tests/unit/viewer/test_snapshot_connection_state.py (or add cases to test_snapshot_incidents.py) +``` + +--- + +## Conventions + +- TDD every task: failing test → confirm fail → implement → confirm pass → commit. +- Run tests via `uv run pytest -v`. +- Commits follow the existing Conventional Commits style. Every commit includes the trailer `Co-Authored-By: Claude Opus 4.7 (1M context) ` (HEREDOC form). +- Implementer is agnostic to task order only within blocks marked "can be parallelized"; otherwise follow task number order. + +--- + +## Task 1 — Add `observe_connection_state` hook to Detector protocol + base + +**Files:** +- Modify: `src/syncfield/health/detector.py` +- Modify: `tests/unit/health/test_detector_base.py` + +- [ ] **Step 1: Write failing test** + +Append to `tests/unit/health/test_detector_base.py`: + +```python +def test_detector_base_observe_connection_state_default_is_noop(): + d = NoopDetector() + # Does not raise; returns None. + assert d.observe_connection_state("cam", "connected", 100) is None +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +uv run pytest tests/unit/health/test_detector_base.py::test_detector_base_observe_connection_state_default_is_noop -v +``` +Expected: `AttributeError: 'NoopDetector' object has no attribute 'observe_connection_state'`. + +- [ ] **Step 3: Implement** + +In `src/syncfield/health/detector.py`, add the method to the `Detector` Protocol (alongside the other five `observe_*` methods): + +```python + def observe_connection_state(self, stream_id: str, new_state: str, at_ns: int) -> None: ... +``` + +And to `DetectorBase` (as a no-op, matching the other `observe_*` defaults): + +```python + def observe_connection_state(self, stream_id: str, new_state: str, at_ns: int) -> None: + pass +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +uv run pytest tests/unit/health/test_detector_base.py -v +``` +Expected: all tests pass (previous 3 + new 1 = 4 passed). + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/detector.py tests/unit/health/test_detector_base.py +git commit -m "$(cat <<'EOF' +feat(health): add Detector.observe_connection_state hook + +Adds a no-op default for the per-stream connection-state observer that +NoDataDetector will override. Protocol + base mirror the existing +observe_sample / observe_health / observe_state shape. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 2 — HealthWorker: `_connection_states` ingress queue + fan-out + +**Files:** +- Modify: `src/syncfield/health/worker.py` +- Modify: `tests/unit/health/test_health_worker.py` + +- [ ] **Step 1: Write failing test** + +Append to `tests/unit/health/test_health_worker.py` (inside the existing module, at top-level): + +```python +def test_worker_drains_connection_state_queue_and_fans_out(): + class Spy(DetectorBase): + name = "conn-spy" + default_severity = Severity.INFO + + def __init__(self): + self.calls = [] + + def observe_connection_state(self, stream_id, new_state, at_ns): + self.calls.append((stream_id, new_state, at_ns)) + + tr = IncidentTracker() + spy = Spy() + w = HealthWorker(tracker=tr, detectors=[spy], tick_hz=100) + w.start() + try: + w.push_connection_state("cam", "connecting", 1) + w.push_connection_state("cam", "connected", 2) + assert _wait_until(lambda: len(spy.calls) == 2) + finally: + w.stop() + + assert spy.calls == [("cam", "connecting", 1), ("cam", "connected", 2)] +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +uv run pytest tests/unit/health/test_health_worker.py::test_worker_drains_connection_state_queue_and_fans_out -v +``` +Expected: `AttributeError: 'HealthWorker' object has no attribute 'push_connection_state'`. + +- [ ] **Step 3: Implement** + +In `src/syncfield/health/worker.py`: + +1. Add a new message dataclass near the other `_*Msg`: + +```python +@dataclass(frozen=True) +class _ConnectionStateMsg: + stream_id: str + new_state: str + at_ns: int +``` + +2. Add the queue field in `HealthWorker.__init__` (alongside the existing four): + +```python + self._connection_states: "queue.SimpleQueue[_ConnectionStateMsg]" = queue.SimpleQueue() +``` + +3. Add the ingress method: + +```python + def push_connection_state(self, stream_id: str, new_state: str, at_ns: int) -> None: + self._connection_states.put(_ConnectionStateMsg(stream_id, new_state, at_ns)) +``` + +4. Extend `_drain_once` with a final block: + +```python + for msg in _drain_queue(self._connection_states): + for d in self._detectors: + d.observe_connection_state(msg.stream_id, msg.new_state, msg.at_ns) +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +uv run pytest tests/unit/health/test_health_worker.py -v +``` +Expected: 5 passed (previous 4 + new 1). + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/worker.py tests/unit/health/test_health_worker.py +git commit -m "$(cat <<'EOF' +feat(health): add per-stream connection-state ingress to HealthWorker + +New _ConnectionStateMsg + push_connection_state() + _drain_once fan-out +so NoDataDetector (next commit) can track per-stream CONNECTING → +CONNECTED transitions via the standard observer pattern. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 3 — HealthSystem: `observe_connection_state` passthrough + +**Files:** +- Modify: `src/syncfield/health/system.py` +- Modify: `tests/unit/health/test_health_system.py` + +- [ ] **Step 1: Write failing test** + +Append to `tests/unit/health/test_health_system.py`: + +```python +def test_health_system_observe_connection_state_routes_to_worker(): + class Spy(DetectorBase): + name = "conn-spy" + default_severity = Severity.INFO + + def __init__(self): + self.calls = [] + + def observe_connection_state(self, stream_id, new_state, at_ns): + self.calls.append((stream_id, new_state, at_ns)) + + hs = HealthSystem() + spy = Spy() + hs.register(spy) + + hs.start() + try: + hs.observe_connection_state("cam", "connecting", 10) + hs.observe_connection_state("cam", "connected", 20) + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline and len(spy.calls) < 2: + time.sleep(0.02) + finally: + hs.stop() + assert spy.calls == [("cam", "connecting", 10), ("cam", "connected", 20)] +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +uv run pytest tests/unit/health/test_health_system.py::test_health_system_observe_connection_state_routes_to_worker -v +``` +Expected: `AttributeError: 'HealthSystem' object has no attribute 'observe_connection_state'`. + +- [ ] **Step 3: Implement** + +In `src/syncfield/health/system.py`, add the passthrough method (right next to the other `observe_*` methods): + +```python + def observe_connection_state(self, stream_id: str, new_state: str, at_ns: int) -> None: + if self._worker is not None: + self._worker.push_connection_state(stream_id, new_state, at_ns) +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +uv run pytest tests/unit/health/test_health_system.py -v +``` +Expected: all existing tests + new one all pass. + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/system.py tests/unit/health/test_health_system.py +git commit -m "$(cat <<'EOF' +feat(health): expose observe_connection_state on HealthSystem + +Passthrough to the worker's ingress queue, matching the existing +observe_sample / observe_health / observe_state shape. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 4 — NoDataDetector (core logic + unit tests) + +**Files:** +- Create: `src/syncfield/health/detectors/no_data.py` +- Create: `tests/unit/health/detectors/test_no_data.py` + +- [ ] **Step 1: Write failing test** + +`tests/unit/health/detectors/test_no_data.py`: + +```python +from syncfield.health.detectors.no_data import NoDataDetector +from syncfield.health.types import Incident +from syncfield.types import SampleEvent + + +def _s(stream: str, t_ns: int) -> SampleEvent: + return SampleEvent(stream_id=stream, frame_number=0, capture_ns=t_ns) + + +def test_no_fire_before_threshold(): + d = NoDataDetector(threshold_ns=1000) + d.observe_connection_state("cam", "connected", at_ns=100) + assert list(d.tick(now_ns=500)) == [] # 400 ns elapsed, under 1000 + + +def test_fires_after_threshold_without_sample(): + d = NoDataDetector(threshold_ns=1000) + d.observe_connection_state("cam", "connected", at_ns=100) + out = list(d.tick(now_ns=2000)) # 1900 ns elapsed + assert len(out) == 1 + ev = out[0] + assert ev.stream_id == "cam" + assert ev.fingerprint == "cam:no-data" + assert ev.source == "detector:no-data" + assert "no data" in (ev.detail or "").lower() + + +def test_does_not_refire_while_still_no_data(): + d = NoDataDetector(threshold_ns=1000) + d.observe_connection_state("cam", "connected", at_ns=100) + first = list(d.tick(now_ns=2000)) + second = list(d.tick(now_ns=3000)) + assert len(first) == 1 + assert len(second) == 0 + + +def test_close_condition_satisfied_once_sample_arrives(): + d = NoDataDetector(threshold_ns=1000) + d.observe_connection_state("cam", "connected", at_ns=100) + events = list(d.tick(now_ns=2000)) + inc = Incident.opened_from(events[0], title="x") + + assert d.close_condition(inc, now_ns=2100) is False # still no sample + + d.observe_sample("cam", _s("cam", 2200)) + assert d.close_condition(inc, now_ns=2300) is True + + +def test_resets_bookkeeping_on_non_connected_state(): + d = NoDataDetector(threshold_ns=1000) + d.observe_connection_state("cam", "connected", at_ns=100) + list(d.tick(now_ns=2000)) # fires + + d.observe_connection_state("cam", "failed", at_ns=2500) + # Back to connected → fresh clock, no duplicate fire. + d.observe_connection_state("cam", "connected", at_ns=3000) + assert list(d.tick(now_ns=3500)) == [] # only 500 ns since new connected + + +def test_per_stream_independent_state(): + d = NoDataDetector(threshold_ns=1000) + d.observe_connection_state("a", "connected", at_ns=100) + d.observe_connection_state("b", "connected", at_ns=100) + d.observe_sample("b", _s("b", 200)) + + out = list(d.tick(now_ns=2000)) + assert len(out) == 1 + assert out[0].stream_id == "a" +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +uv run pytest tests/unit/health/detectors/test_no_data.py -v +``` +Expected: `ModuleNotFoundError: No module named 'syncfield.health.detectors.no_data'`. + +- [ ] **Step 3: Implement** + +`src/syncfield/health/detectors/no_data.py`: + +```python +"""NoDataDetector — fires when a stream is connected but never emits a sample. + +Complements StreamStallDetector (which requires prior samples). Catches +the "connected but silent" case such as an OAK pipeline that fails to +pump frames even though its device connected. Resets bookkeeping on +any non-connected state transition so a reconnect starts a fresh clock. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from typing import Dict, List, Set + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.types import Incident +from syncfield.types import HealthEvent, HealthEventKind, SampleEvent + + +class NoDataDetector(DetectorBase): + name = "no-data" + default_severity = Severity.ERROR + + def __init__(self, threshold_ns: int = 5_000_000_000) -> None: + self._threshold_ns = threshold_ns + self._connected_at: Dict[str, int] = {} + self._has_sample: Set[str] = set() + self._fire_active: Dict[str, bool] = {} + + def observe_connection_state(self, stream_id: str, new_state: str, at_ns: int) -> None: + if new_state == "connected": + self._connected_at[stream_id] = at_ns + self._has_sample.discard(stream_id) + self._fire_active[stream_id] = False + else: + # idle / connecting / failed / disconnected → reset everything. + self._connected_at.pop(stream_id, None) + self._has_sample.discard(stream_id) + self._fire_active.pop(stream_id, None) + + def observe_sample(self, stream_id: str, sample: SampleEvent) -> None: + self._has_sample.add(stream_id) + + def tick(self, now_ns: int) -> Iterator[HealthEvent]: + out: List[HealthEvent] = [] + for stream_id, connected_at in self._connected_at.items(): + if stream_id in self._has_sample: + continue + elapsed = now_ns - connected_at + if elapsed >= self._threshold_ns and not self._fire_active.get(stream_id, False): + self._fire_active[stream_id] = True + out.append(HealthEvent( + stream_id=stream_id, + kind=HealthEventKind.ERROR, + at_ns=now_ns, + detail=f"Connected {elapsed / 1e9:.1f}s ago but no data received", + severity=self.default_severity, + source=f"detector:{self.name}", + fingerprint=f"{stream_id}:{self.name}", + data={"connected_at_ns": connected_at, "elapsed_ns": elapsed}, + )) + return iter(out) + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + return incident.stream_id in self._has_sample +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +uv run pytest tests/unit/health/detectors/test_no_data.py -v +``` +Expected: 6 passed. + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/detectors/no_data.py tests/unit/health/detectors/test_no_data.py +git commit -m "$(cat <<'EOF' +feat(health): add NoDataDetector for connected-but-silent streams + +Fires when a stream is in 'connected' state for N seconds without any +sample. Complements StreamStallDetector which requires prior samples. +Catches the OAK "black square" symptom where connect() succeeds but +the pipeline never pumps frames. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 5 — Register NoDataDetector in HealthSystem default suite + +**Files:** +- Modify: `src/syncfield/health/system.py` +- Modify: `tests/unit/health/test_health_system.py` (extend existing `test_health_system_installs_default_detectors`) + +- [ ] **Step 1: Update the assertion** + +In `tests/unit/health/test_health_system.py`, find `test_health_system_installs_default_detectors` and add `"no-data"` to the expected detector set: + +```python + for expected in ( + "adapter", + "stream-stall", + "fps-drop", + "jitter", + "startup-failure", + "backpressure", + "no-data", + ): + assert expected in names, f"missing default detector: {expected}" +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +uv run pytest tests/unit/health/test_health_system.py::test_health_system_installs_default_detectors -v +``` +Expected: `AssertionError: missing default detector: no-data`. + +- [ ] **Step 3: Implement** + +In `src/syncfield/health/system.py`, add the import and registration: + +```python +from syncfield.health.detectors.no_data import NoDataDetector +``` + +Inside `_install_default_detectors`, after `self.register(BackpressureDetector())`, add: + +```python + self.register(NoDataDetector()) +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +uv run pytest tests/unit/health/test_health_system.py -v +``` +Expected: all pass. + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/system.py tests/unit/health/test_health_system.py +git commit -m "$(cat <<'EOF' +feat(health): register NoDataDetector in default suite + +Now installed automatically on HealthSystem construction alongside the +other six default detectors. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 6 — Orchestrator: per-stream state dicts + `_set_stream_state` helper + +**Files:** +- Modify: `src/syncfield/orchestrator.py` +- Test: no dedicated test file for this task — the test lands in Task 8 (partial connect). + +- [ ] **Step 1: Add state containers to `__init__`** + +In `src/syncfield/orchestrator.py`, inside `SessionOrchestrator.__init__`, after the line `self._connected_streams: List[Stream] = []` (around line 360), add: + +```python + # Per-stream connection state for partial-connect semantics. + # Keys are stream ids; values are one of: + # "idle" | "connecting" | "connected" | "failed" | "disconnected". + self._stream_states: dict[str, str] = {} + # Populated only when a stream's connect() raised. + self._stream_errors: dict[str, str] = {} +``` + +- [ ] **Step 2: Add `_set_stream_state` helper** + +Add a new method on the class (near the other private helpers, after `_persist_incident`): + +```python + def _set_stream_state(self, stream_id: str, new_state: str) -> None: + """Update per-stream connection state and forward to HealthSystem. + + The health worker may not be running yet (e.g. we call this from + add() when the session is IDLE) — observe_connection_state is a + no-op in that case. + """ + self._stream_states[stream_id] = new_state + self.health.observe_connection_state(stream_id, new_state, time.monotonic_ns()) +``` + +- [ ] **Step 3: Wire into `add()`** + +Find `SessionOrchestrator.add(stream)` (around line 1456). At the end of the method body (after any existing wiring such as `stream.on_sample(...)` / `stream.on_health(...)`), add: + +```python + self._set_stream_state(stream.id, "idle") +``` + +- [ ] **Step 4: Quick sanity check** + +```bash +uv run pytest tests/unit/test_orchestrator.py::TestAdd -v +``` +Expected: all existing `TestAdd` tests still pass (no regression). + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/orchestrator.py +git commit -m "$(cat <<'EOF' +feat(orchestrator): add per-stream connection-state dicts + helper + +_stream_states and _stream_errors become the source of truth for +per-stream status. _set_stream_state is the single update point — it +also forwards to HealthSystem.observe_connection_state so NoDataDetector +sees the transitions. add() initializes each new stream to 'idle'. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 7 — Orchestrator: partial-connect rewrite (happy path + error path + success signal) + +**Files:** +- Modify: `src/syncfield/orchestrator.py` (the `connect()` method, around lines 1590-1655) +- Test: `tests/unit/test_orchestrator_partial_connect.py` (new) + +- [ ] **Step 1: Write failing test** + +`tests/unit/test_orchestrator_partial_connect.py`: + +```python +"""Partial-connect semantics for SessionOrchestrator. + +Relies on the FakeStream helper in syncfield.testing, which supports +`fail_on_start=True` to raise from its connect() path. +""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +from syncfield.orchestrator import SessionOrchestrator +from syncfield.testing import FakeStream +from syncfield.types import SessionState + + +def test_one_stream_fails_others_still_connected(tmp_path: Path): + sess = SessionOrchestrator(host_id="h", output_dir=tmp_path) + sess.add(FakeStream("good_a")) + sess.add(FakeStream("bad", fail_on_start=True)) + sess.add(FakeStream("good_b")) + + sess.connect() + + assert sess.state_name == SessionState.CONNECTED.value + assert sess._stream_states["good_a"] == "connected" + assert sess._stream_states["bad"] == "failed" + assert sess._stream_states["good_b"] == "connected" + assert "bad" in sess._stream_errors + assert sess._stream_errors["bad"] # non-empty message + + +def test_all_streams_failing_raises_and_returns_to_idle(tmp_path: Path): + sess = SessionOrchestrator(host_id="h", output_dir=tmp_path) + sess.add(FakeStream("a", fail_on_start=True)) + sess.add(FakeStream("b", fail_on_start=True)) + + with pytest.raises(RuntimeError, match="no streams"): + sess.connect() + + assert sess.state_name == SessionState.IDLE.value + assert sess._stream_states["a"] == "failed" + assert sess._stream_states["b"] == "failed" + + +def test_startup_failure_event_reaches_health_system(tmp_path: Path): + # Spy detector that captures health events it observes. + from syncfield.health.detector import DetectorBase + from syncfield.health.severity import Severity + + class Spy(DetectorBase): + name = "startup-spy" + default_severity = Severity.INFO + + def __init__(self): + self.events = [] + + def observe_health(self, stream_id, event): + self.events.append(event) + + sess = SessionOrchestrator(host_id="h", output_dir=tmp_path) + spy = Spy() + sess.health.register(spy) + sess.add(FakeStream("good")) + sess.add(FakeStream("bad", fail_on_start=True)) + + sess.connect() + + # Give the worker a tick to drain the health queue. + import time + deadline = time.monotonic() + 1.0 + while time.monotonic() < deadline and not any( + e.fingerprint == "bad:startup-failure" for e in spy.events + ): + time.sleep(0.02) + + failure_events = [e for e in spy.events if e.fingerprint == "bad:startup-failure"] + assert failure_events, "no startup-failure event observed" + ev = failure_events[0] + assert ev.data.get("phase") == "connect" + assert ev.data.get("outcome") == "error" + assert ev.data.get("error") +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +uv run pytest tests/unit/test_orchestrator_partial_connect.py -v +``` +Expected: multiple failures — the first test hits the current all-or-nothing rollback and the session ends up in IDLE. + +- [ ] **Step 3: Implement** + +In `src/syncfield/orchestrator.py`, replace the existing `connect()` body's try/except block (lines 1623-1643 in the current file — find the section starting `connected: List[Stream] = []` and ending just before `self._connected_streams = connected`) with: + +```python + connected: List[Stream] = [] + for stream in self._streams.values(): + self._set_stream_state(stream.id, "connecting") + try: + stream.prepare() + stream.connect() + except Exception as exc: + self._stream_errors[stream.id] = str(exc) + self._set_stream_state(stream.id, "failed") + stream._emit_health(HealthEvent( + stream_id=stream.id, + kind=HealthEventKind.ERROR, + at_ns=time.monotonic_ns(), + detail=str(exc), + severity=Severity.ERROR, + source="orchestrator", + fingerprint=f"{stream.id}:startup-failure", + data={"phase": "connect", "outcome": "error", "error": str(exc)}, + )) + continue + connected.append(stream) + self._stream_errors.pop(stream.id, None) + self._set_stream_state(stream.id, "connected") + stream._emit_health(HealthEvent( + stream_id=stream.id, + kind=HealthEventKind.HEARTBEAT, + at_ns=time.monotonic_ns(), + detail="connected", + severity=Severity.INFO, + source="orchestrator", + fingerprint=f"{stream.id}:startup-success", + data={"phase": "connect", "outcome": "success"}, + )) + + if not connected: + self._transition(SessionState.IDLE) + if self._log_writer is not None: + self._log_writer.close() + self._log_writer = None + raise RuntimeError( + "connect() failed: no streams connected — every adapter raised. " + "Inspect per-stream errors via session._stream_errors." + ) +``` + +Imports at the top of the file need to include `Severity` if not already present: + +```python +from syncfield.health.severity import Severity +``` + +(`HealthEvent` and `HealthEventKind` should already be imported.) + +- [ ] **Step 4: Run, confirm pass** + +```bash +uv run pytest tests/unit/test_orchestrator_partial_connect.py -v +``` +Expected: 3 passed. + +Also run the full orchestrator suite to surface regressions: + +```bash +uv run pytest tests/unit/test_orchestrator.py -v +``` +Expected: all pre-existing tests still pass. + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/orchestrator.py tests/unit/test_orchestrator_partial_connect.py +git commit -m "$(cat <<'EOF' +feat(orchestrator): partial-connect — survive per-stream connect() failure + +One stream raising no longer rolls the whole session back to IDLE. We +record the error, emit a structured HealthEvent (phase=connect, +outcome=error) that activates the previously-dormant +StartupFailureDetector, and continue with the rest. Session only fails +if every adapter raises. Success path emits the complementary +outcome=success signal so the detector's recovery tracking works. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 8 — Orchestrator: `disconnect()` skips failed streams, updates state + +**Files:** +- Modify: `src/syncfield/orchestrator.py` (the `disconnect()` method around line 2100) +- Test: `tests/unit/test_orchestrator_partial_connect.py` (extend) + +- [ ] **Step 1: Write failing test** + +Append to `tests/unit/test_orchestrator_partial_connect.py`: + +```python +def test_disconnect_does_not_call_stream_disconnect_on_failed(tmp_path: Path): + from syncfield.testing import FakeStream + + class CountingFakeStream(FakeStream): + def __init__(self, stream_id, fail_on_start=False): + super().__init__(stream_id, fail_on_start=fail_on_start) + self.disconnect_calls = 0 + + def disconnect(self): + self.disconnect_calls += 1 + super().disconnect() + + sess = SessionOrchestrator(host_id="h", output_dir=tmp_path) + good = CountingFakeStream("good") + bad = CountingFakeStream("bad", fail_on_start=True) + sess.add(good) + sess.add(bad) + + sess.connect() + sess.disconnect() + + assert good.disconnect_calls == 1 + assert bad.disconnect_calls == 0 + assert sess._stream_states["good"] == "disconnected" + assert sess._stream_states["bad"] == "disconnected" + assert sess._stream_errors == {} +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +uv run pytest tests/unit/test_orchestrator_partial_connect.py::test_disconnect_does_not_call_stream_disconnect_on_failed -v +``` +Expected: at minimum `sess._stream_errors != {}` (errors never cleared), or `bad.disconnect_calls > 0` because the rollback helper iterates `_connected_streams` — which is correct for good, but we also need to flip each stream's state. + +- [ ] **Step 3: Implement** + +Locate `SessionOrchestrator.disconnect()` (around line 2100). Replace the body of the `with self._lock:` block (the part that calls `_rollback_disconnect_streams(self._connected_streams)`) with: + +```python + if self._state not in (SessionState.CONNECTED, SessionState.STOPPED): + raise RuntimeError( + f"disconnect() requires CONNECTED or STOPPED state; " + f"current state is {self._state.value}" + ) + # Only streams that were 'connected' (or 'recording' → 'stopped') + # ever opened hardware; 'failed' streams never did, so skip them. + _rollback_disconnect_streams(self._connected_streams) + self._connected_streams = [] + + # Flip every stream's snapshot-visible state to 'disconnected', + # regardless of whether we called disconnect() on it. Clear any + # per-stream errors since the session is returning to a clean slate. + for stream_id in list(self._stream_states.keys()): + self._set_stream_state(stream_id, "disconnected") + self._stream_errors.clear() + + # Keep auto-injected audio stream registered (visible in viewer) + # but disconnected. It will be reconnected on next connect(). + + # Multi-host infrastructure (advertiser, browser, control plane) + # stays up across disconnect(). It was brought up at __init__ + # and is only torn down by shutdown() or the atexit handler. +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +uv run pytest tests/unit/test_orchestrator_partial_connect.py -v +``` +Expected: 4 passed. + +```bash +uv run pytest tests/unit/test_orchestrator.py -v +``` +Expected: no regression. + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/orchestrator.py tests/unit/test_orchestrator_partial_connect.py +git commit -m "$(cat <<'EOF' +feat(orchestrator): disconnect() handles partial-connect survivors + +_connected_streams only ever held successful streams, so the existing +rollback helper is already correct — but we now flip every tracked +stream's state to 'disconnected' and clear per-stream errors so the +viewer reflects a clean slate after teardown. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 9 — `StreamSnapshot.connection_state` + `connection_error` fields + +**Files:** +- Modify: `src/syncfield/viewer/state.py` +- Test: `tests/unit/viewer/test_snapshot_incidents.py` (extend) + +- [ ] **Step 1: Write failing test** + +Append to `tests/unit/viewer/test_snapshot_incidents.py`: + +```python +def test_stream_snapshot_has_connection_state_fields(): + import dataclasses + from syncfield.viewer.state import StreamSnapshot + + fields = {f.name: f for f in dataclasses.fields(StreamSnapshot)} + assert "connection_state" in fields + assert "connection_error" in fields + + # Defaults when constructed minimally. + snap = StreamSnapshot( + id="cam", kind="video", provides_audio_track=False, produces_file=False, + frame_count=0, last_sample_at_ns=None, effective_hz=0.0, + latest_frame=None, plot_points={}, latest_pose={}, + ) + assert snap.connection_state == "idle" + assert snap.connection_error is None +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +uv run pytest tests/unit/viewer/test_snapshot_incidents.py::test_stream_snapshot_has_connection_state_fields -v +``` +Expected: `AssertionError: 'connection_state' not in fields`. + +- [ ] **Step 3: Implement** + +In `src/syncfield/viewer/state.py`, find the `StreamSnapshot` dataclass. Add two new fields **at the end** of the dataclass (after `live_preview: bool = True`), so constructors that supply only positional arguments for existing fields keep working: + +```python + connection_state: str = "idle" + connection_error: Optional[str] = None +``` + +Ensure `Optional` is imported (it already is in that file). + +- [ ] **Step 4: Run, confirm pass** + +```bash +uv run pytest tests/unit/viewer/test_snapshot_incidents.py -v +``` +Expected: all pass. + +Regression check: + +```bash +uv run pytest tests/unit/viewer -v +``` + +If any test constructs `StreamSnapshot` positionally and breaks — fix inline by switching to keyword arguments in that test. Add a note to the commit message if so. + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/viewer/state.py tests/unit/viewer/test_snapshot_incidents.py +git commit -m "$(cat <<'EOF' +feat(viewer): add connection_state / connection_error to StreamSnapshot + +Defaults ('idle' / None) preserve existing behavior for any construction +that doesn't set them. Poller (next task) wires the orchestrator's per- +stream state into these fields. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 10 — Poller: propagate orchestrator state into snapshot + +**Files:** +- Modify: `src/syncfield/viewer/poller.py` +- Test: `tests/unit/viewer/test_poller.py` (extend) + +- [ ] **Step 1: Write failing test** + +Append to `tests/unit/viewer/test_poller.py` (use the existing test harness pattern — find how other tests construct a session and poller): + +```python +def test_poller_snapshot_includes_connection_state(tmp_path): + from syncfield.orchestrator import SessionOrchestrator + from syncfield.testing import FakeStream + from syncfield.viewer.poller import SessionPoller + + sess = SessionOrchestrator(host_id="h", output_dir=tmp_path) + sess.add(FakeStream("good")) + sess.add(FakeStream("bad", fail_on_start=True)) + + poller = SessionPoller(sess) + sess.connect() + + snap = poller.snapshot() + assert snap.streams["good"].connection_state == "connected" + assert snap.streams["good"].connection_error is None + assert snap.streams["bad"].connection_state == "failed" + assert snap.streams["bad"].connection_error # non-empty +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +uv run pytest tests/unit/viewer/test_poller.py::test_poller_snapshot_includes_connection_state -v +``` +Expected: `AssertionError` on `connection_state == "connected"` (current default is `"idle"` because the poller doesn't set it). + +- [ ] **Step 3: Implement** + +In `src/syncfield/viewer/poller.py`, locate the method that builds per-stream `StreamSnapshot` objects (look for `StreamSnapshot(` inside `_build_snapshot` or equivalent). Add the two new fields to the constructor call, pulling from the orchestrator: + +```python + connection_state=self._session._stream_states.get(stream.id, "idle"), + connection_error=self._session._stream_errors.get(stream.id), +``` + +If the poller holds the session under a different attribute name (e.g. `self._orchestrator`), use that — read the surrounding code. + +- [ ] **Step 4: Run, confirm pass** + +```bash +uv run pytest tests/unit/viewer/test_poller.py -v +``` +Expected: all pass. + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/viewer/poller.py tests/unit/viewer/test_poller.py +git commit -m "$(cat <<'EOF' +feat(viewer): poller reads orchestrator stream state into snapshot + +Every per-stream StreamSnapshot now carries connection_state and +connection_error, sourced from the orchestrator's per-stream dicts +populated by _set_stream_state. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 11 — Server: serialize new stream fields onto the wire + +**Files:** +- Modify: `src/syncfield/viewer/server.py` +- Test: `tests/unit/test_viewer_aggregation_snapshot.py` (extend) or `tests/unit/viewer/test_server_snapshot_serialization.py` (create if missing) + +- [ ] **Step 1: Locate the serializer** + +```bash +grep -n "snapshot_to_dict\|def _to_wire\|snapshot\.streams\|_serialize_stream" src/syncfield/viewer/server.py | head -10 +``` + +Record the function name + file location. Usually a helper that maps each `StreamSnapshot` to a JSON-friendly dict. + +- [ ] **Step 2: Write failing test** + +Append to `tests/unit/test_viewer_aggregation_snapshot.py` (it already has a `_make_snapshot_mock()` helper from the Task 20 refactor): + +```python +def test_serialized_stream_includes_connection_state(): + from syncfield.viewer.server import snapshot_to_dict # or whatever name + from syncfield.viewer.state import StreamSnapshot, SessionSnapshot + + stream_snap = StreamSnapshot( + id="cam", kind="video", provides_audio_track=False, produces_file=False, + frame_count=0, last_sample_at_ns=None, effective_hz=0.0, + latest_frame=None, plot_points={}, latest_pose={}, + connection_state="failed", connection_error="Device not visible", + ) + sess_snap = SessionSnapshot( + host_id="h", state="idle", output_dir="/tmp", + sync_point_monotonic_ns=None, sync_point_wall_clock_ns=None, + chirp_start_ns=None, chirp_stop_ns=None, chirp_enabled=False, + elapsed_s=0.0, streams={"cam": stream_snap}, + active_incidents=[], resolved_incidents=[], + ) + out = snapshot_to_dict(sess_snap) + assert out["streams"]["cam"]["connection_state"] == "failed" + assert out["streams"]["cam"]["connection_error"] == "Device not visible" +``` + +(Adapt `snapshot_to_dict` to the actual function name discovered in Step 1.) + +- [ ] **Step 3: Run, confirm fail** + +```bash +uv run pytest tests/unit/test_viewer_aggregation_snapshot.py -k connection_state -v +``` +Expected: `KeyError` or missing field. + +- [ ] **Step 4: Implement** + +In `src/syncfield/viewer/server.py`, in the per-stream serializer helper, add the two keys next to the existing `frame_count` / `effective_hz` keys: + +```python + "connection_state": stream.connection_state, + "connection_error": stream.connection_error, +``` + +- [ ] **Step 5: Run, confirm pass** + +```bash +uv run pytest tests/unit -q --timeout 30 +``` +Expected: all pass. + +- [ ] **Step 6: Commit** + +```bash +git add src/syncfield/viewer/server.py tests/unit/test_viewer_aggregation_snapshot.py +git commit -m "$(cat <<'EOF' +feat(viewer): emit connection_state / connection_error over WebSocket + +Frontend consumes these to select the right StreamCard overlay and to +populate the degraded-state header chip. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 12 — Frontend TypeScript types + +**Files:** +- Modify: `src/syncfield/viewer/frontend/src/lib/types.ts` + +- [ ] **Step 1: Add the ConnectionState type + fields** + +Add near the existing `Severity` type declaration: + +```ts +export type ConnectionState = + | "idle" + | "connecting" + | "connected" + | "failed" + | "disconnected"; +``` + +In the `StreamSnapshot` interface (the one mirroring the Python dataclass), add at the end: + +```ts + connection_state: ConnectionState; + connection_error: string | null; +``` + +- [ ] **Step 2: Typecheck** + +```bash +cd src/syncfield/viewer/frontend && npx tsc --noEmit +``` +Expected: clean typecheck (no errors unless downstream code is already missing a field — we fix those in Tasks 13-15). + +- [ ] **Step 3: Commit** + +```bash +git add src/syncfield/viewer/frontend/src/lib/types.ts +git commit -m "$(cat <<'EOF' +feat(viewer-fe): mirror ConnectionState + StreamSnapshot additions + +ConnectionState enum and two new fields on StreamSnapshot so the +overlay branching in StreamCard (next commit) is type-safe. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 13 — Frontend overlay components + +**Files:** +- Create: `src/syncfield/viewer/frontend/src/components/stream-overlays.tsx` + +- [ ] **Step 1: Create the component file** + +```tsx +import { useState } from "react"; + +export function ConnectingOverlay() { + return ( +
+
+ + Connecting… +
+
+ ); +} + +export function WaitingForDataOverlay() { + return ( +
+
+ Connected · waiting for first frame +
+
+ ); +} + +export function FailedOverlay({ error }: { error: string }) { + const [expanded, setExpanded] = useState(false); + return ( + + ); +} +``` + +- [ ] **Step 2: Typecheck + build** + +```bash +cd src/syncfield/viewer/frontend && npx tsc --noEmit && npm run build +``` +Expected: clean. + +- [ ] **Step 3: Commit** + +```bash +git add src/syncfield/viewer/frontend/src/components/stream-overlays.tsx \ + src/syncfield/viewer/frontend/src/lib/types.ts \ + src/syncfield/viewer/static/ # if the build emits updated bundle (skip if gitignored) +# Drop the static/ line above if the working tree shows no changes there. +git commit -m "$(cat <<'EOF' +feat(viewer-fe): add Connecting / WaitingForData / Failed overlays + +Three small presentational components used by StreamCard (next commit) +to replace the browser broken-image fallback. FailedOverlay is click- +to-expand for long error messages. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +Note: `src/syncfield/viewer/static/` is gitignored; the build output stays local. Do not force-add it. + +--- + +## Task 14 — StreamCard: branch body on `connection_state` + +**Files:** +- Modify: `src/syncfield/viewer/frontend/src/components/stream-card.tsx` + +- [ ] **Step 1: Import overlays** + +Add near the top of the file: + +```tsx +import { + ConnectingOverlay, + WaitingForDataOverlay, + FailedOverlay, +} from "./stream-overlays"; +``` + +- [ ] **Step 2: Insert body branching** + +Find the part of `StreamCard` that currently renders the video `` / `VideoPreview`. Wrap it so the branch decides what to render: + +```tsx +function StreamCardBody({ stream }: { stream: StreamSnapshot }) { + if (stream.connection_state === "connecting") { + return ; + } + if (stream.connection_state === "failed") { + return ; + } + if (stream.connection_state === "connected" && stream.frame_count === 0 && stream.kind === "video") { + return ; + } + // Existing render path (VideoPreview / SensorChart / AudioLevel / etc.) + return ; // adapt to real name +} +``` + +Replace the existing `` / `` usage inside `StreamCard` with ``. If `StreamCard` inlines the video element rather than delegating, extract the pre-existing body into a small helper named `ExistingStreamBody` first so the branch above works cleanly. + +- [ ] **Step 3: Typecheck + visual smoke test** + +```bash +cd src/syncfield/viewer/frontend && npx tsc --noEmit && npm run build +``` + +Manual smoke (optional if you have the dev server running): + +```bash +# terminal 1 +cd src/syncfield/viewer/frontend && npm run dev +# terminal 2 +python examples/mac_iphone_dual_oak/record.py +``` +Press Connect. Expected visual: +- `mac_webcam` / `iphone` / `host_audio` → normal body (video or audio chart) +- `oak_lite` / `oak_d` → red "Failed to connect" overlay with the depthai error message + +- [ ] **Step 4: Commit** + +```bash +git add src/syncfield/viewer/frontend/src/components/stream-card.tsx +git commit -m "$(cat <<'EOF' +feat(viewer-fe): branch StreamCard body on connection_state + +Connecting / waiting-for-first-frame / failed now render explicit +overlays instead of the browser's broken-image placeholder. Healthy +streams still render the existing video / sensor / audio body. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 15 — Header: degraded-state chip + +**Files:** +- Modify: `src/syncfield/viewer/frontend/src/components/header.tsx` + +- [ ] **Step 1: Locate the chip** + +```bash +grep -n "Ready\|SessionState\|state.*chip\|state.*label" src/syncfield/viewer/frontend/src/components/header.tsx | head +``` + +Find where the state label (`Ready` / `Recording` / …) is composed. + +- [ ] **Step 2: Add the counter logic** + +At the top of the component, after the snapshot prop is destructured: + +```tsx +const streams = Object.values(snapshot.streams); +const total = streams.length; +const connected = streams.filter((s) => s.connection_state === "connected").length; +const showCount = total > 0 && connected < total; +const label = showCount ? `${stateLabel} (${connected}/${total})` : stateLabel; +const chipTone = showCount ? "warning" : "normal"; +``` + +Use `chipTone` to choose the className. If the header currently has a Tailwind class like `bg-emerald-500/10`, add: + +```tsx +const toneClass = chipTone === "warning" + ? "bg-yellow-500/15 text-yellow-300 border border-yellow-500/40" + : "bg-emerald-500/10 text-emerald-300 border border-emerald-500/30"; +``` + +Apply `toneClass` to the chip's container element, and render `{label}` instead of `stateLabel` inside it. + +- [ ] **Step 3: Typecheck + build** + +```bash +cd src/syncfield/viewer/frontend && npx tsc --noEmit && npm run build +``` + +- [ ] **Step 4: Commit** + +```bash +git add src/syncfield/viewer/frontend/src/components/header.tsx +git commit -m "$(cat <<'EOF' +feat(viewer-fe): show degraded counter on header state chip + +'Ready (3/5)' in yellow when one or more streams are not in 'connected' +state. Normal emerald chip when everything is healthy. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 16 — Integration: partial-connect end-to-end + +**Files:** +- Create: `tests/integration/health/test_partial_connect.py` + +- [ ] **Step 1: Write the test** + +```python +"""End-to-end: real SessionOrchestrator + FakeStream mix survives one +stream failing to connect, and incidents.jsonl captures it.""" +from __future__ import annotations + +import json +import time +from pathlib import Path + +import pytest + +from syncfield.orchestrator import SessionOrchestrator +from syncfield.testing import FakeStream +from syncfield.types import SessionState + + +@pytest.mark.slow +def test_partial_connect_end_to_end(tmp_path: Path): + sess = SessionOrchestrator(host_id="h", output_dir=tmp_path) + sess.add(FakeStream("good_a")) + sess.add(FakeStream("bad", fail_on_start=True)) + sess.add(FakeStream("good_b")) + + sess.connect() + assert sess.state_name == SessionState.CONNECTED.value + + # Give the health worker a moment to ingest the startup-failure event. + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline: + if any(i.fingerprint == "bad:startup-failure" for i in sess.health.open_incidents()): + break + time.sleep(0.05) + + open_fps = [i.fingerprint for i in sess.health.open_incidents()] + assert "bad:startup-failure" in open_fps + + sess.start(countdown_s=0) + time.sleep(0.5) + sess.stop() + sess.disconnect() + + # incidents.jsonl should contain the startup-failure fingerprint. + out = list(tmp_path.rglob("incidents.jsonl")) + assert out, "no incidents.jsonl written" + lines = [json.loads(l) for l in out[0].read_text().strip().splitlines() if l] + fingerprints = {l["fingerprint"] for l in lines} + assert "bad:startup-failure" in fingerprints +``` + +- [ ] **Step 2: Run** + +```bash +uv run pytest tests/integration/health/test_partial_connect.py -v +``` +Expected: 1 passed. + +- [ ] **Step 3: Commit** + +```bash +git add tests/integration/health/test_partial_connect.py +git commit -m "$(cat <<'EOF' +test(health): integration test for partial connect + incident persistence + +Real SessionOrchestrator, mix of passing + failing FakeStreams; asserts +the session reaches CONNECTED, the failed stream surfaces as an open +startup-failure incident, and the fingerprint lands in incidents.jsonl. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 17 — Integration: NoDataDetector on a real orchestrator + +**Files:** +- Create: `tests/integration/health/test_no_data_detector.py` + +- [ ] **Step 1: Write the test** + +```python +"""End-to-end: a stream that connects but never emits a sample triggers +the no-data incident within the configured threshold.""" +from __future__ import annotations + +import threading +import time +from pathlib import Path + +import pytest + +from syncfield.orchestrator import SessionOrchestrator +from syncfield.stream import StreamBase +from syncfield.types import FinalizationReport, SampleEvent, StreamCapabilities + + +class SilentFakeStream(StreamBase): + """FakeStream variant that connects successfully but emits no samples until asked.""" + + def __init__(self, stream_id: str): + super().__init__(stream_id=stream_id, kind="sensor", capabilities=StreamCapabilities()) + self._stop = threading.Event() + self._gate = threading.Event() # held closed until tests allow flow + self._thread: threading.Thread | None = None + self._frame = 0 + + def connect(self): + self._stop.clear() + self._thread = threading.Thread(target=self._run, daemon=True) + self._thread.start() + + def disconnect(self): + self._stop.set() + if self._thread: + self._thread.join(timeout=1.0) + self._thread = None + + def start_recording(self, session_clock): + pass + + def stop_recording(self) -> FinalizationReport: + return FinalizationReport( + stream_id=self.id, status="completed", frame_count=self._frame, + file_path=None, first_sample_at_ns=0, last_sample_at_ns=0, + health_events=[], error=None, + ) + + def allow_samples(self): + self._gate.set() + + def _run(self): + while not self._stop.is_set(): + if self._gate.is_set(): + self._frame += 1 + self._emit_sample(SampleEvent( + stream_id=self.id, frame_number=self._frame, + capture_ns=time.monotonic_ns(), + )) + time.sleep(0.05) + + +@pytest.mark.slow +def test_no_data_incident_opens_then_closes_when_samples_arrive(tmp_path: Path): + sess = SessionOrchestrator(host_id="h", output_dir=tmp_path) + stream = SilentFakeStream("cam") + sess.add(stream) + + # Shrink threshold for the test via direct detector access. + for d in sess.health.iter_detectors(): + if d.name == "no-data": + d._threshold_ns = int(1e9) # 1s + break + + sess.connect() + + # After 1.5s, no-data incident should be open. + time.sleep(1.5) + open_fps = [i.fingerprint for i in sess.health.open_incidents()] + assert "cam:no-data" in open_fps + + # Let samples flow → incident closes within ~1 tick. + stream.allow_samples() + time.sleep(0.5) + + sess.stop() + sess.disconnect() + + resolved_fps = [i.fingerprint for i in sess.health.resolved_incidents()] + assert "cam:no-data" in resolved_fps +``` + +- [ ] **Step 2: Run** + +```bash +uv run pytest tests/integration/health/test_no_data_detector.py -v +``` +Expected: 1 passed (~2s wall clock). + +- [ ] **Step 3: Commit** + +```bash +git add tests/integration/health/test_no_data_detector.py +git commit -m "$(cat <<'EOF' +test(health): integration test for NoDataDetector on real orchestrator + +SilentFakeStream connects but withholds samples until the test opens +its gate. Asserts the no-data incident opens within 1.5s of connect +and closes within one tick of samples resuming. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 18 — Final regression sweep + push + +- [ ] **Step 1: Run the full suite** + +```bash +uv run pytest tests/unit tests/integration/health -q --timeout 60 +``` +Expected: ≥870 passing (baseline before these tasks was 867; this plan adds ~6 new tests). + +- [ ] **Step 2: Typecheck + build frontend** + +```bash +cd src/syncfield/viewer/frontend && npx tsc --noEmit && npm run build +``` +Expected: clean. + +- [ ] **Step 3: Push** + +```bash +git push +``` + +- [ ] **Step 4: Verify CI re-runs against the updated branch** + +```bash +gh pr checks 20 +``` + +Expected: CI kicks off; same pre-existing failures as before (3 Insta360 + audio + meta_quest tests) — this plan does not affect them. + +--- + +## Self-Review Checklist + +**Spec coverage (every spec requirement traces to a task):** + +- Partial connect semantics (§ Goals 1) → Task 7 +- Structured startup-failure event emission (§ Goals 2) → Task 7 (error path + success path) +- Per-stream ConnectionState + error in orchestrator (§ Data model) → Tasks 6, 8 +- `StreamSnapshot.connection_state` / `connection_error` (§ Data model) → Task 9 +- NoDataDetector (§ New detector) → Tasks 4, 5 +- `Detector.observe_connection_state` hook (§ Architecture) → Task 1 +- HealthWorker ingress queue (§ Architecture) → Task 2 +- HealthSystem passthrough (§ Architecture) → Task 3 +- Poller reads orchestrator state (§ Viewer changes) → Task 10 +- Server WS serialization (§ Viewer changes) → Task 11 +- Frontend ConnectionState + fields (§ Viewer changes) → Task 12 +- Overlay components (§ Viewer changes) → Task 13 +- StreamCard branch selection (§ Viewer changes) → Task 14 +- Header degraded chip (§ Viewer changes) → Task 15 +- Unit tests (§ Testing strategy) → Tasks 4, 7, 8, 9, 10, 11 +- Integration tests (§ Testing strategy) → Tasks 16, 17 + +**Type consistency:** `ConnectionState` strings `"idle" | "connecting" | "connected" | "failed" | "disconnected"` — same in Python, same in TypeScript, same in every test assertion. Fingerprints `{stream_id}:startup-failure` and `{stream_id}:startup-success` consistent across orchestrator + detector + tests + spec. + +**No placeholders:** every step shows a concrete code block or command. From d51059948fde9946853cbb9d05ff41cc39050e15 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 15:56:29 +0900 Subject: [PATCH 46/64] feat(health): add Detector.observe_connection_state hook Adds a no-op default for the per-stream connection-state observer that NoDataDetector will override. Protocol + base mirror the existing observe_sample / observe_health / observe_state shape. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/syncfield/health/detector.py | 4 ++++ tests/unit/health/test_detector_base.py | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/src/syncfield/health/detector.py b/src/syncfield/health/detector.py index 5116313..2350918 100644 --- a/src/syncfield/health/detector.py +++ b/src/syncfield/health/detector.py @@ -29,6 +29,7 @@ def observe_sample(self, stream_id: str, sample: SampleEvent) -> None: ... def observe_health(self, stream_id: str, event: HealthEvent) -> None: ... def observe_state(self, old: SessionState, new: SessionState) -> None: ... def observe_writer_stats(self, stream_id: str, stats: WriterStats) -> None: ... + def observe_connection_state(self, stream_id: str, new_state: str, at_ns: int) -> None: ... def tick(self, now_ns: int) -> Iterator[HealthEvent]: ... def close_condition(self, incident: Incident, now_ns: int) -> bool: ... @@ -79,6 +80,9 @@ def observe_state(self, old: SessionState, new: SessionState) -> None: def observe_writer_stats(self, stream_id: str, stats: WriterStats) -> None: pass + def observe_connection_state(self, stream_id: str, new_state: str, at_ns: int) -> None: + pass + def tick(self, now_ns: int) -> Iterator[HealthEvent]: return iter(()) diff --git a/tests/unit/health/test_detector_base.py b/tests/unit/health/test_detector_base.py index 4f4b5d2..8dfad16 100644 --- a/tests/unit/health/test_detector_base.py +++ b/tests/unit/health/test_detector_base.py @@ -44,3 +44,9 @@ class Grandchild(NoopDetector): # subclassing DetectorBase again (which must itself declare attrs). # The more important invariant is covered by the existing # test_detector_base_requires_name_and_severity. + + +def test_detector_base_observe_connection_state_default_is_noop(): + d = NoopDetector() + # Does not raise; returns None. + assert d.observe_connection_state("cam", "connected", 100) is None From 86a9774a59141b8d1ee43def6798f8d6b6803530 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 15:58:20 +0900 Subject: [PATCH 47/64] feat(health): add per-stream connection-state ingress to HealthWorker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New _ConnectionStateMsg + push_connection_state() + _drain_once fan-out so NoDataDetector (next commit) can track per-stream CONNECTING → CONNECTED transitions via the standard observer pattern. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/syncfield/health/worker.py | 14 ++++++++++++++ tests/unit/health/test_health_worker.py | 25 +++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/src/syncfield/health/worker.py b/src/syncfield/health/worker.py index 7c313cc..d58c62a 100644 --- a/src/syncfield/health/worker.py +++ b/src/syncfield/health/worker.py @@ -45,6 +45,13 @@ class _WriterStatsMsg: stats: WriterStats +@dataclass(frozen=True) +class _ConnectionStateMsg: + stream_id: str + new_state: str + at_ns: int + + class HealthWorker: def __init__( self, @@ -65,6 +72,7 @@ def __init__( self._healths: "queue.SimpleQueue[_HealthMsg]" = queue.SimpleQueue() self._states: "queue.SimpleQueue[_StateMsg]" = queue.SimpleQueue() self._writer_stats: "queue.SimpleQueue[_WriterStatsMsg]" = queue.SimpleQueue() + self._connection_states: "queue.SimpleQueue[_ConnectionStateMsg]" = queue.SimpleQueue() self._stop = threading.Event() self._thread: threading.Thread | None = None @@ -83,6 +91,9 @@ def push_state(self, old: SessionState, new: SessionState) -> None: def push_writer_stats(self, stream_id: str, stats: WriterStats) -> None: self._writer_stats.put(_WriterStatsMsg(stream_id, stats)) + def push_connection_state(self, stream_id: str, new_state: str, at_ns: int) -> None: + self._connection_states.put(_ConnectionStateMsg(stream_id, new_state, at_ns)) + # --- lifecycle ------------------------------------------------------- def start(self) -> None: @@ -139,6 +150,9 @@ def _drain_once(self) -> None: for msg in _drain_queue(self._writer_stats): for d in self._detectors: d.observe_writer_stats(msg.stream_id, msg.stats) + for msg in _drain_queue(self._connection_states): + for d in self._detectors: + d.observe_connection_state(msg.stream_id, msg.new_state, msg.at_ns) def _fire_detector_ticks(self) -> None: now = time.monotonic_ns() diff --git a/tests/unit/health/test_health_worker.py b/tests/unit/health/test_health_worker.py index 733e23a..b69150f 100644 --- a/tests/unit/health/test_health_worker.py +++ b/tests/unit/health/test_health_worker.py @@ -126,3 +126,28 @@ def test_worker_stop_is_idempotent(): w.start() w.stop() w.stop() # does not raise + + +def test_worker_drains_connection_state_queue_and_fans_out(): + class Spy(DetectorBase): + name = "conn-spy" + default_severity = Severity.INFO + + def __init__(self): + self.calls = [] + + def observe_connection_state(self, stream_id, new_state, at_ns): + self.calls.append((stream_id, new_state, at_ns)) + + tr = IncidentTracker() + spy = Spy() + w = HealthWorker(tracker=tr, detectors=[spy], tick_hz=100) + w.start() + try: + w.push_connection_state("cam", "connecting", 1) + w.push_connection_state("cam", "connected", 2) + assert _wait_until(lambda: len(spy.calls) == 2) + finally: + w.stop() + + assert spy.calls == [("cam", "connecting", 1), ("cam", "connected", 2)] From 1ea763df9d7fe9dc370899bcf24f7a85c794d93d Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 16:00:04 +0900 Subject: [PATCH 48/64] feat(health): expose observe_connection_state on HealthSystem Passthrough to the worker's ingress queue, matching the existing observe_sample / observe_health / observe_state shape. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/syncfield/health/system.py | 4 ++++ tests/unit/health/test_health_system.py | 28 +++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/src/syncfield/health/system.py b/src/syncfield/health/system.py index a0fb9c4..e03968c 100644 --- a/src/syncfield/health/system.py +++ b/src/syncfield/health/system.py @@ -88,6 +88,10 @@ def observe_writer_stats(self, stream_id: str, stats: WriterStats) -> None: if self._worker is not None: self._worker.push_writer_stats(stream_id, stats) + def observe_connection_state(self, stream_id: str, new_state: str, at_ns: int) -> None: + if self._worker is not None: + self._worker.push_connection_state(stream_id, new_state, at_ns) + # --- lifecycle ------------------------------------------------------- def start(self) -> None: diff --git a/tests/unit/health/test_health_system.py b/tests/unit/health/test_health_system.py index e8606ff..0f09dec 100644 --- a/tests/unit/health/test_health_system.py +++ b/tests/unit/health/test_health_system.py @@ -108,3 +108,31 @@ def test_register_stream_propagates_target_hz_to_detectors(): assert fps._target_getter("cam") == 30.0 assert jitter._target_getter("cam") == 30.0 assert fps._target_getter("unknown") is None + + +def test_health_system_observe_connection_state_routes_to_worker(): + import time as _time + class Spy(DetectorBase): + name = "conn-spy" + default_severity = Severity.INFO + + def __init__(self): + self.calls = [] + + def observe_connection_state(self, stream_id, new_state, at_ns): + self.calls.append((stream_id, new_state, at_ns)) + + hs = HealthSystem() + spy = Spy() + hs.register(spy) + + hs.start() + try: + hs.observe_connection_state("cam", "connecting", 10) + hs.observe_connection_state("cam", "connected", 20) + deadline = _time.monotonic() + 2.0 + while _time.monotonic() < deadline and len(spy.calls) < 2: + _time.sleep(0.02) + finally: + hs.stop() + assert spy.calls == [("cam", "connecting", 10), ("cam", "connected", 20)] From f64d8c618521a4160b2b544fa215b49fc0ae80c9 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 16:02:06 +0900 Subject: [PATCH 49/64] feat(health): add NoDataDetector for connected-but-silent streams Fires when a stream is in 'connected' state for N seconds without any sample. Complements StreamStallDetector which requires prior samples. Catches the OAK "black square" symptom where connect() succeeds but the pipeline never pumps frames. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/syncfield/health/detectors/no_data.py | 65 ++++++++++++++++++++ tests/unit/health/detectors/test_no_data.py | 68 +++++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 src/syncfield/health/detectors/no_data.py create mode 100644 tests/unit/health/detectors/test_no_data.py diff --git a/src/syncfield/health/detectors/no_data.py b/src/syncfield/health/detectors/no_data.py new file mode 100644 index 0000000..fdc3a08 --- /dev/null +++ b/src/syncfield/health/detectors/no_data.py @@ -0,0 +1,65 @@ +"""NoDataDetector — fires when a stream is connected but never emits a sample. + +Complements StreamStallDetector (which requires prior samples). Catches +the "connected but silent" case such as an OAK pipeline that fails to +pump frames even though its device connected. Resets bookkeeping on +any non-connected state transition so a reconnect starts a fresh clock. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from typing import Dict, List, Set + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.types import Incident +from syncfield.types import HealthEvent, HealthEventKind, SampleEvent + + +class NoDataDetector(DetectorBase): + name = "no-data" + default_severity = Severity.ERROR + + def __init__(self, threshold_ns: int = 5_000_000_000) -> None: + self._threshold_ns = threshold_ns + self._connected_at: Dict[str, int] = {} + self._has_sample: Set[str] = set() + self._fire_active: Dict[str, bool] = {} + + def observe_connection_state(self, stream_id: str, new_state: str, at_ns: int) -> None: + if new_state == "connected": + self._connected_at[stream_id] = at_ns + self._has_sample.discard(stream_id) + self._fire_active[stream_id] = False + else: + # idle / connecting / failed / disconnected → reset everything. + self._connected_at.pop(stream_id, None) + self._has_sample.discard(stream_id) + self._fire_active.pop(stream_id, None) + + def observe_sample(self, stream_id: str, sample: SampleEvent) -> None: + self._has_sample.add(stream_id) + + def tick(self, now_ns: int) -> Iterator[HealthEvent]: + out: List[HealthEvent] = [] + for stream_id, connected_at in self._connected_at.items(): + if stream_id in self._has_sample: + continue + elapsed = now_ns - connected_at + if elapsed >= self._threshold_ns and not self._fire_active.get(stream_id, False): + self._fire_active[stream_id] = True + out.append(HealthEvent( + stream_id=stream_id, + kind=HealthEventKind.ERROR, + at_ns=now_ns, + detail=f"Connected {elapsed / 1e9:.1f}s ago but no data received", + severity=self.default_severity, + source=f"detector:{self.name}", + fingerprint=f"{stream_id}:{self.name}", + data={"connected_at_ns": connected_at, "elapsed_ns": elapsed}, + )) + return iter(out) + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + return incident.stream_id in self._has_sample diff --git a/tests/unit/health/detectors/test_no_data.py b/tests/unit/health/detectors/test_no_data.py new file mode 100644 index 0000000..b8e44bf --- /dev/null +++ b/tests/unit/health/detectors/test_no_data.py @@ -0,0 +1,68 @@ +from syncfield.health.detectors.no_data import NoDataDetector +from syncfield.health.types import Incident +from syncfield.types import SampleEvent + + +def _s(stream: str, t_ns: int) -> SampleEvent: + return SampleEvent(stream_id=stream, frame_number=0, capture_ns=t_ns) + + +def test_no_fire_before_threshold(): + d = NoDataDetector(threshold_ns=1000) + d.observe_connection_state("cam", "connected", at_ns=100) + assert list(d.tick(now_ns=500)) == [] # 400 ns elapsed, under 1000 + + +def test_fires_after_threshold_without_sample(): + d = NoDataDetector(threshold_ns=1000) + d.observe_connection_state("cam", "connected", at_ns=100) + out = list(d.tick(now_ns=2000)) # 1900 ns elapsed + assert len(out) == 1 + ev = out[0] + assert ev.stream_id == "cam" + assert ev.fingerprint == "cam:no-data" + assert ev.source == "detector:no-data" + assert "no data" in (ev.detail or "").lower() + + +def test_does_not_refire_while_still_no_data(): + d = NoDataDetector(threshold_ns=1000) + d.observe_connection_state("cam", "connected", at_ns=100) + first = list(d.tick(now_ns=2000)) + second = list(d.tick(now_ns=3000)) + assert len(first) == 1 + assert len(second) == 0 + + +def test_close_condition_satisfied_once_sample_arrives(): + d = NoDataDetector(threshold_ns=1000) + d.observe_connection_state("cam", "connected", at_ns=100) + events = list(d.tick(now_ns=2000)) + inc = Incident.opened_from(events[0], title="x") + + assert d.close_condition(inc, now_ns=2100) is False # still no sample + + d.observe_sample("cam", _s("cam", 2200)) + assert d.close_condition(inc, now_ns=2300) is True + + +def test_resets_bookkeeping_on_non_connected_state(): + d = NoDataDetector(threshold_ns=1000) + d.observe_connection_state("cam", "connected", at_ns=100) + list(d.tick(now_ns=2000)) # fires + + d.observe_connection_state("cam", "failed", at_ns=2500) + # Back to connected → fresh clock, no duplicate fire. + d.observe_connection_state("cam", "connected", at_ns=3000) + assert list(d.tick(now_ns=3500)) == [] # only 500 ns since new connected + + +def test_per_stream_independent_state(): + d = NoDataDetector(threshold_ns=1000) + d.observe_connection_state("a", "connected", at_ns=100) + d.observe_connection_state("b", "connected", at_ns=100) + d.observe_sample("b", _s("b", 200)) + + out = list(d.tick(now_ns=2000)) + assert len(out) == 1 + assert out[0].stream_id == "a" From 4b7b7901c5e3c2935bece025eabfa06b4faa8e5a Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 16:03:33 +0900 Subject: [PATCH 50/64] feat(health): register NoDataDetector in default suite Now installed automatically on HealthSystem construction alongside the other six default detectors. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/syncfield/health/system.py | 2 ++ tests/unit/health/test_health_system.py | 1 + 2 files changed, 3 insertions(+) diff --git a/src/syncfield/health/system.py b/src/syncfield/health/system.py index e03968c..2a4182e 100644 --- a/src/syncfield/health/system.py +++ b/src/syncfield/health/system.py @@ -9,6 +9,7 @@ from syncfield.health.detectors.backpressure import BackpressureDetector from syncfield.health.detectors.fps_drop import FpsDropDetector from syncfield.health.detectors.jitter import JitterDetector +from syncfield.health.detectors.no_data import NoDataDetector from syncfield.health.detectors.startup_failure import StartupFailureDetector from syncfield.health.detectors.stream_stall import StreamStallDetector from syncfield.health.registry import DetectorRegistry @@ -134,3 +135,4 @@ def _install_default_detectors(self) -> None: self.register(JitterDetector(target_getter=target_getter)) self.register(StartupFailureDetector()) self.register(BackpressureDetector()) + self.register(NoDataDetector()) diff --git a/tests/unit/health/test_health_system.py b/tests/unit/health/test_health_system.py index 0f09dec..9e1ed21 100644 --- a/tests/unit/health/test_health_system.py +++ b/tests/unit/health/test_health_system.py @@ -43,6 +43,7 @@ def test_health_system_installs_default_detectors(): "jitter", "startup-failure", "backpressure", + "no-data", ): assert expected in names, f"missing default detector: {expected}" From 84236bb9674330f23da27f6c17fc515709c5e0ab Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 16:05:46 +0900 Subject: [PATCH 51/64] feat(orchestrator): add per-stream connection-state dicts + helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _stream_states and _stream_errors become the source of truth for per-stream status. _set_stream_state is the single update point — it also forwards to HealthSystem.observe_connection_state so NoDataDetector sees the transitions. add() initializes each new stream to 'idle'. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/syncfield/orchestrator.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/syncfield/orchestrator.py b/src/syncfield/orchestrator.py index 4f3a941..fd3952c 100644 --- a/src/syncfield/orchestrator.py +++ b/src/syncfield/orchestrator.py @@ -359,6 +359,13 @@ def __init__( # opened a device. self._connected_streams: List[Stream] = [] + # Per-stream connection state for partial-connect semantics. + # Keys are stream ids; values are one of: + # "idle" | "connecting" | "connected" | "failed" | "disconnected". + self._stream_states: dict[str, str] = {} + # Populated only when a stream's connect() raised. + self._stream_errors: dict[str, str] = {} + # Auto-injected host audio stream (if any). Tracked so it can # be removed on disconnect. self._auto_audio_stream: Optional[Stream] = None @@ -1525,6 +1532,8 @@ def add(self, stream: Stream) -> None: if self._auto_audio_stream is None and not stream.capabilities.provides_audio_track: self._maybe_preregister_host_audio() + self._set_stream_state(stream.id, "idle") + def remove(self, stream_id: str) -> None: """Unregister a previously added stream. @@ -2442,6 +2451,16 @@ def _persist_incident(self, incident) -> None: # Never let telemetry persistence crash the recording. pass + def _set_stream_state(self, stream_id: str, new_state: str) -> None: + """Update per-stream connection state and forward to HealthSystem. + + The health worker may not be running yet (e.g. this is called + from add() when the session is IDLE) — observe_connection_state + is a no-op in that case. + """ + self._stream_states[stream_id] = new_state + self.health.observe_connection_state(stream_id, new_state, time.monotonic_ns()) + # ------------------------------------------------------------------ # Episode lifecycle # ------------------------------------------------------------------ From 8cf253a8435462f596f68813942b52a4b1fe37b5 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 16:12:48 +0900 Subject: [PATCH 52/64] =?UTF-8?q?feat(orchestrator):=20partial-connect=20?= =?UTF-8?q?=E2=80=94=20survive=20per-stream=20connect()=20failure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One stream raising no longer rolls the whole session back to IDLE. We record the error, emit a structured HealthEvent (phase=connect, outcome=error) that activates the previously-dormant StartupFailureDetector, and continue with the rest. Session only fails if every adapter raises. Success path emits the complementary outcome=success signal so the detector's recovery tracking works. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/syncfield/orchestrator.py | 49 +++++++++--- src/syncfield/testing.py | 4 + .../unit/test_orchestrator_partial_connect.py | 80 +++++++++++++++++++ 3 files changed, 122 insertions(+), 11 deletions(-) create mode 100644 tests/unit/test_orchestrator_partial_connect.py diff --git a/src/syncfield/orchestrator.py b/src/syncfield/orchestrator.py index fd3952c..f66c771 100644 --- a/src/syncfield/orchestrator.py +++ b/src/syncfield/orchestrator.py @@ -80,6 +80,7 @@ from typing import Any, Callable, Dict, List, Optional, Union from syncfield.clock import SessionClock +from syncfield.health.severity import Severity from syncfield.multihost.advertiser import SessionAdvertiser from syncfield.multihost.browser import SessionBrowser from syncfield.multihost.types import SessionAnnouncement @@ -1628,28 +1629,54 @@ def connect(self) -> None: self._transition(SessionState.CONNECTING) + # Start the health worker early so events emitted during the + # connect loop (including per-stream failure events) reach + # registered detectors. Idempotent — safe to call again below. + self.health.start() + connected: List[Stream] = [] - try: - for stream in self._streams.values(): + for stream in self._streams.values(): + self._set_stream_state(stream.id, "connecting") + try: stream.prepare() stream.connect() - connected.append(stream) - # Emit a health event so the viewer's Health Events - # panel confirms each device connected successfully. + except Exception as exc: + self._stream_errors[stream.id] = str(exc) + self._set_stream_state(stream.id, "failed") stream._emit_health(HealthEvent( stream_id=stream.id, - kind=HealthEventKind.HEARTBEAT, + kind=HealthEventKind.ERROR, at_ns=time.monotonic_ns(), - detail="connected", + detail=str(exc), + severity=Severity.ERROR, + source="orchestrator", + fingerprint=f"{stream.id}:startup-failure", + data={"phase": "connect", "outcome": "error", "error": str(exc)}, )) - except Exception as exc: - self._log_rollback(exc, len(connected)) - _rollback_disconnect_streams(connected) + continue + connected.append(stream) + self._stream_errors.pop(stream.id, None) + self._set_stream_state(stream.id, "connected") + stream._emit_health(HealthEvent( + stream_id=stream.id, + kind=HealthEventKind.HEARTBEAT, + at_ns=time.monotonic_ns(), + detail="connected", + severity=Severity.INFO, + source="orchestrator", + fingerprint=f"{stream.id}:startup-success", + data={"phase": "connect", "outcome": "success"}, + )) + + if not connected: self._transition(SessionState.IDLE) if self._log_writer is not None: self._log_writer.close() self._log_writer = None - raise + raise RuntimeError( + "connect() failed: no streams connected — every adapter raised. " + "Inspect per-stream errors via session._stream_errors." + ) self._connected_streams = connected diff --git a/src/syncfield/testing.py b/src/syncfield/testing.py index c1c9dce..914be36 100644 --- a/src/syncfield/testing.py +++ b/src/syncfield/testing.py @@ -85,6 +85,10 @@ def prepare(self) -> None: if self._fail_on_prepare: raise RuntimeError("fake failure in prepare") + def connect(self) -> None: + if self._fail_on_start: + raise RuntimeError("fake failure in connect") + def start(self, session_clock: SessionClock) -> None: self.start_calls += 1 if self._fail_on_start: diff --git a/tests/unit/test_orchestrator_partial_connect.py b/tests/unit/test_orchestrator_partial_connect.py new file mode 100644 index 0000000..eb494da --- /dev/null +++ b/tests/unit/test_orchestrator_partial_connect.py @@ -0,0 +1,80 @@ +"""Partial-connect semantics for SessionOrchestrator. + +Relies on the FakeStream helper in syncfield.testing, which supports +`fail_on_start=True` to raise from its connect() path. +""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +from syncfield.orchestrator import SessionOrchestrator +from syncfield.testing import FakeStream +from syncfield.types import SessionState + + +def test_one_stream_fails_others_still_connected(tmp_path: Path): + sess = SessionOrchestrator(host_id="h", output_dir=tmp_path) + sess.add(FakeStream("good_a")) + sess.add(FakeStream("bad", fail_on_start=True)) + sess.add(FakeStream("good_b")) + + sess.connect() + + assert sess.state is SessionState.CONNECTED + assert sess._stream_states["good_a"] == "connected" + assert sess._stream_states["bad"] == "failed" + assert sess._stream_states["good_b"] == "connected" + assert "bad" in sess._stream_errors + assert sess._stream_errors["bad"] + + +def test_all_streams_failing_raises_and_returns_to_idle(tmp_path: Path): + sess = SessionOrchestrator(host_id="h", output_dir=tmp_path) + sess.add(FakeStream("a", fail_on_start=True)) + sess.add(FakeStream("b", fail_on_start=True)) + + with pytest.raises(RuntimeError, match="no streams"): + sess.connect() + + assert sess.state is SessionState.IDLE + assert sess._stream_states["a"] == "failed" + assert sess._stream_states["b"] == "failed" + + +def test_startup_failure_event_reaches_health_system(tmp_path: Path): + from syncfield.health.detector import DetectorBase + from syncfield.health.severity import Severity + + class Spy(DetectorBase): + name = "startup-spy" + default_severity = Severity.INFO + + def __init__(self): + self.events = [] + + def observe_health(self, stream_id, event): + self.events.append(event) + + sess = SessionOrchestrator(host_id="h", output_dir=tmp_path) + spy = Spy() + sess.health.register(spy) + sess.add(FakeStream("good")) + sess.add(FakeStream("bad", fail_on_start=True)) + + sess.connect() + + import time + deadline = time.monotonic() + 1.0 + while time.monotonic() < deadline and not any( + e.fingerprint == "bad:startup-failure" for e in spy.events + ): + time.sleep(0.02) + + failure_events = [e for e in spy.events if e.fingerprint == "bad:startup-failure"] + assert failure_events, "no startup-failure event observed" + ev = failure_events[0] + assert ev.data.get("phase") == "connect" + assert ev.data.get("outcome") == "error" + assert ev.data.get("error") From ae671b25e19b8662e12ff88ee1681e064c81bff3 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 16:15:45 +0900 Subject: [PATCH 53/64] test(orchestrator): update pre-existing tests for partial-connect semantics Tests in TestStartRollback that encoded the old all-or-nothing rollback are now inverted: a single failing stream is isolated, not session-ending. Assertions updated to reflect that the failed stream goes to 'failed' state while the rest proceed to recording. Co-Authored-By: Claude Opus 4.7 (1M context) --- tests/unit/test_orchestrator.py | 41 ++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/tests/unit/test_orchestrator.py b/tests/unit/test_orchestrator.py index 0f40bb9..c7987fb 100644 --- a/tests/unit/test_orchestrator.py +++ b/tests/unit/test_orchestrator.py @@ -322,16 +322,17 @@ def test_failure_during_start_rolls_back_prior_streams(self, tmp_path): assert session.state is SessionState.IDLE - def test_failure_during_prepare_stops_earlier_streams(self, tmp_path): - """A failure in ``prepare()`` happens during the connect phase, - which runs all preparations before any stream starts recording. - The rollback therefore calls ``disconnect()`` on streams that - connected — and ``start()`` is never reached on any of them. - - This differs from the 0.1 behaviour where ``prepare()`` and - ``start()`` interleaved per stream; the 0.2 orchestrator splits - the two phases so all devices connect before any begin writing, - matching the egonaut lab recorder's 2-phase model. + def test_failure_during_prepare_isolated_to_failing_stream(self, tmp_path): + """A prepare() failure in the connect phase is now isolated to the + failing stream — partial-connect semantics. + + The failing stream is marked ``"failed"`` with an entry in + ``_stream_errors``. Streams that connected successfully proceed to + recording, and ``start()`` does not raise. + + The old all-or-nothing rollback behaviour (every stream rolled back + to IDLE when any stream failed) was replaced in the partial-connect + refactor so a single bad camera no longer blocks the entire session. """ session = _session(tmp_path) good = FakeStream("a") @@ -339,17 +340,21 @@ def test_failure_during_prepare_stops_earlier_streams(self, tmp_path): session.add(good) session.add(bad) - with pytest.raises(RuntimeError, match="fake failure in prepare"): - session.start() + # start() must NOT raise — the session recovers with one fewer stream + session.start() - # prepare() ran on both in the connect phase + # Both streams entered the connect phase assert good.prepare_calls == 1 assert bad.prepare_calls == 1 - # start_recording() was never invoked because the connect phase failed - assert good.start_calls == 0 - assert bad.start_calls == 0 - # Rollback returned the auto-connected session to IDLE - assert session.state is SessionState.IDLE + # The failing stream is isolated: marked failed, error recorded + assert session._stream_states["b"] == "failed" + assert "b" in session._stream_errors + assert session._stream_errors["b"] + # The healthy stream connected and was started + assert session._stream_states["a"] == "connected" + assert good.start_calls > 0 + # The session is recording with the surviving stream + assert session.state is SessionState.RECORDING class TestFourPhaseLifecycle: From f63161c0c3a7c2f9cf86f52b0c84e7a1b2e9d462 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 16:23:24 +0900 Subject: [PATCH 54/64] feat(orchestrator): skip failed streams in start/stop recording loops MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After partial-connect, failed streams have no open hardware. Iterating them in start_recording / stop_recording would call lifecycle methods on devices that aren't open — at best a no-op, at worst a cascading error. Switch those loops to iterate self._connected_streams (set by connect() to only the successful streams). Co-Authored-By: Claude Opus 4.7 (1M context) --- src/syncfield/orchestrator.py | 8 ++-- tests/unit/test_orchestrator.py | 37 +++++++++++++------ .../test_orchestrator_config_distribution.py | 3 ++ .../unit/test_orchestrator_partial_connect.py | 35 ++++++++++++++++++ 4 files changed, 68 insertions(+), 15 deletions(-) diff --git a/src/syncfield/orchestrator.py b/src/syncfield/orchestrator.py index f66c771..f84d5e3 100644 --- a/src/syncfield/orchestrator.py +++ b/src/syncfield/orchestrator.py @@ -1326,7 +1326,7 @@ def _rollback_after_distribute_failure(self) -> None: # 1. Stop each stream that's currently recording. Must happen # BEFORE closing writers so any in-flight samples are # flushed before the file handles go away. - recording_streams = list(self._streams.values()) + recording_streams = list(self._connected_streams) _rollback_stop_recording(recording_streams) # 2. Close sample writers — matches the happy stop() flow so @@ -1866,7 +1866,7 @@ def _tick_with_beep(n: int) -> None: recording: List[Stream] = [] try: - for stream in self._streams.values(): + for stream in self._connected_streams: stream.start_recording(self._session_clock) recording.append(stream) except Exception as exc: @@ -2166,7 +2166,7 @@ def _finalize_streams(self) -> List[FinalizationReport]: before moving on to the next. """ finalizations: List[FinalizationReport] = [] - for stream in self._streams.values(): + for stream in self._connected_streams: try: report = stream.stop_recording() except Exception as exc: @@ -2213,7 +2213,7 @@ def _open_sample_writers(self) -> None: capture thread become no-ops instead of writing to a closed writer. """ - for stream in self._streams.values(): + for stream in self._connected_streams: writer: SampleWriter if stream.kind == "sensor": writer = SensorWriter(stream.id, self._output_dir) diff --git a/tests/unit/test_orchestrator.py b/tests/unit/test_orchestrator.py index c7987fb..d2fa053 100644 --- a/tests/unit/test_orchestrator.py +++ b/tests/unit/test_orchestrator.py @@ -299,6 +299,11 @@ def start(self, session_clock): # type: ignore[override] class TestStartRollback: def test_failure_during_start_rolls_back_prior_streams(self, tmp_path): + """Under partial-connect semantics a stream with fail_on_start=True + fails at connect() time — it is never added to _connected_streams and + is therefore skipped during start_recording(). The two healthy streams + both connect and start successfully, so start() no longer raises. + """ session = _session(tmp_path) good1 = FakeStream("a") bad = FakeStream("b", fail_on_start=True) @@ -307,20 +312,21 @@ def test_failure_during_start_rolls_back_prior_streams(self, tmp_path): session.add(bad) session.add(good2) - with pytest.raises(RuntimeError, match="fake failure in start"): - session.start() + # start() must NOT raise — partial-connect isolated the failure + session.start() - # good1 was started → must be rolled back (stop called) + # good1 and good2 connected and started assert good1.start_calls == 1 - assert good1.stop_calls == 1 - # bad raised during start → stop should NOT be called on it - assert bad.start_calls == 1 + assert good2.start_calls == 1 + # bad failed during connect() — start_recording was never called on it + assert bad.start_calls == 0 assert bad.stop_calls == 0 - # good2 never reached start - assert good2.start_calls == 0 - assert good2.stop_calls == 0 - assert session.state is SessionState.IDLE + assert session.state is SessionState.RECORDING + + session.stop() + # Auto-connect path lands in STOPPED (not IDLE) after stop() + assert session.state is SessionState.STOPPED def test_failure_during_prepare_isolated_to_failing_stream(self, tmp_path): """A prepare() failure in the connect phase is now isolated to the @@ -1361,9 +1367,18 @@ def test_session_log_flushes_during_recording(self, tmp_path): session.stop() def test_rollback_is_logged(self, tmp_path): + """A stream that connects successfully but raises during start_recording() + must cause a rollback log entry. Under partial-connect semantics + fail_on_start=True causes connect() to fail (not start_recording()), so + we use a custom subclass that overrides start_recording() directly. + """ + class FailOnStartRecording(FakeStream): + def start_recording(self, session_clock): + raise RuntimeError("fake failure in start_recording") + session = _session(tmp_path) session.add(FakeStream("a")) - session.add(FakeStream("b", fail_on_start=True)) + session.add(FailOnStartRecording("b")) # Capture the episode dir before start() — rollback happens inside # start() and _prepare_next_episode() rotates output_dir in that path. episode_dir = session.output_dir diff --git a/tests/unit/test_orchestrator_config_distribution.py b/tests/unit/test_orchestrator_config_distribution.py index 066fd47..a9e243c 100644 --- a/tests/unit/test_orchestrator_config_distribution.py +++ b/tests/unit/test_orchestrator_config_distribution.py @@ -272,6 +272,9 @@ def test_stops_recording_streams(self, tmp_path) -> None: mic = FakeStream("mic", kind="audio") session.add(cam) session.add(mic) + # Simulate streams that connected successfully — rollback only + # operates on _connected_streams, not the full _streams registry. + session._connected_streams = [cam, mic] session._state = SessionState.RECORDING session._rollback_after_distribute_failure() diff --git a/tests/unit/test_orchestrator_partial_connect.py b/tests/unit/test_orchestrator_partial_connect.py index eb494da..d5b2d82 100644 --- a/tests/unit/test_orchestrator_partial_connect.py +++ b/tests/unit/test_orchestrator_partial_connect.py @@ -78,3 +78,38 @@ def observe_health(self, stream_id, event): assert ev.data.get("phase") == "connect" assert ev.data.get("outcome") == "error" assert ev.data.get("error") + + +def test_failed_stream_is_skipped_during_recording(tmp_path: Path): + from syncfield.testing import FakeStream + + class CountingFakeStream(FakeStream): + def __init__(self, stream_id, fail_on_start=False): + super().__init__(stream_id, fail_on_start=fail_on_start) + self.start_recording_calls = 0 + self.stop_recording_calls = 0 + + def start_recording(self, session_clock): + self.start_recording_calls += 1 + return super().start_recording(session_clock) + + def stop_recording(self): + self.stop_recording_calls += 1 + return super().stop_recording() + + sess = SessionOrchestrator(host_id="h", output_dir=tmp_path) + good = CountingFakeStream("good") + bad = CountingFakeStream("bad", fail_on_start=True) + sess.add(good) + sess.add(bad) + + sess.connect() + sess.start(countdown_s=0) + sess.stop() + + assert good.start_recording_calls > 0 + assert good.stop_recording_calls > 0 + # The failed stream's hardware was never opened, so start/stop_recording + # must not have been called on it. + assert bad.start_recording_calls == 0 + assert bad.stop_recording_calls == 0 From 87734e1dd67d3d3645e5dda6eaa13b439512b3bb Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 16:28:44 +0900 Subject: [PATCH 55/64] feat(orchestrator): disconnect() flips all tracked streams to 'disconnected' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _connected_streams only ever held successful streams, so the existing rollback helper is already correct — but every stream the session knows about (including previously-failed ones) should have its snapshot state flipped to 'disconnected' and its error entry cleared so the viewer renders a clean slate after teardown. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/syncfield/orchestrator.py | 8 ++++++ .../unit/test_orchestrator_partial_connect.py | 28 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/src/syncfield/orchestrator.py b/src/syncfield/orchestrator.py index f84d5e3..5378565 100644 --- a/src/syncfield/orchestrator.py +++ b/src/syncfield/orchestrator.py @@ -2142,6 +2142,14 @@ def disconnect(self) -> None: _rollback_disconnect_streams(self._connected_streams) self._connected_streams = [] + # Flip every tracked stream's state to 'disconnected' so the + # viewer reflects a clean slate — this includes streams that + # were 'failed' (they never opened hardware, but their UI tile + # should no longer display a red error overlay after teardown). + for stream_id in list(self._stream_states.keys()): + self._set_stream_state(stream_id, "disconnected") + self._stream_errors.clear() + # Keep auto-injected audio stream registered (visible in viewer) # but disconnected. It will be reconnected on next connect(). diff --git a/tests/unit/test_orchestrator_partial_connect.py b/tests/unit/test_orchestrator_partial_connect.py index d5b2d82..9d8e9cc 100644 --- a/tests/unit/test_orchestrator_partial_connect.py +++ b/tests/unit/test_orchestrator_partial_connect.py @@ -113,3 +113,31 @@ def stop_recording(self): # must not have been called on it. assert bad.start_recording_calls == 0 assert bad.stop_recording_calls == 0 + + +def test_disconnect_does_not_call_stream_disconnect_on_failed(tmp_path: Path): + from syncfield.testing import FakeStream + + class CountingFakeStream(FakeStream): + def __init__(self, stream_id, fail_on_start=False): + super().__init__(stream_id, fail_on_start=fail_on_start) + self.disconnect_calls = 0 + + def disconnect(self): + self.disconnect_calls += 1 + super().disconnect() + + sess = SessionOrchestrator(host_id="h", output_dir=tmp_path) + good = CountingFakeStream("good") + bad = CountingFakeStream("bad", fail_on_start=True) + sess.add(good) + sess.add(bad) + + sess.connect() + sess.disconnect() + + assert good.disconnect_calls == 1 + assert bad.disconnect_calls == 0 + assert sess._stream_states["good"] == "disconnected" + assert sess._stream_states["bad"] == "disconnected" + assert sess._stream_errors == {} From 82d1b25bb8e24173775cee6d1a113e26907ffc83 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 16:29:39 +0900 Subject: [PATCH 56/64] feat(viewer): add connection_state / connection_error to StreamSnapshot Defaults ('idle' / None) preserve existing behavior for any construction that doesn't set them. Poller (next task) wires the orchestrator's per- stream state into these fields. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/syncfield/viewer/state.py | 2 ++ tests/unit/viewer/test_snapshot_incidents.py | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/src/syncfield/viewer/state.py b/src/syncfield/viewer/state.py index 81ed64d..e18add1 100644 --- a/src/syncfield/viewer/state.py +++ b/src/syncfield/viewer/state.py @@ -88,6 +88,8 @@ class StreamSnapshot: plot_points: Dict[str, Tuple[List[float], List[float]]] latest_pose: Dict[str, List[float]] live_preview: bool = True + connection_state: str = "idle" + connection_error: Optional[str] = None # --------------------------------------------------------------------------- diff --git a/tests/unit/viewer/test_snapshot_incidents.py b/tests/unit/viewer/test_snapshot_incidents.py index c37c4cd..bd65191 100644 --- a/tests/unit/viewer/test_snapshot_incidents.py +++ b/tests/unit/viewer/test_snapshot_incidents.py @@ -34,3 +34,20 @@ def test_stream_snapshot_no_longer_has_health_count(): def test_session_snapshot_no_longer_has_health_log(): fields = {f.name for f in dataclasses.fields(SessionSnapshot)} assert "health_log" not in fields + + +def test_stream_snapshot_has_connection_state_fields(): + import dataclasses + from syncfield.viewer.state import StreamSnapshot + + fields = {f.name: f for f in dataclasses.fields(StreamSnapshot)} + assert "connection_state" in fields + assert "connection_error" in fields + + snap = StreamSnapshot( + id="cam", kind="video", provides_audio_track=False, produces_file=False, + frame_count=0, last_sample_at_ns=None, effective_hz=0.0, + latest_frame=None, plot_points={}, latest_pose={}, + ) + assert snap.connection_state == "idle" + assert snap.connection_error is None From 3a680fc7597cd0bfd691768040b74d2ba8447fa0 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 16:30:57 +0900 Subject: [PATCH 57/64] feat(viewer): poller reads orchestrator stream state into snapshot Every per-stream StreamSnapshot now carries connection_state and connection_error, sourced from the orchestrator's per-stream dicts populated by _set_stream_state. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/syncfield/viewer/poller.py | 2 ++ tests/unit/viewer/test_poller.py | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/src/syncfield/viewer/poller.py b/src/syncfield/viewer/poller.py index 954f9e6..6553094 100644 --- a/src/syncfield/viewer/poller.py +++ b/src/syncfield/viewer/poller.py @@ -212,6 +212,8 @@ def _build_snapshot(self) -> SessionSnapshot: plot_points=plot_points, latest_pose=latest_pose, live_preview=getattr(stream.capabilities, "live_preview", True), + connection_state=session._stream_states.get(stream_id, "idle"), # type: ignore[attr-defined] + connection_error=session._stream_errors.get(stream_id), # type: ignore[attr-defined] ) # Snapshot incidents — take a consistent copy under the incidents lock. diff --git a/tests/unit/viewer/test_poller.py b/tests/unit/viewer/test_poller.py index c272183..f876033 100644 --- a/tests/unit/viewer/test_poller.py +++ b/tests/unit/viewer/test_poller.py @@ -103,3 +103,23 @@ def test_start_stop_and_get_snapshot_thread(self, tmp_path): assert snap.host_id == "test_rig" finally: poller.stop() + + def test_poller_snapshot_includes_connection_state(self, tmp_path): + """Verify that StreamSnapshot carries connection_state and connection_error + from the orchestrator's per-stream tracking.""" + session = sf.SessionOrchestrator( + host_id="h", + output_dir=tmp_path, + sync_tone=sf.SyncToneConfig.silent(), + ) + session.add(FakeStream("good")) + session.add(FakeStream("bad", fail_on_start=True)) + + poller = SessionPoller(session) + session.connect() + + snap = poller._build_snapshot() + assert snap.streams["good"].connection_state == "connected" + assert snap.streams["good"].connection_error is None + assert snap.streams["bad"].connection_state == "failed" + assert snap.streams["bad"].connection_error is not None From 8942ad11b0dc50b1e61d40a07e67e610ac04d752 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 16:33:43 +0900 Subject: [PATCH 58/64] feat(viewer): emit connection_state / connection_error over WebSocket Frontend consumes these to select the right StreamCard overlay and to populate the degraded-state header chip. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/syncfield/viewer/server.py | 2 ++ .../unit/test_viewer_aggregation_snapshot.py | 36 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/src/syncfield/viewer/server.py b/src/syncfield/viewer/server.py index 5c1a547..8f3011b 100644 --- a/src/syncfield/viewer/server.py +++ b/src/syncfield/viewer/server.py @@ -83,6 +83,8 @@ def snapshot_to_dict(snapshot: SessionSnapshot) -> Dict[str, Any]: "last_sample_ms_ago": last_sample_ms_ago, "provides_audio_track": s.provides_audio_track, "produces_file": s.produces_file, + "connection_state": s.connection_state, + "connection_error": s.connection_error, "capabilities": { "live_preview": getattr(s, "live_preview", True), "provides_audio_track": s.provides_audio_track, diff --git a/tests/unit/test_viewer_aggregation_snapshot.py b/tests/unit/test_viewer_aggregation_snapshot.py index ebe529a..926cf64 100644 --- a/tests/unit/test_viewer_aggregation_snapshot.py +++ b/tests/unit/test_viewer_aggregation_snapshot.py @@ -49,3 +49,39 @@ def test_snapshot_serializes_active_job(): assert d["aggregation"]["active_job"]["current_bytes"] == 5_000_000 assert d["aggregation"]["queue_length"] == 1 assert len(d["aggregation"]["recent_jobs"]) == 1 + + +def test_serialized_stream_includes_connection_state(): + from syncfield.viewer.state import StreamSnapshot, SessionSnapshot + + stream_snap = StreamSnapshot( + id="cam", + kind="video", + provides_audio_track=False, + produces_file=False, + frame_count=0, + last_sample_at_ns=None, + effective_hz=0.0, + latest_frame=None, + plot_points={}, + latest_pose={}, + connection_state="failed", + connection_error="Device not visible", + ) + sess_snap = SessionSnapshot( + host_id="h", + state="idle", + output_dir="/tmp", + sync_point_monotonic_ns=None, + sync_point_wall_clock_ns=None, + chirp_start_ns=None, + chirp_stop_ns=None, + chirp_enabled=False, + elapsed_s=0.0, + streams={"cam": stream_snap}, + active_incidents=[], + resolved_incidents=[], + ) + out = snapshot_to_dict(sess_snap) + assert out["streams"]["cam"]["connection_state"] == "failed" + assert out["streams"]["cam"]["connection_error"] == "Device not visible" From 6d9feef395ba9874cace223471b8717004cc2082 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 16:34:52 +0900 Subject: [PATCH 59/64] feat(viewer-fe): mirror ConnectionState + StreamSnapshot additions ConnectionState enum and two new fields on StreamSnapshot so the overlay branching in StreamCard (next commit) is type-safe. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/syncfield/viewer/frontend/src/lib/types.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/syncfield/viewer/frontend/src/lib/types.ts b/src/syncfield/viewer/frontend/src/lib/types.ts index 5f793e2..a965808 100644 --- a/src/syncfield/viewer/frontend/src/lib/types.ts +++ b/src/syncfield/viewer/frontend/src/lib/types.ts @@ -21,6 +21,8 @@ export interface StreamSnapshot { produces_file: boolean; /** Stream capabilities declared by the adapter. May be absent on older servers. */ capabilities?: StreamCapabilities; + connection_state: ConnectionState; + connection_error: string | null; } export interface ChirpInfo { @@ -31,6 +33,13 @@ export interface ChirpInfo { export type Severity = "info" | "warning" | "error" | "critical"; +export type ConnectionState = + | "idle" + | "connecting" + | "connected" + | "failed" + | "disconnected"; + export interface IncidentArtifact { kind: string; path: string; From 964e708e2f8c0d8521266e3a1980135d39b6c81f Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 16:35:11 +0900 Subject: [PATCH 60/64] feat(viewer-fe): add Connecting / WaitingForData / Failed overlays Three small presentational components used by StreamCard (next commit) to replace the browser broken-image fallback. FailedOverlay is click- to-expand for long error messages. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../src/components/stream-overlays.tsx | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 src/syncfield/viewer/frontend/src/components/stream-overlays.tsx diff --git a/src/syncfield/viewer/frontend/src/components/stream-overlays.tsx b/src/syncfield/viewer/frontend/src/components/stream-overlays.tsx new file mode 100644 index 0000000..8e0f4c9 --- /dev/null +++ b/src/syncfield/viewer/frontend/src/components/stream-overlays.tsx @@ -0,0 +1,52 @@ +import { useState } from "react"; + +export function ConnectingOverlay() { + return ( +
+
+ + Connecting… +
+
+ ); +} + +export function WaitingForDataOverlay() { + return ( +
+
+ Connected · waiting for first frame +
+
+ ); +} + +export function FailedOverlay({ error }: { error: string }) { + const [expanded, setExpanded] = useState(false); + return ( + + ); +} From 5f3deae6683753c6b7459486cb852ca06fc9cb4a Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 16:36:46 +0900 Subject: [PATCH 61/64] feat(viewer-fe): branch StreamCard body on connection_state Connecting / waiting-for-first-frame / failed now render explicit overlays instead of the browser's broken-image placeholder. Healthy streams still render the existing video / sensor / audio body. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../frontend/src/components/stream-card.tsx | 48 ++++++++++++++----- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/src/syncfield/viewer/frontend/src/components/stream-card.tsx b/src/syncfield/viewer/frontend/src/components/stream-card.tsx index 745d106..4b1fdb2 100644 --- a/src/syncfield/viewer/frontend/src/components/stream-card.tsx +++ b/src/syncfield/viewer/frontend/src/components/stream-card.tsx @@ -8,6 +8,11 @@ import { StandaloneRecorderPanel, type StandaloneRecorderStream, } from "./standalone-recorder-panel"; +import { + ConnectingOverlay, + WaitingForDataOverlay, + FailedOverlay, +} from "./stream-overlays"; interface StreamCardProps { stream: StreamSnapshot; @@ -196,19 +201,9 @@ export function StreamCard({ {stream.produces_file && file} - {/* Body — varies by stream kind */} + {/* Body — varies by connection state, then stream kind */}
- {stream.kind === "video" ? ( - - ) : stream.kind === "audio" ? ( - - ) : stream.kind === "sensor" ? ( - - ) : ( -
- No preview -
- )} +
{/* Footer stats */} @@ -221,6 +216,35 @@ export function StreamCard({ ); } +// --------------------------------------------------------------------------- +// StreamCardBody — branches on connection_state before kind-based rendering +// --------------------------------------------------------------------------- + +function StreamCardBody({ stream }: { stream: StreamSnapshot }) { + if (stream.connection_state === "connecting") { + return ; + } + if (stream.connection_state === "failed") { + return ; + } + if ( + stream.connection_state === "connected" && + stream.kind === "video" && + stream.frame_count === 0 + ) { + return ; + } + // Healthy / idle / disconnected — fall through to kind-based rendering. + if (stream.kind === "video") return ; + if (stream.kind === "audio") return ; + if (stream.kind === "sensor") return ; + return ( +
+ No preview +
+ ); +} + // --------------------------------------------------------------------------- // Helper — pick the aggregation job relevant to a specific stream // --------------------------------------------------------------------------- From d5fec4116c1ac45d80c6c37436d482fe49e0aa6e Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 16:38:09 +0900 Subject: [PATCH 62/64] feat(viewer-fe): show degraded counter on header state chip 'Ready (3/5)' in yellow when one or more streams are not in 'connected' state. Normal tone when everything is healthy. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../viewer/frontend/src/components/header.tsx | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/syncfield/viewer/frontend/src/components/header.tsx b/src/syncfield/viewer/frontend/src/components/header.tsx index 11ef09b..c2e4293 100644 --- a/src/syncfield/viewer/frontend/src/components/header.tsx +++ b/src/syncfield/viewer/frontend/src/components/header.tsx @@ -61,6 +61,13 @@ export function Header({ const elapsed = snapshot?.elapsed_s ?? 0; const isRecording = state === "recording"; + const streams = Object.values(snapshot?.streams ?? {}); + const total = streams.length; + const connected = streams.filter((s) => s.connection_state === "connected").length; + const showCount = total > 0 && connected < total; + const stateLabel = friendlyState(state); + const displayedLabel = showCount ? `${stateLabel} (${connected}/${total})` : stateLabel; + return (
- {friendlyState(state)} + {displayedLabel} From 1f4488add692c767037300e787c938ac7e0b77d9 Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 16:40:42 +0900 Subject: [PATCH 63/64] test(health): integration test for partial connect + incident persistence Real SessionOrchestrator, mix of passing + failing FakeStreams; asserts the session reaches CONNECTED, the failed stream surfaces as an open startup-failure incident, and the fingerprint lands in incidents.jsonl. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../health/test_partial_connect.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 tests/integration/health/test_partial_connect.py diff --git a/tests/integration/health/test_partial_connect.py b/tests/integration/health/test_partial_connect.py new file mode 100644 index 0000000..4c30e23 --- /dev/null +++ b/tests/integration/health/test_partial_connect.py @@ -0,0 +1,45 @@ +"""End-to-end: real SessionOrchestrator + FakeStream mix survives one +stream failing to connect, and incidents.jsonl captures it.""" +from __future__ import annotations + +import json +import time +from pathlib import Path + +import pytest + +from syncfield.orchestrator import SessionOrchestrator +from syncfield.testing import FakeStream +from syncfield.types import SessionState + + +@pytest.mark.slow +def test_partial_connect_end_to_end(tmp_path: Path): + sess = SessionOrchestrator(host_id="h", output_dir=tmp_path) + sess.add(FakeStream("good_a")) + sess.add(FakeStream("bad", fail_on_start=True)) + sess.add(FakeStream("good_b")) + + sess.connect() + assert sess.state is SessionState.CONNECTED + + # Give the health worker a moment to ingest the startup-failure event. + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline: + if any(i.fingerprint == "bad:startup-failure" for i in sess.health.open_incidents()): + break + time.sleep(0.05) + + open_fps = [i.fingerprint for i in sess.health.open_incidents()] + assert "bad:startup-failure" in open_fps + + sess.start(countdown_s=0) + time.sleep(0.5) + sess.stop() + sess.disconnect() + + out = list(tmp_path.rglob("incidents.jsonl")) + assert out, "no incidents.jsonl written" + lines = [json.loads(l) for l in out[0].read_text().strip().splitlines() if l] + fingerprints = {l["fingerprint"] for l in lines} + assert "bad:startup-failure" in fingerprints From 1b804eb92d81e2501eb4d753d32f809f110074bf Mon Sep 17 00:00:00 2001 From: styu12 Date: Wed, 22 Apr 2026 16:41:29 +0900 Subject: [PATCH 64/64] test(health): integration test for NoDataDetector on real orchestrator SilentFakeStream connects but withholds samples until the test opens its gate. Asserts the no-data incident opens within 1.5s of connect and closes within one tick of samples resuming. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../health/test_no_data_detector.py | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 tests/integration/health/test_no_data_detector.py diff --git a/tests/integration/health/test_no_data_detector.py b/tests/integration/health/test_no_data_detector.py new file mode 100644 index 0000000..5c01af3 --- /dev/null +++ b/tests/integration/health/test_no_data_detector.py @@ -0,0 +1,85 @@ +"""End-to-end: a stream that connects but never emits a sample triggers +the no-data incident within the configured threshold.""" +from __future__ import annotations + +import threading +import time +from pathlib import Path + +import pytest + +from syncfield.orchestrator import SessionOrchestrator +from syncfield.stream import StreamBase +from syncfield.types import FinalizationReport, SampleEvent, StreamCapabilities + + +class SilentFakeStream(StreamBase): + """FakeStream variant that connects successfully but emits no samples until asked.""" + + def __init__(self, stream_id: str): + super().__init__(id=stream_id, kind="sensor", capabilities=StreamCapabilities()) + self._stop = threading.Event() + self._gate = threading.Event() + self._thread: threading.Thread | None = None + self._frame = 0 + + def connect(self): + self._stop.clear() + self._thread = threading.Thread(target=self._run, daemon=True) + self._thread.start() + + def disconnect(self): + self._stop.set() + if self._thread: + self._thread.join(timeout=1.0) + self._thread = None + + def start_recording(self, session_clock): + pass + + def stop_recording(self) -> FinalizationReport: + return FinalizationReport( + stream_id=self.id, status="completed", frame_count=self._frame, + file_path=None, first_sample_at_ns=0, last_sample_at_ns=0, + health_events=[], error=None, + ) + + def allow_samples(self): + self._gate.set() + + def _run(self): + while not self._stop.is_set(): + if self._gate.is_set(): + self._frame += 1 + self._emit_sample(SampleEvent( + stream_id=self.id, frame_number=self._frame, + capture_ns=time.monotonic_ns(), + )) + time.sleep(0.05) + + +@pytest.mark.slow +def test_no_data_incident_opens_then_closes_when_samples_arrive(tmp_path: Path): + sess = SessionOrchestrator(host_id="h", output_dir=tmp_path) + stream = SilentFakeStream("cam") + sess.add(stream) + + for d in sess.health.iter_detectors(): + if d.name == "no-data": + d._threshold_ns = int(1e9) + break + + sess.connect() + + time.sleep(1.5) + open_fps = [i.fingerprint for i in sess.health.open_incidents()] + assert "cam:no-data" in open_fps + + stream.allow_samples() + time.sleep(0.5) + + # Session was never started into RECORDING — disconnect directly from CONNECTED. + sess.disconnect() + + resolved_fps = [i.fingerprint for i in sess.health.resolved_incidents()] + assert "cam:no-data" in resolved_fps