diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4a87aa7..3d5f682 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,6 +16,22 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Set up Node (for viewer frontend) + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Build viewer frontend + working-directory: src/syncfield/viewer/frontend + run: | + # Same lockfile-regeneration dance as publish.yml: optional + # platform-specific deps (e.g. rollup's @rollup/rollup-linux-*) + # aren't re-resolved when a macOS-generated lockfile runs on + # Ubuntu. Regenerating from package.json on CI works. + rm -f package-lock.json + npm install --no-audit --no-fund + npm run build + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: @@ -24,7 +40,21 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -e ".[dev]" || pip install -e . && pip install pytest + # [all] brings in every optional adapter dep (bleak, aiohttp, + # depthai, zeroconf, httpx) so tests that import adapters at + # collection time don't ImportError. + pip install -e ".[all]" + # `dev` lives in PEP 735 [dependency-groups], not + # [project.optional-dependencies], so .[dev] wouldn't resolve. + # Install pytest + its plugins explicitly — keep this list in + # sync with [dependency-groups].dev in pyproject.toml. + pip install \ + "pytest>=8.4.2" \ + "pytest-aiohttp>=1.1.0" \ + "pytest-asyncio>=1.2.0" \ + "pytest-mock>=3.12.0" \ + "pytest-timeout>=2.4.0" \ + "pytest-xdist>=3.6.0" - name: Run tests run: pytest -v diff --git a/docs/superpowers/plans/2026-04-22-health-telemetry.md b/docs/superpowers/plans/2026-04-22-health-telemetry.md new file mode 100644 index 0000000..dc849f0 --- /dev/null +++ b/docs/superpowers/plans/2026-04-22-health-telemetry.md @@ -0,0 +1,4292 @@ +# Health Telemetry Platform Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build a sensor-agnostic, platform-level health telemetry system that continuously observes every stream during a recording, raises Sentry-style Incidents on stalls / FPS drops / jitter / startup failures / writer backpressure / adapter faults, and surfaces them live in the viewer and in `FinalizationReport.incidents` + `incidents.jsonl` post-session. + +**Architecture:** New `src/syncfield/health/` package. Streams push samples and raw `HealthEvent`s through thread-safe ingress queues into a single daemon `HealthWorker` that ticks at 20 Hz. On each tick every registered `Detector` is polled; emitted `HealthEvent`s feed an `IncidentTracker` that groups by `fingerprint`, owns open/close lifecycle, and flushes to disk. Default detectors are installed automatically; new ones can be added via `session.health.register(...)`. OAK's native depthai logger is bridged into the same channel so X_LINK_ERROR / crash / reconnect events become structured incidents with crash-dump artifacts attached. + +**Tech Stack:** Python 3.9+, `dataclasses`, `threading`, `queue.SimpleQueue`, `logging` (for depthai bridge). Frontend: existing React + TypeScript viewer. + +**Spec:** `docs/superpowers/specs/2026-04-22-health-telemetry-design.md` + +--- + +## File Structure + +### New (backend) + +``` +src/syncfield/health/ +├── __init__.py # public API +├── severity.py # Severity enum + ordering helpers +├── types.py # Incident, IncidentArtifact, IncidentSnapshot, WriterStats +├── detector.py # Detector Protocol + DetectorBase +├── registry.py # DetectorRegistry +├── tracker.py # IncidentTracker +├── worker.py # HealthWorker (daemon thread + ingress queues) +├── system.py # HealthSystem (facade users touch) +└── detectors/ + ├── __init__.py + ├── adapter_passthrough.py # AdapterEventPassthrough + ├── stream_stall.py # StreamStallDetector + ├── fps_drop.py # FpsDropDetector + ├── jitter.py # JitterDetector + ├── startup_failure.py # StartupFailureDetector + ├── backpressure.py # BackpressureDetector + └── depthai_bridge.py # DepthAILoggerBridge (soft-imports depthai) +``` + +### Modified (backend) + +- `src/syncfield/types.py` — add `severity` / `source` / `fingerprint` / `data` to `HealthEvent`; add `target_hz` to `StreamCapabilities`; add `incidents` to `FinalizationReport`. +- `src/syncfield/writer.py` — add `SessionLogWriter.log_incident()`; emit `WriterStats`. +- `src/syncfield/orchestrator.py` — construct `HealthSystem`, wire sample / health / state / writer-stats observers, start/stop worker, embed incidents in `FinalizationReport`. +- `src/syncfield/adapters/oak_camera.py` — declare `target_hz`, install `DepthAILoggerBridge`, attach `crash_dump` artifacts. +- `src/syncfield/adapters/uvc_webcam.py`, `host_audio.py`, `meta_quest_camera/*`, `ble_imu.py`, `insta360_go3s/stream.py`, `polling_sensor.py` — declare `target_hz` where known. +- `src/syncfield/viewer/state.py` — replace `HealthEntry` / `StreamSnapshot.health_count` with `IncidentSnapshot` / top-level `active_incidents` / `resolved_incidents`. +- `src/syncfield/viewer/poller.py` — subscribe to `HealthSystem` incident callbacks, populate new snapshot fields. +- `src/syncfield/viewer/server.py` — serialize incidents into WebSocket payload. + +### New (frontend) + +- `src/syncfield/viewer/frontend/src/components/incident-panel.tsx` + +### Deleted (frontend) + +- `src/syncfield/viewer/frontend/src/components/health-table.tsx` + +### Modified (frontend) + +- `src/syncfield/viewer/frontend/src/lib/types.ts` — mirror `Severity`, `IncidentSnapshot`; remove `HealthEntry`, `health_count`. +- `src/syncfield/viewer/frontend/src/components/stream-card.tsx` — severity badge. +- `src/syncfield/viewer/frontend/src/App.tsx` — mount `IncidentPanel` in place of `HealthTable`. + +### Tests + +``` +tests/unit/health/ +├── __init__.py +├── test_severity.py +├── test_incident_types.py +├── test_detector_base.py +├── test_registry.py +├── test_incident_tracker.py +├── test_health_worker.py +├── test_health_system.py +└── detectors/ + ├── __init__.py + ├── test_adapter_passthrough.py + ├── test_stream_stall.py + ├── test_fps_drop.py + ├── test_jitter.py + ├── test_startup_failure.py + ├── test_backpressure.py + └── test_depthai_bridge.py + +tests/integration/health/ +├── __init__.py +├── test_orchestrator_health_integration.py +└── test_incidents_jsonl_roundtrip.py +``` + +--- + +## Conventions used in this plan + +- Every task is **TDD**: write a failing test, verify failure, implement, verify pass, commit. +- Run tests with `pytest -v` (project `pyproject.toml` sets `testpaths=["tests"]`, `pythonpath=["src"]`). +- Each task ends with a commit using Conventional Commits (`feat:`, `refactor:`, `test:`). +- Test helpers use `tests/helpers/` where one exists; new helpers live in `tests/unit/health/_helpers.py`. +- `time.monotonic_ns()` returns an int. All `_ns` fields are ints. +- We will use `queue.SimpleQueue` for lock-free ingress (Python's own fast, unbounded, thread-safe FIFO). + +--- + +## Phase 1 — Core types & skeleton + +### Task 1: Severity enum + +**Files:** +- Create: `src/syncfield/health/__init__.py` (empty for now) +- Create: `src/syncfield/health/severity.py` +- Test: `tests/unit/health/__init__.py` (empty), `tests/unit/health/test_severity.py` + +- [ ] **Step 1: Create the (empty) package init files** + +```bash +mkdir -p src/syncfield/health tests/unit/health +touch src/syncfield/health/__init__.py tests/unit/health/__init__.py +``` + +- [ ] **Step 2: Write failing test** + +Create `tests/unit/health/test_severity.py`: + +```python +from syncfield.health.severity import Severity, max_severity + + +def test_severity_values(): + assert Severity.INFO.value == "info" + assert Severity.WARNING.value == "warning" + assert Severity.ERROR.value == "error" + assert Severity.CRITICAL.value == "critical" + + +def test_severity_ordering(): + # INFO < WARNING < ERROR < CRITICAL + order = [Severity.INFO, Severity.WARNING, Severity.ERROR, Severity.CRITICAL] + for a, b in zip(order, order[1:]): + assert a.rank < b.rank + + +def test_max_severity_picks_highest(): + assert max_severity(Severity.INFO, Severity.WARNING) == Severity.WARNING + assert max_severity(Severity.ERROR, Severity.WARNING) == Severity.ERROR + assert max_severity(Severity.CRITICAL, Severity.INFO, Severity.ERROR) == Severity.CRITICAL + + +def test_max_severity_requires_at_least_one(): + import pytest + with pytest.raises(ValueError): + max_severity() +``` + +- [ ] **Step 3: Run, confirm fail** + +```bash +pytest tests/unit/health/test_severity.py -v +``` +Expected: `ModuleNotFoundError: No module named 'syncfield.health.severity'` + +- [ ] **Step 4: Implement** + +`src/syncfield/health/severity.py`: + +```python +"""Severity levels for health events and incidents. + +Ordered INFO < WARNING < ERROR < CRITICAL. Use :func:`max_severity` to +pick the highest of several levels — incidents escalate to the max +severity of their constituent events. +""" + +from __future__ import annotations + +from enum import Enum + + +class Severity(str, Enum): + INFO = "info" + WARNING = "warning" + ERROR = "error" + CRITICAL = "critical" + + @property + def rank(self) -> int: + return _RANK[self] + + +_RANK = { + Severity.INFO: 0, + Severity.WARNING: 1, + Severity.ERROR: 2, + Severity.CRITICAL: 3, +} + + +def max_severity(*levels: Severity) -> Severity: + if not levels: + raise ValueError("max_severity requires at least one Severity") + return max(levels, key=lambda s: s.rank) +``` + +- [ ] **Step 5: Run, confirm pass** + +```bash +pytest tests/unit/health/test_severity.py -v +``` +Expected: 4 passed. + +- [ ] **Step 6: Commit** + +```bash +git add src/syncfield/health/ tests/unit/health/ +git commit -m "feat(health): add Severity enum with rank + max_severity helper" +``` + +--- + +### Task 2: Enrich HealthEvent with severity / source / fingerprint / data + +**Files:** +- Modify: `src/syncfield/types.py:250-272` (`HealthEvent` dataclass + `to_dict`) +- Test: `tests/unit/test_types.py` (add cases) + +Spec requires new fields on `HealthEvent`. We are intentionally breaking the existing format; all writes will carry the new fields and readers (viewer, session log consumers) update in later tasks. + +- [ ] **Step 1: Write failing test** + +Append to `tests/unit/test_types.py`: + +```python +from syncfield.health.severity import Severity +from syncfield.types import HealthEvent, HealthEventKind + + +def test_health_event_has_enrichment_fields_with_defaults(): + ev = HealthEvent( + stream_id="cam", + kind=HealthEventKind.ERROR, + at_ns=1_000, + detail="boom", + ) + # new fields default to safe values when caller does not set them. + assert ev.severity == Severity.INFO + assert ev.source == "unknown" + assert ev.fingerprint == "" + assert ev.data == {} + + +def test_health_event_to_dict_includes_new_fields(): + ev = HealthEvent( + stream_id="cam", + kind=HealthEventKind.ERROR, + at_ns=1_000, + detail="boom", + severity=Severity.ERROR, + source="adapter:oak", + fingerprint="cam:adapter:xlink-error", + data={"stream": "__x_0_1"}, + ) + d = ev.to_dict() + assert d["severity"] == "error" + assert d["source"] == "adapter:oak" + assert d["fingerprint"] == "cam:adapter:xlink-error" + assert d["data"] == {"stream": "__x_0_1"} +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/test_types.py -k health_event -v +``` +Expected: `TypeError` (frozen dataclass rejecting unknown kwargs) or AttributeError on `.severity`. + +- [ ] **Step 3: Implement** + +In `src/syncfield/types.py`, replace the `HealthEvent` dataclass and its `to_dict`: + +```python +from syncfield.health.severity import Severity # top of file, next to other imports + +@dataclass(frozen=True) +class HealthEvent: + """A stream reports a health observation. + + ``severity`` / ``source`` / ``fingerprint`` / ``data`` enable the + incident-tracking layer in :mod:`syncfield.health` to group many raw + events into a single Sentry-style Incident. Adapters that don't care + can leave them at their safe defaults; the platform will fill them + in before the event reaches the IncidentTracker. + """ + + stream_id: str + kind: HealthEventKind + at_ns: int + detail: str | None = None + severity: Severity = Severity.INFO + source: str = "unknown" + fingerprint: str = "" + data: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "stream_id": self.stream_id, + "kind": self.kind.value, + "at_ns": self.at_ns, + "detail": self.detail, + "severity": self.severity.value, + "source": self.source, + "fingerprint": self.fingerprint, + "data": self.data, + } +``` + +Add `from dataclasses import field` if missing. `field` is needed for the mutable default on `data`. + +- [ ] **Step 4: Run, confirm pass** + +```bash +pytest tests/unit/test_types.py -v +``` +Expected: all pass, including existing tests (defaults are backward-compatible at the Python level; only the JSON format changes). + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/types.py tests/unit/test_types.py +git commit -m "feat(types): enrich HealthEvent with severity/source/fingerprint/data" +``` + +--- + +### Task 3: WriterStats + IncidentArtifact + Incident + IncidentSnapshot + +**Files:** +- Create: `src/syncfield/health/types.py` +- Test: `tests/unit/health/test_incident_types.py` + +- [ ] **Step 1: Write failing test** + +`tests/unit/health/test_incident_types.py`: + +```python +from syncfield.health.severity import Severity +from syncfield.health.types import ( + Incident, + IncidentArtifact, + IncidentSnapshot, + WriterStats, +) +from syncfield.types import HealthEvent, HealthEventKind + + +def _ev(at_ns: int, severity: Severity = Severity.ERROR) -> HealthEvent: + return HealthEvent( + stream_id="cam", + kind=HealthEventKind.ERROR, + at_ns=at_ns, + detail="x", + severity=severity, + source="detector:stream-stall", + fingerprint="cam:stream-stall", + ) + + +def test_writer_stats_fields(): + s = WriterStats( + stream_id="cam", + at_ns=100, + queue_depth=3, + queue_capacity=16, + dropped=0, + ) + assert s.queue_fullness == 3 / 16 + assert s.stream_id == "cam" + + +def test_writer_stats_zero_capacity_is_empty(): + s = WriterStats(stream_id="cam", at_ns=0, queue_depth=0, queue_capacity=0, dropped=0) + assert s.queue_fullness == 0.0 + + +def test_incident_from_first_event_initializes_fields(): + first = _ev(100) + inc = Incident.opened_from(first, title="Stream stalled (silence 2.0s)") + assert inc.stream_id == "cam" + assert inc.fingerprint == "cam:stream-stall" + assert inc.severity == Severity.ERROR + assert inc.title == "Stream stalled (silence 2.0s)" + assert inc.opened_at_ns == 100 + assert inc.closed_at_ns is None + assert inc.event_count == 1 + assert inc.first_event == first + assert inc.last_event == first + assert inc.artifacts == [] + + +def test_incident_record_event_escalates_severity_and_updates_last(): + inc = Incident.opened_from(_ev(100, severity=Severity.WARNING), title="t") + inc.record_event(_ev(200, severity=Severity.ERROR)) + assert inc.event_count == 2 + assert inc.severity == Severity.ERROR + assert inc.last_event.at_ns == 200 + assert inc.last_event_at_ns == 200 + + +def test_incident_close(): + inc = Incident.opened_from(_ev(100), title="t") + inc.close(at_ns=500) + assert inc.closed_at_ns == 500 + assert inc.is_open is False + + +def test_incident_attach_artifact(): + inc = Incident.opened_from(_ev(100), title="t") + inc.attach(IncidentArtifact(kind="crash_dump", path="/tmp/x.json")) + assert inc.artifacts[0].kind == "crash_dump" + assert inc.artifacts[0].path == "/tmp/x.json" + + +def test_incident_snapshot_shape(): + inc = Incident.opened_from(_ev(100), title="t") + snap = IncidentSnapshot.from_incident(inc, now_ns=1_000_000_100) + assert snap.id == inc.id + assert snap.stream_id == "cam" + assert snap.severity == "error" + assert snap.is_open is True + assert snap.ago_s >= 0 +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/health/test_incident_types.py -v +``` +Expected: `ModuleNotFoundError`. + +- [ ] **Step 3: Implement** + +`src/syncfield/health/types.py`: + +```python +"""Data classes for the health/incident layer. + +These are plain, explicit structs — the :mod:`syncfield.health` runtime +mutates :class:`Incident` objects in-place on the worker thread. The +viewer receives immutable :class:`IncidentSnapshot`\\ s instead. +""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass, field +from typing import Any, List + +from syncfield.health.severity import Severity, max_severity +from syncfield.types import HealthEvent + + +@dataclass(frozen=True) +class WriterStats: + """One observation of a per-stream writer's queue.""" + + stream_id: str + at_ns: int + queue_depth: int + queue_capacity: int + dropped: int + + @property + def queue_fullness(self) -> float: + if self.queue_capacity <= 0: + return 0.0 + return self.queue_depth / self.queue_capacity + + +@dataclass(frozen=True) +class IncidentArtifact: + """A piece of evidence attached to an Incident (crash dump, log excerpt, ...).""" + + kind: str + path: str + detail: str | None = None + + def to_dict(self) -> dict[str, Any]: + return {"kind": self.kind, "path": self.path, "detail": self.detail} + + +@dataclass +class Incident: + """A grouped, open/close-tracked sequence of HealthEvents sharing a fingerprint. + + Mutable because the worker thread updates ``last_event`` / ``event_count`` + / ``severity`` on every matching event. The viewer never sees this + class directly — it reads :class:`IncidentSnapshot` instead. + """ + + id: str + stream_id: str + fingerprint: str + title: str + severity: Severity + source: str + opened_at_ns: int + closed_at_ns: int | None + last_event_at_ns: int + event_count: int + first_event: HealthEvent + last_event: HealthEvent + artifacts: List[IncidentArtifact] = field(default_factory=list) + data: dict[str, Any] = field(default_factory=dict) + + @classmethod + def opened_from(cls, event: HealthEvent, *, title: str) -> "Incident": + return cls( + id=uuid.uuid4().hex, + stream_id=event.stream_id, + fingerprint=event.fingerprint, + title=title, + severity=event.severity, + source=event.source, + opened_at_ns=event.at_ns, + closed_at_ns=None, + last_event_at_ns=event.at_ns, + event_count=1, + first_event=event, + last_event=event, + ) + + @property + def is_open(self) -> bool: + return self.closed_at_ns is None + + def record_event(self, event: HealthEvent) -> None: + self.event_count += 1 + self.last_event = event + self.last_event_at_ns = event.at_ns + self.severity = max_severity(self.severity, event.severity) + + def close(self, *, at_ns: int) -> None: + self.closed_at_ns = at_ns + + def attach(self, artifact: IncidentArtifact) -> None: + self.artifacts.append(artifact) + + def to_dict(self) -> dict[str, Any]: + return { + "id": self.id, + "stream_id": self.stream_id, + "fingerprint": self.fingerprint, + "title": self.title, + "severity": self.severity.value, + "source": self.source, + "opened_at_ns": self.opened_at_ns, + "closed_at_ns": self.closed_at_ns, + "last_event_at_ns": self.last_event_at_ns, + "event_count": self.event_count, + "first_event": self.first_event.to_dict(), + "last_event": self.last_event.to_dict(), + "artifacts": [a.to_dict() for a in self.artifacts], + "data": self.data, + } + + +@dataclass(frozen=True) +class IncidentSnapshot: + """Read-only view of an Incident, for the viewer's WebSocket payload.""" + + id: str + stream_id: str + fingerprint: str + title: str + severity: str + source: str + opened_at_ns: int + closed_at_ns: int | None + event_count: int + detail: str | None + ago_s: float + artifacts: List[dict] + + @property + def is_open(self) -> bool: + return self.closed_at_ns is None + + @classmethod + def from_incident(cls, inc: Incident, *, now_ns: int) -> "IncidentSnapshot": + anchor = inc.closed_at_ns if inc.closed_at_ns is not None else inc.last_event_at_ns + ago_s = max(0.0, (now_ns - anchor) / 1e9) + return cls( + id=inc.id, + stream_id=inc.stream_id, + fingerprint=inc.fingerprint, + title=inc.title, + severity=inc.severity.value, + source=inc.source, + opened_at_ns=inc.opened_at_ns, + closed_at_ns=inc.closed_at_ns, + event_count=inc.event_count, + detail=inc.last_event.detail, + ago_s=ago_s, + artifacts=[a.to_dict() for a in inc.artifacts], + ) +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +pytest tests/unit/health/test_incident_types.py -v +``` +Expected: 7 passed. + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/types.py tests/unit/health/test_incident_types.py +git commit -m "feat(health): add WriterStats, Incident, IncidentArtifact, IncidentSnapshot" +``` + +--- + +### Task 4: Detector Protocol + DetectorBase + +**Files:** +- Create: `src/syncfield/health/detector.py` +- Test: `tests/unit/health/test_detector_base.py` + +- [ ] **Step 1: Write failing test** + +`tests/unit/health/test_detector_base.py`: + +```python +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.types import WriterStats +from syncfield.types import HealthEvent, HealthEventKind, SampleEvent, SessionState + + +class NoopDetector(DetectorBase): + name = "noop" + default_severity = Severity.WARNING + + +def test_detector_base_defaults_are_noops(): + d = NoopDetector() + # All observers accept calls without raising. + d.observe_sample("cam", SampleEvent(stream_id="cam", frame_number=1, capture_ns=100)) + d.observe_health("cam", HealthEvent(stream_id="cam", kind=HealthEventKind.WARNING, at_ns=1)) + d.observe_state(SessionState.IDLE, SessionState.CONNECTED) + d.observe_writer_stats("cam", WriterStats("cam", 1, 0, 0, 0)) + # tick yields nothing by default. + assert list(d.tick(now_ns=100)) == [] + # close_condition defaults to True so pass-through close can rely on it. + from syncfield.health.types import Incident + ev = HealthEvent(stream_id="cam", kind=HealthEventKind.WARNING, at_ns=1) + inc = Incident.opened_from(ev, title="x") + assert isinstance(d.close_condition(inc, now_ns=10), bool) + + +def test_detector_base_requires_name_and_severity(): + import pytest + + with pytest.raises(TypeError): + DetectorBase() # abstract base: name / default_severity unset on the class +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/health/test_detector_base.py -v +``` +Expected: `ModuleNotFoundError`. + +- [ ] **Step 3: Implement** + +`src/syncfield/health/detector.py`: + +```python +"""Detector protocol + base class. + +A :class:`Detector` observes the stream (samples, adapter health events, +session state, writer stats) and may emit :class:`HealthEvent` on each +``tick()``. The :class:`IncidentTracker` groups emitted events by +fingerprint, opens incidents, and consults ``close_condition`` to know +when an open incident should resolve. + +Most detectors subclass :class:`DetectorBase` and override only the +observe/tick hooks they care about; the base provides safe no-op +defaults for the rest. +""" + +from __future__ import annotations + +from typing import Iterable, Iterator, Protocol, runtime_checkable + +from syncfield.health.severity import Severity +from syncfield.health.types import Incident, WriterStats +from syncfield.types import HealthEvent, SampleEvent, SessionState + + +@runtime_checkable +class Detector(Protocol): + name: str + default_severity: Severity + + def observe_sample(self, stream_id: str, sample: SampleEvent) -> None: ... + def observe_health(self, stream_id: str, event: HealthEvent) -> None: ... + def observe_state(self, old: SessionState, new: SessionState) -> None: ... + def observe_writer_stats(self, stream_id: str, stats: WriterStats) -> None: ... + def tick(self, now_ns: int) -> Iterable[HealthEvent]: ... + def close_condition(self, incident: Incident, now_ns: int) -> bool: ... + + +class DetectorBase: + """No-op defaults for every Detector hook. + + Subclasses set ``name`` and ``default_severity`` at the class level + and override only the hooks that matter for their rule. + """ + + name: str + default_severity: Severity + + def __init_subclass__(cls, **kwargs: object) -> None: + super().__init_subclass__(**kwargs) + # Guard against subclasses that forget to set required class attrs. + for attr in ("name", "default_severity"): + if not hasattr(cls, attr): + raise TypeError( + f"Detector subclass {cls.__name__} must set class attribute '{attr}'" + ) + + def __new__(cls, *args: object, **kwargs: object) -> "DetectorBase": + if cls is DetectorBase: + raise TypeError("DetectorBase is abstract; subclass it") + return super().__new__(cls) + + def observe_sample(self, stream_id: str, sample: SampleEvent) -> None: + pass + + def observe_health(self, stream_id: str, event: HealthEvent) -> None: + pass + + def observe_state(self, old: SessionState, new: SessionState) -> None: + pass + + def observe_writer_stats(self, stream_id: str, stats: WriterStats) -> None: + pass + + def tick(self, now_ns: int) -> Iterator[HealthEvent]: + return iter(()) + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + # Conservative default: keep open. Subclasses override to close. + return False +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +pytest tests/unit/health/test_detector_base.py -v +``` +Expected: 2 passed. + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/detector.py tests/unit/health/test_detector_base.py +git commit -m "feat(health): add Detector protocol + DetectorBase" +``` + +--- + +### Task 5: DetectorRegistry + +**Files:** +- Create: `src/syncfield/health/registry.py` +- Test: `tests/unit/health/test_registry.py` + +- [ ] **Step 1: Write failing test** + +`tests/unit/health/test_registry.py`: + +```python +import pytest + +from syncfield.health.detector import DetectorBase +from syncfield.health.registry import DetectorRegistry +from syncfield.health.severity import Severity + + +class Det(DetectorBase): + name = "d1" + default_severity = Severity.WARNING + + +class Det2(DetectorBase): + name = "d2" + default_severity = Severity.ERROR + + +def test_register_and_iterate(): + reg = DetectorRegistry() + d1 = Det() + d2 = Det2() + reg.register(d1) + reg.register(d2) + assert list(reg) == [d1, d2] + + +def test_register_duplicate_name_raises(): + reg = DetectorRegistry() + reg.register(Det()) + with pytest.raises(ValueError, match="already registered"): + reg.register(Det()) + + +def test_unregister_removes_by_name(): + reg = DetectorRegistry() + d1 = Det() + reg.register(d1) + reg.unregister("d1") + assert list(reg) == [] + + +def test_unregister_unknown_is_noop(): + reg = DetectorRegistry() + reg.unregister("nope") # does not raise +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/health/test_registry.py -v +``` +Expected: `ModuleNotFoundError`. + +- [ ] **Step 3: Implement** + +`src/syncfield/health/registry.py`: + +```python +"""Registry of active Detectors for a session.""" + +from __future__ import annotations + +from typing import Iterator, List + +from syncfield.health.detector import Detector + + +class DetectorRegistry: + def __init__(self) -> None: + self._detectors: List[Detector] = [] + + def register(self, detector: Detector) -> None: + if any(d.name == detector.name for d in self._detectors): + raise ValueError(f"Detector '{detector.name}' is already registered") + self._detectors.append(detector) + + def unregister(self, name: str) -> None: + self._detectors = [d for d in self._detectors if d.name != name] + + def __iter__(self) -> Iterator[Detector]: + return iter(list(self._detectors)) + + def __len__(self) -> int: + return len(self._detectors) +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +pytest tests/unit/health/test_registry.py -v +``` +Expected: 4 passed. + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/registry.py tests/unit/health/test_registry.py +git commit -m "feat(health): add DetectorRegistry" +``` + +--- + +### Task 6: IncidentTracker + +**Files:** +- Create: `src/syncfield/health/tracker.py` +- Test: `tests/unit/health/test_incident_tracker.py` + +`IncidentTracker` owns the open/close lifecycle for all incidents in a session. It never runs in user code directly — the `HealthWorker` owns and drives it. + +- [ ] **Step 1: Write failing test** + +`tests/unit/health/test_incident_tracker.py`: + +```python +from typing import List + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.tracker import IncidentTracker +from syncfield.health.types import Incident +from syncfield.types import HealthEvent, HealthEventKind + + +def _ev(at_ns: int, fingerprint: str = "cam:stall", severity: Severity = Severity.ERROR, + detail: str = "x") -> HealthEvent: + return HealthEvent( + stream_id="cam", + kind=HealthEventKind.ERROR, + at_ns=at_ns, + detail=detail, + severity=severity, + source="detector:stream-stall", + fingerprint=fingerprint, + ) + + +class AlwaysCloseAfter(DetectorBase): + name = "stream-stall" + default_severity = Severity.ERROR + + def __init__(self, close_after_ns: int) -> None: + self._close_after = close_after_ns + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + return now_ns - incident.last_event_at_ns >= self._close_after + + +def test_tracker_opens_incident_on_first_matching_event(): + tr = IncidentTracker() + tr.bind_detector(AlwaysCloseAfter(close_after_ns=1000)) + opened: List[Incident] = [] + tr.on_opened = opened.append + + tr.ingest(_ev(100)) + + assert len(tr.open_incidents()) == 1 + assert opened and opened[0].stream_id == "cam" + + +def test_tracker_groups_same_fingerprint_into_one_incident(): + tr = IncidentTracker() + tr.bind_detector(AlwaysCloseAfter(close_after_ns=1_000_000_000)) + + tr.ingest(_ev(100, severity=Severity.WARNING)) + tr.ingest(_ev(200, severity=Severity.ERROR)) # escalate + tr.ingest(_ev(300, severity=Severity.ERROR)) + + opens = tr.open_incidents() + assert len(opens) == 1 + inc = opens[0] + assert inc.event_count == 3 + assert inc.severity == Severity.ERROR + assert inc.last_event_at_ns == 300 + + +def test_tracker_closes_incident_when_detector_close_condition_fires(): + tr = IncidentTracker() + tr.bind_detector(AlwaysCloseAfter(close_after_ns=500)) + closed: List[Incident] = [] + tr.on_closed = closed.append + + tr.ingest(_ev(100)) + tr.tick(now_ns=200) # 100 ns since last event, not yet + assert tr.resolved_incidents() == [] + + tr.tick(now_ns=700) # 600 ns since last event → close + assert len(tr.resolved_incidents()) == 1 + assert tr.open_incidents() == [] + assert closed and closed[0].closed_at_ns == 700 + + +def test_tracker_reopens_after_close_on_same_fingerprint(): + tr = IncidentTracker() + tr.bind_detector(AlwaysCloseAfter(close_after_ns=100)) + + tr.ingest(_ev(100)) + tr.tick(now_ns=500) # closed + assert tr.open_incidents() == [] + + tr.ingest(_ev(1000)) # new incident, new id + opens = tr.open_incidents() + assert len(opens) == 1 + assert len(tr.resolved_incidents()) == 1 + assert opens[0].id != tr.resolved_incidents()[0].id + + +def test_tracker_unbound_fingerprint_falls_back_to_passthrough_close(): + # When an event arrives with a fingerprint whose detector is not bound, + # the tracker still groups it, using the default passthrough close + # window (30s of quiet). + tr = IncidentTracker(passthrough_close_ns=500) + + tr.ingest(_ev(100, fingerprint="cam:adapter:xlink")) + tr.tick(now_ns=400) + assert tr.open_incidents() + tr.tick(now_ns=1000) # 900 ns since last event → closes + assert tr.resolved_incidents() + + +def test_tracker_flush_callbacks_fire_on_update_too(): + tr = IncidentTracker() + tr.bind_detector(AlwaysCloseAfter(close_after_ns=1_000_000_000)) + updates: List[Incident] = [] + tr.on_updated = updates.append + + tr.ingest(_ev(100)) # opens + tr.ingest(_ev(200)) # updates + tr.ingest(_ev(300)) # updates + assert len(updates) == 2 +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/health/test_incident_tracker.py -v +``` +Expected: `ModuleNotFoundError`. + +- [ ] **Step 3: Implement** + +`src/syncfield/health/tracker.py`: + +```python +"""IncidentTracker — groups HealthEvents into Incidents and manages open/close. + +Runs on the HealthWorker thread. Public methods are *not* thread-safe on +their own; the worker serializes access. +""" + +from __future__ import annotations + +from typing import Callable, Dict, List, Optional + +from syncfield.health.detector import Detector +from syncfield.health.types import Incident +from syncfield.types import HealthEvent + +Callback = Callable[[Incident], None] + + +class IncidentTracker: + def __init__(self, passthrough_close_ns: int = 30 * 1_000_000_000) -> None: + self._by_fingerprint: Dict[str, Incident] = {} + self._resolved: List[Incident] = [] + self._detectors_by_name: Dict[str, Detector] = {} + self._passthrough_close_ns = passthrough_close_ns + + self.on_opened: Optional[Callback] = None + self.on_updated: Optional[Callback] = None + self.on_closed: Optional[Callback] = None + + # --- detector wiring ------------------------------------------------- + + def bind_detector(self, detector: Detector) -> None: + self._detectors_by_name[detector.name] = detector + + # --- event ingestion ------------------------------------------------- + + def ingest(self, event: HealthEvent) -> None: + open_inc = self._by_fingerprint.get(event.fingerprint) + if open_inc is None: + inc = Incident.opened_from(event, title=_title_from(event)) + self._by_fingerprint[event.fingerprint] = inc + self._fire(self.on_opened, inc) + return + open_inc.record_event(event) + self._fire(self.on_updated, open_inc) + + # --- tick — evaluate close conditions -------------------------------- + + def tick(self, now_ns: int) -> None: + to_close: List[str] = [] + for fp, inc in self._by_fingerprint.items(): + detector = self._detector_for(inc) + if detector is not None: + should_close = detector.close_condition(inc, now_ns) + else: + should_close = (now_ns - inc.last_event_at_ns) >= self._passthrough_close_ns + if should_close: + to_close.append(fp) + + for fp in to_close: + inc = self._by_fingerprint.pop(fp) + inc.close(at_ns=now_ns) + self._resolved.append(inc) + self._fire(self.on_closed, inc) + + def close_all(self, *, at_ns: int) -> None: + """Used at session stop to resolve any still-open incidents.""" + for fp in list(self._by_fingerprint.keys()): + inc = self._by_fingerprint.pop(fp) + inc.close(at_ns=at_ns) + self._resolved.append(inc) + self._fire(self.on_closed, inc) + + # --- read-only views ------------------------------------------------- + + def open_incidents(self) -> List[Incident]: + return list(self._by_fingerprint.values()) + + def resolved_incidents(self) -> List[Incident]: + return list(self._resolved) + + # --- helpers --------------------------------------------------------- + + def _detector_for(self, inc: Incident) -> Optional[Detector]: + # Fingerprint convention: ":[:suffix]". + parts = inc.fingerprint.split(":", 2) + if len(parts) < 2: + return None + return self._detectors_by_name.get(parts[1]) + + @staticmethod + def _fire(cb: Optional[Callback], inc: Incident) -> None: + if cb is not None: + cb(inc) + + +def _title_from(event: HealthEvent) -> str: + # Prefer the first event's detail as the title; fall back to + # ": " if the detail is missing. + if event.detail: + return event.detail + return f"{event.source}: {event.fingerprint}" +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +pytest tests/unit/health/test_incident_tracker.py -v +``` +Expected: 6 passed. + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/tracker.py tests/unit/health/test_incident_tracker.py +git commit -m "feat(health): add IncidentTracker with fingerprint grouping and close conditions" +``` + +--- + +### Task 7: HealthWorker (thread + ingress queues) + +**Files:** +- Create: `src/syncfield/health/worker.py` +- Test: `tests/unit/health/test_health_worker.py` + +The worker owns the thread, ingress queues, detector polling, and tracker. It is started on session `start()` and stopped on `stop()`. + +- [ ] **Step 1: Write failing test** + +`tests/unit/health/test_health_worker.py`: + +```python +import threading +import time + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.tracker import IncidentTracker +from syncfield.health.types import WriterStats +from syncfield.health.worker import HealthWorker +from syncfield.types import HealthEvent, HealthEventKind, SampleEvent, SessionState + + +class RecordingDetector(DetectorBase): + name = "recorder" + default_severity = Severity.INFO + + def __init__(self) -> None: + self.samples = [] + self.healths = [] + self.states = [] + self.writer_stats = [] + self.ticks = 0 + + def observe_sample(self, stream_id, sample): + self.samples.append((stream_id, sample.capture_ns)) + + def observe_health(self, stream_id, event): + self.healths.append((stream_id, event.at_ns)) + + def observe_state(self, old, new): + self.states.append((old, new)) + + def observe_writer_stats(self, stream_id, stats): + self.writer_stats.append((stream_id, stats.queue_depth)) + + def tick(self, now_ns): + self.ticks += 1 + return iter(()) + + +def _wait_until(pred, timeout=1.0, interval=0.01): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if pred(): + return True + time.sleep(interval) + return False + + +def test_worker_drains_all_ingress_queues_on_tick(): + tr = IncidentTracker() + det = RecordingDetector() + tr.bind_detector(det) + w = HealthWorker(tracker=tr, detectors=[det], tick_hz=100) + + w.start() + try: + w.push_sample("cam", SampleEvent(stream_id="cam", frame_number=1, capture_ns=42)) + w.push_health("cam", HealthEvent(stream_id="cam", kind=HealthEventKind.WARNING, at_ns=1)) + w.push_state(SessionState.IDLE, SessionState.CONNECTED) + w.push_writer_stats("cam", WriterStats("cam", 1, 2, 16, 0)) + + assert _wait_until(lambda: det.samples and det.healths and det.states and det.writer_stats) + finally: + w.stop() + + assert det.samples[0] == ("cam", 42) + assert det.healths[0] == ("cam", 1) + assert det.states[0] == (SessionState.IDLE, SessionState.CONNECTED) + assert det.writer_stats[0] == ("cam", 2) + + +def test_worker_ticks_at_roughly_configured_rate(): + tr = IncidentTracker() + det = RecordingDetector() + w = HealthWorker(tracker=tr, detectors=[det], tick_hz=50) + w.start() + try: + time.sleep(0.2) # ~10 ticks + finally: + w.stop() + # Loose bound to avoid flakiness under loaded CI. + assert det.ticks >= 5 + + +def test_worker_feeds_detector_tick_output_into_tracker(): + class EmitsOneAndDone(DetectorBase): + name = "emit" + default_severity = Severity.WARNING + + def __init__(self): + self.fired = False + + def tick(self, now_ns): + if self.fired: + return iter(()) + self.fired = True + return iter([HealthEvent( + stream_id="cam", kind=HealthEventKind.WARNING, at_ns=now_ns, + detail="synthetic", severity=Severity.WARNING, + source="detector:emit", fingerprint="cam:emit", + )]) + + def close_condition(self, inc, now_ns): + return False + + tr = IncidentTracker() + det = EmitsOneAndDone() + tr.bind_detector(det) + w = HealthWorker(tracker=tr, detectors=[det], tick_hz=100) + w.start() + try: + assert _wait_until(lambda: len(tr.open_incidents()) == 1) + finally: + w.stop() + + +def test_worker_stop_is_idempotent(): + tr = IncidentTracker() + det = RecordingDetector() + w = HealthWorker(tracker=tr, detectors=[det], tick_hz=50) + w.start() + w.stop() + w.stop() # does not raise +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/health/test_health_worker.py -v +``` +Expected: `ModuleNotFoundError`. + +- [ ] **Step 3: Implement** + +`src/syncfield/health/worker.py`: + +```python +"""HealthWorker — the dedicated thread that drives detectors + tracker. + +Capture threads push samples / health events / state transitions / +writer stats into :class:`queue.SimpleQueue`\\ s. The worker drains them +every tick, fans out to each registered Detector, runs each Detector's +``tick`` to emit synthetic events, and feeds everything into the +IncidentTracker. +""" + +from __future__ import annotations + +import queue +import threading +import time +from dataclasses import dataclass +from typing import Iterable, List, Tuple + +from syncfield.health.detector import Detector +from syncfield.health.tracker import IncidentTracker +from syncfield.health.types import WriterStats +from syncfield.types import HealthEvent, SampleEvent, SessionState + + +@dataclass(frozen=True) +class _SampleMsg: + stream_id: str + sample: SampleEvent + + +@dataclass(frozen=True) +class _HealthMsg: + stream_id: str + event: HealthEvent + + +@dataclass(frozen=True) +class _StateMsg: + old: SessionState + new: SessionState + + +@dataclass(frozen=True) +class _WriterStatsMsg: + stream_id: str + stats: WriterStats + + +class HealthWorker: + def __init__( + self, + *, + tracker: IncidentTracker, + detectors: Iterable[Detector], + tick_hz: float = 20.0, + ) -> None: + self._tracker = tracker + self._detectors: List[Detector] = list(detectors) + self._tick_interval = 1.0 / tick_hz + + self._samples: "queue.SimpleQueue[_SampleMsg]" = queue.SimpleQueue() + self._healths: "queue.SimpleQueue[_HealthMsg]" = queue.SimpleQueue() + self._states: "queue.SimpleQueue[_StateMsg]" = queue.SimpleQueue() + self._writer_stats: "queue.SimpleQueue[_WriterStatsMsg]" = queue.SimpleQueue() + + self._stop = threading.Event() + self._thread: threading.Thread | None = None + + # --- ingress (called from capture threads) --------------------------- + + def push_sample(self, stream_id: str, sample: SampleEvent) -> None: + self._samples.put(_SampleMsg(stream_id, sample)) + + def push_health(self, stream_id: str, event: HealthEvent) -> None: + self._healths.put(_HealthMsg(stream_id, event)) + + def push_state(self, old: SessionState, new: SessionState) -> None: + self._states.put(_StateMsg(old, new)) + + def push_writer_stats(self, stream_id: str, stats: WriterStats) -> None: + self._writer_stats.put(_WriterStatsMsg(stream_id, stats)) + + # --- lifecycle ------------------------------------------------------- + + def start(self) -> None: + if self._thread is not None and self._thread.is_alive(): + return + self._stop.clear() + self._thread = threading.Thread( + target=self._run, name="syncfield-health", daemon=True + ) + self._thread.start() + + def stop(self) -> None: + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=2.0) + self._thread = None + + # --- main loop ------------------------------------------------------- + + def _run(self) -> None: + next_deadline = time.monotonic() + while not self._stop.is_set(): + self._drain_once() + self._fire_detector_ticks() + self._tracker.tick(now_ns=time.monotonic_ns()) + + next_deadline += self._tick_interval + sleep_for = next_deadline - time.monotonic() + if sleep_for > 0: + # Use Event.wait so stop() can interrupt immediately. + self._stop.wait(timeout=sleep_for) + else: + # Running behind; reset the schedule anchor. + next_deadline = time.monotonic() + + # Drain any straggling messages after stop so tests see a consistent state. + self._drain_once() + + def _drain_once(self) -> None: + for msg in _drain_queue(self._samples): + for d in self._detectors: + d.observe_sample(msg.stream_id, msg.sample) + for msg in _drain_queue(self._healths): + for d in self._detectors: + d.observe_health(msg.stream_id, msg.event) + self._tracker.ingest(msg.event) + for msg in _drain_queue(self._states): + for d in self._detectors: + d.observe_state(msg.old, msg.new) + for msg in _drain_queue(self._writer_stats): + for d in self._detectors: + d.observe_writer_stats(msg.stream_id, msg.stats) + + def _fire_detector_ticks(self) -> None: + now = time.monotonic_ns() + for d in self._detectors: + for event in d.tick(now): + self._tracker.ingest(event) + + +def _drain_queue(q: "queue.SimpleQueue") -> List: + out = [] + while True: + try: + out.append(q.get_nowait()) + except queue.Empty: + return out +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +pytest tests/unit/health/test_health_worker.py -v +``` +Expected: 4 passed. + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/worker.py tests/unit/health/test_health_worker.py +git commit -m "feat(health): add HealthWorker (daemon thread with ingress queues and tick loop)" +``` + +--- + +### Task 8: HealthSystem facade + +**Files:** +- Create: `src/syncfield/health/system.py` +- Modify: `src/syncfield/health/__init__.py` — public API exports +- Test: `tests/unit/health/test_health_system.py` + +- [ ] **Step 1: Write failing test** + +`tests/unit/health/test_health_system.py`: + +```python +from syncfield.health import HealthSystem, Severity +from syncfield.health.detector import DetectorBase +from syncfield.types import HealthEvent, HealthEventKind, SampleEvent, SessionState + + +class Custom(DetectorBase): + name = "custom" + default_severity = Severity.WARNING + + +def test_health_system_boots_and_accepts_inputs(): + hs = HealthSystem() + hs.start() + try: + hs.observe_sample("cam", SampleEvent(stream_id="cam", frame_number=1, capture_ns=1)) + hs.observe_health("cam", HealthEvent(stream_id="cam", kind=HealthEventKind.WARNING, at_ns=1)) + hs.observe_state(SessionState.IDLE, SessionState.CONNECTED) + finally: + hs.stop() + + +def test_health_system_register_and_unregister(): + hs = HealthSystem() + d = Custom() + hs.register(d) + assert any(x.name == "custom" for x in hs.iter_detectors()) + hs.unregister("custom") + assert not any(x.name == "custom" for x in hs.iter_detectors()) + + +def test_health_system_installs_default_detectors(): + hs = HealthSystem() + names = {d.name for d in hs.iter_detectors()} + for expected in ( + "adapter-passthrough", + "stream-stall", + "fps-drop", + "jitter", + "startup-failure", + "backpressure", + ): + assert expected in names, f"missing default detector: {expected}" + + +def test_health_system_callbacks_fire_on_open_and_close(): + hs = HealthSystem(passthrough_close_ns=1) # close instantly for the test + opened, closed = [], [] + hs.on_incident_opened = opened.append + hs.on_incident_closed = closed.append + + hs.start() + try: + hs.observe_health("cam", HealthEvent( + stream_id="cam", kind=HealthEventKind.ERROR, at_ns=1, + severity=Severity.ERROR, source="adapter:test", + fingerprint="cam:adapter:xlink-error", + )) + import time + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline: + if opened and closed: + break + time.sleep(0.02) + finally: + hs.stop() + assert opened, "incident was not opened" + assert closed, "incident was not closed" +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/health/test_health_system.py -v +``` +Expected: `ImportError: cannot import name 'HealthSystem'`. + +- [ ] **Step 3: Implement** + +`src/syncfield/health/system.py`: + +```python +"""HealthSystem — the single handle the orchestrator + user code touch.""" + +from __future__ import annotations + +from typing import Callable, Iterable, Iterator, Optional + +from syncfield.health.detector import Detector +from syncfield.health.detectors.adapter_passthrough import AdapterEventPassthrough +from syncfield.health.detectors.backpressure import BackpressureDetector +from syncfield.health.detectors.fps_drop import FpsDropDetector +from syncfield.health.detectors.jitter import JitterDetector +from syncfield.health.detectors.startup_failure import StartupFailureDetector +from syncfield.health.detectors.stream_stall import StreamStallDetector +from syncfield.health.registry import DetectorRegistry +from syncfield.health.tracker import IncidentTracker +from syncfield.health.types import Incident, WriterStats +from syncfield.health.worker import HealthWorker +from syncfield.types import HealthEvent, SampleEvent, SessionState + + +class HealthSystem: + """Composes Registry + Tracker + Worker into a single user-facing facade.""" + + def __init__( + self, + *, + tick_hz: float = 20.0, + passthrough_close_ns: int = 30 * 1_000_000_000, + ) -> None: + self._registry = DetectorRegistry() + self._tracker = IncidentTracker(passthrough_close_ns=passthrough_close_ns) + self._worker: Optional[HealthWorker] = None + self._tick_hz = tick_hz + + self.on_incident_opened: Optional[Callable[[Incident], None]] = None + self.on_incident_updated: Optional[Callable[[Incident], None]] = None + self.on_incident_closed: Optional[Callable[[Incident], None]] = None + + self._tracker.on_opened = lambda inc: self._fire("on_incident_opened", inc) + self._tracker.on_updated = lambda inc: self._fire("on_incident_updated", inc) + self._tracker.on_closed = lambda inc: self._fire("on_incident_closed", inc) + + self._install_default_detectors() + + # --- registry -------------------------------------------------------- + + def register(self, detector: Detector) -> None: + self._registry.register(detector) + self._tracker.bind_detector(detector) + + def unregister(self, name: str) -> None: + self._registry.unregister(name) + + def iter_detectors(self) -> Iterator[Detector]: + return iter(self._registry) + + # --- observer inputs ------------------------------------------------- + + def observe_sample(self, stream_id: str, sample: SampleEvent) -> None: + if self._worker is not None: + self._worker.push_sample(stream_id, sample) + + def observe_health(self, stream_id: str, event: HealthEvent) -> None: + if self._worker is not None: + self._worker.push_health(stream_id, event) + + def observe_state(self, old: SessionState, new: SessionState) -> None: + if self._worker is not None: + self._worker.push_state(old, new) + + def observe_writer_stats(self, stream_id: str, stats: WriterStats) -> None: + if self._worker is not None: + self._worker.push_writer_stats(stream_id, stats) + + # --- lifecycle ------------------------------------------------------- + + def start(self) -> None: + self._worker = HealthWorker( + tracker=self._tracker, + detectors=list(self._registry), + tick_hz=self._tick_hz, + ) + self._worker.start() + + def stop(self, *, close_open_incidents: bool = True, now_ns: Optional[int] = None) -> None: + if self._worker is not None: + self._worker.stop() + self._worker = None + if close_open_incidents: + import time + self._tracker.close_all(at_ns=now_ns if now_ns is not None else time.monotonic_ns()) + + # --- read-only views ------------------------------------------------- + + def open_incidents(self) -> Iterable[Incident]: + return self._tracker.open_incidents() + + def resolved_incidents(self) -> Iterable[Incident]: + return self._tracker.resolved_incidents() + + # --- helpers --------------------------------------------------------- + + def _install_default_detectors(self) -> None: + self.register(AdapterEventPassthrough()) + self.register(StreamStallDetector()) + self.register(FpsDropDetector()) + self.register(JitterDetector()) + self.register(StartupFailureDetector()) + self.register(BackpressureDetector()) + + def _fire(self, attr: str, inc: Incident) -> None: + cb = getattr(self, attr, None) + if cb is not None: + cb(inc) +``` + +Public API in `src/syncfield/health/__init__.py`: + +```python +"""syncfield.health — platform health telemetry.""" + +from syncfield.health.detector import Detector, DetectorBase +from syncfield.health.registry import DetectorRegistry +from syncfield.health.severity import Severity, max_severity +from syncfield.health.system import HealthSystem +from syncfield.health.tracker import IncidentTracker +from syncfield.health.types import ( + Incident, + IncidentArtifact, + IncidentSnapshot, + WriterStats, +) + +__all__ = [ + "Detector", + "DetectorBase", + "DetectorRegistry", + "HealthSystem", + "Incident", + "IncidentArtifact", + "IncidentSnapshot", + "IncidentTracker", + "Severity", + "WriterStats", + "max_severity", +] +``` + +**Note**: because `HealthSystem._install_default_detectors` imports all six default detector modules, those modules must exist *before* this task's tests can pass. Implement Tasks 9–14 first, or inline placeholder classes temporarily. **Correct order**: skip this task's implementation step (the test) until after Tasks 9–14. Proceed to Task 9 now; return here for the implementation + verification after the detectors exist. + +- [ ] **Step 4: Defer implementation — skip to Task 9** + +Revisit after Tasks 9–14 land. Run then: + +```bash +pytest tests/unit/health/test_health_system.py -v +``` +Expected: 4 passed. + +- [ ] **Step 5: Commit (after Tasks 9–14 complete)** + +```bash +git add src/syncfield/health/system.py src/syncfield/health/__init__.py tests/unit/health/test_health_system.py +git commit -m "feat(health): add HealthSystem facade with default detector install + lifecycle" +``` + +--- + +## Phase 2 — Platform detectors + +### Task 9: AdapterEventPassthrough + +**Files:** +- Create: `src/syncfield/health/detectors/__init__.py` (empty) +- Create: `src/syncfield/health/detectors/adapter_passthrough.py` +- Test: `tests/unit/health/detectors/__init__.py` (empty), `tests/unit/health/detectors/test_adapter_passthrough.py` + +The passthrough does not emit events from `tick()`; it exists so adapter-emitted events that start with `source="adapter:..."` (fingerprints like `cam:adapter:xlink-error`) still have a Detector for the tracker to consult on close — using a stale-quiet window. + +- [ ] **Step 1: Write failing test** + +```python +# tests/unit/health/detectors/test_adapter_passthrough.py +from syncfield.health.detectors.adapter_passthrough import AdapterEventPassthrough +from syncfield.health.severity import Severity +from syncfield.health.types import Incident +from syncfield.types import HealthEvent, HealthEventKind + + +def _adapter_ev(at_ns: int) -> HealthEvent: + return HealthEvent( + stream_id="cam", + kind=HealthEventKind.ERROR, + at_ns=at_ns, + detail="x", + severity=Severity.ERROR, + source="adapter:oak", + fingerprint="cam:adapter:xlink-error", + ) + + +def test_tick_emits_nothing(): + d = AdapterEventPassthrough() + assert list(d.tick(now_ns=1000)) == [] + + +def test_close_condition_respects_quiet_window(): + d = AdapterEventPassthrough(quiet_ns=500) + inc = Incident.opened_from(_adapter_ev(100), title="x") + assert d.close_condition(inc, now_ns=400) is False # 300 < 500 + inc.record_event(_adapter_ev(900)) + assert d.close_condition(inc, now_ns=1000) is False # 100 < 500 + assert d.close_condition(inc, now_ns=1500) is True # 600 >= 500 +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/health/detectors/test_adapter_passthrough.py -v +``` +Expected: `ModuleNotFoundError`. + +- [ ] **Step 3: Implement** + +`src/syncfield/health/detectors/adapter_passthrough.py`: + +```python +"""Passthrough detector: owns close semantics for adapter-emitted events. + +Fingerprint convention ``:adapter:`` routes to this +detector in the tracker. It never emits synthetic events; its only job +is saying "if no new adapter event arrived for N seconds, close the +incident". +""" + +from __future__ import annotations + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.types import Incident + + +class AdapterEventPassthrough(DetectorBase): + name = "adapter" + default_severity = Severity.WARNING + + def __init__(self, quiet_ns: int = 30 * 1_000_000_000) -> None: + self._quiet_ns = quiet_ns + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + return (now_ns - incident.last_event_at_ns) >= self._quiet_ns +``` + +**Note**: the fingerprint middle-token is `adapter`, not the detector's own name, so the tracker's `_detector_for` lookup (`parts[1]`) finds this class by `name="adapter"`. + +- [ ] **Step 4: Run, confirm pass** + +```bash +pytest tests/unit/health/detectors/test_adapter_passthrough.py -v +``` +Expected: 2 passed. + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/detectors/ tests/unit/health/detectors/ +git commit -m "feat(health): add AdapterEventPassthrough detector" +``` + +--- + +### Task 10: StreamStallDetector + +**Files:** +- Create: `src/syncfield/health/detectors/stream_stall.py` +- Test: `tests/unit/health/detectors/test_stream_stall.py` + +Fires when a stream that previously produced samples has been silent for ≥ `stall_threshold_ns` (default 2.0 s). Closes after samples flow for ≥ `recovery_ns` (default 1.0 s). + +- [ ] **Step 1: Write failing test** + +```python +# tests/unit/health/detectors/test_stream_stall.py +from syncfield.health.detectors.stream_stall import StreamStallDetector +from syncfield.health.types import Incident +from syncfield.types import SampleEvent + + +def _sample(stream_id: str, capture_ns: int) -> SampleEvent: + return SampleEvent(stream_id=stream_id, frame_number=1, capture_ns=capture_ns) + + +def test_no_fire_before_seeing_any_sample(): + d = StreamStallDetector(stall_threshold_ns=1000) + assert list(d.tick(now_ns=10_000)) == [] + + +def test_fires_when_silent_longer_than_threshold(): + d = StreamStallDetector(stall_threshold_ns=1000) + d.observe_sample("cam", _sample("cam", capture_ns=100)) + events = list(d.tick(now_ns=2000)) # 1900 ns of silence + assert len(events) == 1 + ev = events[0] + assert ev.stream_id == "cam" + assert ev.fingerprint == "cam:stream-stall" + assert ev.source == "detector:stream-stall" + assert "silence" in (ev.detail or "").lower() + + +def test_does_not_refire_while_still_stalled(): + d = StreamStallDetector(stall_threshold_ns=1000) + d.observe_sample("cam", _sample("cam", capture_ns=100)) + fired_once = list(d.tick(now_ns=2000)) + fired_twice = list(d.tick(now_ns=3000)) + assert len(fired_once) == 1 + assert len(fired_twice) == 0 + + +def test_refires_after_recovery_then_new_stall(): + d = StreamStallDetector(stall_threshold_ns=1000, recovery_ns=500) + d.observe_sample("cam", _sample("cam", capture_ns=0)) + list(d.tick(now_ns=2000)) # fires stall + + # recovery: samples flow for ≥ recovery_ns + for t in range(3000, 4100, 100): + d.observe_sample("cam", _sample("cam", capture_ns=t)) + inc = Incident.opened_from(list(d.tick(now_ns=4100))[0], title="x") \ + if False else None # placeholder; close_condition test follows + # Now silence again. + new_events = list(d.tick(now_ns=6000)) + assert len(new_events) == 1 # second stall → new event + + +def test_close_condition_requires_recent_sample_flow(): + d = StreamStallDetector(stall_threshold_ns=1000, recovery_ns=500) + d.observe_sample("cam", _sample("cam", capture_ns=0)) + events = list(d.tick(now_ns=2000)) + inc = Incident.opened_from(events[0], title="x") + + # Still silent — do not close. + assert d.close_condition(inc, now_ns=2500) is False + + # Samples arrive across a 600 ns window → recovery_ns=500 satisfied. + d.observe_sample("cam", _sample("cam", capture_ns=2600)) + d.observe_sample("cam", _sample("cam", capture_ns=3200)) + assert d.close_condition(inc, now_ns=3300) is True + + +def test_per_stream_independent_state(): + d = StreamStallDetector(stall_threshold_ns=1000) + d.observe_sample("a", _sample("a", capture_ns=100)) + d.observe_sample("b", _sample("b", capture_ns=100)) + # stream b stays alive + d.observe_sample("b", _sample("b", capture_ns=1800)) + events = list(d.tick(now_ns=2500)) + assert len(events) == 1 + assert events[0].stream_id == "a" +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/health/detectors/test_stream_stall.py -v +``` + +- [ ] **Step 3: Implement** + +`src/syncfield/health/detectors/stream_stall.py`: + +```python +"""StreamStallDetector — fires when a stream stops producing samples.""" + +from __future__ import annotations + +from typing import Dict, Iterator, List + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.types import Incident +from syncfield.types import HealthEvent, HealthEventKind, SampleEvent + + +class StreamStallDetector(DetectorBase): + name = "stream-stall" + default_severity = Severity.ERROR + + def __init__( + self, + stall_threshold_ns: int = 2_000_000_000, + recovery_ns: int = 1_000_000_000, + ) -> None: + self._stall_threshold_ns = stall_threshold_ns + self._recovery_ns = recovery_ns + # Per-stream most-recent sample monotonic time. + self._last_sample_at: Dict[str, int] = {} + # Per-stream: are we currently firing? prevents duplicates per stall. + self._stall_active: Dict[str, bool] = {} + + # --- observers ------------------------------------------------------- + + def observe_sample(self, stream_id: str, sample: SampleEvent) -> None: + self._last_sample_at[stream_id] = sample.capture_ns + # A new sample ends any active stall bookkeeping. + self._stall_active[stream_id] = False + + # --- tick ------------------------------------------------------------ + + def tick(self, now_ns: int) -> Iterator[HealthEvent]: + emitted: List[HealthEvent] = [] + for stream_id, last in self._last_sample_at.items(): + silence_ns = now_ns - last + if silence_ns >= self._stall_threshold_ns and not self._stall_active.get(stream_id, False): + self._stall_active[stream_id] = True + emitted.append(HealthEvent( + stream_id=stream_id, + kind=HealthEventKind.ERROR, + at_ns=now_ns, + detail=f"Stream stalled (silence {silence_ns / 1e9:.1f}s)", + severity=self.default_severity, + source=f"detector:{self.name}", + fingerprint=f"{stream_id}:{self.name}", + data={"silence_ns": silence_ns}, + )) + return iter(emitted) + + # --- close condition ------------------------------------------------- + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + last = self._last_sample_at.get(incident.stream_id) + if last is None: + return False + return (now_ns - last) < self._stall_threshold_ns \ + and (now_ns - incident.last_event_at_ns) >= self._recovery_ns +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +pytest tests/unit/health/detectors/test_stream_stall.py -v +``` + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/detectors/stream_stall.py tests/unit/health/detectors/test_stream_stall.py +git commit -m "feat(health): add StreamStallDetector (platform-level silence detection)" +``` + +--- + +### Task 11: FpsDropDetector + +**Files:** +- Create: `src/syncfield/health/detectors/fps_drop.py` +- Test: `tests/unit/health/detectors/test_fps_drop.py` + +Maintains a per-stream rolling FPS estimate over a 1-second sliding window. Fires when the observed FPS has been below 70 % of target for ≥ 3 s. Without a declared `target_hz`, learns a baseline from the first 10 s of samples after a 5 s warmup. + +Since target_hz is declared by the Stream, the detector accepts a `targets: dict[str, float | None]` dependency it can query. `HealthSystem` supplies a getter. + +- [ ] **Step 1: Write failing test** + +```python +# tests/unit/health/detectors/test_fps_drop.py +from syncfield.health.detectors.fps_drop import FpsDropDetector +from syncfield.types import SampleEvent + + +def _s(stream: str, t_ns: int) -> SampleEvent: + return SampleEvent(stream_id=stream, frame_number=0, capture_ns=t_ns) + + +def test_no_fire_if_fps_tracks_target(): + d = FpsDropDetector(target_getter=lambda sid: 30.0) + # emit 30 samples over 1 second + for i in range(30): + d.observe_sample("cam", _s("cam", i * int(1e9 / 30))) + assert list(d.tick(now_ns=int(1.1e9))) == [] + + +def test_fires_when_observed_below_70_percent_for_3s(): + d = FpsDropDetector( + target_getter=lambda sid: 30.0, + drop_ratio=0.70, + sustain_ns=3 * 1_000_000_000, + ) + + # 10 fps for 3.5 seconds — fps is 10, target 30, ratio 0.33. + interval = int(1e9 / 10) + t = 0 + while t <= int(3.5e9): + d.observe_sample("cam", _s("cam", t)) + t += interval + + emitted = list(d.tick(now_ns=int(3.6e9))) + assert len(emitted) == 1 + assert emitted[0].fingerprint == "cam:fps-drop" + assert emitted[0].data["target_hz"] == 30.0 + assert emitted[0].data["observed_hz"] < 15.0 + + +def test_does_not_fire_without_target_before_warmup(): + d = FpsDropDetector( + target_getter=lambda sid: None, + baseline_warmup_ns=5_000_000_000, + ) + # 10 fps, but only for 1s — under warmup. + t = 0 + for _ in range(10): + d.observe_sample("cam", _s("cam", t)) + t += int(1e8) + assert list(d.tick(now_ns=int(1.1e9))) == [] + + +def test_learns_baseline_then_fires_on_subsequent_drop(): + d = FpsDropDetector( + target_getter=lambda sid: None, + baseline_warmup_ns=1_000_000_000, + baseline_window_ns=2_000_000_000, + drop_ratio=0.7, + sustain_ns=1_000_000_000, + ) + # 3 s @ 30 fps → baseline ≈ 30. + t = 0 + while t <= int(3e9): + d.observe_sample("cam", _s("cam", t)) + t += int(1e9 / 30) + # 1.5 s of 10 fps → drop. + end = t + int(1.5e9) + while t <= end: + d.observe_sample("cam", _s("cam", t)) + t += int(1e8) + emitted = list(d.tick(now_ns=t)) + assert len(emitted) == 1 +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/health/detectors/test_fps_drop.py -v +``` + +- [ ] **Step 3: Implement** + +`src/syncfield/health/detectors/fps_drop.py`: + +```python +"""FpsDropDetector — target-relative or baseline-learning FPS drop detector.""" + +from __future__ import annotations + +from collections import deque +from typing import Callable, Deque, Dict, Iterator, List, Optional + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.types import Incident +from syncfield.types import HealthEvent, HealthEventKind, SampleEvent + +TargetGetter = Callable[[str], Optional[float]] + +_WINDOW_NS = 1_000_000_000 # rolling 1s FPS window + + +class FpsDropDetector(DetectorBase): + name = "fps-drop" + default_severity = Severity.WARNING + + def __init__( + self, + target_getter: TargetGetter = lambda sid: None, + drop_ratio: float = 0.70, + sustain_ns: int = 3_000_000_000, + recovery_ratio: float = 0.90, + recovery_ns: int = 5_000_000_000, + baseline_warmup_ns: int = 5_000_000_000, + baseline_window_ns: int = 10_000_000_000, + ) -> None: + self._target_getter = target_getter + self._drop_ratio = drop_ratio + self._sustain_ns = sustain_ns + self._recovery_ratio = recovery_ratio + self._recovery_ns = recovery_ns + self._baseline_warmup_ns = baseline_warmup_ns + self._baseline_window_ns = baseline_window_ns + + self._samples: Dict[str, Deque[int]] = {} + self._first_seen_at: Dict[str, int] = {} + self._baseline: Dict[str, float] = {} + # When did the stream first drop below threshold in the current dip? + self._dip_began_at: Dict[str, Optional[int]] = {} + self._fire_active: Dict[str, bool] = {} + # Same thing for recovery tracking. + self._recovery_began_at: Dict[str, Optional[int]] = {} + + # --- observers ------------------------------------------------------- + + def observe_sample(self, stream_id: str, sample: SampleEvent) -> None: + buf = self._samples.setdefault(stream_id, deque()) + buf.append(sample.capture_ns) + self._first_seen_at.setdefault(stream_id, sample.capture_ns) + # Trim older than baseline_window. + cutoff = sample.capture_ns - self._baseline_window_ns + while buf and buf[0] < cutoff: + buf.popleft() + + # --- tick ------------------------------------------------------------ + + def tick(self, now_ns: int) -> Iterator[HealthEvent]: + out: List[HealthEvent] = [] + for stream_id, buf in list(self._samples.items()): + target = self._effective_target(stream_id, now_ns) + observed = self._observed_fps(buf, now_ns) + if target is None or observed is None: + continue + + if observed < target * self._drop_ratio: + began = self._dip_began_at.get(stream_id) + if began is None: + self._dip_began_at[stream_id] = now_ns + began = now_ns + elapsed = now_ns - began + if elapsed >= self._sustain_ns and not self._fire_active.get(stream_id, False): + self._fire_active[stream_id] = True + out.append(HealthEvent( + stream_id=stream_id, + kind=HealthEventKind.WARNING, + at_ns=now_ns, + detail=f"FPS drop ({observed:.1f} Hz, target {target:.1f} Hz)", + severity=self.default_severity, + source=f"detector:{self.name}", + fingerprint=f"{stream_id}:{self.name}", + data={"observed_hz": observed, "target_hz": target}, + )) + else: + self._dip_began_at[stream_id] = None + self._fire_active[stream_id] = False + return iter(out) + + # --- close condition ------------------------------------------------- + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + stream_id = incident.stream_id + target = self._effective_target(stream_id, now_ns) + observed = self._observed_fps(self._samples.get(stream_id, deque()), now_ns) + if target is None or observed is None: + return False + if observed < target * self._recovery_ratio: + self._recovery_began_at[stream_id] = None + return False + began = self._recovery_began_at.get(stream_id) + if began is None: + self._recovery_began_at[stream_id] = now_ns + return False + return (now_ns - began) >= self._recovery_ns + + # --- helpers --------------------------------------------------------- + + def _effective_target(self, stream_id: str, now_ns: int) -> Optional[float]: + declared = self._target_getter(stream_id) + if declared is not None: + return float(declared) + first = self._first_seen_at.get(stream_id) + if first is None: + return None + if (now_ns - first) < self._baseline_warmup_ns: + return None + cached = self._baseline.get(stream_id) + if cached is not None: + return cached + observed = self._observed_fps(self._samples.get(stream_id, deque()), now_ns) + if observed is not None: + self._baseline[stream_id] = observed + return self._baseline.get(stream_id) + + @staticmethod + def _observed_fps(buf: Deque[int], now_ns: int) -> Optional[float]: + if not buf: + return None + cutoff = now_ns - _WINDOW_NS + count = sum(1 for t in buf if t >= cutoff) + if count == 0: + return 0.0 + return count / (_WINDOW_NS / 1e9) +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +pytest tests/unit/health/detectors/test_fps_drop.py -v +``` + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/detectors/fps_drop.py tests/unit/health/detectors/test_fps_drop.py +git commit -m "feat(health): add FpsDropDetector (target-relative or baseline-learning)" +``` + +--- + +### Task 12: JitterDetector + +**Files:** +- Create: `src/syncfield/health/detectors/jitter.py` +- Test: `tests/unit/health/detectors/test_jitter.py` + +Tracks last 60 inter-sample intervals per stream. Fires when p95 interval exceeds `jitter_ratio * expected_interval` (from target_hz) for `sustain_ns`. Closes when p95 returns to ≤ 1.2× expected for `recovery_ns`. + +- [ ] **Step 1: Write failing test** + +```python +# tests/unit/health/detectors/test_jitter.py +from syncfield.health.detectors.jitter import JitterDetector +from syncfield.types import SampleEvent + + +def _s(t_ns: int) -> SampleEvent: + return SampleEvent(stream_id="cam", frame_number=0, capture_ns=t_ns) + + +def test_steady_30hz_does_not_fire(): + d = JitterDetector(target_getter=lambda sid: 30.0) + step = int(1e9 / 30) + t = 0 + for _ in range(120): + d.observe_sample("cam", _s(t)) + t += step + assert list(d.tick(now_ns=t)) == [] + + +def test_irregular_intervals_fire_when_p95_exceeds_ratio(): + d = JitterDetector( + target_getter=lambda sid: 30.0, + jitter_ratio=2.0, + sustain_ns=500_000_000, + ) + step = int(1e9 / 30) + big = step * 4 # 4× target interval + t = 0 + # 60 samples alternating between normal and 4× intervals. + for i in range(60): + d.observe_sample("cam", _s(t)) + t += big if i % 2 == 0 else step + + # Give sustain time to elapse with more irregular samples. + for _ in range(20): + d.observe_sample("cam", _s(t)) + t += big + + emitted = list(d.tick(now_ns=t + 500_000_000)) + assert len(emitted) == 1 + assert emitted[0].fingerprint == "cam:jitter" +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/health/detectors/test_jitter.py -v +``` + +- [ ] **Step 3: Implement** + +`src/syncfield/health/detectors/jitter.py`: + +```python +"""JitterDetector — p95-based inter-sample interval anomaly detector.""" + +from __future__ import annotations + +from collections import deque +from typing import Callable, Deque, Dict, Iterator, List, Optional + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.types import Incident +from syncfield.types import HealthEvent, HealthEventKind, SampleEvent + +TargetGetter = Callable[[str], Optional[float]] + + +def _p95(values: List[int]) -> int: + if not values: + return 0 + sorted_v = sorted(values) + idx = max(0, int(0.95 * (len(sorted_v) - 1))) + return sorted_v[idx] + + +class JitterDetector(DetectorBase): + name = "jitter" + default_severity = Severity.WARNING + + def __init__( + self, + target_getter: TargetGetter = lambda sid: None, + window: int = 60, + jitter_ratio: float = 2.0, + sustain_ns: int = 3_000_000_000, + recovery_ratio: float = 1.2, + recovery_ns: int = 10_000_000_000, + ) -> None: + self._target_getter = target_getter + self._window = window + self._jitter_ratio = jitter_ratio + self._sustain_ns = sustain_ns + self._recovery_ratio = recovery_ratio + self._recovery_ns = recovery_ns + + self._last_at: Dict[str, int] = {} + self._intervals: Dict[str, Deque[int]] = {} + self._bad_began_at: Dict[str, Optional[int]] = {} + self._fire_active: Dict[str, bool] = {} + self._recovery_began_at: Dict[str, Optional[int]] = {} + + def observe_sample(self, stream_id: str, sample: SampleEvent) -> None: + last = self._last_at.get(stream_id) + if last is not None: + buf = self._intervals.setdefault(stream_id, deque(maxlen=self._window)) + buf.append(sample.capture_ns - last) + self._last_at[stream_id] = sample.capture_ns + + def tick(self, now_ns: int) -> Iterator[HealthEvent]: + out: List[HealthEvent] = [] + for stream_id, buf in list(self._intervals.items()): + target_hz = self._target_getter(stream_id) + if target_hz is None or target_hz <= 0 or len(buf) < max(10, self._window // 2): + continue + expected = 1e9 / target_hz + p95 = _p95(list(buf)) + + if p95 > expected * self._jitter_ratio: + began = self._bad_began_at.get(stream_id) + if began is None: + self._bad_began_at[stream_id] = now_ns + began = now_ns + if (now_ns - began) >= self._sustain_ns and not self._fire_active.get(stream_id, False): + self._fire_active[stream_id] = True + out.append(HealthEvent( + stream_id=stream_id, + kind=HealthEventKind.WARNING, + at_ns=now_ns, + detail=f"Jitter spike (p95 {p95/1e6:.1f} ms, expected {expected/1e6:.1f} ms)", + severity=self.default_severity, + source=f"detector:{self.name}", + fingerprint=f"{stream_id}:{self.name}", + data={"p95_ns": p95, "expected_ns": int(expected)}, + )) + else: + self._bad_began_at[stream_id] = None + self._fire_active[stream_id] = False + return iter(out) + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + stream_id = incident.stream_id + buf = self._intervals.get(stream_id) + target_hz = self._target_getter(stream_id) + if not buf or target_hz is None or target_hz <= 0: + return False + expected = 1e9 / target_hz + p95 = _p95(list(buf)) + if p95 > expected * self._recovery_ratio: + self._recovery_began_at[stream_id] = None + return False + began = self._recovery_began_at.get(stream_id) + if began is None: + self._recovery_began_at[stream_id] = now_ns + return False + return (now_ns - began) >= self._recovery_ns +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +pytest tests/unit/health/detectors/test_jitter.py -v +``` + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/detectors/jitter.py tests/unit/health/detectors/test_jitter.py +git commit -m "feat(health): add JitterDetector (p95-interval spike detector)" +``` + +--- + +### Task 13: StartupFailureDetector + +**Files:** +- Create: `src/syncfield/health/detectors/startup_failure.py` +- Test: `tests/unit/health/detectors/test_startup_failure.py` + +Fires when an adapter-emitted `HealthEvent` carries `data["phase"] in {"connect", "start_recording"}` with `kind == ERROR`. This relies on orchestrator convention (Task 15 wires this up from `SessionOrchestrator` exception handlers). + +- [ ] **Step 1: Write failing test** + +```python +# tests/unit/health/detectors/test_startup_failure.py +from syncfield.health.detectors.startup_failure import StartupFailureDetector +from syncfield.health.severity import Severity +from syncfield.types import HealthEvent, HealthEventKind + + +def _ev_for(phase: str, kind=HealthEventKind.ERROR) -> HealthEvent: + return HealthEvent( + stream_id="cam", kind=kind, at_ns=100, detail="boom", + severity=Severity.ERROR, source="orchestrator", + fingerprint=f"cam:adapter:startup-{phase}", + data={"phase": phase}, + ) + + +def test_fires_on_connect_phase_error(): + d = StartupFailureDetector() + d.observe_health("cam", _ev_for("connect")) + events = list(d.tick(now_ns=500)) + assert len(events) == 1 + assert events[0].fingerprint == "cam:startup-failure" + assert events[0].data["phase"] == "connect" + + +def test_ignores_non_startup_phases(): + d = StartupFailureDetector() + d.observe_health("cam", HealthEvent( + stream_id="cam", kind=HealthEventKind.ERROR, at_ns=1, detail="x", + severity=Severity.ERROR, source="adapter:foo", fingerprint="cam:adapter:xlink", + data={}, + )) + assert list(d.tick(now_ns=100)) == [] + + +def test_closes_after_phase_success_signal(): + d = StartupFailureDetector() + d.observe_health("cam", _ev_for("connect")) + list(d.tick(now_ns=100)) + from syncfield.health.types import Incident + inc = Incident.opened_from(_ev_for("connect"), title="x") + + # Before success, not closed. + assert d.close_condition(inc, now_ns=200) is False + + # Success signal arrives. + d.observe_health("cam", HealthEvent( + stream_id="cam", kind=HealthEventKind.HEARTBEAT, at_ns=300, detail="connected", + severity=Severity.INFO, source="orchestrator", fingerprint="cam:adapter:startup-success", + data={"phase": "connect", "outcome": "success"}, + )) + assert d.close_condition(inc, now_ns=400) is True +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/health/detectors/test_startup_failure.py -v +``` + +- [ ] **Step 3: Implement** + +`src/syncfield/health/detectors/startup_failure.py`: + +```python +"""StartupFailureDetector — fires when connect/start_recording raises. + +Relies on orchestrator-emitted HealthEvents with ``data["phase"]`` in +{``"connect"``, ``"start_recording"``}. A subsequent success event with +``data["outcome"] == "success"`` closes the incident. +""" + +from __future__ import annotations + +from typing import Dict, Iterator, List, Set + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.types import Incident +from syncfield.types import HealthEvent, HealthEventKind + +_STARTUP_PHASES = {"connect", "start_recording"} + + +class StartupFailureDetector(DetectorBase): + name = "startup-failure" + default_severity = Severity.ERROR + + def __init__(self) -> None: + self._pending_failures: Dict[str, HealthEvent] = {} + self._recovered: Set[str] = set() + + def observe_health(self, stream_id: str, event: HealthEvent) -> None: + phase = event.data.get("phase") if event.data else None + if phase not in _STARTUP_PHASES: + return + outcome = event.data.get("outcome") if event.data else None + if event.kind == HealthEventKind.ERROR and outcome != "success": + self._pending_failures[stream_id] = event + self._recovered.discard(stream_id) + elif outcome == "success": + self._recovered.add(stream_id) + + def tick(self, now_ns: int) -> Iterator[HealthEvent]: + out: List[HealthEvent] = [] + for stream_id, origin in list(self._pending_failures.items()): + out.append(HealthEvent( + stream_id=stream_id, + kind=HealthEventKind.ERROR, + at_ns=now_ns, + detail=origin.detail or "Startup failure", + severity=self.default_severity, + source=f"detector:{self.name}", + fingerprint=f"{stream_id}:{self.name}", + data={"phase": origin.data.get("phase"), "origin_at_ns": origin.at_ns}, + )) + del self._pending_failures[stream_id] + return iter(out) + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + return incident.stream_id in self._recovered +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +pytest tests/unit/health/detectors/test_startup_failure.py -v +``` + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/detectors/startup_failure.py tests/unit/health/detectors/test_startup_failure.py +git commit -m "feat(health): add StartupFailureDetector (connect / start_recording errors)" +``` + +--- + +### Task 14: BackpressureDetector + +**Files:** +- Create: `src/syncfield/health/detectors/backpressure.py` +- Test: `tests/unit/health/detectors/test_backpressure.py` + +Fires when writer queue fullness ≥ 0.80 for 2 s, OR when `dropped` counter increments. Closes when fullness < 0.30 for 5 s and no drops in that window. + +- [ ] **Step 1: Write failing test** + +```python +# tests/unit/health/detectors/test_backpressure.py +from syncfield.health.detectors.backpressure import BackpressureDetector +from syncfield.health.types import Incident, WriterStats + + +def _stat(at_ns, depth, cap=16, dropped=0): + return WriterStats(stream_id="cam", at_ns=at_ns, queue_depth=depth, queue_capacity=cap, dropped=dropped) + + +def test_does_not_fire_with_normal_fullness(): + d = BackpressureDetector() + for t in range(0, int(3e9), int(2.5e8)): + d.observe_writer_stats("cam", _stat(t, depth=2)) + assert list(d.tick(now_ns=int(3e9))) == [] + + +def test_fires_when_queue_sustained_above_threshold(): + d = BackpressureDetector(fullness_threshold=0.8, sustain_ns=int(2e9)) + for t in range(0, int(3e9), int(2.5e8)): + d.observe_writer_stats("cam", _stat(t, depth=14)) # 14/16 = 0.875 + emitted = list(d.tick(now_ns=int(3e9))) + assert len(emitted) == 1 + assert emitted[0].fingerprint == "cam:backpressure" + + +def test_fires_on_any_drop_increment(): + d = BackpressureDetector() + d.observe_writer_stats("cam", _stat(0, depth=1, dropped=0)) + d.observe_writer_stats("cam", _stat(int(1e8), depth=1, dropped=5)) + emitted = list(d.tick(now_ns=int(2e8))) + assert len(emitted) == 1 + + +def test_close_condition_requires_low_and_no_new_drops(): + d = BackpressureDetector(fullness_threshold=0.8, sustain_ns=int(2e9), + recovery_ratio=0.3, recovery_ns=int(1e9)) + for t in range(0, int(3e9), int(2.5e8)): + d.observe_writer_stats("cam", _stat(t, depth=14)) + events = list(d.tick(now_ns=int(3e9))) + inc = Incident.opened_from(events[0], title="x") + + # Recovery in progress. + d.observe_writer_stats("cam", _stat(int(3.5e9), depth=2)) + assert d.close_condition(inc, now_ns=int(4e9)) is False # only 500 ms of recovery + + d.observe_writer_stats("cam", _stat(int(5e9), depth=2)) + assert d.close_condition(inc, now_ns=int(5e9)) is True # 1.5 s of recovery +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/health/detectors/test_backpressure.py -v +``` + +- [ ] **Step 3: Implement** + +`src/syncfield/health/detectors/backpressure.py`: + +```python +"""BackpressureDetector — writer queue saturation + drop-counter detector.""" + +from __future__ import annotations + +from typing import Dict, Iterator, List, Optional + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.types import Incident, WriterStats +from syncfield.types import HealthEvent, HealthEventKind + + +class BackpressureDetector(DetectorBase): + name = "backpressure" + default_severity = Severity.WARNING + + def __init__( + self, + fullness_threshold: float = 0.80, + sustain_ns: int = 2_000_000_000, + recovery_ratio: float = 0.30, + recovery_ns: int = 5_000_000_000, + ) -> None: + self._threshold = fullness_threshold + self._sustain_ns = sustain_ns + self._recovery_ratio = recovery_ratio + self._recovery_ns = recovery_ns + + self._latest: Dict[str, WriterStats] = {} + self._bad_began_at: Dict[str, Optional[int]] = {} + self._last_dropped: Dict[str, int] = {} + self._pending_drop_spike: Dict[str, bool] = {} + self._fire_active: Dict[str, bool] = {} + self._recovery_began_at: Dict[str, Optional[int]] = {} + + def observe_writer_stats(self, stream_id: str, stats: WriterStats) -> None: + self._latest[stream_id] = stats + prev = self._last_dropped.get(stream_id, 0) + if stats.dropped > prev: + self._pending_drop_spike[stream_id] = True + self._last_dropped[stream_id] = stats.dropped + + def tick(self, now_ns: int) -> Iterator[HealthEvent]: + out: List[HealthEvent] = [] + for stream_id, stats in self._latest.items(): + fire_now = False + detail = "" + + if self._pending_drop_spike.pop(stream_id, False): + fire_now = True + detail = f"Writer dropped frames (total {stats.dropped})" + + if stats.queue_fullness >= self._threshold: + began = self._bad_began_at.get(stream_id) + if began is None: + self._bad_began_at[stream_id] = now_ns + began = now_ns + if (now_ns - began) >= self._sustain_ns and not self._fire_active.get(stream_id, False): + fire_now = True + self._fire_active[stream_id] = True + detail = f"Writer queue {stats.queue_depth}/{stats.queue_capacity} full" + else: + self._bad_began_at[stream_id] = None + self._fire_active[stream_id] = False + + if fire_now: + out.append(HealthEvent( + stream_id=stream_id, + kind=HealthEventKind.WARNING, + at_ns=now_ns, + detail=detail, + severity=self.default_severity, + source=f"detector:{self.name}", + fingerprint=f"{stream_id}:{self.name}", + data={ + "queue_depth": stats.queue_depth, + "queue_capacity": stats.queue_capacity, + "dropped": stats.dropped, + }, + )) + return iter(out) + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + stats = self._latest.get(incident.stream_id) + if stats is None: + return False + if stats.queue_fullness > self._recovery_ratio: + self._recovery_began_at[incident.stream_id] = None + return False + # Require that no new drops occurred since incident opened. + if self._last_dropped.get(incident.stream_id, 0) > incident.data.get("dropped_at_open", stats.dropped): + return False + began = self._recovery_began_at.get(incident.stream_id) + if began is None: + self._recovery_began_at[incident.stream_id] = now_ns + return False + return (now_ns - began) >= self._recovery_ns +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +pytest tests/unit/health/detectors/test_backpressure.py -v +``` + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/detectors/backpressure.py tests/unit/health/detectors/test_backpressure.py +git commit -m "feat(health): add BackpressureDetector (writer queue fullness + drop counter)" +``` + +--- + +### Task 14.5: Complete HealthSystem (Task 8 follow-up) + +With Tasks 9–14 done, `HealthSystem` can now import all six default detectors. + +- [ ] **Step 1: Run the deferred HealthSystem test suite** + +```bash +pytest tests/unit/health/test_health_system.py -v +``` +Expected: 4 passed (all tests from Task 8). + +- [ ] **Step 2: Commit (if not already done in Task 8)** + +```bash +git add src/syncfield/health/system.py src/syncfield/health/__init__.py tests/unit/health/test_health_system.py +git commit -m "feat(health): add HealthSystem facade with default detector install" +``` + +--- + +## Phase 3 — Orchestrator & writer integration + +### Task 15: Add target_hz to StreamCapabilities + FinalizationReport.incidents + +**Files:** +- Modify: `src/syncfield/types.py:176-206` (`StreamCapabilities`) +- Modify: `src/syncfield/types.py:294-328` (`FinalizationReport`) +- Test: `tests/unit/test_types_capabilities.py` (add cases), `tests/unit/test_types_finalization.py` (add cases) + +- [ ] **Step 1: Write failing tests** + +Append to `tests/unit/test_types_capabilities.py`: + +```python +from syncfield.types import StreamCapabilities + + +def test_target_hz_defaults_to_none(): + caps = StreamCapabilities() + assert caps.target_hz is None + + +def test_target_hz_round_trips_to_dict(): + caps = StreamCapabilities(target_hz=30.0) + d = caps.to_dict() + assert d["target_hz"] == 30.0 +``` + +Append to `tests/unit/test_types_finalization.py`: + +```python +from syncfield.health.types import Incident +from syncfield.health.severity import Severity +from syncfield.types import FinalizationReport, HealthEvent, HealthEventKind + + +def test_finalization_report_incidents_default_empty(): + r = FinalizationReport( + stream_id="cam", status="completed", frame_count=10, file_path=None, + first_sample_at_ns=0, last_sample_at_ns=100, health_events=[], error=None, + ) + assert r.incidents == [] + + +def test_finalization_report_accepts_incidents(): + ev = HealthEvent( + stream_id="cam", kind=HealthEventKind.ERROR, at_ns=1, detail="x", + severity=Severity.ERROR, source="detector:stream-stall", + fingerprint="cam:stream-stall", + ) + inc = Incident.opened_from(ev, title="stall") + r = FinalizationReport( + stream_id="cam", status="completed", frame_count=10, file_path=None, + first_sample_at_ns=0, last_sample_at_ns=100, health_events=[], error=None, + incidents=[inc], + ) + assert r.incidents == [inc] +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/test_types_capabilities.py tests/unit/test_types_finalization.py -v +``` + +- [ ] **Step 3: Implement** + +In `src/syncfield/types.py`, add `target_hz: float | None = None` to `StreamCapabilities` (keep `live_preview` last-but-one), and update `to_dict`: + +```python +@dataclass(frozen=True) +class StreamCapabilities: + provides_audio_track: bool = False + supports_precise_timestamps: bool = False + is_removable: bool = False + produces_file: bool = False + target_hz: float | None = None + live_preview: bool = True + + def to_dict(self) -> dict[str, Any]: + return { + "provides_audio_track": self.provides_audio_track, + "supports_precise_timestamps": self.supports_precise_timestamps, + "is_removable": self.is_removable, + "produces_file": self.produces_file, + "target_hz": self.target_hz, + "live_preview": self.live_preview, + } +``` + +Update `FinalizationReport` to add `incidents: list[Incident] = field(default_factory=list)`: + +```python +from dataclasses import field # ensure imported at top of file + +@dataclass +class FinalizationReport: + stream_id: str + status: Literal["completed", "partial", "failed", "pending_aggregation"] + frame_count: int + file_path: Path | None + first_sample_at_ns: int | None + last_sample_at_ns: int | None + health_events: list[HealthEvent] + error: str | None + jitter_p95_ns: int | None = None + jitter_p99_ns: int | None = None + incidents: "list" = field(default_factory=list) # type: ignore[assignment] +``` + +The `Incident` forward reference avoids a circular import (`types.py` → `health.types` → `types.py`). Consumers that need the typed list import `Incident` separately. + +- [ ] **Step 4: Run, confirm pass** + +```bash +pytest tests/unit/test_types_capabilities.py tests/unit/test_types_finalization.py -v +``` + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/types.py tests/unit/test_types_capabilities.py tests/unit/test_types_finalization.py +git commit -m "feat(types): add StreamCapabilities.target_hz and FinalizationReport.incidents" +``` + +--- + +### Task 16: SessionLogWriter — log_incident + incidents.jsonl path + +**Files:** +- Modify: `src/syncfield/writer.py:112-161` (`SessionLogWriter`) +- Test: `tests/unit/test_writer.py` (add cases) + +Write incidents to `/incidents.jsonl` on every open/update/close. The file is append-only; each line is the full incident state at write time, keyed by `id`. + +- [ ] **Step 1: Write failing test** + +Append to `tests/unit/test_writer.py`: + +```python +import json + +from syncfield.health.severity import Severity +from syncfield.health.types import Incident +from syncfield.types import HealthEvent, HealthEventKind +from syncfield.writer import SessionLogWriter + + +def _ev(at_ns: int) -> HealthEvent: + return HealthEvent( + stream_id="cam", kind=HealthEventKind.ERROR, at_ns=at_ns, detail="x", + severity=Severity.ERROR, source="detector:stream-stall", + fingerprint="cam:stream-stall", + ) + + +def test_log_incident_appends_to_incidents_jsonl(tmp_path): + w = SessionLogWriter(tmp_path) + w.open() + try: + inc = Incident.opened_from(_ev(100), title="stall") + w.log_incident(inc) + inc.record_event(_ev(200)) + w.log_incident(inc) + inc.close(at_ns=300) + w.log_incident(inc) + finally: + w.close() + + path = tmp_path / "incidents.jsonl" + assert path.exists() + lines = path.read_text().strip().splitlines() + assert len(lines) == 3 + first = json.loads(lines[0]) + assert first["id"] == inc.id + assert first["event_count"] == 1 + last = json.loads(lines[-1]) + assert last["closed_at_ns"] == 300 +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/test_writer.py -k incident -v +``` + +- [ ] **Step 3: Implement** + +In `src/syncfield/writer.py`, extend `SessionLogWriter`: + +```python +class SessionLogWriter: + def __init__(self, output_dir: Path) -> None: + self._path = output_dir / "session_log.jsonl" + self._incidents_path = output_dir / "incidents.jsonl" + self._handle: IO[str] | None = None + self._incidents_handle: IO[str] | None = None + + @property + def incidents_path(self) -> Path: + return self._incidents_path + + def open(self) -> None: + if self._handle is None: + self._handle = open(self._path, "w") + if self._incidents_handle is None: + self._incidents_handle = open(self._incidents_path, "w") + + # existing log_event / log_health unchanged ... + + def log_incident(self, incident) -> None: + """Append *incident*'s current full state as one JSON line.""" + if self._incidents_handle is None: + raise RuntimeError("SessionLogWriter is not open") + self._incidents_handle.write( + json.dumps(incident.to_dict(), separators=(",", ":")) + "\n" + ) + self._incidents_handle.flush() + + def close(self) -> None: + if self._handle is not None: + self._handle.close() + self._handle = None + if self._incidents_handle is not None: + self._incidents_handle.close() + self._incidents_handle = None +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +pytest tests/unit/test_writer.py -v +``` + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/writer.py tests/unit/test_writer.py +git commit -m "feat(writer): add SessionLogWriter.log_incident + incidents.jsonl" +``` + +--- + +### Task 17: Wire SessionOrchestrator to HealthSystem + +**Files:** +- Modify: `src/syncfield/orchestrator.py` — multiple sites +- Test: `tests/integration/health/__init__.py`, `tests/integration/health/test_orchestrator_health_integration.py` + +This is the biggest integration step. Each sub-step has its own runnable verification. + +- [ ] **Step 1: Locate and read the target methods** + +```bash +grep -n "def __init__\|def _on_stream_sample\|def _on_stream_health\|def _set_state\|def start\|def stop\|def add\|def disconnect" src/syncfield/orchestrator.py | head -40 +``` + +Make a short note of the line numbers for: `__init__`, `add` (stream registration), `_on_stream_sample`, `_on_stream_health`, `_set_state` (or the state-transition helper), `start`, `stop`, `disconnect`, `_build_finalization_reports` (or similar). + +- [ ] **Step 2: Write the failing integration test** + +`tests/integration/health/__init__.py` is empty. `tests/integration/health/test_orchestrator_health_integration.py`: + +```python +"""Integration: real SessionOrchestrator + FakeStream → incidents flow end-to-end.""" + +import json +import time +from pathlib import Path + +from syncfield.orchestrator import SessionOrchestrator +from syncfield.types import StreamCapabilities + +# Reuse the existing FakeStream helper if one exists; otherwise define a +# minimal StreamBase subclass inline. +try: + from tests.helpers.fake_stream import FakeStream # type: ignore +except ModuleNotFoundError: + from syncfield.stream import StreamBase + from syncfield.types import SampleEvent + import threading + import time as _t + + class FakeStream(StreamBase): + def __init__(self, stream_id: str, target_hz: float | None = None): + super().__init__( + stream_id=stream_id, + kind="sensor", + capabilities=StreamCapabilities(target_hz=target_hz), + ) + self._stop = threading.Event() + self._thread: threading.Thread | None = None + self._frame = 0 + self._interval = 1.0 / 30.0 + + def connect(self): + self._stop.clear() + self._thread = threading.Thread(target=self._run, daemon=True) + self._thread.start() + + def disconnect(self): + self._stop.set() + if self._thread: + self._thread.join(timeout=1.0) + + def start_recording(self, session_clock): + pass + + def stop_recording(self): + from syncfield.types import FinalizationReport + return FinalizationReport( + stream_id=self.id, status="completed", frame_count=self._frame, + file_path=None, first_sample_at_ns=0, last_sample_at_ns=0, + health_events=[], error=None, + ) + + def pause_samples(self): + # Stop emitting without stopping the thread — simulate stall. + self._stop_samples = True + + def resume_samples(self): + self._stop_samples = False + + def _run(self): + self._stop_samples = False + while not self._stop.is_set(): + if not getattr(self, "_stop_samples", False): + self._frame += 1 + self._emit_sample(SampleEvent( + stream_id=self.id, + frame_number=self._frame, + capture_ns=time.monotonic_ns(), + )) + _t.sleep(self._interval) + + +def test_stall_incident_open_and_close(tmp_path: Path): + sess = SessionOrchestrator(host_id="test", output_dir=tmp_path) + stream = FakeStream("cam", target_hz=30.0) + sess.add(stream) + + sess.connect() + sess.start(countdown_s=0) + + # Induce stall. + stream.pause_samples() + time.sleep(3.0) + + opens = [i for i in sess.health.open_incidents() if i.fingerprint == "cam:stream-stall"] + assert opens, "stall incident did not open" + + # Recover. + stream.resume_samples() + time.sleep(2.5) + + sess.stop() + sess.disconnect() + + resolved = [i for i in sess.health.resolved_incidents() if i.fingerprint == "cam:stream-stall"] + assert resolved, "stall incident did not resolve" + + +def test_incidents_jsonl_written(tmp_path: Path): + sess = SessionOrchestrator(host_id="test", output_dir=tmp_path) + stream = FakeStream("cam", target_hz=30.0) + sess.add(stream) + + sess.connect() + sess.start(countdown_s=0) + stream.pause_samples() + time.sleep(2.5) + sess.stop() + sess.disconnect() + + out_files = list(tmp_path.glob("**/incidents.jsonl")) + assert out_files, "no incidents.jsonl written" + lines = out_files[0].read_text().strip().splitlines() + assert any(json.loads(l)["fingerprint"] == "cam:stream-stall" for l in lines) +``` + +- [ ] **Step 3: Run, confirm fail** + +```bash +pytest tests/integration/health/test_orchestrator_health_integration.py -v +``` +Expected: `AttributeError: 'SessionOrchestrator' object has no attribute 'health'`. + +- [ ] **Step 4: Wire `HealthSystem` into `SessionOrchestrator.__init__`** + +In `src/syncfield/orchestrator.py`, inside `__init__` (after the existing self.* attribute assignments, before any `_bring_*` helper): + +```python +from syncfield.health import HealthSystem + +# ... inside __init__ ... +self.health = HealthSystem() +self.health.on_incident_opened = self._persist_incident +self.health.on_incident_updated = self._persist_incident +self.health.on_incident_closed = self._persist_incident +``` + +And add the persistence helper method on the class: + +```python +def _persist_incident(self, incident) -> None: + writer = getattr(self, "_session_log_writer", None) + if writer is not None: + try: + writer.log_incident(incident) + except Exception: + # Never let telemetry persistence crash the recording. + pass +``` + +- [ ] **Step 5: Connect per-stream sample + health + writer-stats observers** + +Find the place where `add()` registers callbacks (look for `.on_sample(` and `.on_health(`). Extend both: + +```python +def add(self, stream) -> None: + # ... existing wiring ... + stream.on_sample(lambda s: self.health.observe_sample(stream.id, s)) + stream.on_health(lambda h: self._on_stream_health(h)) # unchanged if already here +``` + +And in `_on_stream_health`, forward to the health system **after** persisting to the session log: + +```python +def _on_stream_health(self, event): + writer = getattr(self, "_session_log_writer", None) + if writer is not None: + writer.log_health(event) + self._buffered_health.append(event) # existing + self.health.observe_health(event.stream_id, event) +``` + +Find the state-transition helper (look for a method that updates `self._state` and fires listeners — often `_transition` or `_set_state`). Wrap the transition: + +```python +def _set_state(self, new_state: SessionState) -> None: + old = self._state + self._state = new_state + self.health.observe_state(old, new_state) + # ... existing listener notifications ... +``` + +- [ ] **Step 6: Start / stop the worker + flush remaining incidents on stop** + +In `start()`, after state transitions are ready for recording, call `self.health.start()`. In `stop()`, after final per-stream `stop_recording()` calls but before closing the session log, call `self.health.stop()`. Then collect resolved + open incidents for each stream into its `FinalizationReport.incidents`: + +```python +def start(self, countdown_s: float = 3.0) -> None: + # ... existing start logic up through transition to RECORDING ... + self.health.start() + +def stop(self): + # ... existing stop logic: run each stream's stop_recording() ... + + self.health.stop() + + # Attach incidents to each stream's FinalizationReport. + all_incidents = list(self.health.open_incidents()) + list(self.health.resolved_incidents()) + by_stream: dict[str, list] = {} + for inc in all_incidents: + by_stream.setdefault(inc.stream_id, []).append(inc) + for stream_id, report in self._finalization_reports.items(): + report.incidents = by_stream.get(stream_id, []) + + # ... existing log close + return ... +``` + +- [ ] **Step 7: Pump WriterStats from the recording writer** + +In whichever writer pipeline produces per-frame writes (`src/syncfield/writer.py` — look for the frame-writing path around `VideoWriter` / `SensorWriter`), push a `WriterStats` after each flush. If queues are not used, emit `WriterStats` with `queue_depth=0, queue_capacity=1, dropped=0` — the detector will simply never fire, which is correct. + +If the current writer is synchronous (no queue), skip this sub-step and leave `BackpressureDetector` as a no-op; its integration tests already exercise it directly. Add a TODO comment in `writer.py`: + +```python +# TODO(health): when the writer moves to a queued async path, push +# WriterStats into self._health_system.observe_writer_stats(stream_id, ...) +# on every flush. For the synchronous path, fullness stays at 0. +``` + +- [ ] **Step 8: Run the integration tests, confirm pass** + +```bash +pytest tests/integration/health/test_orchestrator_health_integration.py -v +``` +Expected: 2 passed. (Tests take ~6 s each because they exercise real timing.) + +If a test is flaky under CI timing, increase the `time.sleep()` margins inside the test by 500 ms; keep the detector thresholds at production defaults. + +- [ ] **Step 9: Make sure existing orchestrator tests still pass** + +```bash +pytest tests/ -x -q -k "orchestrator" +``` +Expected: all existing tests pass. If the `health_count` or `problem_count` tests break — those are covered in Task 19 (viewer refactor). Defer; proceed to commit. + +- [ ] **Step 10: Commit** + +```bash +git add src/syncfield/orchestrator.py tests/integration/health/ +git commit -m "feat(orchestrator): wire SessionOrchestrator to HealthSystem (samples, health, state, incidents)" +``` + +--- + +## Phase 4 — OAK bridge & adapter integration + +### Task 18: DepthAILoggerBridge + +**Files:** +- Create: `src/syncfield/health/detectors/depthai_bridge.py` +- Test: `tests/unit/health/detectors/test_depthai_bridge.py` + +The bridge is a `logging.Handler` that converts depthai error / warning records into `HealthEvent`s pushed directly into `HealthSystem.observe_health`. It is **not** a Detector (it doesn't own a close condition — the `AdapterEventPassthrough` does that for the resulting fingerprints). + +- [ ] **Step 1: Write failing test** + +```python +# tests/unit/health/detectors/test_depthai_bridge.py +import logging +from typing import List + +from syncfield.health.detectors.depthai_bridge import DepthAILoggerBridge +from syncfield.types import HealthEvent + + +def _mk_record(msg: str, level: int = logging.ERROR, name: str = "depthai") -> logging.LogRecord: + return logging.LogRecord( + name=name, level=level, pathname="", lineno=0, msg=msg, args=(), exc_info=None, + ) + + +def test_xlink_error_maps_to_xlink_fingerprint(): + captured: List[HealthEvent] = [] + bridge = DepthAILoggerBridge(stream_id="oak-main", sink=lambda sid, ev: captured.append(ev)) + rec = _mk_record("Communication exception - possible device error. Original message 'Couldn't read data from stream: '__x_0_1' (X_LINK_ERROR)'") + bridge.emit(rec) + assert len(captured) == 1 + ev = captured[0] + assert ev.stream_id == "oak-main" + assert ev.fingerprint == "oak-main:adapter:xlink-error" + assert ev.source == "adapter:oak" + assert ev.data.get("stream") == "__x_0_1" + + +def test_device_crash_attaches_crash_dump_path(): + captured: List[HealthEvent] = [] + bridge = DepthAILoggerBridge(stream_id="oak-main", sink=lambda sid, ev: captured.append(ev)) + rec = _mk_record("Device with id 194430 has crashed. Crash dump logs are stored in: /tmp/crash/crash_dump.json - please report to developers.") + bridge.emit(rec) + ev = captured[0] + assert ev.fingerprint == "oak-main:adapter:device-crash" + assert ev.data.get("crash_dump_path") == "/tmp/crash/crash_dump.json" + + +def test_reconnect_attempt_and_success_have_distinct_fingerprints(): + captured: List[HealthEvent] = [] + bridge = DepthAILoggerBridge(stream_id="oak-main", sink=lambda sid, ev: captured.append(ev)) + bridge.emit(_mk_record("Attempting to reconnect. Timeout is 10000ms", level=logging.WARNING)) + bridge.emit(_mk_record("Reconnection successful", level=logging.WARNING)) + fps = [c.fingerprint for c in captured] + assert "oak-main:adapter:reconnect-attempt" in fps + assert "oak-main:adapter:reconnect-success" in fps + + +def test_unrecognized_error_falls_back_to_warning_unparsed(): + captured: List[HealthEvent] = [] + bridge = DepthAILoggerBridge(stream_id="oak-main", sink=lambda sid, ev: captured.append(ev)) + bridge.emit(_mk_record("Something totally new and unrecognized", level=logging.ERROR)) + assert len(captured) == 1 + assert captured[0].source == "adapter:oak:unparsed-log" + + +def test_info_records_are_ignored(): + captured = [] + bridge = DepthAILoggerBridge(stream_id="oak-main", sink=lambda sid, ev: captured.append(ev)) + bridge.emit(_mk_record("Some info", level=logging.INFO)) + assert captured == [] +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/health/detectors/test_depthai_bridge.py -v +``` + +- [ ] **Step 3: Implement** + +`src/syncfield/health/detectors/depthai_bridge.py`: + +```python +"""DepthAILoggerBridge — translate depthai Python log records into HealthEvents. + +Installed as a standard :class:`logging.Handler` on the depthai logger. +Does not subclass DetectorBase — it is a translator, not a detector. +Its outputs are fingerprinted as ``:adapter:`` so +the AdapterEventPassthrough detector owns their open/close lifecycle. +""" + +from __future__ import annotations + +import logging +import re +import time +from typing import Callable, Optional + +from syncfield.health.severity import Severity +from syncfield.types import HealthEvent, HealthEventKind + +Sink = Callable[[str, HealthEvent], None] + +_XLINK_RE = re.compile(r"X_LINK_ERROR.*stream: '([^']+)'|stream: '([^']+)'.*X_LINK_ERROR") +_CRASH_RE = re.compile(r"Device with id (\S+) has crashed\. Crash dump logs are stored in: (\S+)") +_RECONNECT_TRY_RE = re.compile(r"Attempting to reconnect", re.IGNORECASE) +_RECONNECT_OK_RE = re.compile(r"Reconnection successful", re.IGNORECASE) +_CONN_CLOSED_RE = re.compile(r"Closed connection", re.IGNORECASE) + + +class DepthAILoggerBridge(logging.Handler): + def __init__(self, stream_id: str, sink: Sink) -> None: + super().__init__(level=logging.WARNING) + self._stream_id = stream_id + self._sink = sink + + def emit(self, record: logging.LogRecord) -> None: + if record.levelno < logging.WARNING: + return + msg = record.getMessage() + now = time.monotonic_ns() + + parsed = self._parse(msg, record.levelno, now) + if parsed is None: + parsed = HealthEvent( + stream_id=self._stream_id, + kind=HealthEventKind.WARNING, + at_ns=now, + detail=msg, + severity=Severity.WARNING, + source="adapter:oak:unparsed-log", + fingerprint=f"{self._stream_id}:adapter:unparsed-log", + data={"raw": msg, "levelname": record.levelname}, + ) + try: + self._sink(self._stream_id, parsed) + except Exception: + # Never let bridge failures crash the logging path. + pass + + def _parse(self, msg: str, levelno: int, now: int) -> Optional[HealthEvent]: + crash = _CRASH_RE.search(msg) + if crash: + device_id, path = crash.group(1), crash.group(2) + return HealthEvent( + stream_id=self._stream_id, + kind=HealthEventKind.ERROR, + at_ns=now, + detail="OAK device crashed", + severity=Severity.CRITICAL, + source="adapter:oak", + fingerprint=f"{self._stream_id}:adapter:device-crash", + data={"device_id": device_id, "crash_dump_path": path}, + ) + xlink = _XLINK_RE.search(msg) + if xlink: + stream = xlink.group(1) or xlink.group(2) + return HealthEvent( + stream_id=self._stream_id, + kind=HealthEventKind.ERROR, + at_ns=now, + detail="XLink communication error", + severity=Severity.ERROR, + source="adapter:oak", + fingerprint=f"{self._stream_id}:adapter:xlink-error", + data={"stream": stream}, + ) + if _RECONNECT_OK_RE.search(msg): + return HealthEvent( + stream_id=self._stream_id, + kind=HealthEventKind.RECONNECT, + at_ns=now, + detail="Reconnection successful", + severity=Severity.INFO, + source="adapter:oak", + fingerprint=f"{self._stream_id}:adapter:reconnect-success", + ) + if _RECONNECT_TRY_RE.search(msg): + return HealthEvent( + stream_id=self._stream_id, + kind=HealthEventKind.RECONNECT, + at_ns=now, + detail="Attempting reconnect", + severity=Severity.WARNING, + source="adapter:oak", + fingerprint=f"{self._stream_id}:adapter:reconnect-attempt", + ) + if _CONN_CLOSED_RE.search(msg): + return HealthEvent( + stream_id=self._stream_id, + kind=HealthEventKind.WARNING, + at_ns=now, + detail="Connection closed", + severity=Severity.WARNING, + source="adapter:oak", + fingerprint=f"{self._stream_id}:adapter:connection-closed", + ) + return None +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +pytest tests/unit/health/detectors/test_depthai_bridge.py -v +``` +Expected: 5 passed. + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/detectors/depthai_bridge.py tests/unit/health/detectors/test_depthai_bridge.py +git commit -m "feat(health): add DepthAILoggerBridge (logging.Handler → HealthEvent)" +``` + +--- + +### Task 19: OakCameraStream — declare target_hz, install bridge, capture crash_dump + +**Files:** +- Modify: `src/syncfield/adapters/oak_camera.py` +- Test: `tests/unit/adapters/test_oak_camera_health.py` (new) + +- [ ] **Step 1: Locate the current OAK setup sites** + +```bash +grep -n "StreamCapabilities\|def __init__\|def connect\|def disconnect\|fps\b" src/syncfield/adapters/oak_camera.py | head -30 +``` + +Record line numbers for: `StreamCapabilities(...)` construction, `connect()` entry, `disconnect()` entry, the `fps=` argument passed to depthai (this is the physical target we surface as `target_hz`). + +- [ ] **Step 2: Write the failing test** + +`tests/unit/adapters/test_oak_camera_health.py`: + +```python +"""Unit tests for OAK adapter health wiring (no real hardware required).""" + +import logging + +import pytest + + +pytest.importorskip("depthai") # skip entire module if the oak extra isn't installed + + +def test_oak_declares_target_hz(): + from syncfield.adapters.oak_camera import OakCameraStream + s = OakCameraStream(stream_id="oak-main", fps=30) + assert s.capabilities.target_hz == 30.0 + + +def test_oak_connect_installs_logger_bridge(monkeypatch, tmp_path): + from syncfield.adapters.oak_camera import OakCameraStream + + captured = [] + s = OakCameraStream(stream_id="oak-main", fps=30, output_dir=tmp_path) + s.on_health(lambda ev: captured.append(ev)) + + # Don't actually build a depthai pipeline — stub the inner connect body. + monkeypatch.setattr(s, "_open_device_pipeline", lambda: None, raising=False) + s._install_depthai_bridge() # exercised directly since connect() may fail without hw + + logging.getLogger("depthai").error( + "Communication exception - Original message 'Couldn't read data from stream: '__x_0_1' (X_LINK_ERROR)'" + ) + + assert any(ev.fingerprint == "oak-main:adapter:xlink-error" for ev in captured) + + s._uninstall_depthai_bridge() +``` + +- [ ] **Step 3: Run, confirm fail** + +```bash +pytest tests/unit/adapters/test_oak_camera_health.py -v +``` + +- [ ] **Step 4: Implement the wiring** + +In `src/syncfield/adapters/oak_camera.py`: + +- Add `target_hz` to the `StreamCapabilities(...)` construction inside `__init__`, using the existing `fps` argument: `StreamCapabilities(produces_file=True, supports_precise_timestamps=True, is_removable=True, target_hz=float(fps))`. +- Add two helpers: + +```python +def _install_depthai_bridge(self) -> None: + from syncfield.health.detectors.depthai_bridge import DepthAILoggerBridge + if getattr(self, "_depthai_bridge", None) is not None: + return + self._depthai_bridge = DepthAILoggerBridge( + stream_id=self.id, + sink=lambda sid, ev: self._emit_health(ev), + ) + logging.getLogger("depthai").addHandler(self._depthai_bridge) + +def _uninstall_depthai_bridge(self) -> None: + bridge = getattr(self, "_depthai_bridge", None) + if bridge is None: + return + logging.getLogger("depthai").removeHandler(bridge) + self._depthai_bridge = None +``` + +- Call `self._install_depthai_bridge()` at the top of `connect()` and `self._uninstall_depthai_bridge()` at the bottom of `disconnect()`. + +- Where `StreamBase._emit_health` is invoked with a crash detail today, prefer letting the bridge handle it. The bridge's own `device-crash` fingerprint already attaches `crash_dump_path` in `event.data`. In the orchestrator's `_persist_incident` (Task 17 step 4), when an incident's `fingerprint` ends with `:device-crash` and `first_event.data["crash_dump_path"]` exists, attach an `IncidentArtifact`: + +```python +# In orchestrator._persist_incident, before writer.log_incident: +from syncfield.health.types import IncidentArtifact + +if incident.fingerprint.endswith(":device-crash"): + path = incident.first_event.data.get("crash_dump_path") + if path and not any(a.kind == "crash_dump" for a in incident.artifacts): + incident.attach(IncidentArtifact(kind="crash_dump", path=str(path))) +``` + +- [ ] **Step 5: Run, confirm pass** + +```bash +pytest tests/unit/adapters/test_oak_camera_health.py -v +``` + +- [ ] **Step 6: Commit** + +```bash +git add src/syncfield/adapters/oak_camera.py src/syncfield/orchestrator.py tests/unit/adapters/test_oak_camera_health.py +git commit -m "feat(oak): declare target_hz, bridge depthai logger, attach crash_dump artifacts" +``` + +--- + +## Phase 5 — Viewer (server) + +### Task 20: Replace HealthEntry with IncidentSnapshot in poller state + +**Files:** +- Modify: `src/syncfield/viewer/state.py` +- Modify: `src/syncfield/viewer/poller.py` (if poller constructs snapshots — else just state.py) +- Test: `tests/unit/viewer/test_snapshot_incidents.py` (new) + +- [ ] **Step 1: Write failing test** + +```python +# tests/unit/viewer/test_snapshot_incidents.py +from syncfield.health.severity import Severity +from syncfield.health.types import Incident, IncidentSnapshot +from syncfield.types import HealthEvent, HealthEventKind +from syncfield.viewer.state import SessionSnapshot, StreamSnapshot + + +def _ev(at_ns: int) -> HealthEvent: + return HealthEvent( + stream_id="cam", kind=HealthEventKind.ERROR, at_ns=at_ns, detail="x", + severity=Severity.ERROR, source="detector:stream-stall", + fingerprint="cam:stream-stall", + ) + + +def test_session_snapshot_has_incident_fields(): + snap = SessionSnapshot( + host_id="h", state="recording", output_dir="/tmp", + sync_point_monotonic_ns=None, sync_point_wall_clock_ns=None, + chirp_start_ns=None, chirp_stop_ns=None, chirp_enabled=False, + elapsed_s=0.0, streams={}, active_incidents=[], resolved_incidents=[], + ) + assert snap.active_incidents == [] + assert snap.resolved_incidents == [] + + +def test_stream_snapshot_no_longer_has_health_count(): + # health_count and problem_count are removed; StreamSnapshot should not accept them. + import dataclasses + fields = {f.name for f in dataclasses.fields(StreamSnapshot)} + assert "health_count" not in fields + assert "problem_count" not in fields +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +pytest tests/unit/viewer/test_snapshot_incidents.py -v +``` + +- [ ] **Step 3: Implement in `src/syncfield/viewer/state.py`** + +- Remove the `HealthEntry` dataclass and all references to it. +- Remove `StreamSnapshot.health_count` (and `problem_count` if present). +- Replace `SessionSnapshot.health_log: List[HealthEntry]` with: + +```python +active_incidents: List[IncidentSnapshot] = field(default_factory=list) +resolved_incidents: List[IncidentSnapshot] = field(default_factory=list) +``` + +Remove `StreamStatsBuffer._health` (the `HealthEntry` deque) and the `observe_health` / `snapshot_health` methods that produced it. + +Imports: + +```python +from syncfield.health.types import IncidentSnapshot +``` + +- [ ] **Step 4: Update the poller** (`src/syncfield/viewer/poller.py`) + +Find where the poller is constructed or initialized with the session — it needs to subscribe to `session.health`: + +```python +# in the poller's __init__, after storing self._session: +session.health.on_incident_opened = self._ingest_incident +session.health.on_incident_updated = self._ingest_incident +session.health.on_incident_closed = self._ingest_incident +``` + +Keep two bounded lists, updated from the callback (callbacks fire on the health worker thread — use a lock): + +```python +import threading +from syncfield.health.types import IncidentSnapshot + +self._incidents_lock = threading.Lock() +self._open_by_id: dict[str, Incident] = {} +self._resolved: deque[Incident] = deque(maxlen=20) + +def _ingest_incident(self, incident): + with self._incidents_lock: + if incident.is_open: + self._open_by_id[incident.id] = incident + else: + self._open_by_id.pop(incident.id, None) + self._resolved.append(incident) +``` + +And when producing the snapshot: + +```python +import time + +with self._incidents_lock: + now = time.monotonic_ns() + active = [IncidentSnapshot.from_incident(i, now_ns=now) for i in self._open_by_id.values()] + resolved = [IncidentSnapshot.from_incident(i, now_ns=now) for i in self._resolved] + +snapshot = SessionSnapshot( + # ... existing fields ... + active_incidents=active, + resolved_incidents=resolved, +) +``` + +Remove `health_log=` / `health_count=` from the construction. Remove any existing `observe_health` poller registration on streams (the orchestrator now owns that). + +- [ ] **Step 5: Run, confirm pass** + +```bash +pytest tests/unit/viewer/ -v +``` + +If existing viewer tests reference `health_count`, `problem_count`, `health_log`, or `HealthEntry`, update them to use the new fields. Search: + +```bash +grep -rn "health_count\|problem_count\|health_log\|HealthEntry" tests/ src/syncfield/viewer/ +``` + +Fix each site in the same commit. + +- [ ] **Step 6: Commit** + +```bash +git add src/syncfield/viewer/ tests/unit/viewer/ +git commit -m "refactor(viewer): replace HealthEntry/health_count with IncidentSnapshot fields" +``` + +--- + +### Task 21: WebSocket serializer — emit incident fields + +**Files:** +- Modify: `src/syncfield/viewer/server.py` — the WebSocket snapshot encoder +- Test: `tests/unit/viewer/test_server_snapshot_serialization.py` (new, or extend existing) + +- [ ] **Step 1: Locate the serializer** + +```bash +grep -n "SessionSnapshot\|snapshot.*dict\|jsonable\|json.dumps" src/syncfield/viewer/server.py | head -20 +``` + +Find the function/method that converts a `SessionSnapshot` to the dict shipped over WebSocket. + +- [ ] **Step 2: Write failing test** + +```python +# tests/unit/viewer/test_server_snapshot_serialization.py +from syncfield.health.severity import Severity +from syncfield.health.types import Incident, IncidentSnapshot +from syncfield.types import HealthEvent, HealthEventKind +from syncfield.viewer.state import SessionSnapshot +from syncfield.viewer.server import snapshot_to_wire # or whatever the real name is + + +def _inc_snap(open_: bool = True) -> IncidentSnapshot: + ev = HealthEvent( + stream_id="cam", kind=HealthEventKind.ERROR, at_ns=1, detail="x", + severity=Severity.ERROR, source="detector:stream-stall", + fingerprint="cam:stream-stall", + ) + inc = Incident.opened_from(ev, title="stall") + if not open_: + inc.close(at_ns=2) + return IncidentSnapshot.from_incident(inc, now_ns=1_000) + + +def test_snapshot_to_wire_emits_incident_fields(): + snap = SessionSnapshot( + host_id="h", state="recording", output_dir="/tmp", + sync_point_monotonic_ns=None, sync_point_wall_clock_ns=None, + chirp_start_ns=None, chirp_stop_ns=None, chirp_enabled=False, + elapsed_s=0.0, streams={}, + active_incidents=[_inc_snap(open_=True)], + resolved_incidents=[_inc_snap(open_=False)], + ) + wire = snapshot_to_wire(snap) + assert isinstance(wire["active_incidents"], list) + assert wire["active_incidents"][0]["severity"] == "error" + assert wire["active_incidents"][0]["fingerprint"] == "cam:stream-stall" + assert wire["resolved_incidents"][0]["closed_at_ns"] == 2 +``` + +(Replace the import name if the actual helper is different — adapt based on step 1's grep.) + +- [ ] **Step 3: Implement** + +Update the snapshot-to-wire helper: + +```python +def snapshot_to_wire(snap: SessionSnapshot) -> dict: + return { + # ... existing fields ... + "active_incidents": [_incident_to_wire(i) for i in snap.active_incidents], + "resolved_incidents": [_incident_to_wire(i) for i in snap.resolved_incidents], + } + +def _incident_to_wire(snap: IncidentSnapshot) -> dict: + return { + "id": snap.id, + "stream_id": snap.stream_id, + "fingerprint": snap.fingerprint, + "title": snap.title, + "severity": snap.severity, + "source": snap.source, + "opened_at_ns": snap.opened_at_ns, + "closed_at_ns": snap.closed_at_ns, + "event_count": snap.event_count, + "detail": snap.detail, + "ago_s": snap.ago_s, + "artifacts": snap.artifacts, + } +``` + +Remove any serialization of the old `health_log` / `health_count`. + +- [ ] **Step 4: Run, confirm pass** + +```bash +pytest tests/unit/viewer/test_server_snapshot_serialization.py -v +``` + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/viewer/server.py tests/unit/viewer/test_server_snapshot_serialization.py +git commit -m "feat(viewer): serialize incidents into WebSocket snapshot payload" +``` + +--- + +## Phase 6 — Viewer (frontend) + +### Task 22: Mirror Severity + IncidentSnapshot TypeScript types + +**Files:** +- Modify: `src/syncfield/viewer/frontend/src/lib/types.ts` + +- [ ] **Step 1: Open the file and read the current shape** + +```bash +sed -n '1,60p' src/syncfield/viewer/frontend/src/lib/types.ts +``` + +- [ ] **Step 2: Remove HealthEntry + health_count; add new types** + +Replace the `HealthEntry` type and any `StreamSnapshot.health_count` / `problem_count` fields with: + +```ts +export type Severity = "info" | "warning" | "error" | "critical"; + +export interface IncidentArtifact { + kind: string; + path: string; + detail: string | null; +} + +export interface IncidentSnapshot { + id: string; + stream_id: string; + fingerprint: string; + title: string; + severity: Severity; + source: string; + opened_at_ns: number; + closed_at_ns: number | null; + event_count: number; + detail: string | null; + ago_s: number; + artifacts: IncidentArtifact[]; +} + +export interface SessionSnapshot { + // ...existing fields (host_id, state, streams, elapsed_s, etc.)... + active_incidents: IncidentSnapshot[]; + resolved_incidents: IncidentSnapshot[]; +} +``` + +Delete the old `HealthEntry` export entirely. Update `StreamSnapshot` to drop `health_count` / `problem_count`. + +- [ ] **Step 3: Typecheck** + +```bash +cd src/syncfield/viewer/frontend && npx tsc --noEmit +``` +Expected: errors listing every call-site that still references `HealthEntry` / `health_count`. We fix those in Tasks 23 and 24. Commit now with the type changes — subsequent tasks compile green. + +```bash +git add src/syncfield/viewer/frontend/src/lib/types.ts +git commit -m "refactor(viewer-fe): mirror IncidentSnapshot + Severity; drop HealthEntry" +``` + +--- + +### Task 23: Delete health-table.tsx, add incident-panel.tsx + +**Files:** +- Delete: `src/syncfield/viewer/frontend/src/components/health-table.tsx` +- Create: `src/syncfield/viewer/frontend/src/components/incident-panel.tsx` +- Modify: `src/syncfield/viewer/frontend/src/App.tsx` — replace `` with `` + +- [ ] **Step 1: Create the new component** + +`src/syncfield/viewer/frontend/src/components/incident-panel.tsx`: + +```tsx +import { useState } from "react"; +import type { IncidentSnapshot, Severity } from "../lib/types"; + +const SEVERITY_ICON: Record = { + info: "·", + warning: "⚠", + error: "⛔", + critical: "⛔", +}; + +const SEVERITY_COLOR: Record = { + info: "text-slate-400", + warning: "text-yellow-400", + error: "text-orange-400", + critical: "text-red-500", +}; + +function formatAgo(s: number): string { + if (s < 60) return `${Math.round(s)}s ago`; + if (s < 3600) return `${Math.round(s / 60)}m ago`; + return `${Math.round(s / 3600)}h ago`; +} + +function IncidentCard({ inc, isOpen }: { inc: IncidentSnapshot; isOpen: boolean }) { + const [expanded, setExpanded] = useState(false); + return ( + + ); +} + +export function IncidentPanel({ + active, + resolved, +}: { + active: IncidentSnapshot[]; + resolved: IncidentSnapshot[]; +}) { + return ( +
+
+ Active Issues ({active.length}) +
+ {active.length === 0 ? ( +
None — all clear.
+ ) : ( +
+ {active.map((inc) => ( + + ))} +
+ )} +
+ Resolved this session ({resolved.length}) +
+ {resolved.length === 0 ? ( +
None.
+ ) : ( + resolved.map((inc) => ) + )} +
+ ); +} +``` + +- [ ] **Step 2: Replace the mount site** + +In `src/syncfield/viewer/frontend/src/App.tsx`, find the `` usage and replace with: + +```tsx +import { IncidentPanel } from "./components/incident-panel"; + +// inside render, wherever HealthTable lived: + +``` + +Delete the `import { HealthTable } from "./components/health-table";` line. + +- [ ] **Step 3: Delete the old file** + +```bash +rm src/syncfield/viewer/frontend/src/components/health-table.tsx +``` + +- [ ] **Step 4: Typecheck and build** + +```bash +cd src/syncfield/viewer/frontend && npx tsc --noEmit && npm run build +``` +Expected: clean type-check. The bundled assets update in `static/` (or wherever the existing pipeline ships them). + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/viewer/frontend/src/components/incident-panel.tsx \ + src/syncfield/viewer/frontend/src/App.tsx \ + src/syncfield/viewer/static/ # if generated bundle is versioned +git rm src/syncfield/viewer/frontend/src/components/health-table.tsx +git commit -m "feat(viewer-fe): add IncidentPanel (replaces HealthTable)" +``` + +--- + +### Task 24: Stream card severity badge + +**Files:** +- Modify: `src/syncfield/viewer/frontend/src/components/stream-card.tsx` + +- [ ] **Step 1: Replace the red-dot / health_count display** + +Find where `stream.health_count` (or `problem_count`) was rendered. Replace with a per-stream severity count computed from `active_incidents`: + +```tsx +import type { IncidentSnapshot, Severity } from "../lib/types"; + +function streamIncidentStats(streamId: string, active: IncidentSnapshot[]) { + const mine = active.filter((i) => i.stream_id === streamId); + const count = mine.length; + const highest: Severity | null = mine.reduce((acc, i) => { + if (acc === null) return i.severity; + const order: Severity[] = ["info", "warning", "error", "critical"]; + return order.indexOf(i.severity) > order.indexOf(acc) ? i.severity : acc; + }, null); + return { count, highest }; +} + +const BADGE_COLOR: Record = { + info: "bg-slate-500", + warning: "bg-yellow-500", + error: "bg-orange-500", + critical: "bg-red-500", +}; + +export function StreamCard({ stream, activeIncidents }: { + stream: StreamSnapshot; + activeIncidents: IncidentSnapshot[]; +}) { + const { count, highest } = streamIncidentStats(stream.id, activeIncidents); + return ( +
+ {/* ...existing header... */} + {count > 0 && highest && ( + + {count} + + )} + {/* ...rest of card... */} +
+ ); +} +``` + +Propagate `activeIncidents` from the parent that already has the snapshot. + +- [ ] **Step 2: Typecheck** + +```bash +cd src/syncfield/viewer/frontend && npx tsc --noEmit +``` + +- [ ] **Step 3: Commit** + +```bash +git add src/syncfield/viewer/frontend/src/components/stream-card.tsx \ + src/syncfield/viewer/frontend/src/App.tsx # if parent changed +git commit -m "feat(viewer-fe): stream card severity badge from active incidents" +``` + +--- + +## Phase 7 — target_hz rollout on other adapters + +### Task 25: Declare target_hz on all known-target adapters + +**Files:** +- Modify: `src/syncfield/adapters/uvc_webcam.py` +- Modify: `src/syncfield/adapters/host_audio.py` +- Modify: `src/syncfield/adapters/meta_quest_camera/stream.py` (or wherever its `StreamCapabilities` are built) +- Modify: `src/syncfield/adapters/ble_imu.py` (if a stable rate is known) +- Modify: `src/syncfield/adapters/insta360_go3s/stream.py` (if a live preview rate exists — otherwise skip) +- Modify: `src/syncfield/adapters/polling_sensor.py`, `push_sensor.py` (if they accept a rate arg) + +- [ ] **Step 1: For each adapter, locate its StreamCapabilities construction** + +```bash +grep -rn "StreamCapabilities(" src/syncfield/adapters/ +``` + +- [ ] **Step 2: For each, thread the existing rate argument (`fps`, `rate_hz`, `target_hz`) into `target_hz=`** + +Example for `uvc_webcam.py`: + +```python +# before: +capabilities=StreamCapabilities(produces_file=True, is_removable=True) +# after: +capabilities=StreamCapabilities(produces_file=True, is_removable=True, target_hz=float(self._fps)) +``` + +For `host_audio.py`, the "sample rate" is huge (48 kHz) — that isn't a per-sample-emission rate. Audio streams emit one `SampleEvent` per chunk, so set `target_hz` to `sample_rate / block_size`. If the adapter doesn't expose a block-rate cleanly, leave `target_hz=None` and rely on the baseline-learning fallback. + +- [ ] **Step 3: Run the full suite to catch regressions** + +```bash +pytest tests/ -q +``` + +- [ ] **Step 4: Commit** + +```bash +git add src/syncfield/adapters/ +git commit -m "feat(adapters): declare target_hz on UVC / audio / Meta Quest / BLE IMU / sensors" +``` + +--- + +## Phase 8 — Manual verification + +### Task 26: Manual verification on real OAK hardware + +This task is a human-run checklist, not automated code. Its purpose is to confirm the OAK-motivated failure modes surface correctly. + +- [ ] **Step 1: Run a normal session, no incidents expected** + +```bash +python examples/oak_live_preview.py # or whichever example runs an OAK-only session +``` +Start a recording, let it run for 30 s, stop. Open the viewer. + +Expected: **Active Issues (0)**, **Resolved this session (0)**. + +- [ ] **Step 2: Induce an XLink stall** + +With a recording running, physically unplug the OAK's USB cable. Wait 3 seconds. Re-plug. + +Expected in the viewer within 3 s of unplug: +- A **stream-stall** incident appears under Active Issues with title like "Stream stalled (silence 2.0s)", severity=`error`. +- A **xlink-error** incident appears separately, severity=`error`. +- After reconnection succeeds and samples flow for ~1 s, the stall incident moves to Resolved. + +- [ ] **Step 3: Induce a crash (if reproducible)** + +If a deterministic crash-reproducer exists (e.g., requesting an unsupported pipeline config mid-run), trigger it. + +Expected: +- A **device-crash** incident appears under Active Issues with severity=`critical` and an attached artifact chip showing `crash_dump`. +- The incident's expanded view shows the `crash_dump.json` absolute path. + +- [ ] **Step 4: Stop and inspect persisted artifacts** + +After `stop()`: + +```bash +cat $(find data_leader -name incidents.jsonl | head -1) +``` + +Expected: one JSON line per incident open/update/close. Crash incident's line includes `"artifacts": [{"kind": "crash_dump", "path": "..."}]`. + +- [ ] **Step 5: Cross-check FinalizationReport** + +Inside the example script, print `report.incidents` for each stream. Confirm the open/close states match the viewer's Active/Resolved counts at stop time. + +- [ ] **Step 6: Capture screenshots for the PR** + +Take a screenshot of the viewer during the induced stall and after recovery. Attach both to the PR description. + +--- + +## Self-Review Checklist + +After completing the plan, run this checklist before merging: + +1. **Spec coverage**: + - Goal 1 (live detection): Tasks 9–14, 17, 20–24 ✓ + - Goal 2 (post-session report): Tasks 16, 17 (FinalizationReport), 20 (persistence) ✓ + - Goal 3 (sensor-agnostic baseline): Tasks 8, 9–14 ✓ + - Goal 4 (pluggable detectors): Tasks 4, 5, 8 (`register()`) ✓ + - Goal 5 (default-on): Task 8 (`_install_default_detectors`), Task 19 (OAK bridge auto-install) ✓ + - Goal 6 (artifact capture): Task 18 (crash dump path), Task 19 (IncidentArtifact attach) ✓ + - Goal 7 (zero hot-path impact): Task 7 (lock-free SimpleQueues, dedicated daemon) ✓ + +2. **No placeholders**: every step shows complete code or a concrete command. No "TBD" / "similar to" / "handle edge cases". + +3. **Type consistency**: `HealthSystem.observe_*`, `Detector.observe_*`, `IncidentTracker.ingest/tick`, `SessionLogWriter.log_incident`, `SessionSnapshot.active_incidents`/`resolved_incidents` — all names line up across tasks. + +--- + +## Plan complete and saved to `docs/superpowers/plans/2026-04-22-health-telemetry.md`. + +**Two execution options:** + +1. **Subagent-Driven (recommended)** — fresh subagent per task, review between tasks, fast iteration. +2. **Inline Execution** — execute tasks in this session with checkpoints for review. + +Which approach? diff --git a/docs/superpowers/plans/2026-04-22-partial-connect-ux.md b/docs/superpowers/plans/2026-04-22-partial-connect-ux.md new file mode 100644 index 0000000..2f5fd23 --- /dev/null +++ b/docs/superpowers/plans/2026-04-22-partial-connect-ux.md @@ -0,0 +1,1708 @@ +# Partial Connect UX Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace the all-or-nothing `SessionOrchestrator.connect()` with per-stream partial connect, surface per-stream connection state + error in the snapshot, fire a new `NoDataDetector` for the "connected but no sample ever" case, and redesign the viewer's `StreamCard` to render state-aware overlays (connecting / waiting / failed) plus a degraded-state header chip. + +**Architecture:** Orchestrator catches each `stream.connect()` independently, records a per-stream `ConnectionState` on `self._stream_states`, and emits structured `HealthEvent`s that feed the existing (previously dormant) `StartupFailureDetector`. A new `Detector.observe_connection_state` hook is added to the health protocol so `NoDataDetector` can track per-stream "entered `connected` at" timestamps. Snapshot gains two new fields; the viewer branches its `StreamCard` body on the connection state and shows a yellow `Ready (n/total)` chip when any stream is failed. + +**Tech Stack:** Python 3.9+ (stdlib + existing health/ package), React + TypeScript + Tailwind for the viewer. No new external dependencies. + +**Spec:** `docs/superpowers/specs/2026-04-22-partial-connect-ux-design.md` + +--- + +## File Structure + +### New (backend) + +``` +src/syncfield/health/detectors/no_data.py # NoDataDetector +``` + +### Modified (backend) + +- `src/syncfield/health/detector.py` — add `observe_connection_state` hook to Protocol + DetectorBase. +- `src/syncfield/health/worker.py` — new `_connection_states` ingress queue + fan-out to detectors. +- `src/syncfield/health/system.py` — `observe_connection_state` passthrough; register `NoDataDetector`. +- `src/syncfield/orchestrator.py` — `_stream_states` / `_stream_errors` dicts, `_set_stream_state` helper, partial-connect rewrite in `connect()`, failed-stream skip in `disconnect()`. +- `src/syncfield/viewer/state.py` — `StreamSnapshot.connection_state` / `connection_error` fields. +- `src/syncfield/viewer/poller.py` — read orchestrator's state dicts into each snapshot. +- `src/syncfield/viewer/server.py` — WebSocket serializer includes new stream fields. + +### New (frontend) + +``` +src/syncfield/viewer/frontend/src/components/stream-overlays.tsx +``` + +### Modified (frontend) + +- `src/syncfield/viewer/frontend/src/lib/types.ts` — `ConnectionState` type + `StreamSnapshot` field additions. +- `src/syncfield/viewer/frontend/src/components/stream-card.tsx` — branch body on `connection_state`. +- `src/syncfield/viewer/frontend/src/components/header.tsx` — degraded-state chip. + +### Tests (new + extended) + +``` +tests/unit/health/detectors/test_no_data.py +tests/unit/health/test_worker_connection_state.py (or add cases to test_health_worker.py) +tests/unit/test_orchestrator_partial_connect.py (new test class) +tests/integration/health/test_partial_connect.py +tests/integration/health/test_no_data_detector.py +tests/unit/viewer/test_snapshot_connection_state.py (or add cases to test_snapshot_incidents.py) +``` + +--- + +## Conventions + +- TDD every task: failing test → confirm fail → implement → confirm pass → commit. +- Run tests via `uv run pytest -v`. +- Commits follow the existing Conventional Commits style. Every commit includes the trailer `Co-Authored-By: Claude Opus 4.7 (1M context) ` (HEREDOC form). +- Implementer is agnostic to task order only within blocks marked "can be parallelized"; otherwise follow task number order. + +--- + +## Task 1 — Add `observe_connection_state` hook to Detector protocol + base + +**Files:** +- Modify: `src/syncfield/health/detector.py` +- Modify: `tests/unit/health/test_detector_base.py` + +- [ ] **Step 1: Write failing test** + +Append to `tests/unit/health/test_detector_base.py`: + +```python +def test_detector_base_observe_connection_state_default_is_noop(): + d = NoopDetector() + # Does not raise; returns None. + assert d.observe_connection_state("cam", "connected", 100) is None +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +uv run pytest tests/unit/health/test_detector_base.py::test_detector_base_observe_connection_state_default_is_noop -v +``` +Expected: `AttributeError: 'NoopDetector' object has no attribute 'observe_connection_state'`. + +- [ ] **Step 3: Implement** + +In `src/syncfield/health/detector.py`, add the method to the `Detector` Protocol (alongside the other five `observe_*` methods): + +```python + def observe_connection_state(self, stream_id: str, new_state: str, at_ns: int) -> None: ... +``` + +And to `DetectorBase` (as a no-op, matching the other `observe_*` defaults): + +```python + def observe_connection_state(self, stream_id: str, new_state: str, at_ns: int) -> None: + pass +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +uv run pytest tests/unit/health/test_detector_base.py -v +``` +Expected: all tests pass (previous 3 + new 1 = 4 passed). + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/detector.py tests/unit/health/test_detector_base.py +git commit -m "$(cat <<'EOF' +feat(health): add Detector.observe_connection_state hook + +Adds a no-op default for the per-stream connection-state observer that +NoDataDetector will override. Protocol + base mirror the existing +observe_sample / observe_health / observe_state shape. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 2 — HealthWorker: `_connection_states` ingress queue + fan-out + +**Files:** +- Modify: `src/syncfield/health/worker.py` +- Modify: `tests/unit/health/test_health_worker.py` + +- [ ] **Step 1: Write failing test** + +Append to `tests/unit/health/test_health_worker.py` (inside the existing module, at top-level): + +```python +def test_worker_drains_connection_state_queue_and_fans_out(): + class Spy(DetectorBase): + name = "conn-spy" + default_severity = Severity.INFO + + def __init__(self): + self.calls = [] + + def observe_connection_state(self, stream_id, new_state, at_ns): + self.calls.append((stream_id, new_state, at_ns)) + + tr = IncidentTracker() + spy = Spy() + w = HealthWorker(tracker=tr, detectors=[spy], tick_hz=100) + w.start() + try: + w.push_connection_state("cam", "connecting", 1) + w.push_connection_state("cam", "connected", 2) + assert _wait_until(lambda: len(spy.calls) == 2) + finally: + w.stop() + + assert spy.calls == [("cam", "connecting", 1), ("cam", "connected", 2)] +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +uv run pytest tests/unit/health/test_health_worker.py::test_worker_drains_connection_state_queue_and_fans_out -v +``` +Expected: `AttributeError: 'HealthWorker' object has no attribute 'push_connection_state'`. + +- [ ] **Step 3: Implement** + +In `src/syncfield/health/worker.py`: + +1. Add a new message dataclass near the other `_*Msg`: + +```python +@dataclass(frozen=True) +class _ConnectionStateMsg: + stream_id: str + new_state: str + at_ns: int +``` + +2. Add the queue field in `HealthWorker.__init__` (alongside the existing four): + +```python + self._connection_states: "queue.SimpleQueue[_ConnectionStateMsg]" = queue.SimpleQueue() +``` + +3. Add the ingress method: + +```python + def push_connection_state(self, stream_id: str, new_state: str, at_ns: int) -> None: + self._connection_states.put(_ConnectionStateMsg(stream_id, new_state, at_ns)) +``` + +4. Extend `_drain_once` with a final block: + +```python + for msg in _drain_queue(self._connection_states): + for d in self._detectors: + d.observe_connection_state(msg.stream_id, msg.new_state, msg.at_ns) +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +uv run pytest tests/unit/health/test_health_worker.py -v +``` +Expected: 5 passed (previous 4 + new 1). + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/worker.py tests/unit/health/test_health_worker.py +git commit -m "$(cat <<'EOF' +feat(health): add per-stream connection-state ingress to HealthWorker + +New _ConnectionStateMsg + push_connection_state() + _drain_once fan-out +so NoDataDetector (next commit) can track per-stream CONNECTING → +CONNECTED transitions via the standard observer pattern. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 3 — HealthSystem: `observe_connection_state` passthrough + +**Files:** +- Modify: `src/syncfield/health/system.py` +- Modify: `tests/unit/health/test_health_system.py` + +- [ ] **Step 1: Write failing test** + +Append to `tests/unit/health/test_health_system.py`: + +```python +def test_health_system_observe_connection_state_routes_to_worker(): + class Spy(DetectorBase): + name = "conn-spy" + default_severity = Severity.INFO + + def __init__(self): + self.calls = [] + + def observe_connection_state(self, stream_id, new_state, at_ns): + self.calls.append((stream_id, new_state, at_ns)) + + hs = HealthSystem() + spy = Spy() + hs.register(spy) + + hs.start() + try: + hs.observe_connection_state("cam", "connecting", 10) + hs.observe_connection_state("cam", "connected", 20) + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline and len(spy.calls) < 2: + time.sleep(0.02) + finally: + hs.stop() + assert spy.calls == [("cam", "connecting", 10), ("cam", "connected", 20)] +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +uv run pytest tests/unit/health/test_health_system.py::test_health_system_observe_connection_state_routes_to_worker -v +``` +Expected: `AttributeError: 'HealthSystem' object has no attribute 'observe_connection_state'`. + +- [ ] **Step 3: Implement** + +In `src/syncfield/health/system.py`, add the passthrough method (right next to the other `observe_*` methods): + +```python + def observe_connection_state(self, stream_id: str, new_state: str, at_ns: int) -> None: + if self._worker is not None: + self._worker.push_connection_state(stream_id, new_state, at_ns) +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +uv run pytest tests/unit/health/test_health_system.py -v +``` +Expected: all existing tests + new one all pass. + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/system.py tests/unit/health/test_health_system.py +git commit -m "$(cat <<'EOF' +feat(health): expose observe_connection_state on HealthSystem + +Passthrough to the worker's ingress queue, matching the existing +observe_sample / observe_health / observe_state shape. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 4 — NoDataDetector (core logic + unit tests) + +**Files:** +- Create: `src/syncfield/health/detectors/no_data.py` +- Create: `tests/unit/health/detectors/test_no_data.py` + +- [ ] **Step 1: Write failing test** + +`tests/unit/health/detectors/test_no_data.py`: + +```python +from syncfield.health.detectors.no_data import NoDataDetector +from syncfield.health.types import Incident +from syncfield.types import SampleEvent + + +def _s(stream: str, t_ns: int) -> SampleEvent: + return SampleEvent(stream_id=stream, frame_number=0, capture_ns=t_ns) + + +def test_no_fire_before_threshold(): + d = NoDataDetector(threshold_ns=1000) + d.observe_connection_state("cam", "connected", at_ns=100) + assert list(d.tick(now_ns=500)) == [] # 400 ns elapsed, under 1000 + + +def test_fires_after_threshold_without_sample(): + d = NoDataDetector(threshold_ns=1000) + d.observe_connection_state("cam", "connected", at_ns=100) + out = list(d.tick(now_ns=2000)) # 1900 ns elapsed + assert len(out) == 1 + ev = out[0] + assert ev.stream_id == "cam" + assert ev.fingerprint == "cam:no-data" + assert ev.source == "detector:no-data" + assert "no data" in (ev.detail or "").lower() + + +def test_does_not_refire_while_still_no_data(): + d = NoDataDetector(threshold_ns=1000) + d.observe_connection_state("cam", "connected", at_ns=100) + first = list(d.tick(now_ns=2000)) + second = list(d.tick(now_ns=3000)) + assert len(first) == 1 + assert len(second) == 0 + + +def test_close_condition_satisfied_once_sample_arrives(): + d = NoDataDetector(threshold_ns=1000) + d.observe_connection_state("cam", "connected", at_ns=100) + events = list(d.tick(now_ns=2000)) + inc = Incident.opened_from(events[0], title="x") + + assert d.close_condition(inc, now_ns=2100) is False # still no sample + + d.observe_sample("cam", _s("cam", 2200)) + assert d.close_condition(inc, now_ns=2300) is True + + +def test_resets_bookkeeping_on_non_connected_state(): + d = NoDataDetector(threshold_ns=1000) + d.observe_connection_state("cam", "connected", at_ns=100) + list(d.tick(now_ns=2000)) # fires + + d.observe_connection_state("cam", "failed", at_ns=2500) + # Back to connected → fresh clock, no duplicate fire. + d.observe_connection_state("cam", "connected", at_ns=3000) + assert list(d.tick(now_ns=3500)) == [] # only 500 ns since new connected + + +def test_per_stream_independent_state(): + d = NoDataDetector(threshold_ns=1000) + d.observe_connection_state("a", "connected", at_ns=100) + d.observe_connection_state("b", "connected", at_ns=100) + d.observe_sample("b", _s("b", 200)) + + out = list(d.tick(now_ns=2000)) + assert len(out) == 1 + assert out[0].stream_id == "a" +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +uv run pytest tests/unit/health/detectors/test_no_data.py -v +``` +Expected: `ModuleNotFoundError: No module named 'syncfield.health.detectors.no_data'`. + +- [ ] **Step 3: Implement** + +`src/syncfield/health/detectors/no_data.py`: + +```python +"""NoDataDetector — fires when a stream is connected but never emits a sample. + +Complements StreamStallDetector (which requires prior samples). Catches +the "connected but silent" case such as an OAK pipeline that fails to +pump frames even though its device connected. Resets bookkeeping on +any non-connected state transition so a reconnect starts a fresh clock. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from typing import Dict, List, Set + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.types import Incident +from syncfield.types import HealthEvent, HealthEventKind, SampleEvent + + +class NoDataDetector(DetectorBase): + name = "no-data" + default_severity = Severity.ERROR + + def __init__(self, threshold_ns: int = 5_000_000_000) -> None: + self._threshold_ns = threshold_ns + self._connected_at: Dict[str, int] = {} + self._has_sample: Set[str] = set() + self._fire_active: Dict[str, bool] = {} + + def observe_connection_state(self, stream_id: str, new_state: str, at_ns: int) -> None: + if new_state == "connected": + self._connected_at[stream_id] = at_ns + self._has_sample.discard(stream_id) + self._fire_active[stream_id] = False + else: + # idle / connecting / failed / disconnected → reset everything. + self._connected_at.pop(stream_id, None) + self._has_sample.discard(stream_id) + self._fire_active.pop(stream_id, None) + + def observe_sample(self, stream_id: str, sample: SampleEvent) -> None: + self._has_sample.add(stream_id) + + def tick(self, now_ns: int) -> Iterator[HealthEvent]: + out: List[HealthEvent] = [] + for stream_id, connected_at in self._connected_at.items(): + if stream_id in self._has_sample: + continue + elapsed = now_ns - connected_at + if elapsed >= self._threshold_ns and not self._fire_active.get(stream_id, False): + self._fire_active[stream_id] = True + out.append(HealthEvent( + stream_id=stream_id, + kind=HealthEventKind.ERROR, + at_ns=now_ns, + detail=f"Connected {elapsed / 1e9:.1f}s ago but no data received", + severity=self.default_severity, + source=f"detector:{self.name}", + fingerprint=f"{stream_id}:{self.name}", + data={"connected_at_ns": connected_at, "elapsed_ns": elapsed}, + )) + return iter(out) + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + return incident.stream_id in self._has_sample +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +uv run pytest tests/unit/health/detectors/test_no_data.py -v +``` +Expected: 6 passed. + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/detectors/no_data.py tests/unit/health/detectors/test_no_data.py +git commit -m "$(cat <<'EOF' +feat(health): add NoDataDetector for connected-but-silent streams + +Fires when a stream is in 'connected' state for N seconds without any +sample. Complements StreamStallDetector which requires prior samples. +Catches the OAK "black square" symptom where connect() succeeds but +the pipeline never pumps frames. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 5 — Register NoDataDetector in HealthSystem default suite + +**Files:** +- Modify: `src/syncfield/health/system.py` +- Modify: `tests/unit/health/test_health_system.py` (extend existing `test_health_system_installs_default_detectors`) + +- [ ] **Step 1: Update the assertion** + +In `tests/unit/health/test_health_system.py`, find `test_health_system_installs_default_detectors` and add `"no-data"` to the expected detector set: + +```python + for expected in ( + "adapter", + "stream-stall", + "fps-drop", + "jitter", + "startup-failure", + "backpressure", + "no-data", + ): + assert expected in names, f"missing default detector: {expected}" +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +uv run pytest tests/unit/health/test_health_system.py::test_health_system_installs_default_detectors -v +``` +Expected: `AssertionError: missing default detector: no-data`. + +- [ ] **Step 3: Implement** + +In `src/syncfield/health/system.py`, add the import and registration: + +```python +from syncfield.health.detectors.no_data import NoDataDetector +``` + +Inside `_install_default_detectors`, after `self.register(BackpressureDetector())`, add: + +```python + self.register(NoDataDetector()) +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +uv run pytest tests/unit/health/test_health_system.py -v +``` +Expected: all pass. + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/health/system.py tests/unit/health/test_health_system.py +git commit -m "$(cat <<'EOF' +feat(health): register NoDataDetector in default suite + +Now installed automatically on HealthSystem construction alongside the +other six default detectors. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 6 — Orchestrator: per-stream state dicts + `_set_stream_state` helper + +**Files:** +- Modify: `src/syncfield/orchestrator.py` +- Test: no dedicated test file for this task — the test lands in Task 8 (partial connect). + +- [ ] **Step 1: Add state containers to `__init__`** + +In `src/syncfield/orchestrator.py`, inside `SessionOrchestrator.__init__`, after the line `self._connected_streams: List[Stream] = []` (around line 360), add: + +```python + # Per-stream connection state for partial-connect semantics. + # Keys are stream ids; values are one of: + # "idle" | "connecting" | "connected" | "failed" | "disconnected". + self._stream_states: dict[str, str] = {} + # Populated only when a stream's connect() raised. + self._stream_errors: dict[str, str] = {} +``` + +- [ ] **Step 2: Add `_set_stream_state` helper** + +Add a new method on the class (near the other private helpers, after `_persist_incident`): + +```python + def _set_stream_state(self, stream_id: str, new_state: str) -> None: + """Update per-stream connection state and forward to HealthSystem. + + The health worker may not be running yet (e.g. we call this from + add() when the session is IDLE) — observe_connection_state is a + no-op in that case. + """ + self._stream_states[stream_id] = new_state + self.health.observe_connection_state(stream_id, new_state, time.monotonic_ns()) +``` + +- [ ] **Step 3: Wire into `add()`** + +Find `SessionOrchestrator.add(stream)` (around line 1456). At the end of the method body (after any existing wiring such as `stream.on_sample(...)` / `stream.on_health(...)`), add: + +```python + self._set_stream_state(stream.id, "idle") +``` + +- [ ] **Step 4: Quick sanity check** + +```bash +uv run pytest tests/unit/test_orchestrator.py::TestAdd -v +``` +Expected: all existing `TestAdd` tests still pass (no regression). + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/orchestrator.py +git commit -m "$(cat <<'EOF' +feat(orchestrator): add per-stream connection-state dicts + helper + +_stream_states and _stream_errors become the source of truth for +per-stream status. _set_stream_state is the single update point — it +also forwards to HealthSystem.observe_connection_state so NoDataDetector +sees the transitions. add() initializes each new stream to 'idle'. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 7 — Orchestrator: partial-connect rewrite (happy path + error path + success signal) + +**Files:** +- Modify: `src/syncfield/orchestrator.py` (the `connect()` method, around lines 1590-1655) +- Test: `tests/unit/test_orchestrator_partial_connect.py` (new) + +- [ ] **Step 1: Write failing test** + +`tests/unit/test_orchestrator_partial_connect.py`: + +```python +"""Partial-connect semantics for SessionOrchestrator. + +Relies on the FakeStream helper in syncfield.testing, which supports +`fail_on_start=True` to raise from its connect() path. +""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +from syncfield.orchestrator import SessionOrchestrator +from syncfield.testing import FakeStream +from syncfield.types import SessionState + + +def test_one_stream_fails_others_still_connected(tmp_path: Path): + sess = SessionOrchestrator(host_id="h", output_dir=tmp_path) + sess.add(FakeStream("good_a")) + sess.add(FakeStream("bad", fail_on_start=True)) + sess.add(FakeStream("good_b")) + + sess.connect() + + assert sess.state_name == SessionState.CONNECTED.value + assert sess._stream_states["good_a"] == "connected" + assert sess._stream_states["bad"] == "failed" + assert sess._stream_states["good_b"] == "connected" + assert "bad" in sess._stream_errors + assert sess._stream_errors["bad"] # non-empty message + + +def test_all_streams_failing_raises_and_returns_to_idle(tmp_path: Path): + sess = SessionOrchestrator(host_id="h", output_dir=tmp_path) + sess.add(FakeStream("a", fail_on_start=True)) + sess.add(FakeStream("b", fail_on_start=True)) + + with pytest.raises(RuntimeError, match="no streams"): + sess.connect() + + assert sess.state_name == SessionState.IDLE.value + assert sess._stream_states["a"] == "failed" + assert sess._stream_states["b"] == "failed" + + +def test_startup_failure_event_reaches_health_system(tmp_path: Path): + # Spy detector that captures health events it observes. + from syncfield.health.detector import DetectorBase + from syncfield.health.severity import Severity + + class Spy(DetectorBase): + name = "startup-spy" + default_severity = Severity.INFO + + def __init__(self): + self.events = [] + + def observe_health(self, stream_id, event): + self.events.append(event) + + sess = SessionOrchestrator(host_id="h", output_dir=tmp_path) + spy = Spy() + sess.health.register(spy) + sess.add(FakeStream("good")) + sess.add(FakeStream("bad", fail_on_start=True)) + + sess.connect() + + # Give the worker a tick to drain the health queue. + import time + deadline = time.monotonic() + 1.0 + while time.monotonic() < deadline and not any( + e.fingerprint == "bad:startup-failure" for e in spy.events + ): + time.sleep(0.02) + + failure_events = [e for e in spy.events if e.fingerprint == "bad:startup-failure"] + assert failure_events, "no startup-failure event observed" + ev = failure_events[0] + assert ev.data.get("phase") == "connect" + assert ev.data.get("outcome") == "error" + assert ev.data.get("error") +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +uv run pytest tests/unit/test_orchestrator_partial_connect.py -v +``` +Expected: multiple failures — the first test hits the current all-or-nothing rollback and the session ends up in IDLE. + +- [ ] **Step 3: Implement** + +In `src/syncfield/orchestrator.py`, replace the existing `connect()` body's try/except block (lines 1623-1643 in the current file — find the section starting `connected: List[Stream] = []` and ending just before `self._connected_streams = connected`) with: + +```python + connected: List[Stream] = [] + for stream in self._streams.values(): + self._set_stream_state(stream.id, "connecting") + try: + stream.prepare() + stream.connect() + except Exception as exc: + self._stream_errors[stream.id] = str(exc) + self._set_stream_state(stream.id, "failed") + stream._emit_health(HealthEvent( + stream_id=stream.id, + kind=HealthEventKind.ERROR, + at_ns=time.monotonic_ns(), + detail=str(exc), + severity=Severity.ERROR, + source="orchestrator", + fingerprint=f"{stream.id}:startup-failure", + data={"phase": "connect", "outcome": "error", "error": str(exc)}, + )) + continue + connected.append(stream) + self._stream_errors.pop(stream.id, None) + self._set_stream_state(stream.id, "connected") + stream._emit_health(HealthEvent( + stream_id=stream.id, + kind=HealthEventKind.HEARTBEAT, + at_ns=time.monotonic_ns(), + detail="connected", + severity=Severity.INFO, + source="orchestrator", + fingerprint=f"{stream.id}:startup-success", + data={"phase": "connect", "outcome": "success"}, + )) + + if not connected: + self._transition(SessionState.IDLE) + if self._log_writer is not None: + self._log_writer.close() + self._log_writer = None + raise RuntimeError( + "connect() failed: no streams connected — every adapter raised. " + "Inspect per-stream errors via session._stream_errors." + ) +``` + +Imports at the top of the file need to include `Severity` if not already present: + +```python +from syncfield.health.severity import Severity +``` + +(`HealthEvent` and `HealthEventKind` should already be imported.) + +- [ ] **Step 4: Run, confirm pass** + +```bash +uv run pytest tests/unit/test_orchestrator_partial_connect.py -v +``` +Expected: 3 passed. + +Also run the full orchestrator suite to surface regressions: + +```bash +uv run pytest tests/unit/test_orchestrator.py -v +``` +Expected: all pre-existing tests still pass. + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/orchestrator.py tests/unit/test_orchestrator_partial_connect.py +git commit -m "$(cat <<'EOF' +feat(orchestrator): partial-connect — survive per-stream connect() failure + +One stream raising no longer rolls the whole session back to IDLE. We +record the error, emit a structured HealthEvent (phase=connect, +outcome=error) that activates the previously-dormant +StartupFailureDetector, and continue with the rest. Session only fails +if every adapter raises. Success path emits the complementary +outcome=success signal so the detector's recovery tracking works. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 8 — Orchestrator: `disconnect()` skips failed streams, updates state + +**Files:** +- Modify: `src/syncfield/orchestrator.py` (the `disconnect()` method around line 2100) +- Test: `tests/unit/test_orchestrator_partial_connect.py` (extend) + +- [ ] **Step 1: Write failing test** + +Append to `tests/unit/test_orchestrator_partial_connect.py`: + +```python +def test_disconnect_does_not_call_stream_disconnect_on_failed(tmp_path: Path): + from syncfield.testing import FakeStream + + class CountingFakeStream(FakeStream): + def __init__(self, stream_id, fail_on_start=False): + super().__init__(stream_id, fail_on_start=fail_on_start) + self.disconnect_calls = 0 + + def disconnect(self): + self.disconnect_calls += 1 + super().disconnect() + + sess = SessionOrchestrator(host_id="h", output_dir=tmp_path) + good = CountingFakeStream("good") + bad = CountingFakeStream("bad", fail_on_start=True) + sess.add(good) + sess.add(bad) + + sess.connect() + sess.disconnect() + + assert good.disconnect_calls == 1 + assert bad.disconnect_calls == 0 + assert sess._stream_states["good"] == "disconnected" + assert sess._stream_states["bad"] == "disconnected" + assert sess._stream_errors == {} +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +uv run pytest tests/unit/test_orchestrator_partial_connect.py::test_disconnect_does_not_call_stream_disconnect_on_failed -v +``` +Expected: at minimum `sess._stream_errors != {}` (errors never cleared), or `bad.disconnect_calls > 0` because the rollback helper iterates `_connected_streams` — which is correct for good, but we also need to flip each stream's state. + +- [ ] **Step 3: Implement** + +Locate `SessionOrchestrator.disconnect()` (around line 2100). Replace the body of the `with self._lock:` block (the part that calls `_rollback_disconnect_streams(self._connected_streams)`) with: + +```python + if self._state not in (SessionState.CONNECTED, SessionState.STOPPED): + raise RuntimeError( + f"disconnect() requires CONNECTED or STOPPED state; " + f"current state is {self._state.value}" + ) + # Only streams that were 'connected' (or 'recording' → 'stopped') + # ever opened hardware; 'failed' streams never did, so skip them. + _rollback_disconnect_streams(self._connected_streams) + self._connected_streams = [] + + # Flip every stream's snapshot-visible state to 'disconnected', + # regardless of whether we called disconnect() on it. Clear any + # per-stream errors since the session is returning to a clean slate. + for stream_id in list(self._stream_states.keys()): + self._set_stream_state(stream_id, "disconnected") + self._stream_errors.clear() + + # Keep auto-injected audio stream registered (visible in viewer) + # but disconnected. It will be reconnected on next connect(). + + # Multi-host infrastructure (advertiser, browser, control plane) + # stays up across disconnect(). It was brought up at __init__ + # and is only torn down by shutdown() or the atexit handler. +``` + +- [ ] **Step 4: Run, confirm pass** + +```bash +uv run pytest tests/unit/test_orchestrator_partial_connect.py -v +``` +Expected: 4 passed. + +```bash +uv run pytest tests/unit/test_orchestrator.py -v +``` +Expected: no regression. + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/orchestrator.py tests/unit/test_orchestrator_partial_connect.py +git commit -m "$(cat <<'EOF' +feat(orchestrator): disconnect() handles partial-connect survivors + +_connected_streams only ever held successful streams, so the existing +rollback helper is already correct — but we now flip every tracked +stream's state to 'disconnected' and clear per-stream errors so the +viewer reflects a clean slate after teardown. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 9 — `StreamSnapshot.connection_state` + `connection_error` fields + +**Files:** +- Modify: `src/syncfield/viewer/state.py` +- Test: `tests/unit/viewer/test_snapshot_incidents.py` (extend) + +- [ ] **Step 1: Write failing test** + +Append to `tests/unit/viewer/test_snapshot_incidents.py`: + +```python +def test_stream_snapshot_has_connection_state_fields(): + import dataclasses + from syncfield.viewer.state import StreamSnapshot + + fields = {f.name: f for f in dataclasses.fields(StreamSnapshot)} + assert "connection_state" in fields + assert "connection_error" in fields + + # Defaults when constructed minimally. + snap = StreamSnapshot( + id="cam", kind="video", provides_audio_track=False, produces_file=False, + frame_count=0, last_sample_at_ns=None, effective_hz=0.0, + latest_frame=None, plot_points={}, latest_pose={}, + ) + assert snap.connection_state == "idle" + assert snap.connection_error is None +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +uv run pytest tests/unit/viewer/test_snapshot_incidents.py::test_stream_snapshot_has_connection_state_fields -v +``` +Expected: `AssertionError: 'connection_state' not in fields`. + +- [ ] **Step 3: Implement** + +In `src/syncfield/viewer/state.py`, find the `StreamSnapshot` dataclass. Add two new fields **at the end** of the dataclass (after `live_preview: bool = True`), so constructors that supply only positional arguments for existing fields keep working: + +```python + connection_state: str = "idle" + connection_error: Optional[str] = None +``` + +Ensure `Optional` is imported (it already is in that file). + +- [ ] **Step 4: Run, confirm pass** + +```bash +uv run pytest tests/unit/viewer/test_snapshot_incidents.py -v +``` +Expected: all pass. + +Regression check: + +```bash +uv run pytest tests/unit/viewer -v +``` + +If any test constructs `StreamSnapshot` positionally and breaks — fix inline by switching to keyword arguments in that test. Add a note to the commit message if so. + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/viewer/state.py tests/unit/viewer/test_snapshot_incidents.py +git commit -m "$(cat <<'EOF' +feat(viewer): add connection_state / connection_error to StreamSnapshot + +Defaults ('idle' / None) preserve existing behavior for any construction +that doesn't set them. Poller (next task) wires the orchestrator's per- +stream state into these fields. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 10 — Poller: propagate orchestrator state into snapshot + +**Files:** +- Modify: `src/syncfield/viewer/poller.py` +- Test: `tests/unit/viewer/test_poller.py` (extend) + +- [ ] **Step 1: Write failing test** + +Append to `tests/unit/viewer/test_poller.py` (use the existing test harness pattern — find how other tests construct a session and poller): + +```python +def test_poller_snapshot_includes_connection_state(tmp_path): + from syncfield.orchestrator import SessionOrchestrator + from syncfield.testing import FakeStream + from syncfield.viewer.poller import SessionPoller + + sess = SessionOrchestrator(host_id="h", output_dir=tmp_path) + sess.add(FakeStream("good")) + sess.add(FakeStream("bad", fail_on_start=True)) + + poller = SessionPoller(sess) + sess.connect() + + snap = poller.snapshot() + assert snap.streams["good"].connection_state == "connected" + assert snap.streams["good"].connection_error is None + assert snap.streams["bad"].connection_state == "failed" + assert snap.streams["bad"].connection_error # non-empty +``` + +- [ ] **Step 2: Run, confirm fail** + +```bash +uv run pytest tests/unit/viewer/test_poller.py::test_poller_snapshot_includes_connection_state -v +``` +Expected: `AssertionError` on `connection_state == "connected"` (current default is `"idle"` because the poller doesn't set it). + +- [ ] **Step 3: Implement** + +In `src/syncfield/viewer/poller.py`, locate the method that builds per-stream `StreamSnapshot` objects (look for `StreamSnapshot(` inside `_build_snapshot` or equivalent). Add the two new fields to the constructor call, pulling from the orchestrator: + +```python + connection_state=self._session._stream_states.get(stream.id, "idle"), + connection_error=self._session._stream_errors.get(stream.id), +``` + +If the poller holds the session under a different attribute name (e.g. `self._orchestrator`), use that — read the surrounding code. + +- [ ] **Step 4: Run, confirm pass** + +```bash +uv run pytest tests/unit/viewer/test_poller.py -v +``` +Expected: all pass. + +- [ ] **Step 5: Commit** + +```bash +git add src/syncfield/viewer/poller.py tests/unit/viewer/test_poller.py +git commit -m "$(cat <<'EOF' +feat(viewer): poller reads orchestrator stream state into snapshot + +Every per-stream StreamSnapshot now carries connection_state and +connection_error, sourced from the orchestrator's per-stream dicts +populated by _set_stream_state. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 11 — Server: serialize new stream fields onto the wire + +**Files:** +- Modify: `src/syncfield/viewer/server.py` +- Test: `tests/unit/test_viewer_aggregation_snapshot.py` (extend) or `tests/unit/viewer/test_server_snapshot_serialization.py` (create if missing) + +- [ ] **Step 1: Locate the serializer** + +```bash +grep -n "snapshot_to_dict\|def _to_wire\|snapshot\.streams\|_serialize_stream" src/syncfield/viewer/server.py | head -10 +``` + +Record the function name + file location. Usually a helper that maps each `StreamSnapshot` to a JSON-friendly dict. + +- [ ] **Step 2: Write failing test** + +Append to `tests/unit/test_viewer_aggregation_snapshot.py` (it already has a `_make_snapshot_mock()` helper from the Task 20 refactor): + +```python +def test_serialized_stream_includes_connection_state(): + from syncfield.viewer.server import snapshot_to_dict # or whatever name + from syncfield.viewer.state import StreamSnapshot, SessionSnapshot + + stream_snap = StreamSnapshot( + id="cam", kind="video", provides_audio_track=False, produces_file=False, + frame_count=0, last_sample_at_ns=None, effective_hz=0.0, + latest_frame=None, plot_points={}, latest_pose={}, + connection_state="failed", connection_error="Device not visible", + ) + sess_snap = SessionSnapshot( + host_id="h", state="idle", output_dir="/tmp", + sync_point_monotonic_ns=None, sync_point_wall_clock_ns=None, + chirp_start_ns=None, chirp_stop_ns=None, chirp_enabled=False, + elapsed_s=0.0, streams={"cam": stream_snap}, + active_incidents=[], resolved_incidents=[], + ) + out = snapshot_to_dict(sess_snap) + assert out["streams"]["cam"]["connection_state"] == "failed" + assert out["streams"]["cam"]["connection_error"] == "Device not visible" +``` + +(Adapt `snapshot_to_dict` to the actual function name discovered in Step 1.) + +- [ ] **Step 3: Run, confirm fail** + +```bash +uv run pytest tests/unit/test_viewer_aggregation_snapshot.py -k connection_state -v +``` +Expected: `KeyError` or missing field. + +- [ ] **Step 4: Implement** + +In `src/syncfield/viewer/server.py`, in the per-stream serializer helper, add the two keys next to the existing `frame_count` / `effective_hz` keys: + +```python + "connection_state": stream.connection_state, + "connection_error": stream.connection_error, +``` + +- [ ] **Step 5: Run, confirm pass** + +```bash +uv run pytest tests/unit -q --timeout 30 +``` +Expected: all pass. + +- [ ] **Step 6: Commit** + +```bash +git add src/syncfield/viewer/server.py tests/unit/test_viewer_aggregation_snapshot.py +git commit -m "$(cat <<'EOF' +feat(viewer): emit connection_state / connection_error over WebSocket + +Frontend consumes these to select the right StreamCard overlay and to +populate the degraded-state header chip. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 12 — Frontend TypeScript types + +**Files:** +- Modify: `src/syncfield/viewer/frontend/src/lib/types.ts` + +- [ ] **Step 1: Add the ConnectionState type + fields** + +Add near the existing `Severity` type declaration: + +```ts +export type ConnectionState = + | "idle" + | "connecting" + | "connected" + | "failed" + | "disconnected"; +``` + +In the `StreamSnapshot` interface (the one mirroring the Python dataclass), add at the end: + +```ts + connection_state: ConnectionState; + connection_error: string | null; +``` + +- [ ] **Step 2: Typecheck** + +```bash +cd src/syncfield/viewer/frontend && npx tsc --noEmit +``` +Expected: clean typecheck (no errors unless downstream code is already missing a field — we fix those in Tasks 13-15). + +- [ ] **Step 3: Commit** + +```bash +git add src/syncfield/viewer/frontend/src/lib/types.ts +git commit -m "$(cat <<'EOF' +feat(viewer-fe): mirror ConnectionState + StreamSnapshot additions + +ConnectionState enum and two new fields on StreamSnapshot so the +overlay branching in StreamCard (next commit) is type-safe. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 13 — Frontend overlay components + +**Files:** +- Create: `src/syncfield/viewer/frontend/src/components/stream-overlays.tsx` + +- [ ] **Step 1: Create the component file** + +```tsx +import { useState } from "react"; + +export function ConnectingOverlay() { + return ( +
+
+ + Connecting… +
+
+ ); +} + +export function WaitingForDataOverlay() { + return ( +
+
+ Connected · waiting for first frame +
+
+ ); +} + +export function FailedOverlay({ error }: { error: string }) { + const [expanded, setExpanded] = useState(false); + return ( + + ); +} +``` + +- [ ] **Step 2: Typecheck + build** + +```bash +cd src/syncfield/viewer/frontend && npx tsc --noEmit && npm run build +``` +Expected: clean. + +- [ ] **Step 3: Commit** + +```bash +git add src/syncfield/viewer/frontend/src/components/stream-overlays.tsx \ + src/syncfield/viewer/frontend/src/lib/types.ts \ + src/syncfield/viewer/static/ # if the build emits updated bundle (skip if gitignored) +# Drop the static/ line above if the working tree shows no changes there. +git commit -m "$(cat <<'EOF' +feat(viewer-fe): add Connecting / WaitingForData / Failed overlays + +Three small presentational components used by StreamCard (next commit) +to replace the browser broken-image fallback. FailedOverlay is click- +to-expand for long error messages. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +Note: `src/syncfield/viewer/static/` is gitignored; the build output stays local. Do not force-add it. + +--- + +## Task 14 — StreamCard: branch body on `connection_state` + +**Files:** +- Modify: `src/syncfield/viewer/frontend/src/components/stream-card.tsx` + +- [ ] **Step 1: Import overlays** + +Add near the top of the file: + +```tsx +import { + ConnectingOverlay, + WaitingForDataOverlay, + FailedOverlay, +} from "./stream-overlays"; +``` + +- [ ] **Step 2: Insert body branching** + +Find the part of `StreamCard` that currently renders the video `` / `VideoPreview`. Wrap it so the branch decides what to render: + +```tsx +function StreamCardBody({ stream }: { stream: StreamSnapshot }) { + if (stream.connection_state === "connecting") { + return ; + } + if (stream.connection_state === "failed") { + return ; + } + if (stream.connection_state === "connected" && stream.frame_count === 0 && stream.kind === "video") { + return ; + } + // Existing render path (VideoPreview / SensorChart / AudioLevel / etc.) + return ; // adapt to real name +} +``` + +Replace the existing `` / `` usage inside `StreamCard` with ``. If `StreamCard` inlines the video element rather than delegating, extract the pre-existing body into a small helper named `ExistingStreamBody` first so the branch above works cleanly. + +- [ ] **Step 3: Typecheck + visual smoke test** + +```bash +cd src/syncfield/viewer/frontend && npx tsc --noEmit && npm run build +``` + +Manual smoke (optional if you have the dev server running): + +```bash +# terminal 1 +cd src/syncfield/viewer/frontend && npm run dev +# terminal 2 +python examples/mac_iphone_dual_oak/record.py +``` +Press Connect. Expected visual: +- `mac_webcam` / `iphone` / `host_audio` → normal body (video or audio chart) +- `oak_lite` / `oak_d` → red "Failed to connect" overlay with the depthai error message + +- [ ] **Step 4: Commit** + +```bash +git add src/syncfield/viewer/frontend/src/components/stream-card.tsx +git commit -m "$(cat <<'EOF' +feat(viewer-fe): branch StreamCard body on connection_state + +Connecting / waiting-for-first-frame / failed now render explicit +overlays instead of the browser's broken-image placeholder. Healthy +streams still render the existing video / sensor / audio body. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 15 — Header: degraded-state chip + +**Files:** +- Modify: `src/syncfield/viewer/frontend/src/components/header.tsx` + +- [ ] **Step 1: Locate the chip** + +```bash +grep -n "Ready\|SessionState\|state.*chip\|state.*label" src/syncfield/viewer/frontend/src/components/header.tsx | head +``` + +Find where the state label (`Ready` / `Recording` / …) is composed. + +- [ ] **Step 2: Add the counter logic** + +At the top of the component, after the snapshot prop is destructured: + +```tsx +const streams = Object.values(snapshot.streams); +const total = streams.length; +const connected = streams.filter((s) => s.connection_state === "connected").length; +const showCount = total > 0 && connected < total; +const label = showCount ? `${stateLabel} (${connected}/${total})` : stateLabel; +const chipTone = showCount ? "warning" : "normal"; +``` + +Use `chipTone` to choose the className. If the header currently has a Tailwind class like `bg-emerald-500/10`, add: + +```tsx +const toneClass = chipTone === "warning" + ? "bg-yellow-500/15 text-yellow-300 border border-yellow-500/40" + : "bg-emerald-500/10 text-emerald-300 border border-emerald-500/30"; +``` + +Apply `toneClass` to the chip's container element, and render `{label}` instead of `stateLabel` inside it. + +- [ ] **Step 3: Typecheck + build** + +```bash +cd src/syncfield/viewer/frontend && npx tsc --noEmit && npm run build +``` + +- [ ] **Step 4: Commit** + +```bash +git add src/syncfield/viewer/frontend/src/components/header.tsx +git commit -m "$(cat <<'EOF' +feat(viewer-fe): show degraded counter on header state chip + +'Ready (3/5)' in yellow when one or more streams are not in 'connected' +state. Normal emerald chip when everything is healthy. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 16 — Integration: partial-connect end-to-end + +**Files:** +- Create: `tests/integration/health/test_partial_connect.py` + +- [ ] **Step 1: Write the test** + +```python +"""End-to-end: real SessionOrchestrator + FakeStream mix survives one +stream failing to connect, and incidents.jsonl captures it.""" +from __future__ import annotations + +import json +import time +from pathlib import Path + +import pytest + +from syncfield.orchestrator import SessionOrchestrator +from syncfield.testing import FakeStream +from syncfield.types import SessionState + + +@pytest.mark.slow +def test_partial_connect_end_to_end(tmp_path: Path): + sess = SessionOrchestrator(host_id="h", output_dir=tmp_path) + sess.add(FakeStream("good_a")) + sess.add(FakeStream("bad", fail_on_start=True)) + sess.add(FakeStream("good_b")) + + sess.connect() + assert sess.state_name == SessionState.CONNECTED.value + + # Give the health worker a moment to ingest the startup-failure event. + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline: + if any(i.fingerprint == "bad:startup-failure" for i in sess.health.open_incidents()): + break + time.sleep(0.05) + + open_fps = [i.fingerprint for i in sess.health.open_incidents()] + assert "bad:startup-failure" in open_fps + + sess.start(countdown_s=0) + time.sleep(0.5) + sess.stop() + sess.disconnect() + + # incidents.jsonl should contain the startup-failure fingerprint. + out = list(tmp_path.rglob("incidents.jsonl")) + assert out, "no incidents.jsonl written" + lines = [json.loads(l) for l in out[0].read_text().strip().splitlines() if l] + fingerprints = {l["fingerprint"] for l in lines} + assert "bad:startup-failure" in fingerprints +``` + +- [ ] **Step 2: Run** + +```bash +uv run pytest tests/integration/health/test_partial_connect.py -v +``` +Expected: 1 passed. + +- [ ] **Step 3: Commit** + +```bash +git add tests/integration/health/test_partial_connect.py +git commit -m "$(cat <<'EOF' +test(health): integration test for partial connect + incident persistence + +Real SessionOrchestrator, mix of passing + failing FakeStreams; asserts +the session reaches CONNECTED, the failed stream surfaces as an open +startup-failure incident, and the fingerprint lands in incidents.jsonl. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 17 — Integration: NoDataDetector on a real orchestrator + +**Files:** +- Create: `tests/integration/health/test_no_data_detector.py` + +- [ ] **Step 1: Write the test** + +```python +"""End-to-end: a stream that connects but never emits a sample triggers +the no-data incident within the configured threshold.""" +from __future__ import annotations + +import threading +import time +from pathlib import Path + +import pytest + +from syncfield.orchestrator import SessionOrchestrator +from syncfield.stream import StreamBase +from syncfield.types import FinalizationReport, SampleEvent, StreamCapabilities + + +class SilentFakeStream(StreamBase): + """FakeStream variant that connects successfully but emits no samples until asked.""" + + def __init__(self, stream_id: str): + super().__init__(stream_id=stream_id, kind="sensor", capabilities=StreamCapabilities()) + self._stop = threading.Event() + self._gate = threading.Event() # held closed until tests allow flow + self._thread: threading.Thread | None = None + self._frame = 0 + + def connect(self): + self._stop.clear() + self._thread = threading.Thread(target=self._run, daemon=True) + self._thread.start() + + def disconnect(self): + self._stop.set() + if self._thread: + self._thread.join(timeout=1.0) + self._thread = None + + def start_recording(self, session_clock): + pass + + def stop_recording(self) -> FinalizationReport: + return FinalizationReport( + stream_id=self.id, status="completed", frame_count=self._frame, + file_path=None, first_sample_at_ns=0, last_sample_at_ns=0, + health_events=[], error=None, + ) + + def allow_samples(self): + self._gate.set() + + def _run(self): + while not self._stop.is_set(): + if self._gate.is_set(): + self._frame += 1 + self._emit_sample(SampleEvent( + stream_id=self.id, frame_number=self._frame, + capture_ns=time.monotonic_ns(), + )) + time.sleep(0.05) + + +@pytest.mark.slow +def test_no_data_incident_opens_then_closes_when_samples_arrive(tmp_path: Path): + sess = SessionOrchestrator(host_id="h", output_dir=tmp_path) + stream = SilentFakeStream("cam") + sess.add(stream) + + # Shrink threshold for the test via direct detector access. + for d in sess.health.iter_detectors(): + if d.name == "no-data": + d._threshold_ns = int(1e9) # 1s + break + + sess.connect() + + # After 1.5s, no-data incident should be open. + time.sleep(1.5) + open_fps = [i.fingerprint for i in sess.health.open_incidents()] + assert "cam:no-data" in open_fps + + # Let samples flow → incident closes within ~1 tick. + stream.allow_samples() + time.sleep(0.5) + + sess.stop() + sess.disconnect() + + resolved_fps = [i.fingerprint for i in sess.health.resolved_incidents()] + assert "cam:no-data" in resolved_fps +``` + +- [ ] **Step 2: Run** + +```bash +uv run pytest tests/integration/health/test_no_data_detector.py -v +``` +Expected: 1 passed (~2s wall clock). + +- [ ] **Step 3: Commit** + +```bash +git add tests/integration/health/test_no_data_detector.py +git commit -m "$(cat <<'EOF' +test(health): integration test for NoDataDetector on real orchestrator + +SilentFakeStream connects but withholds samples until the test opens +its gate. Asserts the no-data incident opens within 1.5s of connect +and closes within one tick of samples resuming. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Task 18 — Final regression sweep + push + +- [ ] **Step 1: Run the full suite** + +```bash +uv run pytest tests/unit tests/integration/health -q --timeout 60 +``` +Expected: ≥870 passing (baseline before these tasks was 867; this plan adds ~6 new tests). + +- [ ] **Step 2: Typecheck + build frontend** + +```bash +cd src/syncfield/viewer/frontend && npx tsc --noEmit && npm run build +``` +Expected: clean. + +- [ ] **Step 3: Push** + +```bash +git push +``` + +- [ ] **Step 4: Verify CI re-runs against the updated branch** + +```bash +gh pr checks 20 +``` + +Expected: CI kicks off; same pre-existing failures as before (3 Insta360 + audio + meta_quest tests) — this plan does not affect them. + +--- + +## Self-Review Checklist + +**Spec coverage (every spec requirement traces to a task):** + +- Partial connect semantics (§ Goals 1) → Task 7 +- Structured startup-failure event emission (§ Goals 2) → Task 7 (error path + success path) +- Per-stream ConnectionState + error in orchestrator (§ Data model) → Tasks 6, 8 +- `StreamSnapshot.connection_state` / `connection_error` (§ Data model) → Task 9 +- NoDataDetector (§ New detector) → Tasks 4, 5 +- `Detector.observe_connection_state` hook (§ Architecture) → Task 1 +- HealthWorker ingress queue (§ Architecture) → Task 2 +- HealthSystem passthrough (§ Architecture) → Task 3 +- Poller reads orchestrator state (§ Viewer changes) → Task 10 +- Server WS serialization (§ Viewer changes) → Task 11 +- Frontend ConnectionState + fields (§ Viewer changes) → Task 12 +- Overlay components (§ Viewer changes) → Task 13 +- StreamCard branch selection (§ Viewer changes) → Task 14 +- Header degraded chip (§ Viewer changes) → Task 15 +- Unit tests (§ Testing strategy) → Tasks 4, 7, 8, 9, 10, 11 +- Integration tests (§ Testing strategy) → Tasks 16, 17 + +**Type consistency:** `ConnectionState` strings `"idle" | "connecting" | "connected" | "failed" | "disconnected"` — same in Python, same in TypeScript, same in every test assertion. Fingerprints `{stream_id}:startup-failure` and `{stream_id}:startup-success` consistent across orchestrator + detector + tests + spec. + +**No placeholders:** every step shows a concrete code block or command. diff --git a/docs/superpowers/specs/2026-04-22-health-telemetry-design.md b/docs/superpowers/specs/2026-04-22-health-telemetry-design.md new file mode 100644 index 0000000..d1c7a36 --- /dev/null +++ b/docs/superpowers/specs/2026-04-22-health-telemetry-design.md @@ -0,0 +1,369 @@ +# Health Telemetry Platform — Design Spec + +- **Date**: 2026-04-22 +- **Status**: Approved for implementation planning +- **Owner**: syncfield-python +- **Related**: `src/syncfield/orchestrator.py` (session lifecycle), `src/syncfield/stream.py` (Stream protocol, existing `HealthEvent`), `src/syncfield/adapters/oak_camera.py` (motivating failure modes), `src/syncfield/viewer/` (live + post-session surface) + +## Summary + +Add a sensor-agnostic, platform-level **health telemetry system** to syncfield. During a recording the system continuously observes every active stream (sample cadence, silence gaps, adapter-reported faults, writer backpressure) and raises **Incidents** — Sentry-style grouped objects with severity, open/close lifecycle, and attached artifacts — that surface in the viewer in real time and are persisted for post-session review. New detectors can be added without touching any adapter; new adapters inherit the full baseline check suite for free. + +The first motivating hardware is OAK (Luxonis DepthAI), where native log output (`X_LINK_ERROR`, `Device has crashed`, crash-dump paths, `Reconnection successful`) today vanishes into stderr. The design bridges that native logger into the unified telemetry channel, and the same detector set catches identical symptoms (stall, FPS drop) on any future adapter. + +## Goals + +1. **Live detection, live display**: while recording, surface hardware crashes, stream stalls, FPS drops, jitter spikes, startup failures, writer backpressure, and adapter-reported faults within ~1s of occurrence. Render them as a first-class **Active Issues** panel in the viewer. +2. **Post-session incident report**: persist a structured `incidents.jsonl` per session and expose `FinalizationReport.incidents`, so users can answer "what went wrong, when, on which stream, with what evidence?" after the fact. +3. **Sensor-agnostic baseline**: every stream — current and future — automatically gets stall detection, FPS-drop detection, jitter detection, startup-failure detection, and adapter-event pass-through, with zero adapter code changes. +4. **Pluggable detectors**: adding a new detection rule (temperature, bandwidth, battery, anomaly) is a single-file, single-class addition registered at startup. No adapter modifications. +5. **Default-on**: users never need to wire up health telemetry; a freshly constructed `SessionOrchestrator` has the full baseline running. +6. **Artifact capture**: when a device provides crash evidence (e.g., OAK's `crash_dump.json`), attach the path to the incident so the user can ship it to the vendor. +7. **Zero impact on capture hot path**: detectors observe via lock-free hand-off; all detection work runs on a dedicated worker thread. + +## Non-goals + +- **Multihost fan-in (leader-side fleet view)**: follower incidents are written to the follower's session log and collected post-stop via the existing file-pull path. Real-time follower→leader streaming of health events is deferred. +- **Cross-session history / search**: no database, no aggregation across sessions. Each session is self-contained. +- **Automatic remediation**: the platform detects and reports. It does not auto-reconnect devices, swap to backup streams, or halt recording on failure. +- **Frame-content anomaly detection**: no checks for black frames, codec corruption, audio silence content, or ML-based quality scoring. +- **Host resource monitoring**: CPU/GPU/memory/thermal of the host machine are out of scope. +- **Cross-session markdown summary**: `incidents.jsonl` is structured; a human-readable post-mortem generator is deferred. +- **Backward compatibility of existing health surfaces**: `StreamSnapshot.health_count` / `problem_count` are removed in favor of the new incident-based fields. Existing `session_log.jsonl` consumers must migrate. There are no external consumers today. + +## Current state (what exists) + +- `HealthEvent` dataclass with `HealthEventKind = {HEARTBEAT, DROP, RECONNECT, WARNING, ERROR}` in `src/syncfield/types.py:250-273`. +- `StreamBase._emit_health(event)` for adapters to push into the session log — `src/syncfield/stream.py:237-241`. +- `SessionLogWriter.log_health(event)` persists to `session_log.jsonl`. +- `SessionOrchestrator._on_stream_health()` routes events to log + buffered collection — `src/syncfield/orchestrator.py:2354-2363`. +- Viewer `HealthTable` component renders last 20 raw events as a timeline. +- `FinalizationReport.health_events[]` includes raw events post-stop. + +**What is missing, and what this spec adds**: no automatic detection (all events are adapter-emitted); no severity; no grouping; no open/close semantics; no detector plugin model; no artifact attachment; no Sentry-style UI surface; no per-adapter target-hz hint for comparing observed vs. expected cadence. + +## Architecture Overview + +``` +┌──────────────────────────────────────────────────────────────────────────┐ +│ SessionOrchestrator │ +│ │ +│ ┌──────────┐ on_sample ┌──────────────────────────┐ │ +│ │ Streams │──────────────►│ │ │ +│ └──────────┘ on_health │ │ │ +│ │ │ HealthSystem │ │ +│ │ state changes ───►│ │ │ +│ │ │ ┌────────────────────┐ │ │ +│ ▼ │ │ DetectorRegistry │ │ │ +│ ┌──────────┐ queue stats │ │ (default + user) │ │ │ +│ │ Writer │──────────────►│ └────────────────────┘ │ │ +│ └──────────┘ │ │ │ │ +│ │ ▼ │ │ +│ │ ┌────────────────────┐ │ │ +│ │ │ HealthWorker │ │ ◄── 20 Hz │ +│ │ │ (thread, ticks) │ │ tick │ +│ │ └────────────────────┘ │ │ +│ │ │ │ │ +│ │ ▼ │ │ +│ │ ┌────────────────────┐ │ │ +│ │ │ IncidentTracker │ │ │ +│ │ │ open / close / │ │ │ +│ │ │ fingerprint group │ │ │ +│ │ └────────────────────┘ │ │ +│ │ │ │ │ +│ └───────────┼──────────────┘ │ +│ │ │ +│ ┌────────────────────────┼────────────────────────┐ │ +│ │ │ │ │ +│ ▼ ▼ ▼ │ +│ session_log.jsonl incidents.jsonl SessionSnapshot +│ (raw HealthEvents) (Incident records) → WebSocket │ +│ │ │ +└─────────────────────────────────────────┼──────────────────────────────┘ + ▼ + FinalizationReport.incidents + (exposed from SDK) +``` + +**Threading model**: streams continue to call `on_sample` / `_emit_health` from capture threads. These fan into lock-free deques drained by a single `HealthWorker` daemon thread (20 Hz). All detector ticks and incident-tracker state live on that thread — there are no locks on the capture hot path beyond the deque push. + +## Data model + +### Severity + +```python +class Severity(str, Enum): + INFO = "info" # heartbeat, reconnect-success, config warning + WARNING = "warning" # mild FPS dip, jitter, macOS deprecation warning + ERROR = "error" # disconnect, X_LINK_ERROR, encoding fail, silence > 2s + CRITICAL = "critical" # device crash, multi-stream simultaneous down +``` + +### HealthEvent (enriched, existing class; new fields) + +```python +@dataclass(frozen=True) +class HealthEvent: + stream_id: str + kind: HealthEventKind # existing + at_ns: int # existing, time.monotonic_ns() + detail: str | None # existing + + # new fields (no backward compat shim — session_log.jsonl format changes): + severity: Severity + source: str # "adapter:oak" | "detector:stream-stall" | ... + fingerprint: str # stable grouping key, e.g. "oak-main:stream-stall" + data: dict[str, Any] = {} # structured context: {"observed_hz": 12, "target_hz": 30} +``` + +The `fingerprint` is what groups many raw events into one Incident. Detectors own the fingerprint formula (typically `f"{stream_id}:{detector.name}"`, optionally refined for sub-categorization). Adapter-emitted events use `f"{stream_id}:adapter:{kind.value}"` via the pass-through detector. + +### Incident (new) + +```python +@dataclass +class Incident: + id: str # ulid, stable for the lifetime of the session + stream_id: str + fingerprint: str # same as the events it groups + title: str # human-readable, e.g., "Stream stalled (silence 9s)" + severity: Severity # = max severity across grouped events + source: str # name of the first detector/adapter that fired + + opened_at_ns: int + closed_at_ns: int | None # None = still open / active + last_event_at_ns: int # most recent event in the group + + event_count: int + first_event: HealthEvent # snapshot for display + last_event: HealthEvent # snapshot for display + + artifacts: list[IncidentArtifact] = [] # attached evidence + data: dict[str, Any] = {} # aggregated context (e.g., min_observed_hz) + +@dataclass(frozen=True) +class IncidentArtifact: + kind: str # "crash_dump" | "log_excerpt" | ... + path: str # absolute path, or URI + detail: str | None = None +``` + +An incident is **open** while its condition persists. The tracker closes it when the owning detector's `close_condition` returns True (for detector-owned incidents) or when a quiet period elapses without a new matching event (for pass-through adapter incidents; default 30s). + +## Detector system + +### Detector protocol + +```python +class Detector(Protocol): + name: str # "stream-stall", "fps-drop", ... + default_severity: Severity # severity of events this detector raises + + def observe_sample(self, stream_id: str, sample: SampleEvent) -> None: ... + def observe_health(self, stream_id: str, event: HealthEvent) -> None: ... + def observe_state(self, old: SessionState, new: SessionState) -> None: ... + def observe_writer_stats(self, stream_id: str, stats: WriterStats) -> None: ... + + def tick(self, now_ns: int) -> Iterable[HealthEvent]: ... + def close_condition(self, incident: Incident, now_ns: int) -> bool: ... +``` + +- `observe_*` feed the detector its raw signals; they must be fast and non-blocking. +- `tick` is called by the `HealthWorker` at 20 Hz; this is where silence/gap-based detections fire. +- `close_condition` decides when an open incident with this detector's fingerprint can be closed. + +A `DetectorBase` ABC provides no-op defaults for every hook so subclasses only implement what they need. + +### Default detector suite (registered automatically on `SessionOrchestrator` construction) + +| Detector | Signal | Fires when | Closes when | +|---|---|---|---| +| `StreamStallDetector` | `observe_sample` (timestamps) + `tick` | no sample for 2.0s on a stream that previously had samples | sample resumes and keeps flowing 1.0s | +| `FpsDropDetector` | `observe_sample` + `tick` | effective FPS < 70% of `target_hz` for 3.0s (or, without `target_hz`, < 70% of learned 10s-baseline after a 5s warmup) | FPS ≥ 90% for 5.0s | +| `JitterDetector` | `observe_sample` + `tick` | p95 inter-sample gap over last 60 samples > 2× expected | p95 ≤ 1.2× expected for 10.0s | +| `StartupFailureDetector` | `observe_health` (for adapter-raised startup errors) + `observe_state` | `connect()` or `start_recording()` raises / times out | subsequent retry reports success | +| `BackpressureDetector` | `observe_writer_stats` | writer queue depth > 80% of capacity for 2.0s, OR writer drop counter increments | queue depth < 30% for 5.0s | +| `AdapterEventPassthrough` | `observe_health` | any adapter-emitted `HealthEvent` not already owned by another detector | 30s quiet on the fingerprint | +| `DepthAILoggerBridge` | native depthai Python logger handler | depthai error/warning log arrives (X_LINK_ERROR, "Device has crashed", reconnect events) → converted to `HealthEvent` → fed into `AdapterEventPassthrough` | (stateless — it is a translator, not a detector) | + +`DepthAILoggerBridge` is not technically a detector in the "fire from tick" sense — it is a logging-handler adapter. For symmetry it lives next to detectors and registers via the same registry, but it only produces `HealthEvent`s; `AdapterEventPassthrough` groups them into incidents. The bridge is installed **automatically** when the `oak` optional extra is present (detected by importability of `depthai`), matching the project's default-on philosophy. + +### DetectorRegistry + +```python +class DetectorRegistry: + def register(self, detector: Detector) -> None: ... + def unregister(self, name: str) -> None: ... + def __iter__(self) -> Iterator[Detector]: ... +``` + +Users register additional detectors before `session.connect()`: + +```python +session = SessionOrchestrator(...) +session.health.register(MyTemperatureDetector(...)) +``` + +Default detectors are installed by `HealthSystem` in `__init__`; users can `unregister` them by name to opt out. + +### HealthWorker & IncidentTracker + +- `HealthWorker` is a daemon thread that owns the tick loop and all mutable state. It exposes thread-safe ingress queues (`Queue` or lock-free deque) for samples, health events, state transitions, and writer stats. +- `IncidentTracker` (running on the worker thread) consumes every `HealthEvent` produced by detectors or observed via pass-through, groups by fingerprint, opens new incidents, updates existing ones, and runs `close_condition` each tick for every open incident. +- Closed incidents stay in the tracker's memory (for the viewer's "Resolved this session" list) and are flushed to `incidents.jsonl` on each state change and on `stop()`. + +## Integration points + +### `src/syncfield/types.py` +- Add `Severity` enum. +- Enrich `HealthEvent` with `severity`, `source`, `fingerprint`, `data`. +- Add `Incident`, `IncidentArtifact`, `IncidentSnapshot` (read-only view used in WebSocket payloads). +- Add `WriterStats` dataclass (queue depth, drop count, bytes flushed). +- `FinalizationReport` gains `incidents: list[Incident]`. `health_events` field remains. + +### `src/syncfield/stream.py` +- `StreamCapabilities` gains `target_hz: float | None = None`. +- No API change to `StreamBase._emit_health`; it continues to push `HealthEvent` and now the system assigns `severity`/`fingerprint`/`source` server-side if the adapter did not. + +### `src/syncfield/health/` (new package) +``` +src/syncfield/health/ +├── __init__.py # public: HealthSystem, Severity, Incident, Detector, DetectorBase +├── types.py # Incident, IncidentArtifact, IncidentSnapshot, WriterStats +├── severity.py # Severity enum + helpers +├── system.py # HealthSystem (user-facing facade, owns worker + registry + tracker) +├── worker.py # HealthWorker (thread, tick loop, ingress queues) +├── registry.py # DetectorRegistry +├── tracker.py # IncidentTracker +├── detector.py # Detector protocol + DetectorBase +└── detectors/ + ├── __init__.py + ├── stream_stall.py + ├── fps_drop.py + ├── jitter.py + ├── startup_failure.py + ├── backpressure.py + ├── adapter_passthrough.py + └── depthai_bridge.py # soft-imports depthai; registered if importable +``` + +### `src/syncfield/orchestrator.py` +- `SessionOrchestrator.__init__` constructs `self.health = HealthSystem(session_id=..., clock=...)`. +- On adding a stream: subscribe `health.observe_sample` and `health.observe_health` to that stream. +- On every `SessionState` transition: `health.observe_state(old, new)`. +- On each writer flush: pump `WriterStats` into `health.observe_writer_stats`. +- `start()` boots the worker thread; `stop()` drains, closes still-open incidents with `closed_at_ns = now`, and embeds `incidents` into `FinalizationReport`. +- Existing `_on_stream_health` is simplified — it now just forwards to `health`. `session_log.jsonl` writing is owned by `SessionLogWriter`, which also gains `log_incident(incident)`. + +### `src/syncfield/writer.py` +- `SessionLogWriter` gains `log_incident(incident)` writing to `incidents.jsonl`. +- On every incident open/update/close, `IncidentTracker` calls `writer.log_incident(...)`. +- The writer also emits `WriterStats` snapshots at a low frequency (e.g., every 250ms) into `HealthSystem` for backpressure detection. + +### `src/syncfield/adapters/oak_camera.py` +- Declare `target_hz` in `StreamCapabilities`. +- At `connect()`: attach `DepthAILoggerBridge` to the depthai Python logger (scoped to this stream instance). +- At crash detection (depthai signals device crash or the bridge observes the "Crash dump logs are stored in" line): emit a `HealthEvent` with `kind=ERROR`, `severity=CRITICAL`, `data={"crash_dump_path": "..."}`. The tracker attaches it as an `IncidentArtifact(kind="crash_dump", path=...)`. +- No other OAK-specific detection code. `StreamStallDetector` and `FpsDropDetector` handle the "frozen for 9s during depthai reconnect" case generically. + +### `src/syncfield/viewer/state.py` & `server.py` +- `SessionSnapshot` gains: + - `active_incidents: list[IncidentSnapshot]` + - `resolved_incidents: list[IncidentSnapshot]` (capped at 20 most recent) +- `StreamSnapshot.health_count` and `StreamSnapshot.problem_count` **removed**; per-stream incident counts derived client-side by filtering `active_incidents` by `stream_id`. +- Poller subscribes to `HealthSystem` for incident opened/updated/closed notifications (via a simple callback protocol) and merges them into the snapshot. + +### `src/syncfield/viewer/frontend/src/` +- Delete `components/health-table.tsx`. +- Add `components/incident-panel.tsx` (Option α): + - Two collapsible sections: **Active Issues (N)**, **Resolved this session (N)**. + - Each incident card: severity icon · stream id · title · opened/recovered relative time · event count · artifact chips. + - Click to expand → raw `HealthEvent` list for that fingerprint pulled from `health_log` (retained in snapshot, capped at 200 most recent overall). +- Stream cards: replace red dot count with severity-colored badge (critical=red, error=orange, warning=yellow) whose number reflects the count of open incidents on that stream. +- Shared type file `lib/types.ts` mirrors `IncidentSnapshot` / `Severity`. + +## OAK — concrete event → incident mapping + +Reference sample from the user's session log: + +``` +[depthai] [error] Communication exception ... 'Couldn't read data from stream: '__x_0_1' (X_LINK_ERROR)' +[host] [warning] Closed connection +[host] [warning] Attempting to reconnect. Timeout is 10000ms +[depthai] [error] Device with id ... has crashed. Crash dump logs are stored in: /path/to/crash_dump.json +[host] [warning] Reconnection successful +``` + +Pipeline: + +1. `DepthAILoggerBridge` converts each native line into a `HealthEvent`: + - `X_LINK_ERROR` → `kind=ERROR`, `severity=ERROR`, `source="adapter:oak"`, `fingerprint="oak-main:adapter:xlink-error"`, `data={"stream": "__x_0_1"}`. + - "Closed connection" → `kind=WARNING`, `severity=WARNING`, `fingerprint="oak-main:adapter:connection-closed"`. + - "Attempting to reconnect" → `kind=RECONNECT`, `severity=INFO`, `fingerprint="oak-main:adapter:reconnect-attempt"`. + - "Device has crashed" → `kind=ERROR`, `severity=CRITICAL`, `fingerprint="oak-main:adapter:device-crash"`, `data={"crash_dump_path": "/path/to/crash_dump.json"}`. + - "Reconnection successful" → `kind=RECONNECT`, `severity=INFO`, `fingerprint="oak-main:adapter:reconnect-success"`. +2. `AdapterEventPassthrough` groups the four non-info fingerprints into incidents. +3. Concurrently, `StreamStallDetector` observes no sample for 2s and opens its own incident `fingerprint="oak-main:stream-stall"` with `severity=ERROR`. +4. When samples resume (bridge sees "Reconnection successful" and stream yields samples again), `StreamStallDetector.close_condition` returns True after 1s of steady flow → the stall incident closes. +5. The device-crash incident gains an `IncidentArtifact(kind="crash_dump", path=...)` attached from its event's `data`. +6. On `stop()`, all these incidents flush to `incidents.jsonl`. + +The macOS `NSCameraUseContinuityCameraDeviceType` warning is emitted by the UVC adapter similarly (UVC adapter adds a tiny stderr-watcher or converts via existing `_emit_health`) as `severity=INFO`, `fingerprint="uvc-cam:adapter:continuity-deprecation"`; surfaces as a single resolved informational incident. + +## Persistence & artifacts + +Per-session directory additions: + +``` +session// +├── session_log.jsonl # existing — raw HealthEvents (format updated, not backward-compat) +├── incidents.jsonl # new — one Incident per line; appended on open/update/close +├── manifest.json # existing — gains "incidents_count", "active_at_stop_count" +└── /... # existing per-stream files; crash dumps referenced via absolute path +``` + +- `incidents.jsonl` is append-only. Each line is the current full state of the incident at write time; readers compact by `incident.id`. +- `FinalizationReport.incidents` is the in-memory compacted list (unique by `id`, latest state). + +## Testing strategy + +Every new module below `health/` ships with unit tests. The following are the load-bearing test suites: + +**`tests/health/test_detectors.py`** — one class per detector. Uses a `FakeClock` and a synthetic sample stream generator. Each test asserts: (a) the detector fires on the trigger, (b) does not fire on noise, (c) `close_condition` returns True after recovery, (d) fingerprint is stable across runs. + +**`tests/health/test_incident_tracker.py`** — grouping, opening, closing, reopening (same fingerprint after close), severity escalation (an open WARNING incident that receives an ERROR event upgrades), artifact attachment, JSONL flush on close. + +**`tests/health/test_health_system_integration.py`** — end-to-end with a `FakeStream` (a `StreamBase` subclass that emits synthetic samples on a driven clock): (a) stall then recover → one incident opens and closes; (b) sustained FPS drop → one incident; (c) adapter emits `HealthEvent(kind=ERROR)` → pass-through creates incident; (d) crash event carrying `data["crash_dump_path"]` produces artifact. + +**`tests/health/test_depthai_bridge.py`** — feeds synthetic depthai log records (as `logging.LogRecord`) and asserts the correct `HealthEvent`s come out. No real depthai device required. + +**`tests/orchestrator/test_orchestrator_health_integration.py`** — real `SessionOrchestrator` with `FakeStream` registered; drives the full lifecycle and asserts `FinalizationReport.incidents` content and `incidents.jsonl` content match. + +**`tests/viewer/test_snapshot_incidents.py`** — poller snapshot correctly includes `active_incidents` / `resolved_incidents`; stream filtering works. + +Frontend: at least a component test for `IncidentPanel` rendering Active + Resolved sections and a click-to-expand interaction. + +## Implementation phases (for the writing-plans step) + +1. **Core types & skeleton** — `Severity`, enriched `HealthEvent`, `Incident`, `IncidentArtifact`, `WriterStats`. `health/` package skeleton with `HealthSystem`, `HealthWorker`, `DetectorRegistry`, `IncidentTracker`, `DetectorBase`. Unit tests for tracker & worker. +2. **Platform detectors** — `StreamStallDetector`, `FpsDropDetector`, `JitterDetector`, `StartupFailureDetector`, `BackpressureDetector`, `AdapterEventPassthrough`. Full unit-test coverage. +3. **Orchestrator & writer integration** — wire `SessionOrchestrator` to `HealthSystem`, remove `StreamSnapshot.health_count`/`problem_count`, add writer stats emission, `incidents.jsonl` flushing, `FinalizationReport.incidents`. Integration tests with `FakeStream`. +4. **OAK bridge & `target_hz`** — `DepthAILoggerBridge`, crash-dump artifact attachment, declare `target_hz` on `OakCameraStream`. Bridge unit tests. +5. **Viewer — server** — `IncidentSnapshot` in `SessionSnapshot`; poller ingests from `HealthSystem`; remove obsolete fields. Snapshot tests. +6. **Viewer — frontend** — new `incident-panel.tsx`, deleted `health-table.tsx`, stream-card badge update, types mirrored. Component tests. +7. **Per-adapter target_hz rollout** — add `target_hz` to every adapter that has a known target (`UVCWebcamStream`, `HostAudioStream`, `MetaQuestCameraStream`, `Go3SStream` where relevant, etc.). No other changes. +8. **Manual verification checklist** — reproduce the OAK crash sequence on a real rig; confirm incidents open, close, and artifact attaches; screenshot `IncidentPanel`. + +## Open design questions (to resolve during planning) + +None blocking. Low-stakes items to be resolved during implementation: + +- Exact `target_hz`-learning window length when unset (currently proposed 5s warmup, 10s rolling baseline). +- Whether `DetectorRegistry.unregister` is public API for v1 or internal-only. +- Whether `IncidentSnapshot` includes the full `first_event` / `last_event`, or just their `detail` strings (wire size tradeoff). + +## Risks + +- **Noisy detectors**: a poorly tuned `JitterDetector` could fire constantly on cameras with naturally irregular cadence. Mitigation: per-adapter `jitter_tolerance` hint in `StreamCapabilities`, default conservative. +- **Depthai logger bridge fragility**: parses string patterns; a depthai version bump could change log format. Mitigation: pattern-match liberally and fall back to `severity=WARNING` with `source="adapter:oak:unparsed-log"` rather than dropping the line. Bridge has its own targeted tests. +- **Thread starvation**: if the worker tick blocks, detections are delayed. Mitigation: all detector `tick` / `observe_*` methods must be non-blocking; enforced by tests that time individual calls. diff --git a/docs/superpowers/specs/2026-04-22-partial-connect-ux-design.md b/docs/superpowers/specs/2026-04-22-partial-connect-ux-design.md new file mode 100644 index 0000000..dd8acbc --- /dev/null +++ b/docs/superpowers/specs/2026-04-22-partial-connect-ux-design.md @@ -0,0 +1,323 @@ +# Partial Connect UX — Design Spec + +- **Date**: 2026-04-22 +- **Status**: Approved for implementation planning +- **Owner**: syncfield-python +- **Related spec**: `docs/superpowers/specs/2026-04-22-health-telemetry-design.md` (this extends it) + +## Summary + +Remove the "all-or-nothing" failure mode of `SessionOrchestrator.connect()`. When one stream's `connect()` raises (common case: an OAK camera with a specific `device_id` isn't plugged in), continue connecting the rest, mark the failed stream, and surface the failure in the viewer as a first-class visual state. Also detect the complementary case where a stream connects successfully but never emits a single sample (the observed OAK "black square" symptom), so the user is not left guessing whether the camera is alive. Together these changes close the silent-failure hole that remains even with the Phase-1 health-telemetry framework in place. + +## Goals + +1. **Partial connect resilience**: if any subset of streams fails to connect, the rest still transition to `CONNECTED` and recording can proceed with the survivors. The session only aborts when *every* stream fails. +2. **Structured startup failure emission**: each `stream.connect()` exception produces a `HealthEvent` shaped for `StartupFailureDetector` (`data["phase"]="connect"`, `data["outcome"]="error"`). Successes emit the matching success signal so the detector closes its incident if a stream recovers on a later connect. +3. **Per-stream connection state** visible to the viewer (`idle | connecting | connected | failed | disconnected`) plus a `connection_error` message for the failed case. +4. **Viewer state-aware stream card**: distinct visual for connecting / waiting-for-first-frame / failed, replacing the current grey "broken image" placeholder. Header chip reflects degraded state as `Ready (3/5)`. +5. **No-data detection**: a new detector fires when a stream has been in `connected` state for ≥ 5 s without emitting any sample. Closes the moment the first sample arrives. + +## Non-goals + +- **Automatic retry / reconnect policy**. A failed stream stays failed until the user explicitly triggers Disconnect → Connect again. Auto-retry is follow-up work. +- **Retry button on the failed card**. Same reason. The existing Disconnect/Connect flow already gets the user there in two clicks; adding a per-card retry button requires partial-reconnect semantics we are not designing yet. +- **Error category taxonomy / normalized error codes**. We show the exception message verbatim in the `connection_error` field for v1. Mapping "device not visible" vs "USB permission denied" vs "XLink handshake timeout" into structured categories is future work. +- **Crash-dump viewer integration**. Crash-dump paths remain surfaced as `IncidentArtifact` entries (delivered in Phase-1 OAK work). A "click to open" handler in the viewer is not in scope here. +- **Recovery UI for mid-recording failures**. This spec targets the connect phase. Detectors already cover mid-recording stalls / device crashes via Phase-1 work; they do not change here. + +## Current state (what exists before this change) + +- `SessionOrchestrator.connect()` iterates streams sequentially and calls `_rollback_disconnect_streams(connected)` on the first exception, re-raising it to the caller (`src/syncfield/orchestrator.py:1623-1643`). One failure collapses the whole session back to `IDLE`. +- On successful connect, the orchestrator emits a `HealthEvent(kind=HEARTBEAT, detail="connected")` but with no `phase` / `outcome` fields — so `StartupFailureDetector` never closes anything. +- On failure, the orchestrator emits **nothing** before re-raising — `StartupFailureDetector` never sees the error, so it is permanently dormant despite being registered. +- `StreamSnapshot` has no per-stream connection state. Frontend conflates "connecting", "connected but no data", and "failed" into one grey dot. +- `VideoPreview` renders the browser broken-image icon when `/stream/video/{id}` returns empty — no text, no visual indication of failure. +- OAK's `connect()` takes ~2.4 s (three enumeration retries × 0.8 s) before raising when its declared `device_id` is not present (`src/syncfield/adapters/oak_camera.py:316-366`). + +## Architecture Overview + +``` + ┌──────────────────────────────────────────┐ + │ SessionOrchestrator.connect() │ + │ │ + │ for stream in self._streams.values(): │ + │ _set_stream_state(sid, "connecting") │ + │ try: │ + │ stream.prepare() │ + │ stream.connect() │ + │ except Exception as exc: │ + │ _set_stream_state(sid, "failed") │ + │ _stream_errors[sid] = str(exc) │ + │ emit phase=connect outcome=error │ + │ continue │ + │ _set_stream_state(sid, "connected") │ + │ emit phase=connect outcome=success │ + │ │ + │ if no "connected" streams: │ + │ raise RuntimeError + IDLE │ + │ else: transition → CONNECTED │ + └────────────────────┬─────────────────────┘ + │ + _set_stream_state ──┼──► health.observe_connection_state(sid, new_state) + │ + ┌────────────────────▼─────────────────────┐ + │ HealthSystem / HealthWorker │ + │ │ + │ ingress: _ConnectionStateMsg (NEW) │ + │ → detector.observe_connection_state │ + │ │ + │ existing path: │ + │ on_sample → detector.observe_sample │ + │ on_health → tracker.ingest + │ + │ detector.observe_health │ + │ │ + │ NoDataDetector tick: │ + │ if connected_at[sid] + 5s ≤ now │ + │ and sid not in _has_sample: │ + │ emit {sid}:no-data incident │ + └────────────────────┬─────────────────────┘ + │ + ┌────────────────────▼─────────────────────┐ + │ SessionSnapshot (viewer wire) │ + │ │ + │ streams[sid].connection_state (NEW) │ + │ streams[sid].connection_error (NEW) │ + │ active_incidents / resolved_incidents │ + │ (unchanged — already carry startup + │ + │ no-data incidents) │ + └────────────────────┬─────────────────────┘ + │ + ┌────────────────────▼─────────────────────┐ + │ Viewer frontend │ + │ │ + │ StreamCard selects body by state: │ + │ "connecting" → ConnectingOverlay │ + │ "failed" → FailedOverlay + error │ + │ "connected" + frame_count === 0 │ + │ → WaitingForDataOverlay │ + │ else → │ + │ │ + │ Header state chip: │ + │ "Ready (3/5)" if connected < total │ + │ "Ready" if all connected │ + │ IncidentPanel + severity badge: unchanged│ + └──────────────────────────────────────────┘ +``` + +## Data model + +### `ConnectionState` (new) + +```python +ConnectionState = Literal[ + "idle", # stream added but connect() not yet called + "connecting", # connect() is running + "connected", # connect() returned successfully + "failed", # connect() raised; error recorded in _stream_errors + "disconnected", # disconnect() ran, or rolled back after global failure +] +``` + +Tracked on `SessionOrchestrator` as `self._stream_states: dict[str, ConnectionState]` and `self._stream_errors: dict[str, str]` (populated only for `failed`). Cleared on `disconnect()` back to `disconnected`. + +Transitions are a narrow DAG: +``` +idle → connecting → connected → disconnected + → failed → disconnected +``` +`failed` never transitions to `connected` within the same connect cycle — a subsequent `disconnect()` followed by `connect()` is a new cycle. + +### `StreamSnapshot` — additions + +```python +connection_state: str # one of ConnectionState literals +connection_error: str | None # set iff connection_state == "failed" +``` + +### `StartupFailureDetector` event contract (no code change; existing design) + +Orchestrator emits on connect failure — using the detector's own fingerprint so both the orchestrator's synchronous event and the detector's tick-emitted event land on the same incident: +```python +HealthEvent( + stream_id=stream.id, + kind=HealthEventKind.ERROR, + at_ns=time.monotonic_ns(), + detail=str(exc), + severity=Severity.ERROR, + source="orchestrator", + fingerprint=f"{stream.id}:startup-failure", + data={"phase": "connect", "outcome": "error", "error": str(exc)}, +) +``` + +On success: +```python +HealthEvent( + stream_id=stream.id, + kind=HealthEventKind.HEARTBEAT, + at_ns=time.monotonic_ns(), + detail="connected", + severity=Severity.INFO, + source="orchestrator", + fingerprint=f"{stream.id}:startup-success", + data={"phase": "connect", "outcome": "success"}, +) +``` + +Rationale for fingerprints: +- `{stream.id}:startup-failure` routes to `StartupFailureDetector` via `IncidentTracker._detector_for` (fingerprint middle token equals the detector's `name`). The detector owns `close_condition` which consults its own `_recovered` set, so the incident closes only after a matching success signal arrives. +- The success event deliberately uses a **different** fingerprint (`:startup-success`), so it does not itself open an incident on the failure one. Its purpose is purely to feed `StartupFailureDetector.observe_health`, which keys off `data["outcome"] == "success"` and adds the stream to `_recovered`. The event still flows through the tracker, but the tracker's per-fingerprint grouping means `:startup-success` events open at most a trivial INFO incident that auto-closes via the passthrough quiet window (no user-visible noise given severity=INFO). + +## New detector: `NoDataDetector` + +Lives at `src/syncfield/health/detectors/no_data.py`. Registered by default in `HealthSystem._install_default_detectors`. + +- Fires `HealthEvent(fingerprint=f"{stream_id}:no-data", severity=ERROR)` when a stream has been in `connected` state for ≥ `threshold_ns` (default 5 s) without any sample. +- Closes the incident the moment the first sample arrives (`close_condition` returns `stream_id in self._has_sample`). +- Per-stream state machine keyed by `stream_id`: `_connected_at`, `_has_sample`, `_fire_active`. Reset to empty on `observe_connection_state(stream_id, new_state)` whenever the new state is `failed / disconnected / idle`. + +### `Detector.observe_connection_state` (new hook) + +Added to the protocol and to `DetectorBase` as a no-op. Existing detectors keep working unchanged. `NoDataDetector` is the only one that overrides it for v1. + +### Worker ingress + +`HealthWorker` gains a fifth queue: +```python +_connection_states: queue.SimpleQueue[_ConnectionStateMsg] +``` +drained on each tick the same way the other four are. `HealthSystem.observe_connection_state(stream_id, new_state)` pushes into it. Called by `SessionOrchestrator._set_stream_state`. + +## Viewer changes + +### Backend (`viewer/state.py`, `viewer/poller.py`, `viewer/server.py`) + +- `StreamSnapshot`: add `connection_state: str = "idle"` and `connection_error: str | None = None` fields. +- Poller reads `orchestrator._stream_states` and `orchestrator._stream_errors` on each tick and populates the snapshot. +- Server WebSocket serializer includes both fields in each stream entry. + +### Frontend types (`lib/types.ts`) + +```ts +export type ConnectionState = + | "idle" | "connecting" | "connected" | "failed" | "disconnected"; + +export interface StreamSnapshot { + // ...existing fields... + connection_state: ConnectionState; + connection_error: string | null; +} +``` + +### `StreamCard` body selection + +```tsx +function StreamCardBody({ stream }: { stream: StreamSnapshot }) { + if (stream.connection_state === "connecting") return ; + if (stream.connection_state === "failed") { + return ; + } + if (stream.connection_state === "connected" && stream.frame_count === 0) { + return ; + } + return ; +} +``` + +### New overlay components (`components/stream-overlays.tsx`) + +- `ConnectingOverlay` — neutral grey background, pulse-animated dot + "Connecting…" text. +- `WaitingForDataOverlay` — soft yellow tint, "Connected · waiting for first frame". No animation (it should feel temporary; the real alarm is the no-data incident that opens after 5 s). +- `FailedOverlay` — red background, "Failed to connect" header, monospace error text (two visible lines, clickable to expand to full text), hint "Press Discover Devices or Disconnect + Connect to retry". + +### Header state chip (`components/header.tsx`) + +```tsx +const total = Object.keys(snapshot.streams).length; +const connected = Object.values(snapshot.streams) + .filter((s) => s.connection_state === "connected").length; +const label = connected === total || total === 0 + ? stateLabel + : `${stateLabel} (${connected}/${total})`; +const tone = connected < total ? "warning" : "normal"; +``` +Warning-tone chip is yellow; normal chip keeps its current styling. + +## Orchestrator integration points + +All changes in `src/syncfield/orchestrator.py`: + +1. **`__init__`**: initialize `self._stream_states: dict[str, ConnectionState] = {}` and `self._stream_errors: dict[str, str] = {}`. +2. **`add(stream)`**: after existing wiring, set `self._stream_states[stream.id] = "idle"`. +3. **`connect()`**: replace the current all-or-nothing loop (lines 1623-1643) with the partial-connect loop in the Architecture Overview. Use a single helper `_set_stream_state(stream_id, new_state)` that updates `_stream_states` *and* calls `self.health.observe_connection_state(stream_id, new_state)`. +4. **`disconnect()`**: only call `stream.disconnect()` on streams whose `_stream_states[sid]` is `connected` (or was up to `stop_recording`). Transition failed streams directly to `disconnected`. Clear `_stream_errors`. +5. **`stop()`**: unchanged by this spec — it already handles only streams that were recording. Failed streams never entered the recording set. + +Remove the now-dead `_rollback_disconnect_streams` helper if nothing else uses it, otherwise leave it untouched. + +## Persistence & artifacts + +No new files. The startup-failure and no-data incidents flow through the existing `incidents.jsonl` sidecar via the callbacks wired in Phase 1. + +## Testing strategy + +### Unit tests + +**`tests/unit/health/detectors/test_no_data.py`** — new file. Covers: +- No fire when `connected` for less than `threshold_ns`. +- Fires after threshold with no sample. +- Closes after first sample arrives. +- Does not re-fire for the same stream while still in the connected-no-data state (uses `_fire_active`). +- Resets bookkeeping on state transition back to `failed` / `disconnected` / `idle`. +- Per-stream independent state. + +**`tests/unit/health/test_health_worker.py`** (extend) — verify the worker drains a `_connection_states` queue and fans out to `observe_connection_state` on every registered detector. + +**`tests/unit/health/test_detector_base.py`** (extend) — confirm the new `observe_connection_state` hook is a safe no-op on the base class. + +**`tests/unit/test_orchestrator.py`** (new test class `TestPartialConnect`) — the orchestrator tests already use `FakeStream` with `fail_on_start` / `fail_on_connect` flags. Cover: +- One stream raising in connect: session ends up in `CONNECTED`; that stream's `connection_state == "failed"`; others are `"connected"`. +- All streams raising: session returns to `IDLE` and the aggregate error is raised. +- `_stream_errors[sid]` contains the exception message string. +- `StartupFailureDetector` sees a `phase="connect"` event (spy detector). +- `disconnect()` does not call `stream.disconnect()` on streams whose state is `failed`. + +### Integration tests + +**`tests/integration/health/test_partial_connect.py`** — new file. Spins a real `SessionOrchestrator` with one `FakeStream` that always raises in `connect()` and two normal `FakeStream`s. Drives the full connect → record → stop → disconnect lifecycle and asserts: +- Session reaches `CONNECTED` (not `IDLE`). +- The startup-failure incident is in `sess.health.open_incidents()` after connect. +- `incidents.jsonl` contains the fingerprint. +- Working streams produce samples; failed stream does not. + +**`tests/integration/health/test_no_data_detector.py`** — a `FakeStream` whose `connect()` returns but whose sample-emission thread is inhibited. After 6 s of wall-clock, the no-data incident is open. Resuming sample emission closes the incident within one tick window. + +### Frontend + +Component test for `StreamCard` branch selection: one test per state (`connecting` / `failed` / `connected-no-frames` / `connected-with-frames`). Assert the correct overlay component is rendered. The overlays themselves get a trivial render test each. + +Header chip rendering: three cases (`total === 0`, `connected === total`, `connected < total`). + +## Scope for this PR (`feat/health-telemetry`) + +Adds to the existing PR on top of the Phase 1 health-telemetry work. Ordered by dependency: + +1. **Data model**: `ConnectionState` literal type, `StreamSnapshot.connection_state` / `connection_error` fields. +2. **`Detector.observe_connection_state` hook** on protocol + base. +3. **`HealthWorker` new ingress queue**; `HealthSystem.observe_connection_state` passthrough. +4. **`NoDataDetector`** + default-register in `HealthSystem`. +5. **`SessionOrchestrator.connect()` partial-connect rewrite** + `_set_stream_state` helper + structured startup events. +6. **Poller + server** snapshot serialization. +7. **Frontend overlays + card branch + header chip**. +8. **All tests listed above**. + +## Open questions / deferred decisions + +- Exact `threshold_ns` for `NoDataDetector` — proposed 5 s. Short enough to catch a silent OAK within the window a user is watching; long enough to absorb the natural warm-up of cameras that take a second or two to produce the first frame after connect. +- `FailedOverlay` copy — current draft is "Failed to connect" + verbatim error + "Press Discover Devices or Disconnect + Connect to retry". Final copy is a UX polish pass, not a design gate. + +## Risks + +- **Orchestrator `disconnect()` paths** need a careful read. If any bookkeeping currently assumes "every stream the orchestrator knows is fully connected", it may drop frames or mis-close files for a stream that was `failed` and never entered recording. The partial-connect rewrite must explicitly skip `failed` streams everywhere a loop iterates `self._streams.values()`. +- **Severity=INFO incident from success events** — the success signal uses `{stream_id}:startup-success` and creates a short-lived INFO incident that auto-closes via the passthrough quiet window. At `severity=INFO` it does not appear in the IncidentPanel's Active Issues (which filters by severity visually in the existing UI), but it does land in `incidents.jsonl`. If that persisted-log noise is unwanted, we can suppress the tracker's ingest for events whose fingerprint ends with `:startup-success`. Deferred until we see real post-session logs and decide whether it's useful telemetry or noise. +- **Frontend overlay stacking with `severity badge`** — the per-stream badge (Phase 1) paints on the card header; the overlays paint on the body. They should not collide, but the implementation must verify the card layout renders sensibly in all five states. diff --git a/src/syncfield/adapters/_generic.py b/src/syncfield/adapters/_generic.py index 8a075fe..dc8c25c 100644 --- a/src/syncfield/adapters/_generic.py +++ b/src/syncfield/adapters/_generic.py @@ -58,12 +58,17 @@ def reset_recording_stats(self) -> None: self._last_at_ns = None -def _default_sensor_capabilities(*, precise: bool) -> StreamCapabilities: +def _default_sensor_capabilities( + *, + precise: bool, + target_hz: Optional[float] = None, +) -> StreamCapabilities: return StreamCapabilities( provides_audio_track=False, supports_precise_timestamps=precise, is_removable=False, produces_file=False, # orchestrator handles JSONL persistence + target_hz=target_hz, ) @@ -71,7 +76,8 @@ def _resolve_capabilities( user: Optional[StreamCapabilities], *, precise: bool, + target_hz: Optional[float] = None, ) -> StreamCapabilities: if user is not None: return user - return _default_sensor_capabilities(precise=precise) + return _default_sensor_capabilities(precise=precise, target_hz=target_hz) diff --git a/src/syncfield/adapters/host_audio.py b/src/syncfield/adapters/host_audio.py index 306229c..2b99518 100644 --- a/src/syncfield/adapters/host_audio.py +++ b/src/syncfield/adapters/host_audio.py @@ -119,6 +119,7 @@ def __init__( supports_precise_timestamps=True, is_removable=True, produces_file=True, + target_hz=1.0 / METRICS_INTERVAL_S, ), ) self._output_dir = Path(output_dir) diff --git a/src/syncfield/adapters/meta_quest_camera/stream.py b/src/syncfield/adapters/meta_quest_camera/stream.py index 153a886..f88e99b 100644 --- a/src/syncfield/adapters/meta_quest_camera/stream.py +++ b/src/syncfield/adapters/meta_quest_camera/stream.py @@ -116,6 +116,7 @@ def __init__( supports_precise_timestamps=True, is_removable=True, produces_file=True, + target_hz=float(fps), ), ) self._quest_host = quest_host diff --git a/src/syncfield/adapters/oak_camera.py b/src/syncfield/adapters/oak_camera.py index cfb721f..1385195 100644 --- a/src/syncfield/adapters/oak_camera.py +++ b/src/syncfield/adapters/oak_camera.py @@ -172,6 +172,7 @@ def __init__( supports_precise_timestamps=True, is_removable=True, produces_file=True, + target_hz=float(rgb_fps), ), ) self._output_dir = Path(output_dir) @@ -332,6 +333,8 @@ def connect(self) -> None: attached devices after :attr:`_ENUMERATE_RETRIES` probe attempts. """ + self._install_depthai_bridge() + if self._thread is not None and self._thread.is_alive(): return @@ -493,6 +496,33 @@ def _finalize_mp4(self) -> bool: pass return True + def _install_depthai_bridge(self) -> None: + """Install a logging.Handler that converts depthai native log records into HealthEvents. + + Idempotent. Registered on depthai's module logger so every internal + warning/error that depthai emits during this stream's lifetime is + routed to the IncidentTracker via :meth:`_emit_health`. + """ + import logging as _logging + from syncfield.health.detectors.depthai_bridge import DepthAILoggerBridge + + if getattr(self, "_depthai_bridge", None) is not None: + return + self._depthai_bridge = DepthAILoggerBridge( + stream_id=self.id, + sink=lambda _sid, ev: self._emit_health(ev), + ) + _logging.getLogger("depthai").addHandler(self._depthai_bridge) + + def _uninstall_depthai_bridge(self) -> None: + """Remove the depthai logging handler. Idempotent.""" + import logging as _logging + bridge = getattr(self, "_depthai_bridge", None) + if bridge is None: + return + _logging.getLogger("depthai").removeHandler(bridge) + self._depthai_bridge = None + def disconnect(self) -> None: """Stop the capture thread and release the DepthAI pipeline. @@ -505,6 +535,7 @@ def disconnect(self) -> None: self._thread.join(timeout=3.0) self._thread = None self._release_pipeline() + self._uninstall_depthai_bridge() # ------------------------------------------------------------------ # Legacy one-shot lifecycle diff --git a/src/syncfield/adapters/polling_sensor.py b/src/syncfield/adapters/polling_sensor.py index b4f120c..f6ce224 100644 --- a/src/syncfield/adapters/polling_sensor.py +++ b/src/syncfield/adapters/polling_sensor.py @@ -41,7 +41,9 @@ def __init__( super().__init__( id=id, kind="sensor", - capabilities=_resolve_capabilities(capabilities, precise=True), + capabilities=_resolve_capabilities( + capabilities, precise=True, target_hz=float(hz) + ), ) self._validate_arity(read, expects_handle=open is not None) self._read = read diff --git a/src/syncfield/adapters/uvc_webcam.py b/src/syncfield/adapters/uvc_webcam.py index 3bf6f21..114abfd 100644 --- a/src/syncfield/adapters/uvc_webcam.py +++ b/src/syncfield/adapters/uvc_webcam.py @@ -91,6 +91,7 @@ def __init__( supports_precise_timestamps=True, is_removable=True, produces_file=True, + target_hz=(float(fps) if fps is not None else None), ), ) self._device_index = device_index diff --git a/src/syncfield/health/__init__.py b/src/syncfield/health/__init__.py new file mode 100644 index 0000000..661d5b3 --- /dev/null +++ b/src/syncfield/health/__init__.py @@ -0,0 +1,69 @@ +"""syncfield.health — platform health telemetry.""" + +from __future__ import annotations + +import sys as _sys + +# Guard flag — checked via __dict__ to avoid re-entering __getattr__. +_health_imported: bool = False + + +def __getattr__(name: str) -> object: + # Lazy-load to avoid a circular import: + # syncfield.types → syncfield.health.severity + # → syncfield.health (this __init__) → syncfield.health.detector + # → syncfield.health.types → syncfield.types (partially initialised) + _import_all() + try: + return _sys.modules[__name__].__dict__[name] + except KeyError: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None + + +def _import_all() -> None: + """Populate the module namespace on first access.""" + mod = _sys.modules[__name__] + # Use __dict__ directly to avoid triggering __getattr__ again. + if mod.__dict__.get("_health_imported"): + return + mod.__dict__["_health_imported"] = True + + from syncfield.health.detector import Detector, DetectorBase + from syncfield.health.registry import DetectorRegistry + from syncfield.health.severity import Severity, max_severity + from syncfield.health.system import HealthSystem + from syncfield.health.tracker import IncidentTracker + from syncfield.health.types import ( + Incident, + IncidentArtifact, + IncidentSnapshot, + WriterStats, + ) + + ns = mod.__dict__ + ns["Detector"] = Detector + ns["DetectorBase"] = DetectorBase + ns["DetectorRegistry"] = DetectorRegistry + ns["Severity"] = Severity + ns["max_severity"] = max_severity + ns["HealthSystem"] = HealthSystem + ns["IncidentTracker"] = IncidentTracker + ns["Incident"] = Incident + ns["IncidentArtifact"] = IncidentArtifact + ns["IncidentSnapshot"] = IncidentSnapshot + ns["WriterStats"] = WriterStats + + +__all__ = [ + "Detector", + "DetectorBase", + "DetectorRegistry", + "HealthSystem", + "Incident", + "IncidentArtifact", + "IncidentSnapshot", + "IncidentTracker", + "Severity", + "WriterStats", + "max_severity", +] diff --git a/src/syncfield/health/detector.py b/src/syncfield/health/detector.py new file mode 100644 index 0000000..2350918 --- /dev/null +++ b/src/syncfield/health/detector.py @@ -0,0 +1,91 @@ +"""Detector protocol + base class. + +A :class:`Detector` observes the stream (samples, adapter health events, +session state, writer stats) and may emit :class:`HealthEvent` on each +``tick()``. The :class:`IncidentTracker` groups emitted events by +fingerprint, opens incidents, and consults ``close_condition`` to know +when an open incident should resolve. + +Most detectors subclass :class:`DetectorBase` and override only the +observe/tick hooks they care about; the base provides safe no-op +defaults for the rest. +""" + +from __future__ import annotations + +from typing import Iterator, Protocol, runtime_checkable + +from syncfield.health.severity import Severity +from syncfield.health.types import Incident, WriterStats +from syncfield.types import HealthEvent, SampleEvent, SessionState + + +@runtime_checkable +class Detector(Protocol): + name: str + default_severity: Severity + + def observe_sample(self, stream_id: str, sample: SampleEvent) -> None: ... + def observe_health(self, stream_id: str, event: HealthEvent) -> None: ... + def observe_state(self, old: SessionState, new: SessionState) -> None: ... + def observe_writer_stats(self, stream_id: str, stats: WriterStats) -> None: ... + def observe_connection_state(self, stream_id: str, new_state: str, at_ns: int) -> None: ... + def tick(self, now_ns: int) -> Iterator[HealthEvent]: ... + def close_condition(self, incident: Incident, now_ns: int) -> bool: ... + + +class DetectorBase: + """No-op defaults for every Detector hook. + + Subclasses set ``name`` and ``default_severity`` at the class level + and override only the hooks that matter for their rule. + """ + + name: str + default_severity: Severity + + def __init_subclass__(cls, **kwargs: object) -> None: + super().__init_subclass__(**kwargs) + # Require each subclass chain to end at a concrete class that declares + # these attrs. We walk the MRO up to (but not including) DetectorBase + # and assert at least one class in that chain sets each attribute + # as an own attribute (via __dict__). This prevents a grandchild from + # silently inheriting a default it shouldn't while still allowing + # legitimate intermediate base classes. + for attr in ("name", "default_severity"): + declared = any( + attr in klass.__dict__ + for klass in cls.__mro__ + if klass is not DetectorBase and klass is not object + ) + if not declared: + raise TypeError( + f"Detector subclass {cls.__name__} must set class attribute '{attr}'" + ) + + def __new__(cls, *args: object, **kwargs: object) -> "DetectorBase": + if cls is DetectorBase: + raise TypeError("DetectorBase is abstract; subclass it") + return super().__new__(cls) + + def observe_sample(self, stream_id: str, sample: SampleEvent) -> None: + pass + + def observe_health(self, stream_id: str, event: HealthEvent) -> None: + pass + + def observe_state(self, old: SessionState, new: SessionState) -> None: + pass + + def observe_writer_stats(self, stream_id: str, stats: WriterStats) -> None: + pass + + def observe_connection_state(self, stream_id: str, new_state: str, at_ns: int) -> None: + pass + + def tick(self, now_ns: int) -> Iterator[HealthEvent]: + return iter(()) + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + # Conservative default: keep open. Subclasses override to close. + return False diff --git a/src/syncfield/health/detectors/__init__.py b/src/syncfield/health/detectors/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/syncfield/health/detectors/adapter_passthrough.py b/src/syncfield/health/detectors/adapter_passthrough.py new file mode 100644 index 0000000..34732e8 --- /dev/null +++ b/src/syncfield/health/detectors/adapter_passthrough.py @@ -0,0 +1,24 @@ +"""Passthrough detector: owns close semantics for adapter-emitted events. + +Fingerprint convention ``:adapter:`` routes to this +detector in the tracker. It never emits synthetic events; its only job +is saying "if no new adapter event arrived for N seconds, close the +incident". +""" + +from __future__ import annotations + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.types import Incident + + +class AdapterEventPassthrough(DetectorBase): + name = "adapter" + default_severity = Severity.WARNING + + def __init__(self, quiet_ns: int = 30 * 1_000_000_000) -> None: + self._quiet_ns = quiet_ns + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + return (now_ns - incident.last_event_at_ns) >= self._quiet_ns diff --git a/src/syncfield/health/detectors/backpressure.py b/src/syncfield/health/detectors/backpressure.py new file mode 100644 index 0000000..d1aef42 --- /dev/null +++ b/src/syncfield/health/detectors/backpressure.py @@ -0,0 +1,105 @@ +"""BackpressureDetector — writer queue saturation + drop-counter detector.""" + +from __future__ import annotations + +from collections.abc import Iterator +from typing import Dict, List, Optional + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.types import Incident, WriterStats +from syncfield.types import HealthEvent, HealthEventKind + + +class BackpressureDetector(DetectorBase): + name = "backpressure" + default_severity = Severity.WARNING + + def __init__( + self, + fullness_threshold: float = 0.80, + sustain_ns: int = 2_000_000_000, + recovery_ratio: float = 0.30, + recovery_ns: int = 5_000_000_000, + ) -> None: + self._threshold = fullness_threshold + self._sustain_ns = sustain_ns + self._recovery_ratio = recovery_ratio + self._recovery_ns = recovery_ns + + self._latest: Dict[str, WriterStats] = {} + self._first_bad_observed_at: Dict[str, Optional[int]] = {} + self._last_dropped: Dict[str, int] = {} + self._pending_drop_spike: Dict[str, bool] = {} + self._fire_active: Dict[str, bool] = {} + self._recovery_began_at: Dict[str, Optional[int]] = {} + + def observe_writer_stats(self, stream_id: str, stats: WriterStats) -> None: + self._latest[stream_id] = stats + prev = self._last_dropped.get(stream_id, 0) + if stats.dropped > prev: + self._pending_drop_spike[stream_id] = True + self._last_dropped[stream_id] = stats.dropped + + # Track when fullness first exceeds threshold + if stats.queue_fullness >= self._threshold: + if self._first_bad_observed_at.get(stream_id) is None: + self._first_bad_observed_at[stream_id] = stats.at_ns + else: + self._first_bad_observed_at[stream_id] = None + + def tick(self, now_ns: int) -> Iterator[HealthEvent]: + out: List[HealthEvent] = [] + for stream_id, stats in self._latest.items(): + fire_now = False + detail = "" + + if self._pending_drop_spike.pop(stream_id, False): + fire_now = True + detail = f"Writer dropped frames (total {stats.dropped})" + + if stats.queue_fullness >= self._threshold: + began = self._first_bad_observed_at.get(stream_id) + if began is not None: + if (now_ns - began) >= self._sustain_ns and not self._fire_active.get(stream_id, False): + fire_now = True + self._fire_active[stream_id] = True + detail = f"Writer queue {stats.queue_depth}/{stats.queue_capacity} full" + else: + self._fire_active[stream_id] = False + + if fire_now: + out.append(HealthEvent( + stream_id=stream_id, + kind=HealthEventKind.WARNING, + at_ns=now_ns, + detail=detail, + severity=self.default_severity, + source=f"detector:{self.name}", + fingerprint=f"{stream_id}:{self.name}", + data={ + "queue_depth": stats.queue_depth, + "queue_capacity": stats.queue_capacity, + "dropped": stats.dropped, + "dropped_at_open": stats.dropped, + }, + )) + return iter(out) + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + stats = self._latest.get(incident.stream_id) + if stats is None: + return False + if stats.queue_fullness > self._recovery_ratio: + self._recovery_began_at[incident.stream_id] = None + return False + # No new drops since incident opened. + opened_dropped = incident.data.get("dropped_at_open") + current_dropped = self._last_dropped.get(incident.stream_id, 0) + if opened_dropped is not None and current_dropped > opened_dropped: + return False + began = self._recovery_began_at.get(incident.stream_id) + if began is None: + self._recovery_began_at[incident.stream_id] = now_ns + return False + return (now_ns - began) >= self._recovery_ns diff --git a/src/syncfield/health/detectors/depthai_bridge.py b/src/syncfield/health/detectors/depthai_bridge.py new file mode 100644 index 0000000..81ae755 --- /dev/null +++ b/src/syncfield/health/detectors/depthai_bridge.py @@ -0,0 +1,115 @@ +"""DepthAILoggerBridge — translate depthai Python log records into HealthEvents. + +Installed as a standard :class:`logging.Handler` on the depthai logger. +Does not subclass DetectorBase — it is a translator, not a detector. +Its outputs are fingerprinted as ``:adapter:`` so +the AdapterEventPassthrough detector owns their open/close lifecycle. +""" + +from __future__ import annotations + +import logging +import re +import time +from typing import Callable, Optional + +from syncfield.health.severity import Severity +from syncfield.types import HealthEvent, HealthEventKind + +Sink = Callable[[str, HealthEvent], None] + +_XLINK_RE = re.compile(r"X_LINK_ERROR.*stream: '([^']+)'|stream: '([^']+)'.*X_LINK_ERROR") +_CRASH_RE = re.compile(r"Device with id (\S+) has crashed\. Crash dump logs are stored in: (\S+)") +_RECONNECT_TRY_RE = re.compile(r"Attempting to reconnect", re.IGNORECASE) +_RECONNECT_OK_RE = re.compile(r"Reconnection successful", re.IGNORECASE) +_CONN_CLOSED_RE = re.compile(r"Closed connection", re.IGNORECASE) + + +class DepthAILoggerBridge(logging.Handler): + def __init__(self, stream_id: str, sink: Sink) -> None: + super().__init__(level=logging.WARNING) + self._stream_id = stream_id + self._sink = sink + + def emit(self, record: logging.LogRecord) -> None: + if record.levelno < logging.WARNING: + return + msg = record.getMessage() + now = time.monotonic_ns() + + parsed = self._parse(msg, record.levelno, now) + if parsed is None: + parsed = HealthEvent( + stream_id=self._stream_id, + kind=HealthEventKind.WARNING, + at_ns=now, + detail=msg, + severity=Severity.WARNING, + source="adapter:oak:unparsed-log", + fingerprint=f"{self._stream_id}:adapter:unparsed-log", + data={"raw": msg, "levelname": record.levelname}, + ) + try: + self._sink(self._stream_id, parsed) + except Exception: + # Never let bridge failures crash the logging path. + pass + + def _parse(self, msg: str, levelno: int, now: int) -> Optional[HealthEvent]: + crash = _CRASH_RE.search(msg) + if crash: + device_id, path = crash.group(1), crash.group(2) + return HealthEvent( + stream_id=self._stream_id, + kind=HealthEventKind.ERROR, + at_ns=now, + detail="OAK device crashed", + severity=Severity.CRITICAL, + source="adapter:oak", + fingerprint=f"{self._stream_id}:adapter:device-crash", + data={"device_id": device_id, "crash_dump_path": path}, + ) + xlink = _XLINK_RE.search(msg) + if xlink: + stream = xlink.group(1) or xlink.group(2) + return HealthEvent( + stream_id=self._stream_id, + kind=HealthEventKind.ERROR, + at_ns=now, + detail="XLink communication error", + severity=Severity.ERROR, + source="adapter:oak", + fingerprint=f"{self._stream_id}:adapter:xlink-error", + data={"stream": stream}, + ) + if _RECONNECT_OK_RE.search(msg): + return HealthEvent( + stream_id=self._stream_id, + kind=HealthEventKind.RECONNECT, + at_ns=now, + detail="Reconnection successful", + severity=Severity.INFO, + source="adapter:oak", + fingerprint=f"{self._stream_id}:adapter:reconnect-success", + ) + if _RECONNECT_TRY_RE.search(msg): + return HealthEvent( + stream_id=self._stream_id, + kind=HealthEventKind.RECONNECT, + at_ns=now, + detail="Attempting reconnect", + severity=Severity.WARNING, + source="adapter:oak", + fingerprint=f"{self._stream_id}:adapter:reconnect-attempt", + ) + if _CONN_CLOSED_RE.search(msg): + return HealthEvent( + stream_id=self._stream_id, + kind=HealthEventKind.WARNING, + at_ns=now, + detail="Connection closed", + severity=Severity.WARNING, + source="adapter:oak", + fingerprint=f"{self._stream_id}:adapter:connection-closed", + ) + return None diff --git a/src/syncfield/health/detectors/fps_drop.py b/src/syncfield/health/detectors/fps_drop.py new file mode 100644 index 0000000..968f629 --- /dev/null +++ b/src/syncfield/health/detectors/fps_drop.py @@ -0,0 +1,153 @@ +"""FpsDropDetector — target-relative or baseline-learning FPS drop detector.""" + +from __future__ import annotations + +from collections import deque +from collections.abc import Iterator +from typing import Callable, Deque, Dict, List, Optional + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.types import Incident +from syncfield.types import HealthEvent, HealthEventKind, SampleEvent + +TargetGetter = Callable[[str], Optional[float]] + +_WINDOW_NS = 1_000_000_000 # rolling 1s FPS window + + +class FpsDropDetector(DetectorBase): + name = "fps-drop" + default_severity = Severity.WARNING + + def __init__( + self, + target_getter: TargetGetter = lambda sid: None, + drop_ratio: float = 0.70, + sustain_ns: int = 3_000_000_000, + recovery_ratio: float = 0.90, + recovery_ns: int = 5_000_000_000, + baseline_warmup_ns: int = 5_000_000_000, + baseline_window_ns: int = 10_000_000_000, + ) -> None: + self._target_getter = target_getter + self._drop_ratio = drop_ratio + self._sustain_ns = sustain_ns + self._recovery_ratio = recovery_ratio + self._recovery_ns = recovery_ns + self._baseline_warmup_ns = baseline_warmup_ns + self._baseline_window_ns = baseline_window_ns + + self._samples: Dict[str, Deque[int]] = {} + self._first_seen_at: Dict[str, int] = {} + self._baseline: Dict[str, float] = {} + self._baseline_locked: Dict[str, bool] = {} # True once baseline is frozen + # When did the stream first drop below threshold in the current dip? + self._dip_began_at: Dict[str, Optional[int]] = {} + self._fire_active: Dict[str, bool] = {} + # Same thing for recovery tracking. + self._recovery_began_at: Dict[str, Optional[int]] = {} + + # --- observers ------------------------------------------------------- + + def observe_sample(self, stream_id: str, sample: SampleEvent) -> None: + buf = self._samples.setdefault(stream_id, deque()) + buf.append(sample.capture_ns) + self._first_seen_at.setdefault(stream_id, sample.capture_ns) + # Trim older than baseline_window. + cutoff = sample.capture_ns - self._baseline_window_ns + while buf and buf[0] < cutoff: + buf.popleft() + + # --- tick ------------------------------------------------------------ + + def tick(self, now_ns: int) -> Iterator[HealthEvent]: + out: List[HealthEvent] = [] + for stream_id, buf in list(self._samples.items()): + target = self._effective_target(stream_id, now_ns) + observed = self._observed_fps(buf, now_ns) + if target is None or observed is None: + continue + + if observed < target * self._drop_ratio: + began = self._dip_began_at.get(stream_id) + if began is None: + self._dip_began_at[stream_id] = now_ns + began = now_ns + if (now_ns - began) >= self._sustain_ns and not self._fire_active.get(stream_id, False): + self._fire_active[stream_id] = True + out.append(HealthEvent( + stream_id=stream_id, + kind=HealthEventKind.WARNING, + at_ns=now_ns, + detail=f"FPS drop ({observed:.1f} Hz, target {target:.1f} Hz)", + severity=self.default_severity, + source=f"detector:{self.name}", + fingerprint=f"{stream_id}:{self.name}", + data={"observed_hz": observed, "target_hz": target}, + )) + else: + self._dip_began_at[stream_id] = None + self._fire_active[stream_id] = False + return iter(out) + + # --- close condition ------------------------------------------------- + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + stream_id = incident.stream_id + target = self._effective_target(stream_id, now_ns) + observed = self._observed_fps(self._samples.get(stream_id, deque()), now_ns) + if target is None or observed is None: + return False + if observed < target * self._recovery_ratio: + self._recovery_began_at[stream_id] = None + return False + began = self._recovery_began_at.get(stream_id) + if began is None: + self._recovery_began_at[stream_id] = now_ns + return False + return (now_ns - began) >= self._recovery_ns + + # --- helpers --------------------------------------------------------- + + def _effective_target(self, stream_id: str, now_ns: int) -> Optional[float]: + declared = self._target_getter(stream_id) + if declared is not None: + return float(declared) + first = self._first_seen_at.get(stream_id) + if first is None: + return None + if (now_ns - first) < self._baseline_warmup_ns: + return None + cached = self._baseline.get(stream_id) + if cached is not None: + return cached + # Calculate and lock baseline once warmup completes + # Lock it to the FPS observed right after warmup completes + if not self._baseline_locked.get(stream_id, False): + buf = self._samples.get(stream_id, deque()) + if not buf: + return None + # Calculate FPS from immediately after warmup to now + warmup_end = first + self._baseline_warmup_ns + count = sum(1 for t in buf if t >= warmup_end) + if count == 0: + return None + # Use the available time since warmup end + elapsed = min(now_ns - warmup_end, self._baseline_window_ns) + if elapsed <= 0: + return None + observed = count / (elapsed / 1e9) + self._baseline[stream_id] = observed + self._baseline_locked[stream_id] = True + return self._baseline.get(stream_id) + + @staticmethod + def _observed_fps(buf: Deque[int], now_ns: int) -> Optional[float]: + if not buf: + return None + cutoff = now_ns - _WINDOW_NS + count = sum(1 for t in buf if t >= cutoff) + if count == 0: + return 0.0 + return count / (_WINDOW_NS / 1e9) diff --git a/src/syncfield/health/detectors/jitter.py b/src/syncfield/health/detectors/jitter.py new file mode 100644 index 0000000..cbaddb3 --- /dev/null +++ b/src/syncfield/health/detectors/jitter.py @@ -0,0 +1,110 @@ +"""JitterDetector — p95-based inter-sample interval anomaly detector.""" + +from __future__ import annotations + +from collections import deque +from collections.abc import Iterator +from typing import Callable, Deque, Dict, List, Optional + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.types import Incident +from syncfield.types import HealthEvent, HealthEventKind, SampleEvent + +TargetGetter = Callable[[str], Optional[float]] + + +def _p95(values: List[int]) -> int: + if not values: + return 0 + sorted_v = sorted(values) + idx = max(0, int(0.95 * (len(sorted_v) - 1))) + return sorted_v[idx] + + +class JitterDetector(DetectorBase): + name = "jitter" + default_severity = Severity.WARNING + + def __init__( + self, + target_getter: TargetGetter = lambda sid: None, + window: int = 60, + jitter_ratio: float = 2.0, + sustain_ns: int = 3_000_000_000, + recovery_ratio: float = 1.2, + recovery_ns: int = 10_000_000_000, + ) -> None: + self._target_getter = target_getter + self._window = window + self._jitter_ratio = jitter_ratio + self._sustain_ns = sustain_ns + self._recovery_ratio = recovery_ratio + self._recovery_ns = recovery_ns + + self._last_at: Dict[str, int] = {} + self._intervals: Dict[str, Deque[int]] = {} + self._bad_began_at: Dict[str, Optional[int]] = {} + self._fire_active: Dict[str, bool] = {} + self._recovery_began_at: Dict[str, Optional[int]] = {} + + def observe_sample(self, stream_id: str, sample: SampleEvent) -> None: + last = self._last_at.get(stream_id) + if last is not None: + buf = self._intervals.setdefault(stream_id, deque(maxlen=self._window)) + buf.append(sample.capture_ns - last) + self._last_at[stream_id] = sample.capture_ns + + def tick(self, now_ns: int) -> Iterator[HealthEvent]: + out: List[HealthEvent] = [] + for stream_id, buf in list(self._intervals.items()): + target_hz = self._target_getter(stream_id) + if target_hz is None or target_hz <= 0 or len(buf) < max(10, self._window // 2): + continue + expected = 1e9 / target_hz + p95 = _p95(list(buf)) + + if p95 > expected * self._jitter_ratio: + began = self._bad_began_at.get(stream_id) + if began is None: + # Backdate bad_began_at to the start of the current window. + # This is the sum of all intervals in the buffer, which represents + # the elapsed time from the start of this window to now. + window_elapsed = sum(buf) + self._bad_began_at[stream_id] = now_ns - window_elapsed + began = self._bad_began_at[stream_id] + + elapsed = now_ns - began + if elapsed >= self._sustain_ns and not self._fire_active.get(stream_id, False): + self._fire_active[stream_id] = True + out.append(HealthEvent( + stream_id=stream_id, + kind=HealthEventKind.WARNING, + at_ns=now_ns, + detail=f"Jitter spike (p95 {p95/1e6:.1f} ms, expected {expected/1e6:.1f} ms)", + severity=self.default_severity, + source=f"detector:{self.name}", + fingerprint=f"{stream_id}:{self.name}", + data={"p95_ns": p95, "expected_ns": int(expected)}, + )) + else: + self._bad_began_at[stream_id] = None + self._fire_active[stream_id] = False + return iter(out) + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + stream_id = incident.stream_id + buf = self._intervals.get(stream_id) + target_hz = self._target_getter(stream_id) + if not buf or target_hz is None or target_hz <= 0: + return False + expected = 1e9 / target_hz + p95 = _p95(list(buf)) + if p95 > expected * self._recovery_ratio: + self._recovery_began_at[stream_id] = None + return False + began = self._recovery_began_at.get(stream_id) + if began is None: + self._recovery_began_at[stream_id] = now_ns + return False + return (now_ns - began) >= self._recovery_ns diff --git a/src/syncfield/health/detectors/no_data.py b/src/syncfield/health/detectors/no_data.py new file mode 100644 index 0000000..fdc3a08 --- /dev/null +++ b/src/syncfield/health/detectors/no_data.py @@ -0,0 +1,65 @@ +"""NoDataDetector — fires when a stream is connected but never emits a sample. + +Complements StreamStallDetector (which requires prior samples). Catches +the "connected but silent" case such as an OAK pipeline that fails to +pump frames even though its device connected. Resets bookkeeping on +any non-connected state transition so a reconnect starts a fresh clock. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from typing import Dict, List, Set + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.types import Incident +from syncfield.types import HealthEvent, HealthEventKind, SampleEvent + + +class NoDataDetector(DetectorBase): + name = "no-data" + default_severity = Severity.ERROR + + def __init__(self, threshold_ns: int = 5_000_000_000) -> None: + self._threshold_ns = threshold_ns + self._connected_at: Dict[str, int] = {} + self._has_sample: Set[str] = set() + self._fire_active: Dict[str, bool] = {} + + def observe_connection_state(self, stream_id: str, new_state: str, at_ns: int) -> None: + if new_state == "connected": + self._connected_at[stream_id] = at_ns + self._has_sample.discard(stream_id) + self._fire_active[stream_id] = False + else: + # idle / connecting / failed / disconnected → reset everything. + self._connected_at.pop(stream_id, None) + self._has_sample.discard(stream_id) + self._fire_active.pop(stream_id, None) + + def observe_sample(self, stream_id: str, sample: SampleEvent) -> None: + self._has_sample.add(stream_id) + + def tick(self, now_ns: int) -> Iterator[HealthEvent]: + out: List[HealthEvent] = [] + for stream_id, connected_at in self._connected_at.items(): + if stream_id in self._has_sample: + continue + elapsed = now_ns - connected_at + if elapsed >= self._threshold_ns and not self._fire_active.get(stream_id, False): + self._fire_active[stream_id] = True + out.append(HealthEvent( + stream_id=stream_id, + kind=HealthEventKind.ERROR, + at_ns=now_ns, + detail=f"Connected {elapsed / 1e9:.1f}s ago but no data received", + severity=self.default_severity, + source=f"detector:{self.name}", + fingerprint=f"{stream_id}:{self.name}", + data={"connected_at_ns": connected_at, "elapsed_ns": elapsed}, + )) + return iter(out) + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + return incident.stream_id in self._has_sample diff --git a/src/syncfield/health/detectors/startup_failure.py b/src/syncfield/health/detectors/startup_failure.py new file mode 100644 index 0000000..6f6e144 --- /dev/null +++ b/src/syncfield/health/detectors/startup_failure.py @@ -0,0 +1,60 @@ +"""StartupFailureDetector — fires when connect/start_recording raises. + +Relies on orchestrator-emitted HealthEvents with ``data["phase"]`` in +{``"connect"``, ``"start_recording"``}. A subsequent success event with +``data["outcome"] == "success"`` closes the incident. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from typing import Dict, List, Set + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.types import Incident +from syncfield.types import HealthEvent, HealthEventKind + +_STARTUP_PHASES = {"connect", "start_recording"} + + +class StartupFailureDetector(DetectorBase): + name = "startup-failure" + default_severity = Severity.ERROR + + def __init__(self) -> None: + self._pending_failures: Dict[str, HealthEvent] = {} + self._recovered: Set[str] = set() + + def observe_health(self, stream_id: str, event: HealthEvent) -> None: + phase = event.data.get("phase") if event.data else None + if phase not in _STARTUP_PHASES: + return + outcome = event.data.get("outcome") if event.data else None + if event.kind == HealthEventKind.ERROR and outcome != "success": + self._pending_failures[stream_id] = event + self._recovered.discard(stream_id) + elif outcome == "success": + self._recovered.add(stream_id) + # Clear any stale pending failure so the next tick doesn't emit it + # as a spurious post-recovery event. + self._pending_failures.pop(stream_id, None) + + def tick(self, now_ns: int) -> Iterator[HealthEvent]: + out: List[HealthEvent] = [] + for stream_id, origin in list(self._pending_failures.items()): + out.append(HealthEvent( + stream_id=stream_id, + kind=HealthEventKind.ERROR, + at_ns=now_ns, + detail=origin.detail or "Startup failure", + severity=self.default_severity, + source=f"detector:{self.name}", + fingerprint=f"{stream_id}:{self.name}", + data={"phase": origin.data.get("phase"), "origin_at_ns": origin.at_ns}, + )) + del self._pending_failures[stream_id] + return iter(out) + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + return incident.stream_id in self._recovered diff --git a/src/syncfield/health/detectors/stream_stall.py b/src/syncfield/health/detectors/stream_stall.py new file mode 100644 index 0000000..b26df2b --- /dev/null +++ b/src/syncfield/health/detectors/stream_stall.py @@ -0,0 +1,64 @@ +"""StreamStallDetector — fires when a stream stops producing samples.""" + +from __future__ import annotations + +from collections.abc import Iterator +from typing import Dict, List + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.types import Incident +from syncfield.types import HealthEvent, HealthEventKind, SampleEvent + + +class StreamStallDetector(DetectorBase): + name = "stream-stall" + default_severity = Severity.ERROR + + def __init__( + self, + stall_threshold_ns: int = 2_000_000_000, + recovery_ns: int = 1_000_000_000, + ) -> None: + self._stall_threshold_ns = stall_threshold_ns + self._recovery_ns = recovery_ns + # Per-stream most-recent sample monotonic time. + self._last_sample_at: Dict[str, int] = {} + # Per-stream: are we currently firing? prevents duplicates per stall. + self._stall_active: Dict[str, bool] = {} + + # --- observers ------------------------------------------------------- + + def observe_sample(self, stream_id: str, sample: SampleEvent) -> None: + self._last_sample_at[stream_id] = sample.capture_ns + # A new sample ends any active stall bookkeeping. + self._stall_active[stream_id] = False + + # --- tick ------------------------------------------------------------ + + def tick(self, now_ns: int) -> Iterator[HealthEvent]: + emitted: List[HealthEvent] = [] + for stream_id, last in self._last_sample_at.items(): + silence_ns = now_ns - last + if silence_ns >= self._stall_threshold_ns and not self._stall_active.get(stream_id, False): + self._stall_active[stream_id] = True + emitted.append(HealthEvent( + stream_id=stream_id, + kind=HealthEventKind.ERROR, + at_ns=now_ns, + detail=f"Stream stalled (silence {silence_ns / 1e9:.1f}s)", + severity=self.default_severity, + source=f"detector:{self.name}", + fingerprint=f"{stream_id}:{self.name}", + data={"silence_ns": silence_ns}, + )) + return iter(emitted) + + # --- close condition ------------------------------------------------- + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + last = self._last_sample_at.get(incident.stream_id) + if last is None: + return False + return (now_ns - last) < self._stall_threshold_ns \ + and (now_ns - incident.last_event_at_ns) >= self._recovery_ns diff --git a/src/syncfield/health/registry.py b/src/syncfield/health/registry.py new file mode 100644 index 0000000..ce66fc8 --- /dev/null +++ b/src/syncfield/health/registry.py @@ -0,0 +1,26 @@ +"""Registry of active Detectors for a session.""" + +from __future__ import annotations + +from typing import Iterator, List + +from syncfield.health.detector import Detector + + +class DetectorRegistry: + def __init__(self) -> None: + self._detectors: List[Detector] = [] + + def register(self, detector: Detector) -> None: + if any(d.name == detector.name for d in self._detectors): + raise ValueError(f"Detector '{detector.name}' is already registered") + self._detectors.append(detector) + + def unregister(self, name: str) -> None: + self._detectors = [d for d in self._detectors if d.name != name] + + def __iter__(self) -> Iterator[Detector]: + return iter(list(self._detectors)) + + def __len__(self) -> int: + return len(self._detectors) diff --git a/src/syncfield/health/severity.py b/src/syncfield/health/severity.py new file mode 100644 index 0000000..af43190 --- /dev/null +++ b/src/syncfield/health/severity.py @@ -0,0 +1,38 @@ +"""Severity levels for health events and incidents. + +Ordered INFO < WARNING < ERROR < CRITICAL. Use :func:`max_severity` to +pick the highest of several levels — incidents escalate to the max +severity of their constituent events. +""" + +from __future__ import annotations + +from enum import Enum + + +class Severity(str, Enum): + INFO = "info" + WARNING = "warning" + ERROR = "error" + CRITICAL = "critical" + + @property + def rank(self) -> int: + return _RANK[self] + + +# `(str, Enum)` inherits alphabetical string comparison, which would order +# critical < error < info < warning — wrong for severity. `_RANK` pins the +# intended order explicitly and gives `.rank` O(1) lookup. +_RANK = { + Severity.INFO: 0, + Severity.WARNING: 1, + Severity.ERROR: 2, + Severity.CRITICAL: 3, +} + + +def max_severity(*levels: Severity) -> Severity: + if not levels: + raise ValueError("max_severity requires at least one Severity") + return max(levels, key=lambda s: s.rank) diff --git a/src/syncfield/health/system.py b/src/syncfield/health/system.py new file mode 100644 index 0000000..2a4182e --- /dev/null +++ b/src/syncfield/health/system.py @@ -0,0 +1,138 @@ +"""HealthSystem — the single handle the orchestrator + user code touch.""" + +from __future__ import annotations + +from typing import Callable, Dict, Iterable, Iterator, Optional + +from syncfield.health.detector import Detector +from syncfield.health.detectors.adapter_passthrough import AdapterEventPassthrough +from syncfield.health.detectors.backpressure import BackpressureDetector +from syncfield.health.detectors.fps_drop import FpsDropDetector +from syncfield.health.detectors.jitter import JitterDetector +from syncfield.health.detectors.no_data import NoDataDetector +from syncfield.health.detectors.startup_failure import StartupFailureDetector +from syncfield.health.detectors.stream_stall import StreamStallDetector +from syncfield.health.registry import DetectorRegistry +from syncfield.health.tracker import IncidentTracker +from syncfield.health.types import Incident, WriterStats +from syncfield.health.worker import HealthWorker +from syncfield.types import HealthEvent, SampleEvent, SessionState + + +class HealthSystem: + """Composes Registry + Tracker + Worker into a single user-facing facade.""" + + def __init__( + self, + *, + tick_hz: float = 20.0, + passthrough_close_ns: int = 30 * 1_000_000_000, + ) -> None: + self._registry = DetectorRegistry() + self._tracker = IncidentTracker(passthrough_close_ns=passthrough_close_ns) + self._worker: Optional[HealthWorker] = None + self._tick_hz = tick_hz + self._target_hz_by_stream: Dict[str, Optional[float]] = {} + + self._install_default_detectors() + + # --- incident callbacks ---------------------------------------------- + + def on_incident_opened(self, cb: Callable[[Incident], None]) -> None: + """Register a callback fired when a new incident opens.""" + self._tracker.add_on_opened(cb) + + def on_incident_updated(self, cb: Callable[[Incident], None]) -> None: + """Register a callback fired when an open incident receives a new event.""" + self._tracker.add_on_updated(cb) + + def on_incident_closed(self, cb: Callable[[Incident], None]) -> None: + """Register a callback fired when an incident is resolved.""" + self._tracker.add_on_closed(cb) + + # --- registry -------------------------------------------------------- + + def register(self, detector: Detector) -> None: + import warnings + self._registry.register(detector) + self._tracker.bind_detector(detector) + if self._worker is not None: + warnings.warn( + f"Detector '{detector.name}' registered after HealthSystem.start(); " + "it will be bound for close-condition routing but will NOT be ticked " + "until the system is stopped and restarted.", + RuntimeWarning, + stacklevel=2, + ) + + def unregister(self, name: str) -> None: + self._registry.unregister(name) + + def iter_detectors(self) -> Iterator[Detector]: + return iter(self._registry) + + # --- observer inputs ------------------------------------------------- + + def observe_sample(self, stream_id: str, sample: SampleEvent) -> None: + if self._worker is not None: + self._worker.push_sample(stream_id, sample) + + def observe_health(self, stream_id: str, event: HealthEvent) -> None: + if self._worker is not None: + self._worker.push_health(stream_id, event) + + def observe_state(self, old: SessionState, new: SessionState) -> None: + if self._worker is not None: + self._worker.push_state(old, new) + + def observe_writer_stats(self, stream_id: str, stats: WriterStats) -> None: + if self._worker is not None: + self._worker.push_writer_stats(stream_id, stats) + + def observe_connection_state(self, stream_id: str, new_state: str, at_ns: int) -> None: + if self._worker is not None: + self._worker.push_connection_state(stream_id, new_state, at_ns) + + # --- lifecycle ------------------------------------------------------- + + def start(self) -> None: + if self._worker is not None: + return # already running — idempotent + self._worker = HealthWorker( + tracker=self._tracker, + detectors=list(self._registry), + tick_hz=self._tick_hz, + ) + self._worker.start() + + def stop(self, *, close_open_incidents: bool = True, now_ns: Optional[int] = None) -> None: + if self._worker is not None: + self._worker.stop() + self._worker = None + if close_open_incidents: + import time + self._tracker.close_all(at_ns=now_ns if now_ns is not None else time.monotonic_ns()) + + # --- read-only views ------------------------------------------------- + + def open_incidents(self) -> Iterable[Incident]: + return self._tracker.open_incidents() + + def resolved_incidents(self) -> Iterable[Incident]: + return self._tracker.resolved_incidents() + + # --- helpers --------------------------------------------------------- + + def register_stream(self, stream_id: str, target_hz: Optional[float]) -> None: + """Declare the expected target_hz for a stream. Detectors consult this.""" + self._target_hz_by_stream[stream_id] = target_hz + + def _install_default_detectors(self) -> None: + target_getter = lambda sid: self._target_hz_by_stream.get(sid) + self.register(AdapterEventPassthrough()) + self.register(StreamStallDetector()) + self.register(FpsDropDetector(target_getter=target_getter)) + self.register(JitterDetector(target_getter=target_getter)) + self.register(StartupFailureDetector()) + self.register(BackpressureDetector()) + self.register(NoDataDetector()) diff --git a/src/syncfield/health/tracker.py b/src/syncfield/health/tracker.py new file mode 100644 index 0000000..fe59594 --- /dev/null +++ b/src/syncfield/health/tracker.py @@ -0,0 +1,119 @@ +"""IncidentTracker — groups HealthEvents into Incidents and manages open/close. + +Runs on the HealthWorker thread. Public methods are *not* thread-safe on +their own; the worker serializes access. +""" + +from __future__ import annotations + +from typing import Callable, Dict, List, Optional + +from syncfield.health.detector import Detector +from syncfield.health.types import Incident +from syncfield.types import HealthEvent + +Callback = Callable[[Incident], None] + + +class IncidentTracker: + def __init__(self, passthrough_close_ns: int = 30 * 1_000_000_000) -> None: + self._by_fingerprint: Dict[str, Incident] = {} + self._resolved: List[Incident] = [] + self._detectors_by_name: Dict[str, Detector] = {} + self._passthrough_close_ns = passthrough_close_ns + + self._on_opened: List[Callback] = [] + self._on_updated: List[Callback] = [] + self._on_closed: List[Callback] = [] + + def add_on_opened(self, cb: Callback) -> None: + self._on_opened.append(cb) + + def add_on_updated(self, cb: Callback) -> None: + self._on_updated.append(cb) + + def add_on_closed(self, cb: Callback) -> None: + self._on_closed.append(cb) + + # --- detector wiring ------------------------------------------------- + + def bind_detector(self, detector: Detector) -> None: + self._detectors_by_name[detector.name] = detector + + # --- event ingestion ------------------------------------------------- + + def ingest(self, event: HealthEvent) -> None: + if not event.fingerprint: + raise ValueError( + "HealthEvent.fingerprint is required before reaching the IncidentTracker; " + "the platform fills it in for adapter events, detectors set their own." + ) + open_inc = self._by_fingerprint.get(event.fingerprint) + if open_inc is None: + inc = Incident.opened_from(event, title=_title_from(event)) + self._by_fingerprint[event.fingerprint] = inc + self._fire(self._on_opened, inc) + return + open_inc.record_event(event) + self._fire(self._on_updated, open_inc) + + # --- tick — evaluate close conditions -------------------------------- + + def tick(self, now_ns: int) -> None: + to_close: List[str] = [] + for fp, inc in self._by_fingerprint.items(): + detector = self._detector_for(inc) + if detector is not None: + should_close = detector.close_condition(inc, now_ns) + else: + should_close = (now_ns - inc.last_event_at_ns) >= self._passthrough_close_ns + if should_close: + to_close.append(fp) + + for fp in to_close: + inc = self._by_fingerprint.pop(fp) + inc.close(at_ns=now_ns) + self._resolved.append(inc) + self._fire(self._on_closed, inc) + + def close_all(self, *, at_ns: int) -> None: + """Used at session stop to resolve any still-open incidents.""" + for fp in list(self._by_fingerprint.keys()): + inc = self._by_fingerprint.pop(fp) + inc.close(at_ns=at_ns) + self._resolved.append(inc) + self._fire(self._on_closed, inc) + + # --- read-only views ------------------------------------------------- + + def open_incidents(self) -> List[Incident]: + return list(self._by_fingerprint.values()) + + def resolved_incidents(self) -> List[Incident]: + return list(self._resolved) + + # --- helpers --------------------------------------------------------- + + def _detector_for(self, inc: Incident) -> Optional[Detector]: + # Fingerprint convention: ":[:suffix]". + parts = inc.fingerprint.split(":", 2) + if len(parts) < 2: + return None + return self._detectors_by_name.get(parts[1]) + + @staticmethod + def _fire(callbacks: List[Callback], inc: Incident) -> None: + for cb in callbacks: + try: + cb(inc) + except Exception: + # A listener failure must not break the tracker. + pass + + +def _title_from(event: HealthEvent) -> str: + # Prefer the first event's detail as the title; fall back to + # ": " if the detail is missing. + if event.detail: + return event.detail + return f"{event.source}: {event.fingerprint}" diff --git a/src/syncfield/health/types.py b/src/syncfield/health/types.py new file mode 100644 index 0000000..629ec18 --- /dev/null +++ b/src/syncfield/health/types.py @@ -0,0 +1,161 @@ +"""Data classes for the health/incident layer. + +These are plain, explicit structs — the :mod:`syncfield.health` runtime +mutates :class:`Incident` objects in-place on the worker thread. The +viewer receives immutable :class:`IncidentSnapshot`\\ s instead. +""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass, field +from typing import Any, List + +from syncfield.health.severity import Severity, max_severity +from syncfield.types import HealthEvent + + +@dataclass(frozen=True) +class WriterStats: + """One observation of a per-stream writer's queue.""" + + stream_id: str + at_ns: int + queue_depth: int + queue_capacity: int + dropped: int + + @property + def queue_fullness(self) -> float: + if self.queue_capacity <= 0: + return 0.0 + return self.queue_depth / self.queue_capacity + + +@dataclass(frozen=True) +class IncidentArtifact: + """A piece of evidence attached to an Incident (crash dump, log excerpt, ...).""" + + kind: str + path: str + detail: str | None = None + + def to_dict(self) -> dict[str, Any]: + return {"kind": self.kind, "path": self.path, "detail": self.detail} + + +@dataclass +class Incident: + """A grouped, open/close-tracked sequence of HealthEvents sharing a fingerprint. + + Mutable because the worker thread updates ``last_event`` / ``event_count`` + / ``severity`` on every matching event. The viewer never sees this + class directly — it reads :class:`IncidentSnapshot` instead. + """ + + id: str + stream_id: str + fingerprint: str + title: str + severity: Severity + source: str + opened_at_ns: int + closed_at_ns: int | None + last_event_at_ns: int + event_count: int + first_event: HealthEvent + last_event: HealthEvent + artifacts: List[IncidentArtifact] = field(default_factory=list) + data: dict[str, Any] = field(default_factory=dict) + + @classmethod + def opened_from(cls, event: HealthEvent, *, title: str) -> "Incident": + return cls( + id=uuid.uuid4().hex, + stream_id=event.stream_id, + fingerprint=event.fingerprint, + title=title, + severity=event.severity, + source=event.source, + opened_at_ns=event.at_ns, + closed_at_ns=None, + last_event_at_ns=event.at_ns, + event_count=1, + first_event=event, + last_event=event, + ) + + @property + def is_open(self) -> bool: + return self.closed_at_ns is None + + def record_event(self, event: HealthEvent) -> None: + self.event_count += 1 + self.last_event = event + self.last_event_at_ns = event.at_ns + self.severity = max_severity(self.severity, event.severity) + + def close(self, *, at_ns: int) -> None: + self.closed_at_ns = at_ns + + def attach(self, artifact: IncidentArtifact) -> None: + self.artifacts.append(artifact) + + def to_dict(self) -> dict[str, Any]: + return { + "id": self.id, + "stream_id": self.stream_id, + "fingerprint": self.fingerprint, + "title": self.title, + "severity": self.severity.value, + "source": self.source, + "opened_at_ns": self.opened_at_ns, + "closed_at_ns": self.closed_at_ns, + "last_event_at_ns": self.last_event_at_ns, + "event_count": self.event_count, + "first_event": self.first_event.to_dict(), + "last_event": self.last_event.to_dict(), + "artifacts": [a.to_dict() for a in self.artifacts], + "data": dict(self.data), + } + + +@dataclass(frozen=True) +class IncidentSnapshot: + """Read-only view of an Incident, for the viewer's WebSocket payload.""" + + id: str + stream_id: str + fingerprint: str + title: str + severity: str + source: str + opened_at_ns: int + closed_at_ns: int | None + event_count: int + detail: str | None + ago_s: float + artifacts: List[dict[str, Any]] + + @property + def is_open(self) -> bool: + return self.closed_at_ns is None + + @classmethod + def from_incident(cls, inc: Incident, *, now_ns: int) -> "IncidentSnapshot": + anchor = inc.closed_at_ns if inc.closed_at_ns is not None else inc.last_event_at_ns + ago_s = max(0.0, (now_ns - anchor) / 1e9) + return cls( + id=inc.id, + stream_id=inc.stream_id, + fingerprint=inc.fingerprint, + title=inc.title, + severity=inc.severity.value, + source=inc.source, + opened_at_ns=inc.opened_at_ns, + closed_at_ns=inc.closed_at_ns, + event_count=inc.event_count, + detail=inc.last_event.detail, + ago_s=ago_s, + artifacts=[a.to_dict() for a in inc.artifacts], + ) diff --git a/src/syncfield/health/worker.py b/src/syncfield/health/worker.py new file mode 100644 index 0000000..d58c62a --- /dev/null +++ b/src/syncfield/health/worker.py @@ -0,0 +1,181 @@ +"""HealthWorker — the dedicated thread that drives detectors + tracker. + +Capture threads push samples / health events / state transitions / +writer stats into :class:`queue.SimpleQueue`\\ s. The worker drains them +every tick, fans out to each registered Detector, runs each Detector's +``tick`` to emit synthetic events, and feeds everything into the +IncidentTracker. +""" + +from __future__ import annotations + +import queue +import threading +import time +from collections.abc import Iterable +from dataclasses import dataclass + +from syncfield.health.detector import Detector +from syncfield.health.tracker import IncidentTracker +from syncfield.health.types import WriterStats +from syncfield.types import HealthEvent, SampleEvent, SessionState + + +@dataclass(frozen=True) +class _SampleMsg: + stream_id: str + sample: SampleEvent + + +@dataclass(frozen=True) +class _HealthMsg: + stream_id: str + event: HealthEvent + + +@dataclass(frozen=True) +class _StateMsg: + old: SessionState + new: SessionState + + +@dataclass(frozen=True) +class _WriterStatsMsg: + stream_id: str + stats: WriterStats + + +@dataclass(frozen=True) +class _ConnectionStateMsg: + stream_id: str + new_state: str + at_ns: int + + +class HealthWorker: + def __init__( + self, + *, + tracker: IncidentTracker, + detectors: Iterable[Detector], + tick_hz: float = 20.0, + ) -> None: + self._tracker = tracker + self._detectors: list[Detector] = list(detectors) + self._tick_interval = 1.0 / tick_hz + + # SimpleQueue is unbounded. Producer rate is bounded in practice by + # hardware frame rate (tens of Hz) and the worker drains at tick_hz + # (default 20 Hz) plus post-stop. No back-pressure is needed for the + # intended load; if that changes, swap to queue.Queue with maxsize. + self._samples: "queue.SimpleQueue[_SampleMsg]" = queue.SimpleQueue() + self._healths: "queue.SimpleQueue[_HealthMsg]" = queue.SimpleQueue() + self._states: "queue.SimpleQueue[_StateMsg]" = queue.SimpleQueue() + self._writer_stats: "queue.SimpleQueue[_WriterStatsMsg]" = queue.SimpleQueue() + self._connection_states: "queue.SimpleQueue[_ConnectionStateMsg]" = queue.SimpleQueue() + + self._stop = threading.Event() + self._thread: threading.Thread | None = None + + # --- ingress (called from capture threads) --------------------------- + + def push_sample(self, stream_id: str, sample: SampleEvent) -> None: + self._samples.put(_SampleMsg(stream_id, sample)) + + def push_health(self, stream_id: str, event: HealthEvent) -> None: + self._healths.put(_HealthMsg(stream_id, event)) + + def push_state(self, old: SessionState, new: SessionState) -> None: + self._states.put(_StateMsg(old, new)) + + def push_writer_stats(self, stream_id: str, stats: WriterStats) -> None: + self._writer_stats.put(_WriterStatsMsg(stream_id, stats)) + + def push_connection_state(self, stream_id: str, new_state: str, at_ns: int) -> None: + self._connection_states.put(_ConnectionStateMsg(stream_id, new_state, at_ns)) + + # --- lifecycle ------------------------------------------------------- + + def start(self) -> None: + if self._thread is not None and self._thread.is_alive(): + return + self._stop.clear() + self._thread = threading.Thread( + target=self._run, name="syncfield-health", daemon=True + ) + self._thread.start() + + def stop(self) -> None: + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=2.0) + self._thread = None + + # --- main loop ------------------------------------------------------- + + def _run(self) -> None: + next_deadline = time.monotonic() + while not self._stop.is_set(): + self._drain_once() + self._fire_detector_ticks() + self._tracker.tick(now_ns=time.monotonic_ns()) + + next_deadline += self._tick_interval + now = time.monotonic() + sleep_for = next_deadline - now + if sleep_for > 0: + # Event.wait lets stop() cut short the sleep. + self._stop.wait(timeout=sleep_for) + else: + # Running behind; reset anchor to the clock reading we just took. + next_deadline = now + + # Drain any stragglers so post-stop state is consistent. We deliberately + # do NOT call tracker.tick() here — incidents still open at stop time + # are resolved by SessionOrchestrator via IncidentTracker.close_all(), + # not by one last opportunistic close_condition pass. + self._drain_once() + + def _drain_once(self) -> None: + for msg in _drain_queue(self._samples): + for d in self._detectors: + d.observe_sample(msg.stream_id, msg.sample) + for msg in _drain_queue(self._healths): + for d in self._detectors: + d.observe_health(msg.stream_id, msg.event) + self._safe_ingest(msg.event) + for msg in _drain_queue(self._states): + for d in self._detectors: + d.observe_state(msg.old, msg.new) + for msg in _drain_queue(self._writer_stats): + for d in self._detectors: + d.observe_writer_stats(msg.stream_id, msg.stats) + for msg in _drain_queue(self._connection_states): + for d in self._detectors: + d.observe_connection_state(msg.stream_id, msg.new_state, msg.at_ns) + + def _fire_detector_ticks(self) -> None: + now = time.monotonic_ns() + for d in self._detectors: + for event in d.tick(now): + self._safe_ingest(event) + + def _safe_ingest(self, event: HealthEvent) -> None: + """Ingest but never let a malformed event crash the worker thread.""" + try: + self._tracker.ingest(event) + except Exception as exc: # noqa: BLE001 — telemetry must not crash + import logging + logging.getLogger(__name__).warning( + "IncidentTracker.ingest dropped event: %s (fingerprint=%r, source=%r)", + exc, event.fingerprint, event.source, + ) + + +def _drain_queue(q: "queue.SimpleQueue") -> list: + out = [] + while True: + try: + out.append(q.get_nowait()) + except queue.Empty: + return out diff --git a/src/syncfield/orchestrator.py b/src/syncfield/orchestrator.py index d4bfd5a..5378565 100644 --- a/src/syncfield/orchestrator.py +++ b/src/syncfield/orchestrator.py @@ -80,6 +80,7 @@ from typing import Any, Callable, Dict, List, Optional, Union from syncfield.clock import SessionClock +from syncfield.health.severity import Severity from syncfield.multihost.advertiser import SessionAdvertiser from syncfield.multihost.browser import SessionBrowser from syncfield.multihost.types import SessionAnnouncement @@ -359,6 +360,13 @@ def __init__( # opened a device. self._connected_streams: List[Stream] = [] + # Per-stream connection state for partial-connect semantics. + # Keys are stream ids; values are one of: + # "idle" | "connecting" | "connected" | "failed" | "disconnected". + self._stream_states: dict[str, str] = {} + # Populated only when a stream's connect() raised. + self._stream_errors: dict[str, str] = {} + # Auto-injected host audio stream (if any). Tracked so it can # be removed on disconnect. self._auto_audio_stream: Optional[Stream] = None @@ -392,6 +400,17 @@ def __init__( self._sample_writers: Dict[str, SampleWriter] = {} self._sample_handler_active: Dict[str, List[bool]] = {} + # Health telemetry — constructed once per orchestrator instance, + # started/stopped alongside each recording cycle. + from syncfield.health import HealthSystem + self.health = HealthSystem() + self.health.on_incident_opened(self._persist_incident) + self.health.on_incident_updated(self._persist_incident) + self.health.on_incident_closed(self._persist_incident) + + # Throttle tracker for writer stats emission — keyed by stream_id. + self._last_writer_stats_emit_at: Dict[str, int] = {} + # Multi-host: start control plane + advertiser (or follower browser) # at construction time so cluster discovery is independent of device # lifecycle. Failures here raise out of __init__ (the constructor is @@ -1307,7 +1326,7 @@ def _rollback_after_distribute_failure(self) -> None: # 1. Stop each stream that's currently recording. Must happen # BEFORE closing writers so any in-flight samples are # flushed before the file handles go away. - recording_streams = list(self._streams.values()) + recording_streams = list(self._connected_streams) _rollback_stop_recording(recording_streams) # 2. Close sample writers — matches the happy stop() flow so @@ -1502,6 +1521,11 @@ def add(self, stream: Stream) -> None: self._streams[stream.id] = stream stream.on_health(self._on_stream_health) + stream.on_sample(lambda s, _sid=stream.id: ( + self.health.observe_sample(_sid, s), + self._emit_writer_stats(_sid), + )) + self.health.register_stream(stream.id, stream.capabilities.target_hz) # After the first non-audio stream is registered, check whether # to pre-register a host audio stream so it appears in the @@ -1509,6 +1533,8 @@ def add(self, stream: Stream) -> None: if self._auto_audio_stream is None and not stream.capabilities.provides_audio_track: self._maybe_preregister_host_audio() + self._set_stream_state(stream.id, "idle") + def remove(self, stream_id: str) -> None: """Unregister a previously added stream. @@ -1603,28 +1629,54 @@ def connect(self) -> None: self._transition(SessionState.CONNECTING) + # Start the health worker early so events emitted during the + # connect loop (including per-stream failure events) reach + # registered detectors. Idempotent — safe to call again below. + self.health.start() + connected: List[Stream] = [] - try: - for stream in self._streams.values(): + for stream in self._streams.values(): + self._set_stream_state(stream.id, "connecting") + try: stream.prepare() stream.connect() - connected.append(stream) - # Emit a health event so the viewer's Health Events - # panel confirms each device connected successfully. + except Exception as exc: + self._stream_errors[stream.id] = str(exc) + self._set_stream_state(stream.id, "failed") stream._emit_health(HealthEvent( stream_id=stream.id, - kind=HealthEventKind.HEARTBEAT, + kind=HealthEventKind.ERROR, at_ns=time.monotonic_ns(), - detail="connected", + detail=str(exc), + severity=Severity.ERROR, + source="orchestrator", + fingerprint=f"{stream.id}:startup-failure", + data={"phase": "connect", "outcome": "error", "error": str(exc)}, )) - except Exception as exc: - self._log_rollback(exc, len(connected)) - _rollback_disconnect_streams(connected) + continue + connected.append(stream) + self._stream_errors.pop(stream.id, None) + self._set_stream_state(stream.id, "connected") + stream._emit_health(HealthEvent( + stream_id=stream.id, + kind=HealthEventKind.HEARTBEAT, + at_ns=time.monotonic_ns(), + detail="connected", + severity=Severity.INFO, + source="orchestrator", + fingerprint=f"{stream.id}:startup-success", + data={"phase": "connect", "outcome": "success"}, + )) + + if not connected: self._transition(SessionState.IDLE) if self._log_writer is not None: self._log_writer.close() self._log_writer = None - raise + raise RuntimeError( + "connect() failed: no streams connected — every adapter raised. " + "Inspect per-stream errors via session._stream_errors." + ) self._connected_streams = connected @@ -1633,6 +1685,10 @@ def connect(self) -> None: # user having to add an audio stream manually. self._maybe_inject_host_audio() + # Start the health worker so samples flowing from stream.connect() + # onward are observed. Idempotent — safe to call again in start(). + self.health.start() + self._transition(SessionState.CONNECTED) # ------------------------------------------------------------------ @@ -1810,7 +1866,7 @@ def _tick_with_beep(n: int) -> None: recording: List[Stream] = [] try: - for stream in self._streams.values(): + for stream in self._connected_streams: stream.start_recording(self._session_clock) recording.append(stream) except Exception as exc: @@ -1848,6 +1904,7 @@ def _tick_with_beep(n: int) -> None: # the recorded audio track, so we wait until every stream # has enabled file writing before playing it. self._maybe_play_start_chirp() + self.health.start() self._transition(SessionState.RECORDING) # Distribute the leader's SessionConfig to every preparing @@ -1918,6 +1975,20 @@ def stop(self) -> SessionReport: finalizations = self._finalize_streams() + # Stop the health worker and close any still-open incidents. + # Must run AFTER finalize_streams (so stall detectors see the + # stream stop) but BEFORE closing _log_writer (so + # _persist_incident can still flush to incidents.jsonl). + self.health.stop() + all_incidents = list(self.health.open_incidents()) + list( + self.health.resolved_incidents() + ) + by_stream: Dict[str, list] = {} + for inc in all_incidents: + by_stream.setdefault(inc.stream_id, []).append(inc) + for report in finalizations: + report.incidents = by_stream.get(report.stream_id, []) + # Leader: flip advert status to stopped BEFORE closing the # advertiser so every follower on the network observes the # transition. Close happens further down after artifacts @@ -2071,6 +2142,14 @@ def disconnect(self) -> None: _rollback_disconnect_streams(self._connected_streams) self._connected_streams = [] + # Flip every tracked stream's state to 'disconnected' so the + # viewer reflects a clean slate — this includes streams that + # were 'failed' (they never opened hardware, but their UI tile + # should no longer display a red error overlay after teardown). + for stream_id in list(self._stream_states.keys()): + self._set_stream_state(stream_id, "disconnected") + self._stream_errors.clear() + # Keep auto-injected audio stream registered (visible in viewer) # but disconnected. It will be reconnected on next connect(). @@ -2078,6 +2157,9 @@ def disconnect(self) -> None: # stays up across disconnect(). It was brought up at __init__ # and is only torn down by shutdown() or the atexit handler. + # Stop health worker — no-op if already stopped by stop(). + self.health.stop(close_open_incidents=False) + self._transition(SessionState.IDLE) if self._log_writer is not None: self._log_writer.close() @@ -2092,7 +2174,7 @@ def _finalize_streams(self) -> List[FinalizationReport]: before moving on to the next. """ finalizations: List[FinalizationReport] = [] - for stream in self._streams.values(): + for stream in self._connected_streams: try: report = stream.stop_recording() except Exception as exc: @@ -2139,7 +2221,7 @@ def _open_sample_writers(self) -> None: capture thread become no-ops instead of writing to a closed writer. """ - for stream in self._streams.values(): + for stream in self._connected_streams: writer: SampleWriter if stream.kind == "sensor": writer = SensorWriter(stream.id, self._output_dir) @@ -2328,6 +2410,7 @@ def _transition(self, new_state: SessionState) -> None: """ old = self._state self._state = new_state + self.health.observe_state(old, new_state) if self._log_writer is not None: self._log_writer.log_event( { @@ -2361,6 +2444,57 @@ def _on_stream_health(self, event: HealthEvent) -> None: """ if self._log_writer is not None: self._log_writer.log_health(event) + self.health.observe_health(event.stream_id, event) + + def _emit_writer_stats(self, stream_id: str) -> None: + """Emit WriterStats to the health system at most ~10 Hz per stream.""" + import time as _t + from syncfield.health.types import WriterStats + now = _t.monotonic_ns() + last = self._last_writer_stats_emit_at.get(stream_id, 0) + if now - last < 100_000_000: # 100 ms throttle + return + self._last_writer_stats_emit_at[stream_id] = now + self.health.observe_writer_stats( + stream_id, + WriterStats( + stream_id=stream_id, at_ns=now, + queue_depth=0, queue_capacity=1, dropped=0, + ), + ) + + def _persist_incident(self, incident) -> None: + """Forward incidents to the session log writer + attach crash dumps. + + Called from the HealthWorker thread — must not raise or the worker dies. + """ + # Attach crash_dump artifacts for device-crash incidents (idempotent). + if incident.fingerprint.endswith(":device-crash"): + path = ( + incident.first_event.data.get("crash_dump_path") + if incident.first_event.data + else None + ) + if path and not any(a.kind == "crash_dump" for a in incident.artifacts): + from syncfield.health.types import IncidentArtifact + incident.attach(IncidentArtifact(kind="crash_dump", path=str(path))) + + if self._log_writer is not None: + try: + self._log_writer.log_incident(incident) + except Exception: + # Never let telemetry persistence crash the recording. + pass + + def _set_stream_state(self, stream_id: str, new_state: str) -> None: + """Update per-stream connection state and forward to HealthSystem. + + The health worker may not be running yet (e.g. this is called + from add() when the session is IDLE) — observe_connection_state + is a no-op in that case. + """ + self._stream_states[stream_id] = new_state + self.health.observe_connection_state(stream_id, new_state, time.monotonic_ns()) # ------------------------------------------------------------------ # Episode lifecycle diff --git a/src/syncfield/testing.py b/src/syncfield/testing.py index c4650cd..914be36 100644 --- a/src/syncfield/testing.py +++ b/src/syncfield/testing.py @@ -20,6 +20,18 @@ ) +def _severity_for(kind: HealthEventKind): + from syncfield.health.severity import Severity + mapping = { + HealthEventKind.HEARTBEAT: Severity.INFO, + HealthEventKind.RECONNECT: Severity.INFO, + HealthEventKind.DROP: Severity.WARNING, + HealthEventKind.WARNING: Severity.WARNING, + HealthEventKind.ERROR: Severity.ERROR, + } + return mapping.get(kind, Severity.WARNING) + + class FakeStream(StreamBase): """Programmable in-memory :class:`~syncfield.stream.Stream` used by tests. @@ -73,6 +85,10 @@ def prepare(self) -> None: if self._fail_on_prepare: raise RuntimeError("fake failure in prepare") + def connect(self) -> None: + if self._fail_on_start: + raise RuntimeError("fake failure in connect") + def start(self, session_clock: SessionClock) -> None: self.start_calls += 1 if self._fail_on_start: @@ -110,4 +126,12 @@ def push_health( detail: Optional[str] = None, ) -> None: """Emit a synthetic health event through the callback path.""" - self._emit_health(HealthEvent(self.id, kind, at_ns, detail)) + self._emit_health(HealthEvent( + stream_id=self.id, + kind=kind, + at_ns=at_ns, + detail=detail, + severity=_severity_for(kind), + source="adapter:test", + fingerprint=f"{self.id}:adapter:{kind.value}", + )) diff --git a/src/syncfield/types.py b/src/syncfield/types.py index 80747cf..e97c43c 100644 --- a/src/syncfield/types.py +++ b/src/syncfield/types.py @@ -14,6 +14,8 @@ from pathlib import Path from typing import Any, Literal, Union +from syncfield.health.severity import Severity + # Sensor channel value type. # Leaf values are always numeric (float | int). # Structure can be nested dicts or lists. @@ -194,6 +196,7 @@ class StreamCapabilities: supports_precise_timestamps: bool = False is_removable: bool = False produces_file: bool = False + target_hz: float | None = None live_preview: bool = True def to_dict(self) -> dict[str, Any]: @@ -202,6 +205,7 @@ def to_dict(self) -> dict[str, Any]: "supports_precise_timestamps": self.supports_precise_timestamps, "is_removable": self.is_removable, "produces_file": self.produces_file, + "target_hz": self.target_hz, "live_preview": self.live_preview, } @@ -249,19 +253,23 @@ class HealthEventKind(Enum): @dataclass(frozen=True) class HealthEvent: - """A stream reports a health observation to the orchestrator. + """A stream reports a health observation. - Attributes: - stream_id: Stream that emitted the event. - kind: Category of the event. - at_ns: ``time.monotonic_ns()`` when the event was observed. - detail: Optional free-form description. + ``severity`` / ``source`` / ``fingerprint`` / ``data`` enable the + incident-tracking layer in :mod:`syncfield.health` to group many raw + events into a single Sentry-style Incident. Adapters that don't care + can leave them at their safe defaults; the platform will fill them + in before the event reaches the IncidentTracker. """ stream_id: str kind: HealthEventKind at_ns: int detail: str | None = None + severity: Severity = Severity.INFO + source: str = "unknown" + fingerprint: str = "" + data: dict[str, Any] = field(default_factory=dict) def to_dict(self) -> dict[str, Any]: return { @@ -269,6 +277,10 @@ def to_dict(self) -> dict[str, Any]: "kind": self.kind.value, "at_ns": self.at_ns, "detail": self.detail, + "severity": self.severity.value, + "source": self.source, + "fingerprint": self.fingerprint, + "data": dict(self.data), } @@ -326,6 +338,7 @@ class FinalizationReport: error: str | None jitter_p95_ns: int | None = None jitter_p99_ns: int | None = None + incidents: list = field(default_factory=list) @dataclass(frozen=True) diff --git a/src/syncfield/viewer/frontend/src/App.tsx b/src/syncfield/viewer/frontend/src/App.tsx index 232389c..0db4b30 100644 --- a/src/syncfield/viewer/frontend/src/App.tsx +++ b/src/syncfield/viewer/frontend/src/App.tsx @@ -7,7 +7,7 @@ import { Header } from "@/components/header"; import { ControlPanel } from "@/components/control-panel"; import { SessionClock } from "@/components/session-clock"; import { StreamCard } from "@/components/stream-card"; -import { HealthTable } from "@/components/health-table"; +import { IncidentPanel } from "@/components/incident-panel"; import { CountdownOverlay } from "@/components/countdown-overlay"; import { DiscoveryModal } from "@/components/discovery-modal"; import { ClusterPanel } from "@/components/cluster-panel"; @@ -169,6 +169,7 @@ function RecordView({ stream={stream} canRemove={canRemove} onRemove={handleRemoveStream} + activeIncidents={snapshot?.active_incidents ?? []} sessionState={state} aggregation={snapshot?.aggregation} onRetryAggregation={(jobId) => @@ -194,12 +195,12 @@ function RecordView({
{cluster.available && } {streamList.length > 0 && ( - <> -
-

Health Events

-
- - +
+ +
)}
)} diff --git a/src/syncfield/viewer/frontend/src/components/header.tsx b/src/syncfield/viewer/frontend/src/components/header.tsx index 11ef09b..c2e4293 100644 --- a/src/syncfield/viewer/frontend/src/components/header.tsx +++ b/src/syncfield/viewer/frontend/src/components/header.tsx @@ -61,6 +61,13 @@ export function Header({ const elapsed = snapshot?.elapsed_s ?? 0; const isRecording = state === "recording"; + const streams = Object.values(snapshot?.streams ?? {}); + const total = streams.length; + const connected = streams.filter((s) => s.connection_state === "connected").length; + const showCount = total > 0 && connected < total; + const stateLabel = friendlyState(state); + const displayedLabel = showCount ? `${stateLabel} (${connected}/${total})` : stateLabel; + return (
- {friendlyState(state)} + {displayedLabel} diff --git a/src/syncfield/viewer/frontend/src/components/health-table.tsx b/src/syncfield/viewer/frontend/src/components/health-table.tsx deleted file mode 100644 index af96162..0000000 --- a/src/syncfield/viewer/frontend/src/components/health-table.tsx +++ /dev/null @@ -1,79 +0,0 @@ -import type { HealthEntry } from "@/lib/types"; -import { cn } from "@/lib/utils"; - -interface HealthTableProps { - entries: HealthEntry[]; -} - -const KIND_ICONS: Record = { - heartbeat: "●", - warning: "⚠", - error: "✗", - drop: "↓", - reconnect: "↻", -}; - -const KIND_COLORS: Record = { - error: "text-destructive", - warning: "text-warning", - drop: "text-destructive", - reconnect: "text-success", - heartbeat: "text-success", -}; - -/** - * Compact health event timeline for the sidebar. - */ -export function HealthTable({ entries }: HealthTableProps) { - if (entries.length === 0) { - return ( -
- No events yet -
- ); - } - - const sorted = [...entries].reverse(); - - return ( -
    - {sorted.map((entry, i) => ( -
  • - {/* Icon */} - - {KIND_ICONS[entry.kind] ?? "·"} - - - {/* Content */} -
    -
    - - {entry.stream_id} - - - {formatAgo(entry.ago_s)} - -
    - {entry.detail && ( -
    - {entry.detail} -
    - )} -
    -
    • - ))} -
- ); -} - -function formatAgo(seconds: number): string { - if (seconds < 1) return "just now"; - if (seconds < 60) return `${Math.round(seconds)}s ago`; - if (seconds < 3600) return `${Math.round(seconds / 60)}m ago`; - return `${Math.round(seconds / 3600)}h ago`; -} diff --git a/src/syncfield/viewer/frontend/src/components/incident-panel.tsx b/src/syncfield/viewer/frontend/src/components/incident-panel.tsx new file mode 100644 index 0000000..96e1a63 --- /dev/null +++ b/src/syncfield/viewer/frontend/src/components/incident-panel.tsx @@ -0,0 +1,97 @@ +import { useState } from "react"; +import type { IncidentSnapshot, Severity } from "../lib/types"; + +const SEVERITY_ICON: Record = { + info: "·", + warning: "⚠", + error: "⛔", + critical: "⛔", +}; + +const SEVERITY_COLOR: Record = { + info: "text-slate-400", + warning: "text-yellow-400", + error: "text-orange-400", + critical: "text-red-500", +}; + +function formatAgo(s: number): string { + if (s < 60) return `${Math.round(s)}s ago`; + if (s < 3600) return `${Math.round(s / 60)}m ago`; + return `${Math.round(s / 3600)}h ago`; +} + +function IncidentCard({ inc, isOpen }: { inc: IncidentSnapshot; isOpen: boolean }) { + const [expanded, setExpanded] = useState(false); + return ( + + ); +} + +export function IncidentPanel({ + active, + resolved, +}: { + active: IncidentSnapshot[]; + resolved: IncidentSnapshot[]; +}) { + return ( +
+
+ Active Issues ({active.length}) +
+ {active.length === 0 ? ( +
None — all clear.
+ ) : ( +
+ {active.map((inc) => ( + + ))} +
+ )} +
+ Resolved this session ({resolved.length}) +
+ {resolved.length === 0 ? ( +
None.
+ ) : ( + resolved.map((inc) => ) + )} +
+ ); +} diff --git a/src/syncfield/viewer/frontend/src/components/stream-card.tsx b/src/syncfield/viewer/frontend/src/components/stream-card.tsx index 9e267a5..4b1fdb2 100644 --- a/src/syncfield/viewer/frontend/src/components/stream-card.tsx +++ b/src/syncfield/viewer/frontend/src/components/stream-card.tsx @@ -1,4 +1,4 @@ -import type { AggregationSnapshotWS, StreamSnapshot } from "@/lib/types"; +import type { AggregationSnapshotWS, IncidentSnapshot, Severity, StreamSnapshot } from "@/lib/types"; import { formatCount, formatHz } from "@/lib/format"; import { cn } from "@/lib/utils"; import { AudioLevelChart } from "./audio-level-chart"; @@ -8,11 +8,18 @@ import { StandaloneRecorderPanel, type StandaloneRecorderStream, } from "./standalone-recorder-panel"; +import { + ConnectingOverlay, + WaitingForDataOverlay, + FailedOverlay, +} from "./stream-overlays"; interface StreamCardProps { stream: StreamSnapshot; canRemove: boolean; onRemove: (streamId: string) => void; + /** Active incidents from the session snapshot — used to derive per-stream severity badge. */ + activeIncidents?: IncidentSnapshot[]; /** Session state string — forwarded to StandaloneRecorderPanel for recording detection. */ sessionState?: string; /** Top-level aggregation snapshot from the WS payload — used by StandaloneRecorderPanel. */ @@ -21,6 +28,30 @@ interface StreamCardProps { onRetryAggregation?: (jobId: string) => void; } +// --------------------------------------------------------------------------- +// Per-stream incident helpers +// --------------------------------------------------------------------------- + +const SEVERITY_ORDER: Severity[] = ["info", "warning", "error", "critical"]; +const BADGE_COLOR: Record = { + info: "bg-slate-500", + warning: "bg-yellow-500", + error: "bg-orange-500", + critical: "bg-red-500", +}; + +function streamIncidentStats(streamId: string, active: IncidentSnapshot[]) { + const mine = active.filter((i) => i.stream_id === streamId); + const count = mine.length; + let highest: Severity | null = null; + for (const i of mine) { + if (highest === null || SEVERITY_ORDER.indexOf(i.severity) > SEVERITY_ORDER.indexOf(highest)) { + highest = i.severity; + } + } + return { count, highest }; +} + /** * Per-stream card with variant body by kind. * @@ -35,10 +66,12 @@ export function StreamCard({ stream, canRemove, onRemove, + activeIncidents = [], sessionState, aggregation, onRetryAggregation, }: StreamCardProps) { + const { count: incidentCount, highest: incidentSeverity } = streamIncidentStats(stream.id, activeIncidents); // Dispatch to StandaloneRecorderPanel for video streams without live preview // (e.g. Insta360 Go3S which downloads files via BLE/Wi-Fi after recording). const isStandalone = @@ -66,6 +99,13 @@ export function StreamCard({ {stream.id} + {incidentCount > 0 && incidentSeverity && ( + + {incidentCount} + + )}
{canRemove && (
); @@ -136,6 +168,13 @@ export function StreamCard({ {stream.id} + {incidentCount > 0 && incidentSeverity && ( + + {incidentCount} + + )}
{canRemove && (
- {/* Body — varies by stream kind */} + {/* Body — varies by connection state, then stream kind */}
- {stream.kind === "video" ? ( - - ) : stream.kind === "audio" ? ( - - ) : stream.kind === "sensor" ? ( - - ) : ( -
- No preview -
- )} +
{/* Footer stats */} @@ -182,19 +211,40 @@ export function StreamCard({ {formatCount(stream.frame_count)} {formatHz(stream.effective_hz)} - {stream.problem_count > 0 && ( - <> - - - {stream.problem_count} issue{stream.problem_count > 1 ? "s" : ""} - - - )} ); } +// --------------------------------------------------------------------------- +// StreamCardBody — branches on connection_state before kind-based rendering +// --------------------------------------------------------------------------- + +function StreamCardBody({ stream }: { stream: StreamSnapshot }) { + if (stream.connection_state === "connecting") { + return ; + } + if (stream.connection_state === "failed") { + return ; + } + if ( + stream.connection_state === "connected" && + stream.kind === "video" && + stream.frame_count === 0 + ) { + return ; + } + // Healthy / idle / disconnected — fall through to kind-based rendering. + if (stream.kind === "video") return ; + if (stream.kind === "audio") return ; + if (stream.kind === "sensor") return ; + return ( +
+ No preview +
+ ); +} + // --------------------------------------------------------------------------- // Helper — pick the aggregation job relevant to a specific stream // --------------------------------------------------------------------------- diff --git a/src/syncfield/viewer/frontend/src/components/stream-overlays.tsx b/src/syncfield/viewer/frontend/src/components/stream-overlays.tsx new file mode 100644 index 0000000..8e0f4c9 --- /dev/null +++ b/src/syncfield/viewer/frontend/src/components/stream-overlays.tsx @@ -0,0 +1,52 @@ +import { useState } from "react"; + +export function ConnectingOverlay() { + return ( +
+
+ + Connecting… +
+
+ ); +} + +export function WaitingForDataOverlay() { + return ( +
+
+ Connected · waiting for first frame +
+
+ ); +} + +export function FailedOverlay({ error }: { error: string }) { + const [expanded, setExpanded] = useState(false); + return ( + + ); +} diff --git a/src/syncfield/viewer/frontend/src/lib/types.ts b/src/syncfield/viewer/frontend/src/lib/types.ts index 556d64b..a965808 100644 --- a/src/syncfield/viewer/frontend/src/lib/types.ts +++ b/src/syncfield/viewer/frontend/src/lib/types.ts @@ -19,11 +19,10 @@ export interface StreamSnapshot { last_sample_ms_ago: number | null; provides_audio_track: boolean; produces_file: boolean; - health_count: number; - /** Count of non-heartbeat events (warnings/errors/drops). */ - problem_count: number; /** Stream capabilities declared by the adapter. May be absent on older servers. */ capabilities?: StreamCapabilities; + connection_state: ConnectionState; + connection_error: string | null; } export interface ChirpInfo { @@ -32,13 +31,36 @@ export interface ChirpInfo { stop_ns: number | null; } -export interface HealthEntry { - stream_id: string; +export type Severity = "info" | "warning" | "error" | "critical"; + +export type ConnectionState = + | "idle" + | "connecting" + | "connected" + | "failed" + | "disconnected"; + +export interface IncidentArtifact { kind: string; - ago_s: number; + path: string; detail: string | null; } +export interface IncidentSnapshot { + id: string; + stream_id: string; + fingerprint: string; + title: string; + severity: Severity; + source: string; + opened_at_ns: number; + closed_at_ns: number | null; + event_count: number; + detail: string | null; + ago_s: number; + artifacts: IncidentArtifact[]; +} + // --------------------------------------------------------------------------- // Aggregation types (Insta360 Go3S) // --------------------------------------------------------------------------- @@ -74,7 +96,8 @@ export interface SessionSnapshot { elapsed_s: number; chirp: ChirpInfo; streams: Record; - health_log: HealthEntry[]; + active_incidents: IncidentSnapshot[]; + resolved_incidents: IncidentSnapshot[]; output_dir: string; /** Aggregation state for Go3S streams; present when a Go3S adapter is active. */ aggregation?: AggregationSnapshotWS; diff --git a/src/syncfield/viewer/poller.py b/src/syncfield/viewer/poller.py index 0a7b116..6553094 100644 --- a/src/syncfield/viewer/poller.py +++ b/src/syncfield/viewer/poller.py @@ -16,16 +16,17 @@ from __future__ import annotations import threading -import time +import time as _time +from collections import deque from pathlib import Path -from typing import Dict, List, Optional +from typing import Dict, Optional +from syncfield.health.types import Incident, IncidentSnapshot from syncfield.orchestrator import SessionOrchestrator from syncfield.stream import Stream -from syncfield.types import HealthEvent, SampleEvent, SessionState +from syncfield.types import SampleEvent, SessionState from syncfield.viewer.state import ( - HealthEntry, SessionSnapshot, StreamSnapshot, StreamStatsBuffer, @@ -66,6 +67,15 @@ def __init__( self._recording_started_at: Optional[float] = None self._last_observed_state: SessionState = SessionState.IDLE + # Incident tracking — fed by HealthSystem callbacks. + self._incidents_lock = threading.Lock() + self._open_by_id: dict[str, Incident] = {} + self._resolved: deque[Incident] = deque(maxlen=20) + + session.health.on_incident_opened(self._ingest_incident) + session.health.on_incident_updated(self._ingest_incident) + session.health.on_incident_closed(self._ingest_incident) + # ------------------------------------------------------------------ # Public API # ------------------------------------------------------------------ @@ -96,7 +106,7 @@ def get_snapshot(self) -> Optional[SessionSnapshot]: # ------------------------------------------------------------------ def _register_callbacks(self) -> None: - """Attach on_sample / on_health to each registered stream. + """Attach on_sample to each registered stream. Re-entrant: callbacks for streams we've already registered for are skipped by tracking which stream ids already have a buffer. @@ -107,7 +117,6 @@ def _register_callbacks(self) -> None: buffer = StreamStatsBuffer() self._stats[stream_id] = buffer stream.on_sample(self._make_sample_callback(stream_id, buffer)) - stream.on_health(self._make_health_callback(stream_id, buffer)) @staticmethod def _make_sample_callback(stream_id: str, buffer: StreamStatsBuffer): @@ -116,19 +125,14 @@ def _on_sample(event: SampleEvent) -> None: return _on_sample - @staticmethod - def _make_health_callback(stream_id: str, buffer: StreamStatsBuffer): - def _on_health(event: HealthEvent) -> None: - buffer.observe_health( - HealthEntry( - stream_id=stream_id, - kind=event.kind.value, - at_ns=event.at_ns, - detail=event.detail, - ) - ) - - return _on_health + def _ingest_incident(self, incident: Incident) -> None: + """Called from the HealthSystem worker thread whenever an incident changes.""" + with self._incidents_lock: + if incident.is_open: + self._open_by_id[incident.id] = incident + else: + self._open_by_id.pop(incident.id, None) + self._resolved.append(incident) # ------------------------------------------------------------------ # Poll loop @@ -157,7 +161,7 @@ def _build_snapshot(self) -> SessionSnapshot: # Track the recording start time so we can compute elapsed seconds. current_state: SessionState = session.state - now = time.time() + now = _time.time() if ( current_state is SessionState.RECORDING and self._last_observed_state is not SessionState.RECORDING @@ -171,7 +175,7 @@ def _build_snapshot(self) -> SessionSnapshot: if self._recording_started_at is not None: elapsed_s = max(0.0, now - self._recording_started_at) - now_ns = time.monotonic_ns() + now_ns = _time.monotonic_ns() streams_snapshot: Dict[str, StreamSnapshot] = {} for stream_id, stream in session._streams.items(): # type: ignore[attr-defined] buffer = self._stats.get(stream_id) @@ -207,12 +211,22 @@ def _build_snapshot(self) -> SessionSnapshot: latest_frame=latest_frame, plot_points=plot_points, latest_pose=latest_pose, - health_count=len(buffer._health), live_preview=getattr(stream.capabilities, "live_preview", True), + connection_state=session._stream_states.get(stream_id, "idle"), # type: ignore[attr-defined] + connection_error=session._stream_errors.get(stream_id), # type: ignore[attr-defined] ) - # Merge health events into a session-wide, time-sorted log. - health_log = self._collect_health_log() + # Snapshot incidents — take a consistent copy under the incidents lock. + with self._incidents_lock: + now_ns = _time.monotonic_ns() + active = [ + IncidentSnapshot.from_incident(i, now_ns=now_ns) + for i in self._open_by_id.values() + ] + resolved = [ + IncidentSnapshot.from_incident(i, now_ns=now_ns) + for i in self._resolved + ] # Session-level sync point + chirp fields. sync_point = getattr(session, "_sync_point", None) @@ -233,7 +247,8 @@ def _build_snapshot(self) -> SessionSnapshot: chirp_enabled=chirp_enabled, elapsed_s=elapsed_s, streams=streams_snapshot, - health_log=health_log, + active_incidents=active, + resolved_incidents=resolved, ) @staticmethod @@ -241,12 +256,3 @@ def _safe_latest_frame(stream: Stream): """Read ``stream.latest_frame`` if the adapter exposes it.""" frame = getattr(stream, "latest_frame", None) return frame - - def _collect_health_log(self) -> List[HealthEntry]: - """Merge per-stream health deques into a time-sorted global log.""" - merged: List[HealthEntry] = [] - for buffer in self._stats.values(): - merged.extend(buffer.snapshot_health()) - merged.sort(key=lambda e: e.at_ns) - # Cap to the most recent N so a long session doesn't blow up the table. - return merged[-50:] diff --git a/src/syncfield/viewer/server.py b/src/syncfield/viewer/server.py index ffc9d02..8f3011b 100644 --- a/src/syncfield/viewer/server.py +++ b/src/syncfield/viewer/server.py @@ -37,7 +37,7 @@ from syncfield.orchestrator import SessionOrchestrator from syncfield.viewer.poller import SessionPoller -from syncfield.viewer.state import AggregationSnapshot, HealthEntry, SessionSnapshot, StreamSnapshot +from syncfield.viewer.state import AggregationSnapshot, SessionSnapshot, StreamSnapshot logger = logging.getLogger(__name__) @@ -70,14 +70,6 @@ def snapshot_to_dict(snapshot: SessionSnapshot) -> Dict[str, Any]: now_ns = time.monotonic_ns() streams: Dict[str, Any] = {} - # Count non-heartbeat events per stream (heartbeats are informational) - problem_count_by_stream: Dict[str, int] = {} - for h in snapshot.health_log: - if h.kind != "heartbeat": - problem_count_by_stream[h.stream_id] = ( - problem_count_by_stream.get(h.stream_id, 0) + 1 - ) - for sid, s in snapshot.streams.items(): last_sample_ms_ago: Optional[float] = None if s.last_sample_at_ns is not None: @@ -91,8 +83,8 @@ def snapshot_to_dict(snapshot: SessionSnapshot) -> Dict[str, Any]: "last_sample_ms_ago": last_sample_ms_ago, "provides_audio_track": s.provides_audio_track, "produces_file": s.produces_file, - "health_count": s.health_count, - "problem_count": problem_count_by_stream.get(sid, 0), + "connection_state": s.connection_state, + "connection_error": s.connection_error, "capabilities": { "live_preview": getattr(s, "live_preview", True), "provides_audio_track": s.provides_audio_track, @@ -102,16 +94,21 @@ def snapshot_to_dict(snapshot: SessionSnapshot) -> Dict[str, Any]: }, } - health_log: List[Dict[str, Any]] = [] - for h in snapshot.health_log: - # Convert monotonic ns to "seconds ago" for display - ago_s = round((now_ns - h.at_ns) / 1e9, 1) if h.at_ns else 0 - health_log.append({ - "stream_id": h.stream_id, - "kind": h.kind, - "ago_s": ago_s, - "detail": h.detail, - }) + def _serialize_incident(inc) -> Dict[str, Any]: + return { + "id": inc.id, + "stream_id": inc.stream_id, + "fingerprint": inc.fingerprint, + "title": inc.title, + "severity": inc.severity, + "source": inc.source, + "opened_at_ns": inc.opened_at_ns, + "closed_at_ns": inc.closed_at_ns, + "event_count": inc.event_count, + "detail": inc.detail, + "ago_s": round(inc.ago_s, 1), + "artifacts": inc.artifacts, + } return { "type": "snapshot", @@ -124,7 +121,8 @@ def snapshot_to_dict(snapshot: SessionSnapshot) -> Dict[str, Any]: "stop_ns": snapshot.chirp_stop_ns, }, "streams": streams, - "health_log": health_log, + "active_incidents": [_serialize_incident(i) for i in snapshot.active_incidents], + "resolved_incidents": [_serialize_incident(i) for i in snapshot.resolved_incidents], "output_dir": snapshot.output_dir, "aggregation": _serialize_aggregation(getattr(snapshot, "aggregation", None)), } diff --git a/src/syncfield/viewer/state.py b/src/syncfield/viewer/state.py index 84c6b34..e18add1 100644 --- a/src/syncfield/viewer/state.py +++ b/src/syncfield/viewer/state.py @@ -23,6 +23,8 @@ from dataclasses import dataclass, field from typing import Any, Deque, Dict, List, Optional, Tuple +from syncfield.health.types import IncidentSnapshot + # --------------------------------------------------------------------------- # Aggregation snapshot @@ -73,8 +75,6 @@ class StreamSnapshot: exposes the latest vector/list sample so panels that render a single-frame pose (3-D hand skeleton, quaternion axes…) have data to draw. Empty for streams that emit only scalars. - health_count: Number of health events this stream has buffered so - far. Useful for showing a red dot on degraded streams. """ id: str @@ -87,27 +87,9 @@ class StreamSnapshot: latest_frame: Any # numpy array or None — kept as Any so numpy is optional plot_points: Dict[str, Tuple[List[float], List[float]]] latest_pose: Dict[str, List[float]] - health_count: int live_preview: bool = True - - -# --------------------------------------------------------------------------- -# Health event -# --------------------------------------------------------------------------- - - -@dataclass(frozen=True) -class HealthEntry: - """A single health event surfaced by any stream. - - Simpler than :class:`~syncfield.types.HealthEvent` because the viewer - only needs strings for display and a monotonic ordering key. - """ - - stream_id: str - kind: str # "heartbeat" | "drop" | "reconnect" | "warning" | "error" - at_ns: int - detail: Optional[str] + connection_state: str = "idle" + connection_error: Optional[str] = None # --------------------------------------------------------------------------- @@ -131,7 +113,8 @@ class SessionSnapshot: chirp_enabled: Whether :class:`SyncToneConfig` had chirp enabled. elapsed_s: Wall-clock seconds since ``start()``, or 0 if idle. streams: Ordered map ``stream_id -> StreamSnapshot``. - health_log: Most recent health events across all streams (newest last). + active_incidents: Currently open incidents across all streams. + resolved_incidents: Recently closed incidents (newest last, capped at 20). """ host_id: str @@ -144,7 +127,8 @@ class SessionSnapshot: chirp_enabled: bool elapsed_s: float streams: Dict[str, StreamSnapshot] - health_log: List[HealthEntry] + active_incidents: List[IncidentSnapshot] = field(default_factory=list) + resolved_incidents: List[IncidentSnapshot] = field(default_factory=list) aggregation: Optional[AggregationSnapshot] = None @@ -163,7 +147,6 @@ class StreamStatsBuffer: """ max_plot_samples: int = 300 - max_health: int = 20 # Rolling fps window (monotonic ns) _fps_window: Deque[int] = field(default_factory=lambda: deque(maxlen=30)) @@ -179,9 +162,6 @@ class StreamStatsBuffer: # too large to buffer thousands of frames of. _latest_pose: Dict[str, List[float]] = field(default_factory=dict) - # Health events produced by this stream (capped) - _health: Deque[HealthEntry] = field(default_factory=lambda: deque(maxlen=20)) - def observe_sample(self, capture_ns: int, channels: Optional[Dict[str, Any]]) -> None: """Record one sample. Called from the stream's callback thread. @@ -244,9 +224,6 @@ def observe_sample(self, capture_ns: int, channels: Optional[Dict[str, Any]]) -> if name not in plottable: buf.append(float("nan")) - def observe_health(self, event: HealthEntry) -> None: - self._health.append(event) - def snapshot_fps(self, now_ns: int) -> float: """Effective Hz over the last second of samples. @@ -290,9 +267,6 @@ def snapshot_pose(self) -> Dict[str, List[float]]: """ return {name: list(values) for name, values in self._latest_pose.items()} - def snapshot_health(self) -> List[HealthEntry]: - return list(self._health) - # --------------------------------------------------------------------------- # Free helpers diff --git a/src/syncfield/writer.py b/src/syncfield/writer.py index 20a5534..8657186 100644 --- a/src/syncfield/writer.py +++ b/src/syncfield/writer.py @@ -116,21 +116,31 @@ class SessionLogWriter: crash mid-recording and the core service can reconstruct partial sessions from the file. - Output file: ``session_log.jsonl`` + Output files: + - ``session_log.jsonl`` — state transitions and health events + - ``incidents.jsonl`` — incident lifecycle events """ def __init__(self, output_dir: Path) -> None: self._path = output_dir / "session_log.jsonl" + self._incidents_path = output_dir / "incidents.jsonl" self._handle: IO[str] | None = None + self._incidents_handle: IO[str] | None = None @property def path(self) -> Path: return self._path + @property + def incidents_path(self) -> Path: + return self._incidents_path + def open(self) -> None: - """Open the log file for writing. Idempotent on an already-open writer.""" + """Open the log files for writing. Idempotent on an already-open writer.""" if self._handle is None: self._handle = open(self._path, "w") + if self._incidents_handle is None: + self._incidents_handle = open(self._incidents_path, "w") def log_event(self, event: dict[str, Any]) -> None: """Serialize *event* as a single JSON line and flush. @@ -155,10 +165,24 @@ def log_health(self, event: HealthEvent) -> None: } ) + def log_incident(self, incident: Any) -> None: + """Serialize an :class:`Incident` as a single JSON line and flush. + + Raises: + RuntimeError: If the writer has not been opened. + """ + if self._incidents_handle is None: + raise RuntimeError("SessionLogWriter is not open") + self._incidents_handle.write(json.dumps(incident.to_dict(), separators=(",", ":")) + "\n") + self._incidents_handle.flush() + def close(self) -> None: if self._handle is not None: self._handle.close() self._handle = None + if self._incidents_handle is not None: + self._incidents_handle.close() + self._incidents_handle = None def write_sync_point( diff --git a/tests/integration/health/__init__.py b/tests/integration/health/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/health/test_no_data_detector.py b/tests/integration/health/test_no_data_detector.py new file mode 100644 index 0000000..5c01af3 --- /dev/null +++ b/tests/integration/health/test_no_data_detector.py @@ -0,0 +1,85 @@ +"""End-to-end: a stream that connects but never emits a sample triggers +the no-data incident within the configured threshold.""" +from __future__ import annotations + +import threading +import time +from pathlib import Path + +import pytest + +from syncfield.orchestrator import SessionOrchestrator +from syncfield.stream import StreamBase +from syncfield.types import FinalizationReport, SampleEvent, StreamCapabilities + + +class SilentFakeStream(StreamBase): + """FakeStream variant that connects successfully but emits no samples until asked.""" + + def __init__(self, stream_id: str): + super().__init__(id=stream_id, kind="sensor", capabilities=StreamCapabilities()) + self._stop = threading.Event() + self._gate = threading.Event() + self._thread: threading.Thread | None = None + self._frame = 0 + + def connect(self): + self._stop.clear() + self._thread = threading.Thread(target=self._run, daemon=True) + self._thread.start() + + def disconnect(self): + self._stop.set() + if self._thread: + self._thread.join(timeout=1.0) + self._thread = None + + def start_recording(self, session_clock): + pass + + def stop_recording(self) -> FinalizationReport: + return FinalizationReport( + stream_id=self.id, status="completed", frame_count=self._frame, + file_path=None, first_sample_at_ns=0, last_sample_at_ns=0, + health_events=[], error=None, + ) + + def allow_samples(self): + self._gate.set() + + def _run(self): + while not self._stop.is_set(): + if self._gate.is_set(): + self._frame += 1 + self._emit_sample(SampleEvent( + stream_id=self.id, frame_number=self._frame, + capture_ns=time.monotonic_ns(), + )) + time.sleep(0.05) + + +@pytest.mark.slow +def test_no_data_incident_opens_then_closes_when_samples_arrive(tmp_path: Path): + sess = SessionOrchestrator(host_id="h", output_dir=tmp_path) + stream = SilentFakeStream("cam") + sess.add(stream) + + for d in sess.health.iter_detectors(): + if d.name == "no-data": + d._threshold_ns = int(1e9) + break + + sess.connect() + + time.sleep(1.5) + open_fps = [i.fingerprint for i in sess.health.open_incidents()] + assert "cam:no-data" in open_fps + + stream.allow_samples() + time.sleep(0.5) + + # Session was never started into RECORDING — disconnect directly from CONNECTED. + sess.disconnect() + + resolved_fps = [i.fingerprint for i in sess.health.resolved_incidents()] + assert "cam:no-data" in resolved_fps diff --git a/tests/integration/health/test_orchestrator_health_integration.py b/tests/integration/health/test_orchestrator_health_integration.py new file mode 100644 index 0000000..0792169 --- /dev/null +++ b/tests/integration/health/test_orchestrator_health_integration.py @@ -0,0 +1,170 @@ +"""Integration: real SessionOrchestrator + FakeStream → incidents flow end-to-end.""" +from __future__ import annotations + +import json +import threading +import time +from pathlib import Path + +import pytest + +from syncfield.orchestrator import SessionOrchestrator +from syncfield.stream import StreamBase +from syncfield.clock import SessionClock +from syncfield.types import FinalizationReport, SampleEvent, StreamCapabilities + + +class FakeStream(StreamBase): + def __init__(self, stream_id: str, target_hz: float | None = None): + super().__init__( + id=stream_id, + kind="sensor", + capabilities=StreamCapabilities(target_hz=target_hz), + ) + self._interval = 1.0 / 30.0 + self._stop_thread = threading.Event() + self._pause = threading.Event() + self._thread: threading.Thread | None = None + self._frame = 0 + + def connect(self): + self._stop_thread.clear() + self._thread = threading.Thread(target=self._run, daemon=True) + self._thread.start() + + def disconnect(self): + self._stop_thread.set() + if self._thread: + self._thread.join(timeout=1.0) + self._thread = None + + def start_recording(self, session_clock: SessionClock) -> None: + pass + + def stop_recording(self) -> FinalizationReport: + return FinalizationReport( + stream_id=self.id, status="completed", frame_count=self._frame, + file_path=None, first_sample_at_ns=0, last_sample_at_ns=0, + health_events=[], error=None, + ) + + def pause_samples(self): + self._pause.set() + + def resume_samples(self): + self._pause.clear() + + def _run(self): + while not self._stop_thread.is_set(): + if not self._pause.is_set(): + self._frame += 1 + self._emit_sample(SampleEvent( + stream_id=self.id, frame_number=self._frame, + capture_ns=time.monotonic_ns(), + )) + time.sleep(self._interval) + + +@pytest.mark.slow +def test_stall_incident_open_and_close(tmp_path: Path): + sess = SessionOrchestrator(host_id="test", output_dir=tmp_path) + stream = FakeStream("cam", target_hz=30.0) + sess.add(stream) + + sess.connect() + sess.start(countdown_s=0) + + # Induce stall for 3s — StreamStallDetector default threshold is 2s. + stream.pause_samples() + time.sleep(3.0) + + opens = [i for i in sess.health.open_incidents() if i.fingerprint == "cam:stream-stall"] + assert opens, "stall incident did not open" + + # Recover. + stream.resume_samples() + time.sleep(2.5) + + sess.stop() + sess.disconnect() + + resolved = [i for i in sess.health.resolved_incidents() if i.fingerprint == "cam:stream-stall"] + assert resolved, "stall incident did not resolve" + + +@pytest.mark.slow +def test_incidents_jsonl_written(tmp_path: Path): + sess = SessionOrchestrator(host_id="test", output_dir=tmp_path) + stream = FakeStream("cam", target_hz=30.0) + sess.add(stream) + + sess.connect() + sess.start(countdown_s=0) + stream.pause_samples() + time.sleep(2.5) + sess.stop() + sess.disconnect() + + # Locate incidents.jsonl — the orchestrator places it inside a session-specific subdir. + out_files = list(tmp_path.rglob("incidents.jsonl")) + assert out_files, "no incidents.jsonl written" + lines = out_files[0].read_text().strip().splitlines() + fingerprints = [json.loads(l)["fingerprint"] for l in lines] + assert any(fp == "cam:stream-stall" for fp in fingerprints), \ + f"stall fingerprint missing — found: {set(fingerprints)}" + + +@pytest.mark.slow +def test_incidents_jsonl_written_with_poller_wired(tmp_path: Path): + """Regression: SessionPoller must not clobber SessionOrchestrator's persist listener.""" + from syncfield.viewer.poller import SessionPoller + + sess = SessionOrchestrator(host_id="test", output_dir=tmp_path) + stream = FakeStream("cam", target_hz=30.0) + sess.add(stream) + + # Spin up a poller as the viewer would. + poller = SessionPoller(sess) + + sess.connect() + sess.start(countdown_s=0) + stream.pause_samples() + time.sleep(2.5) + sess.stop() + sess.disconnect() + + out = list(tmp_path.rglob("incidents.jsonl")) + assert out, "no incidents.jsonl written — poller likely clobbered persist listener" + lines = out[0].read_text().strip().splitlines() + assert any('"fingerprint": "cam:stream-stall"' in l or '"fingerprint":"cam:stream-stall"' in l for l in lines) + + +@pytest.mark.slow +def test_orchestrator_feeds_writer_stats_to_backpressure_detector(tmp_path: Path): + from syncfield.health.detector import DetectorBase + from syncfield.health.severity import Severity + + class WriterStatsSpy(DetectorBase): + name = "writer-stats-spy" + default_severity = Severity.INFO + + def __init__(self): + self.calls = 0 + + def observe_writer_stats(self, stream_id, stats): + self.calls += 1 + + sess = SessionOrchestrator(host_id="test", output_dir=tmp_path) + spy = WriterStatsSpy() + sess.health.register(spy) + stream = FakeStream("cam", target_hz=30.0) + sess.add(stream) + + sess.connect() + sess.start(countdown_s=0) + time.sleep(0.6) + sess.stop() + sess.disconnect() + + # At 10 Hz throttle + 600ms → ~5-6 calls expected. + assert spy.calls >= 2, f"writer stats not emitted (got {spy.calls})" diff --git a/tests/integration/health/test_partial_connect.py b/tests/integration/health/test_partial_connect.py new file mode 100644 index 0000000..4c30e23 --- /dev/null +++ b/tests/integration/health/test_partial_connect.py @@ -0,0 +1,45 @@ +"""End-to-end: real SessionOrchestrator + FakeStream mix survives one +stream failing to connect, and incidents.jsonl captures it.""" +from __future__ import annotations + +import json +import time +from pathlib import Path + +import pytest + +from syncfield.orchestrator import SessionOrchestrator +from syncfield.testing import FakeStream +from syncfield.types import SessionState + + +@pytest.mark.slow +def test_partial_connect_end_to_end(tmp_path: Path): + sess = SessionOrchestrator(host_id="h", output_dir=tmp_path) + sess.add(FakeStream("good_a")) + sess.add(FakeStream("bad", fail_on_start=True)) + sess.add(FakeStream("good_b")) + + sess.connect() + assert sess.state is SessionState.CONNECTED + + # Give the health worker a moment to ingest the startup-failure event. + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline: + if any(i.fingerprint == "bad:startup-failure" for i in sess.health.open_incidents()): + break + time.sleep(0.05) + + open_fps = [i.fingerprint for i in sess.health.open_incidents()] + assert "bad:startup-failure" in open_fps + + sess.start(countdown_s=0) + time.sleep(0.5) + sess.stop() + sess.disconnect() + + out = list(tmp_path.rglob("incidents.jsonl")) + assert out, "no incidents.jsonl written" + lines = [json.loads(l) for l in out[0].read_text().strip().splitlines() if l] + fingerprints = {l["fingerprint"] for l in lines} + assert "bad:startup-failure" in fingerprints diff --git a/tests/unit/adapters/insta360_go3s/test_aggregation_queue.py b/tests/unit/adapters/insta360_go3s/test_aggregation_queue.py index 1039808..c27cbd5 100644 --- a/tests/unit/adapters/insta360_go3s/test_aggregation_queue.py +++ b/tests/unit/adapters/insta360_go3s/test_aggregation_queue.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import asyncio from pathlib import Path from typing import Any diff --git a/tests/unit/adapters/insta360_go3s/test_ble_camera.py b/tests/unit/adapters/insta360_go3s/test_ble_camera.py index db3572a..2daa839 100644 --- a/tests/unit/adapters/insta360_go3s/test_ble_camera.py +++ b/tests/unit/adapters/insta360_go3s/test_ble_camera.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import asyncio import time from typing import Callable diff --git a/tests/unit/adapters/test_oak_camera_health.py b/tests/unit/adapters/test_oak_camera_health.py new file mode 100644 index 0000000..e16f7eb --- /dev/null +++ b/tests/unit/adapters/test_oak_camera_health.py @@ -0,0 +1,43 @@ +"""Unit tests for OAK adapter health wiring (no real hardware required).""" + +import logging +import pytest + + +pytest.importorskip("depthai") # skip entire module if the oak extra isn't installed + + +def test_oak_declares_target_hz(tmp_path): + from syncfield.adapters.oak_camera import OakCameraStream + s = OakCameraStream(id="oak-main", rgb_fps=30, output_dir=tmp_path) + assert s.capabilities.target_hz == 30.0 + + +def test_oak_bridge_install_routes_depthai_errors_to_emit_health(tmp_path): + from syncfield.adapters.oak_camera import OakCameraStream + + captured = [] + s = OakCameraStream(id="oak-main", rgb_fps=30, output_dir=tmp_path) + s.on_health(lambda ev: captured.append(ev)) + + # Install the bridge directly (connect() builds a pipeline we can't easily mock here). + s._install_depthai_bridge() + try: + logging.getLogger("depthai").error( + "Communication exception - Original message 'Couldn't read data from stream: '__x_0_1' (X_LINK_ERROR)'" + ) + finally: + s._uninstall_depthai_bridge() + + assert any(ev.fingerprint == "oak-main:adapter:xlink-error" for ev in captured), \ + f"no xlink-error event captured; got fingerprints: {[c.fingerprint for c in captured]}" + + +def test_oak_bridge_uninstall_is_idempotent(tmp_path): + from syncfield.adapters.oak_camera import OakCameraStream + + s = OakCameraStream(id="oak-main", rgb_fps=30, output_dir=tmp_path) + s._uninstall_depthai_bridge() # no-op before install + s._install_depthai_bridge() + s._uninstall_depthai_bridge() + s._uninstall_depthai_bridge() # no-op after uninstall diff --git a/tests/unit/health/__init__.py b/tests/unit/health/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/health/detectors/__init__.py b/tests/unit/health/detectors/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/health/detectors/test_adapter_passthrough.py b/tests/unit/health/detectors/test_adapter_passthrough.py new file mode 100644 index 0000000..2ab615d --- /dev/null +++ b/tests/unit/health/detectors/test_adapter_passthrough.py @@ -0,0 +1,30 @@ +from syncfield.health.detectors.adapter_passthrough import AdapterEventPassthrough +from syncfield.health.severity import Severity +from syncfield.health.types import Incident +from syncfield.types import HealthEvent, HealthEventKind + + +def _adapter_ev(at_ns: int) -> HealthEvent: + return HealthEvent( + stream_id="cam", + kind=HealthEventKind.ERROR, + at_ns=at_ns, + detail="x", + severity=Severity.ERROR, + source="adapter:oak", + fingerprint="cam:adapter:xlink-error", + ) + + +def test_tick_emits_nothing(): + d = AdapterEventPassthrough() + assert list(d.tick(now_ns=1000)) == [] + + +def test_close_condition_respects_quiet_window(): + d = AdapterEventPassthrough(quiet_ns=500) + inc = Incident.opened_from(_adapter_ev(100), title="x") + assert d.close_condition(inc, now_ns=400) is False # 300 < 500 + inc.record_event(_adapter_ev(900)) + assert d.close_condition(inc, now_ns=1000) is False # 100 < 500 + assert d.close_condition(inc, now_ns=1500) is True # 600 >= 500 diff --git a/tests/unit/health/detectors/test_backpressure.py b/tests/unit/health/detectors/test_backpressure.py new file mode 100644 index 0000000..0d208e0 --- /dev/null +++ b/tests/unit/health/detectors/test_backpressure.py @@ -0,0 +1,46 @@ +from syncfield.health.detectors.backpressure import BackpressureDetector +from syncfield.health.types import Incident, WriterStats + + +def _stat(at_ns, depth, cap=16, dropped=0): + return WriterStats(stream_id="cam", at_ns=at_ns, queue_depth=depth, queue_capacity=cap, dropped=dropped) + + +def test_does_not_fire_with_normal_fullness(): + d = BackpressureDetector() + for t in range(0, int(3e9), int(2.5e8)): + d.observe_writer_stats("cam", _stat(t, depth=2)) + assert list(d.tick(now_ns=int(3e9))) == [] + + +def test_fires_when_queue_sustained_above_threshold(): + d = BackpressureDetector(fullness_threshold=0.8, sustain_ns=int(2e9)) + for t in range(0, int(3e9), int(2.5e8)): + d.observe_writer_stats("cam", _stat(t, depth=14)) # 14/16 = 0.875 + emitted = list(d.tick(now_ns=int(3e9))) + assert len(emitted) == 1 + assert emitted[0].fingerprint == "cam:backpressure" + + +def test_fires_on_any_drop_increment(): + d = BackpressureDetector() + d.observe_writer_stats("cam", _stat(0, depth=1, dropped=0)) + d.observe_writer_stats("cam", _stat(int(1e8), depth=1, dropped=5)) + emitted = list(d.tick(now_ns=int(2e8))) + assert len(emitted) == 1 + + +def test_close_condition_requires_low_and_no_new_drops(): + d = BackpressureDetector(fullness_threshold=0.8, sustain_ns=int(2e9), + recovery_ratio=0.3, recovery_ns=int(1e9)) + for t in range(0, int(3e9), int(2.5e8)): + d.observe_writer_stats("cam", _stat(t, depth=14)) + events = list(d.tick(now_ns=int(3e9))) + inc = Incident.opened_from(events[0], title="x") + + # Recovery in progress. + d.observe_writer_stats("cam", _stat(int(3.5e9), depth=2)) + assert d.close_condition(inc, now_ns=int(4e9)) is False # only 500 ms of recovery + + d.observe_writer_stats("cam", _stat(int(5e9), depth=2)) + assert d.close_condition(inc, now_ns=int(5e9)) is True # 1.5 s of recovery diff --git a/tests/unit/health/detectors/test_depthai_bridge.py b/tests/unit/health/detectors/test_depthai_bridge.py new file mode 100644 index 0000000..d89d55b --- /dev/null +++ b/tests/unit/health/detectors/test_depthai_bridge.py @@ -0,0 +1,59 @@ +import logging +from typing import List + +from syncfield.health.detectors.depthai_bridge import DepthAILoggerBridge +from syncfield.types import HealthEvent + + +def _mk_record(msg: str, level: int = logging.ERROR, name: str = "depthai") -> logging.LogRecord: + return logging.LogRecord( + name=name, level=level, pathname="", lineno=0, msg=msg, args=(), exc_info=None, + ) + + +def test_xlink_error_maps_to_xlink_fingerprint(): + captured: List[HealthEvent] = [] + bridge = DepthAILoggerBridge(stream_id="oak-main", sink=lambda sid, ev: captured.append(ev)) + rec = _mk_record("Communication exception - possible device error. Original message 'Couldn't read data from stream: '__x_0_1' (X_LINK_ERROR)'") + bridge.emit(rec) + assert len(captured) == 1 + ev = captured[0] + assert ev.stream_id == "oak-main" + assert ev.fingerprint == "oak-main:adapter:xlink-error" + assert ev.source == "adapter:oak" + assert ev.data.get("stream") == "__x_0_1" + + +def test_device_crash_attaches_crash_dump_path(): + captured: List[HealthEvent] = [] + bridge = DepthAILoggerBridge(stream_id="oak-main", sink=lambda sid, ev: captured.append(ev)) + rec = _mk_record("Device with id 194430 has crashed. Crash dump logs are stored in: /tmp/crash/crash_dump.json - please report to developers.") + bridge.emit(rec) + ev = captured[0] + assert ev.fingerprint == "oak-main:adapter:device-crash" + assert ev.data.get("crash_dump_path") == "/tmp/crash/crash_dump.json" + + +def test_reconnect_attempt_and_success_have_distinct_fingerprints(): + captured: List[HealthEvent] = [] + bridge = DepthAILoggerBridge(stream_id="oak-main", sink=lambda sid, ev: captured.append(ev)) + bridge.emit(_mk_record("Attempting to reconnect. Timeout is 10000ms", level=logging.WARNING)) + bridge.emit(_mk_record("Reconnection successful", level=logging.WARNING)) + fps = [c.fingerprint for c in captured] + assert "oak-main:adapter:reconnect-attempt" in fps + assert "oak-main:adapter:reconnect-success" in fps + + +def test_unrecognized_error_falls_back_to_warning_unparsed(): + captured: List[HealthEvent] = [] + bridge = DepthAILoggerBridge(stream_id="oak-main", sink=lambda sid, ev: captured.append(ev)) + bridge.emit(_mk_record("Something totally new and unrecognized", level=logging.ERROR)) + assert len(captured) == 1 + assert captured[0].source == "adapter:oak:unparsed-log" + + +def test_info_records_are_ignored(): + captured = [] + bridge = DepthAILoggerBridge(stream_id="oak-main", sink=lambda sid, ev: captured.append(ev)) + bridge.emit(_mk_record("Some info", level=logging.INFO)) + assert captured == [] diff --git a/tests/unit/health/detectors/test_fps_drop.py b/tests/unit/health/detectors/test_fps_drop.py new file mode 100644 index 0000000..28adcb2 --- /dev/null +++ b/tests/unit/health/detectors/test_fps_drop.py @@ -0,0 +1,75 @@ +from syncfield.health.detectors.fps_drop import FpsDropDetector +from syncfield.types import SampleEvent + + +def _s(stream: str, t_ns: int) -> SampleEvent: + return SampleEvent(stream_id=stream, frame_number=0, capture_ns=t_ns) + + +def test_no_fire_if_fps_tracks_target(): + d = FpsDropDetector(target_getter=lambda sid: 30.0) + # emit 30 samples over 1 second + for i in range(30): + d.observe_sample("cam", _s("cam", i * int(1e9 / 30))) + assert list(d.tick(now_ns=int(1.1e9))) == [] + + +def test_fires_when_observed_below_70_percent_for_3s(): + d = FpsDropDetector( + target_getter=lambda sid: 30.0, + drop_ratio=0.70, + sustain_ns=3 * 1_000_000_000, + ) + + # 10 fps for 3.5 seconds — fps is 10, target 30, ratio 0.33. + interval = int(1e9 / 10) + t = 0 + events = [] + while t <= int(3.5e9): + d.observe_sample("cam", _s("cam", t)) + events.extend(list(d.tick(now_ns=t))) + t += interval + + # Eventually, after 3s of sustained low fps, at least one event fires. + assert len(events) >= 1 + assert events[0].fingerprint == "cam:fps-drop" + assert events[0].data["target_hz"] == 30.0 + + +def test_does_not_fire_without_target_before_warmup(): + d = FpsDropDetector( + target_getter=lambda sid: None, + baseline_warmup_ns=5_000_000_000, + ) + # 10 fps, but only for 1s — under warmup. + t = 0 + for _ in range(10): + d.observe_sample("cam", _s("cam", t)) + t += int(1e8) + assert list(d.tick(now_ns=int(1.1e9))) == [] + + +def test_learns_baseline_then_fires_on_subsequent_drop(): + d = FpsDropDetector( + target_getter=lambda sid: None, + baseline_warmup_ns=1_000_000_000, + baseline_window_ns=2_000_000_000, + drop_ratio=0.7, + sustain_ns=1_000_000_000, + ) + # 3 s @ 30 fps → baseline ≈ 30. + t = 0 + events = [] + while t <= int(3e9): + d.observe_sample("cam", _s("cam", t)) + events.extend(list(d.tick(now_ns=t))) + t += int(1e9 / 30) + # 1.5 s of 10 fps → drop. + end = t + int(1.5e9) + while t <= end: + d.observe_sample("cam", _s("cam", t)) + events.extend(list(d.tick(now_ns=t))) + t += int(1e8) + # At least one fps-drop event should have fired during the low-fps period. + drop_events = [e for e in events if e.fingerprint == "cam:fps-drop"] + assert len(drop_events) >= 1 diff --git a/tests/unit/health/detectors/test_jitter.py b/tests/unit/health/detectors/test_jitter.py new file mode 100644 index 0000000..2d157dd --- /dev/null +++ b/tests/unit/health/detectors/test_jitter.py @@ -0,0 +1,40 @@ +from syncfield.health.detectors.jitter import JitterDetector +from syncfield.types import SampleEvent + + +def _s(t_ns: int) -> SampleEvent: + return SampleEvent(stream_id="cam", frame_number=0, capture_ns=t_ns) + + +def test_steady_30hz_does_not_fire(): + d = JitterDetector(target_getter=lambda sid: 30.0) + step = int(1e9 / 30) + t = 0 + for _ in range(120): + d.observe_sample("cam", _s(t)) + t += step + assert list(d.tick(now_ns=t)) == [] + + +def test_irregular_intervals_fire_when_p95_exceeds_ratio(): + d = JitterDetector( + target_getter=lambda sid: 30.0, + jitter_ratio=2.0, + sustain_ns=500_000_000, + ) + step = int(1e9 / 30) + big = step * 4 # 4× target interval + t = 0 + # 60 samples alternating between normal and 4× intervals. + for i in range(60): + d.observe_sample("cam", _s(t)) + t += big if i % 2 == 0 else step + + # Give sustain time to elapse with more irregular samples. + for _ in range(20): + d.observe_sample("cam", _s(t)) + t += big + + emitted = list(d.tick(now_ns=t + 500_000_000)) + assert len(emitted) == 1 + assert emitted[0].fingerprint == "cam:jitter" diff --git a/tests/unit/health/detectors/test_no_data.py b/tests/unit/health/detectors/test_no_data.py new file mode 100644 index 0000000..b8e44bf --- /dev/null +++ b/tests/unit/health/detectors/test_no_data.py @@ -0,0 +1,68 @@ +from syncfield.health.detectors.no_data import NoDataDetector +from syncfield.health.types import Incident +from syncfield.types import SampleEvent + + +def _s(stream: str, t_ns: int) -> SampleEvent: + return SampleEvent(stream_id=stream, frame_number=0, capture_ns=t_ns) + + +def test_no_fire_before_threshold(): + d = NoDataDetector(threshold_ns=1000) + d.observe_connection_state("cam", "connected", at_ns=100) + assert list(d.tick(now_ns=500)) == [] # 400 ns elapsed, under 1000 + + +def test_fires_after_threshold_without_sample(): + d = NoDataDetector(threshold_ns=1000) + d.observe_connection_state("cam", "connected", at_ns=100) + out = list(d.tick(now_ns=2000)) # 1900 ns elapsed + assert len(out) == 1 + ev = out[0] + assert ev.stream_id == "cam" + assert ev.fingerprint == "cam:no-data" + assert ev.source == "detector:no-data" + assert "no data" in (ev.detail or "").lower() + + +def test_does_not_refire_while_still_no_data(): + d = NoDataDetector(threshold_ns=1000) + d.observe_connection_state("cam", "connected", at_ns=100) + first = list(d.tick(now_ns=2000)) + second = list(d.tick(now_ns=3000)) + assert len(first) == 1 + assert len(second) == 0 + + +def test_close_condition_satisfied_once_sample_arrives(): + d = NoDataDetector(threshold_ns=1000) + d.observe_connection_state("cam", "connected", at_ns=100) + events = list(d.tick(now_ns=2000)) + inc = Incident.opened_from(events[0], title="x") + + assert d.close_condition(inc, now_ns=2100) is False # still no sample + + d.observe_sample("cam", _s("cam", 2200)) + assert d.close_condition(inc, now_ns=2300) is True + + +def test_resets_bookkeeping_on_non_connected_state(): + d = NoDataDetector(threshold_ns=1000) + d.observe_connection_state("cam", "connected", at_ns=100) + list(d.tick(now_ns=2000)) # fires + + d.observe_connection_state("cam", "failed", at_ns=2500) + # Back to connected → fresh clock, no duplicate fire. + d.observe_connection_state("cam", "connected", at_ns=3000) + assert list(d.tick(now_ns=3500)) == [] # only 500 ns since new connected + + +def test_per_stream_independent_state(): + d = NoDataDetector(threshold_ns=1000) + d.observe_connection_state("a", "connected", at_ns=100) + d.observe_connection_state("b", "connected", at_ns=100) + d.observe_sample("b", _s("b", 200)) + + out = list(d.tick(now_ns=2000)) + assert len(out) == 1 + assert out[0].stream_id == "a" diff --git a/tests/unit/health/detectors/test_startup_failure.py b/tests/unit/health/detectors/test_startup_failure.py new file mode 100644 index 0000000..2472856 --- /dev/null +++ b/tests/unit/health/detectors/test_startup_failure.py @@ -0,0 +1,65 @@ +from syncfield.health.detectors.startup_failure import StartupFailureDetector +from syncfield.health.severity import Severity +from syncfield.types import HealthEvent, HealthEventKind + + +def _ev_for(phase: str, kind=HealthEventKind.ERROR) -> HealthEvent: + return HealthEvent( + stream_id="cam", kind=kind, at_ns=100, detail="boom", + severity=Severity.ERROR, source="orchestrator", + fingerprint=f"cam:adapter:startup-{phase}", + data={"phase": phase}, + ) + + +def test_fires_on_connect_phase_error(): + d = StartupFailureDetector() + d.observe_health("cam", _ev_for("connect")) + events = list(d.tick(now_ns=500)) + assert len(events) == 1 + assert events[0].fingerprint == "cam:startup-failure" + assert events[0].data["phase"] == "connect" + + +def test_ignores_non_startup_phases(): + d = StartupFailureDetector() + d.observe_health("cam", HealthEvent( + stream_id="cam", kind=HealthEventKind.ERROR, at_ns=1, detail="x", + severity=Severity.ERROR, source="adapter:foo", fingerprint="cam:adapter:xlink", + data={}, + )) + assert list(d.tick(now_ns=100)) == [] + + +def test_closes_after_phase_success_signal(): + d = StartupFailureDetector() + d.observe_health("cam", _ev_for("connect")) + list(d.tick(now_ns=100)) + from syncfield.health.types import Incident + inc = Incident.opened_from(_ev_for("connect"), title="x") + + # Before success, not closed. + assert d.close_condition(inc, now_ns=200) is False + + # Success signal arrives. + d.observe_health("cam", HealthEvent( + stream_id="cam", kind=HealthEventKind.HEARTBEAT, at_ns=300, detail="connected", + severity=Severity.INFO, source="orchestrator", fingerprint="cam:adapter:startup-success", + data={"phase": "connect", "outcome": "success"}, + )) + assert d.close_condition(inc, now_ns=400) is True + + +def test_success_clears_pending_so_no_stale_fire(): + d = StartupFailureDetector() + # Error first, then success. + d.observe_health("cam", _ev_for("connect")) + d.observe_health("cam", HealthEvent( + stream_id="cam", kind=HealthEventKind.HEARTBEAT, at_ns=50, detail="connected", + severity=Severity.INFO, source="orchestrator", + fingerprint="cam:adapter:startup-success", + data={"phase": "connect", "outcome": "success"}, + )) + + # Next tick should emit nothing — success arrived before any tick fired. + assert list(d.tick(now_ns=200)) == [] diff --git a/tests/unit/health/detectors/test_stream_stall.py b/tests/unit/health/detectors/test_stream_stall.py new file mode 100644 index 0000000..06f2cf4 --- /dev/null +++ b/tests/unit/health/detectors/test_stream_stall.py @@ -0,0 +1,72 @@ +from syncfield.health.detectors.stream_stall import StreamStallDetector +from syncfield.health.types import Incident +from syncfield.types import SampleEvent + + +def _sample(stream_id: str, capture_ns: int) -> SampleEvent: + return SampleEvent(stream_id=stream_id, frame_number=1, capture_ns=capture_ns) + + +def test_no_fire_before_seeing_any_sample(): + d = StreamStallDetector(stall_threshold_ns=1000) + assert list(d.tick(now_ns=10_000)) == [] + + +def test_fires_when_silent_longer_than_threshold(): + d = StreamStallDetector(stall_threshold_ns=1000) + d.observe_sample("cam", _sample("cam", capture_ns=100)) + events = list(d.tick(now_ns=2000)) # 1900 ns of silence + assert len(events) == 1 + ev = events[0] + assert ev.stream_id == "cam" + assert ev.fingerprint == "cam:stream-stall" + assert ev.source == "detector:stream-stall" + assert "silence" in (ev.detail or "").lower() + + +def test_does_not_refire_while_still_stalled(): + d = StreamStallDetector(stall_threshold_ns=1000) + d.observe_sample("cam", _sample("cam", capture_ns=100)) + fired_once = list(d.tick(now_ns=2000)) + fired_twice = list(d.tick(now_ns=3000)) + assert len(fired_once) == 1 + assert len(fired_twice) == 0 + + +def test_refires_after_recovery_then_new_stall(): + d = StreamStallDetector(stall_threshold_ns=1000, recovery_ns=500) + d.observe_sample("cam", _sample("cam", capture_ns=0)) + list(d.tick(now_ns=2000)) # fires stall + + # recovery: samples flow for ≥ recovery_ns + for t in range(3000, 4100, 100): + d.observe_sample("cam", _sample("cam", capture_ns=t)) + # Silence again. + new_events = list(d.tick(now_ns=6000)) + assert len(new_events) == 1 # second stall → new event + + +def test_close_condition_requires_recent_sample_flow(): + d = StreamStallDetector(stall_threshold_ns=1000, recovery_ns=500) + d.observe_sample("cam", _sample("cam", capture_ns=0)) + events = list(d.tick(now_ns=2000)) + inc = Incident.opened_from(events[0], title="x") + + # Still silent — do not close. + assert d.close_condition(inc, now_ns=2500) is False + + # Samples arrive across a 600 ns window → recovery_ns=500 satisfied. + d.observe_sample("cam", _sample("cam", capture_ns=2600)) + d.observe_sample("cam", _sample("cam", capture_ns=3200)) + assert d.close_condition(inc, now_ns=3300) is True + + +def test_per_stream_independent_state(): + d = StreamStallDetector(stall_threshold_ns=1000) + d.observe_sample("a", _sample("a", capture_ns=100)) + d.observe_sample("b", _sample("b", capture_ns=100)) + # stream b stays alive + d.observe_sample("b", _sample("b", capture_ns=1800)) + events = list(d.tick(now_ns=2500)) + assert len(events) == 1 + assert events[0].stream_id == "a" diff --git a/tests/unit/health/test_detector_base.py b/tests/unit/health/test_detector_base.py new file mode 100644 index 0000000..8dfad16 --- /dev/null +++ b/tests/unit/health/test_detector_base.py @@ -0,0 +1,52 @@ +import pytest + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.types import WriterStats +from syncfield.types import HealthEvent, HealthEventKind, SampleEvent, SessionState + + +class NoopDetector(DetectorBase): + name = "noop" + default_severity = Severity.WARNING + + +def test_detector_base_defaults_are_noops(): + d = NoopDetector() + # All observers accept calls without raising. + d.observe_sample("cam", SampleEvent(stream_id="cam", frame_number=1, capture_ns=100)) + d.observe_health("cam", HealthEvent(stream_id="cam", kind=HealthEventKind.WARNING, at_ns=1)) + d.observe_state(SessionState.IDLE, SessionState.CONNECTED) + d.observe_writer_stats("cam", WriterStats("cam", 1, 0, 0, 0)) + # tick yields nothing by default. + assert list(d.tick(now_ns=100)) == [] + # close_condition defaults to False (conservative: keep open; subclasses override). + from syncfield.health.types import Incident + ev = HealthEvent(stream_id="cam", kind=HealthEventKind.WARNING, at_ns=1) + inc = Incident.opened_from(ev, title="x") + assert d.close_condition(inc, now_ns=10) is False + + +def test_detector_base_requires_name_and_severity(): + with pytest.raises(TypeError): + DetectorBase() # abstract base: name / default_severity unset on the class + + +def test_grandchild_subclass_must_redeclare_if_needed(): + # Sanity: once a parent sets name/default_severity, grandchildren inheriting + # them pass the check (legitimate use case). + class Grandchild(NoopDetector): + pass + Grandchild() # should not raise + + # A subclass that overrides nothing but lacks both required attrs + # cannot exist — we cannot construct such a test directly without + # subclassing DetectorBase again (which must itself declare attrs). + # The more important invariant is covered by the existing + # test_detector_base_requires_name_and_severity. + + +def test_detector_base_observe_connection_state_default_is_noop(): + d = NoopDetector() + # Does not raise; returns None. + assert d.observe_connection_state("cam", "connected", 100) is None diff --git a/tests/unit/health/test_health_system.py b/tests/unit/health/test_health_system.py new file mode 100644 index 0000000..9e1ed21 --- /dev/null +++ b/tests/unit/health/test_health_system.py @@ -0,0 +1,139 @@ +import time + +from syncfield.health import HealthSystem, Severity +from syncfield.health.detector import DetectorBase +from syncfield.types import HealthEvent, HealthEventKind, SampleEvent, SessionState + + +class Custom(DetectorBase): + name = "custom" + default_severity = Severity.WARNING + + +def test_health_system_boots_and_accepts_inputs(): + hs = HealthSystem() + hs.start() + try: + hs.observe_sample("cam", SampleEvent(stream_id="cam", frame_number=1, capture_ns=1)) + hs.observe_health("cam", HealthEvent( + stream_id="cam", kind=HealthEventKind.WARNING, at_ns=1, + severity=Severity.WARNING, source="test", fingerprint="cam:adapter:test", + )) + hs.observe_state(SessionState.IDLE, SessionState.CONNECTED) + finally: + hs.stop() + + +def test_health_system_register_and_unregister(): + hs = HealthSystem() + d = Custom() + hs.register(d) + assert any(x.name == "custom" for x in hs.iter_detectors()) + hs.unregister("custom") + assert not any(x.name == "custom" for x in hs.iter_detectors()) + + +def test_health_system_installs_default_detectors(): + hs = HealthSystem() + names = {d.name for d in hs.iter_detectors()} + for expected in ( + "adapter", + "stream-stall", + "fps-drop", + "jitter", + "startup-failure", + "backpressure", + "no-data", + ): + assert expected in names, f"missing default detector: {expected}" + + +def test_health_system_callbacks_fire_on_open_and_close(): + hs = HealthSystem(passthrough_close_ns=1) # close instantly for the test + opened, closed = [], [] + hs.on_incident_opened(opened.append) + hs.on_incident_closed(closed.append) + + hs.start() + try: + hs.observe_health("cam", HealthEvent( + stream_id="cam", kind=HealthEventKind.ERROR, at_ns=1, + severity=Severity.ERROR, source="adapter:test", + fingerprint="cam:adapter:xlink-error", + )) + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline: + if opened and closed: + break + time.sleep(0.02) + finally: + hs.stop() + assert opened, "incident was not opened" + assert closed, "incident was not closed" + + +def test_health_system_double_start_is_idempotent(): + hs = HealthSystem() + hs.start() + first_worker = hs._worker + hs.start() # should not spin up a new thread + second_worker = hs._worker + try: + assert first_worker is second_worker, "start() should be idempotent" + finally: + hs.stop() + + +def test_health_system_register_after_start_warns(): + import warnings + + hs = HealthSystem() + hs.start() + try: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + hs.register(Custom()) + assert any(issubclass(w.category, RuntimeWarning) for w in caught) + assert any("after HealthSystem.start()" in str(w.message) for w in caught) + finally: + hs.stop() + + +def test_register_stream_propagates_target_hz_to_detectors(): + hs = HealthSystem() + hs.register_stream("cam", 30.0) + + fps = next(d for d in hs.iter_detectors() if d.name == "fps-drop") + jitter = next(d for d in hs.iter_detectors() if d.name == "jitter") + + assert fps._target_getter("cam") == 30.0 + assert jitter._target_getter("cam") == 30.0 + assert fps._target_getter("unknown") is None + + +def test_health_system_observe_connection_state_routes_to_worker(): + import time as _time + class Spy(DetectorBase): + name = "conn-spy" + default_severity = Severity.INFO + + def __init__(self): + self.calls = [] + + def observe_connection_state(self, stream_id, new_state, at_ns): + self.calls.append((stream_id, new_state, at_ns)) + + hs = HealthSystem() + spy = Spy() + hs.register(spy) + + hs.start() + try: + hs.observe_connection_state("cam", "connecting", 10) + hs.observe_connection_state("cam", "connected", 20) + deadline = _time.monotonic() + 2.0 + while _time.monotonic() < deadline and len(spy.calls) < 2: + _time.sleep(0.02) + finally: + hs.stop() + assert spy.calls == [("cam", "connecting", 10), ("cam", "connected", 20)] diff --git a/tests/unit/health/test_health_worker.py b/tests/unit/health/test_health_worker.py new file mode 100644 index 0000000..b69150f --- /dev/null +++ b/tests/unit/health/test_health_worker.py @@ -0,0 +1,153 @@ +import threading +import time + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.tracker import IncidentTracker +from syncfield.health.types import WriterStats +from syncfield.health.worker import HealthWorker +from syncfield.types import HealthEvent, HealthEventKind, SampleEvent, SessionState + + +class RecordingDetector(DetectorBase): + name = "recorder" + default_severity = Severity.INFO + + def __init__(self) -> None: + self.samples = [] + self.healths = [] + self.states = [] + self.writer_stats = [] + self.ticks = 0 + + def observe_sample(self, stream_id, sample): + self.samples.append((stream_id, sample.capture_ns)) + + def observe_health(self, stream_id, event): + self.healths.append((stream_id, event.at_ns)) + + def observe_state(self, old, new): + self.states.append((old, new)) + + def observe_writer_stats(self, stream_id, stats): + self.writer_stats.append((stream_id, stats.queue_depth)) + + def tick(self, now_ns): + self.ticks += 1 + return iter(()) + + +def _wait_until(pred, timeout=1.0, interval=0.01): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if pred(): + return True + time.sleep(interval) + return False + + +def test_worker_drains_all_ingress_queues_on_tick(): + tr = IncidentTracker() + det = RecordingDetector() + tr.bind_detector(det) + w = HealthWorker(tracker=tr, detectors=[det], tick_hz=100) + + w.start() + try: + w.push_sample("cam", SampleEvent(stream_id="cam", frame_number=1, capture_ns=42)) + # IMPORTANT: health events must now carry a non-empty fingerprint because + # Task 6 added a guard in IncidentTracker.ingest. Provide one. + w.push_health("cam", HealthEvent( + stream_id="cam", kind=HealthEventKind.WARNING, at_ns=1, + severity=Severity.WARNING, source="test", fingerprint="cam:test:ingest", + )) + w.push_state(SessionState.IDLE, SessionState.CONNECTED) + w.push_writer_stats("cam", WriterStats("cam", 1, 2, 16, 0)) + + assert _wait_until(lambda: det.samples and det.healths and det.states and det.writer_stats) + finally: + w.stop() + + assert det.samples[0] == ("cam", 42) + assert det.healths[0] == ("cam", 1) + assert det.states[0] == (SessionState.IDLE, SessionState.CONNECTED) + assert det.writer_stats[0] == ("cam", 2) + + +def test_worker_ticks_at_roughly_configured_rate(): + tr = IncidentTracker() + det = RecordingDetector() + w = HealthWorker(tracker=tr, detectors=[det], tick_hz=50) + w.start() + try: + time.sleep(0.2) # ~10 ticks + finally: + w.stop() + # Loose bound to avoid flakiness under loaded CI. + assert det.ticks >= 5 + + +def test_worker_feeds_detector_tick_output_into_tracker(): + class EmitsOneAndDone(DetectorBase): + name = "emit" + default_severity = Severity.WARNING + + def __init__(self): + self.fired = False + + def tick(self, now_ns): + if self.fired: + return iter(()) + self.fired = True + return iter([HealthEvent( + stream_id="cam", kind=HealthEventKind.WARNING, at_ns=now_ns, + detail="synthetic", severity=Severity.WARNING, + source="detector:emit", fingerprint="cam:emit", + )]) + + def close_condition(self, inc, now_ns): + return False + + tr = IncidentTracker() + det = EmitsOneAndDone() + tr.bind_detector(det) + w = HealthWorker(tracker=tr, detectors=[det], tick_hz=100) + w.start() + try: + assert _wait_until(lambda: len(tr.open_incidents()) == 1) + finally: + w.stop() + + +def test_worker_stop_is_idempotent(): + tr = IncidentTracker() + det = RecordingDetector() + w = HealthWorker(tracker=tr, detectors=[det], tick_hz=50) + w.start() + w.stop() + w.stop() # does not raise + + +def test_worker_drains_connection_state_queue_and_fans_out(): + class Spy(DetectorBase): + name = "conn-spy" + default_severity = Severity.INFO + + def __init__(self): + self.calls = [] + + def observe_connection_state(self, stream_id, new_state, at_ns): + self.calls.append((stream_id, new_state, at_ns)) + + tr = IncidentTracker() + spy = Spy() + w = HealthWorker(tracker=tr, detectors=[spy], tick_hz=100) + w.start() + try: + w.push_connection_state("cam", "connecting", 1) + w.push_connection_state("cam", "connected", 2) + assert _wait_until(lambda: len(spy.calls) == 2) + finally: + w.stop() + + assert spy.calls == [("cam", "connecting", 1), ("cam", "connected", 2)] diff --git a/tests/unit/health/test_incident_tracker.py b/tests/unit/health/test_incident_tracker.py new file mode 100644 index 0000000..bbe1be7 --- /dev/null +++ b/tests/unit/health/test_incident_tracker.py @@ -0,0 +1,128 @@ +from typing import List + +import pytest + +from syncfield.health.detector import DetectorBase +from syncfield.health.severity import Severity +from syncfield.health.tracker import IncidentTracker +from syncfield.health.types import Incident +from syncfield.types import HealthEvent, HealthEventKind + + +def _ev(at_ns: int, fingerprint: str = "cam:stall", severity: Severity = Severity.ERROR, + detail: str = "x") -> HealthEvent: + return HealthEvent( + stream_id="cam", + kind=HealthEventKind.ERROR, + at_ns=at_ns, + detail=detail, + severity=severity, + source="detector:stream-stall", + fingerprint=fingerprint, + ) + + +class AlwaysCloseAfter(DetectorBase): + name = "stall" + default_severity = Severity.ERROR + + def __init__(self, close_after_ns: int) -> None: + self._close_after = close_after_ns + + def close_condition(self, incident: Incident, now_ns: int) -> bool: + return now_ns - incident.last_event_at_ns >= self._close_after + + +def test_tracker_opens_incident_on_first_matching_event(): + tr = IncidentTracker() + tr.bind_detector(AlwaysCloseAfter(close_after_ns=1000)) + opened: List[Incident] = [] + tr.add_on_opened(opened.append) + + tr.ingest(_ev(100)) + + assert len(tr.open_incidents()) == 1 + assert opened and opened[0].stream_id == "cam" + + +def test_tracker_groups_same_fingerprint_into_one_incident(): + tr = IncidentTracker() + tr.bind_detector(AlwaysCloseAfter(close_after_ns=1_000_000_000)) + + tr.ingest(_ev(100, severity=Severity.WARNING)) + tr.ingest(_ev(200, severity=Severity.ERROR)) # escalate + tr.ingest(_ev(300, severity=Severity.ERROR)) + + opens = tr.open_incidents() + assert len(opens) == 1 + inc = opens[0] + assert inc.event_count == 3 + assert inc.severity == Severity.ERROR + assert inc.last_event_at_ns == 300 + + +def test_tracker_closes_incident_when_detector_close_condition_fires(): + tr = IncidentTracker() + tr.bind_detector(AlwaysCloseAfter(close_after_ns=500)) + closed: List[Incident] = [] + tr.add_on_closed(closed.append) + + tr.ingest(_ev(100)) + tr.tick(now_ns=200) # 100 ns since last event, not yet + assert tr.resolved_incidents() == [] + + tr.tick(now_ns=700) # 600 ns since last event → close + assert len(tr.resolved_incidents()) == 1 + assert tr.open_incidents() == [] + assert closed and closed[0].closed_at_ns == 700 + + +def test_tracker_reopens_after_close_on_same_fingerprint(): + tr = IncidentTracker() + tr.bind_detector(AlwaysCloseAfter(close_after_ns=100)) + + tr.ingest(_ev(100)) + tr.tick(now_ns=500) # closed + assert tr.open_incidents() == [] + + tr.ingest(_ev(1000)) # new incident, new id + opens = tr.open_incidents() + assert len(opens) == 1 + assert len(tr.resolved_incidents()) == 1 + assert opens[0].id != tr.resolved_incidents()[0].id + + +def test_tracker_unbound_fingerprint_falls_back_to_passthrough_close(): + # When an event arrives with a fingerprint whose detector is not bound, + # the tracker still groups it, using the default passthrough close + # window (30s of quiet). + tr = IncidentTracker(passthrough_close_ns=500) + + tr.ingest(_ev(100, fingerprint="cam:adapter:xlink")) + tr.tick(now_ns=400) + assert tr.open_incidents() + tr.tick(now_ns=1000) # 900 ns since last event → closes + assert tr.resolved_incidents() + + +def test_tracker_flush_callbacks_fire_on_update_too(): + tr = IncidentTracker() + tr.bind_detector(AlwaysCloseAfter(close_after_ns=1_000_000_000)) + updates: List[Incident] = [] + tr.add_on_updated(updates.append) + + tr.ingest(_ev(100)) # opens + tr.ingest(_ev(200)) # updates + tr.ingest(_ev(300)) # updates + assert len(updates) == 2 + + +def test_tracker_rejects_empty_fingerprint(): + tr = IncidentTracker() + with pytest.raises(ValueError, match="fingerprint"): + tr.ingest(HealthEvent( + stream_id="cam", + kind=HealthEventKind.ERROR, + at_ns=1, + # fingerprint defaults to "" + )) diff --git a/tests/unit/health/test_incident_types.py b/tests/unit/health/test_incident_types.py new file mode 100644 index 0000000..20d2ea6 --- /dev/null +++ b/tests/unit/health/test_incident_types.py @@ -0,0 +1,98 @@ +from syncfield.health.severity import Severity +from syncfield.health.types import ( + Incident, + IncidentArtifact, + IncidentSnapshot, + WriterStats, +) +from syncfield.types import HealthEvent, HealthEventKind + + +def _ev(at_ns: int, severity: Severity = Severity.ERROR) -> HealthEvent: + return HealthEvent( + stream_id="cam", + kind=HealthEventKind.ERROR, + at_ns=at_ns, + detail="x", + severity=severity, + source="detector:stream-stall", + fingerprint="cam:stream-stall", + ) + + +def test_writer_stats_fields(): + s = WriterStats( + stream_id="cam", + at_ns=100, + queue_depth=3, + queue_capacity=16, + dropped=0, + ) + assert s.queue_fullness == 3 / 16 + assert s.stream_id == "cam" + + +def test_writer_stats_zero_capacity_is_empty(): + s = WriterStats(stream_id="cam", at_ns=0, queue_depth=0, queue_capacity=0, dropped=0) + assert s.queue_fullness == 0.0 + + +def test_incident_from_first_event_initializes_fields(): + first = _ev(100) + inc = Incident.opened_from(first, title="Stream stalled (silence 2.0s)") + assert inc.stream_id == "cam" + assert inc.fingerprint == "cam:stream-stall" + assert inc.severity == Severity.ERROR + assert inc.title == "Stream stalled (silence 2.0s)" + assert inc.opened_at_ns == 100 + assert inc.closed_at_ns is None + assert inc.event_count == 1 + assert inc.first_event == first + assert inc.last_event == first + assert inc.artifacts == [] + + +def test_incident_record_event_escalates_severity_and_updates_last(): + inc = Incident.opened_from(_ev(100, severity=Severity.WARNING), title="t") + inc.record_event(_ev(200, severity=Severity.ERROR)) + assert inc.event_count == 2 + assert inc.severity == Severity.ERROR + assert inc.last_event.at_ns == 200 + assert inc.last_event_at_ns == 200 + + +def test_incident_close(): + inc = Incident.opened_from(_ev(100), title="t") + inc.close(at_ns=500) + assert inc.closed_at_ns == 500 + assert inc.is_open is False + + +def test_incident_attach_artifact(): + inc = Incident.opened_from(_ev(100), title="t") + inc.attach(IncidentArtifact(kind="crash_dump", path="/tmp/x.json")) + assert inc.artifacts[0].kind == "crash_dump" + assert inc.artifacts[0].path == "/tmp/x.json" + + +def test_incident_snapshot_shape(): + inc = Incident.opened_from(_ev(100), title="t") + snap = IncidentSnapshot.from_incident(inc, now_ns=1_000_000_100) + assert snap.id == inc.id + assert snap.stream_id == "cam" + assert snap.severity == "error" + assert snap.is_open is True + assert snap.ago_s >= 0 + + +def test_incident_snapshot_uses_closed_at_as_anchor_when_closed(): + inc = Incident.opened_from(_ev(1_000_000_000), title="t") + inc.record_event(_ev(1_100_000_000)) # last_event_at_ns = 1.1s + inc.close(at_ns=1_500_000_000) # closed 500ms later + + # now_ns is 2s after close + snap = IncidentSnapshot.from_incident(inc, now_ns=3_500_000_000) + assert snap.is_open is False + assert snap.closed_at_ns == 1_500_000_000 + # ago_s is measured from closed_at_ns, not last_event_at_ns. + assert abs(snap.ago_s - 2.0) < 0.01 diff --git a/tests/unit/health/test_registry.py b/tests/unit/health/test_registry.py new file mode 100644 index 0000000..2d631fe --- /dev/null +++ b/tests/unit/health/test_registry.py @@ -0,0 +1,44 @@ +import pytest + +from syncfield.health.detector import DetectorBase +from syncfield.health.registry import DetectorRegistry +from syncfield.health.severity import Severity + + +class Det(DetectorBase): + name = "d1" + default_severity = Severity.WARNING + + +class Det2(DetectorBase): + name = "d2" + default_severity = Severity.ERROR + + +def test_register_and_iterate(): + reg = DetectorRegistry() + d1 = Det() + d2 = Det2() + reg.register(d1) + reg.register(d2) + assert list(reg) == [d1, d2] + + +def test_register_duplicate_name_raises(): + reg = DetectorRegistry() + reg.register(Det()) + with pytest.raises(ValueError, match="already registered"): + reg.register(Det()) + + +def test_unregister_removes_by_name(): + reg = DetectorRegistry() + d1 = Det() + reg.register(d1) + reg.unregister("d1") + assert list(reg) == [] + + +def test_unregister_unknown_is_noop(): + reg = DetectorRegistry() + reg.unregister("nope") # does not raise diff --git a/tests/unit/health/test_severity.py b/tests/unit/health/test_severity.py new file mode 100644 index 0000000..b352ec6 --- /dev/null +++ b/tests/unit/health/test_severity.py @@ -0,0 +1,29 @@ +import pytest + +from syncfield.health.severity import Severity, max_severity + + +def test_severity_values(): + assert Severity.INFO.value == "info" + assert Severity.WARNING.value == "warning" + assert Severity.ERROR.value == "error" + assert Severity.CRITICAL.value == "critical" + + +def test_severity_ordering(): + # INFO < WARNING < ERROR < CRITICAL + order = [Severity.INFO, Severity.WARNING, Severity.ERROR, Severity.CRITICAL] + for a, b in zip(order, order[1:]): + assert a.rank < b.rank + + +def test_max_severity_picks_highest(): + assert max_severity(Severity.INFO, Severity.WARNING) == Severity.WARNING + assert max_severity(Severity.ERROR, Severity.WARNING) == Severity.ERROR + assert max_severity(Severity.CRITICAL, Severity.INFO, Severity.ERROR) == Severity.CRITICAL + assert max_severity(Severity.ERROR) == Severity.ERROR + + +def test_max_severity_requires_at_least_one(): + with pytest.raises(ValueError): + max_severity() diff --git a/tests/unit/test_orchestrator.py b/tests/unit/test_orchestrator.py index 0f40bb9..d2fa053 100644 --- a/tests/unit/test_orchestrator.py +++ b/tests/unit/test_orchestrator.py @@ -299,6 +299,11 @@ def start(self, session_clock): # type: ignore[override] class TestStartRollback: def test_failure_during_start_rolls_back_prior_streams(self, tmp_path): + """Under partial-connect semantics a stream with fail_on_start=True + fails at connect() time — it is never added to _connected_streams and + is therefore skipped during start_recording(). The two healthy streams + both connect and start successfully, so start() no longer raises. + """ session = _session(tmp_path) good1 = FakeStream("a") bad = FakeStream("b", fail_on_start=True) @@ -307,31 +312,33 @@ def test_failure_during_start_rolls_back_prior_streams(self, tmp_path): session.add(bad) session.add(good2) - with pytest.raises(RuntimeError, match="fake failure in start"): - session.start() + # start() must NOT raise — partial-connect isolated the failure + session.start() - # good1 was started → must be rolled back (stop called) + # good1 and good2 connected and started assert good1.start_calls == 1 - assert good1.stop_calls == 1 - # bad raised during start → stop should NOT be called on it - assert bad.start_calls == 1 + assert good2.start_calls == 1 + # bad failed during connect() — start_recording was never called on it + assert bad.start_calls == 0 assert bad.stop_calls == 0 - # good2 never reached start - assert good2.start_calls == 0 - assert good2.stop_calls == 0 - assert session.state is SessionState.IDLE + assert session.state is SessionState.RECORDING + + session.stop() + # Auto-connect path lands in STOPPED (not IDLE) after stop() + assert session.state is SessionState.STOPPED - def test_failure_during_prepare_stops_earlier_streams(self, tmp_path): - """A failure in ``prepare()`` happens during the connect phase, - which runs all preparations before any stream starts recording. - The rollback therefore calls ``disconnect()`` on streams that - connected — and ``start()`` is never reached on any of them. + def test_failure_during_prepare_isolated_to_failing_stream(self, tmp_path): + """A prepare() failure in the connect phase is now isolated to the + failing stream — partial-connect semantics. - This differs from the 0.1 behaviour where ``prepare()`` and - ``start()`` interleaved per stream; the 0.2 orchestrator splits - the two phases so all devices connect before any begin writing, - matching the egonaut lab recorder's 2-phase model. + The failing stream is marked ``"failed"`` with an entry in + ``_stream_errors``. Streams that connected successfully proceed to + recording, and ``start()`` does not raise. + + The old all-or-nothing rollback behaviour (every stream rolled back + to IDLE when any stream failed) was replaced in the partial-connect + refactor so a single bad camera no longer blocks the entire session. """ session = _session(tmp_path) good = FakeStream("a") @@ -339,17 +346,21 @@ def test_failure_during_prepare_stops_earlier_streams(self, tmp_path): session.add(good) session.add(bad) - with pytest.raises(RuntimeError, match="fake failure in prepare"): - session.start() + # start() must NOT raise — the session recovers with one fewer stream + session.start() - # prepare() ran on both in the connect phase + # Both streams entered the connect phase assert good.prepare_calls == 1 assert bad.prepare_calls == 1 - # start_recording() was never invoked because the connect phase failed - assert good.start_calls == 0 - assert bad.start_calls == 0 - # Rollback returned the auto-connected session to IDLE - assert session.state is SessionState.IDLE + # The failing stream is isolated: marked failed, error recorded + assert session._stream_states["b"] == "failed" + assert "b" in session._stream_errors + assert session._stream_errors["b"] + # The healthy stream connected and was started + assert session._stream_states["a"] == "connected" + assert good.start_calls > 0 + # The session is recording with the surviving stream + assert session.state is SessionState.RECORDING class TestFourPhaseLifecycle: @@ -1356,9 +1367,18 @@ def test_session_log_flushes_during_recording(self, tmp_path): session.stop() def test_rollback_is_logged(self, tmp_path): + """A stream that connects successfully but raises during start_recording() + must cause a rollback log entry. Under partial-connect semantics + fail_on_start=True causes connect() to fail (not start_recording()), so + we use a custom subclass that overrides start_recording() directly. + """ + class FailOnStartRecording(FakeStream): + def start_recording(self, session_clock): + raise RuntimeError("fake failure in start_recording") + session = _session(tmp_path) session.add(FakeStream("a")) - session.add(FakeStream("b", fail_on_start=True)) + session.add(FailOnStartRecording("b")) # Capture the episode dir before start() — rollback happens inside # start() and _prepare_next_episode() rotates output_dir in that path. episode_dir = session.output_dir diff --git a/tests/unit/test_orchestrator_config_distribution.py b/tests/unit/test_orchestrator_config_distribution.py index a5f50e2..a9e243c 100644 --- a/tests/unit/test_orchestrator_config_distribution.py +++ b/tests/unit/test_orchestrator_config_distribution.py @@ -1,4 +1,5 @@ """Leader-side config distribution: build, discover, distribute.""" +from __future__ import annotations from unittest.mock import MagicMock @@ -271,6 +272,9 @@ def test_stops_recording_streams(self, tmp_path) -> None: mic = FakeStream("mic", kind="audio") session.add(cam) session.add(mic) + # Simulate streams that connected successfully — rollback only + # operates on _connected_streams, not the full _streams registry. + session._connected_streams = [cam, mic] session._state = SessionState.RECORDING session._rollback_after_distribute_failure() diff --git a/tests/unit/test_orchestrator_partial_connect.py b/tests/unit/test_orchestrator_partial_connect.py new file mode 100644 index 0000000..9d8e9cc --- /dev/null +++ b/tests/unit/test_orchestrator_partial_connect.py @@ -0,0 +1,143 @@ +"""Partial-connect semantics for SessionOrchestrator. + +Relies on the FakeStream helper in syncfield.testing, which supports +`fail_on_start=True` to raise from its connect() path. +""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +from syncfield.orchestrator import SessionOrchestrator +from syncfield.testing import FakeStream +from syncfield.types import SessionState + + +def test_one_stream_fails_others_still_connected(tmp_path: Path): + sess = SessionOrchestrator(host_id="h", output_dir=tmp_path) + sess.add(FakeStream("good_a")) + sess.add(FakeStream("bad", fail_on_start=True)) + sess.add(FakeStream("good_b")) + + sess.connect() + + assert sess.state is SessionState.CONNECTED + assert sess._stream_states["good_a"] == "connected" + assert sess._stream_states["bad"] == "failed" + assert sess._stream_states["good_b"] == "connected" + assert "bad" in sess._stream_errors + assert sess._stream_errors["bad"] + + +def test_all_streams_failing_raises_and_returns_to_idle(tmp_path: Path): + sess = SessionOrchestrator(host_id="h", output_dir=tmp_path) + sess.add(FakeStream("a", fail_on_start=True)) + sess.add(FakeStream("b", fail_on_start=True)) + + with pytest.raises(RuntimeError, match="no streams"): + sess.connect() + + assert sess.state is SessionState.IDLE + assert sess._stream_states["a"] == "failed" + assert sess._stream_states["b"] == "failed" + + +def test_startup_failure_event_reaches_health_system(tmp_path: Path): + from syncfield.health.detector import DetectorBase + from syncfield.health.severity import Severity + + class Spy(DetectorBase): + name = "startup-spy" + default_severity = Severity.INFO + + def __init__(self): + self.events = [] + + def observe_health(self, stream_id, event): + self.events.append(event) + + sess = SessionOrchestrator(host_id="h", output_dir=tmp_path) + spy = Spy() + sess.health.register(spy) + sess.add(FakeStream("good")) + sess.add(FakeStream("bad", fail_on_start=True)) + + sess.connect() + + import time + deadline = time.monotonic() + 1.0 + while time.monotonic() < deadline and not any( + e.fingerprint == "bad:startup-failure" for e in spy.events + ): + time.sleep(0.02) + + failure_events = [e for e in spy.events if e.fingerprint == "bad:startup-failure"] + assert failure_events, "no startup-failure event observed" + ev = failure_events[0] + assert ev.data.get("phase") == "connect" + assert ev.data.get("outcome") == "error" + assert ev.data.get("error") + + +def test_failed_stream_is_skipped_during_recording(tmp_path: Path): + from syncfield.testing import FakeStream + + class CountingFakeStream(FakeStream): + def __init__(self, stream_id, fail_on_start=False): + super().__init__(stream_id, fail_on_start=fail_on_start) + self.start_recording_calls = 0 + self.stop_recording_calls = 0 + + def start_recording(self, session_clock): + self.start_recording_calls += 1 + return super().start_recording(session_clock) + + def stop_recording(self): + self.stop_recording_calls += 1 + return super().stop_recording() + + sess = SessionOrchestrator(host_id="h", output_dir=tmp_path) + good = CountingFakeStream("good") + bad = CountingFakeStream("bad", fail_on_start=True) + sess.add(good) + sess.add(bad) + + sess.connect() + sess.start(countdown_s=0) + sess.stop() + + assert good.start_recording_calls > 0 + assert good.stop_recording_calls > 0 + # The failed stream's hardware was never opened, so start/stop_recording + # must not have been called on it. + assert bad.start_recording_calls == 0 + assert bad.stop_recording_calls == 0 + + +def test_disconnect_does_not_call_stream_disconnect_on_failed(tmp_path: Path): + from syncfield.testing import FakeStream + + class CountingFakeStream(FakeStream): + def __init__(self, stream_id, fail_on_start=False): + super().__init__(stream_id, fail_on_start=fail_on_start) + self.disconnect_calls = 0 + + def disconnect(self): + self.disconnect_calls += 1 + super().disconnect() + + sess = SessionOrchestrator(host_id="h", output_dir=tmp_path) + good = CountingFakeStream("good") + bad = CountingFakeStream("bad", fail_on_start=True) + sess.add(good) + sess.add(bad) + + sess.connect() + sess.disconnect() + + assert good.disconnect_calls == 1 + assert bad.disconnect_calls == 0 + assert sess._stream_states["good"] == "disconnected" + assert sess._stream_states["bad"] == "disconnected" + assert sess._stream_errors == {} diff --git a/tests/unit/test_types.py b/tests/unit/test_types.py index ed5d368..4105966 100644 --- a/tests/unit/test_types.py +++ b/tests/unit/test_types.py @@ -1,6 +1,11 @@ """Tests for syncfield.types.""" +import pytest +from dataclasses import FrozenInstanceError +from pathlib import Path + from syncfield.types import FrameTimestamp, SensorSample, SyncPoint +from syncfield.health.severity import Severity def test_sync_point_create_now(): @@ -134,10 +139,6 @@ def test_sensor_sample_nested_round_trip(): assert restored.channels["gestures"]["pinch"] == 0.95 -import pytest -from dataclasses import FrozenInstanceError -from pathlib import Path - from syncfield.types import ( ChirpSpec, FinalizationReport, @@ -182,6 +183,7 @@ def test_to_dict_round_trip(self): "supports_precise_timestamps": False, "is_removable": True, "produces_file": True, + "target_hz": None, "live_preview": True, } @@ -220,8 +222,42 @@ def test_to_dict(self): "kind": "drop", "at_ns": 42, "detail": "buffer overflow", + "severity": "info", + "source": "unknown", + "fingerprint": "", + "data": {}, } + def test_health_event_has_enrichment_fields_with_defaults(self): + ev = HealthEvent( + stream_id="cam", + kind=HealthEventKind.ERROR, + at_ns=1_000, + detail="boom", + ) + # new fields default to safe values when caller does not set them. + assert ev.severity == Severity.INFO + assert ev.source == "unknown" + assert ev.fingerprint == "" + assert ev.data == {} + + def test_health_event_to_dict_includes_new_fields(self): + ev = HealthEvent( + stream_id="cam", + kind=HealthEventKind.ERROR, + at_ns=1_000, + detail="boom", + severity=Severity.ERROR, + source="adapter:oak", + fingerprint="cam:adapter:xlink-error", + data={"stream": "__x_0_1"}, + ) + d = ev.to_dict() + assert d["severity"] == "error" + assert d["source"] == "adapter:oak" + assert d["fingerprint"] == "cam:adapter:xlink-error" + assert d["data"] == {"stream": "__x_0_1"} + class TestSampleEvent: def test_minimal(self): @@ -288,3 +324,4 @@ def test_minimal(self): ) assert report.host_id == "rig_01" assert report.finalizations == [] + diff --git a/tests/unit/test_types_capabilities.py b/tests/unit/test_types_capabilities.py index 755cf37..287ed75 100644 --- a/tests/unit/test_types_capabilities.py +++ b/tests/unit/test_types_capabilities.py @@ -16,3 +16,16 @@ def test_to_dict_includes_live_preview(): d = caps.to_dict() assert d["live_preview"] is False assert d["produces_file"] is False + + +def test_target_hz_defaults_to_none(): + from syncfield.types import StreamCapabilities + caps = StreamCapabilities() + assert caps.target_hz is None + + +def test_target_hz_round_trips_to_dict(): + from syncfield.types import StreamCapabilities + caps = StreamCapabilities(target_hz=30.0) + d = caps.to_dict() + assert d["target_hz"] == 30.0 diff --git a/tests/unit/test_types_finalization.py b/tests/unit/test_types_finalization.py index a401716..4fe5c1b 100644 --- a/tests/unit/test_types_finalization.py +++ b/tests/unit/test_types_finalization.py @@ -14,3 +14,31 @@ def test_finalization_report_accepts_pending_aggregation_status(): error=None, ) assert report.status == "pending_aggregation" + + +def test_finalization_report_incidents_default_empty(): + from syncfield.types import FinalizationReport + r = FinalizationReport( + stream_id="cam", status="completed", frame_count=10, file_path=None, + first_sample_at_ns=0, last_sample_at_ns=100, health_events=[], error=None, + ) + assert r.incidents == [] + + +def test_finalization_report_accepts_incidents(): + from syncfield.health.types import Incident + from syncfield.health.severity import Severity + from syncfield.types import FinalizationReport, HealthEvent, HealthEventKind + + ev = HealthEvent( + stream_id="cam", kind=HealthEventKind.ERROR, at_ns=1, detail="x", + severity=Severity.ERROR, source="detector:stream-stall", + fingerprint="cam:stream-stall", + ) + inc = Incident.opened_from(ev, title="stall") + r = FinalizationReport( + stream_id="cam", status="completed", frame_count=10, file_path=None, + first_sample_at_ns=0, last_sample_at_ns=100, health_events=[], error=None, + incidents=[inc], + ) + assert r.incidents == [inc] diff --git a/tests/unit/test_viewer_aggregation_snapshot.py b/tests/unit/test_viewer_aggregation_snapshot.py index 47ff2a4..926cf64 100644 --- a/tests/unit/test_viewer_aggregation_snapshot.py +++ b/tests/unit/test_viewer_aggregation_snapshot.py @@ -9,9 +9,18 @@ from syncfield.viewer.server import snapshot_to_dict -def test_snapshot_includes_aggregation_section_empty_by_default(): +def _make_snapshot_mock(**kwargs): snapshot = MagicMock() - snapshot.aggregation = None + snapshot.streams = {} + snapshot.active_incidents = [] + snapshot.resolved_incidents = [] + for k, v in kwargs.items(): + setattr(snapshot, k, v) + return snapshot + + +def test_snapshot_includes_aggregation_section_empty_by_default(): + snapshot = _make_snapshot_mock(aggregation=None) d = snapshot_to_dict(snapshot) assert "aggregation" in d assert d["aggregation"]["active_job"] is None @@ -30,13 +39,49 @@ def test_snapshot_serializes_active_job(): current_bytes=5_000_000, current_total_bytes=10_000_000, ) - snapshot = MagicMock() - snapshot.aggregation = MagicMock() - snapshot.aggregation.active_job = progress - snapshot.aggregation.queue_length = 1 - snapshot.aggregation.recent_jobs = [progress] + agg_mock = MagicMock() + agg_mock.active_job = progress + agg_mock.queue_length = 1 + agg_mock.recent_jobs = [progress] + snapshot = _make_snapshot_mock(aggregation=agg_mock) d = snapshot_to_dict(snapshot) assert d["aggregation"]["active_job"]["state"] == "running" assert d["aggregation"]["active_job"]["current_bytes"] == 5_000_000 assert d["aggregation"]["queue_length"] == 1 assert len(d["aggregation"]["recent_jobs"]) == 1 + + +def test_serialized_stream_includes_connection_state(): + from syncfield.viewer.state import StreamSnapshot, SessionSnapshot + + stream_snap = StreamSnapshot( + id="cam", + kind="video", + provides_audio_track=False, + produces_file=False, + frame_count=0, + last_sample_at_ns=None, + effective_hz=0.0, + latest_frame=None, + plot_points={}, + latest_pose={}, + connection_state="failed", + connection_error="Device not visible", + ) + sess_snap = SessionSnapshot( + host_id="h", + state="idle", + output_dir="/tmp", + sync_point_monotonic_ns=None, + sync_point_wall_clock_ns=None, + chirp_start_ns=None, + chirp_stop_ns=None, + chirp_enabled=False, + elapsed_s=0.0, + streams={"cam": stream_snap}, + active_incidents=[], + resolved_incidents=[], + ) + out = snapshot_to_dict(sess_snap) + assert out["streams"]["cam"]["connection_state"] == "failed" + assert out["streams"]["cam"]["connection_error"] == "Device not visible" diff --git a/tests/unit/test_writer.py b/tests/unit/test_writer.py index 5abc1f7..e2b59e0 100644 --- a/tests/unit/test_writer.py +++ b/tests/unit/test_writer.py @@ -271,3 +271,37 @@ def test_manifest_omits_session_config_when_none(self, tmp_path): import json manifest = json.loads(path.read_text()) assert "session_config" not in manifest + + +def test_log_incident_appends_to_incidents_jsonl(tmp_path): + from syncfield.health.severity import Severity + from syncfield.health.types import Incident + + def _ev(at_ns: int) -> HealthEvent: + return HealthEvent( + stream_id="cam", kind=HealthEventKind.ERROR, at_ns=at_ns, detail="x", + severity=Severity.ERROR, source="detector:stream-stall", + fingerprint="cam:stream-stall", + ) + + w = SessionLogWriter(tmp_path) + w.open() + try: + inc = Incident.opened_from(_ev(100), title="stall") + w.log_incident(inc) + inc.record_event(_ev(200)) + w.log_incident(inc) + inc.close(at_ns=300) + w.log_incident(inc) + finally: + w.close() + + path = tmp_path / "incidents.jsonl" + assert path.exists() + lines = path.read_text().strip().splitlines() + assert len(lines) == 3 + first = json.loads(lines[0]) + assert first["id"] == inc.id + assert first["event_count"] == 1 + last = json.loads(lines[-1]) + assert last["closed_at_ns"] == 300 diff --git a/tests/unit/viewer/test_poller.py b/tests/unit/viewer/test_poller.py index 1ddb33f..f876033 100644 --- a/tests/unit/viewer/test_poller.py +++ b/tests/unit/viewer/test_poller.py @@ -11,7 +11,6 @@ import syncfield as sf from syncfield.testing import FakeStream -from syncfield.types import HealthEventKind from syncfield.viewer.poller import SessionPoller @@ -79,24 +78,17 @@ def test_push_sample_shows_up_in_next_snapshot(self, tmp_path): finally: session.stop() - def test_push_health_surfaces_in_health_log(self, tmp_path): + def test_snapshot_has_incident_lists(self, tmp_path): + """Verify that SessionSnapshot carries active_incidents / resolved_incidents + (replacing the retired health_log / health_count fields from Task 20).""" session = _make_session(tmp_path) poller = SessionPoller(session, interval_s=0.01) - poller._register_callbacks() - - session.start() - try: - imu = session._streams["imu"] # type: ignore[attr-defined] - imu.push_health(HealthEventKind.WARNING, at_ns=123, detail="burst") - snap = poller._build_snapshot() - assert len(snap.health_log) >= 1 - assert any( - ev.kind == "warning" and ev.detail == "burst" - for ev in snap.health_log - ) - assert snap.streams["imu"].health_count >= 1 - finally: - session.stop() + snap = poller._build_snapshot() + # No incidents yet — both lists should be present and empty. + assert hasattr(snap, "active_incidents") + assert hasattr(snap, "resolved_incidents") + assert snap.active_incidents == [] + assert snap.resolved_incidents == [] def test_start_stop_and_get_snapshot_thread(self, tmp_path): """End-to-end smoke test of the polling thread.""" @@ -111,3 +103,23 @@ def test_start_stop_and_get_snapshot_thread(self, tmp_path): assert snap.host_id == "test_rig" finally: poller.stop() + + def test_poller_snapshot_includes_connection_state(self, tmp_path): + """Verify that StreamSnapshot carries connection_state and connection_error + from the orchestrator's per-stream tracking.""" + session = sf.SessionOrchestrator( + host_id="h", + output_dir=tmp_path, + sync_tone=sf.SyncToneConfig.silent(), + ) + session.add(FakeStream("good")) + session.add(FakeStream("bad", fail_on_start=True)) + + poller = SessionPoller(session) + session.connect() + + snap = poller._build_snapshot() + assert snap.streams["good"].connection_state == "connected" + assert snap.streams["good"].connection_error is None + assert snap.streams["bad"].connection_state == "failed" + assert snap.streams["bad"].connection_error is not None diff --git a/tests/unit/viewer/test_snapshot_incidents.py b/tests/unit/viewer/test_snapshot_incidents.py new file mode 100644 index 0000000..bd65191 --- /dev/null +++ b/tests/unit/viewer/test_snapshot_incidents.py @@ -0,0 +1,53 @@ +import dataclasses + +from syncfield.health.severity import Severity +from syncfield.health.types import Incident, IncidentSnapshot +from syncfield.types import HealthEvent, HealthEventKind +from syncfield.viewer.state import SessionSnapshot, StreamSnapshot + + +def _ev(at_ns: int) -> HealthEvent: + return HealthEvent( + stream_id="cam", kind=HealthEventKind.ERROR, at_ns=at_ns, detail="x", + severity=Severity.ERROR, source="detector:stream-stall", + fingerprint="cam:stream-stall", + ) + + +def test_session_snapshot_has_incident_fields(): + snap = SessionSnapshot( + host_id="h", state="recording", output_dir="/tmp", + sync_point_monotonic_ns=None, sync_point_wall_clock_ns=None, + chirp_start_ns=None, chirp_stop_ns=None, chirp_enabled=False, + elapsed_s=0.0, streams={}, active_incidents=[], resolved_incidents=[], + ) + assert snap.active_incidents == [] + assert snap.resolved_incidents == [] + + +def test_stream_snapshot_no_longer_has_health_count(): + fields = {f.name for f in dataclasses.fields(StreamSnapshot)} + assert "health_count" not in fields + assert "problem_count" not in fields + + +def test_session_snapshot_no_longer_has_health_log(): + fields = {f.name for f in dataclasses.fields(SessionSnapshot)} + assert "health_log" not in fields + + +def test_stream_snapshot_has_connection_state_fields(): + import dataclasses + from syncfield.viewer.state import StreamSnapshot + + fields = {f.name: f for f in dataclasses.fields(StreamSnapshot)} + assert "connection_state" in fields + assert "connection_error" in fields + + snap = StreamSnapshot( + id="cam", kind="video", provides_audio_track=False, produces_file=False, + frame_count=0, last_sample_at_ns=None, effective_hz=0.0, + latest_frame=None, plot_points={}, latest_pose={}, + ) + assert snap.connection_state == "idle" + assert snap.connection_error is None diff --git a/tests/unit/viewer/test_state.py b/tests/unit/viewer/test_state.py index eb2a69c..061eb84 100644 --- a/tests/unit/viewer/test_state.py +++ b/tests/unit/viewer/test_state.py @@ -11,7 +11,7 @@ import pytest -from syncfield.viewer.state import HealthEntry, StreamStatsBuffer +from syncfield.viewer.state import StreamStatsBuffer class TestStreamStatsBufferSamples: @@ -95,29 +95,6 @@ def test_missing_channel_fills_nan_forward(self): assert math.isnan(gx_ys[1]) -class TestStreamStatsBufferHealth: - def test_empty(self): - assert StreamStatsBuffer().snapshot_health() == [] - - def test_records_in_order(self): - buf = StreamStatsBuffer() - a = HealthEntry(stream_id="s", kind="warning", at_ns=10, detail="a") - b = HealthEntry(stream_id="s", kind="error", at_ns=20, detail="b") - buf.observe_health(a) - buf.observe_health(b) - events = buf.snapshot_health() - assert events == [a, b] - - def test_capped(self): - buf = StreamStatsBuffer(max_health=3) - # Override the internal deque size — real constructor caps at 20, - # but we test the cap principle with whatever the dataclass set up. - # The deque default is maxlen=20; this test verifies behaviour at - # whatever cap the buffer ended up with. - for i in range(30): - buf.observe_health( - HealthEntry(stream_id="s", kind="heartbeat", at_ns=i, detail=None) - ) - events = buf.snapshot_health() - assert len(events) <= 20 # matches default max_health - assert events[-1].at_ns == 29 # newest kept +# Health tracking has been moved from StreamStatsBuffer to HealthSystem/IncidentTracker. +# The per-stream HealthEntry buffer and observe_health/snapshot_health methods were +# removed in Task 20. Health-related viewer tests now live in test_snapshot_incidents.py. diff --git a/uv.lock b/uv.lock index ea9da8f..ff85d05 100644 --- a/uv.lock +++ b/uv.lock @@ -2330,7 +2330,7 @@ wheels = [ [[package]] name = "syncfield" -version = "0.3.17" +version = "0.3.19" source = { editable = "." } dependencies = [ { name = "av", version = "15.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },