From 3d1bd8433b6d078a91177729e453bc7c347b7e5c Mon Sep 17 00:00:00 2001 From: styu12 Date: Fri, 10 Apr 2026 17:52:01 -0700 Subject: [PATCH] feat(adapters,viewer): 4-phase stream lifecycle with live preview before recording Squashed from feat/oglo-tactile branch (25 commits). --- examples/README.md | 50 +++- examples/full_rig/README.md | 95 +++++++ examples/full_rig/record.py | 50 ++++ examples/iphone_mac_webcam/README.md | 33 ++- examples/mac_iphone_dual_oak/README.md | 216 ++++++++++++++++ src/syncfield/adapters/oak_camera.py | 313 +++++++++++++++++------ src/syncfield/adapters/oglo_tactile.py | 185 ++++++++++++-- src/syncfield/discovery/_ble.py | 2 +- src/syncfield/tone.py | 32 ++- src/syncfield/types.py | 20 +- src/syncfield/viewer/app.py | 23 +- src/syncfield/viewer/demo.py | 263 ++++++++++++++----- src/syncfield/viewer/fonts.py | 200 +++++++++++++++ src/syncfield/viewer/state.py | 55 +++- tests/unit/adapters/test_oak_camera.py | 101 +++++--- tests/unit/adapters/test_oglo_tactile.py | 29 +++ 16 files changed, 1439 insertions(+), 228 deletions(-) create mode 100644 examples/full_rig/README.md create mode 100644 examples/full_rig/record.py create mode 100644 examples/mac_iphone_dual_oak/README.md create mode 100644 src/syncfield/viewer/fonts.py diff --git a/examples/README.md b/examples/README.md index 15781e8..0ebccbd 100644 --- a/examples/README.md +++ b/examples/README.md @@ -9,22 +9,60 @@ Start with the simplest example that matches hardware you actually have, then sc | Example | Hardware | What it shows | |---|---|---| | [`iphone_mac_webcam/`](./iphone_mac_webcam/) | Mac built-in webcam + iPhone (Continuity Camera) | Shortest end-to-end recipe: two OpenCV video streams through `UVCWebcamStream`, live preview in the desktop viewer, MP4 + timestamps written to disk | +| [`mac_iphone_dual_oak/`](./mac_iphone_dual_oak/) | Mac webcam + iPhone + OAK-D-Lite + OAK-D-S2 | Four video streams on one host: two `UVCWebcamStream`s and two `OakCameraStream`s, each OAK pinned to its DepthAI serial. Optional stereo depth flags on either OAK. | +| [`full_rig/`](./full_rig/) | Mac webcam + iPhone + OAK-D-Lite + OAK-D-S2 + OGLO glove (BLE) | Mixes four video streams with a 100 Hz tactile sensor stream: same four cameras as above plus an `OgloTactileStream` that renders a 5-finger FSR plot card in the viewer. Shows how video + sensor streams share one atomic session. | More recipes will be added as the rigs they target come online. Expected next: -- **`oak_plus_webcam/`** — OAK-D Pro depth camera + Mac webcam (add depth to the dual-camera setup) - **`iphone_imu/`** — iPhone + BLE IMU (`BLEImuGenericStream`) showing mixed video + sensor streams - **`tactile_rig/`** — webcam + tactile sensor via `OgloTactileStream` showing custom-adapter integration - **`multi_host_pair/`** — two Macs on the same WiFi recording together with `LeaderRole` / `FollowerRole` -## How to run any example +## Run any example (`uv run`) -Every example follows the same shape: +Every example is a plain Python script inside this repo, so the easiest way to run one is with `uv run` from the repo root — no virtualenv setup, no `pip install` step. `uv` resolves the extras you pass with `--extra` against the root `pyproject.toml` and executes the script in a temporary env: ```bash -cd examples/ -pip install "syncfield[uvc,audio,viewer]" # extras vary — see the example's README -python record.py # blocking, opens the viewer +# iphone_mac_webcam — Mac webcam + iPhone Continuity Camera +uv run --extra uvc --extra audio --extra viewer \ + python examples/iphone_mac_webcam/record.py + +# mac_iphone_dual_oak — Mac webcam + iPhone + OAK-D-Lite + OAK-D-S2 +uv run --extra uvc --extra oak --extra audio --extra viewer \ + python examples/mac_iphone_dual_oak/record.py + +# full_rig — dual_oak + OGLO tactile glove over BLE +uv run --extra uvc --extra oak --extra ble --extra audio --extra viewer \ + python examples/full_rig/record.py +``` + +The first run for each extra set downloads the wheels into the uv cache (~10–30 s); every subsequent run is instant. + +> **Why `audio` is always there.** SyncField plays a 3/2/1 countdown tick and start/stop sync chirps through `sounddevice`. Without the `audio` extra installed the session runs in total silence and the console prints a WARNING telling you to add it. Every example in this directory includes `audio` in its recommended extras for that reason. + +### Common flags + +Every `record.py` accepts at least: + +```bash +--output-dir ./my_recording # where to write session artifacts (default ./output) +``` + +Individual examples have their own extra flags — see the per-example README. + +### Alternative: install once, then `python` + +If you'd rather install the package into a persistent environment and run `python record.py` directly (closer to how end-users would ship it), use either of: + +```bash +# Plain pip + venv +python -m venv .venv && source .venv/bin/activate +pip install "syncfield[uvc,oak,audio,viewer]" +python examples/mac_iphone_dual_oak/record.py + +# uv sync +uv sync --extra uvc --extra oak --extra audio --extra viewer +uv run python examples/mac_iphone_dual_oak/record.py ``` Inside the viewer, click **Record** to start the session, **Stop** to finish, and close the window to exit. Output files land in `./output/` by default; every example accepts `--output-dir` if you want a different location. diff --git a/examples/full_rig/README.md b/examples/full_rig/README.md new file mode 100644 index 0000000..d7cb9d1 --- /dev/null +++ b/examples/full_rig/README.md @@ -0,0 +1,95 @@ +# Full Rig — Mac + iPhone + Dual OAK + OGLO Glove + +**Five streams on one host.** Mac built-in webcam, iPhone over Continuity Camera, OAK-D-Lite, OAK-D-S2, and an OGLO tactile glove over BLE — all recorded as one atomic SyncField session. + +This is the next step up from [`mac_iphone_dual_oak/`](../mac_iphone_dual_oak/): same four video streams, plus a 100 Hz tactile sensor card that renders a live 5-finger FSR plot in the viewer card row. + +## Hardware checklist + +- [x] Mac with a working webcam +- [x] iPhone with Continuity Camera enabled +- [x] OAK-D-Lite connected via USB +- [x] OAK-D-S2 connected via USB (different bus if possible) +- [x] OGLO tactile glove powered on and advertising over BLE within ~5 m of the Mac +- [x] Bluetooth enabled on the Mac + +## Install + +```bash +pip install "syncfield[uvc,oak,ble,audio,viewer]" +``` + +| Extra | What it's for | +|---|---| +| `uvc` | OpenCV — Mac webcam + iPhone Continuity Camera | +| `oak` | DepthAI v3 — both OAK cameras | +| `ble` | `bleak` — BLE scan + notify subscription for the OGLO glove | +| `audio` | `sounddevice` — 3/2/1 countdown ticks + start/stop sync chirps | +| `viewer` | DearPyGui + NumPy — the bundled desktop viewer | + +## Run + +From the repo root: + +```bash +# Default serials + the currently paired OGLO address on the maintainer's rig +uv run --extra uvc --extra oak --extra ble --extra audio --extra viewer \ + python examples/full_rig/record.py + +# Override any identifier +uv run --extra uvc --extra oak --extra ble --extra audio --extra viewer \ + python examples/full_rig/record.py \ + --oak-lite 19443010813AF02C00 \ + --oak-d 1944301071781C1300 \ + --oglo-address C1718989-5A77-F3EB-B00A-01A758D99D54 \ + --oglo-hand right \ + --output-dir ./my_recording +``` + +Drop `--oglo-address` to fall back to a BLE name-substring scan for `"oglo"` — slower (scans for ~10 s at session start) but works on any Mac even if the CoreBluetooth address changes after re-pairing. + +## Finding the OGLO BLE address on a new Mac + +On macOS the BLE "address" returned by `bleak` is a per-host CoreBluetooth UUID — stable across reboots but different on every Mac. Discover yours with: + +```bash +uv run --extra ble python -c " +import asyncio, bleak +SERVICE = '4652535f-424c-4500-0000-000000000001' +async def main(): + results = await bleak.BleakScanner.discover(timeout=10, return_adv=True) + for addr, (d, ad) in results.items(): + if SERVICE.lower() in [s.lower() for s in (ad.service_uuids or [])]: + print(f'OGLO → address={addr} local_name={ad.local_name!r}') +asyncio.run(main()) +" +``` + +Copy the printed address into the `--oglo-address` flag (or into `DEFAULT_OGLO_ADDRESS` at the top of `record.py`). + +## Output + +``` +output/ +├── mac_webcam.mp4 mac_webcam.timestamps.jsonl +├── iphone.mp4 iphone.timestamps.jsonl +├── oak_lite.mp4 oak_lite.timestamps.jsonl +├── oak_d.mp4 oak_d.timestamps.jsonl +├── oglo.timestamps.jsonl (no .mp4 — OGLO is a sensor stream) +├── sync_point.json +├── manifest.json +└── session_log.jsonl +``` + +The OGLO stream is tagged `kind="sensor"` with `produces_file=False`, so you get one JSONL per tactile sample (thumb/index/middle/ring/pinky + device timestamp in nanoseconds) instead of a video file. In the viewer it renders as a multi-series line plot card alongside the four camera cards. + +## Troubleshooting + +| Symptom | Likely cause | Fix | +|---|---|---| +| `oglo` card stays blank | Glove is off, out of range, or claimed by another app | Power-cycle the glove; make sure the iOS egonaut app isn't connected to it at the same time | +| Session hangs for 10 s at connect | You dropped `--oglo-address` and it's running the fallback name scan | Normal — the scan runs once at connect and the session proceeds after it | +| `OGLO connection failed: no peripheral matched` | Name scan couldn't find the glove | Use the one-liner above to discover the address, then pass `--oglo-address` | +| `RuntimeError: No OAK devices found` | Old viewer still holds the USB handles | Close the previous viewer window, wait ~5 s, retry | + +See [`mac_iphone_dual_oak/README.md`](../mac_iphone_dual_oak/README.md) for OAK-specific troubleshooting and [`iphone_mac_webcam/README.md`](../iphone_mac_webcam/README.md) for webcam troubleshooting. diff --git a/examples/full_rig/record.py b/examples/full_rig/record.py new file mode 100644 index 0000000..58bb521 --- /dev/null +++ b/examples/full_rig/record.py @@ -0,0 +1,50 @@ +"""Record Mac webcam + iPhone + OAK-D-Lite + OAK-D-S2 + OGLO glove. + + pip install "syncfield[uvc,oak,ble,audio,viewer]" + python record.py +""" + +import argparse +from pathlib import Path + +import syncfield as sf +import syncfield.viewer +from syncfield.adapters import ( + OakCameraStream, + OgloTactileStream, + UVCWebcamStream, +) + +DEFAULT_OAK_LITE_SERIAL = "19443010813AF02C00" +DEFAULT_OAK_D_SERIAL = "1944301071781C1300" +DEFAULT_OGLO_ADDRESS = "C1718989-5A77-F3EB-B00A-01A758D99D54" + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--webcam-index", type=int, default=0) + parser.add_argument("--iphone-index", type=int, default=1) + parser.add_argument("--oak-lite", default=DEFAULT_OAK_LITE_SERIAL) + parser.add_argument("--oak-d", default=DEFAULT_OAK_D_SERIAL) + parser.add_argument("--oglo-address", default=DEFAULT_OGLO_ADDRESS) + parser.add_argument("--oglo-hand", default="right", choices=("left", "right")) + parser.add_argument("--output-dir", type=Path, default=Path("./output")) + args = parser.parse_args() + + args.output_dir.mkdir(parents=True, exist_ok=True) + + session = sf.SessionOrchestrator( + host_id="mac_studio", + output_dir=args.output_dir, + ) + session.add(UVCWebcamStream("mac_webcam", args.webcam_index, args.output_dir)) + session.add(UVCWebcamStream("iphone", args.iphone_index, args.output_dir)) + session.add(OakCameraStream("oak_lite", args.output_dir, device_id=args.oak_lite)) + session.add(OakCameraStream("oak_d", args.output_dir, device_id=args.oak_d)) + session.add(OgloTactileStream("oglo", address=args.oglo_address, hand=args.oglo_hand)) + + syncfield.viewer.launch(session) + + +if __name__ == "__main__": + main() diff --git a/examples/iphone_mac_webcam/README.md b/examples/iphone_mac_webcam/README.md index 942574b..179aa57 100644 --- a/examples/iphone_mac_webcam/README.md +++ b/examples/iphone_mac_webcam/README.md @@ -33,21 +33,38 @@ pip install "syncfield[uvc,audio,viewer]" | Extra | What it's for | |---|---| | `uvc` | OpenCV — the `UVCWebcamStream` adapter that drives both cameras | -| `audio` | `sounddevice` — needed by the sync tone / chirp path, even though chirps are skipped in single-host mode | +| `audio` | `sounddevice` — plays the 3/2/1 countdown ticks and start/stop sync chirps through the MacBook speakers | | `viewer` | `dearpygui` + `numpy` — the bundled desktop viewer | ## Run +The shortest way to run this example is `uv run` from the **repo root** — no virtualenv, no install step, `uv` resolves the extras against the root `pyproject.toml`: + ```bash -# Default: webcam at index 0, iPhone at index 1 -python record.py +# Default: Mac webcam at index 0, iPhone at index 1 +uv run --extra uvc --extra audio --extra viewer \ + python examples/iphone_mac_webcam/record.py # Custom indices, output dir, geometry -python record.py \ - --webcam-index 0 \ - --iphone-index 1 \ - --output-dir ./my_recording \ - --width 1920 --height 1080 --fps 30 +uv run --extra uvc --extra audio --extra viewer \ + python examples/iphone_mac_webcam/record.py \ + --webcam-index 0 \ + --iphone-index 1 \ + --output-dir ./my_recording \ + --width 1920 --height 1080 --fps 30 +``` + +> If you forget `--extra audio`, the recording still works but the countdown and chirps are silent and the console prints a WARNING telling you to add it. + +### Alternative: plain `python` after install + +If you'd rather `pip install` into a venv and run the script directly: + +```bash +python -m venv .venv && source .venv/bin/activate +pip install "syncfield[uvc,audio,viewer]" +cd examples/iphone_mac_webcam +python record.py ``` ### Not sure which index is which? diff --git a/examples/mac_iphone_dual_oak/README.md b/examples/mac_iphone_dual_oak/README.md new file mode 100644 index 0000000..dd4cf2c --- /dev/null +++ b/examples/mac_iphone_dual_oak/README.md @@ -0,0 +1,216 @@ +# Mac + iPhone + Dual OAK + +**Four-camera rig running on a single Mac.** The Mac's built-in webcam, an iPhone over Continuity Camera, an OAK-D-Lite, and an OAK-D-S2 — all recorded as one atomic SyncField session with live previews, a countdown, and synchronized start/stop across every device. + +This is the next step up from [`iphone_mac_webcam/`](../iphone_mac_webcam/): same session shape, just with two `OakCameraStream` adapters added alongside the two `UVCWebcamStream`s. The whole rig is driven from a single `SessionOrchestrator`. + +## What you'll see + +When you run `record.py`, the SyncField desktop viewer opens and: + +1. **Connect phase** — all four devices open in parallel; stream cards start showing live previews (gradient / face / OAK color sensor) before you press anything. +2. **Record click** — a big `· 3 ·` → `· 2 ·` → `· 1 ·` countdown appears on the session clock panel. +3. **Recording** — every stream begins writing simultaneously, then the start chirp plays (captured into any audio track present), state chip turns red, timer starts ticking. +4. **Stop click** — the stop chirp plays first (so it lands inside the recorded audio), then every stream finalizes its file. The session returns to `CONNECTED` and you can record another episode without re-opening hardware. +5. **Close** — closing the window disconnects every device cleanly. + +## Hardware checklist + +- [x] **Mac with a working webcam** — built-in FaceTime or any UVC camera at OpenCV index 0 +- [x] **iPhone** signed in to the same Apple ID, Continuity Camera enabled, within Bluetooth range of the Mac +- [x] **OAK-D-Lite** plugged into a USB-C port (or hub) on the Mac +- [x] **OAK-D-S2** plugged into a separate USB bus if possible — two OAKs on the same hub share bandwidth and can drop frames at full resolution +- [x] Ideally all four on **wall power**: Continuity can drop mid-session on battery, and OAKs draw enough current to matter on a laptop + +## Install + +```bash +pip install "syncfield[uvc,oak,audio,viewer]" +``` + +| Extra | What it's for | +|---|---| +| `uvc` | OpenCV — drives the Mac webcam and the iPhone Continuity Camera through `UVCWebcamStream` | +| `oak` | DepthAI v3 — drives both OAK cameras through `OakCameraStream` | +| `audio` | `sounddevice` — plays the 3-2-1 countdown ticks and start/stop chirps through the MacBook speakers. **Without this extra the session runs in total silence** — no error, just a WARNING in the console that says to install it. | +| `viewer` | DearPyGui + NumPy — the bundled desktop viewer | + +If you want depth output from either OAK, the `uvc` extra is also required (the MP4 writer for depth uses OpenCV). Both extras combined are what ship as `syncfield[oak,uvc]`. + +### Why you need the `audio` extra + +SyncField plays four audible cues during a recording session: + +1. **Countdown beep** at each of `3 → 2 → 1` (100 ms C6 tick) +2. **Start chirp** (rising 400 → 2500 Hz sweep, 500 ms) the moment every stream is actually writing +3. **Stop chirp** (falling 2500 → 400 Hz sweep, 500 ms) the moment you press Stop +4. Every one of them is played through the system default output — that's your MacBook speakers unless you've reassigned audio output in macOS + +All four go through `sounddevice`. If the `audio` extra isn't installed, `create_default_player()` falls back to `SilentChirpPlayer` and you'll hear nothing. The console will show a WARNING like: + +``` +WARNING sounddevice unavailable (No module named 'sounddevice'). The countdown + ticks and start/stop chirps will be SILENT. Install the audio extra + to hear them: pip install 'syncfield[audio]' +``` + +If you see that line and no sound plays, run ``pip install 'syncfield[audio]'`` and rerun ``record.py``. + +## Run + +Every command below is copy-pastable from the **repo root** (`syncfield-python/`) using `uv run` — no venv, no install step, `uv` resolves the extras against the root `pyproject.toml` and executes the script in a temporary env. + +### Step 1: list attached OAKs + +First time through, discover the serials of your two OAKs so you can pin each one: + +```bash +uv run --extra uvc --extra oak --extra audio --extra viewer \ + python examples/mac_iphone_dual_oak/record.py --list-oak +``` + +``` +Found 2 OAK device(s): + deviceId=19443010813AF02C00 bus='2.1.4' product=OAK-D-LITE-AF + sensors={: 'OV7251', : 'IMX214', : 'OV7251'} + deviceId=1944301071781C1300 bus='0.1' product=OAK-D-S2-AF + sensors={: 'OV9282', : 'IMX378', : 'OV9282'} +``` + +The `deviceId` field is the persistent DepthAI serial — it never changes across reboots, unlike the USB bus topology (`name`). Copy the serial for each board. + +### Step 2: record + +```bash +# Default serials match the maintainer's rig; no flags needed +uv run --extra uvc --extra oak --extra audio --extra viewer \ + python examples/mac_iphone_dual_oak/record.py + +# Override serials + webcam indices + output directory +uv run --extra uvc --extra oak --extra audio --extra viewer \ + python examples/mac_iphone_dual_oak/record.py \ + --oak-lite 19443010813AF02C00 \ + --oak-d 1944301071781C1300 \ + --webcam-index 0 --iphone-index 1 \ + --output-dir ./my_recording +``` + +> If you forget `--extra audio`, the recording still works but the 3/2/1 countdown ticks and start/stop chirps play silently and the console prints a WARNING telling you to add it. + +### Enabling depth + +Both OAKs default to **RGB-only** so the USB-3 bus has headroom for four simultaneous video streams. To enable stereo depth on either: + +```bash +# Depth on the OAK-D-S2 only +uv run --extra uvc --extra oak --extra audio --extra viewer \ + python examples/mac_iphone_dual_oak/record.py --oak-d-depth + +# Depth on the OAK-D-Lite only +uv run --extra uvc --extra oak --extra audio --extra viewer \ + python examples/mac_iphone_dual_oak/record.py --oak-lite-depth + +# Depth on both +uv run --extra uvc --extra oak --extra audio --extra viewer \ + python examples/mac_iphone_dual_oak/record.py --oak-lite-depth --oak-d-depth +``` + +If you turn on depth for both OAKs on a single USB bus, watch the health events table — drops will appear there if the bandwidth ceiling gets hit. + +### Alternative: plain `python` after install + +If you'd rather `pip install` into a venv and run the script directly: + +```bash +python -m venv .venv && source .venv/bin/activate +pip install "syncfield[uvc,oak,audio,viewer]" +cd examples/mac_iphone_dual_oak +python record.py +``` + +## Output + +``` +output/ +├── mac_webcam.mp4 # Mac webcam video +├── mac_webcam.timestamps.jsonl # Per-frame capture timestamps +├── iphone.mp4 # iPhone Continuity Camera video +├── iphone.timestamps.jsonl +├── oak_lite.mp4 # OAK-D-Lite RGB +├── oak_lite.timestamps.jsonl +├── oak_lite.depth.mp4 # (only if --oak-lite-depth) +├── oak_lite.depth.timestamps.jsonl +├── oak_d.mp4 # OAK-D-S2 RGB +├── oak_d.timestamps.jsonl +├── oak_d.depth.mp4 # (only if --oak-d-depth) +├── oak_d.depth.timestamps.jsonl +├── sync_point.json # Session anchor + chirp metadata +├── manifest.json # Stream capabilities + file paths +└── session_log.jsonl # Crash-safe timeline log +``` + +## Architecture at a glance + +``` +┌───────────────────────────────────────────────────────────────────────┐ +│ SessionOrchestrator (host_id = mac_studio) │ +│ │ +│ ┌──────────────────┐ ┌──────────────────┐ │ +│ │ UVCWebcamStream │ │ UVCWebcamStream │ │ +│ │ id=mac_webcam │ │ id=iphone │ │ +│ │ device_index=0 │ │ device_index=1 │ │ +│ │ → mac_webcam.mp4 │ │ → iphone.mp4 │ │ +│ └──────────────────┘ └──────────────────┘ │ +│ │ +│ ┌──────────────────────┐ ┌──────────────────────┐ │ +│ │ OakCameraStream │ │ OakCameraStream │ │ +│ │ id=oak_lite │ │ id=oak_d │ │ +│ │ device_id=1944…2C00 │ │ device_id=1944…1C00 │ │ +│ │ → oak_lite.mp4 │ │ → oak_d.mp4 │ │ +│ │ (+ .depth.mp4) │ │ (+ .depth.mp4) │ │ +│ └──────────────────────┘ └──────────────────────┘ │ +│ │ +│ Lifecycle: connect → countdown 3/2/1 → start_recording + chirp │ +│ → RECORDING → stop: chirp + stop_recording → CONNECTED │ +└───────────────────────────────────────────────────────────────────────┘ +``` + +Every adapter conforms to the same `Stream` SPI — the orchestrator doesn't know the difference between an OpenCV webcam and a DepthAI pipeline. Adding a fifth stream (a BLE IMU, a tactile sensor, a custom source) is one more `session.add(...)` call. + +## Pinning OAKs by serial + +The `device_id` kwarg on `OakCameraStream` is the single most important flag in this example. Without it, both `OakCameraStream` instances would call `dai.Device()` with no filter, and DepthAI would hand the "first available device" to whichever pipeline opened first — the second pipeline then finds nothing and raises. Pinning each stream to its persistent DepthAI serial removes the race entirely. + +The serials are stable across: + +- USB port swaps +- Reboots +- macOS updates +- DepthAI version bumps + +They're **not** stable across a physical Myriad-X firmware reflash, which is rare enough to ignore. + +## Troubleshooting + +| Symptom | Likely cause | Fix | +|---|---|---| +| `python record.py --list-oak` shows 0 devices | USB enumeration issue | Replug both OAKs, check the lights come on; `depthai` ships with a diagnostic tool you can run | +| Only one OAK appears | USB hub bandwidth limit | Split the two OAKs across separate USB buses; OAK-D-S2 prefers USB 3 SuperSpeed | +| `RuntimeError: X_LINK_ALREADY_OPEN` | Device still held by a previous session | Wait ~10 s for the OAK to recycle, or replug | +| Both previews freeze after ~1 s | Turned depth on for both OAKs on one bus | Run RGB-only (drop the `--*-depth` flags) or move one OAK to a separate bus | +| `ImportError: depthai` | Missing OAK extra | `pip install "syncfield[oak]"` | +| `ImportError: cv2` | Missing UVC extra | `pip install "syncfield[uvc]"` | +| iPhone card stays blank | Continuity Camera dropped | Wake the iPhone, plug it into power, check System Settings → AirPlay & Handoff | +| Health table shows `drop` events | USB bandwidth saturated | Lower resolution / FPS on one OAK, or split across USB buses | + +## Why no chirp timestamps in `sync_point.json`? + +None of the four streams declares `provides_audio_track=True`, so the orchestrator logs `"no audio-capable stream registered; chirp injection disabled"` and the `chirp_*_ns` fields stay `null` in the written artifacts. This is the correct behavior for a single-host video-only rig — the sync service falls back to per-stream timestamp alignment, which is exactly what you want when every stream is on the same monotonic clock. + +The chirp path re-activates the moment you add an audio-capable stream (e.g., a microphone, or once you move to multi-host mode where a peer Mac captures audio). The recording code stays identical. + +## Next steps + +1. **Go multi-host.** Run this same script on a second Mac with `FollowerRole` and add a microphone stream so the chirp cross-correlation kicks in. See the [`multi-host`](../../website/docs/sdk/multi-host.md) docs. +2. **Add a BLE IMU.** Register a `BLEImuGenericStream` alongside the cameras — your sensor plot card appears automatically in the viewer. +3. **Replay through the sync service.** The output directory is exactly what `syncfield-app`'s `/api/v1/sync` endpoint expects. diff --git a/src/syncfield/adapters/oak_camera.py b/src/syncfield/adapters/oak_camera.py index 29b0398..2fb07a3 100644 --- a/src/syncfield/adapters/oak_camera.py +++ b/src/syncfield/adapters/oak_camera.py @@ -12,6 +12,31 @@ Both extras are available together via ``syncfield[all]``. +Lifecycle +--------- + +This adapter implements the 4-phase :class:`~syncfield.Stream` SPI so +the viewer can show live OAK preview frames **before** Record is +pressed: + +* ``prepare()`` — create the output directory. +* ``connect()`` — discover the target device, build and start the + DepthAI pipeline, spawn the capture thread in preview-only mode. + ``latest_frame`` begins updating as soon as the pipeline warms up. +* ``start_recording()`` — open the MP4 (and optional depth binary) + writer and flip the ``_recording`` flag so the running capture loop + starts writing and emitting :class:`SampleEvent`\\ s. +* ``stop_recording()`` — flip the flag back off, close the writers, + return the finalization report. The capture loop **keeps running** + so preview stays live and the operator can start another recording + without rebuilding the pipeline. +* ``disconnect()`` — signal the capture thread, join it, and + release the DepthAI pipeline. + +Legacy ``start()`` / ``stop()`` are still supported for 0.1-era code +paths — they collapse the new lifecycle into a one-shot +``connect + start_recording`` / ``stop_recording + disconnect`` pair. + The adapter is intentionally thinner than the full-featured OakCamera class used inside opengraph-studio/recorder — it ships the 80% common case (RGB + optional depth) so the code stays small and easy to extend. For IMU, stereo @@ -55,17 +80,16 @@ class OakCameraStream(StreamBase): """Captures RGB (and optional depth) from a Luxonis OAK camera. - Lifecycle: - 1. ``prepare()`` discovers a device, builds a DepthAI pipeline with an - RGB ``Camera`` node (and optionally a ``StereoDepth`` node), and - starts the pipeline. - 2. ``start()`` opens the MP4 writer (and depth raw-bin file if - depth is enabled), then spins up a background thread that reads - frames in a tight loop, timestamps each read with - ``time.monotonic_ns()``, writes the frame to disk, and emits a - :class:`~syncfield.types.SampleEvent`. - 3. ``stop()`` signals the thread, joins it, releases the pipeline - and writers, and returns a :class:`FinalizationReport`. + See the module docstring for the full 4-phase lifecycle. In short: + + * ``connect()`` builds and starts the DepthAI pipeline so + :attr:`latest_frame` begins updating (live viewer preview). + * ``start_recording(session_clock)`` opens the MP4 (and optional + depth bin) writer and flips the ``_recording`` flag so the + already-running capture loop begins writing and emitting samples. + * ``stop_recording()`` closes the writers but leaves the pipeline + running so preview stays live. + * ``disconnect()`` tears down the pipeline. Args: id: Stream id (also used as the output file name: ``{id}.mp4``). @@ -131,6 +155,13 @@ def __init__( self._first_at: Optional[int] = None self._last_at: Optional[int] = None + # True while the capture loop should write frames to disk and + # emit ``SampleEvent``. ``connect()`` leaves this False so the + # CONNECTED preview phase never touches the filesystem; + # ``start_recording()`` flips it True; ``stop_recording()`` + # flips it False again while the capture thread keeps running. + self._recording = False + # Live preview support — the viewer reads ``latest_frame`` to render # the stream card thumbnail. ``_frame_lock`` protects handoff between # the capture thread and the reader. @@ -138,42 +169,100 @@ def __init__( self._latest_frame: Any = None # ------------------------------------------------------------------ - # Stream SPI + # Stream SPI — 4-phase lifecycle # ------------------------------------------------------------------ def prepare(self) -> None: - """Discover a device and build the DepthAI pipeline. + """Create the output directory. + + The heavy lifting — device discovery, pipeline build, pipeline + start — happens in :meth:`connect` so the viewer can show a + live preview as soon as the session enters the ``CONNECTED`` + state. ``prepare()`` stays cheap and idempotent. + """ + self._output_dir.mkdir(parents=True, exist_ok=True) + + #: How many times to poll ``dai.Device.getAllAvailableDevices()`` + #: before giving up. The first call often returns only a subset on + #: dual-OAK rigs because XLink enumeration is asynchronous — the + #: second board shows up after 0.5–1 s. Three tries with a short + #: sleep between comfortably covers that gap without extending the + #: happy-path connect time (which still returns on the first call). + _ENUMERATE_RETRIES = 3 + _ENUMERATE_RETRY_DELAY_S = 0.8 + + def _locate_device(self) -> Any: + """Find the target OAK, retrying the XLink enumeration if needed. + + ``dai.Device.getAllAvailableDevices()`` is an asynchronous probe + over XLink. On dual-OAK rigs the first call frequently returns + only one of the two boards, then 500–1000 ms later the second + board appears. When the caller pinned a specific ``device_id`` + and it's missing from the first probe, we re-probe up to + :attr:`_ENUMERATE_RETRIES` times before raising — that's the + difference between a flaky startup and a hard failure the + operator has to replug around. + + When ``device_id`` is ``None`` (pick any), we return on the + first non-empty result so auto-pick stays fast. + + Raises: + RuntimeError: If no device is found after exhausting all + retries, or if the pinned ``device_id`` never appears. + """ + last_seen: list = [] + for attempt in range(self._ENUMERATE_RETRIES): + devices = dai.Device.getAllAvailableDevices() + last_seen = devices + if devices: + if self._device_id is None: + return devices[0] + for dev in devices: + if getattr(dev, "deviceId", None) == self._device_id: + return dev + if attempt < self._ENUMERATE_RETRIES - 1: + time.sleep(self._ENUMERATE_RETRY_DELAY_S) + + if not last_seen: + raise RuntimeError( + "No OAK devices found after " + f"{self._ENUMERATE_RETRIES} enumeration attempts. Check " + "cables, power, and that no other DepthAI process is " + "holding the board." + ) + available = [getattr(d, "deviceId", "?") for d in last_seen] + raise RuntimeError( + f"OAK device_id {self._device_id!r} not found after " + f"{self._ENUMERATE_RETRIES} enumeration attempts. Visible " + f"devices: {available}. If the missing device is physically " + f"attached, unplug and replug its USB cable — Myriad-X " + f"boards can enter a zombie state after an unclean shutdown." + ) - When multiple OAK devices are connected, the ``device_id`` - constructor argument (a ``deviceId`` serial string as returned - by :func:`depthai.Device.getAllAvailableDevices`) selects which - one to open. If omitted, the first available device is used. + def connect(self) -> None: + """Open the DepthAI pipeline and spawn the preview capture thread. + + Discovers the requested device (by ``device_id`` if supplied, + else the first attached OAK), builds the pipeline, starts it, + and spawns the capture thread in **preview-only** mode: + :attr:`latest_frame` updates continuously, but no file is + written and no :class:`SampleEvent` is emitted until + :meth:`start_recording` flips the ``_recording`` flag. + + Idempotent — calling ``connect()`` on an already-connected + stream is a no-op so legacy callers that jump straight to + ``start()`` don't spawn a second capture thread. Raises: RuntimeError: If no OAK devices are connected, or if the requested ``device_id`` is not among the currently - attached devices. + attached devices after :attr:`_ENUMERATE_RETRIES` + probe attempts. """ - self._output_dir.mkdir(parents=True, exist_ok=True) + if self._thread is not None and self._thread.is_alive(): + return - devices = dai.Device.getAllAvailableDevices() - if not devices: - raise RuntimeError("No OAK devices found") - - if self._device_id is not None: - matching = [ - d for d in devices - if getattr(d, "deviceId", None) == self._device_id - ] - if not matching: - available = [getattr(d, "deviceId", "?") for d in devices] - raise RuntimeError( - f"OAK device_id {self._device_id!r} not found. " - f"Available: {available}" - ) - selected = matching[0] - else: - selected = devices[0] + selected = self._locate_device() self._pipeline = self._build_pipeline() # DepthAI v3 build() accepts an optional device info; older @@ -188,8 +277,33 @@ def prepare(self) -> None: # camera settles. Keeps the capture loop's error counters clean. time.sleep(1.0) - def start(self, session_clock: SessionClock) -> None: - """Open output files and launch the background capture thread.""" + # Reset counters so a reconnect starts clean. + self._recording = False + self._frame_count = 0 + self._depth_frame_count = 0 + self._first_at = None + self._last_at = None + self._stop_event.clear() + self._thread = threading.Thread( + target=self._capture_loop, name=f"oak-{self.id}", daemon=True + ) + self._thread.start() + + def start_recording(self, session_clock: SessionClock) -> None: + """Open output files and flip the recording flag. + + The capture thread is already running from :meth:`connect`, so + this is a :class:`cv2.VideoWriter` construction plus a boolean + flip — fast enough to run atomically across every stream in + the orchestrator's start phase. + + If the caller skipped :meth:`connect` (legacy 0.1 ``start()`` + path), the pipeline is started here first so the writer always + has a feeder. + """ + if self._thread is None or not self._thread.is_alive(): + self.connect() + width, height = self._rgb_resolution fourcc = cv2.VideoWriter_fourcc(*"mp4v") self._video_writer = cv2.VideoWriter( @@ -198,29 +312,21 @@ def start(self, session_clock: SessionClock) -> None: if self._depth_enabled: self._depth_file = open(self._depth_path, "wb") - self._stop_event.clear() - self._thread = threading.Thread( - target=self._capture_loop, name=f"oak-{self.id}", daemon=True - ) - self._thread.start() + # Flip the flag LAST so the capture loop doesn't race into a + # half-built writer. + self._recording = True - def stop(self) -> FinalizationReport: - """Signal the thread, release the pipeline, return the report.""" - self._stop_event.set() - if self._thread is not None: - self._thread.join(timeout=3.0) + def stop_recording(self) -> FinalizationReport: + """Flip recording off, close the writers, return the report. + The pipeline stays live so the viewer preview keeps rendering + and the operator can start a fresh recording on the same + session without re-opening hardware. + """ + self._recording = False self._release_writers() - self._release_pipeline() - extra_channels: dict[str, Any] = {} - if self._depth_enabled: - extra_channels["depth_frame_count"] = self._depth_frame_count - extra_channels["depth_path"] = ( - str(self._depth_path) if self._depth_frame_count > 0 else None - ) - - report = FinalizationReport( + return FinalizationReport( stream_id=self.id, status="completed", frame_count=self._frame_count, @@ -230,8 +336,40 @@ def stop(self) -> FinalizationReport: health_events=list(self._collected_health), error=None, ) - # Expose depth stats through the health_events buffer so consumers - # that only look at FinalizationReport still get visibility. + + def disconnect(self) -> None: + """Stop the capture thread and release the DepthAI pipeline. + + Called when the session returns to ``IDLE``. Idempotent — + calling twice is safe. After this call the adapter holds no + DepthAI handles. + """ + self._stop_event.set() + if self._thread is not None: + self._thread.join(timeout=3.0) + self._thread = None + self._release_pipeline() + + # ------------------------------------------------------------------ + # Legacy one-shot lifecycle + # ------------------------------------------------------------------ + + def start(self, session_clock: SessionClock) -> None: + """Legacy one-shot start — ``connect() + start_recording()``. + + Exists so 0.1-era scripts that call ``prepare() → start() → + stop()`` keep working without changes. New callers (the + viewer, the 4-phase orchestrator path) should use + :meth:`connect` and :meth:`start_recording` directly to get + live preview before the first Record click. + """ + self.connect() + self.start_recording(session_clock) + + def stop(self) -> FinalizationReport: + """Legacy one-shot stop — ``stop_recording() + disconnect()``.""" + report = self.stop_recording() + self.disconnect() return report # ------------------------------------------------------------------ @@ -277,11 +415,23 @@ def _build_pipeline(self) -> Any: def _capture_loop(self) -> None: """Body of the background thread — tight read/timestamp/write loop. - The timestamp is captured *immediately* after ``queue.get()`` so - the jitter between the physical frame and the recorded timestamp - stays as small as possible. Depth frames are consumed in the - same tick with ``tryGet()`` so depth and RGB share the same - monotonic anchor. + Two phases, distinguished by the ``_recording`` flag: + + * **Preview** (``_recording == False``) — publish every frame + to ``latest_frame`` so the viewer card stays live, but do + **not** write to the MP4, update frame counters, or emit + ``SampleEvent``. The capture runs continuously from the + moment :meth:`connect` spawned this thread. + * **Recording** (``_recording == True``) — same preview + publish, plus write to the ``VideoWriter``, advance + ``_frame_count``, drain a depth tick, and emit + ``SampleEvent``. The capture timestamp is sampled + *immediately* after ``queue.get()`` so the jitter between + the physical frame and the recorded monotonic time stays + as small as possible. + + The loop exits when ``_stop_event`` fires (from + :meth:`disconnect`). """ while not self._stop_event.is_set(): rgb_msg = self._safe_get_rgb() @@ -290,27 +440,30 @@ def _capture_loop(self) -> None: continue frame = rgb_msg.getCvFrame() - if self._first_at is None: - self._first_at = capture_ns - self._last_at = capture_ns - self._frame_count += 1 - # Publish the latest frame for live preview (viewer reads this). + # Always publish the latest frame for live preview — the + # viewer reads this in both CONNECTED and RECORDING states. with self._frame_lock: self._latest_frame = frame - if self._video_writer is not None: - self._video_writer.write(frame) - self._emit_sample( - SampleEvent( - stream_id=self.id, - frame_number=self._frame_count - 1, - capture_ns=capture_ns, + if self._recording: + if self._first_at is None: + self._first_at = capture_ns + self._last_at = capture_ns + self._frame_count += 1 + + if self._video_writer is not None: + self._video_writer.write(frame) + self._emit_sample( + SampleEvent( + stream_id=self.id, + frame_number=self._frame_count - 1, + capture_ns=capture_ns, + ) ) - ) - if self._depth_enabled: - self._drain_depth_tick() + if self._depth_enabled: + self._drain_depth_tick() def _safe_get_rgb(self) -> Any: """Pull one RGB frame from the queue, swallowing timeouts.""" diff --git a/src/syncfield/adapters/oglo_tactile.py b/src/syncfield/adapters/oglo_tactile.py index df6044d..328c2d5 100644 --- a/src/syncfield/adapters/oglo_tactile.py +++ b/src/syncfield/adapters/oglo_tactile.py @@ -31,6 +31,31 @@ (10 samples × 10 ms) so consumers see uniform 100 Hz spacing instead of a cluster at every batch boundary — critical for downstream jitter analysis. +Lifecycle +--------- + +This adapter implements the 4-phase :class:`~syncfield.Stream` SPI so +the viewer can plot live FSR values **before** Record is pressed: + +* ``prepare()`` — resolve the target peripheral (explicit address + or name scan). +* ``connect()`` — open the BLE client, subscribe to the notify + characteristic, and start the background asyncio loop. Samples + begin flowing into :meth:`_handle_payload` which emits + :class:`SampleEvent` unconditionally so the viewer's sensor card + updates its plot even during the preview phase. +* ``start_recording()`` — flip the ``_recording`` flag so incoming + samples also advance the finalization counters. +* ``stop_recording()`` — flip ``_recording`` back off, snapshot the + counters into a :class:`FinalizationReport`. The BLE session + stays live so the plot keeps updating. +* ``disconnect()`` — signal the asyncio loop to stop, release the + BLE client. + +Legacy ``start()`` / ``stop()`` still work — they collapse the new +lifecycle into a ``connect + start_recording`` / ``stop_recording + +disconnect`` pair for 0.1-era callers. + Requires the optional ``ble`` extra:: pip install 'syncfield[ble]' @@ -147,24 +172,40 @@ def __init__( self._thread: Optional[threading.Thread] = None self._stop_event = threading.Event() + # True while the capture loop should count samples toward the + # finalization report. ``connect()`` leaves this False so the + # CONNECTED preview phase still drives the viewer plot (samples + # are emitted unconditionally via ``_emit_sample``) without + # polluting the recording's frame counters. + self._recording = False self._frame_count = 0 self._first_at: Optional[int] = None self._last_at: Optional[int] = None # ------------------------------------------------------------------ - # Stream SPI + # Stream SPI — 4-phase lifecycle # ------------------------------------------------------------------ def prepare(self) -> None: - """Resolve the target device (explicit address or name scan).""" + """Resolve the target device (explicit address or name scan). + + Heavy connect work (opening the BleakClient, subscribing to + notifications) happens in :meth:`connect` so the viewer can + show live sensor values as soon as the session enters + ``CONNECTED``. This step is cheap and idempotent — repeated + calls are safe. + """ if self._address is not None: # bleak accepts either a BLEDevice or a plain address string. self._device = self._address return + if self._device is not None: + return + # Name-filtered scan. Run synchronously by spinning a throwaway - # asyncio loop — prepare() is called once, before the capture - # loop starts, so it's OK to block here briefly. + # asyncio loop — prepare() runs once before connect() and the + # scan budget is bounded by ``scan_timeout``. self._device = asyncio.run(self._scan_for_glove()) if self._device is None: raise RuntimeError( @@ -172,8 +213,28 @@ def prepare(self) -> None: f"(name filter={self._ble_name!r}, timeout={self._scan_timeout}s)" ) - def start(self, session_clock: SessionClock) -> None: - """Kick off the background asyncio loop that drives the BLE client.""" + def connect(self) -> None: + """Open the BLE session and start the background asyncio loop. + + After this call the ``_on_notify`` handler is subscribed to the + glove's notify characteristic and decoded samples flow through + :meth:`_handle_payload` — which emits :class:`SampleEvent` + unconditionally so the viewer's sensor card's live plot starts + updating immediately, even while the session is still in + ``CONNECTED`` (pre-record) state. + + Idempotent — a second call while the loop thread is already + running is a no-op. + """ + if self._thread is not None and self._thread.is_alive(): + return + if self._device is None: + self.prepare() + + self._recording = False + self._frame_count = 0 + self._first_at = None + self._last_at = None self._stop_event.clear() self._thread = threading.Thread( target=self._run_event_loop, @@ -182,12 +243,32 @@ def start(self, session_clock: SessionClock) -> None: ) self._thread.start() - def stop(self) -> FinalizationReport: - """Signal the loop to exit and collect the finalization report.""" - self._stop_event.set() - if self._thread is not None: - self._thread.join(timeout=3.0) + def start_recording(self, session_clock: SessionClock) -> None: + """Begin counting incoming samples toward the recording report. + + The asyncio loop is already running from :meth:`connect`, so + this is just a boolean flip. The first sample that arrives + after this call lands at ``frame_count == 1``; samples that + arrived during the preview phase are discarded for the + finalization report but were already visible on the live plot + through the ``on_sample`` callbacks. + + If the caller skipped :meth:`connect` (legacy 0.1 ``start()`` + path), the BLE session is started here first so the recording + has a data source. + """ + if self._thread is None or not self._thread.is_alive(): + self.connect() + self._recording = True + + def stop_recording(self) -> FinalizationReport: + """Flip recording off and snapshot the finalization report. + The BLE session **stays live** so the viewer plot keeps + updating and the operator can start another recording on the + same session without rescanning or reconnecting. + """ + self._recording = False return FinalizationReport( stream_id=self.id, status="completed", @@ -199,6 +280,38 @@ def stop(self) -> FinalizationReport: error=None, ) + def disconnect(self) -> None: + """Signal the asyncio loop to stop and release the BLE client. + + Called when the session returns to ``IDLE``. Idempotent — a + second call on an already-disconnected stream is a no-op. + After this call the adapter holds no BLE handles. + """ + self._stop_event.set() + if self._thread is not None: + self._thread.join(timeout=3.0) + self._thread = None + + # ------------------------------------------------------------------ + # Legacy one-shot lifecycle + # ------------------------------------------------------------------ + + def start(self, session_clock: SessionClock) -> None: + """Legacy one-shot start — ``connect() + start_recording()``. + + Exists so 0.1-era scripts that called ``prepare() → start() → + stop()`` keep running unchanged. New callers should use + :meth:`connect` + :meth:`start_recording` directly. + """ + self.connect() + self.start_recording(session_clock) + + def stop(self) -> FinalizationReport: + """Legacy one-shot stop — ``stop_recording() + disconnect()``.""" + report = self.stop_recording() + self.disconnect() + return report + # ------------------------------------------------------------------ # Async runtime on the background thread # ------------------------------------------------------------------ @@ -243,13 +356,26 @@ async def _on_notify(self, characteristic: Any, payload: bytes) -> None: self._handle_payload(bytes(payload)) async def _scan_for_glove(self) -> Any: - """Scan for a peripheral whose advertised name contains ``ble_name``.""" + """Scan for a peripheral whose advertised name contains ``ble_name``. + + Matches against **both** the bleak device ``name`` (which is the + hardware module's peripheral name — on the OGLO board that's + ``"nimble"``) and the advertisement data's ``local_name`` + (which is what the firmware sets to ``"OGLO"``). Checking only + one of the two misses real hardware in the wild. + """ filter_lower = self._ble_name.lower() - devices = await bleak.BleakScanner.discover(timeout=self._scan_timeout) - for device in devices: - name = (getattr(device, "name", None) or "").lower() - if filter_lower in name: - return device + results = await bleak.BleakScanner.discover( + timeout=self._scan_timeout, return_adv=True + ) + for address, (device, adv) in results.items(): + candidates = [ + (getattr(device, "name", None) or ""), + (getattr(adv, "local_name", None) or ""), + ] + for candidate in candidates: + if filter_lower in candidate.lower(): + return device return None # ------------------------------------------------------------------ @@ -296,6 +422,13 @@ def _handle_payload(self, payload: bytes) -> None: # full 100 Hz rate. The MCU hardware clock is linearly interpolated # across the batch so downstream consumers see uniform 10 ms # spacing instead of a cluster at every batch boundary. + # + # Samples are emitted unconditionally so the viewer's sensor + # card plot updates during both the CONNECTED preview phase + # and the RECORDING phase. Finalization counters + # (``_frame_count`` / ``_first_at`` / ``_last_at``) only + # advance while ``_recording`` is True — so the preview + # samples never contaminate the recording's frame total. for i in range(count): offset = _HEADER_SIZE + i * _SAMPLE_SIZE values = struct.unpack( @@ -308,15 +441,23 @@ def _handle_payload(self, payload: bytes) -> None: (timestamp_us + i * _SAMPLE_PERIOD_US) * 1000 ) - if self._first_at is None: - self._first_at = recv_ns - self._last_at = recv_ns - self._frame_count += 1 + if self._recording: + if self._first_at is None: + self._first_at = recv_ns + self._last_at = recv_ns + self._frame_count += 1 + frame_number = self._frame_count - 1 + else: + # Preview phase — pass through a synthetic, ever- + # increasing frame number so subscribers that rely + # on monotonic numbering don't see duplicates, but + # don't advance the real counter. + frame_number = -1 self._emit_sample( SampleEvent( stream_id=self.id, - frame_number=self._frame_count - 1, + frame_number=frame_number, capture_ns=recv_ns, channels=channels, uncertainty_ns=500_000, # ~0.5 ms — MCU clock precision diff --git a/src/syncfield/discovery/_ble.py b/src/syncfield/discovery/_ble.py index 1f9cb41..57b953a 100644 --- a/src/syncfield/discovery/_ble.py +++ b/src/syncfield/discovery/_ble.py @@ -114,6 +114,6 @@ def scan_peripherals(timeout: float = 5.0) -> List[Any]: def clear_cache() -> None: """Invalidate the shared BLE scan cache. Primarily a test hook.""" global _cache, _cache_time - with _cache_lock: + with _scan_lock: _cache = [] _cache_time = 0.0 diff --git a/src/syncfield/tone.py b/src/syncfield/tone.py index be3f030..14745dc 100644 --- a/src/syncfield/tone.py +++ b/src/syncfield/tone.py @@ -58,6 +58,15 @@ from_hz=2500, to_hz=400, duration_ms=500, amplitude=0.8, envelope_ms=15 ) +# Countdown tick — a short flat-frequency beep played once per countdown +# second by :meth:`SessionOrchestrator.start`. The tone is C6 (1046.5 Hz) +# for 100 ms with a 10 ms envelope, which reads as a clean digital "tick" +# on MacBook speakers without being jarring. The operator hears +# beep · beep · beep before the start chirp sweeps in. +_DEFAULT_COUNTDOWN_TICK = ChirpSpec( + from_hz=1047, to_hz=1047, duration_ms=100, amplitude=0.6, envelope_ms=10 +) + def generate_chirp_samples(spec: ChirpSpec, sample_rate: int = 44100) -> List[float]: """Generate mono PCM float samples for a linear FM chirp with cosine envelope. @@ -175,6 +184,11 @@ class SyncToneConfig: streams have started. stop_chirp: Parameters for the chirp played right before the orchestrator stops all streams. + countdown_tick: Optional short beep played once per second + during the ``COUNTDOWN`` phase. Defaults to a 100 ms C6 + tick so the operator hears ``beep · beep · beep`` before + the start chirp sweeps in. Set to ``None`` to silence the + countdown while keeping the start/stop chirps audible. post_start_stabilization_ms: How long to wait after starting every stream before playing the start chirp. pre_stop_tail_margin_ms: Extra wait time (on top of the stop @@ -185,6 +199,9 @@ class SyncToneConfig: enabled: bool = True start_chirp: ChirpSpec = field(default_factory=lambda: _DEFAULT_START_CHIRP) stop_chirp: ChirpSpec = field(default_factory=lambda: _DEFAULT_STOP_CHIRP) + countdown_tick: Optional[ChirpSpec] = field( + default_factory=lambda: _DEFAULT_COUNTDOWN_TICK + ) post_start_stabilization_ms: int = 200 pre_stop_tail_margin_ms: int = 200 @@ -201,7 +218,7 @@ def silent(cls) -> "SyncToneConfig": unacceptable (quiet rooms, meetings) or for headless lab machines with no audio output path. """ - return cls(enabled=False) + return cls(enabled=False, countdown_tick=None) # --------------------------------------------------------------------------- @@ -441,14 +458,19 @@ def create_default_player(sample_rate: int = 44100) -> ChirpPlayer: Returns a :class:`SoundDeviceChirpPlayer` when ``sounddevice`` is importable, else a :class:`SilentChirpPlayer`. Import errors are - logged at INFO — never raised — so the SDK stays usable on headless - machines with no audio output. + logged at WARNING — never raised — so the SDK stays usable on + headless machines with no audio output, but interactive users see + the explicit "install ``syncfield[audio]`` to hear chirps" hint + instead of silently wondering why nothing beeps. """ try: import sounddevice # noqa: F401 except (ImportError, OSError) as exc: - logger.info( - "sounddevice unavailable (%s); chirp playback disabled", exc + logger.warning( + "sounddevice unavailable (%s). The countdown ticks and start/" + "stop chirps will be SILENT. Install the audio extra to hear " + "them: pip install 'syncfield[audio]'", + exc, ) return SilentChirpPlayer() return SoundDeviceChirpPlayer(sample_rate=sample_rate) diff --git a/src/syncfield/types.py b/src/syncfield/types.py index 32b3ae3..bc562f1 100644 --- a/src/syncfield/types.py +++ b/src/syncfield/types.py @@ -190,7 +190,25 @@ def to_dict(self) -> dict[str, Any]: class SessionState(Enum): - """Lifecycle state of a SessionOrchestrator.""" + """Lifecycle state of a SessionOrchestrator. + + A SyncField session walks a small state machine. The 0.2 release + adds a ``CONNECTED`` state so devices can stream live preview data + before the user hits Record — the viewer and CLI both sit in this + state to show frames and sensor values. A brief ``COUNTDOWN`` state + fires between Record-click and actual recording so the user sees a + 3 / 2 / 1 indicator and has time to flick a glance at the rig. + + Typical single-recording transitions:: + + IDLE → CONNECTED → COUNTDOWN → RECORDING → STOPPING → CONNECTED … + + Calling ``disconnect()`` from ``CONNECTED`` returns the session to + ``IDLE``. Applications that want the legacy one-shot behavior can + still call ``start()`` straight from ``IDLE`` — the orchestrator + auto-connects, records, and ``stop()`` runs to ``STOPPED`` at the + end. + """ IDLE = "idle" CONNECTING = "connecting" diff --git a/src/syncfield/viewer/app.py b/src/syncfield/viewer/app.py index c6652cb..1f05ef2 100644 --- a/src/syncfield/viewer/app.py +++ b/src/syncfield/viewer/app.py @@ -32,6 +32,7 @@ from syncfield.orchestrator import SessionOrchestrator from syncfield.viewer import theme +from syncfield.viewer.fonts import FontRegistry, load_fonts from syncfield.viewer.poller import SessionPoller from syncfield.viewer.widgets.layout import ViewerLayout @@ -162,6 +163,7 @@ def __init__( self._viewport_pos = viewport_pos self._poller = SessionPoller(session) self._layout: Optional[ViewerLayout] = None + self._fonts: FontRegistry = FontRegistry() self._running = False self._setup_done = False self._close_requested = False @@ -185,6 +187,13 @@ def setup(self) -> None: resizable=True, ) + # Load system fonts into the DPG registry before any widget is + # created — otherwise the first frame renders with the default + # ASCII-only bitmap font and every non-ASCII glyph flashes as '?'. + self._fonts = load_fonts() + if self._fonts.ui is not None: + dpg.bind_font(self._fonts.ui) + # Bind the global theme before any widgets are created so the # first frame doesn't flash with the default dark theme. global_theme_tag = theme.build_theme() @@ -196,7 +205,7 @@ def setup(self) -> None: [c / 255 for c in theme.BG_APP] ) - self._layout = ViewerLayout(self._session) + self._layout = ViewerLayout(self._session, fonts=self._fonts) self._layout.build() dpg.setup_dearpygui() @@ -240,10 +249,20 @@ def render_one_frame(self) -> None: dpg.render_dearpygui_frame() def close(self) -> None: - """Stop the poller and destroy the DPG context.""" + """Stop the poller, tear down the session, and destroy the DPG context.""" if self._close_requested: return self._close_requested = True + + # Return the session to IDLE before tearing down DPG so any + # connected devices are released cleanly. Runs on the caller's + # thread (typically the main thread during app shutdown). + if self._layout is not None: + try: + self._layout.teardown_session() + except Exception: + logger.exception("Viewer session teardown failed") + self._poller.stop() try: if dpg.is_dearpygui_running(): diff --git a/src/syncfield/viewer/demo.py b/src/syncfield/viewer/demo.py index 9f81df7..e744f93 100644 --- a/src/syncfield/viewer/demo.py +++ b/src/syncfield/viewer/demo.py @@ -18,6 +18,7 @@ import argparse import math +import os import sys import threading import time @@ -48,7 +49,11 @@ class SyntheticVideoStream(StreamBase): Exposes ``latest_frame`` the same way :class:`UVCWebcamStream` and :class:`OakCameraStream` do, so the viewer's video card renders it - correctly without any mocking on the viewer side. + correctly without any mocking on the viewer side. Implements the + full 4-phase lifecycle: the gradient loop runs during ``CONNECTED`` + so preview is live *before* Record is pressed, and ``_recording`` + gates the ``SampleEvent`` emission so only frames captured while + the session is in ``RECORDING`` count toward the finalization. """ def __init__( @@ -76,26 +81,39 @@ def __init__( self._hue_shift = hue_shift self._stop = threading.Event() self._thread: threading.Thread | None = None + # `_recording` toggles whether the capture loop counts frames + # and emits sample events. While False the loop still produces + # `latest_frame` so the viewer has something to show. + self._recording = False self._frame_count = 0 self._first_at: int | None = None self._last_at: int | None = None self._latest_frame: Any = None self._frame_lock = threading.Lock() - def prepare(self) -> None: - pass + # -- 4-phase lifecycle -------------------------------------------------- - def start(self, session_clock) -> None: # type: ignore[override] + def connect(self) -> None: + """Spawn the capture loop so preview frames are available.""" + if self._thread is not None and self._thread.is_alive(): + return self._stop.clear() + self._recording = False + self._frame_count = 0 + self._first_at = None + self._last_at = None self._thread = threading.Thread( target=self._generate_loop, name=f"synth-vid-{self.id}", daemon=True ) self._thread.start() - def stop(self) -> FinalizationReport: - self._stop.set() - if self._thread is not None: - self._thread.join(timeout=2.0) + def start_recording(self, session_clock) -> None: # type: ignore[override] + """Flip the recording flag — atomic and fast.""" + self._recording = True + + def stop_recording(self) -> FinalizationReport: + """Stop emitting samples but leave the capture loop running.""" + self._recording = False return FinalizationReport( stream_id=self.id, status="completed", @@ -107,6 +125,31 @@ def stop(self) -> FinalizationReport: error=None, ) + def disconnect(self) -> None: + """Tear down the capture loop.""" + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=2.0) + self._thread = None + + # -- Legacy one-shot compatibility ------------------------------------- + + def prepare(self) -> None: + pass + + def start(self, session_clock) -> None: # type: ignore[override] + # Compatibility path for any caller still using the legacy + # one-shot lifecycle: connect + start_recording in one call. + self.connect() + self.start_recording(session_clock) + + def stop(self) -> FinalizationReport: + report = self.stop_recording() + self.disconnect() + return report + + # -- Viewer integration ------------------------------------------------ + @property def latest_frame(self) -> Any: with self._frame_lock: @@ -117,7 +160,9 @@ def _generate_loop(self) -> None: The frame is a smooth sinusoidal pattern that drifts across the image — visually distinctive enough that screenshots show real - motion but cheap enough to compute at 30 fps. + motion but cheap enough to compute at 30 fps. Runs continuously + while connected; sample events are only emitted during + ``_recording`` so the frame count reflects recorded frames only. """ xs = np.linspace(0, 2 * math.pi, self._width, dtype=np.float32) ys = np.linspace(0, 2 * math.pi, self._height, dtype=np.float32) @@ -138,18 +183,22 @@ def _generate_loop(self) -> None: with self._frame_lock: self._latest_frame = bgr - if self._first_at is None: - self._first_at = capture_ns - self._last_at = capture_ns - self._frame_count += 1 - self._emit_sample( - SampleEvent( - stream_id=self.id, - frame_number=frame_number, - capture_ns=capture_ns, + # Preview-only frames never touch the counters or the sample + # stream — they just update `latest_frame`. Recording frames + # do both. + if self._recording: + if self._first_at is None: + self._first_at = capture_ns + self._last_at = capture_ns + self._frame_count += 1 + self._emit_sample( + SampleEvent( + stream_id=self.id, + frame_number=frame_number, + capture_ns=capture_ns, + ) ) - ) - frame_number += 1 + frame_number += 1 self._stop.wait(self._period_s) @@ -159,7 +208,14 @@ def _generate_loop(self) -> None: class SyntheticImuStream(StreamBase): - """Fake 9-DOF IMU that produces smooth sinusoidal channels at 100 Hz.""" + """Fake 9-DOF IMU that produces smooth sinusoidal channels at 100 Hz. + + Like :class:`SyntheticVideoStream`, the sample loop runs during + ``CONNECTED`` so the viewer can plot live values before Record is + pressed. Only frames captured while ``_recording`` is ``True`` are + counted into the finalization report and emitted as + :class:`SampleEvent`. + """ def __init__(self, id: str) -> None: super().__init__( @@ -174,24 +230,31 @@ def __init__(self, id: str) -> None: ) self._stop = threading.Event() self._thread: threading.Thread | None = None + self._recording = False self._frame_count = 0 self._first_at: int | None = None self._last_at: int | None = None - def prepare(self) -> None: - pass + # -- 4-phase lifecycle ------------------------------------------------- - def start(self, session_clock) -> None: # type: ignore[override] + def connect(self) -> None: + if self._thread is not None and self._thread.is_alive(): + return self._stop.clear() + self._recording = False + self._frame_count = 0 + self._first_at = None + self._last_at = None self._thread = threading.Thread( target=self._loop, name=f"synth-imu-{self.id}", daemon=True ) self._thread.start() - def stop(self) -> FinalizationReport: - self._stop.set() - if self._thread is not None: - self._thread.join(timeout=2.0) + def start_recording(self, session_clock) -> None: # type: ignore[override] + self._recording = True + + def stop_recording(self) -> FinalizationReport: + self._recording = False return FinalizationReport( stream_id=self.id, status="completed", @@ -203,16 +266,34 @@ def stop(self) -> FinalizationReport: error=None, ) + def disconnect(self) -> None: + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=2.0) + self._thread = None + + # -- Legacy one-shot compatibility ------------------------------------- + + def prepare(self) -> None: + pass + + def start(self, session_clock) -> None: # type: ignore[override] + self.connect() + self.start_recording(session_clock) + + def stop(self) -> FinalizationReport: + report = self.stop_recording() + self.disconnect() + return report + + # -- Capture loop ------------------------------------------------------ + def _loop(self) -> None: period = 0.01 # 100 Hz t0 = time.monotonic() while not self._stop.is_set(): t = time.monotonic() - t0 capture_ns = time.monotonic_ns() - if self._first_at is None: - self._first_at = capture_ns - self._last_at = capture_ns - self._frame_count += 1 channels = { "ax": math.sin(t * 1.3) * 0.8 + math.sin(t * 7.0) * 0.1, "ay": math.cos(t * 1.6) * 0.6, @@ -221,35 +302,41 @@ def _loop(self) -> None: "gy": math.cos(t * 2.3) * 0.5, "gz": math.sin(t * 3.1) * 0.3, } - self._emit_sample( - SampleEvent( - stream_id=self.id, - frame_number=self._frame_count - 1, - capture_ns=capture_ns, - channels=channels, - ) - ) - # Sprinkle in a health event occasionally so the health table - # actually has content in screenshots. - if self._frame_count == 150: - self._emit_health( - HealthEvent( + if self._recording: + if self._first_at is None: + self._first_at = capture_ns + self._last_at = capture_ns + self._frame_count += 1 + self._emit_sample( + SampleEvent( stream_id=self.id, - kind=HealthEventKind.WARNING, - at_ns=capture_ns, - detail="synthetic jitter above threshold", + frame_number=self._frame_count - 1, + capture_ns=capture_ns, + channels=channels, ) ) - if self._frame_count == 320: - self._emit_health( - HealthEvent( - stream_id=self.id, - kind=HealthEventKind.RECONNECT, - at_ns=capture_ns, - detail=None, + + # Sprinkle in a health event occasionally so the health + # table actually has content in screenshots. + if self._frame_count == 150: + self._emit_health( + HealthEvent( + stream_id=self.id, + kind=HealthEventKind.WARNING, + at_ns=capture_ns, + detail="synthetic jitter above threshold", + ) + ) + if self._frame_count == 320: + self._emit_health( + HealthEvent( + stream_id=self.id, + kind=HealthEventKind.RECONNECT, + at_ns=capture_ns, + detail=None, + ) ) - ) self._stop.wait(period) @@ -389,18 +476,48 @@ def _auto_record() -> None: import subprocess def _capture_window_screenshot() -> None: - """Capture the viewer window via AppleScript window discovery. - - On macOS 'screencapture -l ' grabs a specific window - by its CoreGraphics window id. The id is resolved via - AppleScript by matching the frontmost process's window title - against 'SyncField'. This avoids manual coordinate math and - Retina scaling entirely. + """Capture the viewer window to a PNG via ``screencapture``. + + The demo is launched from a terminal, so by the time the + timer fires the frontmost app may still be the shell or the + editor that kicked off the run — not the DPG window. Before + capturing we ask the current Python process (which owns the + viewer window) to activate itself via AppleScript. That + guarantees the window is on top of whatever was previously + frontmost, so the region capture at ``(60, 60)`` lines up + with the pinned viewport. """ args.screenshot.parent.mkdir(parents=True, exist_ok=True) - # Step 1: ask macOS for the window id of the 'SyncField' window - osa = """ + # Step 1: force the SyncField window to the front. We look + # up "python" (which owns this process) and send an 'activate' + # event. Falls through silently if System Events is + # unreachable — the capture will still run, it just may + # grab whatever is on top of the viewer. + activate_script = """ + tell application "System Events" + set pyProcs to every application process whose unix id is %d + if (count of pyProcs) > 0 then + set frontmost of (item 1 of pyProcs) to true + end if + end tell + """ % os.getpid() + try: + subprocess.run( + ["osascript", "-e", activate_script], + capture_output=True, + text=True, + timeout=5, + ) + except Exception as exc: + print(f"viewer activate failed: {exc}", file=sys.stderr) + + # Small settle so the window manager finishes raising the + # viewer above whatever was in front of it before. + time.sleep(0.3) + + # Step 2: log the frontmost window title for debugging. + probe = """ tell application "System Events" set frontApp to first application process whose frontmost is true set frontWin to window 1 of frontApp @@ -409,13 +526,16 @@ def _capture_window_screenshot() -> None: """ try: result = subprocess.run( - ["osascript", "-e", osa], + ["osascript", "-e", probe], capture_output=True, text=True, check=True, timeout=5, ) - print(f"frontmost window title: {result.stdout.strip()!r}", file=sys.stderr) + print( + f"frontmost window title: {result.stdout.strip()!r}", + file=sys.stderr, + ) except Exception as exc: print(f"title probe failed: {exc}", file=sys.stderr) @@ -438,8 +558,15 @@ def _capture_window_screenshot() -> None: try: from syncfield.viewer import theme + # The viewer is pinned at (60, 60) by the screenshot + # harness. macOS window chrome adds a ~28 px title bar + # above the DPG content — we pad the capture region by + # the same amount at the bottom to guarantee the full + # content area is visible even after the viewport grows. + _TITLE_BAR_PX = 28 x, y = 60, 60 - w, h = theme.VIEWPORT_WIDTH, theme.VIEWPORT_HEIGHT + w = theme.VIEWPORT_WIDTH + h = theme.VIEWPORT_HEIGHT + _TITLE_BAR_PX subprocess.run( [ "screencapture", diff --git a/src/syncfield/viewer/fonts.py b/src/syncfield/viewer/fonts.py new file mode 100644 index 0000000..ad3bef6 --- /dev/null +++ b/src/syncfield/viewer/fonts.py @@ -0,0 +1,200 @@ +"""System font loader for the SyncField desktop viewer. + +DearPyGui's built-in font (ProggyClean) is a 13 px ASCII bitmap — it +looks dated and, more importantly, it renders every non-ASCII glyph as +``?``. The viewer uses a handful of geometric symbols (``●``, ``■``, +``→``, ``—``) plus the occasional warning glyph, so a font with the +extended Latin ranges is a correctness requirement, not just polish. + +This module walks a small list of platform-appropriate system font paths +and loads the first one it finds into the DearPyGui font registry at the +typographic scale the layout expects. The returned :class:`FontRegistry` +hands out tags per display role — body text, muted text, display sizes, +monospace for timestamps and device ids. + +If no system font is found the module falls back gracefully: the +returned registry's fields are all ``None`` and the caller keeps the +DearPyGui default font. The UI will still work; exotic glyphs will just +render as placeholders. +""" + +from __future__ import annotations + +import logging +import os +import platform +from dataclasses import dataclass +from typing import List, Optional + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Registry returned to the viewer app +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class FontRegistry: + """Bundle of DearPyGui font tags keyed by display role. + + All fields default to ``None`` so callers can treat "font missing" + the same way as "don't override this element's font" — no special + casing in widget code. + """ + + ui_sm: Optional[int] = None # 12 px — small labels, muted captions + ui: Optional[int] = None # 14 px — body text (bound globally) + ui_md: Optional[int] = None # 16 px — section titles, card headers + ui_lg: Optional[int] = None # 22 px — app title, prominent timers + mono: Optional[int] = None # 14 px — device ids, timestamps + + +# --------------------------------------------------------------------------- +# Platform font catalog +# --------------------------------------------------------------------------- + +# Ordered by preference. First existing file wins. +_SANS_CANDIDATES = { + "Darwin": [ + # San Francisco Pro — Apple's system UI font. Single-file .ttf + # so DearPyGui can load it without .ttc collection handling. + "/System/Library/Fonts/SFNS.ttf", + "/System/Library/Fonts/SFCompact.ttf", + "/System/Library/Fonts/Helvetica.ttc", + "/Library/Fonts/Arial.ttf", + ], + "Linux": [ + "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", + "/usr/share/fonts/TTF/DejaVuSans.ttf", + "/usr/share/fonts/truetype/noto/NotoSans-Regular.ttf", + "/usr/share/fonts/noto/NotoSans-Regular.ttf", + "/usr/share/fonts/liberation/LiberationSans-Regular.ttf", + "/usr/share/fonts/truetype/liberation/LiberationSans-Regular.ttf", + "/usr/share/fonts/truetype/ubuntu/Ubuntu-R.ttf", + ], + "Windows": [ + r"C:\Windows\Fonts\segoeui.ttf", + r"C:\Windows\Fonts\arial.ttf", + ], +} + +_MONO_CANDIDATES = { + "Darwin": [ + "/System/Library/Fonts/SFNSMono.ttf", + "/System/Library/Fonts/Menlo.ttc", + ], + "Linux": [ + "/usr/share/fonts/truetype/dejavu/DejaVuSansMono.ttf", + "/usr/share/fonts/TTF/DejaVuSansMono.ttf", + "/usr/share/fonts/truetype/liberation/LiberationMono-Regular.ttf", + ], + "Windows": [ + r"C:\Windows\Fonts\consola.ttf", + r"C:\Windows\Fonts\lucon.ttf", + ], +} + + +def _find_first_existing(candidates: List[str]) -> Optional[str]: + for path in candidates: + if os.path.isfile(path): + return path + return None + + +# --------------------------------------------------------------------------- +# Typography scale +# --------------------------------------------------------------------------- + +# Keyed by the FontRegistry field name so ``load_fonts`` can splat it in. +_SIZE_SCALE = { + "ui_sm": 13, # captions, muted labels + "ui": 15, # body text (bound globally) + "ui_md": 17, # card titles, controls + "ui_lg": 24, # app title, prominent display +} + + +# --------------------------------------------------------------------------- +# Unicode ranges needed by viewer glyphs +# --------------------------------------------------------------------------- + +# DearPyGui's default range hint covers Basic Latin + Latin-1 Supplement. +# Everything else the viewer uses lives in these blocks — we rasterize +# only what we need so the font atlas stays small. +_EXTRA_RANGES = ( + (0x2000, 0x206F), # General Punctuation — em dash, ellipsis, bullet + (0x2190, 0x21FF), # Arrows — → + (0x25A0, 0x25FF), # Geometric Shapes — ● ■ ◐ □ + (0x2600, 0x26FF), # Miscellaneous Symbols — ⚠ +) + + +# --------------------------------------------------------------------------- +# Public entry point +# --------------------------------------------------------------------------- + + +def load_fonts() -> FontRegistry: + """Load viewer fonts into the DearPyGui font registry. + + Must be called **after** ``dpg.create_context()`` and **before** the + first widget is built. Returns a :class:`FontRegistry` of font tags + the caller can pass to ``dpg.bind_font`` (for the global default) + and ``dpg.bind_item_font`` (for per-widget overrides like the app + title or monospace device ids). + + When no system font is available the returned registry is empty and + the viewer falls back to the DearPyGui built-in font. + """ + import dearpygui.dearpygui as dpg + + system = platform.system() + sans_path = _find_first_existing(_SANS_CANDIDATES.get(system, [])) + mono_path = _find_first_existing(_MONO_CANDIDATES.get(system, [])) + + if sans_path is None: + logger.warning( + "viewer: no system UI font found on %s; exotic glyphs will " + "render as '?' placeholders. Install a TrueType font such as " + "DejaVu Sans or Noto Sans to fix.", + system, + ) + return FontRegistry() + + logger.debug( + "viewer: loading UI font %s (mono=%s)", sans_path, mono_path or "none" + ) + + tags: dict = {} + with dpg.font_registry(): + for role, size in _SIZE_SCALE.items(): + tag = _load_sized_font(dpg, sans_path, size) + if tag is not None: + tags[role] = tag + + if mono_path is not None: + mono_tag = _load_sized_font(dpg, mono_path, 15) + if mono_tag is not None: + tags["mono"] = mono_tag + + return FontRegistry(**tags) + + +def _load_sized_font(dpg_module, font_path: str, size: int) -> Optional[int]: + """Load one ``font_path`` at ``size`` px and register the extra ranges. + + Returns the font tag on success or ``None`` if DearPyGui rejects the + file (corrupt file, unreadable collection index, etc.). Never raises + — a bad font file should never crash the viewer. + """ + try: + with dpg_module.font(font_path, size) as font_tag: + dpg_module.add_font_range_hint(dpg_module.mvFontRangeHint_Default) + for start, end in _EXTRA_RANGES: + dpg_module.add_font_range(start, end) + return font_tag + except Exception as exc: + logger.debug("viewer: failed to load %s @ %dpx: %s", font_path, size, exc) + return None diff --git a/src/syncfield/viewer/state.py b/src/syncfield/viewer/state.py index fb40701..ad56ad7 100644 --- a/src/syncfield/viewer/state.py +++ b/src/syncfield/viewer/state.py @@ -154,16 +154,28 @@ def observe_sample(self, capture_ns: int, channels: Optional[Dict[str, Any]]) -> are bounded — truncation happens inside ``deque.append`` atomically and readers call :meth:`snapshot_fps` / :meth:`snapshot_plot` which make a list copy. + + Auxiliary channels — timestamps, metadata, anything whose name + starts with ``_`` or contains ``timestamp`` — are skipped from + the plot buffer. Those values often live in the nanoseconds + range (~10¹⁸) and would otherwise dominate the auto-scaled Y + axis, squashing real sensor readings (0–65535 for OGLO FSRs) + flat against the baseline. """ self._fps_window.append(capture_ns) self._plot_timestamps.append(capture_ns / 1e9) if channels: + plottable = { + name: value + for name, value in channels.items() + if isinstance(value, (int, float)) + and not _is_auxiliary_channel(name) + } + # Pad any missing channel to align lengths, then append numeric values. current_len = len(self._plot_timestamps) - for name, value in channels.items(): - if not isinstance(value, (int, float)): - continue + for name, value in plottable.items(): buf = self._plot_channels.get(name) if buf is None: buf = deque(maxlen=self.max_plot_samples) @@ -177,7 +189,7 @@ def observe_sample(self, capture_ns: int, channels: Optional[Dict[str, Any]]) -> # Any channel we already track but that's missing from this sample # gets a NaN so it doesn't drift out of alignment. for name, buf in self._plot_channels.items(): - if name not in channels: + if name not in plottable: buf.append(float("nan")) def observe_health(self, event: HealthEntry) -> None: @@ -210,3 +222,38 @@ def snapshot_plot(self) -> Dict[str, Tuple[List[float], List[float]]]: def snapshot_health(self) -> List[HealthEntry]: return list(self._health) + + +# --------------------------------------------------------------------------- +# Free helpers +# --------------------------------------------------------------------------- + + +def _is_auxiliary_channel(name: str) -> bool: + """Return True for channel names that shouldn't appear in the plot. + + Adapters sometimes attach metadata channels alongside real sensor + readings — the OGLO tactile stream, for example, emits + ``device_timestamp_ns`` next to its thumb/index/middle/ring/pinky + FSR values so downstream consumers can recover the MCU hardware + clock. Those auxiliary values often sit in the nanoseconds range + (~10¹⁸) and, if plotted alongside the real 0–65535 readings on a + single auto-scaled Y axis, flatten every real reading into a + straight baseline. + + The rule is deliberately simple so third-party adapters can opt + channels out of plotting by convention alone — without any API + hook — by: + + * prefixing the channel name with an underscore (``_raw``, + ``_calibration``, …), or + * including the substring ``timestamp`` in the channel name + (``device_timestamp_ns``, ``capture_timestamp_us``, …). + """ + if not name: + return False + if name.startswith("_"): + return True + if "timestamp" in name.lower(): + return True + return False diff --git a/tests/unit/adapters/test_oak_camera.py b/tests/unit/adapters/test_oak_camera.py index 791ecd5..ca05528 100644 --- a/tests/unit/adapters/test_oak_camera.py +++ b/tests/unit/adapters/test_oak_camera.py @@ -17,12 +17,19 @@ def _clock() -> SessionClock: return SessionClock(sync_point=SyncPoint.create_now("h")) -def _build_fake_depthai(frame_budget: int = 3) -> MagicMock: +def _build_fake_depthai() -> MagicMock: """Return a MagicMock that looks enough like depthai for the adapter. Models the depthai v3 pipeline API: Pipeline.build() / start() / stop(), Camera node with requestOutput() → OutputQueue.get() → ImgFrame-like object exposing .getCvFrame() and .getTimestamp(). + + The fake queue returns an unlimited stream of frames — in the new + 4-phase lifecycle the capture loop runs across both the preview + (``connect()``) and recording (``start_recording()``) phases, so a + fixed budget would get consumed before any recording happens. + Tests that want to exercise "queue drains" swap in a narrower + side_effect manually. """ fake = MagicMock() @@ -35,17 +42,12 @@ def __init__(self) -> None: def getCvFrame(self) -> MagicMock: return self._cv_frame - # --- Fake output queue: returns a few frames then None -------------- - call_count = {"n": 0} - + # --- Fake output queue: unlimited stream of frames ------------------ def make_queue() -> MagicMock: q = MagicMock() - def fake_get(timeout: float = 0.1) -> _FakeFrame | None: - call_count["n"] += 1 - if call_count["n"] <= frame_budget: - return _FakeFrame() - return None + def fake_get(timeout: float = 0.1) -> _FakeFrame: + return _FakeFrame() q.get.side_effect = fake_get q.tryGet.return_value = None @@ -111,55 +113,86 @@ def test_capabilities(self, mock_depthai, tmp_path): class TestLifecycle: - def test_prepare_builds_and_starts_pipeline(self, mock_depthai, tmp_path): + """Exercise the 4-phase connect → start_recording → stop_recording → disconnect path. + + In 0.2 the pipeline build moved from ``prepare()`` into + ``connect()`` so the viewer can show a live preview before Record + is pressed. These tests pin that split down. + """ + + def test_connect_builds_and_starts_pipeline(self, mock_depthai, tmp_path): fake, _ = mock_depthai from syncfield.adapters.oak_camera import OakCameraStream stream = OakCameraStream("oak", output_dir=tmp_path) stream.prepare() + # prepare() no longer opens the pipeline — only connect() does. + assert fake.Pipeline.call_count == 0 - # Pipeline was constructed, built, and started - fake.Pipeline.assert_called_once() - pipeline = fake.Pipeline.return_value - assert pipeline.build.called - assert pipeline.start.called - - def test_prepare_raises_when_no_devices(self, mock_depthai, tmp_path): + stream.connect() + # Give the capture thread a moment to pull a frame or two. + time.sleep(0.05) + try: + fake.Pipeline.assert_called_once() + pipeline = fake.Pipeline.return_value + assert pipeline.build.called + assert pipeline.start.called + finally: + stream.disconnect() + + def test_connect_raises_when_no_devices(self, mock_depthai, tmp_path): fake, _ = mock_depthai fake.Device.getAllAvailableDevices.return_value = [] from syncfield.adapters.oak_camera import OakCameraStream stream = OakCameraStream("oak", output_dir=tmp_path) + stream.prepare() with pytest.raises(RuntimeError, match="No OAK devices"): - stream.prepare() + stream.connect() - def test_start_stop_produces_file_path(self, mock_depthai, tmp_path): + def test_full_lifecycle_produces_file_path(self, mock_depthai, tmp_path): from syncfield.adapters.oak_camera import OakCameraStream stream = OakCameraStream("oak", output_dir=tmp_path) stream.prepare() - stream.start(_clock()) + stream.connect() + stream.start_recording(_clock()) # Give the background thread time to read the mocked frames time.sleep(0.15) - report = stream.stop() + report = stream.stop_recording() + stream.disconnect() assert report.status == "completed" assert report.file_path is not None assert report.frame_count >= 1 - def test_stop_releases_pipeline(self, mock_depthai, tmp_path): + def test_disconnect_releases_pipeline(self, mock_depthai, tmp_path): fake, _ = mock_depthai from syncfield.adapters.oak_camera import OakCameraStream stream = OakCameraStream("oak", output_dir=tmp_path) stream.prepare() - stream.start(_clock()) + stream.connect() time.sleep(0.05) - stream.stop() + stream.disconnect() pipeline = fake.Pipeline.return_value assert pipeline.stop.called + def test_legacy_start_stop_still_works(self, mock_depthai, tmp_path): + """Old 0.1-era ``prepare() → start() → stop()`` path stays valid.""" + from syncfield.adapters.oak_camera import OakCameraStream + + stream = OakCameraStream("oak", output_dir=tmp_path) + stream.prepare() + stream.start(_clock()) + time.sleep(0.15) + report = stream.stop() + + assert report.status == "completed" + assert report.file_path is not None + assert report.frame_count >= 1 + class TestDepthOption: def test_depth_enabled_declares_depth_output(self, mock_depthai, tmp_path): @@ -173,10 +206,13 @@ def test_depth_enabled_declares_depth_output(self, mock_depthai, tmp_path): depth_enabled=True, ) stream.prepare() - - # pipeline.create was called twice (Camera + StereoDepth) - pipeline = fake.Pipeline.return_value - assert pipeline.create.call_count >= 2 + stream.connect() + try: + # pipeline.create was called twice (Camera + StereoDepth) + pipeline = fake.Pipeline.return_value + assert pipeline.create.call_count >= 2 + finally: + stream.disconnect() def test_depth_disabled_by_default(self, mock_depthai, tmp_path): """Default config builds only the RGB camera node.""" @@ -185,9 +221,12 @@ def test_depth_disabled_by_default(self, mock_depthai, tmp_path): stream = OakCameraStream("oak", output_dir=tmp_path) stream.prepare() - - pipeline = fake.Pipeline.return_value - assert pipeline.create.call_count == 1 # RGB only + stream.connect() + try: + pipeline = fake.Pipeline.return_value + assert pipeline.create.call_count == 1 # RGB only + finally: + stream.disconnect() class TestImportGuard: diff --git a/tests/unit/adapters/test_oglo_tactile.py b/tests/unit/adapters/test_oglo_tactile.py index 955d8c6..563e0ee 100644 --- a/tests/unit/adapters/test_oglo_tactile.py +++ b/tests/unit/adapters/test_oglo_tactile.py @@ -99,6 +99,10 @@ def test_decodes_full_batch(self, mock_bleak): from syncfield.adapters.oglo_tactile import OgloTactileStream stream = OgloTactileStream("tactile_right", address="m", hand="right") + # Put the stream into "recording" mode so decoded samples + # advance the real frame_number counters rather than flowing + # through the preview path (which uses -1 placeholders). + stream._recording = True received: List[SampleEvent] = [] stream.on_sample(received.append) @@ -146,6 +150,7 @@ def test_frame_count_accumulates_across_batches(self, mock_bleak): from syncfield.adapters.oglo_tactile import OgloTactileStream stream = OgloTactileStream("tactile_right", address="m") + stream._recording = True # simulate post-``start_recording`` state for batch_index in range(3): samples = [(1, 2, 3, 4, 5)] * 10 @@ -158,6 +163,29 @@ def test_frame_count_accumulates_across_batches(self, mock_bleak): assert stream._frame_count == 30 + def test_preview_samples_do_not_advance_frame_count(self, mock_bleak): + """Samples decoded while ``_recording == False`` update the + viewer plot but must not pollute the finalization counters. + """ + from syncfield.adapters.oglo_tactile import OgloTactileStream + + stream = OgloTactileStream("tactile_right", address="m") + received: List[SampleEvent] = [] + stream.on_sample(received.append) + + # Stream is in the preview phase — ``_recording`` stays False. + samples = [(1, 2, 3, 4, 5)] * 10 + packet = _build_packet(count=10, timestamp_us=0, samples=samples) + stream._dispatch_notification_for_test(packet) + + assert len(received) == 10 # samples reached on_sample subscribers + assert stream._frame_count == 0 # but no recording advancement + assert stream._first_at is None + assert stream._last_at is None + # Preview samples carry a sentinel frame_number of -1 so + # subscribers can tell them apart from recording samples. + assert all(ev.frame_number == -1 for ev in received) + def test_short_packet_becomes_warning(self, mock_bleak): from syncfield.adapters.oglo_tactile import OgloTactileStream @@ -190,6 +218,7 @@ def test_finalization_report_carries_counts(self, mock_bleak): from syncfield.adapters.oglo_tactile import OgloTactileStream stream = OgloTactileStream("tactile_right", address="m", hand="right") + stream._recording = True # simulate post-``start_recording`` state packet = _build_packet( count=5,