From 6cb31b4a7b1ba64a3a56a6889e321d0c59d9fed0 Mon Sep 17 00:00:00 2001 From: styu12 Date: Tue, 14 Apr 2026 17:24:30 -0700 Subject: [PATCH] feat(examples): Meta Quest 3 egocentric + hand/head pose recording example MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New example `examples/meta_quest/` pairs with the newly-landed MetaQuestCameraStream (PR #10) and the MetaQuestHandStream that's been on main since PR #7. Captures, in one synchronised session: - ``quest_tracking`` (UDP 14043, 72 Hz) — 26-joint hand skeleton × 2 hands + head pose, all timestamped in the ``remote_quest3`` clock domain. - ``quest_cam`` (HTTP 14045) — 720p stereo passthrough MJPEG recorded on the Quest and pulled to output_dir after stop. Viewer extensions to render the hand skeleton in 3-D: 1. State / poller / SSE backend now passes vector-valued channels (e.g. ``hand_joints``: 156 floats) through to the frontend under a new ``pose`` field on each SensorEvent. ``StreamStatsBuffer`` retains only the latest sample per list channel — 3-D pose panels render instantaneous state, buffering 300 × 500-float poses per adapter would blow memory. 2. New ``Quest3PosePanel`` component (React Three Fiber), ported from opengraph-studio/showcase with egocentric framing and the viewer's warm-paper palette. Draws the 26-joint skeleton per hand (cyan / orange) plus an RGB-axis head rig inside the same 0.84-m workspace box used by the showcase. 3. ``SensorPanel`` dispatcher routes streams with ``hand_joints`` payloads to the new 3-D panel; roll/pitch/yaw IMUs still go to the existing cube panel; everything else falls back to the multi-channel line chart. All existing adapters keep their rendering unchanged. 4. Frontend deps: ``three``, ``@react-three/fiber``, ``@react-three/drei``, ``@types/three``. Bundle grows ~1 MB gz which is acceptable for a desktop-launched viewer. Backward compatibility: adapters that emit only scalar channels never populate ``latest_pose``, ``pose`` on the SSE event stays ``null``, and the hook / dispatcher fall through to the existing chart path — zero behaviour change for non-Quest streams. Co-Authored-By: Claude Opus 4.6 --- examples/meta_quest/README.md | 159 +++++++ examples/meta_quest/record.py | 82 ++++ src/syncfield/viewer/frontend/package.json | 14 +- .../src/components/quest3-pose-panel.tsx | 341 +++++++++++++++ .../frontend/src/components/sensor-panel.tsx | 32 +- .../frontend/src/hooks/use-sensor-stream.ts | 12 +- .../viewer/frontend/src/lib/types.ts | 11 + src/syncfield/viewer/frontend/yarn.lock | 398 ++++++++++++++++++ src/syncfield/viewer/poller.py | 2 + src/syncfield/viewer/server.py | 18 +- src/syncfield/viewer/state.py | 40 ++ 11 files changed, 1088 insertions(+), 21 deletions(-) create mode 100644 examples/meta_quest/README.md create mode 100644 examples/meta_quest/record.py create mode 100644 src/syncfield/viewer/frontend/src/components/quest3-pose-panel.tsx diff --git a/examples/meta_quest/README.md b/examples/meta_quest/README.md new file mode 100644 index 0000000..6ef2152 --- /dev/null +++ b/examples/meta_quest/README.md @@ -0,0 +1,159 @@ +# Meta Quest 3 — egocentric camera + head/hand pose + +Captures four synchronised streams from a single Quest 3 headset: + +| Stream | Data | Rate | Transport | +|---|---|---|---| +| `quest_tracking` | 26-joint hand skeleton × 2, head pose | 72 Hz | UDP :14043 | +| `quest_cam` (left + right) | 720p stereo passthrough MP4 | 30 fps | HTTP :14045 | + +All samples land in the `remote_quest3` clock domain with a 10 ms +uncertainty budget so they align with on-host sensors (Mac webcam, +IMU, etc.) via the standard SyncField sync pipeline. + +--- + +## Prerequisites + +### Quest 3 side + +1. Install the **SyncField Quest Sender** companion app on the + headset. Source in + `opengraph-studio/unity/SyncFieldQuest3Sender/` — + build with `scripts/build_and_deploy.sh` (Quest connected via + USB-C, Developer Mode enabled). +2. On first launch, grant two permission prompts: + - Camera + - Horizon Camera (Meta's PCA — required for passthrough RGB frames) +3. With the sender app running, read the Quest's IP from the on-HUD + `Host` line or from `Settings → Wi-Fi → (your network) → IP + address`. Paste it into `QUEST_IP` at the top of `record.py`. + +### Mac side + +```bash +pip install "syncfield[uvc,viewer]" +``` + +`httpx` (used by `MetaQuestCameraStream`) and `numpy` are pulled in by +the viewer extra automatically. + +### Network + +- Quest and Mac on the **same WiFi subnet** (no AP isolation). + Verify with `ping ` from the Mac. +- No firewall blocking UDP 14043 / 14044 or TCP 14045 between Mac and + Quest. + +--- + +## Run + +```bash +python record.py +``` + +The SyncField viewer opens in your browser. + +1. The `quest_tracking` stream card shows a **3-D hand skeleton panel** + — cyan = left hand, orange = right hand, coloured axes = head pose. + If it stays empty, the Quest is not sending (check the HUD, WiFi, + and firewall). +2. The `quest_cam` card shows the live MJPEG preview (320×240 per + eye) while the session is connected. +3. Click **Record** to start. All streams synchronise via the start + chirp. +4. Click **Stop**. The Mac immediately pulls the recorded MP4s + + per-eye timestamps JSONL off the Quest (a few seconds for short + clips, up to tens of seconds for longer sessions). + +--- + +## Output + +``` +output// +├── sync_point.json +├── quest_tracking.jsonl # 72 Hz hand + head samples +├── quest_cam_left.mp4 # 720p MJPEG-AVI container +├── quest_cam_right.mp4 +├── quest_cam_left.timestamps.jsonl # one line per frame, host-monotonic ns +└── quest_cam_right.timestamps.jsonl +``` + +The MP4 files are MJPEG-in-AVI in phase 1 — VLC, ffmpeg, and +QuickTime open them out of the box. Phase 2 will swap in H.264 +hardware encoding on the Quest side; the wire protocol and the Python +adapter both stay the same. + +### Sample JSONL line (tracking) + +```json +{ + "frame_number": 247, + "capture_ns": 123456789012345, + "clock_domain": "remote_quest3", + "uncertainty_ns": 10000000, + "channels": { + "hand_joints": [/* 156 floats */], + "joint_rotations": [/* 208 floats */], + "head_pose": [px, py, pz, qx, qy, qz, qw] + } +} +``` + +### Joint layout + +OpenXR 26-joint order, per hand, 0-indexed: + +``` + 0: Palm 11: MiddleMetacarpal 21: LittleMetacarpal + 1: Wrist 12: MiddleProximal 22: LittleProximal + 2: ThumbMeta 13: MiddleIntermediate 23: LittleIntermediate + 3: ThumbProximal 14: MiddleDistal 24: LittleDistal + 4: ThumbDistal 15: MiddleTip 25: LittleTip + 5: ThumbTip 16: RingMetacarpal + 6: IndexMeta 17: RingProximal + 7: IndexProximal 18: RingIntermediate + 8: IndexInterm. 19: RingDistal + 9: IndexDistal 20: RingTip +10: IndexTip +``` + +Left hand fills `hand_joints[0:78]` (26 × 3), right `hand_joints[78:156]`. +Coordinate system: Unity left-handed Y-up (Quest's native OpenXR frame). + +--- + +## Troubleshooting + +### `quest_tracking` shows "Waiting for data…" +The Quest isn't sending UDP. Check: +- Is the sender app's HUD showing `● SENDING ` (green)? +- On the Quest HUD, is the **Host** line the correct Mac IP? +- Mac firewall allowing inbound UDP 14043? (`System Settings → Network + → Firewall`) + +### `quest_cam` fails on stop with `httpx.ConnectError` +The stop-recording phase is when the Mac hits the Quest HTTP server. +- Is the Quest still on? (If you took the headset off mid-session, + Quest might have gone to sleep and closed the socket.) +- Is `QUEST_IP` correct? Open `http://:14045/status` in a + browser — should return JSON. If it times out, the Quest's HTTP + server is unreachable. +- Phase 1 records on-device and file-pulls at stop; if the session + was very short or very long, try again or inspect `adb logcat -s + Unity` during stop for the actual error. + +### Video files exist but are 0 bytes +The Quest ran out of storage or `horizonos.permission.HEADSET_CAMERA` +was not granted. The sender app's HUD shows +`storage_free_bytes` on the `/status` endpoint; check and clear. Also +reinstall and re-grant permissions with +`scripts/build_and_deploy.sh` (it uninstalls + reinstalls fresh so +the permission prompts fire again). + +### Hand-skeleton panel is mostly empty but tracking Hz looks fine +The Quest is sending UDP with `tracked: false` flags because your +hands are out of the headset's tracking volume. Move them back into +view — the panel should populate within a frame or two. diff --git a/examples/meta_quest/record.py b/examples/meta_quest/record.py new file mode 100644 index 0000000..c8e8091 --- /dev/null +++ b/examples/meta_quest/record.py @@ -0,0 +1,82 @@ +"""Record Meta Quest 3 head pose + hand tracking + stereo egocentric camera. + + pip install "syncfield[uvc,viewer]" + python record.py + +Workflow: + 1. Put on the Quest 3 headset. + 2. Open the SyncField Quest Sender app on the Quest (sideloaded from + ``opengraph-studio/unity/SyncFieldQuest3Sender``). Confirm the HUD + shows ``● SENDING `` in green. + 3. Confirm the Quest and this Mac are on the same WiFi subnet, then + copy the Quest's IP address from the HUD into ``QUEST_IP`` below. + 4. Click **Record** in the viewer → every stream starts in sync. + 5. Click **Stop** → the Mac pulls the recorded stereo MP4 + per-eye + timestamps JSONL off the Quest over HTTP (takes a few seconds). + +What the session captures: + + ``quest_tracking`` (MetaQuestHandStream, UDP 14043) + ``quest_tracking.jsonl`` — 72 Hz samples of 26 OpenXR hand + joints × 3 xyz × 2 hands (156 floats) + + per-joint quaternions (208 floats) + + head pose (pos3 + quat4) + + ``quest_cam`` (MetaQuestCameraStream, HTTP 14045) + ``quest_cam_left.mp4`` + ``quest_cam_right.mp4`` — 720p @ 30 fps passthrough video + ``quest_cam_*.timestamps.jsonl`` — per-frame host-monotonic ns + +Everything lands under ``output//``. Timestamps are +projected into the host monotonic clock domain by the Quest app so +offline sync tooling can align against the Mac's other sensors +without any extra calibration step. + +HUD signals (Quest-side): + ● SENDING 72 Hz · pkts tracking flowing, receiver reachable + ● STALLED no ack last UDP ack >0.5 s ago + ● NO HOST no tracking ack for >2 s; check WiFi + +Pinch both hands for 2 s on the Quest to exit the sender app from +inside a hand-tracking session (the Meta system menu can be awkward +to reach with recording gloves / straps on). +""" + +from pathlib import Path + +import syncfield as sf +import syncfield.viewer +from syncfield.adapters import MetaQuestCameraStream, MetaQuestHandStream + +# Quest 3 IPv4 — copy from the Quest sender app's HUD ("Host" line) or +# check Settings → Wi-Fi → → Details → IP address. Must be on +# the same subnet as this Mac. Camera file pull uses HTTP :14045; there +# is no auto-discovery yet (tracking UDP does auto-discover, but the +# camera HTTP endpoint needs an explicit host). +QUEST_IP = "192.168.4.26" + +session = sf.SessionOrchestrator( + host_id="mac_studio", + output_dir=Path(__file__).parent / "output", +) + +# Head + hand tracking over UDP. Defaults to listening on +# 0.0.0.0:14043 — no IP needed here because the Quest app sends TO +# this Mac after the Mac broadcasts a discovery probe on :14044. +session.add(MetaQuestHandStream( + "quest_tracking", + mode="hand", # or "controller" to map Touch Plus pose into wrist slots +)) + +# Stereo passthrough camera. We pull MP4 + timestamps over HTTP on +# stop_recording() — this path DOES need an explicit Quest IP because +# the Mac initiates the request. +session.add(MetaQuestCameraStream( + "quest_cam", + quest_host=QUEST_IP, + output_dir=session.output_dir, + fps=30, + resolution=(1280, 720), +)) + +syncfield.viewer.launch(session) diff --git a/src/syncfield/viewer/frontend/package.json b/src/syncfield/viewer/frontend/package.json index 2a6c462..ea19b0a 100644 --- a/src/syncfield/viewer/frontend/package.json +++ b/src/syncfield/viewer/frontend/package.json @@ -11,19 +11,23 @@ "test:watch": "vitest" }, "dependencies": { + "@react-three/drei": "^10.7.7", + "@react-three/fiber": "^9.6.0", + "clsx": "^2.1.1", "react": "^19.1.0", "react-dom": "^19.1.0", - "clsx": "^2.1.1", - "tailwind-merge": "^3.3.0" + "tailwind-merge": "^3.3.0", + "three": "^0.183.2" }, "devDependencies": { + "@tailwindcss/vite": "^4.1.8", "@types/react": "^19.1.6", "@types/react-dom": "^19.1.6", + "@types/three": "^0.183.1", "@vitejs/plugin-react": "^4.5.2", + "tailwindcss": "^4.1.8", "typescript": "~5.8.3", "vite": "^6.3.5", - "vitest": "^3.2.1", - "tailwindcss": "^4.1.8", - "@tailwindcss/vite": "^4.1.8" + "vitest": "^3.2.1" } } diff --git a/src/syncfield/viewer/frontend/src/components/quest3-pose-panel.tsx b/src/syncfield/viewer/frontend/src/components/quest3-pose-panel.tsx new file mode 100644 index 0000000..b815b2d --- /dev/null +++ b/src/syncfield/viewer/frontend/src/components/quest3-pose-panel.tsx @@ -0,0 +1,341 @@ +/** + * Quest3PosePanel — 3-D hand skeleton + head rig renderer for the + * MetaQuestHandStream data. + * + * Ported from the opengraph-studio/showcase implementation with + * viewer-specific trim: + * - consumes ``pose`` payload from :hook:`useSensorStream` instead of + * a prop-drilled Quest3Frame, + * - drops the allocentric X-mirror (showcase flipped to match a third- + * person rig; viewer shows the user's own egocentric frame), + * - renders into a flex container so the parent sensor panel can size + * it alongside the other stream cards. + */ + +import type { Quest3Frame } from "@/lib/types"; +import { Line, OrbitControls } from "@react-three/drei"; +import { Canvas } from "@react-three/fiber"; +import { useMemo } from "react"; +import * as THREE from "three"; + +const JOINTS_PER_HAND = 26; +const HAND_FLOATS = JOINTS_PER_HAND * 3; +const WRIST_INDEX = 1; + +// OpenXR 26-joint connection graph (Palm → Wrist → 5 fingers). +const HAND_CONNECTIONS: [number, number][] = [ + [0, 1], [0, 6], [0, 11], [0, 16], [0, 21], + [1, 2], [2, 3], [3, 4], [4, 5], + [1, 6], [6, 7], [7, 8], [8, 9], [9, 10], + [1, 11], [11, 12], [12, 13], [13, 14], [14, 15], + [1, 16], [16, 17], [17, 18], [18, 19], [19, 20], + [1, 21], [21, 22], [22, 23], [23, 24], [24, 25], +]; + +const COLORS = { + left: "#22d3ee", + right: "#fb923c", + head: "#111827", + bg: "#FAF8F6", + grid: "#e5e0d8", + gridStrong: "#d8d0c5", + floor: "#f5f0e7", + wall: "#fcfaf6", + sideWall: "#f2ece1", + x: "#f87171", + y: "#34d399", + z: "#fbbf24", +} as const; + +type Vec3 = [number, number, number]; + +interface ParsedPoseData { + leftTracked: boolean; + rightTracked: boolean; + leftJoints: Vec3[]; + rightJoints: Vec3[]; + head: { position: Vec3; rotation: [number, number, number, number] } | null; + sceneCenter: Vec3; +} + +function parseQuest3Frame(frame: Quest3Frame | null): ParsedPoseData | null { + if (!frame?.hand_joints || frame.hand_joints.length < HAND_FLOATS * 2) return null; + + const leftJoints = splitJointArray(frame.hand_joints.slice(0, HAND_FLOATS)); + const rightJoints = splitJointArray(frame.hand_joints.slice(HAND_FLOATS, HAND_FLOATS * 2)); + + const leftTracked = countValidJoints(leftJoints) > 0; + const rightTracked = countValidJoints(rightJoints) > 0; + + const head = frame.head_pose && frame.head_pose.length >= 7 + ? { + position: [ + frame.head_pose[0] ?? 0, + frame.head_pose[1] ?? 0, + frame.head_pose[2] ?? 0, + ] as Vec3, + rotation: normalizeQuaternion([ + frame.head_pose[3] ?? 0, frame.head_pose[4] ?? 0, + frame.head_pose[5] ?? 0, frame.head_pose[6] ?? 1, + ]), + } + : null; + + const anchors = [ + head?.position, + leftJoints[WRIST_INDEX], + rightJoints[WRIST_INDEX], + ].filter((v): v is Vec3 => Boolean(v) && !isZeroVector(v!)); + + const allPoints = [ + ...leftJoints.filter((j) => !isZeroVector(j) && isFiniteVector(j)), + ...rightJoints.filter((j) => !isZeroVector(j) && isFiniteVector(j)), + ...anchors, + ]; + const avg = averagePoints(anchors.length > 0 ? anchors : [[0, 1.4, 0]]); + const minY = allPoints.length > 0 ? Math.min(...allPoints.map((p) => p[1])) : 0; + const sceneCenter: Vec3 = [avg[0], minY, avg[2]]; + + return { leftTracked, rightTracked, leftJoints, rightJoints, head, sceneCenter }; +} + +function splitJointArray(values: number[]): Vec3[] { + const result: Vec3[] = []; + for (let i = 0; i < JOINTS_PER_HAND; i++) { + const o = i * 3; + result.push([values[o] ?? 0, values[o + 1] ?? 0, values[o + 2] ?? 0]); + } + return result; +} + +function normalizeQuaternion(v: number[]): [number, number, number, number] { + const [x = 0, y = 0, z = 0, w = 1] = v; + const len = Math.hypot(x, y, z, w); + if (len < 1e-6) return [0, 0, 0, 1]; + return [x / len, y / len, z / len, w / len]; +} + +function countValidJoints(joints: Vec3[]): number { + return joints.filter((j) => !isZeroVector(j) && isFiniteVector(j)).length; +} + +function averagePoints(points: Vec3[]): Vec3 { + const s = points.reduce((a, p) => [a[0] + p[0], a[1] + p[1], a[2] + p[2]], [0, 0, 0]); + return [s[0] / points.length, s[1] / points.length, s[2] / points.length]; +} + +function vectorSubtract(a: Vec3, b: Vec3): Vec3 { + return [a[0] - b[0], a[1] - b[1], a[2] - b[2]]; +} + +function vectorScale(v: Vec3, s: number): Vec3 { + return [v[0] * s, v[1] * s, v[2] * s]; +} + +function isFiniteVector(v: Vec3): boolean { + return v.every((n) => Number.isFinite(n) && Math.abs(n) < 10); +} + +function isZeroVector(v: Vec3): boolean { + return v.every((n) => Math.abs(n) < 1e-5); +} + +// --------------------------------------------------------------------------- +// Three.js sub-components +// --------------------------------------------------------------------------- + +function Workspace() { + const floorLines = useMemo(() => { + const lines: Vec3[][] = []; + for (let o = -0.42; o <= 0.4201; o += 0.1) { + lines.push([[-0.42, 0, o], [0.42, 0, o]]); + lines.push([[o, 0, -0.42], [o, 0, 0.42]]); + } + return lines; + }, []); + + const backWallLines = useMemo(() => { + const lines: Vec3[][] = []; + for (let o = -0.42; o <= 0.4201; o += 0.1) { + lines.push([[-0.42, o, -0.42], [0.42, o, -0.42]]); + lines.push([[o, -0.42, -0.42], [o, 0.42, -0.42]]); + } + return lines; + }, []); + + const sideWallLines = useMemo(() => { + const lines: Vec3[][] = []; + for (let o = -0.42; o <= 0.4201; o += 0.1) { + lines.push([[0.42, -0.42, o], [0.42, 0.42, o]]); + lines.push([[0.42, o, -0.42], [0.42, o, 0.42]]); + } + return lines; + }, []); + + return ( + + + + + + + + + + + + + + + {floorLines.map((pts, i) => ( + + ))} + {backWallLines.map((pts, i) => ( + + ))} + {sideWallLines.map((pts, i) => ( + + ))} + + ); +} + +function HandSkeleton({ joints, color, scale }: { joints: Vec3[]; color: string; scale: number }) { + const validJoints = joints.filter((j) => j.some((v) => Math.abs(v) > 1e-5)).length; + if (validJoints < 2) return null; + + return ( + + {HAND_CONNECTIONS.map(([from, to]) => { + const a = joints[from]; + const b = joints[to]; + if (!a || !b || !isFiniteVector(a) || !isFiniteVector(b) || isZeroVector(a) || isZeroVector(b)) return null; + return ; + })} + {joints.map((joint, i) => { + if (!isFiniteVector(joint) || isZeroVector(joint)) return null; + const radius = (i === WRIST_INDEX ? 0.012 : 0.008) * scale; + return ( + + + + + ); + })} + + ); +} + +function HeadRig({ position, rotation, length, scale }: { + position: Vec3; + rotation: [number, number, number, number]; + length: number; + scale: number; +}) { + const quaternion = useMemo( + () => new THREE.Quaternion(rotation[0], rotation[1], rotation[2], rotation[3]).normalize(), + [rotation] + ); + + const axisLines = useMemo(() => { + const x = new THREE.Vector3(length, 0, 0).applyQuaternion(quaternion); + const y = new THREE.Vector3(0, length, 0).applyQuaternion(quaternion); + const z = new THREE.Vector3(0, 0, length).applyQuaternion(quaternion); + return { + x: [position, [position[0] + x.x, position[1] + x.y, position[2] + x.z] as Vec3], + y: [position, [position[0] + y.x, position[1] + y.y, position[2] + y.z] as Vec3], + z: [position, [position[0] + z.x, position[1] + z.y, position[2] + z.z] as Vec3], + }; + }, [length, position, quaternion]); + + return ( + + + + + + + + + + ); +} + +function SceneContent({ pose }: { pose: ParsedPoseData }) { + const rawPoints = [ + ...pose.leftJoints.map((j) => vectorSubtract(j, pose.sceneCenter)), + ...pose.rightJoints.map((j) => vectorSubtract(j, pose.sceneCenter)), + ]; + if (pose.head) rawPoints.push(vectorSubtract(pose.head.position, pose.sceneCenter)); + + const maxRadius = Math.max( + 0.15, + ...rawPoints.filter(isFiniteVector).map((p) => Math.hypot(p[0], p[1], p[2])) + ); + // Scale down aggressively so content stays inside the 0.84-m workspace box. + const fitScale = Math.min(0.8, Math.max(0.2, 0.18 / maxRadius)); + const headAxisLength = Math.max(0.03, 0.06 * fitScale); + const markerScale = Math.max(0.6, Math.min(1.2, fitScale * 1.5)); + + const transformed = (p: Vec3): Vec3 => vectorScale(vectorSubtract(p, pose.sceneCenter), fitScale); + const leftJoints = pose.leftJoints.map(transformed); + const rightJoints = pose.rightJoints.map(transformed); + const headPos = pose.head ? transformed(pose.head.position) : null; + + return ( + + {pose.leftTracked && } + {pose.rightTracked && } + {headPos && pose.head && ( + + )} + + ); +} + +// --------------------------------------------------------------------------- +// Main component +// --------------------------------------------------------------------------- + +interface Quest3PosePanelProps { + /** Pose payload as delivered by :hook:`useSensorStream`. */ + pose: Record | null; +} + +export function Quest3PosePanel({ pose: poseData }: Quest3PosePanelProps) { + const parsed = useMemo(() => { + if (!poseData) return null; + const frame: Quest3Frame = { + hand_joints: poseData.hand_joints, + joint_rotations: poseData.joint_rotations, + head_pose: poseData.head_pose, + }; + return parseQuest3Frame(frame); + }, [poseData]); + + if (!parsed) { + return ( +
+ Waiting for hand / head pose… +
+ ); + } + + return ( +
+ + + + + + + + + + + + + +
+ ); +} diff --git a/src/syncfield/viewer/frontend/src/components/sensor-panel.tsx b/src/syncfield/viewer/frontend/src/components/sensor-panel.tsx index 7e64b97..e3e1e39 100644 --- a/src/syncfield/viewer/frontend/src/components/sensor-panel.tsx +++ b/src/syncfield/viewer/frontend/src/components/sensor-panel.tsx @@ -1,5 +1,6 @@ import { useSensorStream } from "@/hooks/use-sensor-stream"; import { PosePanel } from "./pose-panel"; +import { Quest3PosePanel } from "./quest3-pose-panel"; import { SensorChart } from "./sensor-chart"; interface SensorPanelProps { @@ -7,19 +8,18 @@ interface SensorPanelProps { } /** - * Dispatcher between :component:`PosePanel` and :component:`SensorChart`. + * Dispatcher across the three sensor render paths: * - * The viewer's SSE sensor endpoint is already rate-limited server-side - * to ~10 Hz, so a single shared subscription is both the simplest and - * most efficient design — no separate "probe" connection needed. This - * component opens :hook:`useSensorStream` once, decides which panel to - * render from the first event's channel names, and the chosen panel - * opens its own short subscription for rendering. + * - :component:`Quest3PosePanel` — MetaQuestHandStream samples + * (detected by the ``hand_joints`` list channel). + * - :component:`PosePanel` — roll/pitch/yaw IMUs (e.g. WitMotion). + * - :component:`SensorChart` — fallback multi-channel line chart. * - * Orientation-capable sensors (those that emit ``roll``, ``pitch``, - * and ``yaw`` channels — e.g. WitMotion WT901BLE) get the 3D cube - * view; everything else falls back to the generic multi-channel line - * chart so existing adapters keep working unchanged. + * The viewer's SSE sensor endpoint is rate-limited server-side to + * ~10 Hz, so one shared subscription per stream is enough. We open + * :hook:`useSensorStream` once, look at both the scalar channel names + * and the pose payload to pick a panel, and that panel opens its own + * short subscription for rendering. */ function hasOrientationChannels(names: string[]): boolean { @@ -30,8 +30,12 @@ function hasOrientationChannels(names: string[]): boolean { ); } +function hasQuest3Pose(pose: Record | null): boolean { + return Boolean(pose && Array.isArray(pose.hand_joints) && pose.hand_joints.length > 0); +} + export function SensorPanel({ streamId }: SensorPanelProps) { - const { channels, isConnected } = useSensorStream(streamId); + const { channels, pose, isConnected } = useSensorStream(streamId); const channelNames = Object.keys(channels); if (!isConnected) { @@ -42,6 +46,10 @@ export function SensorPanel({ streamId }: SensorPanelProps) { ); } + if (hasQuest3Pose(pose)) { + return ; + } + if (channelNames.length === 0) { return (
diff --git a/src/syncfield/viewer/frontend/src/hooks/use-sensor-stream.ts b/src/syncfield/viewer/frontend/src/hooks/use-sensor-stream.ts index 3c705f8..6ed4de2 100644 --- a/src/syncfield/viewer/frontend/src/hooks/use-sensor-stream.ts +++ b/src/syncfield/viewer/frontend/src/hooks/use-sensor-stream.ts @@ -9,6 +9,10 @@ interface UseSensorStreamReturn { channels: Record; /** Rolling buffer of x-axis labels (timestamps). */ labels: number[]; + /** Latest vector-valued channels (e.g. ``hand_joints`` from + * MetaQuestHandStream). Not buffered — 3-D pose panels render + * instantaneous state. ``null`` when the stream is scalar-only. */ + pose: Record | null; /** Whether the SSE connection is alive. */ isConnected: boolean; } @@ -23,6 +27,7 @@ interface UseSensorStreamReturn { export function useSensorStream(streamId: string): UseSensorStreamReturn { const [channels, setChannels] = useState>({}); const [labels, setLabels] = useState([]); + const [pose, setPose] = useState | null>(null); const [isConnected, setIsConnected] = useState(false); // Mutable buffers for performance — we only push state on ticks @@ -71,6 +76,11 @@ export function useSensorStream(streamId: string): UseSensorStreamReturn { } } + // Vector-valued channels: ship the entire latest payload + // through — not buffered because pose panels want a single + // instantaneous snapshot per render. + if (data.pose) setPose(data.pose); + // Push a snapshot to React state setChannels({ ...channelBuf.current }); setLabels([...labelBuf.current]); @@ -99,5 +109,5 @@ export function useSensorStream(streamId: string): UseSensorStreamReturn { }; }, [streamId]); - return { channels, labels, isConnected }; + return { channels, labels, pose, isConnected }; } diff --git a/src/syncfield/viewer/frontend/src/lib/types.ts b/src/syncfield/viewer/frontend/src/lib/types.ts index 6b1bb3d..556d64b 100644 --- a/src/syncfield/viewer/frontend/src/lib/types.ts +++ b/src/syncfield/viewer/frontend/src/lib/types.ts @@ -158,9 +158,20 @@ export interface DiscoveredDevice { export interface SensorEvent { channels: Record; + // List / vector-valued channels (e.g. MetaQuestHandStream's + // ``hand_joints``: 156 floats). Present when the adapter emits + // non-scalar samples. ``null`` when the stream is scalar-only. + pose: Record | null; label: number | null; } +export interface Quest3Frame { + hand_joints?: number[]; // 156 floats (26 × 3 × 2 hands) + joint_rotations?: number[]; // 208 floats (26 × 4 × 2 hands) + head_pose?: number[]; // 7 floats (pos3 + quat4 xyzw) + mode?: "hands" | "controller"; +} + // --------------------------------------------------------------------------- // Type guards // --------------------------------------------------------------------------- diff --git a/src/syncfield/viewer/frontend/yarn.lock b/src/syncfield/viewer/frontend/yarn.lock index df5ba2f..aed81a0 100644 --- a/src/syncfield/viewer/frontend/yarn.lock +++ b/src/syncfield/viewer/frontend/yarn.lock @@ -130,6 +130,11 @@ dependencies: "@babel/helper-plugin-utils" "^7.27.1" +"@babel/runtime@^7.17.8", "@babel/runtime@^7.26.0": + version "7.29.2" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.29.2.tgz#9a6e2d05f4b6692e1801cd4fb176ad823930ed5e" + integrity sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g== + "@babel/template@^7.28.6": version "7.28.6" resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.28.6.tgz#0e7e56ecedb78aeef66ce7972b082fce76a23e57" @@ -160,6 +165,11 @@ "@babel/helper-string-parser" "^7.27.1" "@babel/helper-validator-identifier" "^7.28.5" +"@dimforge/rapier3d-compat@~0.12.0": + version "0.12.0" + resolved "https://registry.yarnpkg.com/@dimforge/rapier3d-compat/-/rapier3d-compat-0.12.0.tgz#7b3365e1dfdc5cd957b45afe920b4ac06c7cd389" + integrity sha512-uekIGetywIgopfD97oDL5PfeezkFpNhwlzlaEYNOA0N6ghdsOvh/HYjSMek5Q2O1PYvRSDFcqFVJl4r4ZBwOow== + "@emnapi/core@^1.8.1": version "1.9.2" resolved "https://registry.yarnpkg.com/@emnapi/core/-/core-1.9.2.tgz#3870265ecffc7352d01ead62d8d83d8358a2d034" @@ -476,6 +486,18 @@ "@jridgewell/resolve-uri" "^3.1.0" "@jridgewell/sourcemap-codec" "^1.4.14" +"@mediapipe/tasks-vision@0.10.17": + version "0.10.17" + resolved "https://registry.yarnpkg.com/@mediapipe/tasks-vision/-/tasks-vision-0.10.17.tgz#2c1c73ed81902b21d37336a587b96183bb6882d5" + integrity sha512-CZWV/q6TTe8ta61cZXjfnnHsfWIdFhms03M9T7Cnd5y2mdpylJM0rF1qRq+wsQVRMLz1OYPVEBU9ph2Bx8cxrg== + +"@monogrid/gainmap-js@^3.0.6": + version "3.4.0" + resolved "https://registry.yarnpkg.com/@monogrid/gainmap-js/-/gainmap-js-3.4.0.tgz#9788a92d34530d3da274fef8d3445d008c640c48" + integrity sha512-2Z0FATFHaoYJ8b+Y4y4Hgfn3FRFwuU5zRrk+9dFWp4uGAdHGqVEdP7HP+gLA3X469KXHmfupJaUbKo1b/aDKIg== + dependencies: + promise-worker-transferable "^1.0.4" + "@napi-rs/wasm-runtime@^1.1.1": version "1.1.3" resolved "https://registry.yarnpkg.com/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.3.tgz#1eeb8699770481306e5fcd84471f20fcb6177336" @@ -483,6 +505,49 @@ dependencies: "@tybys/wasm-util" "^0.10.1" +"@react-three/drei@^10.7.7": + version "10.7.7" + resolved "https://registry.yarnpkg.com/@react-three/drei/-/drei-10.7.7.tgz#7ac029ace001307dfc71c61b6284b1c12efe8b80" + integrity sha512-ff+J5iloR0k4tC++QtD/j9u3w5fzfgFAWDtAGQah9pF2B1YgOq/5JxqY0/aVoQG5r3xSZz0cv5tk2YuBob4xEQ== + dependencies: + "@babel/runtime" "^7.26.0" + "@mediapipe/tasks-vision" "0.10.17" + "@monogrid/gainmap-js" "^3.0.6" + "@use-gesture/react" "^10.3.1" + camera-controls "^3.1.0" + cross-env "^7.0.3" + detect-gpu "^5.0.56" + glsl-noise "^0.0.0" + hls.js "^1.5.17" + maath "^0.10.8" + meshline "^3.3.1" + stats-gl "^2.2.8" + stats.js "^0.17.0" + suspend-react "^0.1.3" + three-mesh-bvh "^0.8.3" + three-stdlib "^2.35.6" + troika-three-text "^0.52.4" + tunnel-rat "^0.1.2" + use-sync-external-store "^1.4.0" + utility-types "^3.11.0" + zustand "^5.0.1" + +"@react-three/fiber@^9.6.0": + version "9.6.0" + resolved "https://registry.yarnpkg.com/@react-three/fiber/-/fiber-9.6.0.tgz#3d86b8ab1a765d3a28ce4633381af48e6a9762a9" + integrity sha512-90abYK2q5/qDM+GACs9zRvc5KhEEpEWqWlHSd64zTPNxg+9wCJvTfyD9x2so7hlQhjRYO1Fa6flR3BC/kpTFkA== + dependencies: + "@babel/runtime" "^7.17.8" + "@types/webxr" "*" + base64-js "^1.5.1" + buffer "^6.0.3" + its-fine "^2.0.0" + react-use-measure "^2.1.7" + scheduler "^0.27.0" + suspend-react "^0.1.3" + use-sync-external-store "^1.4.0" + zustand "^5.0.3" + "@rolldown/pluginutils@1.0.0-beta.27": version "1.0.0-beta.27" resolved "https://registry.yarnpkg.com/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.27.tgz#47d2bf4cef6d470b22f5831b420f8964e0bf755f" @@ -720,6 +785,11 @@ "@tailwindcss/oxide" "4.2.2" tailwindcss "4.2.2" +"@tweenjs/tween.js@~23.1.3": + version "23.1.3" + resolved "https://registry.yarnpkg.com/@tweenjs/tween.js/-/tween.js-23.1.3.tgz#eff0245735c04a928bb19c026b58c2a56460539d" + integrity sha512-vJmvvwFxYuGnF2axRtPYocag6Clbb5YS7kLL+SO/TeVFzHqDIWrNKYtcsPMibjDx9O+bu+psAy9NKfWklassUA== + "@tybys/wasm-util@^0.10.1": version "0.10.1" resolved "https://registry.yarnpkg.com/@tybys/wasm-util/-/wasm-util-0.10.1.tgz#ecddd3205cf1e2d5274649ff0eedd2991ed7f414" @@ -773,16 +843,31 @@ resolved "https://registry.yarnpkg.com/@types/deep-eql/-/deep-eql-4.0.2.tgz#334311971d3a07121e7eb91b684a605e7eea9cbd" integrity sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw== +"@types/draco3d@^1.4.0": + version "1.4.10" + resolved "https://registry.yarnpkg.com/@types/draco3d/-/draco3d-1.4.10.tgz#63ec0ba78b30bd58203ec031f4e4f0198c596dca" + integrity sha512-AX22jp8Y7wwaBgAixaSvkoG4M/+PlAcm3Qs4OW8yT9DM4xUpWKeFhLueTAyZF39pviAdcDdeJoACapiAceqNcw== + "@types/estree@1.0.8", "@types/estree@^1.0.0": version "1.0.8" resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.8.tgz#958b91c991b1867ced318bedea0e215ee050726e" integrity sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w== +"@types/offscreencanvas@^2019.6.4": + version "2019.7.3" + resolved "https://registry.yarnpkg.com/@types/offscreencanvas/-/offscreencanvas-2019.7.3.tgz#90267db13f64d6e9ccb5ae3eac92786a7c77a516" + integrity sha512-ieXiYmgSRXUDeOntE1InxjWyvEelZGP63M+cGuquuRLuIKKT1osnkXjxev9B7d1nXSug5vpunx+gNlbVxMlC9A== + "@types/react-dom@^19.1.6": version "19.2.3" resolved "https://registry.yarnpkg.com/@types/react-dom/-/react-dom-19.2.3.tgz#c1e305d15a52a3e508d54dca770d202cb63abf2c" integrity sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ== +"@types/react-reconciler@^0.28.9": + version "0.28.9" + resolved "https://registry.yarnpkg.com/@types/react-reconciler/-/react-reconciler-0.28.9.tgz#d24b4864c384e770c83275b3fe73fba00269c83b" + integrity sha512-HHM3nxyUZ3zAylX8ZEyrDNd2XZOnQ0D5XfunJF5FLQnZbHHYq4UWvW1QfelQNXv1ICNkwYhfxjwfnqivYB6bFg== + "@types/react@^19.1.6": version "19.2.14" resolved "https://registry.yarnpkg.com/@types/react/-/react-19.2.14.tgz#39604929b5e3957e3a6fa0001dafb17c7af70bad" @@ -790,6 +875,41 @@ dependencies: csstype "^3.2.2" +"@types/stats.js@*": + version "0.17.4" + resolved "https://registry.yarnpkg.com/@types/stats.js/-/stats.js-0.17.4.tgz#1933e5ff153a23c7664487833198d685c22e791e" + integrity sha512-jIBvWWShCvlBqBNIZt0KAshWpvSjhkwkEu4ZUcASoAvhmrgAUI2t1dXrjSL4xXVLB4FznPrIsX3nKXFl/Dt4vA== + +"@types/three@*", "@types/three@^0.183.1": + version "0.183.1" + resolved "https://registry.yarnpkg.com/@types/three/-/three-0.183.1.tgz#d812d028b38ad68843725e3e7bd3268607cef150" + integrity sha512-f2Pu5Hrepfgavttdye3PsH5RWyY/AvdZQwIVhrc4uNtvF7nOWJacQKcoVJn0S4f0yYbmAE6AR+ve7xDcuYtMGw== + dependencies: + "@dimforge/rapier3d-compat" "~0.12.0" + "@tweenjs/tween.js" "~23.1.3" + "@types/stats.js" "*" + "@types/webxr" ">=0.5.17" + "@webgpu/types" "*" + fflate "~0.8.2" + meshoptimizer "~1.0.1" + +"@types/webxr@*", "@types/webxr@>=0.5.17", "@types/webxr@^0.5.2": + version "0.5.24" + resolved "https://registry.yarnpkg.com/@types/webxr/-/webxr-0.5.24.tgz#734d5d90dadc5809a53e422726c60337fa2f4a44" + integrity sha512-h8fgEd/DpoS9CBrjEQXR+dIDraopAEfu4wYVNY2tEPwk60stPWhvZMf4Foo5FakuQ7HFZoa8WceaWFervK2Ovg== + +"@use-gesture/core@10.3.1": + version "10.3.1" + resolved "https://registry.yarnpkg.com/@use-gesture/core/-/core-10.3.1.tgz#976c9421e905f0079d49822cfd5c2e56b808fc56" + integrity sha512-WcINiDt8WjqBdUXye25anHiNxPc0VOrlT8F6LLkU6cycrOGUDyY/yyFmsg3k8i5OLvv25llc0QC45GhR/C8llw== + +"@use-gesture/react@^10.3.1": + version "10.3.1" + resolved "https://registry.yarnpkg.com/@use-gesture/react/-/react-10.3.1.tgz#17a743a894d9bd9a0d1980c618f37f0164469867" + integrity sha512-Yy19y6O2GJq8f7CHf7L0nxL8bf4PZCPaVOCgJrusOeFHY1LvHgYXnmnXg6N5iwAnbgbZCDjo60SiM6IPJi9C5g== + dependencies: + "@use-gesture/core" "10.3.1" + "@vitejs/plugin-react@^4.5.2": version "4.7.0" resolved "https://registry.yarnpkg.com/@vitejs/plugin-react/-/plugin-react-4.7.0.tgz#647af4e7bb75ad3add578e762ad984b90f4a24b9" @@ -863,16 +983,33 @@ loupe "^3.1.4" tinyrainbow "^2.0.0" +"@webgpu/types@*": + version "0.1.69" + resolved "https://registry.yarnpkg.com/@webgpu/types/-/types-0.1.69.tgz#6b849bf370a1f29c78bd3aeba8e84c1150b237f2" + integrity sha512-RPmm6kgRbI8e98zSD3RVACvnuktIja5+yLgDAkTmxLr90BEwdTXRQWNLF3ETTTyH/8mKhznZuN5AveXYFEsMGQ== + assertion-error@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/assertion-error/-/assertion-error-2.0.1.tgz#f641a196b335690b1070bf00b6e7593fec190bf7" integrity sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA== +base64-js@^1.3.1, base64-js@^1.5.1: + version "1.5.1" + resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" + integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== + baseline-browser-mapping@^2.10.12: version "2.10.17" resolved "https://registry.yarnpkg.com/baseline-browser-mapping/-/baseline-browser-mapping-2.10.17.tgz#435c101835c314c2d89d768795e1ea79941fafd3" integrity sha512-HdrkN8eVG2CXxeifv/VdJ4A4RSra1DTW8dc/hdxzhGHN8QePs6gKaWM9pHPcpCoxYZJuOZ8drHmbdpLHjCYjLA== +bidi-js@^1.0.2: + version "1.0.3" + resolved "https://registry.yarnpkg.com/bidi-js/-/bidi-js-1.0.3.tgz#6f8bcf3c877c4d9220ddf49b9bb6930c88f877d2" + integrity sha512-RKshQI1R3YQ+n9YJz2QQ147P66ELpa1FQEg20Dk8oW9t2KgLbpDLLp9aGZ7y8WHSshDknG0bknqGw5/tyCs5tw== + dependencies: + require-from-string "^2.0.2" + browserslist@^4.24.0: version "4.28.2" resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.28.2.tgz#f50b65362ef48974ca9f50b3680566d786b811d2" @@ -884,11 +1021,24 @@ browserslist@^4.24.0: node-releases "^2.0.36" update-browserslist-db "^1.2.3" +buffer@^6.0.3: + version "6.0.3" + resolved "https://registry.yarnpkg.com/buffer/-/buffer-6.0.3.tgz#2ace578459cc8fbe2a70aaa8f52ee63b6a74c6c6" + integrity sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA== + dependencies: + base64-js "^1.3.1" + ieee754 "^1.2.1" + cac@^6.7.14: version "6.7.14" resolved "https://registry.yarnpkg.com/cac/-/cac-6.7.14.tgz#804e1e6f506ee363cb0e3ccbb09cad5dd9870959" integrity sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ== +camera-controls@^3.1.0: + version "3.1.2" + resolved "https://registry.yarnpkg.com/camera-controls/-/camera-controls-3.1.2.tgz#fb504fa77ade929aa75c22ba42d0b53f48a81e44" + integrity sha512-xkxfpG2ECZ6Ww5/9+kf4mfg1VEYAoe9aDSY+IwF0UEs7qEzwy0aVRfs2grImIECs/PoBtWFrh7RXsQkwG922JA== + caniuse-lite@^1.0.30001782: version "1.0.30001787" resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001787.tgz#fd25c5e42e2d35df5c75eddda00d15d9c0c68f81" @@ -920,6 +1070,22 @@ convert-source-map@^2.0.0: resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-2.0.0.tgz#4b560f649fc4e918dd0ab75cf4961e8bc882d82a" integrity sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg== +cross-env@^7.0.3: + version "7.0.3" + resolved "https://registry.yarnpkg.com/cross-env/-/cross-env-7.0.3.tgz#865264b29677dc015ba8418918965dd232fc54cf" + integrity sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw== + dependencies: + cross-spawn "^7.0.1" + +cross-spawn@^7.0.1: + version "7.0.6" + resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.6.tgz#8a58fe78f00dcd70c370451759dfbfaf03e8ee9f" + integrity sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA== + dependencies: + path-key "^3.1.0" + shebang-command "^2.0.0" + which "^2.0.1" + csstype@^3.2.2: version "3.2.3" resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.2.3.tgz#ec48c0f3e993e50648c86da559e2610995cf989a" @@ -937,11 +1103,23 @@ deep-eql@^5.0.1: resolved "https://registry.yarnpkg.com/deep-eql/-/deep-eql-5.0.2.tgz#4b756d8d770a9257300825d52a2c2cff99c3a341" integrity sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q== +detect-gpu@^5.0.56: + version "5.0.70" + resolved "https://registry.yarnpkg.com/detect-gpu/-/detect-gpu-5.0.70.tgz#db2202d3cd440714ba6e789ff8b62d1b584eabf7" + integrity sha512-bqerEP1Ese6nt3rFkwPnGbsUF9a4q+gMmpTVVOEzoCyeCc+y7/RvJnQZJx1JwhgQI5Ntg0Kgat8Uu7XpBqnz1w== + dependencies: + webgl-constants "^1.1.1" + detect-libc@^2.0.3: version "2.1.2" resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-2.1.2.tgz#689c5dcdc1900ef5583a4cb9f6d7b473742074ad" integrity sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ== +draco3d@^1.4.1: + version "1.5.7" + resolved "https://registry.yarnpkg.com/draco3d/-/draco3d-1.5.7.tgz#94f9bce293eb8920c159dc91a4ce9124a9e899e0" + integrity sha512-m6WCKt/erDXcw+70IJXnG7M3awwQPAsZvJGX5zY7beBqpELw6RDGkYVU0W43AFxye4pDZ5i2Lbyc/NNGqwjUVQ== + electron-to-chromium@^1.5.328: version "1.5.335" resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.5.335.tgz#0b957cea44ef86795c227c616d16b4803d119daa" @@ -1046,6 +1224,16 @@ fdir@^6.4.4, fdir@^6.5.0: resolved "https://registry.yarnpkg.com/fdir/-/fdir-6.5.0.tgz#ed2ab967a331ade62f18d077dae192684d50d350" integrity sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg== +fflate@^0.6.9: + version "0.6.10" + resolved "https://registry.yarnpkg.com/fflate/-/fflate-0.6.10.tgz#5f40f9659205936a2d18abf88b2e7781662b6d43" + integrity sha512-IQrh3lEPM93wVCEczc9SaAOvkmcoQn/G8Bo1e8ZPlY3X3bnAxWaBdvTdvM1hP62iZp0BXWDy4vTAy4fF0+Dlpg== + +fflate@~0.8.2: + version "0.8.2" + resolved "https://registry.yarnpkg.com/fflate/-/fflate-0.8.2.tgz#fc8631f5347812ad6028bbe4a2308b2792aa1dea" + integrity sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A== + fsevents@~2.3.2, fsevents@~2.3.3: version "2.3.3" resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6" @@ -1056,11 +1244,48 @@ gensync@^1.0.0-beta.2: resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0" integrity sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg== +glsl-noise@^0.0.0: + version "0.0.0" + resolved "https://registry.yarnpkg.com/glsl-noise/-/glsl-noise-0.0.0.tgz#367745f3a33382c0eeec4cb54b7e99cfc1d7670b" + integrity sha512-b/ZCF6amfAUb7dJM/MxRs7AetQEahYzJ8PtgfrmEdtw6uyGOr+ZSGtgjFm6mfsBkxJ4d2W7kg+Nlqzqvn3Bc0w== + graceful-fs@^4.2.4: version "4.2.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ== +hls.js@^1.5.17: + version "1.6.16" + resolved "https://registry.yarnpkg.com/hls.js/-/hls.js-1.6.16.tgz#e2e7e79b68357cb3f8aa82dec0a36c9c2701f5c0" + integrity sha512-VSIRpLfRwlAAdGL4wiTucx2ScRipo0ed1FBatWkyt832jC4CReKstga6yIhYVwGu9LOBjuX9wzmRMeQdBJtzEA== + +ieee754@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" + integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== + +immediate@~3.0.5: + version "3.0.6" + resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b" + integrity sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ== + +is-promise@^2.1.0: + version "2.2.2" + resolved "https://registry.yarnpkg.com/is-promise/-/is-promise-2.2.2.tgz#39ab959ccbf9a774cf079f7b40c7a26f763135f1" + integrity sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ== + +isexe@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" + integrity sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw== + +its-fine@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/its-fine/-/its-fine-2.0.0.tgz#a90b18a3ee4c211a1fb6faac2abcc2b682ce1f21" + integrity sha512-KLViCmWx94zOvpLwSlsx6yOCeMhZYaxrJV87Po5k/FoZzcPSahvK5qJ7fYhS61sZi5ikmh2S3Hz55A2l3U69ng== + dependencies: + "@types/react-reconciler" "^0.28.9" + jiti@^2.6.1: version "2.6.1" resolved "https://registry.yarnpkg.com/jiti/-/jiti-2.6.1.tgz#178ef2fc9a1a594248c20627cd820187a4d78d92" @@ -1086,6 +1311,13 @@ json5@^2.2.3: resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.3.tgz#78cd6f1a19bdc12b73db5ad0c61efd66c1e29283" integrity sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg== +lie@^3.0.2: + version "3.3.0" + resolved "https://registry.yarnpkg.com/lie/-/lie-3.3.0.tgz#dcf82dee545f46074daf200c7c1c5a08e0f40f6a" + integrity sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ== + dependencies: + immediate "~3.0.5" + lightningcss-android-arm64@1.32.0: version "1.32.0" resolved "https://registry.yarnpkg.com/lightningcss-android-arm64/-/lightningcss-android-arm64-1.32.0.tgz#f033885116dfefd9c6f54787523e3514b61e1968" @@ -1172,6 +1404,11 @@ lru-cache@^5.1.1: dependencies: yallist "^3.0.2" +maath@^0.10.8: + version "0.10.8" + resolved "https://registry.yarnpkg.com/maath/-/maath-0.10.8.tgz#cf647544430141bf6982da6e878abb6c4b804e24" + integrity sha512-tRvbDF0Pgqz+9XUa4jjfgAQ8/aPKmQdWXilFu2tMy4GWj4NOsx99HlULO4IeREfbO3a0sA145DZYyvXPkybm0g== + magic-string@^0.30.17, magic-string@^0.30.21: version "0.30.21" resolved "https://registry.yarnpkg.com/magic-string/-/magic-string-0.30.21.tgz#56763ec09a0fa8091df27879fd94d19078c00d91" @@ -1179,6 +1416,16 @@ magic-string@^0.30.17, magic-string@^0.30.21: dependencies: "@jridgewell/sourcemap-codec" "^1.5.5" +meshline@^3.3.1: + version "3.3.1" + resolved "https://registry.yarnpkg.com/meshline/-/meshline-3.3.1.tgz#20decfd5cdd25c8469e862ddf0ab1ad167759734" + integrity sha512-/TQj+JdZkeSUOl5Mk2J7eLcYTLiQm2IDzmlSvYm7ov15anEcDJ92GHqqazxTSreeNgfnYu24kiEvvv0WlbCdFQ== + +meshoptimizer@~1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/meshoptimizer/-/meshoptimizer-1.0.1.tgz#c3ef0d509a8b84ac562493dba5a108fd67fa76dc" + integrity sha512-Vix+QlA1YYT3FwmBBZ+49cE5y/b+pRrcXKqGpS5ouh33d3lSp2PoTpCw19E0cKDFWalembrHnIaZetf27a+W2g== + ms@^2.1.3: version "2.1.3" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" @@ -1194,6 +1441,11 @@ node-releases@^2.0.36: resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.37.tgz#9bd4f10b77ba39c2b9402d4e8399c482a797f671" integrity sha512-1h5gKZCF+pO/o3Iqt5Jp7wc9rH3eJJ0+nh/CIoiRwjRxde/hAHyLPXYN4V3CqKAbiZPSeJFSWHmJsbkicta0Eg== +path-key@^3.1.0: + version "3.1.1" + resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375" + integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q== + pathe@^2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/pathe/-/pathe-2.0.3.tgz#3ecbec55421685b70a9da872b2cff3e1cbed1716" @@ -1223,6 +1475,19 @@ postcss@^8.5.3, postcss@^8.5.6: picocolors "^1.1.1" source-map-js "^1.2.1" +potpack@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/potpack/-/potpack-1.0.2.tgz#23b99e64eb74f5741ffe7656b5b5c4ddce8dfc14" + integrity sha512-choctRBIV9EMT9WGAZHn3V7t0Z2pMQyl0EZE6pFc/6ml3ssw7Dlf/oAOvFwjm1HVsqfQN8GfeFyJ+d8tRzqueQ== + +promise-worker-transferable@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/promise-worker-transferable/-/promise-worker-transferable-1.0.4.tgz#2c72861ba053e5ae42b487b4a83b1ed3ae3786e8" + integrity sha512-bN+0ehEnrXfxV2ZQvU2PetO0n4gqBD4ulq3MI1WOPLgr7/Mg9yRQkX5+0v1vagr74ZTsl7XtzlaYDo2EuCeYJw== + dependencies: + is-promise "^2.1.0" + lie "^3.0.2" + react-dom@^19.1.0: version "19.2.5" resolved "https://registry.yarnpkg.com/react-dom/-/react-dom-19.2.5.tgz#b8768b10837d0b8e9ca5b9e2d58dff3d880ea25e" @@ -1235,11 +1500,21 @@ react-refresh@^0.17.0: resolved "https://registry.yarnpkg.com/react-refresh/-/react-refresh-0.17.0.tgz#b7e579c3657f23d04eccbe4ad2e58a8ed51e7e53" integrity sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ== +react-use-measure@^2.1.7: + version "2.1.7" + resolved "https://registry.yarnpkg.com/react-use-measure/-/react-use-measure-2.1.7.tgz#36b8a2e7fd2fa58109ab851b3addcb0aad66ad1d" + integrity sha512-KrvcAo13I/60HpwGO5jpW7E9DfusKyLPLvuHlUyP5zqnmAPhNc6qTRjUQrdTADl0lpPpDVU2/Gg51UlOGHXbdg== + react@^19.1.0: version "19.2.5" resolved "https://registry.yarnpkg.com/react/-/react-19.2.5.tgz#c888ab8b8ef33e2597fae8bdb2d77edbdb42858b" integrity sha512-llUJLzz1zTUBrskt2pwZgLq59AemifIftw4aB7JxOqf1HY2FDaGDxgwpAPVzHU1kdWabH7FauP4i1oEeer2WCA== +require-from-string@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/require-from-string/-/require-from-string-2.0.2.tgz#89a7fdd938261267318eafe14f9c32e598c36909" + integrity sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw== + rollup@^4.34.9, rollup@^4.43.0: version "4.60.1" resolved "https://registry.yarnpkg.com/rollup/-/rollup-4.60.1.tgz#b4aa2bcb3a5e1437b5fad40d43fe42d4bde7a42d" @@ -1284,6 +1559,18 @@ semver@^6.3.1: resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== +shebang-command@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-2.0.0.tgz#ccd0af4f8835fbdc265b82461aaf0c36663f34ea" + integrity sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA== + dependencies: + shebang-regex "^3.0.0" + +shebang-regex@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172" + integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== + siginfo@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/siginfo/-/siginfo-2.0.0.tgz#32e76c70b79724e3bb567cb9d543eb858ccfaf30" @@ -1299,6 +1586,19 @@ stackback@0.0.2: resolved "https://registry.yarnpkg.com/stackback/-/stackback-0.0.2.tgz#1ac8a0d9483848d1695e418b6d031a3c3ce68e3b" integrity sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw== +stats-gl@^2.2.8: + version "2.4.2" + resolved "https://registry.yarnpkg.com/stats-gl/-/stats-gl-2.4.2.tgz#28a6c869fc3a36a8be608ef21df63c0aad99d1ba" + integrity sha512-g5O9B0hm9CvnM36+v7SFl39T7hmAlv541tU81ME8YeSb3i1CIP5/QdDeSB3A0la0bKNHpxpwxOVRo2wFTYEosQ== + dependencies: + "@types/three" "*" + three "^0.170.0" + +stats.js@^0.17.0: + version "0.17.0" + resolved "https://registry.yarnpkg.com/stats.js/-/stats.js-0.17.0.tgz#b1c3dc46d94498b578b7fd3985b81ace7131cc7d" + integrity sha512-hNKz8phvYLPEcRkeG1rsGmV5ChMjKDAWU7/OJJdDErPBNChQXxCo3WZurGpnWc6gZhAzEPFad1aVgyOANH1sMw== + std-env@^3.9.0: version "3.10.0" resolved "https://registry.yarnpkg.com/std-env/-/std-env-3.10.0.tgz#d810b27e3a073047b2b5e40034881f5ea6f9c83b" @@ -1311,6 +1611,11 @@ strip-literal@^3.0.0: dependencies: js-tokens "^9.0.1" +suspend-react@^0.1.3: + version "0.1.3" + resolved "https://registry.yarnpkg.com/suspend-react/-/suspend-react-0.1.3.tgz#a52f49d21cfae9a2fb70bd0c68413d3f9d90768e" + integrity sha512-aqldKgX9aZqpoDp3e8/BZ8Dm7x1pJl+qI3ZKxDN0i/IQTWUwBx/ManmlVJ3wowqbno6c2bmiIfs+Um6LbsjJyQ== + tailwind-merge@^3.3.0: version "3.5.0" resolved "https://registry.yarnpkg.com/tailwind-merge/-/tailwind-merge-3.5.0.tgz#06502f4496ba15151445d97d916a26564d50d1ca" @@ -1326,6 +1631,33 @@ tapable@^2.3.0: resolved "https://registry.yarnpkg.com/tapable/-/tapable-2.3.2.tgz#86755feabad08d82a26b891db044808c6ad00f15" integrity sha512-1MOpMXuhGzGL5TTCZFItxCc0AARf1EZFQkGqMm7ERKj8+Hgr5oLvJOVFcC+lRmR8hCe2S3jC4T5D7Vg/d7/fhA== +three-mesh-bvh@^0.8.3: + version "0.8.3" + resolved "https://registry.yarnpkg.com/three-mesh-bvh/-/three-mesh-bvh-0.8.3.tgz#c5e72472e7f062ff79084157a25c122d73184163" + integrity sha512-4G5lBaF+g2auKX3P0yqx+MJC6oVt6sB5k+CchS6Ob0qvH0YIhuUk1eYr7ktsIpY+albCqE80/FVQGV190PmiAg== + +three-stdlib@^2.35.6: + version "2.36.1" + resolved "https://registry.yarnpkg.com/three-stdlib/-/three-stdlib-2.36.1.tgz#7aad805f3de1b8f792274226132bda809fcb15c5" + integrity sha512-XyGQrFmNQ5O/IoKm556ftwKsBg11TIb301MB5dWNicziQBEs2g3gtOYIf7pFiLa0zI2gUwhtCjv9fmjnxKZ1Cg== + dependencies: + "@types/draco3d" "^1.4.0" + "@types/offscreencanvas" "^2019.6.4" + "@types/webxr" "^0.5.2" + draco3d "^1.4.1" + fflate "^0.6.9" + potpack "^1.0.1" + +three@^0.170.0: + version "0.170.0" + resolved "https://registry.yarnpkg.com/three/-/three-0.170.0.tgz#6087f97aab79e9e9312f9c89fcef6808642dfbb7" + integrity sha512-FQK+LEpYc0fBD+J8g6oSEyyNzjp+Q7Ks1C568WWaoMRLW+TkNNWmenWeGgJjV105Gd+p/2ql1ZcjYvNiPZBhuQ== + +three@^0.183.2: + version "0.183.2" + resolved "https://registry.yarnpkg.com/three/-/three-0.183.2.tgz#606e3195bf210ef8d1eaaca2ab8c59d92d2bbc18" + integrity sha512-di3BsL2FEQ1PA7Hcvn4fyJOlxRRgFYBpMTcyOgkwJIaDOdJMebEFPA+t98EvjuljDx4hNulAGwF6KIjtwI5jgQ== + tinybench@^2.9.0: version "2.9.0" resolved "https://registry.yarnpkg.com/tinybench/-/tinybench-2.9.0.tgz#103c9f8ba6d7237a47ab6dd1dcff77251863426b" @@ -1359,11 +1691,38 @@ tinyspy@^4.0.3: resolved "https://registry.yarnpkg.com/tinyspy/-/tinyspy-4.0.4.tgz#d77a002fb53a88aa1429b419c1c92492e0c81f78" integrity sha512-azl+t0z7pw/z958Gy9svOTuzqIk6xq+NSheJzn5MMWtWTFywIacg2wUlzKFGtt3cthx0r2SxMK0yzJOR0IES7Q== +troika-three-text@^0.52.4: + version "0.52.4" + resolved "https://registry.yarnpkg.com/troika-three-text/-/troika-three-text-0.52.4.tgz#f7b2389a2067d9506a5757457771cf4f6356e738" + integrity sha512-V50EwcYGruV5rUZ9F4aNsrytGdKcXKALjEtQXIOBfhVoZU9VAqZNIoGQ3TMiooVqFAbR1w15T+f+8gkzoFzawg== + dependencies: + bidi-js "^1.0.2" + troika-three-utils "^0.52.4" + troika-worker-utils "^0.52.0" + webgl-sdf-generator "1.1.1" + +troika-three-utils@^0.52.4: + version "0.52.4" + resolved "https://registry.yarnpkg.com/troika-three-utils/-/troika-three-utils-0.52.4.tgz#9292019e93cab97582af1cf491c4c895e5c03b66" + integrity sha512-NORAStSVa/BDiG52Mfudk4j1FG4jC4ILutB3foPnfGbOeIs9+G5vZLa0pnmnaftZUGm4UwSoqEpWdqvC7zms3A== + +troika-worker-utils@^0.52.0: + version "0.52.0" + resolved "https://registry.yarnpkg.com/troika-worker-utils/-/troika-worker-utils-0.52.0.tgz#ba5525fc444345006ebab0bc9cabdd66f1561e66" + integrity sha512-W1CpvTHykaPH5brv5VHLfQo9D1OYuo0cSBEUQFFT/nBUzM8iD6Lq2/tgG/f1OelbAS1WtaTPQzE5uM49egnngw== + tslib@^2.4.0, tslib@^2.8.1: version "2.8.1" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.8.1.tgz#612efe4ed235d567e8aba5f2a5fab70280ade83f" integrity sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w== +tunnel-rat@^0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/tunnel-rat/-/tunnel-rat-0.1.2.tgz#1717efbc474ea2d8aa05a91622457a6e201c0aeb" + integrity sha512-lR5VHmkPhzdhrM092lI2nACsLO4QubF0/yoOhzX7c+wIpbN1GjHNzCc91QlpxBi+cnx8vVJ+Ur6vL5cEoQPFpQ== + dependencies: + zustand "^4.3.2" + typescript@~5.8.3: version "5.8.3" resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.8.3.tgz#92f8a3e5e3cf497356f4178c34cd65a7f5e8440e" @@ -1377,6 +1736,16 @@ update-browserslist-db@^1.2.3: escalade "^3.2.0" picocolors "^1.1.1" +use-sync-external-store@^1.2.2, use-sync-external-store@^1.4.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/use-sync-external-store/-/use-sync-external-store-1.6.0.tgz#b174bfa65cb2b526732d9f2ac0a408027876f32d" + integrity sha512-Pp6GSwGP/NrPIrxVFAIkOQeyw8lFenOHijQWkUTrDvrF4ALqylP2C/KCkeS9dpUM3KvYRQhna5vt7IL95+ZQ9w== + +utility-types@^3.11.0: + version "3.11.0" + resolved "https://registry.yarnpkg.com/utility-types/-/utility-types-3.11.0.tgz#607c40edb4f258915e901ea7995607fdf319424c" + integrity sha512-6Z7Ma2aVEWisaL6TvBCy7P8rm2LQoPv6dJ7ecIaIixHcwfbJ0x7mWdbcwlIM5IGQxPZSFYeqRCqlOOeKoJYMkw== + vite-node@3.2.4: version "3.2.4" resolved "https://registry.yarnpkg.com/vite-node/-/vite-node-3.2.4.tgz#f3676d94c4af1e76898c162c92728bca65f7bb07" @@ -1445,6 +1814,23 @@ vitest@^3.2.1: vite-node "3.2.4" why-is-node-running "^2.3.0" +webgl-constants@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/webgl-constants/-/webgl-constants-1.1.1.tgz#f9633ee87fea56647a60b9ce735cbdfb891c6855" + integrity sha512-LkBXKjU5r9vAW7Gcu3T5u+5cvSvh5WwINdr0C+9jpzVB41cjQAP5ePArDtk/WHYdVj0GefCgM73BA7FlIiNtdg== + +webgl-sdf-generator@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/webgl-sdf-generator/-/webgl-sdf-generator-1.1.1.tgz#3e1b422b3d87cd3cc77f2602c9db63bc0f6accbd" + integrity sha512-9Z0JcMTFxeE+b2x1LJTdnaT8rT8aEp7MVxkNwoycNmJWwPdzoXzMh0BjJSh/AEFP+KPYZUli814h8bJZFIZ2jA== + +which@^2.0.1: + version "2.0.2" + resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1" + integrity sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA== + dependencies: + isexe "^2.0.0" + why-is-node-running@^2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/why-is-node-running/-/why-is-node-running-2.3.0.tgz#a3f69a97107f494b3cdc3bdddd883a7d65cebf04" @@ -1457,3 +1843,15 @@ yallist@^3.0.2: version "3.1.1" resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd" integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g== + +zustand@^4.3.2: + version "4.5.7" + resolved "https://registry.yarnpkg.com/zustand/-/zustand-4.5.7.tgz#7d6bb2026a142415dd8be8891d7870e6dbe65f55" + integrity sha512-CHOUy7mu3lbD6o6LJLfllpjkzhHXSBlX8B9+qPddUsIfeF5S/UZ5q0kmCsnRqT1UHFQZchNFDDzMbQsuesHWlw== + dependencies: + use-sync-external-store "^1.2.2" + +zustand@^5.0.1, zustand@^5.0.3: + version "5.0.12" + resolved "https://registry.yarnpkg.com/zustand/-/zustand-5.0.12.tgz#ed36f647aa89965c4019b671dfc23ef6c6e3af8c" + integrity sha512-i77ae3aZq4dhMlRhJVCYgMLKuSiZAaUPAct2AksxQ+gOtimhGMdXljRT21P5BNpeT4kXlLIckvkPM029OljD7g== diff --git a/src/syncfield/viewer/poller.py b/src/syncfield/viewer/poller.py index b746d3b..0a7b116 100644 --- a/src/syncfield/viewer/poller.py +++ b/src/syncfield/viewer/poller.py @@ -180,6 +180,7 @@ def _build_snapshot(self) -> SessionSnapshot: self._stats[stream_id] = buffer plot_points = buffer.snapshot_plot() if stream.kind != "video" else {} + latest_pose = buffer.snapshot_pose() if stream.kind != "video" else {} effective_hz = buffer.snapshot_fps(now_ns) latest_frame = self._safe_latest_frame(stream) @@ -205,6 +206,7 @@ def _build_snapshot(self) -> SessionSnapshot: effective_hz=effective_hz, latest_frame=latest_frame, plot_points=plot_points, + latest_pose=latest_pose, health_count=len(buffer._health), live_preview=getattr(stream.capabilities, "live_preview", True), ) diff --git a/src/syncfield/viewer/server.py b/src/syncfield/viewer/server.py index 3c3c66c..de986a2 100644 --- a/src/syncfield/viewer/server.py +++ b/src/syncfield/viewer/server.py @@ -1990,12 +1990,21 @@ async def _mjpeg_generator(self, stream_id: str): # ------------------------------------------------------------------ async def _sse_generator(self, stream_id: str): - """Yield sensor data as Server-Sent Events.""" + """Yield sensor data as Server-Sent Events. + + Two payload shapes ride the same event: + + * ``channels`` — scalar latest values (rolling chart) + * ``pose`` — list-valued latest samples (e.g. 156-float + ``hand_joints`` from MetaQuestHandStream). + Kept optional so existing sensor-chart callers + that only consume scalars stay backward-compatible. + """ while True: snapshot = self._poller.get_snapshot() if snapshot is not None: stream = snapshot.streams.get(stream_id) - if stream is not None and stream.plot_points: + if stream is not None and (stream.plot_points or stream.latest_pose): channels: Dict[str, float] = {} label: Optional[float] = None for ch_name, (xs, ys) in stream.plot_points.items(): @@ -2004,9 +2013,12 @@ async def _sse_generator(self, stream_id: str): if xs and label is None: label = xs[-1] - if channels: + pose = stream.latest_pose if stream.latest_pose else None + + if channels or pose: event_data = json.dumps({ "channels": channels, + "pose": pose, "label": label, }) yield f"data: {event_data}\n\n" diff --git a/src/syncfield/viewer/state.py b/src/syncfield/viewer/state.py index ab58ead..84c6b34 100644 --- a/src/syncfield/viewer/state.py +++ b/src/syncfield/viewer/state.py @@ -66,6 +66,13 @@ class StreamSnapshot: plot_points: For sensor streams, a dict of ``channel_name -> (x_list, y_list)`` rolling buffers of numeric values. Empty for video streams. + latest_pose: The most recent *non-scalar* channel values in flat + form (e.g. ``"hand_joints" -> [156 floats]``, ``"head_pose" + -> [7 floats]`` from :class:`MetaQuestHandStream`). Unlike + ``plot_points`` which tracks rolling scalar histories, this + exposes the latest vector/list sample so panels that render + a single-frame pose (3-D hand skeleton, quaternion axes…) + have data to draw. Empty for streams that emit only scalars. health_count: Number of health events this stream has buffered so far. Useful for showing a red dot on degraded streams. """ @@ -79,6 +86,7 @@ class StreamSnapshot: effective_hz: float latest_frame: Any # numpy array or None — kept as Any so numpy is optional plot_points: Dict[str, Tuple[List[float], List[float]]] + latest_pose: Dict[str, List[float]] health_count: int live_preview: bool = True @@ -164,6 +172,13 @@ class StreamStatsBuffer: _plot_timestamps: Deque[float] = field(default_factory=lambda: deque(maxlen=300)) _plot_channels: Dict[str, Deque[float]] = field(default_factory=dict) + # Latest list/vector-valued channels (e.g. 156-float hand_joints + # from MetaQuestHandStream). We only retain the most recent value + # because the receiver — a 3-D pose panel — wants instantaneous + # state, not a history, and the payload (~500 floats per pose) is + # too large to buffer thousands of frames of. + _latest_pose: Dict[str, List[float]] = field(default_factory=dict) + # Health events produced by this stream (capped) _health: Deque[HealthEntry] = field(default_factory=lambda: deque(maxlen=20)) @@ -193,6 +208,23 @@ def observe_sample(self, capture_ns: int, channels: Optional[Dict[str, Any]]) -> and not _is_auxiliary_channel(name) } + # List/tuple-valued channels (e.g. MetaQuestHandStream's + # hand_joints: 156 floats per sample) — retain only the + # latest value. 3-D pose panels want an instantaneous + # snapshot; keeping a rolling history would explode memory + # (~500 floats × 300 samples × 1 adapter = 150k floats). + for name, value in channels.items(): + if _is_auxiliary_channel(name): + continue + if isinstance(value, (list, tuple)): + # Cast to a plain list of floats so the SSE JSON + # serializer never sees numpy scalars or similar. + try: + self._latest_pose[name] = [float(v) for v in value] + except (TypeError, ValueError): + # Best-effort: skip non-numeric list payloads. + continue + # Pad any missing channel to align lengths, then append numeric values. current_len = len(self._plot_timestamps) for name, value in plottable.items(): @@ -250,6 +282,14 @@ def snapshot_plot(self) -> Dict[str, Tuple[List[float], List[float]]]: out[name] = (x, y) return out + def snapshot_pose(self) -> Dict[str, List[float]]: + """Return the most-recent list-valued channel samples. + + Returns a plain dict so the viewer render thread can serialise + it straight to JSON without touching the poller's state. + """ + return {name: list(values) for name, values in self._latest_pose.items()} + def snapshot_health(self) -> List[HealthEntry]: return list(self._health)