From 742811f0fd4cd798a74c6f3c40a4af634d9ed430 Mon Sep 17 00:00:00 2001 From: styu12 Date: Tue, 14 Apr 2026 18:16:02 -0700 Subject: [PATCH] fix(meta_quest): auto-discover + expose latest_frame so viewer isn't empty MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On-device diagnosis against a live Quest session showed two reasons the viewer sat empty even though the Unity sender app was running and the adapters were "connected": 1. Quest sender wasn't reaching the Mac's UDP :14043 at all. The Quest app ships with a hardcoded default target IP of ``172.30.1.51`` (a dev LAN from the original project template) and relies on auto-discovery to find the recorder at runtime. MetaQuestHandStream never responded to those discovery probes, so the Quest kept firing UDP into the void while the viewer cheerfully said "connected" (one stale packet from an old session had set the heartbeat). Add a background UDP :14044 responder that replies to the ``SYNCFIELD_DISCOVER_RECORDER_V1`` broadcasts Unity's UDPTrackingSender already emits every 2 s. Reply shape matches the ``RecorderDiscoveryResponse`` parser in the Unity client. We pick the "right" local IP per responder by opening a scratch UDP socket toward the Quest's address and reading getsockname — no packets sent, but the kernel chooses the correct outbound interface, which is the interface the Quest should target. Single-process multi-Quest setups would fight over :14044, so a bind failure silently falls back to "manual IP only" with a log hint. Normal single-Quest use now auto-resolves. 2. MetaQuestCameraStream declared ``kind="video"`` but exposed only ``latest_frame_left`` / ``latest_frame_right``, not the ``latest_frame`` attribute the viewer's StreamSnapshot polls for every video stream. Result: the camera card rendered black even while MjpegPreviewConsumer was happily decoding JPEGs into the per-eye slots. Add a ``latest_frame`` property that returns the left eye (falling back to right) so the existing video panel lights up without any viewer-side changes. Phase-1 caveat still applies: both eyes get the primary-camera view until Meta XR 2.4 exposes per-eye acquisition cleanly (spec §9 Q1). Once that lands ``latest_frame`` can switch to a side-by-side composite. Co-Authored-By: Claude Opus 4.6 --- src/syncfield/adapters/meta_quest.py | 137 ++++++++++++++++++ .../adapters/meta_quest_camera/stream.py | 18 +++ 2 files changed, 155 insertions(+) diff --git a/src/syncfield/adapters/meta_quest.py b/src/syncfield/adapters/meta_quest.py index 8dd1729..e5b5071 100644 --- a/src/syncfield/adapters/meta_quest.py +++ b/src/syncfield/adapters/meta_quest.py @@ -66,6 +66,15 @@ ROTATIONS_DIM = NUM_JOINTS * NUM_QUAT * NUM_HANDS # 208 HEAD_POSE_DIM = 7 # pos(3) + quat(4) DEFAULT_PORT = 14043 +DEFAULT_DISCOVERY_PORT = 14044 + +# Discovery protocol spoken by the Unity companion app +# (opengraph-studio/unity/SyncFieldQuest3Sender/Assets/Scripts/ +# UDPTrackingSender.cs). The Quest broadcasts the probe string on +# UDP :14044 every ~2 s; whoever responds with a well-formed +# RecorderDiscoveryResponse becomes the Quest's tracking target. +DISCOVERY_PROBE = b"SYNCFIELD_DISCOVER_RECORDER_V1" +DISCOVERY_RESPONSE_TYPE = "syncfield_recorder" # OpenXR standard joint names (26 per hand) JOINT_NAMES = [ @@ -152,6 +161,9 @@ def __init__( self._socket: Optional[socket.socket] = None self._receive_thread: Optional[threading.Thread] = None + self._discovery_socket: Optional[socket.socket] = None + self._discovery_thread: Optional[threading.Thread] = None + self._discovery_port = DEFAULT_DISCOVERY_PORT self._stop_event = threading.Event() self._recording = False self._frame_count = 0 @@ -193,6 +205,11 @@ def connect(self) -> None: self.id, self._host, self._port, self._mode, ) + # Spin up the discovery responder so the Quest companion app can + # auto-resolve our IP. Without this the user has to type the + # Mac's IP into the Quest HUD every time the network changes. + self._start_discovery_responder() + def start_recording(self, session_clock: SessionClock) -> None: self._recording = True self._frame_count = 0 @@ -218,6 +235,15 @@ def disconnect(self) -> None: if self._receive_thread is not None: self._receive_thread.join(timeout=2.0) self._receive_thread = None + if self._discovery_thread is not None: + self._discovery_thread.join(timeout=2.0) + self._discovery_thread = None + if self._discovery_socket is not None: + try: + self._discovery_socket.close() + except Exception: + pass + self._discovery_socket = None if self._socket is not None: try: self._socket.close() @@ -225,6 +251,117 @@ def disconnect(self) -> None: pass self._socket = None + # ------------------------------------------------------------------ + # Discovery responder (UDP :14044) + # ------------------------------------------------------------------ + + def _start_discovery_responder(self) -> None: + """Respond to Quest companion-app discovery broadcasts. + + The Quest sender broadcasts :data:`DISCOVERY_PROBE` every ~2 s + on :data:`DEFAULT_DISCOVERY_PORT` until a recorder replies. + We listen on that port and reply with a JSON payload naming + ourselves as the tracker target — Quest then locks onto our + IP automatically and starts sending on :data:`DEFAULT_PORT`. + + Multiple ``MetaQuestHandStream`` instances in one process would + fight over :14044, so we silently skip the bind if another + responder already owns the port. Single-Quest setups — the + common case — just work. + """ + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(("0.0.0.0", self._discovery_port)) + sock.settimeout(1.0) + except OSError as exc: + logger.info( + "[%s] Discovery responder disabled (port %d busy: %s). " + "If your Quest can't auto-find this Mac, set the receiver " + "IP manually in the Quest app HUD.", + self.id, self._discovery_port, exc, + ) + return + + self._discovery_socket = sock + self._discovery_thread = threading.Thread( + target=self._discovery_loop, + name=f"quest3-discovery-{self.id}", + daemon=True, + ) + self._discovery_thread.start() + logger.info( + "[%s] Quest 3 discovery responder listening on :%d", + self.id, self._discovery_port, + ) + + def _discovery_loop(self) -> None: + while not self._stop_event.is_set(): + sock = self._discovery_socket + if sock is None: + break + try: + data, addr = sock.recvfrom(512) + except TimeoutError: + continue + except Exception: + if self._stop_event.is_set(): + return + continue + if data.strip() != DISCOVERY_PROBE: + continue + response = self._build_discovery_response(addr[0]) + try: + sock.sendto(response, addr) + logger.info( + "[%s] Answered Quest discovery probe from %s", + self.id, addr[0], + ) + except Exception as exc: + logger.warning("[%s] Discovery reply failed: %s", self.id, exc) + + def _build_discovery_response(self, quest_ip: str) -> bytes: + """Build the JSON response shape the Unity app's + ``RecorderDiscoveryResponse`` parser expects. + + We source our own IP by opening a UDP socket toward the Quest + and reading the resulting local endpoint — that's the address + we'd send out of, which is the address the Quest should target. + """ + recorder_ip = self._resolve_local_ip_for(quest_ip) + payload = { + "type": DISCOVERY_RESPONSE_TYPE, + "recorder_ip": recorder_ip, + "tracker_port": self._port, + "api_port": 0, + "hostname": socket.gethostname(), + "label": "SyncField (Python)", + "active_config": "", + "ts_ms": int(time.time() * 1000), + } + return json.dumps(payload).encode("utf-8") + + @staticmethod + def _resolve_local_ip_for(remote_ip: str) -> str: + """Return the local IPv4 the OS would use to reach ``remote_ip``. + + Creating a UDP socket and calling ``connect()`` doesn't send any + packets; it just makes the kernel pick the right outbound + interface. Reading ``getsockname`` after that gives the IP the + Quest should target. + """ + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + s.connect((remote_ip, 1)) + return s.getsockname()[0] + except Exception: + return "127.0.0.1" + finally: + try: + s.close() + except Exception: + pass + # Legacy one-shot compatibility def prepare(self) -> None: pass diff --git a/src/syncfield/adapters/meta_quest_camera/stream.py b/src/syncfield/adapters/meta_quest_camera/stream.py index b7d9009..8173d60 100644 --- a/src/syncfield/adapters/meta_quest_camera/stream.py +++ b/src/syncfield/adapters/meta_quest_camera/stream.py @@ -264,3 +264,21 @@ def latest_frame_right(self): if self._preview_right is None: return None return self._preview_right.latest_frame + + @property + def latest_frame(self): + """Viewer-compat: return the left eye so the standard video panel + has something to render. The viewer's ``StreamSnapshot`` polls + ``stream.latest_frame`` for any adapter declaring + ``kind="video"``; without this proxy the camera card would sit + black even while the MJPEG preview is actively streaming. + + The left eye is authoritative for now — phase 1 records the + same primary-camera view on both slots (see spec §9 Q1). Once + per-eye acquisition lands we can switch this to a side-by-side + composite or expose a user-selectable eye. + """ + left = self.latest_frame_left + if left is not None: + return left + return self.latest_frame_right