diff --git a/addon/globalPlugins/remoteClient/__init__.py b/addon/globalPlugins/remoteClient/__init__.py index 375b04d..6cd5c6e 100644 --- a/addon/globalPlugins/remoteClient/__init__.py +++ b/addon/globalPlugins/remoteClient/__init__.py @@ -204,6 +204,7 @@ def __init__(self, *args, **kwargs): self.script_screenshot, self.script_screenshot_powershell, self.script_toggle_screen_share, + self.script_toggle_remote_sound, self.script_toggle_remote_mouse, ) self.is_connect_dialog_open = False @@ -1003,15 +1004,41 @@ def script_toggle_screen_share(self, gesture): ui.message(_("Not connected.")) return configuration.record_activity() - was_active = session.screen_share.active + # The keyboard follows the picture rather than the session: sound may well go on + # arriving after the screen has been dismissed, and typing blind into a machine + # nobody is watching is exactly what must not happen. + was_watching = session.screen_share.video_requested ui.message(session.screen_share.toggle()) if session is not self.master_session: return - if not was_active and session.screen_share.active: + if not was_watching and session.screen_share.video_requested: self._take_control_for_screen_share(gesture) - elif was_active and not session.screen_share.active and self.screen_share_took_control: + elif was_watching and not session.screen_share.video_requested and self.screen_share_took_control: self._switch_to_local_control() + @script( + # Translators: toggle remote sound gesture description + _("Plays or stops playing the sound of the controlled computer on this one"), + gesture="kb:control+shift+NVDA+j", + **speakOnDemand) + def script_toggle_remote_sound(self, gesture): + """Start or stop hearing what the controlled computer plays. + + Unlike the screen, sound is useful without anything to look at and without the + keyboard following it, so this neither takes control nor gives it back. It can be + turned on while a screen is already being watched, and the picture then stays. + """ + session = None + if self.master_session is not None and self._is_master_connected(): + session = self.master_session + elif self.slave_session is not None and self._is_slave_connected(): + session = self.slave_session + if session is None or session.screen_share is None: + ui.message(_("Not connected.")) + return + configuration.record_activity() + ui.message(session.screen_share.toggle_audio()) + def _take_control_for_screen_share(self, gesture): """Send the keyboard to the controlled computer now that its screen is requested.""" if self.sending_keys: diff --git a/addon/globalPlugins/remoteClient/audio_bridge.py b/addon/globalPlugins/remoteClient/audio_bridge.py new file mode 100644 index 0000000..20136ab --- /dev/null +++ b/addon/globalPlugins/remoteClient/audio_bridge.py @@ -0,0 +1,162 @@ +"""The native helper which captures the sound of this computer without NVDA in it. + +The browser can capture the system mix, but only all of it. What this computer plays +includes NVDA, so the sound sent to the watching computer used to carry the remote +NVDA speaking, which is why the forwarded speech had to be silenced while it flowed. + +Windows can do better. Since build 20348 an audio client can be activated in process +loopback mode, capturing everything except one process tree. No web API exposes it, +so a small native helper does the capture and hands the samples to the page over a +loopback WebSocket, which turns them back into a WebRTC track. + +The helper repeats the protections of local_bridge: it binds to 127.0.0.1 alone, the +port is chosen by the system for each session, every connection must carry the session +token, and an Origin header that is not the page's own is refused. The token goes in +through the standard input of the helper rather than its command line, which every +other process on the machine can read. + +When any of this is unavailable - an older Windows, a missing helper, a refusal from +the audio engine - is_available() answers no and the caller falls back to the browser +capture, with the forwarded speech silenced as before. +""" + +import base64 +import os +import subprocess +import sys +import threading +from logging import getLogger + +logger = getLogger("audio_bridge") + +#: Build which introduced AUDIOCLIENT_ACTIVATION_TYPE_PROCESS_LOOPBACK. +MIN_WINDOWS_BUILD = 20348 + +#: Where the helper sits, next to this module. +HELPER_SUBDIR = "bin" +HELPER_NAME = "nvda_audio_capture.exe" + +#: The helper prints its port as soon as it is listening. It has nothing to do +#: beforehand, so waiting long would only hide a failure. +START_TIMEOUT = 5.0 + +TOKEN_BYTES = 32 + + +def helper_path(): + """Return the absolute path of the helper, or None when it is not there.""" + path = os.path.join(os.path.abspath(os.path.dirname(__file__)), HELPER_SUBDIR, HELPER_NAME) + return path if os.path.isfile(path) else None + + +def is_available(): + """Whether this computer can capture its sound with NVDA left out of it.""" + try: + if sys.getwindowsversion().build < MIN_WINDOWS_BUILD: + return False + except Exception: + logger.debug("Unable to read the Windows build", exc_info=True) + return False + return helper_path() is not None + + +def _make_token(): + """Return an unguessable token for one session, as local_bridge does.""" + return base64.urlsafe_b64encode(os.urandom(TOKEN_BYTES)).decode("ascii").rstrip("=") + + +class AudioBridge: + """One run of the native helper, serving one page.""" + + def __init__(self): + self._process = None + self.url = None + + @property + def running(self): + return self._process is not None and self._process.poll() is None + + def start(self, origin): + """Start capturing and return the address the page must connect to. + + `origin` is the local origin the page is served from, and the only one the + helper will accept. Returns None when the helper could not be started, which + is never fatal: the caller falls back to the browser capture. + """ + if self.running: + return self.url + path = helper_path() + if path is None or not origin: + return None + token = _make_token() + try: + self._process = subprocess.Popen( + [ + path, + "--serve", + "--origin", + origin, + # Excluding this very process is the whole point. The mode excludes + # the children too, so anything NVDA starts to speak is covered. + "--pid", + str(os.getpid()), + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0), + ) + self._process.stdin.write((token + "\n").encode("ascii")) + self._process.stdin.flush() + except Exception: + logger.exception("Unable to start the audio capture helper") + self.stop() + return None + + port = self._read_port() + if port is None: + logger.error("The audio capture helper did not report a port") + self.stop() + return None + self.url = "ws://127.0.0.1:%d/?token=%s" % (port, token) + logger.debug("Audio capture helper listening on port %d", port) + return self.url + + def _read_port(self): + """Read the "PORT n" line the helper prints once it listens.""" + result = {} + + def read(): + try: + result["line"] = self._process.stdout.readline().decode("ascii", "replace") + except Exception: + logger.debug("Unable to read from the audio capture helper", exc_info=True) + + reader = threading.Thread(target=read, name="audio_bridge_start", daemon=True) + reader.start() + reader.join(START_TIMEOUT) + line = (result.get("line") or "").strip() + if not line.startswith("PORT "): + return None + try: + return int(line.split()[1]) + except (IndexError, ValueError): + return None + + def stop(self): + """Stop capturing. Safe to call at any point, including twice.""" + process, self._process = self._process, None + self.url = None + if process is None: + return + try: + if process.poll() is None: + process.terminate() + except Exception: + logger.debug("Unable to stop the audio capture helper", exc_info=True) + for stream in (process.stdin, process.stdout): + try: + if stream is not None: + stream.close() + except Exception: + pass diff --git a/addon/globalPlugins/remoteClient/bin/nvda_audio_capture.exe b/addon/globalPlugins/remoteClient/bin/nvda_audio_capture.exe new file mode 100644 index 0000000..99d2308 Binary files /dev/null and b/addon/globalPlugins/remoteClient/bin/nvda_audio_capture.exe differ diff --git a/addon/globalPlugins/remoteClient/capabilities.py b/addon/globalPlugins/remoteClient/capabilities.py index 3d2d1c1..4c52e4e 100644 --- a/addon/globalPlugins/remoteClient/capabilities.py +++ b/addon/globalPlugins/remoteClient/capabilities.py @@ -32,6 +32,13 @@ #: WebRTC screen sharing of the controlled computer, with optional mouse control. FEATURE_SCREEN_SHARE = "screen_share" +#: Streaming of the sound the controlled computer plays, carried as an audio track +#: on the same WebRTC session as screen sharing. Announced separately from +#: FEATURE_SCREEN_SHARE, because a computer willing to show its screen may still +#: refuse to let everything it plays be heard, and the reverse is just as useful: +#: the sound can be sent on its own, without any picture. +FEATURE_AUDIO_SHARE = "audio_share" + #: Optional features implemented by this build. LOCAL_FEATURES = (FEATURE_CHUNKED_FILE_TRANSFER,) @@ -52,6 +59,8 @@ def available_features(): from . import screen_share if screen_share.is_available(): features.append(FEATURE_SCREEN_SHARE) + if screen_share.is_audio_share_allowed(): + features.append(FEATURE_AUDIO_SHARE) return features diff --git a/addon/globalPlugins/remoteClient/configuration.py b/addon/globalPlugins/remoteClient/configuration.py index 3e1dc99..f14a186 100644 --- a/addon/globalPlugins/remoteClient/configuration.py +++ b/addon/globalPlugins/remoteClient/configuration.py @@ -88,6 +88,8 @@ max_fps = integer(default=15) max_width = integer(default=1600) quality = option("low", "balanced", "high", default="balanced") + share_audio = boolean(default=True) + mute_remote_speech_with_audio = boolean(default=True) [keep_awake] enabled = boolean(default=True) diff --git a/addon/globalPlugins/remoteClient/dialogs.py b/addon/globalPlugins/remoteClient/dialogs.py index b70f9cd..1718d13 100644 --- a/addon/globalPlugins/remoteClient/dialogs.py +++ b/addon/globalPlugins/remoteClient/dialogs.py @@ -637,6 +637,12 @@ def makeSettings(self, sizer): _("High, for a fast connection"), ]) sizer.Add(self.screen_share_quality) + # Translators: A checkbox in add-on options dialog to allow the controlling computer to hear this one. + self.screen_share_audio = wx.CheckBox(self, wx.ID_ANY, label=_("Allow sending the sound of this computer, after confirmation")) + sizer.Add(self.screen_share_audio) + # Translators: A checkbox in add-on options dialog to stop speaking what the controlled computer reports when the sound it sends already contains its own screen reader. + self.screen_share_mute_speech = wx.CheckBox(self, wx.ID_ANY, label=_("When the sound of the controlled computer already contains its own screen reader, do not also speak what it reports")) + sizer.Add(self.screen_share_mute_speech) # Translators: a text field in add-on options dialog to set the portcheck service URL sizer.Add(wx.StaticText(self, wx.ID_ANY, label=_("Portcheck &service URL: "))) self.portcheck = wx.TextCtrl(self, wx.ID_ANY) @@ -675,6 +681,8 @@ def _update_screen_share_controls(self): self.screen_share_max_fps.Enable(enabled) self.screen_share_max_width.Enable(enabled) self.screen_share_quality.Enable(enabled) + self.screen_share_audio.Enable(enabled) + self.screen_share_mute_speech.Enable(enabled) def on_autoconnect(self, evt): if self.autoconnect.GetValue() and not self._autoconnect_was_enabled: @@ -794,6 +802,8 @@ def onPanelActivated(self): self.max_received_size.SetValue(int(file_transfer_section['max_received_size_mb'])) self.screen_share_enabled.SetValue(bool(config['screen_share']['enabled'])) self.screen_share_max_fps.SetValue(int(config['screen_share']['max_fps'])) + self.screen_share_audio.SetValue(bool(config['screen_share']['share_audio'])) + self.screen_share_mute_speech.SetValue(bool(config['screen_share']['mute_remote_speech_with_audio'])) max_width = int(config['screen_share']['max_width']) if max_width not in SCREEN_SHARE_WIDTHS: max_width = SCREEN_SHARE_DEFAULT_WIDTH @@ -923,6 +933,8 @@ def onSave(self): config['file_transfer']['max_received_size_mb'] = int(self.max_received_size.GetValue()) config['screen_share']['enabled'] = self.screen_share_enabled.GetValue() config['screen_share']['max_fps'] = int(self.screen_share_max_fps.GetValue()) + config['screen_share']['share_audio'] = self.screen_share_audio.GetValue() + config['screen_share']['mute_remote_speech_with_audio'] = self.screen_share_mute_speech.GetValue() config['screen_share']['max_width'] = SCREEN_SHARE_WIDTHS[self.screen_share_max_width.GetSelection()] config['screen_share']['quality'] = SCREEN_SHARE_QUALITIES[self.screen_share_quality.GetSelection()] config['updates']['check_at_startup'] = self.check_updates.GetValue() diff --git a/addon/globalPlugins/remoteClient/edge.py b/addon/globalPlugins/remoteClient/edge.py index 5c0cba0..9d3cfae 100644 --- a/addon/globalPlugins/remoteClient/edge.py +++ b/addon/globalPlugins/remoteClient/edge.py @@ -133,6 +133,10 @@ def _build_arguments(browser, url, profile, off_screen): # See the module docstring: this is what removes the source picker, and the # reason the profile above is throwaway. "--use-fake-ui-for-media-stream", + # The picture is displayed muted, but the sound of the controlled computer has + # to play by itself: there is nobody to click in a window kept off screen, and + # the watching user may not be able to see it at all. + "--autoplay-policy=no-user-gesture-required", # Not for the average frame rate, which holds without them, but to suppress # the freezes measured on a window that is not on screen. "--disable-background-timer-throttling", diff --git a/addon/globalPlugins/remoteClient/edge_engine.py b/addon/globalPlugins/remoteClient/edge_engine.py index f08dfd1..dfebfe8 100644 --- a/addon/globalPlugins/remoteClient/edge_engine.py +++ b/addon/globalPlugins/remoteClient/edge_engine.py @@ -61,6 +61,15 @@ def running(self): # The bridge, not the browser process: see LocalBridge.silence. return self._bridge.running + @property + def origin(self): + """The local origin the page is served from, None while nothing is running. + + The audio capture helper needs it: it is the only origin it will accept a + connection from. + """ + return self._bridge.origin + def start(self, role): """Open the browser on the signalling page. Raises RuntimeError when unusable.""" if self.running: diff --git a/addon/globalPlugins/remoteClient/local_machine.py b/addon/globalPlugins/remoteClient/local_machine.py index 635a840..e5fb212 100644 --- a/addon/globalPlugins/remoteClient/local_machine.py +++ b/addon/globalPlugins/remoteClient/local_machine.py @@ -59,6 +59,12 @@ def setSpeechCancelledToFalse(): class LocalMachine: def __init__(self): self.is_muted = False + # True while the sound of the controlled computer is streamed to this one. + # What that computer plays already includes its own NVDA, so announcing the + # speech it also forwards would say everything twice, a fraction of a second + # apart. Braille is deliberately left alone: it is not audible, and losing it + # would cost a braille user their only view of the remote machine. + self.audio_streaming = False self.receiving_braille = False self._cached_sizes = None if buildVersion.version_year >= 2023: @@ -70,7 +76,7 @@ def terminate(self): def play_wave(self, fileName): """Instructed by remote machine to play a wave file.""" - if self.is_muted: + if self.is_muted or self.audio_streaming: return if os.path.exists(fileName): # ignore async / asynchronous from kwargs: @@ -78,7 +84,7 @@ def play_wave(self, fileName): nvwave.playWaveFile(fileName=fileName, asynchronous=True) def beep(self, hz, length, left, right, **kwargs): - if self.is_muted: + if self.is_muted or self.audio_streaming: return tones.beep(hz, length, left, right) @@ -98,7 +104,7 @@ def speak( priority=speech.priorities.Spri.NORMAL, **kwargs, ): - if self.is_muted: + if self.is_muted or self.audio_streaming: return setSpeechCancelledToFalse() if not configuration.get_config()["ui"]["allow_speech_commands"]: diff --git a/addon/globalPlugins/remoteClient/screen_share.py b/addon/globalPlugins/remoteClient/screen_share.py index 852bc88..5286bb0 100644 --- a/addon/globalPlugins/remoteClient/screen_share.py +++ b/addon/globalPlugins/remoteClient/screen_share.py @@ -27,7 +27,7 @@ import gui import ui -from . import capabilities, configuration, edge_engine +from . import audio_bridge, capabilities, configuration, edge_engine from .transport import TransportEvents logger = getLogger("screen_share") @@ -78,6 +78,26 @@ def is_available(): return is_enabled() and edge_engine.is_available() +def is_audio_share_allowed(): + """Whether this computer accepts to let the sound it plays be heard remotely. + + Sharing sound is not the same promise as sharing a picture. What a computer plays + carries the other side of a telephone call, a video someone is watching or a voice + message, none of which the person watching a screen would otherwise get. It + therefore has a setting of its own, and its own question to the user. + + The browser is still what captures and encodes, so this can only be offered where + screen sharing itself is usable. + """ + if not is_available(): + return False + try: + return bool(configuration.get_config()["screen_share"]["share_audio"]) + except Exception: + logger.debug("Unable to read the audio sharing configuration", exc_info=True) + return False + + def is_input_control_allowed(): """Whether this computer accepts to be driven with the remote mouse. @@ -114,10 +134,34 @@ def __init__(self, transport, negotiator, role): #: Identifier of the peer this session is held with. self.peer_id = None self.helper = edge_engine.EdgeEngine(self._handle_helper_event, self._handle_helper_exit) + #: Native capture of the sound of this computer with NVDA left out of it. Used + #: when this computer is the one sharing; falls back to the browser capture, + #: which cannot leave NVDA out, when the helper or the Windows build is missing. + self.audio_helper = audio_bridge.AudioBridge() + #: Whether the sound arriving in this session is free of the remote NVDA. When + #: it is not, the forwarded speech has to be silenced or everything is said + #: twice; when it is, the speech keeps coming through the relay, which is both + #: faster and rendered with the settings of whoever is listening. + self.audio_excludes_nvda = False + #: Whether the link dropped without the session ending. A WebRTC connection can + #: sit in that state for tens of seconds before it either recovers or gives up, + #: and no media flows meanwhile. + self.audio_interrupted = False #: ICE servers given by the relay, used as a fallback when a direct link fails. self.ice_servers = [] #: Whether the peer agreed, for this session, to be driven with the mouse. self.input_allowed = False + #: What this session was asked to carry. Set as soon as the request goes out, so + #: that a second keystroke acts on what the user just asked for rather than on + #: what the far end has not had time to answer yet. + self.video_requested = False + self.audio_requested = False + #: What it actually carries, once the far end has said so. + self.video_active = False + self.audio_active = False + #: Set by the session, so that sound arriving from the controlled computer can + #: silence the speech that computer also forwards. Left None on the controlled side. + self.local_machine = None #: Set by the controlled session, so that accepting to share this screen also #: hands the mouse over without asking a second question. self.input_receiver = None @@ -147,39 +191,167 @@ def active(self): return self.state != STATE_IDLE def toggle(self): - """Start the session when there is none, stop the current one otherwise. + """Start or stop watching the screen of the controlled computer. Returns a message to report to the user. """ - if self.active: + if self.role != ROLE_VIEWER: + # The controlled computer chooses nothing here: it can only end a session it + # accepted. Asking for one is the business of the computer being helped. + return self._end_from_publisher() + if self.video_requested: + # Sound the user asked for separately is not theirs to lose here, so only the + # picture goes away while some is still arriving. + if self.audio_requested: + return self._restart(want_video=False, want_audio=True) self.stop() # Translators: message spoken when screen sharing is turned off return _("Screen sharing stopped") - return self.start() + return self._restart(want_video=True, want_audio=self.audio_requested) + + def toggle_audio(self): + """Start or stop hearing what the controlled computer plays. + + Returns a message to report to the user. + """ + if self.role != ROLE_VIEWER: + return self._end_from_publisher() + if self.audio_requested: + if self.video_requested: + return self._restart(want_video=True, want_audio=False) + self.stop() + # Translators: message spoken when the sound of the controlled computer is turned off + return _("Remote sound stopped") + return self._restart(want_video=self.video_requested, want_audio=True) + + def _end_from_publisher(self): + """End the session from the computer which is being watched or listened to.""" + if not self.active: + # Translators: message spoken when screen sharing is requested from the wrong computer + return _("Screen sharing can only be started from the controlling computer") + # Read before stopping, which clears both. + sharing_audio = self.audio_requested + sharing_video = self.video_requested + self.stop() + if sharing_audio and not sharing_video: + # Translators: message spoken when the sound of the controlled computer is turned off + return _("Remote sound stopped") + # Translators: message spoken when screen sharing is turned off + return _("Screen sharing stopped") + + def _restart(self, want_video, want_audio): + """Ask for a session carrying exactly these two streams. + + Adding sound to a session, or taking the picture away from one, is done by asking + for a new session rather than by renegotiating the one in progress. The controlled + computer then gets to answer the question which matches what is really being asked + of it, which it could not do if the streams changed underneath it. + """ + if self.active: + self.stop() + return self.start(want_video=want_video, want_audio=want_audio) - def start(self): - """Ask the controlled computer to share its screen. Returns a message to report.""" + def start(self, want_video=True, want_audio=False): + """Ask the controlled computer for a session. Returns a message to report.""" if self.role != ROLE_VIEWER: # Translators: message spoken when screen sharing is requested from the wrong computer return _("Screen sharing can only be started from the controlling computer") if not is_available(): # Translators: message spoken when screen sharing cannot run on this computer return _("Screen sharing is not available on this computer") + if not want_video and not want_audio: + self.stop() + # Translators: message spoken when screen sharing is turned off + return _("Screen sharing stopped") peers = self.negotiator.peers_supporting(capabilities.FEATURE_SCREEN_SHARE) if not peers: # Translators: message spoken when the other computer cannot share its screen return _("The other computer does not support screen sharing") + if want_audio and not self.negotiator.peers_supporting(capabilities.FEATURE_AUDIO_SHARE): + if not want_video: + # Translators: message spoken when the other computer cannot send the sound it plays + return _("The other computer does not support sending its sound") + want_audio = False self.peer_id = peers[0] self.state = STATE_REQUESTING + self.video_requested = want_video + self.audio_requested = want_audio # The relay only hands out TURN credentials to clients which asked for them, # and they expire, so they are requested for each session rather than kept. self._request_turn_credentials() # Seeing a screen without being able to point at it is of little use, so the mouse # is always asked for. The controlled computer alone decides whether to grant it. - self._send(MSG_REQUEST, allow_input=True) + # There is nothing to point at when only the sound was asked for. + self._send( + MSG_REQUEST, + allow_input=want_video, + want_video=want_video, + want_audio=want_audio, + ) + if not want_video: + # Translators: message spoken when the sound of the controlled computer has been requested + return _("Remote sound requested") + if want_audio: + # Translators: message spoken when screen sharing with sound has been requested + return _("Screen sharing with sound requested") # Translators: message spoken when screen sharing has been requested return _("Screen sharing requested") + def _set_audio_active(self, active): + """Record that sound is or is no longer flowing, and silence forwarded speech. + + Whether the speech has to be silenced depends on how the far end captured its + sound. Captured by the browser, that sound contains its own NVDA, and announcing + the speech forwarded on top of it says everything twice a fraction of a second + apart, which is worse than either on its own. Captured by the native helper, NVDA + is left out, and the forwarded speech is then the better of the two: it arrives + without the delay of the audio stream, and is spoken with the synthesiser, voice + and rate of whoever is listening rather than those of the far end. + + The speech is always restored when the sound stops, whatever ended the session. + """ + self.audio_active = bool(active) + if not self.audio_active: + self.audio_interrupted = False + self._apply_speech_muting() + + def _apply_speech_muting(self): + """Silence or restore the forwarded speech, from the current state of the session.""" + if self.local_machine is None: + return + mute = self.audio_active + if mute and self.audio_excludes_nvda: + mute = False + if mute and self.audio_interrupted: + # No sound is arriving, so there is nothing left to say twice, and the + # forwarded speech is the only thing still telling this user what the other + # computer is doing. Silencing it here would leave them with neither. + mute = False + if mute: + try: + mute = bool(configuration.get_config()["screen_share"]["mute_remote_speech_with_audio"]) + except Exception: + logger.debug("Unable to read the remote speech setting", exc_info=True) + mute = True + self.local_machine.audio_streaming = mute + + def _handle_interruption(self): + """The link dropped without the session ending: hand the speech back. + + A connection which goes to "disconnected" has not failed. The browser often + recovers on its own, so tearing the session down would be wrong. But no media + flows meanwhile, and that state can last tens of seconds before it turns into a + failure, which is long enough for someone relying on the sound to be left with + nothing at all and no idea why. + """ + if not self.active or self.audio_interrupted: + return + self.audio_interrupted = True + self._apply_speech_muting() + if self.audio_active: + # Translators: message spoken when the sound of the watched computer stops arriving + ui.message(_("Sound interrupted")) + def stop(self, notify_peer=True): """End the current session, telling the peer about it unless it asked for it.""" if not self.active: @@ -190,6 +362,13 @@ def stop(self, notify_peer=True): self.peer_id = None self.ice_servers = [] self.input_allowed = False + self.video_requested = False + self.audio_requested = False + self.video_active = False + self._set_audio_active(False) + self.audio_excludes_nvda = False + self.audio_interrupted = False + self.audio_helper.stop() self.helper.stop() def terminate(self): @@ -198,8 +377,12 @@ def terminate(self): # Signalling received from the peer. - def handle_request(self, origin=None, allow_input=False, **kwargs): - """The controlling computer asks this one to share its screen.""" + def handle_request(self, origin=None, allow_input=False, want_video=True, want_audio=False, **kwargs): + """The controlling computer asks this one for its screen, its sound, or both. + + A controlling computer built before sound was carried names neither stream, so + the picture is what a request without them asks for. + """ if not self._accept_from(origin): return if self.role != ROLE_PUBLISHER or not is_available(): @@ -208,31 +391,58 @@ def handle_request(self, origin=None, allow_input=False, **kwargs): if self.active: self._refuse(origin, "busy") return + send_video = bool(want_video) + # Sound is only ever sent when this computer allows it, whatever was asked for. + send_audio = bool(want_audio) and is_audio_share_allowed() + if not send_video and not send_audio: + # Either nothing was asked for, or the only thing asked for was refused here. + self._refuse(origin, "unavailable") + return # Remote input is only ever granted when this computer allows it, whatever the - # controlling computer asked for. - allow_input = bool(allow_input) and is_input_control_allowed() - wx.CallAfter(self._ask_permission, origin, allow_input) - - def _ask_permission(self, origin, allow_input): - if allow_input: + # controlling computer asked for, and there is nothing to point at without a picture. + allow_input = bool(allow_input) and send_video and is_input_control_allowed() + # The relay only hands out ICE servers to clients which asked for them, and stop() + # drops the ones the previous session used. Without this, every session after the + # first negotiates on local addresses alone, which only ever links up two computers + # already on the same network. The question asked below leaves ample time for the + # answer to come back. + self._request_turn_credentials() + wx.CallAfter(self._ask_permission, origin, allow_input, send_video, send_audio) + + def _ask_permission(self, origin, allow_input, send_video, send_audio): + if send_video and send_audio and allow_input: + # Translators: question asked before this screen and this sound are shared, with mouse control + question = _("Do you want to share your screen and your sound? The controlling computer will see this screen, hear everything this computer plays, and will be able to use its mouse.") + elif send_video and send_audio: + # Translators: question asked before this screen and this sound are shared + question = _("Do you want to share your screen and your sound? The controlling computer will see this screen and hear everything this computer plays.") + elif send_audio: + # Translators: question asked before the sound of this computer is shared + question = _("Do you want to share your sound? The controlling computer will hear everything this computer plays, including calls and videos.") + elif allow_input: # Translators: question asked before this screen is shared, with mouse control question = _("Do you want to share your screen? The controlling computer will see this screen and will be able to use its mouse.") else: # Translators: question asked before this screen is shared question = _("Do you want to share your screen? The controlling computer will see this screen.") + if send_audio and not send_video: + # Translators: title of the remote sound request dialog + caption = _("Sound sharing request") + else: + # Translators: title of the screen sharing request dialog + caption = _("Screen sharing request") answer = gui.messageBox( parent=gui.mainFrame, - # Translators: title of the screen sharing request dialog - caption=_("Screen sharing request"), + caption=caption, message=question, style=wx.YES_NO | wx.NO_DEFAULT | wx.ICON_QUESTION, ) if answer == wx.YES: - self._accept_request(origin, allow_input) + self._accept_request(origin, allow_input, send_video, send_audio) else: self._refuse(origin, "declined") - def _accept_request(self, origin, allow_input): + def _accept_request(self, origin, allow_input, send_video=True, send_audio=False): if self.active: # The user took long enough to answer that another session started. self._refuse(origin, "busy") @@ -251,22 +461,69 @@ def _accept_request(self, origin, allow_input): # The user has just answered the only question there is, so the mouse events # which follow must not open a second one. self.input_receiver.granted = True - self._send(MSG_RESPONSE, accepted=True, allow_input=allow_input) + self.video_requested = send_video + self.audio_requested = send_audio + self.video_active = send_video + self.audio_active = send_audio + # Captured natively, the sound leaves NVDA out; captured by the browser it cannot. + # Which one it is decides whether the watching computer has to silence the speech + # this one forwards, so it is told, and an older peer which does not understand + # the answer simply keeps silencing as before. + audio_url = None + if send_audio: + audio_url = self.audio_helper.start(self.helper.origin) + if audio_url is None: + logger.debug("No native audio capture here, falling back to the browser") + self._send( + MSG_RESPONSE, + accepted=True, + allow_input=allow_input, + video=send_video, + audio=send_audio, + audio_excludes_nvda=audio_url is not None, + ) self.helper.send( command="start", role=ROLE_PUBLISHER, allow_input=allow_input, ice_servers=self.ice_servers, + send_video=send_video, + send_audio=send_audio, + audio_ws=audio_url, **_capture_settings() ) - # Translators: message spoken on the controlled computer when it starts sharing its screen - ui.message(_("Sharing this screen")) + if send_video and send_audio: + # Translators: message spoken on the controlled computer when it starts sharing its screen and its sound + ui.message(_("Sharing this screen and this sound")) + elif send_audio: + # Translators: message spoken on the controlled computer when it starts sharing its sound + ui.message(_("Sharing this sound")) + else: + # Translators: message spoken on the controlled computer when it starts sharing its screen + ui.message(_("Sharing this screen")) def _refuse(self, origin, reason): self._send(MSG_RESPONSE, target=origin, accepted=False, reason=reason) - def handle_response(self, origin=None, accepted=False, allow_input=False, reason="", **kwargs): - """The controlled computer answered our request.""" + def handle_response( + self, + origin=None, + accepted=False, + allow_input=False, + reason="", + video=True, + audio=False, + audio_excludes_nvda=False, + **kwargs, + ): + """The controlled computer answered our request. + + A controlled computer built before sound was carried names neither stream, and + answers a picture, which is the only thing it could have been asked for. One + built before the native capture existed says nothing about NVDA being left out + of its sound, and the default of no is then the safe reading: the speech it + forwards gets silenced, exactly as it used to be. + """ if not self._accept_from(origin) or origin != self.peer_id: return if self.state != STATE_REQUESTING: @@ -286,11 +543,16 @@ def handle_response(self, origin=None, accepted=False, allow_input=False, reason ui.message(_("Unable to start screen sharing")) return self.input_allowed = bool(allow_input) + self.video_active = bool(video) + self.audio_excludes_nvda = bool(audio_excludes_nvda) + self._set_audio_active(audio) self.helper.send( command="start", role=ROLE_VIEWER, allow_input=self.input_allowed, ice_servers=self.ice_servers, + receive_video=self.video_active, + receive_audio=self.audio_active, ) def handle_stop(self, origin=None, **kwargs): @@ -352,8 +614,19 @@ def _handle_helper_event(self, event): self._forward_input(event) elif kind == "connected": self.state = STATE_ACTIVE - # Translators: message spoken when the screen sharing picture starts flowing - wx.CallAfter(ui.message, _("Screen sharing started")) + if self.audio_interrupted: + # Coming back from a passing cut rather than starting: the speech that + # was handed back has to be held again, and announcing a start would + # be wrong. + self.audio_interrupted = False + wx.CallAfter(self._apply_speech_muting) + # Translators: message spoken when the screen sharing link comes back after a cut + wx.CallAfter(ui.message, _("Connection restored")) + else: + # Translators: message spoken when the screen sharing picture starts flowing + wx.CallAfter(ui.message, _("Screen sharing started")) + elif kind == "interrupted": + wx.CallAfter(self._handle_interruption) elif kind == "failed": logger.warning("Screen sharing failed: %s", event.get("reason", "")) wx.CallAfter(self._report_failure) diff --git a/addon/globalPlugins/remoteClient/session.py b/addon/globalPlugins/remoteClient/session.py index 9e093ef..d68ede1 100644 --- a/addon/globalPlugins/remoteClient/session.py +++ b/addon/globalPlugins/remoteClient/session.py @@ -67,6 +67,9 @@ def __init__(self, local_machine, transport: RelayTransport): self.screen_share = screen_share.ScreenShareManager( transport, self.capabilities, self.SCREEN_SHARE_ROLE ) + # Sound arriving from the controlled computer already carries its NVDA, so the + # manager needs to reach the speech this session would otherwise also announce. + self.screen_share.local_machine = local_machine self.client_count = 1 def handle_version_mismatch(self, **kwargs): diff --git a/addon/globalPlugins/remoteClient/web/screen_share.html b/addon/globalPlugins/remoteClient/web/screen_share.html index c5663c2..38541db 100644 --- a/addon/globalPlugins/remoteClient/web/screen_share.html +++ b/addon/globalPlugins/remoteClient/web/screen_share.html @@ -41,6 +41,13 @@ let pc = null; let stream = null; +/* The native capture, when this computer has one. See openAudioBridge. */ +let audioBridge = null; +/* Whether this session carries sound, and whether it carries a picture. Sound can + be sent on its own, in which case the same element simply plays with nothing to + display. */ +let receiveAudio = false; +let receiveVideo = true; let since = 0; let stopped = false; /* Candidates that arrive before the remote description does cannot be added yet. */ @@ -83,6 +90,16 @@ if (pc.connectionState === "connected") { post({ event: "connected" }); show(""); + } else if (pc.connectionState === "disconnected") { + /* A passing network cut. Nothing is broken yet and the browser may well + recover on its own, so the session is left alone. But the media has + stopped, and that has to be said: on the watching side the forwarded + speech is held back while sound is arriving, so staying quiet here + would leave that user with neither the sound nor the speech, and no + sign that anything went wrong. It can take tens of seconds before this + turns into "failed". */ + post({ event: "interrupted" }); + show("Connection interrupted..."); } else if (pc.connectionState === "failed") { fail("The connection could not be established"); } else if (pc.connectionState === "closed") { @@ -92,6 +109,19 @@ if (!isPublisher) { pc.ontrack = (event) => { video.srcObject = event.streams[0]; + /* The element is muted in the markup so that a picture always starts on its + own, whatever the autoplay policy. Sound is what the user explicitly asked + for, so it is unmuted here, and the browser is started with the flag that + lets it play without a click nobody could give in an off screen window. */ + video.muted = !receiveAudio; + const played = video.play(); + if (played && played.catch) { + played.catch((error) => { + /* Losing the sound is worth reporting; losing the picture is not, since + a muted element always plays. */ + if (receiveAudio) { fail("The sound could not be played: " + error); } + }); + } }; } } @@ -109,16 +139,145 @@ return QUALITY[name] || QUALITY.balanced; } +/* The native helper captures the system mix with NVDA left out, which no web API can + * do, and pushes 48 kHz stereo 16 bit frames over a loopback WebSocket. They are fed + * to an AudioWorklet whose output goes to a MediaStreamDestination, and the track that + * comes out is handed to the connection like any other. + * + * Timing is the whole difficulty. Frames arrive over a socket, in bursts, with no + * relation to the audio clock. A small buffer absorbs that: too small and every hiccup + * is a hole in the sound, too large and delay piles up for nothing. */ +const BRIDGE_RATE = 48000; +const BRIDGE_CHANNELS = 2; +const BRIDGE_TARGET_MS = 30; +const BRIDGE_MAX_MS = 400; + +const BRIDGE_WORKLET = ` +class BridgeProcessor extends AudioWorkletProcessor { + constructor(options) { + super(); + const p = options.processorOptions; + this.channels = p.channels; + this.capacity = Math.floor(p.rate * p.maxMs / 1000) * this.channels; + this.ring = new Float32Array(this.capacity); + this.read = 0; + this.write = 0; + this.filled = 0; + this.startAt = Math.floor(p.rate * p.targetMs / 1000) * this.channels; + this.started = false; + this.port.onmessage = (event) => { this.push(event.data); }; + } + push(samples) { + /* Dropping the oldest rather than the newest bounds the delay when the socket + runs ahead: the sound stays late by at most maxMs, never more. */ + const room = this.capacity - this.filled; + if (samples.length > room) { + const drop = samples.length - room; + this.read = (this.read + drop) % this.capacity; + this.filled -= drop; + } + for (let i = 0; i < samples.length; i++) { + this.ring[this.write] = samples[i] / 32768; + this.write = (this.write + 1) % this.capacity; + } + this.filled += samples.length; + } + process(inputs, outputs) { + const out = outputs[0]; + const frames = out[0].length; + if (!this.started) { + if (this.filled < this.startAt) { return true; } + this.started = true; + } + const wanted = frames * this.channels; + if (this.filled >= wanted) { + for (let f = 0; f < frames; f++) { + for (let c = 0; c < this.channels; c++) { + out[c][f] = this.ring[this.read]; + this.read = (this.read + 1) % this.capacity; + } + } + this.filled -= wanted; + } else { + /* Run dry. The outputs stay at zero, silence rather than a click, and the + buffer is refilled to its target before playing again. Without this the + level settles just above empty, where every scheduling hiccup is heard; + one short resynchronisation is far less unpleasant than a stutter. */ + this.started = false; + } + return true; + } +} +registerProcessor("bridge", BridgeProcessor); +`; + +async function openAudioBridge(url) { + const context = new AudioContext({ sampleRate: BRIDGE_RATE, latencyHint: "interactive" }); + const moduleUrl = URL.createObjectURL(new Blob([BRIDGE_WORKLET], { type: "application/javascript" })); + try { + await context.audioWorklet.addModule(moduleUrl); + } finally { + URL.revokeObjectURL(moduleUrl); + } + const node = new AudioWorkletNode(context, "bridge", { + numberOfInputs: 0, + numberOfOutputs: 1, + outputChannelCount: [BRIDGE_CHANNELS], + processorOptions: { + rate: BRIDGE_RATE, + channels: BRIDGE_CHANNELS, + targetMs: BRIDGE_TARGET_MS, + maxMs: BRIDGE_MAX_MS, + }, + }); + const destination = context.createMediaStreamDestination(); + node.connect(destination); + + const socket = new WebSocket(url); + socket.binaryType = "arraybuffer"; + await new Promise((resolve, reject) => { + socket.onopen = resolve; + socket.onerror = () => reject(new Error("the local audio bridge is unreachable")); + }); + socket.onmessage = (event) => { node.port.postMessage(new Int16Array(event.data)); }; + + return { + track: destination.stream.getAudioTracks()[0], + close: () => { + try { socket.close(); } catch (e) { /* already gone */ } + try { node.disconnect(); } catch (e) { /* already gone */ } + try { context.close(); } catch (e) { /* already gone */ } + }, + }; +} + async function startPublisher(command) { video.remove(); + /* A command sent by an older add-on names neither stream, and asks for a picture. */ + const sendVideo = command.send_video !== false; + const sendAudio = !!command.send_audio; const quality = qualityOf(command.quality); - const maxFps = Math.max(1, Math.min(30, command.max_fps || 15)); + /* When the add-on could start the native helper it hands over its address here, and + the sound then comes from it rather than from the browser. That capture leaves + NVDA out, which the browser cannot do, and it needs no display surface. */ + const audioWs = command.audio_ws || null; + const nativeAudio = sendAudio && audioWs !== null; + const browserAudio = sendAudio && !nativeAudio; + /* getDisplayMedia has no audio only form: the sound of the computer is only ever + handed over alongside a display surface. So when the sound comes from the browser + and the picture was not asked for, the capture still runs and its track is simply + never added, nothing being encoded or sent. With the native helper that detour + disappears: an audio only session captures no screen at all. */ + const needCapture = sendVideo || browserAudio; + const maxFps = sendVideo ? Math.max(1, Math.min(30, command.max_fps || 15)) : 1; /* Encoding a 4K desktop in real time saturates any processor, and the picture is displayed far smaller than that on the other side anyway. Asking the capture itself for a smaller frame is what costs least: the browser scales it down before the encoder ever sees it. The ratio is preserved, only a ceiling is given. */ - const maxWidth = Math.max(640, Math.min(3840, command.max_width || quality.width)); + const maxWidth = sendVideo + ? Math.max(640, Math.min(3840, command.max_width || quality.width)) + : 640; const constraints = { video: { frameRate: maxFps, @@ -127,17 +286,35 @@ has no idea where the mouse they are driving actually is. */ cursor: "always", }, - audio: false, + /* This is the sound the computer plays, taken from the system mix, not a + microphone. The three processing steps below belong to a voice call: they + would fight the echo of a video against itself, ride the volume up and down + and eat anything they took for noise. Turned off, the mix arrives in stereo, + as it sounds on the machine it comes from. */ + audio: browserAudio + ? { echoCancellation: false, autoGainControl: false, noiseSuppression: false } + : false, }; - try { - stream = await navigator.mediaDevices.getDisplayMedia(constraints); - } catch (error) { - /* The add-on starts the browser with the flag that suppresses the source picker, - so this normally cannot happen. If a future version of the browser drops that - flag, the failure has to travel back rather than leave a window waiting for - a click nobody can see. */ - fail("Screen capture was refused: " + (error && error.name ? error.name : error)); - return; + if (browserAudio) { + /* Asks for the sound of the whole computer rather than that of one tab. */ + constraints.systemAudio = "include"; + /* The person sitting at the controlled computer must keep hearing their own + machine while it is being listened to. */ + constraints.audio.suppressLocalAudioPlayback = false; + } + if (needCapture) { + try { + stream = await navigator.mediaDevices.getDisplayMedia(constraints); + } catch (error) { + /* The add-on starts the browser with the flag that suppresses the source + picker, so this normally cannot happen. If a future version of the browser + drops that flag, the failure has to travel back rather than leave a window + waiting for a click nobody can see. */ + fail("Screen capture was refused: " + (error && error.name ? error.name : error)); + return; + } + } else { + stream = new MediaStream(); } stream.getVideoTracks().forEach((track) => { /* What is being watched is text, not a film. This tells the encoder to keep the @@ -151,15 +328,88 @@ /* The user can also stop the capture from the browser indicator. */ track.onended = () => { post({ event: "closed" }); }; }); - stream.getTracks().forEach((track) => pc.addTrack(track, stream)); + if (browserAudio && !stream.getAudioTracks().length) { + /* The capture succeeded but the browser handed over no sound. Saying so is far + better than a session which looks established and stays silent. */ + fail("This computer did not allow its sound to be captured"); + return; + } + if (nativeAudio) { + try { + audioBridge = await openAudioBridge(audioWs); + } catch (error) { + /* The helper announced itself to the add-on a moment ago, so this is not + expected. Saying so beats a session which looks established and stays + silent. */ + fail("The sound of this computer could not be captured: " + error); + return; + } + stream.addTrack(audioBridge.track); + } + stream.getAudioTracks().forEach((track) => { + /* Not speech: the mix carries music, alert sounds and video as readily as a + voice, and must not be narrowed to the band of a telephone call. */ + track.contentHint = "music"; + }); + stream.getTracks().forEach((track) => { + /* The video track is kept running even when it is not wanted, because stopping + it would end the capture the sound is taken from. It is simply never offered. */ + if (track.kind === "video" && !sendVideo) { return; } + pc.addTrack(track, stream); + }); /* After addTrack, and not before: the senders this walks through do not exist yet while the track has not been added. */ applyQuality(quality, maxFps); + applyAudioQuality(); const offer = await pc.createOffer(); + offer.sdp = withStereoOpus(offer.sdp); await pc.setLocalDescription(offer); post({ event: "offer", sdp: pc.localDescription.sdp }); } +/* Stereo, at a rate which carries music rather than only speech. This is a small + * fraction of what the picture costs, and the link already carries that. */ +const AUDIO_BITRATE = 128000; + +function applyAudioQuality() { + pc.getSenders().forEach((sender) => { + if (!sender.track || sender.track.kind !== "audio") { return; } + const parameters = sender.getParameters(); + if (!parameters.encodings || !parameters.encodings.length) { + parameters.encodings = [{}]; + } + parameters.encodings[0].maxBitrate = AUDIO_BITRATE; + sender.setParameters(parameters).catch(() => {}); + }); +} + +/* Opus is negotiated in mono unless the description says otherwise, whatever the track + feeding it holds. "stereo=1" states that this end can decode two channels, which is + what makes the other end encode them, and "sprop-stereo=1" announces that this end + sends two. A browser adds neither on its own, so the sound captured in stereo would + be mixed down before it ever left. Both computers run this, so both directions carry + two channels; one running an older build simply announces nothing and gets the mono + it understands. */ +function withStereoOpus(sdp) { + /* The payload number Opus was given is not fixed, so it is read rather than assumed. */ + const rtpmap = sdp.match(/^a=rtpmap:(\d+) opus\/48000\/2/mi); + if (!rtpmap) { return sdp; } + const payload = rtpmap[1]; + const wanted = "stereo=1;sprop-stereo=1;maxaveragebitrate=" + AUDIO_BITRATE; + /* A carriage return ends every line here, so it must be kept out of what is matched. */ + const fmtp = new RegExp("^a=fmtp:" + payload + " ([^\\r\\n]*)", "mi"); + if (fmtp.test(sdp)) { + return sdp.replace(fmtp, (line, existing) => ( + existing.indexOf("stereo=") !== -1 + ? line + : "a=fmtp:" + payload + " " + existing + ";" + wanted + )); + } + /* Opus was offered without any parameters at all: give it a line of its own. */ + const eol = sdp.indexOf("\r\n") !== -1 ? "\r\n" : "\n"; + return sdp.replace(rtpmap[0], rtpmap[0] + eol + "a=fmtp:" + payload + " " + wanted); +} + function applyQuality(quality, maxFps) { pc.getSenders().forEach((sender) => { if (!sender.track || sender.track.kind !== "video") { return; } @@ -283,14 +533,22 @@ if (isPublisher) { await startPublisher(command); } else { + /* Set before the first track arrives: ontrack reads them. */ + receiveAudio = !!command.receive_audio; + receiveVideo = command.receive_video !== false; if (command.allow_input) { enableInput(); } - show("Waiting for the picture..."); + if (!receiveVideo) { + show("Waiting for the sound..."); + } else { + show("Waiting for the picture..."); + } } } else if (name === "offer") { if (!pc) { return; } await pc.setRemoteDescription({ type: "offer", sdp: command.sdp }); await drainCandidates(); const answer = await pc.createAnswer(); + answer.sdp = withStereoOpus(answer.sdp); await pc.setLocalDescription(answer); post({ event: "answer", sdp: pc.localDescription.sdp }); } else if (name === "answer") { @@ -322,6 +580,10 @@ function shutdown() { stopped = true; + if (audioBridge) { + audioBridge.close(); + audioBridge = null; + } if (stream) { stream.getTracks().forEach((track) => track.stop()); stream = null; diff --git a/addon/locale/fr/LC_MESSAGES/nvda.mo b/addon/locale/fr/LC_MESSAGES/nvda.mo index fa1efe6..8cf4869 100644 Binary files a/addon/locale/fr/LC_MESSAGES/nvda.mo and b/addon/locale/fr/LC_MESSAGES/nvda.mo differ diff --git a/addon/locale/fr/LC_MESSAGES/nvda.po b/addon/locale/fr/LC_MESSAGES/nvda.po index d28feeb..7ee6c30 100644 --- a/addon/locale/fr/LC_MESSAGES/nvda.po +++ b/addon/locale/fr/LC_MESSAGES/nvda.po @@ -1574,6 +1574,74 @@ msgstr "" "Kerberos, améliorer la compatibilité avec les proxys et fournir deux " "méthodes de capture d'écran distante." +msgid "Allow sending the sound of this computer, after confirmation" +msgstr "Autoriser l'envoi du son de cet ordinateur, après confirmation" + +msgid "Connection restored" +msgstr "Connexion rétablie" + +msgid "" +"Do you want to share your screen and your sound? The controlling " +"computer will see this screen and hear everything this computer plays." +msgstr "" +"Voulez-vous partager votre écran et votre son ? L'ordinateur qui vous " +"contrôle verra cet écran et entendra tout ce que cet ordinateur joue." + +msgid "" +"Do you want to share your screen and your sound? The controlling " +"computer will see this screen, hear everything this computer plays, " +"and will be able to use its mouse." +msgstr "" +"Voulez-vous partager votre écran et votre son ? L'ordinateur qui vous " +"contrôle verra cet écran, entendra tout ce que cet ordinateur joue, et " +"pourra utiliser sa souris." + +msgid "" +"Do you want to share your sound? The controlling computer will hear " +"everything this computer plays, including calls and videos." +msgstr "" +"Voulez-vous partager votre son ? L'ordinateur qui vous contrôle " +"entendra tout ce que cet ordinateur joue, y compris les appels et les " +"vidéos." + +msgid "" +"Plays or stops playing the sound of the controlled computer on this " +"one" +msgstr "" +"Joue ou arrête de jouer sur cet ordinateur le son de l'ordinateur " +"contrôlé" + +msgid "Remote sound requested" +msgstr "Son distant demandé" + +msgid "Remote sound stopped" +msgstr "Son distant arrêté" + +msgid "Screen sharing with sound requested" +msgstr "Partage d'écran avec son demandé" + +msgid "Sharing this screen and this sound" +msgstr "Partage de cet écran et de ce son en cours" + +msgid "Sharing this sound" +msgstr "Partage de ce son en cours" + +msgid "Sound interrupted" +msgstr "Son interrompu" + +msgid "Sound sharing request" +msgstr "Demande de partage du son" + +msgid "The other computer does not support sending its sound" +msgstr "L'autre ordinateur ne peut pas envoyer son son" + +msgid "" +"When the sound of the controlled computer already contains its own " +"screen reader, do not also speak what it reports" +msgstr "" +"Lorsque le son de l'ordinateur contrôlé contient déjà son propre " +"lecteur d'écran, ne pas annoncer en plus ce qu'il rapporte" + #~ msgid "This file is too large. Only files smaller than 10 MB are supported." #~ msgstr "" #~ "Ce fichier est trop grand. Seuls les fichiers inférieurs à 10 Mo sont " diff --git a/native-audio-capture/CMakeLists.txt b/native-audio-capture/CMakeLists.txt new file mode 100644 index 0000000..d83cb55 --- /dev/null +++ b/native-audio-capture/CMakeLists.txt @@ -0,0 +1,24 @@ +cmake_minimum_required(VERSION 3.20) +project(nvda_audio_capture CXX) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +add_executable(nvda_audio_capture src/main.cpp) +target_link_libraries(nvda_audio_capture PRIVATE ole32 oleaut32 ws2_32 bcrypt crypt32) + +if(MINGW) + # -municode selects the wmain entry point. + # -static drops the libstdc++ and libgcc DLL dependencies, so the executable + # can be shipped inside an add-on without anything to install alongside it. + # + # The rest is about size. The add-on repository refuses files over 500 kB + # (check-added-large-files in .pre-commit-config.yaml), and a statically linked + # C++ binary sails past that without trying. Optimising for size, dropping + # exceptions and RTTI, giving each function its own section and letting the + # linker discard the unreachable ones brings it back well under. + target_compile_options(nvda_audio_capture PRIVATE + -municode -Os -fno-exceptions -fno-rtti -ffunction-sections -fdata-sections) + target_link_options(nvda_audio_capture PRIVATE + -municode -static -s -Wl,--gc-sections) +endif() diff --git a/native-audio-capture/README.md b/native-audio-capture/README.md new file mode 100644 index 0000000..dce7d7f --- /dev/null +++ b/native-audio-capture/README.md @@ -0,0 +1,73 @@ +# Native audio capture helper + +Source of `addon/globalPlugins/remoteClient/bin/nvda_audio_capture.exe`, the small +program the add-on starts when the controlled computer shares its sound. + +## Why it exists + +A web page can only capture the audio the browser itself renders, or the whole system +mix through screen capture. Neither lets NVDA be left out, so the sound shared with the +other end carries the remote NVDA's own speech over everything else — which is exactly +what the person watching does not want to hear. + +Windows 10 build 20348 and later expose a *process loopback* activation of +`IAudioClient`: given a process id, it captures either only what that process tree +renders, or everything except it. The second mode is the one that matters here. That API +is unreachable from a page, so this helper does the capture natively and serves the +samples to the page over a loopback WebSocket, which turns them back into a WebRTC track. + +When the helper is unavailable — an older Windows, or a build where it is missing — the +add-on falls back to the browser capture, and the sound then includes the remote NVDA. +Nothing breaks; only the exclusion is lost. + +## Building + +Requires CMake 3.20 or later and a MinGW-w64 g++. The shipped binary was built with the +toolchain bundled in Strawberry Perl: + +```sh +cmake -B build -S . -G Ninja -DCMAKE_BUILD_TYPE=Release +cmake --build build +``` + +Then copy the result next to the add-on sources: + +```sh +cp build/nvda_audio_capture.exe ../addon/globalPlugins/remoteClient/bin/ +``` + +The compile and link options are chosen for size and are commented in `CMakeLists.txt`: +the executable is linked statically so it can ship inside an add-on with nothing to +install alongside it, which pushes a C++ binary well past the 500 kB ceiling that +`check-added-large-files` enforces. Optimising for size, dropping exceptions and RTTI and +letting the linker discard unreachable sections brings it back to about 256 kB. + +## Checking it on a single machine + +The default mode needs no session and no second computer. It reports the captured level +once per second, so that the exclusion can be confirmed on its own: + +```sh +# Everything except NVDA. Play some music: the level should follow it. +./build/nvda_audio_capture.exe --seconds 10 + +# The opposite, to confirm the right process is being targeted. Make NVDA +# speak: the level should follow the speech and nothing else. +./build/nvda_audio_capture.exe --seconds 10 --include +``` + +`--out file.wav` writes what was captured, and `--pid N` or `--process name.exe` targets +something other than `nvda.exe`. A level of -120 dBFS means the stream carried nothing at +all, which is not the same as a stream of silence — around -96 dBFS is the noise floor of +16 bit samples and shows the capture is live. + +## Serving mode + +`--serve` is what the add-on uses. It binds a WebSocket on `127.0.0.1`, prints `PORT n` +on standard output and streams 48 kHz stereo 16 bit PCM once the page connects. + +It repeats, deliberately, the four protections of `local_bridge.py`: the socket binds to +`127.0.0.1` alone, the port is chosen by the system for each session, every connection +must carry the session token, compared in constant time, and an `Origin` header that is +not the exact expected one is refused. The token arrives on standard input rather than on +the command line, which other processes on the machine can read. diff --git a/native-audio-capture/src/main.cpp b/native-audio-capture/src/main.cpp new file mode 100644 index 0000000..5e981cb --- /dev/null +++ b/native-audio-capture/src/main.cpp @@ -0,0 +1,636 @@ +// nvda_audio_capture - capture the Windows audio mix with one process tree excluded, +// and hand it to the browser page over a loopback WebSocket. +// +// Windows 10 build 20348 and later expose a "process loopback" activation of +// IAudioClient. Given a process id it can capture either only what that process +// tree renders, or everything except what it renders. The second mode is what we +// need: the whole system mix minus NVDA, so the sound of the controlled computer +// can be streamed without carrying NVDA's own speech along with it. +// +// A web page cannot reach this API, which is why the browser capture used today +// unavoidably contains the remote NVDA. This program does the capture natively and +// serves the samples to the page, which turns them back into a WebRTC track. +// +// Two modes: +// measure (default) report the level once per second, optionally write a WAV. +// Everything can be checked on a single machine this way. +// --serve bind a WebSocket on 127.0.0.1, print the chosen port, wait for the +// page and stream 48 kHz stereo 16 bit PCM to it. +// +// The serving mode repeats, deliberately, the four protections of local_bridge.py: +// the socket binds to 127.0.0.1 alone, the port is chosen by the system for each +// session, every connection must carry the session token, compared in constant +// time, and an Origin header that is not the exact expected one is refused. The +// token arrives on standard input rather than on the command line, which other +// processes on the machine can read. + +#include +#include + +#define WIN32_LEAN_AND_MEAN +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +// --------------------------------------------------------------------------- +// Declarations missing from the MinGW headers (Windows SDK 10.0.20348+). +// The API itself lives in mmdevapi.dll and ships with Windows; only the header +// describing these structures is absent, so we restate it here. +// --------------------------------------------------------------------------- + +#ifndef VIRTUAL_AUDIO_DEVICE_PROCESS_LOOPBACK +#define VIRTUAL_AUDIO_DEVICE_PROCESS_LOOPBACK L"VAD\\Process_Loopback" +#endif + +enum ActivationType { + ACTIVATION_TYPE_DEFAULT = 0, + ACTIVATION_TYPE_PROCESS_LOOPBACK = 1, +}; + +enum LoopbackMode { + LOOPBACK_INCLUDE_TARGET_PROCESS_TREE = 0, + LOOPBACK_EXCLUDE_TARGET_PROCESS_TREE = 1, +}; + +struct ProcessLoopbackParams { + DWORD TargetProcessId; + LoopbackMode ProcessLoopbackMode; +}; + +struct ActivationParams { + ActivationType Type; + union { + ProcessLoopbackParams ProcessLoopback; + }; +}; + +typedef HRESULT(STDAPICALLTYPE *PfnActivateAudioInterfaceAsync)( + LPCWSTR, REFIID, PROPVARIANT *, + IActivateAudioInterfaceCompletionHandler *, + IActivateAudioInterfaceAsyncOperation **); + +static const GUID kIID_IAudioClient = + {0x1CB9AD4C, 0xDBFA, 0x4C32, {0xB1, 0x78, 0xC2, 0xF5, 0x68, 0xA7, 0x03, 0xB2}}; +static const GUID kIID_IAudioCaptureClient = + {0xC8ADBD64, 0xE71E, 0x48A0, {0xA4, 0xDE, 0x18, 0x5C, 0x39, 0x5C, 0xD3, 0x17}}; +static const GUID kIID_ICompletionHandler = + {0x41D949AB, 0x9862, 0x444A, {0x80, 0xF6, 0xC2, 0x61, 0x33, 0x4D, 0xA5, 0xEB}}; +static const GUID kIID_IAgileObject = + {0x94EA2B94, 0xE9CC, 0x49E0, {0xC0, 0xFF, 0xEE, 0x64, 0xCA, 0x8F, 0x5B, 0x90}}; + +static const int kRate = 48000; +static const int kChannels = 2; + +// --------------------------------------------------------------------------- +// Activation is asynchronous and answers through this handler. +// --------------------------------------------------------------------------- + +class Handler final : public IActivateAudioInterfaceCompletionHandler, public IAgileObject { +public: + HANDLE done = CreateEventW(nullptr, TRUE, FALSE, nullptr); + HRESULT result = E_FAIL; + IAudioClient *client = nullptr; + + HRESULT STDMETHODCALLTYPE QueryInterface(REFIID riid, void **out) override { + if (out == nullptr) return E_POINTER; + if (IsEqualGUID(riid, IID_IUnknown) || IsEqualGUID(riid, kIID_ICompletionHandler)) { + *out = static_cast(this); + } else if (IsEqualGUID(riid, kIID_IAgileObject)) { + *out = static_cast(this); + } else { + *out = nullptr; + return E_NOINTERFACE; + } + AddRef(); + return S_OK; + } + + ULONG STDMETHODCALLTYPE AddRef() override { return InterlockedIncrement(&refs_); } + + ULONG STDMETHODCALLTYPE Release() override { + LONG n = InterlockedDecrement(&refs_); + if (n == 0) delete this; + return n; + } + + HRESULT STDMETHODCALLTYPE ActivateCompleted(IActivateAudioInterfaceAsyncOperation *op) override { + IUnknown *unknown = nullptr; + HRESULT activation = S_OK; + result = op->GetActivateResult(&activation, &unknown); + if (SUCCEEDED(result)) result = activation; + if (SUCCEEDED(result) && unknown != nullptr) { + result = unknown->QueryInterface(kIID_IAudioClient, reinterpret_cast(&client)); + } + if (unknown != nullptr) unknown->Release(); + SetEvent(done); + return S_OK; + } + +private: + ~Handler() { + if (done != nullptr) CloseHandle(done); + } + LONG refs_ = 1; +}; + +// --------------------------------------------------------------------------- +// Small helpers. +// --------------------------------------------------------------------------- + +static DWORD find_process(const wchar_t *name) { + HANDLE snap = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0); + if (snap == INVALID_HANDLE_VALUE) return 0; + PROCESSENTRY32W entry; + entry.dwSize = sizeof(entry); + DWORD found = 0; + if (Process32FirstW(snap, &entry)) { + do { + if (_wcsicmp(entry.szExeFile, name) == 0) { + found = entry.th32ProcessID; + break; + } + } while (Process32NextW(snap, &entry)); + } + CloseHandle(snap); + return found; +} + +static void write_wav(const char *path, const std::vector &samples) { + FILE *f = fopen(path, "wb"); + if (f == nullptr) { + printf("Unable to write %s\n", path); + return; + } + const unsigned data_bytes = static_cast(samples.size() * sizeof(short)); + const unsigned byte_rate = static_cast(kRate * kChannels * 2); + const unsigned short block_align = static_cast(kChannels * 2); + const unsigned riff = 36 + data_bytes; + const unsigned fmt_size = 16; + const unsigned short pcm = 1; + const unsigned short ch = static_cast(kChannels); + const unsigned short bits = 16; + const unsigned rate_u = static_cast(kRate); + fwrite("RIFF", 1, 4, f); + fwrite(&riff, 4, 1, f); + fwrite("WAVEfmt ", 1, 8, f); + fwrite(&fmt_size, 4, 1, f); + fwrite(&pcm, 2, 1, f); + fwrite(&ch, 2, 1, f); + fwrite(&rate_u, 4, 1, f); + fwrite(&byte_rate, 4, 1, f); + fwrite(&block_align, 2, 1, f); + fwrite(&bits, 2, 1, f); + fwrite("data", 1, 4, f); + fwrite(&data_bytes, 4, 1, f); + fwrite(samples.data(), 1, data_bytes, f); + fclose(f); + printf("Wrote %s (%.1f s)\n", path, static_cast(samples.size()) / (kRate * kChannels)); +} + +static double to_dbfs(double rms) { + if (rms <= 0.0000001) return -120.0; + return 20.0 * log10(rms); +} + +// --------------------------------------------------------------------------- +// Just enough WebSocket to push binary frames at one local page. +// --------------------------------------------------------------------------- + +static bool sha1(const std::string &in, BYTE out[20]) { + BCRYPT_ALG_HANDLE alg = nullptr; + if (BCryptOpenAlgorithmProvider(&alg, BCRYPT_SHA1_ALGORITHM, nullptr, 0) != 0) return false; + BCRYPT_HASH_HANDLE hash = nullptr; + bool ok = BCryptCreateHash(alg, &hash, nullptr, 0, nullptr, 0, 0) == 0; + if (ok) ok = BCryptHashData(hash, reinterpret_cast(const_cast(in.data())), + static_cast(in.size()), 0) == 0; + if (ok) ok = BCryptFinishHash(hash, out, 20, 0) == 0; + if (hash != nullptr) BCryptDestroyHash(hash); + BCryptCloseAlgorithmProvider(alg, 0); + return ok; +} + +static std::string base64(const BYTE *data, DWORD len) { + DWORD chars = 0; + if (!CryptBinaryToStringA(data, len, CRYPT_STRING_BASE64 | CRYPT_STRING_NOCRLF, nullptr, &chars)) { + return std::string(); + } + std::string out(chars, '\0'); + if (!CryptBinaryToStringA(data, len, CRYPT_STRING_BASE64 | CRYPT_STRING_NOCRLF, &out[0], &chars)) { + return std::string(); + } + out.resize(strlen(out.c_str())); + return out; +} + +/// Compare without leaking, through timing, how many characters matched. +static bool same_secret(const std::string &a, const std::string &b) { + if (a.size() != b.size()) return false; + unsigned char diff = 0; + for (size_t i = 0; i < a.size(); i++) { + diff |= static_cast(a[i] ^ b[i]); + } + return diff == 0; +} + +static std::string header_value(const std::string &request, const std::string &name) { + std::string lower; + lower.reserve(request.size()); + for (char c : request) lower.push_back(static_cast(tolower(static_cast(c)))); + const std::string needle = "\r\n" + name + ":"; + const size_t at = lower.find(needle); + if (at == std::string::npos) return std::string(); + size_t start = at + needle.size(); + while (start < request.size() && (request[start] == ' ' || request[start] == '\t')) start++; + const size_t end = request.find("\r\n", start); + if (end == std::string::npos) return std::string(); + return request.substr(start, end - start); +} + +static std::string query_token(const std::string &request) { + const size_t line_end = request.find("\r\n"); + if (line_end == std::string::npos) return std::string(); + const std::string line = request.substr(0, line_end); + const size_t at = line.find("token="); + if (at == std::string::npos) return std::string(); + size_t end = at + 6; + while (end < line.size() && line[end] != '&' && line[end] != ' ') end++; + return line.substr(at + 6, end - (at + 6)); +} + +static bool send_all(SOCKET s, const char *data, size_t len) { + size_t sent = 0; + while (sent < len) { + const int n = send(s, data + sent, static_cast(len - sent), 0); + if (n <= 0) return false; + sent += static_cast(n); + } + return true; +} + +static bool ws_send_binary(SOCKET s, const void *payload, size_t len) { + char header[10]; + size_t header_len = 0; + header[0] = static_cast(0x82); // FIN + binary opcode + if (len < 126) { + header[1] = static_cast(len); + header_len = 2; + } else if (len <= 0xFFFF) { + header[1] = 126; + header[2] = static_cast((len >> 8) & 0xFF); + header[3] = static_cast(len & 0xFF); + header_len = 4; + } else { + header[1] = 127; + for (int i = 0; i < 8; i++) { + header[2 + i] = static_cast((static_cast(len) >> ((7 - i) * 8)) & 0xFF); + } + header_len = 10; + } + if (!send_all(s, header, header_len)) return false; + return send_all(s, static_cast(payload), len); +} + +/// Read the request, check the session token and the origin, answer the upgrade. +static bool ws_accept(SOCKET s, const std::string &token, const std::string &origin) { + std::string request; + char buf[1024]; + while (request.find("\r\n\r\n") == std::string::npos) { + if (request.size() > 8192) return false; + const int n = recv(s, buf, sizeof(buf), 0); + if (n <= 0) return false; + request.append(buf, static_cast(n)); + } + + if (!same_secret(query_token(request), token)) { + fprintf(stderr, "Connection refused: invalid token.\n"); + return false; + } + const std::string got_origin = header_value(request, "origin"); + if (got_origin != origin) { + fprintf(stderr, "Connection refused: unexpected origin \"%s\".\n", got_origin.c_str()); + return false; + } + const std::string key = header_value(request, "sec-websocket-key"); + if (key.empty()) return false; + + BYTE digest[20]; + if (!sha1(key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11", digest)) return false; + const std::string accept = base64(digest, 20); + + const std::string response = + "HTTP/1.1 101 Switching Protocols\r\n" + "Upgrade: websocket\r\n" + "Connection: Upgrade\r\n" + "Sec-WebSocket-Accept: " + accept + "\r\n\r\n"; + return send_all(s, response.data(), response.size()); +} + +// --------------------------------------------------------------------------- +// Capture setup, shared by both modes. +// --------------------------------------------------------------------------- + +struct Capture { + IAudioClient *client = nullptr; + IAudioCaptureClient *capture = nullptr; + HANDLE ready = nullptr; +}; + +static bool start_capture(DWORD pid, bool include, Capture *out) { + ActivationParams params{}; + params.Type = ACTIVATION_TYPE_PROCESS_LOOPBACK; + params.ProcessLoopback.TargetProcessId = pid; + params.ProcessLoopback.ProcessLoopbackMode = + include ? LOOPBACK_INCLUDE_TARGET_PROCESS_TREE : LOOPBACK_EXCLUDE_TARGET_PROCESS_TREE; + + PROPVARIANT activation{}; + activation.vt = VT_BLOB; + activation.blob.cbSize = sizeof(params); + activation.blob.pBlobData = reinterpret_cast(¶ms); + + HMODULE dll = LoadLibraryW(L"mmdevapi.dll"); + if (dll == nullptr) { + fprintf(stderr, "mmdevapi.dll not found.\n"); + return false; + } + const auto activate = reinterpret_cast( + reinterpret_cast(GetProcAddress(dll, "ActivateAudioInterfaceAsync"))); + if (activate == nullptr) { + fprintf(stderr, "ActivateAudioInterfaceAsync is missing: this Windows is too old.\n" + "Process loopback needs Windows 10 build 20348 or later.\n"); + return false; + } + + Handler *handler = new Handler(); + IActivateAudioInterfaceAsyncOperation *op = nullptr; + HRESULT hr = activate(VIRTUAL_AUDIO_DEVICE_PROCESS_LOOPBACK, kIID_IAudioClient, &activation, handler, &op); + if (FAILED(hr)) { + fprintf(stderr, "ActivateAudioInterfaceAsync failed: 0x%08lX\n", static_cast(hr)); + return false; + } + WaitForSingleObject(handler->done, 5000); + if (op != nullptr) op->Release(); + if (FAILED(handler->result) || handler->client == nullptr) { + fprintf(stderr, "Activation refused: 0x%08lX\n", static_cast(handler->result)); + fprintf(stderr, "If that is 0x88890008, process loopback is not available on this machine.\n"); + return false; + } + out->client = handler->client; + handler->Release(); + + // The pseudo device has no mix format to ask for: we impose one. + WAVEFORMATEX wfx{}; + wfx.wFormatTag = WAVE_FORMAT_PCM; + wfx.nChannels = static_cast(kChannels); + wfx.nSamplesPerSec = kRate; + wfx.wBitsPerSample = 16; + wfx.nBlockAlign = static_cast(kChannels * 2); + wfx.nAvgBytesPerSec = kRate * wfx.nBlockAlign; + wfx.cbSize = 0; + + hr = out->client->Initialize(AUDCLNT_SHAREMODE_SHARED, + AUDCLNT_STREAMFLAGS_LOOPBACK | AUDCLNT_STREAMFLAGS_EVENTCALLBACK, + 200000 /* 20 ms, en unites de 100 ns */, 0, &wfx, nullptr); + if (FAILED(hr)) { + fprintf(stderr, "IAudioClient::Initialize failed: 0x%08lX\n", static_cast(hr)); + return false; + } + out->ready = CreateEventW(nullptr, FALSE, FALSE, nullptr); + out->client->SetEventHandle(out->ready); + hr = out->client->GetService(kIID_IAudioCaptureClient, reinterpret_cast(&out->capture)); + if (FAILED(hr)) { + fprintf(stderr, "GetService failed: 0x%08lX\n", static_cast(hr)); + return false; + } + return true; +} + +// --------------------------------------------------------------------------- + +static int run_serve(Capture &cap, const std::string &origin) { + std::string token; + { + char line[512]; + if (fgets(line, sizeof(line), stdin) == nullptr) { + fprintf(stderr, "No token received on standard input.\n"); + return 1; + } + token = line; + while (!token.empty() && (token.back() == '\n' || token.back() == '\r')) token.pop_back(); + } + if (token.empty()) { + fprintf(stderr, "Empty token.\n"); + return 1; + } + + WSADATA wsa; + if (WSAStartup(MAKEWORD(2, 2), &wsa) != 0) { + fprintf(stderr, "WSAStartup failed.\n"); + return 1; + } + const SOCKET listener = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (listener == INVALID_SOCKET) { + fprintf(stderr, "socket() failed.\n"); + return 1; + } + sockaddr_in addr{}; + addr.sin_family = AF_INET; + addr.sin_port = 0; // the system picks, so the port cannot be guessed in advance + InetPtonA(AF_INET, "127.0.0.1", &addr.sin_addr); // loopback alone, never every interface + if (bind(listener, reinterpret_cast(&addr), sizeof(addr)) != 0 || listen(listener, 1) != 0) { + fprintf(stderr, "bind/listen failed.\n"); + return 1; + } + sockaddr_in bound{}; + int bound_len = sizeof(bound); + getsockname(listener, reinterpret_cast(&bound), &bound_len); + + // NVDA reads this line to know where to point the page. + printf("PORT %d\n", ntohs(bound.sin_port)); + fflush(stdout); + + const SOCKET client = accept(listener, nullptr, nullptr); + closesocket(listener); + if (client == INVALID_SOCKET) { + fprintf(stderr, "accept() failed.\n"); + return 1; + } + if (!ws_accept(client, token, origin)) { + closesocket(client); + return 1; + } + fprintf(stderr, "Page connected, streaming.\n"); + + cap.client->Start(); + std::vector silence(static_cast(kRate / 100) * kChannels, 0); + bool alive = true; + while (alive) { + WaitForSingleObject(cap.ready, 200); + for (;;) { + BYTE *data = nullptr; + UINT32 frames = 0; + DWORD flags = 0; + const HRESULT hr = cap.capture->GetBuffer(&data, &frames, &flags, nullptr, nullptr); + if (hr == AUDCLNT_S_BUFFER_EMPTY || FAILED(hr) || frames == 0) break; + const size_t bytes = static_cast(frames) * kChannels * 2; + // Silent packets still have to go out: the page rebuilds a stream from + // this and its clock must keep being fed, or the track stalls. + if ((flags & AUDCLNT_BUFFERFLAGS_SILENT) != 0) { + if (silence.size() * 2 < bytes) silence.assign(bytes / 2, 0); + alive = ws_send_binary(client, silence.data(), bytes); + } else { + alive = ws_send_binary(client, data, bytes); + } + cap.capture->ReleaseBuffer(frames); + if (!alive) break; + } + } + cap.client->Stop(); + closesocket(client); + fprintf(stderr, "Page disconnected, stopping.\n"); + return 0; +} + +static int run_measure(Capture &cap, DWORD pid, bool include, int seconds, const std::string &out) { + printf(include ? "Capturing ONLY the sound of PID %lu, for %d s.\n" + : "Capturing all the system sound EXCEPT that of PID %lu, for %d s.\n", + pid, seconds); + printf("Level per second, in dBFS. -120 means complete silence.\n\n"); + + cap.client->Start(); + + std::vector recorded; + const DWORD started = GetTickCount(); + double window_sum = 0.0; + long window_count = 0; + double peak = -120.0; + int printed = 0; + + while (static_cast((GetTickCount() - started) / 1000) < seconds) { + WaitForSingleObject(cap.ready, 200); + for (;;) { + BYTE *data = nullptr; + UINT32 frames = 0; + DWORD flags = 0; + const HRESULT hr = cap.capture->GetBuffer(&data, &frames, &flags, nullptr, nullptr); + if (hr == AUDCLNT_S_BUFFER_EMPTY || FAILED(hr) || frames == 0) break; + const short *pcm = reinterpret_cast(data); + const size_t count = static_cast(frames) * kChannels; + if ((flags & AUDCLNT_BUFFERFLAGS_SILENT) != 0) { + window_count += static_cast(count); + if (!out.empty()) recorded.insert(recorded.end(), count, 0); + } else { + for (size_t i = 0; i < count; i++) { + const double v = pcm[i] / 32768.0; + window_sum += v * v; + } + window_count += static_cast(count); + if (!out.empty()) recorded.insert(recorded.end(), pcm, pcm + count); + } + cap.capture->ReleaseBuffer(frames); + } + const int elapsed = static_cast((GetTickCount() - started) / 1000); + if (elapsed > printed && window_count > 0) { + const double db = to_dbfs(sqrt(window_sum / window_count)); + if (db > peak) peak = db; + printf(" %2d s : %7.1f dBFS\n", elapsed, db); + fflush(stdout); + window_sum = 0.0; + window_count = 0; + printed = elapsed; + } + } + + cap.client->Stop(); + printf("\nLoudest level observed: %.1f dBFS\n", peak); + if (peak < -90.0) printf("Silence: nothing was captured.\n"); + if (!out.empty()) write_wav(out.c_str(), recorded); + return 0; +} + +// --------------------------------------------------------------------------- + +int wmain(int argc, wchar_t **argv) { + DWORD pid = 0; + int seconds = 10; + bool include = false; + bool serve = false; + std::string out; + std::string origin; + + for (int i = 1; i < argc; i++) { + const std::wstring a = argv[i]; + auto narrow = [&](int index) { + char buf[512]; + WideCharToMultiByte(CP_UTF8, 0, argv[index], -1, buf, sizeof(buf), nullptr, nullptr); + return std::string(buf); + }; + if (a == L"--pid" && i + 1 < argc) { + pid = static_cast(_wtoi(argv[++i])); + } else if (a == L"--process" && i + 1 < argc) { + pid = find_process(argv[++i]); + } else if (a == L"--seconds" && i + 1 < argc) { + seconds = _wtoi(argv[++i]); + } else if (a == L"--include") { + include = true; + } else if (a == L"--serve") { + serve = true; + } else if (a == L"--origin" && i + 1 < argc) { + origin = narrow(++i); + } else if (a == L"--out" && i + 1 < argc) { + out = narrow(++i); + } else { + printf("Usage: nvda_audio_capture [--process nvda.exe | --pid N]\n" + " [--seconds N] [--include] [--out file.wav]\n" + " [--serve --origin http://127.0.0.1:PORT]\n" + "\n" + "By default, captures all the system sound EXCEPT that of the target\n" + "process, and reports the level once per second.\n" + "--include does the opposite: only that process, which is handy to check\n" + "the right one is being targeted.\n" + "--serve opens a WebSocket on 127.0.0.1, prints \"PORT n\" on standard\n" + "output and streams the PCM. The session token is read from standard\n" + "input, and the given origin is the only one accepted.\n"); + return 1; + } + } + + if (serve && origin.empty()) { + fprintf(stderr, "--serve requires --origin.\n"); + return 1; + } + + if (pid == 0) { + pid = find_process(L"nvda.exe"); + if (pid == 0) { + fprintf(stderr, "NVDA not found. Use --pid or --process.\n"); + return 1; + } + if (!serve) printf("NVDA found, PID %lu\n", pid); + } + + const HRESULT hr = CoInitializeEx(nullptr, COINIT_MULTITHREADED); + if (FAILED(hr)) { + fprintf(stderr, "CoInitializeEx failed: 0x%08lX\n", static_cast(hr)); + return 1; + } + + Capture cap; + if (!start_capture(pid, include, &cap)) return 1; + + const int rc = serve ? run_serve(cap, origin) : run_measure(cap, pid, include, seconds, out); + + if (cap.capture != nullptr) cap.capture->Release(); + if (cap.client != nullptr) cap.client->Release(); + CoUninitialize(); + return rc; +} diff --git a/protocol.md b/protocol.md index 6c9615f..613f460 100644 --- a/protocol.md +++ b/protocol.md @@ -461,18 +461,27 @@ capabilities, network reliability and transfer timeouts. Transfers work in both directions: the controlling and the controlled computer use the same code and either of them may start a transfer. -### Screen Sharing +### Screen and Sound Sharing -The controlling computer may display the screen of the controlled one over a -peer to peer WebRTC link. The pictures never travel through the relay: it only -carries the few messages needed to set the link up, and the two computers then -talk to each other directly, falling back on a TURN server when the network -leaves them no other route. +The controlling computer may display the screen of the controlled one, hear the +sound it plays, or both, over a peer to peer WebRTC link. Neither the pictures +nor the sound travel through the relay: it only carries the few messages needed +to set the link up, and the two computers then talk to each other directly, +falling back on a TURN server when the network leaves them no other route. + +The two streams are independent. Sound may be asked for on its own, without any +picture, and a session already showing a screen may be replaced by one which also +carries sound. Each combination is a separate question to the user of the +controlled computer, which is why a change of streams is a new session rather than +a renegotiation of the one in progress. This feature is optional at every level. A relay built without it, or started without `-screen-share`, simply forwards the messages like any other, and the clients then fail to establish anything. A client which does not announce -`screen_share` in its `telenvda_capabilities` is never asked to share anything. +`screen_share` in its `telenvda_capabilities` is never asked to share anything, +and one which does not announce `audio_share` is never asked for its sound. The +two are announced separately: a computer willing to show its screen may still +refuse to let everything it plays be heard. #### Relay capabilities @@ -575,7 +584,9 @@ credentials are derived from a secret the clients never see and expire after "type": { "const": "screen_share_request" }, "target": { "type": "integer" }, "origin": { "type": "integer" }, - "allow_input": { "type": "boolean", "description": "Whether the sender is willing to drive the mouse. The receiver still decides on its own." } + "allow_input": { "type": "boolean", "description": "Whether the sender is willing to drive the mouse. The receiver still decides on its own." }, + "want_video": { "type": "boolean", "description": "Whether the picture is asked for. Absent means true, which is what a client predating sound sharing asks for." }, + "want_audio": { "type": "boolean", "description": "Whether the sound the controlled computer plays is asked for. Absent means false." } }, "required": ["type", "target"] }, @@ -587,6 +598,8 @@ credentials are derived from a secret the clients never see and expire after "origin": { "type": "integer" }, "accepted": { "type": "boolean" }, "allow_input": { "type": "boolean", "description": "Whether mouse control was actually granted." }, + "video": { "type": "boolean", "description": "Whether the picture is actually being sent. Absent means true." }, + "audio": { "type": "boolean", "description": "Whether the sound is actually being sent. Absent means false." }, "reason": { "enum": ["declined", "busy", "unavailable"], "description": "Present when the request was refused." } }, "required": ["type", "target", "accepted"] @@ -643,14 +656,23 @@ Both ends send `webrtc_candidate` as routes are discovered, without waiting for the description exchange to complete. Either end may send `screen_share_stop`, and a session ends by itself when the relay connection drops. -`allow_input` never grants anything on its own. The controlling computer states -what it would like, but the controlled computer only ever grants what its own -configuration allows, and the answer says what was really granted. Only mouse -actions can be replayed this way: no keyboard input travels over this link. - -The pictures themselves are carried on a WebRTC data channel rather than a media -track, as still frames compressed to JPEG and split into chunks. That format is -private to the two helper programs and is not part of this protocol. +`allow_input`, `want_video` and `want_audio` never grant anything on their own. +The controlling computer states what it would like, but the controlled computer +only ever grants what its own configuration allows, and the answer says what was +really granted. A request whose every stream is refused is answered as a refusal, +with `unavailable`. Only mouse actions can be replayed this way: no keyboard input +travels over this link. + +Both streams are ordinary WebRTC media tracks, encoded and carried by the two +helper programs. The picture is video, and the sound is a stereo audio track taken +from the mix the controlled computer plays, not from a microphone. Which codecs +are used, and how they are configured, is a matter between the two helpers and is +not part of this protocol. + +A controlled computer cannot leave its own screen reader out of the sound it +sends: the mix it captures is the whole of what the machine plays. A controlling +computer which plays that sound should therefore stop announcing the speech the +same session forwards to it, or the remote screen reader is heard twice. ### Braille Support diff --git a/readme.md b/readme.md index 83a0b67..750e03f 100644 --- a/readme.md +++ b/readme.md @@ -26,6 +26,8 @@ version 2 or later. * Two remote screenshot workflows described below. * Optional peer to peer screen sharing of the controlled computer, with mouse control when its user allows it. +* Optional peer to peer sharing of the sound the controlled computer plays, + either alongside the picture or on its own. ## Installation @@ -219,6 +221,62 @@ no mouse event is applied before that question has been answered, and the answer only lasts for the session. No keyboard input ever travels over this link. +## Hearing the controlled computer + +The controlling computer can also hear what the controlled one plays. Press +**NVDA+Control+Shift+J** to start or stop it. What is sent is the sound of the +whole computer, as it comes out of its speakers: music, videos, alert sounds and +the other side of a call. Its own screen reader is left out, so what arrives is +the sound of the machine rather than a recording of NVDA talking over it. + +The sound can be used on its own, without watching anything. When no picture was +asked for, none is sent and none is encoded, so a sound only session costs a +fraction of what a shared screen costs. It can also be turned on while a screen +is already being watched, and the picture then continues. Because adding or +removing the sound changes what the controlled computer is being asked for, its +user is asked again. + +It travels the same way as the picture, directly between the two computers, in +stereo, and is never recorded at either end. It needs the same Chromium browser +and the same relay support as screen sharing, and both computers must run a +version of TeleNVDA that supports it. A computer running an older version is +simply never asked for its sound. + +Before any sound is sent, the controlled computer asks its user *Do you want to +share your sound? The controlling computer will hear everything this computer +plays, including calls and videos.* Nothing is sent before that question has +been answered, and the answer only lasts for the session. + +Leaving the screen reader out is done by a small helper shipped with the add-on, +which captures the mix of the whole machine minus one program. It needs Windows +11, or Windows 10 from version 21H2, since that is when Windows gained the +ability to exclude a program from a capture. The speech the controlled computer +reports separately then keeps being spoken as usual, through the relay, which is +both quicker than the sound stream and rendered with the synthesiser, voice and +rate of whoever is listening. + +On an older Windows, or if the helper cannot start, the browser captures the +whole mix as before, screen reader included. The speech that computer also +reports is then not spoken a second time, so it is heard as it really sounds +rather than announced twice a fraction of a second apart. Braille is unaffected +either way and keeps working throughout. + +If the link drops without ending the session, the sound stops but the speech is +handed back for as long as the interruption lasts, announced by *Sound +interrupted* and then *Connection restored*. Neither the sound nor the speech is +left silently missing. + +Two options are available in the add-on settings: + +* **Allow sending the sound of this computer, after confirmation**, which + refuses the request outright when cleared, on this computer only. Clearing it + does not affect screen sharing. +* **When the sound of the controlled computer already contains its own screen + reader, do not also speak what it reports**, which stops that screen reader + from being heard twice. It only has an effect when the far end could not leave + its screen reader out, which is why the condition is part of the wording. + Clear it to hear both. + ## Controlling the remote computer Press **NVDA+Alt+Tab** (Insert+Alt+Tab with the default NVDA key) to switch