Share the sound of the controlled computer

addon/globalPlugins/remoteClient/__init__.py

@@ -204,6 +204,7 @@ def __init__(self, *args, **kwargs):
 			self.script_screenshot,
 			self.script_screenshot_powershell,
 			self.script_toggle_screen_share,
+			self.script_toggle_remote_sound,
 			self.script_toggle_remote_mouse,
 		)
 		self.is_connect_dialog_open = False
@@ -1003,15 +1004,41 @@ def script_toggle_screen_share(self, gesture):
 			ui.message(_("Not connected."))
 			return
 		configuration.record_activity()
-		was_active = session.screen_share.active
+		# The keyboard follows the picture rather than the session: sound may well go on
+		# arriving after the screen has been dismissed, and typing blind into a machine
+		# nobody is watching is exactly what must not happen.
+		was_watching = session.screen_share.video_requested
 		ui.message(session.screen_share.toggle())
 		if session is not self.master_session:
 			return
-		if not was_active and session.screen_share.active:
+		if not was_watching and session.screen_share.video_requested:
 			self._take_control_for_screen_share(gesture)
-		elif was_active and not session.screen_share.active and self.screen_share_took_control:
+		elif was_watching and not session.screen_share.video_requested and self.screen_share_took_control:
 			self._switch_to_local_control()
 
+	@script(
+		# Translators: toggle remote sound gesture description
+		_("Plays or stops playing the sound of the controlled computer on this one"),
+		gesture="kb:control+shift+NVDA+j",
+		**speakOnDemand)
+	def script_toggle_remote_sound(self, gesture):
+		"""Start or stop hearing what the controlled computer plays.
+
+		Unlike the screen, sound is useful without anything to look at and without the
+		keyboard following it, so this neither takes control nor gives it back. It can be
+		turned on while a screen is already being watched, and the picture then stays.
+		"""
+		session = None
+		if self.master_session is not None and self._is_master_connected():
+			session = self.master_session
+		elif self.slave_session is not None and self._is_slave_connected():
+			session = self.slave_session
+		if session is None or session.screen_share is None:
+			ui.message(_("Not connected."))
+			return
+		configuration.record_activity()
+		ui.message(session.screen_share.toggle_audio())
+
 	def _take_control_for_screen_share(self, gesture):
 		"""Send the keyboard to the controlled computer now that its screen is requested."""
 		if self.sending_keys:

addon/globalPlugins/remoteClient/audio_bridge.py

@@ -0,0 +1,162 @@
+"""The native helper which captures the sound of this computer without NVDA in it.
+
+The browser can capture the system mix, but only all of it. What this computer plays
+includes NVDA, so the sound sent to the watching computer used to carry the remote
+NVDA speaking, which is why the forwarded speech had to be silenced while it flowed.
+
+Windows can do better. Since build 20348 an audio client can be activated in process
+loopback mode, capturing everything except one process tree. No web API exposes it,
+so a small native helper does the capture and hands the samples to the page over a
+loopback WebSocket, which turns them back into a WebRTC track.
+
+The helper repeats the protections of local_bridge: it binds to 127.0.0.1 alone, the
+port is chosen by the system for each session, every connection must carry the session
+token, and an Origin header that is not the page's own is refused. The token goes in
+through the standard input of the helper rather than its command line, which every
+other process on the machine can read.
+
+When any of this is unavailable - an older Windows, a missing helper, a refusal from
+the audio engine - is_available() answers no and the caller falls back to the browser
+capture, with the forwarded speech silenced as before.
+"""
+
+import base64
+import os
+import subprocess
+import sys
+import threading
+from logging import getLogger
+
+logger = getLogger("audio_bridge")
+
+#: Build which introduced AUDIOCLIENT_ACTIVATION_TYPE_PROCESS_LOOPBACK.
+MIN_WINDOWS_BUILD = 20348
+
+#: Where the helper sits, next to this module.
+HELPER_SUBDIR = "bin"
+HELPER_NAME = "nvda_audio_capture.exe"
+
+#: The helper prints its port as soon as it is listening. It has nothing to do
+#: beforehand, so waiting long would only hide a failure.
+START_TIMEOUT = 5.0
+
+TOKEN_BYTES = 32
+
+
+def helper_path():
+	"""Return the absolute path of the helper, or None when it is not there."""
+	path = os.path.join(os.path.abspath(os.path.dirname(__file__)), HELPER_SUBDIR, HELPER_NAME)
+	return path if os.path.isfile(path) else None
+
+
+def is_available():
+	"""Whether this computer can capture its sound with NVDA left out of it."""
+	try:
+		if sys.getwindowsversion().build < MIN_WINDOWS_BUILD:
+			return False
+	except Exception:
+		logger.debug("Unable to read the Windows build", exc_info=True)
+		return False
+	return helper_path() is not None
+
+
+def _make_token():
+	"""Return an unguessable token for one session, as local_bridge does."""
+	return base64.urlsafe_b64encode(os.urandom(TOKEN_BYTES)).decode("ascii").rstrip("=")
+
+
+class AudioBridge:
+	"""One run of the native helper, serving one page."""
+
+	def __init__(self):
+		self._process = None
+		self.url = None
+
+	@property
+	def running(self):
+		return self._process is not None and self._process.poll() is None
+
+	def start(self, origin):
+		"""Start capturing and return the address the page must connect to.
+
+		`origin` is the local origin the page is served from, and the only one the
+		helper will accept. Returns None when the helper could not be started, which
+		is never fatal: the caller falls back to the browser capture.
+		"""
+		if self.running:
+			return self.url
+		path = helper_path()
+		if path is None or not origin:
+			return None
+		token = _make_token()
+		try:
+			self._process = subprocess.Popen(
+				[
+					path,
+					"--serve",
+					"--origin",
+					origin,
+					# Excluding this very process is the whole point. The mode excludes
+					# the children too, so anything NVDA starts to speak is covered.
+					"--pid",
+					str(os.getpid()),
+				],
+				stdin=subprocess.PIPE,
+				stdout=subprocess.PIPE,
+				stderr=subprocess.DEVNULL,
+				creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0),
+			)
+			self._process.stdin.write((token + "\n").encode("ascii"))
+			self._process.stdin.flush()
+		except Exception:
+			logger.exception("Unable to start the audio capture helper")
+			self.stop()
+			return None
+
+		port = self._read_port()
+		if port is None:
+			logger.error("The audio capture helper did not report a port")
+			self.stop()
+			return None
+		self.url = "ws://127.0.0.1:%d/?token=%s" % (port, token)
+		logger.debug("Audio capture helper listening on port %d", port)
+		return self.url
+
+	def _read_port(self):
+		"""Read the "PORT n" line the helper prints once it listens."""
+		result = {}
+
+		def read():
+			try:
+				result["line"] = self._process.stdout.readline().decode("ascii", "replace")
+			except Exception:
+				logger.debug("Unable to read from the audio capture helper", exc_info=True)
+
+		reader = threading.Thread(target=read, name="audio_bridge_start", daemon=True)
+		reader.start()
+		reader.join(START_TIMEOUT)
+		line = (result.get("line") or "").strip()
+		if not line.startswith("PORT "):
+			return None
+		try:
+			return int(line.split()[1])
+		except (IndexError, ValueError):
+			return None
+
+	def stop(self):
+		"""Stop capturing. Safe to call at any point, including twice."""
+		process, self._process = self._process, None
+		self.url = None
+		if process is None:
+			return
+		try:
+			if process.poll() is None:
+				process.terminate()
+		except Exception:
+			logger.debug("Unable to stop the audio capture helper", exc_info=True)
+		for stream in (process.stdin, process.stdout):
+			try:
+				if stream is not None:
+					stream.close()
+			except Exception:
+				pass

addon/globalPlugins/remoteClient/capabilities.py

@@ -32,6 +32,13 @@
 #: WebRTC screen sharing of the controlled computer, with optional mouse control.
 FEATURE_SCREEN_SHARE = "screen_share"
 
+#: Streaming of the sound the controlled computer plays, carried as an audio track
+#: on the same WebRTC session as screen sharing. Announced separately from
+#: FEATURE_SCREEN_SHARE, because a computer willing to show its screen may still
+#: refuse to let everything it plays be heard, and the reverse is just as useful:
+#: the sound can be sent on its own, without any picture.
+FEATURE_AUDIO_SHARE = "audio_share"
+
 #: Optional features implemented by this build.
 LOCAL_FEATURES = (FEATURE_CHUNKED_FILE_TRANSFER,)
 
@@ -52,6 +59,8 @@ def available_features():
 	from . import screen_share
 	if screen_share.is_available():
 		features.append(FEATURE_SCREEN_SHARE)
+		if screen_share.is_audio_share_allowed():
+			features.append(FEATURE_AUDIO_SHARE)
 	return features
 
 

addon/globalPlugins/remoteClient/configuration.py

@@ -88,6 +88,8 @@
 	max_fps = integer(default=15)
 	max_width = integer(default=1600)
 	quality = option("low", "balanced", "high", default="balanced")
+	share_audio = boolean(default=True)
+	mute_remote_speech_with_audio = boolean(default=True)
 
 [keep_awake]
 	enabled = boolean(default=True)

addon/globalPlugins/remoteClient/dialogs.py

@@ -637,6 +637,12 @@ def makeSettings(self, sizer):
 			_("High, for a fast connection"),
 		])
 		sizer.Add(self.screen_share_quality)
+		# Translators: A checkbox in add-on options dialog to allow the controlling computer to hear this one.
+		self.screen_share_audio = wx.CheckBox(self, wx.ID_ANY, label=_("Allow sending the sound of this computer, after confirmation"))
+		sizer.Add(self.screen_share_audio)
+		# Translators: A checkbox in add-on options dialog to stop speaking what the controlled computer reports when the sound it sends already contains its own screen reader.
+		self.screen_share_mute_speech = wx.CheckBox(self, wx.ID_ANY, label=_("When the sound of the controlled computer already contains its own screen reader, do not also speak what it reports"))
+		sizer.Add(self.screen_share_mute_speech)
 		# Translators: a text field in add-on options dialog to set the portcheck service URL
 		sizer.Add(wx.StaticText(self, wx.ID_ANY, label=_("Portcheck &service URL: ")))
 		self.portcheck = wx.TextCtrl(self, wx.ID_ANY)
@@ -675,6 +681,8 @@ def _update_screen_share_controls(self):
 		self.screen_share_max_fps.Enable(enabled)
 		self.screen_share_max_width.Enable(enabled)
 		self.screen_share_quality.Enable(enabled)
+		self.screen_share_audio.Enable(enabled)
+		self.screen_share_mute_speech.Enable(enabled)
 
 	def on_autoconnect(self, evt):
 		if self.autoconnect.GetValue() and not self._autoconnect_was_enabled:
@@ -794,6 +802,8 @@ def onPanelActivated(self):
 		self.max_received_size.SetValue(int(file_transfer_section['max_received_size_mb']))
 		self.screen_share_enabled.SetValue(bool(config['screen_share']['enabled']))
 		self.screen_share_max_fps.SetValue(int(config['screen_share']['max_fps']))
+		self.screen_share_audio.SetValue(bool(config['screen_share']['share_audio']))
+		self.screen_share_mute_speech.SetValue(bool(config['screen_share']['mute_remote_speech_with_audio']))
 		max_width = int(config['screen_share']['max_width'])
 		if max_width not in SCREEN_SHARE_WIDTHS:
 			max_width = SCREEN_SHARE_DEFAULT_WIDTH
@@ -923,6 +933,8 @@ def onSave(self):
 		config['file_transfer']['max_received_size_mb'] = int(self.max_received_size.GetValue())
 		config['screen_share']['enabled'] = self.screen_share_enabled.GetValue()
 		config['screen_share']['max_fps'] = int(self.screen_share_max_fps.GetValue())
+		config['screen_share']['share_audio'] = self.screen_share_audio.GetValue()
+		config['screen_share']['mute_remote_speech_with_audio'] = self.screen_share_mute_speech.GetValue()
 		config['screen_share']['max_width'] = SCREEN_SHARE_WIDTHS[self.screen_share_max_width.GetSelection()]
 		config['screen_share']['quality'] = SCREEN_SHARE_QUALITIES[self.screen_share_quality.GetSelection()]
 		config['updates']['check_at_startup'] = self.check_updates.GetValue()

addon/globalPlugins/remoteClient/edge.py

@@ -133,6 +133,10 @@ def _build_arguments(browser, url, profile, off_screen):
 		# See the module docstring: this is what removes the source picker, and the
 		# reason the profile above is throwaway.
 		"--use-fake-ui-for-media-stream",
+		# The picture is displayed muted, but the sound of the controlled computer has
+		# to play by itself: there is nobody to click in a window kept off screen, and
+		# the watching user may not be able to see it at all.
+		"--autoplay-policy=no-user-gesture-required",
 		# Not for the average frame rate, which holds without them, but to suppress
 		# the freezes measured on a window that is not on screen.
 		"--disable-background-timer-throttling",

addon/globalPlugins/remoteClient/edge_engine.py

@@ -61,6 +61,15 @@ def running(self):
 		# The bridge, not the browser process: see LocalBridge.silence.
 		return self._bridge.running
 
+	@property
+	def origin(self):
+		"""The local origin the page is served from, None while nothing is running.
+
+		The audio capture helper needs it: it is the only origin it will accept a
+		connection from.
+		"""
+		return self._bridge.origin
+
 	def start(self, role):
 		"""Open the browser on the signalling page. Raises RuntimeError when unusable."""
 		if self.running:

addon/globalPlugins/remoteClient/local_machine.py

@@ -59,6 +59,12 @@ def setSpeechCancelledToFalse():
 class LocalMachine:
 	def __init__(self):
 		self.is_muted = False
+		# True while the sound of the controlled computer is streamed to this one.
+		# What that computer plays already includes its own NVDA, so announcing the
+		# speech it also forwards would say everything twice, a fraction of a second
+		# apart. Braille is deliberately left alone: it is not audible, and losing it
+		# would cost a braille user their only view of the remote machine.
+		self.audio_streaming = False
 		self.receiving_braille = False
 		self._cached_sizes = None
 		if buildVersion.version_year >= 2023:
@@ -70,15 +76,15 @@ def terminate(self):
 
 	def play_wave(self, fileName):
 		"""Instructed by remote machine to play a wave file."""
-		if self.is_muted:
+		if self.is_muted or self.audio_streaming:
 			return
 		if os.path.exists(fileName):
 			# ignore async / asynchronous from kwargs:
 			# playWaveFile should play asynchronously from TeleNVDA.
 			nvwave.playWaveFile(fileName=fileName, asynchronous=True)
 
 	def beep(self, hz, length, left, right, **kwargs):
-		if self.is_muted:
+		if self.is_muted or self.audio_streaming:
 			return
 		tones.beep(hz, length, left, right)
 
@@ -98,7 +104,7 @@ def speak(
 		priority=speech.priorities.Spri.NORMAL,
 		**kwargs,
 	):
-		if self.is_muted:
+		if self.is_muted or self.audio_streaming:
 			return
 		setSpeechCancelledToFalse()
 		if not configuration.get_config()["ui"]["allow_speech_commands"]: