diff --git a/src-tauri/crates/berd-call/PROTOCOL.md b/src-tauri/crates/berd-call/PROTOCOL.md index e1b7b7e62..d38454cab 100644 --- a/src-tauri/crates/berd-call/PROTOCOL.md +++ b/src-tauri/crates/berd-call/PROTOCOL.md @@ -72,6 +72,14 @@ OpenAI snapshots contain `model`, `voice`, and `rate`; Siri contains `voice`, and `rate`. Credentials, endpoints, and bundle paths never appear on stdout. Detailed backend errors are diagnostics on stderr only; protocol rejection and fatal messages are sanitized at the stdout boundary. +`session.tts` describes the voice that speaks replies, and `set_tts_settings` +changes it: in Expert-Spokesperson sessions, that is the configured TTS backend +with `--allow-direct-agent-speech` and otherwise the Spokesperson's own voice. With +`--allow-direct-agent-speech`, `session.spokesperson` is the Spokesperson's own OpenAI +snapshot, with its own revision, and `set_spokesperson_settings` changes it the +way `set_tts_settings` changes the Spokesperson's voice in other sessions. A reply in +the Expert's own voice reports the same `admitted`, `speech_completed`, +`speech_interrupted`, and `speech_failed` lifecycle as a conventional reply. `session.input_during_tts` is the authoritative effective assistant-input policy and has its own revision. Status-sound settings are per-update parameters, not session snapshot configuration. The runtime owns cue cadence and playback; it @@ -195,9 +203,10 @@ it never admits a replacement while old host audio may still be active. {"type":"set_input_muted","id":u64,"active":bool} {"type":"set_conversation_status","id":u64,"status":"working"|"waiting","settings":StatusSoundSettings} {"type":"set_tts_settings","id":u64,"expected_revision":u64,"settings":TtsSettings} +{"type":"set_spokesperson_settings","id":u64,"expected_revision":u64,"settings":TtsSettings} {"type":"set_input_during_tts","id":u64,"expected_revision":u64,"policy":"allow_barge_in"|"suppress_input"} {"type":"reset_input","id":u64} -{"type":"prepare_speak","id":u64,"acknowledgement":u64|null,"text":string} +{"type":"prepare_speak","id":u64,"acknowledgement":u64|null,"text":string,"resolved_handoff_ids":[string]?,"voice":string?,"language":string?,"relay":bool?} {"type":"output_ready","id":u64,"speech_id":u64} audio acknowledgements listed above {"type":"query_state","id":u64,"after":u64} @@ -207,6 +216,19 @@ audio acknowledgements listed above ``` Unknown fields are rejected. IDs are positive. Speak text is at most 16 KiB. +`prepare_speak.voice` speaks that one utterance in another voice of the reply +backend, keeping its other settings; `language` changes a Siri voice's locale +too. A voice the backend doesn't offer, or a voice for a call whose replies +are spoken by the Spokesperson, is `not_admitted(invalid_voice)`. In a call +whose replies have their own voice, a reply needs the same `acknowledgement` +as a directive. With `relay: true` (what +`berd-call message-spokesperson` sends), the reply is a directive in any +Expert-Spokesperson call, even one whose replies have their own voice: the +Spokesperson says it, on the same `acknowledgement` and `resolved_handoff_ids` +terms. A call whose replies have their own voice holds no handoff open: the +Expert still hears each one, but `unresolved_handoff_ids` stays empty, no +reminder or renewal waits on them, and naming one in `resolved_handoff_ids` is +`not_admitted(invalid_handoff)`. The parent cannot author speaking state or finalized input; those are derived only from PCM by the child runtime. @@ -241,6 +263,11 @@ linearization point. A speech reservation holds a configuration lease: speech admitted before the response retains its old backend/settings, while later admission receives the new revision. Pocket's public model identifier cannot be changed without selecting and validating another bundle at process startup. +In a `berd-call start` realtime call, a Spokesperson voice or rate change that +the conversation overtakes while its new session is being built is not +rejected: the candidate is discarded and the change is tried again, with the +conversation as it then stands, once the call is idle. Its one result comes +when it finally applies or fails. `set_input_during_tts` changes the policy for later admissions. Its expected revision must be positive. The correlated result is always authoritative: @@ -281,6 +308,21 @@ Expert-Spokesperson sessions use `origin` to distinguish user transcripts, Spokesperson transcripts, and handoffs. User finals interrupt reserved or playing assistant output only after storage and emission. Final text is at most 64 KiB. +Expert-Spokesperson sessions also emit two previews for hosts that show the +conversation as it happens: + +```text +{"type":"user_transcribed","token":u64,"text":string} +{"type":"spokesperson_generated","text":string} +``` + +`user_transcribed` follows a user final's `live_event` with its token and plain +text. `spokesperson_generated` carries a reply's whole text once the provider +finishes generating it, before playback, and only after no user speech is still +being transcribed. A reply already spoken or cut off by then gets none. The +Spokesperson transcript recorded after playback stays authoritative. Neither +message changes Expert delivery. + ## Confirmation and admission `acknowledgement` is a request-local causal cutoff. `null` uses the stored @@ -324,7 +366,7 @@ cancellation and shutdown terminals. ```text {"type":"pending","id":u64,"utterances":[{"token":u64,"text":string}]} -{"type":"not_admitted","id":u64,"reason":"paused"|"in_progress"|"cancelled"|"empty_text"} +{"type":"not_admitted","id":u64,"reason":"paused"|"in_progress"|"cancelled"|"empty_text"|"invalid_handoff"|"invalid_voice"} {"type":"state","id":u64,"confirmed_token":u64,"utterances_after":[{"token":u64,"text":string}]} {"type":"cancel_result","id":u64,"outcome":"cancelled"|"stale","speech_id":u64|null} {"type":"speech_started","id":u64,"speech_id":u64} diff --git a/src-tauri/crates/berd-call/README.md b/src-tauri/crates/berd-call/README.md index 6eb5cd441..c372cd788 100644 --- a/src-tauri/crates/berd-call/README.md +++ b/src-tauri/crates/berd-call/README.md @@ -57,14 +57,25 @@ Start a call with the same backend flags accepted by `session`: berd-call start --voice Aaron --language en-US --stream berd-call status berd-call speak --re 7 "I found the cause." +berd-call speak --voice Nora "Sol here: the tests pass." berd-call stop ``` +`speak --voice` speaks one reply in another voice of the call's reply backend +without changing the call's voice, so agents sharing a call can sound distinct. + `start` stays in the foreground and owns the child session process. The control server listens only on `127.0.0.1:5222` by default; all four commands accept `--port` when more than one local call must be addressed. `--stream` writes one TSV row per delivered transcript event after the `cursor`, `role`, and `text` -header. The standalone host intentionally uses only the default devices. It +header. In Expert-Spokesperson mode the stream hears the call as Berd's Expert +does: a batch of every row since the last one after each Spokesperson reply, +and at each handoff, ending with the `handoff` row. `--stream live` instead shows the conversation as it happens: a +`user` row as soon as the words are transcribed, and a `spokesperson_generated` +row with cursor `0` as soon as a reply is generated, before it is spoken. The +`spokesperson` or `spokesperson_interrupted` row after playback carries the +reply's cursor and what was heard. A conventional call has no handoffs, so it +always streams each row. A `--codex` task gets the same batches as `--stream`. The standalone host intentionally uses only the default devices. It suppresses recognition during playback when the default output identifies itself as built-in speakers, preventing the call from transcribing its own speech; headphone-like outputs retain barge-in. @@ -98,6 +109,11 @@ complete `berd-call session` options. The call keeps its control port, stream, blocking policy, mute state, and input-during-TTS policy; the stream emits a lifecycle row because transcript cursors restart with the new session. +`berd-call settings --spokesperson true|false`, or Spokesperson in the menu bar, +restarts the call the same way in the other mode. Agents keep their current +voice, options only the other mode takes are left out, and in a `--stream live` +call the Spokesperson comes back with `--allow-direct-agent-speech`. + This crate owns the neutral PCM output contract and backend-neutral TTS stream used by Berd, plus the April ONNX runtime and text chunking used by Berd's native voice commands. It also owns the concrete Parakeet model loader and complete @@ -133,11 +149,14 @@ berd-call session --tts-backend pocket --model-dir /path/to/native-voice-v2 --vo berd-call session --stt-backend parakeet --stt-model-dir /path/to/parakeet berd-call session --stt-backend openai berd-call session --tts-backend openai --mode expert-spokesperson --realtime-url ws://127.0.0.1:18870/v1/realtime +berd-call session --voice Aaron --language en-US --mode expert-spokesperson --allow-direct-agent-speech berd-call session --tts-backend openai --tts-url https://proxy.example/v1/audio/speech berd-call session --stt-backend openai --stt-url wss://proxy.example/v1/realtime?intent=transcription ``` -The three URL flags take full endpoints and override only their matching service for that call. Expert–Spokesperson uses `--realtime-url` for both input and output; `--stt-url` and `--tts-url` apply only to conventional mode. Without an override, each service uses its OpenAI endpoint (or its existing environment override). `OPENAI_API_KEY` supplies the CLI credential; the desktop app stores keys separately by endpoint URL in Keychain. +In an Expert–Spokesperson call, the Expert reaches you through the Spokesperson: `berd-call message-spokesperson --re CURSOR [--resolves HANDOFF_ID]... TEXT` has it tell you in its own voice. Like the directives of Berd's Expert, the message must acknowledge the latest input (the Spokesperson's own speech doesn't count), or it comes back `spoke:false` with what was missed, and a handoff it names settles once the Spokesperson has said it. `speak` is refused unless the call was started with `--allow-direct-agent-speech`. With it, the Expert can also speak for itself: berd-call plays each `speak` reply through the configured TTS backend, taking turns with the Spokesperson's audio, and your speech interrupts either of them. Afterwards the Spokesperson gets a silent note of what you heard, so it can follow up without repeating it. A reply has to acknowledge everything you've said, as `speak --re` always does. Such a call holds no handoff open: each still reaches the Expert as a `handoff` row, but nothing waits for an answer, reminds the Expert, or holds back renewing or recovering the Spokesperson's session, and there is nothing for `--resolves` to name. `--voice`, `--rate`, `status` and `settings --tts` describe and change the Expert's voice. The Spokesperson keeps its own voice, shown as `session.spokesperson`: choose it with `--spokesperson-voice NAME`, at `start` or live with `settings`, and the menu bar offers both voices. It speaks as close to the agents' `--rate` as it can (0.25–1.5×), so the menu bar's one Speech Rate sets both; `--spokesperson-rate` gives it a speed of its own. Saved settings keep the Spokesperson's voice for the next call whose replies have their own, but not `--allow-direct-agent-speech`, so pass that on each `start`. + +The three URL flags take full endpoints and override only their matching service for that call. Expert–Spokesperson uses `--realtime-url` for both input and output; `--stt-url` applies only to conventional mode, and `--tts-url` also applies to the Expert's own voice. Without an override, each service uses its OpenAI endpoint (or its existing environment override). `OPENAI_API_KEY` supplies the CLI credential; the desktop app stores keys separately by endpoint URL in Keychain. The default Siri backend still requires an exact installed voice name and language. Missing or unavailable Siri voice configuration and an unavailable @@ -254,7 +273,9 @@ retries cleanup. This avoids reporting a failed install after the model has already been applied. `voices.list` returns `backend`, normalized `languageFilter`, -`availableLanguages`, and exact voice records. `voices.download` returns the +`availableLanguages`, and exact voice records. A voice is `installed` only when +sirittsd reports it downloaded and its assets are on disk: sirittsd can claim +a voice whose assets are missing, then silently speak it in another voice. `voices.download` returns the canonical voice, `installed: true`, and `availabilityWaitSeconds`; a missing exact catalog identity fails with `voice_not_found` before any native download request. macOS status and install results contain `supported`, `locale`, diff --git a/src-tauri/crates/berd-call/native/siri_tts_bridge.m b/src-tauri/crates/berd-call/native/siri_tts_bridge.m index ce06b7a13..1f73dc5da 100644 --- a/src-tauri/crates/berd-call/native/siri_tts_bridge.m +++ b/src-tauri/crates/berd-call/native/siri_tts_bridge.m @@ -997,10 +997,14 @@ static void BerdDownloadedVoices( NSString *name = [resource valueForKey:@"name"] ?: @""; if (!name.length || !language.length) continue; NSString *key = [NSString stringWithFormat:@"%@|%@", name, normalizedLanguage]; + // sirittsd can report a voice as downloaded whose assets aren't on disk, + // and then speaks it in another voice. + NSString *content = [resource valueForKey:@"contentPath"]; byKey[key] = @{ @"name" : name, @"language" : language, @"sizeBytes" : [resource valueForKey:@"assetSize"] ?: @0, + @"content" : @(content.length && [NSFileManager.defaultManager fileExistsAtPath:content]), }; } return [[byKey allValues] sortedArrayUsingComparator: @@ -1142,7 +1146,9 @@ static BOOL BerdTriggerDownload(NSString *language, NSError **error) { NSString *key = [NSString stringWithFormat:@"%@|%@", candidate[@"name"], normalizedLanguage]; NSMutableDictionary *voice = [candidate mutableCopy]; - voice[@"installed"] = installedSnapshot[key] ?: @NO; + [voice removeObjectForKey:@"content"]; + voice[@"installed"] = + [installedSnapshot[key] boolValue] && [candidate[@"content"] boolValue] ? @YES : @NO; [result addObject:voice]; } NSData *json = [NSJSONSerialization dataWithJSONObject:result options:0 error:&error]; diff --git a/src-tauri/crates/berd-call/src/cli_help.rs b/src-tauri/crates/berd-call/src/cli_help.rs index 157e0a26d..bef26a44d 100644 --- a/src-tauri/crates/berd-call/src/cli_help.rs +++ b/src-tauri/crates/berd-call/src/cli_help.rs @@ -8,6 +8,7 @@ Usage: Call runtime: start Run a microphone-backed local voice call speak Speak through the active local call + message-spokesperson Have the Spokesperson tell the user status Inspect the active local call catch-up Read finalized input after a cursor wait-for-input Wait for finalized input after a cursor @@ -33,14 +34,17 @@ Run `berd-call help ` for command-specific usage."#; const START_HELP: &str = r#"Run a local voice call backed by the shared session runtime. Usage: - berd-call start [--port PORT] [--stream | --codex] [--non-blocking] + berd-call start [--port PORT] [--stream [live] | --codex] [--non-blocking] [--no-menu-bar] --voice NAME --language BCP47 [session options] The foreground process owns the default microphone and audio output. Session options are the same as `berd-call session`, except the PCM descriptor is owned -internally. `--stream` prints cursor, role, and text as TSV. `--codex` delivers -the same records to the calling Codex task (requires CODEX_THREAD_ID). A +internally. `--stream` prints cursor, role, and text as TSV. In +Expert-Spokesperson mode it prints a batch after every Spokesperson reply and +at each handoff, as Berd's Expert hears the call; `--stream live` prints each +row as it happens. `--codex` delivers the same batches as `--stream` to the +calling Codex task (requires CODEX_THREAD_ID). A menu-bar item controls speech rate, input muting, and ending the call; pass `--no-menu-bar` to omit it. @@ -55,12 +59,30 @@ const SPEAK_HELP: &str = r#"Speak through the active local voice call. Usage: berd-call speak [--port PORT] [--re CURSOR] - [--resolves HANDOFF_ID]... TEXT + [--resolves HANDOFF_ID]... [--voice NAME [--language BCP47]] TEXT Speech waits for admission. The session setting controls whether it also waits for delivery. Non-blocking sessions require --stream or --codex and emit speech_result rows only for interruptions, failures, or pending user input. Interrupted results include -estimatedSpokenText, a best-effort prefix based on delivered audio."#; +estimatedSpokenText, a best-effort prefix based on delivered audio. --voice speaks +this text in another voice of the call's reply backend without changing its saved +voice, so several agents can share one call in distinct voices. An +Expert-Spokesperson call refuses speak unless started with +--allow-direct-agent-speech; otherwise use message-spokesperson. With it, a +reply acknowledging the latest input is spoken in the Expert's own voice, and +handoffs are never held open, so there is nothing for --resolves to name."#; + +const MESSAGE_SPOKESPERSON_HELP: &str = r#"Message the Spokesperson of the active realtime call. + +Usage: + berd-call message-spokesperson [--port PORT] --re CURSOR + [--resolves HANDOFF_ID]... TEXT + +The Spokesperson tells the user in its own voice, even in a call with +--allow-direct-agent-speech, where `speak` uses the Expert's. Like a directive +from Berd's Expert, the message must acknowledge the latest input (the +Spokesperson's own speech doesn't count), each --resolves settles that handoff +once the Spokesperson has said it, and the result is the same as `speak`'s."#; const SETTINGS_HELP: &str = r#"Update settings for the active local voice call. @@ -68,13 +90,24 @@ Usage: berd-call settings [--port PORT] --non-blocking true|false berd-call settings [--port PORT] --rate FLOAT berd-call settings [--port PORT] --voice NAME [--language BCP47] + berd-call settings [--port PORT] --spokesperson-voice NAME + berd-call settings [--port PORT] --spokesperson-rate FLOAT berd-call settings [--port PORT] --tts JSON berd-call settings [--port PORT] --input-during-tts allow|suppress berd-call settings [--port PORT] --muted true|false + berd-call settings [--port PORT] --spokesperson true|false berd-call settings [--port PORT] --restart [session options] -The setting applies to subsequent speak calls. Non-blocking delivery requires -a call started with --stream or --codex. Successful delivery produces no notification."#; +The setting applies to subsequent speak calls. In a call started with +--allow-direct-agent-speech, --voice and --rate change the agents' voice, and +--spokesperson-voice changes the Spokesperson's (an OpenAI realtime voice). +--rate also sets the Spokesperson's speed, as close as it can go (0.25 to +1.5); --spokesperson-rate sets only the Spokesperson's. --spokesperson turns the Spokesperson on +or off, restarting the call's session in the other mode with the same voice; +options only the other mode takes are dropped, and in a call started with +--stream live, agents keep their own voice beside the Spokesperson's. +Non-blocking delivery requires a call started with --stream or --codex. +Successful delivery produces no notification."#; const STATUS_HELP: &str = r#"Inspect the active local voice call. @@ -110,13 +143,20 @@ Usage: Each form also accepts: [--rate FLOAT] [--stt-backend macos|parakeet|openai] [--stt-model-dir PATH] - [--mode conventional|expert-spokesperson] + [--mode conventional|expert-spokesperson] [--allow-direct-agent-speech] + [--spokesperson-voice NAME] [--spokesperson-rate 0.25..1.5] [--realtime-url WS_URL] [--stt-url WS_URL] [--tts-url HTTP_URL] The host owns microphone capture, audio playback, transcript delivery, and agent integration. See PROTOCOL.md for the framed stdin, stdout, and PCM contracts. Endpoint URL flags override only the matching service for this call -and default to OpenAI when no environment override is set."#; +and default to OpenAI when no environment override is set. + +In expert-spokesperson mode, the Expert speaks through the Spokesperson. +--allow-direct-agent-speech also lets it speak to the user itself, through the +configured TTS backend, in its own voice beside the Spokesperson's. --spokesperson-voice then +chooses the Spokesperson's own voice. It speaks as close to --rate as it can, unless +--spokesperson-rate chooses its own speed."#; const SYNTHESIZE_HELP: &str = r#"Render text through a configured TTS backend into a new WAV file. @@ -195,6 +235,7 @@ fn help_for(topic: &[&str]) -> Option<&'static str> { [] => Some(TOP_LEVEL_HELP), ["start"] => Some(START_HELP), ["speak"] => Some(SPEAK_HELP), + ["message-spokesperson"] => Some(MESSAGE_SPOKESPERSON_HELP), ["status"] => Some(STATUS_HELP), ["catch-up"] | ["wait-for-input"] => Some(POLL_INPUT_HELP), ["settings"] => Some(SETTINGS_HELP), diff --git a/src-tauri/crates/berd-call/src/configured_tts.rs b/src-tauri/crates/berd-call/src/configured_tts.rs index eaab4d423..09d156b96 100644 --- a/src-tauri/crates/berd-call/src/configured_tts.rs +++ b/src-tauri/crates/berd-call/src/configured_tts.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::fmt; use std::path::PathBuf; use std::sync::{Arc, Mutex}; @@ -214,6 +215,33 @@ impl TtsConfiguration { } } + /// This configuration in another of its backend's voices. + fn with_voice(mut self, name: &str, locale: Option<&str>) -> Result { + match &mut self { + Self::Siri { + voice, language, .. + } => { + *voice = name.into(); + if let Some(locale) = locale { + *language = locale.into(); + } + } + Self::OpenAi { voice, .. } | Self::Pocket { voice, .. } => { + if locale.is_some() { + return Err("only Siri voices take a language".into()); + } + *voice = name.into(); + } + } + if let Self::OpenAi { voice, .. } = &self { + if !crate::openai_realtime_protocol::OPENAI_REALTIME_VOICE_IDS.contains(&voice.as_str()) + { + return Err(format!("OpenAI has no voice named {voice}")); + } + } + Ok(self) + } + fn build(&self) -> Result, String> { validate_settings(&self.settings())?; match self { @@ -291,6 +319,8 @@ struct ConfiguredTtsState { configuration: TtsConfiguration, backend: Arc, snapshot: TtsConfigurationSnapshot, + /// Backends for other voices, built for single utterances at this revision. + voices: HashMap<(String, Option), TtsConfigurationLease>, } pub struct ConfiguredTtsSlot { @@ -364,6 +394,7 @@ impl ConfiguredTtsSlot { revision: 1, settings, }, + voices: HashMap::new(), }), } } @@ -386,6 +417,46 @@ impl ConfiguredTtsSlot { }) } + /// Whether `lease_voice` already has this voice built. + pub fn has_voice(&self, voice: &str, language: Option<&str>) -> bool { + self.inner.lock().is_ok_and(|state| { + state + .voices + .contains_key(&(voice.to_string(), language.map(str::to_string))) + }) + } + + /// The configured backend speaking one utterance in another of its voices, + /// keeping the other settings. Building it doesn't hold the slot lock. + pub fn lease_voice( + &self, + voice: &str, + language: Option<&str>, + ) -> Result { + let key = (voice.to_string(), language.map(str::to_string)); + let poisoned = || "configured TTS lock was poisoned".to_string(); + let (configuration, revision) = { + let state = self.inner.lock().map_err(|_| poisoned())?; + if let Some(lease) = state.voices.get(&key) { + return Ok(lease.clone()); + } + (state.configuration.clone(), state.snapshot.revision) + }; + let configuration = configuration.with_voice(voice, language)?; + let lease = TtsConfigurationLease { + backend: configuration.build()?, + snapshot: TtsConfigurationSnapshot { + revision, + settings: configuration.settings(), + }, + }; + let mut state = self.inner.lock().map_err(|_| poisoned())?; + if state.snapshot.revision == revision { + state.voices.insert(key, lease.clone()); + } + Ok(lease) + } + /// Builds and validates a replacement without holding the slot lock. pub fn prepare_replacement( &self, @@ -475,6 +546,7 @@ impl ConfiguredTtsSlot { })?; state.configuration = replacement.configuration; state.backend = replacement.backend; + state.voices.clear(); state.snapshot = TtsConfigurationSnapshot { revision, settings: replacement.settings, @@ -554,6 +626,38 @@ mod tests { } } + #[test] + fn one_utterance_can_use_another_voice_without_changing_the_slot() { + let settings = |voice: &str| TtsSettings::OpenAi { + model: "gpt-4o-mini-tts".into(), + voice: voice.into(), + rate: 1.25, + }; + let slot = ConfiguredTtsSlot::new_for_test(settings("marin")); + let cedar = slot.lease_voice("cedar", None).unwrap(); + assert_eq!(cedar.snapshot().settings, settings("cedar")); + assert!(Arc::ptr_eq( + cedar.backend(), + slot.lease_voice("cedar", None).unwrap().backend() + )); + assert_eq!(slot.snapshot().unwrap().settings, settings("marin")); + assert!(slot + .lease_voice("nobody", None) + .err() + .unwrap() + .contains("no voice named nobody")); + assert!(slot + .lease_voice("cedar", Some("en-US")) + .err() + .unwrap() + .contains("language")); + slot.replace_for_test(1, settings("ash")).unwrap(); + assert!(!Arc::ptr_eq( + cedar.backend(), + slot.lease_voice("cedar", None).unwrap().backend() + )); + } + #[test] fn snapshot_is_sanitized_and_revisioned() { let slot = ConfiguredTtsSlot::new(TtsConfiguration::openai( diff --git a/src-tauri/crates/berd-call/src/host_control.rs b/src-tauri/crates/berd-call/src/host_control.rs index f3d56d714..c288c8e34 100644 --- a/src-tauri/crates/berd-call/src/host_control.rs +++ b/src-tauri/crates/berd-call/src/host_control.rs @@ -28,18 +28,24 @@ pub(crate) trait HostControl: Send + Sync + 'static { text: String, acknowledgement: Option, resolved_handoff_ids: Vec, + voice: Option, + language: Option, + relay: bool, ) -> Result; fn stop(&self) -> Result; fn set_non_blocking(&self, enabled: bool) -> Result; fn set_tts(&self, settings: berd_call::TtsSettings) -> Result; fn set_rate(&self, rate: f32) -> Result; fn set_voice(&self, voice: String, language: Option) -> Result; + fn set_spokesperson(&self, voice: Option, rate: Option) -> Result; fn set_input_during_tts( &self, policy: berd_call::input::InputDuringTtsPolicy, ) -> Result; fn set_muted(&self, muted: bool) -> Result; fn restart(&self, session_arguments: Vec) -> Result; + /// Restarts the call's session with the Spokesperson on or off. + fn set_spokesperson_mode(&self, on: bool) -> Result; } pub(crate) struct ControlServer { @@ -114,6 +120,12 @@ pub(crate) enum ControlRequest { acknowledgement: Option, #[serde(rename = "resolvedHandoffIds")] resolved_handoff_ids: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + voice: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + language: Option, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + relay: bool, }, Stop, NonBlocking { @@ -129,6 +141,11 @@ pub(crate) enum ControlRequest { voice: String, language: Option, }, + /// The Spokesperson's voice or speed, when replies have their own. + Spokesperson { + voice: Option, + rate: Option, + }, InputDuringTts { policy: berd_call::input::InputDuringTtsPolicy, }, @@ -139,6 +156,9 @@ pub(crate) enum ControlRequest { #[serde(rename = "sessionArguments")] session_arguments: Vec, }, + SpokespersonMode { + on: bool, + }, } #[derive(Deserialize, Serialize)] @@ -187,7 +207,9 @@ pub(crate) fn begin_request(port: u16, request: ControlRequest) -> Result Result text, acknowledgement, resolved_handoff_ids, + voice, + language, + relay, } => { - validate_speak(&text, &resolved_handoff_ids)?; - control.speak(text, acknowledgement, resolved_handoff_ids) + validate_speak( + &text, + &resolved_handoff_ids, + voice.as_deref(), + language.is_some(), + )?; + if relay && voice.is_some() { + return Err("a message to the Spokesperson takes no voice".into()); + } + control.speak( + text, + acknowledgement, + resolved_handoff_ids, + voice, + language, + relay, + ) } ControlRequest::Stop => control.stop(), ControlRequest::TtsSettings { settings } => control.set_tts(settings), ControlRequest::Rate { rate } => control.set_rate(rate), ControlRequest::Voice { voice, language } => control.set_voice(voice, language), + ControlRequest::Spokesperson { voice, rate } => { + control.set_spokesperson(voice, rate) + } ControlRequest::NonBlocking { enabled } => control.set_non_blocking(enabled), ControlRequest::InputDuringTts { policy } => control.set_input_during_tts(policy), ControlRequest::Muted { muted } => control.set_muted(muted), ControlRequest::Restart { session_arguments } => control.restart(session_arguments), + ControlRequest::SpokespersonMode { on } => control.set_spokesperson_mode(on), }, ); let response = match result { @@ -300,7 +344,15 @@ fn write_json_line( .map_err(|error| format!("could not write berd-call {label}: {error}")) } -fn validate_speak(text: &str, resolved_handoff_ids: &[String]) -> Result<(), String> { +fn validate_speak( + text: &str, + resolved_handoff_ids: &[String], + voice: Option<&str>, + language: bool, +) -> Result<(), String> { + if voice.is_some_and(|voice| voice.trim().is_empty()) || language && voice.is_none() { + return Err("speak --language requires a nonempty --voice".into()); + } if text.trim().is_empty() { return Err("speak text must not be empty".into()); } @@ -349,6 +401,13 @@ mod tests { fn set_voice(&self, voice: String, language: Option) -> Result { Ok(json!({"voice": voice, "language": language})) } + fn set_spokesperson( + &self, + voice: Option, + rate: Option, + ) -> Result { + Ok(json!({"spokesperson": {"voice": voice, "rate": rate}})) + } fn set_non_blocking(&self, enabled: bool) -> Result { Ok(json!({"nonBlocking": enabled})) } @@ -364,6 +423,9 @@ mod tests { fn restart(&self, session_arguments: Vec) -> Result { Ok(json!({"sessionArguments": session_arguments})) } + fn set_spokesperson_mode(&self, on: bool) -> Result { + Ok(json!({"spokesperson": on})) + } fn status(&self) -> Result { Ok(json!({"running": !self.stopped.load(Ordering::SeqCst)})) } @@ -373,11 +435,17 @@ mod tests { text: String, acknowledgement: Option, resolved_handoff_ids: Vec, + voice: Option, + language: Option, + relay: bool, ) -> Result { Ok(json!({ "spoken":text, "acknowledgement":acknowledgement, "resolvedHandoffIds":resolved_handoff_ids, + "voice":voice, + "language":language, + "relay":relay, })) } @@ -424,6 +492,9 @@ mod tests { text: "hello".into(), acknowledgement: Some(7), resolved_handoff_ids: vec!["call-1".into()], + voice: Some("Nora".into()), + language: Some("en-GB".into()), + relay: false, } ) .unwrap(), @@ -431,6 +502,9 @@ mod tests { "spoken":"hello", "acknowledgement":7, "resolvedHandoffIds":["call-1"], + "voice":"Nora", + "language":"en-GB", + "relay":false, }) ); assert_eq!( @@ -442,9 +516,12 @@ mod tests { #[test] fn speak_validation_rejects_invalid_values() { - assert!(validate_speak(" ", &[]).is_err()); - assert!(validate_speak(&"x".repeat(16 * 1024 + 1), &[]).is_err()); - assert!(validate_speak("hello", &["".into()]).is_err()); - assert!(validate_speak("hello", &["same".into(), "same".into()]).is_err()); + assert!(validate_speak(" ", &[], None, false).is_err()); + assert!(validate_speak(&"x".repeat(16 * 1024 + 1), &[], None, false).is_err()); + assert!(validate_speak("hello", &["".into()], None, false).is_err()); + assert!(validate_speak("hello", &["same".into(), "same".into()], None, false).is_err()); + assert!(validate_speak("hello", &[], None, true).is_err()); + assert!(validate_speak("hello", &[], Some(" "), false).is_err()); + assert!(validate_speak("hello", &[], Some("Nora"), true).is_ok()); } } diff --git a/src-tauri/crates/berd-call/src/host_session.rs b/src-tauri/crates/berd-call/src/host_session.rs index d827ae2fc..c358edb38 100644 --- a/src-tauri/crates/berd-call/src/host_session.rs +++ b/src-tauri/crates/berd-call/src/host_session.rs @@ -1,6 +1,6 @@ #![cfg(target_os = "macos")] -use std::collections::VecDeque; +use std::collections::{HashSet, VecDeque}; use std::fs::File; use std::io::{BufRead, BufReader, Read, Write}; use std::os::fd::{FromRawFd, RawFd}; @@ -145,7 +145,7 @@ pub(crate) fn run(options: StartOptions) -> Result<(), String> { /// Where live transcript records go: nowhere, stdout TSV, or a Codex task. pub(crate) enum Transcript { Silent, - Stdout, + Stdout { live: bool }, Codex(CodexRelay), } @@ -153,7 +153,7 @@ impl Transcript { fn for_options(options: &StartOptions) -> Result { Ok(match options.transcript { TranscriptDestination::None => Self::Silent, - TranscriptDestination::Stdout => Self::Stdout, + TranscriptDestination::Stdout { live } => Self::Stdout { live }, TranscriptDestination::Codex => { let target = CodexTarget::from_environment()?; let executable = std::env::current_exe().map_err(|error| { @@ -174,7 +174,7 @@ impl Transcript { fn start(&self) -> Result<(), String> { match self { - Self::Stdout => print_flushed("cursor\trole\ttext"), + Self::Stdout { .. } => print_flushed("cursor\trole\ttext"), Self::Codex(_) => { println!("Berd Call is delivering transcripts to this Codex task."); Ok(()) @@ -192,7 +192,7 @@ impl Transcript { ) -> Result<(), String> { match self { Self::Silent => Ok(()), - Self::Stdout => print_flushed(&format!( + Self::Stdout { .. } => print_flushed(&format!( "{}\t{role}\t{}", cursor.unwrap_or_default(), stream_text(text) @@ -213,7 +213,7 @@ impl Transcript { fn finish(&self, text: &str) { match self { Self::Silent => {} - Self::Stdout => { + Self::Stdout { .. } => { let _ = self.record(None, "lifecycle", None, text); } Self::Codex(relay) => relay.finish(Some(CodexRecord { @@ -342,7 +342,7 @@ impl SessionActor { SessionStart { saved: self.saved.take().map(|(path, mut saved)| { saved.arguments = saved_settings::persistable_arguments(&restart.arguments[1..]); - saved.tts = None; + saved.tts = restart.tts; (path, saved) }), arguments: restart.arguments, @@ -462,6 +462,10 @@ fn start_session( stop_requested: Arc::clone(stop_requested), muted: Arc::clone(&actor.muted), transcript: Arc::clone(transcript), + speaks: !expert_spokesperson + || arguments + .iter() + .any(|argument| argument == crate::DIRECT_AGENT_SPEECH), session_arguments: arguments, }); Ok(StartedSession { @@ -482,6 +486,43 @@ fn default_input_during_tts_policy() -> InputDuringTtsPolicy { } } +/// Session options for these speech settings, keeping a Pocket model directory from `arguments`. +fn tts_arguments(settings: &berd_call::TtsSettings, arguments: &[String]) -> Vec { + let pocket_model_dir = arguments + .chunks_exact(2) + .find(|pair| pair[0] == "--model-dir") + .map(|pair| pair[1].clone()); + let backend = match settings { + berd_call::TtsSettings::Siri { .. } => "siri", + berd_call::TtsSettings::Pocket { .. } => "pocket", + berd_call::TtsSettings::OpenAi { .. } => "openai", + }; + let mut replacement = vec![ + "--tts-backend".to_string(), + backend.to_string(), + "--rate".to_string(), + settings.rate().to_string(), + ]; + match settings { + berd_call::TtsSettings::Siri { + voice, language, .. + } => replacement.extend([ + "--voice".into(), + voice.clone(), + "--language".into(), + language.clone(), + ]), + berd_call::TtsSettings::Pocket { voice, .. } => { + if let Some(model_dir) = pocket_model_dir { + replacement.extend(["--model-dir".into(), model_dir]); + } + replacement.extend(["--voice".into(), voice.clone()]) + } + berd_call::TtsSettings::OpenAi { .. } => {} + } + replacement +} + #[derive(Clone)] struct ReadyState { session: Arc>, @@ -508,10 +549,18 @@ enum ControlCommand { expected_revision: u64, response: SyncSender>, }, + SpokespersonSettings { + settings: berd_call::TtsSettings, + expected_revision: u64, + response: SyncSender>, + }, Speak { text: String, acknowledgement: Option, resolved_handoff_ids: Vec, + voice: Option, + language: Option, + relay: bool, non_blocking: bool, response: SyncSender>, }, @@ -521,6 +570,8 @@ enum ControlCommand { Restart { arguments: Vec, expert_spokesperson: bool, + // Speech settings to restore in the new session, beyond what its options can say. + tts: Option, response: SyncSender>, }, } @@ -528,6 +579,7 @@ enum ControlCommand { struct PendingRestart { arguments: Vec, expert_spokesperson: bool, + tts: Option, response: SyncSender>, } @@ -539,10 +591,32 @@ struct SessionControl { stop_requested: Arc, muted: Arc, transcript: Arc, + // Whether the Expert may speak to the user rather than only through the Spokesperson. + speaks: bool, session_arguments: Vec, } impl SessionControl { + fn restart_with( + &self, + session_arguments: Vec, + tts: Option, + ) -> Result { + let expert_spokesperson = crate::validate_session_arguments(&session_arguments)?; + let (response, result) = mpsc::sync_channel(1); + self.commands + .send(ControlCommand::Restart { + arguments: session_arguments, + expert_spokesperson, + tts, + response, + }) + .map_err(|_| "voice call is not running")?; + result + .recv() + .map_err(|_| "voice call stopped during restart".to_string())? + } + fn request( &self, command: impl FnOnce(SyncSender>) -> ControlCommand, @@ -587,22 +661,56 @@ impl HostControl for SessionControl { } fn set_rate(&self, rate: f32) -> Result { - let (settings, expected_revision) = { + let (previous, expected_revision, spokesperson) = { let session = self .ready .session .lock() .map_err(|_| "session settings lock failed")?; + // The Spokesperson keeps as close to the agents' speed as it can. + let spokesperson = session.spokesperson.as_ref().map(|spokesperson| { + ( + spokesperson + .settings + .clone() + .with_rate(crate::spokesperson_speed(rate)), + spokesperson.revision, + ) + }); ( - session.tts.settings.clone().with_rate(rate), + session.tts.settings.clone(), session.tts.revision, + spokesperson, ) }; - self.request(|response| ControlCommand::TtsSettings { + let status = self.request(|response| ControlCommand::TtsSettings { + settings: previous.clone().with_rate(rate), + expected_revision, + response, + })?; + // A rate the agents' voice refuses leaves the Spokesperson's alone too. + let Some((settings, expected_revision)) = + spokesperson.filter(|_| status["outcome"] == "applied") + else { + return Ok(status); + }; + let result = self.request(|response| ControlCommand::SpokespersonSettings { settings, expected_revision, response, - }) + }); + // One the Spokesperson refuses puts the agents' back, so the voices never disagree. + if let (false, Some(expected_revision)) = ( + matches!(&result, Ok(value) if value["outcome"] == "applied"), + status["snapshot"]["revision"].as_u64(), + ) { + let _ = self.request(|response| ControlCommand::TtsSettings { + settings: previous, + expected_revision, + response, + }); + } + result } fn set_voice(&self, voice: String, language: Option) -> Result { @@ -644,6 +752,38 @@ impl HostControl for SessionControl { }) } + fn set_spokesperson(&self, voice: Option, rate: Option) -> Result { + let (mut settings, expected_revision) = { + let session = self + .ready + .session + .lock() + .map_err(|_| "session settings lock failed")?; + let spokesperson = session + .spokesperson + .as_ref() + .ok_or("this call has no separate Spokesperson voice; use --voice and --rate")?; + (spokesperson.settings.clone(), spokesperson.revision) + }; + if let ( + Some(voice), + berd_call::TtsSettings::OpenAi { + voice: selected, .. + }, + ) = (voice, &mut settings) + { + *selected = voice; + } + if let Some(rate) = rate { + settings = settings.with_rate(rate); + } + self.request(|response| ControlCommand::SpokespersonSettings { + settings, + expected_revision, + response, + }) + } + fn set_input_during_tts(&self, policy: InputDuringTtsPolicy) -> Result { let expected_revision = self .ready @@ -665,18 +805,45 @@ impl HostControl for SessionControl { } fn restart(&self, session_arguments: Vec) -> Result { - let expert_spokesperson = crate::validate_session_arguments(&session_arguments)?; - let (response, result) = mpsc::sync_channel(1); - self.commands - .send(ControlCommand::Restart { - arguments: session_arguments, - expert_spokesperson, - response, - }) - .map_err(|_| "voice call is not running")?; - result - .recv() - .map_err(|_| "voice call stopped during restart".to_string())? + self.restart_with(session_arguments, None) + } + + fn set_spokesperson_mode(&self, on: bool) -> Result { + let realtime = crate::validate_session_arguments(&self.session_arguments)?; + // Without the one flag that takes no value, so the rest pair up. + let arguments: Vec<_> = self.session_arguments[1..] + .iter() + .filter(|argument| *argument != crate::DIRECT_AGENT_SPEECH) + .cloned() + .collect(); + let arguments = &arguments[..]; + if on == realtime { + return self.status(); + } + let direct = matches!(*self.transcript, Transcript::Stdout { live: true }); + let live = self + .ready + .session + .lock() + .map_err(|_| "session settings lock failed")? + .tts + .settings + .clone(); + // Agents keep the voice they have now, if they speak before and after; otherwise the Spokesperson's voice is + // its own, and only their speed carries over. + let tts = (self.speaks && (!on || direct)).then(|| live.clone()); + let arguments = crate::saved_settings::merge_arguments( + arguments, + &match &tts { + Some(settings) => tts_arguments(settings, arguments), + None => vec!["--rate".into(), live.rate().to_string()], + }, + ); + let mut session_arguments = vec!["session".to_string()]; + session_arguments.extend(crate::saved_settings::with_spokesperson( + &arguments, on, direct, + )); + self.restart_with(session_arguments, tts) } fn set_non_blocking(&self, enabled: bool) -> Result { @@ -702,13 +869,25 @@ impl HostControl for SessionControl { text: String, acknowledgement: Option, resolved_handoff_ids: Vec, + voice: Option, + language: Option, + relay: bool, ) -> Result { + if !relay && !self.speaks { + return Err(format!( + "the Expert speaks through the Spokesperson: use message-spokesperson, or start the call with {}", + crate::DIRECT_AGENT_SPEECH + )); + } let (tx, rx) = mpsc::sync_channel(1); self.commands .send(ControlCommand::Speak { text, acknowledgement, resolved_handoff_ids, + voice, + language, + relay, non_blocking: self.non_blocking.load(Ordering::SeqCst), response: tx, }) @@ -752,6 +931,7 @@ enum AudioCommand { enum SavedPreference { SessionStart, Tts, + Spokesperson, InputPolicy, } @@ -767,10 +947,14 @@ struct SessionActor { next_id: u64, pending_speak: Option, pending_settings: std::collections::HashMap>>, + // Settings requests answered with the Spokesperson's voice rather than the call's. + spokesperson_settings: HashSet, waiting_speaks: VecDeque, session: Arc>, transcript: Arc, expert_spokesperson: bool, + // User rows already streamed when transcribed, left out of the Expert batch. + transcribed: HashSet, stopping: bool, restart: Option, muted: Arc, @@ -827,10 +1011,12 @@ impl SessionActor { next_id: 2, pending_speak: None, pending_settings: std::collections::HashMap::new(), + spokesperson_settings: HashSet::new(), waiting_speaks: VecDeque::new(), session, transcript, expert_spokesperson, + transcribed: HashSet::new(), stopping: false, restart: None, muted: Arc::new(AtomicBool::new(false)), @@ -1024,10 +1210,30 @@ impl SessionActor { )?; self.pending_settings.insert(id, response); } + ControlCommand::SpokespersonSettings { + settings, + expected_revision, + response, + } => { + let id = self.next_id(); + send_request( + &self.writer, + &SessionRequest::SetSpokespersonSettings { + id, + expected_revision, + settings, + }, + )?; + self.spokesperson_settings.insert(id); + self.pending_settings.insert(id, response); + } ControlCommand::Speak { text, acknowledgement, resolved_handoff_ids, + voice, + language, + relay, non_blocking, response, } => { @@ -1040,6 +1246,9 @@ impl SessionActor { text, acknowledgement, resolved_handoff_ids, + voice, + language, + relay, non_blocking, response, }); @@ -1053,6 +1262,9 @@ impl SessionActor { acknowledgement, text: text.clone(), resolved_handoff_ids, + voice, + language, + relay, }, )?; self.pending_speak = Some(PendingSpeak { @@ -1065,6 +1277,7 @@ impl SessionActor { ControlCommand::Restart { arguments, expert_spokesperson, + tts, response, } => { if self.stopping { @@ -1076,6 +1289,7 @@ impl SessionActor { self.restart = Some(PendingRestart { arguments, expert_spokesperson, + tts, response, }); } @@ -1145,13 +1359,23 @@ impl SessionActor { snapshot, message, } => { + let spokesperson = self.spokesperson_settings.remove(&id); if let Ok(mut session) = self.session.lock() { - if snapshot.revision >= session.tts.revision { - session.tts = snapshot.clone(); + let voice = if spokesperson { + session.spokesperson.as_mut() + } else { + Some(&mut session.tts) + }; + if let Some(voice) = voice.filter(|voice| snapshot.revision >= voice.revision) { + *voice = snapshot.clone(); } } if outcome == berd_call::protocol::TtsSettingsOutcome::Applied { - self.persist_settings(SavedPreference::Tts); + self.persist_settings(if spokesperson { + SavedPreference::Spokesperson + } else { + SavedPreference::Tts + }); } if let Some(response) = self.pending_settings.remove(&id) { let _ = response.send(Ok( @@ -1191,6 +1415,21 @@ impl SessionActor { text, origin, } if !self.expert_spokesperson => self.stream_live_event(token, origin, &text)?, + // Only a live stream shows the conversation as it happens. + SessionMessage::UserTranscribed { token, text } + if self.expert_spokesperson + && matches!(*self.transcript, Transcript::Stdout { live: true }) => + { + self.transcript.record(Some(token), "user", None, &text)?; + self.transcribed.insert(token); + } + SessionMessage::SpokespersonGenerated { text } + if self.expert_spokesperson + && matches!(*self.transcript, Transcript::Stdout { live: true }) => + { + self.transcript + .record(None, "spokesperson_generated", None, &text)? + } SessionMessage::ExpertDelivery { events, .. } if self.expert_spokesperson => { self.stream_expert_delivery(&events)? } @@ -1306,8 +1545,11 @@ impl SessionActor { self.transcript.record(Some(token), role, None, text) } - fn stream_expert_delivery(&self, events: &[RealtimeExpertDeliveryEvent]) -> Result<(), String> { - for event in events { + fn stream_expert_delivery( + &mut self, + events: &[RealtimeExpertDeliveryEvent], + ) -> Result<(), String> { + for event in unstreamed(events, &mut self.transcribed) { self.transcript.record( Some(event.cursor), expert_delivery_role_name(event.role), @@ -1350,41 +1592,28 @@ impl SessionActor { let Ok(session) = self.session.lock() else { return; }; - saved.tts = Some(session.tts.settings.clone()); - let settings = &session.tts.settings; - let pocket_model_dir = saved - .arguments - .chunks_exact(2) - .find(|pair| pair[0] == "--model-dir") - .map(|pair| pair[1].clone()); - let backend = match settings { - berd_call::TtsSettings::Siri { .. } => "siri", - berd_call::TtsSettings::Pocket { .. } => "pocket", - berd_call::TtsSettings::OpenAi { .. } => "openai", - }; - let mut replacement = vec![ - "--tts-backend".to_string(), - backend.to_string(), - "--rate".to_string(), - settings.rate().to_string(), - ]; - match settings { - berd_call::TtsSettings::Siri { - voice, language, .. - } => replacement.extend([ - "--voice".into(), - voice.clone(), - "--language".into(), - language.clone(), - ]), - berd_call::TtsSettings::Pocket { voice, .. } => { - if let Some(model_dir) = pocket_model_dir { - replacement.extend(["--model-dir".into(), model_dir]); - } - replacement.extend(["--voice".into(), voice.clone()]) + if let SavedPreference::Spokesperson = changed { + let Some(spokesperson) = &session.spokesperson else { + return; + }; + let replacement = vec![ + "--spokesperson-voice".to_string(), + spokesperson.settings.voice().to_string(), + "--spokesperson-rate".to_string(), + spokesperson.settings.rate().to_string(), + ]; + saved.arguments = + crate::saved_settings::merge_arguments(&saved.arguments, &replacement); + if let Err(error) = crate::saved_settings::update(path, |latest| { + latest.arguments = + crate::saved_settings::merge_arguments(&latest.arguments, &replacement) + }) { + eprintln!("berd-call: live settings applied but could not be saved: {error}"); } - berd_call::TtsSettings::OpenAi { .. } => {} + return; } + saved.tts = Some(session.tts.settings.clone()); + let replacement = tts_arguments(&session.tts.settings, &saved.arguments); saved.arguments = crate::saved_settings::merge_arguments(&saved.arguments, &replacement); if let Err(error) = crate::saved_settings::update(path, |latest| match changed { SavedPreference::SessionStart => { @@ -1397,6 +1626,7 @@ impl SessionActor { latest.tts = saved.tts.clone(); } SavedPreference::InputPolicy => latest.input_policy = saved.input_policy, + SavedPreference::Spokesperson => unreachable!("saved above"), }) { eprintln!("berd-call: live settings applied but could not be saved: {error}"); } @@ -2169,6 +2399,16 @@ impl AudioPlaybackState { } } + /// What the host played of this speech, once it has ended it. The session may not have heard yet, so its records for it can still arrive. + fn retired(&self, speech_id: u64) -> Option { + match self { + Self::Finished(retired) if retired.speech_id == speech_id => { + Some(retired.played_frames) + } + _ => None, + } + } + fn finish(&mut self, speech_id: u64, played_frames: u64) { *self = Self::Finished(RetiredAudio { speech_id, @@ -2383,6 +2623,9 @@ fn handle_audio_record( AUDIO_CHUNK_KIND if payload.len() >= 16 && (payload.len() - 16).is_multiple_of(4) => { let speech_id = le_u64(&payload[0..8])?; let sequence = le_u64(&payload[8..16])?; + if playback.retired(speech_id).is_some() { + return Ok(()); + } let active = playback .playing_mut(speech_id) .ok_or_else(|| "voice session sent audio for an inactive speech".to_string())?; @@ -2415,6 +2658,9 @@ fn handle_audio_record( AUDIO_END_KIND if payload.len() == 24 => { let speech_id = le_u64(&payload[0..8])?; let sequence = le_u64(&payload[8..16])?; + if playback.retired(speech_id).is_some() { + return Ok(()); + } let active = playback .playing_mut(speech_id) .ok_or_else(|| "voice session ended inactive audio".to_string())?; @@ -2423,6 +2669,15 @@ fn handle_audio_record( } AUDIO_CANCEL_KIND if payload.len() == 8 => { let speech_id = le_u64(payload)?; + if let Some(played_frames) = playback.retired(speech_id) { + return send_request( + writer, + &SessionRequest::AudioCancelled { + speech_id, + played_frames, + }, + ); + } let active = playback .playing_mut(speech_id) .ok_or_else(|| "voice session cancelled inactive audio".to_string())?; @@ -2534,9 +2789,20 @@ fn not_admitted_reason_name(reason: NotAdmittedReason) -> &'static str { NotAdmittedReason::Cancelled => "cancelled", NotAdmittedReason::EmptyText => "empty_text", NotAdmittedReason::InvalidHandoff => "invalid_handoff", + NotAdmittedReason::InvalidVoice => "invalid_voice", } } +/// The batch's events not already streamed as they happened. +fn unstreamed<'a>( + events: &'a [RealtimeExpertDeliveryEvent], + transcribed: &'a mut HashSet, +) -> impl Iterator { + events.iter().filter(move |event| { + !(event.role == RealtimeExpertDeliveryRole::User && transcribed.remove(&event.cursor)) + }) +} + fn expert_delivery_role_name(role: RealtimeExpertDeliveryRole) -> &'static str { match role { RealtimeExpertDeliveryRole::User => "user", @@ -2551,6 +2817,27 @@ fn expert_delivery_role_name(role: RealtimeExpertDeliveryRole) -> &'static str { mod tests { use super::*; + #[test] + fn expert_batches_leave_out_user_rows_already_streamed() { + let event = |cursor, role| RealtimeExpertDeliveryEvent { + cursor, + role, + text: format!("event {cursor}"), + handoff_id: None, + }; + let events = [ + event(1, RealtimeExpertDeliveryRole::User), + event(2, RealtimeExpertDeliveryRole::User), + event(3, RealtimeExpertDeliveryRole::Spokesperson), + ]; + let mut transcribed = HashSet::from([1, 3]); + let cursors: Vec<_> = unstreamed(&events, &mut transcribed) + .map(|event| event.cursor) + .collect(); + assert_eq!(cursors, [2, 3]); + assert_eq!(transcribed, HashSet::from([3])); + } + #[test] fn microphone_teardown_reports_a_blocked_forwarder() { let (_release, blocked) = mpsc::sync_channel::<()>(1); @@ -2674,6 +2961,7 @@ mod tests { PendingRestart { arguments: vec!["session".into()], expert_spokesperson: false, + tts: None, response, }, InputDuringTtsPolicy::AllowBargeIn, @@ -2728,6 +3016,7 @@ mod tests { PendingRestart { arguments: arguments.clone(), expert_spokesperson: false, + tts: None, response, }, InputDuringTtsPolicy::AllowBargeIn, @@ -2772,6 +3061,7 @@ mod tests { .handle_command(ControlCommand::Restart { arguments: vec!["session".into()], expert_spokesperson: false, + tts: None, response, }) .unwrap(); @@ -3161,6 +3451,9 @@ mod tests { text: "next".into(), acknowledgement: Some(7), resolved_handoff_ids: Vec::new(), + voice: None, + language: None, + relay: false, non_blocking: true, response: next_tx, }) @@ -3177,6 +3470,7 @@ mod tests { fn test_snapshot() -> VoiceSessionSnapshot { VoiceSessionSnapshot { + spokesperson: None, tts: berd_call::TtsConfigurationSnapshot { revision: 1, settings: berd_call::TtsSettings::Siri { @@ -3205,7 +3499,7 @@ mod tests { commands, audio_commands, session, - Arc::new(Transcript::Stdout), + Arc::new(Transcript::Stdout { live: false }), false, ) } @@ -3578,6 +3872,79 @@ mod tests { second_child.wait().unwrap(); } + #[test] + fn spokesperson_results_change_and_save_only_its_voice() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("settings.json"); + let mut child = Command::new("/bin/cat") + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .spawn() + .unwrap(); + let writer = Arc::new(Mutex::new(child.stdin.take().unwrap())); + let (_events, receiver) = mpsc::sync_channel(1); + let (_commands, commands) = mpsc::sync_channel(1); + let (audio, _audio_rx) = mpsc::sync_channel(1); + let mut actor = test_actor(writer, receiver, commands, audio); + let openai = |voice: &str, revision| berd_call::TtsConfigurationSnapshot { + revision, + settings: berd_call::TtsSettings::OpenAi { + model: "gpt-realtime".into(), + voice: voice.into(), + rate: 1.25, + }, + }; + actor.session.lock().unwrap().spokesperson = Some(openai("marin", 1)); + actor.saved = Some(( + path.clone(), + crate::saved_settings::SavedSettings { + arguments: ["--voice", "Aaron", "--language", "en-US"] + .map(str::to_string) + .to_vec(), + ..crate::saved_settings::SavedSettings::default() + }, + )); + let (response, result) = mpsc::sync_channel(1); + actor + .handle_command(ControlCommand::SpokespersonSettings { + settings: openai("cedar", 1).settings, + expected_revision: 1, + response, + }) + .unwrap(); + let id = *actor.spokesperson_settings.iter().next().unwrap(); + actor + .handle_event(SessionMessage::TtsSettingsResult { + id, + outcome: berd_call::protocol::TtsSettingsOutcome::Applied, + snapshot: openai("cedar", 2), + message: None, + }) + .unwrap(); + assert_eq!(result.recv().unwrap().unwrap()["outcome"], "applied"); + let session = actor.session.lock().unwrap().clone(); + assert_eq!(session.spokesperson, Some(openai("cedar", 2))); + assert_eq!(session.tts, test_snapshot().tts); + let saved = crate::saved_settings::load(&path).unwrap(); + assert_eq!(saved.tts, None); + assert_eq!( + saved.arguments, + [ + "--voice", + "Aaron", + "--language", + "en-US", + "--spokesperson-voice", + "cedar", + "--spokesperson-rate", + "1.25" + ] + .map(str::to_string) + ); + drop(actor); + child.wait().unwrap(); + } + #[test] fn pocket_tts_save_preserves_the_active_bundle_path() { let directory = tempfile::tempdir().unwrap(); @@ -4000,6 +4367,7 @@ mod tests { PendingRestart { arguments: vec!["session".into()], expert_spokesperson: false, + tts: None, response, }, InputDuringTtsPolicy::AllowBargeIn, @@ -4059,6 +4427,7 @@ mod tests { stop_requested: Arc::clone(&stop_requested), muted: Arc::new(AtomicBool::new(false)), transcript: Arc::new(Transcript::Silent), + speaks: true, session_arguments: vec!["session".into()], }; drop(receiver); @@ -4066,6 +4435,194 @@ mod tests { assert!(stop_requested.load(Ordering::SeqCst)); } + #[test] + fn turning_the_spokesperson_on_restarts_the_call_with_the_agents_current_voice() { + let toggle = |transcript, rate| { + let mut snapshot = test_snapshot(); + snapshot.tts.settings = snapshot.tts.settings.with_rate(rate); + let (commands, receiver) = mpsc::sync_channel(1); + let control = SessionControl { + commands, + running: Arc::new(AtomicBool::new(true)), + ready: ReadyState { + session: Arc::new(Mutex::new(snapshot)), + }, + non_blocking: Arc::new(AtomicBool::new(false)), + stop_requested: Arc::new(AtomicBool::new(false)), + muted: Arc::new(AtomicBool::new(false)), + transcript: Arc::new(transcript), + speaks: true, + session_arguments: [ + "session", + "--voice", + "Samantha", + "--language", + "en-GB", + "--rate", + "2", + "--stt-backend", + "openai", + "--stt-url", + "ws://127.0.0.1:1/stt", + ] + .map(str::to_string) + .to_vec(), + }; + let worker = thread::spawn(move || match receiver.recv().unwrap() { + ControlCommand::Restart { + arguments, + expert_spokesperson, + tts, + response, + } => { + response.send(Ok(json!({}))).unwrap(); + assert!(expert_spokesperson); + (arguments, tts) + } + _ => panic!("expected a restart"), + }); + control.set_spokesperson_mode(true).unwrap(); + worker.join().unwrap() + }; + // Agents with their own voice keep the one they have now. + let (arguments, tts) = toggle(Transcript::Stdout { live: true }, 1.25); + assert_eq!( + arguments, + [ + "session", + "--stt-backend", + "openai", + "--tts-backend", + "siri", + "--rate", + "1.25", + "--voice", + "Aaron", + "--language", + "en-US", + "--mode", + "expert-spokesperson", + crate::DIRECT_AGENT_SPEECH, + ] + ); + assert_eq!(tts, Some(test_snapshot().tts.settings.with_rate(1.25))); + // Otherwise only their speed carries over, which the Spokesperson keeps as close to as it can. + let (arguments, tts) = toggle(Transcript::Silent, 1.75); + assert_eq!( + arguments, + [ + "session", + "--voice", + "Samantha", + "--language", + "en-GB", + "--stt-backend", + "openai", + "--rate", + "1.75", + "--mode", + "expert-spokesperson", + ] + ); + assert_eq!(tts, None); + assert!(crate::validate_session_arguments(&arguments).unwrap()); + } + + #[test] + fn turning_the_spokesperson_off_drops_direct_speech_wherever_it_was_given() { + let (commands, receiver) = mpsc::sync_channel(1); + let control = SessionControl { + commands, + running: Arc::new(AtomicBool::new(true)), + ready: ReadyState { + session: Arc::new(Mutex::new(test_snapshot())), + }, + non_blocking: Arc::new(AtomicBool::new(false)), + stop_requested: Arc::new(AtomicBool::new(false)), + muted: Arc::new(AtomicBool::new(false)), + transcript: Arc::new(Transcript::Stdout { live: true }), + speaks: true, + session_arguments: [ + "session", + "--mode", + "expert-spokesperson", + crate::DIRECT_AGENT_SPEECH, + "--voice", + "Samantha", + "--language", + "en-GB", + ] + .map(str::to_string) + .to_vec(), + }; + let worker = thread::spawn(move || match receiver.recv().unwrap() { + ControlCommand::Restart { + arguments, + expert_spokesperson, + response, + .. + } => { + response.send(Ok(json!({}))).unwrap(); + assert!(!expert_spokesperson); + arguments + } + _ => panic!("expected a restart"), + }); + control.set_spokesperson_mode(false).unwrap(); + assert_eq!( + worker.join().unwrap(), + [ + "session", + "--tts-backend", + "siri", + "--rate", + "1", + "--voice", + "Aaron", + "--language", + "en-US", + "--mode", + "conventional", + ] + ); + } + + #[test] + fn the_expert_speaks_only_through_the_spokesperson_unless_allowed() { + let (commands, receiver) = mpsc::sync_channel(1); + let control = SessionControl { + commands, + running: Arc::new(AtomicBool::new(true)), + ready: ReadyState { + session: Arc::new(Mutex::new(test_snapshot())), + }, + non_blocking: Arc::new(AtomicBool::new(false)), + stop_requested: Arc::new(AtomicBool::new(false)), + muted: Arc::new(AtomicBool::new(false)), + transcript: Arc::new(Transcript::Silent), + speaks: false, + session_arguments: vec!["session".into()], + }; + let refusal = control + .speak("Hello".into(), None, Vec::new(), None, None, false) + .unwrap_err(); + assert!(refusal.contains("message-spokesperson"), "{refusal}"); + assert!(receiver.try_recv().is_err()); + let worker = thread::spawn(move || match receiver.recv().unwrap() { + ControlCommand::Speak { + relay, response, .. + } => { + response.send(Ok(json!({}))).unwrap(); + relay + } + _ => panic!("expected a message to the Spokesperson"), + }); + control + .speak("Hello".into(), None, Vec::new(), None, None, true) + .unwrap(); + assert!(worker.join().unwrap()); + } + #[test] fn voice_updates_preserve_the_latest_rate_and_revision() { let (commands, receiver) = mpsc::sync_channel(1); @@ -4081,6 +4638,7 @@ mod tests { stop_requested: Arc::new(AtomicBool::new(false)), muted: Arc::new(AtomicBool::new(false)), transcript: Arc::new(Transcript::Silent), + speaks: true, session_arguments: vec!["session".into()], }; let worker = thread::spawn(move || match receiver.recv().unwrap() { @@ -4123,6 +4681,7 @@ mod tests { stop_requested: Arc::new(AtomicBool::new(false)), muted: Arc::new(AtomicBool::new(false)), transcript: Arc::new(Transcript::Silent), + speaks: true, session_arguments: vec!["session".into()], }; let worker = thread::spawn(move || match receiver.recv().unwrap() { @@ -4150,6 +4709,135 @@ mod tests { ); } + #[test] + fn spokesperson_updates_keep_its_other_setting_and_revision() { + let (commands, receiver) = mpsc::sync_channel(1); + let mut snapshot = test_snapshot(); + let openai = |voice: &str, rate| berd_call::TtsSettings::OpenAi { + model: "gpt-realtime".into(), + voice: voice.into(), + rate, + }; + let session = Arc::new(Mutex::new(snapshot.clone())); + let control = SessionControl { + commands, + running: Arc::new(AtomicBool::new(true)), + ready: ReadyState { + session: Arc::clone(&session), + }, + non_blocking: Arc::new(AtomicBool::new(false)), + stop_requested: Arc::new(AtomicBool::new(false)), + muted: Arc::new(AtomicBool::new(false)), + transcript: Arc::new(Transcript::Silent), + speaks: true, + session_arguments: vec!["session".into()], + }; + let refusal = control.set_spokesperson(None, Some(1.5)).unwrap_err(); + assert!( + refusal.contains("no separate Spokesperson voice"), + "{refusal}" + ); + snapshot.spokesperson = Some(berd_call::TtsConfigurationSnapshot { + revision: 3, + settings: openai("marin", 1.25), + }); + *session.lock().unwrap() = snapshot; + let worker = thread::spawn(move || { + (0..2) + .map(|_| match receiver.recv().unwrap() { + ControlCommand::SpokespersonSettings { + settings, + expected_revision, + response, + } => { + response.send(Ok(json!({}))).unwrap(); + (settings, expected_revision) + } + _ => panic!("expected a Spokesperson settings command"), + }) + .collect::>() + }); + control.set_spokesperson(None, Some(1.5)).unwrap(); + control + .set_spokesperson(Some("cedar".into()), None) + .unwrap(); + assert_eq!( + worker.join().unwrap(), + [(openai("marin", 1.5), 3), (openai("cedar", 1.25), 3)] + ); + } + + #[test] + fn the_speech_rate_keeps_the_spokesperson_as_close_to_the_agents_as_it_can() { + let rates = |spokesperson, outcome: &'static str, spokesperson_outcome: &'static str| { + let (commands, receiver) = mpsc::sync_channel(1); + let mut snapshot = test_snapshot(); + snapshot.spokesperson = spokesperson; + let control = SessionControl { + commands, + running: Arc::new(AtomicBool::new(true)), + ready: ReadyState { + session: Arc::new(Mutex::new(snapshot)), + }, + non_blocking: Arc::new(AtomicBool::new(false)), + stop_requested: Arc::new(AtomicBool::new(false)), + muted: Arc::new(AtomicBool::new(false)), + transcript: Arc::new(Transcript::Silent), + speaks: true, + session_arguments: vec!["session".into()], + }; + let worker = thread::spawn(move || { + receiver + .iter() + .map(|command| match command { + ControlCommand::TtsSettings { + settings, response, .. + } => { + response + .send(Ok(json!({"outcome": outcome, "snapshot": {"revision": 2}}))) + .unwrap(); + ("agents", settings.rate()) + } + ControlCommand::SpokespersonSettings { + settings, response, .. + } => { + response + .send(Ok(json!({"outcome": spokesperson_outcome}))) + .unwrap(); + ("spokesperson", settings.rate()) + } + _ => panic!("expected a speech settings command"), + }) + .collect::>() + }); + control.set_rate(2.0).unwrap(); + drop(control); + worker.join().unwrap() + }; + let spokesperson = berd_call::TtsConfigurationSnapshot { + revision: 3, + settings: berd_call::TtsSettings::OpenAi { + model: "gpt-realtime".into(), + voice: "marin".into(), + rate: 1.0, + }, + }; + assert_eq!( + rates(Some(spokesperson.clone()), "applied", "applied"), + [("agents", 2.0), ("spokesperson", 1.5)] + ); + // A Spokesperson that refuses its speed puts the agents' back. + assert_eq!( + rates(Some(spokesperson.clone()), "applied", "rejected"), + [("agents", 2.0), ("spokesperson", 1.5), ("agents", 1.0)] + ); + assert_eq!( + rates(Some(spokesperson), "rejected", "applied"), + [("agents", 2.0)] + ); + assert_eq!(rates(None, "applied", "applied"), [("agents", 2.0)]); + } + #[test] fn stop_cancels_a_pending_restart() { let mut child = Command::new("/bin/cat") @@ -4167,6 +4855,7 @@ mod tests { .handle_command(ControlCommand::Restart { arguments: vec!["session".into()], expert_spokesperson: false, + tts: None, response: restart_tx, }) .unwrap(); @@ -4182,6 +4871,7 @@ mod tests { .handle_command(ControlCommand::Restart { arguments: vec!["session".into()], expert_spokesperson: false, + tts: None, response: late_tx, }) .unwrap(); @@ -4266,6 +4956,17 @@ mod tests { assert_eq!(playback.played_frames(18), 0); } + #[test] + fn only_the_ended_speech_is_retired() { + let playback = AudioPlaybackState::Finished(RetiredAudio { + speech_id: 17, + played_frames: 4_096, + }); + assert_eq!(playback.retired(17), Some(4_096)); + assert_eq!(playback.retired(18), None); + assert_eq!(AudioPlaybackState::Idle.retired(17), None); + } + #[test] fn suspension_settle_time_preserves_route_safety_duration() { assert_eq!( diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 7e9d0cb1a..3a116ed65 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -81,7 +81,7 @@ mod system_input_mute; use session_audio::{ AudioHostAck, AudioOutputControlRequest, AudioPipeTransport, RemotePcmAudioOutput, - AUDIO_CANCELLED, + AUDIO_CANCELLED, AUDIO_HOST_FAILED, }; use session_framing::{ @@ -110,6 +110,14 @@ enum Input { Request(SessionRequest), Pcm(Box), Invalid(String), + /// A speech request whose voice is now built. + VoiceBuilt(SessionRequest), + /// A speech request's voice could not be built. + VoiceUnavailable { + id: u64, + voice: String, + message: String, + }, Eof, } @@ -130,6 +138,8 @@ enum PlaybackEvent { #[derive(Debug)] struct PlaybackFailure { message: String, + /// What the host had played when playback failed. + delivery: DeliveryProgress, output_quiescent: bool, } @@ -189,6 +199,12 @@ struct SessionConfig { stt: SttBackendConfig, mode: SessionMode, endpoints: SessionEndpointOverrides, + /// In Expert-Spokesperson mode, speak the Expert's replies with `tts` + /// rather than having the Spokesperson say them. + expert_tts: bool, + /// The Spokesperson's own voice and speed when replies have theirs. + spokesperson_voice: Option, + spokesperson_rate: Option, } #[derive(Clone, Debug, Default, PartialEq)] @@ -533,6 +549,8 @@ fn main() { port, acknowledgement, resolved_handoff_ids, + voice, + language, text, } = parse_or_exit(parse_speak_control_args(&args), &args); let response = host_control::request( @@ -541,11 +559,31 @@ fn main() { text, acknowledgement, resolved_handoff_ids, + voice, + language, + relay: false, }, ) .unwrap_or_else(|error| operational_error("speak", error)); print_pretty_json(&response).unwrap_or_else(|error| operational_error("speak", error)); } + Some("message-spokesperson") => { + let options = parse_or_exit(parse_message_spokesperson_args(&args), &args); + let response = host_control::request( + options.port, + host_control::ControlRequest::Speak { + text: options.text, + acknowledgement: options.acknowledgement, + resolved_handoff_ids: options.resolved_handoff_ids, + voice: None, + language: None, + relay: true, + }, + ) + .unwrap_or_else(|error| operational_error("message-spokesperson", error)); + print_pretty_json(&response) + .unwrap_or_else(|error| operational_error("message-spokesperson", error)); + } Some("status") => { let port = parse_or_exit(parse_control_port(&args), &args); let response = host_control::request(port, host_control::ControlRequest::Status) @@ -732,7 +770,11 @@ fn request_bundled_app_update(app_path: &Path) { #[derive(Clone, Copy, Debug, PartialEq)] enum TranscriptDestination { None, - Stdout, + /// `live` streams an Expert-Spokesperson call as it happens; otherwise + /// its rows wait for the next handoff. + Stdout { + live: bool, + }, Codex, } @@ -777,12 +819,16 @@ fn parse_start_args(args: &[String]) -> Result { if transcript != TranscriptDestination::None { return Err("choose one of --stream or --codex for transcript delivery".into()); } - transcript = if flag == "--stream" { - TranscriptDestination::Stdout - } else { + index += 1; + transcript = if flag == "--codex" { TranscriptDestination::Codex + } else { + let live = args.get(index).map(String::as_str) == Some("live"); + if live { + index += 1; + } + TranscriptDestination::Stdout { live } }; - index += 1; } "--no-menu-bar" if menu_bar => { menu_bar = false; @@ -850,6 +896,11 @@ fn parse_saved_start_args_at(args: &[String], path: PathBuf) -> Result Result, resolved_handoff_ids: Vec, + voice: Option, + language: Option, text: String, } @@ -903,6 +959,7 @@ fn parse_speak_control_args(args: &[String]) -> Result Result return Err("--re may be provided only once".into()), + flag @ ("--voice" | "--language") => { + let slot = if flag == "--voice" { + &mut voice + } else { + &mut language + }; + if slot.is_some() { + return Err(format!("{flag} may be provided only once").into()); + } + *slot = Some( + args.get(index + 1) + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| format!("{flag} requires a value"))? + .clone(), + ); + index += 2; + } "--resolves" => { let value = args .get(index + 1) @@ -958,14 +1032,31 @@ fn parse_speak_control_args(args: &[String]) -> Result MAX_SPEAK_TEXT_BYTES { return Err("speak text is larger than 16 KiB".into()); } + if language.is_some() && voice.is_none() { + return Err("--language requires --voice".into()); + } Ok(SpeakOptions { port, acknowledgement, resolved_handoff_ids, + voice, + language, text, }) } +/// `message-spokesperson` takes only a port and the text. +fn parse_message_spokesperson_args(args: &[String]) -> Result { + let options = parse_speak_control_args(args)?; + if options.voice.is_some() { + return Err("message-spokesperson takes no --voice".into()); + } + if options.acknowledgement.is_none() { + return Err("message-spokesperson requires --re CURSOR".into()); + } + Ok(options) +} + fn parse_host_settings_args( args: &[String], ) -> Result<(u16, host_control::ControlRequest), ParseFailure> { @@ -1017,6 +1108,9 @@ fn parse_host_settings_args( "--muted" => host_control::ControlRequest::Muted { muted: parse_bool_setting(flag, args.get(index + 1))?, }, + "--spokesperson" => host_control::ControlRequest::SpokespersonMode { + on: parse_bool_setting(flag, args.get(index + 1))?, + }, "--input-during-tts" => host_control::ControlRequest::InputDuringTts { policy: match args.get(index + 1).map(String::as_str) { Some("allow") => berd_call::input::InputDuringTtsPolicy::AllowBargeIn, @@ -1039,6 +1133,24 @@ fn parse_host_settings_args( .clone(), language: None, }, + "--spokesperson-voice" => host_control::ControlRequest::Spokesperson { + voice: Some( + args.get(index + 1) + .filter(|value| !value.trim().is_empty()) + .ok_or("--spokesperson-voice requires a name")? + .clone(), + ), + rate: None, + }, + "--spokesperson-rate" => host_control::ControlRequest::Spokesperson { + voice: None, + rate: Some( + args.get(index + 1) + .and_then(|value| value.parse::().ok()) + .filter(|rate| rate.is_finite() && *rate > 0.0) + .ok_or("--spokesperson-rate requires a positive number")?, + ), + }, "--tts" => { let value = args .get(index + 1) @@ -1070,7 +1182,7 @@ fn parse_host_settings_args( } const SETTINGS_CHOICE_ERROR: &str = - "provide exactly one of --non-blocking, --rate, --voice, --tts, --input-during-tts, --muted, or --restart"; + "provide exactly one of --non-blocking, --rate, --voice, --spokesperson-voice, --spokesperson-rate, --tts, --input-during-tts, --muted, --spokesperson, or --restart"; fn parse_bool_setting(flag: &str, value: Option<&String>) -> Result { match value.map(String::as_str) { @@ -1929,6 +2041,10 @@ fn expert_session_status_cues_suppressed( fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String> { let (control_tx, control_rx) = mpsc::channel(); let (pcm_tx, pcm_rx) = mpsc::sync_channel(INPUT_QUEUE_CAPACITY); + let mut voice_builds = VoiceBuilds { + controls: control_tx.clone(), + pending: HashSet::new(), + }; thread::spawn(move || read_framed_requests(io::stdin().lock(), control_tx, pcm_tx)); let audio_transport = Arc::new(unsafe { AudioPipeTransport::from_raw_fd(pcm_output_fd)? }); let (playback_tx, playback_rx) = mpsc::channel(); @@ -1948,7 +2064,7 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String let mut next_input_token = 1_u64; let mut pending_control = None; let mut processed_pcm = 0_u64; - let mut held: Option = None; + let mut held: Option = None; let mut active: Option = None; let mut status_sound_runtime = StatusSoundRuntime::default(); @@ -2034,6 +2150,9 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String ) else { continue; }; + let Some(input) = voice_builds.filter(input, tts_slot.as_ref(), &mut writer)? else { + continue; + }; match input { Input::Invalid(message) => { write_protocol_fatal(&mut writer, "invalid session input", &message)?; @@ -2043,6 +2162,9 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String } return Ok(()); } + Input::VoiceBuilt(_) | Input::VoiceUnavailable { .. } => { + unreachable!("voice builds are filtered") + } Input::Eof => { if let Some(current) = active.as_mut() { if let Some(flag) = ¤t.active { @@ -2088,7 +2210,7 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String "session is shutting down", &mut writer, )?; - if let Some(held) = held.take() { + if let Some((held, _)) = held.take() { write_message( &mut writer, &SessionMessage::NotAdmitted { @@ -2178,6 +2300,7 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String let session = VoiceSessionSnapshot { tts: slot.snapshot()?, input_during_tts: input_policy.snapshot()?, + spokesperson: None, }; tts_slot = Some(slot); input_during_tts_slot = Some(input_policy); @@ -2227,6 +2350,20 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String &mut writer, )?; } + Input::Request(SessionRequest::SetSpokespersonSettings { id, .. }) => { + write_message( + &mut writer, + &SessionMessage::TtsSettingsResult { + id, + outcome: TtsSettingsOutcome::Rejected, + snapshot: tts_slot + .as_ref() + .expect("hello initialized TTS") + .snapshot()?, + message: Some("a conventional call has no Spokesperson".into()), + }, + )?; + } Input::Request(SessionRequest::SetTtsSettings { id, expected_revision, @@ -2307,12 +2444,20 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String acknowledgement, text, resolved_handoff_ids: _, + voice, + language, + relay: _, }) => { let request = PrepareRequest { id, acknowledgement, text, }; + let Some(voice) = + speak_voice(tts_slot.as_deref(), voice, language, id, &mut writer)? + else { + continue; + }; if held.is_some() { write_message( &mut writer, @@ -2324,6 +2469,7 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String } else { process_prepare( request, + voice, &mut core, tts_slot.as_deref().expect("hello initialized TTS"), input_during_tts_slot @@ -2599,6 +2745,15 @@ enum LivePlaybackInput { type LivePlaybackResult = Result<(String, u64), (String, u64, String)>; +/// How an Expert reply spoken in its own voice ended. +enum ReplyOutcome { + Completed, + Interrupted { spoken_through_utf8: u64 }, + Failed(PlaybackFailure), +} + +type ReplyResult = (String, u64, ReplyOutcome); + struct LivePlayback { response_id: String, speech_id: u64, @@ -2613,6 +2768,10 @@ struct PendingExpertPrepare { acknowledgement: Option, text: String, resolved_handoff_ids: Vec, + /// The Expert's voice for this reply, when not its usual one. + voice: Option, + /// The Spokesperson says it, even when the Expert has its own voice. + relay: bool, } struct PendingSpokespersonSettingsUpdate { @@ -2640,6 +2799,32 @@ fn rollback_spokesperson_voice_update( } } +fn is_settings_update(pending: &Option) -> bool { + pending + .as_ref() + .is_some_and(|update| matches!(update.purpose(), VoiceUpdatePurpose::Settings)) +} + +/// Puts back a settings change the conversation overtook while its session +/// was being built, to be tried again, with the conversation as it then +/// stands, once the call is idle. +fn requeue_spokesperson_voice_update( + pending: &mut Option, + old_runtime: &OpenAiSpokespersonRuntime, + queue: &mut VoiceUpdateQueue, +) -> Result<(), String> { + let update = pending.take().expect("matched settings update"); + let request = PendingSpokespersonSettingsUpdate { + id: update.id, + base_revision: update.base_revision, + settings: update.settings.clone(), + }; + update.abort(old_runtime)?; + queue + .try_enqueue(false, request) + .map_err(|_| "a Spokesperson settings update was already queued".to_string()) +} + fn activate_spokesperson_voice_update( pending: &mut Option, runtime: &mut Option, @@ -2848,6 +3033,14 @@ struct LiveResponse { prepare_id: Option, speech_id: Option, expert_text: Option, + /// An Expert reply spoken in its own voice. It has no provider + /// response; its whole text is played at once. + reply: bool, + /// An Expert reply's voice, leased when it was admitted. + voice: Option, + /// When an Expert reply still awaits the parent's `output_ready`, the time + /// it must arrive by. + ready_by: Option, delivery: RealtimeAudioDelivery, pending_audio: VecDeque>, pending_frames: usize, @@ -2865,6 +3058,9 @@ impl LiveResponse { prepare_id, speech_id, expert_text: None, + reply: false, + voice: None, + ready_by: None, delivery: RealtimeAudioDelivery::default(), pending_audio: VecDeque::new(), pending_frames: 0, @@ -2969,30 +3165,50 @@ fn spokesperson_pcm_allowed( && input_policy.policy == berd_call::input::InputDuringTtsPolicy::SuppressInput) } -fn set_spokesperson_input_muted( - id: u64, - muted: bool, - input_muted: &mut bool, - mut reset_input: impl FnMut() -> Result<(), String>, - writer: &mut impl Write, -) -> Result<(), String> { - *input_muted = muted; - if muted { - reset_input()?; - } - write_message( - writer, - &SessionMessage::InputMuteApplied { id, active: muted }, - ) +/// Input controls waiting on the provider to clear its input buffer, so microphone audio keeps flowing meanwhile. +/// Each clear goes out where its control arrived; controls answer in order once no clear is outstanding. +#[derive(Default)] +struct InputResets { + clearing: VecDeque>>, + replies: Vec, } -fn reset_spokesperson_input( - id: u64, - mut reset_input: impl FnMut() -> Result<(), String>, - writer: &mut impl Write, -) -> Result<(), String> { - reset_input()?; - write_message(writer, &SessionMessage::InputResetApplied { id }) +impl InputResets { + fn idle(&self) -> bool { + self.clearing.is_empty() + } + + fn apply( + &mut self, + reply: SessionMessage, + clear: bool, + reset_input: impl FnOnce() -> Result>, String>, + writer: &mut impl Write, + ) -> Result<(), String> { + if clear { + self.clearing.push_back(reset_input()?); + } else if self.idle() { + return write_message(writer, &reply); + } + self.replies.push(reply); + Ok(()) + } + + fn poll(&mut self, writer: &mut impl Write) -> Result<(), String> { + while let Some(clearing) = self.clearing.front() { + match clearing.try_recv() { + Err(mpsc::TryRecvError::Empty) => return Ok(()), + Err(mpsc::TryRecvError::Disconnected) => { + return Err("Spokesperson runtime is closed".into()) + } + Ok(result) => result?, + } + self.clearing.pop_front(); + } + self.replies + .drain(..) + .try_for_each(|reply| write_message(writer, &reply)) + } } fn set_spokesperson_input_policy( @@ -3060,6 +3276,7 @@ fn queued_spokesperson_settings_are_ready( responses: &HashMap, directive_speeches: &HashMap, cancelled_directives: &HashSet, + expert_tts: bool, ) -> bool { let work = spokesperson_host_work( gate, @@ -3068,7 +3285,8 @@ fn queued_spokesperson_settings_are_ready( directive_speeches, cancelled_directives, ); - gate.lifecycle.queued_settings_are_ready(work) + // A held directive would make the change unsafe, so it goes first. + !(work.pending_expert_prepare && !expert_tts) && gate.lifecycle.queued_settings_are_ready(work) } fn validate_queued_spokesperson_settings( @@ -3087,10 +3305,17 @@ fn validate_queued_spokesperson_settings( fn spokesperson_voice_update_is_safe( update: &VoiceUpdateTransaction, core: &RealtimeExpertSpokespersonSession, + expert_tts: bool, snapshot: &berd_call::TtsConfigurationSnapshot, gate: &ExpertTurnGate, work: RealtimeHostWork, ) -> bool { + // When the Expert has its own voice, a held reply waits for the + // replacement, and adds nothing to what it was given. + let work = RealtimeHostWork { + pending_expert_prepare: work.pending_expert_prepare && !expert_tts, + ..work + }; gate.lifecycle.voice_update_is_safe( update.purpose(), update.semantic_revision, @@ -3113,17 +3338,19 @@ fn unavailable_spokesperson_title(connection_lost: bool, quiescent: bool) -> &'s fn apply_spokesperson_startup_settings( session: &SessionConfig, spokesperson: &mut OpenAiSpokespersonConfig, -) -> Result<(), String> { - let rate = match &session.tts { - TtsBackendConfig::OpenAi { rate } - | TtsBackendConfig::Siri { rate, .. } - | TtsBackendConfig::Pocket { rate, .. } => *rate, - }; - if !(0.25..=1.5).contains(&rate) { - return Err("Expert-Spokesperson rate must be between 0.25 and 1.5".into()); - } - spokesperson.session.speed = Some(rate); - Ok(()) +) { + let (TtsBackendConfig::OpenAi { rate } + | TtsBackendConfig::Siri { rate, .. } + | TtsBackendConfig::Pocket { rate, .. }) = &session.tts; + if let Some(voice) = &session.spokesperson_voice { + spokesperson.session.voice = Some(voice.clone()); + } + // The Spokesperson keeps as close to the agents' speed as it can, unless given its own. + spokesperson.session.speed = Some( + session + .spokesperson_rate + .unwrap_or_else(|| spokesperson_speed(*rate)), + ); } fn run_expert_spokesperson_session( @@ -3132,13 +3359,28 @@ fn run_expert_spokesperson_session( ) -> Result<(), String> { let (control_tx, control_rx) = mpsc::channel(); let (pcm_tx, pcm_rx) = mpsc::sync_channel(INPUT_QUEUE_CAPACITY); + let mut voice_builds = VoiceBuilds { + controls: control_tx.clone(), + pending: HashSet::new(), + }; thread::spawn(move || read_framed_requests(io::stdin().lock(), control_tx, pcm_tx)); let audio_transport = Arc::new(unsafe { AudioPipeTransport::from_raw_fd(pcm_output_fd)? }); let (audio_control_tx, audio_control_rx) = mpsc::channel(); let (playback_tx, playback_rx) = mpsc::channel::(); + let (reply_tx, reply_rx) = mpsc::channel::(); + // The Expert's own voice, with `--allow-direct-agent-speech`. + let mut reply_tts: Option> = None; + let (tts_configuration_tx, tts_configuration_rx) = mpsc::channel::(); + let mut tts_update: Option = None; + let mut next_tts_update_attempt = 0_u64; let stdout = io::stdout(); let mut writer = BufWriter::new(stdout.lock()); + // An Expert that may speak for itself answers handoffs when it likes, so + // nothing holds them open. let mut core = RealtimeExpertSpokespersonSession::new(0, "external"); + if config.expert_tts { + core = core.without_handoff_tracking(); + } let mut runtime: Option = None; let mut runtime_events: Option> = None; let mut runtime_config: Option = None; @@ -3153,15 +3395,19 @@ fn run_expert_spokesperson_session( let mut waiting_responses = VecDeque::::new(); let mut active: Option = None; let mut turn_gate = ExpertTurnGate::new(spokesperson_renew_after()); + // Generated replies waiting to be shown until the words they answer are. + let mut generated = Vec::::new(); let mut session_tts: Option = None; let mut input_during_tts_slot: Option = None; let mut input_muted = false; + let mut input_resets = InputResets::default(); let mut queued_tts_settings = VoiceUpdateQueue::::default(); let mut pending_voice_update: Option = None; let mut status_sound_runtime = StatusSoundRuntime::default(); loop { if initialized { + input_resets.poll(&mut writer)?; let work = spokesperson_host_work( &turn_gate, active.as_ref(), @@ -3194,7 +3440,11 @@ fn run_expert_spokesperson_session( )?); } } - if let Some(update) = pending_voice_update.as_ref() { + // A voice change waits for input resets, which clear the same provider input buffer it does. + if let Some(update) = pending_voice_update + .as_ref() + .filter(|_| input_resets.idle()) + { let work = spokesperson_host_work( &turn_gate, active.as_ref(), @@ -3205,6 +3455,7 @@ fn run_expert_spokesperson_session( let safe = spokesperson_voice_update_is_safe( update, &core, + reply_tts.is_some(), session_tts.as_ref().expect("initialized TTS snapshot"), &turn_gate, work, @@ -3226,6 +3477,15 @@ fn run_expert_spokesperson_session( )?; turn_gate.lifecycle.session_started(Instant::now()); } + VoiceUpdateAction::Reject(_) + if !safe && is_settings_update(&pending_voice_update) => + { + requeue_spokesperson_voice_update( + &mut pending_voice_update, + runtime.as_ref().expect("initialized runtime"), + &mut queued_tts_settings, + )?; + } VoiceUpdateAction::Reject(message) => { let expiry_cause = pending_voice_update.as_ref().and_then(|update| { if let VoiceUpdatePurpose::SessionRecovery { cause } = update.purpose() { @@ -3266,6 +3526,7 @@ fn run_expert_spokesperson_session( &responses, &directive_speeches, &cancelled_directives, + reply_tts.is_some(), ); if let Some(request) = queued_tts_settings.take_ready(pending_voice_update.is_some(), queued_settings_ready) @@ -3353,6 +3614,14 @@ fn run_expert_spokesperson_session( &mut cancelled_directives, &mut writer, )?; + drop_waiting_replies( + &preexisting_responses, + active_response_id.as_deref(), + &mut responses, + &mut core, + runtime.as_ref().expect("initialized runtime"), + &mut writer, + )?; } else { turn_gate.finish_user_speaking(); } @@ -3363,6 +3632,12 @@ fn run_expert_spokesperson_session( } SpokespersonEvent::UserTurnDiscarded { item_id } => { turn_gate.discard_user_turn(&item_id); + show_generated_replies( + &mut generated, + &responses, + &turn_gate, + &mut writer, + )?; } SpokespersonEvent::UserFinal { item_id, text } => { turn_gate.resolve_user_final(&item_id); @@ -3372,7 +3647,20 @@ fn run_expert_spokesperson_session( LiveSideEvent::UserTranscript { text: text.clone() }, &mut writer, )?; + write_message( + &mut writer, + &SessionMessage::UserTranscribed { + token: emitted_live_token, + text: text.clone(), + }, + )?; core.record_user_turn(text); + show_generated_replies( + &mut generated, + &responses, + &turn_gate, + &mut writer, + )?; } SpokespersonEvent::ResponseStarted { response_id } => { turn_gate.response_started(&response_id, responses.len())?; @@ -3431,7 +3719,19 @@ fn run_expert_spokesperson_session( response.playback_complete = true; } } + if status == SpokespersonResponseStatus::Completed + && response.delivery.has_transcript() + && !response.playback_complete + { + generated.push(response_id.clone()); + } } + show_generated_replies( + &mut generated, + &responses, + &turn_gate, + &mut writer, + )?; publish_live_response_if_complete( &response_id, &mut responses, @@ -3493,12 +3793,8 @@ fn run_expert_spokesperson_session( .expect("streaming response state exists") .queue_audio(samples, total_pending_frames)?; } - Err(mpsc::TrySendError::Disconnected(_)) => { - return Err( - "Spokesperson playback worker closed while streaming" - .into(), - ) - } + // The worker has stopped; its result settles this reply. + Err(mpsc::TrySendError::Disconnected(_)) => {} Err(mpsc::TrySendError::Full(LivePlaybackInput::Finish)) => { unreachable!("streaming sends samples") } @@ -3615,8 +3911,8 @@ fn run_expert_spokesperson_session( } } SpokespersonEvent::InputCutoverFinished { request_id, result } => { - let action = pending_voice_update.as_ref().map_or( - VoiceBarrierAction::Ignore, + let (action, safe) = pending_voice_update.as_ref().map_or( + (VoiceBarrierAction::Ignore, true), |update| { let work = spokesperson_host_work( &turn_gate, @@ -3628,16 +3924,18 @@ fn run_expert_spokesperson_session( let safe = spokesperson_voice_update_is_safe( update, &core, + reply_tts.is_some(), session_tts.as_ref().expect("initialized TTS snapshot"), &turn_gate, work, ); - match update.next_action(Instant::now(), safe) { + let action = match update.next_action(Instant::now(), safe) { VoiceUpdateAction::Reject(message) => { VoiceBarrierAction::Reject(message) } _ => update.finish_barrier(request_id, result, safe), - } + }; + (action, safe) }, ); if action == VoiceBarrierAction::Activate { @@ -3650,6 +3948,15 @@ fn run_expert_spokesperson_session( &mut writer, )?; turn_gate.lifecycle.session_started(Instant::now()); + } else if matches!(action, VoiceBarrierAction::Reject(_)) + && !safe + && is_settings_update(&pending_voice_update) + { + requeue_spokesperson_voice_update( + &mut pending_voice_update, + runtime.as_ref().expect("initialized runtime"), + &mut queued_tts_settings, + )?; } else if let VoiceBarrierAction::Reject(message) = action { rollback_spokesperson_voice_update( &mut pending_voice_update, @@ -3915,7 +4222,8 @@ fn run_expert_spokesperson_session( else { continue; }; - if message == AUDIO_CANCELLED { + // A reply the host stopped playing ends where it was cut off, as if you had interrupted it; the next gets a new player. + if message == AUDIO_CANCELLED || message == AUDIO_HOST_FAILED { apply_external_coordinator_event( &mut core, runtime.as_ref().expect("initialized runtime"), @@ -3926,6 +4234,10 @@ fn run_expert_spokesperson_session( )?; if let Some(response) = responses.get_mut(&response_id) { emit_live_interrupted_terminal(response, &playback, &mut writer)?; + if message == AUDIO_HOST_FAILED { + // The model hears only what played, as when the user talks over it. + require_live_response_truncation(response)?; + } send_live_response_truncation( &response_id, response, @@ -3957,6 +4269,95 @@ fn run_expert_spokesperson_session( } } } + poll_tts_configuration_update( + Instant::now(), + &tts_configuration_rx, + reply_tts.as_deref(), + &mut tts_update, + &mut writer, + )?; + while let Ok((response_id, speech_id, outcome)) = reply_rx.try_recv() { + if take_matching_live_playback(&mut active, &response_id, speech_id).is_none() { + continue; + } + let Some(mut response) = responses.remove(&response_id) else { + continue; + }; + let id = response + .prepare_id + .expect("an Expert reply has a prepare ID"); + let text = response.expert_text.take().unwrap_or_default(); + let mut output_quiescent = true; + let (message, heard) = match outcome { + ReplyOutcome::Completed => ( + SessionMessage::SpeechCompleted { id, speech_id }, + text.len(), + ), + ReplyOutcome::Interrupted { + spoken_through_utf8, + } => ( + SessionMessage::SpeechInterrupted { + id, + speech_id, + spoken_through_utf8, + }, + usize::try_from(spoken_through_utf8).unwrap_or(text.len()), + ), + ReplyOutcome::Failed(failure) => { + output_quiescent = failure.output_quiescent; + ( + SessionMessage::SpeechFailed { + id, + speech_id, + message: failure.message, + }, + estimated_spoken_through_utf8(&text, &failure.delivery), + ) + } + }; + let heard = (0..=heard.min(text.len())) + .rev() + .find(|&end| text.is_char_boundary(end)) + .unwrap_or_default(); + let interrupted = heard < text.len(); + if response.claim_speech_terminal() { + write_message(&mut writer, &message)?; + } + if !output_quiescent { + return Err("remote PCM output did not reach a quiescent terminal".into()); + } + finish_expert_reply( + &mut core, + runtime.as_ref().expect("initialized runtime"), + &text[..heard], + interrupted, + )?; + } + let now = Instant::now(); + let expired: Vec<_> = responses + .iter() + .filter(|(_, response)| response.ready_by.is_some_and(|by| by <= now)) + .map(|(response_id, _)| response_id.clone()) + .collect(); + for response_id in expired { + let response = responses.remove(&response_id).expect("reply exists"); + write_message( + &mut writer, + &SessionMessage::SpeechFailed { + id: response + .prepare_id + .expect("an Expert reply has a prepare ID"), + speech_id: response.speech_id.expect("an Expert reply has a speech ID"), + message: "output readiness timed out".into(), + }, + )?; + finish_expert_reply( + &mut core, + runtime.as_ref().expect("initialized runtime"), + "", + true, + )?; + } if initialized && !turn_gate.input_blocks_output() { start_next_live_playback( &mut active, @@ -3966,6 +4367,7 @@ fn run_expert_spokesperson_session( &audio_transport, &audio_control_tx, &playback_tx, + &reply_tx, &mut core, runtime.as_ref().expect("initialized runtime"), &mut writer, @@ -3980,7 +4382,7 @@ fn run_expert_spokesperson_session( if let Some(request) = turn_gate.take_ready( active.is_some() || pending_voice_update.is_some() - || queued_tts_settings.is_pending(), + || reply_tts.is_some() && queued_tts_settings.is_pending(), responses.len(), ) { submit_expert_prepare( @@ -3988,6 +4390,8 @@ fn run_expert_spokesperson_session( &mut core, runtime.as_ref().expect("initialized runtime"), &mut directive_speeches, + reply_tts.as_deref(), + &mut responses, &mut next_speech_id, &mut writer, )?; @@ -4008,11 +4412,17 @@ fn run_expert_spokesperson_session( ) else { continue; }; + let Some(input) = voice_builds.filter(input, reply_tts.as_ref(), &mut writer)? else { + continue; + }; match input { Input::Invalid(message) => { write_protocol_fatal(&mut writer, "invalid session input", &message)?; break; } + Input::VoiceBuilt(_) | Input::VoiceUnavailable { .. } => { + unreachable!("voice builds are filtered") + } Input::Eof => break, Input::Request(SessionRequest::Shutdown) => { if let Some(request) = queued_tts_settings.take() { @@ -4023,6 +4433,12 @@ fn run_expert_spokesperson_session( &mut writer, )?; } + reject_tts_configuration_update( + &mut tts_update, + reply_tts.as_deref(), + "session is shutting down", + &mut writer, + )?; if let Some(delivery) = core.flush_expert_events("Voice conversation ended") { write_message( &mut writer, @@ -4106,7 +4522,13 @@ fn run_expert_spokesperson_session( if let Some(url) = &config.endpoints.realtime { spokesperson_config.endpoint.clone_from(url); } - apply_spokesperson_startup_settings(&config, &mut spokesperson_config)?; + apply_spokesperson_startup_settings(&config, &mut spokesperson_config); + if config.expert_tts { + reply_tts = Some(Arc::new(create_tts_slot( + &config.tts, + config.endpoints.tts.as_deref(), + )?)); + } let tts = berd_call::TtsConfigurationSnapshot { revision: 1, settings: TtsSettings::OpenAi { @@ -4117,8 +4539,12 @@ fn run_expert_spokesperson_session( }; let input_policy = InputDuringTtsSlot::new(input_during_tts); let snapshot = VoiceSessionSnapshot { - tts: tts.clone(), + tts: match &reply_tts { + Some(slot) => slot.snapshot()?, + None => tts.clone(), + }, input_during_tts: input_policy.snapshot()?, + spokesperson: reply_tts.is_some().then(|| tts.clone()), }; let (created, events) = OpenAiSpokespersonRuntime::spawn_observed(spokesperson_config.clone())?; @@ -4170,12 +4596,22 @@ fn run_expert_spokesperson_session( acknowledgement, text, resolved_handoff_ids, + voice, + language, + relay, }) => { + let Some(voice) = + speak_voice(reply_tts.as_deref(), voice, language, id, &mut writer)? + else { + continue; + }; let request = PendingExpertPrepare { id, acknowledgement, text, resolved_handoff_ids, + voice, + relay, }; let routing = route_expert_prepare( &mut turn_gate, @@ -4186,7 +4622,8 @@ fn run_expert_spokesperson_session( active.as_ref(), &responses, ), - pending_voice_update.is_some() || queued_tts_settings.is_pending(), + pending_voice_update.is_some() + || reply_tts.is_some() && queued_tts_settings.is_pending(), active.is_some(), responses.len(), ); @@ -4196,6 +4633,8 @@ fn run_expert_spokesperson_session( &mut core, runtime.as_ref().expect("initialized runtime"), &mut directive_speeches, + reply_tts.as_deref(), + &mut responses, &mut next_speech_id, &mut writer, )?, @@ -4210,12 +4649,24 @@ fn run_expert_spokesperson_session( } } Input::Request(SessionRequest::OutputReady { id, speech_id }) => { - let valid = directive_speeches - .get(&id) - .is_some_and(|directive| directive.speech_id == speech_id) - || responses.values().any(|response| { - response.prepare_id == Some(id) && response.speech_id == Some(speech_id) - }); + let matches = |response: &LiveResponse| { + response.prepare_id == Some(id) && response.speech_id == Some(speech_id) + }; + let reply = responses + .iter_mut() + .find(|(_, response)| response.ready_by.is_some() && matches(response)); + let valid = if let Some((response_id, response)) = reply { + response.ready_by = None; + waiting_responses.push_back(response_id.clone()); + true + } else { + directive_speeches + .get(&id) + .is_some_and(|directive| directive.speech_id == speech_id) + || responses + .values() + .any(|response| !response.reply && matches(response)) + }; write_message( &mut writer, &SessionMessage::OutputReadyResult { @@ -4482,6 +4933,21 @@ fn run_expert_spokesperson_session( { cancel_live_playback(&mut active); CancelOutcome::Cancelled + } else if let Some(response_id) = responses + .iter() + .find(|(_, response)| response.reply && response.prepare_id == Some(id)) + .map(|(response_id, _)| response_id.clone()) + { + waiting_responses.retain(|waiting| *waiting != response_id); + drop_waiting_replies( + &HashSet::from([response_id]), + None, + &mut responses, + &mut core, + runtime.as_ref().expect("initialized runtime"), + &mut writer, + )?; + CancelOutcome::Cancelled } else { CancelOutcome::Stale }; @@ -4550,10 +5016,10 @@ fn run_expert_spokesperson_session( &mut writer, )?; } - set_spokesperson_input_muted( - id, + input_muted = muted; + input_resets.apply( + SessionMessage::InputMuteApplied { id, active: muted }, muted, - &mut input_muted, || runtime.as_ref().expect("initialized runtime").reset_input(), &mut writer, )?; @@ -4568,8 +5034,9 @@ fn run_expert_spokesperson_session( &mut writer, )?; } - reset_spokesperson_input( - id, + input_resets.apply( + SessionMessage::InputResetApplied { id }, + true, || runtime.as_ref().expect("initialized runtime").reset_input(), &mut writer, )?; @@ -4602,7 +5069,49 @@ fn run_expert_spokesperson_session( id, expected_revision, settings, - }) => { + }) if reply_tts.is_some() => { + let slot = Arc::clone(reply_tts.as_ref().expect("checked reply voice")); + if tts_update.is_some() { + write_message( + &mut writer, + &SessionMessage::TtsSettingsResult { + id, + outcome: TtsSettingsOutcome::Rejected, + snapshot: slot.snapshot()?, + message: Some("another TTS configuration update is in progress".into()), + }, + )?; + } else { + let attempt = next_tts_update_attempt; + next_tts_update_attempt += 1; + tts_update = Some(ActiveTtsConfigurationUpdate { + attempt, + id, + deadline: Instant::now() + TTS_CONFIGURATION_TIMEOUT, + }); + let sender = tts_configuration_tx.clone(); + thread::spawn(move || { + let result = slot.prepare_replacement(expected_revision, settings); + let _ = sender.send(TtsConfigurationEvent { + attempt, + id, + result, + }); + }); + } + } + Input::Request( + SessionRequest::SetTtsSettings { + id, + expected_revision, + settings, + } + | SessionRequest::SetSpokespersonSettings { + id, + expected_revision, + settings, + }, + ) => { let request = PendingSpokespersonSettingsUpdate { id, base_revision: expected_revision, @@ -4650,6 +5159,113 @@ fn run_expert_spokesperson_session( Ok(()) } +/// Speaks an Expert reply with its own voice in the Spokesperson's playback slot. +#[allow(clippy::too_many_arguments)] +fn spawn_reply_playback( + response_id: String, + speech_id: u64, + prepare_id: Option, + text: String, + backend: Arc, + transport: Arc, + control: mpsc::Sender, + completed: mpsc::Sender, +) -> Result { + let active = Arc::new(AtomicBool::new(true)); + let output = Arc::new(RemotePcmAudioOutput::new( + speech_id, + backend.pcm_spec(), + transport, + Arc::clone(&active), + control, + )?); + let worker_output = Arc::clone(&output); + let worker_active = Arc::clone(&active); + let worker_response_id = response_id.clone(); + thread::spawn(move || { + let outcome = match play_tts(&text, backend.as_ref(), &worker_output, &worker_active) { + Ok((true, _)) => ReplyOutcome::Completed, + Ok((false, delivery)) => ReplyOutcome::Interrupted { + spoken_through_utf8: u64::try_from(estimated_spoken_through_utf8(&text, &delivery)) + .expect("speech text is bounded well below u64"), + }, + Err(failure) => ReplyOutcome::Failed(failure), + }; + let _ = completed.send((worker_response_id, speech_id, outcome)); + }); + // A reply's audio comes from its worker, not from provider deltas. + let (sender, _) = mpsc::sync_channel(1); + Ok(LivePlayback { + response_id, + speech_id, + prepare_id, + output, + active, + sender, + }) +} + +/// Ends Expert replies that never started playing, as interrupted before any was heard. +fn drop_waiting_replies( + response_ids: &HashSet, + active_response_id: Option<&str>, + responses: &mut HashMap, + core: &mut RealtimeExpertSpokespersonSession, + runtime: &OpenAiSpokespersonRuntime, + writer: &mut impl Write, +) -> Result<(), String> { + for response_id in response_ids { + if Some(response_id.as_str()) == active_response_id + || !responses + .get(response_id) + .is_some_and(|response| response.reply) + { + continue; + } + let response = responses.remove(response_id).expect("reply exists"); + write_message( + writer, + &SessionMessage::SpeechInterrupted { + id: response + .prepare_id + .expect("an Expert reply has a prepare ID"), + speech_id: response.speech_id.expect("an Expert reply has a speech ID"), + spoken_through_utf8: 0, + }, + )?; + finish_expert_reply(core, runtime, "", true)?; + } + Ok(()) +} + +/// Tells the Spokesperson what the user heard of an Expert reply. +fn finish_expert_reply( + core: &mut RealtimeExpertSpokespersonSession, + runtime: &OpenAiSpokespersonRuntime, + heard: &str, + interrupted: bool, +) -> Result<(), String> { + let heard = heard.trim(); + let message = match (heard.is_empty(), interrupted) { + (true, _) => "The user interrupted the Expert's spoken reply before hearing any of it.".into(), + (false, false) => format!("The Expert said this to the user in its own voice:\n{heard}"), + (false, true) => format!( + "The Expert said this to the user in its own voice before the user interrupted:\n{heard}" + ), + }; + let coordination = core.request_expert_message(RealtimeExpertMessage { + message, + mode: RealtimeExpertMessageMode::Context, + event_id: None, + directive_id: None, + resolved_handoff_ids: Vec::new(), + })?; + for event in coordination.events { + runtime.send(SpokespersonCommand::Provider(event))?; + } + Ok(()) +} + fn spawn_live_playback( response_id: String, speech_id: u64, @@ -4706,14 +5322,18 @@ fn spawn_live_playback( }) } +#[allow(clippy::too_many_arguments)] fn submit_expert_prepare( request: PendingExpertPrepare, core: &mut RealtimeExpertSpokespersonSession, runtime: &OpenAiSpokespersonRuntime, directive_speeches: &mut HashMap, + reply_tts: Option<&ConfiguredTtsSlot>, + responses: &mut HashMap, next_speech_id: &mut u64, writer: &mut impl Write, ) -> Result<(), String> { + let own_voice = reply_tts.is_some() && !request.relay; if !core .unknown_handoff_ids(&request.resolved_handoff_ids) .is_empty() @@ -4748,26 +5368,45 @@ fn submit_expert_prepare( } => { let speech_id = *next_speech_id; *next_speech_id += 1; - directive_speeches.insert( - request.id, - DirectiveSpeech { - prepare_id: request.id, - speech_id, - text: message.clone(), - }, - ); core.record_expert_turn(message.clone()); - let resolved_handoff_ids = request.resolved_handoff_ids; - core.mark_handoffs_resolving(&resolved_handoff_ids)?; - let coordination = core.request_expert_message(RealtimeExpertMessage { - message, - mode: RealtimeExpertMessageMode::Say, - event_id: None, - directive_id: Some(request.id), - resolved_handoff_ids, - })?; - for event in coordination.events { - runtime.send(SpokespersonCommand::Provider(event))?; + if own_voice { + // Played in turn with the Spokesperson's responses, without one of its own, + // once the parent is ready for its output. + let response_id = format!("expert-reply-{speech_id}"); + let mut response = LiveResponse::new(Some(request.id), Some(speech_id)); + response.expert_text = Some(message); + response.reply = true; + response.voice = Some( + request + .voice + .map_or_else(|| reply_tts.expect("own voice has a slot").lease(), Ok)?, + ); + response.ready_by = Some(Instant::now() + Duration::from_secs(2)); + response.audio_done = true; + response.finish_sent = true; + response.server_finished = true; + responses.insert(response_id, response); + } else { + let resolved_handoff_ids = request.resolved_handoff_ids; + core.mark_handoffs_resolving(&resolved_handoff_ids)?; + directive_speeches.insert( + request.id, + DirectiveSpeech { + prepare_id: request.id, + speech_id, + text: message.clone(), + }, + ); + let coordination = core.request_expert_message(RealtimeExpertMessage { + message, + mode: RealtimeExpertMessageMode::Say, + event_id: None, + directive_id: Some(request.id), + resolved_handoff_ids, + })?; + for event in coordination.events { + runtime.send(SpokespersonCommand::Provider(event))?; + } } write_message( writer, @@ -4902,6 +5541,7 @@ fn start_next_live_playback( audio_transport: &Arc, audio_control_tx: &mpsc::Sender, playback_tx: &mpsc::Sender, + reply_tx: &mpsc::Sender, core: &mut RealtimeExpertSpokespersonSession, runtime: &OpenAiSpokespersonRuntime, writer: &mut impl Write, @@ -4913,7 +5553,29 @@ fn start_next_live_playback( let Some(response) = responses.get_mut(&response_id) else { continue; }; - if response.interrupted || response.playback_complete || response.pending_audio.is_empty() { + if response.interrupted || response.playback_complete { + continue; + } + if response.reply { + *active = Some(spawn_reply_playback( + response_id, + response.speech_id.expect("an Expert reply has a speech ID"), + response.prepare_id, + response.expert_text.clone().unwrap_or_default(), + Arc::clone( + response + .voice + .as_ref() + .expect("an Expert reply leases its voice when admitted") + .backend(), + ), + Arc::clone(audio_transport), + audio_control_tx.clone(), + reply_tx.clone(), + )?); + break; + } + if response.pending_audio.is_empty() { continue; } let speech_id = response.speech_id.unwrap_or_else(|| { @@ -4986,8 +5648,11 @@ fn flush_active_live_playback( response.pending_audio.push_front(samples); break; } + // The worker has stopped, and the result it sent first settles this reply. Err(mpsc::TrySendError::Disconnected(_)) => { - return Err("Spokesperson playback worker closed while streaming".into()) + response.pending_audio.clear(); + response.pending_frames = 0; + return Ok(()); } Err(mpsc::TrySendError::Full(LivePlaybackInput::Finish)) => { unreachable!("audio queue contains samples") @@ -4997,10 +5662,10 @@ fn flush_active_live_playback( if response.audio_done && response.pending_audio.is_empty() && !response.finish_sent { match playback.sender.try_send(LivePlaybackInput::Finish) { Ok(()) => response.finish_sent = true, - Err(mpsc::TrySendError::Full(LivePlaybackInput::Finish)) => {} - Err(mpsc::TrySendError::Disconnected(_)) => { - return Err("Spokesperson playback worker closed before finish".into()) - } + Err( + mpsc::TrySendError::Full(LivePlaybackInput::Finish) + | mpsc::TrySendError::Disconnected(_), + ) => {} Err(mpsc::TrySendError::Full(LivePlaybackInput::Samples(_))) => { unreachable!("finish queue contains finish") } @@ -5148,6 +5813,34 @@ fn publish_live_response_if_complete( Ok(()) } +/// Shows each generated reply before it is spoken, once no user words are +/// still being transcribed, so a live transcript reads in order. A reply +/// already spoken or cut off is left to its authoritative transcript. +fn show_generated_replies( + generated: &mut Vec, + responses: &HashMap, + turn_gate: &ExpertTurnGate, + writer: &mut impl Write, +) -> Result<(), String> { + if turn_gate.lifecycle.transcribing() { + return Ok(()); + } + for response_id in generated.drain(..) { + if let Some(response) = responses + .get(&response_id) + .filter(|response| !response.playback_complete && !response.interrupted) + { + write_message( + writer, + &SessionMessage::SpokespersonGenerated { + text: response.delivery.transcript(), + }, + )?; + } + } + Ok(()) +} + fn live_truncation_pending(responses: &HashMap) -> bool { responses .values() @@ -5318,6 +6011,17 @@ fn finish_unready_input_runtime(runtime: VoiceInputRuntime) -> Result<(), String .map_err(|error| format!("finish unready voice input runtime: {error}")) } +/// Lets the Expert speak to the user in its own voice in an +/// Expert-Spokesperson call, rather than only through the Spokesperson. +pub(crate) const DIRECT_AGENT_SPEECH: &str = "--allow-direct-agent-speech"; +/// The speeds the Spokesperson's realtime voice supports. +pub(crate) const SPOKESPERSON_RATES: std::ops::RangeInclusive = 0.25..=1.5; + +/// The Spokesperson's closest supported speed to `rate`. +pub(crate) fn spokesperson_speed(rate: f32) -> f32 { + rate.clamp(*SPOKESPERSON_RATES.start(), *SPOKESPERSON_RATES.end()) +} + fn parse_args(args: &[String]) -> Result { if args.get(1).map(String::as_str) != Some("session") { return Err("the only supported command is session".into()); @@ -5331,12 +6035,20 @@ fn parse_args(args: &[String]) -> Result { let mut stt_model_dir = None; let mut mode = SessionMode::Conventional; let mut endpoints = SessionEndpointOverrides::default(); + let mut expert_tts = false; + let mut spokesperson_voice = None; + let mut spokesperson_rate = None; let mut index = 2; while index < args.len() { let flag = args[index].as_str(); if is_help_flag(flag) { return Err(ParseFailure::HelpRequested); } + if flag == DIRECT_AGENT_SPEECH { + expert_tts = true; + index += 1; + continue; + } let value = args .get(index + 1) .ok_or_else(|| format!("{flag} requires a value"))?; @@ -5366,6 +6078,14 @@ fn parse_args(args: &[String]) -> Result { _ => return Err("--mode must be conventional or expert-spokesperson".into()), } } + "--spokesperson-voice" => spokesperson_voice = Some(value.clone()), + "--spokesperson-rate" => { + spokesperson_rate = Some( + value + .parse::() + .map_err(|_| "--spokesperson-rate must be a number".to_string())?, + ) + } "--pcm-output-fd" => {} _ => return Err(format!("unknown argument: {flag}").into()), } @@ -5376,8 +6096,26 @@ fn parse_args(args: &[String]) -> Result { if endpoints.realtime.is_some() && mode != SessionMode::ExpertSpokesperson { return Err("--realtime-url requires --mode expert-spokesperson".into()); } + if expert_tts && mode != SessionMode::ExpertSpokesperson { + return Err(format!("{DIRECT_AGENT_SPEECH} requires --mode expert-spokesperson").into()); + } + if (spokesperson_voice.is_some() || spokesperson_rate.is_some()) && !expert_tts { + return Err(format!( + "--spokesperson-voice and --spokesperson-rate require {DIRECT_AGENT_SPEECH}" + ) + .into()); + } + if let Some(voice) = &spokesperson_voice { + if !berd_call::openai_realtime_protocol::OPENAI_REALTIME_VOICE_IDS.contains(&voice.as_str()) + { + return Err(format!("OpenAI has no realtime voice named {voice}").into()); + } + } + if spokesperson_rate.is_some_and(|rate| !SPOKESPERSON_RATES.contains(&rate)) { + return Err("--spokesperson-rate must be between 0.25 and 1.5".into()); + } if mode == SessionMode::ExpertSpokesperson - && (endpoints.tts.is_some() || endpoints.stt.is_some()) + && (endpoints.stt.is_some() || endpoints.tts.is_some() && !expert_tts) { return Err("--tts-url and --stt-url apply only to conventional mode; Expert-Spokesperson uses --realtime-url".into()); } @@ -5392,6 +6130,9 @@ fn parse_args(args: &[String]) -> Result { stt, mode, endpoints, + expert_tts, + spokesperson_voice, + spokesperson_rate, }) } @@ -5412,6 +6153,10 @@ fn parse_pcm_output_fd(args: &[String]) -> Result { let mut value = None; let mut index = 2; while index < args.len() { + if args[index] == DIRECT_AGENT_SPEECH { + index += 1; + continue; + } if args[index] == "--pcm-output-fd" { if value.is_some() { return Err("--pcm-output-fd may be provided only once".into()); @@ -6585,20 +7330,152 @@ fn reject_tts_configuration_update( ) } +/// A held reply, with the voice it asked for. +type HeldPrepare = (PrepareRequest, Option); + +/// The voice one reply asked for, or `None` after telling the host it can't +/// have it. +fn speak_voice( + slot: Option<&ConfiguredTtsSlot>, + voice: Option, + language: Option, + id: u64, + writer: &mut impl Write, +) -> Result>, String> { + let Some(voice) = voice else { + return Ok(Some(None)); + }; + let lease = slot + .ok_or_else(|| "this call's replies have no voice of their own".to_string()) + .and_then(|slot| slot.lease_voice(&voice, language.as_deref())); + match lease { + Ok(lease) => Ok(Some(Some(lease))), + Err(message) => { + refuse_voice(id, &voice, &message, writer)?; + Ok(None) + } + } +} + +fn refuse_voice( + id: u64, + voice: &str, + message: &str, + writer: &mut impl Write, +) -> Result<(), String> { + eprintln!("berd-call: speech {id} can't use voice {voice}: {message}"); + write_message( + writer, + &SessionMessage::NotAdmitted { + id, + reason: NotAdmittedReason::InvalidVoice, + }, + ) +} + +/// Speech requests waiting on a worker to build a voice the session hasn't used yet, since that +/// can take seconds and the session must keep serving audio and input meanwhile. +struct VoiceBuilds { + controls: mpsc::Sender, + pending: HashSet, +} + +impl VoiceBuilds { + /// Takes the requests that wait on a voice, and hands each back once its voice is built. + fn filter( + &mut self, + input: Input, + slot: Option<&Arc>, + writer: &mut impl Write, + ) -> Result, String> { + match input { + Input::VoiceBuilt(request) => { + let SessionRequest::PrepareSpeak { id, .. } = request else { + unreachable!("only speech waits on a voice") + }; + Ok(self.pending.remove(&id).then_some(Input::Request(request))) + } + Input::VoiceUnavailable { id, voice, message } => { + if self.pending.remove(&id) { + refuse_voice(id, &voice, &message, writer)?; + } + Ok(None) + } + Input::Request(SessionRequest::Cancel { id }) if self.pending.remove(&id) => { + write_message( + writer, + &SessionMessage::CancelResult { + id, + outcome: CancelOutcome::Cancelled, + speech_id: None, + }, + )?; + write_message( + writer, + &SessionMessage::NotAdmitted { + id, + reason: NotAdmittedReason::Cancelled, + }, + )?; + Ok(None) + } + Input::Request(request @ SessionRequest::PrepareSpeak { id, .. }) => { + let missing = slot.and_then(|slot| { + missing_voice(slot, &request).map(|voice| (Arc::clone(slot), voice)) + }); + let Some((slot, (voice, language))) = missing else { + return Ok(Some(Input::Request(request))); + }; + let controls = self.controls.clone(); + self.pending.insert(id); + thread::spawn(move || { + let input = match slot.lease_voice(&voice, language.as_deref()) { + Ok(_) => Input::VoiceBuilt(request), + Err(message) => Input::VoiceUnavailable { id, voice, message }, + }; + let _ = controls.send(OrderedControl { + after_pcm: 0, + input, + }); + }); + Ok(None) + } + input => Ok(Some(input)), + } + } +} + +/// The voice and language a speech request asks for, when the slot hasn't built them yet. +fn missing_voice( + slot: &ConfiguredTtsSlot, + request: &SessionRequest, +) -> Option<(String, Option)> { + let SessionRequest::PrepareSpeak { + voice: Some(voice), + language, + .. + } = request + else { + return None; + }; + (!slot.has_voice(voice, language.as_deref())).then(|| (voice.clone(), language.clone())) +} + #[allow(clippy::too_many_arguments)] fn process_prepare( request: PrepareRequest, + voice: Option, core: &mut SessionCore, tts_slot: &ConfiguredTtsSlot, input_during_tts_slot: &InputDuringTtsSlot, active: &mut Option, - held: &mut Option, + held: &mut Option, writer: &mut impl Write, ) -> Result<(), String> { let id = request.id; match core.prepare(request.clone()) { PrepareOutcome::Hold => { - *held = Some(request); + *held = Some((request, voice)); } PrepareOutcome::Pending(utterances) => { write_message(writer, &SessionMessage::Pending { id, utterances })?; @@ -6611,7 +7488,7 @@ fn process_prepare( confirmed_token, text, } => { - let tts = tts_slot.lease()?; + let tts = voice.map_or_else(|| tts_slot.lease(), Ok)?; let input_during_tts = input_during_tts_slot.snapshot()?; *active = Some(ActivePlayback { prepare_id: id, @@ -6639,7 +7516,7 @@ fn process_prepare( } fn reevaluate_held( - held: &mut Option, + held: &mut Option, core: &mut SessionCore, tts_slot: Option<&ConfiguredTtsSlot>, input_during_tts_slot: Option<&InputDuringTtsSlot>, @@ -6647,9 +7524,10 @@ fn reevaluate_held( writer: &mut impl Write, ) -> Result<(), String> { if !core.user_speaking() && !core.recognition_pending() && active.is_none() { - if let Some(pending_prepare) = held.take() { + if let Some((pending_prepare, voice)) = held.take() { process_prepare( pending_prepare, + voice, core, tts_slot.expect("held prepare requires initialized TTS"), input_during_tts_slot @@ -6984,12 +7862,12 @@ fn handle_reset_input( fn handle_cancel( id: u64, - held: &mut Option, + held: &mut Option, core: &mut SessionCore, active: &mut Option, writer: &mut impl Write, ) -> Result<(), String> { - if held.as_ref().is_some_and(|held| held.id == id) { + if held.as_ref().is_some_and(|(held, _)| held.id == id) { held.take(); write_message( writer, @@ -7281,6 +8159,7 @@ fn validate_request(request: SessionRequest) -> Result { | SessionRequest::SetInputMuted { id, .. } | SessionRequest::SetConversationStatus { id, .. } | SessionRequest::SetTtsSettings { id, .. } + | SessionRequest::SetSpokespersonSettings { id, .. } | SessionRequest::SetInputDuringTts { id, .. } | SessionRequest::ResetInput { id } | SessionRequest::PrepareSpeak { id, .. } @@ -7351,6 +8230,10 @@ fn validate_request(request: SessionRequest) -> Result { SessionRequest::SetTtsSettings { expected_revision: 0, .. + } + | SessionRequest::SetSpokespersonSettings { + expected_revision: 0, + .. } => return Err("expected TTS revision must be positive".into()), SessionRequest::SetInputDuringTts { expected_revision: 0, @@ -7436,17 +8319,16 @@ fn play_tts( active: &AtomicBool, ) -> Result<(bool, DeliveryProgress), PlaybackFailure> { if let Err(message) = output.start() { + let delivery = DeliveryProgress { + sample_rate: backend.pcm_spec().sample_rate, + segments: Vec::new(), + }; if message == AUDIO_CANCELLED { - return Ok(( - false, - DeliveryProgress { - sample_rate: backend.pcm_spec().sample_rate, - segments: Vec::new(), - }, - )); + return Ok((false, delivery)); } return Err(PlaybackFailure { message, + delivery, output_quiescent: output.failure_is_quiescent(), }); } @@ -7487,46 +8369,65 @@ fn synthesize_to_output_with_finish( use berd_call::{DrainPolicy, DrainTimeoutOutcome, OutboundOutcome, OutboundPlayback}; let spec = backend.pcm_spec(); + let unplayed = || DeliveryProgress { + sample_rate: spec.sample_rate, + segments: Vec::new(), + }; let initial_frames = usize::try_from(spec.sample_rate / 5).map_err(|_| PlaybackFailure { message: "TTS sample rate is too large".into(), + delivery: unplayed(), output_quiescent: false, })?; let mut playback = OutboundPlayback::new(output, active, spec.sample_rate, initial_frames) .map_err(|message| PlaybackFailure { message, + delivery: unplayed(), output_quiescent: false, })?; - if playback - .synthesize_segment(backend, text, &mut |_| Ok(()), on_started, &mut |_| Ok(())) - .map_err(|failure| PlaybackFailure { - message: failure.message, - output_quiescent: failure.output_quiescent, - })? - == OutboundOutcome::Interrupted - { - return Ok((false, playback.snapshot())); - } - if let Err(message) = finish_writes() { - let output_quiescent = output.cancel_and_snapshot().is_ok(); - return Err(PlaybackFailure { - message, - output_quiescent, - }); + let completed = (|| { + if playback + .synthesize_segment(backend, text, &mut |_| Ok(()), on_started, &mut |_| Ok(())) + .map_err(|failure| PlaybackFailure { + message: failure.message, + delivery: failure.delivery, + output_quiescent: failure.output_quiescent, + })? + == OutboundOutcome::Interrupted + { + return Ok(false); + } + if let Err(message) = finish_writes() { + let output_quiescent = output.cancel_and_snapshot().is_ok(); + return Err(PlaybackFailure { + message, + delivery: playback.snapshot(), + output_quiescent, + }); + } + let outcome = playback + .finish( + DrainPolicy { + timeout: Some(Duration::from_secs(2)), + timeout_outcome: DrainTimeoutOutcome::Fail, + ..DrainPolicy::default() + }, + &mut |_| Ok(()), + ) + .map_err(|failure| PlaybackFailure { + message: failure.message, + delivery: failure.delivery, + output_quiescent: failure.output_quiescent, + })?; + Ok(outcome == OutboundOutcome::Completed) + })(); + match completed { + Ok(completed) => Ok((completed, playback.snapshot())), + // Speech the host stopped playing, as when the speakers change, ends where it was cut off, as if you had talked over it. + Err(failure) if failure.message == AUDIO_HOST_FAILED && failure.output_quiescent => { + Ok((false, playback.snapshot())) + } + Err(failure) => Err(failure), } - let outcome = playback - .finish( - DrainPolicy { - timeout: Some(Duration::from_secs(2)), - timeout_outcome: DrainTimeoutOutcome::Fail, - ..DrainPolicy::default() - }, - &mut |_| Ok(()), - ) - .map_err(|failure| PlaybackFailure { - message: failure.message, - output_quiescent: failure.output_quiescent, - })?; - Ok((outcome == OutboundOutcome::Completed, playback.snapshot())) } #[cfg(test)] @@ -7758,6 +8659,8 @@ mod tests { acknowledgement: Some(1), text: "answer".into(), resolved_handoff_ids: Vec::new(), + voice: None, + relay: false, }; assert!(matches!( gate.defer_if_busy(request, false, 1), @@ -7797,6 +8700,8 @@ mod tests { acknowledgement: Some(1), text: "first".into(), resolved_handoff_ids: Vec::new(), + voice: None, + relay: false, }, false, 1, @@ -7810,6 +8715,8 @@ mod tests { acknowledgement: Some(1), text: "second".into(), resolved_handoff_ids: Vec::new(), + voice: None, + relay: false, }, false, 1, @@ -7840,6 +8747,8 @@ mod tests { acknowledgement: Some(1), text: "second".into(), resolved_handoff_ids: Vec::new(), + voice: None, + relay: false, }, expert_output_reserved(&directives, &HashSet::new(), None, &responses), false, @@ -8063,6 +8972,40 @@ mod tests { assert_eq!(emitted[1]["type"], "expert_delivery"); } + #[test] + fn generated_replies_wait_for_the_words_they_answer_and_are_shown_once() { + let reply = |text: &str| { + let mut response = LiveResponse::new(None, None); + response + .delivery + .replace_transcript("assistant", 0, 0, text.into()) + .unwrap(); + response + }; + let mut spoken = reply("Already spoken."); + spoken.playback_complete = true; + let responses = HashMap::from([ + ("response-a".into(), reply("Twelve.")), + ("response-b".into(), spoken), + ]); + let mut turn_gate = ExpertTurnGate::new(Duration::from_secs(60)); + turn_gate.begin_user_speaking("user-1".into()); + turn_gate.finish_user_speaking(); + let mut generated = vec!["response-a".to_string(), "response-b".to_string()]; + let mut output = Vec::new(); + + show_generated_replies(&mut generated, &responses, &turn_gate, &mut output).unwrap(); + assert!(output.is_empty()); + + turn_gate.resolve_user_final("user-1"); + show_generated_replies(&mut generated, &responses, &turn_gate, &mut output).unwrap(); + show_generated_replies(&mut generated, &responses, &turn_gate, &mut output).unwrap(); + let emitted = messages(&output); + assert_eq!(emitted.len(), 1); + assert_eq!(emitted[0]["type"], "spokesperson_generated"); + assert_eq!(emitted[0]["text"], "Twelve."); + } + #[test] fn late_audio_after_interruption_requires_zero_ms_truncation_before_release() { let mut response = LiveResponse::new(None, None); @@ -8145,31 +9088,99 @@ mod tests { } #[test] - fn spokesperson_input_controls_are_nonfatal_revisioned_and_gate_pcm() { - let mut muted = false; - let mut reset_count = 0; + fn input_controls_answer_in_order_once_the_provider_has_cleared_their_input() { + let clears = std::cell::RefCell::new(Vec::new()); + let reset_input = || { + let (completed, result) = mpsc::sync_channel(1); + clears.borrow_mut().push(completed); + Ok(result) + }; + let mut resets = InputResets::default(); let mut output = Vec::new(); - set_spokesperson_input_muted( - 1, - true, - &mut muted, - || { - reset_count += 1; - Ok(()) - }, - &mut output, - ) - .unwrap(); - reset_spokesperson_input( - 2, - || { - reset_count += 1; - Ok(()) - }, - &mut output, - ) - .unwrap(); + resets + .apply( + SessionMessage::InputMuteApplied { + id: 1, + active: true, + }, + true, + reset_input, + &mut output, + ) + .unwrap(); + resets + .apply( + SessionMessage::InputResetApplied { id: 2 }, + true, + reset_input, + &mut output, + ) + .unwrap(); + resets + .apply( + SessionMessage::InputMuteApplied { + id: 3, + active: false, + }, + false, + reset_input, + &mut output, + ) + .unwrap(); + // The reset asked for during the first clear gets a clear of its own, at once. + resets.poll(&mut output).unwrap(); + assert_eq!(clears.borrow().len(), 2); + assert!(output.is_empty()); + clears.borrow()[0].send(Ok(())).unwrap(); + resets.poll(&mut output).unwrap(); + assert!(output.is_empty()); + clears.borrow()[1].send(Ok(())).unwrap(); + resets.poll(&mut output).unwrap(); + assert!(resets.idle()); + assert_eq!( + messages(&output), + [ + json!({"type":"input_mute_applied","id":1,"active":true}), + json!({"type":"input_reset_applied","id":2}), + json!({"type":"input_mute_applied","id":3,"active":false}), + ] + ); + + output.clear(); + resets + .apply( + SessionMessage::InputMuteApplied { + id: 4, + active: false, + }, + false, + reset_input, + &mut output, + ) + .unwrap(); + assert_eq!(clears.borrow().len(), 2); + assert_eq!( + messages(&output), + [json!({"type":"input_mute_applied","id":4,"active":false})] + ); + + resets + .apply( + SessionMessage::InputResetApplied { id: 5 }, + true, + reset_input, + &mut output, + ) + .unwrap(); + clears.borrow()[2] + .send(Err("provider refused".into())) + .unwrap(); + assert_eq!(resets.poll(&mut output).unwrap_err(), "provider refused"); + } + #[test] + fn spokesperson_input_policy_is_nonfatal_revisioned_and_gates_pcm() { + let mut output = Vec::new(); let slot = InputDuringTtsSlot::new(InputDuringTtsPolicy::AllowBargeIn); set_spokesperson_input_policy( 3, @@ -8182,8 +9193,6 @@ mod tests { set_spokesperson_input_policy(4, 1, InputDuringTtsPolicy::AllowBargeIn, &slot, &mut output) .unwrap(); - assert!(muted); - assert_eq!(reset_count, 2); assert!(!spokesperson_pcm_allowed( false, true, @@ -8200,15 +9209,10 @@ mod tests { slot.snapshot().unwrap() )); let messages = messages(&output); - assert_eq!( - messages[0], - json!({"type":"input_mute_applied","id":1,"active":true}) - ); - assert_eq!(messages[1], json!({"type":"input_reset_applied","id":2})); - assert_eq!(messages[2]["outcome"], "applied"); - assert_eq!(messages[2]["snapshot"]["revision"], 2); - assert_eq!(messages[3]["outcome"], "rejected"); - assert_eq!(messages[3]["snapshot"]["revision"], 2); + assert_eq!(messages[0]["outcome"], "applied"); + assert_eq!(messages[0]["snapshot"]["revision"], 2); + assert_eq!(messages[1]["outcome"], "rejected"); + assert_eq!(messages[1]["snapshot"]["revision"], 2); } #[test] @@ -8332,6 +9336,8 @@ mod tests { acknowledgement: Some(1), text: "answer after the user".into(), resolved_handoff_ids: Vec::new(), + voice: None, + relay: false, }, false, 1, @@ -8345,6 +9351,23 @@ mod tests { assert_eq!(gate.take_ready(false, 0).map(|request| request.id), Some(7)); } + #[test] + fn audio_for_a_playback_worker_that_stopped_waits_for_its_result() { + let (playback, _controls) = live_playback_fixture(None); + let (sender, receiver) = mpsc::sync_channel(1); + drop(receiver); + let active = Some(LivePlayback { sender, ..playback }); + let mut response = LiveResponse::new(None, Some(3)); + response.queue_audio(vec![0.25; 32], 0).unwrap(); + mark_live_response_server_finished(&mut response); + let mut responses = HashMap::from([("response-a".into(), response)]); + + // The worker sent the result that settles this reply before it stopped taking audio. + flush_active_live_playback(&active, &mut responses).unwrap(); + + assert!(responses["response-a"].pending_audio.is_empty()); + } + #[test] fn response_done_finishes_playback_without_a_separate_audio_done() { let (playback, _controls) = live_playback_fixture(None); @@ -8756,6 +9779,109 @@ mod tests { test_tts_slot().lease().unwrap() } + #[test] + fn a_new_voice_is_built_off_the_session_loop_and_its_request_handed_back() { + let slot = Arc::new(test_tts_slot()); + let (controls, handed_back) = mpsc::channel::(); + let mut builds = VoiceBuilds { + controls, + pending: HashSet::new(), + }; + let mut writer = Vec::new(); + let mut filter = |input, slot| builds.filter(input, slot, &mut writer).unwrap(); + let prepare = |id, voice: &str| { + Input::Request(SessionRequest::PrepareSpeak { + id, + acknowledgement: None, + text: "Hello.".into(), + resolved_handoff_ids: Vec::new(), + voice: Some(voice.into()), + language: None, + relay: false, + }) + }; + let next = || { + handed_back + .recv_timeout(Duration::from_secs(2)) + .unwrap() + .input + }; + assert!(filter(prepare(1, "coral"), Some(&slot)).is_none()); + assert!(matches!( + filter(next(), Some(&slot)), + Some(Input::Request(SessionRequest::PrepareSpeak { id: 1, .. })) + )); + assert!(matches!( + filter(prepare(2, "coral"), Some(&slot)), + Some(Input::Request(SessionRequest::PrepareSpeak { id: 2, .. })) + )); + assert!(filter(prepare(3, "nobody"), Some(&slot)).is_none()); + assert!(filter(next(), Some(&slot)).is_none()); + // A request cancelled while its voice is built is not revived by the build. + assert!(filter(prepare(4, "marin"), Some(&slot)).is_none()); + assert!(filter( + Input::Request(SessionRequest::Cancel { id: 4 }), + Some(&slot) + ) + .is_none()); + assert!(filter(next(), Some(&slot)).is_none()); + assert!(filter(prepare(5, "cedar"), None).is_some()); + drop(filter); + let types: Vec<_> = String::from_utf8(writer) + .unwrap() + .lines() + .map(|line| { + let message: serde_json::Value = serde_json::from_str(line).unwrap(); + format!( + "{} {} {}", + message["type"], + message["id"], + message["reason"] + .as_str() + .or(message["outcome"].as_str()) + .unwrap() + ) + }) + .collect(); + assert_eq!( + types, + [ + r#""not_admitted" 3 invalid_voice"#, + r#""cancel_result" 4 cancelled"#, + r#""not_admitted" 4 cancelled"#, + ] + ); + } + + #[test] + fn speech_in_another_voice_is_refused_when_the_voice_is_unavailable() { + let slot = test_tts_slot(); + let mut writer = Vec::new(); + assert!(speak_voice(Some(&slot), None, None, 1, &mut writer) + .unwrap() + .unwrap() + .is_none()); + let coral = speak_voice(Some(&slot), Some("coral".into()), None, 2, &mut writer) + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(coral.snapshot().settings.voice(), "coral"); + assert!(writer.is_empty()); + for (id, slot, voice) in [(3, Some(&slot), "nobody"), (4, None, "coral")] { + assert!(speak_voice(slot, Some(voice.into()), None, id, &mut writer) + .unwrap() + .is_none()); + } + let refusals = String::from_utf8(writer).unwrap(); + assert_eq!( + refusals.lines().collect::>(), + [ + r#"{"type":"not_admitted","id":3,"reason":"invalid_voice"}"#, + r#"{"type":"not_admitted","id":4,"reason":"invalid_voice"}"#, + ] + ); + } + fn test_input_policy_slot() -> InputDuringTtsSlot { InputDuringTtsSlot::new(InputDuringTtsPolicy::AllowBargeIn) } @@ -9006,6 +10132,7 @@ mod tests { session: VoiceSessionSnapshot { tts: snapshot.clone(), input_during_tts: test_input_policy(), + spokesperson: None, }, }) .unwrap(); @@ -9592,6 +10719,9 @@ mod tests { stt: SttBackendConfig::Macos, mode: SessionMode::Conventional, endpoints: SessionEndpointOverrides::default(), + expert_tts: false, + spokesperson_voice: None, + spokesperson_rate: None, } ); @@ -9602,6 +10732,9 @@ mod tests { stt: SttBackendConfig::Macos, mode: SessionMode::Conventional, endpoints: SessionEndpointOverrides::default(), + expert_tts: false, + spokesperson_voice: None, + spokesperson_rate: None, } ); } @@ -9678,6 +10811,9 @@ mod tests { stt: SttBackendConfig::Macos, mode: SessionMode::Conventional, endpoints: SessionEndpointOverrides::default(), + expert_tts: false, + spokesperson_voice: None, + spokesperson_rate: None, } ); assert!(parse_args(&args(&[ @@ -9733,6 +10869,9 @@ mod tests { stt: SttBackendConfig::Macos, mode: SessionMode::ExpertSpokesperson, endpoints: SessionEndpointOverrides::default(), + expert_tts: false, + spokesperson_voice: None, + spokesperson_rate: None, }; let mut realtime = OpenAiSpokespersonConfig { endpoint: "ws://localhost".into(), @@ -9747,12 +10886,103 @@ mod tests { semantic_transcript: Vec::new(), }; - apply_spokesperson_startup_settings(&session, &mut realtime).unwrap(); + apply_spokesperson_startup_settings(&session, &mut realtime); assert_eq!(realtime.speed(), 1.5); assert_eq!(realtime.voice(), "marin"); } + #[test] + fn expert_voice_is_an_expert_spokesperson_choice_with_its_own_tts() { + let parse = |extra: &[&str]| { + let mut all = vec!["berd-call", "session"]; + all.extend_from_slice(extra); + parse_args(&args(&all)) + }; + let expert = |extra: &[&str]| { + let mut all = vec!["--mode", "expert-spokesperson"]; + all.extend_from_slice(extra); + parse(&all) + }; + assert!(!expert(&["--tts-backend", "openai"]).unwrap().expert_tts); + let own = expert(&[ + "--tts-backend", + "openai", + "--rate", + "2.0", + "--allow-direct-agent-speech", + "--tts-url", + "http://127.0.0.1:18870/v1/audio/speech", + ]) + .unwrap(); + assert!(own.expert_tts); + assert_eq!( + own.endpoints.tts.as_deref(), + Some("http://127.0.0.1:18870/v1/audio/speech") + ); + let mut realtime = OpenAiSpokespersonConfig { + endpoint: "ws://localhost".into(), + api_key: "test-key".into(), + session: berd_call::openai_realtime_protocol::RealtimeSpokespersonSessionOptions { + model: Some("test-model".into()), + voice: Some("marin".into()), + ..Default::default() + }, + semantic_transcript: Vec::new(), + }; + apply_spokesperson_startup_settings(&own, &mut realtime); + // The Spokesperson keeps as close to the replies' rate as it can. + assert_eq!(realtime.speed(), 1.5); + let chosen = expert(&[ + "--tts-backend", + "openai", + "--allow-direct-agent-speech", + "--spokesperson-voice", + "cedar", + "--spokesperson-rate", + "1.25", + ]) + .unwrap(); + apply_spokesperson_startup_settings(&chosen, &mut realtime); + assert_eq!((realtime.voice(), realtime.speed()), ("cedar", 1.25)); + for (extra, error) in [ + (&["--spokesperson-voice", "cedar"][..], DIRECT_AGENT_SPEECH), + ( + &[ + "--allow-direct-agent-speech", + "--spokesperson-voice", + "nobody", + ], + "no realtime voice named nobody", + ), + ( + &["--allow-direct-agent-speech", "--spokesperson-rate", "2"], + "between 0.25 and 1.5", + ), + ] { + let all = [&["--tts-backend", "openai"][..], extra].concat(); + assert!(expert(&all).unwrap_err().contains(error), "{extra:?}"); + } + let read_aloud = expert(&["--tts-backend", "openai", "--rate", "2.0"]).unwrap(); + apply_spokesperson_startup_settings(&read_aloud, &mut realtime); + assert_eq!(realtime.speed(), 1.5); + assert!( + parse(&["--tts-backend", "openai", "--allow-direct-agent-speech"]) + .unwrap_err() + .contains("--mode expert-spokesperson") + ); + assert_eq!( + saved_settings::persistable_arguments(&args(&[ + "--mode", + "expert-spokesperson", + "--allow-direct-agent-speech", + "--voice", + "Aaron" + ])), + args(&["--mode", "expert-spokesperson", "--voice", "Aaron"]) + ); + } + #[test] fn session_endpoint_overrides_are_independent_and_require_matching_modes() { let realtime = parse_args(&args(&[ @@ -9861,6 +11091,9 @@ mod tests { stt: SttBackendConfig::Macos, mode: SessionMode::Conventional, endpoints: SessionEndpointOverrides::default(), + expert_tts: false, + spokesperson_voice: None, + spokesperson_rate: None, } ); assert!(parse_args(&args(&[ @@ -9922,6 +11155,9 @@ mod tests { }, mode: SessionMode::Conventional, endpoints: SessionEndpointOverrides::default(), + expert_tts: false, + spokesperson_voice: None, + spokesperson_rate: None, } ); assert!(parse_args(&args(&[ @@ -10366,6 +11602,9 @@ mod tests { stt: SttBackendConfig::OpenAi, mode: SessionMode::Conventional, endpoints: SessionEndpointOverrides::default(), + expert_tts: false, + spokesperson_voice: None, + spokesperson_rate: None, } ); } @@ -10383,6 +11622,18 @@ mod tests { assert!( parse_pcm_output_fd(&args(&["berd-call", "session", "--pcm-output-fd", "2"])).is_err() ); + // As the host spawns it, after a flag that takes no value. + assert_eq!( + parse_pcm_output_fd(&args(&[ + "berd-call", + "session", + DIRECT_AGENT_SPEECH, + "--pcm-output-fd", + "9" + ])) + .unwrap(), + 9 + ); assert!(parse_pcm_output_fd(&args(&[ "berd-call", "session", @@ -10546,6 +11797,9 @@ mod tests { acknowledgement: None, text: "answer".into(), resolved_handoff_ids: vec![" ".into()], + voice: None, + language: None, + relay: false, }; assert_eq!( validate_request(empty).unwrap_err(), @@ -10596,6 +11850,7 @@ mod tests { acknowledgement: None, text: "reply".into(), }, + None, &mut core, &test_tts_slot(), &input_policy, @@ -10641,6 +11896,7 @@ mod tests { acknowledgement: None, text: "old voice".into(), }, + None, &mut core, &slot, &input_policy, @@ -11355,7 +12611,10 @@ mod tests { ])) .unwrap(); assert_eq!(parsed.port, 5300); - assert_eq!(parsed.transcript, TranscriptDestination::Stdout); + assert_eq!( + parsed.transcript, + TranscriptDestination::Stdout { live: false } + ); assert!(!parsed.expert_spokesperson); assert_eq!( parsed.session_arguments, @@ -11385,6 +12644,17 @@ mod tests { let parsed = start(&["--codex", "--non-blocking"]).unwrap(); assert_eq!(parsed.transcript, TranscriptDestination::Codex); assert!(parsed.non_blocking); + for (flags, live) in [ + (&["--stream"][..], false), + (&["--stream", "live", "--non-blocking"], true), + ] { + let parsed = start(flags).unwrap(); + assert_eq!(parsed.transcript, TranscriptDestination::Stdout { live }); + assert_eq!( + parsed.session_arguments[1..], + ["--voice", "Aaron", "--language", "en-US"] + ); + } assert!(start(&["--codex", "--stream"]).is_err()); assert!(start(&["--codex", "--codex"]).is_err()); assert!(start(&["--non-blocking"]).is_err()); @@ -11408,6 +12678,23 @@ mod tests { assert!(start(&["--no-menu-bar", "--no-menu-bar"]).is_err()); } + #[test] + fn spokesperson_settings_name_its_voice_or_rate() { + let parse = |flag: &str, value: &str| { + parse_host_settings_args(&args(&["berd-call", "settings", flag, value])) + .map(|(_, request)| request) + }; + assert!(matches!( + parse("--spokesperson-voice", "cedar").unwrap(), + host_control::ControlRequest::Spokesperson { voice: Some(voice), rate: None } if voice == "cedar" + )); + assert!(matches!( + parse("--spokesperson-rate", "1.25").unwrap(), + host_control::ControlRequest::Spokesperson { voice: None, rate: Some(rate) } if rate == 1.25 + )); + assert!(parse("--spokesperson-rate", "fast").is_err()); + } + #[test] fn settings_rate_changes_only_the_speech_rate() { let (_, request) = @@ -11418,6 +12705,55 @@ mod tests { ); } + #[test] + fn saved_spokesperson_voice_waits_for_calls_whose_replies_have_their_own() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("settings.json"); + let chosen = args(&[ + "--spokesperson-voice", + "cedar", + "--spokesperson-rate", + "1.25", + ]); + saved_settings::save( + &path, + &saved_settings::SavedSettings { + arguments: [ + args(&["--voice", "Aaron", "--language", "en-US"]), + chosen.clone(), + ] + .concat(), + ..saved_settings::SavedSettings::default() + }, + ) + .unwrap(); + let start = |extra: &[&str]| { + parse_saved_start_args_at( + &args(&[&["berd-call", "start"], extra].concat()), + path.clone(), + ) + .unwrap() + }; + let conventional = start(&[]); + assert!(!conventional + .session_arguments + .iter() + .any(|flag| flag.starts_with("--spokesperson"))); + assert!(conventional.saved.unwrap().1.arguments.starts_with(&chosen)); + let own = start(&[ + "--mode", + "expert-spokesperson", + "--allow-direct-agent-speech", + ]); + assert!(own + .session_arguments + .ends_with(&args(&["--allow-direct-agent-speech"]))); + assert!(own + .session_arguments + .windows(2) + .any(|pair| pair == ["--spokesperson-voice", "cedar"])); + } + #[test] fn saved_start_restores_preferences_but_not_agent_routing() { let directory = tempfile::tempdir().unwrap(); @@ -11594,6 +12930,12 @@ mod tests { .1, host_control::ControlRequest::Muted { muted: true } )); + assert!(matches!( + parse_host_settings_args(&args(&["berd-call", "settings", "--spokesperson", "false"])) + .unwrap() + .1, + host_control::ControlRequest::SpokespersonMode { on: false } + )); assert!(matches!( parse_host_settings_args(&args(&[ "berd-call", @@ -11652,6 +12994,10 @@ mod tests { "5300", "--re", "17", + "--voice", + "Nora", + "--language", + "en-GB", "hello", ])) .unwrap(), @@ -11659,9 +13005,47 @@ mod tests { port: 5300, acknowledgement: Some(17), resolved_handoff_ids: Vec::new(), + voice: Some("Nora".into()), + language: Some("en-GB".into()), text: "hello".into(), } ); + assert_eq!( + parse_message_spokesperson_args(&args(&[ + "berd-call", + "message-spokesperson", + "--port", + "5300", + "--re", + "4", + "--resolves", + "handoff-1", + "Tell them it's fixed." + ])) + .unwrap(), + SpeakOptions { + port: 5300, + acknowledgement: Some(4), + resolved_handoff_ids: vec!["handoff-1".into()], + voice: None, + language: None, + text: "Tell them it's fixed.".into(), + } + ); + for extra in [&["--voice", "Nora"][..], &[]] { + let mut given = args(&["berd-call", "message-spokesperson"]); + given.extend(args(extra)); + given.push("hello".into()); + assert!(parse_message_spokesperson_args(&given).is_err()); + } + assert!(parse_speak_control_args(&args(&[ + "berd-call", + "speak", + "--language", + "en-GB", + "hello" + ])) + .is_err()); assert_eq!( parse_control_port(&args(&["berd-call", "status", "--port", "5301"])).unwrap(), 5301 diff --git a/src-tauri/crates/berd-call/src/menu_bar.rs b/src-tauri/crates/berd-call/src/menu_bar.rs index 276370887..a2d101bc9 100644 --- a/src-tauri/crates/berd-call/src/menu_bar.rs +++ b/src-tauri/crates/berd-call/src/menu_bar.rs @@ -64,8 +64,10 @@ pub(crate) enum MenuAction { language: Option, }, Rate(f32), + SpokespersonVoice(String), Muted(bool), InputDuringTts(InputDuringTtsPolicy), + SpokespersonMode(bool), Stop, } @@ -107,6 +109,8 @@ struct MenuState { status: Option, voices: Vec, action_error: Option, + // The mode a Spokesperson switch is restarting the call in, shown until the switch finishes. + switching: Option, catalog_error: Option, } @@ -121,33 +125,8 @@ fn menu_model(state: &MenuState) -> Vec { }; let muted = status["muted"].as_bool().unwrap_or(false); let suppressing = input_policy(status) == Some(InputDuringTtsPolicy::SuppressInput); - let rate = status["session"]["tts"]["rate"].as_f64().unwrap_or(1.0) as f32; - let rates = RATES - .iter() - .filter(|&&option| rate_range(status).contains(&option)) - .map(|&option| MenuNode { - checked: (option - rate).abs() < 0.01, - ..item(rate_title(option), Some(MenuAction::Rate(option))) - }) - .collect(); - let voice = status["session"]["tts"]["voice"] - .as_str() - .unwrap_or("Unknown"); - let language = status["session"]["tts"]["language"].as_str(); - let voices = state - .voices - .iter() - .map(|entry| MenuNode { - checked: entry.voice == voice && entry.language.as_deref() == language, - ..item( - entry.title.clone(), - Some(MenuAction::Voice { - voice: entry.voice.clone(), - language: entry.language.clone(), - }), - ) - }) - .collect(); + let tts = &status["session"]["tts"]; + let spokesperson = &status["session"]["spokesperson"]; let mut nodes = vec![item(format!("Berd Call · {}", mode_name(status)), None)]; for error in [&state.action_error, &state.catalog_error] .into_iter() @@ -155,16 +134,33 @@ fn menu_model(state: &MenuState) -> Vec { { nodes.push(item(format!("Control error: {error}"), None)); } + let expert = state + .switching + .unwrap_or(mode_name(status) == "Expert-Spokesperson"); nodes.extend([ SEPARATOR, MenuNode { - children: voices, - ..item(format!("Voice: {voice}"), None) - }, - MenuNode { - children: rates, - ..item(format!("Speech Rate: {}", rate_title(rate)), None) + checked: expert, + ..item("Spokesperson", Some(MenuAction::SpokespersonMode(!expert))) }, + ]); + // The Spokesperson's own voice sits with it, and replies with their own voice get a section beside it. + if expert && !spokesperson.is_null() { + nodes.push(voice_menu( + "Spokesperson ", + spokesperson, + &available_voices("openai").unwrap_or_default(), + |entry| MenuAction::SpokespersonVoice(entry.voice.clone()), + )); + } + let agents = if spokesperson.is_null() { "" } else { "Agent " }; + nodes.extend([ + SEPARATOR, + voice_menu(agents, tts, &state.voices, |entry| MenuAction::Voice { + voice: entry.voice.clone(), + language: entry.language.clone(), + }), + rate_menu(tts, spokesperson, rate_range(status)), MenuNode { checked: suppressing, ..item( @@ -188,6 +184,48 @@ fn menu_model(state: &MenuState) -> Vec { nodes } +/// A voice's choices, titled with whose voice it is. +fn voice_menu( + whose: &str, + tts: &Value, + voices: &[VoiceChoice], + choose: impl Fn(&VoiceChoice) -> MenuAction, +) -> MenuNode { + let voice = tts["voice"].as_str().unwrap_or("Unknown"); + let language = tts["language"].as_str(); + MenuNode { + children: voices + .iter() + .map(|entry| MenuNode { + checked: entry.voice == voice && entry.language.as_deref() == language, + ..item(entry.title.clone(), Some(choose(entry))) + }) + .collect(), + ..item(format!("{whose}Voice: {voice}"), None) + } +} + +/// One speed for every voice, noting the Spokesperson's when it can't go as fast. +fn rate_menu(tts: &Value, spokesperson: &Value, range: std::ops::RangeInclusive) -> MenuNode { + let speed = |tts: &Value| tts["rate"].as_f64().unwrap_or(1.0) as f32; + let rate = speed(tts); + let mut title = format!("Speech Rate: {}", rate_title(rate)); + if !spokesperson.is_null() && (speed(spokesperson) - rate).abs() >= 0.01 { + title += &format!(" (Spokesperson {})", rate_title(speed(spokesperson))); + } + MenuNode { + children: RATES + .iter() + .filter(|option| range.contains(option)) + .map(|&option| MenuNode { + checked: (option - rate).abs() < 0.01, + ..item(rate_title(option), Some(MenuAction::Rate(option))) + }) + .collect(), + ..item(title, None) + } +} + /// Translates a menu action into the control request the CLI would send. pub(crate) fn control_request(action: &MenuAction) -> ControlRequest { match action { @@ -196,16 +234,21 @@ pub(crate) fn control_request(action: &MenuAction) -> ControlRequest { language: language.clone(), }, MenuAction::Rate(rate) => ControlRequest::Rate { rate: *rate }, + MenuAction::SpokespersonVoice(voice) => ControlRequest::Spokesperson { + voice: Some(voice.clone()), + rate: None, + }, MenuAction::Muted(muted) => ControlRequest::Muted { muted: *muted }, MenuAction::InputDuringTts(policy) => ControlRequest::InputDuringTts { policy: *policy }, + MenuAction::SpokespersonMode(on) => ControlRequest::SpokespersonMode { on: *on }, MenuAction::Stop => ControlRequest::Stop, } } /// The rates `berd-call` accepts for the call's TTS backend and mode. fn rate_range(status: &Value) -> std::ops::RangeInclusive { - if mode_name(status) == "Expert-Spokesperson" { - return 0.25..=1.5; + if mode_name(status) == "Expert-Spokesperson" && status["session"]["spokesperson"].is_null() { + return crate::SPOKESPERSON_RATES; } match status["session"]["tts"]["backend"].as_str() { Some("siri") => 0.5..=2.0, @@ -455,6 +498,11 @@ fn spawn_coordinator(port: u16, state: Arc>) -> mpsc::SyncSende if latest_by_domain.get(&domain) != Some(&generation) { continue; } + if domain == ActionDomain::Mode { + if let Ok(mut state) = state.lock() { + state.switching = None; + } + } if let Some(error) = menu_action_error(result) { errors.insert(domain, (generation, error)); } else { @@ -486,7 +534,10 @@ fn spawn_coordinator(port: u16, state: Arc>) -> mpsc::SyncSende latest }); if let Ok(mut state) = state.lock() { - state.status = latest; + // The call doesn't answer while it restarts in the other mode. + if latest.is_some() || state.switching.is_none() { + state.status = latest; + } state.voices = catalog .as_ref() .map(|(_, voices)| voices.clone()) @@ -539,6 +590,7 @@ enum ActionDomain { Tts, Mute, InputDuringTts, + Mode, Stop, } @@ -546,9 +598,11 @@ fn action_domain(request: &ControlRequest) -> ActionDomain { match request { ControlRequest::TtsSettings { .. } | ControlRequest::Rate { .. } - | ControlRequest::Voice { .. } => ActionDomain::Tts, + | ControlRequest::Voice { .. } + | ControlRequest::Spokesperson { .. } => ActionDomain::Tts, ControlRequest::Muted { .. } => ActionDomain::Mute, ControlRequest::InputDuringTts { .. } => ActionDomain::InputDuringTts, + ControlRequest::SpokespersonMode { .. } => ActionDomain::Mode, ControlRequest::Stop => ActionDomain::Stop, _ => unreachable!("menu only sends supported actions"), } @@ -560,6 +614,7 @@ fn revisioned_tts_action(request: &ControlRequest) -> bool { ControlRequest::TtsSettings { .. } | ControlRequest::Rate { .. } | ControlRequest::Voice { .. } + | ControlRequest::Spokesperson { .. } ) } @@ -571,6 +626,11 @@ fn begin_menu_action( results: &mpsc::Sender<(u64, ActionDomain, Result)>, state: &Mutex, ) { + if let ControlRequest::SpokespersonMode { on } = request { + if let Ok(mut state) = state.lock() { + state.switching = Some(on); + } + } match host_control::begin_request(port, request) { Ok(pending) => { let results = results.clone(); @@ -733,10 +793,98 @@ mod tests { .. } )); + assert!(matches!( + find(&nodes, "Spokesperson"), + MenuNode { + checked: true, + action: Some(MenuAction::SpokespersonMode(false)), + .. + } + )); + assert!(matches!( + find( + &menu_model(&MenuState { + status: Some(self::status(false, "allow_barge_in", None)), + ..MenuState::default() + }), + "Spokesperson" + ), + MenuNode { + checked: false, + action: Some(MenuAction::SpokespersonMode(true)), + .. + } + )); + // A switch shows its mode at once, before the call has restarted in it. + assert!( + find( + &menu_model(&MenuState { + status: Some(self::status(false, "allow_barge_in", None)), + switching: Some(true), + ..MenuState::default() + }), + "Spokesperson" + ) + .checked + ); assert_eq!(icon_symbol(Some(&status)), "mic.slash"); find(&menu_model(&MenuState::default()), "End Call"); } + #[test] + fn replies_with_their_own_voice_put_the_spokespersons_beside_it() { + let mut status = status(false, "allow_barge_in", Some("expert-spokesperson")); + status["session"]["spokesperson"] = json!({"backend": "openai", "model": "gpt-realtime", "voice": "marin", "rate": 1.0, "revision": 1}); + let nodes = menu_model(&MenuState { + status: Some(status), + ..MenuState::default() + }); + let titles = |wanted: &str| -> Vec { + find(&nodes, wanted) + .children + .iter() + .map(|node| node.title.clone()) + .collect() + }; + // One speed for everyone, noting the Spokesperson's when it differs. + let rates = titles("Speech Rate: 1.25× (Spokesperson 1×)"); + assert_eq!( + rates, + ["0.5×", "0.75×", "1×", "1.25×", "1.5×", "1.75×", "2×"] + ); + let cedar = find(&nodes, "Spokesperson Voice: marin") + .children + .iter() + .find(|node| node.title == "cedar") + .unwrap(); + assert_eq!( + serde_json::to_value(control_request(cedar.action.as_ref().unwrap())).unwrap(), + json!({"command": "spokesperson", "voice": "cedar", "rate": null}) + ); + find(&nodes, "Agent Voice: Aaron"); + // The Spokesperson's voice is in its section, and the agents' in the next. + let sections: Vec<_> = nodes[1..] + .split(|node| node.title.is_empty()) + .map(|section| { + section + .iter() + .map(|node| node.title.as_str()) + .collect::>() + }) + .collect(); + assert_eq!( + sections[1..3], + [ + vec!["Spokesperson", "Spokesperson Voice: marin"], + vec![ + "Agent Voice: Aaron", + "Speech Rate: 1.25× (Spokesperson 1×)", + "Mute Input During TTS" + ], + ] + ); + } + #[test] fn actions_become_the_same_requests_as_the_cli() { assert_eq!( diff --git a/src-tauri/crates/berd-call/src/openai_realtime_protocol.rs b/src-tauri/crates/berd-call/src/openai_realtime_protocol.rs index 2bc4afc6f..7a68cef77 100644 --- a/src-tauri/crates/berd-call/src/openai_realtime_protocol.rs +++ b/src-tauri/crates/berd-call/src/openai_realtime_protocol.rs @@ -1287,6 +1287,7 @@ pub struct RealtimeExpertSpokespersonSession { responses: RealtimeResponseCoordinator, conversation: ExpertSpokespersonCore, open_handoffs: HashMap, + tracks_handoffs: bool, call_scope: String, pending_expert_events: Vec, } @@ -1387,11 +1388,21 @@ impl RealtimeExpertSpokespersonSession { responses: RealtimeResponseCoordinator::default(), conversation: ExpertSpokespersonCore::new(initial_cursor), open_handoffs: HashMap::new(), + tracks_handoffs: true, call_scope: call_scope.into(), pending_expert_events: Vec::new(), } } + /// A session whose handoffs reach the Expert but are never held open: + /// nothing waits on them, so nothing reminds, resolves or keeps them. + pub fn without_handoff_tracking(self) -> Self { + Self { + tracks_handoffs: false, + ..self + } + } + pub fn handle_provider_event( &mut self, event: &Value, @@ -1726,14 +1737,16 @@ impl RealtimeExpertSpokespersonSession { let handoff_id = require_non_empty(handoff_id, "handoff id")?; let message = require_non_empty(message, "handoff message")?; let expert_message = expert_handoff_message(&handoff_id, cursor, &message); - self.open_handoffs.insert( - handoff_id, - RealtimeOpenHandoff { - message, - reminder_attempts: 0, - resolving: false, - }, - ); + if self.tracks_handoffs { + self.open_handoffs.insert( + handoff_id, + RealtimeOpenHandoff { + message, + reminder_attempts: 0, + resolving: false, + }, + ); + } Ok(expert_message) } @@ -3083,6 +3096,24 @@ mod tests { ); } + #[test] + fn untracked_handoffs_reach_the_expert_and_stay_closed() { + let mut session = + RealtimeExpertSpokespersonSession::new(0, "test-call").without_handoff_tracking(); + let (_, delivery) = session + .record_live_event_with_delivery(LiveSideEvent::Handoff { + call_id: "call-1".into(), + message: "First request".into(), + }) + .unwrap(); + assert_eq!(delivery.unwrap().handoff_ids, ["handoff-test-call-1"]); + assert!(!session.has_unresolved_handoff()); + assert_eq!( + session.complete_expert_turn(&[], 3), + RealtimeHandoffReminder::None + ); + } + #[test] fn native_session_records_every_semantic_turn_for_replacement() { let mut session = RealtimeExpertSpokespersonSession::new(0, "test-call"); diff --git a/src-tauri/crates/berd-call/src/openai_spokesperson.rs b/src-tauri/crates/berd-call/src/openai_spokesperson.rs index ba57624f6..2c6c62737 100644 --- a/src-tauri/crates/berd-call/src/openai_spokesperson.rs +++ b/src-tauri/crates/berd-call/src/openai_spokesperson.rs @@ -226,7 +226,7 @@ pub enum SpokespersonResponseStatus { pub struct OpenAiSpokespersonRuntime { commands: mpsc::UnboundedSender, - audio: mpsc::Sender>, + audio: mpsc::Sender, worker: Option>, } @@ -367,8 +367,10 @@ impl OpenAiSpokespersonRuntime { command => command, }; match command { - SpokespersonCommand::InputPcm48Khz(samples) => { - self.audio.try_send(samples).map_err(|error| match error { + // Input and the clears that discard it share one queue, so they stay in order. + command @ (SpokespersonCommand::InputPcm48Khz(_) + | SpokespersonCommand::ResetInput { .. }) => { + self.audio.try_send(command).map_err(|error| match error { mpsc::error::TrySendError::Full(_) => "Spokesperson input queue is full".into(), mpsc::error::TrySendError::Closed(_) => "Spokesperson runtime is closed".into(), }) @@ -380,12 +382,11 @@ impl OpenAiSpokespersonRuntime { } } - pub fn reset_input(&self) -> Result<(), String> { + /// Clears the provider's input buffer of earlier input but not later input; the result arrives once the provider confirms. + pub fn reset_input(&self) -> Result>, String> { let (completed, result) = std::sync::mpsc::sync_channel(1); self.send(SpokespersonCommand::ResetInput { completed })?; - result - .recv_timeout(std::time::Duration::from_secs(5)) - .map_err(|_| "Spokesperson input reset timed out".to_string())? + Ok(result) } pub fn abort_input_cutover(&self) -> Result<(), String> { @@ -425,7 +426,7 @@ impl Drop for OpenAiSpokespersonRuntime { fn reap_spokesperson_worker( worker: thread::JoinHandle<()>, - audio_lifetime: mpsc::Sender>, + audio_lifetime: mpsc::Sender, ) { let _ = thread::Builder::new() .name("berd-call-spokesperson-reaper".into()) @@ -445,10 +446,21 @@ async fn run( run_inner(config, commands, audio_rx, events, false).await } +/// The next command, or `None` once the runtime is shut down. +async fn next_command( + commands: &mut mpsc::UnboundedReceiver, + audio: &mut mpsc::Receiver, +) -> Result, String> { + tokio::select! { + command = commands.recv() => Ok(command), + command = audio.recv() => command.map(Some).ok_or_else(|| "Spokesperson input queue is closed".into()), + } +} + async fn run_inner( mut config: OpenAiSpokespersonConfig, mut commands: mpsc::UnboundedReceiver, - mut audio: mpsc::Receiver>, + mut audio: mpsc::Receiver, events: &std::sync::mpsc::Sender, forward_provider_events: bool, ) -> Result<(), String> { @@ -488,7 +500,8 @@ async fn run_inner( let mut cancellation_events = HashMap::::new(); let mut pending_truncations = HashMap::<(String, u64), PendingTruncation>::new(); let mut pending_input_cutover: Option = None; - let mut pending_input_reset: Option = None; + // Clears in the order sent; the provider confirms them in that order. + let mut pending_input_resets = VecDeque::::new(); let mut started_input_items = HashSet::::new(); let mut committed_input_items = HashSet::::new(); let mut seed_turns = VecDeque::from(std::mem::take(&mut config.semantic_transcript)); @@ -524,7 +537,7 @@ async fn run_inner( let cutover_deadline = pending_input_cutover .as_ref() .map(|cutover| cutover.deadline); - let reset_deadline = pending_input_reset.as_ref().map(|reset| reset.deadline); + let reset_deadline = pending_input_resets.front().map(|reset| reset.deadline); tokio::select! { _ = async { match shutdown_not_before { @@ -549,11 +562,11 @@ async fn run_inner( None => std::future::pending().await, } }, if reset_deadline.is_some() => { - if pending_input_reset - .as_ref() + if pending_input_resets + .front() .is_some_and(|reset| tokio::time::Instant::now() >= reset.deadline) { - let reset = pending_input_reset.take().expect("expired input reset"); + let reset = pending_input_resets.pop_front().expect("expired input reset"); let _ = reset.completed.send(Err("Spokesperson input reset timed out".into())); return Err("Spokesperson input reset timed out; provider input state is indeterminate".into()); } @@ -590,18 +603,8 @@ async fn run_inner( return Err("Spokesperson output truncation timed out; server context is indeterminate".into()); } } - samples = audio.recv(), if shutdown_deadline.is_none() => { - let Some(samples) = samples else { - return Err("Spokesperson input queue is closed".into()); - }; - let pcm = downsample_pcm16(&samples); - send_json(&mut socket, serde_json::json!({ - "type": "input_audio_buffer.append", - "audio": BASE64.encode(pcm), - })).await?; - } - command = commands.recv(), if shutdown_deadline.is_none() => { - match command { + command = next_command(&mut commands, &mut audio), if shutdown_deadline.is_none() => { + match command? { Some(SpokespersonCommand::Provider(event)) => { send_json(&mut socket, event).await?; } @@ -613,7 +616,7 @@ async fn run_inner( })).await?; } Some(SpokespersonCommand::ResetInput { completed }) => { - if pending_input_cutover.is_some() || pending_input_reset.is_some() { + if pending_input_cutover.is_some() { let _ = completed.send(Err( "another Spokesperson input clear is in progress".into(), )); @@ -625,7 +628,7 @@ async fn run_inner( "event_id": event_id, "type": "input_audio_buffer.clear", })).await?; - pending_input_reset = Some(PendingInputReset { + pending_input_resets.push_back(PendingInputReset { event_id, deadline: tokio::time::Instant::now() + CONTROL_ACK_TIMEOUT, completed, @@ -646,7 +649,7 @@ async fn run_inner( } } Some(SpokespersonCommand::BeginInputCutover { request_id }) => { - if pending_input_cutover.is_some() || pending_input_reset.is_some() { + if pending_input_cutover.is_some() || !pending_input_resets.is_empty() { return Err("Spokesperson input cutover was requested twice".into()); } let event_id = format!("berd-cutover-{next_control_event_id}"); @@ -719,7 +722,8 @@ async fn run_inner( Some(SpokespersonCommand::Shutdown) | None => { commands.close(); audio.close(); - while let Ok(samples) = audio.try_recv() { + while let Ok(command) = audio.try_recv() { + let SpokespersonCommand::InputPcm48Khz(samples) = command else { continue }; let pcm = downsample_pcm16(&samples); send_json(&mut socket, serde_json::json!({ "type": "input_audio_buffer.append", @@ -926,7 +930,7 @@ async fn run_inner( &committed_input_items, events, )?; - } else if let Some(reset) = pending_input_reset.take() { + } else if let Some(reset) = pending_input_resets.pop_front() { let _ = reset.completed.send(Ok(())); } } @@ -1039,11 +1043,9 @@ async fn run_inner( } "error" => { let message = value.pointer("/error/message").and_then(|value| value.as_str()).unwrap_or("OpenAI Realtime failed").to_string(); - if pending_input_reset.as_ref().is_some_and(|reset| { - value.pointer("/error/event_id").and_then(|value| value.as_str()) - == Some(reset.event_id.as_str()) - }) { - let reset = pending_input_reset.take().expect("matched input reset"); + let event_id = value.pointer("/error/event_id").and_then(|value| value.as_str()); + if let Some(index) = pending_input_resets.iter().position(|reset| Some(reset.event_id.as_str()) == event_id) { + let reset = pending_input_resets.remove(index).expect("matched input reset"); let _ = reset.completed.send(Err(message)); } else if pending_input_cutover.as_ref().is_some_and(|cutover| { value.pointer("/error/event_id").and_then(|value| value.as_str()) @@ -1942,6 +1944,93 @@ mod tests { server.await.unwrap(); } + #[tokio::test(flavor = "multi_thread")] + async fn input_reset_clears_input_sent_before_it_but_not_after() { + const FRAMES: usize = 10; + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("ws://{}/", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = accept_hdr_async(stream, require_test_authorization) + .await + .unwrap(); + let update = receive_json(&mut socket).await; + acknowledge_initial_session(&mut socket, &update, "test-model").await; + let mut types = Vec::new(); + for _ in 0..=2 * FRAMES { + types.push(receive_json(&mut socket).await["type"].clone()); + } + send_json(&mut socket, json!({"type":"input_audio_buffer.cleared"})).await; + let _ = socket.next().await; + types + }); + let (runtime, events) = + OpenAiSpokespersonRuntime::spawn(test_config(endpoint, "test-voice", 1.0, Vec::new())) + .unwrap(); + assert!(matches!( + events.recv_timeout(Duration::from_secs(2)).unwrap(), + SpokespersonEvent::Ready + )); + let pcm = || SpokespersonCommand::InputPcm48Khz(vec![0.5; 960]); + (0..FRAMES).for_each(|_| runtime.send(pcm()).unwrap()); + let reset = runtime.reset_input().unwrap(); + (0..FRAMES).for_each(|_| runtime.send(pcm()).unwrap()); + tokio::task::spawn_blocking(move || reset.recv_timeout(Duration::from_secs(2))) + .await + .unwrap() + .unwrap() + .unwrap(); + runtime.finish().unwrap(); + let append = json!("input_audio_buffer.append"); + let mut expected = vec![append.clone(); 2 * FRAMES]; + expected.insert(FRAMES, json!("input_audio_buffer.clear")); + assert_eq!(server.await.unwrap(), expected); + } + + #[tokio::test(flavor = "multi_thread")] + async fn input_resets_sent_together_are_each_answered_in_turn() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("ws://{}/", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = accept_hdr_async(stream, require_test_authorization) + .await + .unwrap(); + let update = receive_json(&mut socket).await; + acknowledge_initial_session(&mut socket, &update, "test-model").await; + let first = receive_json(&mut socket).await; + let second = receive_json(&mut socket).await; + assert_eq!(first["type"], "input_audio_buffer.clear"); + assert_eq!(second["type"], "input_audio_buffer.clear"); + send_json( + &mut socket, + json!({"type":"error","error":{"message":"refused","event_id":second["event_id"]}}), + ) + .await; + send_json(&mut socket, json!({"type":"input_audio_buffer.cleared"})).await; + let _ = socket.next().await; + }); + let (runtime, events) = + OpenAiSpokespersonRuntime::spawn(test_config(endpoint, "test-voice", 1.0, Vec::new())) + .unwrap(); + assert!(matches!( + events.recv_timeout(Duration::from_secs(2)).unwrap(), + SpokespersonEvent::Ready + )); + let first = runtime.reset_input().unwrap(); + let second = runtime.reset_input().unwrap(); + let wait = |reset: std::sync::mpsc::Receiver>| { + reset.recv_timeout(Duration::from_secs(2)).unwrap() + }; + let (first, second) = tokio::task::spawn_blocking(move || (wait(first), wait(second))) + .await + .unwrap(); + assert_eq!(first, Ok(())); + assert_eq!(second, Err("refused".into())); + runtime.finish().unwrap(); + server.await.unwrap(); + } + #[test] fn output_truncation_timeout_is_bounded() { let now = tokio::time::Instant::now(); diff --git a/src-tauri/crates/berd-call/src/protocol.rs b/src-tauri/crates/berd-call/src/protocol.rs index 9767ee5c2..cf6d578d5 100644 --- a/src-tauri/crates/berd-call/src/protocol.rs +++ b/src-tauri/crates/berd-call/src/protocol.rs @@ -32,6 +32,12 @@ pub enum SessionRequest { expected_revision: u64, settings: TtsSettings, }, + /// Changes the Spokesperson's voice in a session whose replies have their own. + SetSpokespersonSettings { + id: u64, + expected_revision: u64, + settings: TtsSettings, + }, SetInputDuringTts { id: u64, expected_revision: u64, @@ -46,6 +52,13 @@ pub enum SessionRequest { text: String, #[serde(default)] resolved_handoff_ids: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + voice: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + language: Option, + /// The Spokesperson says it, even in a call where the Expert has its own voice. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + relay: bool, }, OutputReady { id: u64, @@ -139,6 +152,7 @@ pub enum NotAdmittedReason { Cancelled, EmptyText, InvalidHandoff, + InvalidVoice, } #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] @@ -157,8 +171,13 @@ pub enum OutputReadyOutcome { #[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] pub struct VoiceSessionSnapshot { + /// The voice that speaks replies: the conventional backend, or the + /// Spokesperson's own voice when it reads them. pub tts: TtsConfigurationSnapshot, pub input_during_tts: InputDuringTtsSnapshot, + /// The Spokesperson's voice, when replies have their own. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub spokesperson: Option, } #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] @@ -235,6 +254,18 @@ pub enum SessionMessage { #[serde(skip_serializing_if = "Option::is_none")] origin: Option, }, + /// The user's words as soon as they are transcribed, for hosts that show + /// the conversation as it happens. `live_event` stays authoritative. + UserTranscribed { + token: u64, + text: String, + }, + /// The Spokesperson's whole reply once it is generated, before it is + /// spoken. The transcript recorded after playback stays authoritative and + /// says how much was heard when the user cut it off. + SpokespersonGenerated { + text: String, + }, /// A crate-selected batch that is ready to wake the external Expert. /// `live_event` remains the authoritative transcript projection; this /// message alone controls when the host schedules an Expert turn. @@ -371,6 +402,9 @@ mod tests { acknowledgement: Some(0), text: "hi".into(), resolved_handoff_ids: Vec::new(), + voice: None, + language: None, + relay: false, } ); assert_eq!( @@ -390,10 +424,18 @@ mod tests { revision: 1, policy: InputDuringTtsPolicy::AllowBargeIn, }, + spokesperson: Some(TtsConfigurationSnapshot { + revision: 1, + settings: TtsSettings::OpenAi { + model: "gpt-realtime".into(), + voice: "cedar".into(), + rate: 1.0, + }, + }), }, }) .unwrap(), - r#"{"type":"ready","id":4,"protocol":5,"session":{"tts":{"revision":1,"backend":"openai","model":"gpt-4o-mini-tts","voice":"marin","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"}}}"# + r#"{"type":"ready","id":4,"protocol":5,"session":{"tts":{"revision":1,"backend":"openai","model":"gpt-4o-mini-tts","voice":"marin","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"},"spokesperson":{"revision":1,"backend":"openai","model":"gpt-realtime","voice":"cedar","rate":1.0}}}"# ); assert_eq!( serde_json::from_str::( diff --git a/src-tauri/crates/berd-call/src/realtime_host_lifecycle.rs b/src-tauri/crates/berd-call/src/realtime_host_lifecycle.rs index f34009e76..ac70ea306 100644 --- a/src-tauri/crates/berd-call/src/realtime_host_lifecycle.rs +++ b/src-tauri/crates/berd-call/src/realtime_host_lifecycle.rs @@ -90,6 +90,11 @@ impl RealtimeHostLifecycle { self.activity.user_speaking } + /// Whether user speech has started that is not yet transcribed or discarded. + pub fn transcribing(&self) -> bool { + !self.activity.pending_user_items.is_empty() + } + pub fn is_busy(&self, work: RealtimeHostWork) -> bool { self.activity.is_busy(work) } diff --git a/src-tauri/crates/berd-call/src/saved_settings.rs b/src-tauri/crates/berd-call/src/saved_settings.rs index 8587b52b1..ea9674b87 100644 --- a/src-tauri/crates/berd-call/src/saved_settings.rs +++ b/src-tauri/crates/berd-call/src/saved_settings.rs @@ -99,6 +99,8 @@ fn validate_arguments(arguments: &[String]) -> Result<(), String> { | "--rate" | "--model-dir" | "--stt-model-dir" + | "--spokesperson-voice" + | "--spokesperson-rate" ) }) { @@ -126,6 +128,11 @@ pub(crate) fn merge_arguments(saved: &[String], explicit: &[String]) -> Vec Vec { + let arguments: Vec<_> = arguments + .iter() + .filter(|argument| *argument != crate::DIRECT_AGENT_SPEECH) + .cloned() + .collect(); arguments .chunks_exact(2) .filter(|pair| { @@ -139,6 +146,32 @@ pub(crate) fn persistable_arguments(arguments: &[String]) -> Vec { .collect() } +fn spokesperson_flag(pair: &[String]) -> bool { + matches!( + pair[0].as_str(), + "--spokesperson-voice" | "--spokesperson-rate" + ) +} + +/// Leaves out the Spokesperson's own voice, which only calls whose replies have theirs take. +pub(crate) fn without_spokesperson(arguments: &[String]) -> Vec { + arguments + .chunks_exact(2) + .filter(|pair| !spokesperson_flag(pair)) + .flatten() + .cloned() + .collect() +} + +pub(crate) fn only_spokesperson(arguments: &[String]) -> Vec { + arguments + .chunks_exact(2) + .filter(|pair| spokesperson_flag(pair)) + .flatten() + .cloned() + .collect() +} + pub(crate) fn without_tts(arguments: &[String]) -> Vec { arguments .chunks_exact(2) @@ -153,6 +186,44 @@ pub(crate) fn without_tts(arguments: &[String]) -> Vec { .collect() } +/// The same call's session options with the Spokesperson on or off. Options only one mode takes are left out of the +/// other; with the Spokesperson on, `direct` lets agents also speak in their own voice. +/// The Spokesperson keeps as close to the agents' speed as it can. +pub(crate) fn with_spokesperson(arguments: &[String], on: bool, direct: bool) -> Vec { + let mut result: Vec<_> = arguments + .iter() + .filter(|argument| *argument != crate::DIRECT_AGENT_SPEECH) + .cloned() + .collect::>() + .chunks_exact(2) + .filter(|pair| match pair[0].as_str() { + "--mode" => false, + "--realtime-url" => on, + "--stt-url" => !on, + "--tts-url" => !on || direct, + // The Spokesperson's speed follows the agents'. + "--spokesperson-rate" => false, + _ => on || !spokesperson_flag(pair), + }) + .flatten() + .cloned() + .collect(); + result.extend([ + "--mode".into(), + if on { + "expert-spokesperson" + } else { + "conventional" + } + .into(), + ]); + if on && direct { + // Last, since it takes no value. + result.push(crate::DIRECT_AGENT_SPEECH.into()); + } + result +} + #[cfg(test)] mod tests { use super::*; @@ -227,4 +298,65 @@ mod tests { ["--mode", "chained", "--stt-backend", "macos"] ); } + + #[test] + fn turning_the_spokesperson_off_and_on_keeps_the_options_both_modes_take() { + let strings = |values: &[&str]| { + values + .iter() + .map(|value| value.to_string()) + .collect::>() + }; + let realtime = strings(&[ + "--mode", + "expert-spokesperson", + "--voice", + "Aaron", + "--rate", + "2", + "--spokesperson-voice", + "cedar", + "--spokesperson-rate", + "1", + "--realtime-url", + "wss://realtime.test/", + "--allow-direct-agent-speech", + ]); + let conventional = with_spokesperson(&realtime, false, true); + assert_eq!( + conventional, + strings(&["--voice", "Aaron", "--rate", "2", "--mode", "conventional"]) + ); + assert_eq!( + with_spokesperson( + &[ + conventional.clone(), + strings(&["--stt-url", "wss://stt.test/"]) + ] + .concat(), + true, + true + ), + strings(&[ + "--voice", + "Aaron", + "--rate", + "2", + "--mode", + "expert-spokesperson", + "--allow-direct-agent-speech" + ]) + ); + assert_eq!( + with_spokesperson(&conventional, true, false), + strings(&[ + "--voice", + "Aaron", + "--rate", + "2", + "--mode", + "expert-spokesperson" + ]) + ); + } } diff --git a/src-tauri/crates/berd-call/src/session_audio.rs b/src-tauri/crates/berd-call/src/session_audio.rs index 0368d6a9d..7f2aabbc6 100644 --- a/src-tauri/crates/berd-call/src/session_audio.rs +++ b/src-tauri/crates/berd-call/src/session_audio.rs @@ -17,12 +17,16 @@ pub const AUDIO_END_KIND: u8 = 3; pub const AUDIO_CANCEL_KIND: u8 = 4; pub const AUDIO_FRAME_HEADER_BYTES: usize = 8; pub const MAX_AUDIO_CHUNK_FRAMES: usize = 4096; -const MIN_ACCEPTED_AUDIO_RUNWAY_MS: f64 = 400.0; +/// Audio the host may hold beyond what it reports played. It reports a chunk played only once the output device has played it, +/// up to a few hundred milliseconds after it is heard on Bluetooth headphones, so this must outlast that delay or playback runs dry. +const MIN_ACCEPTED_AUDIO_RUNWAY_MS: f64 = 1_000.0; const MAX_ACCEPTED_NOT_PLAYED_CHUNKS: usize = 64; const AUDIO_PIPE_WRITE_TIMEOUT: Duration = Duration::from_secs(2); const AUDIO_OPERATION_TIMEOUT: Duration = Duration::from_secs(3); pub const AUDIO_CANCELLED: &str = "remote PCM output was cancelled"; +/// The host's player stopped, as when the output device changes under it; the host has ended that speech. +pub const AUDIO_HOST_FAILED: &str = "host audio output failed"; #[derive(Clone, Debug, PartialEq)] pub enum AudioHostAck { @@ -619,7 +623,7 @@ impl RemotePcmAudioOutput { Err(state .failure .clone() - .unwrap_or_else(|| "host audio output failed".into())) + .unwrap_or_else(|| AUDIO_HOST_FAILED.into())) } else { Ok(()) } @@ -666,7 +670,7 @@ impl RemotePcmAudioOutput { return Err(state .failure .clone() - .unwrap_or_else(|| "host audio output failed".into())); + .unwrap_or_else(|| AUDIO_HOST_FAILED.into())); } Phase::Cancelling => false, Phase::WaitingBegin @@ -710,7 +714,7 @@ impl RemotePcmAudioOutput { return Err(state .failure .clone() - .unwrap_or_else(|| "host audio output failed".into())) + .unwrap_or_else(|| AUDIO_HOST_FAILED.into())) } Phase::Drained => return Err("remote PCM output is already drained".into()), _ => {} @@ -759,7 +763,7 @@ impl RemotePcmAudioOutput { return Err(state .failure .clone() - .unwrap_or_else(|| "host audio output failed".into())); + .unwrap_or_else(|| AUDIO_HOST_FAILED.into())); } if state.suspension == SuspensionPhase::Suspended && state.suspension_requested { while state.suspension_requested && self.active.load(Ordering::SeqCst) { @@ -889,7 +893,7 @@ impl PcmAudioOutput for RemotePcmAudioOutput { Err(state .failure .clone() - .unwrap_or_else(|| "host audio output failed".into())) + .unwrap_or_else(|| AUDIO_HOST_FAILED.into())) } else { Ok(()) } @@ -912,7 +916,7 @@ fn accepted_audio_runway_frames(spec: TtsPcmSpec) -> u64 { } fn public_host_failure(_message: String) -> String { - "host audio output failed".into() + AUDIO_HOST_FAILED.into() } #[cfg(test)] @@ -1040,8 +1044,8 @@ mod tests { let (output, mut host) = fixture(); start(&output, &mut host); let current = Arc::clone(&output); - let worker = thread::spawn(move || current.write(&vec![0.25; 4096 * 4])); - for sequence in 1..=3 { + let worker = thread::spawn(move || current.write(&vec![0.25; 4096 * 7])); + for sequence in 1..=6 { let (kind, _) = read_record(&mut host); assert_eq!(kind, AUDIO_CHUNK_KIND); let started = output @@ -1062,7 +1066,7 @@ mod tests { let (kind, _) = read_record(&mut host); assert_eq!(kind, AUDIO_CHUNK_KIND); output - .handle_ack(AudioHostAck::ChunkAccepted { sequence: 4 }) + .handle_ack(AudioHostAck::ChunkAccepted { sequence: 7 }) .unwrap(); worker.join().unwrap().unwrap(); } @@ -1525,14 +1529,14 @@ mod tests { sample_rate: 48_000, playback_rate: 1.5, }, - 8, + 18, ), ( TtsPcmSpec { sample_rate: 24_000, playback_rate: 2.0, }, - 5, + 12, ), ] { let (output, mut host) = @@ -1579,6 +1583,86 @@ mod tests { worker.join().unwrap().unwrap(); } + /// The silence, in milliseconds, a host playing in real time leaves in 3 s of audio written at once, when it reports each chunk played `lag` after it is heard, as the output device does. + fn silence_when_played_reports_lag(lag: Duration) -> f64 { + let (output, mut host) = fixture(); + start(&output, &mut host); + // When each chunk is heard, and its frames. + let heard = Arc::new(Mutex::new(Vec::<(Instant, Instant, u64)>::new())); + let writing = Arc::new(AtomicBool::new(true)); + let reporter = { + let (output, heard, writing) = ( + Arc::clone(&output), + Arc::clone(&heard), + Arc::clone(&writing), + ); + thread::spawn(move || { + while writing.load(Ordering::SeqCst) { + let now = Instant::now(); + let done: u64 = heard + .lock() + .unwrap() + .iter() + .filter(|(_, end, _)| *end + lag <= now) + .map(|(_, _, frames)| frames) + .sum(); + // Less the host's delivery safety, 100 ms. + let played = done.saturating_sub(2_400); + if played > output.played_frames() { + output + .handle_ack(AudioHostAck::Played { + played_frames: played, + }) + .unwrap(); + } + thread::sleep(Duration::from_millis(1)); + } + }) + }; + let writer = { + let output = Arc::clone(&output); + thread::spawn(move || { + output.write(&vec![0.25; 72_000])?; + output.finish_writes() + }) + }; + loop { + let (kind, payload) = read_record(&mut host); + if kind == AUDIO_END_KIND { + break; + } + let frames = (payload.len() as u64 - 16) / 4; + let mut heard = heard.lock().unwrap(); + let start = heard + .last() + .map_or_else(Instant::now, |(_, end, _)| (*end).max(Instant::now())); + heard.push(( + start, + start + Duration::from_micros(frames * 1_000 / 24), + frames, + )); + drop(heard); + let sequence = u64::from_le_bytes(payload[8..16].try_into().unwrap()); + output + .handle_ack(AudioHostAck::ChunkAccepted { sequence }) + .unwrap(); + } + writer.join().unwrap().unwrap(); + writing.store(false, Ordering::SeqCst); + reporter.join().unwrap(); + let heard = heard.lock().unwrap(); + heard + .windows(2) + .map(|pair| pair[1].0.duration_since(pair[0].1).as_secs_f64() * 1_000.0) + .sum() + } + + #[test] + fn playback_outlasts_bluetooth_played_reports() { + let silence = silence_when_played_reports_lag(Duration::from_millis(300)); + assert!(silence < 20.0, "{silence:.0} ms of silence"); + } + #[test] fn empty_output_fails_before_emitting_audio_end() { let (output, mut host) = fixture(); diff --git a/src-tauri/crates/berd-call/tests/session_protocol.rs b/src-tauri/crates/berd-call/tests/session_protocol.rs index 89c8339a3..6f0e9e460 100644 --- a/src-tauri/crates/berd-call/tests/session_protocol.rs +++ b/src-tauri/crates/berd-call/tests/session_protocol.rs @@ -32,11 +32,15 @@ struct ExpertSpokespersonTestSession { output: mpsc::Receiver, stderr: mpsc::Receiver, audio_host: Option>, + ready: Value, } struct AudioCancellationGate { observed: mpsc::SyncSender<()>, release: mpsc::Receiver<()>, + /// Hold the drain instead, reading nothing after it, as a hung host would. Once released, + /// the host goes on without ever draining that speech. + at_drain: bool, } impl ExpertSpokespersonTestSession { @@ -54,6 +58,35 @@ impl ExpertSpokespersonTestSession { played_frame_limit: Option, played_ready: Option>, cancellation_gate: Option, + ) -> Self { + Self::launch( + endpoint, + renew_after_ms, + played_frame_limit, + played_ready, + cancellation_gate, + false, + &[], + ) + } + + /// A session whose host player stops once it has played this much of the first speech, as when the output device changes under it. + fn start_with_failing_audio(endpoint: String, played_frames: u64) -> Self { + Self::launch(endpoint, None, Some(played_frames), None, None, true, &[]) + } + + fn start_with_args(endpoint: String, args: &[&str]) -> Self { + Self::launch(endpoint, None, None, None, None, false, args) + } + + fn launch( + endpoint: String, + renew_after_ms: Option, + played_frame_limit: Option, + played_ready: Option>, + cancellation_gate: Option, + fail_at_limit: bool, + args: &[&str], ) -> Self { let (mut command, _pcm, audio_host) = session_command(); if let Some(renew_after_ms) = renew_after_ms { @@ -72,6 +105,7 @@ impl ExpertSpokespersonTestSession { "--realtime-url", &endpoint, ]) + .args(args) .env("OPENAI_API_KEY", "test-key") .env("OPENAI_REALTIME_ENDPOINT", "ws://127.0.0.1:1/unused") .env("OPENAI_REALTIME_MODEL", "test-model") @@ -92,6 +126,7 @@ impl ExpertSpokespersonTestSession { played_frame_limit, played_ready, cancellation_gate, + fail_at_limit, ); let mut session = Self { child, @@ -99,11 +134,13 @@ impl ExpertSpokespersonTestSession { output, stderr, audio_host: Some(audio_host), + ready: Value::Null, }; session.send(json!({ "type":"hello","id":1,"input_during_tts":"allow_barge_in" })); - assert_eq!(session.recv(Duration::from_secs(2))["type"], "ready"); + session.ready = session.recv(Duration::from_secs(2)); + assert_eq!(session.ready["type"], "ready"); session } @@ -299,7 +336,7 @@ fn spawn_audio_host( reader: UnixStream, stdin: Arc>, ) -> std::thread::JoinHandle<()> { - spawn_audio_host_with_played_limit(reader, stdin, None, None, None) + spawn_audio_host_with_played_limit(reader, stdin, None, None, None, false) } fn spawn_audio_host_with_played_limit( @@ -308,9 +345,12 @@ fn spawn_audio_host_with_played_limit( played_frame_limit: Option, mut played_ready: Option>, mut cancellation_gate: Option, + mut fail_at_limit: bool, ) -> std::thread::JoinHandle<()> { std::thread::spawn(move || { let mut current = None::<(u64, u64, u64)>; + // The speech whose player failed, and what it played: like the real host, it ignores what the session sent before hearing. + let mut failed = None::<(u64, u64)>; loop { let mut header = [0_u8; 8]; match reader.read_exact(&mut header) { @@ -333,6 +373,9 @@ fn spawn_audio_host_with_played_limit( 2 => { assert!(payload.len() >= 20); let speech_id = u64::from_le_bytes(payload[..8].try_into().unwrap()); + if failed.is_some_and(|(id, _)| id == speech_id) { + continue; + } let sequence = u64::from_le_bytes(payload[8..16].try_into().unwrap()); let frames = u64::try_from((payload.len() - 16) / 4).unwrap(); let state = current.as_mut().expect("chunk follows begin"); @@ -350,6 +393,16 @@ fn spawn_audio_host_with_played_limit( &mut *writer, &json!({"type":"audio_played","speech_id":speech_id,"played_frames":played_frame_limit.map_or(state.2, |limit| state.2.min(limit))}), ); + if reached_limit && fail_at_limit { + fail_at_limit = false; + let played_frames = played_frame_limit.unwrap(); + failed = Some((speech_id, played_frames)); + current = None; + write_session_json( + &mut *writer, + &json!({"type":"audio_failed","speech_id":speech_id,"played_frames":played_frames,"message":"PCM audio output stopped unexpectedly"}), + ); + } writer.flush().unwrap(); if reached_limit { if let Some(ready) = played_ready.take() { @@ -361,15 +414,34 @@ fn spawn_audio_host_with_played_limit( 3 => { assert_eq!(payload.len(), 24); let speech_id = u64::from_le_bytes(payload[..8].try_into().unwrap()); + if failed.is_some_and(|(id, _)| id == speech_id) { + continue; + } let sequence = u64::from_le_bytes(payload[8..16].try_into().unwrap()); let frames = u64::from_le_bytes(payload[16..24].try_into().unwrap()); assert_eq!(current, Some((speech_id, sequence, frames))); + if let Some(gate) = cancellation_gate.take_if(|gate| gate.at_drain) { + let _ = gate.observed.send(()); + if gate.release.recv().is_err() { + return; + } + continue; + } current = None; json!({"type":"audio_drained","speech_id":speech_id,"sequence":sequence,"played_frames":frames}) } 4 => { assert_eq!(payload.len(), 8); let speech_id = u64::from_le_bytes(payload.try_into().unwrap()); + if let Some((_, played_frames)) = failed.filter(|(id, _)| *id == speech_id) { + let mut writer = stdin.lock().unwrap(); + write_session_json( + &mut *writer, + &json!({"type":"audio_cancelled","speech_id":speech_id,"played_frames":played_frames}), + ); + writer.flush().unwrap(); + continue; + } let played_frames = current .take() .filter(|state| state.0 == speech_id) @@ -923,6 +995,7 @@ fn autonomous_spokesperson_cancel_is_correlated_before_audio_cancellation_and_th Some(AudioCancellationGate { observed: cancel_observed_tx, release: release_cancel_rx, + at_drain: false, }), ); let started = loop { @@ -1138,6 +1211,110 @@ fn one_realtime_response_with_two_audio_parts_completes_without_identity_failure server.join().unwrap(); } +#[test] +fn a_reply_whose_player_stops_ends_where_it_was_heard_and_the_call_goes_on() { + let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); + let server = std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + endpoint_tx + .send(format!("ws://{}/", listener.local_addr().unwrap())) + .unwrap(); + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = accept_async(stream).await.unwrap(); + let update = receive_realtime_json(&mut socket).await; + acknowledge_realtime_session(&mut socket, &update).await; + // Two seconds of audio, of which the host plays a fraction before its player stops; then a short reply. + for (directive, response, bytes) in [("2", "response-cut", 96_000), ("3", "response-next", 4_096)] { + assert_eq!( + receive_realtime_json(&mut socket).await["type"], + "conversation.item.create" + ); + assert_eq!( + receive_realtime_json(&mut socket).await["type"], + "response.create" + ); + send_realtime_json( + &mut socket, + json!({"type":"response.created","response":{"id":response,"metadata":{"berd_expert_directive_id":directive}}}), + ) + .await; + let item = format!("{response}-item"); + for event in [ + json!({"type":"response.output_audio.delta","response_id":response,"item_id":item,"output_index":0,"content_index":0,"delta":BASE64.encode(vec![0_u8; bytes])}), + json!({"type":"response.output_audio.done","response_id":response,"item_id":item,"output_index":0,"content_index":0}), + json!({"type":"response.output_audio_transcript.done","response_id":response,"item_id":item,"output_index":0,"content_index":0,"transcript":"a reply"}), + json!({"type":"response.done","response":{"id":response,"status":"completed"}}), + ] { + send_realtime_json(&mut socket, event).await; + } + if directive == "2" { + // The model learns the user heard the 4,096 frames (170 ms) played before the player stopped. + let truncate = receive_realtime_json(&mut socket).await; + assert_eq!(truncate["type"], "conversation.item.truncate"); + assert_eq!(truncate["item_id"], item); + assert_eq!(truncate["audio_end_ms"], 170); + send_realtime_json( + &mut socket, + json!({"type":"conversation.item.truncated","item_id":item,"content_index":0}), + ) + .await; + } + } + let _ = socket.next().await; + }); + }); + let mut session = + ExpertSpokespersonTestSession::start_with_failing_audio(endpoint_rx.recv().unwrap(), 4_096); + let mut seen = Vec::new(); + let mut until = + |session: &mut ExpertSpokespersonTestSession, what: &str, done: &dyn Fn(&Value) -> bool| { + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + let remaining = deadline + .checked_duration_since(std::time::Instant::now()) + .unwrap_or_else(|| panic!("no {what}: {seen:?}")); + let message = session + .output + .recv_timeout(remaining) + .unwrap_or_else(|error| panic!("no {what} ({error}): {seen:?}")); + assert_ne!(message["type"], "fatal", "unexpected fatal: {message}"); + let found = done(&message); + seen.push(message); + if found { + break; + } + } + }; + let speak = |session: &mut ExpertSpokespersonTestSession, id: u64| { + session.send( + json!({"type":"prepare_speak","id":id,"acknowledgement":null,"text":"say something"}), + ); + let admitted = std::iter::repeat_with(|| session.recv(Duration::from_secs(2))) + .find(|message| message["id"] == id) + .unwrap(); + assert_eq!(admitted["type"], "admitted", "{admitted}"); + session.send(json!({"type":"output_ready","id":id,"speech_id":admitted["speech_id"]})); + }; + speak(&mut session, 2); + until(&mut session, "interruption", &|message| { + message["id"] == 2 && message["type"] == "speech_interrupted" + }); + until(&mut session, "cut transcript", &|message| { + message["type"] == "live_event" && message["origin"] == "spokesperson" + }); + speak(&mut session, 3); + until(&mut session, "next reply", &|message| { + message["id"] == 3 && message["type"] == "speech_completed" + }); + session.shutdown(); + server.join().unwrap(); +} + #[test] fn expert_spokesperson_expiry_recovery_failure_reports_the_specific_terminal() { let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); @@ -1553,7 +1730,104 @@ fn expert_spokesperson_voice_change_swaps_atomically_and_flushes_gated_pcm_once( } #[test] -fn user_activity_during_voice_cutover_rolls_back_and_flushes_gated_pcm_once() { +fn user_activity_during_voice_cutover_rolls_back_then_retries_once_idle() { + retries_a_voice_change_the_user_overtook( + &[], + json!({ + "type":"set_tts_settings", + "id":2, + "expected_revision":1, + "settings":{"backend":"openai","model":"test-model","voice":"new-voice","rate":1.0} + }), + ); +} + +#[test] +fn a_spokesperson_rate_change_the_user_overtook_is_retried_beside_the_agents_voice() { + let (speech_url, _speech_requests) = spawn_speech_server(); + retries_a_voice_change_the_user_overtook( + &["--allow-direct-agent-speech", "--tts-url", &speech_url], + json!({ + "type":"set_spokesperson_settings", + "id":2, + "expected_revision":1, + "settings":{"backend":"openai","model":"test-model","voice":"old-voice","rate":1.5} + }), + ); +} + +#[test] +fn a_message_held_for_a_voice_change_is_said_before_the_change_is_retried() { + let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); + let (candidate_seen_tx, candidate_seen_rx) = mpsc::sync_channel(1); + let (held_tx, held_rx) = mpsc::sync_channel(1); + let (first_tx, first_rx) = mpsc::sync_channel(1); + std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + endpoint_tx + .send(format!("ws://{}/", listener.local_addr().unwrap())) + .unwrap(); + let (old_stream, _) = listener.accept().await.unwrap(); + let mut old = accept_async(old_stream).await.unwrap(); + let initial = receive_realtime_json(&mut old).await; + acknowledge_realtime_session(&mut old, &initial).await; + let (candidate_stream, _) = listener.accept().await.unwrap(); + let mut candidate = accept_async(candidate_stream).await.unwrap(); + let update = receive_realtime_json(&mut candidate).await; + candidate_seen_tx.send(()).unwrap(); + tokio::task::spawn_blocking(move || held_rx.recv().unwrap()) + .await + .unwrap(); + acknowledge_realtime_session(&mut candidate, &update).await; + let said = async { + while !receive_realtime_json(&mut old) + .await + .to_string() + .contains("I checked the repository.") + {} + }; + let first = tokio::select! { + () = said => "said", + _ = listener.accept() => "retried", + }; + first_tx.send(first).unwrap(); + while old.next().await.is_some() {} + }); + }); + let mut session = ExpertSpokespersonTestSession::start(endpoint_rx.recv().unwrap()); + session.send(json!({ + "type":"set_tts_settings", + "id":2, + "expected_revision":1, + "settings":{"backend":"openai","model":"test-model","voice":"new-voice","rate":1.0} + })); + candidate_seen_rx + .recv_timeout(Duration::from_secs(2)) + .unwrap(); + // The message waits for the voice change, and the change can't land while the message waits. + session.send(json!({ + "type":"prepare_speak", + "id":3, + "acknowledgement":null, + "text":"I checked the repository.", + "relay":true + })); + session.flush(); + held_tx.send(()).unwrap(); + assert_eq!( + first_rx.recv_timeout(Duration::from_secs(3)).unwrap(), + "said" + ); +} + +/// Starts a call with `args`, makes the Spokesperson change in `request`, and speaks while its session is being built: the change rolls back, then lands once the call is idle. +fn retries_a_voice_change_the_user_overtook(args: &[&str], request: Value) { + let output = request["settings"].clone(); let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); let (clear_seen_tx, clear_seen_rx) = mpsc::sync_channel(1); let (release_activity_tx, release_activity_rx) = mpsc::sync_channel(1); @@ -1627,17 +1901,36 @@ fn user_activity_during_voice_cutover_rolls_back_and_flushes_gated_pcm_once() { .is_err()); rollback_seen_tx.send(()).unwrap(); let _ = candidate.next().await; + + // Once the conversation is idle, the change is tried again, with the user's words. + let (retry_stream, _) = listener.accept().await.unwrap(); + let mut retry = accept_async(retry_stream).await.unwrap(); + let retry_update = receive_realtime_json(&mut retry).await; + assert_eq!(retry_update["session"]["audio"]["output"]["voice"], output["voice"]); + assert_eq!(retry_update["session"]["audio"]["output"]["speed"], output["rate"]); + acknowledge_realtime_session(&mut retry, &retry_update).await; + let seeds = tokio::spawn(async move { + while let Some(Ok(Message::Text(text))) = retry.next().await { + let seed: Value = serde_json::from_str(&text).unwrap(); + if seed["type"] == "conversation.item.create" { + send_realtime_json( + &mut retry, + json!({"type":"conversation.item.created","item":{"id":seed["item"]["id"]}}), + ) + .await; + } + } + }); + while receive_realtime_json(&mut old).await["type"] != "input_audio_buffer.clear" {} + send_realtime_json(&mut old, json!({"type":"input_audio_buffer.cleared"})).await; + let _ = old.next().await; + seeds.await.unwrap(); }); }); let endpoint = endpoint_rx.recv().unwrap(); - let mut session = ExpertSpokespersonTestSession::start(endpoint); - session.send(json!({ - "type":"set_tts_settings", - "id":2, - "expected_revision":1, - "settings":{"backend":"openai","model":"test-model","voice":"new-voice","rate":1.0} - })); + let mut session = ExpertSpokespersonTestSession::start_with_args(endpoint, args); + session.send(request.clone()); clear_seen_rx.recv_timeout(Duration::from_secs(2)).unwrap(); session.send_pcm(0.25); session.flush(); @@ -1645,18 +1938,20 @@ fn user_activity_during_voice_cutover_rolls_back_and_flushes_gated_pcm_once() { let deadline = std::time::Instant::now() + Duration::from_secs(3); let mut saw_user = false; - let mut saw_rejection = false; - while !(saw_user && saw_rejection) { + let mut applied = false; + while !(saw_user && applied) { let remaining = deadline .checked_duration_since(std::time::Instant::now()) - .expect("rollback did not publish user input and settings rejection promptly"); + .expect("the retried change did not apply promptly after the user's input"); let message = session.recv(remaining); saw_user |= message["type"] == "live_event"; if message["type"] == "tts_settings_result" { - assert_eq!(message["outcome"], "rejected"); - assert_eq!(message["snapshot"]["revision"], 1); - assert_eq!(message["snapshot"]["voice"], "old-voice"); - saw_rejection = true; + assert!(saw_user, "{message}"); + assert_eq!(message["outcome"], "applied", "{message}"); + assert_eq!(message["snapshot"]["revision"], 2); + assert_eq!(message["snapshot"]["voice"], request["settings"]["voice"]); + assert_eq!(message["snapshot"]["rate"], request["settings"]["rate"]); + applied = true; } } rollback_seen_rx @@ -1666,6 +1961,156 @@ fn user_activity_during_voice_cutover_rolls_back_and_flushes_gated_pcm_once() { server.join().unwrap(); } +type RealtimeSocket = tokio_tungstenite::WebSocketStream; + +/// Accepts the realtime session a Spokesperson voice or speed change builds, checks it asks for `voice` at `speed`, and lets the call move to it from `current`. +async fn cut_over( + listener: &TcpListener, + current: &mut RealtimeSocket, + voice: &str, + speed: f64, +) -> RealtimeSocket { + let (stream, _) = listener.accept().await.unwrap(); + let mut candidate = accept_async(stream).await.unwrap(); + let update = receive_realtime_json(&mut candidate).await; + assert_eq!(update["session"]["audio"]["output"]["voice"], voice); + assert_eq!(update["session"]["audio"]["output"]["speed"], speed); + acknowledge_realtime_session(&mut candidate, &update).await; + while receive_realtime_json(current).await["type"] != "input_audio_buffer.clear" {} + send_realtime_json(current, json!({"type":"input_audio_buffer.cleared"})).await; + candidate +} + +/// Serves a call whose Spokesperson changes speed, then voice, and is then muted. +fn serve_spokesperson_changes() -> (String, std::thread::JoinHandle<()>) { + let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); + let server = std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + endpoint_tx + .send(format!("ws://{}/", listener.local_addr().unwrap())) + .unwrap(); + let (stream, _) = listener.accept().await.unwrap(); + let mut first = accept_async(stream).await.unwrap(); + let initial = receive_realtime_json(&mut first).await; + assert_eq!(initial["session"]["audio"]["output"]["voice"], "old-voice"); + acknowledge_realtime_session(&mut first, &initial).await; + let mut faster = cut_over(&listener, &mut first, "old-voice", 1.5).await; + let mut voiced = cut_over(&listener, &mut faster, "new-voice", 1.5).await; + // Muting clears what the Spokesperson has heard. + let clear = loop { + let request = receive_realtime_json(&mut voiced).await; + if request["type"] == "input_audio_buffer.clear" { + break request; + } + }; + assert!(clear["event_id"] + .as_str() + .unwrap() + .starts_with("berd-reset-")); + send_realtime_json(&mut voiced, json!({"type":"input_audio_buffer.cleared"})).await; + while voiced.next().await.is_some() {} + drop((first, faster)); + }); + }); + (endpoint_rx.recv().unwrap(), server) +} + +/// Sends a settings change on a live call and returns its result. +fn change(session: &mut ExpertSpokespersonTestSession, request: Value) -> Value { + session.send(request); + session.recv(Duration::from_secs(3)) +} + +/// Changes the Spokesperson's speed, then its voice, with `change_type`, then the input policy and mute, checking each lands. +fn change_spokesperson_and_input(session: &mut ExpertSpokespersonTestSession, change_type: &str) { + for (id, voice, revision) in [(2, "old-voice", 2), (3, "new-voice", 3)] { + let result = change( + session, + json!({ + "type":change_type, + "id":id, + "expected_revision":revision - 1, + "settings":{"backend":"openai","model":"test-model","voice":voice,"rate":1.5} + }), + ); + assert_eq!(result["type"], "tts_settings_result", "{result}"); + assert_eq!(result["outcome"], "applied", "{result}"); + assert_eq!(result["snapshot"]["revision"], revision); + assert_eq!(result["snapshot"]["voice"], voice); + assert_eq!(result["snapshot"]["rate"], 1.5); + } + let policy = change( + session, + json!({"type":"set_input_during_tts","id":4,"expected_revision":1,"policy":"suppress_input"}), + ); + assert_eq!(policy["type"], "input_during_tts_result", "{policy}"); + assert_eq!(policy["outcome"], "applied"); + assert_eq!( + policy["snapshot"], + json!({"revision":2,"policy":"suppress_input"}) + ); + for (id, active) in [(5, true), (6, false)] { + let muted = change( + session, + json!({"type":"set_input_muted","id":id,"active":active}), + ); + assert_eq!( + muted, + json!({"type":"input_mute_applied","id":id,"active":active}) + ); + } +} + +#[test] +fn every_setting_changes_in_a_live_call_with_one_voice() { + let (endpoint, server) = serve_spokesperson_changes(); + let mut session = ExpertSpokespersonTestSession::start(endpoint); + assert!(session.ready["session"]["spokesperson"].is_null()); + change_spokesperson_and_input(&mut session, "set_tts_settings"); + session.shutdown(); + server.join().unwrap(); +} + +#[test] +fn every_setting_changes_in_a_live_call_with_separate_voices() { + let (speech_url, _speech_requests) = spawn_speech_server(); + let (endpoint, server) = serve_spokesperson_changes(); + let mut session = ExpertSpokespersonTestSession::start_with_args( + endpoint, + &["--allow-direct-agent-speech", "--tts-url", &speech_url], + ); + let model = session.ready["session"]["tts"]["model"].clone(); + assert_eq!( + session.ready["session"]["spokesperson"]["voice"], + "old-voice" + ); + // The agent's own voice, then its speed, change without touching the Spokesperson's session. + for (id, voice, rate, revision) in [(7, "cedar", 1.0, 2), (8, "cedar", 1.5, 3)] { + let result = change( + &mut session, + json!({ + "type":"set_tts_settings", + "id":id, + "expected_revision":revision - 1, + "settings":{"backend":"openai","model":model,"voice":voice,"rate":rate} + }), + ); + assert_eq!(result["type"], "tts_settings_result", "{result}"); + assert_eq!(result["outcome"], "applied", "{result}"); + assert_eq!(result["snapshot"]["revision"], revision); + assert_eq!(result["snapshot"]["voice"], voice); + assert_eq!(result["snapshot"]["rate"], rate); + } + change_spokesperson_and_input(&mut session, "set_spokesperson_settings"); + session.shutdown(); + server.join().unwrap(); +} + #[test] fn expert_spokesperson_voice_candidate_mismatch_preserves_old_snapshot() { let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); @@ -1841,6 +2286,71 @@ fn voice_cutover_pcm_overflow_rolls_back_every_frame_to_the_old_runtime() { server.join().unwrap(); } +/// The host resets input whenever it restarts the microphone, as when the output device changes, and keeps sending audio while the provider confirms. +#[test] +fn microphone_audio_keeps_flowing_while_the_provider_confirms_an_input_reset() { + const FRAMES: usize = 40; + let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); + let (clear_seen_tx, clear_seen_rx) = mpsc::sync_channel(1); + let (appended_tx, appended_rx) = mpsc::sync_channel(1); + let server = std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + endpoint_tx + .send(format!("ws://{}/", listener.local_addr().unwrap())) + .unwrap(); + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = accept_async(stream).await.unwrap(); + let update = receive_realtime_json(&mut socket).await; + acknowledge_realtime_session(&mut socket, &update).await; + let clear = receive_realtime_json(&mut socket).await; + assert_eq!(clear["type"], "input_audio_buffer.clear"); + clear_seen_tx.send(()).unwrap(); + // Longer than the session can buffer microphone audio. + tokio::time::sleep(Duration::from_millis(1_200)).await; + send_realtime_json(&mut socket, json!({"type":"input_audio_buffer.cleared"})).await; + let mut appended = 0; + while let Some(Ok(Message::Text(text))) = socket.next().await { + let message: Value = serde_json::from_str(&text).unwrap(); + if message["type"] == "input_audio_buffer.append" + && BASE64 + .decode(message["audio"].as_str().unwrap()) + .unwrap() + .iter() + .any(|sample| *sample != 0) + { + appended += 1; + } + } + appended_tx.send(appended).unwrap(); + }); + }); + let endpoint = endpoint_rx.recv().unwrap(); + let mut session = ExpertSpokespersonTestSession::start(endpoint); + session.send(json!({"type":"reset_input","id":3})); + clear_seen_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + for _ in 0..FRAMES { + session.send_pcm(0.25); + session.flush(); + std::thread::sleep(Duration::from_millis(20)); + } + assert_eq!( + session.recv(Duration::from_secs(3)), + json!({"type":"input_reset_applied","id":3}) + ); + let messages = session.shutdown_and_collect(); + assert!( + messages.iter().all(|message| message["type"] != "fatal"), + "{messages:?}" + ); + server.join().unwrap(); + assert_eq!(appended_rx.recv().unwrap(), FRAMES); +} + #[test] fn reset_input_during_voice_cutover_rolls_back_then_clears_the_old_runtime() { let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); @@ -2241,6 +2751,7 @@ fn voice_cutover_timeout_is_terminal_and_never_reports_a_settings_result() { fn live_event_promptly_aborts_a_stalled_voice_candidate_and_replies_on_old_runtime() { let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); let (candidate_seen_tx, candidate_seen_rx) = mpsc::sync_channel(1); + let (retry_seen_tx, retry_seen_rx) = mpsc::sync_channel(1); let server = std::thread::spawn(move || { tokio::runtime::Builder::new_current_thread() .enable_all() @@ -2287,6 +2798,9 @@ fn live_event_promptly_aborts_a_stalled_voice_candidate_and_replies_on_old_runti .expect("stalled candidate should be cancelled promptly") .unwrap(); assert!(!pending_handshake.is_empty()); + // The change is tried again once the user's turn is over. + let (_retry, _) = listener.accept().await.unwrap(); + retry_seen_tx.send(()).unwrap(); let _ = old.next().await; }); }); @@ -2301,21 +2815,18 @@ fn live_event_promptly_aborts_a_stalled_voice_candidate_and_replies_on_old_runti candidate_seen_rx .recv_timeout(Duration::from_secs(2)) .unwrap(); - let mut saw_user = false; - let mut saw_rejection = false; let deadline = std::time::Instant::now() + Duration::from_secs(2); - while !(saw_user && saw_rejection) { + loop { let remaining = deadline .checked_duration_since(std::time::Instant::now()) - .expect("session did not report user input and voice rejection promptly"); + .expect("session did not report user input promptly"); let message = session.recv(remaining); - saw_user |= message["type"] == "live_event"; - if message["type"] == "tts_settings_result" { - assert_eq!(message["outcome"], "rejected"); - assert_eq!(message["snapshot"]["voice"], "old-voice"); - saw_rejection = true; + assert_ne!(message["type"], "tts_settings_result", "{message}"); + if message["type"] == "live_event" { + break; } } + retry_seen_rx.recv_timeout(Duration::from_secs(2)).unwrap(); session.shutdown(); server.join().unwrap(); } @@ -2654,3 +3165,991 @@ fn siri_remote_output_supports_consecutive_turns_and_cancellation() { reader.join().unwrap(); audio_host.join().unwrap(); } + +#[test] +fn speech_whose_player_stops_is_interrupted_where_it_was_heard_and_the_call_goes_on() { + let (speech_url, _speech_requests) = spawn_speech_server(); + let (mut command, pcm_guard, host) = session_command(); + let child = command + .args([ + "--tts-backend", + "openai", + "--tts-url", + &speech_url, + "--stt-backend", + "openai", + "--stt-url", + &spawn_transcription_server(), + ]) + .env("OPENAI_API_KEY", "test-key") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + drop(pcm_guard); + let mut child = ChildGuard(Some(child)); + let process = child.0.as_mut().unwrap(); + let stdin = Arc::new(Mutex::new(process.stdin.take().unwrap())); + let output = spawn_session_message_reader(process.stdout.take().unwrap()); + // The host's player stops partway through the first speech, as when the speakers change. + let audio_host = + spawn_audio_host_with_played_limit(host, Arc::clone(&stdin), Some(1_200), None, None, true); + let send = |message: Value| { + let mut stdin = stdin.lock().unwrap(); + write_session_json(&mut *stdin, &message); + stdin.flush().unwrap(); + }; + let terminal = |id: u64| loop { + let message = output.recv_timeout(Duration::from_secs(5)).unwrap(); + match message["type"].as_str() { + Some("admitted") => send(json!({ + "type":"output_ready","id":id,"speech_id":message["speech_id"] + })), + Some("output_ready_result" | "speech_started") => {} + _ => break message, + } + }; + send(json!({"type":"hello","id":1,"input_during_tts":"allow_barge_in"})); + let ready = output.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!(ready["type"], "ready", "{ready}"); + let text = "A reply cut off by a change of speakers."; + send(json!({"type":"prepare_speak","id":2,"acknowledgement":null,"text":text})); + let interrupted = terminal(2); + assert_eq!(interrupted["type"], "speech_interrupted", "{interrupted}"); + let heard = interrupted["spoken_through_utf8"].as_u64().unwrap(); + assert!(heard < text.len() as u64, "{interrupted}"); + send(json!({"type":"prepare_speak","id":3,"acknowledgement":null,"text":"Still here."})); + assert_eq!(terminal(3)["type"], "speech_completed"); + send(json!({"type":"shutdown"})); + drop(stdin); + assert!(child.0.take().unwrap().wait().unwrap().success()); + audio_host.join().unwrap(); +} + +/// Accepts one OpenAI transcription session that never hears anything. +fn spawn_transcription_server() -> String { + let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); + std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + endpoint_tx + .send(format!("ws://{}/", listener.local_addr().unwrap())) + .unwrap(); + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = accept_async(stream).await.unwrap(); + assert_eq!( + receive_realtime_json(&mut socket).await["type"], + "session.update" + ); + send_realtime_json(&mut socket, json!({"type":"session.updated"})).await; + while let Some(Ok(_)) = socket.next().await {} + }); + }); + endpoint_rx.recv().unwrap() +} + +/// Serves OpenAI speech requests with a little silence, reporting each request body. +fn spawn_speech_server() -> (String, mpsc::Receiver) { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/v1/audio/speech", listener.local_addr().unwrap()); + let (sender, receiver) = mpsc::channel(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let mut stream = stream.unwrap(); + let mut request = Vec::new(); + let mut byte = [0_u8; 1]; + while !request.ends_with(b"\r\n\r\n") { + stream.read_exact(&mut byte).unwrap(); + request.push(byte[0]); + } + let head = String::from_utf8(request).unwrap().to_ascii_lowercase(); + let length = head + .lines() + .find_map(|line| line.strip_prefix("content-length:")) + .unwrap() + .trim() + .parse() + .unwrap(); + let mut body = vec![0_u8; length]; + stream.read_exact(&mut body).unwrap(); + let _ = sender.send(serde_json::from_slice(&body).unwrap()); + let audio = vec![1_u8; 4_800]; + write!( + stream, + "HTTP/1.1 200 OK\r\ncontent-type: audio/pcm\r\ncontent-length: {}\r\nconnection: close\r\n\r\n", + audio.len() + ) + .unwrap(); + stream.write_all(&audio).unwrap(); + } + }); + (url, receiver) +} + +#[test] +fn expert_voice_keeps_a_separate_spokesperson_voice_control() { + let (speech_url, _speech_requests) = spawn_speech_server(); + let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); + let server = std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + endpoint_tx + .send(format!("ws://{}/", listener.local_addr().unwrap())) + .unwrap(); + let (old_stream, _) = listener.accept().await.unwrap(); + let mut old = accept_async(old_stream).await.unwrap(); + let initial = receive_realtime_json(&mut old).await; + assert_eq!(initial["session"]["audio"]["output"]["voice"], "cedar"); + assert_eq!(initial["session"]["audio"]["output"]["speed"], 1.25); + acknowledge_realtime_session(&mut old, &initial).await; + + let (candidate_stream, _) = listener.accept().await.unwrap(); + let mut candidate = accept_async(candidate_stream).await.unwrap(); + let replacement = receive_realtime_json(&mut candidate).await; + assert_eq!( + replacement["session"]["audio"]["output"]["voice"], + "new-voice" + ); + assert_eq!(replacement["session"]["audio"]["output"]["speed"], 1.25); + acknowledge_realtime_session(&mut candidate, &replacement).await; + while receive_realtime_json(&mut old).await["type"] != "input_audio_buffer.clear" {} + send_realtime_json(&mut old, json!({"type":"input_audio_buffer.cleared"})).await; + while candidate.next().await.is_some() {} + }); + }); + let endpoint = endpoint_rx.recv().unwrap(); + let mut session = ExpertSpokespersonTestSession::start_with_args( + endpoint, + &[ + "--allow-direct-agent-speech", + "--tts-url", + &speech_url, + "--spokesperson-voice", + "cedar", + "--spokesperson-rate", + "1.25", + ], + ); + assert_eq!(session.ready["session"]["tts"]["backend"], "openai"); + assert_eq!(session.ready["session"]["spokesperson"]["voice"], "cedar"); + session.send(json!({ + "type":"set_spokesperson_settings", + "id":2, + "expected_revision":1, + "settings":{"backend":"openai","model":"test-model","voice":"new-voice","rate":1.25} + })); + let applied = session.recv(Duration::from_secs(3)); + assert_eq!(applied["type"], "tts_settings_result", "{applied}"); + assert_eq!(applied["id"], 2); + assert_eq!(applied["outcome"], "applied", "{applied}"); + assert_eq!(applied["snapshot"]["revision"], 2); + assert_eq!(applied["snapshot"]["voice"], "new-voice"); + session.shutdown(); + server.join().unwrap(); +} + +enum ReplyInterruption { + UserSpeech, + Cancel, +} + +/// Stops an Expert reply partway through playback and returns what the session reports. +fn interrupt_expert_reply(interruption: ReplyInterruption) -> (Value, usize) { + let (speech_url, _speech_requests) = spawn_speech_server(); + let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); + let (user_speaks, user_speech) = mpsc::sync_channel(1); + std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + endpoint_tx + .send(format!("ws://{}/", listener.local_addr().unwrap())) + .unwrap(); + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = accept_async(stream).await.unwrap(); + let initial = receive_realtime_json(&mut socket).await; + acknowledge_realtime_session(&mut socket, &initial).await; + if user_speech.recv().is_ok() { + send_realtime_json( + &mut socket, + json!({"type":"input_audio_buffer.speech_started","item_id":"user-1"}), + ) + .await; + } + while socket.next().await.is_some() {} + }); + }); + let (observed, playing) = mpsc::sync_channel(1); + let (release_drain, release) = mpsc::sync_channel(1); + // The host has played part of the reply and is still playing it. + let mut session = ExpertSpokespersonTestSession::launch( + endpoint_rx.recv().unwrap(), + None, + Some(1_000), + None, + Some(AudioCancellationGate { + observed, + release, + at_drain: true, + }), + false, + &["--allow-direct-agent-speech", "--tts-url", &speech_url], + ); + let text = "I checked the repository and found three failing tests in the parser."; + session.send(json!({"type":"prepare_speak","id":2,"acknowledgement":null,"text":text})); + let admitted = session.recv(Duration::from_secs(2)); + assert_eq!(admitted["type"], "admitted", "{admitted}"); + session.send(json!({"type":"output_ready","id":2,"speech_id":admitted["speech_id"]})); + playing.recv_timeout(Duration::from_secs(2)).unwrap(); + release_drain.send(()).unwrap(); + match interruption { + ReplyInterruption::UserSpeech => user_speaks.send(()).unwrap(), + ReplyInterruption::Cancel => session.send(json!({"type":"cancel","id":2})), + } + let interrupted = loop { + let message = session.recv(Duration::from_secs(2)); + assert!( + !matches!( + message["type"].as_str(), + Some("fatal" | "speech_failed" | "speech_completed") + ), + "{message}" + ); + if message["type"] == "speech_interrupted" { + break message; + } + }; + drop(user_speaks); + session.shutdown(); + (interrupted, text.len()) +} + +#[test] +fn user_speech_interrupts_an_expert_reply_where_it_was_heard() { + let (interrupted, length) = interrupt_expert_reply(ReplyInterruption::UserSpeech); + assert_eq!(interrupted["id"], 2); + assert!(interrupted["spoken_through_utf8"].as_u64().unwrap() < length as u64); +} + +#[test] +fn cancel_interrupts_an_expert_reply_where_it_was_heard() { + let (interrupted, length) = interrupt_expert_reply(ReplyInterruption::Cancel); + assert_eq!(interrupted["id"], 2); + assert!(interrupted["spoken_through_utf8"].as_u64().unwrap() < length as u64); +} + +#[test] +fn expert_voice_reply_whose_audio_never_settles_ends_the_session() { + let (speech_url, _speech_requests) = spawn_speech_server(); + let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); + std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + endpoint_tx + .send(format!("ws://{}/", listener.local_addr().unwrap())) + .unwrap(); + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = accept_async(stream).await.unwrap(); + let initial = receive_realtime_json(&mut socket).await; + acknowledge_realtime_session(&mut socket, &initial).await; + while socket.next().await.is_some() {} + }); + }); + let (observed, cancel_observed) = mpsc::sync_channel(1); + let (release_cancel, release) = mpsc::sync_channel(1); + // The host never confirms the drain, nor the cancellation that follows it. + let mut session = ExpertSpokespersonTestSession::launch( + endpoint_rx.recv().unwrap(), + None, + None, + None, + Some(AudioCancellationGate { + observed, + release, + at_drain: true, + }), + false, + &["--allow-direct-agent-speech", "--tts-url", &speech_url], + ); + session.send(json!({ + "type":"prepare_speak", + "id":2, + "acknowledgement":null, + "text":"I checked the repository." + })); + let admitted = session.recv(Duration::from_secs(2)); + assert_eq!(admitted["type"], "admitted", "{admitted}"); + session.send(json!({"type":"output_ready","id":2,"speech_id":admitted["speech_id"]})); + let deadline = std::time::Instant::now() + Duration::from_secs(8); + let mut messages = Vec::new(); + loop { + let remaining = deadline + .checked_duration_since(std::time::Instant::now()) + .unwrap_or_else(|| panic!("the session went on: {messages:?}")); + match session.output.recv_timeout(remaining) { + Ok(message) => messages.push(message), + Err(mpsc::RecvTimeoutError::Disconnected) => break, + Err(mpsc::RecvTimeoutError::Timeout) => panic!("the session went on: {messages:?}"), + } + } + assert!( + messages + .iter() + .any(|message| message["type"] == "speech_failed" && message["id"] == 2), + "the reply did not fail: {messages:?}" + ); + let status = session.child.0.as_mut().unwrap().wait().unwrap(); + assert!(!status.success()); + cancel_observed.try_recv().unwrap(); + drop(release_cancel); +} + +#[test] +fn expert_voice_speaks_replies_itself_and_tells_the_spokesperson_what_was_heard() { + let (speech_url, speech_requests) = spawn_speech_server(); + let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); + let (context_tx, context_rx) = mpsc::sync_channel(1); + let server = std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + endpoint_tx + .send(format!("ws://{}/", listener.local_addr().unwrap())) + .unwrap(); + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = accept_async(stream).await.unwrap(); + let initial = receive_realtime_json(&mut socket).await; + acknowledge_realtime_session(&mut socket, &initial).await; + send_realtime_json( + &mut socket, + json!({ + "type":"response.function_call_arguments.done", + "response_id":"response-handoff", + "call_id":"handoff-reply", + "name":"handoff", + "arguments":"{\"message\":\"inspect the repository\"}" + }), + ) + .await; + let mut mentions = Vec::new(); + let context = loop { + let next = receive_realtime_json(&mut socket).await; + let text = next.to_string(); + if text.contains("in its own voice") { + break next; + } + if text.contains("I checked the repository.") { + mentions.push(next); + } + }; + let after = tokio::time::timeout(Duration::from_millis(300), async { + loop { + let next = receive_realtime_json(&mut socket).await; + if next["type"] == "response.create" { + break next; + } + } + }) + .await + .ok(); + context_tx.send((context, mentions, after)).unwrap(); + while socket.next().await.is_some() {} + }); + }); + let endpoint = endpoint_rx.recv().unwrap(); + let mut session = ExpertSpokespersonTestSession::start_with_args( + endpoint, + &["--allow-direct-agent-speech", "--tts-url", &speech_url], + ); + assert_eq!(session.ready["session"]["tts"]["backend"], "openai"); + assert_eq!( + session.ready["session"]["spokesperson"], + json!({"revision":1,"backend":"openai","model":"test-model","voice":"old-voice","rate":1.0}) + ); + assert_eq!(session.recv(Duration::from_secs(2))["origin"], "handoff"); + assert_eq!( + session.recv(Duration::from_secs(2))["handoff_ids"], + json!(["handoff-external-1"]) + ); + + session.send(json!({ + "type":"prepare_speak", + "id":2, + "acknowledgement":1, + "text":"I checked the repository.", + "voice":"coral" + })); + let admitted = session.recv(Duration::from_secs(2)); + assert_eq!(admitted["type"], "admitted", "{admitted}"); + session.send(json!({ + "type":"output_ready", + "id":2, + "speech_id":admitted["speech_id"] + })); + let deadline = std::time::Instant::now() + Duration::from_secs(3); + loop { + let remaining = deadline + .checked_duration_since(std::time::Instant::now()) + .expect("the Expert's reply did not complete"); + let message = session.recv(remaining); + assert_ne!(message["type"], "fatal", "{message}"); + assert_ne!(message["type"], "speech_failed", "{message}"); + if message["type"] == "speech_completed" { + assert_eq!(message["id"], 2); + assert_eq!(message["speech_id"], admitted["speech_id"]); + break; + } + } + let speech = speech_requests + .recv_timeout(Duration::from_secs(1)) + .unwrap(); + assert_eq!(speech["input"], "I checked the repository."); + assert_eq!(speech["voice"], "coral"); + session.send(json!({ + "type":"prepare_speak", + "id":3, + "acknowledgement":1, + "text":"Who am I?", + "voice":"nobody" + })); + let rejected = session.recv(Duration::from_secs(2)); + assert_eq!( + rejected, + json!({"type":"not_admitted","id":3,"reason":"invalid_voice"}) + ); + + let (context, mentions, after) = context_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + assert_eq!(context["type"], "conversation.item.create", "{context}"); + assert!(context.to_string().contains( + "The Expert said this to the user in its own voice:\\nI checked the repository." + )); + assert!( + mentions.is_empty(), + "the Spokesperson was asked to say it: {mentions:?}" + ); + assert!( + after.is_none(), + "the context note started a response: {after:?}" + ); + session.shutdown(); + server.join().unwrap(); +} + +#[test] +fn expert_voice_reply_waits_for_output_ready_before_playing() { + let (speech_url, speech_requests) = spawn_speech_server(); + let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); + std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + endpoint_tx + .send(format!("ws://{}/", listener.local_addr().unwrap())) + .unwrap(); + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = accept_async(stream).await.unwrap(); + let initial = receive_realtime_json(&mut socket).await; + acknowledge_realtime_session(&mut socket, &initial).await; + while socket.next().await.is_some() {} + }); + }); + let mut session = ExpertSpokespersonTestSession::start_with_args( + endpoint_rx.recv().unwrap(), + &["--allow-direct-agent-speech", "--tts-url", &speech_url], + ); + session.send(json!({ + "type":"prepare_speak", + "id":2, + "acknowledgement":null, + "text":"I checked the repository." + })); + let admitted = session.recv(Duration::from_secs(2)); + assert_eq!(admitted["type"], "admitted", "{admitted}"); + let speech_id = admitted["speech_id"].as_u64().unwrap(); + if let Ok(message) = session.output.recv_timeout(Duration::from_millis(500)) { + panic!("the reply went on before output_ready: {message}"); + } + assert!( + speech_requests.try_recv().is_err(), + "the reply was synthesized before output_ready" + ); + for (id, speech_id) in [(2, speech_id + 1), (3, speech_id)] { + session.send(json!({"type":"output_ready","id":id,"speech_id":speech_id})); + assert_eq!( + session.recv(Duration::from_secs(2)), + json!({"type":"output_ready_result","id":id,"speech_id":speech_id,"outcome":"stale"}) + ); + } + session.send(json!({"type":"output_ready","id":2,"speech_id":speech_id})); + assert_eq!( + session.recv(Duration::from_secs(2)), + json!({"type":"output_ready_result","id":2,"speech_id":speech_id,"outcome":"accepted"}) + ); + let mut started = false; + loop { + let message = session.recv(Duration::from_secs(3)); + match message["type"].as_str() { + Some("speech_started") => started = true, + Some("speech_completed") => break, + _ => panic!("unexpected message: {message}"), + } + } + assert!(started); + session.send(json!({"type":"output_ready","id":2,"speech_id":speech_id})); + assert_eq!(session.recv(Duration::from_secs(2))["outcome"], "stale"); + session.shutdown(); +} + +/// Starts a direct-speech call against a realtime server that accepts it and then listens. +fn start_direct_speech_session(speech_url: &str) -> ExpertSpokespersonTestSession { + let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); + std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + endpoint_tx + .send(format!("ws://{}/", listener.local_addr().unwrap())) + .unwrap(); + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = accept_async(stream).await.unwrap(); + let initial = receive_realtime_json(&mut socket).await; + acknowledge_realtime_session(&mut socket, &initial).await; + while socket.next().await.is_some() {} + }); + }); + ExpertSpokespersonTestSession::start_with_args( + endpoint_rx.recv().unwrap(), + &["--allow-direct-agent-speech", "--tts-url", speech_url], + ) +} + +#[test] +fn expert_voice_reply_without_output_ready_fails_and_frees_the_slot() { + let (speech_url, speech_requests) = spawn_speech_server(); + let mut session = start_direct_speech_session(&speech_url); + for id in [2, 3] { + session.send(json!({ + "type":"prepare_speak", + "id":id, + "acknowledgement":null, + "text":"I checked the repository." + })); + let admitted = session.recv(Duration::from_secs(2)); + assert_eq!(admitted["type"], "admitted", "{admitted}"); + if id == 3 { + break; + } + assert_eq!( + session.recv(Duration::from_secs(3)), + json!({ + "type":"speech_failed", + "id":id, + "speech_id":admitted["speech_id"], + "message":"output readiness timed out" + }) + ); + } + assert!(speech_requests.try_recv().is_err()); + session.shutdown(); +} + +#[test] +fn expert_voice_reply_keeps_the_voice_it_was_admitted_with() { + let (speech_url, speech_requests) = spawn_speech_server(); + let mut session = start_direct_speech_session(&speech_url); + let tts = session.ready["session"]["tts"].clone(); + session.send(json!({ + "type":"prepare_speak", + "id":2, + "acknowledgement":null, + "text":"I checked the repository." + })); + let admitted = session.recv(Duration::from_secs(2)); + assert_eq!(admitted["type"], "admitted", "{admitted}"); + session.send(json!({ + "type":"set_tts_settings", + "id":3, + "expected_revision":tts["revision"], + "settings":{"backend":"openai","model":tts["model"],"voice":"cedar","rate":1.0} + })); + let result = session.recv(Duration::from_secs(2)); + assert_eq!(result["outcome"], "applied", "{result}"); + session.send(json!({"type":"output_ready","id":2,"speech_id":admitted["speech_id"]})); + assert_eq!(session.recv(Duration::from_secs(2))["outcome"], "accepted"); + let request = speech_requests + .recv_timeout(Duration::from_secs(3)) + .unwrap(); + assert_eq!(request["voice"], tts["voice"], "{request}"); + session.shutdown(); +} + +/// Stops the host's player once it has played `played_frames` of an Expert reply, as when the +/// speakers change, returning how the reply ended and what the Spokesperson was told. +fn stop_expert_reply_after(played_frames: u64) -> (Value, String) { + let (speech_url, _speech_requests) = spawn_speech_server(); + let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); + let (context_tx, context_rx) = mpsc::sync_channel(1); + std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + endpoint_tx + .send(format!("ws://{}/", listener.local_addr().unwrap())) + .unwrap(); + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = accept_async(stream).await.unwrap(); + let initial = receive_realtime_json(&mut socket).await; + acknowledge_realtime_session(&mut socket, &initial).await; + let context = loop { + let text = receive_realtime_json(&mut socket).await.to_string(); + if text.contains("spoken reply") || text.contains("in its own voice") { + break text; + } + }; + context_tx.send(context).unwrap(); + while socket.next().await.is_some() {} + }); + }); + let (observed, playing) = mpsc::sync_channel(1); + let (release_drain, release) = mpsc::sync_channel(1); + let mut session = ExpertSpokespersonTestSession::launch( + endpoint_rx.recv().unwrap(), + None, + Some(played_frames), + None, + Some(AudioCancellationGate { + observed, + release, + at_drain: true, + }), + false, + &["--allow-direct-agent-speech", "--tts-url", &speech_url], + ); + session.send(json!({ + "type":"prepare_speak", + "id":2, + "acknowledgement":null, + "text":"I checked the repository and found three failing tests in the parser." + })); + let admitted = session.recv(Duration::from_secs(2)); + assert_eq!(admitted["type"], "admitted", "{admitted}"); + session.send(json!({"type":"output_ready","id":2,"speech_id":admitted["speech_id"]})); + playing.recv_timeout(Duration::from_secs(2)).unwrap(); + release_drain.send(()).unwrap(); + session.send(json!({ + "type":"audio_failed", + "speech_id":admitted["speech_id"], + "played_frames":played_frames, + "message":"output device disappeared" + })); + let ended = loop { + let message = session.recv(Duration::from_secs(2)); + assert!( + !matches!( + message["type"].as_str(), + Some("fatal" | "speech_completed" | "speech_failed") + ), + "{message}" + ); + if message["type"] == "speech_interrupted" { + break message; + } + }; + let context = context_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + session.shutdown(); + (ended, context) +} + +#[test] +fn an_expert_reply_whose_player_stops_tells_the_spokesperson_what_was_heard() { + let (ended, context) = stop_expert_reply_after(1_000); + assert_eq!(ended["id"], 2); + assert!( + context.contains("in its own voice") && context.contains("I checked the repository"), + "{context}" + ); + assert!(!context.contains("in the parser"), "{context}"); +} + +#[test] +fn an_expert_reply_whose_player_stops_unheard_tells_the_spokesperson_so() { + let (ended, context) = stop_expert_reply_after(0); + assert_eq!(ended["id"], 2); + assert!(context.contains("before hearing any of it"), "{context}"); +} + +#[test] +fn a_message_to_the_spokesperson_is_a_directive_it_says() { + let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); + let (relay_tx, relay_rx) = mpsc::sync_channel(1); + let server = std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + endpoint_tx + .send(format!("ws://{}/", listener.local_addr().unwrap())) + .unwrap(); + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = accept_async(stream).await.unwrap(); + let initial = receive_realtime_json(&mut socket).await; + acknowledge_realtime_session(&mut socket, &initial).await; + for (call_id, message) in [ + ("handoff-1", "inspect the repository"), + ("handoff-2", "check the tests"), + ] { + send_realtime_json( + &mut socket, + json!({ + "type":"response.function_call_arguments.done", + "response_id":format!("response-{call_id}"), + "call_id":call_id, + "name":"handoff", + "arguments":json!({"message":message}).to_string() + }), + ) + .await; + while receive_realtime_json(&mut socket) + .await + .pointer("/item/type") + != Some(&json!("function_call_output")) + {} + } + while let Some(Ok(Message::Text(text))) = socket.next().await { + if text.contains("I checked the repository.") { + let _ = relay_tx.try_send(serde_json::from_str::(&text).unwrap()); + } + } + }); + }); + let mut session = ExpertSpokespersonTestSession::start(endpoint_rx.recv().unwrap()); + let handoffs = loop { + session.send(json!({"type":"query_state","id":1,"after":0})); + let state = loop { + let message = session.recv(Duration::from_secs(2)); + assert_ne!(message["type"], "fatal", "{message}"); + if message["type"] == "state" { + break message; + } + }; + if let Some(ids) = state["unresolved_handoff_ids"] + .as_array() + .filter(|ids| ids.len() == 2) + { + break ids.clone(); + } + std::thread::sleep(Duration::from_millis(20)); + }; + + // A stale message waits for what it missed, like a directive. + let prepare = |acknowledgement: &Value| { + json!({ + "type":"prepare_speak", + "id":2, + "acknowledgement":acknowledgement, + "resolved_handoff_ids":[&handoffs[0]], + "text":"I checked the repository.", + "relay":true + }) + }; + let reply = |session: &mut ExpertSpokespersonTestSession, request| { + session.send(request); + loop { + let message = session.recv(Duration::from_secs(2)); + assert_ne!(message["type"], "fatal", "{message}"); + if message["id"] == 2 { + break message; + } + } + }; + let pending = reply(&mut session, prepare(&json!(0))); + assert_eq!(pending["type"], "pending", "{pending}"); + let latest = &pending["utterances"].as_array().unwrap().last().unwrap()["token"]; + // Current, it is the Spokesperson's to say, and answers only the handoff it names. + let admitted = reply(&mut session, prepare(latest)); + assert_eq!(admitted["type"], "admitted", "{admitted}"); + let relayed = relay_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + assert!( + relayed.to_string().contains("Speak it naturally"), + "{relayed}" + ); + session.send(json!({"type":"query_state","id":3,"after":0})); + let state = loop { + let message = session.recv(Duration::from_secs(2)); + if message["type"] == "state" { + break message; + } + }; + assert_eq!( + state["unresolved_handoff_ids"], + json!([&handoffs[1]]), + "{state}" + ); + session.shutdown(); + server.join().unwrap(); +} + +#[test] +fn handoffs_hold_nothing_open_when_the_expert_may_speak() { + let (speech_url, speech_requests) = spawn_speech_server(); + let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); + let (renewed_tx, renewed_rx) = mpsc::sync_channel(1); + let server = std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + endpoint_tx + .send(format!("ws://{}/", listener.local_addr().unwrap())) + .unwrap(); + let (old_stream, _) = listener.accept().await.unwrap(); + let mut old = accept_async(old_stream).await.unwrap(); + let initial = receive_realtime_json(&mut old).await; + acknowledge_realtime_session(&mut old, &initial).await; + send_realtime_json( + &mut old, + json!({ + "type":"conversation.item.input_audio_transcription.completed", + "item_id":"user-1", + "transcript":"What changed in the repository?" + }), + ) + .await; + send_realtime_json( + &mut old, + json!({ + "type":"response.function_call_arguments.done", + "response_id":"response-handoff", + "call_id":"handoff-1", + "name":"handoff", + "arguments":"{\"message\":\"inspect the repository\"}" + }), + ) + .await; + while receive_realtime_json(&mut old).await.pointer("/item/type") + != Some(&json!("function_call_output")) + {} + send_realtime_json( + &mut old, + json!({ + "type":"error", + "error":{"code":"session_expired","message":"provider session expired"} + }), + ) + .await; + + // The handoff was never held open, so the call renews. + let (stream, _) = listener.accept().await.unwrap(); + let mut replacement = accept_async(stream).await.unwrap(); + let update = receive_realtime_json(&mut replacement).await; + acknowledge_realtime_session(&mut replacement, &update).await; + renewed_tx.send(()).unwrap(); + while let Some(Ok(Message::Text(text))) = replacement.next().await { + let seed: Value = serde_json::from_str(&text).unwrap(); + if seed["type"] == "conversation.item.create" { + send_realtime_json( + &mut replacement, + json!({"type":"conversation.item.created","item":{"id":seed["item"]["id"]}}), + ) + .await; + } + } + }); + }); + let endpoint = endpoint_rx.recv().unwrap(); + let mut session = ExpertSpokespersonTestSession::start_with_args( + endpoint, + &["--allow-direct-agent-speech", "--tts-url", &speech_url], + ); + renewed_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + + // A reply written before everything since waits for it, and is said once + // it acknowledges the latest. + session.send(json!({ + "type":"prepare_speak", + "id":2, + "acknowledgement":0, + "text":"I checked the repository." + })); + let pending = loop { + let message = session.recv(Duration::from_secs(2)); + assert_ne!(message["type"], "fatal", "{message}"); + if message["id"] == 2 { + break message; + } + }; + assert_eq!(pending["type"], "pending", "{pending}"); + let latest = pending["utterances"].as_array().unwrap().last().unwrap()["token"].clone(); + session.send(json!({ + "type":"prepare_speak", + "id":2, + "acknowledgement":latest, + "text":"I checked the repository." + })); + let admitted = loop { + let message = session.recv(Duration::from_secs(2)); + assert_ne!(message["type"], "fatal", "{message}"); + if message["id"] == 2 { + break message; + } + }; + assert_eq!(admitted["type"], "admitted", "{admitted}"); + session.send(json!({"type":"query_state","id":3,"after":0})); + session.send(json!({ + "type":"output_ready", + "id":2, + "speech_id":admitted["speech_id"] + })); + let (mut state, mut completed) = (None, false); + while state.is_none() || !completed { + let message = session.recv(Duration::from_secs(3)); + assert_ne!(message["type"], "fatal", "{message}"); + assert_ne!(message["type"], "speech_failed", "{message}"); + match message["type"].as_str() { + Some("state") => state = Some(message), + Some("speech_completed") => completed = true, + _ => {} + } + } + assert_eq!(state.unwrap().get("unresolved_handoff_ids"), None); + assert_eq!( + speech_requests + .recv_timeout(Duration::from_secs(1)) + .unwrap()["input"], + "I checked the repository." + ); + session.shutdown(); + server.join().unwrap(); +}