From 79facf377fcc39fefefd6d3eeb28b3498631865e Mon Sep 17 00:00:00 2001 From: John Tennant Date: Mon, 19 Oct 2026 15:44:43 -0400 Subject: [PATCH 01/49] berd-call: stream realtime calls live or at handoffs An Expert driving a realtime call from the CLI should hear the conversation the way a GPT-Live delegate does: only when the Spokesperson hands off, with everything said since the last handoff. Until now `start --stream` and `--codex` also flushed after every Spokesperson reply, waking the Expert on each turn, while a host that shows the call as it happens saw the user's words only once the reply had played. Bare `--stream` (or `--stream handoffs`) and `--codex` now hold rows until a handoff. `--stream live` shows the conversation as it happens: a `user` row as soon as the words are transcribed, and a `spokesperson_generated` row as soon as a reply is generated, before it is spoken and only once no user speech is still being transcribed. The session emits these as `user_transcribed` and `spokesperson_generated` previews; the transcript recorded after playback stays authoritative. Co-Authored-By: Claude Opus 5.5 --- src-tauri/crates/berd-call/PROTOCOL.md | 15 ++ src-tauri/crates/berd-call/README.md | 11 +- src-tauri/crates/berd-call/src/cli_help.rs | 9 +- .../crates/berd-call/src/host_session.rs | 128 +++++++++++++++-- src-tauri/crates/berd-call/src/main.rs | 131 +++++++++++++++++- src-tauri/crates/berd-call/src/protocol.rs | 12 ++ .../berd-call/src/realtime_host_lifecycle.rs | 5 + 7 files changed, 290 insertions(+), 21 deletions(-) diff --git a/src-tauri/crates/berd-call/PROTOCOL.md b/src-tauri/crates/berd-call/PROTOCOL.md index e1b7b7e62..1c8922022 100644 --- a/src-tauri/crates/berd-call/PROTOCOL.md +++ b/src-tauri/crates/berd-call/PROTOCOL.md @@ -281,6 +281,21 @@ Expert-Spokesperson sessions use `origin` to distinguish user transcripts, Spokesperson transcripts, and handoffs. User finals interrupt reserved or playing assistant output only after storage and emission. Final text is at most 64 KiB. +Expert-Spokesperson sessions also emit two previews for hosts that show the +conversation as it happens: + +```text +{"type":"user_transcribed","token":u64,"text":string} +{"type":"spokesperson_generated","text":string} +``` + +`user_transcribed` follows a user final's `live_event` with its token and plain +text. `spokesperson_generated` carries a reply's whole text once the provider +finishes generating it, before playback, and only after no user speech is still +being transcribed. A reply already spoken or cut off by then gets none. The +Spokesperson transcript recorded after playback stays authoritative. Neither +message changes Expert delivery. + ## Confirmation and admission `acknowledgement` is a request-local causal cutoff. `null` uses the stored diff --git a/src-tauri/crates/berd-call/README.md b/src-tauri/crates/berd-call/README.md index 6eb5cd441..752a84bc1 100644 --- a/src-tauri/crates/berd-call/README.md +++ b/src-tauri/crates/berd-call/README.md @@ -64,7 +64,16 @@ berd-call stop server listens only on `127.0.0.1:5222` by default; all four commands accept `--port` when more than one local call must be addressed. `--stream` writes one TSV row per delivered transcript event after the `cursor`, `role`, and `text` -header. The standalone host intentionally uses only the default devices. It +header. In Expert-Spokesperson mode the Expert, like a GPT-Live delegate, hears +the conversation only when the Spokesperson hands off: the stream prints +nothing until then, and then every row since the last handoff, ending with the +`handoff` row. `--stream live` instead shows the conversation as it happens: a +`user` row as soon as the words are transcribed, and a `spokesperson_generated` +row with cursor `0` as soon as a reply is generated, before it is spoken. The +`spokesperson` or `spokesperson_interrupted` row after playback carries the +reply's cursor and what was heard. A conventional call has no handoffs, so it +always streams each row. A `--codex` task, like `--stream`, wakes only at +handoffs. The standalone host intentionally uses only the default devices. It suppresses recognition during playback when the default output identifies itself as built-in speakers, preventing the call from transcribing its own speech; headphone-like outputs retain barge-in. diff --git a/src-tauri/crates/berd-call/src/cli_help.rs b/src-tauri/crates/berd-call/src/cli_help.rs index 157e0a26d..cc5448b85 100644 --- a/src-tauri/crates/berd-call/src/cli_help.rs +++ b/src-tauri/crates/berd-call/src/cli_help.rs @@ -33,14 +33,17 @@ Run `berd-call help ` for command-specific usage."#; const START_HELP: &str = r#"Run a local voice call backed by the shared session runtime. Usage: - berd-call start [--port PORT] [--stream | --codex] [--non-blocking] + berd-call start [--port PORT] [--stream [handoffs|live] | --codex] [--non-blocking] [--no-menu-bar] --voice NAME --language BCP47 [session options] The foreground process owns the default microphone and audio output. Session options are the same as `berd-call session`, except the PCM descriptor is owned -internally. `--stream` prints cursor, role, and text as TSV. `--codex` delivers -the same records to the calling Codex task (requires CODEX_THREAD_ID). A +internally. `--stream` prints cursor, role, and text as TSV. In +Expert-Spokesperson mode it prints nothing until the Spokesperson hands off, +then everything since the last handoff; `--stream live` prints each row as it +happens. `--codex` delivers the handoff batches to the calling Codex task +(requires CODEX_THREAD_ID). A menu-bar item controls speech rate, input muting, and ending the call; pass `--no-menu-bar` to omit it. diff --git a/src-tauri/crates/berd-call/src/host_session.rs b/src-tauri/crates/berd-call/src/host_session.rs index d827ae2fc..e28968fcf 100644 --- a/src-tauri/crates/berd-call/src/host_session.rs +++ b/src-tauri/crates/berd-call/src/host_session.rs @@ -1,6 +1,6 @@ #![cfg(target_os = "macos")] -use std::collections::VecDeque; +use std::collections::{HashSet, VecDeque}; use std::fs::File; use std::io::{BufRead, BufReader, Read, Write}; use std::os::fd::{FromRawFd, RawFd}; @@ -145,7 +145,7 @@ pub(crate) fn run(options: StartOptions) -> Result<(), String> { /// Where live transcript records go: nowhere, stdout TSV, or a Codex task. pub(crate) enum Transcript { Silent, - Stdout, + Stdout { live: bool }, Codex(CodexRelay), } @@ -153,7 +153,7 @@ impl Transcript { fn for_options(options: &StartOptions) -> Result { Ok(match options.transcript { TranscriptDestination::None => Self::Silent, - TranscriptDestination::Stdout => Self::Stdout, + TranscriptDestination::Stdout { live } => Self::Stdout { live }, TranscriptDestination::Codex => { let target = CodexTarget::from_environment()?; let executable = std::env::current_exe().map_err(|error| { @@ -174,7 +174,7 @@ impl Transcript { fn start(&self) -> Result<(), String> { match self { - Self::Stdout => print_flushed("cursor\trole\ttext"), + Self::Stdout { .. } => print_flushed("cursor\trole\ttext"), Self::Codex(_) => { println!("Berd Call is delivering transcripts to this Codex task."); Ok(()) @@ -192,7 +192,7 @@ impl Transcript { ) -> Result<(), String> { match self { Self::Silent => Ok(()), - Self::Stdout => print_flushed(&format!( + Self::Stdout { .. } => print_flushed(&format!( "{}\t{role}\t{}", cursor.unwrap_or_default(), stream_text(text) @@ -213,7 +213,7 @@ impl Transcript { fn finish(&self, text: &str) { match self { Self::Silent => {} - Self::Stdout => { + Self::Stdout { .. } => { let _ = self.record(None, "lifecycle", None, text); } Self::Codex(relay) => relay.finish(Some(CodexRecord { @@ -771,6 +771,10 @@ struct SessionActor { session: Arc>, transcript: Arc, expert_spokesperson: bool, + // User rows already streamed when transcribed, left out of the Expert batch. + transcribed: HashSet, + // Rows a handoffs stream or Codex task gets only when the Spokesperson hands off. + held: Vec, stopping: bool, restart: Option, muted: Arc, @@ -831,6 +835,8 @@ impl SessionActor { session, transcript, expert_spokesperson, + transcribed: HashSet::new(), + held: Vec::new(), stopping: false, restart: None, muted: Arc::new(AtomicBool::new(false)), @@ -1191,8 +1197,27 @@ impl SessionActor { text, origin, } if !self.expert_spokesperson => self.stream_live_event(token, origin, &text)?, - SessionMessage::ExpertDelivery { events, .. } if self.expert_spokesperson => { - self.stream_expert_delivery(&events)? + // Only a live stream shows the conversation as it happens. + SessionMessage::UserTranscribed { token, text } + if self.expert_spokesperson + && matches!(*self.transcript, Transcript::Stdout { live: true }) => + { + self.transcript.record(Some(token), "user", None, &text)?; + self.transcribed.insert(token); + } + SessionMessage::SpokespersonGenerated { text } + if self.expert_spokesperson + && matches!(*self.transcript, Transcript::Stdout { live: true }) => + { + self.transcript + .record(None, "spokesperson_generated", None, &text)? + } + SessionMessage::ExpertDelivery { + events, + handoff_ids, + .. + } if self.expert_spokesperson => { + self.stream_expert_delivery(events, !handoff_ids.is_empty())? } SessionMessage::Pending { id, utterances } if self.pending_prepare_id() == Some(id) => { self.respond_speak(Ok(json!({"spoke":false,"utterances":utterances}))); @@ -1306,8 +1331,21 @@ impl SessionActor { self.transcript.record(Some(token), role, None, text) } - fn stream_expert_delivery(&self, events: &[RealtimeExpertDeliveryEvent]) -> Result<(), String> { - for event in events { + fn stream_expert_delivery( + &mut self, + events: Vec, + handoff: bool, + ) -> Result<(), String> { + self.held.extend(events); + if !handoff + && matches!( + *self.transcript, + Transcript::Stdout { live: false } | Transcript::Codex(_) + ) + { + return Ok(()); + } + for event in unstreamed(&std::mem::take(&mut self.held), &mut self.transcribed) { self.transcript.record( Some(event.cursor), expert_delivery_role_name(event.role), @@ -2537,6 +2575,16 @@ fn not_admitted_reason_name(reason: NotAdmittedReason) -> &'static str { } } +/// The batch's events not already streamed as they happened. +fn unstreamed<'a>( + events: &'a [RealtimeExpertDeliveryEvent], + transcribed: &'a mut HashSet, +) -> impl Iterator { + events.iter().filter(move |event| { + !(event.role == RealtimeExpertDeliveryRole::User && transcribed.remove(&event.cursor)) + }) +} + fn expert_delivery_role_name(role: RealtimeExpertDeliveryRole) -> &'static str { match role { RealtimeExpertDeliveryRole::User => "user", @@ -2551,6 +2599,64 @@ fn expert_delivery_role_name(role: RealtimeExpertDeliveryRole) -> &'static str { mod tests { use super::*; + #[test] + fn expert_batches_leave_out_user_rows_already_streamed() { + let event = |cursor, role| RealtimeExpertDeliveryEvent { + cursor, + role, + text: format!("event {cursor}"), + handoff_id: None, + }; + let events = [ + event(1, RealtimeExpertDeliveryRole::User), + event(2, RealtimeExpertDeliveryRole::User), + event(3, RealtimeExpertDeliveryRole::Spokesperson), + ]; + let mut transcribed = HashSet::from([1, 3]); + let cursors: Vec<_> = unstreamed(&events, &mut transcribed) + .map(|event| event.cursor) + .collect(); + assert_eq!(cursors, [2, 3]); + assert_eq!(transcribed, HashSet::from([3])); + } + + #[test] + fn handoffs_stream_holds_rows_until_a_handoff() { + let mut child = Command::new("/bin/cat") + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .spawn() + .unwrap(); + let writer = Arc::new(Mutex::new(child.stdin.take().unwrap())); + let (_event_tx, events) = mpsc::sync_channel(1); + let (_command_tx, commands) = mpsc::sync_channel(1); + let (audio, _audio_rx) = mpsc::sync_channel(1); + let mut actor = test_actor(writer, events, commands, audio); + let event = |cursor, role| RealtimeExpertDeliveryEvent { + cursor, + role, + text: format!("event {cursor}"), + handoff_id: None, + }; + actor + .stream_expert_delivery( + vec![ + event(1, RealtimeExpertDeliveryRole::User), + event(2, RealtimeExpertDeliveryRole::Spokesperson), + ], + false, + ) + .unwrap(); + assert_eq!(actor.held.len(), 2); + actor + .stream_expert_delivery(vec![event(3, RealtimeExpertDeliveryRole::Handoff)], true) + .unwrap(); + assert!(actor.held.is_empty()); + drop(actor); + let _ = child.kill(); + let _ = child.wait(); + } + #[test] fn microphone_teardown_reports_a_blocked_forwarder() { let (_release, blocked) = mpsc::sync_channel::<()>(1); @@ -3205,7 +3311,7 @@ mod tests { commands, audio_commands, session, - Arc::new(Transcript::Stdout), + Arc::new(Transcript::Stdout { live: false }), false, ) } diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 7e9d0cb1a..7346405ec 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -732,7 +732,11 @@ fn request_bundled_app_update(app_path: &Path) { #[derive(Clone, Copy, Debug, PartialEq)] enum TranscriptDestination { None, - Stdout, + /// `live` streams an Expert-Spokesperson call as it happens; otherwise + /// its rows wait for the next handoff. + Stdout { + live: bool, + }, Codex, } @@ -777,12 +781,17 @@ fn parse_start_args(args: &[String]) -> Result { if transcript != TranscriptDestination::None { return Err("choose one of --stream or --codex for transcript delivery".into()); } - transcript = if flag == "--stream" { - TranscriptDestination::Stdout - } else { + index += 1; + transcript = if flag == "--codex" { TranscriptDestination::Codex + } else { + let mode = args.get(index).map(String::as_str); + let live = mode == Some("live"); + if matches!(mode, Some("live" | "handoffs")) { + index += 1; + } + TranscriptDestination::Stdout { live } }; - index += 1; } "--no-menu-bar" if menu_bar => { menu_bar = false; @@ -3153,6 +3162,8 @@ fn run_expert_spokesperson_session( let mut waiting_responses = VecDeque::::new(); let mut active: Option = None; let mut turn_gate = ExpertTurnGate::new(spokesperson_renew_after()); + // Generated replies waiting to be shown until the words they answer are. + let mut generated = Vec::::new(); let mut session_tts: Option = None; let mut input_during_tts_slot: Option = None; let mut input_muted = false; @@ -3363,6 +3374,12 @@ fn run_expert_spokesperson_session( } SpokespersonEvent::UserTurnDiscarded { item_id } => { turn_gate.discard_user_turn(&item_id); + show_generated_replies( + &mut generated, + &responses, + &turn_gate, + &mut writer, + )?; } SpokespersonEvent::UserFinal { item_id, text } => { turn_gate.resolve_user_final(&item_id); @@ -3372,7 +3389,20 @@ fn run_expert_spokesperson_session( LiveSideEvent::UserTranscript { text: text.clone() }, &mut writer, )?; + write_message( + &mut writer, + &SessionMessage::UserTranscribed { + token: emitted_live_token, + text: text.clone(), + }, + )?; core.record_user_turn(text); + show_generated_replies( + &mut generated, + &responses, + &turn_gate, + &mut writer, + )?; } SpokespersonEvent::ResponseStarted { response_id } => { turn_gate.response_started(&response_id, responses.len())?; @@ -3431,7 +3461,19 @@ fn run_expert_spokesperson_session( response.playback_complete = true; } } + if status == SpokespersonResponseStatus::Completed + && response.delivery.has_transcript() + && !response.playback_complete + { + generated.push(response_id.clone()); + } } + show_generated_replies( + &mut generated, + &responses, + &turn_gate, + &mut writer, + )?; publish_live_response_if_complete( &response_id, &mut responses, @@ -5148,6 +5190,34 @@ fn publish_live_response_if_complete( Ok(()) } +/// Shows each generated reply before it is spoken, once no user words are +/// still being transcribed, so a live transcript reads in order. A reply +/// already spoken or cut off is left to its authoritative transcript. +fn show_generated_replies( + generated: &mut Vec, + responses: &HashMap, + turn_gate: &ExpertTurnGate, + writer: &mut impl Write, +) -> Result<(), String> { + if turn_gate.lifecycle.transcribing() { + return Ok(()); + } + for response_id in generated.drain(..) { + if let Some(response) = responses + .get(&response_id) + .filter(|response| !response.playback_complete && !response.interrupted) + { + write_message( + writer, + &SessionMessage::SpokespersonGenerated { + text: response.delivery.transcript(), + }, + )?; + } + } + Ok(()) +} + fn live_truncation_pending(responses: &HashMap) -> bool { responses .values() @@ -8063,6 +8133,40 @@ mod tests { assert_eq!(emitted[1]["type"], "expert_delivery"); } + #[test] + fn generated_replies_wait_for_the_words_they_answer_and_are_shown_once() { + let reply = |text: &str| { + let mut response = LiveResponse::new(None, None); + response + .delivery + .replace_transcript("assistant", 0, 0, text.into()) + .unwrap(); + response + }; + let mut spoken = reply("Already spoken."); + spoken.playback_complete = true; + let responses = HashMap::from([ + ("response-a".into(), reply("Twelve.")), + ("response-b".into(), spoken), + ]); + let mut turn_gate = ExpertTurnGate::new(Duration::from_secs(60)); + turn_gate.begin_user_speaking("user-1".into()); + turn_gate.finish_user_speaking(); + let mut generated = vec!["response-a".to_string(), "response-b".to_string()]; + let mut output = Vec::new(); + + show_generated_replies(&mut generated, &responses, &turn_gate, &mut output).unwrap(); + assert!(output.is_empty()); + + turn_gate.resolve_user_final("user-1"); + show_generated_replies(&mut generated, &responses, &turn_gate, &mut output).unwrap(); + show_generated_replies(&mut generated, &responses, &turn_gate, &mut output).unwrap(); + let emitted = messages(&output); + assert_eq!(emitted.len(), 1); + assert_eq!(emitted[0]["type"], "spokesperson_generated"); + assert_eq!(emitted[0]["text"], "Twelve."); + } + #[test] fn late_audio_after_interruption_requires_zero_ms_truncation_before_release() { let mut response = LiveResponse::new(None, None); @@ -11355,7 +11459,10 @@ mod tests { ])) .unwrap(); assert_eq!(parsed.port, 5300); - assert_eq!(parsed.transcript, TranscriptDestination::Stdout); + assert_eq!( + parsed.transcript, + TranscriptDestination::Stdout { live: false } + ); assert!(!parsed.expert_spokesperson); assert_eq!( parsed.session_arguments, @@ -11385,6 +11492,18 @@ mod tests { let parsed = start(&["--codex", "--non-blocking"]).unwrap(); assert_eq!(parsed.transcript, TranscriptDestination::Codex); assert!(parsed.non_blocking); + for (flags, live) in [ + (&["--stream"][..], false), + (&["--stream", "handoffs"], false), + (&["--stream", "live", "--non-blocking"], true), + ] { + let parsed = start(flags).unwrap(); + assert_eq!(parsed.transcript, TranscriptDestination::Stdout { live }); + assert_eq!( + parsed.session_arguments[1..], + ["--voice", "Aaron", "--language", "en-US"] + ); + } assert!(start(&["--codex", "--stream"]).is_err()); assert!(start(&["--codex", "--codex"]).is_err()); assert!(start(&["--non-blocking"]).is_err()); diff --git a/src-tauri/crates/berd-call/src/protocol.rs b/src-tauri/crates/berd-call/src/protocol.rs index 9767ee5c2..32eca4e91 100644 --- a/src-tauri/crates/berd-call/src/protocol.rs +++ b/src-tauri/crates/berd-call/src/protocol.rs @@ -235,6 +235,18 @@ pub enum SessionMessage { #[serde(skip_serializing_if = "Option::is_none")] origin: Option, }, + /// The user's words as soon as they are transcribed, for hosts that show + /// the conversation as it happens. `live_event` stays authoritative. + UserTranscribed { + token: u64, + text: String, + }, + /// The Spokesperson's whole reply once it is generated, before it is + /// spoken. The transcript recorded after playback stays authoritative and + /// says how much was heard when the user cut it off. + SpokespersonGenerated { + text: String, + }, /// A crate-selected batch that is ready to wake the external Expert. /// `live_event` remains the authoritative transcript projection; this /// message alone controls when the host schedules an Expert turn. diff --git a/src-tauri/crates/berd-call/src/realtime_host_lifecycle.rs b/src-tauri/crates/berd-call/src/realtime_host_lifecycle.rs index f34009e76..ac70ea306 100644 --- a/src-tauri/crates/berd-call/src/realtime_host_lifecycle.rs +++ b/src-tauri/crates/berd-call/src/realtime_host_lifecycle.rs @@ -90,6 +90,11 @@ impl RealtimeHostLifecycle { self.activity.user_speaking } + /// Whether user speech has started that is not yet transcribed or discarded. + pub fn transcribing(&self) -> bool { + !self.activity.pending_user_items.is_empty() + } + pub fn is_busy(&self, work: RealtimeHostWork) -> bool { self.activity.is_busy(work) } From a200e60b04cf4171854a5d182e6c8cf0e9d11702 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Mon, 19 Oct 2026 16:41:35 -0400 Subject: [PATCH 02/49] berd-call: let the Expert speak in its own voice in realtime calls With `--expert-voice tts`, an Expert-Spokesperson session builds the configured TTS backend beside the realtime connection. Each reply plays through that backend in the Spokesperson's playback slot instead of being read aloud by the realtime model, so the same turn gates and barge-in apply to both voices. Afterwards the realtime session gets a silent note of what the user heard, and the handoffs the reply answers settle once any of it was heard. `session.tts` and `set_tts_settings` now describe the Expert's voice in that mode, and every Expert-Spokesperson session reports the realtime model and voice as `session.spokesperson`. Co-Authored-By: Claude Opus 5.5 Signed-off-by: John Tennant --- src-tauri/crates/berd-call/PROTOCOL.md | 6 + src-tauri/crates/berd-call/README.md | 5 +- src-tauri/crates/berd-call/src/cli_help.rs | 8 +- .../crates/berd-call/src/host_session.rs | 1 + src-tauri/crates/berd-call/src/main.rs | 466 +++++++++++++++++- .../berd-call/src/openai_realtime_protocol.rs | 10 + src-tauri/crates/berd-call/src/protocol.rs | 17 +- .../crates/berd-call/src/saved_settings.rs | 2 +- .../berd-call/tests/session_protocol.rs | 187 ++++++- 9 files changed, 674 insertions(+), 28 deletions(-) diff --git a/src-tauri/crates/berd-call/PROTOCOL.md b/src-tauri/crates/berd-call/PROTOCOL.md index 1c8922022..c1655f70a 100644 --- a/src-tauri/crates/berd-call/PROTOCOL.md +++ b/src-tauri/crates/berd-call/PROTOCOL.md @@ -72,6 +72,12 @@ OpenAI snapshots contain `model`, `voice`, and `rate`; Siri contains `voice`, and `rate`. Credentials, endpoints, and bundle paths never appear on stdout. Detailed backend errors are diagnostics on stderr only; protocol rejection and fatal messages are sanitized at the stdout boundary. +In Expert-Spokesperson sessions, `session.spokesperson` holds the realtime +`model` and `voice`. `session.tts` describes the voice that speaks the Expert's +replies: the configured TTS backend with `--expert-voice tts`, otherwise the +Spokesperson's own voice, and `set_tts_settings` changes that voice. A reply in +the Expert's own voice reports the same `admitted`, `speech_completed`, +`speech_interrupted`, and `speech_failed` lifecycle as a conventional reply. `session.input_during_tts` is the authoritative effective assistant-input policy and has its own revision. Status-sound settings are per-update parameters, not session snapshot configuration. The runtime owns cue cadence and playback; it diff --git a/src-tauri/crates/berd-call/README.md b/src-tauri/crates/berd-call/README.md index 752a84bc1..36ce6b8a1 100644 --- a/src-tauri/crates/berd-call/README.md +++ b/src-tauri/crates/berd-call/README.md @@ -142,11 +142,14 @@ berd-call session --tts-backend pocket --model-dir /path/to/native-voice-v2 --vo berd-call session --stt-backend parakeet --stt-model-dir /path/to/parakeet berd-call session --stt-backend openai berd-call session --tts-backend openai --mode expert-spokesperson --realtime-url ws://127.0.0.1:18870/v1/realtime +berd-call session --voice Aaron --language en-US --mode expert-spokesperson --expert-voice tts berd-call session --tts-backend openai --tts-url https://proxy.example/v1/audio/speech berd-call session --stt-backend openai --stt-url wss://proxy.example/v1/realtime?intent=transcription ``` -The three URL flags take full endpoints and override only their matching service for that call. Expert–Spokesperson uses `--realtime-url` for both input and output; `--stt-url` and `--tts-url` apply only to conventional mode. Without an override, each service uses its OpenAI endpoint (or its existing environment override). `OPENAI_API_KEY` supplies the CLI credential; the desktop app stores keys separately by endpoint URL in Keychain. +By default the Spokesperson reads the Expert's replies aloud in its own voice. With `--expert-voice tts`, the Expert speaks for itself: berd-call plays each reply through the configured TTS backend, taking turns with the Spokesperson's audio, and your speech interrupts either of them. Afterwards the Spokesperson gets a silent note of what you heard, so it can follow up without repeating it. A reply settles the handoffs it answers once you hear any of it. Its rate applies to the Expert's voice, and the Spokesperson's speed is that rate limited to 0.25–1.5. `status` and `settings --tts` describe and change the Expert's voice; the Spokesperson's model and voice appear as `session.spokesperson`. Saved settings don't keep `--expert-voice`, so pass it on each `start`. + +The three URL flags take full endpoints and override only their matching service for that call. Expert–Spokesperson uses `--realtime-url` for both input and output; `--stt-url` applies only to conventional mode, and `--tts-url` also applies to the Expert's own voice. Without an override, each service uses its OpenAI endpoint (or its existing environment override). `OPENAI_API_KEY` supplies the CLI credential; the desktop app stores keys separately by endpoint URL in Keychain. The default Siri backend still requires an exact installed voice name and language. Missing or unavailable Siri voice configuration and an unavailable diff --git a/src-tauri/crates/berd-call/src/cli_help.rs b/src-tauri/crates/berd-call/src/cli_help.rs index cc5448b85..f4e0d906b 100644 --- a/src-tauri/crates/berd-call/src/cli_help.rs +++ b/src-tauri/crates/berd-call/src/cli_help.rs @@ -113,13 +113,17 @@ Usage: Each form also accepts: [--rate FLOAT] [--stt-backend macos|parakeet|openai] [--stt-model-dir PATH] - [--mode conventional|expert-spokesperson] + [--mode conventional|expert-spokesperson] [--expert-voice tts|spokesperson] [--realtime-url WS_URL] [--stt-url WS_URL] [--tts-url HTTP_URL] The host owns microphone capture, audio playback, transcript delivery, and agent integration. See PROTOCOL.md for the framed stdin, stdout, and PCM contracts. Endpoint URL flags override only the matching service for this call -and default to OpenAI when no environment override is set."#; +and default to OpenAI when no environment override is set. + +In expert-spokesperson mode, --expert-voice tts speaks the Expert's replies +through the configured TTS backend, in its own voice beside the Spokesperson's, +instead of having the Spokesperson read them aloud."#; const SYNTHESIZE_HELP: &str = r#"Render text through a configured TTS backend into a new WAV file. diff --git a/src-tauri/crates/berd-call/src/host_session.rs b/src-tauri/crates/berd-call/src/host_session.rs index e28968fcf..132615ff2 100644 --- a/src-tauri/crates/berd-call/src/host_session.rs +++ b/src-tauri/crates/berd-call/src/host_session.rs @@ -3283,6 +3283,7 @@ mod tests { fn test_snapshot() -> VoiceSessionSnapshot { VoiceSessionSnapshot { + spokesperson: None, tts: berd_call::TtsConfigurationSnapshot { revision: 1, settings: berd_call::TtsSettings::Siri { diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 7346405ec..217e8565c 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -36,8 +36,8 @@ use berd_call::openai_spokesperson::{ }; use berd_call::protocol::{ CancelOutcome, DismissHandoffsOutcome, ExpertTurnOutcome, InputDuringTtsOutcome, - NotAdmittedReason, OutputReadyOutcome, SessionMessage, SessionRequest, TtsSettingsOutcome, - VoiceSessionSnapshot, + NotAdmittedReason, OutputReadyOutcome, SessionMessage, SessionRequest, + SpokespersonVoiceSnapshot, TtsSettingsOutcome, VoiceSessionSnapshot, }; use berd_call::realtime_audio_delivery::RealtimeAudioDelivery; use berd_call::realtime_host_lifecycle::RealtimeSessionLossAction; @@ -189,6 +189,9 @@ struct SessionConfig { stt: SttBackendConfig, mode: SessionMode, endpoints: SessionEndpointOverrides, + /// In Expert-Spokesperson mode, speak the Expert's replies with `tts` + /// rather than having the Spokesperson say them. + expert_tts: bool, } #[derive(Clone, Debug, Default, PartialEq)] @@ -2187,6 +2190,7 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String let session = VoiceSessionSnapshot { tts: slot.snapshot()?, input_during_tts: input_policy.snapshot()?, + spokesperson: None, }; tts_slot = Some(slot); input_during_tts_slot = Some(input_policy); @@ -2608,6 +2612,15 @@ enum LivePlaybackInput { type LivePlaybackResult = Result<(String, u64), (String, u64, String)>; +/// How an Expert reply spoken in its own voice ended. +enum ReplyOutcome { + Completed, + Interrupted { spoken_through_utf8: u64 }, + Failed(String), +} + +type ReplyResult = (String, u64, ReplyOutcome); + struct LivePlayback { response_id: String, speech_id: u64, @@ -2857,6 +2870,9 @@ struct LiveResponse { prepare_id: Option, speech_id: Option, expert_text: Option, + /// An Expert reply spoken in its own voice, with the handoffs it answers. + /// It has no provider response; its whole text is played at once. + reply: Option>, delivery: RealtimeAudioDelivery, pending_audio: VecDeque>, pending_frames: usize, @@ -2874,6 +2890,7 @@ impl LiveResponse { prepare_id, speech_id, expert_text: None, + reply: None, delivery: RealtimeAudioDelivery::default(), pending_audio: VecDeque::new(), pending_frames: 0, @@ -3128,6 +3145,12 @@ fn apply_spokesperson_startup_settings( | TtsBackendConfig::Siri { rate, .. } | TtsBackendConfig::Pocket { rate, .. } => *rate, }; + // A rate for the Expert's own voice need not suit the Spokesperson's. + let rate = if session.expert_tts { + rate.clamp(0.25, 1.5) + } else { + rate + }; if !(0.25..=1.5).contains(&rate) { return Err("Expert-Spokesperson rate must be between 0.25 and 1.5".into()); } @@ -3145,6 +3168,12 @@ fn run_expert_spokesperson_session( let audio_transport = Arc::new(unsafe { AudioPipeTransport::from_raw_fd(pcm_output_fd)? }); let (audio_control_tx, audio_control_rx) = mpsc::channel(); let (playback_tx, playback_rx) = mpsc::channel::(); + let (reply_tx, reply_rx) = mpsc::channel::(); + // The Expert's own voice, with `--expert-voice tts`. + let mut reply_tts: Option> = None; + let (tts_configuration_tx, tts_configuration_rx) = mpsc::channel::(); + let mut tts_update: Option = None; + let mut next_tts_update_attempt = 0_u64; let stdout = io::stdout(); let mut writer = BufWriter::new(stdout.lock()); let mut core = RealtimeExpertSpokespersonSession::new(0, "external"); @@ -3364,6 +3393,14 @@ fn run_expert_spokesperson_session( &mut cancelled_directives, &mut writer, )?; + drop_waiting_replies( + &preexisting_responses, + active_response_id.as_deref(), + &mut responses, + &mut core, + runtime.as_ref().expect("initialized runtime"), + &mut writer, + )?; } else { turn_gate.finish_user_speaking(); } @@ -3999,6 +4036,64 @@ fn run_expert_spokesperson_session( } } } + poll_tts_configuration_update( + Instant::now(), + &tts_configuration_rx, + reply_tts.as_deref(), + &mut tts_update, + &mut writer, + )?; + while let Ok((response_id, speech_id, outcome)) = reply_rx.try_recv() { + if take_matching_live_playback(&mut active, &response_id, speech_id).is_none() { + continue; + } + let Some(mut response) = responses.remove(&response_id) else { + continue; + }; + let id = response + .prepare_id + .expect("an Expert reply has a prepare ID"); + let text = response.expert_text.take().unwrap_or_default(); + let (message, heard) = match outcome { + ReplyOutcome::Completed => ( + SessionMessage::SpeechCompleted { id, speech_id }, + text.len(), + ), + ReplyOutcome::Interrupted { + spoken_through_utf8, + } => ( + SessionMessage::SpeechInterrupted { + id, + speech_id, + spoken_through_utf8, + }, + usize::try_from(spoken_through_utf8).unwrap_or(text.len()), + ), + ReplyOutcome::Failed(message) => ( + SessionMessage::SpeechFailed { + id, + speech_id, + message, + }, + 0, + ), + }; + let heard = (0..=heard.min(text.len())) + .rev() + .find(|&end| text.is_char_boundary(end)) + .unwrap_or_default(); + let interrupted = heard < text.len(); + if response.claim_speech_terminal() { + write_message(&mut writer, &message)?; + } + finish_expert_reply( + &mut core, + runtime.as_ref().expect("initialized runtime"), + &text[..heard], + interrupted, + response.reply.as_deref().unwrap_or_default(), + )?; + } if initialized && !turn_gate.input_blocks_output() { start_next_live_playback( &mut active, @@ -4008,6 +4103,8 @@ fn run_expert_spokesperson_session( &audio_transport, &audio_control_tx, &playback_tx, + reply_tts.as_deref(), + &reply_tx, &mut core, runtime.as_ref().expect("initialized runtime"), &mut writer, @@ -4030,6 +4127,9 @@ fn run_expert_spokesperson_session( &mut core, runtime.as_ref().expect("initialized runtime"), &mut directive_speeches, + reply_tts.is_some(), + &mut responses, + &mut waiting_responses, &mut next_speech_id, &mut writer, )?; @@ -4065,6 +4165,12 @@ fn run_expert_spokesperson_session( &mut writer, )?; } + reject_tts_configuration_update( + &mut tts_update, + reply_tts.as_deref(), + "session is shutting down", + &mut writer, + )?; if let Some(delivery) = core.flush_expert_events("Voice conversation ended") { write_message( &mut writer, @@ -4149,6 +4255,12 @@ fn run_expert_spokesperson_session( spokesperson_config.endpoint.clone_from(url); } apply_spokesperson_startup_settings(&config, &mut spokesperson_config)?; + if config.expert_tts { + reply_tts = Some(Arc::new(create_tts_slot( + &config.tts, + config.endpoints.tts.as_deref(), + )?)); + } let tts = berd_call::TtsConfigurationSnapshot { revision: 1, settings: TtsSettings::OpenAi { @@ -4159,8 +4271,15 @@ fn run_expert_spokesperson_session( }; let input_policy = InputDuringTtsSlot::new(input_during_tts); let snapshot = VoiceSessionSnapshot { - tts: tts.clone(), + tts: match &reply_tts { + Some(slot) => slot.snapshot()?, + None => tts.clone(), + }, input_during_tts: input_policy.snapshot()?, + spokesperson: Some(SpokespersonVoiceSnapshot { + model: spokesperson_config.model().into(), + voice: spokesperson_config.voice().into(), + }), }; let (created, events) = OpenAiSpokespersonRuntime::spawn_observed(spokesperson_config.clone())?; @@ -4238,6 +4357,9 @@ fn run_expert_spokesperson_session( &mut core, runtime.as_ref().expect("initialized runtime"), &mut directive_speeches, + reply_tts.is_some(), + &mut responses, + &mut waiting_responses, &mut next_speech_id, &mut writer, )?, @@ -4524,6 +4646,23 @@ fn run_expert_spokesperson_session( { cancel_live_playback(&mut active); CancelOutcome::Cancelled + } else if let Some(response_id) = responses + .iter() + .find(|(_, response)| { + response.reply.is_some() && response.prepare_id == Some(id) + }) + .map(|(response_id, _)| response_id.clone()) + { + waiting_responses.retain(|waiting| *waiting != response_id); + drop_waiting_replies( + &HashSet::from([response_id]), + None, + &mut responses, + &mut core, + runtime.as_ref().expect("initialized runtime"), + &mut writer, + )?; + CancelOutcome::Cancelled } else { CancelOutcome::Stale }; @@ -4640,6 +4779,41 @@ fn run_expert_spokesperson_session( &mut writer, )?; } + Input::Request(SessionRequest::SetTtsSettings { + id, + expected_revision, + settings, + }) if reply_tts.is_some() => { + let slot = Arc::clone(reply_tts.as_ref().expect("checked reply voice")); + if tts_update.is_some() { + write_message( + &mut writer, + &SessionMessage::TtsSettingsResult { + id, + outcome: TtsSettingsOutcome::Rejected, + snapshot: slot.snapshot()?, + message: Some("another TTS configuration update is in progress".into()), + }, + )?; + } else { + let attempt = next_tts_update_attempt; + next_tts_update_attempt += 1; + tts_update = Some(ActiveTtsConfigurationUpdate { + attempt, + id, + deadline: Instant::now() + TTS_CONFIGURATION_TIMEOUT, + }); + let sender = tts_configuration_tx.clone(); + thread::spawn(move || { + let result = slot.prepare_replacement(expected_revision, settings); + let _ = sender.send(TtsConfigurationEvent { + attempt, + id, + result, + }); + }); + } + } Input::Request(SessionRequest::SetTtsSettings { id, expected_revision, @@ -4692,6 +4866,133 @@ fn run_expert_spokesperson_session( Ok(()) } +/// Speaks an Expert reply with its own voice in the Spokesperson's playback slot. +#[allow(clippy::too_many_arguments)] +fn spawn_reply_playback( + response_id: String, + speech_id: u64, + prepare_id: Option, + text: String, + backend: Arc, + transport: Arc, + control: mpsc::Sender, + completed: mpsc::Sender, +) -> Result { + let active = Arc::new(AtomicBool::new(true)); + let output = Arc::new(RemotePcmAudioOutput::new( + speech_id, + backend.pcm_spec(), + transport, + Arc::clone(&active), + control, + )?); + let worker_output = Arc::clone(&output); + let worker_active = Arc::clone(&active); + let worker_response_id = response_id.clone(); + thread::spawn(move || { + let outcome = match play_tts(&text, backend.as_ref(), &worker_output, &worker_active) { + Ok((true, _)) => ReplyOutcome::Completed, + Ok((false, delivery)) => ReplyOutcome::Interrupted { + spoken_through_utf8: u64::try_from(estimated_spoken_through_utf8(&text, &delivery)) + .expect("speech text is bounded well below u64"), + }, + Err(failure) => ReplyOutcome::Failed(failure.message), + }; + let _ = completed.send((worker_response_id, speech_id, outcome)); + }); + // A reply's audio comes from its worker, not from provider deltas. + let (sender, _) = mpsc::sync_channel(1); + Ok(LivePlayback { + response_id, + speech_id, + prepare_id, + output, + active, + sender, + }) +} + +/// Ends Expert replies that never started playing, as interrupted before any was heard. +fn drop_waiting_replies( + response_ids: &HashSet, + active_response_id: Option<&str>, + responses: &mut HashMap, + core: &mut RealtimeExpertSpokespersonSession, + runtime: &OpenAiSpokespersonRuntime, + writer: &mut impl Write, +) -> Result<(), String> { + for response_id in response_ids { + if Some(response_id.as_str()) == active_response_id + || !responses + .get(response_id) + .is_some_and(|response| response.reply.is_some()) + { + continue; + } + let response = responses.remove(response_id).expect("reply exists"); + write_message( + writer, + &SessionMessage::SpeechInterrupted { + id: response + .prepare_id + .expect("an Expert reply has a prepare ID"), + speech_id: response.speech_id.expect("an Expert reply has a speech ID"), + spoken_through_utf8: 0, + }, + )?; + finish_expert_reply( + core, + runtime, + "", + true, + response.reply.as_deref().unwrap_or_default(), + )?; + } + Ok(()) +} + +/// Tells the Spokesperson what the user heard of an Expert reply, and settles +/// the handoffs it answers. +fn finish_expert_reply( + core: &mut RealtimeExpertSpokespersonSession, + runtime: &OpenAiSpokespersonRuntime, + heard: &str, + interrupted: bool, + handoff_ids: &[String], +) -> Result<(), String> { + let heard = heard.trim(); + // Handoffs settled elsewhere while the reply played need no settling here. + let unknown = core.unknown_handoff_ids(handoff_ids); + let open: Vec = handoff_ids + .iter() + .filter(|id| !unknown.contains(id)) + .cloned() + .collect(); + if heard.is_empty() { + core.release_handoffs(&open); + } else { + core.dismiss_handoffs(&open)?; + } + let message = match (heard.is_empty(), interrupted) { + (true, _) => "The user interrupted the Expert's spoken reply before hearing any of it.".into(), + (false, false) => format!("The Expert said this to the user in its own voice:\n{heard}"), + (false, true) => format!( + "The Expert said this to the user in its own voice before the user interrupted:\n{heard}" + ), + }; + let coordination = core.request_expert_message(RealtimeExpertMessage { + message, + mode: RealtimeExpertMessageMode::Context, + event_id: None, + directive_id: None, + resolved_handoff_ids: Vec::new(), + })?; + for event in coordination.events { + runtime.send(SpokespersonCommand::Provider(event))?; + } + Ok(()) +} + fn spawn_live_playback( response_id: String, speech_id: u64, @@ -4748,11 +5049,15 @@ fn spawn_live_playback( }) } +#[allow(clippy::too_many_arguments)] fn submit_expert_prepare( request: PendingExpertPrepare, core: &mut RealtimeExpertSpokespersonSession, runtime: &OpenAiSpokespersonRuntime, directive_speeches: &mut HashMap, + expert_tts: bool, + responses: &mut HashMap, + waiting_responses: &mut VecDeque, next_speech_id: &mut u64, writer: &mut impl Write, ) -> Result<(), String> { @@ -4790,26 +5095,39 @@ fn submit_expert_prepare( } => { let speech_id = *next_speech_id; *next_speech_id += 1; - directive_speeches.insert( - request.id, - DirectiveSpeech { - prepare_id: request.id, - speech_id, - text: message.clone(), - }, - ); core.record_expert_turn(message.clone()); let resolved_handoff_ids = request.resolved_handoff_ids; core.mark_handoffs_resolving(&resolved_handoff_ids)?; - let coordination = core.request_expert_message(RealtimeExpertMessage { - message, - mode: RealtimeExpertMessageMode::Say, - event_id: None, - directive_id: Some(request.id), - resolved_handoff_ids, - })?; - for event in coordination.events { - runtime.send(SpokespersonCommand::Provider(event))?; + if expert_tts { + // Played in turn with the Spokesperson's responses, without one of its own. + let response_id = format!("expert-reply-{speech_id}"); + let mut response = LiveResponse::new(Some(request.id), Some(speech_id)); + response.expert_text = Some(message); + response.reply = Some(resolved_handoff_ids); + response.audio_done = true; + response.finish_sent = true; + response.server_finished = true; + responses.insert(response_id.clone(), response); + waiting_responses.push_back(response_id); + } else { + directive_speeches.insert( + request.id, + DirectiveSpeech { + prepare_id: request.id, + speech_id, + text: message.clone(), + }, + ); + let coordination = core.request_expert_message(RealtimeExpertMessage { + message, + mode: RealtimeExpertMessageMode::Say, + event_id: None, + directive_id: Some(request.id), + resolved_handoff_ids, + })?; + for event in coordination.events { + runtime.send(SpokespersonCommand::Provider(event))?; + } } write_message( writer, @@ -4944,6 +5262,8 @@ fn start_next_live_playback( audio_transport: &Arc, audio_control_tx: &mpsc::Sender, playback_tx: &mpsc::Sender, + reply_tts: Option<&ConfiguredTtsSlot>, + reply_tx: &mpsc::Sender, core: &mut RealtimeExpertSpokespersonSession, runtime: &OpenAiSpokespersonRuntime, writer: &mut impl Write, @@ -4955,7 +5275,24 @@ fn start_next_live_playback( let Some(response) = responses.get_mut(&response_id) else { continue; }; - if response.interrupted || response.playback_complete || response.pending_audio.is_empty() { + if response.interrupted || response.playback_complete { + continue; + } + if response.reply.is_some() { + let tts = reply_tts.ok_or("an Expert reply has no voice of its own")?; + *active = Some(spawn_reply_playback( + response_id, + response.speech_id.expect("an Expert reply has a speech ID"), + response.prepare_id, + response.expert_text.clone().unwrap_or_default(), + Arc::clone(tts.lease()?.backend()), + Arc::clone(audio_transport), + audio_control_tx.clone(), + reply_tx.clone(), + )?); + break; + } + if response.pending_audio.is_empty() { continue; } let speech_id = response.speech_id.unwrap_or_else(|| { @@ -5401,6 +5738,7 @@ fn parse_args(args: &[String]) -> Result { let mut stt_model_dir = None; let mut mode = SessionMode::Conventional; let mut endpoints = SessionEndpointOverrides::default(); + let mut expert_tts = false; let mut index = 2; while index < args.len() { let flag = args[index].as_str(); @@ -5436,6 +5774,13 @@ fn parse_args(args: &[String]) -> Result { _ => return Err("--mode must be conventional or expert-spokesperson".into()), } } + "--expert-voice" => { + expert_tts = match value.as_str() { + "tts" => true, + "spokesperson" => false, + _ => return Err("--expert-voice must be tts or spokesperson".into()), + } + } "--pcm-output-fd" => {} _ => return Err(format!("unknown argument: {flag}").into()), } @@ -5446,8 +5791,11 @@ fn parse_args(args: &[String]) -> Result { if endpoints.realtime.is_some() && mode != SessionMode::ExpertSpokesperson { return Err("--realtime-url requires --mode expert-spokesperson".into()); } + if expert_tts && mode != SessionMode::ExpertSpokesperson { + return Err("--expert-voice requires --mode expert-spokesperson".into()); + } if mode == SessionMode::ExpertSpokesperson - && (endpoints.tts.is_some() || endpoints.stt.is_some()) + && (endpoints.stt.is_some() || endpoints.tts.is_some() && !expert_tts) { return Err("--tts-url and --stt-url apply only to conventional mode; Expert-Spokesperson uses --realtime-url".into()); } @@ -5462,6 +5810,7 @@ fn parse_args(args: &[String]) -> Result { stt, mode, endpoints, + expert_tts, }) } @@ -9110,6 +9459,7 @@ mod tests { session: VoiceSessionSnapshot { tts: snapshot.clone(), input_during_tts: test_input_policy(), + spokesperson: None, }, }) .unwrap(); @@ -9696,6 +10046,7 @@ mod tests { stt: SttBackendConfig::Macos, mode: SessionMode::Conventional, endpoints: SessionEndpointOverrides::default(), + expert_tts: false, } ); @@ -9706,6 +10057,7 @@ mod tests { stt: SttBackendConfig::Macos, mode: SessionMode::Conventional, endpoints: SessionEndpointOverrides::default(), + expert_tts: false, } ); } @@ -9782,6 +10134,7 @@ mod tests { stt: SttBackendConfig::Macos, mode: SessionMode::Conventional, endpoints: SessionEndpointOverrides::default(), + expert_tts: false, } ); assert!(parse_args(&args(&[ @@ -9837,6 +10190,7 @@ mod tests { stt: SttBackendConfig::Macos, mode: SessionMode::ExpertSpokesperson, endpoints: SessionEndpointOverrides::default(), + expert_tts: false, }; let mut realtime = OpenAiSpokespersonConfig { endpoint: "ws://localhost".into(), @@ -9857,6 +10211,71 @@ mod tests { assert_eq!(realtime.voice(), "marin"); } + #[test] + fn expert_voice_is_an_expert_spokesperson_choice_with_its_own_tts() { + let parse = |extra: &[&str]| { + let mut all = vec!["berd-call", "session"]; + all.extend_from_slice(extra); + parse_args(&args(&all)) + }; + let expert = |extra: &[&str]| { + let mut all = vec!["--mode", "expert-spokesperson"]; + all.extend_from_slice(extra); + parse(&all) + }; + assert!(!expert(&["--tts-backend", "openai"]).unwrap().expert_tts); + assert!( + !expert(&["--tts-backend", "openai", "--expert-voice", "spokesperson"]) + .unwrap() + .expert_tts + ); + let own = expert(&[ + "--tts-backend", + "openai", + "--rate", + "2.0", + "--expert-voice", + "tts", + "--tts-url", + "http://127.0.0.1:18870/v1/audio/speech", + ]) + .unwrap(); + assert!(own.expert_tts); + assert_eq!( + own.endpoints.tts.as_deref(), + Some("http://127.0.0.1:18870/v1/audio/speech") + ); + let mut realtime = OpenAiSpokespersonConfig { + endpoint: "ws://localhost".into(), + api_key: "test-key".into(), + session: berd_call::openai_realtime_protocol::RealtimeSpokespersonSessionOptions { + model: Some("test-model".into()), + voice: Some("marin".into()), + ..Default::default() + }, + semantic_transcript: Vec::new(), + }; + apply_spokesperson_startup_settings(&own, &mut realtime).unwrap(); + assert_eq!(realtime.speed(), 1.5); + let read_aloud = expert(&["--tts-backend", "openai", "--rate", "2.0"]).unwrap(); + assert!(apply_spokesperson_startup_settings(&read_aloud, &mut realtime).is_err()); + assert!(expert(&["--expert-voice", "loud"]) + .unwrap_err() + .contains("tts or spokesperson")); + assert!(parse(&["--tts-backend", "openai", "--expert-voice", "tts"]) + .unwrap_err() + .contains("--mode expert-spokesperson")); + assert_eq!( + saved_settings::persistable_arguments(&args(&[ + "--mode", + "expert-spokesperson", + "--expert-voice", + "tts" + ])), + args(&["--mode", "expert-spokesperson"]) + ); + } + #[test] fn session_endpoint_overrides_are_independent_and_require_matching_modes() { let realtime = parse_args(&args(&[ @@ -9965,6 +10384,7 @@ mod tests { stt: SttBackendConfig::Macos, mode: SessionMode::Conventional, endpoints: SessionEndpointOverrides::default(), + expert_tts: false, } ); assert!(parse_args(&args(&[ @@ -10026,6 +10446,7 @@ mod tests { }, mode: SessionMode::Conventional, endpoints: SessionEndpointOverrides::default(), + expert_tts: false, } ); assert!(parse_args(&args(&[ @@ -10470,6 +10891,7 @@ mod tests { stt: SttBackendConfig::OpenAi, mode: SessionMode::Conventional, endpoints: SessionEndpointOverrides::default(), + expert_tts: false, } ); } diff --git a/src-tauri/crates/berd-call/src/openai_realtime_protocol.rs b/src-tauri/crates/berd-call/src/openai_realtime_protocol.rs index 2bc4afc6f..570299a09 100644 --- a/src-tauri/crates/berd-call/src/openai_realtime_protocol.rs +++ b/src-tauri/crates/berd-call/src/openai_realtime_protocol.rs @@ -1783,6 +1783,16 @@ impl RealtimeExpertSpokespersonSession { Ok(()) } + /// Leaves handoffs open for another reply once the one meant to answer + /// them went unheard. + pub fn release_handoffs(&mut self, handoff_ids: &[String]) { + for handoff_id in handoff_ids { + if let Some(handoff) = self.open_handoffs.get_mut(handoff_id) { + handoff.resolving = false; + } + } + } + pub fn dismiss_handoffs(&mut self, handoff_ids: &[String]) -> Result<(), String> { let unknown = self.unknown_handoff_ids(handoff_ids); if !unknown.is_empty() { diff --git a/src-tauri/crates/berd-call/src/protocol.rs b/src-tauri/crates/berd-call/src/protocol.rs index 32eca4e91..e27904bd0 100644 --- a/src-tauri/crates/berd-call/src/protocol.rs +++ b/src-tauri/crates/berd-call/src/protocol.rs @@ -157,8 +157,19 @@ pub enum OutputReadyOutcome { #[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] pub struct VoiceSessionSnapshot { + /// The voice that speaks replies: the conventional backend, or the + /// Spokesperson's own voice when it reads them. pub tts: TtsConfigurationSnapshot, pub input_during_tts: InputDuringTtsSnapshot, + /// The Spokesperson's voice, in Expert-Spokesperson sessions. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub spokesperson: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +pub struct SpokespersonVoiceSnapshot { + pub model: String, + pub voice: String, } #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] @@ -402,10 +413,14 @@ mod tests { revision: 1, policy: InputDuringTtsPolicy::AllowBargeIn, }, + spokesperson: Some(SpokespersonVoiceSnapshot { + model: "gpt-realtime".into(), + voice: "cedar".into(), + }), }, }) .unwrap(), - r#"{"type":"ready","id":4,"protocol":5,"session":{"tts":{"revision":1,"backend":"openai","model":"gpt-4o-mini-tts","voice":"marin","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"}}}"# + r#"{"type":"ready","id":4,"protocol":5,"session":{"tts":{"revision":1,"backend":"openai","model":"gpt-4o-mini-tts","voice":"marin","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"},"spokesperson":{"model":"gpt-realtime","voice":"cedar"}}}"# ); assert_eq!( serde_json::from_str::( diff --git a/src-tauri/crates/berd-call/src/saved_settings.rs b/src-tauri/crates/berd-call/src/saved_settings.rs index 8587b52b1..0caf56779 100644 --- a/src-tauri/crates/berd-call/src/saved_settings.rs +++ b/src-tauri/crates/berd-call/src/saved_settings.rs @@ -131,7 +131,7 @@ pub(crate) fn persistable_arguments(arguments: &[String]) -> Vec { .filter(|pair| { !matches!( pair[0].as_str(), - "--realtime-url" | "--stt-url" | "--tts-url" + "--realtime-url" | "--stt-url" | "--tts-url" | "--expert-voice" ) }) .flatten() diff --git a/src-tauri/crates/berd-call/tests/session_protocol.rs b/src-tauri/crates/berd-call/tests/session_protocol.rs index 89c8339a3..cd9ea0ca6 100644 --- a/src-tauri/crates/berd-call/tests/session_protocol.rs +++ b/src-tauri/crates/berd-call/tests/session_protocol.rs @@ -32,6 +32,7 @@ struct ExpertSpokespersonTestSession { output: mpsc::Receiver, stderr: mpsc::Receiver, audio_host: Option>, + ready: Value, } struct AudioCancellationGate { @@ -54,6 +55,28 @@ impl ExpertSpokespersonTestSession { played_frame_limit: Option, played_ready: Option>, cancellation_gate: Option, + ) -> Self { + Self::launch( + endpoint, + renew_after_ms, + played_frame_limit, + played_ready, + cancellation_gate, + &[], + ) + } + + fn start_with_args(endpoint: String, args: &[&str]) -> Self { + Self::launch(endpoint, None, None, None, None, args) + } + + fn launch( + endpoint: String, + renew_after_ms: Option, + played_frame_limit: Option, + played_ready: Option>, + cancellation_gate: Option, + args: &[&str], ) -> Self { let (mut command, _pcm, audio_host) = session_command(); if let Some(renew_after_ms) = renew_after_ms { @@ -72,6 +95,7 @@ impl ExpertSpokespersonTestSession { "--realtime-url", &endpoint, ]) + .args(args) .env("OPENAI_API_KEY", "test-key") .env("OPENAI_REALTIME_ENDPOINT", "ws://127.0.0.1:1/unused") .env("OPENAI_REALTIME_MODEL", "test-model") @@ -99,11 +123,13 @@ impl ExpertSpokespersonTestSession { output, stderr, audio_host: Some(audio_host), + ready: Value::Null, }; session.send(json!({ "type":"hello","id":1,"input_during_tts":"allow_barge_in" })); - assert_eq!(session.recv(Duration::from_secs(2))["type"], "ready"); + session.ready = session.recv(Duration::from_secs(2)); + assert_eq!(session.ready["type"], "ready"); session } @@ -2654,3 +2680,162 @@ fn siri_remote_output_supports_consecutive_turns_and_cancellation() { reader.join().unwrap(); audio_host.join().unwrap(); } + +/// Serves OpenAI speech requests with a little silence, reporting each request body. +fn spawn_speech_server() -> (String, mpsc::Receiver) { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/v1/audio/speech", listener.local_addr().unwrap()); + let (sender, receiver) = mpsc::channel(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let mut stream = stream.unwrap(); + let mut request = Vec::new(); + let mut byte = [0_u8; 1]; + while !request.ends_with(b"\r\n\r\n") { + stream.read_exact(&mut byte).unwrap(); + request.push(byte[0]); + } + let head = String::from_utf8(request).unwrap().to_ascii_lowercase(); + let length = head + .lines() + .find_map(|line| line.strip_prefix("content-length:")) + .unwrap() + .trim() + .parse() + .unwrap(); + let mut body = vec![0_u8; length]; + stream.read_exact(&mut body).unwrap(); + let _ = sender.send(serde_json::from_slice(&body).unwrap()); + let audio = vec![1_u8; 4_800]; + write!( + stream, + "HTTP/1.1 200 OK\r\ncontent-type: audio/pcm\r\ncontent-length: {}\r\nconnection: close\r\n\r\n", + audio.len() + ) + .unwrap(); + stream.write_all(&audio).unwrap(); + } + }); + (url, receiver) +} + +#[test] +fn expert_voice_speaks_replies_itself_and_tells_the_spokesperson_what_was_heard() { + let (speech_url, speech_requests) = spawn_speech_server(); + let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); + let (context_tx, context_rx) = mpsc::sync_channel(1); + let server = std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + endpoint_tx + .send(format!("ws://{}/", listener.local_addr().unwrap())) + .unwrap(); + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = accept_async(stream).await.unwrap(); + let initial = receive_realtime_json(&mut socket).await; + acknowledge_realtime_session(&mut socket, &initial).await; + send_realtime_json( + &mut socket, + json!({ + "type":"response.function_call_arguments.done", + "response_id":"response-handoff", + "call_id":"handoff-reply", + "name":"handoff", + "arguments":"{\"message\":\"inspect the repository\"}" + }), + ) + .await; + let mut mentions = Vec::new(); + let context = loop { + let next = receive_realtime_json(&mut socket).await; + let text = next.to_string(); + if text.contains("in its own voice") { + break next; + } + if text.contains("I checked the repository.") { + mentions.push(next); + } + }; + let after = tokio::time::timeout(Duration::from_millis(300), async { + loop { + let next = receive_realtime_json(&mut socket).await; + if next["type"] == "response.create" { + break next; + } + } + }) + .await + .ok(); + context_tx.send((context, mentions, after)).unwrap(); + while socket.next().await.is_some() {} + }); + }); + let endpoint = endpoint_rx.recv().unwrap(); + let mut session = ExpertSpokespersonTestSession::start_with_args( + endpoint, + &["--expert-voice", "tts", "--tts-url", &speech_url], + ); + assert_eq!(session.ready["session"]["tts"]["backend"], "openai"); + assert_eq!( + session.ready["session"]["spokesperson"], + json!({"model":"test-model","voice":"old-voice"}) + ); + assert_eq!(session.recv(Duration::from_secs(2))["origin"], "handoff"); + assert_eq!( + session.recv(Duration::from_secs(2))["handoff_ids"], + json!(["handoff-external-1"]) + ); + + session.send(json!({ + "type":"prepare_speak", + "id":2, + "acknowledgement":1, + "text":"I checked the repository.", + "resolved_handoff_ids":["handoff-external-1"] + })); + let admitted = session.recv(Duration::from_secs(2)); + assert_eq!(admitted["type"], "admitted", "{admitted}"); + session.send(json!({ + "type":"output_ready", + "id":2, + "speech_id":admitted["speech_id"] + })); + let deadline = std::time::Instant::now() + Duration::from_secs(3); + loop { + let remaining = deadline + .checked_duration_since(std::time::Instant::now()) + .expect("the Expert's reply did not complete"); + let message = session.recv(remaining); + assert_ne!(message["type"], "fatal", "{message}"); + assert_ne!(message["type"], "speech_failed", "{message}"); + if message["type"] == "speech_completed" { + assert_eq!(message["id"], 2); + assert_eq!(message["speech_id"], admitted["speech_id"]); + break; + } + } + let speech = speech_requests + .recv_timeout(Duration::from_secs(1)) + .unwrap(); + assert_eq!(speech["input"], "I checked the repository."); + + let (context, mentions, after) = context_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + assert_eq!(context["type"], "conversation.item.create", "{context}"); + assert!(context.to_string().contains( + "The Expert said this to the user in its own voice:\\nI checked the repository." + )); + assert!( + mentions.is_empty(), + "the Spokesperson was asked to say it: {mentions:?}" + ); + assert!( + after.is_none(), + "the context note started a response: {after:?}" + ); + session.shutdown(); + server.join().unwrap(); +} From c754eacad5e3a549b1cbe006b27ed7659858f03a Mon Sep 17 00:00:00 2001 From: John Tennant Date: Mon, 19 Oct 2026 17:05:00 -0400 Subject: [PATCH 03/49] berd-call: give the Spokesperson its own voice controls beside the Expert's With --expert-voice tts, the one voice and speed control changed the Expert's voice, leaving no way to change the Spokesperson's during a call. The Spokesperson now has its own controls in these calls: - `set_spokesperson_settings` swaps in a replacement realtime session, the same way `set_tts_settings` changes the Spokesperson's voice elsewhere. - `settings --spokesperson-voice` and `--spokesperson-rate`, plus matching `start` options, set its voice and speed. - The menu bar shows Agent and Spokesperson voice and speed menus side by side, each limited to its own range of speeds. `session.spokesperson` is now the Spokesperson's full revisioned OpenAI snapshot. It appears only in calls whose replies have their own voice, since elsewhere `session.tts` is already the Spokesperson's. The Spokesperson's speed no longer follows the Expert's rate. The voice and speed it was last given are saved, and come back in the next call whose replies have their own voice. Co-Authored-By: Claude Opus 5.5 Signed-off-by: John Tennant --- src-tauri/crates/berd-call/PROTOCOL.md | 11 +- src-tauri/crates/berd-call/README.md | 2 +- src-tauri/crates/berd-call/src/cli_help.rs | 11 +- .../crates/berd-call/src/host_control.rs | 17 ++ .../crates/berd-call/src/host_session.rs | 168 ++++++++++++- src-tauri/crates/berd-call/src/main.rs | 224 ++++++++++++++++-- src-tauri/crates/berd-call/src/menu_bar.rs | 153 +++++++++--- src-tauri/crates/berd-call/src/protocol.rs | 28 ++- .../crates/berd-call/src/saved_settings.rs | 28 +++ .../berd-call/tests/session_protocol.rs | 69 +++++- 10 files changed, 629 insertions(+), 82 deletions(-) diff --git a/src-tauri/crates/berd-call/PROTOCOL.md b/src-tauri/crates/berd-call/PROTOCOL.md index c1655f70a..0d1d20639 100644 --- a/src-tauri/crates/berd-call/PROTOCOL.md +++ b/src-tauri/crates/berd-call/PROTOCOL.md @@ -72,10 +72,12 @@ OpenAI snapshots contain `model`, `voice`, and `rate`; Siri contains `voice`, and `rate`. Credentials, endpoints, and bundle paths never appear on stdout. Detailed backend errors are diagnostics on stderr only; protocol rejection and fatal messages are sanitized at the stdout boundary. -In Expert-Spokesperson sessions, `session.spokesperson` holds the realtime -`model` and `voice`. `session.tts` describes the voice that speaks the Expert's -replies: the configured TTS backend with `--expert-voice tts`, otherwise the -Spokesperson's own voice, and `set_tts_settings` changes that voice. A reply in +`session.tts` describes the voice that speaks replies, and `set_tts_settings` +changes it: in Expert-Spokesperson sessions, that is the configured TTS backend +with `--expert-voice tts` and otherwise the Spokesperson's own voice. With +`--expert-voice tts`, `session.spokesperson` is the Spokesperson's own OpenAI +snapshot, with its own revision, and `set_spokesperson_settings` changes it the +way `set_tts_settings` changes the Spokesperson's voice in other sessions. A reply in the Expert's own voice reports the same `admitted`, `speech_completed`, `speech_interrupted`, and `speech_failed` lifecycle as a conventional reply. `session.input_during_tts` is the authoritative effective assistant-input @@ -201,6 +203,7 @@ it never admits a replacement while old host audio may still be active. {"type":"set_input_muted","id":u64,"active":bool} {"type":"set_conversation_status","id":u64,"status":"working"|"waiting","settings":StatusSoundSettings} {"type":"set_tts_settings","id":u64,"expected_revision":u64,"settings":TtsSettings} +{"type":"set_spokesperson_settings","id":u64,"expected_revision":u64,"settings":TtsSettings} {"type":"set_input_during_tts","id":u64,"expected_revision":u64,"policy":"allow_barge_in"|"suppress_input"} {"type":"reset_input","id":u64} {"type":"prepare_speak","id":u64,"acknowledgement":u64|null,"text":string} diff --git a/src-tauri/crates/berd-call/README.md b/src-tauri/crates/berd-call/README.md index 36ce6b8a1..d48af1818 100644 --- a/src-tauri/crates/berd-call/README.md +++ b/src-tauri/crates/berd-call/README.md @@ -147,7 +147,7 @@ berd-call session --tts-backend openai --tts-url https://proxy.example/v1/audio/ berd-call session --stt-backend openai --stt-url wss://proxy.example/v1/realtime?intent=transcription ``` -By default the Spokesperson reads the Expert's replies aloud in its own voice. With `--expert-voice tts`, the Expert speaks for itself: berd-call plays each reply through the configured TTS backend, taking turns with the Spokesperson's audio, and your speech interrupts either of them. Afterwards the Spokesperson gets a silent note of what you heard, so it can follow up without repeating it. A reply settles the handoffs it answers once you hear any of it. Its rate applies to the Expert's voice, and the Spokesperson's speed is that rate limited to 0.25–1.5. `status` and `settings --tts` describe and change the Expert's voice; the Spokesperson's model and voice appear as `session.spokesperson`. Saved settings don't keep `--expert-voice`, so pass it on each `start`. +By default the Spokesperson reads the Expert's replies aloud in its own voice. With `--expert-voice tts`, the Expert speaks for itself: berd-call plays each reply through the configured TTS backend, taking turns with the Spokesperson's audio, and your speech interrupts either of them. Afterwards the Spokesperson gets a silent note of what you heard, so it can follow up without repeating it. A reply settles the handoffs it answers once you hear any of it. `--voice`, `--rate`, `status` and `settings --tts` describe and change the Expert's voice. The Spokesperson keeps its own, shown as `session.spokesperson`: choose it with `--spokesperson-voice NAME` and `--spokesperson-rate 0.25–1.5`, at `start` or live with `settings`, and the menu bar offers both voices. Saved settings keep the Spokesperson's voice for the next call whose replies have their own, but not `--expert-voice`, so pass that on each `start`. The three URL flags take full endpoints and override only their matching service for that call. Expert–Spokesperson uses `--realtime-url` for both input and output; `--stt-url` applies only to conventional mode, and `--tts-url` also applies to the Expert's own voice. Without an override, each service uses its OpenAI endpoint (or its existing environment override). `OPENAI_API_KEY` supplies the CLI credential; the desktop app stores keys separately by endpoint URL in Keychain. diff --git a/src-tauri/crates/berd-call/src/cli_help.rs b/src-tauri/crates/berd-call/src/cli_help.rs index f4e0d906b..4d5edab41 100644 --- a/src-tauri/crates/berd-call/src/cli_help.rs +++ b/src-tauri/crates/berd-call/src/cli_help.rs @@ -71,12 +71,17 @@ Usage: berd-call settings [--port PORT] --non-blocking true|false berd-call settings [--port PORT] --rate FLOAT berd-call settings [--port PORT] --voice NAME [--language BCP47] + berd-call settings [--port PORT] --spokesperson-voice NAME + berd-call settings [--port PORT] --spokesperson-rate FLOAT berd-call settings [--port PORT] --tts JSON berd-call settings [--port PORT] --input-during-tts allow|suppress berd-call settings [--port PORT] --muted true|false berd-call settings [--port PORT] --restart [session options] -The setting applies to subsequent speak calls. Non-blocking delivery requires +The setting applies to subsequent speak calls. In a call started with +--expert-voice tts, --voice and --rate change the agents' voice, and +--spokesperson-voice and --spokesperson-rate change the Spokesperson's (an +OpenAI realtime voice, 0.25 to 1.5). Non-blocking delivery requires a call started with --stream or --codex. Successful delivery produces no notification."#; const STATUS_HELP: &str = r#"Inspect the active local voice call. @@ -114,6 +119,7 @@ Each form also accepts: [--rate FLOAT] [--stt-backend macos|parakeet|openai] [--stt-model-dir PATH] [--mode conventional|expert-spokesperson] [--expert-voice tts|spokesperson] + [--spokesperson-voice NAME] [--spokesperson-rate 0.25..1.5] [--realtime-url WS_URL] [--stt-url WS_URL] [--tts-url HTTP_URL] The host owns microphone capture, audio playback, transcript delivery, and @@ -123,7 +129,8 @@ and default to OpenAI when no environment override is set. In expert-spokesperson mode, --expert-voice tts speaks the Expert's replies through the configured TTS backend, in its own voice beside the Spokesperson's, -instead of having the Spokesperson read them aloud."#; +instead of having the Spokesperson read them aloud. --spokesperson-voice and +--spokesperson-rate then choose the Spokesperson's own voice and speed."#; const SYNTHESIZE_HELP: &str = r#"Render text through a configured TTS backend into a new WAV file. diff --git a/src-tauri/crates/berd-call/src/host_control.rs b/src-tauri/crates/berd-call/src/host_control.rs index f3d56d714..cd00eb516 100644 --- a/src-tauri/crates/berd-call/src/host_control.rs +++ b/src-tauri/crates/berd-call/src/host_control.rs @@ -34,6 +34,7 @@ pub(crate) trait HostControl: Send + Sync + 'static { fn set_tts(&self, settings: berd_call::TtsSettings) -> Result; fn set_rate(&self, rate: f32) -> Result; fn set_voice(&self, voice: String, language: Option) -> Result; + fn set_spokesperson(&self, voice: Option, rate: Option) -> Result; fn set_input_during_tts( &self, policy: berd_call::input::InputDuringTtsPolicy, @@ -129,6 +130,11 @@ pub(crate) enum ControlRequest { voice: String, language: Option, }, + /// The Spokesperson's voice or speed, when replies have their own. + Spokesperson { + voice: Option, + rate: Option, + }, InputDuringTts { policy: berd_call::input::InputDuringTtsPolicy, }, @@ -187,6 +193,7 @@ pub(crate) fn begin_request(port: u16, request: ControlRequest) -> Result Result ControlRequest::TtsSettings { settings } => control.set_tts(settings), ControlRequest::Rate { rate } => control.set_rate(rate), ControlRequest::Voice { voice, language } => control.set_voice(voice, language), + ControlRequest::Spokesperson { voice, rate } => { + control.set_spokesperson(voice, rate) + } ControlRequest::NonBlocking { enabled } => control.set_non_blocking(enabled), ControlRequest::InputDuringTts { policy } => control.set_input_during_tts(policy), ControlRequest::Muted { muted } => control.set_muted(muted), @@ -349,6 +359,13 @@ mod tests { fn set_voice(&self, voice: String, language: Option) -> Result { Ok(json!({"voice": voice, "language": language})) } + fn set_spokesperson( + &self, + voice: Option, + rate: Option, + ) -> Result { + Ok(json!({"spokesperson": {"voice": voice, "rate": rate}})) + } fn set_non_blocking(&self, enabled: bool) -> Result { Ok(json!({"nonBlocking": enabled})) } diff --git a/src-tauri/crates/berd-call/src/host_session.rs b/src-tauri/crates/berd-call/src/host_session.rs index 132615ff2..ac01d0fa5 100644 --- a/src-tauri/crates/berd-call/src/host_session.rs +++ b/src-tauri/crates/berd-call/src/host_session.rs @@ -508,6 +508,11 @@ enum ControlCommand { expected_revision: u64, response: SyncSender>, }, + SpokespersonSettings { + settings: berd_call::TtsSettings, + expected_revision: u64, + response: SyncSender>, + }, Speak { text: String, acknowledgement: Option, @@ -644,6 +649,38 @@ impl HostControl for SessionControl { }) } + fn set_spokesperson(&self, voice: Option, rate: Option) -> Result { + let (mut settings, expected_revision) = { + let session = self + .ready + .session + .lock() + .map_err(|_| "session settings lock failed")?; + let spokesperson = session + .spokesperson + .as_ref() + .ok_or("this call has no separate Spokesperson voice; use --voice and --rate")?; + (spokesperson.settings.clone(), spokesperson.revision) + }; + if let ( + Some(voice), + berd_call::TtsSettings::OpenAi { + voice: selected, .. + }, + ) = (voice, &mut settings) + { + *selected = voice; + } + if let Some(rate) = rate { + settings = settings.with_rate(rate); + } + self.request(|response| ControlCommand::SpokespersonSettings { + settings, + expected_revision, + response, + }) + } + fn set_input_during_tts(&self, policy: InputDuringTtsPolicy) -> Result { let expected_revision = self .ready @@ -752,6 +789,7 @@ enum AudioCommand { enum SavedPreference { SessionStart, Tts, + Spokesperson, InputPolicy, } @@ -767,6 +805,8 @@ struct SessionActor { next_id: u64, pending_speak: Option, pending_settings: std::collections::HashMap>>, + // Settings requests answered with the Spokesperson's voice rather than the call's. + spokesperson_settings: HashSet, waiting_speaks: VecDeque, session: Arc>, transcript: Arc, @@ -831,6 +871,7 @@ impl SessionActor { next_id: 2, pending_speak: None, pending_settings: std::collections::HashMap::new(), + spokesperson_settings: HashSet::new(), waiting_speaks: VecDeque::new(), session, transcript, @@ -1030,6 +1071,23 @@ impl SessionActor { )?; self.pending_settings.insert(id, response); } + ControlCommand::SpokespersonSettings { + settings, + expected_revision, + response, + } => { + let id = self.next_id(); + send_request( + &self.writer, + &SessionRequest::SetSpokespersonSettings { + id, + expected_revision, + settings, + }, + )?; + self.spokesperson_settings.insert(id); + self.pending_settings.insert(id, response); + } ControlCommand::Speak { text, acknowledgement, @@ -1151,13 +1209,23 @@ impl SessionActor { snapshot, message, } => { + let spokesperson = self.spokesperson_settings.remove(&id); if let Ok(mut session) = self.session.lock() { - if snapshot.revision >= session.tts.revision { - session.tts = snapshot.clone(); + let voice = if spokesperson { + session.spokesperson.as_mut() + } else { + Some(&mut session.tts) + }; + if let Some(voice) = voice.filter(|voice| snapshot.revision >= voice.revision) { + *voice = snapshot.clone(); } } if outcome == berd_call::protocol::TtsSettingsOutcome::Applied { - self.persist_settings(SavedPreference::Tts); + self.persist_settings(if spokesperson { + SavedPreference::Spokesperson + } else { + SavedPreference::Tts + }); } if let Some(response) = self.pending_settings.remove(&id) { let _ = response.send(Ok( @@ -1388,6 +1456,26 @@ impl SessionActor { let Ok(session) = self.session.lock() else { return; }; + if let SavedPreference::Spokesperson = changed { + let Some(spokesperson) = &session.spokesperson else { + return; + }; + let replacement = vec![ + "--spokesperson-voice".to_string(), + spokesperson.settings.voice().to_string(), + "--spokesperson-rate".to_string(), + spokesperson.settings.rate().to_string(), + ]; + saved.arguments = + crate::saved_settings::merge_arguments(&saved.arguments, &replacement); + if let Err(error) = crate::saved_settings::update(path, |latest| { + latest.arguments = + crate::saved_settings::merge_arguments(&latest.arguments, &replacement) + }) { + eprintln!("berd-call: live settings applied but could not be saved: {error}"); + } + return; + } saved.tts = Some(session.tts.settings.clone()); let settings = &session.tts.settings; let pocket_model_dir = saved @@ -1435,6 +1523,7 @@ impl SessionActor { latest.tts = saved.tts.clone(); } SavedPreference::InputPolicy => latest.input_policy = saved.input_policy, + SavedPreference::Spokesperson => unreachable!("saved above"), }) { eprintln!("berd-call: live settings applied but could not be saved: {error}"); } @@ -3685,6 +3774,79 @@ mod tests { second_child.wait().unwrap(); } + #[test] + fn spokesperson_results_change_and_save_only_its_voice() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("settings.json"); + let mut child = Command::new("/bin/cat") + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .spawn() + .unwrap(); + let writer = Arc::new(Mutex::new(child.stdin.take().unwrap())); + let (_events, receiver) = mpsc::sync_channel(1); + let (_commands, commands) = mpsc::sync_channel(1); + let (audio, _audio_rx) = mpsc::sync_channel(1); + let mut actor = test_actor(writer, receiver, commands, audio); + let openai = |voice: &str, revision| berd_call::TtsConfigurationSnapshot { + revision, + settings: berd_call::TtsSettings::OpenAi { + model: "gpt-realtime".into(), + voice: voice.into(), + rate: 1.25, + }, + }; + actor.session.lock().unwrap().spokesperson = Some(openai("marin", 1)); + actor.saved = Some(( + path.clone(), + crate::saved_settings::SavedSettings { + arguments: ["--voice", "Aaron", "--language", "en-US"] + .map(str::to_string) + .to_vec(), + ..crate::saved_settings::SavedSettings::default() + }, + )); + let (response, result) = mpsc::sync_channel(1); + actor + .handle_command(ControlCommand::SpokespersonSettings { + settings: openai("cedar", 1).settings, + expected_revision: 1, + response, + }) + .unwrap(); + let id = *actor.spokesperson_settings.iter().next().unwrap(); + actor + .handle_event(SessionMessage::TtsSettingsResult { + id, + outcome: berd_call::protocol::TtsSettingsOutcome::Applied, + snapshot: openai("cedar", 2), + message: None, + }) + .unwrap(); + assert_eq!(result.recv().unwrap().unwrap()["outcome"], "applied"); + let session = actor.session.lock().unwrap().clone(); + assert_eq!(session.spokesperson, Some(openai("cedar", 2))); + assert_eq!(session.tts, test_snapshot().tts); + let saved = crate::saved_settings::load(&path).unwrap(); + assert_eq!(saved.tts, None); + assert_eq!( + saved.arguments, + [ + "--voice", + "Aaron", + "--language", + "en-US", + "--spokesperson-voice", + "cedar", + "--spokesperson-rate", + "1.25" + ] + .map(str::to_string) + ); + drop(actor); + child.wait().unwrap(); + } + #[test] fn pocket_tts_save_preserves_the_active_bundle_path() { let directory = tempfile::tempdir().unwrap(); diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 217e8565c..5e9c639b5 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -36,8 +36,8 @@ use berd_call::openai_spokesperson::{ }; use berd_call::protocol::{ CancelOutcome, DismissHandoffsOutcome, ExpertTurnOutcome, InputDuringTtsOutcome, - NotAdmittedReason, OutputReadyOutcome, SessionMessage, SessionRequest, - SpokespersonVoiceSnapshot, TtsSettingsOutcome, VoiceSessionSnapshot, + NotAdmittedReason, OutputReadyOutcome, SessionMessage, SessionRequest, TtsSettingsOutcome, + VoiceSessionSnapshot, }; use berd_call::realtime_audio_delivery::RealtimeAudioDelivery; use berd_call::realtime_host_lifecycle::RealtimeSessionLossAction; @@ -192,6 +192,9 @@ struct SessionConfig { /// In Expert-Spokesperson mode, speak the Expert's replies with `tts` /// rather than having the Spokesperson say them. expert_tts: bool, + /// The Spokesperson's own voice and speed when replies have theirs. + spokesperson_voice: Option, + spokesperson_rate: Option, } #[derive(Clone, Debug, Default, PartialEq)] @@ -862,6 +865,14 @@ fn parse_saved_start_args_at(args: &[String], path: PathBuf) -> Result Result host_control::ControlRequest::Spokesperson { + voice: Some( + args.get(index + 1) + .filter(|value| !value.trim().is_empty()) + .ok_or("--spokesperson-voice requires a name")? + .clone(), + ), + rate: None, + }, + "--spokesperson-rate" => host_control::ControlRequest::Spokesperson { + voice: None, + rate: Some( + args.get(index + 1) + .and_then(|value| value.parse::().ok()) + .filter(|rate| rate.is_finite() && *rate > 0.0) + .ok_or("--spokesperson-rate requires a positive number")?, + ), + }, "--tts" => { let value = args .get(index + 1) @@ -1082,7 +1114,7 @@ fn parse_host_settings_args( } const SETTINGS_CHOICE_ERROR: &str = - "provide exactly one of --non-blocking, --rate, --voice, --tts, --input-during-tts, --muted, or --restart"; + "provide exactly one of --non-blocking, --rate, --voice, --spokesperson-voice, --spokesperson-rate, --tts, --input-during-tts, --muted, or --restart"; fn parse_bool_setting(flag: &str, value: Option<&String>) -> Result { match value.map(String::as_str) { @@ -2240,6 +2272,20 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String &mut writer, )?; } + Input::Request(SessionRequest::SetSpokespersonSettings { id, .. }) => { + write_message( + &mut writer, + &SessionMessage::TtsSettingsResult { + id, + outcome: TtsSettingsOutcome::Rejected, + snapshot: tts_slot + .as_ref() + .expect("hello initialized TTS") + .snapshot()?, + message: Some("a conventional call has no Spokesperson".into()), + }, + )?; + } Input::Request(SessionRequest::SetTtsSettings { id, expected_revision, @@ -3140,17 +3186,16 @@ fn apply_spokesperson_startup_settings( session: &SessionConfig, spokesperson: &mut OpenAiSpokespersonConfig, ) -> Result<(), String> { + // Replies with their own voice leave the Spokesperson's speed to it. let rate = match &session.tts { + _ if session.expert_tts => session.spokesperson_rate.unwrap_or(spokesperson.speed()), TtsBackendConfig::OpenAi { rate } | TtsBackendConfig::Siri { rate, .. } | TtsBackendConfig::Pocket { rate, .. } => *rate, }; - // A rate for the Expert's own voice need not suit the Spokesperson's. - let rate = if session.expert_tts { - rate.clamp(0.25, 1.5) - } else { - rate - }; + if let Some(voice) = &session.spokesperson_voice { + spokesperson.session.voice = Some(voice.clone()); + } if !(0.25..=1.5).contains(&rate) { return Err("Expert-Spokesperson rate must be between 0.25 and 1.5".into()); } @@ -4276,10 +4321,7 @@ fn run_expert_spokesperson_session( None => tts.clone(), }, input_during_tts: input_policy.snapshot()?, - spokesperson: Some(SpokespersonVoiceSnapshot { - model: spokesperson_config.model().into(), - voice: spokesperson_config.voice().into(), - }), + spokesperson: reply_tts.is_some().then(|| tts.clone()), }; let (created, events) = OpenAiSpokespersonRuntime::spawn_observed(spokesperson_config.clone())?; @@ -4814,11 +4856,18 @@ fn run_expert_spokesperson_session( }); } } - Input::Request(SessionRequest::SetTtsSettings { - id, - expected_revision, - settings, - }) => { + Input::Request( + SessionRequest::SetTtsSettings { + id, + expected_revision, + settings, + } + | SessionRequest::SetSpokespersonSettings { + id, + expected_revision, + settings, + }, + ) => { let request = PendingSpokespersonSettingsUpdate { id, base_revision: expected_revision, @@ -5739,6 +5788,8 @@ fn parse_args(args: &[String]) -> Result { let mut mode = SessionMode::Conventional; let mut endpoints = SessionEndpointOverrides::default(); let mut expert_tts = false; + let mut spokesperson_voice = None; + let mut spokesperson_rate = None; let mut index = 2; while index < args.len() { let flag = args[index].as_str(); @@ -5781,6 +5832,14 @@ fn parse_args(args: &[String]) -> Result { _ => return Err("--expert-voice must be tts or spokesperson".into()), } } + "--spokesperson-voice" => spokesperson_voice = Some(value.clone()), + "--spokesperson-rate" => { + spokesperson_rate = Some( + value + .parse::() + .map_err(|_| "--spokesperson-rate must be a number".to_string())?, + ) + } "--pcm-output-fd" => {} _ => return Err(format!("unknown argument: {flag}").into()), } @@ -5794,6 +5853,20 @@ fn parse_args(args: &[String]) -> Result { if expert_tts && mode != SessionMode::ExpertSpokesperson { return Err("--expert-voice requires --mode expert-spokesperson".into()); } + if (spokesperson_voice.is_some() || spokesperson_rate.is_some()) && !expert_tts { + return Err( + "--spokesperson-voice and --spokesperson-rate require --expert-voice tts".into(), + ); + } + if let Some(voice) = &spokesperson_voice { + if !berd_call::openai_realtime_protocol::OPENAI_REALTIME_VOICE_IDS.contains(&voice.as_str()) + { + return Err(format!("OpenAI has no realtime voice named {voice}").into()); + } + } + if spokesperson_rate.is_some_and(|rate| !(0.25..=1.5).contains(&rate)) { + return Err("--spokesperson-rate must be between 0.25 and 1.5".into()); + } if mode == SessionMode::ExpertSpokesperson && (endpoints.stt.is_some() || endpoints.tts.is_some() && !expert_tts) { @@ -5811,6 +5884,8 @@ fn parse_args(args: &[String]) -> Result { mode, endpoints, expert_tts, + spokesperson_voice, + spokesperson_rate, }) } @@ -7700,6 +7775,7 @@ fn validate_request(request: SessionRequest) -> Result { | SessionRequest::SetInputMuted { id, .. } | SessionRequest::SetConversationStatus { id, .. } | SessionRequest::SetTtsSettings { id, .. } + | SessionRequest::SetSpokespersonSettings { id, .. } | SessionRequest::SetInputDuringTts { id, .. } | SessionRequest::ResetInput { id } | SessionRequest::PrepareSpeak { id, .. } @@ -7770,6 +7846,10 @@ fn validate_request(request: SessionRequest) -> Result { SessionRequest::SetTtsSettings { expected_revision: 0, .. + } + | SessionRequest::SetSpokespersonSettings { + expected_revision: 0, + .. } => return Err("expected TTS revision must be positive".into()), SessionRequest::SetInputDuringTts { expected_revision: 0, @@ -10047,6 +10127,8 @@ mod tests { mode: SessionMode::Conventional, endpoints: SessionEndpointOverrides::default(), expert_tts: false, + spokesperson_voice: None, + spokesperson_rate: None, } ); @@ -10058,6 +10140,8 @@ mod tests { mode: SessionMode::Conventional, endpoints: SessionEndpointOverrides::default(), expert_tts: false, + spokesperson_voice: None, + spokesperson_rate: None, } ); } @@ -10135,6 +10219,8 @@ mod tests { mode: SessionMode::Conventional, endpoints: SessionEndpointOverrides::default(), expert_tts: false, + spokesperson_voice: None, + spokesperson_rate: None, } ); assert!(parse_args(&args(&[ @@ -10191,6 +10277,8 @@ mod tests { mode: SessionMode::ExpertSpokesperson, endpoints: SessionEndpointOverrides::default(), expert_tts: false, + spokesperson_voice: None, + spokesperson_rate: None, }; let mut realtime = OpenAiSpokespersonConfig { endpoint: "ws://localhost".into(), @@ -10256,7 +10344,35 @@ mod tests { semantic_transcript: Vec::new(), }; apply_spokesperson_startup_settings(&own, &mut realtime).unwrap(); - assert_eq!(realtime.speed(), 1.5); + // The replies' rate is theirs, not the Spokesperson's. + assert_eq!(realtime.speed(), 1.0); + let chosen = expert(&[ + "--tts-backend", + "openai", + "--expert-voice", + "tts", + "--spokesperson-voice", + "cedar", + "--spokesperson-rate", + "1.25", + ]) + .unwrap(); + apply_spokesperson_startup_settings(&chosen, &mut realtime).unwrap(); + assert_eq!((realtime.voice(), realtime.speed()), ("cedar", 1.25)); + for (extra, error) in [ + (&["--spokesperson-voice", "cedar"][..], "--expert-voice tts"), + ( + &["--expert-voice", "tts", "--spokesperson-voice", "nobody"], + "no realtime voice named nobody", + ), + ( + &["--expert-voice", "tts", "--spokesperson-rate", "2"], + "between 0.25 and 1.5", + ), + ] { + let all = [&["--tts-backend", "openai"][..], extra].concat(); + assert!(expert(&all).unwrap_err().contains(error), "{extra:?}"); + } let read_aloud = expert(&["--tts-backend", "openai", "--rate", "2.0"]).unwrap(); assert!(apply_spokesperson_startup_settings(&read_aloud, &mut realtime).is_err()); assert!(expert(&["--expert-voice", "loud"]) @@ -10385,6 +10501,8 @@ mod tests { mode: SessionMode::Conventional, endpoints: SessionEndpointOverrides::default(), expert_tts: false, + spokesperson_voice: None, + spokesperson_rate: None, } ); assert!(parse_args(&args(&[ @@ -10447,6 +10565,8 @@ mod tests { mode: SessionMode::Conventional, endpoints: SessionEndpointOverrides::default(), expert_tts: false, + spokesperson_voice: None, + spokesperson_rate: None, } ); assert!(parse_args(&args(&[ @@ -10892,6 +11012,8 @@ mod tests { mode: SessionMode::Conventional, endpoints: SessionEndpointOverrides::default(), expert_tts: false, + spokesperson_voice: None, + spokesperson_rate: None, } ); } @@ -11949,6 +12071,23 @@ mod tests { assert!(start(&["--no-menu-bar", "--no-menu-bar"]).is_err()); } + #[test] + fn spokesperson_settings_name_its_voice_or_rate() { + let parse = |flag: &str, value: &str| { + parse_host_settings_args(&args(&["berd-call", "settings", flag, value])) + .map(|(_, request)| request) + }; + assert!(matches!( + parse("--spokesperson-voice", "cedar").unwrap(), + host_control::ControlRequest::Spokesperson { voice: Some(voice), rate: None } if voice == "cedar" + )); + assert!(matches!( + parse("--spokesperson-rate", "1.25").unwrap(), + host_control::ControlRequest::Spokesperson { voice: None, rate: Some(rate) } if rate == 1.25 + )); + assert!(parse("--spokesperson-rate", "fast").is_err()); + } + #[test] fn settings_rate_changes_only_the_speech_rate() { let (_, request) = @@ -11959,6 +12098,51 @@ mod tests { ); } + #[test] + fn saved_spokesperson_voice_waits_for_calls_whose_replies_have_their_own() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("settings.json"); + let chosen = args(&[ + "--spokesperson-voice", + "cedar", + "--spokesperson-rate", + "1.25", + ]); + saved_settings::save( + &path, + &saved_settings::SavedSettings { + arguments: [ + args(&["--voice", "Aaron", "--language", "en-US"]), + chosen.clone(), + ] + .concat(), + ..saved_settings::SavedSettings::default() + }, + ) + .unwrap(); + let start = |extra: &[&str]| { + parse_saved_start_args_at( + &args(&[&["berd-call", "start"], extra].concat()), + path.clone(), + ) + .unwrap() + }; + let conventional = start(&[]); + assert!(!conventional + .session_arguments + .iter() + .any(|flag| flag.starts_with("--spokesperson"))); + assert!(conventional.saved.unwrap().1.arguments.starts_with(&chosen)); + let own = start(&["--mode", "expert-spokesperson", "--expert-voice", "tts"]); + assert!(own + .session_arguments + .ends_with(&args(&["--expert-voice", "tts"]))); + assert!(own + .session_arguments + .windows(2) + .any(|pair| pair == ["--spokesperson-voice", "cedar"])); + } + #[test] fn saved_start_restores_preferences_but_not_agent_routing() { let directory = tempfile::tempdir().unwrap(); diff --git a/src-tauri/crates/berd-call/src/menu_bar.rs b/src-tauri/crates/berd-call/src/menu_bar.rs index 276370887..be5eebab1 100644 --- a/src-tauri/crates/berd-call/src/menu_bar.rs +++ b/src-tauri/crates/berd-call/src/menu_bar.rs @@ -28,6 +28,7 @@ use crate::host_control::{self, ControlRequest}; use crate::StartOptions; const RATES: [f32; 7] = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75, 2.0]; +const SPOKESPERSON_RATES: std::ops::RangeInclusive = 0.25..=1.5; const REFRESH_INTERVAL: Duration = Duration::from_millis(100); /// Runs the call on a worker thread while the main thread owns the menu bar. @@ -64,6 +65,8 @@ pub(crate) enum MenuAction { language: Option, }, Rate(f32), + SpokespersonVoice(String), + SpokespersonRate(f32), Muted(bool), InputDuringTts(InputDuringTtsPolicy), Stop, @@ -121,33 +124,8 @@ fn menu_model(state: &MenuState) -> Vec { }; let muted = status["muted"].as_bool().unwrap_or(false); let suppressing = input_policy(status) == Some(InputDuringTtsPolicy::SuppressInput); - let rate = status["session"]["tts"]["rate"].as_f64().unwrap_or(1.0) as f32; - let rates = RATES - .iter() - .filter(|&&option| rate_range(status).contains(&option)) - .map(|&option| MenuNode { - checked: (option - rate).abs() < 0.01, - ..item(rate_title(option), Some(MenuAction::Rate(option))) - }) - .collect(); - let voice = status["session"]["tts"]["voice"] - .as_str() - .unwrap_or("Unknown"); - let language = status["session"]["tts"]["language"].as_str(); - let voices = state - .voices - .iter() - .map(|entry| MenuNode { - checked: entry.voice == voice && entry.language.as_deref() == language, - ..item( - entry.title.clone(), - Some(MenuAction::Voice { - voice: entry.voice.clone(), - language: entry.language.clone(), - }), - ) - }) - .collect(); + let tts = &status["session"]["tts"]; + let spokesperson = &status["session"]["spokesperson"]; let mut nodes = vec![item(format!("Berd Call · {}", mode_name(status)), None)]; for error in [&state.action_error, &state.catalog_error] .into_iter() @@ -155,16 +133,31 @@ fn menu_model(state: &MenuState) -> Vec { { nodes.push(item(format!("Control error: {error}"), None)); } - nodes.extend([ - SEPARATOR, - MenuNode { - children: voices, - ..item(format!("Voice: {voice}"), None) - }, - MenuNode { - children: rates, - ..item(format!("Speech Rate: {}", rate_title(rate)), None) + nodes.push(SEPARATOR); + // Replies with their own voice put the Spokesperson's beside it. + let agents = if spokesperson.is_null() { "" } else { "Agent " }; + nodes.extend(voice_menu( + agents, + tts, + &state.voices, + rate_range(status), + |entry| MenuAction::Voice { + voice: entry.voice.clone(), + language: entry.language.clone(), }, + MenuAction::Rate, + )); + if !spokesperson.is_null() { + nodes.extend(voice_menu( + "Spokesperson ", + spokesperson, + &available_voices("openai").unwrap_or_default(), + SPOKESPERSON_RATES, + |entry| MenuAction::SpokespersonVoice(entry.voice.clone()), + MenuAction::SpokespersonRate, + )); + } + nodes.extend([ MenuNode { checked: suppressing, ..item( @@ -188,6 +181,43 @@ fn menu_model(state: &MenuState) -> Vec { nodes } +/// A voice's choices and speeds, titled with whose voice it is. +fn voice_menu( + whose: &str, + tts: &Value, + voices: &[VoiceChoice], + range: std::ops::RangeInclusive, + choose: impl Fn(&VoiceChoice) -> MenuAction, + rate_action: impl Fn(f32) -> MenuAction, +) -> [MenuNode; 2] { + let voice = tts["voice"].as_str().unwrap_or("Unknown"); + let language = tts["language"].as_str(); + let rate = tts["rate"].as_f64().unwrap_or(1.0) as f32; + [ + MenuNode { + children: voices + .iter() + .map(|entry| MenuNode { + checked: entry.voice == voice && entry.language.as_deref() == language, + ..item(entry.title.clone(), Some(choose(entry))) + }) + .collect(), + ..item(format!("{whose}Voice: {voice}"), None) + }, + MenuNode { + children: RATES + .iter() + .filter(|option| range.contains(option)) + .map(|&option| MenuNode { + checked: (option - rate).abs() < 0.01, + ..item(rate_title(option), Some(rate_action(option))) + }) + .collect(), + ..item(format!("{whose}Speech Rate: {}", rate_title(rate)), None) + }, + ] +} + /// Translates a menu action into the control request the CLI would send. pub(crate) fn control_request(action: &MenuAction) -> ControlRequest { match action { @@ -196,6 +226,14 @@ pub(crate) fn control_request(action: &MenuAction) -> ControlRequest { language: language.clone(), }, MenuAction::Rate(rate) => ControlRequest::Rate { rate: *rate }, + MenuAction::SpokespersonVoice(voice) => ControlRequest::Spokesperson { + voice: Some(voice.clone()), + rate: None, + }, + MenuAction::SpokespersonRate(rate) => ControlRequest::Spokesperson { + voice: None, + rate: Some(*rate), + }, MenuAction::Muted(muted) => ControlRequest::Muted { muted: *muted }, MenuAction::InputDuringTts(policy) => ControlRequest::InputDuringTts { policy: *policy }, MenuAction::Stop => ControlRequest::Stop, @@ -204,8 +242,8 @@ pub(crate) fn control_request(action: &MenuAction) -> ControlRequest { /// The rates `berd-call` accepts for the call's TTS backend and mode. fn rate_range(status: &Value) -> std::ops::RangeInclusive { - if mode_name(status) == "Expert-Spokesperson" { - return 0.25..=1.5; + if mode_name(status) == "Expert-Spokesperson" && status["session"]["spokesperson"].is_null() { + return SPOKESPERSON_RATES; } match status["session"]["tts"]["backend"].as_str() { Some("siri") => 0.5..=2.0, @@ -546,7 +584,8 @@ fn action_domain(request: &ControlRequest) -> ActionDomain { match request { ControlRequest::TtsSettings { .. } | ControlRequest::Rate { .. } - | ControlRequest::Voice { .. } => ActionDomain::Tts, + | ControlRequest::Voice { .. } + | ControlRequest::Spokesperson { .. } => ActionDomain::Tts, ControlRequest::Muted { .. } => ActionDomain::Mute, ControlRequest::InputDuringTts { .. } => ActionDomain::InputDuringTts, ControlRequest::Stop => ActionDomain::Stop, @@ -560,6 +599,7 @@ fn revisioned_tts_action(request: &ControlRequest) -> bool { ControlRequest::TtsSettings { .. } | ControlRequest::Rate { .. } | ControlRequest::Voice { .. } + | ControlRequest::Spokesperson { .. } ) } @@ -737,6 +777,41 @@ mod tests { find(&menu_model(&MenuState::default()), "End Call"); } + #[test] + fn replies_with_their_own_voice_put_the_spokespersons_beside_it() { + let mut status = status(false, "allow_barge_in", Some("expert-spokesperson")); + status["session"]["spokesperson"] = json!({"backend": "openai", "model": "gpt-realtime", "voice": "marin", "rate": 1.0, "revision": 1}); + let nodes = menu_model(&MenuState { + status: Some(status), + ..MenuState::default() + }); + let titles = |wanted: &str| -> Vec { + find(&nodes, wanted) + .children + .iter() + .map(|node| node.title.clone()) + .collect() + }; + assert_eq!( + titles("Agent Speech Rate: 1.25×"), + ["0.5×", "0.75×", "1×", "1.25×", "1.5×", "1.75×", "2×"] + ); + assert_eq!( + titles("Spokesperson Speech Rate: 1×"), + ["0.5×", "0.75×", "1×", "1.25×", "1.5×"] + ); + let cedar = find(&nodes, "Spokesperson Voice: marin") + .children + .iter() + .find(|node| node.title == "cedar") + .unwrap(); + assert_eq!( + serde_json::to_value(control_request(cedar.action.as_ref().unwrap())).unwrap(), + json!({"command": "spokesperson", "voice": "cedar", "rate": null}) + ); + find(&nodes, "Agent Voice: Aaron"); + } + #[test] fn actions_become_the_same_requests_as_the_cli() { assert_eq!( diff --git a/src-tauri/crates/berd-call/src/protocol.rs b/src-tauri/crates/berd-call/src/protocol.rs index e27904bd0..550aaeaaa 100644 --- a/src-tauri/crates/berd-call/src/protocol.rs +++ b/src-tauri/crates/berd-call/src/protocol.rs @@ -32,6 +32,12 @@ pub enum SessionRequest { expected_revision: u64, settings: TtsSettings, }, + /// Changes the Spokesperson's voice in a session whose replies have their own. + SetSpokespersonSettings { + id: u64, + expected_revision: u64, + settings: TtsSettings, + }, SetInputDuringTts { id: u64, expected_revision: u64, @@ -161,15 +167,9 @@ pub struct VoiceSessionSnapshot { /// Spokesperson's own voice when it reads them. pub tts: TtsConfigurationSnapshot, pub input_during_tts: InputDuringTtsSnapshot, - /// The Spokesperson's voice, in Expert-Spokesperson sessions. + /// The Spokesperson's voice, when replies have their own. #[serde(default, skip_serializing_if = "Option::is_none")] - pub spokesperson: Option, -} - -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] -pub struct SpokespersonVoiceSnapshot { - pub model: String, - pub voice: String, + pub spokesperson: Option, } #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] @@ -413,14 +413,18 @@ mod tests { revision: 1, policy: InputDuringTtsPolicy::AllowBargeIn, }, - spokesperson: Some(SpokespersonVoiceSnapshot { - model: "gpt-realtime".into(), - voice: "cedar".into(), + spokesperson: Some(TtsConfigurationSnapshot { + revision: 1, + settings: TtsSettings::OpenAi { + model: "gpt-realtime".into(), + voice: "cedar".into(), + rate: 1.0, + }, }), }, }) .unwrap(), - r#"{"type":"ready","id":4,"protocol":5,"session":{"tts":{"revision":1,"backend":"openai","model":"gpt-4o-mini-tts","voice":"marin","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"},"spokesperson":{"model":"gpt-realtime","voice":"cedar"}}}"# + r#"{"type":"ready","id":4,"protocol":5,"session":{"tts":{"revision":1,"backend":"openai","model":"gpt-4o-mini-tts","voice":"marin","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"},"spokesperson":{"revision":1,"backend":"openai","model":"gpt-realtime","voice":"cedar","rate":1.0}}}"# ); assert_eq!( serde_json::from_str::( diff --git a/src-tauri/crates/berd-call/src/saved_settings.rs b/src-tauri/crates/berd-call/src/saved_settings.rs index 0caf56779..e9ff2f081 100644 --- a/src-tauri/crates/berd-call/src/saved_settings.rs +++ b/src-tauri/crates/berd-call/src/saved_settings.rs @@ -99,6 +99,8 @@ fn validate_arguments(arguments: &[String]) -> Result<(), String> { | "--rate" | "--model-dir" | "--stt-model-dir" + | "--spokesperson-voice" + | "--spokesperson-rate" ) }) { @@ -139,6 +141,32 @@ pub(crate) fn persistable_arguments(arguments: &[String]) -> Vec { .collect() } +fn spokesperson_flag(pair: &[String]) -> bool { + matches!( + pair[0].as_str(), + "--spokesperson-voice" | "--spokesperson-rate" + ) +} + +/// Leaves out the Spokesperson's own voice, which only calls whose replies have theirs take. +pub(crate) fn without_spokesperson(arguments: &[String]) -> Vec { + arguments + .chunks_exact(2) + .filter(|pair| !spokesperson_flag(pair)) + .flatten() + .cloned() + .collect() +} + +pub(crate) fn only_spokesperson(arguments: &[String]) -> Vec { + arguments + .chunks_exact(2) + .filter(|pair| spokesperson_flag(pair)) + .flatten() + .cloned() + .collect() +} + pub(crate) fn without_tts(arguments: &[String]) -> Vec { arguments .chunks_exact(2) diff --git a/src-tauri/crates/berd-call/tests/session_protocol.rs b/src-tauri/crates/berd-call/tests/session_protocol.rs index cd9ea0ca6..c86cc1147 100644 --- a/src-tauri/crates/berd-call/tests/session_protocol.rs +++ b/src-tauri/crates/berd-call/tests/session_protocol.rs @@ -2719,6 +2719,73 @@ fn spawn_speech_server() -> (String, mpsc::Receiver) { (url, receiver) } +#[test] +fn expert_voice_keeps_a_separate_spokesperson_voice_control() { + let (speech_url, _speech_requests) = spawn_speech_server(); + let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); + let server = std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + endpoint_tx + .send(format!("ws://{}/", listener.local_addr().unwrap())) + .unwrap(); + let (old_stream, _) = listener.accept().await.unwrap(); + let mut old = accept_async(old_stream).await.unwrap(); + let initial = receive_realtime_json(&mut old).await; + assert_eq!(initial["session"]["audio"]["output"]["voice"], "cedar"); + assert_eq!(initial["session"]["audio"]["output"]["speed"], 1.25); + acknowledge_realtime_session(&mut old, &initial).await; + + let (candidate_stream, _) = listener.accept().await.unwrap(); + let mut candidate = accept_async(candidate_stream).await.unwrap(); + let replacement = receive_realtime_json(&mut candidate).await; + assert_eq!( + replacement["session"]["audio"]["output"]["voice"], + "new-voice" + ); + assert_eq!(replacement["session"]["audio"]["output"]["speed"], 1.25); + acknowledge_realtime_session(&mut candidate, &replacement).await; + while receive_realtime_json(&mut old).await["type"] != "input_audio_buffer.clear" {} + send_realtime_json(&mut old, json!({"type":"input_audio_buffer.cleared"})).await; + while candidate.next().await.is_some() {} + }); + }); + let endpoint = endpoint_rx.recv().unwrap(); + let mut session = ExpertSpokespersonTestSession::start_with_args( + endpoint, + &[ + "--expert-voice", + "tts", + "--tts-url", + &speech_url, + "--spokesperson-voice", + "cedar", + "--spokesperson-rate", + "1.25", + ], + ); + assert_eq!(session.ready["session"]["tts"]["backend"], "openai"); + assert_eq!(session.ready["session"]["spokesperson"]["voice"], "cedar"); + session.send(json!({ + "type":"set_spokesperson_settings", + "id":2, + "expected_revision":1, + "settings":{"backend":"openai","model":"test-model","voice":"new-voice","rate":1.25} + })); + let applied = session.recv(Duration::from_secs(3)); + assert_eq!(applied["type"], "tts_settings_result", "{applied}"); + assert_eq!(applied["id"], 2); + assert_eq!(applied["outcome"], "applied", "{applied}"); + assert_eq!(applied["snapshot"]["revision"], 2); + assert_eq!(applied["snapshot"]["voice"], "new-voice"); + session.shutdown(); + server.join().unwrap(); +} + #[test] fn expert_voice_speaks_replies_itself_and_tells_the_spokesperson_what_was_heard() { let (speech_url, speech_requests) = spawn_speech_server(); @@ -2782,7 +2849,7 @@ fn expert_voice_speaks_replies_itself_and_tells_the_spokesperson_what_was_heard( assert_eq!(session.ready["session"]["tts"]["backend"], "openai"); assert_eq!( session.ready["session"]["spokesperson"], - json!({"model":"test-model","voice":"old-voice"}) + json!({"revision":1,"backend":"openai","model":"test-model","voice":"old-voice","rate":1.0}) ); assert_eq!(session.recv(Duration::from_secs(2))["origin"], "handoff"); assert_eq!( From e553b5f618f8b9456dbf16c0c88c64af39efa3c6 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Mon, 19 Oct 2026 17:12:33 -0400 Subject: [PATCH 04/49] berd-call: let one reply speak in another voice `speak --voice NAME [--language BCP47]` speaks a single utterance in another voice of the call's reply backend, keeping its other settings and leaving the call's voice and saved settings alone. Agents sharing a call can then sound distinct. A voice the backend doesn't offer, or one for a call whose replies the Spokesperson speaks, is not admitted with the new `invalid_voice` reason. Built backends are cached per voice until the call's settings change. Co-Authored-By: Claude Opus 5.5 Signed-off-by: John Tennant --- src-tauri/crates/berd-call/PROTOCOL.md | 8 +- src-tauri/crates/berd-call/README.md | 4 + src-tauri/crates/berd-call/src/cli_help.rs | 6 +- .../crates/berd-call/src/configured_tts.rs | 95 ++++++++++ .../crates/berd-call/src/host_control.rs | 46 ++++- .../crates/berd-call/src/host_session.rs | 15 ++ src-tauri/crates/berd-call/src/main.rs | 165 ++++++++++++++++-- src-tauri/crates/berd-call/src/protocol.rs | 7 + .../berd-call/tests/session_protocol.rs | 16 +- 9 files changed, 340 insertions(+), 22 deletions(-) diff --git a/src-tauri/crates/berd-call/PROTOCOL.md b/src-tauri/crates/berd-call/PROTOCOL.md index 0d1d20639..410801034 100644 --- a/src-tauri/crates/berd-call/PROTOCOL.md +++ b/src-tauri/crates/berd-call/PROTOCOL.md @@ -206,7 +206,7 @@ it never admits a replacement while old host audio may still be active. {"type":"set_spokesperson_settings","id":u64,"expected_revision":u64,"settings":TtsSettings} {"type":"set_input_during_tts","id":u64,"expected_revision":u64,"policy":"allow_barge_in"|"suppress_input"} {"type":"reset_input","id":u64} -{"type":"prepare_speak","id":u64,"acknowledgement":u64|null,"text":string} +{"type":"prepare_speak","id":u64,"acknowledgement":u64|null,"text":string,"resolved_handoff_ids":[string]?,"voice":string?,"language":string?} {"type":"output_ready","id":u64,"speech_id":u64} audio acknowledgements listed above {"type":"query_state","id":u64,"after":u64} @@ -216,6 +216,10 @@ audio acknowledgements listed above ``` Unknown fields are rejected. IDs are positive. Speak text is at most 16 KiB. +`prepare_speak.voice` speaks that one utterance in another voice of the reply +backend, keeping its other settings; `language` changes a Siri voice's locale +too. A voice the backend doesn't offer, or a voice for a call whose replies +are spoken by the Spokesperson, is `not_admitted(invalid_voice)`. The parent cannot author speaking state or finalized input; those are derived only from PCM by the child runtime. @@ -348,7 +352,7 @@ cancellation and shutdown terminals. ```text {"type":"pending","id":u64,"utterances":[{"token":u64,"text":string}]} -{"type":"not_admitted","id":u64,"reason":"paused"|"in_progress"|"cancelled"|"empty_text"} +{"type":"not_admitted","id":u64,"reason":"paused"|"in_progress"|"cancelled"|"empty_text"|"invalid_handoff"|"invalid_voice"} {"type":"state","id":u64,"confirmed_token":u64,"utterances_after":[{"token":u64,"text":string}]} {"type":"cancel_result","id":u64,"outcome":"cancelled"|"stale","speech_id":u64|null} {"type":"speech_started","id":u64,"speech_id":u64} diff --git a/src-tauri/crates/berd-call/README.md b/src-tauri/crates/berd-call/README.md index d48af1818..049ca53e7 100644 --- a/src-tauri/crates/berd-call/README.md +++ b/src-tauri/crates/berd-call/README.md @@ -57,9 +57,13 @@ Start a call with the same backend flags accepted by `session`: berd-call start --voice Aaron --language en-US --stream berd-call status berd-call speak --re 7 "I found the cause." +berd-call speak --voice Nora "Sol here: the tests pass." berd-call stop ``` +`speak --voice` speaks one reply in another voice of the call's reply backend +without changing the call's voice, so agents sharing a call can sound distinct. + `start` stays in the foreground and owns the child session process. The control server listens only on `127.0.0.1:5222` by default; all four commands accept `--port` when more than one local call must be addressed. `--stream` writes one diff --git a/src-tauri/crates/berd-call/src/cli_help.rs b/src-tauri/crates/berd-call/src/cli_help.rs index 4d5edab41..64125af97 100644 --- a/src-tauri/crates/berd-call/src/cli_help.rs +++ b/src-tauri/crates/berd-call/src/cli_help.rs @@ -58,12 +58,14 @@ const SPEAK_HELP: &str = r#"Speak through the active local voice call. Usage: berd-call speak [--port PORT] [--re CURSOR] - [--resolves HANDOFF_ID]... TEXT + [--resolves HANDOFF_ID]... [--voice NAME [--language BCP47]] TEXT Speech waits for admission. The session setting controls whether it also waits for delivery. Non-blocking sessions require --stream or --codex and emit speech_result rows only for interruptions, failures, or pending user input. Interrupted results include -estimatedSpokenText, a best-effort prefix based on delivered audio."#; +estimatedSpokenText, a best-effort prefix based on delivered audio. --voice speaks +this text in another voice of the call's reply backend without changing its saved +voice, so several agents can share one call in distinct voices."#; const SETTINGS_HELP: &str = r#"Update settings for the active local voice call. diff --git a/src-tauri/crates/berd-call/src/configured_tts.rs b/src-tauri/crates/berd-call/src/configured_tts.rs index eaab4d423..c55f467f5 100644 --- a/src-tauri/crates/berd-call/src/configured_tts.rs +++ b/src-tauri/crates/berd-call/src/configured_tts.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::fmt; use std::path::PathBuf; use std::sync::{Arc, Mutex}; @@ -214,6 +215,33 @@ impl TtsConfiguration { } } + /// This configuration in another of its backend's voices. + fn with_voice(mut self, name: &str, locale: Option<&str>) -> Result { + match &mut self { + Self::Siri { + voice, language, .. + } => { + *voice = name.into(); + if let Some(locale) = locale { + *language = locale.into(); + } + } + Self::OpenAi { voice, .. } | Self::Pocket { voice, .. } => { + if locale.is_some() { + return Err("only Siri voices take a language".into()); + } + *voice = name.into(); + } + } + if let Self::OpenAi { voice, .. } = &self { + if !crate::openai_realtime_protocol::OPENAI_REALTIME_VOICE_IDS.contains(&voice.as_str()) + { + return Err(format!("OpenAI has no voice named {voice}")); + } + } + Ok(self) + } + fn build(&self) -> Result, String> { validate_settings(&self.settings())?; match self { @@ -291,6 +319,8 @@ struct ConfiguredTtsState { configuration: TtsConfiguration, backend: Arc, snapshot: TtsConfigurationSnapshot, + /// Backends for other voices, built for single utterances at this revision. + voices: HashMap<(String, Option), TtsConfigurationLease>, } pub struct ConfiguredTtsSlot { @@ -364,6 +394,7 @@ impl ConfiguredTtsSlot { revision: 1, settings, }, + voices: HashMap::new(), }), } } @@ -386,6 +417,37 @@ impl ConfiguredTtsSlot { }) } + /// The configured backend speaking one utterance in another of its voices, + /// keeping the other settings. Building it doesn't hold the slot lock. + pub fn lease_voice( + &self, + voice: &str, + language: Option<&str>, + ) -> Result { + let key = (voice.to_string(), language.map(str::to_string)); + let poisoned = || "configured TTS lock was poisoned".to_string(); + let (configuration, revision) = { + let state = self.inner.lock().map_err(|_| poisoned())?; + if let Some(lease) = state.voices.get(&key) { + return Ok(lease.clone()); + } + (state.configuration.clone(), state.snapshot.revision) + }; + let configuration = configuration.with_voice(voice, language)?; + let lease = TtsConfigurationLease { + backend: configuration.build()?, + snapshot: TtsConfigurationSnapshot { + revision, + settings: configuration.settings(), + }, + }; + let mut state = self.inner.lock().map_err(|_| poisoned())?; + if state.snapshot.revision == revision { + state.voices.insert(key, lease.clone()); + } + Ok(lease) + } + /// Builds and validates a replacement without holding the slot lock. pub fn prepare_replacement( &self, @@ -475,6 +537,7 @@ impl ConfiguredTtsSlot { })?; state.configuration = replacement.configuration; state.backend = replacement.backend; + state.voices.clear(); state.snapshot = TtsConfigurationSnapshot { revision, settings: replacement.settings, @@ -554,6 +617,38 @@ mod tests { } } + #[test] + fn one_utterance_can_use_another_voice_without_changing_the_slot() { + let settings = |voice: &str| TtsSettings::OpenAi { + model: "gpt-4o-mini-tts".into(), + voice: voice.into(), + rate: 1.25, + }; + let slot = ConfiguredTtsSlot::new_for_test(settings("marin")); + let cedar = slot.lease_voice("cedar", None).unwrap(); + assert_eq!(cedar.snapshot().settings, settings("cedar")); + assert!(Arc::ptr_eq( + cedar.backend(), + slot.lease_voice("cedar", None).unwrap().backend() + )); + assert_eq!(slot.snapshot().unwrap().settings, settings("marin")); + assert!(slot + .lease_voice("nobody", None) + .err() + .unwrap() + .contains("no voice named nobody")); + assert!(slot + .lease_voice("cedar", Some("en-US")) + .err() + .unwrap() + .contains("language")); + slot.replace_for_test(1, settings("ash")).unwrap(); + assert!(!Arc::ptr_eq( + cedar.backend(), + slot.lease_voice("cedar", None).unwrap().backend() + )); + } + #[test] fn snapshot_is_sanitized_and_revisioned() { let slot = ConfiguredTtsSlot::new(TtsConfiguration::openai( diff --git a/src-tauri/crates/berd-call/src/host_control.rs b/src-tauri/crates/berd-call/src/host_control.rs index cd00eb516..e98cadecf 100644 --- a/src-tauri/crates/berd-call/src/host_control.rs +++ b/src-tauri/crates/berd-call/src/host_control.rs @@ -28,6 +28,8 @@ pub(crate) trait HostControl: Send + Sync + 'static { text: String, acknowledgement: Option, resolved_handoff_ids: Vec, + voice: Option, + language: Option, ) -> Result; fn stop(&self) -> Result; fn set_non_blocking(&self, enabled: bool) -> Result; @@ -115,6 +117,10 @@ pub(crate) enum ControlRequest { acknowledgement: Option, #[serde(rename = "resolvedHandoffIds")] resolved_handoff_ids: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + voice: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + language: Option, }, Stop, NonBlocking { @@ -231,9 +237,16 @@ fn handle_connection(mut stream: TcpStream, control: &dyn HostControl) -> Result text, acknowledgement, resolved_handoff_ids, + voice, + language, } => { - validate_speak(&text, &resolved_handoff_ids)?; - control.speak(text, acknowledgement, resolved_handoff_ids) + validate_speak( + &text, + &resolved_handoff_ids, + voice.as_deref(), + language.is_some(), + )?; + control.speak(text, acknowledgement, resolved_handoff_ids, voice, language) } ControlRequest::Stop => control.stop(), ControlRequest::TtsSettings { settings } => control.set_tts(settings), @@ -310,7 +323,15 @@ fn write_json_line( .map_err(|error| format!("could not write berd-call {label}: {error}")) } -fn validate_speak(text: &str, resolved_handoff_ids: &[String]) -> Result<(), String> { +fn validate_speak( + text: &str, + resolved_handoff_ids: &[String], + voice: Option<&str>, + language: bool, +) -> Result<(), String> { + if voice.is_some_and(|voice| voice.trim().is_empty()) || language && voice.is_none() { + return Err("speak --language requires a nonempty --voice".into()); + } if text.trim().is_empty() { return Err("speak text must not be empty".into()); } @@ -390,11 +411,15 @@ mod tests { text: String, acknowledgement: Option, resolved_handoff_ids: Vec, + voice: Option, + language: Option, ) -> Result { Ok(json!({ "spoken":text, "acknowledgement":acknowledgement, "resolvedHandoffIds":resolved_handoff_ids, + "voice":voice, + "language":language, })) } @@ -441,6 +466,8 @@ mod tests { text: "hello".into(), acknowledgement: Some(7), resolved_handoff_ids: vec!["call-1".into()], + voice: Some("Nora".into()), + language: Some("en-GB".into()), } ) .unwrap(), @@ -448,6 +475,8 @@ mod tests { "spoken":"hello", "acknowledgement":7, "resolvedHandoffIds":["call-1"], + "voice":"Nora", + "language":"en-GB", }) ); assert_eq!( @@ -459,9 +488,12 @@ mod tests { #[test] fn speak_validation_rejects_invalid_values() { - assert!(validate_speak(" ", &[]).is_err()); - assert!(validate_speak(&"x".repeat(16 * 1024 + 1), &[]).is_err()); - assert!(validate_speak("hello", &["".into()]).is_err()); - assert!(validate_speak("hello", &["same".into(), "same".into()]).is_err()); + assert!(validate_speak(" ", &[], None, false).is_err()); + assert!(validate_speak(&"x".repeat(16 * 1024 + 1), &[], None, false).is_err()); + assert!(validate_speak("hello", &["".into()], None, false).is_err()); + assert!(validate_speak("hello", &["same".into(), "same".into()], None, false).is_err()); + assert!(validate_speak("hello", &[], None, true).is_err()); + assert!(validate_speak("hello", &[], Some(" "), false).is_err()); + assert!(validate_speak("hello", &[], Some("Nora"), true).is_ok()); } } diff --git a/src-tauri/crates/berd-call/src/host_session.rs b/src-tauri/crates/berd-call/src/host_session.rs index ac01d0fa5..71b09af77 100644 --- a/src-tauri/crates/berd-call/src/host_session.rs +++ b/src-tauri/crates/berd-call/src/host_session.rs @@ -517,6 +517,8 @@ enum ControlCommand { text: String, acknowledgement: Option, resolved_handoff_ids: Vec, + voice: Option, + language: Option, non_blocking: bool, response: SyncSender>, }, @@ -739,6 +741,8 @@ impl HostControl for SessionControl { text: String, acknowledgement: Option, resolved_handoff_ids: Vec, + voice: Option, + language: Option, ) -> Result { let (tx, rx) = mpsc::sync_channel(1); self.commands @@ -746,6 +750,8 @@ impl HostControl for SessionControl { text, acknowledgement, resolved_handoff_ids, + voice, + language, non_blocking: self.non_blocking.load(Ordering::SeqCst), response: tx, }) @@ -1092,6 +1098,8 @@ impl SessionActor { text, acknowledgement, resolved_handoff_ids, + voice, + language, non_blocking, response, } => { @@ -1104,6 +1112,8 @@ impl SessionActor { text, acknowledgement, resolved_handoff_ids, + voice, + language, non_blocking, response, }); @@ -1117,6 +1127,8 @@ impl SessionActor { acknowledgement, text: text.clone(), resolved_handoff_ids, + voice, + language, }, )?; self.pending_speak = Some(PendingSpeak { @@ -2661,6 +2673,7 @@ fn not_admitted_reason_name(reason: NotAdmittedReason) -> &'static str { NotAdmittedReason::Cancelled => "cancelled", NotAdmittedReason::EmptyText => "empty_text", NotAdmittedReason::InvalidHandoff => "invalid_handoff", + NotAdmittedReason::InvalidVoice => "invalid_voice", } } @@ -3356,6 +3369,8 @@ mod tests { text: "next".into(), acknowledgement: Some(7), resolved_handoff_ids: Vec::new(), + voice: None, + language: None, non_blocking: true, response: next_tx, }) diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 5e9c639b5..ebac91061 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -539,6 +539,8 @@ fn main() { port, acknowledgement, resolved_handoff_ids, + voice, + language, text, } = parse_or_exit(parse_speak_control_args(&args), &args); let response = host_control::request( @@ -547,6 +549,8 @@ fn main() { text, acknowledgement, resolved_handoff_ids, + voice, + language, }, ) .unwrap_or_else(|error| operational_error("speak", error)); @@ -921,6 +925,8 @@ struct SpeakOptions { port: u16, acknowledgement: Option, resolved_handoff_ids: Vec, + voice: Option, + language: Option, text: String, } @@ -929,6 +935,7 @@ fn parse_speak_control_args(args: &[String]) -> Result Result return Err("--re may be provided only once".into()), + flag @ ("--voice" | "--language") => { + let slot = if flag == "--voice" { + &mut voice + } else { + &mut language + }; + if slot.is_some() { + return Err(format!("{flag} may be provided only once").into()); + } + *slot = Some( + args.get(index + 1) + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| format!("{flag} requires a value"))? + .clone(), + ); + index += 2; + } "--resolves" => { let value = args .get(index + 1) @@ -984,10 +1008,15 @@ fn parse_speak_control_args(args: &[String]) -> Result MAX_SPEAK_TEXT_BYTES { return Err("speak text is larger than 16 KiB".into()); } + if language.is_some() && voice.is_none() { + return Err("--language requires --voice".into()); + } Ok(SpeakOptions { port, acknowledgement, resolved_handoff_ids, + voice, + language, text, }) } @@ -1992,7 +2021,7 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String let mut next_input_token = 1_u64; let mut pending_control = None; let mut processed_pcm = 0_u64; - let mut held: Option = None; + let mut held: Option = None; let mut active: Option = None; let mut status_sound_runtime = StatusSoundRuntime::default(); @@ -2132,7 +2161,7 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String "session is shutting down", &mut writer, )?; - if let Some(held) = held.take() { + if let Some((held, _)) = held.take() { write_message( &mut writer, &SessionMessage::NotAdmitted { @@ -2366,12 +2395,19 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String acknowledgement, text, resolved_handoff_ids: _, + voice, + language, }) => { let request = PrepareRequest { id, acknowledgement, text, }; + let Some(voice) = + speak_voice(tts_slot.as_deref(), voice, language, id, &mut writer)? + else { + continue; + }; if held.is_some() { write_message( &mut writer, @@ -2383,6 +2419,7 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String } else { process_prepare( request, + voice, &mut core, tts_slot.as_deref().expect("hello initialized TTS"), input_during_tts_slot @@ -2681,6 +2718,8 @@ struct PendingExpertPrepare { acknowledgement: Option, text: String, resolved_handoff_ids: Vec, + /// The Expert's voice for this reply, when not its usual one. + voice: Option, } struct PendingSpokespersonSettingsUpdate { @@ -2919,6 +2958,8 @@ struct LiveResponse { /// An Expert reply spoken in its own voice, with the handoffs it answers. /// It has no provider response; its whole text is played at once. reply: Option>, + /// The voice it asked for, when not the Expert's usual one. + voice: Option, delivery: RealtimeAudioDelivery, pending_audio: VecDeque>, pending_frames: usize, @@ -2937,6 +2978,7 @@ impl LiveResponse { speech_id, expert_text: None, reply: None, + voice: None, delivery: RealtimeAudioDelivery::default(), pending_audio: VecDeque::new(), pending_frames: 0, @@ -4373,12 +4415,20 @@ fn run_expert_spokesperson_session( acknowledgement, text, resolved_handoff_ids, + voice, + language, }) => { + let Some(voice) = + speak_voice(reply_tts.as_deref(), voice, language, id, &mut writer)? + else { + continue; + }; let request = PendingExpertPrepare { id, acknowledgement, text, resolved_handoff_ids, + voice, }; let routing = route_expert_prepare( &mut turn_gate, @@ -5153,6 +5203,7 @@ fn submit_expert_prepare( let mut response = LiveResponse::new(Some(request.id), Some(speech_id)); response.expert_text = Some(message); response.reply = Some(resolved_handoff_ids); + response.voice = request.voice; response.audio_done = true; response.finish_sent = true; response.server_finished = true; @@ -5334,7 +5385,13 @@ fn start_next_live_playback( response.speech_id.expect("an Expert reply has a speech ID"), response.prepare_id, response.expert_text.clone().unwrap_or_default(), - Arc::clone(tts.lease()?.backend()), + Arc::clone( + response + .voice + .clone() + .map_or_else(|| tts.lease(), Ok)? + .backend(), + ), Arc::clone(audio_transport), audio_control_tx.clone(), reply_tx.clone(), @@ -7079,20 +7136,55 @@ fn reject_tts_configuration_update( ) } +/// A held reply, with the voice it asked for. +type HeldPrepare = (PrepareRequest, Option); + +/// The voice one reply asked for, or `None` after telling the host it can't +/// have it. +fn speak_voice( + slot: Option<&ConfiguredTtsSlot>, + voice: Option, + language: Option, + id: u64, + writer: &mut impl Write, +) -> Result>, String> { + let Some(voice) = voice else { + return Ok(Some(None)); + }; + let lease = slot + .ok_or_else(|| "this call's replies have no voice of their own".to_string()) + .and_then(|slot| slot.lease_voice(&voice, language.as_deref())); + match lease { + Ok(lease) => Ok(Some(Some(lease))), + Err(message) => { + eprintln!("berd-call: speech {id} can't use voice {voice}: {message}"); + write_message( + writer, + &SessionMessage::NotAdmitted { + id, + reason: NotAdmittedReason::InvalidVoice, + }, + )?; + Ok(None) + } + } +} + #[allow(clippy::too_many_arguments)] fn process_prepare( request: PrepareRequest, + voice: Option, core: &mut SessionCore, tts_slot: &ConfiguredTtsSlot, input_during_tts_slot: &InputDuringTtsSlot, active: &mut Option, - held: &mut Option, + held: &mut Option, writer: &mut impl Write, ) -> Result<(), String> { let id = request.id; match core.prepare(request.clone()) { PrepareOutcome::Hold => { - *held = Some(request); + *held = Some((request, voice)); } PrepareOutcome::Pending(utterances) => { write_message(writer, &SessionMessage::Pending { id, utterances })?; @@ -7105,7 +7197,7 @@ fn process_prepare( confirmed_token, text, } => { - let tts = tts_slot.lease()?; + let tts = voice.map_or_else(|| tts_slot.lease(), Ok)?; let input_during_tts = input_during_tts_slot.snapshot()?; *active = Some(ActivePlayback { prepare_id: id, @@ -7133,7 +7225,7 @@ fn process_prepare( } fn reevaluate_held( - held: &mut Option, + held: &mut Option, core: &mut SessionCore, tts_slot: Option<&ConfiguredTtsSlot>, input_during_tts_slot: Option<&InputDuringTtsSlot>, @@ -7141,9 +7233,10 @@ fn reevaluate_held( writer: &mut impl Write, ) -> Result<(), String> { if !core.user_speaking() && !core.recognition_pending() && active.is_none() { - if let Some(pending_prepare) = held.take() { + if let Some((pending_prepare, voice)) = held.take() { process_prepare( pending_prepare, + voice, core, tts_slot.expect("held prepare requires initialized TTS"), input_during_tts_slot @@ -7478,12 +7571,12 @@ fn handle_reset_input( fn handle_cancel( id: u64, - held: &mut Option, + held: &mut Option, core: &mut SessionCore, active: &mut Option, writer: &mut impl Write, ) -> Result<(), String> { - if held.as_ref().is_some_and(|held| held.id == id) { + if held.as_ref().is_some_and(|(held, _)| held.id == id) { held.take(); write_message( writer, @@ -8257,6 +8350,7 @@ mod tests { acknowledgement: Some(1), text: "answer".into(), resolved_handoff_ids: Vec::new(), + voice: None, }; assert!(matches!( gate.defer_if_busy(request, false, 1), @@ -8296,6 +8390,7 @@ mod tests { acknowledgement: Some(1), text: "first".into(), resolved_handoff_ids: Vec::new(), + voice: None, }, false, 1, @@ -8309,6 +8404,7 @@ mod tests { acknowledgement: Some(1), text: "second".into(), resolved_handoff_ids: Vec::new(), + voice: None, }, false, 1, @@ -8339,6 +8435,7 @@ mod tests { acknowledgement: Some(1), text: "second".into(), resolved_handoff_ids: Vec::new(), + voice: None, }, expert_output_reserved(&directives, &HashSet::new(), None, &responses), false, @@ -8865,6 +8962,7 @@ mod tests { acknowledgement: Some(1), text: "answer after the user".into(), resolved_handoff_ids: Vec::new(), + voice: None, }, false, 1, @@ -9289,6 +9387,35 @@ mod tests { test_tts_slot().lease().unwrap() } + #[test] + fn speech_in_another_voice_is_refused_when_the_voice_is_unavailable() { + let slot = test_tts_slot(); + let mut writer = Vec::new(); + assert!(speak_voice(Some(&slot), None, None, 1, &mut writer) + .unwrap() + .unwrap() + .is_none()); + let coral = speak_voice(Some(&slot), Some("coral".into()), None, 2, &mut writer) + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(coral.snapshot().settings.voice(), "coral"); + assert!(writer.is_empty()); + for (id, slot, voice) in [(3, Some(&slot), "nobody"), (4, None, "coral")] { + assert!(speak_voice(slot, Some(voice.into()), None, id, &mut writer) + .unwrap() + .is_none()); + } + let refusals = String::from_utf8(writer).unwrap(); + assert_eq!( + refusals.lines().collect::>(), + [ + r#"{"type":"not_admitted","id":3,"reason":"invalid_voice"}"#, + r#"{"type":"not_admitted","id":4,"reason":"invalid_voice"}"#, + ] + ); + } + fn test_input_policy_slot() -> InputDuringTtsSlot { InputDuringTtsSlot::new(InputDuringTtsPolicy::AllowBargeIn) } @@ -11194,6 +11321,8 @@ mod tests { acknowledgement: None, text: "answer".into(), resolved_handoff_ids: vec![" ".into()], + voice: None, + language: None, }; assert_eq!( validate_request(empty).unwrap_err(), @@ -11244,6 +11373,7 @@ mod tests { acknowledgement: None, text: "reply".into(), }, + None, &mut core, &test_tts_slot(), &input_policy, @@ -11289,6 +11419,7 @@ mod tests { acknowledgement: None, text: "old voice".into(), }, + None, &mut core, &slot, &input_policy, @@ -12377,6 +12508,10 @@ mod tests { "5300", "--re", "17", + "--voice", + "Nora", + "--language", + "en-GB", "hello", ])) .unwrap(), @@ -12384,9 +12519,19 @@ mod tests { port: 5300, acknowledgement: Some(17), resolved_handoff_ids: Vec::new(), + voice: Some("Nora".into()), + language: Some("en-GB".into()), text: "hello".into(), } ); + assert!(parse_speak_control_args(&args(&[ + "berd-call", + "speak", + "--language", + "en-GB", + "hello" + ])) + .is_err()); assert_eq!( parse_control_port(&args(&["berd-call", "status", "--port", "5301"])).unwrap(), 5301 diff --git a/src-tauri/crates/berd-call/src/protocol.rs b/src-tauri/crates/berd-call/src/protocol.rs index 550aaeaaa..02e58c261 100644 --- a/src-tauri/crates/berd-call/src/protocol.rs +++ b/src-tauri/crates/berd-call/src/protocol.rs @@ -52,6 +52,10 @@ pub enum SessionRequest { text: String, #[serde(default)] resolved_handoff_ids: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + voice: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + language: Option, }, OutputReady { id: u64, @@ -145,6 +149,7 @@ pub enum NotAdmittedReason { Cancelled, EmptyText, InvalidHandoff, + InvalidVoice, } #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] @@ -394,6 +399,8 @@ mod tests { acknowledgement: Some(0), text: "hi".into(), resolved_handoff_ids: Vec::new(), + voice: None, + language: None, } ); assert_eq!( diff --git a/src-tauri/crates/berd-call/tests/session_protocol.rs b/src-tauri/crates/berd-call/tests/session_protocol.rs index c86cc1147..d2af1fa6c 100644 --- a/src-tauri/crates/berd-call/tests/session_protocol.rs +++ b/src-tauri/crates/berd-call/tests/session_protocol.rs @@ -2862,7 +2862,8 @@ fn expert_voice_speaks_replies_itself_and_tells_the_spokesperson_what_was_heard( "id":2, "acknowledgement":1, "text":"I checked the repository.", - "resolved_handoff_ids":["handoff-external-1"] + "resolved_handoff_ids":["handoff-external-1"], + "voice":"coral" })); let admitted = session.recv(Duration::from_secs(2)); assert_eq!(admitted["type"], "admitted", "{admitted}"); @@ -2889,6 +2890,19 @@ fn expert_voice_speaks_replies_itself_and_tells_the_spokesperson_what_was_heard( .recv_timeout(Duration::from_secs(1)) .unwrap(); assert_eq!(speech["input"], "I checked the repository."); + assert_eq!(speech["voice"], "coral"); + session.send(json!({ + "type":"prepare_speak", + "id":3, + "acknowledgement":1, + "text":"Who am I?", + "voice":"nobody" + })); + let rejected = session.recv(Duration::from_secs(2)); + assert_eq!( + rejected, + json!({"type":"not_admitted","id":3,"reason":"invalid_voice"}) + ); let (context, mentions, after) = context_rx.recv_timeout(Duration::from_secs(2)).unwrap(); assert_eq!(context["type"], "conversation.item.create", "{context}"); From 54f6b853a391c3a30d383fbcfd27475c5b99d23d Mon Sep 17 00:00:00 2001 From: John Tennant Date: Mon, 19 Oct 2026 17:17:56 -0400 Subject: [PATCH 05/49] berd-call: list Siri voices whose assets are missing as not installed sirittsd reports en-GB-C and en-GB-D as downloaded on a Mac where their assets aren't on disk, and then speaks both in the same low-quality substitute (byte-identical renders). Every voice that really speaks has a TTSAXResourceManager contentPath; these have none. The catalog now counts a voice as installed only when sirittsd reports it downloaded and its content path exists, so the menu bar, Berd's settings and the plugin stop offering it, and `voices download` fetches it instead. Co-Authored-By: Claude Opus 5.5 Signed-off-by: John Tennant --- src-tauri/crates/berd-call/README.md | 4 +++- src-tauri/crates/berd-call/native/siri_tts_bridge.m | 8 +++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src-tauri/crates/berd-call/README.md b/src-tauri/crates/berd-call/README.md index 049ca53e7..77cb30a97 100644 --- a/src-tauri/crates/berd-call/README.md +++ b/src-tauri/crates/berd-call/README.md @@ -270,7 +270,9 @@ retries cleanup. This avoids reporting a failed install after the model has already been applied. `voices.list` returns `backend`, normalized `languageFilter`, -`availableLanguages`, and exact voice records. `voices.download` returns the +`availableLanguages`, and exact voice records. A voice is `installed` only when +sirittsd reports it downloaded and its assets are on disk: sirittsd can claim +a voice whose assets are missing, then silently speak it in another voice. `voices.download` returns the canonical voice, `installed: true`, and `availabilityWaitSeconds`; a missing exact catalog identity fails with `voice_not_found` before any native download request. macOS status and install results contain `supported`, `locale`, diff --git a/src-tauri/crates/berd-call/native/siri_tts_bridge.m b/src-tauri/crates/berd-call/native/siri_tts_bridge.m index ce06b7a13..1f73dc5da 100644 --- a/src-tauri/crates/berd-call/native/siri_tts_bridge.m +++ b/src-tauri/crates/berd-call/native/siri_tts_bridge.m @@ -997,10 +997,14 @@ static void BerdDownloadedVoices( NSString *name = [resource valueForKey:@"name"] ?: @""; if (!name.length || !language.length) continue; NSString *key = [NSString stringWithFormat:@"%@|%@", name, normalizedLanguage]; + // sirittsd can report a voice as downloaded whose assets aren't on disk, + // and then speaks it in another voice. + NSString *content = [resource valueForKey:@"contentPath"]; byKey[key] = @{ @"name" : name, @"language" : language, @"sizeBytes" : [resource valueForKey:@"assetSize"] ?: @0, + @"content" : @(content.length && [NSFileManager.defaultManager fileExistsAtPath:content]), }; } return [[byKey allValues] sortedArrayUsingComparator: @@ -1142,7 +1146,9 @@ static BOOL BerdTriggerDownload(NSString *language, NSError **error) { NSString *key = [NSString stringWithFormat:@"%@|%@", candidate[@"name"], normalizedLanguage]; NSMutableDictionary *voice = [candidate mutableCopy]; - voice[@"installed"] = installedSnapshot[key] ?: @NO; + [voice removeObjectForKey:@"content"]; + voice[@"installed"] = + [installedSnapshot[key] boolValue] && [candidate[@"content"] boolValue] ? @YES : @NO; [result addObject:voice]; } NSData *json = [NSJSONSerialization dataWithJSONObject:result options:0 error:&error]; From d7568a6792edf68cd1a851d47e5eebdab693c7cf Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 11:21:49 -0400 Subject: [PATCH 06/49] End the session when an Expert reply's audio never settles A reply whose playback failed without the host confirming the cancellation left old audio possibly still playing, yet the session went on admitting output. Carry the playback failure's quiescence into the reply result and end the session after reporting the failure, as conventional playback already does. Signed-off-by: John Tennant Co-Authored-By: Claude Opus 5.5 --- src-tauri/crates/berd-call/src/main.rs | 27 ++++--- .../berd-call/tests/session_protocol.rs | 77 +++++++++++++++++++ 2 files changed, 94 insertions(+), 10 deletions(-) diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index ebac91061..baf47a42c 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -2699,7 +2699,7 @@ type LivePlaybackResult = Result<(String, u64), (String, u64, String)>; enum ReplyOutcome { Completed, Interrupted { spoken_through_utf8: u64 }, - Failed(String), + Failed(PlaybackFailure), } type ReplyResult = (String, u64, ReplyOutcome); @@ -4141,6 +4141,7 @@ fn run_expert_spokesperson_session( .prepare_id .expect("an Expert reply has a prepare ID"); let text = response.expert_text.take().unwrap_or_default(); + let mut output_quiescent = true; let (message, heard) = match outcome { ReplyOutcome::Completed => ( SessionMessage::SpeechCompleted { id, speech_id }, @@ -4156,14 +4157,17 @@ fn run_expert_spokesperson_session( }, usize::try_from(spoken_through_utf8).unwrap_or(text.len()), ), - ReplyOutcome::Failed(message) => ( - SessionMessage::SpeechFailed { - id, - speech_id, - message, - }, - 0, - ), + ReplyOutcome::Failed(failure) => { + output_quiescent = failure.output_quiescent; + ( + SessionMessage::SpeechFailed { + id, + speech_id, + message: failure.message, + }, + 0, + ) + } }; let heard = (0..=heard.min(text.len())) .rev() @@ -4173,6 +4177,9 @@ fn run_expert_spokesperson_session( if response.claim_speech_terminal() { write_message(&mut writer, &message)?; } + if !output_quiescent { + return Err("remote PCM output did not reach a quiescent terminal".into()); + } finish_expert_reply( &mut core, runtime.as_ref().expect("initialized runtime"), @@ -4995,7 +5002,7 @@ fn spawn_reply_playback( spoken_through_utf8: u64::try_from(estimated_spoken_through_utf8(&text, &delivery)) .expect("speech text is bounded well below u64"), }, - Err(failure) => ReplyOutcome::Failed(failure.message), + Err(failure) => ReplyOutcome::Failed(failure), }; let _ = completed.send((worker_response_id, speech_id, outcome)); }); diff --git a/src-tauri/crates/berd-call/tests/session_protocol.rs b/src-tauri/crates/berd-call/tests/session_protocol.rs index d2af1fa6c..573e7720c 100644 --- a/src-tauri/crates/berd-call/tests/session_protocol.rs +++ b/src-tauri/crates/berd-call/tests/session_protocol.rs @@ -38,6 +38,8 @@ struct ExpertSpokespersonTestSession { struct AudioCancellationGate { observed: mpsc::SyncSender<()>, release: mpsc::Receiver<()>, + /// Hold the drain instead, as a hung host would, without reading anything after it. + at_drain: bool, } impl ExpertSpokespersonTestSession { @@ -391,6 +393,11 @@ fn spawn_audio_host_with_played_limit( let frames = u64::from_le_bytes(payload[16..24].try_into().unwrap()); assert_eq!(current, Some((speech_id, sequence, frames))); current = None; + if let Some(gate) = cancellation_gate.take_if(|gate| gate.at_drain) { + let _ = gate.observed.send(()); + let _ = gate.release.recv(); + return; + } json!({"type":"audio_drained","speech_id":speech_id,"sequence":sequence,"played_frames":frames}) } 4 => { @@ -949,6 +956,7 @@ fn autonomous_spokesperson_cancel_is_correlated_before_audio_cancellation_and_th Some(AudioCancellationGate { observed: cancel_observed_tx, release: release_cancel_rx, + at_drain: false, }), ); let started = loop { @@ -2786,6 +2794,75 @@ fn expert_voice_keeps_a_separate_spokesperson_voice_control() { server.join().unwrap(); } +#[test] +fn expert_voice_reply_whose_audio_never_settles_ends_the_session() { + let (speech_url, _speech_requests) = spawn_speech_server(); + let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); + std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + endpoint_tx + .send(format!("ws://{}/", listener.local_addr().unwrap())) + .unwrap(); + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = accept_async(stream).await.unwrap(); + let initial = receive_realtime_json(&mut socket).await; + acknowledge_realtime_session(&mut socket, &initial).await; + while socket.next().await.is_some() {} + }); + }); + let (observed, cancel_observed) = mpsc::sync_channel(1); + let (release_cancel, release) = mpsc::sync_channel(1); + // The host never confirms the drain, nor the cancellation that follows it. + let mut session = ExpertSpokespersonTestSession::launch( + endpoint_rx.recv().unwrap(), + None, + None, + None, + Some(AudioCancellationGate { + observed, + release, + at_drain: true, + }), + &["--expert-voice", "tts", "--tts-url", &speech_url], + ); + session.send(json!({ + "type":"prepare_speak", + "id":2, + "acknowledgement":null, + "text":"I checked the repository." + })); + let admitted = session.recv(Duration::from_secs(2)); + assert_eq!(admitted["type"], "admitted", "{admitted}"); + session.send(json!({"type":"output_ready","id":2,"speech_id":admitted["speech_id"]})); + let deadline = std::time::Instant::now() + Duration::from_secs(8); + let mut messages = Vec::new(); + loop { + let remaining = deadline + .checked_duration_since(std::time::Instant::now()) + .unwrap_or_else(|| panic!("the session went on: {messages:?}")); + match session.output.recv_timeout(remaining) { + Ok(message) => messages.push(message), + Err(mpsc::RecvTimeoutError::Disconnected) => break, + Err(mpsc::RecvTimeoutError::Timeout) => panic!("the session went on: {messages:?}"), + } + } + assert!( + messages + .iter() + .any(|message| message["type"] == "speech_failed" && message["id"] == 2), + "the reply did not fail: {messages:?}" + ); + let status = session.child.0.as_mut().unwrap().wait().unwrap(); + assert!(!status.success()); + cancel_observed.try_recv().unwrap(); + release_cancel.send(()).unwrap(); +} + #[test] fn expert_voice_speaks_replies_itself_and_tells_the_spokesperson_what_was_heard() { let (speech_url, speech_requests) = spawn_speech_server(); From b1ced00bb45570800a04aadfe0159c01019775bb Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 11:25:31 -0400 Subject: [PATCH 07/49] Build an utterance's new voice off the session loop `speak --voice` built and validated a voice it hadn't used yet on the session loop. Siri validation can wait seconds for sirittsd, so a reply already playing could miss its audio acknowledgement deadlines and microphone input could back up. Build the voice on a worker and hand the request back to the loop once it's ready, or refuse it there if the voice can't be built. Signed-off-by: John Tennant Co-Authored-By: Claude Opus 5.5 --- .../crates/berd-call/src/configured_tts.rs | 9 ++ src-tauri/crates/berd-call/src/main.rs | 116 ++++++++++++++++-- 2 files changed, 117 insertions(+), 8 deletions(-) diff --git a/src-tauri/crates/berd-call/src/configured_tts.rs b/src-tauri/crates/berd-call/src/configured_tts.rs index c55f467f5..09d156b96 100644 --- a/src-tauri/crates/berd-call/src/configured_tts.rs +++ b/src-tauri/crates/berd-call/src/configured_tts.rs @@ -417,6 +417,15 @@ impl ConfiguredTtsSlot { }) } + /// Whether `lease_voice` already has this voice built. + pub fn has_voice(&self, voice: &str, language: Option<&str>) -> bool { + self.inner.lock().is_ok_and(|state| { + state + .voices + .contains_key(&(voice.to_string(), language.map(str::to_string))) + }) + } + /// The configured backend speaking one utterance in another of its voices, /// keeping the other settings. Building it doesn't hold the slot lock. pub fn lease_voice( diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index baf47a42c..f7158c982 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -110,6 +110,12 @@ enum Input { Request(SessionRequest), Pcm(Box), Invalid(String), + /// A speech request's voice could not be built. + VoiceUnavailable { + id: u64, + voice: String, + message: String, + }, Eof, } @@ -2002,6 +2008,7 @@ fn expert_session_status_cues_suppressed( fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String> { let (control_tx, control_rx) = mpsc::channel(); let (pcm_tx, pcm_rx) = mpsc::sync_channel(INPUT_QUEUE_CAPACITY); + let voice_tx = control_tx.clone(); thread::spawn(move || read_framed_requests(io::stdin().lock(), control_tx, pcm_tx)); let audio_transport = Arc::new(unsafe { AudioPipeTransport::from_raw_fd(pcm_output_fd)? }); let (playback_tx, playback_rx) = mpsc::channel(); @@ -2107,6 +2114,9 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String ) else { continue; }; + let Some(input) = build_voice_later(input, tts_slot.as_ref(), &voice_tx) else { + continue; + }; match input { Input::Invalid(message) => { write_protocol_fatal(&mut writer, "invalid session input", &message)?; @@ -2116,6 +2126,9 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String } return Ok(()); } + Input::VoiceUnavailable { id, voice, message } => { + refuse_voice(id, &voice, &message, &mut writer)?; + } Input::Eof => { if let Some(current) = active.as_mut() { if let Some(flag) = ¤t.active { @@ -3251,6 +3264,7 @@ fn run_expert_spokesperson_session( ) -> Result<(), String> { let (control_tx, control_rx) = mpsc::channel(); let (pcm_tx, pcm_rx) = mpsc::sync_channel(INPUT_QUEUE_CAPACITY); + let voice_tx = control_tx.clone(); thread::spawn(move || read_framed_requests(io::stdin().lock(), control_tx, pcm_tx)); let audio_transport = Arc::new(unsafe { AudioPipeTransport::from_raw_fd(pcm_output_fd)? }); let (audio_control_tx, audio_control_rx) = mpsc::channel(); @@ -4244,11 +4258,17 @@ fn run_expert_spokesperson_session( ) else { continue; }; + let Some(input) = build_voice_later(input, reply_tts.as_ref(), &voice_tx) else { + continue; + }; match input { Input::Invalid(message) => { write_protocol_fatal(&mut writer, "invalid session input", &message)?; break; } + Input::VoiceUnavailable { id, voice, message } => { + refuse_voice(id, &voice, &message, &mut writer)?; + } Input::Eof => break, Input::Request(SessionRequest::Shutdown) => { if let Some(request) = queued_tts_settings.take() { @@ -7164,19 +7184,65 @@ fn speak_voice( match lease { Ok(lease) => Ok(Some(Some(lease))), Err(message) => { - eprintln!("berd-call: speech {id} can't use voice {voice}: {message}"); - write_message( - writer, - &SessionMessage::NotAdmitted { - id, - reason: NotAdmittedReason::InvalidVoice, - }, - )?; + refuse_voice(id, &voice, &message, writer)?; Ok(None) } } } +fn refuse_voice( + id: u64, + voice: &str, + message: &str, + writer: &mut impl Write, +) -> Result<(), String> { + eprintln!("berd-call: speech {id} can't use voice {voice}: {message}"); + write_message( + writer, + &SessionMessage::NotAdmitted { + id, + reason: NotAdmittedReason::InvalidVoice, + }, + ) +} + +/// Builds a speech request's new voice on a worker, since that can take seconds, so the session +/// keeps serving audio and input meanwhile; the worker hands the request back once it's built. +fn build_voice_later( + input: Input, + slot: Option<&Arc>, + controls: &mpsc::Sender, +) -> Option { + let ( + Some(slot), + Input::Request(SessionRequest::PrepareSpeak { + id, + voice: Some(voice), + language, + .. + }), + ) = (slot, &input) + else { + return Some(input); + }; + if slot.has_voice(voice, language.as_deref()) { + return Some(input); + } + let (id, voice, language) = (*id, voice.clone(), language.clone()); + let (slot, controls) = (Arc::clone(slot), controls.clone()); + thread::spawn(move || { + let input = match slot.lease_voice(&voice, language.as_deref()) { + Ok(_) => input, + Err(message) => Input::VoiceUnavailable { id, voice, message }, + }; + let _ = controls.send(OrderedControl { + after_pcm: 0, + input, + }); + }); + None +} + #[allow(clippy::too_many_arguments)] fn process_prepare( request: PrepareRequest, @@ -9394,6 +9460,40 @@ mod tests { test_tts_slot().lease().unwrap() } + #[test] + fn a_new_voice_is_built_off_the_session_loop_and_its_request_handed_back() { + let slot = Arc::new(test_tts_slot()); + let (controls, handed_back) = mpsc::channel::(); + let prepare = |id, voice: &str| { + Input::Request(SessionRequest::PrepareSpeak { + id, + acknowledgement: None, + text: "Hello.".into(), + resolved_handoff_ids: Vec::new(), + voice: Some(voice.into()), + language: None, + }) + }; + let next = || { + handed_back + .recv_timeout(Duration::from_secs(2)) + .unwrap() + .input + }; + assert!(build_voice_later(prepare(1, "coral"), Some(&slot), &controls).is_none()); + assert!(matches!( + next(), + Input::Request(SessionRequest::PrepareSpeak { id: 1, .. }) + )); + assert!(matches!( + build_voice_later(prepare(2, "coral"), Some(&slot), &controls), + Some(Input::Request(SessionRequest::PrepareSpeak { id: 2, .. })) + )); + assert!(build_voice_later(prepare(3, "nobody"), Some(&slot), &controls).is_none()); + assert!(matches!(next(), Input::VoiceUnavailable { id: 3, .. })); + assert!(build_voice_later(prepare(4, "coral"), None, &controls).is_some()); + } + #[test] fn speech_in_another_voice_is_refused_when_the_voice_is_unavailable() { let slot = test_tts_slot(); From 04b73fa88910461f492a4d57b7efeba6dd80b4aa Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 11:33:02 -0400 Subject: [PATCH 08/49] Let cancel reach speech waiting on its voice A speech request whose voice was still being built was invisible to `cancel`, which answered stale; the request then came back and played. Track the requests waiting on a voice, cancel them there, and drop a cancelled one when its voice arrives. Signed-off-by: John Tennant Co-Authored-By: Claude Opus 5.5 --- src-tauri/crates/berd-call/src/main.rs | 194 +++++++++++++++++++------ 1 file changed, 146 insertions(+), 48 deletions(-) diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index f7158c982..3ee3b3138 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -110,6 +110,8 @@ enum Input { Request(SessionRequest), Pcm(Box), Invalid(String), + /// A speech request whose voice is now built. + VoiceBuilt(SessionRequest), /// A speech request's voice could not be built. VoiceUnavailable { id: u64, @@ -2008,7 +2010,10 @@ fn expert_session_status_cues_suppressed( fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String> { let (control_tx, control_rx) = mpsc::channel(); let (pcm_tx, pcm_rx) = mpsc::sync_channel(INPUT_QUEUE_CAPACITY); - let voice_tx = control_tx.clone(); + let mut voice_builds = VoiceBuilds { + controls: control_tx.clone(), + pending: HashSet::new(), + }; thread::spawn(move || read_framed_requests(io::stdin().lock(), control_tx, pcm_tx)); let audio_transport = Arc::new(unsafe { AudioPipeTransport::from_raw_fd(pcm_output_fd)? }); let (playback_tx, playback_rx) = mpsc::channel(); @@ -2114,7 +2119,7 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String ) else { continue; }; - let Some(input) = build_voice_later(input, tts_slot.as_ref(), &voice_tx) else { + let Some(input) = voice_builds.filter(input, tts_slot.as_ref(), &mut writer)? else { continue; }; match input { @@ -2126,8 +2131,8 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String } return Ok(()); } - Input::VoiceUnavailable { id, voice, message } => { - refuse_voice(id, &voice, &message, &mut writer)?; + Input::VoiceBuilt(_) | Input::VoiceUnavailable { .. } => { + unreachable!("voice builds are filtered") } Input::Eof => { if let Some(current) = active.as_mut() { @@ -3264,7 +3269,10 @@ fn run_expert_spokesperson_session( ) -> Result<(), String> { let (control_tx, control_rx) = mpsc::channel(); let (pcm_tx, pcm_rx) = mpsc::sync_channel(INPUT_QUEUE_CAPACITY); - let voice_tx = control_tx.clone(); + let mut voice_builds = VoiceBuilds { + controls: control_tx.clone(), + pending: HashSet::new(), + }; thread::spawn(move || read_framed_requests(io::stdin().lock(), control_tx, pcm_tx)); let audio_transport = Arc::new(unsafe { AudioPipeTransport::from_raw_fd(pcm_output_fd)? }); let (audio_control_tx, audio_control_rx) = mpsc::channel(); @@ -4258,7 +4266,7 @@ fn run_expert_spokesperson_session( ) else { continue; }; - let Some(input) = build_voice_later(input, reply_tts.as_ref(), &voice_tx) else { + let Some(input) = voice_builds.filter(input, reply_tts.as_ref(), &mut writer)? else { continue; }; match input { @@ -4266,8 +4274,8 @@ fn run_expert_spokesperson_session( write_protocol_fatal(&mut writer, "invalid session input", &message)?; break; } - Input::VoiceUnavailable { id, voice, message } => { - refuse_voice(id, &voice, &message, &mut writer)?; + Input::VoiceBuilt(_) | Input::VoiceUnavailable { .. } => { + unreachable!("voice builds are filtered") } Input::Eof => break, Input::Request(SessionRequest::Shutdown) => { @@ -7206,41 +7214,92 @@ fn refuse_voice( ) } -/// Builds a speech request's new voice on a worker, since that can take seconds, so the session -/// keeps serving audio and input meanwhile; the worker hands the request back once it's built. -fn build_voice_later( - input: Input, - slot: Option<&Arc>, - controls: &mpsc::Sender, -) -> Option { - let ( - Some(slot), - Input::Request(SessionRequest::PrepareSpeak { - id, - voice: Some(voice), - language, - .. - }), - ) = (slot, &input) +/// Speech requests waiting on a worker to build a voice the session hasn't used yet, since that +/// can take seconds and the session must keep serving audio and input meanwhile. +struct VoiceBuilds { + controls: mpsc::Sender, + pending: HashSet, +} + +impl VoiceBuilds { + /// Takes the requests that wait on a voice, and hands each back once its voice is built. + fn filter( + &mut self, + input: Input, + slot: Option<&Arc>, + writer: &mut impl Write, + ) -> Result, String> { + match input { + Input::VoiceBuilt(request) => { + let SessionRequest::PrepareSpeak { id, .. } = request else { + unreachable!("only speech waits on a voice") + }; + Ok(self.pending.remove(&id).then_some(Input::Request(request))) + } + Input::VoiceUnavailable { id, voice, message } => { + if self.pending.remove(&id) { + refuse_voice(id, &voice, &message, writer)?; + } + Ok(None) + } + Input::Request(SessionRequest::Cancel { id }) if self.pending.remove(&id) => { + write_message( + writer, + &SessionMessage::CancelResult { + id, + outcome: CancelOutcome::Cancelled, + speech_id: None, + }, + )?; + write_message( + writer, + &SessionMessage::NotAdmitted { + id, + reason: NotAdmittedReason::Cancelled, + }, + )?; + Ok(None) + } + Input::Request(request @ SessionRequest::PrepareSpeak { id, .. }) => { + let missing = slot.and_then(|slot| { + missing_voice(slot, &request).map(|voice| (Arc::clone(slot), voice)) + }); + let Some((slot, (voice, language))) = missing else { + return Ok(Some(Input::Request(request))); + }; + let controls = self.controls.clone(); + self.pending.insert(id); + thread::spawn(move || { + let input = match slot.lease_voice(&voice, language.as_deref()) { + Ok(_) => Input::VoiceBuilt(request), + Err(message) => Input::VoiceUnavailable { id, voice, message }, + }; + let _ = controls.send(OrderedControl { + after_pcm: 0, + input, + }); + }); + Ok(None) + } + input => Ok(Some(input)), + } + } +} + +/// The voice and language a speech request asks for, when the slot hasn't built them yet. +fn missing_voice( + slot: &ConfiguredTtsSlot, + request: &SessionRequest, +) -> Option<(String, Option)> { + let SessionRequest::PrepareSpeak { + voice: Some(voice), + language, + .. + } = request else { - return Some(input); + return None; }; - if slot.has_voice(voice, language.as_deref()) { - return Some(input); - } - let (id, voice, language) = (*id, voice.clone(), language.clone()); - let (slot, controls) = (Arc::clone(slot), controls.clone()); - thread::spawn(move || { - let input = match slot.lease_voice(&voice, language.as_deref()) { - Ok(_) => input, - Err(message) => Input::VoiceUnavailable { id, voice, message }, - }; - let _ = controls.send(OrderedControl { - after_pcm: 0, - input, - }); - }); - None + (!slot.has_voice(voice, language.as_deref())).then(|| (voice.clone(), language.clone())) } #[allow(clippy::too_many_arguments)] @@ -9464,6 +9523,12 @@ mod tests { fn a_new_voice_is_built_off_the_session_loop_and_its_request_handed_back() { let slot = Arc::new(test_tts_slot()); let (controls, handed_back) = mpsc::channel::(); + let mut builds = VoiceBuilds { + controls, + pending: HashSet::new(), + }; + let mut writer = Vec::new(); + let mut filter = |input, slot| builds.filter(input, slot, &mut writer).unwrap(); let prepare = |id, voice: &str| { Input::Request(SessionRequest::PrepareSpeak { id, @@ -9480,18 +9545,51 @@ mod tests { .unwrap() .input }; - assert!(build_voice_later(prepare(1, "coral"), Some(&slot), &controls).is_none()); + assert!(filter(prepare(1, "coral"), Some(&slot)).is_none()); assert!(matches!( - next(), - Input::Request(SessionRequest::PrepareSpeak { id: 1, .. }) + filter(next(), Some(&slot)), + Some(Input::Request(SessionRequest::PrepareSpeak { id: 1, .. })) )); assert!(matches!( - build_voice_later(prepare(2, "coral"), Some(&slot), &controls), + filter(prepare(2, "coral"), Some(&slot)), Some(Input::Request(SessionRequest::PrepareSpeak { id: 2, .. })) )); - assert!(build_voice_later(prepare(3, "nobody"), Some(&slot), &controls).is_none()); - assert!(matches!(next(), Input::VoiceUnavailable { id: 3, .. })); - assert!(build_voice_later(prepare(4, "coral"), None, &controls).is_some()); + assert!(filter(prepare(3, "nobody"), Some(&slot)).is_none()); + assert!(filter(next(), Some(&slot)).is_none()); + // A request cancelled while its voice is built is not revived by the build. + assert!(filter(prepare(4, "marin"), Some(&slot)).is_none()); + assert!(filter( + Input::Request(SessionRequest::Cancel { id: 4 }), + Some(&slot) + ) + .is_none()); + assert!(filter(next(), Some(&slot)).is_none()); + assert!(filter(prepare(5, "cedar"), None).is_some()); + drop(filter); + let types: Vec<_> = String::from_utf8(writer) + .unwrap() + .lines() + .map(|line| { + let message: serde_json::Value = serde_json::from_str(line).unwrap(); + format!( + "{} {} {}", + message["type"], + message["id"], + message["reason"] + .as_str() + .or(message["outcome"].as_str()) + .unwrap() + ) + }) + .collect(); + assert_eq!( + types, + [ + r#""not_admitted" 3 invalid_voice"#, + r#""cancel_result" 4 cancelled"#, + r#""not_admitted" 4 cancelled"#, + ] + ); } #[test] From a9696911fa00ce95fcece04657cbd287cab6ab32 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 11:34:20 -0400 Subject: [PATCH 09/49] Test that user speech and cancel stop an Expert reply The test host can now hold a drain and then keep serving, so a reply stays playing until something stops it. Signed-off-by: John Tennant Co-Authored-By: Claude Opus 5.5 --- .../berd-call/tests/session_protocol.rs | 105 +++++++++++++++++- 1 file changed, 100 insertions(+), 5 deletions(-) diff --git a/src-tauri/crates/berd-call/tests/session_protocol.rs b/src-tauri/crates/berd-call/tests/session_protocol.rs index 573e7720c..0228b9f67 100644 --- a/src-tauri/crates/berd-call/tests/session_protocol.rs +++ b/src-tauri/crates/berd-call/tests/session_protocol.rs @@ -38,7 +38,8 @@ struct ExpertSpokespersonTestSession { struct AudioCancellationGate { observed: mpsc::SyncSender<()>, release: mpsc::Receiver<()>, - /// Hold the drain instead, as a hung host would, without reading anything after it. + /// Hold the drain instead, reading nothing after it, as a hung host would. Once released, + /// the host goes on without ever draining that speech. at_drain: bool, } @@ -392,12 +393,14 @@ fn spawn_audio_host_with_played_limit( let sequence = u64::from_le_bytes(payload[8..16].try_into().unwrap()); let frames = u64::from_le_bytes(payload[16..24].try_into().unwrap()); assert_eq!(current, Some((speech_id, sequence, frames))); - current = None; if let Some(gate) = cancellation_gate.take_if(|gate| gate.at_drain) { let _ = gate.observed.send(()); - let _ = gate.release.recv(); - return; + if gate.release.recv().is_err() { + return; + } + continue; } + current = None; json!({"type":"audio_drained","speech_id":speech_id,"sequence":sequence,"played_frames":frames}) } 4 => { @@ -2794,6 +2797,98 @@ fn expert_voice_keeps_a_separate_spokesperson_voice_control() { server.join().unwrap(); } +enum ReplyInterruption { + UserSpeech, + Cancel, +} + +/// Stops an Expert reply partway through playback and returns what the session reports. +fn interrupt_expert_reply(interruption: ReplyInterruption) -> (Value, usize) { + let (speech_url, _speech_requests) = spawn_speech_server(); + let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); + let (user_speaks, user_speech) = mpsc::sync_channel(1); + std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + endpoint_tx + .send(format!("ws://{}/", listener.local_addr().unwrap())) + .unwrap(); + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = accept_async(stream).await.unwrap(); + let initial = receive_realtime_json(&mut socket).await; + acknowledge_realtime_session(&mut socket, &initial).await; + if user_speech.recv().is_ok() { + send_realtime_json( + &mut socket, + json!({"type":"input_audio_buffer.speech_started","item_id":"user-1"}), + ) + .await; + } + while socket.next().await.is_some() {} + }); + }); + let (observed, playing) = mpsc::sync_channel(1); + let (release_drain, release) = mpsc::sync_channel(1); + // The host has played part of the reply and is still playing it. + let mut session = ExpertSpokespersonTestSession::launch( + endpoint_rx.recv().unwrap(), + None, + Some(1_000), + None, + Some(AudioCancellationGate { + observed, + release, + at_drain: true, + }), + &["--expert-voice", "tts", "--tts-url", &speech_url], + ); + let text = "I checked the repository and found three failing tests in the parser."; + session.send(json!({"type":"prepare_speak","id":2,"acknowledgement":null,"text":text})); + let admitted = session.recv(Duration::from_secs(2)); + assert_eq!(admitted["type"], "admitted", "{admitted}"); + session.send(json!({"type":"output_ready","id":2,"speech_id":admitted["speech_id"]})); + playing.recv_timeout(Duration::from_secs(2)).unwrap(); + release_drain.send(()).unwrap(); + match interruption { + ReplyInterruption::UserSpeech => user_speaks.send(()).unwrap(), + ReplyInterruption::Cancel => session.send(json!({"type":"cancel","id":2})), + } + let interrupted = loop { + let message = session.recv(Duration::from_secs(2)); + assert!( + !matches!( + message["type"].as_str(), + Some("fatal" | "speech_failed" | "speech_completed") + ), + "{message}" + ); + if message["type"] == "speech_interrupted" { + break message; + } + }; + drop(user_speaks); + session.shutdown(); + (interrupted, text.len()) +} + +#[test] +fn user_speech_interrupts_an_expert_reply_where_it_was_heard() { + let (interrupted, length) = interrupt_expert_reply(ReplyInterruption::UserSpeech); + assert_eq!(interrupted["id"], 2); + assert!(interrupted["spoken_through_utf8"].as_u64().unwrap() < length as u64); +} + +#[test] +fn cancel_interrupts_an_expert_reply_where_it_was_heard() { + let (interrupted, length) = interrupt_expert_reply(ReplyInterruption::Cancel); + assert_eq!(interrupted["id"], 2); + assert!(interrupted["spoken_through_utf8"].as_u64().unwrap() < length as u64); +} + #[test] fn expert_voice_reply_whose_audio_never_settles_ends_the_session() { let (speech_url, _speech_requests) = spawn_speech_server(); @@ -2860,7 +2955,7 @@ fn expert_voice_reply_whose_audio_never_settles_ends_the_session() { let status = session.child.0.as_mut().unwrap().wait().unwrap(); assert!(!status.success()); cancel_observed.try_recv().unwrap(); - release_cancel.send(()).unwrap(); + drop(release_cancel); } #[test] From 4d1cc303f3483846787d66ca59eefbddd9215c7d Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 13:15:11 -0400 Subject: [PATCH 10/49] Wait for output_ready before playing an Expert reply An admitted Expert-voice reply was queued for playback at once, so it could start before the parent sent output_ready. It now waits: a matching output_ready marks it ready and queues it, and the accepted result is written before playback can begin. Mismatched, repeated or dropped requests are stale. Signed-off-by: John Tennant Co-Authored-By: Claude Opus 5.5 --- src-tauri/crates/berd-call/src/main.rs | 36 ++++++---- .../berd-call/tests/session_protocol.rs | 68 +++++++++++++++++++ 2 files changed, 92 insertions(+), 12 deletions(-) diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 3ee3b3138..1cd81fce9 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -2978,6 +2978,8 @@ struct LiveResponse { reply: Option>, /// The voice it asked for, when not the Expert's usual one. voice: Option, + /// Whether the parent has accepted the reply's output, so it may play. + output_ready: bool, delivery: RealtimeAudioDelivery, pending_audio: VecDeque>, pending_frames: usize, @@ -2997,6 +2999,7 @@ impl LiveResponse { expert_text: None, reply: None, voice: None, + output_ready: false, delivery: RealtimeAudioDelivery::default(), pending_audio: VecDeque::new(), pending_frames: 0, @@ -4245,7 +4248,6 @@ fn run_expert_spokesperson_session( &mut directive_speeches, reply_tts.is_some(), &mut responses, - &mut waiting_responses, &mut next_speech_id, &mut writer, )?; @@ -4486,7 +4488,6 @@ fn run_expert_spokesperson_session( &mut directive_speeches, reply_tts.is_some(), &mut responses, - &mut waiting_responses, &mut next_speech_id, &mut writer, )?, @@ -4501,12 +4502,24 @@ fn run_expert_spokesperson_session( } } Input::Request(SessionRequest::OutputReady { id, speech_id }) => { - let valid = directive_speeches - .get(&id) - .is_some_and(|directive| directive.speech_id == speech_id) - || responses.values().any(|response| { - response.prepare_id == Some(id) && response.speech_id == Some(speech_id) - }); + let matches = |response: &LiveResponse| { + response.prepare_id == Some(id) && response.speech_id == Some(speech_id) + }; + let reply = responses.iter_mut().find(|(_, response)| { + response.reply.is_some() && !response.output_ready && matches(response) + }); + let valid = if let Some((response_id, response)) = reply { + response.output_ready = true; + waiting_responses.push_back(response_id.clone()); + true + } else { + directive_speeches + .get(&id) + .is_some_and(|directive| directive.speech_id == speech_id) + || responses + .values() + .any(|response| response.reply.is_none() && matches(response)) + }; write_message( &mut writer, &SessionMessage::OutputReadyResult { @@ -5191,7 +5204,6 @@ fn submit_expert_prepare( directive_speeches: &mut HashMap, expert_tts: bool, responses: &mut HashMap, - waiting_responses: &mut VecDeque, next_speech_id: &mut u64, writer: &mut impl Write, ) -> Result<(), String> { @@ -5233,7 +5245,8 @@ fn submit_expert_prepare( let resolved_handoff_ids = request.resolved_handoff_ids; core.mark_handoffs_resolving(&resolved_handoff_ids)?; if expert_tts { - // Played in turn with the Spokesperson's responses, without one of its own. + // Played in turn with the Spokesperson's responses, without one of its own, + // once the parent is ready for its output. let response_id = format!("expert-reply-{speech_id}"); let mut response = LiveResponse::new(Some(request.id), Some(speech_id)); response.expert_text = Some(message); @@ -5242,8 +5255,7 @@ fn submit_expert_prepare( response.audio_done = true; response.finish_sent = true; response.server_finished = true; - responses.insert(response_id.clone(), response); - waiting_responses.push_back(response_id); + responses.insert(response_id, response); } else { directive_speeches.insert( request.id, diff --git a/src-tauri/crates/berd-call/tests/session_protocol.rs b/src-tauri/crates/berd-call/tests/session_protocol.rs index 0228b9f67..090b3d2c2 100644 --- a/src-tauri/crates/berd-call/tests/session_protocol.rs +++ b/src-tauri/crates/berd-call/tests/session_protocol.rs @@ -3092,3 +3092,71 @@ fn expert_voice_speaks_replies_itself_and_tells_the_spokesperson_what_was_heard( session.shutdown(); server.join().unwrap(); } + +#[test] +fn expert_voice_reply_waits_for_output_ready_before_playing() { + let (speech_url, speech_requests) = spawn_speech_server(); + let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); + std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + endpoint_tx + .send(format!("ws://{}/", listener.local_addr().unwrap())) + .unwrap(); + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = accept_async(stream).await.unwrap(); + let initial = receive_realtime_json(&mut socket).await; + acknowledge_realtime_session(&mut socket, &initial).await; + while socket.next().await.is_some() {} + }); + }); + let mut session = ExpertSpokespersonTestSession::start_with_args( + endpoint_rx.recv().unwrap(), + &["--expert-voice", "tts", "--tts-url", &speech_url], + ); + session.send(json!({ + "type":"prepare_speak", + "id":2, + "acknowledgement":null, + "text":"I checked the repository." + })); + let admitted = session.recv(Duration::from_secs(2)); + assert_eq!(admitted["type"], "admitted", "{admitted}"); + let speech_id = admitted["speech_id"].as_u64().unwrap(); + if let Ok(message) = session.output.recv_timeout(Duration::from_millis(500)) { + panic!("the reply went on before output_ready: {message}"); + } + assert!( + speech_requests.try_recv().is_err(), + "the reply was synthesized before output_ready" + ); + for (id, speech_id) in [(2, speech_id + 1), (3, speech_id)] { + session.send(json!({"type":"output_ready","id":id,"speech_id":speech_id})); + assert_eq!( + session.recv(Duration::from_secs(2)), + json!({"type":"output_ready_result","id":id,"speech_id":speech_id,"outcome":"stale"}) + ); + } + session.send(json!({"type":"output_ready","id":2,"speech_id":speech_id})); + assert_eq!( + session.recv(Duration::from_secs(2)), + json!({"type":"output_ready_result","id":2,"speech_id":speech_id,"outcome":"accepted"}) + ); + let mut started = false; + loop { + let message = session.recv(Duration::from_secs(3)); + match message["type"].as_str() { + Some("speech_started") => started = true, + Some("speech_completed") => break, + _ => panic!("unexpected message: {message}"), + } + } + assert!(started); + session.send(json!({"type":"output_ready","id":2,"speech_id":speech_id})); + assert_eq!(session.recv(Duration::from_secs(2))["outcome"], "stale"); + session.shutdown(); +} From ef2eadd5c70daa27d55c9b23c5de4972516ed7d3 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 13:15:12 -0400 Subject: [PATCH 11/49] Tell the Spokesperson what was heard of a failed Expert reply A failed Expert reply was treated as unheard, so its handoffs were released and the Spokesperson was told the user heard none of it. Playback failures now carry what had been played, so the heard prefix reaches the Spokesperson and an answered handoff is dismissed. Signed-off-by: John Tennant Co-Authored-By: Claude Opus 5.5 --- src-tauri/crates/berd-call/src/main.rs | 26 ++-- .../berd-call/tests/session_protocol.rs | 123 ++++++++++++++++++ 2 files changed, 141 insertions(+), 8 deletions(-) diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 1cd81fce9..ed85acaf7 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -138,6 +138,8 @@ enum PlaybackEvent { #[derive(Debug)] struct PlaybackFailure { message: String, + /// What the host had played when playback failed. + delivery: DeliveryProgress, output_quiescent: bool, } @@ -4190,7 +4192,7 @@ fn run_expert_spokesperson_session( speech_id, message: failure.message, }, - 0, + estimated_spoken_through_utf8(&text, &failure.delivery), ) } }; @@ -8172,17 +8174,16 @@ fn play_tts( active: &AtomicBool, ) -> Result<(bool, DeliveryProgress), PlaybackFailure> { if let Err(message) = output.start() { + let delivery = DeliveryProgress { + sample_rate: backend.pcm_spec().sample_rate, + segments: Vec::new(), + }; if message == AUDIO_CANCELLED { - return Ok(( - false, - DeliveryProgress { - sample_rate: backend.pcm_spec().sample_rate, - segments: Vec::new(), - }, - )); + return Ok((false, delivery)); } return Err(PlaybackFailure { message, + delivery, output_quiescent: output.failure_is_quiescent(), }); } @@ -8223,19 +8224,26 @@ fn synthesize_to_output_with_finish( use berd_call::{DrainPolicy, DrainTimeoutOutcome, OutboundOutcome, OutboundPlayback}; let spec = backend.pcm_spec(); + let unplayed = || DeliveryProgress { + sample_rate: spec.sample_rate, + segments: Vec::new(), + }; let initial_frames = usize::try_from(spec.sample_rate / 5).map_err(|_| PlaybackFailure { message: "TTS sample rate is too large".into(), + delivery: unplayed(), output_quiescent: false, })?; let mut playback = OutboundPlayback::new(output, active, spec.sample_rate, initial_frames) .map_err(|message| PlaybackFailure { message, + delivery: unplayed(), output_quiescent: false, })?; if playback .synthesize_segment(backend, text, &mut |_| Ok(()), on_started, &mut |_| Ok(())) .map_err(|failure| PlaybackFailure { message: failure.message, + delivery: failure.delivery, output_quiescent: failure.output_quiescent, })? == OutboundOutcome::Interrupted @@ -8246,6 +8254,7 @@ fn synthesize_to_output_with_finish( let output_quiescent = output.cancel_and_snapshot().is_ok(); return Err(PlaybackFailure { message, + delivery: playback.snapshot(), output_quiescent, }); } @@ -8260,6 +8269,7 @@ fn synthesize_to_output_with_finish( ) .map_err(|failure| PlaybackFailure { message: failure.message, + delivery: failure.delivery, output_quiescent: failure.output_quiescent, })?; Ok((outcome == OutboundOutcome::Completed, playback.snapshot())) diff --git a/src-tauri/crates/berd-call/tests/session_protocol.rs b/src-tauri/crates/berd-call/tests/session_protocol.rs index 090b3d2c2..dd2da7a78 100644 --- a/src-tauri/crates/berd-call/tests/session_protocol.rs +++ b/src-tauri/crates/berd-call/tests/session_protocol.rs @@ -3160,3 +3160,126 @@ fn expert_voice_reply_waits_for_output_ready_before_playing() { assert_eq!(session.recv(Duration::from_secs(2))["outcome"], "stale"); session.shutdown(); } + +/// Fails an Expert reply answering a handoff once the host has played `played_frames` of it, +/// returning the failure, what the Spokesperson was told, and the handoffs left open. +fn fail_expert_reply_after(played_frames: u64) -> (Value, String, Value) { + let (speech_url, _speech_requests) = spawn_speech_server(); + let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); + let (context_tx, context_rx) = mpsc::sync_channel(1); + std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + endpoint_tx + .send(format!("ws://{}/", listener.local_addr().unwrap())) + .unwrap(); + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = accept_async(stream).await.unwrap(); + let initial = receive_realtime_json(&mut socket).await; + acknowledge_realtime_session(&mut socket, &initial).await; + send_realtime_json( + &mut socket, + json!({ + "type":"response.function_call_arguments.done", + "response_id":"response-handoff", + "call_id":"handoff-reply", + "name":"handoff", + "arguments":"{\"message\":\"inspect the repository\"}" + }), + ) + .await; + let context = loop { + let text = receive_realtime_json(&mut socket).await.to_string(); + if text.contains("spoken reply") || text.contains("in its own voice") { + break text; + } + }; + context_tx.send(context).unwrap(); + while socket.next().await.is_some() {} + }); + }); + let (observed, playing) = mpsc::sync_channel(1); + let (release_drain, release) = mpsc::sync_channel(1); + let mut session = ExpertSpokespersonTestSession::launch( + endpoint_rx.recv().unwrap(), + None, + Some(played_frames), + None, + Some(AudioCancellationGate { + observed, + release, + at_drain: true, + }), + &["--expert-voice", "tts", "--tts-url", &speech_url], + ); + assert_eq!(session.recv(Duration::from_secs(2))["origin"], "handoff"); + assert_eq!( + session.recv(Duration::from_secs(2))["handoff_ids"], + json!(["handoff-external-1"]) + ); + session.send(json!({ + "type":"prepare_speak", + "id":2, + "acknowledgement":1, + "text":"I checked the repository and found three failing tests in the parser.", + "resolved_handoff_ids":["handoff-external-1"] + })); + let admitted = session.recv(Duration::from_secs(2)); + assert_eq!(admitted["type"], "admitted", "{admitted}"); + session.send(json!({"type":"output_ready","id":2,"speech_id":admitted["speech_id"]})); + playing.recv_timeout(Duration::from_secs(2)).unwrap(); + release_drain.send(()).unwrap(); + session.send(json!({ + "type":"audio_failed", + "speech_id":admitted["speech_id"], + "played_frames":played_frames, + "message":"output device disappeared" + })); + let failed = loop { + let message = session.recv(Duration::from_secs(2)); + assert!( + !matches!( + message["type"].as_str(), + Some("fatal" | "speech_completed" | "speech_interrupted") + ), + "{message}" + ); + if message["type"] == "speech_failed" { + break message; + } + }; + let context = context_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + session.send(json!({"type":"query_state","id":3,"after":0})); + let state = loop { + let message = session.recv(Duration::from_secs(2)); + if message["type"] == "state" { + break message; + } + }; + session.shutdown(); + (failed, context, state["unresolved_handoff_ids"].clone()) +} + +#[test] +fn a_failed_expert_reply_tells_the_spokesperson_what_was_heard() { + let (failed, context, unresolved) = fail_expert_reply_after(1_000); + assert_eq!(failed["id"], 2); + assert!( + context.contains("in its own voice") && context.contains("I checked the repository"), + "{context}" + ); + assert!(!context.contains("in the parser"), "{context}"); + assert!(unresolved.is_null(), "{unresolved}"); +} + +#[test] +fn an_expert_reply_that_fails_unheard_leaves_its_handoffs_open() { + let (failed, context, unresolved) = fail_expert_reply_after(0); + assert_eq!(failed["id"], 2); + assert!(context.contains("before hearing any of it"), "{context}"); + assert_eq!(unresolved, json!(["handoff-external-1"])); +} From af09dab27497a0443ca70eac8f81c183a7b90c2f Mon Sep 17 00:00:00 2001 From: John Tennant Date: Mon, 19 Oct 2026 20:25:50 -0400 Subject: [PATCH 12/49] berd-call: let any reply in the Expert's own voice answer every handoff With --expert-voice tts, a reply could be dropped: one written before the user's latest words came back pending, and one naming a handoff already settled was not admitted. An open handoff also kept the Spokesperson's session from renewing, and ended the call if the provider expired it. Such a reply is now always admitted, whatever its cursor, and settles every open handoff as soon as it is, so --resolves is ignored. Open handoffs no longer hold back renewal, recovery or a voice change, and the replacement session is told what they asked so it doesn't hand them off again. A reply that arrives during a replacement waits for it rather than failing it. Calls whose replies the Spokesperson says, and the Berd app, behave as before. Co-Authored-By: Claude Opus 5.5 Signed-off-by: John Tennant --- src-tauri/crates/berd-call/PROTOCOL.md | 7 +- src-tauri/crates/berd-call/README.md | 2 +- src-tauri/crates/berd-call/src/cli_help.rs | 4 +- .../berd-call/src/expert_spokesperson.rs | 5 + src-tauri/crates/berd-call/src/main.rs | 93 ++++++------- .../berd-call/src/openai_realtime_protocol.rs | 78 +++++++++-- .../berd-call/tests/session_protocol.rs | 126 ++++++++++++++++++ 7 files changed, 256 insertions(+), 59 deletions(-) diff --git a/src-tauri/crates/berd-call/PROTOCOL.md b/src-tauri/crates/berd-call/PROTOCOL.md index 410801034..b598059cb 100644 --- a/src-tauri/crates/berd-call/PROTOCOL.md +++ b/src-tauri/crates/berd-call/PROTOCOL.md @@ -219,7 +219,12 @@ Unknown fields are rejected. IDs are positive. Speak text is at most 16 KiB. `prepare_speak.voice` speaks that one utterance in another voice of the reply backend, keeping its other settings; `language` changes a Siri voice's locale too. A voice the backend doesn't offer, or a voice for a call whose replies -are spoken by the Spokesperson, is `not_admitted(invalid_voice)`. +are spoken by the Spokesperson, is `not_admitted(invalid_voice)`. In a call +whose replies have their own voice, a reply is admitted however stale its +`acknowledgement`, never `pending`, and answers every open handoff once +admitted, so `resolved_handoff_ids` is ignored. Open handoffs there don't hold +back a renewal or recovery of the Spokesperson's session; the replacement is +told what they asked. The parent cannot author speaking state or finalized input; those are derived only from PCM by the child runtime. diff --git a/src-tauri/crates/berd-call/README.md b/src-tauri/crates/berd-call/README.md index 77cb30a97..f842d5ea3 100644 --- a/src-tauri/crates/berd-call/README.md +++ b/src-tauri/crates/berd-call/README.md @@ -151,7 +151,7 @@ berd-call session --tts-backend openai --tts-url https://proxy.example/v1/audio/ berd-call session --stt-backend openai --stt-url wss://proxy.example/v1/realtime?intent=transcription ``` -By default the Spokesperson reads the Expert's replies aloud in its own voice. With `--expert-voice tts`, the Expert speaks for itself: berd-call plays each reply through the configured TTS backend, taking turns with the Spokesperson's audio, and your speech interrupts either of them. Afterwards the Spokesperson gets a silent note of what you heard, so it can follow up without repeating it. A reply settles the handoffs it answers once you hear any of it. `--voice`, `--rate`, `status` and `settings --tts` describe and change the Expert's voice. The Spokesperson keeps its own, shown as `session.spokesperson`: choose it with `--spokesperson-voice NAME` and `--spokesperson-rate 0.25–1.5`, at `start` or live with `settings`, and the menu bar offers both voices. Saved settings keep the Spokesperson's voice for the next call whose replies have their own, but not `--expert-voice`, so pass that on each `start`. +By default the Spokesperson reads the Expert's replies aloud in its own voice. With `--expert-voice tts`, the Expert speaks for itself: berd-call plays each reply through the configured TTS backend, taking turns with the Spokesperson's audio, and your speech interrupts either of them. Afterwards the Spokesperson gets a silent note of what you heard, so it can follow up without repeating it. Every reply is spoken, even one written before what you said since, and it settles every open handoff once admitted, so `--resolves` isn't needed. Open handoffs no longer hold back renewing or recovering the Spokesperson's session; the new session is told what they asked, so it doesn't hand them off again. `--voice`, `--rate`, `status` and `settings --tts` describe and change the Expert's voice. The Spokesperson keeps its own, shown as `session.spokesperson`: choose it with `--spokesperson-voice NAME` and `--spokesperson-rate 0.25–1.5`, at `start` or live with `settings`, and the menu bar offers both voices. Saved settings keep the Spokesperson's voice for the next call whose replies have their own, but not `--expert-voice`, so pass that on each `start`. The three URL flags take full endpoints and override only their matching service for that call. Expert–Spokesperson uses `--realtime-url` for both input and output; `--stt-url` applies only to conventional mode, and `--tts-url` also applies to the Expert's own voice. Without an override, each service uses its OpenAI endpoint (or its existing environment override). `OPENAI_API_KEY` supplies the CLI credential; the desktop app stores keys separately by endpoint URL in Keychain. diff --git a/src-tauri/crates/berd-call/src/cli_help.rs b/src-tauri/crates/berd-call/src/cli_help.rs index 64125af97..da80b19f2 100644 --- a/src-tauri/crates/berd-call/src/cli_help.rs +++ b/src-tauri/crates/berd-call/src/cli_help.rs @@ -65,7 +65,9 @@ for delivery. Non-blocking sessions require --stream or --codex and emit speech_ rows only for interruptions, failures, or pending user input. Interrupted results include estimatedSpokenText, a best-effort prefix based on delivered audio. --voice speaks this text in another voice of the call's reply backend without changing its saved -voice, so several agents can share one call in distinct voices."#; +voice, so several agents can share one call in distinct voices. In a call with +--expert-voice tts, every reply is spoken however stale --re is, and answers +every open handoff, so --resolves is ignored."#; const SETTINGS_HELP: &str = r#"Update settings for the active local voice call. diff --git a/src-tauri/crates/berd-call/src/expert_spokesperson.rs b/src-tauri/crates/berd-call/src/expert_spokesperson.rs index ef9cf322a..433e99877 100644 --- a/src-tauri/crates/berd-call/src/expert_spokesperson.rs +++ b/src-tauri/crates/berd-call/src/expert_spokesperson.rs @@ -155,6 +155,11 @@ impl ExpertSpokespersonCore { self.pipe.cursor(RealtimePipePeer::Expert) } + /// The newest message the Expert has been sent, read or not. + pub fn delivered_token(&self) -> u64 { + self.pipe.delivery_cursor(RealtimePipePeer::Expert) + } + pub fn next_live_token(&self) -> u64 { self.pipe.next_message_id() } diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index ed85acaf7..e073c9108 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -2975,9 +2975,9 @@ struct LiveResponse { prepare_id: Option, speech_id: Option, expert_text: Option, - /// An Expert reply spoken in its own voice, with the handoffs it answers. - /// It has no provider response; its whole text is played at once. - reply: Option>, + /// An Expert reply spoken in its own voice. It has no provider + /// response; its whole text is played at once. + reply: bool, /// The voice it asked for, when not the Expert's usual one. voice: Option, /// Whether the parent has accepted the reply's output, so it may play. @@ -2999,7 +2999,7 @@ impl LiveResponse { prepare_id, speech_id, expert_text: None, - reply: None, + reply: false, voice: None, output_ready: false, delivery: RealtimeAudioDelivery::default(), @@ -3224,21 +3224,35 @@ fn validate_queued_spokesperson_settings( fn spokesperson_voice_update_is_safe( update: &VoiceUpdateTransaction, core: &RealtimeExpertSpokespersonSession, + expert_tts: bool, snapshot: &berd_call::TtsConfigurationSnapshot, gate: &ExpertTurnGate, work: RealtimeHostWork, ) -> bool { + // A reply in the Expert's own voice waits for the replacement, and adds + // nothing to what it was given. + let work = RealtimeHostWork { + pending_expert_prepare: work.pending_expert_prepare && !expert_tts, + ..work + }; gate.lifecycle.voice_update_is_safe( update.purpose(), update.semantic_revision, core.semantic_revision(), update.base_revision, snapshot.revision, - core.has_unresolved_handoff(), + handoffs_hold_session(core, expert_tts), work, ) } +/// Whether open handoffs keep the Spokesperson's session from being +/// replaced: only while it says their replies itself. Replies in the +/// Expert's own voice need nothing from it. +fn handoffs_hold_session(core: &RealtimeExpertSpokespersonSession, expert_tts: bool) -> bool { + !expert_tts && core.has_unresolved_handoff() +} + fn unavailable_spokesperson_title(connection_lost: bool, quiescent: bool) -> &'static str { match (connection_lost, quiescent) { (true, false) => "Spokesperson connection was lost during an active turn", @@ -3326,7 +3340,7 @@ fn run_expert_spokesperson_session( if turn_gate.lifecycle.renewal_is_due( Instant::now(), work, - core.has_unresolved_handoff(), + handoffs_hold_session(&core, reply_tts.is_some()), pending_voice_update.is_some(), ) { let snapshot = session_tts.as_ref().expect("initialized TTS snapshot"); @@ -3343,7 +3357,7 @@ fn run_expert_spokesperson_session( runtime_config .as_ref() .expect("initialized Spokesperson config"), - core.semantic_transcript(), + core.replacement_transcript(), VoiceUpdatePurpose::Renewal, )?); } @@ -3359,6 +3373,7 @@ fn run_expert_spokesperson_session( let safe = spokesperson_voice_update_is_safe( update, &core, + reply_tts.is_some(), session_tts.as_ref().expect("initialized TTS snapshot"), &turn_gate, work, @@ -3438,7 +3453,7 @@ fn run_expert_spokesperson_session( runtime_config .as_ref() .expect("initialized Spokesperson config"), - core.semantic_transcript(), + core.replacement_transcript(), ) { Ok(update) => Some(update), Err(message) => { @@ -3821,6 +3836,7 @@ fn run_expert_spokesperson_session( let safe = spokesperson_voice_update_is_safe( update, &core, + reply_tts.is_some(), session_tts.as_ref().expect("initialized TTS snapshot"), &turn_gate, work, @@ -3942,7 +3958,7 @@ fn run_expert_spokesperson_session( &cancelled_directives, ); let quiescent = turn_gate.lifecycle.settings_are_quiescent(work) - && !core.has_unresolved_handoff(); + && !handoffs_hold_session(&core, reply_tts.is_some()); if !quiescent { write_protocol_fatal( &mut writer, @@ -3957,7 +3973,7 @@ fn run_expert_spokesperson_session( .as_ref() .map(VoiceUpdateTransaction::purpose), work, - core.has_unresolved_handoff(), + handoffs_hold_session(&core, reply_tts.is_some()), ); let start_recovery = match recovery_action { RealtimeSessionLossAction::ContinuePendingRecovery => { @@ -3999,7 +4015,7 @@ fn run_expert_spokesperson_session( runtime_config .as_ref() .expect("initialized Spokesperson config"), - core.semantic_transcript(), + core.replacement_transcript(), VoiceUpdatePurpose::SessionRecovery { cause: message.clone(), }, @@ -4212,7 +4228,6 @@ fn run_expert_spokesperson_session( runtime.as_ref().expect("initialized runtime"), &text[..heard], interrupted, - response.reply.as_deref().unwrap_or_default(), )?; } if initialized && !turn_gate.input_blocks_output() { @@ -4790,9 +4805,7 @@ fn run_expert_spokesperson_session( CancelOutcome::Cancelled } else if let Some(response_id) = responses .iter() - .find(|(_, response)| { - response.reply.is_some() && response.prepare_id == Some(id) - }) + .find(|(_, response)| response.reply && response.prepare_id == Some(id)) .map(|(response_id, _)| response_id.clone()) { waiting_responses.retain(|waiting| *waiting != response_id); @@ -5074,7 +5087,7 @@ fn drop_waiting_replies( if Some(response_id.as_str()) == active_response_id || !responses .get(response_id) - .is_some_and(|response| response.reply.is_some()) + .is_some_and(|response| response.reply) { continue; } @@ -5089,39 +5102,19 @@ fn drop_waiting_replies( spoken_through_utf8: 0, }, )?; - finish_expert_reply( - core, - runtime, - "", - true, - response.reply.as_deref().unwrap_or_default(), - )?; + finish_expert_reply(core, runtime, "", true)?; } Ok(()) } -/// Tells the Spokesperson what the user heard of an Expert reply, and settles -/// the handoffs it answers. +/// Tells the Spokesperson what the user heard of an Expert reply. fn finish_expert_reply( core: &mut RealtimeExpertSpokespersonSession, runtime: &OpenAiSpokespersonRuntime, heard: &str, interrupted: bool, - handoff_ids: &[String], ) -> Result<(), String> { let heard = heard.trim(); - // Handoffs settled elsewhere while the reply played need no settling here. - let unknown = core.unknown_handoff_ids(handoff_ids); - let open: Vec = handoff_ids - .iter() - .filter(|id| !unknown.contains(id)) - .cloned() - .collect(); - if heard.is_empty() { - core.release_handoffs(&open); - } else { - core.dismiss_handoffs(&open)?; - } let message = match (heard.is_empty(), interrupted) { (true, _) => "The user interrupted the Expert's spoken reply before hearing any of it.".into(), (false, false) => format!("The Expert said this to the user in its own voice:\n{heard}"), @@ -5209,9 +5202,12 @@ fn submit_expert_prepare( next_speech_id: &mut u64, writer: &mut impl Write, ) -> Result<(), String> { - if !core - .unknown_handoff_ids(&request.resolved_handoff_ids) - .is_empty() + // A reply in the Expert's own voice answers every open handoff, so the + // ones it names don't matter. + if !expert_tts + && !core + .unknown_handoff_ids(&request.resolved_handoff_ids) + .is_empty() { return write_message( writer, @@ -5221,7 +5217,12 @@ fn submit_expert_prepare( }, ); } - match core.prepare_expert_directive(request.acknowledgement, request.text) { + let outcome = if expert_tts { + core.prepare_expert_reply(request.text) + } else { + core.prepare_expert_directive(request.acknowledgement, request.text) + }; + match outcome { ExpertDirectiveOutcome::Pending(events) => write_message( writer, &SessionMessage::Pending { @@ -5244,21 +5245,21 @@ fn submit_expert_prepare( let speech_id = *next_speech_id; *next_speech_id += 1; core.record_expert_turn(message.clone()); - let resolved_handoff_ids = request.resolved_handoff_ids; - core.mark_handoffs_resolving(&resolved_handoff_ids)?; if expert_tts { // Played in turn with the Spokesperson's responses, without one of its own, // once the parent is ready for its output. let response_id = format!("expert-reply-{speech_id}"); let mut response = LiveResponse::new(Some(request.id), Some(speech_id)); response.expert_text = Some(message); - response.reply = Some(resolved_handoff_ids); + response.reply = true; response.voice = request.voice; response.audio_done = true; response.finish_sent = true; response.server_finished = true; responses.insert(response_id, response); } else { + let resolved_handoff_ids = request.resolved_handoff_ids; + core.mark_handoffs_resolving(&resolved_handoff_ids)?; directive_speeches.insert( request.id, DirectiveSpeech { @@ -5427,7 +5428,7 @@ fn start_next_live_playback( if response.interrupted || response.playback_complete { continue; } - if response.reply.is_some() { + if response.reply { let tts = reply_tts.ok_or("an Expert reply has no voice of its own")?; *active = Some(spawn_reply_playback( response_id, diff --git a/src-tauri/crates/berd-call/src/openai_realtime_protocol.rs b/src-tauri/crates/berd-call/src/openai_realtime_protocol.rs index 570299a09..c037297fb 100644 --- a/src-tauri/crates/berd-call/src/openai_realtime_protocol.rs +++ b/src-tauri/crates/berd-call/src/openai_realtime_protocol.rs @@ -1587,6 +1587,20 @@ impl RealtimeExpertSpokespersonSession { }) } + /// Prepares an Expert reply spoken in its own voice: it is admitted + /// however much was said since it was written, and answers every open + /// handoff. + pub fn prepare_expert_reply(&mut self, message: String) -> ExpertDirectiveOutcome { + let outcome = self.conversation.prepare_directive(ExpertDirective { + acknowledgement: Some(self.conversation.delivered_token()), + message, + }); + if matches!(outcome, ExpertDirectiveOutcome::Accepted { .. }) { + self.open_handoffs.clear(); + } + outcome + } + pub fn record_live_event_with_delivery( &mut self, event: LiveSideEvent, @@ -1680,6 +1694,25 @@ impl RealtimeExpertSpokespersonSession { self.conversation.semantic_transcript() } + /// The transcript a replacement Spokesperson session starts from: the + /// conversation so far, and the handoffs still waiting for a reply, so it + /// doesn't hand them off again. + pub fn replacement_transcript(&self) -> Vec { + let mut turns = self.semantic_transcript(); + let mut open: Vec<_> = self.open_handoffs.iter().collect(); + open.sort_by_key(|(id, _)| *id); + if !open.is_empty() { + turns.push(SemanticTurn::Expert(format!( + "The Expert is still working on these handoffs and will answer in its own voice, so do not hand them off again:\n{}", + open.iter() + .map(|(_, handoff)| format!("- {}", handoff.message)) + .collect::>() + .join("\n") + ))); + } + turns + } + pub fn request_expert_message( &mut self, message: RealtimeExpertMessage, @@ -1783,16 +1816,6 @@ impl RealtimeExpertSpokespersonSession { Ok(()) } - /// Leaves handoffs open for another reply once the one meant to answer - /// them went unheard. - pub fn release_handoffs(&mut self, handoff_ids: &[String]) { - for handoff_id in handoff_ids { - if let Some(handoff) = self.open_handoffs.get_mut(handoff_id) { - handoff.resolving = false; - } - } - } - pub fn dismiss_handoffs(&mut self, handoff_ids: &[String]) -> Result<(), String> { let unknown = self.unknown_handoff_ids(handoff_ids); if !unknown.is_empty() { @@ -3093,6 +3116,41 @@ mod tests { ); } + #[test] + fn expert_reply_answers_every_handoff_however_late() { + let mut session = RealtimeExpertSpokespersonSession::new(0, "test-call"); + session + .register_handoff("handoff-1", 1, "First request") + .unwrap(); + session + .register_handoff("handoff-2", 1, "Second request") + .unwrap(); + let note = SemanticTurn::Expert( + "The Expert is still working on these handoffs and will answer in its own voice, so do not hand them off again:\n- First request\n- Second request".into(), + ); + assert_eq!(session.replacement_transcript(), [note]); + session + .record_live_event_with_delivery(LiveSideEvent::UserTranscript { + text: "And one more thing".into(), + }) + .unwrap(); + + // A directive written before the user's latest words waits for them. + assert!(matches!( + session.prepare_expert_directive(None, "Stale".into()), + ExpertDirectiveOutcome::Pending(_) + )); + assert!(matches!( + session.prepare_expert_reply("The answer".into()), + ExpertDirectiveOutcome::Accepted { + confirmed_token: 1, + .. + } + )); + assert!(!session.has_unresolved_handoff()); + assert_eq!(session.replacement_transcript(), []); + } + #[test] fn native_session_records_every_semantic_turn_for_replacement() { let mut session = RealtimeExpertSpokespersonSession::new(0, "test-call"); diff --git a/src-tauri/crates/berd-call/tests/session_protocol.rs b/src-tauri/crates/berd-call/tests/session_protocol.rs index dd2da7a78..72d79e988 100644 --- a/src-tauri/crates/berd-call/tests/session_protocol.rs +++ b/src-tauri/crates/berd-call/tests/session_protocol.rs @@ -3283,3 +3283,129 @@ fn an_expert_reply_that_fails_unheard_leaves_its_handoffs_open() { assert!(context.contains("before hearing any of it"), "{context}"); assert_eq!(unresolved, json!(["handoff-external-1"])); } + +#[test] +fn expert_voice_handoffs_survive_provider_expiry_and_any_late_reply_answers_them() { + let (speech_url, speech_requests) = spawn_speech_server(); + let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); + let (note_tx, note_rx) = mpsc::sync_channel(1); + let server = std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + endpoint_tx + .send(format!("ws://{}/", listener.local_addr().unwrap())) + .unwrap(); + let (old_stream, _) = listener.accept().await.unwrap(); + let mut old = accept_async(old_stream).await.unwrap(); + let initial = receive_realtime_json(&mut old).await; + acknowledge_realtime_session(&mut old, &initial).await; + send_realtime_json( + &mut old, + json!({ + "type":"conversation.item.input_audio_transcription.completed", + "item_id":"user-1", + "transcript":"What changed in the repository?" + }), + ) + .await; + send_realtime_json( + &mut old, + json!({ + "type":"response.function_call_arguments.done", + "response_id":"response-handoff", + "call_id":"handoff-1", + "name":"handoff", + "arguments":"{\"message\":\"inspect the repository\"}" + }), + ) + .await; + while receive_realtime_json(&mut old).await.pointer("/item/type") + != Some(&json!("function_call_output")) + {} + send_realtime_json( + &mut old, + json!({ + "type":"error", + "error":{"code":"session_expired","message":"provider session expired"} + }), + ) + .await; + + // The open handoff doesn't stop the replacement, which is told of it. + let (stream, _) = listener.accept().await.unwrap(); + let mut replacement = accept_async(stream).await.unwrap(); + let update = receive_realtime_json(&mut replacement).await; + acknowledge_realtime_session(&mut replacement, &update).await; + while let Some(Ok(Message::Text(text))) = replacement.next().await { + let seed: Value = serde_json::from_str(&text).unwrap(); + if seed["type"] != "conversation.item.create" { + continue; + } + send_realtime_json( + &mut replacement, + json!({"type":"conversation.item.created","item":{"id":seed["item"]["id"]}}), + ) + .await; + if text.contains("do not hand them off again") { + let _ = note_tx.try_send(seed); + } + } + }); + }); + let endpoint = endpoint_rx.recv().unwrap(); + let mut session = ExpertSpokespersonTestSession::start_with_args( + endpoint, + &["--expert-voice", "tts", "--tts-url", &speech_url], + ); + let note = note_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + assert!( + note.to_string().contains("- inspect the repository"), + "{note}" + ); + + // A reply written before everything since, naming no handoff, is still spoken. + session.send(json!({ + "type":"prepare_speak", + "id":2, + "acknowledgement":0, + "text":"I checked the repository." + })); + let admitted = loop { + let message = session.recv(Duration::from_secs(2)); + assert_ne!(message["type"], "fatal", "{message}"); + if message["id"] == 2 { + break message; + } + }; + assert_eq!(admitted["type"], "admitted", "{admitted}"); + session.send(json!({"type":"query_state","id":3,"after":0})); + session.send(json!({ + "type":"output_ready", + "id":2, + "speech_id":admitted["speech_id"] + })); + let (mut state, mut completed) = (None, false); + while state.is_none() || !completed { + let message = session.recv(Duration::from_secs(3)); + assert_ne!(message["type"], "fatal", "{message}"); + assert_ne!(message["type"], "speech_failed", "{message}"); + match message["type"].as_str() { + Some("state") => state = Some(message), + Some("speech_completed") => completed = true, + _ => {} + } + } + assert_eq!(state.unwrap().get("unresolved_handoff_ids"), None); + assert_eq!( + speech_requests + .recv_timeout(Duration::from_secs(1)) + .unwrap()["input"], + "I checked the repository." + ); + session.shutdown(); + server.join().unwrap(); +} From e1c8d0f8d3fc737562d6818ed5924ab1e24481fc Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 07:26:26 -0400 Subject: [PATCH 13/49] berd-call: let the Expert message the Spokesperson in an own-voice call In a call with --expert-voice tts, every speak played in the Expert's voice, so an Expert couldn't have the Spokesperson tell the user something instead. `berd-call message-spokesperson TEXT` (protocol: `prepare_speak.relay`) hands the text to the Spokesperson to say, on the same terms as an own-voice reply there: admitted however stale, and answering every open handoff once admitted. Co-Authored-By: Claude Opus 5.5 Signed-off-by: John Tennant --- src-tauri/crates/berd-call/PROTOCOL.md | 6 +- src-tauri/crates/berd-call/README.md | 2 +- src-tauri/crates/berd-call/src/cli_help.rs | 12 ++ .../crates/berd-call/src/host_control.rs | 20 +++- .../crates/berd-call/src/host_session.rs | 7 ++ src-tauri/crates/berd-call/src/main.rs | 77 ++++++++++++- .../berd-call/src/openai_realtime_protocol.rs | 7 +- src-tauri/crates/berd-call/src/protocol.rs | 4 + .../berd-call/tests/session_protocol.rs | 106 ++++++++++++++++++ 9 files changed, 228 insertions(+), 13 deletions(-) diff --git a/src-tauri/crates/berd-call/PROTOCOL.md b/src-tauri/crates/berd-call/PROTOCOL.md index b598059cb..eae984e98 100644 --- a/src-tauri/crates/berd-call/PROTOCOL.md +++ b/src-tauri/crates/berd-call/PROTOCOL.md @@ -206,7 +206,7 @@ it never admits a replacement while old host audio may still be active. {"type":"set_spokesperson_settings","id":u64,"expected_revision":u64,"settings":TtsSettings} {"type":"set_input_during_tts","id":u64,"expected_revision":u64,"policy":"allow_barge_in"|"suppress_input"} {"type":"reset_input","id":u64} -{"type":"prepare_speak","id":u64,"acknowledgement":u64|null,"text":string,"resolved_handoff_ids":[string]?,"voice":string?,"language":string?} +{"type":"prepare_speak","id":u64,"acknowledgement":u64|null,"text":string,"resolved_handoff_ids":[string]?,"voice":string?,"language":string?,"relay":bool?} {"type":"output_ready","id":u64,"speech_id":u64} audio acknowledgements listed above {"type":"query_state","id":u64,"after":u64} @@ -222,7 +222,9 @@ too. A voice the backend doesn't offer, or a voice for a call whose replies are spoken by the Spokesperson, is `not_admitted(invalid_voice)`. In a call whose replies have their own voice, a reply is admitted however stale its `acknowledgement`, never `pending`, and answers every open handoff once -admitted, so `resolved_handoff_ids` is ignored. Open handoffs there don't hold +admitted, so `resolved_handoff_ids` is ignored. With `relay: true` (what +`berd-call message-spokesperson` sends), the Spokesperson says such a reply instead, as it would in a call without its own +voice, and it reports the same lifecycle. Open handoffs there don't hold back a renewal or recovery of the Spokesperson's session; the replacement is told what they asked. The parent cannot author speaking state or finalized input; those are derived diff --git a/src-tauri/crates/berd-call/README.md b/src-tauri/crates/berd-call/README.md index f842d5ea3..f05af6c83 100644 --- a/src-tauri/crates/berd-call/README.md +++ b/src-tauri/crates/berd-call/README.md @@ -151,7 +151,7 @@ berd-call session --tts-backend openai --tts-url https://proxy.example/v1/audio/ berd-call session --stt-backend openai --stt-url wss://proxy.example/v1/realtime?intent=transcription ``` -By default the Spokesperson reads the Expert's replies aloud in its own voice. With `--expert-voice tts`, the Expert speaks for itself: berd-call plays each reply through the configured TTS backend, taking turns with the Spokesperson's audio, and your speech interrupts either of them. Afterwards the Spokesperson gets a silent note of what you heard, so it can follow up without repeating it. Every reply is spoken, even one written before what you said since, and it settles every open handoff once admitted, so `--resolves` isn't needed. Open handoffs no longer hold back renewing or recovering the Spokesperson's session; the new session is told what they asked, so it doesn't hand them off again. `--voice`, `--rate`, `status` and `settings --tts` describe and change the Expert's voice. The Spokesperson keeps its own, shown as `session.spokesperson`: choose it with `--spokesperson-voice NAME` and `--spokesperson-rate 0.25–1.5`, at `start` or live with `settings`, and the menu bar offers both voices. Saved settings keep the Spokesperson's voice for the next call whose replies have their own, but not `--expert-voice`, so pass that on each `start`. +By default the Spokesperson reads the Expert's replies aloud in its own voice. With `--expert-voice tts`, the Expert speaks for itself: berd-call plays each reply through the configured TTS backend, taking turns with the Spokesperson's audio, and your speech interrupts either of them. Afterwards the Spokesperson gets a silent note of what you heard, so it can follow up without repeating it. Every reply is spoken, even one written before what you said since, and it settles every open handoff once admitted, so `--resolves` isn't needed. `berd-call message-spokesperson TEXT` has the Spokesperson tell you instead, in its own voice, on the same terms. Open handoffs no longer hold back renewing or recovering the Spokesperson's session; the new session is told what they asked, so it doesn't hand them off again. `--voice`, `--rate`, `status` and `settings --tts` describe and change the Expert's voice. The Spokesperson keeps its own, shown as `session.spokesperson`: choose it with `--spokesperson-voice NAME` and `--spokesperson-rate 0.25–1.5`, at `start` or live with `settings`, and the menu bar offers both voices. Saved settings keep the Spokesperson's voice for the next call whose replies have their own, but not `--expert-voice`, so pass that on each `start`. The three URL flags take full endpoints and override only their matching service for that call. Expert–Spokesperson uses `--realtime-url` for both input and output; `--stt-url` applies only to conventional mode, and `--tts-url` also applies to the Expert's own voice. Without an override, each service uses its OpenAI endpoint (or its existing environment override). `OPENAI_API_KEY` supplies the CLI credential; the desktop app stores keys separately by endpoint URL in Keychain. diff --git a/src-tauri/crates/berd-call/src/cli_help.rs b/src-tauri/crates/berd-call/src/cli_help.rs index da80b19f2..0aaa7de06 100644 --- a/src-tauri/crates/berd-call/src/cli_help.rs +++ b/src-tauri/crates/berd-call/src/cli_help.rs @@ -8,6 +8,7 @@ Usage: Call runtime: start Run a microphone-backed local voice call speak Speak through the active local call + message-spokesperson Have the Spokesperson tell the user status Inspect the active local call catch-up Read finalized input after a cursor wait-for-input Wait for finalized input after a cursor @@ -69,6 +70,16 @@ voice, so several agents can share one call in distinct voices. In a call with --expert-voice tts, every reply is spoken however stale --re is, and answers every open handoff, so --resolves is ignored."#; +const MESSAGE_SPOKESPERSON_HELP: &str = r#"Message the Spokesperson of the active realtime call. + +Usage: + berd-call message-spokesperson [--port PORT] TEXT + +The Spokesperson tells the user in its own voice, even in a call with +--expert-voice tts, where `speak` uses the Expert's. Like `speak` there, it is +admitted however much the user has said since, answers every open handoff, and +reports the same result."#; + const SETTINGS_HELP: &str = r#"Update settings for the active local voice call. Usage: @@ -213,6 +224,7 @@ fn help_for(topic: &[&str]) -> Option<&'static str> { [] => Some(TOP_LEVEL_HELP), ["start"] => Some(START_HELP), ["speak"] => Some(SPEAK_HELP), + ["message-spokesperson"] => Some(MESSAGE_SPOKESPERSON_HELP), ["status"] => Some(STATUS_HELP), ["catch-up"] | ["wait-for-input"] => Some(POLL_INPUT_HELP), ["settings"] => Some(SETTINGS_HELP), diff --git a/src-tauri/crates/berd-call/src/host_control.rs b/src-tauri/crates/berd-call/src/host_control.rs index e98cadecf..3cd9ea101 100644 --- a/src-tauri/crates/berd-call/src/host_control.rs +++ b/src-tauri/crates/berd-call/src/host_control.rs @@ -30,6 +30,7 @@ pub(crate) trait HostControl: Send + Sync + 'static { resolved_handoff_ids: Vec, voice: Option, language: Option, + relay: bool, ) -> Result; fn stop(&self) -> Result; fn set_non_blocking(&self, enabled: bool) -> Result; @@ -121,6 +122,8 @@ pub(crate) enum ControlRequest { voice: Option, #[serde(default, skip_serializing_if = "Option::is_none")] language: Option, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + relay: bool, }, Stop, NonBlocking { @@ -239,6 +242,7 @@ fn handle_connection(mut stream: TcpStream, control: &dyn HostControl) -> Result resolved_handoff_ids, voice, language, + relay, } => { validate_speak( &text, @@ -246,7 +250,17 @@ fn handle_connection(mut stream: TcpStream, control: &dyn HostControl) -> Result voice.as_deref(), language.is_some(), )?; - control.speak(text, acknowledgement, resolved_handoff_ids, voice, language) + if relay && voice.is_some() { + return Err("a message to the Spokesperson takes no voice".into()); + } + control.speak( + text, + acknowledgement, + resolved_handoff_ids, + voice, + language, + relay, + ) } ControlRequest::Stop => control.stop(), ControlRequest::TtsSettings { settings } => control.set_tts(settings), @@ -413,6 +427,7 @@ mod tests { resolved_handoff_ids: Vec, voice: Option, language: Option, + relay: bool, ) -> Result { Ok(json!({ "spoken":text, @@ -420,6 +435,7 @@ mod tests { "resolvedHandoffIds":resolved_handoff_ids, "voice":voice, "language":language, + "relay":relay, })) } @@ -468,6 +484,7 @@ mod tests { resolved_handoff_ids: vec!["call-1".into()], voice: Some("Nora".into()), language: Some("en-GB".into()), + relay: false, } ) .unwrap(), @@ -477,6 +494,7 @@ mod tests { "resolvedHandoffIds":["call-1"], "voice":"Nora", "language":"en-GB", + "relay":false, }) ); assert_eq!( diff --git a/src-tauri/crates/berd-call/src/host_session.rs b/src-tauri/crates/berd-call/src/host_session.rs index 71b09af77..973a7dc7c 100644 --- a/src-tauri/crates/berd-call/src/host_session.rs +++ b/src-tauri/crates/berd-call/src/host_session.rs @@ -519,6 +519,7 @@ enum ControlCommand { resolved_handoff_ids: Vec, voice: Option, language: Option, + relay: bool, non_blocking: bool, response: SyncSender>, }, @@ -743,6 +744,7 @@ impl HostControl for SessionControl { resolved_handoff_ids: Vec, voice: Option, language: Option, + relay: bool, ) -> Result { let (tx, rx) = mpsc::sync_channel(1); self.commands @@ -752,6 +754,7 @@ impl HostControl for SessionControl { resolved_handoff_ids, voice, language, + relay, non_blocking: self.non_blocking.load(Ordering::SeqCst), response: tx, }) @@ -1100,6 +1103,7 @@ impl SessionActor { resolved_handoff_ids, voice, language, + relay, non_blocking, response, } => { @@ -1114,6 +1118,7 @@ impl SessionActor { resolved_handoff_ids, voice, language, + relay, non_blocking, response, }); @@ -1129,6 +1134,7 @@ impl SessionActor { resolved_handoff_ids, voice, language, + relay, }, )?; self.pending_speak = Some(PendingSpeak { @@ -3371,6 +3377,7 @@ mod tests { resolved_handoff_ids: Vec::new(), voice: None, language: None, + relay: false, non_blocking: true, response: next_tx, }) diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index e073c9108..d4b134ff5 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -561,11 +561,29 @@ fn main() { resolved_handoff_ids, voice, language, + relay: false, }, ) .unwrap_or_else(|error| operational_error("speak", error)); print_pretty_json(&response).unwrap_or_else(|error| operational_error("speak", error)); } + Some("message-spokesperson") => { + let (port, text) = parse_or_exit(parse_message_spokesperson_args(&args), &args); + let response = host_control::request( + port, + host_control::ControlRequest::Speak { + text, + acknowledgement: None, + resolved_handoff_ids: Vec::new(), + voice: None, + language: None, + relay: true, + }, + ) + .unwrap_or_else(|error| operational_error("message-spokesperson", error)); + print_pretty_json(&response) + .unwrap_or_else(|error| operational_error("message-spokesperson", error)); + } Some("status") => { let port = parse_or_exit(parse_control_port(&args), &args); let response = host_control::request(port, host_control::ControlRequest::Status) @@ -1031,6 +1049,18 @@ fn parse_speak_control_args(args: &[String]) -> Result Result<(u16, String), ParseFailure> { + let options = parse_speak_control_args(args)?; + if options.acknowledgement.is_some() + || !options.resolved_handoff_ids.is_empty() + || options.voice.is_some() + { + return Err("message-spokesperson accepts only --port PORT and TEXT".into()); + } + Ok((options.port, options.text)) +} + fn parse_host_settings_args( args: &[String], ) -> Result<(u16, host_control::ControlRequest), ParseFailure> { @@ -2417,6 +2447,7 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String resolved_handoff_ids: _, voice, language, + relay: _, }) => { let request = PrepareRequest { id, @@ -2740,6 +2771,8 @@ struct PendingExpertPrepare { resolved_handoff_ids: Vec, /// The Expert's voice for this reply, when not its usual one. voice: Option, + /// The Spokesperson says it, even when the Expert has its own voice. + relay: bool, } struct PendingSpokespersonSettingsUpdate { @@ -3229,8 +3262,8 @@ fn spokesperson_voice_update_is_safe( gate: &ExpertTurnGate, work: RealtimeHostWork, ) -> bool { - // A reply in the Expert's own voice waits for the replacement, and adds - // nothing to what it was given. + // When the Expert has its own voice, a held reply waits for the + // replacement, and adds nothing to what it was given. let work = RealtimeHostWork { pending_expert_prepare: work.pending_expert_prepare && !expert_tts, ..work @@ -4471,6 +4504,7 @@ fn run_expert_spokesperson_session( resolved_handoff_ids, voice, language, + relay, }) => { let Some(voice) = speak_voice(reply_tts.as_deref(), voice, language, id, &mut writer)? @@ -4483,6 +4517,7 @@ fn run_expert_spokesperson_session( text, resolved_handoff_ids, voice, + relay, }; let routing = route_expert_prepare( &mut turn_gate, @@ -5202,8 +5237,8 @@ fn submit_expert_prepare( next_speech_id: &mut u64, writer: &mut impl Write, ) -> Result<(), String> { - // A reply in the Expert's own voice answers every open handoff, so the - // ones it names don't matter. + // When the Expert has its own voice, any reply answers every open + // handoff, so the ones it names don't matter. if !expert_tts && !core .unknown_handoff_ids(&request.resolved_handoff_ids) @@ -5245,7 +5280,7 @@ fn submit_expert_prepare( let speech_id = *next_speech_id; *next_speech_id += 1; core.record_expert_turn(message.clone()); - if expert_tts { + if expert_tts && !request.relay { // Played in turn with the Spokesperson's responses, without one of its own, // once the parent is ready for its output. let response_id = format!("expert-reply-{speech_id}"); @@ -5258,7 +5293,12 @@ fn submit_expert_prepare( response.server_finished = true; responses.insert(response_id, response); } else { - let resolved_handoff_ids = request.resolved_handoff_ids; + // A relayed reply already answered every handoff when admitted. + let resolved_handoff_ids = if expert_tts { + Vec::new() + } else { + request.resolved_handoff_ids + }; core.mark_handoffs_resolving(&resolved_handoff_ids)?; directive_speeches.insert( request.id, @@ -8506,6 +8546,7 @@ mod tests { text: "answer".into(), resolved_handoff_ids: Vec::new(), voice: None, + relay: false, }; assert!(matches!( gate.defer_if_busy(request, false, 1), @@ -8546,6 +8587,7 @@ mod tests { text: "first".into(), resolved_handoff_ids: Vec::new(), voice: None, + relay: false, }, false, 1, @@ -8560,6 +8602,7 @@ mod tests { text: "second".into(), resolved_handoff_ids: Vec::new(), voice: None, + relay: false, }, false, 1, @@ -8591,6 +8634,7 @@ mod tests { text: "second".into(), resolved_handoff_ids: Vec::new(), voice: None, + relay: false, }, expert_output_reserved(&directives, &HashSet::new(), None, &responses), false, @@ -9118,6 +9162,7 @@ mod tests { text: "answer after the user".into(), resolved_handoff_ids: Vec::new(), voice: None, + relay: false, }, false, 1, @@ -11551,6 +11596,7 @@ mod tests { resolved_handoff_ids: vec![" ".into()], voice: None, language: None, + relay: false, }; assert_eq!( validate_request(empty).unwrap_err(), @@ -12752,6 +12798,25 @@ mod tests { text: "hello".into(), } ); + assert_eq!( + parse_message_spokesperson_args(&args(&[ + "berd-call", + "message-spokesperson", + "--port", + "5300", + "Tell them it's fixed." + ])) + .unwrap(), + (5300, "Tell them it's fixed.".into()) + ); + assert!(parse_message_spokesperson_args(&args(&[ + "berd-call", + "message-spokesperson", + "--voice", + "Nora", + "hello" + ])) + .is_err()); assert!(parse_speak_control_args(&args(&[ "berd-call", "speak", diff --git a/src-tauri/crates/berd-call/src/openai_realtime_protocol.rs b/src-tauri/crates/berd-call/src/openai_realtime_protocol.rs index c037297fb..c3299cd1d 100644 --- a/src-tauri/crates/berd-call/src/openai_realtime_protocol.rs +++ b/src-tauri/crates/berd-call/src/openai_realtime_protocol.rs @@ -1587,9 +1587,10 @@ impl RealtimeExpertSpokespersonSession { }) } - /// Prepares an Expert reply spoken in its own voice: it is admitted - /// however much was said since it was written, and answers every open - /// handoff. + /// Prepares an Expert reply in a call where the Expert has its own voice, + /// whether it speaks the reply or the Spokesperson relays it: it is + /// admitted however much was said since it was written, and answers + /// every open handoff. pub fn prepare_expert_reply(&mut self, message: String) -> ExpertDirectiveOutcome { let outcome = self.conversation.prepare_directive(ExpertDirective { acknowledgement: Some(self.conversation.delivered_token()), diff --git a/src-tauri/crates/berd-call/src/protocol.rs b/src-tauri/crates/berd-call/src/protocol.rs index 02e58c261..cf6d578d5 100644 --- a/src-tauri/crates/berd-call/src/protocol.rs +++ b/src-tauri/crates/berd-call/src/protocol.rs @@ -56,6 +56,9 @@ pub enum SessionRequest { voice: Option, #[serde(default, skip_serializing_if = "Option::is_none")] language: Option, + /// The Spokesperson says it, even in a call where the Expert has its own voice. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + relay: bool, }, OutputReady { id: u64, @@ -401,6 +404,7 @@ mod tests { resolved_handoff_ids: Vec::new(), voice: None, language: None, + relay: false, } ); assert_eq!( diff --git a/src-tauri/crates/berd-call/tests/session_protocol.rs b/src-tauri/crates/berd-call/tests/session_protocol.rs index 72d79e988..4a50ad928 100644 --- a/src-tauri/crates/berd-call/tests/session_protocol.rs +++ b/src-tauri/crates/berd-call/tests/session_protocol.rs @@ -3284,6 +3284,112 @@ fn an_expert_reply_that_fails_unheard_leaves_its_handoffs_open() { assert_eq!(unresolved, json!(["handoff-external-1"])); } +#[test] +fn a_relayed_reply_is_said_by_the_spokesperson_and_answers_every_handoff() { + let (speech_url, speech_requests) = spawn_speech_server(); + let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); + let (relay_tx, relay_rx) = mpsc::sync_channel(1); + let server = std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + endpoint_tx + .send(format!("ws://{}/", listener.local_addr().unwrap())) + .unwrap(); + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = accept_async(stream).await.unwrap(); + let initial = receive_realtime_json(&mut socket).await; + acknowledge_realtime_session(&mut socket, &initial).await; + for (call_id, message) in [ + ("handoff-1", "inspect the repository"), + ("handoff-2", "check the tests"), + ] { + send_realtime_json( + &mut socket, + json!({ + "type":"response.function_call_arguments.done", + "response_id":format!("response-{call_id}"), + "call_id":call_id, + "name":"handoff", + "arguments":json!({"message":message}).to_string() + }), + ) + .await; + while receive_realtime_json(&mut socket) + .await + .pointer("/item/type") + != Some(&json!("function_call_output")) + {} + } + while let Some(Ok(Message::Text(text))) = socket.next().await { + if text.contains("I checked the repository.") { + let _ = relay_tx.try_send(serde_json::from_str::(&text).unwrap()); + } + } + }); + }); + let endpoint = endpoint_rx.recv().unwrap(); + let mut session = ExpertSpokespersonTestSession::start_with_args( + endpoint, + &["--expert-voice", "tts", "--tts-url", &speech_url], + ); + loop { + session.send(json!({"type":"query_state","id":1,"after":0})); + let state = loop { + let message = session.recv(Duration::from_secs(2)); + assert_ne!(message["type"], "fatal", "{message}"); + if message["type"] == "state" { + break message; + } + }; + if state["unresolved_handoff_ids"] + .as_array() + .is_some_and(|ids| ids.len() == 2) + { + break; + } + std::thread::sleep(Duration::from_millis(20)); + } + + // Stale, naming no handoff, and still the Spokesperson's to say. + session.send(json!({ + "type":"prepare_speak", + "id":2, + "acknowledgement":0, + "text":"I checked the repository.", + "relay":true + })); + let admitted = loop { + let message = session.recv(Duration::from_secs(2)); + assert_ne!(message["type"], "fatal", "{message}"); + if message["id"] == 2 { + break message; + } + }; + assert_eq!(admitted["type"], "admitted", "{admitted}"); + let relayed = relay_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + assert!( + relayed.to_string().contains("Speak it naturally"), + "{relayed}" + ); + session.send(json!({"type":"query_state","id":3,"after":0})); + let state = loop { + let message = session.recv(Duration::from_secs(2)); + if message["type"] == "state" { + break message; + } + }; + assert_eq!(state.get("unresolved_handoff_ids"), None, "{state}"); + assert!(speech_requests + .recv_timeout(Duration::from_millis(300)) + .is_err()); + session.shutdown(); + server.join().unwrap(); +} + #[test] fn expert_voice_handoffs_survive_provider_expiry_and_any_late_reply_answers_them() { let (speech_url, speech_requests) = spawn_speech_server(); From 7e2a9a4d958f72582f715bce0890b5cd78a5c722 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 07:29:18 -0400 Subject: [PATCH 14/49] berd-call: let a message to the Spokesperson answer handoffs in any call `message-spokesperson` is how an Expert answers through the Spokesperson, so it now works the same in a call without --expert-voice tts: admitted however much the user has said since, and answering every open handoff, with no --re or --resolves. `speak` there keeps today's rules. Co-Authored-By: Claude Opus 5.5 Signed-off-by: John Tennant --- src-tauri/crates/berd-call/PROTOCOL.md | 5 +++-- src-tauri/crates/berd-call/README.md | 2 +- src-tauri/crates/berd-call/src/cli_help.rs | 6 +++--- src-tauri/crates/berd-call/src/main.rs | 13 +++++++------ .../berd-call/src/openai_realtime_protocol.rs | 7 +++---- .../crates/berd-call/tests/session_protocol.rs | 10 ++++++++-- 6 files changed, 25 insertions(+), 18 deletions(-) diff --git a/src-tauri/crates/berd-call/PROTOCOL.md b/src-tauri/crates/berd-call/PROTOCOL.md index eae984e98..0b7215e3f 100644 --- a/src-tauri/crates/berd-call/PROTOCOL.md +++ b/src-tauri/crates/berd-call/PROTOCOL.md @@ -223,8 +223,9 @@ are spoken by the Spokesperson, is `not_admitted(invalid_voice)`. In a call whose replies have their own voice, a reply is admitted however stale its `acknowledgement`, never `pending`, and answers every open handoff once admitted, so `resolved_handoff_ids` is ignored. With `relay: true` (what -`berd-call message-spokesperson` sends), the Spokesperson says such a reply instead, as it would in a call without its own -voice, and it reports the same lifecycle. Open handoffs there don't hold +`berd-call message-spokesperson` sends), the Spokesperson says the reply in any +Expert-Spokesperson call, on those same terms, with the same lifecycle. Open +handoffs in a call whose replies have their own voice don't hold back a renewal or recovery of the Spokesperson's session; the replacement is told what they asked. The parent cannot author speaking state or finalized input; those are derived diff --git a/src-tauri/crates/berd-call/README.md b/src-tauri/crates/berd-call/README.md index f05af6c83..ae292f9fc 100644 --- a/src-tauri/crates/berd-call/README.md +++ b/src-tauri/crates/berd-call/README.md @@ -151,7 +151,7 @@ berd-call session --tts-backend openai --tts-url https://proxy.example/v1/audio/ berd-call session --stt-backend openai --stt-url wss://proxy.example/v1/realtime?intent=transcription ``` -By default the Spokesperson reads the Expert's replies aloud in its own voice. With `--expert-voice tts`, the Expert speaks for itself: berd-call plays each reply through the configured TTS backend, taking turns with the Spokesperson's audio, and your speech interrupts either of them. Afterwards the Spokesperson gets a silent note of what you heard, so it can follow up without repeating it. Every reply is spoken, even one written before what you said since, and it settles every open handoff once admitted, so `--resolves` isn't needed. `berd-call message-spokesperson TEXT` has the Spokesperson tell you instead, in its own voice, on the same terms. Open handoffs no longer hold back renewing or recovering the Spokesperson's session; the new session is told what they asked, so it doesn't hand them off again. `--voice`, `--rate`, `status` and `settings --tts` describe and change the Expert's voice. The Spokesperson keeps its own, shown as `session.spokesperson`: choose it with `--spokesperson-voice NAME` and `--spokesperson-rate 0.25–1.5`, at `start` or live with `settings`, and the menu bar offers both voices. Saved settings keep the Spokesperson's voice for the next call whose replies have their own, but not `--expert-voice`, so pass that on each `start`. +By default the Spokesperson reads the Expert's replies aloud in its own voice. `berd-call message-spokesperson TEXT` always has it tell you, in any Expert–Spokesperson call: the message is admitted however much you've said since, and answers every open handoff, with no `--re` or `--resolves`. With `--expert-voice tts`, the Expert speaks for itself: berd-call plays each reply through the configured TTS backend, taking turns with the Spokesperson's audio, and your speech interrupts either of them. Afterwards the Spokesperson gets a silent note of what you heard, so it can follow up without repeating it. Every reply is spoken, even one written before what you said since, and it settles every open handoff once admitted, so `--resolves` isn't needed. Open handoffs no longer hold back renewing or recovering the Spokesperson's session; the new session is told what they asked, so it doesn't hand them off again. `--voice`, `--rate`, `status` and `settings --tts` describe and change the Expert's voice. The Spokesperson keeps its own, shown as `session.spokesperson`: choose it with `--spokesperson-voice NAME` and `--spokesperson-rate 0.25–1.5`, at `start` or live with `settings`, and the menu bar offers both voices. Saved settings keep the Spokesperson's voice for the next call whose replies have their own, but not `--expert-voice`, so pass that on each `start`. The three URL flags take full endpoints and override only their matching service for that call. Expert–Spokesperson uses `--realtime-url` for both input and output; `--stt-url` applies only to conventional mode, and `--tts-url` also applies to the Expert's own voice. Without an override, each service uses its OpenAI endpoint (or its existing environment override). `OPENAI_API_KEY` supplies the CLI credential; the desktop app stores keys separately by endpoint URL in Keychain. diff --git a/src-tauri/crates/berd-call/src/cli_help.rs b/src-tauri/crates/berd-call/src/cli_help.rs index 0aaa7de06..05c77ad6a 100644 --- a/src-tauri/crates/berd-call/src/cli_help.rs +++ b/src-tauri/crates/berd-call/src/cli_help.rs @@ -76,9 +76,9 @@ Usage: berd-call message-spokesperson [--port PORT] TEXT The Spokesperson tells the user in its own voice, even in a call with ---expert-voice tts, where `speak` uses the Expert's. Like `speak` there, it is -admitted however much the user has said since, answers every open handoff, and -reports the same result."#; +--expert-voice tts, where `speak` uses the Expert's. It is admitted however much +the user has said since, answers every open handoff, and reports the same result +as `speak`."#; const SETTINGS_HELP: &str = r#"Update settings for the active local voice call. diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index d4b134ff5..7cabd79f3 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -5237,9 +5237,10 @@ fn submit_expert_prepare( next_speech_id: &mut u64, writer: &mut impl Write, ) -> Result<(), String> { - // When the Expert has its own voice, any reply answers every open - // handoff, so the ones it names don't matter. - if !expert_tts + // A message to the Spokesperson, or any reply when the Expert has its own + // voice, answers every open handoff, so the ones it names don't matter. + let answers_all = expert_tts || request.relay; + if !answers_all && !core .unknown_handoff_ids(&request.resolved_handoff_ids) .is_empty() @@ -5252,7 +5253,7 @@ fn submit_expert_prepare( }, ); } - let outcome = if expert_tts { + let outcome = if answers_all { core.prepare_expert_reply(request.text) } else { core.prepare_expert_directive(request.acknowledgement, request.text) @@ -5293,8 +5294,8 @@ fn submit_expert_prepare( response.server_finished = true; responses.insert(response_id, response); } else { - // A relayed reply already answered every handoff when admitted. - let resolved_handoff_ids = if expert_tts { + // A message to the Spokesperson already answered every handoff when admitted. + let resolved_handoff_ids = if answers_all { Vec::new() } else { request.resolved_handoff_ids diff --git a/src-tauri/crates/berd-call/src/openai_realtime_protocol.rs b/src-tauri/crates/berd-call/src/openai_realtime_protocol.rs index c3299cd1d..ff2148772 100644 --- a/src-tauri/crates/berd-call/src/openai_realtime_protocol.rs +++ b/src-tauri/crates/berd-call/src/openai_realtime_protocol.rs @@ -1587,10 +1587,9 @@ impl RealtimeExpertSpokespersonSession { }) } - /// Prepares an Expert reply in a call where the Expert has its own voice, - /// whether it speaks the reply or the Spokesperson relays it: it is - /// admitted however much was said since it was written, and answers - /// every open handoff. + /// Prepares an Expert message to the Spokesperson, or any reply when the + /// Expert has its own voice: it is admitted however much was said since + /// it was written, and answers every open handoff. pub fn prepare_expert_reply(&mut self, message: String) -> ExpertDirectiveOutcome { let outcome = self.conversation.prepare_directive(ExpertDirective { acknowledgement: Some(self.conversation.delivered_token()), diff --git a/src-tauri/crates/berd-call/tests/session_protocol.rs b/src-tauri/crates/berd-call/tests/session_protocol.rs index 4a50ad928..0b7c91d2d 100644 --- a/src-tauri/crates/berd-call/tests/session_protocol.rs +++ b/src-tauri/crates/berd-call/tests/session_protocol.rs @@ -3285,7 +3285,12 @@ fn an_expert_reply_that_fails_unheard_leaves_its_handoffs_open() { } #[test] -fn a_relayed_reply_is_said_by_the_spokesperson_and_answers_every_handoff() { +fn a_message_to_the_spokesperson_is_said_by_it_and_answers_every_handoff() { + message_the_spokesperson(false); + message_the_spokesperson(true); +} + +fn message_the_spokesperson(expert_voice: bool) { let (speech_url, speech_requests) = spawn_speech_server(); let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); let (relay_tx, relay_rx) = mpsc::sync_channel(1); @@ -3332,9 +3337,10 @@ fn a_relayed_reply_is_said_by_the_spokesperson_and_answers_every_handoff() { }); }); let endpoint = endpoint_rx.recv().unwrap(); + let own = ["--expert-voice", "tts", "--tts-url", &speech_url]; let mut session = ExpertSpokespersonTestSession::start_with_args( endpoint, - &["--expert-voice", "tts", "--tts-url", &speech_url], + if expert_voice { &own } else { &[] }, ); loop { session.send(json!({"type":"query_state","id":1,"after":0})); From ee04961521c4f91c16049a145a58e8c7a1bffe0c Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 07:40:30 -0400 Subject: [PATCH 15/49] berd-call: let the Expert speak directly only with --allow-direct-agent-speech In an Expert-Spokesperson call, the Expert now reaches the user through the Spokesperson with `message-spokesperson`, and `speak` is refused unless the call was started with --allow-direct-agent-speech. That flag replaces --expert-voice tts. With it, a reply in the Expert's own voice must acknowledge the latest input, as `speak --re` does everywhere, and once admitted it still answers every open handoff. A message to the Spokesperson stays admitted however stale. Co-Authored-By: Claude Opus 5.5 Signed-off-by: John Tennant --- src-tauri/crates/berd-call/PROTOCOL.md | 13 +-- src-tauri/crates/berd-call/README.md | 4 +- src-tauri/crates/berd-call/src/cli_help.rs | 20 +++-- .../crates/berd-call/src/host_session.rs | 51 +++++++++++ src-tauri/crates/berd-call/src/main.rs | 85 ++++++++++--------- .../berd-call/src/openai_realtime_protocol.rs | 34 +++++--- .../crates/berd-call/src/saved_settings.rs | 7 +- .../berd-call/tests/session_protocol.rs | 29 +++++-- 8 files changed, 165 insertions(+), 78 deletions(-) diff --git a/src-tauri/crates/berd-call/PROTOCOL.md b/src-tauri/crates/berd-call/PROTOCOL.md index 0b7215e3f..fbb87488b 100644 --- a/src-tauri/crates/berd-call/PROTOCOL.md +++ b/src-tauri/crates/berd-call/PROTOCOL.md @@ -74,8 +74,8 @@ Detailed backend errors are diagnostics on stderr only; protocol rejection and fatal messages are sanitized at the stdout boundary. `session.tts` describes the voice that speaks replies, and `set_tts_settings` changes it: in Expert-Spokesperson sessions, that is the configured TTS backend -with `--expert-voice tts` and otherwise the Spokesperson's own voice. With -`--expert-voice tts`, `session.spokesperson` is the Spokesperson's own OpenAI +with `--allow-direct-agent-speech` and otherwise the Spokesperson's own voice. With +`--allow-direct-agent-speech`, `session.spokesperson` is the Spokesperson's own OpenAI snapshot, with its own revision, and `set_spokesperson_settings` changes it the way `set_tts_settings` changes the Spokesperson's voice in other sessions. A reply in the Expert's own voice reports the same `admitted`, `speech_completed`, @@ -220,11 +220,12 @@ Unknown fields are rejected. IDs are positive. Speak text is at most 16 KiB. backend, keeping its other settings; `language` changes a Siri voice's locale too. A voice the backend doesn't offer, or a voice for a call whose replies are spoken by the Spokesperson, is `not_admitted(invalid_voice)`. In a call -whose replies have their own voice, a reply is admitted however stale its -`acknowledgement`, never `pending`, and answers every open handoff once -admitted, so `resolved_handoff_ids` is ignored. With `relay: true` (what +whose replies have their own voice, a reply needs the same `acknowledgement` +as a directive, and answers every open handoff once admitted, so +`resolved_handoff_ids` is ignored. With `relay: true` (what `berd-call message-spokesperson` sends), the Spokesperson says the reply in any -Expert-Spokesperson call, on those same terms, with the same lifecycle. Open +Expert-Spokesperson call; it is admitted however stale its `acknowledgement`, +never `pending`, and answers every open handoff. Open handoffs in a call whose replies have their own voice don't hold back a renewal or recovery of the Spokesperson's session; the replacement is told what they asked. diff --git a/src-tauri/crates/berd-call/README.md b/src-tauri/crates/berd-call/README.md index ae292f9fc..4da9813e0 100644 --- a/src-tauri/crates/berd-call/README.md +++ b/src-tauri/crates/berd-call/README.md @@ -146,12 +146,12 @@ berd-call session --tts-backend pocket --model-dir /path/to/native-voice-v2 --vo berd-call session --stt-backend parakeet --stt-model-dir /path/to/parakeet berd-call session --stt-backend openai berd-call session --tts-backend openai --mode expert-spokesperson --realtime-url ws://127.0.0.1:18870/v1/realtime -berd-call session --voice Aaron --language en-US --mode expert-spokesperson --expert-voice tts +berd-call session --voice Aaron --language en-US --mode expert-spokesperson --allow-direct-agent-speech berd-call session --tts-backend openai --tts-url https://proxy.example/v1/audio/speech berd-call session --stt-backend openai --stt-url wss://proxy.example/v1/realtime?intent=transcription ``` -By default the Spokesperson reads the Expert's replies aloud in its own voice. `berd-call message-spokesperson TEXT` always has it tell you, in any Expert–Spokesperson call: the message is admitted however much you've said since, and answers every open handoff, with no `--re` or `--resolves`. With `--expert-voice tts`, the Expert speaks for itself: berd-call plays each reply through the configured TTS backend, taking turns with the Spokesperson's audio, and your speech interrupts either of them. Afterwards the Spokesperson gets a silent note of what you heard, so it can follow up without repeating it. Every reply is spoken, even one written before what you said since, and it settles every open handoff once admitted, so `--resolves` isn't needed. Open handoffs no longer hold back renewing or recovering the Spokesperson's session; the new session is told what they asked, so it doesn't hand them off again. `--voice`, `--rate`, `status` and `settings --tts` describe and change the Expert's voice. The Spokesperson keeps its own, shown as `session.spokesperson`: choose it with `--spokesperson-voice NAME` and `--spokesperson-rate 0.25–1.5`, at `start` or live with `settings`, and the menu bar offers both voices. Saved settings keep the Spokesperson's voice for the next call whose replies have their own, but not `--expert-voice`, so pass that on each `start`. +In an Expert–Spokesperson call, the Expert reaches you through the Spokesperson: `berd-call message-spokesperson TEXT` has it tell you in its own voice. The message is admitted however much you've said since, and answers every open handoff, with no `--re` or `--resolves`. `speak` is refused unless the call was started with `--allow-direct-agent-speech`. With it, the Expert can also speak for itself: berd-call plays each `speak` reply through the configured TTS backend, taking turns with the Spokesperson's audio, and your speech interrupts either of them. Afterwards the Spokesperson gets a silent note of what you heard, so it can follow up without repeating it. A reply has to acknowledge everything you've said, as `speak --re` always does, and it settles every open handoff once admitted, so `--resolves` isn't needed. Open handoffs no longer hold back renewing or recovering the Spokesperson's session; the new session is told what they asked, so it doesn't hand them off again. `--voice`, `--rate`, `status` and `settings --tts` describe and change the Expert's voice. The Spokesperson keeps its own, shown as `session.spokesperson`: choose it with `--spokesperson-voice NAME` and `--spokesperson-rate 0.25–1.5`, at `start` or live with `settings`, and the menu bar offers both voices. Saved settings keep the Spokesperson's voice for the next call whose replies have their own, but not `--allow-direct-agent-speech`, so pass that on each `start`. The three URL flags take full endpoints and override only their matching service for that call. Expert–Spokesperson uses `--realtime-url` for both input and output; `--stt-url` applies only to conventional mode, and `--tts-url` also applies to the Expert's own voice. Without an override, each service uses its OpenAI endpoint (or its existing environment override). `OPENAI_API_KEY` supplies the CLI credential; the desktop app stores keys separately by endpoint URL in Keychain. diff --git a/src-tauri/crates/berd-call/src/cli_help.rs b/src-tauri/crates/berd-call/src/cli_help.rs index 05c77ad6a..86711efab 100644 --- a/src-tauri/crates/berd-call/src/cli_help.rs +++ b/src-tauri/crates/berd-call/src/cli_help.rs @@ -66,9 +66,11 @@ for delivery. Non-blocking sessions require --stream or --codex and emit speech_ rows only for interruptions, failures, or pending user input. Interrupted results include estimatedSpokenText, a best-effort prefix based on delivered audio. --voice speaks this text in another voice of the call's reply backend without changing its saved -voice, so several agents can share one call in distinct voices. In a call with ---expert-voice tts, every reply is spoken however stale --re is, and answers -every open handoff, so --resolves is ignored."#; +voice, so several agents can share one call in distinct voices. An +Expert-Spokesperson call refuses speak unless started with +--allow-direct-agent-speech; otherwise use message-spokesperson. With it, a +reply acknowledging the latest input is spoken in the Expert's own voice and +answers every open handoff, so --resolves is ignored."#; const MESSAGE_SPOKESPERSON_HELP: &str = r#"Message the Spokesperson of the active realtime call. @@ -76,7 +78,7 @@ Usage: berd-call message-spokesperson [--port PORT] TEXT The Spokesperson tells the user in its own voice, even in a call with ---expert-voice tts, where `speak` uses the Expert's. It is admitted however much +--allow-direct-agent-speech, where `speak` uses the Expert's. It is admitted however much the user has said since, answers every open handoff, and reports the same result as `speak`."#; @@ -94,7 +96,7 @@ Usage: berd-call settings [--port PORT] --restart [session options] The setting applies to subsequent speak calls. In a call started with ---expert-voice tts, --voice and --rate change the agents' voice, and +--allow-direct-agent-speech, --voice and --rate change the agents' voice, and --spokesperson-voice and --spokesperson-rate change the Spokesperson's (an OpenAI realtime voice, 0.25 to 1.5). Non-blocking delivery requires a call started with --stream or --codex. Successful delivery produces no notification."#; @@ -133,7 +135,7 @@ Usage: Each form also accepts: [--rate FLOAT] [--stt-backend macos|parakeet|openai] [--stt-model-dir PATH] - [--mode conventional|expert-spokesperson] [--expert-voice tts|spokesperson] + [--mode conventional|expert-spokesperson] [--allow-direct-agent-speech] [--spokesperson-voice NAME] [--spokesperson-rate 0.25..1.5] [--realtime-url WS_URL] [--stt-url WS_URL] [--tts-url HTTP_URL] @@ -142,9 +144,9 @@ agent integration. See PROTOCOL.md for the framed stdin, stdout, and PCM contracts. Endpoint URL flags override only the matching service for this call and default to OpenAI when no environment override is set. -In expert-spokesperson mode, --expert-voice tts speaks the Expert's replies -through the configured TTS backend, in its own voice beside the Spokesperson's, -instead of having the Spokesperson read them aloud. --spokesperson-voice and +In expert-spokesperson mode, the Expert speaks through the Spokesperson. +--allow-direct-agent-speech also lets it speak to the user itself, through the +configured TTS backend, in its own voice beside the Spokesperson's. --spokesperson-voice and --spokesperson-rate then choose the Spokesperson's own voice and speed."#; const SYNTHESIZE_HELP: &str = r#"Render text through a configured TTS backend into a new WAV file. diff --git a/src-tauri/crates/berd-call/src/host_session.rs b/src-tauri/crates/berd-call/src/host_session.rs index 973a7dc7c..3cdc50d3f 100644 --- a/src-tauri/crates/berd-call/src/host_session.rs +++ b/src-tauri/crates/berd-call/src/host_session.rs @@ -462,6 +462,10 @@ fn start_session( stop_requested: Arc::clone(stop_requested), muted: Arc::clone(&actor.muted), transcript: Arc::clone(transcript), + speaks: !expert_spokesperson + || arguments + .iter() + .any(|argument| argument == crate::DIRECT_AGENT_SPEECH), session_arguments: arguments, }); Ok(StartedSession { @@ -547,6 +551,8 @@ struct SessionControl { stop_requested: Arc, muted: Arc, transcript: Arc, + // Whether the Expert may speak to the user rather than only through the Spokesperson. + speaks: bool, session_arguments: Vec, } @@ -746,6 +752,12 @@ impl HostControl for SessionControl { language: Option, relay: bool, ) -> Result { + if !relay && !self.speaks { + return Err(format!( + "the Expert speaks through the Spokesperson: use message-spokesperson, or start the call with {}", + crate::DIRECT_AGENT_SPEECH + )); + } let (tx, rx) = mpsc::sync_channel(1); self.commands .send(ControlCommand::Speak { @@ -4350,6 +4362,7 @@ mod tests { stop_requested: Arc::clone(&stop_requested), muted: Arc::new(AtomicBool::new(false)), transcript: Arc::new(Transcript::Silent), + speaks: true, session_arguments: vec!["session".into()], }; drop(receiver); @@ -4357,6 +4370,42 @@ mod tests { assert!(stop_requested.load(Ordering::SeqCst)); } + #[test] + fn the_expert_speaks_only_through_the_spokesperson_unless_allowed() { + let (commands, receiver) = mpsc::sync_channel(1); + let control = SessionControl { + commands, + running: Arc::new(AtomicBool::new(true)), + ready: ReadyState { + session: Arc::new(Mutex::new(test_snapshot())), + }, + non_blocking: Arc::new(AtomicBool::new(false)), + stop_requested: Arc::new(AtomicBool::new(false)), + muted: Arc::new(AtomicBool::new(false)), + transcript: Arc::new(Transcript::Silent), + speaks: false, + session_arguments: vec!["session".into()], + }; + let refusal = control + .speak("Hello".into(), None, Vec::new(), None, None, false) + .unwrap_err(); + assert!(refusal.contains("message-spokesperson"), "{refusal}"); + assert!(receiver.try_recv().is_err()); + let worker = thread::spawn(move || match receiver.recv().unwrap() { + ControlCommand::Speak { + relay, response, .. + } => { + response.send(Ok(json!({}))).unwrap(); + relay + } + _ => panic!("expected a message to the Spokesperson"), + }); + control + .speak("Hello".into(), None, Vec::new(), None, None, true) + .unwrap(); + assert!(worker.join().unwrap()); + } + #[test] fn voice_updates_preserve_the_latest_rate_and_revision() { let (commands, receiver) = mpsc::sync_channel(1); @@ -4372,6 +4421,7 @@ mod tests { stop_requested: Arc::new(AtomicBool::new(false)), muted: Arc::new(AtomicBool::new(false)), transcript: Arc::new(Transcript::Silent), + speaks: true, session_arguments: vec!["session".into()], }; let worker = thread::spawn(move || match receiver.recv().unwrap() { @@ -4414,6 +4464,7 @@ mod tests { stop_requested: Arc::new(AtomicBool::new(false)), muted: Arc::new(AtomicBool::new(false)), transcript: Arc::new(Transcript::Silent), + speaks: true, session_arguments: vec!["session".into()], }; let worker = thread::spawn(move || match receiver.recv().unwrap() { diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 7cabd79f3..79ba32fac 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -899,10 +899,7 @@ fn parse_saved_start_args_at(args: &[String], path: PathBuf) -> Result(); let (reply_tx, reply_rx) = mpsc::channel::(); - // The Expert's own voice, with `--expert-voice tts`. + // The Expert's own voice, with `--allow-direct-agent-speech`. let mut reply_tts: Option> = None; let (tts_configuration_tx, tts_configuration_rx) = mpsc::channel::(); let mut tts_update: Option = None; @@ -5253,8 +5250,10 @@ fn submit_expert_prepare( }, ); } - let outcome = if answers_all { - core.prepare_expert_reply(request.text) + let outcome = if request.relay { + core.prepare_expert_message(request.text) + } else if expert_tts { + core.prepare_expert_reply(request.acknowledgement, request.text) } else { core.prepare_expert_directive(request.acknowledgement, request.text) }; @@ -5922,6 +5921,10 @@ fn finish_unready_input_runtime(runtime: VoiceInputRuntime) -> Result<(), String .map_err(|error| format!("finish unready voice input runtime: {error}")) } +/// Lets the Expert speak to the user in its own voice in an +/// Expert-Spokesperson call, rather than only through the Spokesperson. +pub(crate) const DIRECT_AGENT_SPEECH: &str = "--allow-direct-agent-speech"; + fn parse_args(args: &[String]) -> Result { if args.get(1).map(String::as_str) != Some("session") { return Err("the only supported command is session".into()); @@ -5944,6 +5947,11 @@ fn parse_args(args: &[String]) -> Result { if is_help_flag(flag) { return Err(ParseFailure::HelpRequested); } + if flag == DIRECT_AGENT_SPEECH { + expert_tts = true; + index += 1; + continue; + } let value = args .get(index + 1) .ok_or_else(|| format!("{flag} requires a value"))?; @@ -5973,13 +5981,6 @@ fn parse_args(args: &[String]) -> Result { _ => return Err("--mode must be conventional or expert-spokesperson".into()), } } - "--expert-voice" => { - expert_tts = match value.as_str() { - "tts" => true, - "spokesperson" => false, - _ => return Err("--expert-voice must be tts or spokesperson".into()), - } - } "--spokesperson-voice" => spokesperson_voice = Some(value.clone()), "--spokesperson-rate" => { spokesperson_rate = Some( @@ -5999,12 +6000,13 @@ fn parse_args(args: &[String]) -> Result { return Err("--realtime-url requires --mode expert-spokesperson".into()); } if expert_tts && mode != SessionMode::ExpertSpokesperson { - return Err("--expert-voice requires --mode expert-spokesperson".into()); + return Err(format!("{DIRECT_AGENT_SPEECH} requires --mode expert-spokesperson").into()); } if (spokesperson_voice.is_some() || spokesperson_rate.is_some()) && !expert_tts { - return Err( - "--spokesperson-voice and --spokesperson-rate require --expert-voice tts".into(), - ); + return Err(format!( + "--spokesperson-voice and --spokesperson-rate require {DIRECT_AGENT_SPEECH}" + ) + .into()); } if let Some(voice) = &spokesperson_voice { if !berd_call::openai_realtime_protocol::OPENAI_REALTIME_VOICE_IDS.contains(&voice.as_str()) @@ -10713,18 +10715,12 @@ mod tests { parse(&all) }; assert!(!expert(&["--tts-backend", "openai"]).unwrap().expert_tts); - assert!( - !expert(&["--tts-backend", "openai", "--expert-voice", "spokesperson"]) - .unwrap() - .expert_tts - ); let own = expert(&[ "--tts-backend", "openai", "--rate", "2.0", - "--expert-voice", - "tts", + "--allow-direct-agent-speech", "--tts-url", "http://127.0.0.1:18870/v1/audio/speech", ]) @@ -10750,8 +10746,7 @@ mod tests { let chosen = expert(&[ "--tts-backend", "openai", - "--expert-voice", - "tts", + "--allow-direct-agent-speech", "--spokesperson-voice", "cedar", "--spokesperson-rate", @@ -10761,13 +10756,17 @@ mod tests { apply_spokesperson_startup_settings(&chosen, &mut realtime).unwrap(); assert_eq!((realtime.voice(), realtime.speed()), ("cedar", 1.25)); for (extra, error) in [ - (&["--spokesperson-voice", "cedar"][..], "--expert-voice tts"), + (&["--spokesperson-voice", "cedar"][..], DIRECT_AGENT_SPEECH), ( - &["--expert-voice", "tts", "--spokesperson-voice", "nobody"], + &[ + "--allow-direct-agent-speech", + "--spokesperson-voice", + "nobody", + ], "no realtime voice named nobody", ), ( - &["--expert-voice", "tts", "--spokesperson-rate", "2"], + &["--allow-direct-agent-speech", "--spokesperson-rate", "2"], "between 0.25 and 1.5", ), ] { @@ -10776,20 +10775,20 @@ mod tests { } let read_aloud = expert(&["--tts-backend", "openai", "--rate", "2.0"]).unwrap(); assert!(apply_spokesperson_startup_settings(&read_aloud, &mut realtime).is_err()); - assert!(expert(&["--expert-voice", "loud"]) - .unwrap_err() - .contains("tts or spokesperson")); - assert!(parse(&["--tts-backend", "openai", "--expert-voice", "tts"]) - .unwrap_err() - .contains("--mode expert-spokesperson")); + assert!( + parse(&["--tts-backend", "openai", "--allow-direct-agent-speech"]) + .unwrap_err() + .contains("--mode expert-spokesperson") + ); assert_eq!( saved_settings::persistable_arguments(&args(&[ "--mode", "expert-spokesperson", - "--expert-voice", - "tts" + "--allow-direct-agent-speech", + "--voice", + "Aaron" ])), - args(&["--mode", "expert-spokesperson"]) + args(&["--mode", "expert-spokesperson", "--voice", "Aaron"]) ); } @@ -12539,10 +12538,14 @@ mod tests { .iter() .any(|flag| flag.starts_with("--spokesperson"))); assert!(conventional.saved.unwrap().1.arguments.starts_with(&chosen)); - let own = start(&["--mode", "expert-spokesperson", "--expert-voice", "tts"]); + let own = start(&[ + "--mode", + "expert-spokesperson", + "--allow-direct-agent-speech", + ]); assert!(own .session_arguments - .ends_with(&args(&["--expert-voice", "tts"]))); + .ends_with(&args(&["--allow-direct-agent-speech"]))); assert!(own .session_arguments .windows(2) diff --git a/src-tauri/crates/berd-call/src/openai_realtime_protocol.rs b/src-tauri/crates/berd-call/src/openai_realtime_protocol.rs index ff2148772..d6482ba5f 100644 --- a/src-tauri/crates/berd-call/src/openai_realtime_protocol.rs +++ b/src-tauri/crates/berd-call/src/openai_realtime_protocol.rs @@ -1587,14 +1587,22 @@ impl RealtimeExpertSpokespersonSession { }) } - /// Prepares an Expert message to the Spokesperson, or any reply when the - /// Expert has its own voice: it is admitted however much was said since - /// it was written, and answers every open handoff. - pub fn prepare_expert_reply(&mut self, message: String) -> ExpertDirectiveOutcome { - let outcome = self.conversation.prepare_directive(ExpertDirective { - acknowledgement: Some(self.conversation.delivered_token()), - message, - }); + /// Prepares an Expert message to the Spokesperson: it is admitted however + /// much was said since it was written, and answers every open handoff. + pub fn prepare_expert_message(&mut self, message: String) -> ExpertDirectiveOutcome { + let acknowledgement = Some(self.conversation.delivered_token()); + self.prepare_expert_reply(acknowledgement, message) + } + + /// Prepares a reply in the Expert's own voice: like a directive, it waits + /// for whatever the Expert hasn't acknowledged, and once admitted it + /// answers every open handoff. + pub fn prepare_expert_reply( + &mut self, + acknowledgement: Option, + message: String, + ) -> ExpertDirectiveOutcome { + let outcome = self.prepare_expert_directive(acknowledgement, message); if matches!(outcome, ExpertDirectiveOutcome::Accepted { .. }) { self.open_handoffs.clear(); } @@ -3117,7 +3125,7 @@ mod tests { } #[test] - fn expert_reply_answers_every_handoff_however_late() { + fn expert_message_answers_every_handoff_however_late() { let mut session = RealtimeExpertSpokespersonSession::new(0, "test-call"); session .register_handoff("handoff-1", 1, "First request") @@ -3135,13 +3143,15 @@ mod tests { }) .unwrap(); - // A directive written before the user's latest words waits for them. + // A reply written before the user's latest words waits for them, and + // answers no handoff until it is admitted. assert!(matches!( - session.prepare_expert_directive(None, "Stale".into()), + session.prepare_expert_reply(None, "Stale".into()), ExpertDirectiveOutcome::Pending(_) )); + assert!(session.has_unresolved_handoff()); assert!(matches!( - session.prepare_expert_reply("The answer".into()), + session.prepare_expert_message("The answer".into()), ExpertDirectiveOutcome::Accepted { confirmed_token: 1, .. diff --git a/src-tauri/crates/berd-call/src/saved_settings.rs b/src-tauri/crates/berd-call/src/saved_settings.rs index e9ff2f081..cc83d2396 100644 --- a/src-tauri/crates/berd-call/src/saved_settings.rs +++ b/src-tauri/crates/berd-call/src/saved_settings.rs @@ -128,12 +128,17 @@ pub(crate) fn merge_arguments(saved: &[String], explicit: &[String]) -> Vec Vec { + let arguments: Vec<_> = arguments + .iter() + .filter(|argument| *argument != crate::DIRECT_AGENT_SPEECH) + .cloned() + .collect(); arguments .chunks_exact(2) .filter(|pair| { !matches!( pair[0].as_str(), - "--realtime-url" | "--stt-url" | "--tts-url" | "--expert-voice" + "--realtime-url" | "--stt-url" | "--tts-url" ) }) .flatten() diff --git a/src-tauri/crates/berd-call/tests/session_protocol.rs b/src-tauri/crates/berd-call/tests/session_protocol.rs index 0b7c91d2d..457cd69c6 100644 --- a/src-tauri/crates/berd-call/tests/session_protocol.rs +++ b/src-tauri/crates/berd-call/tests/session_protocol.rs @@ -2769,8 +2769,7 @@ fn expert_voice_keeps_a_separate_spokesperson_voice_control() { let mut session = ExpertSpokespersonTestSession::start_with_args( endpoint, &[ - "--expert-voice", - "tts", + "--allow-direct-agent-speech", "--tts-url", &speech_url, "--spokesperson-voice", @@ -3016,7 +3015,7 @@ fn expert_voice_speaks_replies_itself_and_tells_the_spokesperson_what_was_heard( let endpoint = endpoint_rx.recv().unwrap(); let mut session = ExpertSpokespersonTestSession::start_with_args( endpoint, - &["--expert-voice", "tts", "--tts-url", &speech_url], + &["--allow-direct-agent-speech", "--tts-url", &speech_url], ); assert_eq!(session.ready["session"]["tts"]["backend"], "openai"); assert_eq!( @@ -3337,7 +3336,7 @@ fn message_the_spokesperson(expert_voice: bool) { }); }); let endpoint = endpoint_rx.recv().unwrap(); - let own = ["--expert-voice", "tts", "--tts-url", &speech_url]; + let own = ["--allow-direct-agent-speech", "--tts-url", &speech_url]; let mut session = ExpertSpokespersonTestSession::start_with_args( endpoint, if expert_voice { &own } else { &[] }, @@ -3397,7 +3396,7 @@ fn message_the_spokesperson(expert_voice: bool) { } #[test] -fn expert_voice_handoffs_survive_provider_expiry_and_any_late_reply_answers_them() { +fn expert_voice_handoffs_survive_provider_expiry_and_a_current_reply_answers_them() { let (speech_url, speech_requests) = spawn_speech_server(); let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); let (note_tx, note_rx) = mpsc::sync_channel(1); @@ -3471,7 +3470,7 @@ fn expert_voice_handoffs_survive_provider_expiry_and_any_late_reply_answers_them let endpoint = endpoint_rx.recv().unwrap(); let mut session = ExpertSpokespersonTestSession::start_with_args( endpoint, - &["--expert-voice", "tts", "--tts-url", &speech_url], + &["--allow-direct-agent-speech", "--tts-url", &speech_url], ); let note = note_rx.recv_timeout(Duration::from_secs(3)).unwrap(); assert!( @@ -3479,13 +3478,29 @@ fn expert_voice_handoffs_survive_provider_expiry_and_any_late_reply_answers_them "{note}" ); - // A reply written before everything since, naming no handoff, is still spoken. + // A reply written before everything since waits for it, and once it + // acknowledges the latest, naming no handoff, it answers them all. session.send(json!({ "type":"prepare_speak", "id":2, "acknowledgement":0, "text":"I checked the repository." })); + let pending = loop { + let message = session.recv(Duration::from_secs(2)); + assert_ne!(message["type"], "fatal", "{message}"); + if message["id"] == 2 { + break message; + } + }; + assert_eq!(pending["type"], "pending", "{pending}"); + let latest = pending["utterances"].as_array().unwrap().last().unwrap()["token"].clone(); + session.send(json!({ + "type":"prepare_speak", + "id":2, + "acknowledgement":latest, + "text":"I checked the repository." + })); let admitted = loop { let message = session.recv(Duration::from_secs(2)); assert_ne!(message["type"], "fatal", "{message}"); From 156d8f82985f102abd7ac8dcb9b6bbd49b19d8fc Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 07:54:15 -0400 Subject: [PATCH 16/49] berd-call: let a Codex task hear the call as Berd's Expert does A `--codex` task heard a realtime call only at handoffs. It now gets a batch after every Spokesperson reply and at each handoff, as Berd's Expert and berd-call before the handoffs stream did, so it can follow the conversation and answer what was said last. `--stream` (handoffs) still holds rows until a handoff, and `--stream live` is unchanged. Co-Authored-By: Claude Opus 5.5 Signed-off-by: John Tennant --- src-tauri/crates/berd-call/README.md | 4 ++-- src-tauri/crates/berd-call/src/cli_help.rs | 3 ++- src-tauri/crates/berd-call/src/host_session.rs | 5 +---- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src-tauri/crates/berd-call/README.md b/src-tauri/crates/berd-call/README.md index 4da9813e0..b5ea6d21c 100644 --- a/src-tauri/crates/berd-call/README.md +++ b/src-tauri/crates/berd-call/README.md @@ -76,8 +76,8 @@ nothing until then, and then every row since the last handoff, ending with the row with cursor `0` as soon as a reply is generated, before it is spoken. The `spokesperson` or `spokesperson_interrupted` row after playback carries the reply's cursor and what was heard. A conventional call has no handoffs, so it -always streams each row. A `--codex` task, like `--stream`, wakes only at -handoffs. The standalone host intentionally uses only the default devices. It +always streams each row. A `--codex` task hears the call as Berd's Expert +does: a batch after every Spokesperson reply, and at each handoff. The standalone host intentionally uses only the default devices. It suppresses recognition during playback when the default output identifies itself as built-in speakers, preventing the call from transcribing its own speech; headphone-like outputs retain barge-in. diff --git a/src-tauri/crates/berd-call/src/cli_help.rs b/src-tauri/crates/berd-call/src/cli_help.rs index 86711efab..d304643f3 100644 --- a/src-tauri/crates/berd-call/src/cli_help.rs +++ b/src-tauri/crates/berd-call/src/cli_help.rs @@ -43,7 +43,8 @@ options are the same as `berd-call session`, except the PCM descriptor is owned internally. `--stream` prints cursor, role, and text as TSV. In Expert-Spokesperson mode it prints nothing until the Spokesperson hands off, then everything since the last handoff; `--stream live` prints each row as it -happens. `--codex` delivers the handoff batches to the calling Codex task +happens. `--codex` delivers a batch to the calling Codex task after every +Spokesperson reply and at each handoff, as Berd's Expert hears the call (requires CODEX_THREAD_ID). A menu-bar item controls speech rate, input muting, and ending the call; pass `--no-menu-bar` to omit it. diff --git a/src-tauri/crates/berd-call/src/host_session.rs b/src-tauri/crates/berd-call/src/host_session.rs index 3cdc50d3f..6038be16f 100644 --- a/src-tauri/crates/berd-call/src/host_session.rs +++ b/src-tauri/crates/berd-call/src/host_session.rs @@ -1436,10 +1436,7 @@ impl SessionActor { ) -> Result<(), String> { self.held.extend(events); if !handoff - && matches!( - *self.transcript, - Transcript::Stdout { live: false } | Transcript::Codex(_) - ) + && matches!(*self.transcript, Transcript::Stdout { live: false }) { return Ok(()); } From 0afc16a680f7c2f92e7be606541f8ef63a4ab0ea Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 07:57:12 -0400 Subject: [PATCH 17/49] berd-call: make a message to the Spokesperson an Expert directive `message-spokesperson` was admitted however stale and answered every open handoff. It now takes the terms of a directive from Berd's Expert: `--re CURSOR` is required and must acknowledge the latest input (the Spokesperson's own speech doesn't count), or the message comes back `spoke:false` with what was missed, and each `--resolves HANDOFF_ID` settles that handoff once the Spokesperson has said it. This holds in any Expert-Spokesperson call, including one with --allow-direct-agent-speech, where `speak` still answers every handoff. Co-Authored-By: Claude Opus 5.5 Signed-off-by: John Tennant --- src-tauri/crates/berd-call/PROTOCOL.md | 7 +- src-tauri/crates/berd-call/README.md | 2 +- src-tauri/crates/berd-call/src/cli_help.rs | 10 +-- .../berd-call/src/expert_spokesperson.rs | 5 -- .../crates/berd-call/src/host_session.rs | 4 +- src-tauri/crates/berd-call/src/main.rs | 72 ++++++++++--------- .../berd-call/src/openai_realtime_protocol.rs | 11 +-- .../berd-call/tests/session_protocol.rs | 55 ++++++++------ 8 files changed, 86 insertions(+), 80 deletions(-) diff --git a/src-tauri/crates/berd-call/PROTOCOL.md b/src-tauri/crates/berd-call/PROTOCOL.md index fbb87488b..9e10acd4a 100644 --- a/src-tauri/crates/berd-call/PROTOCOL.md +++ b/src-tauri/crates/berd-call/PROTOCOL.md @@ -223,9 +223,10 @@ are spoken by the Spokesperson, is `not_admitted(invalid_voice)`. In a call whose replies have their own voice, a reply needs the same `acknowledgement` as a directive, and answers every open handoff once admitted, so `resolved_handoff_ids` is ignored. With `relay: true` (what -`berd-call message-spokesperson` sends), the Spokesperson says the reply in any -Expert-Spokesperson call; it is admitted however stale its `acknowledgement`, -never `pending`, and answers every open handoff. Open +`berd-call message-spokesperson` sends), the reply is a directive in any +Expert-Spokesperson call, even one whose replies have their own voice: the +Spokesperson says it, on the same `acknowledgement` and `resolved_handoff_ids` +terms. Open handoffs in a call whose replies have their own voice don't hold back a renewal or recovery of the Spokesperson's session; the replacement is told what they asked. diff --git a/src-tauri/crates/berd-call/README.md b/src-tauri/crates/berd-call/README.md index b5ea6d21c..40f6b25c7 100644 --- a/src-tauri/crates/berd-call/README.md +++ b/src-tauri/crates/berd-call/README.md @@ -151,7 +151,7 @@ berd-call session --tts-backend openai --tts-url https://proxy.example/v1/audio/ berd-call session --stt-backend openai --stt-url wss://proxy.example/v1/realtime?intent=transcription ``` -In an Expert–Spokesperson call, the Expert reaches you through the Spokesperson: `berd-call message-spokesperson TEXT` has it tell you in its own voice. The message is admitted however much you've said since, and answers every open handoff, with no `--re` or `--resolves`. `speak` is refused unless the call was started with `--allow-direct-agent-speech`. With it, the Expert can also speak for itself: berd-call plays each `speak` reply through the configured TTS backend, taking turns with the Spokesperson's audio, and your speech interrupts either of them. Afterwards the Spokesperson gets a silent note of what you heard, so it can follow up without repeating it. A reply has to acknowledge everything you've said, as `speak --re` always does, and it settles every open handoff once admitted, so `--resolves` isn't needed. Open handoffs no longer hold back renewing or recovering the Spokesperson's session; the new session is told what they asked, so it doesn't hand them off again. `--voice`, `--rate`, `status` and `settings --tts` describe and change the Expert's voice. The Spokesperson keeps its own, shown as `session.spokesperson`: choose it with `--spokesperson-voice NAME` and `--spokesperson-rate 0.25–1.5`, at `start` or live with `settings`, and the menu bar offers both voices. Saved settings keep the Spokesperson's voice for the next call whose replies have their own, but not `--allow-direct-agent-speech`, so pass that on each `start`. +In an Expert–Spokesperson call, the Expert reaches you through the Spokesperson: `berd-call message-spokesperson --re CURSOR [--resolves HANDOFF_ID]... TEXT` has it tell you in its own voice. Like the directives of Berd's Expert, the message must acknowledge the latest input (the Spokesperson's own speech doesn't count), or it comes back `spoke:false` with what was missed, and a handoff it names settles once the Spokesperson has said it. `speak` is refused unless the call was started with `--allow-direct-agent-speech`. With it, the Expert can also speak for itself: berd-call plays each `speak` reply through the configured TTS backend, taking turns with the Spokesperson's audio, and your speech interrupts either of them. Afterwards the Spokesperson gets a silent note of what you heard, so it can follow up without repeating it. A reply has to acknowledge everything you've said, as `speak --re` always does, and it settles every open handoff once admitted, so `--resolves` isn't needed. Open handoffs no longer hold back renewing or recovering the Spokesperson's session; the new session is told what they asked, so it doesn't hand them off again. `--voice`, `--rate`, `status` and `settings --tts` describe and change the Expert's voice. The Spokesperson keeps its own, shown as `session.spokesperson`: choose it with `--spokesperson-voice NAME` and `--spokesperson-rate 0.25–1.5`, at `start` or live with `settings`, and the menu bar offers both voices. Saved settings keep the Spokesperson's voice for the next call whose replies have their own, but not `--allow-direct-agent-speech`, so pass that on each `start`. The three URL flags take full endpoints and override only their matching service for that call. Expert–Spokesperson uses `--realtime-url` for both input and output; `--stt-url` applies only to conventional mode, and `--tts-url` also applies to the Expert's own voice. Without an override, each service uses its OpenAI endpoint (or its existing environment override). `OPENAI_API_KEY` supplies the CLI credential; the desktop app stores keys separately by endpoint URL in Keychain. diff --git a/src-tauri/crates/berd-call/src/cli_help.rs b/src-tauri/crates/berd-call/src/cli_help.rs index d304643f3..dabffbff0 100644 --- a/src-tauri/crates/berd-call/src/cli_help.rs +++ b/src-tauri/crates/berd-call/src/cli_help.rs @@ -76,12 +76,14 @@ answers every open handoff, so --resolves is ignored."#; const MESSAGE_SPOKESPERSON_HELP: &str = r#"Message the Spokesperson of the active realtime call. Usage: - berd-call message-spokesperson [--port PORT] TEXT + berd-call message-spokesperson [--port PORT] --re CURSOR + [--resolves HANDOFF_ID]... TEXT The Spokesperson tells the user in its own voice, even in a call with ---allow-direct-agent-speech, where `speak` uses the Expert's. It is admitted however much -the user has said since, answers every open handoff, and reports the same result -as `speak`."#; +--allow-direct-agent-speech, where `speak` uses the Expert's. Like a directive +from Berd's Expert, the message must acknowledge the latest input (the +Spokesperson's own speech doesn't count), each --resolves settles that handoff +once the Spokesperson has said it, and the result is the same as `speak`'s."#; const SETTINGS_HELP: &str = r#"Update settings for the active local voice call. diff --git a/src-tauri/crates/berd-call/src/expert_spokesperson.rs b/src-tauri/crates/berd-call/src/expert_spokesperson.rs index 433e99877..ef9cf322a 100644 --- a/src-tauri/crates/berd-call/src/expert_spokesperson.rs +++ b/src-tauri/crates/berd-call/src/expert_spokesperson.rs @@ -155,11 +155,6 @@ impl ExpertSpokespersonCore { self.pipe.cursor(RealtimePipePeer::Expert) } - /// The newest message the Expert has been sent, read or not. - pub fn delivered_token(&self) -> u64 { - self.pipe.delivery_cursor(RealtimePipePeer::Expert) - } - pub fn next_live_token(&self) -> u64 { self.pipe.next_message_id() } diff --git a/src-tauri/crates/berd-call/src/host_session.rs b/src-tauri/crates/berd-call/src/host_session.rs index 6038be16f..a039ef1e0 100644 --- a/src-tauri/crates/berd-call/src/host_session.rs +++ b/src-tauri/crates/berd-call/src/host_session.rs @@ -1435,9 +1435,7 @@ impl SessionActor { handoff: bool, ) -> Result<(), String> { self.held.extend(events); - if !handoff - && matches!(*self.transcript, Transcript::Stdout { live: false }) - { + if !handoff && matches!(*self.transcript, Transcript::Stdout { live: false }) { return Ok(()); } for event in unstreamed(&std::mem::take(&mut self.held), &mut self.transcribed) { diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 79ba32fac..330560db1 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -568,13 +568,13 @@ fn main() { print_pretty_json(&response).unwrap_or_else(|error| operational_error("speak", error)); } Some("message-spokesperson") => { - let (port, text) = parse_or_exit(parse_message_spokesperson_args(&args), &args); + let options = parse_or_exit(parse_message_spokesperson_args(&args), &args); let response = host_control::request( - port, + options.port, host_control::ControlRequest::Speak { - text, - acknowledgement: None, - resolved_handoff_ids: Vec::new(), + text: options.text, + acknowledgement: options.acknowledgement, + resolved_handoff_ids: options.resolved_handoff_ids, voice: None, language: None, relay: true, @@ -1047,15 +1047,15 @@ fn parse_speak_control_args(args: &[String]) -> Result Result<(u16, String), ParseFailure> { +fn parse_message_spokesperson_args(args: &[String]) -> Result { let options = parse_speak_control_args(args)?; - if options.acknowledgement.is_some() - || !options.resolved_handoff_ids.is_empty() - || options.voice.is_some() - { - return Err("message-spokesperson accepts only --port PORT and TEXT".into()); + if options.voice.is_some() { + return Err("message-spokesperson takes no --voice".into()); + } + if options.acknowledgement.is_none() { + return Err("message-spokesperson requires --re CURSOR".into()); } - Ok((options.port, options.text)) + Ok(options) } fn parse_host_settings_args( @@ -5234,10 +5234,10 @@ fn submit_expert_prepare( next_speech_id: &mut u64, writer: &mut impl Write, ) -> Result<(), String> { - // A message to the Spokesperson, or any reply when the Expert has its own - // voice, answers every open handoff, so the ones it names don't matter. - let answers_all = expert_tts || request.relay; - if !answers_all + // A reply in the Expert's own voice answers every open handoff, so the + // ones it names don't matter. A message to the Spokesperson is a directive. + let own_voice = expert_tts && !request.relay; + if !own_voice && !core .unknown_handoff_ids(&request.resolved_handoff_ids) .is_empty() @@ -5250,9 +5250,7 @@ fn submit_expert_prepare( }, ); } - let outcome = if request.relay { - core.prepare_expert_message(request.text) - } else if expert_tts { + let outcome = if own_voice { core.prepare_expert_reply(request.acknowledgement, request.text) } else { core.prepare_expert_directive(request.acknowledgement, request.text) @@ -5280,7 +5278,7 @@ fn submit_expert_prepare( let speech_id = *next_speech_id; *next_speech_id += 1; core.record_expert_turn(message.clone()); - if expert_tts && !request.relay { + if own_voice { // Played in turn with the Spokesperson's responses, without one of its own, // once the parent is ready for its output. let response_id = format!("expert-reply-{speech_id}"); @@ -5293,12 +5291,7 @@ fn submit_expert_prepare( response.server_finished = true; responses.insert(response_id, response); } else { - // A message to the Spokesperson already answered every handoff when admitted. - let resolved_handoff_ids = if answers_all { - Vec::new() - } else { - request.resolved_handoff_ids - }; + let resolved_handoff_ids = request.resolved_handoff_ids; core.mark_handoffs_resolving(&resolved_handoff_ids)?; directive_speeches.insert( request.id, @@ -12808,19 +12801,28 @@ mod tests { "message-spokesperson", "--port", "5300", + "--re", + "4", + "--resolves", + "handoff-1", "Tell them it's fixed." ])) .unwrap(), - (5300, "Tell them it's fixed.".into()) + SpeakOptions { + port: 5300, + acknowledgement: Some(4), + resolved_handoff_ids: vec!["handoff-1".into()], + voice: None, + language: None, + text: "Tell them it's fixed.".into(), + } ); - assert!(parse_message_spokesperson_args(&args(&[ - "berd-call", - "message-spokesperson", - "--voice", - "Nora", - "hello" - ])) - .is_err()); + for extra in [&["--voice", "Nora"][..], &[]] { + let mut given = args(&["berd-call", "message-spokesperson"]); + given.extend(args(extra)); + given.push("hello".into()); + assert!(parse_message_spokesperson_args(&given).is_err()); + } assert!(parse_speak_control_args(&args(&[ "berd-call", "speak", diff --git a/src-tauri/crates/berd-call/src/openai_realtime_protocol.rs b/src-tauri/crates/berd-call/src/openai_realtime_protocol.rs index d6482ba5f..d79829184 100644 --- a/src-tauri/crates/berd-call/src/openai_realtime_protocol.rs +++ b/src-tauri/crates/berd-call/src/openai_realtime_protocol.rs @@ -1587,13 +1587,6 @@ impl RealtimeExpertSpokespersonSession { }) } - /// Prepares an Expert message to the Spokesperson: it is admitted however - /// much was said since it was written, and answers every open handoff. - pub fn prepare_expert_message(&mut self, message: String) -> ExpertDirectiveOutcome { - let acknowledgement = Some(self.conversation.delivered_token()); - self.prepare_expert_reply(acknowledgement, message) - } - /// Prepares a reply in the Expert's own voice: like a directive, it waits /// for whatever the Expert hasn't acknowledged, and once admitted it /// answers every open handoff. @@ -3125,7 +3118,7 @@ mod tests { } #[test] - fn expert_message_answers_every_handoff_however_late() { + fn expert_reply_answers_every_handoff_once_current() { let mut session = RealtimeExpertSpokespersonSession::new(0, "test-call"); session .register_handoff("handoff-1", 1, "First request") @@ -3151,7 +3144,7 @@ mod tests { )); assert!(session.has_unresolved_handoff()); assert!(matches!( - session.prepare_expert_message("The answer".into()), + session.prepare_expert_reply(Some(1), "The answer".into()), ExpertDirectiveOutcome::Accepted { confirmed_token: 1, .. diff --git a/src-tauri/crates/berd-call/tests/session_protocol.rs b/src-tauri/crates/berd-call/tests/session_protocol.rs index 457cd69c6..930d49b53 100644 --- a/src-tauri/crates/berd-call/tests/session_protocol.rs +++ b/src-tauri/crates/berd-call/tests/session_protocol.rs @@ -3284,7 +3284,7 @@ fn an_expert_reply_that_fails_unheard_leaves_its_handoffs_open() { } #[test] -fn a_message_to_the_spokesperson_is_said_by_it_and_answers_every_handoff() { +fn a_message_to_the_spokesperson_is_a_directive_it_says() { message_the_spokesperson(false); message_the_spokesperson(true); } @@ -3341,7 +3341,7 @@ fn message_the_spokesperson(expert_voice: bool) { endpoint, if expert_voice { &own } else { &[] }, ); - loop { + let handoffs = loop { session.send(json!({"type":"query_state","id":1,"after":0})); let state = loop { let message = session.recv(Duration::from_secs(2)); @@ -3350,30 +3350,41 @@ fn message_the_spokesperson(expert_voice: bool) { break message; } }; - if state["unresolved_handoff_ids"] + if let Some(ids) = state["unresolved_handoff_ids"] .as_array() - .is_some_and(|ids| ids.len() == 2) + .filter(|ids| ids.len() == 2) { - break; + break ids.clone(); } std::thread::sleep(Duration::from_millis(20)); - } + }; - // Stale, naming no handoff, and still the Spokesperson's to say. - session.send(json!({ - "type":"prepare_speak", - "id":2, - "acknowledgement":0, - "text":"I checked the repository.", - "relay":true - })); - let admitted = loop { - let message = session.recv(Duration::from_secs(2)); - assert_ne!(message["type"], "fatal", "{message}"); - if message["id"] == 2 { - break message; + // A stale message waits for what it missed, like a directive. + let prepare = |acknowledgement: &Value| { + json!({ + "type":"prepare_speak", + "id":2, + "acknowledgement":acknowledgement, + "resolved_handoff_ids":[&handoffs[0]], + "text":"I checked the repository.", + "relay":true + }) + }; + let mut reply = |session: &mut ExpertSpokespersonTestSession, request| { + session.send(request); + loop { + let message = session.recv(Duration::from_secs(2)); + assert_ne!(message["type"], "fatal", "{message}"); + if message["id"] == 2 { + break message; + } } }; + let pending = reply(&mut session, prepare(&json!(0))); + assert_eq!(pending["type"], "pending", "{pending}"); + let latest = &pending["utterances"].as_array().unwrap().last().unwrap()["token"]; + // Current, it is the Spokesperson's to say, and answers only the handoff it names. + let admitted = reply(&mut session, prepare(latest)); assert_eq!(admitted["type"], "admitted", "{admitted}"); let relayed = relay_rx.recv_timeout(Duration::from_secs(3)).unwrap(); assert!( @@ -3387,7 +3398,11 @@ fn message_the_spokesperson(expert_voice: bool) { break message; } }; - assert_eq!(state.get("unresolved_handoff_ids"), None, "{state}"); + assert_eq!( + state["unresolved_handoff_ids"], + json!([&handoffs[1]]), + "{state}" + ); assert!(speech_requests .recv_timeout(Duration::from_millis(300)) .is_err()); From d43e83e392c8a9bc5cf245813d22307c8b15250a Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 07:58:30 -0400 Subject: [PATCH 18/49] berd-call: stream realtime calls the way Berd's Expert hears them `--stream` (or `--stream handoffs`) printed nothing in an Expert-Spokesperson call until a handoff. It now prints the same batches as `--codex`, after every Spokesperson reply and at each handoff, so the CLI has two ways to hear a realtime call: as Berd's Expert does, or as it happens with `--stream live`. `handoffs` is no longer accepted after `--stream`. Co-Authored-By: Claude Opus 5.5 Signed-off-by: John Tennant --- src-tauri/crates/berd-call/README.md | 10 ++-- src-tauri/crates/berd-call/src/cli_help.rs | 11 ++-- .../crates/berd-call/src/host_session.rs | 57 ++----------------- src-tauri/crates/berd-call/src/main.rs | 6 +- 4 files changed, 15 insertions(+), 69 deletions(-) diff --git a/src-tauri/crates/berd-call/README.md b/src-tauri/crates/berd-call/README.md index 40f6b25c7..4fef37df9 100644 --- a/src-tauri/crates/berd-call/README.md +++ b/src-tauri/crates/berd-call/README.md @@ -68,16 +68,14 @@ without changing the call's voice, so agents sharing a call can sound distinct. server listens only on `127.0.0.1:5222` by default; all four commands accept `--port` when more than one local call must be addressed. `--stream` writes one TSV row per delivered transcript event after the `cursor`, `role`, and `text` -header. In Expert-Spokesperson mode the Expert, like a GPT-Live delegate, hears -the conversation only when the Spokesperson hands off: the stream prints -nothing until then, and then every row since the last handoff, ending with the -`handoff` row. `--stream live` instead shows the conversation as it happens: a +header. In Expert-Spokesperson mode the stream hears the call as Berd's Expert +does: a batch of every row since the last one after each Spokesperson reply, +and at each handoff, ending with the `handoff` row. `--stream live` instead shows the conversation as it happens: a `user` row as soon as the words are transcribed, and a `spokesperson_generated` row with cursor `0` as soon as a reply is generated, before it is spoken. The `spokesperson` or `spokesperson_interrupted` row after playback carries the reply's cursor and what was heard. A conventional call has no handoffs, so it -always streams each row. A `--codex` task hears the call as Berd's Expert -does: a batch after every Spokesperson reply, and at each handoff. The standalone host intentionally uses only the default devices. It +always streams each row. A `--codex` task gets the same batches as `--stream`. The standalone host intentionally uses only the default devices. It suppresses recognition during playback when the default output identifies itself as built-in speakers, preventing the call from transcribing its own speech; headphone-like outputs retain barge-in. diff --git a/src-tauri/crates/berd-call/src/cli_help.rs b/src-tauri/crates/berd-call/src/cli_help.rs index dabffbff0..6be273470 100644 --- a/src-tauri/crates/berd-call/src/cli_help.rs +++ b/src-tauri/crates/berd-call/src/cli_help.rs @@ -34,18 +34,17 @@ Run `berd-call help ` for command-specific usage."#; const START_HELP: &str = r#"Run a local voice call backed by the shared session runtime. Usage: - berd-call start [--port PORT] [--stream [handoffs|live] | --codex] [--non-blocking] + berd-call start [--port PORT] [--stream [live] | --codex] [--non-blocking] [--no-menu-bar] --voice NAME --language BCP47 [session options] The foreground process owns the default microphone and audio output. Session options are the same as `berd-call session`, except the PCM descriptor is owned internally. `--stream` prints cursor, role, and text as TSV. In -Expert-Spokesperson mode it prints nothing until the Spokesperson hands off, -then everything since the last handoff; `--stream live` prints each row as it -happens. `--codex` delivers a batch to the calling Codex task after every -Spokesperson reply and at each handoff, as Berd's Expert hears the call -(requires CODEX_THREAD_ID). A +Expert-Spokesperson mode it prints a batch after every Spokesperson reply and +at each handoff, as Berd's Expert hears the call; `--stream live` prints each +row as it happens. `--codex` delivers the same batches as `--stream` to the +calling Codex task (requires CODEX_THREAD_ID). A menu-bar item controls speech rate, input muting, and ending the call; pass `--no-menu-bar` to omit it. diff --git a/src-tauri/crates/berd-call/src/host_session.rs b/src-tauri/crates/berd-call/src/host_session.rs index a039ef1e0..dd0dfe108 100644 --- a/src-tauri/crates/berd-call/src/host_session.rs +++ b/src-tauri/crates/berd-call/src/host_session.rs @@ -834,8 +834,6 @@ struct SessionActor { expert_spokesperson: bool, // User rows already streamed when transcribed, left out of the Expert batch. transcribed: HashSet, - // Rows a handoffs stream or Codex task gets only when the Spokesperson hands off. - held: Vec, stopping: bool, restart: Option, muted: Arc, @@ -898,7 +896,6 @@ impl SessionActor { transcript, expert_spokesperson, transcribed: HashSet::new(), - held: Vec::new(), stopping: false, restart: None, muted: Arc::new(AtomicBool::new(false)), @@ -1310,12 +1307,8 @@ impl SessionActor { self.transcript .record(None, "spokesperson_generated", None, &text)? } - SessionMessage::ExpertDelivery { - events, - handoff_ids, - .. - } if self.expert_spokesperson => { - self.stream_expert_delivery(events, !handoff_ids.is_empty())? + SessionMessage::ExpertDelivery { events, .. } if self.expert_spokesperson => { + self.stream_expert_delivery(&events)? } SessionMessage::Pending { id, utterances } if self.pending_prepare_id() == Some(id) => { self.respond_speak(Ok(json!({"spoke":false,"utterances":utterances}))); @@ -1431,14 +1424,9 @@ impl SessionActor { fn stream_expert_delivery( &mut self, - events: Vec, - handoff: bool, + events: &[RealtimeExpertDeliveryEvent], ) -> Result<(), String> { - self.held.extend(events); - if !handoff && matches!(*self.transcript, Transcript::Stdout { live: false }) { - return Ok(()); - } - for event in unstreamed(&std::mem::take(&mut self.held), &mut self.transcribed) { + for event in unstreamed(events, &mut self.transcribed) { self.transcript.record( Some(event.cursor), expert_delivery_role_name(event.role), @@ -2735,43 +2723,6 @@ mod tests { assert_eq!(transcribed, HashSet::from([3])); } - #[test] - fn handoffs_stream_holds_rows_until_a_handoff() { - let mut child = Command::new("/bin/cat") - .stdin(Stdio::piped()) - .stdout(Stdio::null()) - .spawn() - .unwrap(); - let writer = Arc::new(Mutex::new(child.stdin.take().unwrap())); - let (_event_tx, events) = mpsc::sync_channel(1); - let (_command_tx, commands) = mpsc::sync_channel(1); - let (audio, _audio_rx) = mpsc::sync_channel(1); - let mut actor = test_actor(writer, events, commands, audio); - let event = |cursor, role| RealtimeExpertDeliveryEvent { - cursor, - role, - text: format!("event {cursor}"), - handoff_id: None, - }; - actor - .stream_expert_delivery( - vec![ - event(1, RealtimeExpertDeliveryRole::User), - event(2, RealtimeExpertDeliveryRole::Spokesperson), - ], - false, - ) - .unwrap(); - assert_eq!(actor.held.len(), 2); - actor - .stream_expert_delivery(vec![event(3, RealtimeExpertDeliveryRole::Handoff)], true) - .unwrap(); - assert!(actor.held.is_empty()); - drop(actor); - let _ = child.kill(); - let _ = child.wait(); - } - #[test] fn microphone_teardown_reports_a_blocked_forwarder() { let (_release, blocked) = mpsc::sync_channel::<()>(1); diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 330560db1..3819280b0 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -823,9 +823,8 @@ fn parse_start_args(args: &[String]) -> Result { transcript = if flag == "--codex" { TranscriptDestination::Codex } else { - let mode = args.get(index).map(String::as_str); - let live = mode == Some("live"); - if matches!(mode, Some("live" | "handoffs")) { + let live = args.get(index).map(String::as_str) == Some("live"); + if live { index += 1; } TranscriptDestination::Stdout { live } @@ -12436,7 +12435,6 @@ mod tests { assert!(parsed.non_blocking); for (flags, live) in [ (&["--stream"][..], false), - (&["--stream", "handoffs"], false), (&["--stream", "live", "--non-blocking"], true), ] { let parsed = start(flags).unwrap(); From b205cdcab707f9475c5f3fbc35b64c167473241f Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 08:03:49 -0400 Subject: [PATCH 19/49] berd-call: hold no handoff open when the Expert may speak for itself With --allow-direct-agent-speech the Spokesperson still hands off and the Expert still hears each handoff, but nothing tracks it: no reminders, no renewal or recovery held back, no note for a replacement session, and nothing for --resolves to name. A reply is said once it acknowledges the latest input, as before. Calls without the flag keep Berd's handoff rules. Co-Authored-By: Claude Opus 5.5 Signed-off-by: John Tennant --- src-tauri/crates/berd-call/PROTOCOL.md | 11 +- src-tauri/crates/berd-call/README.md | 2 +- src-tauri/crates/berd-call/src/cli_help.rs | 4 +- src-tauri/crates/berd-call/src/main.rs | 42 +++---- .../berd-call/src/openai_realtime_protocol.rs | 106 ++++++------------ .../berd-call/tests/session_protocol.rs | 52 +++------ 6 files changed, 72 insertions(+), 145 deletions(-) diff --git a/src-tauri/crates/berd-call/PROTOCOL.md b/src-tauri/crates/berd-call/PROTOCOL.md index 9e10acd4a..1ff842ecf 100644 --- a/src-tauri/crates/berd-call/PROTOCOL.md +++ b/src-tauri/crates/berd-call/PROTOCOL.md @@ -221,15 +221,14 @@ backend, keeping its other settings; `language` changes a Siri voice's locale too. A voice the backend doesn't offer, or a voice for a call whose replies are spoken by the Spokesperson, is `not_admitted(invalid_voice)`. In a call whose replies have their own voice, a reply needs the same `acknowledgement` -as a directive, and answers every open handoff once admitted, so -`resolved_handoff_ids` is ignored. With `relay: true` (what +as a directive. With `relay: true` (what `berd-call message-spokesperson` sends), the reply is a directive in any Expert-Spokesperson call, even one whose replies have their own voice: the Spokesperson says it, on the same `acknowledgement` and `resolved_handoff_ids` -terms. Open -handoffs in a call whose replies have their own voice don't hold -back a renewal or recovery of the Spokesperson's session; the replacement is -told what they asked. +terms. A call whose replies have their own voice holds no handoff open: the +Expert still hears each one, but `unresolved_handoff_ids` stays empty, no +reminder or renewal waits on them, and naming one in `resolved_handoff_ids` is +`not_admitted(invalid_handoff)`. The parent cannot author speaking state or finalized input; those are derived only from PCM by the child runtime. diff --git a/src-tauri/crates/berd-call/README.md b/src-tauri/crates/berd-call/README.md index 4fef37df9..75d8628c6 100644 --- a/src-tauri/crates/berd-call/README.md +++ b/src-tauri/crates/berd-call/README.md @@ -149,7 +149,7 @@ berd-call session --tts-backend openai --tts-url https://proxy.example/v1/audio/ berd-call session --stt-backend openai --stt-url wss://proxy.example/v1/realtime?intent=transcription ``` -In an Expert–Spokesperson call, the Expert reaches you through the Spokesperson: `berd-call message-spokesperson --re CURSOR [--resolves HANDOFF_ID]... TEXT` has it tell you in its own voice. Like the directives of Berd's Expert, the message must acknowledge the latest input (the Spokesperson's own speech doesn't count), or it comes back `spoke:false` with what was missed, and a handoff it names settles once the Spokesperson has said it. `speak` is refused unless the call was started with `--allow-direct-agent-speech`. With it, the Expert can also speak for itself: berd-call plays each `speak` reply through the configured TTS backend, taking turns with the Spokesperson's audio, and your speech interrupts either of them. Afterwards the Spokesperson gets a silent note of what you heard, so it can follow up without repeating it. A reply has to acknowledge everything you've said, as `speak --re` always does, and it settles every open handoff once admitted, so `--resolves` isn't needed. Open handoffs no longer hold back renewing or recovering the Spokesperson's session; the new session is told what they asked, so it doesn't hand them off again. `--voice`, `--rate`, `status` and `settings --tts` describe and change the Expert's voice. The Spokesperson keeps its own, shown as `session.spokesperson`: choose it with `--spokesperson-voice NAME` and `--spokesperson-rate 0.25–1.5`, at `start` or live with `settings`, and the menu bar offers both voices. Saved settings keep the Spokesperson's voice for the next call whose replies have their own, but not `--allow-direct-agent-speech`, so pass that on each `start`. +In an Expert–Spokesperson call, the Expert reaches you through the Spokesperson: `berd-call message-spokesperson --re CURSOR [--resolves HANDOFF_ID]... TEXT` has it tell you in its own voice. Like the directives of Berd's Expert, the message must acknowledge the latest input (the Spokesperson's own speech doesn't count), or it comes back `spoke:false` with what was missed, and a handoff it names settles once the Spokesperson has said it. `speak` is refused unless the call was started with `--allow-direct-agent-speech`. With it, the Expert can also speak for itself: berd-call plays each `speak` reply through the configured TTS backend, taking turns with the Spokesperson's audio, and your speech interrupts either of them. Afterwards the Spokesperson gets a silent note of what you heard, so it can follow up without repeating it. A reply has to acknowledge everything you've said, as `speak --re` always does. Such a call holds no handoff open: each still reaches the Expert as a `handoff` row, but nothing waits for an answer, reminds the Expert, or holds back renewing or recovering the Spokesperson's session, and there is nothing for `--resolves` to name. `--voice`, `--rate`, `status` and `settings --tts` describe and change the Expert's voice. The Spokesperson keeps its own, shown as `session.spokesperson`: choose it with `--spokesperson-voice NAME` and `--spokesperson-rate 0.25–1.5`, at `start` or live with `settings`, and the menu bar offers both voices. Saved settings keep the Spokesperson's voice for the next call whose replies have their own, but not `--allow-direct-agent-speech`, so pass that on each `start`. The three URL flags take full endpoints and override only their matching service for that call. Expert–Spokesperson uses `--realtime-url` for both input and output; `--stt-url` applies only to conventional mode, and `--tts-url` also applies to the Expert's own voice. Without an override, each service uses its OpenAI endpoint (or its existing environment override). `OPENAI_API_KEY` supplies the CLI credential; the desktop app stores keys separately by endpoint URL in Keychain. diff --git a/src-tauri/crates/berd-call/src/cli_help.rs b/src-tauri/crates/berd-call/src/cli_help.rs index 6be273470..5c2ffc270 100644 --- a/src-tauri/crates/berd-call/src/cli_help.rs +++ b/src-tauri/crates/berd-call/src/cli_help.rs @@ -69,8 +69,8 @@ this text in another voice of the call's reply backend without changing its save voice, so several agents can share one call in distinct voices. An Expert-Spokesperson call refuses speak unless started with --allow-direct-agent-speech; otherwise use message-spokesperson. With it, a -reply acknowledging the latest input is spoken in the Expert's own voice and -answers every open handoff, so --resolves is ignored."#; +reply acknowledging the latest input is spoken in the Expert's own voice, and +handoffs are never held open, so there is nothing for --resolves to name."#; const MESSAGE_SPOKESPERSON_HELP: &str = r#"Message the Spokesperson of the active realtime call. diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 3819280b0..87a125304 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -3270,18 +3270,11 @@ fn spokesperson_voice_update_is_safe( core.semantic_revision(), update.base_revision, snapshot.revision, - handoffs_hold_session(core, expert_tts), + core.has_unresolved_handoff(), work, ) } -/// Whether open handoffs keep the Spokesperson's session from being -/// replaced: only while it says their replies itself. Replies in the -/// Expert's own voice need nothing from it. -fn handoffs_hold_session(core: &RealtimeExpertSpokespersonSession, expert_tts: bool) -> bool { - !expert_tts && core.has_unresolved_handoff() -} - fn unavailable_spokesperson_title(connection_lost: bool, quiescent: bool) -> &'static str { match (connection_lost, quiescent) { (true, false) => "Spokesperson connection was lost during an active turn", @@ -3333,7 +3326,12 @@ fn run_expert_spokesperson_session( let mut next_tts_update_attempt = 0_u64; let stdout = io::stdout(); let mut writer = BufWriter::new(stdout.lock()); + // An Expert that may speak for itself answers handoffs when it likes, so + // nothing holds them open. let mut core = RealtimeExpertSpokespersonSession::new(0, "external"); + if config.expert_tts { + core = core.without_handoff_tracking(); + } let mut runtime: Option = None; let mut runtime_events: Option> = None; let mut runtime_config: Option = None; @@ -3369,7 +3367,7 @@ fn run_expert_spokesperson_session( if turn_gate.lifecycle.renewal_is_due( Instant::now(), work, - handoffs_hold_session(&core, reply_tts.is_some()), + core.has_unresolved_handoff(), pending_voice_update.is_some(), ) { let snapshot = session_tts.as_ref().expect("initialized TTS snapshot"); @@ -3386,7 +3384,7 @@ fn run_expert_spokesperson_session( runtime_config .as_ref() .expect("initialized Spokesperson config"), - core.replacement_transcript(), + core.semantic_transcript(), VoiceUpdatePurpose::Renewal, )?); } @@ -3482,7 +3480,7 @@ fn run_expert_spokesperson_session( runtime_config .as_ref() .expect("initialized Spokesperson config"), - core.replacement_transcript(), + core.semantic_transcript(), ) { Ok(update) => Some(update), Err(message) => { @@ -3987,7 +3985,7 @@ fn run_expert_spokesperson_session( &cancelled_directives, ); let quiescent = turn_gate.lifecycle.settings_are_quiescent(work) - && !handoffs_hold_session(&core, reply_tts.is_some()); + && !core.has_unresolved_handoff(); if !quiescent { write_protocol_fatal( &mut writer, @@ -4002,7 +4000,7 @@ fn run_expert_spokesperson_session( .as_ref() .map(VoiceUpdateTransaction::purpose), work, - handoffs_hold_session(&core, reply_tts.is_some()), + core.has_unresolved_handoff(), ); let start_recovery = match recovery_action { RealtimeSessionLossAction::ContinuePendingRecovery => { @@ -4044,7 +4042,7 @@ fn run_expert_spokesperson_session( runtime_config .as_ref() .expect("initialized Spokesperson config"), - core.replacement_transcript(), + core.semantic_transcript(), VoiceUpdatePurpose::SessionRecovery { cause: message.clone(), }, @@ -5233,13 +5231,10 @@ fn submit_expert_prepare( next_speech_id: &mut u64, writer: &mut impl Write, ) -> Result<(), String> { - // A reply in the Expert's own voice answers every open handoff, so the - // ones it names don't matter. A message to the Spokesperson is a directive. let own_voice = expert_tts && !request.relay; - if !own_voice - && !core - .unknown_handoff_ids(&request.resolved_handoff_ids) - .is_empty() + if !core + .unknown_handoff_ids(&request.resolved_handoff_ids) + .is_empty() { return write_message( writer, @@ -5249,12 +5244,7 @@ fn submit_expert_prepare( }, ); } - let outcome = if own_voice { - core.prepare_expert_reply(request.acknowledgement, request.text) - } else { - core.prepare_expert_directive(request.acknowledgement, request.text) - }; - match outcome { + match core.prepare_expert_directive(request.acknowledgement, request.text) { ExpertDirectiveOutcome::Pending(events) => write_message( writer, &SessionMessage::Pending { diff --git a/src-tauri/crates/berd-call/src/openai_realtime_protocol.rs b/src-tauri/crates/berd-call/src/openai_realtime_protocol.rs index d79829184..7a68cef77 100644 --- a/src-tauri/crates/berd-call/src/openai_realtime_protocol.rs +++ b/src-tauri/crates/berd-call/src/openai_realtime_protocol.rs @@ -1287,6 +1287,7 @@ pub struct RealtimeExpertSpokespersonSession { responses: RealtimeResponseCoordinator, conversation: ExpertSpokespersonCore, open_handoffs: HashMap, + tracks_handoffs: bool, call_scope: String, pending_expert_events: Vec, } @@ -1387,11 +1388,21 @@ impl RealtimeExpertSpokespersonSession { responses: RealtimeResponseCoordinator::default(), conversation: ExpertSpokespersonCore::new(initial_cursor), open_handoffs: HashMap::new(), + tracks_handoffs: true, call_scope: call_scope.into(), pending_expert_events: Vec::new(), } } + /// A session whose handoffs reach the Expert but are never held open: + /// nothing waits on them, so nothing reminds, resolves or keeps them. + pub fn without_handoff_tracking(self) -> Self { + Self { + tracks_handoffs: false, + ..self + } + } + pub fn handle_provider_event( &mut self, event: &Value, @@ -1587,21 +1598,6 @@ impl RealtimeExpertSpokespersonSession { }) } - /// Prepares a reply in the Expert's own voice: like a directive, it waits - /// for whatever the Expert hasn't acknowledged, and once admitted it - /// answers every open handoff. - pub fn prepare_expert_reply( - &mut self, - acknowledgement: Option, - message: String, - ) -> ExpertDirectiveOutcome { - let outcome = self.prepare_expert_directive(acknowledgement, message); - if matches!(outcome, ExpertDirectiveOutcome::Accepted { .. }) { - self.open_handoffs.clear(); - } - outcome - } - pub fn record_live_event_with_delivery( &mut self, event: LiveSideEvent, @@ -1695,25 +1691,6 @@ impl RealtimeExpertSpokespersonSession { self.conversation.semantic_transcript() } - /// The transcript a replacement Spokesperson session starts from: the - /// conversation so far, and the handoffs still waiting for a reply, so it - /// doesn't hand them off again. - pub fn replacement_transcript(&self) -> Vec { - let mut turns = self.semantic_transcript(); - let mut open: Vec<_> = self.open_handoffs.iter().collect(); - open.sort_by_key(|(id, _)| *id); - if !open.is_empty() { - turns.push(SemanticTurn::Expert(format!( - "The Expert is still working on these handoffs and will answer in its own voice, so do not hand them off again:\n{}", - open.iter() - .map(|(_, handoff)| format!("- {}", handoff.message)) - .collect::>() - .join("\n") - ))); - } - turns - } - pub fn request_expert_message( &mut self, message: RealtimeExpertMessage, @@ -1760,14 +1737,16 @@ impl RealtimeExpertSpokespersonSession { let handoff_id = require_non_empty(handoff_id, "handoff id")?; let message = require_non_empty(message, "handoff message")?; let expert_message = expert_handoff_message(&handoff_id, cursor, &message); - self.open_handoffs.insert( - handoff_id, - RealtimeOpenHandoff { - message, - reminder_attempts: 0, - resolving: false, - }, - ); + if self.tracks_handoffs { + self.open_handoffs.insert( + handoff_id, + RealtimeOpenHandoff { + message, + reminder_attempts: 0, + resolving: false, + }, + ); + } Ok(expert_message) } @@ -3118,40 +3097,21 @@ mod tests { } #[test] - fn expert_reply_answers_every_handoff_once_current() { - let mut session = RealtimeExpertSpokespersonSession::new(0, "test-call"); - session - .register_handoff("handoff-1", 1, "First request") - .unwrap(); - session - .register_handoff("handoff-2", 1, "Second request") - .unwrap(); - let note = SemanticTurn::Expert( - "The Expert is still working on these handoffs and will answer in its own voice, so do not hand them off again:\n- First request\n- Second request".into(), - ); - assert_eq!(session.replacement_transcript(), [note]); - session - .record_live_event_with_delivery(LiveSideEvent::UserTranscript { - text: "And one more thing".into(), + fn untracked_handoffs_reach_the_expert_and_stay_closed() { + let mut session = + RealtimeExpertSpokespersonSession::new(0, "test-call").without_handoff_tracking(); + let (_, delivery) = session + .record_live_event_with_delivery(LiveSideEvent::Handoff { + call_id: "call-1".into(), + message: "First request".into(), }) .unwrap(); - - // A reply written before the user's latest words waits for them, and - // answers no handoff until it is admitted. - assert!(matches!( - session.prepare_expert_reply(None, "Stale".into()), - ExpertDirectiveOutcome::Pending(_) - )); - assert!(session.has_unresolved_handoff()); - assert!(matches!( - session.prepare_expert_reply(Some(1), "The answer".into()), - ExpertDirectiveOutcome::Accepted { - confirmed_token: 1, - .. - } - )); + assert_eq!(delivery.unwrap().handoff_ids, ["handoff-test-call-1"]); assert!(!session.has_unresolved_handoff()); - assert_eq!(session.replacement_transcript(), []); + assert_eq!( + session.complete_expert_turn(&[], 3), + RealtimeHandoffReminder::None + ); } #[test] diff --git a/src-tauri/crates/berd-call/tests/session_protocol.rs b/src-tauri/crates/berd-call/tests/session_protocol.rs index 930d49b53..8b9f41860 100644 --- a/src-tauri/crates/berd-call/tests/session_protocol.rs +++ b/src-tauri/crates/berd-call/tests/session_protocol.rs @@ -3033,7 +3033,6 @@ fn expert_voice_speaks_replies_itself_and_tells_the_spokesperson_what_was_heard( "id":2, "acknowledgement":1, "text":"I checked the repository.", - "resolved_handoff_ids":["handoff-external-1"], "voice":"coral" })); let admitted = session.recv(Duration::from_secs(2)); @@ -3285,12 +3284,6 @@ fn an_expert_reply_that_fails_unheard_leaves_its_handoffs_open() { #[test] fn a_message_to_the_spokesperson_is_a_directive_it_says() { - message_the_spokesperson(false); - message_the_spokesperson(true); -} - -fn message_the_spokesperson(expert_voice: bool) { - let (speech_url, speech_requests) = spawn_speech_server(); let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); let (relay_tx, relay_rx) = mpsc::sync_channel(1); let server = std::thread::spawn(move || { @@ -3335,12 +3328,7 @@ fn message_the_spokesperson(expert_voice: bool) { } }); }); - let endpoint = endpoint_rx.recv().unwrap(); - let own = ["--allow-direct-agent-speech", "--tts-url", &speech_url]; - let mut session = ExpertSpokespersonTestSession::start_with_args( - endpoint, - if expert_voice { &own } else { &[] }, - ); + let mut session = ExpertSpokespersonTestSession::start(endpoint_rx.recv().unwrap()); let handoffs = loop { session.send(json!({"type":"query_state","id":1,"after":0})); let state = loop { @@ -3370,7 +3358,7 @@ fn message_the_spokesperson(expert_voice: bool) { "relay":true }) }; - let mut reply = |session: &mut ExpertSpokespersonTestSession, request| { + let reply = |session: &mut ExpertSpokespersonTestSession, request| { session.send(request); loop { let message = session.recv(Duration::from_secs(2)); @@ -3403,18 +3391,15 @@ fn message_the_spokesperson(expert_voice: bool) { json!([&handoffs[1]]), "{state}" ); - assert!(speech_requests - .recv_timeout(Duration::from_millis(300)) - .is_err()); session.shutdown(); server.join().unwrap(); } #[test] -fn expert_voice_handoffs_survive_provider_expiry_and_a_current_reply_answers_them() { +fn handoffs_hold_nothing_open_when_the_expert_may_speak() { let (speech_url, speech_requests) = spawn_speech_server(); let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); - let (note_tx, note_rx) = mpsc::sync_channel(1); + let (renewed_tx, renewed_rx) = mpsc::sync_channel(1); let server = std::thread::spawn(move || { tokio::runtime::Builder::new_current_thread() .enable_all() @@ -3461,23 +3446,20 @@ fn expert_voice_handoffs_survive_provider_expiry_and_a_current_reply_answers_the ) .await; - // The open handoff doesn't stop the replacement, which is told of it. + // The handoff was never held open, so the call renews. let (stream, _) = listener.accept().await.unwrap(); let mut replacement = accept_async(stream).await.unwrap(); let update = receive_realtime_json(&mut replacement).await; acknowledge_realtime_session(&mut replacement, &update).await; + renewed_tx.send(()).unwrap(); while let Some(Ok(Message::Text(text))) = replacement.next().await { let seed: Value = serde_json::from_str(&text).unwrap(); - if seed["type"] != "conversation.item.create" { - continue; - } - send_realtime_json( - &mut replacement, - json!({"type":"conversation.item.created","item":{"id":seed["item"]["id"]}}), - ) - .await; - if text.contains("do not hand them off again") { - let _ = note_tx.try_send(seed); + if seed["type"] == "conversation.item.create" { + send_realtime_json( + &mut replacement, + json!({"type":"conversation.item.created","item":{"id":seed["item"]["id"]}}), + ) + .await; } } }); @@ -3487,14 +3469,10 @@ fn expert_voice_handoffs_survive_provider_expiry_and_a_current_reply_answers_the endpoint, &["--allow-direct-agent-speech", "--tts-url", &speech_url], ); - let note = note_rx.recv_timeout(Duration::from_secs(3)).unwrap(); - assert!( - note.to_string().contains("- inspect the repository"), - "{note}" - ); + renewed_rx.recv_timeout(Duration::from_secs(3)).unwrap(); - // A reply written before everything since waits for it, and once it - // acknowledges the latest, naming no handoff, it answers them all. + // A reply written before everything since waits for it, and is said once + // it acknowledges the latest. session.send(json!({ "type":"prepare_speak", "id":2, From f822edc5fc0fe8614c2827b3c5579e2e99a40d55 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 08:18:34 -0400 Subject: [PATCH 20/49] berd-call: find the PCM descriptor after --allow-direct-agent-speech The voice session read its arguments in pairs to find --pcm-output-fd, which the host appends last. The valueless --allow-direct-agent-speech shifted the pairs, so every call started with it failed with "voice session event stream closed during startup". Co-Authored-By: Claude Opus 5.5 Signed-off-by: John Tennant --- src-tauri/crates/berd-call/src/main.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 87a125304..655f7068b 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -6038,6 +6038,10 @@ fn parse_pcm_output_fd(args: &[String]) -> Result { let mut value = None; let mut index = 2; while index < args.len() { + if args[index] == DIRECT_AGENT_SPEECH { + index += 1; + continue; + } if args[index] == "--pcm-output-fd" { if value.is_some() { return Err("--pcm-output-fd may be provided only once".into()); @@ -11413,6 +11417,18 @@ mod tests { assert!( parse_pcm_output_fd(&args(&["berd-call", "session", "--pcm-output-fd", "2"])).is_err() ); + // As the host spawns it, after a flag that takes no value. + assert_eq!( + parse_pcm_output_fd(&args(&[ + "berd-call", + "session", + DIRECT_AGENT_SPEECH, + "--pcm-output-fd", + "9" + ])) + .unwrap(), + 9 + ); assert!(parse_pcm_output_fd(&args(&[ "berd-call", "session", From a3ebee4805daf332990680a5a3c6494894f815dc Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 08:25:14 -0400 Subject: [PATCH 21/49] berd-call: retry a Spokesperson voice change the conversation overtook A Spokesperson voice or rate change builds a new realtime session, and if anyone spoke before it was ready, the change was rejected with "Spokesperson conversation changed during voice replacement", so a menu-bar change made mid-conversation never took. In a berd-call start call it is now tried again once the call is idle, with the conversation as it then stands, and reports one result when it applies. The Berd app's handling is unchanged. Co-Authored-By: Claude Opus 5.5 Signed-off-by: John Tennant --- src-tauri/crates/berd-call/PROTOCOL.md | 5 ++ src-tauri/crates/berd-call/src/main.rs | 53 +++++++++++++++-- .../berd-call/tests/session_protocol.rs | 59 +++++++++++++------ 3 files changed, 96 insertions(+), 21 deletions(-) diff --git a/src-tauri/crates/berd-call/PROTOCOL.md b/src-tauri/crates/berd-call/PROTOCOL.md index 1ff842ecf..d38454cab 100644 --- a/src-tauri/crates/berd-call/PROTOCOL.md +++ b/src-tauri/crates/berd-call/PROTOCOL.md @@ -263,6 +263,11 @@ linearization point. A speech reservation holds a configuration lease: speech admitted before the response retains its old backend/settings, while later admission receives the new revision. Pocket's public model identifier cannot be changed without selecting and validating another bundle at process startup. +In a `berd-call start` realtime call, a Spokesperson voice or rate change that +the conversation overtakes while its new session is being built is not +rejected: the candidate is discarded and the change is tried again, with the +conversation as it then stands, once the call is idle. Its one result comes +when it finally applies or fails. `set_input_during_tts` changes the policy for later admissions. Its expected revision must be positive. The correlated result is always authoritative: diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 655f7068b..74f937c19 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -2796,6 +2796,32 @@ fn rollback_spokesperson_voice_update( } } +fn is_settings_update(pending: &Option) -> bool { + pending + .as_ref() + .is_some_and(|update| matches!(update.purpose(), VoiceUpdatePurpose::Settings)) +} + +/// Puts back a settings change the conversation overtook while its session +/// was being built, to be tried again, with the conversation as it then +/// stands, once the call is idle. +fn requeue_spokesperson_voice_update( + pending: &mut Option, + old_runtime: &OpenAiSpokespersonRuntime, + queue: &mut VoiceUpdateQueue, +) -> Result<(), String> { + let update = pending.take().expect("matched settings update"); + let request = PendingSpokespersonSettingsUpdate { + id: update.id, + base_revision: update.base_revision, + settings: update.settings.clone(), + }; + update.abort(old_runtime)?; + queue + .try_enqueue(false, request) + .map_err(|_| "a Spokesperson settings update was already queued".to_string()) +} + fn activate_spokesperson_voice_update( pending: &mut Option, runtime: &mut Option, @@ -3422,6 +3448,15 @@ fn run_expert_spokesperson_session( )?; turn_gate.lifecycle.session_started(Instant::now()); } + VoiceUpdateAction::Reject(_) + if !safe && is_settings_update(&pending_voice_update) => + { + requeue_spokesperson_voice_update( + &mut pending_voice_update, + runtime.as_ref().expect("initialized runtime"), + &mut queued_tts_settings, + )?; + } VoiceUpdateAction::Reject(message) => { let expiry_cause = pending_voice_update.as_ref().and_then(|update| { if let VoiceUpdatePurpose::SessionRecovery { cause } = update.purpose() { @@ -3850,8 +3885,8 @@ fn run_expert_spokesperson_session( } } SpokespersonEvent::InputCutoverFinished { request_id, result } => { - let action = pending_voice_update.as_ref().map_or( - VoiceBarrierAction::Ignore, + let (action, safe) = pending_voice_update.as_ref().map_or( + (VoiceBarrierAction::Ignore, true), |update| { let work = spokesperson_host_work( &turn_gate, @@ -3868,12 +3903,13 @@ fn run_expert_spokesperson_session( &turn_gate, work, ); - match update.next_action(Instant::now(), safe) { + let action = match update.next_action(Instant::now(), safe) { VoiceUpdateAction::Reject(message) => { VoiceBarrierAction::Reject(message) } _ => update.finish_barrier(request_id, result, safe), - } + }; + (action, safe) }, ); if action == VoiceBarrierAction::Activate { @@ -3886,6 +3922,15 @@ fn run_expert_spokesperson_session( &mut writer, )?; turn_gate.lifecycle.session_started(Instant::now()); + } else if matches!(action, VoiceBarrierAction::Reject(_)) + && !safe + && is_settings_update(&pending_voice_update) + { + requeue_spokesperson_voice_update( + &mut pending_voice_update, + runtime.as_ref().expect("initialized runtime"), + &mut queued_tts_settings, + )?; } else if let VoiceBarrierAction::Reject(message) = action { rollback_spokesperson_voice_update( &mut pending_voice_update, diff --git a/src-tauri/crates/berd-call/tests/session_protocol.rs b/src-tauri/crates/berd-call/tests/session_protocol.rs index 8b9f41860..8993d73bd 100644 --- a/src-tauri/crates/berd-call/tests/session_protocol.rs +++ b/src-tauri/crates/berd-call/tests/session_protocol.rs @@ -1590,7 +1590,7 @@ fn expert_spokesperson_voice_change_swaps_atomically_and_flushes_gated_pcm_once( } #[test] -fn user_activity_during_voice_cutover_rolls_back_and_flushes_gated_pcm_once() { +fn user_activity_during_voice_cutover_rolls_back_then_retries_once_idle() { let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); let (clear_seen_tx, clear_seen_rx) = mpsc::sync_channel(1); let (release_activity_tx, release_activity_rx) = mpsc::sync_channel(1); @@ -1664,6 +1664,29 @@ fn user_activity_during_voice_cutover_rolls_back_and_flushes_gated_pcm_once() { .is_err()); rollback_seen_tx.send(()).unwrap(); let _ = candidate.next().await; + + // Once the conversation is idle, the change is tried again, with the user's words. + let (retry_stream, _) = listener.accept().await.unwrap(); + let mut retry = accept_async(retry_stream).await.unwrap(); + let retry_update = receive_realtime_json(&mut retry).await; + assert_eq!(retry_update["session"]["audio"]["output"]["voice"], "new-voice"); + acknowledge_realtime_session(&mut retry, &retry_update).await; + let seeds = tokio::spawn(async move { + while let Some(Ok(Message::Text(text))) = retry.next().await { + let seed: Value = serde_json::from_str(&text).unwrap(); + if seed["type"] == "conversation.item.create" { + send_realtime_json( + &mut retry, + json!({"type":"conversation.item.created","item":{"id":seed["item"]["id"]}}), + ) + .await; + } + } + }); + while receive_realtime_json(&mut old).await["type"] != "input_audio_buffer.clear" {} + send_realtime_json(&mut old, json!({"type":"input_audio_buffer.cleared"})).await; + let _ = old.next().await; + seeds.await.unwrap(); }); }); let endpoint = endpoint_rx.recv().unwrap(); @@ -1682,18 +1705,19 @@ fn user_activity_during_voice_cutover_rolls_back_and_flushes_gated_pcm_once() { let deadline = std::time::Instant::now() + Duration::from_secs(3); let mut saw_user = false; - let mut saw_rejection = false; - while !(saw_user && saw_rejection) { + let mut applied = false; + while !(saw_user && applied) { let remaining = deadline .checked_duration_since(std::time::Instant::now()) - .expect("rollback did not publish user input and settings rejection promptly"); + .expect("the retried change did not apply promptly after the user's input"); let message = session.recv(remaining); saw_user |= message["type"] == "live_event"; if message["type"] == "tts_settings_result" { - assert_eq!(message["outcome"], "rejected"); - assert_eq!(message["snapshot"]["revision"], 1); - assert_eq!(message["snapshot"]["voice"], "old-voice"); - saw_rejection = true; + assert!(saw_user, "{message}"); + assert_eq!(message["outcome"], "applied", "{message}"); + assert_eq!(message["snapshot"]["revision"], 2); + assert_eq!(message["snapshot"]["voice"], "new-voice"); + applied = true; } } rollback_seen_rx @@ -2278,6 +2302,7 @@ fn voice_cutover_timeout_is_terminal_and_never_reports_a_settings_result() { fn live_event_promptly_aborts_a_stalled_voice_candidate_and_replies_on_old_runtime() { let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); let (candidate_seen_tx, candidate_seen_rx) = mpsc::sync_channel(1); + let (retry_seen_tx, retry_seen_rx) = mpsc::sync_channel(1); let server = std::thread::spawn(move || { tokio::runtime::Builder::new_current_thread() .enable_all() @@ -2324,6 +2349,9 @@ fn live_event_promptly_aborts_a_stalled_voice_candidate_and_replies_on_old_runti .expect("stalled candidate should be cancelled promptly") .unwrap(); assert!(!pending_handshake.is_empty()); + // The change is tried again once the user's turn is over. + let (_retry, _) = listener.accept().await.unwrap(); + retry_seen_tx.send(()).unwrap(); let _ = old.next().await; }); }); @@ -2338,21 +2366,18 @@ fn live_event_promptly_aborts_a_stalled_voice_candidate_and_replies_on_old_runti candidate_seen_rx .recv_timeout(Duration::from_secs(2)) .unwrap(); - let mut saw_user = false; - let mut saw_rejection = false; let deadline = std::time::Instant::now() + Duration::from_secs(2); - while !(saw_user && saw_rejection) { + loop { let remaining = deadline .checked_duration_since(std::time::Instant::now()) - .expect("session did not report user input and voice rejection promptly"); + .expect("session did not report user input promptly"); let message = session.recv(remaining); - saw_user |= message["type"] == "live_event"; - if message["type"] == "tts_settings_result" { - assert_eq!(message["outcome"], "rejected"); - assert_eq!(message["snapshot"]["voice"], "old-voice"); - saw_rejection = true; + assert_ne!(message["type"], "tts_settings_result", "{message}"); + if message["type"] == "live_event" { + break; } } + retry_seen_rx.recv_timeout(Duration::from_secs(2)).unwrap(); session.shutdown(); server.join().unwrap(); } From 963deb74bb2eb2a01e9d2597c8de15c8c55c4962 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 08:43:26 -0400 Subject: [PATCH 22/49] berd-call: gather a lead before playing the Spokesperson's audio OpenAI sends the Spokesperson's audio about as fast as it plays. Playback sent it to the host only in whole 170 ms chunks and started on the first one, so the speaker often ran dry before the next chunk was complete, and any late delta left an audible gap. Gather 300 ms of each reply before it plays, then send each delta as it arrives. A test plays a reply arriving in real time with one 150 ms stall: whole chunks leave a 109 ms silence; the lead plays through. Co-Authored-By: Claude Opus 5.5 Signed-off-by: John Tennant --- src-tauri/crates/berd-call/src/main.rs | 26 ++-- .../crates/berd-call/src/session_audio.rs | 135 +++++++++++++++++- 2 files changed, 147 insertions(+), 14 deletions(-) diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 74f937c19..6d60e8451 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -5209,6 +5209,9 @@ fn finish_expert_reply( Ok(()) } +/// The Spokesperson's audio gathered before each reply plays, 300 ms at 24 kHz: OpenAI sends it about as fast as it plays, so this covers a delta that comes late. +const SPOKESPERSON_AUDIO_LEAD_FRAMES: usize = 7_200; + fn spawn_live_playback( response_id: String, speech_id: u64, @@ -5218,16 +5221,19 @@ fn spawn_live_playback( completed: mpsc::Sender, ) -> Result { let active = Arc::new(AtomicBool::new(true)); - let output = Arc::new(RemotePcmAudioOutput::new( - speech_id, - TtsPcmSpec { - sample_rate: 24_000, - playback_rate: 1.0, - }, - transport, - Arc::clone(&active), - control, - )?); + let output = Arc::new( + RemotePcmAudioOutput::new( + speech_id, + TtsPcmSpec { + sample_rate: 24_000, + playback_rate: 1.0, + }, + transport, + Arc::clone(&active), + control, + )? + .streaming_after(SPOKESPERSON_AUDIO_LEAD_FRAMES), + ); let (sender, receiver) = mpsc::sync_channel::(64); let worker_output = Arc::clone(&output); let worker_response_id = response_id.clone(); diff --git a/src-tauri/crates/berd-call/src/session_audio.rs b/src-tauri/crates/berd-call/src/session_audio.rs index 0368d6a9d..e4fc2543c 100644 --- a/src-tauri/crates/berd-call/src/session_audio.rs +++ b/src-tauri/crates/berd-call/src/session_audio.rs @@ -2,7 +2,7 @@ use std::collections::VecDeque; use std::fs::File; use std::io; use std::os::fd::{AsRawFd, FromRawFd, RawFd}; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::mpsc; use std::sync::{Arc, Condvar, Mutex}; use std::time::{Duration, Instant}; @@ -190,6 +190,8 @@ pub struct RemotePcmAudioOutput { operation_timeout: Duration, minimum_accepted_runway_frames: u64, pending_samples: Mutex>, + /// Frames to gather before streaming each write as it comes, or `None` to send whole chunks. + stream_after: Option, state: Mutex, changed: Condvar, } @@ -236,6 +238,7 @@ impl RemotePcmAudioOutput { operation_timeout, minimum_accepted_runway_frames: accepted_audio_runway_frames(spec), pending_samples: Mutex::new(Vec::with_capacity(MAX_AUDIO_CHUNK_FRAMES)), + stream_after: None, state: Mutex::new(State { phase: Phase::New, begin_accepted: false, @@ -257,6 +260,12 @@ impl RemotePcmAudioOutput { }) } + /// Holds back the first `frames` of audio, then sends each write at once, for a source that arrives about as fast as it plays: what is held covers its late arrivals. + pub fn streaming_after(mut self, frames: usize) -> Self { + self.stream_after = Some(AtomicUsize::new(frames)); + self + } + pub fn start(&self) -> Result<(), String> { { let mut state = self.state.lock().expect("remote output state"); @@ -860,11 +869,18 @@ impl PcmAudioOutput for RemotePcmAudioOutput { .lock() .expect("remote pending PCM lock"); pending.extend_from_slice(samples); - let complete_len = pending.len() / MAX_AUDIO_CHUNK_FRAMES * MAX_AUDIO_CHUNK_FRAMES; - let tail = pending.split_off(complete_len); + let ready = match &self.stream_after { + None => pending.len() / MAX_AUDIO_CHUNK_FRAMES * MAX_AUDIO_CHUNK_FRAMES, + Some(held) if pending.len() >= held.load(Ordering::SeqCst) => { + held.store(0, Ordering::SeqCst); + pending.len() + } + Some(_) => 0, + }; + let tail = pending.split_off(ready); std::mem::replace(&mut *pending, tail) }; - for chunk in complete_samples.chunks_exact(MAX_AUDIO_CHUNK_FRAMES) { + for chunk in complete_samples.chunks(MAX_AUDIO_CHUNK_FRAMES) { self.write_chunk(chunk)?; } Ok(()) @@ -921,6 +937,7 @@ mod tests { use std::io::Read; use std::os::fd::IntoRawFd; use std::os::unix::net::UnixStream; + use std::sync::atomic::AtomicU64; use std::sync::Arc; use std::thread; @@ -1579,6 +1596,116 @@ mod tests { worker.join().unwrap().unwrap(); } + fn remote(stream_after: Option) -> (Arc, UnixStream) { + let (child, host) = UnixStream::pair().unwrap(); + let transport = unsafe { AudioPipeTransport::from_raw_fd(child.into_raw_fd()) }.unwrap(); + let spec = TtsPcmSpec { + sample_rate: 24_000, + playback_rate: 1.0, + }; + let output = RemotePcmAudioOutput::new( + 7, + spec, + Arc::new(transport), + Arc::new(AtomicBool::new(true)), + mpsc::channel().0, + ) + .unwrap(); + ( + Arc::new(match stream_after { + Some(frames) => output.streaming_after(frames), + None => output, + }), + host, + ) + } + + /// The longest silence, in milliseconds, between the first sound and the last that a host playing in real time hears from `output`, as each of `deltas` (its frames, and the millisecond it arrives) is written to it. + fn longest_gap( + (output, mut host): (Arc, UnixStream), + deltas: &[(usize, u64)], + ) -> f64 { + start(&output, &mut host); + let now = Arc::new(AtomicU64::new(0)); + let arrivals = Arc::new(Mutex::new(Vec::<(u64, usize)>::new())); + let player = { + let (output, now, arrivals) = + (Arc::clone(&output), Arc::clone(&now), Arc::clone(&arrivals)); + thread::spawn(move || loop { + let (kind, payload) = read_record(&mut host); + if kind == AUDIO_END_KIND { + return; + } + assert_eq!(kind, AUDIO_CHUNK_KIND); + arrivals + .lock() + .unwrap() + .push((now.load(Ordering::SeqCst), (payload.len() - 16) / 4)); + let sequence = u64::from_le_bytes(payload[8..16].try_into().unwrap()); + output + .handle_ack(AudioHostAck::ChunkAccepted { sequence }) + .unwrap(); + }) + }; + // When each chunk the host was sent starts and stops playing, and its frames. + let played = || { + let arrivals = arrivals.lock().unwrap(); + arrivals + .iter() + .scan(0.0, |end: &mut f64, &(at, frames)| { + let start = end.max(at as f64); + *end = start + frames as f64 / 24.0; + Some((start, *end, frames as u64)) + }) + .collect::>() + }; + for &(frames, at) in deltas { + now.store(at, Ordering::SeqCst); + let done = played() + .iter() + .filter(|(_, end, _)| *end <= at as f64) + .map(|(_, _, frames)| frames) + .sum(); + if done > output.played_frames() { + output + .handle_ack(AudioHostAck::Played { + played_frames: done, + }) + .unwrap(); + } + output.write(&vec![0.25; frames]).unwrap(); + } + output.finish_writes().unwrap(); + player.join().unwrap(); + played() + .windows(2) + .map(|pair| pair[1].0 - pair[0].1) + .fold(0.0, f64::max) + } + + #[test] + fn streaming_after_a_lead_plays_a_realtime_source_through_a_stall() { + // A reply arriving as fast as it plays, in 100 ms deltas, two of them held up until 950 ms. + let deltas: Vec<_> = (0..20) + .map(|index| { + ( + 2400, + if index == 8 || index == 9 { + 950 + } else { + index * 100 + }, + ) + }) + .collect(); + let chunked = longest_gap(remote(None), &deltas); + assert!( + chunked > 100.0, + "whole chunks fell silent for only {chunked} ms" + ); + assert_eq!(longest_gap(remote(Some(7200)), &deltas), 0.0); + } + #[test] fn empty_output_fails_before_emitting_audio_end() { let (output, mut host) = fixture(); From cd90b8c83e8d4528a108d4aed44d55e4b9dabd8b Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 08:48:32 -0400 Subject: [PATCH 23/49] berd-call: test changing every setting in a live call On a live realtime call, in both modes, change the Spokesperson's speed, then its voice, then the input policy, then mute and unmute, and check each one lands. With a separate agent voice, also change the agent's voice and speed. Check that the Spokesperson's realtime session is rebuilt with each new voice and speed. Run the retry test for a change the user overtook in both modes as well. With a separate agent voice, the Spokesperson rate change it retries fails without the retry. On the host, a Spokesperson voice or rate change keeps the other setting and the revision, and is refused when the call has no separate Spokesperson voice. Co-Authored-By: Claude Opus 5.5 Signed-off-by: John Tennant --- .../crates/berd-call/src/host_session.rs | 58 ++++++ .../berd-call/tests/session_protocol.rs | 193 +++++++++++++++++- 2 files changed, 242 insertions(+), 9 deletions(-) diff --git a/src-tauri/crates/berd-call/src/host_session.rs b/src-tauri/crates/berd-call/src/host_session.rs index dd0dfe108..537d5f522 100644 --- a/src-tauri/crates/berd-call/src/host_session.rs +++ b/src-tauri/crates/berd-call/src/host_session.rs @@ -4438,6 +4438,64 @@ mod tests { ); } + #[test] + fn spokesperson_updates_keep_its_other_setting_and_revision() { + let (commands, receiver) = mpsc::sync_channel(1); + let mut snapshot = test_snapshot(); + let openai = |voice: &str, rate| berd_call::TtsSettings::OpenAi { + model: "gpt-realtime".into(), + voice: voice.into(), + rate, + }; + let session = Arc::new(Mutex::new(snapshot.clone())); + let control = SessionControl { + commands, + running: Arc::new(AtomicBool::new(true)), + ready: ReadyState { + session: Arc::clone(&session), + }, + non_blocking: Arc::new(AtomicBool::new(false)), + stop_requested: Arc::new(AtomicBool::new(false)), + muted: Arc::new(AtomicBool::new(false)), + transcript: Arc::new(Transcript::Silent), + speaks: true, + session_arguments: vec!["session".into()], + }; + let refusal = control.set_spokesperson(None, Some(1.5)).unwrap_err(); + assert!( + refusal.contains("no separate Spokesperson voice"), + "{refusal}" + ); + snapshot.spokesperson = Some(berd_call::TtsConfigurationSnapshot { + revision: 3, + settings: openai("marin", 1.25), + }); + *session.lock().unwrap() = snapshot; + let worker = thread::spawn(move || { + (0..2) + .map(|_| match receiver.recv().unwrap() { + ControlCommand::SpokespersonSettings { + settings, + expected_revision, + response, + } => { + response.send(Ok(json!({}))).unwrap(); + (settings, expected_revision) + } + _ => panic!("expected a Spokesperson settings command"), + }) + .collect::>() + }); + control.set_spokesperson(None, Some(1.5)).unwrap(); + control + .set_spokesperson(Some("cedar".into()), None) + .unwrap(); + assert_eq!( + worker.join().unwrap(), + [(openai("marin", 1.5), 3), (openai("cedar", 1.25), 3)] + ); + } + #[test] fn stop_cancels_a_pending_restart() { let mut child = Command::new("/bin/cat") diff --git a/src-tauri/crates/berd-call/tests/session_protocol.rs b/src-tauri/crates/berd-call/tests/session_protocol.rs index 8993d73bd..860dd9f5a 100644 --- a/src-tauri/crates/berd-call/tests/session_protocol.rs +++ b/src-tauri/crates/berd-call/tests/session_protocol.rs @@ -1591,6 +1591,34 @@ fn expert_spokesperson_voice_change_swaps_atomically_and_flushes_gated_pcm_once( #[test] fn user_activity_during_voice_cutover_rolls_back_then_retries_once_idle() { + retries_a_voice_change_the_user_overtook( + &[], + json!({ + "type":"set_tts_settings", + "id":2, + "expected_revision":1, + "settings":{"backend":"openai","model":"test-model","voice":"new-voice","rate":1.0} + }), + ); +} + +#[test] +fn a_spokesperson_rate_change_the_user_overtook_is_retried_beside_the_agents_voice() { + let (speech_url, _speech_requests) = spawn_speech_server(); + retries_a_voice_change_the_user_overtook( + &["--allow-direct-agent-speech", "--tts-url", &speech_url], + json!({ + "type":"set_spokesperson_settings", + "id":2, + "expected_revision":1, + "settings":{"backend":"openai","model":"test-model","voice":"old-voice","rate":1.5} + }), + ); +} + +/// Starts a call with `args`, makes the Spokesperson change in `request`, and speaks while its session is being built: the change rolls back, then lands once the call is idle. +fn retries_a_voice_change_the_user_overtook(args: &[&str], request: Value) { + let output = request["settings"].clone(); let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); let (clear_seen_tx, clear_seen_rx) = mpsc::sync_channel(1); let (release_activity_tx, release_activity_rx) = mpsc::sync_channel(1); @@ -1669,7 +1697,8 @@ fn user_activity_during_voice_cutover_rolls_back_then_retries_once_idle() { let (retry_stream, _) = listener.accept().await.unwrap(); let mut retry = accept_async(retry_stream).await.unwrap(); let retry_update = receive_realtime_json(&mut retry).await; - assert_eq!(retry_update["session"]["audio"]["output"]["voice"], "new-voice"); + assert_eq!(retry_update["session"]["audio"]["output"]["voice"], output["voice"]); + assert_eq!(retry_update["session"]["audio"]["output"]["speed"], output["rate"]); acknowledge_realtime_session(&mut retry, &retry_update).await; let seeds = tokio::spawn(async move { while let Some(Ok(Message::Text(text))) = retry.next().await { @@ -1691,13 +1720,8 @@ fn user_activity_during_voice_cutover_rolls_back_then_retries_once_idle() { }); let endpoint = endpoint_rx.recv().unwrap(); - let mut session = ExpertSpokespersonTestSession::start(endpoint); - session.send(json!({ - "type":"set_tts_settings", - "id":2, - "expected_revision":1, - "settings":{"backend":"openai","model":"test-model","voice":"new-voice","rate":1.0} - })); + let mut session = ExpertSpokespersonTestSession::start_with_args(endpoint, args); + session.send(request.clone()); clear_seen_rx.recv_timeout(Duration::from_secs(2)).unwrap(); session.send_pcm(0.25); session.flush(); @@ -1716,7 +1740,8 @@ fn user_activity_during_voice_cutover_rolls_back_then_retries_once_idle() { assert!(saw_user, "{message}"); assert_eq!(message["outcome"], "applied", "{message}"); assert_eq!(message["snapshot"]["revision"], 2); - assert_eq!(message["snapshot"]["voice"], "new-voice"); + assert_eq!(message["snapshot"]["voice"], request["settings"]["voice"]); + assert_eq!(message["snapshot"]["rate"], request["settings"]["rate"]); applied = true; } } @@ -1727,6 +1752,156 @@ fn user_activity_during_voice_cutover_rolls_back_then_retries_once_idle() { server.join().unwrap(); } +type RealtimeSocket = tokio_tungstenite::WebSocketStream; + +/// Accepts the realtime session a Spokesperson voice or speed change builds, checks it asks for `voice` at `speed`, and lets the call move to it from `current`. +async fn cut_over( + listener: &TcpListener, + current: &mut RealtimeSocket, + voice: &str, + speed: f64, +) -> RealtimeSocket { + let (stream, _) = listener.accept().await.unwrap(); + let mut candidate = accept_async(stream).await.unwrap(); + let update = receive_realtime_json(&mut candidate).await; + assert_eq!(update["session"]["audio"]["output"]["voice"], voice); + assert_eq!(update["session"]["audio"]["output"]["speed"], speed); + acknowledge_realtime_session(&mut candidate, &update).await; + while receive_realtime_json(current).await["type"] != "input_audio_buffer.clear" {} + send_realtime_json(current, json!({"type":"input_audio_buffer.cleared"})).await; + candidate +} + +/// Serves a call whose Spokesperson changes speed, then voice, and is then muted. +fn serve_spokesperson_changes() -> (String, std::thread::JoinHandle<()>) { + let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); + let server = std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + endpoint_tx + .send(format!("ws://{}/", listener.local_addr().unwrap())) + .unwrap(); + let (stream, _) = listener.accept().await.unwrap(); + let mut first = accept_async(stream).await.unwrap(); + let initial = receive_realtime_json(&mut first).await; + assert_eq!(initial["session"]["audio"]["output"]["voice"], "old-voice"); + acknowledge_realtime_session(&mut first, &initial).await; + let mut faster = cut_over(&listener, &mut first, "old-voice", 1.5).await; + let mut voiced = cut_over(&listener, &mut faster, "new-voice", 1.5).await; + // Muting clears what the Spokesperson has heard. + let clear = loop { + let request = receive_realtime_json(&mut voiced).await; + if request["type"] == "input_audio_buffer.clear" { + break request; + } + }; + assert!(clear["event_id"] + .as_str() + .unwrap() + .starts_with("berd-reset-")); + send_realtime_json(&mut voiced, json!({"type":"input_audio_buffer.cleared"})).await; + while voiced.next().await.is_some() {} + drop((first, faster)); + }); + }); + (endpoint_rx.recv().unwrap(), server) +} + +/// Sends a settings change on a live call and returns its result. +fn change(session: &mut ExpertSpokespersonTestSession, request: Value) -> Value { + session.send(request); + session.recv(Duration::from_secs(3)) +} + +/// Changes the Spokesperson's speed, then its voice, with `change_type`, then the input policy and mute, checking each lands. +fn change_spokesperson_and_input(session: &mut ExpertSpokespersonTestSession, change_type: &str) { + for (id, voice, revision) in [(2, "old-voice", 2), (3, "new-voice", 3)] { + let result = change( + session, + json!({ + "type":change_type, + "id":id, + "expected_revision":revision - 1, + "settings":{"backend":"openai","model":"test-model","voice":voice,"rate":1.5} + }), + ); + assert_eq!(result["type"], "tts_settings_result", "{result}"); + assert_eq!(result["outcome"], "applied", "{result}"); + assert_eq!(result["snapshot"]["revision"], revision); + assert_eq!(result["snapshot"]["voice"], voice); + assert_eq!(result["snapshot"]["rate"], 1.5); + } + let policy = change( + session, + json!({"type":"set_input_during_tts","id":4,"expected_revision":1,"policy":"suppress_input"}), + ); + assert_eq!(policy["type"], "input_during_tts_result", "{policy}"); + assert_eq!(policy["outcome"], "applied"); + assert_eq!( + policy["snapshot"], + json!({"revision":2,"policy":"suppress_input"}) + ); + for (id, active) in [(5, true), (6, false)] { + let muted = change( + session, + json!({"type":"set_input_muted","id":id,"active":active}), + ); + assert_eq!( + muted, + json!({"type":"input_mute_applied","id":id,"active":active}) + ); + } +} + +#[test] +fn every_setting_changes_in_a_live_call_with_one_voice() { + let (endpoint, server) = serve_spokesperson_changes(); + let mut session = ExpertSpokespersonTestSession::start(endpoint); + assert!(session.ready["session"]["spokesperson"].is_null()); + change_spokesperson_and_input(&mut session, "set_tts_settings"); + session.shutdown(); + server.join().unwrap(); +} + +#[test] +fn every_setting_changes_in_a_live_call_with_separate_voices() { + let (speech_url, _speech_requests) = spawn_speech_server(); + let (endpoint, server) = serve_spokesperson_changes(); + let mut session = ExpertSpokespersonTestSession::start_with_args( + endpoint, + &["--allow-direct-agent-speech", "--tts-url", &speech_url], + ); + let model = session.ready["session"]["tts"]["model"].clone(); + assert_eq!( + session.ready["session"]["spokesperson"]["voice"], + "old-voice" + ); + // The agent's own voice, then its speed, change without touching the Spokesperson's session. + for (id, voice, rate, revision) in [(7, "cedar", 1.0, 2), (8, "cedar", 1.5, 3)] { + let result = change( + &mut session, + json!({ + "type":"set_tts_settings", + "id":id, + "expected_revision":revision - 1, + "settings":{"backend":"openai","model":model,"voice":voice,"rate":rate} + }), + ); + assert_eq!(result["type"], "tts_settings_result", "{result}"); + assert_eq!(result["outcome"], "applied", "{result}"); + assert_eq!(result["snapshot"]["revision"], revision); + assert_eq!(result["snapshot"]["voice"], voice); + assert_eq!(result["snapshot"]["rate"], rate); + } + change_spokesperson_and_input(&mut session, "set_spokesperson_settings"); + session.shutdown(); + server.join().unwrap(); +} + #[test] fn expert_spokesperson_voice_candidate_mismatch_preserves_old_snapshot() { let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); From 2a053509d1960b3b8076003eb6cbb8a0cb632d96 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 09:00:55 -0400 Subject: [PATCH 24/49] Revert "berd-call: gather a lead before playing the Spokesperson's audio" This reverts commit aad01ee8. Its premise was wrong: measured on six live replies, OpenAI Realtime sends a reply's audio 3 to 5.5 times faster than it plays, in 150 to 400 ms deltas, so a lead covers no late arrival and only delays the first sound. Co-Authored-By: Claude Opus 5.5 Signed-off-by: John Tennant --- src-tauri/crates/berd-call/src/main.rs | 26 ++-- .../crates/berd-call/src/session_audio.rs | 135 +----------------- 2 files changed, 14 insertions(+), 147 deletions(-) diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 6d60e8451..74f937c19 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -5209,9 +5209,6 @@ fn finish_expert_reply( Ok(()) } -/// The Spokesperson's audio gathered before each reply plays, 300 ms at 24 kHz: OpenAI sends it about as fast as it plays, so this covers a delta that comes late. -const SPOKESPERSON_AUDIO_LEAD_FRAMES: usize = 7_200; - fn spawn_live_playback( response_id: String, speech_id: u64, @@ -5221,19 +5218,16 @@ fn spawn_live_playback( completed: mpsc::Sender, ) -> Result { let active = Arc::new(AtomicBool::new(true)); - let output = Arc::new( - RemotePcmAudioOutput::new( - speech_id, - TtsPcmSpec { - sample_rate: 24_000, - playback_rate: 1.0, - }, - transport, - Arc::clone(&active), - control, - )? - .streaming_after(SPOKESPERSON_AUDIO_LEAD_FRAMES), - ); + let output = Arc::new(RemotePcmAudioOutput::new( + speech_id, + TtsPcmSpec { + sample_rate: 24_000, + playback_rate: 1.0, + }, + transport, + Arc::clone(&active), + control, + )?); let (sender, receiver) = mpsc::sync_channel::(64); let worker_output = Arc::clone(&output); let worker_response_id = response_id.clone(); diff --git a/src-tauri/crates/berd-call/src/session_audio.rs b/src-tauri/crates/berd-call/src/session_audio.rs index e4fc2543c..0368d6a9d 100644 --- a/src-tauri/crates/berd-call/src/session_audio.rs +++ b/src-tauri/crates/berd-call/src/session_audio.rs @@ -2,7 +2,7 @@ use std::collections::VecDeque; use std::fs::File; use std::io; use std::os::fd::{AsRawFd, FromRawFd, RawFd}; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc; use std::sync::{Arc, Condvar, Mutex}; use std::time::{Duration, Instant}; @@ -190,8 +190,6 @@ pub struct RemotePcmAudioOutput { operation_timeout: Duration, minimum_accepted_runway_frames: u64, pending_samples: Mutex>, - /// Frames to gather before streaming each write as it comes, or `None` to send whole chunks. - stream_after: Option, state: Mutex, changed: Condvar, } @@ -238,7 +236,6 @@ impl RemotePcmAudioOutput { operation_timeout, minimum_accepted_runway_frames: accepted_audio_runway_frames(spec), pending_samples: Mutex::new(Vec::with_capacity(MAX_AUDIO_CHUNK_FRAMES)), - stream_after: None, state: Mutex::new(State { phase: Phase::New, begin_accepted: false, @@ -260,12 +257,6 @@ impl RemotePcmAudioOutput { }) } - /// Holds back the first `frames` of audio, then sends each write at once, for a source that arrives about as fast as it plays: what is held covers its late arrivals. - pub fn streaming_after(mut self, frames: usize) -> Self { - self.stream_after = Some(AtomicUsize::new(frames)); - self - } - pub fn start(&self) -> Result<(), String> { { let mut state = self.state.lock().expect("remote output state"); @@ -869,18 +860,11 @@ impl PcmAudioOutput for RemotePcmAudioOutput { .lock() .expect("remote pending PCM lock"); pending.extend_from_slice(samples); - let ready = match &self.stream_after { - None => pending.len() / MAX_AUDIO_CHUNK_FRAMES * MAX_AUDIO_CHUNK_FRAMES, - Some(held) if pending.len() >= held.load(Ordering::SeqCst) => { - held.store(0, Ordering::SeqCst); - pending.len() - } - Some(_) => 0, - }; - let tail = pending.split_off(ready); + let complete_len = pending.len() / MAX_AUDIO_CHUNK_FRAMES * MAX_AUDIO_CHUNK_FRAMES; + let tail = pending.split_off(complete_len); std::mem::replace(&mut *pending, tail) }; - for chunk in complete_samples.chunks(MAX_AUDIO_CHUNK_FRAMES) { + for chunk in complete_samples.chunks_exact(MAX_AUDIO_CHUNK_FRAMES) { self.write_chunk(chunk)?; } Ok(()) @@ -937,7 +921,6 @@ mod tests { use std::io::Read; use std::os::fd::IntoRawFd; use std::os::unix::net::UnixStream; - use std::sync::atomic::AtomicU64; use std::sync::Arc; use std::thread; @@ -1596,116 +1579,6 @@ mod tests { worker.join().unwrap().unwrap(); } - fn remote(stream_after: Option) -> (Arc, UnixStream) { - let (child, host) = UnixStream::pair().unwrap(); - let transport = unsafe { AudioPipeTransport::from_raw_fd(child.into_raw_fd()) }.unwrap(); - let spec = TtsPcmSpec { - sample_rate: 24_000, - playback_rate: 1.0, - }; - let output = RemotePcmAudioOutput::new( - 7, - spec, - Arc::new(transport), - Arc::new(AtomicBool::new(true)), - mpsc::channel().0, - ) - .unwrap(); - ( - Arc::new(match stream_after { - Some(frames) => output.streaming_after(frames), - None => output, - }), - host, - ) - } - - /// The longest silence, in milliseconds, between the first sound and the last that a host playing in real time hears from `output`, as each of `deltas` (its frames, and the millisecond it arrives) is written to it. - fn longest_gap( - (output, mut host): (Arc, UnixStream), - deltas: &[(usize, u64)], - ) -> f64 { - start(&output, &mut host); - let now = Arc::new(AtomicU64::new(0)); - let arrivals = Arc::new(Mutex::new(Vec::<(u64, usize)>::new())); - let player = { - let (output, now, arrivals) = - (Arc::clone(&output), Arc::clone(&now), Arc::clone(&arrivals)); - thread::spawn(move || loop { - let (kind, payload) = read_record(&mut host); - if kind == AUDIO_END_KIND { - return; - } - assert_eq!(kind, AUDIO_CHUNK_KIND); - arrivals - .lock() - .unwrap() - .push((now.load(Ordering::SeqCst), (payload.len() - 16) / 4)); - let sequence = u64::from_le_bytes(payload[8..16].try_into().unwrap()); - output - .handle_ack(AudioHostAck::ChunkAccepted { sequence }) - .unwrap(); - }) - }; - // When each chunk the host was sent starts and stops playing, and its frames. - let played = || { - let arrivals = arrivals.lock().unwrap(); - arrivals - .iter() - .scan(0.0, |end: &mut f64, &(at, frames)| { - let start = end.max(at as f64); - *end = start + frames as f64 / 24.0; - Some((start, *end, frames as u64)) - }) - .collect::>() - }; - for &(frames, at) in deltas { - now.store(at, Ordering::SeqCst); - let done = played() - .iter() - .filter(|(_, end, _)| *end <= at as f64) - .map(|(_, _, frames)| frames) - .sum(); - if done > output.played_frames() { - output - .handle_ack(AudioHostAck::Played { - played_frames: done, - }) - .unwrap(); - } - output.write(&vec![0.25; frames]).unwrap(); - } - output.finish_writes().unwrap(); - player.join().unwrap(); - played() - .windows(2) - .map(|pair| pair[1].0 - pair[0].1) - .fold(0.0, f64::max) - } - - #[test] - fn streaming_after_a_lead_plays_a_realtime_source_through_a_stall() { - // A reply arriving as fast as it plays, in 100 ms deltas, two of them held up until 950 ms. - let deltas: Vec<_> = (0..20) - .map(|index| { - ( - 2400, - if index == 8 || index == 9 { - 950 - } else { - index * 100 - }, - ) - }) - .collect(); - let chunked = longest_gap(remote(None), &deltas); - assert!( - chunked > 100.0, - "whole chunks fell silent for only {chunked} ms" - ); - assert_eq!(longest_gap(remote(Some(7200)), &deltas), 0.0); - } - #[test] fn empty_output_fails_before_emitting_audio_end() { let (output, mut host) = fixture(); From d314cdc6692cbd273678ab5cba2fb1f410abea69 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 09:05:19 -0400 Subject: [PATCH 25/49] berd-call: keep a second of audio ahead of the host's played reports The host reports a chunk played only when the output device has played it: about 60 ms after it is heard on a MacBook's speakers, and a few hundred more on Bluetooth headphones. Playback held at most 400 ms beyond what was reported played (less 100 ms of delivery safety), so on headphones the player ran dry while waiting for credit, and the Spokesperson's speech, and an agent's, came out in short gaps. Let the host hold a second beyond its reports, which plays through reports up to about 800 ms late. A test plays 3 s of audio with reports 300 ms late: the old limit leaves 1041 ms of silence; this leaves none. Co-Authored-By: Claude Opus 5.5 Signed-off-by: John Tennant --- .../crates/berd-call/src/session_audio.rs | 94 +++++++++++++++++-- 1 file changed, 88 insertions(+), 6 deletions(-) diff --git a/src-tauri/crates/berd-call/src/session_audio.rs b/src-tauri/crates/berd-call/src/session_audio.rs index 0368d6a9d..210adbe50 100644 --- a/src-tauri/crates/berd-call/src/session_audio.rs +++ b/src-tauri/crates/berd-call/src/session_audio.rs @@ -17,7 +17,9 @@ pub const AUDIO_END_KIND: u8 = 3; pub const AUDIO_CANCEL_KIND: u8 = 4; pub const AUDIO_FRAME_HEADER_BYTES: usize = 8; pub const MAX_AUDIO_CHUNK_FRAMES: usize = 4096; -const MIN_ACCEPTED_AUDIO_RUNWAY_MS: f64 = 400.0; +/// Audio the host may hold beyond what it reports played. It reports a chunk played only once the output device has played it, +/// up to a few hundred milliseconds after it is heard on Bluetooth headphones, so this must outlast that delay or playback runs dry. +const MIN_ACCEPTED_AUDIO_RUNWAY_MS: f64 = 1_000.0; const MAX_ACCEPTED_NOT_PLAYED_CHUNKS: usize = 64; const AUDIO_PIPE_WRITE_TIMEOUT: Duration = Duration::from_secs(2); @@ -1040,8 +1042,8 @@ mod tests { let (output, mut host) = fixture(); start(&output, &mut host); let current = Arc::clone(&output); - let worker = thread::spawn(move || current.write(&vec![0.25; 4096 * 4])); - for sequence in 1..=3 { + let worker = thread::spawn(move || current.write(&vec![0.25; 4096 * 7])); + for sequence in 1..=6 { let (kind, _) = read_record(&mut host); assert_eq!(kind, AUDIO_CHUNK_KIND); let started = output @@ -1062,7 +1064,7 @@ mod tests { let (kind, _) = read_record(&mut host); assert_eq!(kind, AUDIO_CHUNK_KIND); output - .handle_ack(AudioHostAck::ChunkAccepted { sequence: 4 }) + .handle_ack(AudioHostAck::ChunkAccepted { sequence: 7 }) .unwrap(); worker.join().unwrap().unwrap(); } @@ -1525,14 +1527,14 @@ mod tests { sample_rate: 48_000, playback_rate: 1.5, }, - 8, + 18, ), ( TtsPcmSpec { sample_rate: 24_000, playback_rate: 2.0, }, - 5, + 12, ), ] { let (output, mut host) = @@ -1579,6 +1581,86 @@ mod tests { worker.join().unwrap().unwrap(); } + /// The silence, in milliseconds, a host playing in real time leaves in 3 s of audio written at once, when it reports each chunk played `lag` after it is heard, as the output device does. + fn silence_when_played_reports_lag(lag: Duration) -> f64 { + let (output, mut host) = fixture(); + start(&output, &mut host); + // When each chunk is heard, and its frames. + let heard = Arc::new(Mutex::new(Vec::<(Instant, Instant, u64)>::new())); + let writing = Arc::new(AtomicBool::new(true)); + let reporter = { + let (output, heard, writing) = ( + Arc::clone(&output), + Arc::clone(&heard), + Arc::clone(&writing), + ); + thread::spawn(move || { + while writing.load(Ordering::SeqCst) { + let now = Instant::now(); + let done: u64 = heard + .lock() + .unwrap() + .iter() + .filter(|(_, end, _)| *end + lag <= now) + .map(|(_, _, frames)| frames) + .sum(); + // Less the host's delivery safety, 100 ms. + let played = done.saturating_sub(2_400); + if played > output.played_frames() { + output + .handle_ack(AudioHostAck::Played { + played_frames: played, + }) + .unwrap(); + } + thread::sleep(Duration::from_millis(1)); + } + }) + }; + let writer = { + let output = Arc::clone(&output); + thread::spawn(move || { + output.write(&vec![0.25; 72_000])?; + output.finish_writes() + }) + }; + loop { + let (kind, payload) = read_record(&mut host); + if kind == AUDIO_END_KIND { + break; + } + let frames = (payload.len() as u64 - 16) / 4; + let mut heard = heard.lock().unwrap(); + let start = heard + .last() + .map_or_else(Instant::now, |(_, end, _)| (*end).max(Instant::now())); + heard.push(( + start, + start + Duration::from_micros(frames * 1_000 / 24), + frames, + )); + drop(heard); + let sequence = u64::from_le_bytes(payload[8..16].try_into().unwrap()); + output + .handle_ack(AudioHostAck::ChunkAccepted { sequence }) + .unwrap(); + } + writer.join().unwrap().unwrap(); + writing.store(false, Ordering::SeqCst); + reporter.join().unwrap(); + let heard = heard.lock().unwrap(); + heard + .windows(2) + .map(|pair| pair[1].0.duration_since(pair[0].1).as_secs_f64() * 1_000.0) + .sum() + } + + #[test] + fn playback_outlasts_bluetooth_played_reports() { + let silence = silence_when_played_reports_lag(Duration::from_millis(300)); + assert!(silence < 20.0, "{silence:.0} ms of silence"); + } + #[test] fn empty_output_fails_before_emitting_audio_end() { let (output, mut host) = fixture(); From 24123fdb887ece665712fab2a8ecd38f0725b3c6 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 09:50:41 -0400 Subject: [PATCH 26/49] berd-call: let the host ignore records for speech it already ended When the host's player stops, as when the output device changes, the host ends that speech and reports audio_failed. The session can still send chunk, end or cancel records for it before hearing the report. The host treated those as protocol errors and ended the call. It now drops late chunk and end records for the speech it retired and answers a late cancel with what it played. Co-Authored-By: Claude Opus 5.5 Signed-off-by: John Tennant --- .../crates/berd-call/src/host_session.rs | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src-tauri/crates/berd-call/src/host_session.rs b/src-tauri/crates/berd-call/src/host_session.rs index 537d5f522..507bc9e82 100644 --- a/src-tauri/crates/berd-call/src/host_session.rs +++ b/src-tauri/crates/berd-call/src/host_session.rs @@ -2309,6 +2309,16 @@ impl AudioPlaybackState { } } + /// What the host played of this speech, once it has ended it. The session may not have heard yet, so its records for it can still arrive. + fn retired(&self, speech_id: u64) -> Option { + match self { + Self::Finished(retired) if retired.speech_id == speech_id => { + Some(retired.played_frames) + } + _ => None, + } + } + fn finish(&mut self, speech_id: u64, played_frames: u64) { *self = Self::Finished(RetiredAudio { speech_id, @@ -2523,6 +2533,9 @@ fn handle_audio_record( AUDIO_CHUNK_KIND if payload.len() >= 16 && (payload.len() - 16).is_multiple_of(4) => { let speech_id = le_u64(&payload[0..8])?; let sequence = le_u64(&payload[8..16])?; + if playback.retired(speech_id).is_some() { + return Ok(()); + } let active = playback .playing_mut(speech_id) .ok_or_else(|| "voice session sent audio for an inactive speech".to_string())?; @@ -2555,6 +2568,9 @@ fn handle_audio_record( AUDIO_END_KIND if payload.len() == 24 => { let speech_id = le_u64(&payload[0..8])?; let sequence = le_u64(&payload[8..16])?; + if playback.retired(speech_id).is_some() { + return Ok(()); + } let active = playback .playing_mut(speech_id) .ok_or_else(|| "voice session ended inactive audio".to_string())?; @@ -2563,6 +2579,15 @@ fn handle_audio_record( } AUDIO_CANCEL_KIND if payload.len() == 8 => { let speech_id = le_u64(payload)?; + if let Some(played_frames) = playback.retired(speech_id) { + return send_request( + writer, + &SessionRequest::AudioCancelled { + speech_id, + played_frames, + }, + ); + } let active = playback .playing_mut(speech_id) .ok_or_else(|| "voice session cancelled inactive audio".to_string())?; @@ -4612,6 +4637,17 @@ mod tests { assert_eq!(playback.played_frames(18), 0); } + #[test] + fn only_the_ended_speech_is_retired() { + let playback = AudioPlaybackState::Finished(RetiredAudio { + speech_id: 17, + played_frames: 4_096, + }); + assert_eq!(playback.retired(17), Some(4_096)); + assert_eq!(playback.retired(18), None); + assert_eq!(AudioPlaybackState::Idle.retired(17), None); + } + #[test] fn suspension_settle_time_preserves_route_safety_duration() { assert_eq!( From 078da1748bb5cd7f31089791c4f939bf36aa9ab6 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 09:50:42 -0400 Subject: [PATCH 27/49] berd-call: keep a realtime call going when the host's player stops A Spokesperson reply whose host player stopped ended the whole call with "host audio output failed". The session now ends that reply where the host stopped, as if the user had interrupted it: the provider's item is truncated to what played, the transcript is published as interrupted, and the next reply gets a new player. Co-Authored-By: Claude Opus 5.5 Signed-off-by: John Tennant --- src-tauri/crates/berd-call/src/main.rs | 9 +- .../crates/berd-call/src/session_audio.rs | 14 +- .../berd-call/tests/session_protocol.rs | 144 +++++++++++++++++- 3 files changed, 157 insertions(+), 10 deletions(-) diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 74f937c19..9473fd0ba 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -81,7 +81,7 @@ mod system_input_mute; use session_audio::{ AudioHostAck, AudioOutputControlRequest, AudioPipeTransport, RemotePcmAudioOutput, - AUDIO_CANCELLED, + AUDIO_CANCELLED, AUDIO_HOST_FAILED, }; use session_framing::{ @@ -4196,7 +4196,8 @@ fn run_expert_spokesperson_session( else { continue; }; - if message == AUDIO_CANCELLED { + // A reply the host stopped playing ends where it was cut off, as if you had interrupted it; the next gets a new player. + if message == AUDIO_CANCELLED || message == AUDIO_HOST_FAILED { apply_external_coordinator_event( &mut core, runtime.as_ref().expect("initialized runtime"), @@ -4207,6 +4208,10 @@ fn run_expert_spokesperson_session( )?; if let Some(response) = responses.get_mut(&response_id) { emit_live_interrupted_terminal(response, &playback, &mut writer)?; + if message == AUDIO_HOST_FAILED { + // The model hears only what played, as when the user talks over it. + require_live_response_truncation(response)?; + } send_live_response_truncation( &response_id, response, diff --git a/src-tauri/crates/berd-call/src/session_audio.rs b/src-tauri/crates/berd-call/src/session_audio.rs index 210adbe50..7f2aabbc6 100644 --- a/src-tauri/crates/berd-call/src/session_audio.rs +++ b/src-tauri/crates/berd-call/src/session_audio.rs @@ -25,6 +25,8 @@ const MAX_ACCEPTED_NOT_PLAYED_CHUNKS: usize = 64; const AUDIO_PIPE_WRITE_TIMEOUT: Duration = Duration::from_secs(2); const AUDIO_OPERATION_TIMEOUT: Duration = Duration::from_secs(3); pub const AUDIO_CANCELLED: &str = "remote PCM output was cancelled"; +/// The host's player stopped, as when the output device changes under it; the host has ended that speech. +pub const AUDIO_HOST_FAILED: &str = "host audio output failed"; #[derive(Clone, Debug, PartialEq)] pub enum AudioHostAck { @@ -621,7 +623,7 @@ impl RemotePcmAudioOutput { Err(state .failure .clone() - .unwrap_or_else(|| "host audio output failed".into())) + .unwrap_or_else(|| AUDIO_HOST_FAILED.into())) } else { Ok(()) } @@ -668,7 +670,7 @@ impl RemotePcmAudioOutput { return Err(state .failure .clone() - .unwrap_or_else(|| "host audio output failed".into())); + .unwrap_or_else(|| AUDIO_HOST_FAILED.into())); } Phase::Cancelling => false, Phase::WaitingBegin @@ -712,7 +714,7 @@ impl RemotePcmAudioOutput { return Err(state .failure .clone() - .unwrap_or_else(|| "host audio output failed".into())) + .unwrap_or_else(|| AUDIO_HOST_FAILED.into())) } Phase::Drained => return Err("remote PCM output is already drained".into()), _ => {} @@ -761,7 +763,7 @@ impl RemotePcmAudioOutput { return Err(state .failure .clone() - .unwrap_or_else(|| "host audio output failed".into())); + .unwrap_or_else(|| AUDIO_HOST_FAILED.into())); } if state.suspension == SuspensionPhase::Suspended && state.suspension_requested { while state.suspension_requested && self.active.load(Ordering::SeqCst) { @@ -891,7 +893,7 @@ impl PcmAudioOutput for RemotePcmAudioOutput { Err(state .failure .clone() - .unwrap_or_else(|| "host audio output failed".into())) + .unwrap_or_else(|| AUDIO_HOST_FAILED.into())) } else { Ok(()) } @@ -914,7 +916,7 @@ fn accepted_audio_runway_frames(spec: TtsPcmSpec) -> u64 { } fn public_host_failure(_message: String) -> String { - "host audio output failed".into() + AUDIO_HOST_FAILED.into() } #[cfg(test)] diff --git a/src-tauri/crates/berd-call/tests/session_protocol.rs b/src-tauri/crates/berd-call/tests/session_protocol.rs index 860dd9f5a..24b271bc8 100644 --- a/src-tauri/crates/berd-call/tests/session_protocol.rs +++ b/src-tauri/crates/berd-call/tests/session_protocol.rs @@ -65,12 +65,18 @@ impl ExpertSpokespersonTestSession { played_frame_limit, played_ready, cancellation_gate, + false, &[], ) } + /// A session whose host player stops once it has played this much of the first speech, as when the output device changes under it. + fn start_with_failing_audio(endpoint: String, played_frames: u64) -> Self { + Self::launch(endpoint, None, Some(played_frames), None, None, true, &[]) + } + fn start_with_args(endpoint: String, args: &[&str]) -> Self { - Self::launch(endpoint, None, None, None, None, args) + Self::launch(endpoint, None, None, None, None, false, args) } fn launch( @@ -79,6 +85,7 @@ impl ExpertSpokespersonTestSession { played_frame_limit: Option, played_ready: Option>, cancellation_gate: Option, + fail_at_limit: bool, args: &[&str], ) -> Self { let (mut command, _pcm, audio_host) = session_command(); @@ -119,6 +126,7 @@ impl ExpertSpokespersonTestSession { played_frame_limit, played_ready, cancellation_gate, + fail_at_limit, ); let mut session = Self { child, @@ -328,7 +336,7 @@ fn spawn_audio_host( reader: UnixStream, stdin: Arc>, ) -> std::thread::JoinHandle<()> { - spawn_audio_host_with_played_limit(reader, stdin, None, None, None) + spawn_audio_host_with_played_limit(reader, stdin, None, None, None, false) } fn spawn_audio_host_with_played_limit( @@ -337,9 +345,12 @@ fn spawn_audio_host_with_played_limit( played_frame_limit: Option, mut played_ready: Option>, mut cancellation_gate: Option, + mut fail_at_limit: bool, ) -> std::thread::JoinHandle<()> { std::thread::spawn(move || { let mut current = None::<(u64, u64, u64)>; + // The speech whose player failed, and what it played: like the real host, it ignores what the session sent before hearing. + let mut failed = None::<(u64, u64)>; loop { let mut header = [0_u8; 8]; match reader.read_exact(&mut header) { @@ -362,6 +373,9 @@ fn spawn_audio_host_with_played_limit( 2 => { assert!(payload.len() >= 20); let speech_id = u64::from_le_bytes(payload[..8].try_into().unwrap()); + if failed.is_some_and(|(id, _)| id == speech_id) { + continue; + } let sequence = u64::from_le_bytes(payload[8..16].try_into().unwrap()); let frames = u64::try_from((payload.len() - 16) / 4).unwrap(); let state = current.as_mut().expect("chunk follows begin"); @@ -379,6 +393,16 @@ fn spawn_audio_host_with_played_limit( &mut *writer, &json!({"type":"audio_played","speech_id":speech_id,"played_frames":played_frame_limit.map_or(state.2, |limit| state.2.min(limit))}), ); + if reached_limit && fail_at_limit { + fail_at_limit = false; + let played_frames = played_frame_limit.unwrap(); + failed = Some((speech_id, played_frames)); + current = None; + write_session_json( + &mut *writer, + &json!({"type":"audio_failed","speech_id":speech_id,"played_frames":played_frames,"message":"PCM audio output stopped unexpectedly"}), + ); + } writer.flush().unwrap(); if reached_limit { if let Some(ready) = played_ready.take() { @@ -390,6 +414,9 @@ fn spawn_audio_host_with_played_limit( 3 => { assert_eq!(payload.len(), 24); let speech_id = u64::from_le_bytes(payload[..8].try_into().unwrap()); + if failed.is_some_and(|(id, _)| id == speech_id) { + continue; + } let sequence = u64::from_le_bytes(payload[8..16].try_into().unwrap()); let frames = u64::from_le_bytes(payload[16..24].try_into().unwrap()); assert_eq!(current, Some((speech_id, sequence, frames))); @@ -406,6 +433,15 @@ fn spawn_audio_host_with_played_limit( 4 => { assert_eq!(payload.len(), 8); let speech_id = u64::from_le_bytes(payload.try_into().unwrap()); + if let Some((_, played_frames)) = failed.filter(|(id, _)| *id == speech_id) { + let mut writer = stdin.lock().unwrap(); + write_session_json( + &mut *writer, + &json!({"type":"audio_cancelled","speech_id":speech_id,"played_frames":played_frames}), + ); + writer.flush().unwrap(); + continue; + } let played_frames = current .take() .filter(|state| state.0 == speech_id) @@ -1175,6 +1211,110 @@ fn one_realtime_response_with_two_audio_parts_completes_without_identity_failure server.join().unwrap(); } +#[test] +fn a_reply_whose_player_stops_ends_where_it_was_heard_and_the_call_goes_on() { + let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); + let server = std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + endpoint_tx + .send(format!("ws://{}/", listener.local_addr().unwrap())) + .unwrap(); + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = accept_async(stream).await.unwrap(); + let update = receive_realtime_json(&mut socket).await; + acknowledge_realtime_session(&mut socket, &update).await; + // Two seconds of audio, of which the host plays a fraction before its player stops; then a short reply. + for (directive, response, bytes) in [("2", "response-cut", 96_000), ("3", "response-next", 4_096)] { + assert_eq!( + receive_realtime_json(&mut socket).await["type"], + "conversation.item.create" + ); + assert_eq!( + receive_realtime_json(&mut socket).await["type"], + "response.create" + ); + send_realtime_json( + &mut socket, + json!({"type":"response.created","response":{"id":response,"metadata":{"berd_expert_directive_id":directive}}}), + ) + .await; + let item = format!("{response}-item"); + for event in [ + json!({"type":"response.output_audio.delta","response_id":response,"item_id":item,"output_index":0,"content_index":0,"delta":BASE64.encode(vec![0_u8; bytes])}), + json!({"type":"response.output_audio.done","response_id":response,"item_id":item,"output_index":0,"content_index":0}), + json!({"type":"response.output_audio_transcript.done","response_id":response,"item_id":item,"output_index":0,"content_index":0,"transcript":"a reply"}), + json!({"type":"response.done","response":{"id":response,"status":"completed"}}), + ] { + send_realtime_json(&mut socket, event).await; + } + if directive == "2" { + // The model learns the user heard the 4,096 frames (170 ms) played before the player stopped. + let truncate = receive_realtime_json(&mut socket).await; + assert_eq!(truncate["type"], "conversation.item.truncate"); + assert_eq!(truncate["item_id"], item); + assert_eq!(truncate["audio_end_ms"], 170); + send_realtime_json( + &mut socket, + json!({"type":"conversation.item.truncated","item_id":item,"content_index":0}), + ) + .await; + } + } + let _ = socket.next().await; + }); + }); + let mut session = + ExpertSpokespersonTestSession::start_with_failing_audio(endpoint_rx.recv().unwrap(), 4_096); + let mut seen = Vec::new(); + let mut until = + |session: &mut ExpertSpokespersonTestSession, what: &str, done: &dyn Fn(&Value) -> bool| { + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + let remaining = deadline + .checked_duration_since(std::time::Instant::now()) + .unwrap_or_else(|| panic!("no {what}: {seen:?}")); + let message = session + .output + .recv_timeout(remaining) + .unwrap_or_else(|error| panic!("no {what} ({error}): {seen:?}")); + assert_ne!(message["type"], "fatal", "unexpected fatal: {message}"); + let found = done(&message); + seen.push(message); + if found { + break; + } + } + }; + let speak = |session: &mut ExpertSpokespersonTestSession, id: u64| { + session.send( + json!({"type":"prepare_speak","id":id,"acknowledgement":null,"text":"say something"}), + ); + let admitted = std::iter::repeat_with(|| session.recv(Duration::from_secs(2))) + .find(|message| message["id"] == id) + .unwrap(); + assert_eq!(admitted["type"], "admitted", "{admitted}"); + session.send(json!({"type":"output_ready","id":id,"speech_id":admitted["speech_id"]})); + }; + speak(&mut session, 2); + until(&mut session, "interruption", &|message| { + message["id"] == 2 && message["type"] == "speech_interrupted" + }); + until(&mut session, "cut transcript", &|message| { + message["type"] == "live_event" && message["origin"] == "spokesperson" + }); + speak(&mut session, 3); + until(&mut session, "next reply", &|message| { + message["id"] == 3 && message["type"] == "speech_completed" + }); + session.shutdown(); + server.join().unwrap(); +} + #[test] fn expert_spokesperson_expiry_recovery_failure_reports_the_specific_terminal() { let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); From 1f48fc55c9ac5052ed1c604bf59b2bdba93b5a75 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 11:00:05 -0400 Subject: [PATCH 28/49] Keep microphone input flowing while an input reset clears Switching the output device restarts the microphone and resets the Spokesperson's input. The realtime loop waited for the provider to confirm the clear, so microphone audio stopped draining, the 32-frame session queue filled, and the session failed with "session PCM input queue is full". Input resets and mutes now send the clear without waiting. The loop answers them in order once the provider confirms, coalesces a request that arrives while a clear is outstanding into one follow-up clear, and holds a pending voice change until no clear is outstanding. Resets share the input queue with microphone audio, so a clear discards exactly the audio sent before it. Signed-off-by: John Tennant Co-Authored-By: Claude Opus 5.5 --- src-tauri/crates/berd-call/src/main.rs | 233 +++++++++++++----- .../berd-call/src/openai_spokesperson.rs | 90 +++++-- .../berd-call/tests/session_protocol.rs | 65 +++++ 3 files changed, 304 insertions(+), 84 deletions(-) diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 9473fd0ba..65ce8660c 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -3161,30 +3161,63 @@ fn spokesperson_pcm_allowed( && input_policy.policy == berd_call::input::InputDuringTtsPolicy::SuppressInput) } -fn set_spokesperson_input_muted( - id: u64, - muted: bool, - input_muted: &mut bool, - mut reset_input: impl FnMut() -> Result<(), String>, - writer: &mut impl Write, -) -> Result<(), String> { - *input_muted = muted; - if muted { - reset_input()?; - } - write_message( - writer, - &SessionMessage::InputMuteApplied { id, active: muted }, - ) +/// Input controls waiting on the provider to clear its input buffer, so microphone audio keeps flowing meanwhile. +/// Controls answer in order once no clear is outstanding; one that needs a clear while another is outstanding gets the next. +#[derive(Default)] +struct InputResets { + clearing: Option>>, + again: bool, + replies: Vec, } -fn reset_spokesperson_input( - id: u64, - mut reset_input: impl FnMut() -> Result<(), String>, - writer: &mut impl Write, -) -> Result<(), String> { - reset_input()?; - write_message(writer, &SessionMessage::InputResetApplied { id }) +impl InputResets { + fn idle(&self) -> bool { + self.clearing.is_none() + } + + fn apply( + &mut self, + reply: SessionMessage, + clear: bool, + reset_input: impl FnOnce() -> Result>, String>, + writer: &mut impl Write, + ) -> Result<(), String> { + if self.idle() { + if !clear { + return write_message(writer, &reply); + } + self.clearing = Some(reset_input()?); + } else { + self.again |= clear; + } + self.replies.push(reply); + Ok(()) + } + + fn poll( + &mut self, + reset_input: impl FnOnce() -> Result>, String>, + writer: &mut impl Write, + ) -> Result<(), String> { + let Some(clearing) = &self.clearing else { + return Ok(()); + }; + match clearing.try_recv() { + Err(mpsc::TryRecvError::Empty) => return Ok(()), + Err(mpsc::TryRecvError::Disconnected) => { + return Err("Spokesperson runtime is closed".into()) + } + Ok(result) => result?, + } + if std::mem::take(&mut self.again) { + self.clearing = Some(reset_input()?); + return Ok(()); + } + self.clearing = None; + self.replies + .drain(..) + .try_for_each(|reply| write_message(writer, &reply)) + } } fn set_spokesperson_input_policy( @@ -3377,12 +3410,17 @@ fn run_expert_spokesperson_session( let mut session_tts: Option = None; let mut input_during_tts_slot: Option = None; let mut input_muted = false; + let mut input_resets = InputResets::default(); let mut queued_tts_settings = VoiceUpdateQueue::::default(); let mut pending_voice_update: Option = None; let mut status_sound_runtime = StatusSoundRuntime::default(); loop { if initialized { + input_resets.poll( + || runtime.as_ref().expect("initialized runtime").reset_input(), + &mut writer, + )?; let work = spokesperson_host_work( &turn_gate, active.as_ref(), @@ -3415,7 +3453,11 @@ fn run_expert_spokesperson_session( )?); } } - if let Some(update) = pending_voice_update.as_ref() { + // A voice change waits for input resets, which clear the same provider input buffer it does. + if let Some(update) = pending_voice_update + .as_ref() + .filter(|_| input_resets.idle()) + { let work = spokesperson_host_work( &turn_gate, active.as_ref(), @@ -4965,10 +5007,10 @@ fn run_expert_spokesperson_session( &mut writer, )?; } - set_spokesperson_input_muted( - id, + input_muted = muted; + input_resets.apply( + SessionMessage::InputMuteApplied { id, active: muted }, muted, - &mut input_muted, || runtime.as_ref().expect("initialized runtime").reset_input(), &mut writer, )?; @@ -4983,8 +5025,9 @@ fn run_expert_spokesperson_session( &mut writer, )?; } - reset_spokesperson_input( - id, + input_resets.apply( + SessionMessage::InputResetApplied { id }, + true, || runtime.as_ref().expect("initialized runtime").reset_input(), &mut writer, )?; @@ -9013,31 +9056,104 @@ mod tests { } #[test] - fn spokesperson_input_controls_are_nonfatal_revisioned_and_gate_pcm() { - let mut muted = false; - let mut reset_count = 0; + fn input_controls_answer_in_order_once_the_provider_has_cleared_their_input() { + let clears = std::cell::RefCell::new(Vec::new()); + let reset_input = || { + let (completed, result) = mpsc::sync_channel(1); + clears.borrow_mut().push(completed); + Ok(result) + }; + let mut resets = InputResets::default(); let mut output = Vec::new(); - set_spokesperson_input_muted( - 1, - true, - &mut muted, - || { - reset_count += 1; - Ok(()) - }, - &mut output, - ) - .unwrap(); - reset_spokesperson_input( - 2, - || { - reset_count += 1; - Ok(()) - }, - &mut output, - ) - .unwrap(); + resets + .apply( + SessionMessage::InputMuteApplied { + id: 1, + active: true, + }, + true, + reset_input, + &mut output, + ) + .unwrap(); + resets + .apply( + SessionMessage::InputResetApplied { id: 2 }, + true, + reset_input, + &mut output, + ) + .unwrap(); + resets + .apply( + SessionMessage::InputMuteApplied { + id: 3, + active: false, + }, + false, + reset_input, + &mut output, + ) + .unwrap(); + resets.poll(reset_input, &mut output).unwrap(); + assert_eq!(clears.borrow().len(), 1); + assert!(output.is_empty()); + // The reset asked for during the first clear needs a clear of its own. + clears.borrow()[0].send(Ok(())).unwrap(); + resets.poll(reset_input, &mut output).unwrap(); + assert_eq!(clears.borrow().len(), 2); + assert!(output.is_empty()); + clears.borrow()[1].send(Ok(())).unwrap(); + resets.poll(reset_input, &mut output).unwrap(); + assert!(resets.idle()); + assert_eq!( + messages(&output), + [ + json!({"type":"input_mute_applied","id":1,"active":true}), + json!({"type":"input_reset_applied","id":2}), + json!({"type":"input_mute_applied","id":3,"active":false}), + ] + ); + + output.clear(); + resets + .apply( + SessionMessage::InputMuteApplied { + id: 4, + active: false, + }, + false, + reset_input, + &mut output, + ) + .unwrap(); + assert_eq!(clears.borrow().len(), 2); + assert_eq!( + messages(&output), + [json!({"type":"input_mute_applied","id":4,"active":false})] + ); + + resets + .apply( + SessionMessage::InputResetApplied { id: 5 }, + true, + reset_input, + &mut output, + ) + .unwrap(); + clears.borrow()[2] + .send(Err("provider refused".into())) + .unwrap(); + assert_eq!( + resets.poll(reset_input, &mut output).unwrap_err(), + "provider refused" + ); + } + + #[test] + fn spokesperson_input_policy_is_nonfatal_revisioned_and_gates_pcm() { + let mut output = Vec::new(); let slot = InputDuringTtsSlot::new(InputDuringTtsPolicy::AllowBargeIn); set_spokesperson_input_policy( 3, @@ -9050,8 +9166,6 @@ mod tests { set_spokesperson_input_policy(4, 1, InputDuringTtsPolicy::AllowBargeIn, &slot, &mut output) .unwrap(); - assert!(muted); - assert_eq!(reset_count, 2); assert!(!spokesperson_pcm_allowed( false, true, @@ -9068,15 +9182,10 @@ mod tests { slot.snapshot().unwrap() )); let messages = messages(&output); - assert_eq!( - messages[0], - json!({"type":"input_mute_applied","id":1,"active":true}) - ); - assert_eq!(messages[1], json!({"type":"input_reset_applied","id":2})); - assert_eq!(messages[2]["outcome"], "applied"); - assert_eq!(messages[2]["snapshot"]["revision"], 2); - assert_eq!(messages[3]["outcome"], "rejected"); - assert_eq!(messages[3]["snapshot"]["revision"], 2); + assert_eq!(messages[0]["outcome"], "applied"); + assert_eq!(messages[0]["snapshot"]["revision"], 2); + assert_eq!(messages[1]["outcome"], "rejected"); + assert_eq!(messages[1]["snapshot"]["revision"], 2); } #[test] diff --git a/src-tauri/crates/berd-call/src/openai_spokesperson.rs b/src-tauri/crates/berd-call/src/openai_spokesperson.rs index ba57624f6..1141185d6 100644 --- a/src-tauri/crates/berd-call/src/openai_spokesperson.rs +++ b/src-tauri/crates/berd-call/src/openai_spokesperson.rs @@ -226,7 +226,7 @@ pub enum SpokespersonResponseStatus { pub struct OpenAiSpokespersonRuntime { commands: mpsc::UnboundedSender, - audio: mpsc::Sender>, + audio: mpsc::Sender, worker: Option>, } @@ -367,8 +367,10 @@ impl OpenAiSpokespersonRuntime { command => command, }; match command { - SpokespersonCommand::InputPcm48Khz(samples) => { - self.audio.try_send(samples).map_err(|error| match error { + // Input and the clears that discard it share one queue, so they stay in order. + command @ (SpokespersonCommand::InputPcm48Khz(_) + | SpokespersonCommand::ResetInput { .. }) => { + self.audio.try_send(command).map_err(|error| match error { mpsc::error::TrySendError::Full(_) => "Spokesperson input queue is full".into(), mpsc::error::TrySendError::Closed(_) => "Spokesperson runtime is closed".into(), }) @@ -380,12 +382,11 @@ impl OpenAiSpokespersonRuntime { } } - pub fn reset_input(&self) -> Result<(), String> { + /// Clears the provider's input buffer of earlier input but not later input; the result arrives once the provider confirms. + pub fn reset_input(&self) -> Result>, String> { let (completed, result) = std::sync::mpsc::sync_channel(1); self.send(SpokespersonCommand::ResetInput { completed })?; - result - .recv_timeout(std::time::Duration::from_secs(5)) - .map_err(|_| "Spokesperson input reset timed out".to_string())? + Ok(result) } pub fn abort_input_cutover(&self) -> Result<(), String> { @@ -425,7 +426,7 @@ impl Drop for OpenAiSpokespersonRuntime { fn reap_spokesperson_worker( worker: thread::JoinHandle<()>, - audio_lifetime: mpsc::Sender>, + audio_lifetime: mpsc::Sender, ) { let _ = thread::Builder::new() .name("berd-call-spokesperson-reaper".into()) @@ -445,10 +446,21 @@ async fn run( run_inner(config, commands, audio_rx, events, false).await } +/// The next command, or `None` once the runtime is shut down. +async fn next_command( + commands: &mut mpsc::UnboundedReceiver, + audio: &mut mpsc::Receiver, +) -> Result, String> { + tokio::select! { + command = commands.recv() => Ok(command), + command = audio.recv() => command.map(Some).ok_or_else(|| "Spokesperson input queue is closed".into()), + } +} + async fn run_inner( mut config: OpenAiSpokespersonConfig, mut commands: mpsc::UnboundedReceiver, - mut audio: mpsc::Receiver>, + mut audio: mpsc::Receiver, events: &std::sync::mpsc::Sender, forward_provider_events: bool, ) -> Result<(), String> { @@ -590,18 +602,8 @@ async fn run_inner( return Err("Spokesperson output truncation timed out; server context is indeterminate".into()); } } - samples = audio.recv(), if shutdown_deadline.is_none() => { - let Some(samples) = samples else { - return Err("Spokesperson input queue is closed".into()); - }; - let pcm = downsample_pcm16(&samples); - send_json(&mut socket, serde_json::json!({ - "type": "input_audio_buffer.append", - "audio": BASE64.encode(pcm), - })).await?; - } - command = commands.recv(), if shutdown_deadline.is_none() => { - match command { + command = next_command(&mut commands, &mut audio), if shutdown_deadline.is_none() => { + match command? { Some(SpokespersonCommand::Provider(event)) => { send_json(&mut socket, event).await?; } @@ -719,7 +721,8 @@ async fn run_inner( Some(SpokespersonCommand::Shutdown) | None => { commands.close(); audio.close(); - while let Ok(samples) = audio.try_recv() { + while let Ok(command) = audio.try_recv() { + let SpokespersonCommand::InputPcm48Khz(samples) = command else { continue }; let pcm = downsample_pcm16(&samples); send_json(&mut socket, serde_json::json!({ "type": "input_audio_buffer.append", @@ -1942,6 +1945,49 @@ mod tests { server.await.unwrap(); } + #[tokio::test(flavor = "multi_thread")] + async fn input_reset_clears_input_sent_before_it_but_not_after() { + const FRAMES: usize = 10; + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("ws://{}/", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = accept_hdr_async(stream, require_test_authorization) + .await + .unwrap(); + let update = receive_json(&mut socket).await; + acknowledge_initial_session(&mut socket, &update, "test-model").await; + let mut types = Vec::new(); + for _ in 0..=2 * FRAMES { + types.push(receive_json(&mut socket).await["type"].clone()); + } + send_json(&mut socket, json!({"type":"input_audio_buffer.cleared"})).await; + let _ = socket.next().await; + types + }); + let (runtime, events) = + OpenAiSpokespersonRuntime::spawn(test_config(endpoint, "test-voice", 1.0, Vec::new())) + .unwrap(); + assert!(matches!( + events.recv_timeout(Duration::from_secs(2)).unwrap(), + SpokespersonEvent::Ready + )); + let pcm = || SpokespersonCommand::InputPcm48Khz(vec![0.5; 960]); + (0..FRAMES).for_each(|_| runtime.send(pcm()).unwrap()); + let reset = runtime.reset_input().unwrap(); + (0..FRAMES).for_each(|_| runtime.send(pcm()).unwrap()); + tokio::task::spawn_blocking(move || reset.recv_timeout(Duration::from_secs(2))) + .await + .unwrap() + .unwrap() + .unwrap(); + runtime.finish().unwrap(); + let append = json!("input_audio_buffer.append"); + let mut expected = vec![append.clone(); 2 * FRAMES]; + expected.insert(FRAMES, json!("input_audio_buffer.clear")); + assert_eq!(server.await.unwrap(), expected); + } + #[test] fn output_truncation_timeout_is_bounded() { let now = tokio::time::Instant::now(); diff --git a/src-tauri/crates/berd-call/tests/session_protocol.rs b/src-tauri/crates/berd-call/tests/session_protocol.rs index 24b271bc8..e21b0df77 100644 --- a/src-tauri/crates/berd-call/tests/session_protocol.rs +++ b/src-tauri/crates/berd-call/tests/session_protocol.rs @@ -2217,6 +2217,71 @@ fn voice_cutover_pcm_overflow_rolls_back_every_frame_to_the_old_runtime() { server.join().unwrap(); } +/// The host resets input whenever it restarts the microphone, as when the output device changes, and keeps sending audio while the provider confirms. +#[test] +fn microphone_audio_keeps_flowing_while_the_provider_confirms_an_input_reset() { + const FRAMES: usize = 40; + let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); + let (clear_seen_tx, clear_seen_rx) = mpsc::sync_channel(1); + let (appended_tx, appended_rx) = mpsc::sync_channel(1); + let server = std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + endpoint_tx + .send(format!("ws://{}/", listener.local_addr().unwrap())) + .unwrap(); + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = accept_async(stream).await.unwrap(); + let update = receive_realtime_json(&mut socket).await; + acknowledge_realtime_session(&mut socket, &update).await; + let clear = receive_realtime_json(&mut socket).await; + assert_eq!(clear["type"], "input_audio_buffer.clear"); + clear_seen_tx.send(()).unwrap(); + // Longer than the session can buffer microphone audio. + tokio::time::sleep(Duration::from_millis(1_200)).await; + send_realtime_json(&mut socket, json!({"type":"input_audio_buffer.cleared"})).await; + let mut appended = 0; + while let Some(Ok(Message::Text(text))) = socket.next().await { + let message: Value = serde_json::from_str(&text).unwrap(); + if message["type"] == "input_audio_buffer.append" + && BASE64 + .decode(message["audio"].as_str().unwrap()) + .unwrap() + .iter() + .any(|sample| *sample != 0) + { + appended += 1; + } + } + appended_tx.send(appended).unwrap(); + }); + }); + let endpoint = endpoint_rx.recv().unwrap(); + let mut session = ExpertSpokespersonTestSession::start(endpoint); + session.send(json!({"type":"reset_input","id":3})); + clear_seen_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + for _ in 0..FRAMES { + session.send_pcm(0.25); + session.flush(); + std::thread::sleep(Duration::from_millis(20)); + } + assert_eq!( + session.recv(Duration::from_secs(3)), + json!({"type":"input_reset_applied","id":3}) + ); + let messages = session.shutdown_and_collect(); + assert!( + messages.iter().all(|message| message["type"] != "fatal"), + "{messages:?}" + ); + server.join().unwrap(); + assert_eq!(appended_rx.recv().unwrap(), FRAMES); +} + #[test] fn reset_input_during_voice_cutover_rolls_back_then_clears_the_old_runtime() { let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); From 4f54948ebcabfc76a6121d237fc273c512846a34 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 11:29:38 -0400 Subject: [PATCH 29/49] Adapt the Expert reply tests to direct agent speech The tests restacked from below still started sessions with `--expert-voice tts` and built speech requests without `relay`. Signed-off-by: John Tennant Co-Authored-By: Claude Opus 5.5 --- src-tauri/crates/berd-call/src/main.rs | 1 + src-tauri/crates/berd-call/tests/session_protocol.rs | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 65ce8660c..5405fdaf1 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -9753,6 +9753,7 @@ mod tests { resolved_handoff_ids: Vec::new(), voice: Some(voice.into()), language: None, + relay: false, }) }; let next = || { diff --git a/src-tauri/crates/berd-call/tests/session_protocol.rs b/src-tauri/crates/berd-call/tests/session_protocol.rs index e21b0df77..041cb5c57 100644 --- a/src-tauri/crates/berd-call/tests/session_protocol.rs +++ b/src-tauri/crates/berd-call/tests/session_protocol.rs @@ -3327,7 +3327,8 @@ fn expert_voice_reply_whose_audio_never_settles_ends_the_session() { release, at_drain: true, }), - &["--expert-voice", "tts", "--tts-url", &speech_url], + false, + &["--allow-direct-agent-speech", "--tts-url", &speech_url], ); session.send(json!({ "type":"prepare_speak", From 986a17edfca3f0811c53baf8acf0a64ea2791c13 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 11:34:00 -0400 Subject: [PATCH 30/49] Say a held message before retrying the voice change it waited for In Berd mode, a message-spokesperson that arrived while a Spokesperson voice change was being built was held for the change, but the held message made the change unsafe. The change was rejected, queued and restarted at once, so neither could ever go through. Let a held directive go before a queued change starts again, and keep queued changes from holding directives back. Signed-off-by: John Tennant Co-Authored-By: Claude Opus 5.5 --- src-tauri/crates/berd-call/src/main.rs | 10 ++- .../berd-call/tests/session_protocol.rs | 69 +++++++++++++++++++ 2 files changed, 76 insertions(+), 3 deletions(-) diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 5405fdaf1..870ccf737 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -3285,6 +3285,7 @@ fn queued_spokesperson_settings_are_ready( responses: &HashMap, directive_speeches: &HashMap, cancelled_directives: &HashSet, + expert_tts: bool, ) -> bool { let work = spokesperson_host_work( gate, @@ -3293,7 +3294,8 @@ fn queued_spokesperson_settings_are_ready( directive_speeches, cancelled_directives, ); - gate.lifecycle.queued_settings_are_ready(work) + // A held directive would make the change unsafe, so it goes first. + !(work.pending_expert_prepare && !expert_tts) && gate.lifecycle.queued_settings_are_ready(work) } fn validate_queued_spokesperson_settings( @@ -3539,6 +3541,7 @@ fn run_expert_spokesperson_session( &responses, &directive_speeches, &cancelled_directives, + reply_tts.is_some(), ); if let Some(request) = queued_tts_settings.take_ready(pending_voice_update.is_some(), queued_settings_ready) @@ -4374,7 +4377,7 @@ fn run_expert_spokesperson_session( if let Some(request) = turn_gate.take_ready( active.is_some() || pending_voice_update.is_some() - || queued_tts_settings.is_pending(), + || reply_tts.is_some() && queued_tts_settings.is_pending(), responses.len(), ) { submit_expert_prepare( @@ -4614,7 +4617,8 @@ fn run_expert_spokesperson_session( active.as_ref(), &responses, ), - pending_voice_update.is_some() || queued_tts_settings.is_pending(), + pending_voice_update.is_some() + || reply_tts.is_some() && queued_tts_settings.is_pending(), active.is_some(), responses.len(), ); diff --git a/src-tauri/crates/berd-call/tests/session_protocol.rs b/src-tauri/crates/berd-call/tests/session_protocol.rs index 041cb5c57..195c460e4 100644 --- a/src-tauri/crates/berd-call/tests/session_protocol.rs +++ b/src-tauri/crates/berd-call/tests/session_protocol.rs @@ -1756,6 +1756,75 @@ fn a_spokesperson_rate_change_the_user_overtook_is_retried_beside_the_agents_voi ); } +#[test] +fn a_message_held_for_a_voice_change_is_said_before_the_change_is_retried() { + let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); + let (candidate_seen_tx, candidate_seen_rx) = mpsc::sync_channel(1); + let (held_tx, held_rx) = mpsc::sync_channel(1); + let (first_tx, first_rx) = mpsc::sync_channel(1); + std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + endpoint_tx + .send(format!("ws://{}/", listener.local_addr().unwrap())) + .unwrap(); + let (old_stream, _) = listener.accept().await.unwrap(); + let mut old = accept_async(old_stream).await.unwrap(); + let initial = receive_realtime_json(&mut old).await; + acknowledge_realtime_session(&mut old, &initial).await; + let (candidate_stream, _) = listener.accept().await.unwrap(); + let mut candidate = accept_async(candidate_stream).await.unwrap(); + let update = receive_realtime_json(&mut candidate).await; + candidate_seen_tx.send(()).unwrap(); + tokio::task::spawn_blocking(move || held_rx.recv().unwrap()) + .await + .unwrap(); + acknowledge_realtime_session(&mut candidate, &update).await; + let said = async { + while !receive_realtime_json(&mut old) + .await + .to_string() + .contains("I checked the repository.") + {} + }; + let first = tokio::select! { + () = said => "said", + _ = listener.accept() => "retried", + }; + first_tx.send(first).unwrap(); + while old.next().await.is_some() {} + }); + }); + let mut session = ExpertSpokespersonTestSession::start(endpoint_rx.recv().unwrap()); + session.send(json!({ + "type":"set_tts_settings", + "id":2, + "expected_revision":1, + "settings":{"backend":"openai","model":"test-model","voice":"new-voice","rate":1.0} + })); + candidate_seen_rx + .recv_timeout(Duration::from_secs(2)) + .unwrap(); + // The message waits for the voice change, and the change can't land while the message waits. + session.send(json!({ + "type":"prepare_speak", + "id":3, + "acknowledgement":null, + "text":"I checked the repository.", + "relay":true + })); + session.flush(); + held_tx.send(()).unwrap(); + assert_eq!( + first_rx.recv_timeout(Duration::from_secs(3)).unwrap(), + "said" + ); +} + /// Starts a call with `args`, makes the Spokesperson change in `request`, and speaks while its session is being built: the change rolls back, then lands once the call is idle. fn retries_a_voice_change_the_user_overtook(args: &[&str], request: Value) { let output = request["settings"].clone(); From cad868b178762e2862726b685e2830df381362c8 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 11:39:47 -0400 Subject: [PATCH 31/49] Give each input reset its own clear A reset that arrived while another clear was outstanding was folded into one follow-up clear sent only after the first was confirmed, so audio heard between the two controls survived the second reset. Send every clear where its control arrives and answer the queued controls once the provider has confirmed them all, in order. Signed-off-by: John Tennant Co-Authored-By: Claude Opus 5.5 --- src-tauri/crates/berd-call/src/main.rs | 64 ++++++----------- .../berd-call/src/openai_spokesperson.rs | 71 +++++++++++++++---- 2 files changed, 80 insertions(+), 55 deletions(-) diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 870ccf737..f1865c15c 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -3162,17 +3162,16 @@ fn spokesperson_pcm_allowed( } /// Input controls waiting on the provider to clear its input buffer, so microphone audio keeps flowing meanwhile. -/// Controls answer in order once no clear is outstanding; one that needs a clear while another is outstanding gets the next. +/// Each clear goes out where its control arrived; controls answer in order once no clear is outstanding. #[derive(Default)] struct InputResets { - clearing: Option>>, - again: bool, + clearing: VecDeque>>, replies: Vec, } impl InputResets { fn idle(&self) -> bool { - self.clearing.is_none() + self.clearing.is_empty() } fn apply( @@ -3182,38 +3181,26 @@ impl InputResets { reset_input: impl FnOnce() -> Result>, String>, writer: &mut impl Write, ) -> Result<(), String> { - if self.idle() { - if !clear { - return write_message(writer, &reply); - } - self.clearing = Some(reset_input()?); - } else { - self.again |= clear; + if clear { + self.clearing.push_back(reset_input()?); + } else if self.idle() { + return write_message(writer, &reply); } self.replies.push(reply); Ok(()) } - fn poll( - &mut self, - reset_input: impl FnOnce() -> Result>, String>, - writer: &mut impl Write, - ) -> Result<(), String> { - let Some(clearing) = &self.clearing else { - return Ok(()); - }; - match clearing.try_recv() { - Err(mpsc::TryRecvError::Empty) => return Ok(()), - Err(mpsc::TryRecvError::Disconnected) => { - return Err("Spokesperson runtime is closed".into()) + fn poll(&mut self, writer: &mut impl Write) -> Result<(), String> { + while let Some(clearing) = self.clearing.front() { + match clearing.try_recv() { + Err(mpsc::TryRecvError::Empty) => return Ok(()), + Err(mpsc::TryRecvError::Disconnected) => { + return Err("Spokesperson runtime is closed".into()) + } + Ok(result) => result?, } - Ok(result) => result?, - } - if std::mem::take(&mut self.again) { - self.clearing = Some(reset_input()?); - return Ok(()); + self.clearing.pop_front(); } - self.clearing = None; self.replies .drain(..) .try_for_each(|reply| write_message(writer, &reply)) @@ -3419,10 +3406,7 @@ fn run_expert_spokesperson_session( loop { if initialized { - input_resets.poll( - || runtime.as_ref().expect("initialized runtime").reset_input(), - &mut writer, - )?; + input_resets.poll(&mut writer)?; let work = spokesperson_host_work( &turn_gate, active.as_ref(), @@ -9099,17 +9083,15 @@ mod tests { &mut output, ) .unwrap(); - resets.poll(reset_input, &mut output).unwrap(); - assert_eq!(clears.borrow().len(), 1); + // The reset asked for during the first clear gets a clear of its own, at once. + resets.poll(&mut output).unwrap(); + assert_eq!(clears.borrow().len(), 2); assert!(output.is_empty()); - - // The reset asked for during the first clear needs a clear of its own. clears.borrow()[0].send(Ok(())).unwrap(); - resets.poll(reset_input, &mut output).unwrap(); - assert_eq!(clears.borrow().len(), 2); + resets.poll(&mut output).unwrap(); assert!(output.is_empty()); clears.borrow()[1].send(Ok(())).unwrap(); - resets.poll(reset_input, &mut output).unwrap(); + resets.poll(&mut output).unwrap(); assert!(resets.idle()); assert_eq!( messages(&output), @@ -9150,7 +9132,7 @@ mod tests { .send(Err("provider refused".into())) .unwrap(); assert_eq!( - resets.poll(reset_input, &mut output).unwrap_err(), + resets.poll(&mut output).unwrap_err(), "provider refused" ); } diff --git a/src-tauri/crates/berd-call/src/openai_spokesperson.rs b/src-tauri/crates/berd-call/src/openai_spokesperson.rs index 1141185d6..2c6c62737 100644 --- a/src-tauri/crates/berd-call/src/openai_spokesperson.rs +++ b/src-tauri/crates/berd-call/src/openai_spokesperson.rs @@ -500,7 +500,8 @@ async fn run_inner( let mut cancellation_events = HashMap::::new(); let mut pending_truncations = HashMap::<(String, u64), PendingTruncation>::new(); let mut pending_input_cutover: Option = None; - let mut pending_input_reset: Option = None; + // Clears in the order sent; the provider confirms them in that order. + let mut pending_input_resets = VecDeque::::new(); let mut started_input_items = HashSet::::new(); let mut committed_input_items = HashSet::::new(); let mut seed_turns = VecDeque::from(std::mem::take(&mut config.semantic_transcript)); @@ -536,7 +537,7 @@ async fn run_inner( let cutover_deadline = pending_input_cutover .as_ref() .map(|cutover| cutover.deadline); - let reset_deadline = pending_input_reset.as_ref().map(|reset| reset.deadline); + let reset_deadline = pending_input_resets.front().map(|reset| reset.deadline); tokio::select! { _ = async { match shutdown_not_before { @@ -561,11 +562,11 @@ async fn run_inner( None => std::future::pending().await, } }, if reset_deadline.is_some() => { - if pending_input_reset - .as_ref() + if pending_input_resets + .front() .is_some_and(|reset| tokio::time::Instant::now() >= reset.deadline) { - let reset = pending_input_reset.take().expect("expired input reset"); + let reset = pending_input_resets.pop_front().expect("expired input reset"); let _ = reset.completed.send(Err("Spokesperson input reset timed out".into())); return Err("Spokesperson input reset timed out; provider input state is indeterminate".into()); } @@ -615,7 +616,7 @@ async fn run_inner( })).await?; } Some(SpokespersonCommand::ResetInput { completed }) => { - if pending_input_cutover.is_some() || pending_input_reset.is_some() { + if pending_input_cutover.is_some() { let _ = completed.send(Err( "another Spokesperson input clear is in progress".into(), )); @@ -627,7 +628,7 @@ async fn run_inner( "event_id": event_id, "type": "input_audio_buffer.clear", })).await?; - pending_input_reset = Some(PendingInputReset { + pending_input_resets.push_back(PendingInputReset { event_id, deadline: tokio::time::Instant::now() + CONTROL_ACK_TIMEOUT, completed, @@ -648,7 +649,7 @@ async fn run_inner( } } Some(SpokespersonCommand::BeginInputCutover { request_id }) => { - if pending_input_cutover.is_some() || pending_input_reset.is_some() { + if pending_input_cutover.is_some() || !pending_input_resets.is_empty() { return Err("Spokesperson input cutover was requested twice".into()); } let event_id = format!("berd-cutover-{next_control_event_id}"); @@ -929,7 +930,7 @@ async fn run_inner( &committed_input_items, events, )?; - } else if let Some(reset) = pending_input_reset.take() { + } else if let Some(reset) = pending_input_resets.pop_front() { let _ = reset.completed.send(Ok(())); } } @@ -1042,11 +1043,9 @@ async fn run_inner( } "error" => { let message = value.pointer("/error/message").and_then(|value| value.as_str()).unwrap_or("OpenAI Realtime failed").to_string(); - if pending_input_reset.as_ref().is_some_and(|reset| { - value.pointer("/error/event_id").and_then(|value| value.as_str()) - == Some(reset.event_id.as_str()) - }) { - let reset = pending_input_reset.take().expect("matched input reset"); + let event_id = value.pointer("/error/event_id").and_then(|value| value.as_str()); + if let Some(index) = pending_input_resets.iter().position(|reset| Some(reset.event_id.as_str()) == event_id) { + let reset = pending_input_resets.remove(index).expect("matched input reset"); let _ = reset.completed.send(Err(message)); } else if pending_input_cutover.as_ref().is_some_and(|cutover| { value.pointer("/error/event_id").and_then(|value| value.as_str()) @@ -1988,6 +1987,50 @@ mod tests { assert_eq!(server.await.unwrap(), expected); } + #[tokio::test(flavor = "multi_thread")] + async fn input_resets_sent_together_are_each_answered_in_turn() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("ws://{}/", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = accept_hdr_async(stream, require_test_authorization) + .await + .unwrap(); + let update = receive_json(&mut socket).await; + acknowledge_initial_session(&mut socket, &update, "test-model").await; + let first = receive_json(&mut socket).await; + let second = receive_json(&mut socket).await; + assert_eq!(first["type"], "input_audio_buffer.clear"); + assert_eq!(second["type"], "input_audio_buffer.clear"); + send_json( + &mut socket, + json!({"type":"error","error":{"message":"refused","event_id":second["event_id"]}}), + ) + .await; + send_json(&mut socket, json!({"type":"input_audio_buffer.cleared"})).await; + let _ = socket.next().await; + }); + let (runtime, events) = + OpenAiSpokespersonRuntime::spawn(test_config(endpoint, "test-voice", 1.0, Vec::new())) + .unwrap(); + assert!(matches!( + events.recv_timeout(Duration::from_secs(2)).unwrap(), + SpokespersonEvent::Ready + )); + let first = runtime.reset_input().unwrap(); + let second = runtime.reset_input().unwrap(); + let wait = |reset: std::sync::mpsc::Receiver>| { + reset.recv_timeout(Duration::from_secs(2)).unwrap() + }; + let (first, second) = tokio::task::spawn_blocking(move || (wait(first), wait(second))) + .await + .unwrap(); + assert_eq!(first, Ok(())); + assert_eq!(second, Err("refused".into())); + runtime.finish().unwrap(); + server.await.unwrap(); + } + #[test] fn output_truncation_timeout_is_bounded() { let now = tokio::time::Instant::now(); From edd6abf66070ae629dd041a533e79ed65015c52d Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 11:41:28 -0400 Subject: [PATCH 32/49] Adapt the Expert reply interruption tests to direct agent speech Signed-off-by: John Tennant Co-Authored-By: Claude Opus 5.5 --- src-tauri/crates/berd-call/tests/session_protocol.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src-tauri/crates/berd-call/tests/session_protocol.rs b/src-tauri/crates/berd-call/tests/session_protocol.rs index 195c460e4..04597afac 100644 --- a/src-tauri/crates/berd-call/tests/session_protocol.rs +++ b/src-tauri/crates/berd-call/tests/session_protocol.rs @@ -3317,7 +3317,8 @@ fn interrupt_expert_reply(interruption: ReplyInterruption) -> (Value, usize) { release, at_drain: true, }), - &["--expert-voice", "tts", "--tts-url", &speech_url], + false, + &["--allow-direct-agent-speech", "--tts-url", &speech_url], ); let text = "I checked the repository and found three failing tests in the parser."; session.send(json!({"type":"prepare_speak","id":2,"acknowledgement":null,"text":text})); From 33a47fe089a7e8dec4e5141b11eb77bbea829161 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 11:46:45 -0400 Subject: [PATCH 33/49] Format the input reset test Signed-off-by: John Tennant Co-Authored-By: Claude Opus 5.5 --- src-tauri/crates/berd-call/src/main.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index f1865c15c..30855a02e 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -9131,10 +9131,7 @@ mod tests { clears.borrow()[2] .send(Err("provider refused".into())) .unwrap(); - assert_eq!( - resets.poll(&mut output).unwrap_err(), - "provider refused" - ); + assert_eq!(resets.poll(&mut output).unwrap_err(), "provider refused"); } #[test] From d3189a0f0f8972b0c97f3dcd0ba6c2e61785abcd Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 11:52:46 -0400 Subject: [PATCH 34/49] Let a stopped playback worker's result settle its reply When the host's player stops, the Spokesperson playback worker sends its result and stops taking audio. Provider audio that arrived in the meantime hit the closed worker first and ended the call, before the loop read the result that interrupts the reply where it was heard. Drop that audio instead and let the result settle the reply. Signed-off-by: John Tennant Co-Authored-By: Claude Opus 5.5 --- src-tauri/crates/berd-call/src/main.rs | 38 ++++++++++++++++++-------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 30855a02e..c8f67a226 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -3792,12 +3792,8 @@ fn run_expert_spokesperson_session( .expect("streaming response state exists") .queue_audio(samples, total_pending_frames)?; } - Err(mpsc::TrySendError::Disconnected(_)) => { - return Err( - "Spokesperson playback worker closed while streaming" - .into(), - ) - } + // The worker has stopped; its result settles this reply. + Err(mpsc::TrySendError::Disconnected(_)) => {} Err(mpsc::TrySendError::Full(LivePlaybackInput::Finish)) => { unreachable!("streaming sends samples") } @@ -5624,8 +5620,11 @@ fn flush_active_live_playback( response.pending_audio.push_front(samples); break; } + // The worker has stopped, and the result it sent first settles this reply. Err(mpsc::TrySendError::Disconnected(_)) => { - return Err("Spokesperson playback worker closed while streaming".into()) + response.pending_audio.clear(); + response.pending_frames = 0; + return Ok(()); } Err(mpsc::TrySendError::Full(LivePlaybackInput::Finish)) => { unreachable!("audio queue contains samples") @@ -5635,10 +5634,10 @@ fn flush_active_live_playback( if response.audio_done && response.pending_audio.is_empty() && !response.finish_sent { match playback.sender.try_send(LivePlaybackInput::Finish) { Ok(()) => response.finish_sent = true, - Err(mpsc::TrySendError::Full(LivePlaybackInput::Finish)) => {} - Err(mpsc::TrySendError::Disconnected(_)) => { - return Err("Spokesperson playback worker closed before finish".into()) - } + Err( + mpsc::TrySendError::Full(LivePlaybackInput::Finish) + | mpsc::TrySendError::Disconnected(_), + ) => {} Err(mpsc::TrySendError::Full(LivePlaybackInput::Samples(_))) => { unreachable!("finish queue contains finish") } @@ -9307,6 +9306,23 @@ mod tests { assert_eq!(gate.take_ready(false, 0).map(|request| request.id), Some(7)); } + #[test] + fn audio_for_a_playback_worker_that_stopped_waits_for_its_result() { + let (playback, _controls) = live_playback_fixture(None); + let (sender, receiver) = mpsc::sync_channel(1); + drop(receiver); + let active = Some(LivePlayback { sender, ..playback }); + let mut response = LiveResponse::new(None, Some(3)); + response.queue_audio(vec![0.25; 32], 0).unwrap(); + mark_live_response_server_finished(&mut response); + let mut responses = HashMap::from([("response-a".into(), response)]); + + // The worker sent the result that settles this reply before it stopped taking audio. + flush_active_live_playback(&active, &mut responses).unwrap(); + + assert!(responses["response-a"].pending_audio.is_empty()); + } + #[test] fn response_done_finishes_playback_without_a_separate_audio_done() { let (playback, _controls) = live_playback_fixture(None); From 0fc4238fdd61dba0009110d91b351054a6dd1ed7 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 13:20:58 -0400 Subject: [PATCH 35/49] Adapt the Expert reply fixes to direct agent speech The output_ready gate and failure tests restacked from below used the reply's old optional text and started sessions with `--expert-voice tts`. A call that lets the agent speak holds no handoff open, so the failure tests now check only what the Spokesperson is told. Signed-off-by: John Tennant Co-Authored-By: Claude Opus 5.5 --- src-tauri/crates/berd-call/src/main.rs | 4 +- .../berd-call/tests/session_protocol.rs | 49 +++++-------------- 2 files changed, 14 insertions(+), 39 deletions(-) diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index c8f67a226..11ae20517 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -4628,7 +4628,7 @@ fn run_expert_spokesperson_session( response.prepare_id == Some(id) && response.speech_id == Some(speech_id) }; let reply = responses.iter_mut().find(|(_, response)| { - response.reply.is_some() && !response.output_ready && matches(response) + response.reply && !response.output_ready && matches(response) }); let valid = if let Some((response_id, response)) = reply { response.output_ready = true; @@ -4640,7 +4640,7 @@ fn run_expert_spokesperson_session( .is_some_and(|directive| directive.speech_id == speech_id) || responses .values() - .any(|response| response.reply.is_none() && matches(response)) + .any(|response| !response.reply && matches(response)) }; write_message( &mut writer, diff --git a/src-tauri/crates/berd-call/tests/session_protocol.rs b/src-tauri/crates/berd-call/tests/session_protocol.rs index 04597afac..964da9e72 100644 --- a/src-tauri/crates/berd-call/tests/session_protocol.rs +++ b/src-tauri/crates/berd-call/tests/session_protocol.rs @@ -3590,7 +3590,7 @@ fn expert_voice_reply_waits_for_output_ready_before_playing() { }); let mut session = ExpertSpokespersonTestSession::start_with_args( endpoint_rx.recv().unwrap(), - &["--expert-voice", "tts", "--tts-url", &speech_url], + &["--allow-direct-agent-speech", "--tts-url", &speech_url], ); session.send(json!({ "type":"prepare_speak", @@ -3635,9 +3635,9 @@ fn expert_voice_reply_waits_for_output_ready_before_playing() { session.shutdown(); } -/// Fails an Expert reply answering a handoff once the host has played `played_frames` of it, -/// returning the failure, what the Spokesperson was told, and the handoffs left open. -fn fail_expert_reply_after(played_frames: u64) -> (Value, String, Value) { +/// Fails an Expert reply once the host has played `played_frames` of it, returning the +/// failure and what the Spokesperson was told. +fn fail_expert_reply_after(played_frames: u64) -> (Value, String) { let (speech_url, _speech_requests) = spawn_speech_server(); let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); let (context_tx, context_rx) = mpsc::sync_channel(1); @@ -3655,17 +3655,6 @@ fn fail_expert_reply_after(played_frames: u64) -> (Value, String, Value) { let mut socket = accept_async(stream).await.unwrap(); let initial = receive_realtime_json(&mut socket).await; acknowledge_realtime_session(&mut socket, &initial).await; - send_realtime_json( - &mut socket, - json!({ - "type":"response.function_call_arguments.done", - "response_id":"response-handoff", - "call_id":"handoff-reply", - "name":"handoff", - "arguments":"{\"message\":\"inspect the repository\"}" - }), - ) - .await; let context = loop { let text = receive_realtime_json(&mut socket).await.to_string(); if text.contains("spoken reply") || text.contains("in its own voice") { @@ -3688,19 +3677,14 @@ fn fail_expert_reply_after(played_frames: u64) -> (Value, String, Value) { release, at_drain: true, }), - &["--expert-voice", "tts", "--tts-url", &speech_url], - ); - assert_eq!(session.recv(Duration::from_secs(2))["origin"], "handoff"); - assert_eq!( - session.recv(Duration::from_secs(2))["handoff_ids"], - json!(["handoff-external-1"]) + false, + &["--allow-direct-agent-speech", "--tts-url", &speech_url], ); session.send(json!({ "type":"prepare_speak", "id":2, - "acknowledgement":1, - "text":"I checked the repository and found three failing tests in the parser.", - "resolved_handoff_ids":["handoff-external-1"] + "acknowledgement":null, + "text":"I checked the repository and found three failing tests in the parser." })); let admitted = session.recv(Duration::from_secs(2)); assert_eq!(admitted["type"], "admitted", "{admitted}"); @@ -3727,35 +3711,26 @@ fn fail_expert_reply_after(played_frames: u64) -> (Value, String, Value) { } }; let context = context_rx.recv_timeout(Duration::from_secs(2)).unwrap(); - session.send(json!({"type":"query_state","id":3,"after":0})); - let state = loop { - let message = session.recv(Duration::from_secs(2)); - if message["type"] == "state" { - break message; - } - }; session.shutdown(); - (failed, context, state["unresolved_handoff_ids"].clone()) + (failed, context) } #[test] fn a_failed_expert_reply_tells_the_spokesperson_what_was_heard() { - let (failed, context, unresolved) = fail_expert_reply_after(1_000); + let (failed, context) = fail_expert_reply_after(1_000); assert_eq!(failed["id"], 2); assert!( context.contains("in its own voice") && context.contains("I checked the repository"), "{context}" ); assert!(!context.contains("in the parser"), "{context}"); - assert!(unresolved.is_null(), "{unresolved}"); } #[test] -fn an_expert_reply_that_fails_unheard_leaves_its_handoffs_open() { - let (failed, context, unresolved) = fail_expert_reply_after(0); +fn an_expert_reply_that_fails_unheard_tells_the_spokesperson_so() { + let (failed, context) = fail_expert_reply_after(0); assert_eq!(failed["id"], 2); assert!(context.contains("before hearing any of it"), "{context}"); - assert_eq!(unresolved, json!(["handoff-external-1"])); } #[test] From 4ea7f17ab795deaf8ca2438c6a154eb9b2f77312 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 11:14:43 -0400 Subject: [PATCH 36/49] Interrupt speech whose player stops instead of failing it When the speakers change mid-reply, macOS stops the audio engine and the host reports its player failed. A conventional call reported that as `speech_failed: host audio output failed`, so `speak` printed an error even though the call carried on. Treat a quiescent host failure like a barge-in: the speech ends as interrupted where it was heard. Signed-off-by: John Tennant Co-Authored-By: Claude Opus 5.5 --- src-tauri/crates/berd-call/src/main.rs | 76 +++++++++------- .../berd-call/tests/session_protocol.rs | 88 +++++++++++++++++++ 2 files changed, 131 insertions(+), 33 deletions(-) diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 11ae20517..3f5e22c8a 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -8349,40 +8349,50 @@ fn synthesize_to_output_with_finish( delivery: unplayed(), output_quiescent: false, })?; - if playback - .synthesize_segment(backend, text, &mut |_| Ok(()), on_started, &mut |_| Ok(())) - .map_err(|failure| PlaybackFailure { - message: failure.message, - delivery: failure.delivery, - output_quiescent: failure.output_quiescent, - })? - == OutboundOutcome::Interrupted - { - return Ok((false, playback.snapshot())); - } - if let Err(message) = finish_writes() { - let output_quiescent = output.cancel_and_snapshot().is_ok(); - return Err(PlaybackFailure { - message, - delivery: playback.snapshot(), - output_quiescent, - }); + let completed = (|| { + if playback + .synthesize_segment(backend, text, &mut |_| Ok(()), on_started, &mut |_| Ok(())) + .map_err(|failure| PlaybackFailure { + message: failure.message, + delivery: failure.delivery, + output_quiescent: failure.output_quiescent, + })? + == OutboundOutcome::Interrupted + { + return Ok(false); + } + if let Err(message) = finish_writes() { + let output_quiescent = output.cancel_and_snapshot().is_ok(); + return Err(PlaybackFailure { + message, + delivery: playback.snapshot(), + output_quiescent, + }); + } + let outcome = playback + .finish( + DrainPolicy { + timeout: Some(Duration::from_secs(2)), + timeout_outcome: DrainTimeoutOutcome::Fail, + ..DrainPolicy::default() + }, + &mut |_| Ok(()), + ) + .map_err(|failure| PlaybackFailure { + message: failure.message, + delivery: failure.delivery, + output_quiescent: failure.output_quiescent, + })?; + Ok(outcome == OutboundOutcome::Completed) + })(); + match completed { + Ok(completed) => Ok((completed, playback.snapshot())), + // Speech the host stopped playing, as when the speakers change, ends where it was cut off, as if you had talked over it. + Err(failure) if failure.message == AUDIO_HOST_FAILED && failure.output_quiescent => { + Ok((false, playback.snapshot())) + } + Err(failure) => Err(failure), } - let outcome = playback - .finish( - DrainPolicy { - timeout: Some(Duration::from_secs(2)), - timeout_outcome: DrainTimeoutOutcome::Fail, - ..DrainPolicy::default() - }, - &mut |_| Ok(()), - ) - .map_err(|failure| PlaybackFailure { - message: failure.message, - delivery: failure.delivery, - output_quiescent: failure.output_quiescent, - })?; - Ok((outcome == OutboundOutcome::Completed, playback.snapshot())) } #[cfg(test)] diff --git a/src-tauri/crates/berd-call/tests/session_protocol.rs b/src-tauri/crates/berd-call/tests/session_protocol.rs index 964da9e72..b6cb5fcbb 100644 --- a/src-tauri/crates/berd-call/tests/session_protocol.rs +++ b/src-tauri/crates/berd-call/tests/session_protocol.rs @@ -3166,6 +3166,94 @@ fn siri_remote_output_supports_consecutive_turns_and_cancellation() { audio_host.join().unwrap(); } +#[test] +fn speech_whose_player_stops_is_interrupted_where_it_was_heard_and_the_call_goes_on() { + let (speech_url, _speech_requests) = spawn_speech_server(); + let (mut command, pcm_guard, host) = session_command(); + let child = command + .args([ + "--tts-backend", + "openai", + "--tts-url", + &speech_url, + "--stt-backend", + "openai", + "--stt-url", + &spawn_transcription_server(), + ]) + .env("OPENAI_API_KEY", "test-key") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + drop(pcm_guard); + let mut child = ChildGuard(Some(child)); + let process = child.0.as_mut().unwrap(); + let stdin = Arc::new(Mutex::new(process.stdin.take().unwrap())); + let output = spawn_session_message_reader(process.stdout.take().unwrap()); + // The host's player stops partway through the first speech, as when the speakers change. + let audio_host = + spawn_audio_host_with_played_limit(host, Arc::clone(&stdin), Some(1_200), None, None, true); + let send = |message: Value| { + let mut stdin = stdin.lock().unwrap(); + write_session_json(&mut *stdin, &message); + stdin.flush().unwrap(); + }; + let terminal = |id: u64| loop { + let message = output.recv_timeout(Duration::from_secs(5)).unwrap(); + match message["type"].as_str() { + Some("admitted") => send(json!({ + "type":"output_ready","id":id,"speech_id":message["speech_id"] + })), + Some("output_ready_result" | "speech_started") => {} + _ => break message, + } + }; + send(json!({"type":"hello","id":1,"input_during_tts":"allow_barge_in"})); + let ready = output.recv_timeout(Duration::from_secs(5)).unwrap(); + assert_eq!(ready["type"], "ready", "{ready}"); + let text = "A reply cut off by a change of speakers."; + send(json!({"type":"prepare_speak","id":2,"acknowledgement":null,"text":text})); + let interrupted = terminal(2); + assert_eq!(interrupted["type"], "speech_interrupted", "{interrupted}"); + let heard = interrupted["spoken_through_utf8"].as_u64().unwrap(); + assert!(heard < text.len() as u64, "{interrupted}"); + send(json!({"type":"prepare_speak","id":3,"acknowledgement":null,"text":"Still here."})); + assert_eq!(terminal(3)["type"], "speech_completed"); + send(json!({"type":"shutdown"})); + drop(send); + drop(stdin); + assert!(child.0.take().unwrap().wait().unwrap().success()); + audio_host.join().unwrap(); +} + +/// Accepts one OpenAI transcription session that never hears anything. +fn spawn_transcription_server() -> String { + let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); + std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + endpoint_tx + .send(format!("ws://{}/", listener.local_addr().unwrap())) + .unwrap(); + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = accept_async(stream).await.unwrap(); + assert_eq!( + receive_realtime_json(&mut socket).await["type"], + "session.update" + ); + send_realtime_json(&mut socket, json!({"type":"session.updated"})).await; + while let Some(Ok(_)) = socket.next().await {} + }); + }); + endpoint_rx.recv().unwrap() +} + /// Serves OpenAI speech requests with a little silence, reporting each request body. fn spawn_speech_server() -> (String, mpsc::Receiver) { let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); From f014d273fcebc608eccbba60d7248d3a5be3e80c Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 11:43:15 -0400 Subject: [PATCH 37/49] Drop a no-op drop from the player-stop test Signed-off-by: John Tennant Co-Authored-By: Claude Opus 5.5 --- src-tauri/crates/berd-call/tests/session_protocol.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src-tauri/crates/berd-call/tests/session_protocol.rs b/src-tauri/crates/berd-call/tests/session_protocol.rs index b6cb5fcbb..d283222a4 100644 --- a/src-tauri/crates/berd-call/tests/session_protocol.rs +++ b/src-tauri/crates/berd-call/tests/session_protocol.rs @@ -3222,7 +3222,6 @@ fn speech_whose_player_stops_is_interrupted_where_it_was_heard_and_the_call_goes send(json!({"type":"prepare_speak","id":3,"acknowledgement":null,"text":"Still here."})); assert_eq!(terminal(3)["type"], "speech_completed"); send(json!({"type":"shutdown"})); - drop(send); drop(stdin); assert!(child.0.take().unwrap().wait().unwrap().success()); audio_host.join().unwrap(); From d4ab11eba0e932e3e20f0964dd102e41867cc8a0 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 13:23:59 -0400 Subject: [PATCH 38/49] Test an Expert reply whose player stops as interrupted A stopped host player now interrupts speech rather than failing it, so the Expert reply tests from below check that the reply is interrupted where it was heard and that the Spokesperson is told that much. Signed-off-by: John Tennant Co-Authored-By: Claude Opus 5.5 --- .../berd-call/tests/session_protocol.rs | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src-tauri/crates/berd-call/tests/session_protocol.rs b/src-tauri/crates/berd-call/tests/session_protocol.rs index d283222a4..034ba1479 100644 --- a/src-tauri/crates/berd-call/tests/session_protocol.rs +++ b/src-tauri/crates/berd-call/tests/session_protocol.rs @@ -3722,9 +3722,9 @@ fn expert_voice_reply_waits_for_output_ready_before_playing() { session.shutdown(); } -/// Fails an Expert reply once the host has played `played_frames` of it, returning the -/// failure and what the Spokesperson was told. -fn fail_expert_reply_after(played_frames: u64) -> (Value, String) { +/// Stops the host's player once it has played `played_frames` of an Expert reply, as when the +/// speakers change, returning how the reply ended and what the Spokesperson was told. +fn stop_expert_reply_after(played_frames: u64) -> (Value, String) { let (speech_url, _speech_requests) = spawn_speech_server(); let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); let (context_tx, context_rx) = mpsc::sync_channel(1); @@ -3784,28 +3784,28 @@ fn fail_expert_reply_after(played_frames: u64) -> (Value, String) { "played_frames":played_frames, "message":"output device disappeared" })); - let failed = loop { + let ended = loop { let message = session.recv(Duration::from_secs(2)); assert!( !matches!( message["type"].as_str(), - Some("fatal" | "speech_completed" | "speech_interrupted") + Some("fatal" | "speech_completed" | "speech_failed") ), "{message}" ); - if message["type"] == "speech_failed" { + if message["type"] == "speech_interrupted" { break message; } }; let context = context_rx.recv_timeout(Duration::from_secs(2)).unwrap(); session.shutdown(); - (failed, context) + (ended, context) } #[test] -fn a_failed_expert_reply_tells_the_spokesperson_what_was_heard() { - let (failed, context) = fail_expert_reply_after(1_000); - assert_eq!(failed["id"], 2); +fn an_expert_reply_whose_player_stops_tells_the_spokesperson_what_was_heard() { + let (ended, context) = stop_expert_reply_after(1_000); + assert_eq!(ended["id"], 2); assert!( context.contains("in its own voice") && context.contains("I checked the repository"), "{context}" @@ -3814,9 +3814,9 @@ fn a_failed_expert_reply_tells_the_spokesperson_what_was_heard() { } #[test] -fn an_expert_reply_that_fails_unheard_tells_the_spokesperson_so() { - let (failed, context) = fail_expert_reply_after(0); - assert_eq!(failed["id"], 2); +fn an_expert_reply_whose_player_stops_unheard_tells_the_spokesperson_so() { + let (ended, context) = stop_expert_reply_after(0); + assert_eq!(ended["id"], 2); assert!(context.contains("before hearing any of it"), "{context}"); } From e478c810f0882a7744ce919fb10b9f694da20519 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 12:09:40 -0400 Subject: [PATCH 39/49] berd-call: turn the Spokesperson on or off from the menu bar A Spokesperson checkmark in the menu bar, and `berd-call settings --spokesperson true|false`, restart the running call's session in the other mode. Agents keep the voice they have now, options only the other mode takes are left out, and a `--stream live` call gets the Spokesperson back beside the agents' own voice. Signed-off-by: John Tennant Co-Authored-By: Claude Opus 5.5 --- src-tauri/crates/berd-call/README.md | 5 + src-tauri/crates/berd-call/src/cli_help.rs | 9 +- .../crates/berd-call/src/host_control.rs | 10 ++ .../crates/berd-call/src/host_session.rs | 163 ++++++++++++++---- src-tauri/crates/berd-call/src/main.rs | 11 +- src-tauri/crates/berd-call/src/menu_bar.rs | 36 +++- .../crates/berd-call/src/saved_settings.rs | 83 +++++++++ 7 files changed, 279 insertions(+), 38 deletions(-) diff --git a/src-tauri/crates/berd-call/README.md b/src-tauri/crates/berd-call/README.md index 75d8628c6..aa918d9f5 100644 --- a/src-tauri/crates/berd-call/README.md +++ b/src-tauri/crates/berd-call/README.md @@ -109,6 +109,11 @@ complete `berd-call session` options. The call keeps its control port, stream, blocking policy, mute state, and input-during-TTS policy; the stream emits a lifecycle row because transcript cursors restart with the new session. +`berd-call settings --spokesperson true|false`, or Spokesperson in the menu bar, +restarts the call the same way in the other mode. Agents keep their current +voice, options only the other mode takes are left out, and in a `--stream live` +call the Spokesperson comes back with `--allow-direct-agent-speech`. + This crate owns the neutral PCM output contract and backend-neutral TTS stream used by Berd, plus the April ONNX runtime and text chunking used by Berd's native voice commands. It also owns the concrete Parakeet model loader and complete diff --git a/src-tauri/crates/berd-call/src/cli_help.rs b/src-tauri/crates/berd-call/src/cli_help.rs index 5c2ffc270..3aa05d756 100644 --- a/src-tauri/crates/berd-call/src/cli_help.rs +++ b/src-tauri/crates/berd-call/src/cli_help.rs @@ -95,13 +95,18 @@ Usage: berd-call settings [--port PORT] --tts JSON berd-call settings [--port PORT] --input-during-tts allow|suppress berd-call settings [--port PORT] --muted true|false + berd-call settings [--port PORT] --spokesperson true|false berd-call settings [--port PORT] --restart [session options] The setting applies to subsequent speak calls. In a call started with --allow-direct-agent-speech, --voice and --rate change the agents' voice, and --spokesperson-voice and --spokesperson-rate change the Spokesperson's (an -OpenAI realtime voice, 0.25 to 1.5). Non-blocking delivery requires -a call started with --stream or --codex. Successful delivery produces no notification."#; +OpenAI realtime voice, 0.25 to 1.5). --spokesperson turns the Spokesperson on +or off, restarting the call's session in the other mode with the same voice; +options only the other mode takes are dropped, and in a call started with +--stream live, agents keep their own voice beside the Spokesperson's. +Non-blocking delivery requires a call started with --stream or --codex. +Successful delivery produces no notification."#; const STATUS_HELP: &str = r#"Inspect the active local voice call. diff --git a/src-tauri/crates/berd-call/src/host_control.rs b/src-tauri/crates/berd-call/src/host_control.rs index 3cd9ea101..c288c8e34 100644 --- a/src-tauri/crates/berd-call/src/host_control.rs +++ b/src-tauri/crates/berd-call/src/host_control.rs @@ -44,6 +44,8 @@ pub(crate) trait HostControl: Send + Sync + 'static { ) -> Result; fn set_muted(&self, muted: bool) -> Result; fn restart(&self, session_arguments: Vec) -> Result; + /// Restarts the call's session with the Spokesperson on or off. + fn set_spokesperson_mode(&self, on: bool) -> Result; } pub(crate) struct ControlServer { @@ -154,6 +156,9 @@ pub(crate) enum ControlRequest { #[serde(rename = "sessionArguments")] session_arguments: Vec, }, + SpokespersonMode { + on: bool, + }, } #[derive(Deserialize, Serialize)] @@ -204,6 +209,7 @@ pub(crate) fn begin_request(port: u16, request: ControlRequest) -> Result Result ControlRequest::InputDuringTts { policy } => control.set_input_during_tts(policy), ControlRequest::Muted { muted } => control.set_muted(muted), ControlRequest::Restart { session_arguments } => control.restart(session_arguments), + ControlRequest::SpokespersonMode { on } => control.set_spokesperson_mode(on), }, ); let response = match result { @@ -416,6 +423,9 @@ mod tests { fn restart(&self, session_arguments: Vec) -> Result { Ok(json!({"sessionArguments": session_arguments})) } + fn set_spokesperson_mode(&self, on: bool) -> Result { + Ok(json!({"spokesperson": on})) + } fn status(&self) -> Result { Ok(json!({"running": !self.stopped.load(Ordering::SeqCst)})) } diff --git a/src-tauri/crates/berd-call/src/host_session.rs b/src-tauri/crates/berd-call/src/host_session.rs index 507bc9e82..cca4dcdbc 100644 --- a/src-tauri/crates/berd-call/src/host_session.rs +++ b/src-tauri/crates/berd-call/src/host_session.rs @@ -486,6 +486,43 @@ fn default_input_during_tts_policy() -> InputDuringTtsPolicy { } } +/// Session options for these speech settings, keeping a Pocket model directory from `arguments`. +fn tts_arguments(settings: &berd_call::TtsSettings, arguments: &[String]) -> Vec { + let pocket_model_dir = arguments + .chunks_exact(2) + .find(|pair| pair[0] == "--model-dir") + .map(|pair| pair[1].clone()); + let backend = match settings { + berd_call::TtsSettings::Siri { .. } => "siri", + berd_call::TtsSettings::Pocket { .. } => "pocket", + berd_call::TtsSettings::OpenAi { .. } => "openai", + }; + let mut replacement = vec![ + "--tts-backend".to_string(), + backend.to_string(), + "--rate".to_string(), + settings.rate().to_string(), + ]; + match settings { + berd_call::TtsSettings::Siri { + voice, language, .. + } => replacement.extend([ + "--voice".into(), + voice.clone(), + "--language".into(), + language.clone(), + ]), + berd_call::TtsSettings::Pocket { voice, .. } => { + if let Some(model_dir) = pocket_model_dir { + replacement.extend(["--model-dir".into(), model_dir]); + } + replacement.extend(["--voice".into(), voice.clone()]) + } + berd_call::TtsSettings::OpenAi { .. } => {} + } + replacement +} + #[derive(Clone)] struct ReadyState { session: Arc>, @@ -725,6 +762,34 @@ impl HostControl for SessionControl { .map_err(|_| "voice call stopped during restart".to_string())? } + fn set_spokesperson_mode(&self, on: bool) -> Result { + let arguments = &self.session_arguments[1..]; + let realtime = crate::validate_session_arguments(&self.session_arguments)?; + if on == realtime { + return self.status(); + } + // Agents keep the voice they have now; a realtime call's own voice is the Spokesperson's, which a direct call has no use for. + let arguments = if realtime && !self.speaks { + arguments.to_vec() + } else { + let session = self + .ready + .session + .lock() + .map_err(|_| "session settings lock failed")?; + crate::saved_settings::merge_arguments( + arguments, + &tts_arguments(&session.tts.settings, arguments), + ) + }; + let direct = matches!(*self.transcript, Transcript::Stdout { live: true }); + let mut session_arguments = vec!["session".to_string()]; + session_arguments.extend(crate::saved_settings::with_spokesperson( + &arguments, on, direct, + )); + self.restart(session_arguments) + } + fn set_non_blocking(&self, enabled: bool) -> Result { if enabled && !self.transcript.delivers() { return Err(NON_BLOCKING_REQUIRES_DELIVERY.into()); @@ -1490,40 +1555,7 @@ impl SessionActor { return; } saved.tts = Some(session.tts.settings.clone()); - let settings = &session.tts.settings; - let pocket_model_dir = saved - .arguments - .chunks_exact(2) - .find(|pair| pair[0] == "--model-dir") - .map(|pair| pair[1].clone()); - let backend = match settings { - berd_call::TtsSettings::Siri { .. } => "siri", - berd_call::TtsSettings::Pocket { .. } => "pocket", - berd_call::TtsSettings::OpenAi { .. } => "openai", - }; - let mut replacement = vec![ - "--tts-backend".to_string(), - backend.to_string(), - "--rate".to_string(), - settings.rate().to_string(), - ]; - match settings { - berd_call::TtsSettings::Siri { - voice, language, .. - } => replacement.extend([ - "--voice".into(), - voice.clone(), - "--language".into(), - language.clone(), - ]), - berd_call::TtsSettings::Pocket { voice, .. } => { - if let Some(model_dir) = pocket_model_dir { - replacement.extend(["--model-dir".into(), model_dir]); - } - replacement.extend(["--voice".into(), voice.clone()]) - } - berd_call::TtsSettings::OpenAi { .. } => {} - } + let replacement = tts_arguments(&session.tts.settings, &saved.arguments); saved.arguments = crate::saved_settings::merge_arguments(&saved.arguments, &replacement); if let Err(error) = crate::saved_settings::update(path, |latest| match changed { SavedPreference::SessionStart => { @@ -4341,6 +4373,69 @@ mod tests { assert!(stop_requested.load(Ordering::SeqCst)); } + #[test] + fn turning_the_spokesperson_on_restarts_the_call_with_the_agents_current_voice() { + let (commands, receiver) = mpsc::sync_channel(1); + let control = SessionControl { + commands, + running: Arc::new(AtomicBool::new(true)), + ready: ReadyState { + session: Arc::new(Mutex::new(test_snapshot())), + }, + non_blocking: Arc::new(AtomicBool::new(false)), + stop_requested: Arc::new(AtomicBool::new(false)), + muted: Arc::new(AtomicBool::new(false)), + transcript: Arc::new(Transcript::Stdout { live: true }), + speaks: true, + session_arguments: [ + "session", + "--voice", + "Samantha", + "--language", + "en-GB", + "--stt-backend", + "openai", + "--stt-url", + "ws://127.0.0.1:1/stt", + ] + .map(str::to_string) + .to_vec(), + }; + let worker = thread::spawn(move || match receiver.recv().unwrap() { + ControlCommand::Restart { + arguments, + expert_spokesperson, + response, + } => { + response.send(Ok(json!({}))).unwrap(); + (arguments, expert_spokesperson) + } + _ => panic!("expected a restart"), + }); + control.set_spokesperson_mode(true).unwrap(); + let (arguments, expert_spokesperson) = worker.join().unwrap(); + assert!(expert_spokesperson); + assert_eq!( + arguments, + [ + "session", + "--stt-backend", + "openai", + "--tts-backend", + "siri", + "--rate", + "1", + "--voice", + "Aaron", + "--language", + "en-US", + "--mode", + "expert-spokesperson", + crate::DIRECT_AGENT_SPEECH, + ] + ); + } + #[test] fn the_expert_speaks_only_through_the_spokesperson_unless_allowed() { let (commands, receiver) = mpsc::sync_channel(1); diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 3f5e22c8a..7643aa0b1 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -1108,6 +1108,9 @@ fn parse_host_settings_args( "--muted" => host_control::ControlRequest::Muted { muted: parse_bool_setting(flag, args.get(index + 1))?, }, + "--spokesperson" => host_control::ControlRequest::SpokespersonMode { + on: parse_bool_setting(flag, args.get(index + 1))?, + }, "--input-during-tts" => host_control::ControlRequest::InputDuringTts { policy: match args.get(index + 1).map(String::as_str) { Some("allow") => berd_call::input::InputDuringTtsPolicy::AllowBargeIn, @@ -1179,7 +1182,7 @@ fn parse_host_settings_args( } const SETTINGS_CHOICE_ERROR: &str = - "provide exactly one of --non-blocking, --rate, --voice, --spokesperson-voice, --spokesperson-rate, --tts, --input-during-tts, --muted, or --restart"; + "provide exactly one of --non-blocking, --rate, --voice, --spokesperson-voice, --spokesperson-rate, --tts, --input-during-tts, --muted, --spokesperson, or --restart"; fn parse_bool_setting(flag: &str, value: Option<&String>) -> Result { match value.map(String::as_str) { @@ -12894,6 +12897,12 @@ mod tests { .1, host_control::ControlRequest::Muted { muted: true } )); + assert!(matches!( + parse_host_settings_args(&args(&["berd-call", "settings", "--spokesperson", "false"])) + .unwrap() + .1, + host_control::ControlRequest::SpokespersonMode { on: false } + )); assert!(matches!( parse_host_settings_args(&args(&[ "berd-call", diff --git a/src-tauri/crates/berd-call/src/menu_bar.rs b/src-tauri/crates/berd-call/src/menu_bar.rs index be5eebab1..864f93ddd 100644 --- a/src-tauri/crates/berd-call/src/menu_bar.rs +++ b/src-tauri/crates/berd-call/src/menu_bar.rs @@ -69,6 +69,7 @@ pub(crate) enum MenuAction { SpokespersonRate(f32), Muted(bool), InputDuringTts(InputDuringTtsPolicy), + SpokespersonMode(bool), Stop, } @@ -133,7 +134,15 @@ fn menu_model(state: &MenuState) -> Vec { { nodes.push(item(format!("Control error: {error}"), None)); } - nodes.push(SEPARATOR); + let expert = mode_name(status) == "Expert-Spokesperson"; + nodes.extend([ + SEPARATOR, + MenuNode { + checked: expert, + ..item("Spokesperson", Some(MenuAction::SpokespersonMode(!expert))) + }, + SEPARATOR, + ]); // Replies with their own voice put the Spokesperson's beside it. let agents = if spokesperson.is_null() { "" } else { "Agent " }; nodes.extend(voice_menu( @@ -236,6 +245,7 @@ pub(crate) fn control_request(action: &MenuAction) -> ControlRequest { }, MenuAction::Muted(muted) => ControlRequest::Muted { muted: *muted }, MenuAction::InputDuringTts(policy) => ControlRequest::InputDuringTts { policy: *policy }, + MenuAction::SpokespersonMode(on) => ControlRequest::SpokespersonMode { on: *on }, MenuAction::Stop => ControlRequest::Stop, } } @@ -577,6 +587,7 @@ enum ActionDomain { Tts, Mute, InputDuringTts, + Mode, Stop, } @@ -588,6 +599,7 @@ fn action_domain(request: &ControlRequest) -> ActionDomain { | ControlRequest::Spokesperson { .. } => ActionDomain::Tts, ControlRequest::Muted { .. } => ActionDomain::Mute, ControlRequest::InputDuringTts { .. } => ActionDomain::InputDuringTts, + ControlRequest::SpokespersonMode { .. } => ActionDomain::Mode, ControlRequest::Stop => ActionDomain::Stop, _ => unreachable!("menu only sends supported actions"), } @@ -773,6 +785,28 @@ mod tests { .. } )); + assert!(matches!( + find(&nodes, "Spokesperson"), + MenuNode { + checked: true, + action: Some(MenuAction::SpokespersonMode(false)), + .. + } + )); + assert!(matches!( + find( + &menu_model(&MenuState { + status: Some(self::status(false, "allow_barge_in", None)), + ..MenuState::default() + }), + "Spokesperson" + ), + MenuNode { + checked: false, + action: Some(MenuAction::SpokespersonMode(true)), + .. + } + )); assert_eq!(icon_symbol(Some(&status)), "mic.slash"); find(&menu_model(&MenuState::default()), "End Call"); } diff --git a/src-tauri/crates/berd-call/src/saved_settings.rs b/src-tauri/crates/berd-call/src/saved_settings.rs index cc83d2396..d5f1964e4 100644 --- a/src-tauri/crates/berd-call/src/saved_settings.rs +++ b/src-tauri/crates/berd-call/src/saved_settings.rs @@ -186,6 +186,41 @@ pub(crate) fn without_tts(arguments: &[String]) -> Vec { .collect() } +/// The same call's session options with the Spokesperson on or off. Options only one mode takes are left out of the +/// other; with the Spokesperson on, `direct` lets agents also speak in their own voice. +pub(crate) fn with_spokesperson(arguments: &[String], on: bool, direct: bool) -> Vec { + let mut result: Vec<_> = arguments + .iter() + .filter(|argument| *argument != crate::DIRECT_AGENT_SPEECH) + .cloned() + .collect::>() + .chunks_exact(2) + .filter(|pair| match pair[0].as_str() { + "--mode" => false, + "--realtime-url" => on, + "--stt-url" => !on, + "--tts-url" => !on || direct, + _ => on || !spokesperson_flag(pair), + }) + .flatten() + .cloned() + .collect(); + result.extend([ + "--mode".into(), + if on { + "expert-spokesperson" + } else { + "conventional" + } + .into(), + ]); + if on && direct { + // Last, since it takes no value. + result.push(crate::DIRECT_AGENT_SPEECH.into()); + } + result +} + #[cfg(test)] mod tests { use super::*; @@ -260,4 +295,52 @@ mod tests { ["--mode", "chained", "--stt-backend", "macos"] ); } + + #[test] + fn turning_the_spokesperson_off_and_on_keeps_the_options_both_modes_take() { + let strings = |values: &[&str]| { + values + .iter() + .map(|value| value.to_string()) + .collect::>() + }; + let realtime = strings(&[ + "--mode", + "expert-spokesperson", + "--voice", + "Aaron", + "--spokesperson-voice", + "cedar", + "--realtime-url", + "wss://realtime.test/", + "--allow-direct-agent-speech", + ]); + let conventional = with_spokesperson(&realtime, false, true); + assert_eq!( + conventional, + strings(&["--voice", "Aaron", "--mode", "conventional"]) + ); + assert_eq!( + with_spokesperson( + &[ + conventional.clone(), + strings(&["--stt-url", "wss://stt.test/"]) + ] + .concat(), + true, + true + ), + strings(&[ + "--voice", + "Aaron", + "--mode", + "expert-spokesperson", + "--allow-direct-agent-speech" + ]) + ); + assert_eq!( + with_spokesperson(&conventional, true, false), + strings(&["--voice", "Aaron", "--mode", "expert-spokesperson"]) + ); + } } From 87ebe984df2216d70272cecffe90af2f517b0550 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 12:13:35 -0400 Subject: [PATCH 40/49] berd-call: keep the agents' full voice and a valid speed across the toggle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Turning the Spokesperson on or off now restores the agents' live speech settings in the new session, so an OpenAI voice and model chosen during the call survive, which startup options can't express. When agents have no voice of their own after the switch, their speed is left out rather than becoming the Spokesperson's, whose range is narrower; a conventional call at 2× no longer ends when the Spokesperson turns on. Signed-off-by: John Tennant Co-Authored-By: Claude Opus 5.5 --- .../crates/berd-call/src/host_session.rs | 187 +++++++++++------- .../crates/berd-call/src/saved_settings.rs | 2 + 2 files changed, 123 insertions(+), 66 deletions(-) diff --git a/src-tauri/crates/berd-call/src/host_session.rs b/src-tauri/crates/berd-call/src/host_session.rs index cca4dcdbc..bab50fb34 100644 --- a/src-tauri/crates/berd-call/src/host_session.rs +++ b/src-tauri/crates/berd-call/src/host_session.rs @@ -342,7 +342,7 @@ impl SessionActor { SessionStart { saved: self.saved.take().map(|(path, mut saved)| { saved.arguments = saved_settings::persistable_arguments(&restart.arguments[1..]); - saved.tts = None; + saved.tts = restart.tts; (path, saved) }), arguments: restart.arguments, @@ -570,6 +570,8 @@ enum ControlCommand { Restart { arguments: Vec, expert_spokesperson: bool, + // Speech settings to restore in the new session, beyond what its options can say. + tts: Option, response: SyncSender>, }, } @@ -577,6 +579,7 @@ enum ControlCommand { struct PendingRestart { arguments: Vec, expert_spokesperson: bool, + tts: Option, response: SyncSender>, } @@ -594,6 +597,26 @@ struct SessionControl { } impl SessionControl { + fn restart_with( + &self, + session_arguments: Vec, + tts: Option, + ) -> Result { + let expert_spokesperson = crate::validate_session_arguments(&session_arguments)?; + let (response, result) = mpsc::sync_channel(1); + self.commands + .send(ControlCommand::Restart { + arguments: session_arguments, + expert_spokesperson, + tts, + response, + }) + .map_err(|_| "voice call is not running")?; + result + .recv() + .map_err(|_| "voice call stopped during restart".to_string())? + } + fn request( &self, command: impl FnOnce(SyncSender>) -> ControlCommand, @@ -748,18 +771,7 @@ impl HostControl for SessionControl { } fn restart(&self, session_arguments: Vec) -> Result { - let expert_spokesperson = crate::validate_session_arguments(&session_arguments)?; - let (response, result) = mpsc::sync_channel(1); - self.commands - .send(ControlCommand::Restart { - arguments: session_arguments, - expert_spokesperson, - response, - }) - .map_err(|_| "voice call is not running")?; - result - .recv() - .map_err(|_| "voice call stopped during restart".to_string())? + self.restart_with(session_arguments, None) } fn set_spokesperson_mode(&self, on: bool) -> Result { @@ -768,26 +780,33 @@ impl HostControl for SessionControl { if on == realtime { return self.status(); } - // Agents keep the voice they have now; a realtime call's own voice is the Spokesperson's, which a direct call has no use for. - let arguments = if realtime && !self.speaks { - arguments.to_vec() + let direct = matches!(*self.transcript, Transcript::Stdout { live: true }); + // Agents keep the voice they have now, if they speak before and after; otherwise the Spokesperson's voice is its own. + let tts = if self.speaks && (!on || direct) { + Some( + self.ready + .session + .lock() + .map_err(|_| "session settings lock failed")? + .tts + .settings + .clone(), + ) } else { - let session = self - .ready - .session - .lock() - .map_err(|_| "session settings lock failed")?; - crate::saved_settings::merge_arguments( + None + }; + let arguments = match &tts { + Some(settings) => crate::saved_settings::merge_arguments( arguments, - &tts_arguments(&session.tts.settings, arguments), - ) + &tts_arguments(settings, arguments), + ), + None => arguments.to_vec(), }; - let direct = matches!(*self.transcript, Transcript::Stdout { live: true }); let mut session_arguments = vec!["session".to_string()]; session_arguments.extend(crate::saved_settings::with_spokesperson( &arguments, on, direct, )); - self.restart(session_arguments) + self.restart_with(session_arguments, tts) } fn set_non_blocking(&self, enabled: bool) -> Result { @@ -1221,6 +1240,7 @@ impl SessionActor { ControlCommand::Restart { arguments, expert_spokesperson, + tts, response, } => { if self.stopping { @@ -1232,6 +1252,7 @@ impl SessionActor { self.restart = Some(PendingRestart { arguments, expert_spokesperson, + tts, response, }); } @@ -2903,6 +2924,7 @@ mod tests { PendingRestart { arguments: vec!["session".into()], expert_spokesperson: false, + tts: None, response, }, InputDuringTtsPolicy::AllowBargeIn, @@ -2957,6 +2979,7 @@ mod tests { PendingRestart { arguments: arguments.clone(), expert_spokesperson: false, + tts: None, response, }, InputDuringTtsPolicy::AllowBargeIn, @@ -3001,6 +3024,7 @@ mod tests { .handle_command(ControlCommand::Restart { arguments: vec!["session".into()], expert_spokesperson: false, + tts: None, response, }) .unwrap(); @@ -4306,6 +4330,7 @@ mod tests { PendingRestart { arguments: vec!["session".into()], expert_spokesperson: false, + tts: None, response, }, InputDuringTtsPolicy::AllowBargeIn, @@ -4375,46 +4400,55 @@ mod tests { #[test] fn turning_the_spokesperson_on_restarts_the_call_with_the_agents_current_voice() { - let (commands, receiver) = mpsc::sync_channel(1); - let control = SessionControl { - commands, - running: Arc::new(AtomicBool::new(true)), - ready: ReadyState { - session: Arc::new(Mutex::new(test_snapshot())), - }, - non_blocking: Arc::new(AtomicBool::new(false)), - stop_requested: Arc::new(AtomicBool::new(false)), - muted: Arc::new(AtomicBool::new(false)), - transcript: Arc::new(Transcript::Stdout { live: true }), - speaks: true, - session_arguments: [ - "session", - "--voice", - "Samantha", - "--language", - "en-GB", - "--stt-backend", - "openai", - "--stt-url", - "ws://127.0.0.1:1/stt", - ] - .map(str::to_string) - .to_vec(), + let toggle = |transcript, rate| { + let mut snapshot = test_snapshot(); + snapshot.tts.settings = snapshot.tts.settings.with_rate(rate); + let (commands, receiver) = mpsc::sync_channel(1); + let control = SessionControl { + commands, + running: Arc::new(AtomicBool::new(true)), + ready: ReadyState { + session: Arc::new(Mutex::new(snapshot)), + }, + non_blocking: Arc::new(AtomicBool::new(false)), + stop_requested: Arc::new(AtomicBool::new(false)), + muted: Arc::new(AtomicBool::new(false)), + transcript: Arc::new(transcript), + speaks: true, + session_arguments: [ + "session", + "--voice", + "Samantha", + "--language", + "en-GB", + "--rate", + "2", + "--stt-backend", + "openai", + "--stt-url", + "ws://127.0.0.1:1/stt", + ] + .map(str::to_string) + .to_vec(), + }; + let worker = thread::spawn(move || match receiver.recv().unwrap() { + ControlCommand::Restart { + arguments, + expert_spokesperson, + tts, + response, + } => { + response.send(Ok(json!({}))).unwrap(); + assert!(expert_spokesperson); + (arguments, tts) + } + _ => panic!("expected a restart"), + }); + control.set_spokesperson_mode(true).unwrap(); + worker.join().unwrap() }; - let worker = thread::spawn(move || match receiver.recv().unwrap() { - ControlCommand::Restart { - arguments, - expert_spokesperson, - response, - } => { - response.send(Ok(json!({}))).unwrap(); - (arguments, expert_spokesperson) - } - _ => panic!("expected a restart"), - }); - control.set_spokesperson_mode(true).unwrap(); - let (arguments, expert_spokesperson) = worker.join().unwrap(); - assert!(expert_spokesperson); + // Agents with their own voice keep the one they have now. + let (arguments, tts) = toggle(Transcript::Stdout { live: true }, 1.25); assert_eq!( arguments, [ @@ -4424,7 +4458,7 @@ mod tests { "--tts-backend", "siri", "--rate", - "1", + "1.25", "--voice", "Aaron", "--language", @@ -4434,6 +4468,25 @@ mod tests { crate::DIRECT_AGENT_SPEECH, ] ); + assert_eq!(tts, Some(test_snapshot().tts.settings.with_rate(1.25))); + // Otherwise their speed would be the Spokesperson's, which can't go as fast. + let (arguments, tts) = toggle(Transcript::Silent, 2.0); + assert_eq!( + arguments, + [ + "session", + "--voice", + "Samantha", + "--language", + "en-GB", + "--stt-backend", + "openai", + "--mode", + "expert-spokesperson", + ] + ); + assert_eq!(tts, None); + assert!(crate::validate_session_arguments(&arguments).unwrap()); } #[test] @@ -4633,6 +4686,7 @@ mod tests { .handle_command(ControlCommand::Restart { arguments: vec!["session".into()], expert_spokesperson: false, + tts: None, response: restart_tx, }) .unwrap(); @@ -4648,6 +4702,7 @@ mod tests { .handle_command(ControlCommand::Restart { arguments: vec!["session".into()], expert_spokesperson: false, + tts: None, response: late_tx, }) .unwrap(); diff --git a/src-tauri/crates/berd-call/src/saved_settings.rs b/src-tauri/crates/berd-call/src/saved_settings.rs index d5f1964e4..710bd5950 100644 --- a/src-tauri/crates/berd-call/src/saved_settings.rs +++ b/src-tauri/crates/berd-call/src/saved_settings.rs @@ -200,6 +200,8 @@ pub(crate) fn with_spokesperson(arguments: &[String], on: bool, direct: bool) -> "--realtime-url" => on, "--stt-url" => !on, "--tts-url" => !on || direct, + // Without replies of their own, the agents' speed would be the Spokesperson's, which takes a narrower range. + "--rate" => !on || direct, _ => on || !spokesperson_flag(pair), }) .flatten() From cf88d92d457a9e87235320a1b17785872e328310 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 12:40:11 -0400 Subject: [PATCH 41/49] berd-call: tick the Spokesperson checkmark as soon as it's chosen The menu showed the call's mode only once the restarted session answered, so the checkmark lagged a switch by the session's startup time. The menu now shows the chosen mode until the switch finishes, and keeps the last status while the restarting call doesn't answer. Signed-off-by: John Tennant Co-Authored-By: Claude Opus 5.5 --- src-tauri/crates/berd-call/src/menu_bar.rs | 33 ++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/src-tauri/crates/berd-call/src/menu_bar.rs b/src-tauri/crates/berd-call/src/menu_bar.rs index 864f93ddd..e9512f7d3 100644 --- a/src-tauri/crates/berd-call/src/menu_bar.rs +++ b/src-tauri/crates/berd-call/src/menu_bar.rs @@ -111,6 +111,8 @@ struct MenuState { status: Option, voices: Vec, action_error: Option, + // The mode a Spokesperson switch is restarting the call in, shown until the switch finishes. + switching: Option, catalog_error: Option, } @@ -134,7 +136,9 @@ fn menu_model(state: &MenuState) -> Vec { { nodes.push(item(format!("Control error: {error}"), None)); } - let expert = mode_name(status) == "Expert-Spokesperson"; + let expert = state + .switching + .unwrap_or(mode_name(status) == "Expert-Spokesperson"); nodes.extend([ SEPARATOR, MenuNode { @@ -503,6 +507,11 @@ fn spawn_coordinator(port: u16, state: Arc>) -> mpsc::SyncSende if latest_by_domain.get(&domain) != Some(&generation) { continue; } + if domain == ActionDomain::Mode { + if let Ok(mut state) = state.lock() { + state.switching = None; + } + } if let Some(error) = menu_action_error(result) { errors.insert(domain, (generation, error)); } else { @@ -534,7 +543,10 @@ fn spawn_coordinator(port: u16, state: Arc>) -> mpsc::SyncSende latest }); if let Ok(mut state) = state.lock() { - state.status = latest; + // The call doesn't answer while it restarts in the other mode. + if latest.is_some() || state.switching.is_none() { + state.status = latest; + } state.voices = catalog .as_ref() .map(|(_, voices)| voices.clone()) @@ -623,6 +635,11 @@ fn begin_menu_action( results: &mpsc::Sender<(u64, ActionDomain, Result)>, state: &Mutex, ) { + if let ControlRequest::SpokespersonMode { on } = request { + if let Ok(mut state) = state.lock() { + state.switching = Some(on); + } + } match host_control::begin_request(port, request) { Ok(pending) => { let results = results.clone(); @@ -807,6 +824,18 @@ mod tests { .. } )); + // A switch shows its mode at once, before the call has restarted in it. + assert!( + find( + &menu_model(&MenuState { + status: Some(self::status(false, "allow_barge_in", None)), + switching: Some(true), + ..MenuState::default() + }), + "Spokesperson" + ) + .checked + ); assert_eq!(icon_symbol(Some(&status)), "mic.slash"); find(&menu_model(&MenuState::default()), "End Call"); } From 83a7f1e2697c228c7d0a9c24346697ac49240619 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 12:43:00 -0400 Subject: [PATCH 42/49] berd-call: group the Spokesperson's voice under its checkmark With the Spokesperson on, its voice and speed now sit in its menu section, under the checkmark, and the agents' voice, speed and input-during-TTS setting get the next section. Signed-off-by: John Tennant Co-Authored-By: Claude Opus 5.5 --- src-tauri/crates/berd-call/src/menu_bar.rs | 49 ++++++++++++++++------ 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/src-tauri/crates/berd-call/src/menu_bar.rs b/src-tauri/crates/berd-call/src/menu_bar.rs index e9512f7d3..6e5952b82 100644 --- a/src-tauri/crates/berd-call/src/menu_bar.rs +++ b/src-tauri/crates/berd-call/src/menu_bar.rs @@ -145,9 +145,19 @@ fn menu_model(state: &MenuState) -> Vec { checked: expert, ..item("Spokesperson", Some(MenuAction::SpokespersonMode(!expert))) }, - SEPARATOR, ]); - // Replies with their own voice put the Spokesperson's beside it. + // The Spokesperson's own voice sits with it, and replies with their own voice get a section beside it. + if expert && !spokesperson.is_null() { + nodes.extend(voice_menu( + "Spokesperson ", + spokesperson, + &available_voices("openai").unwrap_or_default(), + SPOKESPERSON_RATES, + |entry| MenuAction::SpokespersonVoice(entry.voice.clone()), + MenuAction::SpokespersonRate, + )); + } + nodes.push(SEPARATOR); let agents = if spokesperson.is_null() { "" } else { "Agent " }; nodes.extend(voice_menu( agents, @@ -160,16 +170,6 @@ fn menu_model(state: &MenuState) -> Vec { }, MenuAction::Rate, )); - if !spokesperson.is_null() { - nodes.extend(voice_menu( - "Spokesperson ", - spokesperson, - &available_voices("openai").unwrap_or_default(), - SPOKESPERSON_RATES, - |entry| MenuAction::SpokespersonVoice(entry.voice.clone()), - MenuAction::SpokespersonRate, - )); - } nodes.extend([ MenuNode { checked: suppressing, @@ -873,6 +873,31 @@ mod tests { json!({"command": "spokesperson", "voice": "cedar", "rate": null}) ); find(&nodes, "Agent Voice: Aaron"); + // The Spokesperson's voice is in its section, and the agents' in the next. + let sections: Vec<_> = nodes[1..] + .split(|node| node.title.is_empty()) + .map(|section| { + section + .iter() + .map(|node| node.title.as_str()) + .collect::>() + }) + .collect(); + assert_eq!( + sections[1..3], + [ + vec![ + "Spokesperson", + "Spokesperson Voice: marin", + "Spokesperson Speech Rate: 1×" + ], + vec![ + "Agent Voice: Aaron", + "Agent Speech Rate: 1.25×", + "Mute Input During TTS" + ], + ] + ); } #[test] From b8021f78a940cf136da7df01005322d66decefc2 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 12:54:50 -0400 Subject: [PATCH 43/49] berd-call: one speech rate for agents and the Spokesperson `settings --rate`, and so the menu bar's Speech Rate, now also sets the Spokesperson's speed in a call whose agents have their own voice, as close as the Spokesperson can go (0.25 to 1.5). The menu shows one Speech Rate, noting the Spokesperson's when it differs. At startup the Spokesperson speaks as close to `--rate` as it can, instead of refusing a faster rate, and turning it on keeps the agents' speed. `--spokesperson-rate` still gives the Spokesperson a speed of its own. Signed-off-by: John Tennant Co-Authored-By: Claude Opus 5.5 --- src-tauri/crates/berd-call/README.md | 2 +- src-tauri/crates/berd-call/src/cli_help.rs | 10 +- .../crates/berd-call/src/host_session.rs | 126 ++++++++++++++---- src-tauri/crates/berd-call/src/main.rs | 47 ++++--- src-tauri/crates/berd-call/src/menu_bar.rs | 105 +++++++-------- .../crates/berd-call/src/saved_settings.rs | 22 ++- 6 files changed, 197 insertions(+), 115 deletions(-) diff --git a/src-tauri/crates/berd-call/README.md b/src-tauri/crates/berd-call/README.md index aa918d9f5..c372cd788 100644 --- a/src-tauri/crates/berd-call/README.md +++ b/src-tauri/crates/berd-call/README.md @@ -154,7 +154,7 @@ berd-call session --tts-backend openai --tts-url https://proxy.example/v1/audio/ berd-call session --stt-backend openai --stt-url wss://proxy.example/v1/realtime?intent=transcription ``` -In an Expert–Spokesperson call, the Expert reaches you through the Spokesperson: `berd-call message-spokesperson --re CURSOR [--resolves HANDOFF_ID]... TEXT` has it tell you in its own voice. Like the directives of Berd's Expert, the message must acknowledge the latest input (the Spokesperson's own speech doesn't count), or it comes back `spoke:false` with what was missed, and a handoff it names settles once the Spokesperson has said it. `speak` is refused unless the call was started with `--allow-direct-agent-speech`. With it, the Expert can also speak for itself: berd-call plays each `speak` reply through the configured TTS backend, taking turns with the Spokesperson's audio, and your speech interrupts either of them. Afterwards the Spokesperson gets a silent note of what you heard, so it can follow up without repeating it. A reply has to acknowledge everything you've said, as `speak --re` always does. Such a call holds no handoff open: each still reaches the Expert as a `handoff` row, but nothing waits for an answer, reminds the Expert, or holds back renewing or recovering the Spokesperson's session, and there is nothing for `--resolves` to name. `--voice`, `--rate`, `status` and `settings --tts` describe and change the Expert's voice. The Spokesperson keeps its own, shown as `session.spokesperson`: choose it with `--spokesperson-voice NAME` and `--spokesperson-rate 0.25–1.5`, at `start` or live with `settings`, and the menu bar offers both voices. Saved settings keep the Spokesperson's voice for the next call whose replies have their own, but not `--allow-direct-agent-speech`, so pass that on each `start`. +In an Expert–Spokesperson call, the Expert reaches you through the Spokesperson: `berd-call message-spokesperson --re CURSOR [--resolves HANDOFF_ID]... TEXT` has it tell you in its own voice. Like the directives of Berd's Expert, the message must acknowledge the latest input (the Spokesperson's own speech doesn't count), or it comes back `spoke:false` with what was missed, and a handoff it names settles once the Spokesperson has said it. `speak` is refused unless the call was started with `--allow-direct-agent-speech`. With it, the Expert can also speak for itself: berd-call plays each `speak` reply through the configured TTS backend, taking turns with the Spokesperson's audio, and your speech interrupts either of them. Afterwards the Spokesperson gets a silent note of what you heard, so it can follow up without repeating it. A reply has to acknowledge everything you've said, as `speak --re` always does. Such a call holds no handoff open: each still reaches the Expert as a `handoff` row, but nothing waits for an answer, reminds the Expert, or holds back renewing or recovering the Spokesperson's session, and there is nothing for `--resolves` to name. `--voice`, `--rate`, `status` and `settings --tts` describe and change the Expert's voice. The Spokesperson keeps its own voice, shown as `session.spokesperson`: choose it with `--spokesperson-voice NAME`, at `start` or live with `settings`, and the menu bar offers both voices. It speaks as close to the agents' `--rate` as it can (0.25–1.5×), so the menu bar's one Speech Rate sets both; `--spokesperson-rate` gives it a speed of its own. Saved settings keep the Spokesperson's voice for the next call whose replies have their own, but not `--allow-direct-agent-speech`, so pass that on each `start`. The three URL flags take full endpoints and override only their matching service for that call. Expert–Spokesperson uses `--realtime-url` for both input and output; `--stt-url` applies only to conventional mode, and `--tts-url` also applies to the Expert's own voice. Without an override, each service uses its OpenAI endpoint (or its existing environment override). `OPENAI_API_KEY` supplies the CLI credential; the desktop app stores keys separately by endpoint URL in Keychain. diff --git a/src-tauri/crates/berd-call/src/cli_help.rs b/src-tauri/crates/berd-call/src/cli_help.rs index 3aa05d756..bef26a44d 100644 --- a/src-tauri/crates/berd-call/src/cli_help.rs +++ b/src-tauri/crates/berd-call/src/cli_help.rs @@ -100,8 +100,9 @@ Usage: The setting applies to subsequent speak calls. In a call started with --allow-direct-agent-speech, --voice and --rate change the agents' voice, and ---spokesperson-voice and --spokesperson-rate change the Spokesperson's (an -OpenAI realtime voice, 0.25 to 1.5). --spokesperson turns the Spokesperson on +--spokesperson-voice changes the Spokesperson's (an OpenAI realtime voice). +--rate also sets the Spokesperson's speed, as close as it can go (0.25 to +1.5); --spokesperson-rate sets only the Spokesperson's. --spokesperson turns the Spokesperson on or off, restarting the call's session in the other mode with the same voice; options only the other mode takes are dropped, and in a call started with --stream live, agents keep their own voice beside the Spokesperson's. @@ -153,8 +154,9 @@ and default to OpenAI when no environment override is set. In expert-spokesperson mode, the Expert speaks through the Spokesperson. --allow-direct-agent-speech also lets it speak to the user itself, through the -configured TTS backend, in its own voice beside the Spokesperson's. --spokesperson-voice and ---spokesperson-rate then choose the Spokesperson's own voice and speed."#; +configured TTS backend, in its own voice beside the Spokesperson's. --spokesperson-voice then +chooses the Spokesperson's own voice. It speaks as close to --rate as it can, unless +--spokesperson-rate chooses its own speed."#; const SYNTHESIZE_HELP: &str = r#"Render text through a configured TTS backend into a new WAV file. diff --git a/src-tauri/crates/berd-call/src/host_session.rs b/src-tauri/crates/berd-call/src/host_session.rs index bab50fb34..dabbe41e4 100644 --- a/src-tauri/crates/berd-call/src/host_session.rs +++ b/src-tauri/crates/berd-call/src/host_session.rs @@ -661,18 +661,37 @@ impl HostControl for SessionControl { } fn set_rate(&self, rate: f32) -> Result { - let (settings, expected_revision) = { + let (settings, expected_revision, spokesperson) = { let session = self .ready .session .lock() .map_err(|_| "session settings lock failed")?; + // The Spokesperson keeps as close to the agents' speed as it can. + let spokesperson = session.spokesperson.as_ref().map(|spokesperson| { + ( + spokesperson + .settings + .clone() + .with_rate(crate::spokesperson_speed(rate)), + spokesperson.revision, + ) + }); ( session.tts.settings.clone().with_rate(rate), session.tts.revision, + spokesperson, ) }; - self.request(|response| ControlCommand::TtsSettings { + let status = self.request(|response| ControlCommand::TtsSettings { + settings, + expected_revision, + response, + })?; + let Some((settings, expected_revision)) = spokesperson else { + return Ok(status); + }; + self.request(|response| ControlCommand::SpokespersonSettings { settings, expected_revision, response, @@ -781,27 +800,24 @@ impl HostControl for SessionControl { return self.status(); } let direct = matches!(*self.transcript, Transcript::Stdout { live: true }); - // Agents keep the voice they have now, if they speak before and after; otherwise the Spokesperson's voice is its own. - let tts = if self.speaks && (!on || direct) { - Some( - self.ready - .session - .lock() - .map_err(|_| "session settings lock failed")? - .tts - .settings - .clone(), - ) - } else { - None - }; - let arguments = match &tts { - Some(settings) => crate::saved_settings::merge_arguments( - arguments, - &tts_arguments(settings, arguments), - ), - None => arguments.to_vec(), - }; + let live = self + .ready + .session + .lock() + .map_err(|_| "session settings lock failed")? + .tts + .settings + .clone(); + // Agents keep the voice they have now, if they speak before and after; otherwise the Spokesperson's voice is + // its own, and only their speed carries over. + let tts = (self.speaks && (!on || direct)).then(|| live.clone()); + let arguments = crate::saved_settings::merge_arguments( + arguments, + &match &tts { + Some(settings) => tts_arguments(settings, arguments), + None => vec!["--rate".into(), live.rate().to_string()], + }, + ); let mut session_arguments = vec!["session".to_string()]; session_arguments.extend(crate::saved_settings::with_spokesperson( &arguments, on, direct, @@ -4469,8 +4485,8 @@ mod tests { ] ); assert_eq!(tts, Some(test_snapshot().tts.settings.with_rate(1.25))); - // Otherwise their speed would be the Spokesperson's, which can't go as fast. - let (arguments, tts) = toggle(Transcript::Silent, 2.0); + // Otherwise only their speed carries over, which the Spokesperson keeps as close to as it can. + let (arguments, tts) = toggle(Transcript::Silent, 1.75); assert_eq!( arguments, [ @@ -4481,6 +4497,8 @@ mod tests { "en-GB", "--stt-backend", "openai", + "--rate", + "1.75", "--mode", "expert-spokesperson", ] @@ -4669,6 +4687,64 @@ mod tests { ); } + #[test] + fn the_speech_rate_keeps_the_spokesperson_as_close_to_the_agents_as_it_can() { + let rates = |spokesperson| { + let (commands, receiver) = mpsc::sync_channel(1); + let mut snapshot = test_snapshot(); + snapshot.spokesperson = spokesperson; + let control = SessionControl { + commands, + running: Arc::new(AtomicBool::new(true)), + ready: ReadyState { + session: Arc::new(Mutex::new(snapshot)), + }, + non_blocking: Arc::new(AtomicBool::new(false)), + stop_requested: Arc::new(AtomicBool::new(false)), + muted: Arc::new(AtomicBool::new(false)), + transcript: Arc::new(Transcript::Silent), + speaks: true, + session_arguments: vec!["session".into()], + }; + let worker = thread::spawn(move || { + receiver + .iter() + .map(|command| match command { + ControlCommand::TtsSettings { + settings, response, .. + } => { + response.send(Ok(json!({}))).unwrap(); + ("agents", settings.rate()) + } + ControlCommand::SpokespersonSettings { + settings, response, .. + } => { + response.send(Ok(json!({}))).unwrap(); + ("spokesperson", settings.rate()) + } + _ => panic!("expected a speech settings command"), + }) + .collect::>() + }); + control.set_rate(2.0).unwrap(); + drop(control); + worker.join().unwrap() + }; + let spokesperson = berd_call::TtsConfigurationSnapshot { + revision: 3, + settings: berd_call::TtsSettings::OpenAi { + model: "gpt-realtime".into(), + voice: "marin".into(), + rate: 1.0, + }, + }; + assert_eq!( + rates(Some(spokesperson)), + [("agents", 2.0), ("spokesperson", 1.5)] + ); + assert_eq!(rates(None), [("agents", 2.0)]); + } + #[test] fn stop_cancels_a_pending_restart() { let mut child = Command::new("/bin/cat") diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 7643aa0b1..ee5e196ff 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -3337,22 +3337,19 @@ fn unavailable_spokesperson_title(connection_lost: bool, quiescent: bool) -> &'s fn apply_spokesperson_startup_settings( session: &SessionConfig, spokesperson: &mut OpenAiSpokespersonConfig, -) -> Result<(), String> { - // Replies with their own voice leave the Spokesperson's speed to it. - let rate = match &session.tts { - _ if session.expert_tts => session.spokesperson_rate.unwrap_or(spokesperson.speed()), - TtsBackendConfig::OpenAi { rate } - | TtsBackendConfig::Siri { rate, .. } - | TtsBackendConfig::Pocket { rate, .. } => *rate, - }; +) { + let (TtsBackendConfig::OpenAi { rate } + | TtsBackendConfig::Siri { rate, .. } + | TtsBackendConfig::Pocket { rate, .. }) = &session.tts; if let Some(voice) = &session.spokesperson_voice { spokesperson.session.voice = Some(voice.clone()); } - if !(0.25..=1.5).contains(&rate) { - return Err("Expert-Spokesperson rate must be between 0.25 and 1.5".into()); - } - spokesperson.session.speed = Some(rate); - Ok(()) + // The Spokesperson keeps as close to the agents' speed as it can, unless given its own. + spokesperson.session.speed = Some( + session + .spokesperson_rate + .unwrap_or_else(|| spokesperson_speed(*rate)), + ); } fn run_expert_spokesperson_session( @@ -4500,7 +4497,7 @@ fn run_expert_spokesperson_session( if let Some(url) = &config.endpoints.realtime { spokesperson_config.endpoint.clone_from(url); } - apply_spokesperson_startup_settings(&config, &mut spokesperson_config)?; + apply_spokesperson_startup_settings(&config, &mut spokesperson_config); if config.expert_tts { reply_tts = Some(Arc::new(create_tts_slot( &config.tts, @@ -5989,6 +5986,13 @@ fn finish_unready_input_runtime(runtime: VoiceInputRuntime) -> Result<(), String /// Lets the Expert speak to the user in its own voice in an /// Expert-Spokesperson call, rather than only through the Spokesperson. pub(crate) const DIRECT_AGENT_SPEECH: &str = "--allow-direct-agent-speech"; +/// The speeds the Spokesperson's realtime voice supports. +pub(crate) const SPOKESPERSON_RATES: std::ops::RangeInclusive = 0.25..=1.5; + +/// The Spokesperson's closest supported speed to `rate`. +pub(crate) fn spokesperson_speed(rate: f32) -> f32 { + rate.clamp(*SPOKESPERSON_RATES.start(), *SPOKESPERSON_RATES.end()) +} fn parse_args(args: &[String]) -> Result { if args.get(1).map(String::as_str) != Some("session") { @@ -6079,7 +6083,7 @@ fn parse_args(args: &[String]) -> Result { return Err(format!("OpenAI has no realtime voice named {voice}").into()); } } - if spokesperson_rate.is_some_and(|rate| !(0.25..=1.5).contains(&rate)) { + if spokesperson_rate.is_some_and(|rate| !SPOKESPERSON_RATES.contains(&rate)) { return Err("--spokesperson-rate must be between 0.25 and 1.5".into()); } if mode == SessionMode::ExpertSpokesperson @@ -10854,7 +10858,7 @@ mod tests { semantic_transcript: Vec::new(), }; - apply_spokesperson_startup_settings(&session, &mut realtime).unwrap(); + apply_spokesperson_startup_settings(&session, &mut realtime); assert_eq!(realtime.speed(), 1.5); assert_eq!(realtime.voice(), "marin"); @@ -10898,9 +10902,9 @@ mod tests { }, semantic_transcript: Vec::new(), }; - apply_spokesperson_startup_settings(&own, &mut realtime).unwrap(); - // The replies' rate is theirs, not the Spokesperson's. - assert_eq!(realtime.speed(), 1.0); + apply_spokesperson_startup_settings(&own, &mut realtime); + // The Spokesperson keeps as close to the replies' rate as it can. + assert_eq!(realtime.speed(), 1.5); let chosen = expert(&[ "--tts-backend", "openai", @@ -10911,7 +10915,7 @@ mod tests { "1.25", ]) .unwrap(); - apply_spokesperson_startup_settings(&chosen, &mut realtime).unwrap(); + apply_spokesperson_startup_settings(&chosen, &mut realtime); assert_eq!((realtime.voice(), realtime.speed()), ("cedar", 1.25)); for (extra, error) in [ (&["--spokesperson-voice", "cedar"][..], DIRECT_AGENT_SPEECH), @@ -10932,7 +10936,8 @@ mod tests { assert!(expert(&all).unwrap_err().contains(error), "{extra:?}"); } let read_aloud = expert(&["--tts-backend", "openai", "--rate", "2.0"]).unwrap(); - assert!(apply_spokesperson_startup_settings(&read_aloud, &mut realtime).is_err()); + apply_spokesperson_startup_settings(&read_aloud, &mut realtime); + assert_eq!(realtime.speed(), 1.5); assert!( parse(&["--tts-backend", "openai", "--allow-direct-agent-speech"]) .unwrap_err() diff --git a/src-tauri/crates/berd-call/src/menu_bar.rs b/src-tauri/crates/berd-call/src/menu_bar.rs index 6e5952b82..a2d101bc9 100644 --- a/src-tauri/crates/berd-call/src/menu_bar.rs +++ b/src-tauri/crates/berd-call/src/menu_bar.rs @@ -28,7 +28,6 @@ use crate::host_control::{self, ControlRequest}; use crate::StartOptions; const RATES: [f32; 7] = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75, 2.0]; -const SPOKESPERSON_RATES: std::ops::RangeInclusive = 0.25..=1.5; const REFRESH_INTERVAL: Duration = Duration::from_millis(100); /// Runs the call on a worker thread while the main thread owns the menu bar. @@ -66,7 +65,6 @@ pub(crate) enum MenuAction { }, Rate(f32), SpokespersonVoice(String), - SpokespersonRate(f32), Muted(bool), InputDuringTts(InputDuringTtsPolicy), SpokespersonMode(bool), @@ -148,29 +146,21 @@ fn menu_model(state: &MenuState) -> Vec { ]); // The Spokesperson's own voice sits with it, and replies with their own voice get a section beside it. if expert && !spokesperson.is_null() { - nodes.extend(voice_menu( + nodes.push(voice_menu( "Spokesperson ", spokesperson, &available_voices("openai").unwrap_or_default(), - SPOKESPERSON_RATES, |entry| MenuAction::SpokespersonVoice(entry.voice.clone()), - MenuAction::SpokespersonRate, )); } - nodes.push(SEPARATOR); let agents = if spokesperson.is_null() { "" } else { "Agent " }; - nodes.extend(voice_menu( - agents, - tts, - &state.voices, - rate_range(status), - |entry| MenuAction::Voice { + nodes.extend([ + SEPARATOR, + voice_menu(agents, tts, &state.voices, |entry| MenuAction::Voice { voice: entry.voice.clone(), language: entry.language.clone(), - }, - MenuAction::Rate, - )); - nodes.extend([ + }), + rate_menu(tts, spokesperson, rate_range(status)), MenuNode { checked: suppressing, ..item( @@ -194,41 +184,46 @@ fn menu_model(state: &MenuState) -> Vec { nodes } -/// A voice's choices and speeds, titled with whose voice it is. +/// A voice's choices, titled with whose voice it is. fn voice_menu( whose: &str, tts: &Value, voices: &[VoiceChoice], - range: std::ops::RangeInclusive, choose: impl Fn(&VoiceChoice) -> MenuAction, - rate_action: impl Fn(f32) -> MenuAction, -) -> [MenuNode; 2] { +) -> MenuNode { let voice = tts["voice"].as_str().unwrap_or("Unknown"); let language = tts["language"].as_str(); - let rate = tts["rate"].as_f64().unwrap_or(1.0) as f32; - [ - MenuNode { - children: voices - .iter() - .map(|entry| MenuNode { - checked: entry.voice == voice && entry.language.as_deref() == language, - ..item(entry.title.clone(), Some(choose(entry))) - }) - .collect(), - ..item(format!("{whose}Voice: {voice}"), None) - }, - MenuNode { - children: RATES - .iter() - .filter(|option| range.contains(option)) - .map(|&option| MenuNode { - checked: (option - rate).abs() < 0.01, - ..item(rate_title(option), Some(rate_action(option))) - }) - .collect(), - ..item(format!("{whose}Speech Rate: {}", rate_title(rate)), None) - }, - ] + MenuNode { + children: voices + .iter() + .map(|entry| MenuNode { + checked: entry.voice == voice && entry.language.as_deref() == language, + ..item(entry.title.clone(), Some(choose(entry))) + }) + .collect(), + ..item(format!("{whose}Voice: {voice}"), None) + } +} + +/// One speed for every voice, noting the Spokesperson's when it can't go as fast. +fn rate_menu(tts: &Value, spokesperson: &Value, range: std::ops::RangeInclusive) -> MenuNode { + let speed = |tts: &Value| tts["rate"].as_f64().unwrap_or(1.0) as f32; + let rate = speed(tts); + let mut title = format!("Speech Rate: {}", rate_title(rate)); + if !spokesperson.is_null() && (speed(spokesperson) - rate).abs() >= 0.01 { + title += &format!(" (Spokesperson {})", rate_title(speed(spokesperson))); + } + MenuNode { + children: RATES + .iter() + .filter(|option| range.contains(option)) + .map(|&option| MenuNode { + checked: (option - rate).abs() < 0.01, + ..item(rate_title(option), Some(MenuAction::Rate(option))) + }) + .collect(), + ..item(title, None) + } } /// Translates a menu action into the control request the CLI would send. @@ -243,10 +238,6 @@ pub(crate) fn control_request(action: &MenuAction) -> ControlRequest { voice: Some(voice.clone()), rate: None, }, - MenuAction::SpokespersonRate(rate) => ControlRequest::Spokesperson { - voice: None, - rate: Some(*rate), - }, MenuAction::Muted(muted) => ControlRequest::Muted { muted: *muted }, MenuAction::InputDuringTts(policy) => ControlRequest::InputDuringTts { policy: *policy }, MenuAction::SpokespersonMode(on) => ControlRequest::SpokespersonMode { on: *on }, @@ -257,7 +248,7 @@ pub(crate) fn control_request(action: &MenuAction) -> ControlRequest { /// The rates `berd-call` accepts for the call's TTS backend and mode. fn rate_range(status: &Value) -> std::ops::RangeInclusive { if mode_name(status) == "Expert-Spokesperson" && status["session"]["spokesperson"].is_null() { - return SPOKESPERSON_RATES; + return crate::SPOKESPERSON_RATES; } match status["session"]["tts"]["backend"].as_str() { Some("siri") => 0.5..=2.0, @@ -855,14 +846,12 @@ mod tests { .map(|node| node.title.clone()) .collect() }; + // One speed for everyone, noting the Spokesperson's when it differs. + let rates = titles("Speech Rate: 1.25× (Spokesperson 1×)"); assert_eq!( - titles("Agent Speech Rate: 1.25×"), + rates, ["0.5×", "0.75×", "1×", "1.25×", "1.5×", "1.75×", "2×"] ); - assert_eq!( - titles("Spokesperson Speech Rate: 1×"), - ["0.5×", "0.75×", "1×", "1.25×", "1.5×"] - ); let cedar = find(&nodes, "Spokesperson Voice: marin") .children .iter() @@ -886,14 +875,10 @@ mod tests { assert_eq!( sections[1..3], [ - vec![ - "Spokesperson", - "Spokesperson Voice: marin", - "Spokesperson Speech Rate: 1×" - ], + vec!["Spokesperson", "Spokesperson Voice: marin"], vec![ "Agent Voice: Aaron", - "Agent Speech Rate: 1.25×", + "Speech Rate: 1.25× (Spokesperson 1×)", "Mute Input During TTS" ], ] diff --git a/src-tauri/crates/berd-call/src/saved_settings.rs b/src-tauri/crates/berd-call/src/saved_settings.rs index 710bd5950..ea9674b87 100644 --- a/src-tauri/crates/berd-call/src/saved_settings.rs +++ b/src-tauri/crates/berd-call/src/saved_settings.rs @@ -188,6 +188,7 @@ pub(crate) fn without_tts(arguments: &[String]) -> Vec { /// The same call's session options with the Spokesperson on or off. Options only one mode takes are left out of the /// other; with the Spokesperson on, `direct` lets agents also speak in their own voice. +/// The Spokesperson keeps as close to the agents' speed as it can. pub(crate) fn with_spokesperson(arguments: &[String], on: bool, direct: bool) -> Vec { let mut result: Vec<_> = arguments .iter() @@ -200,8 +201,8 @@ pub(crate) fn with_spokesperson(arguments: &[String], on: bool, direct: bool) -> "--realtime-url" => on, "--stt-url" => !on, "--tts-url" => !on || direct, - // Without replies of their own, the agents' speed would be the Spokesperson's, which takes a narrower range. - "--rate" => !on || direct, + // The Spokesperson's speed follows the agents'. + "--spokesperson-rate" => false, _ => on || !spokesperson_flag(pair), }) .flatten() @@ -311,8 +312,12 @@ mod tests { "expert-spokesperson", "--voice", "Aaron", + "--rate", + "2", "--spokesperson-voice", "cedar", + "--spokesperson-rate", + "1", "--realtime-url", "wss://realtime.test/", "--allow-direct-agent-speech", @@ -320,7 +325,7 @@ mod tests { let conventional = with_spokesperson(&realtime, false, true); assert_eq!( conventional, - strings(&["--voice", "Aaron", "--mode", "conventional"]) + strings(&["--voice", "Aaron", "--rate", "2", "--mode", "conventional"]) ); assert_eq!( with_spokesperson( @@ -335,6 +340,8 @@ mod tests { strings(&[ "--voice", "Aaron", + "--rate", + "2", "--mode", "expert-spokesperson", "--allow-direct-agent-speech" @@ -342,7 +349,14 @@ mod tests { ); assert_eq!( with_spokesperson(&conventional, true, false), - strings(&["--voice", "Aaron", "--mode", "expert-spokesperson"]) + strings(&[ + "--voice", + "Aaron", + "--rate", + "2", + "--mode", + "expert-spokesperson" + ]) ); } } From 8f9f63d6d2aabf5784a11e9c70924e81b61066bb Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 12:58:19 -0400 Subject: [PATCH 44/49] berd-call: leave the Spokesperson's speed alone when the agents' is refused A rate the agents' voice rejects returned the Spokesperson's applied result instead, and changed only the Spokesperson. Signed-off-by: John Tennant Co-Authored-By: Claude Opus 5.5 --- src-tauri/crates/berd-call/src/host_session.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src-tauri/crates/berd-call/src/host_session.rs b/src-tauri/crates/berd-call/src/host_session.rs index dabbe41e4..ff7556ecd 100644 --- a/src-tauri/crates/berd-call/src/host_session.rs +++ b/src-tauri/crates/berd-call/src/host_session.rs @@ -688,7 +688,10 @@ impl HostControl for SessionControl { expected_revision, response, })?; - let Some((settings, expected_revision)) = spokesperson else { + // A rate the agents' voice refuses leaves the Spokesperson's alone too. + let Some((settings, expected_revision)) = + spokesperson.filter(|_| status["outcome"] == "applied") + else { return Ok(status); }; self.request(|response| ControlCommand::SpokespersonSettings { @@ -4689,7 +4692,7 @@ mod tests { #[test] fn the_speech_rate_keeps_the_spokesperson_as_close_to_the_agents_as_it_can() { - let rates = |spokesperson| { + let rates = |spokesperson, outcome: &'static str| { let (commands, receiver) = mpsc::sync_channel(1); let mut snapshot = test_snapshot(); snapshot.spokesperson = spokesperson; @@ -4713,7 +4716,7 @@ mod tests { ControlCommand::TtsSettings { settings, response, .. } => { - response.send(Ok(json!({}))).unwrap(); + response.send(Ok(json!({"outcome": outcome}))).unwrap(); ("agents", settings.rate()) } ControlCommand::SpokespersonSettings { @@ -4739,10 +4742,11 @@ mod tests { }, }; assert_eq!( - rates(Some(spokesperson)), + rates(Some(spokesperson.clone()), "applied"), [("agents", 2.0), ("spokesperson", 1.5)] ); - assert_eq!(rates(None), [("agents", 2.0)]); + assert_eq!(rates(Some(spokesperson), "rejected"), [("agents", 2.0)]); + assert_eq!(rates(None, "applied"), [("agents", 2.0)]); } #[test] From 3b95e78a3833ac41920a63607d17d0266aa140e7 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 13:27:54 -0400 Subject: [PATCH 45/49] Put the agents' rate back when the Spokesperson refuses its own A shared speech rate applied the agents' rate first and then the Spokesperson's. If the Spokesperson refused, the agents kept the new rate, and it was saved, while the command reported the failure. The agents' previous settings are now restored, so a refused change leaves both voices as they were. Signed-off-by: John Tennant Co-Authored-By: Claude Opus 5.5 --- .../crates/berd-call/src/host_session.rs | 46 ++++++++++++++----- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/src-tauri/crates/berd-call/src/host_session.rs b/src-tauri/crates/berd-call/src/host_session.rs index ff7556ecd..97537251f 100644 --- a/src-tauri/crates/berd-call/src/host_session.rs +++ b/src-tauri/crates/berd-call/src/host_session.rs @@ -661,7 +661,7 @@ impl HostControl for SessionControl { } fn set_rate(&self, rate: f32) -> Result { - let (settings, expected_revision, spokesperson) = { + let (previous, expected_revision, spokesperson) = { let session = self .ready .session @@ -678,13 +678,13 @@ impl HostControl for SessionControl { ) }); ( - session.tts.settings.clone().with_rate(rate), + session.tts.settings.clone(), session.tts.revision, spokesperson, ) }; let status = self.request(|response| ControlCommand::TtsSettings { - settings, + settings: previous.clone().with_rate(rate), expected_revision, response, })?; @@ -694,11 +694,23 @@ impl HostControl for SessionControl { else { return Ok(status); }; - self.request(|response| ControlCommand::SpokespersonSettings { + let result = self.request(|response| ControlCommand::SpokespersonSettings { settings, expected_revision, response, - }) + }); + // One the Spokesperson refuses puts the agents' back, so the voices never disagree. + if let (false, Some(expected_revision)) = ( + matches!(&result, Ok(value) if value["outcome"] == "applied"), + status["snapshot"]["revision"].as_u64(), + ) { + let _ = self.request(|response| ControlCommand::TtsSettings { + settings: previous, + expected_revision, + response, + }); + } + result } fn set_voice(&self, voice: String, language: Option) -> Result { @@ -4692,7 +4704,7 @@ mod tests { #[test] fn the_speech_rate_keeps_the_spokesperson_as_close_to_the_agents_as_it_can() { - let rates = |spokesperson, outcome: &'static str| { + let rates = |spokesperson, outcome: &'static str, spokesperson_outcome: &'static str| { let (commands, receiver) = mpsc::sync_channel(1); let mut snapshot = test_snapshot(); snapshot.spokesperson = spokesperson; @@ -4716,13 +4728,17 @@ mod tests { ControlCommand::TtsSettings { settings, response, .. } => { - response.send(Ok(json!({"outcome": outcome}))).unwrap(); + response + .send(Ok(json!({"outcome": outcome, "snapshot": {"revision": 2}}))) + .unwrap(); ("agents", settings.rate()) } ControlCommand::SpokespersonSettings { settings, response, .. } => { - response.send(Ok(json!({}))).unwrap(); + response + .send(Ok(json!({"outcome": spokesperson_outcome}))) + .unwrap(); ("spokesperson", settings.rate()) } _ => panic!("expected a speech settings command"), @@ -4742,11 +4758,19 @@ mod tests { }, }; assert_eq!( - rates(Some(spokesperson.clone()), "applied"), + rates(Some(spokesperson.clone()), "applied", "applied"), [("agents", 2.0), ("spokesperson", 1.5)] ); - assert_eq!(rates(Some(spokesperson), "rejected"), [("agents", 2.0)]); - assert_eq!(rates(None, "applied"), [("agents", 2.0)]); + // A Spokesperson that refuses its speed puts the agents' back. + assert_eq!( + rates(Some(spokesperson.clone()), "applied", "rejected"), + [("agents", 2.0), ("spokesperson", 1.5), ("agents", 1.0)] + ); + assert_eq!( + rates(Some(spokesperson), "rejected", "applied"), + [("agents", 2.0)] + ); + assert_eq!(rates(None, "applied", "applied"), [("agents", 2.0)]); } #[test] From 11e1f7c00c8624a19713d3220a69a61c517d1db3 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 13:33:22 -0400 Subject: [PATCH 46/49] Fail an Expert reply that never gets output_ready A direct Expert reply admitted without the parent's output_ready stayed reserved forever, so every later reply was refused. Give it the two-second readiness deadline PROTOCOL.md sets for all admitted speech: on expiry it fails with speech_failed, the Spokesperson is told none of it was heard, and the slot is free again. Signed-off-by: John Tennant Co-Authored-By: Claude Opus 5.5 --- src-tauri/crates/berd-call/src/main.rs | 37 ++++++++++-- .../berd-call/tests/session_protocol.rs | 56 +++++++++++++++++++ 2 files changed, 88 insertions(+), 5 deletions(-) diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index ee5e196ff..d7b844692 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -3038,8 +3038,9 @@ struct LiveResponse { reply: bool, /// The voice it asked for, when not the Expert's usual one. voice: Option, - /// Whether the parent has accepted the reply's output, so it may play. - output_ready: bool, + /// When an Expert reply still awaits the parent's `output_ready`, the time + /// it must arrive by. + ready_by: Option, delivery: RealtimeAudioDelivery, pending_audio: VecDeque>, pending_frames: usize, @@ -3059,7 +3060,7 @@ impl LiveResponse { expert_text: None, reply: false, voice: None, - output_ready: false, + ready_by: None, delivery: RealtimeAudioDelivery::default(), pending_audio: VecDeque::new(), pending_frames: 0, @@ -4332,6 +4333,31 @@ fn run_expert_spokesperson_session( interrupted, )?; } + let now = Instant::now(); + let expired: Vec<_> = responses + .iter() + .filter(|(_, response)| response.ready_by.is_some_and(|by| by <= now)) + .map(|(response_id, _)| response_id.clone()) + .collect(); + for response_id in expired { + let response = responses.remove(&response_id).expect("reply exists"); + write_message( + &mut writer, + &SessionMessage::SpeechFailed { + id: response + .prepare_id + .expect("an Expert reply has a prepare ID"), + speech_id: response.speech_id.expect("an Expert reply has a speech ID"), + message: "output readiness timed out".into(), + }, + )?; + finish_expert_reply( + &mut core, + runtime.as_ref().expect("initialized runtime"), + "", + true, + )?; + } if initialized && !turn_gate.input_blocks_output() { start_next_live_playback( &mut active, @@ -4628,10 +4654,10 @@ fn run_expert_spokesperson_session( response.prepare_id == Some(id) && response.speech_id == Some(speech_id) }; let reply = responses.iter_mut().find(|(_, response)| { - response.reply && !response.output_ready && matches(response) + response.ready_by.is_some() && matches(response) }); let valid = if let Some((response_id, response)) = reply { - response.output_ready = true; + response.ready_by = None; waiting_responses.push_back(response_id.clone()); true } else { @@ -5352,6 +5378,7 @@ fn submit_expert_prepare( response.expert_text = Some(message); response.reply = true; response.voice = request.voice; + response.ready_by = Some(Instant::now() + Duration::from_secs(2)); response.audio_done = true; response.finish_sent = true; response.server_finished = true; diff --git a/src-tauri/crates/berd-call/tests/session_protocol.rs b/src-tauri/crates/berd-call/tests/session_protocol.rs index 034ba1479..c4f7fd828 100644 --- a/src-tauri/crates/berd-call/tests/session_protocol.rs +++ b/src-tauri/crates/berd-call/tests/session_protocol.rs @@ -3722,6 +3722,62 @@ fn expert_voice_reply_waits_for_output_ready_before_playing() { session.shutdown(); } +/// Starts a direct-speech call against a realtime server that accepts it and then listens. +fn start_direct_speech_session(speech_url: &str) -> ExpertSpokespersonTestSession { + let (endpoint_tx, endpoint_rx) = mpsc::sync_channel(1); + std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async move { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + endpoint_tx + .send(format!("ws://{}/", listener.local_addr().unwrap())) + .unwrap(); + let (stream, _) = listener.accept().await.unwrap(); + let mut socket = accept_async(stream).await.unwrap(); + let initial = receive_realtime_json(&mut socket).await; + acknowledge_realtime_session(&mut socket, &initial).await; + while socket.next().await.is_some() {} + }); + }); + ExpertSpokespersonTestSession::start_with_args( + endpoint_rx.recv().unwrap(), + &["--allow-direct-agent-speech", "--tts-url", speech_url], + ) +} + +#[test] +fn expert_voice_reply_without_output_ready_fails_and_frees_the_slot() { + let (speech_url, speech_requests) = spawn_speech_server(); + let mut session = start_direct_speech_session(&speech_url); + for id in [2, 3] { + session.send(json!({ + "type":"prepare_speak", + "id":id, + "acknowledgement":null, + "text":"I checked the repository." + })); + let admitted = session.recv(Duration::from_secs(2)); + assert_eq!(admitted["type"], "admitted", "{admitted}"); + if id == 3 { + break; + } + assert_eq!( + session.recv(Duration::from_secs(3)), + json!({ + "type":"speech_failed", + "id":id, + "speech_id":admitted["speech_id"], + "message":"output readiness timed out" + }) + ); + } + assert!(speech_requests.try_recv().is_err()); + session.shutdown(); +} + /// Stops the host's player once it has played `played_frames` of an Expert reply, as when the /// speakers change, returning how the reply ended and what the Spokesperson was told. fn stop_expert_reply_after(played_frames: u64) -> (Value, String) { From 53d4789ef205be40b661d6c9f2dd33a49c1d67dd Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 13:33:32 -0400 Subject: [PATCH 47/49] Format the output_ready reply lookup Signed-off-by: John Tennant Co-Authored-By: Claude Opus 5.5 --- src-tauri/crates/berd-call/src/main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index d7b844692..8d49d2982 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -4653,9 +4653,9 @@ fn run_expert_spokesperson_session( let matches = |response: &LiveResponse| { response.prepare_id == Some(id) && response.speech_id == Some(speech_id) }; - let reply = responses.iter_mut().find(|(_, response)| { - response.ready_by.is_some() && matches(response) - }); + let reply = responses + .iter_mut() + .find(|(_, response)| response.ready_by.is_some() && matches(response)); let valid = if let Some((response_id, response)) = reply { response.ready_by = None; waiting_responses.push_back(response_id.clone()); From c5bf2cba1158607985d67496f5af01581cc25a86 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 13:34:13 -0400 Subject: [PATCH 48/49] Lease an Expert reply's voice when it is admitted A direct reply in the Expert's usual voice took its TTS lease only when playback began, so a set_tts_settings applied between admitted and output_ready changed the voice of speech already admitted. PROTOCOL.md says admitted speech keeps the configuration it was admitted with; take the lease in submit_expert_prepare instead, as per-utterance voices already are. Signed-off-by: John Tennant Co-Authored-By: Claude Opus 5.5 --- src-tauri/crates/berd-call/src/main.rs | 23 +++++++------- .../berd-call/tests/session_protocol.rs | 30 +++++++++++++++++++ 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 8d49d2982..3a116ed65 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -3036,7 +3036,7 @@ struct LiveResponse { /// An Expert reply spoken in its own voice. It has no provider /// response; its whole text is played at once. reply: bool, - /// The voice it asked for, when not the Expert's usual one. + /// An Expert reply's voice, leased when it was admitted. voice: Option, /// When an Expert reply still awaits the parent's `output_ready`, the time /// it must arrive by. @@ -4367,7 +4367,6 @@ fn run_expert_spokesperson_session( &audio_transport, &audio_control_tx, &playback_tx, - reply_tts.as_deref(), &reply_tx, &mut core, runtime.as_ref().expect("initialized runtime"), @@ -4391,7 +4390,7 @@ fn run_expert_spokesperson_session( &mut core, runtime.as_ref().expect("initialized runtime"), &mut directive_speeches, - reply_tts.is_some(), + reply_tts.as_deref(), &mut responses, &mut next_speech_id, &mut writer, @@ -4634,7 +4633,7 @@ fn run_expert_spokesperson_session( &mut core, runtime.as_ref().expect("initialized runtime"), &mut directive_speeches, - reply_tts.is_some(), + reply_tts.as_deref(), &mut responses, &mut next_speech_id, &mut writer, @@ -5329,12 +5328,12 @@ fn submit_expert_prepare( core: &mut RealtimeExpertSpokespersonSession, runtime: &OpenAiSpokespersonRuntime, directive_speeches: &mut HashMap, - expert_tts: bool, + reply_tts: Option<&ConfiguredTtsSlot>, responses: &mut HashMap, next_speech_id: &mut u64, writer: &mut impl Write, ) -> Result<(), String> { - let own_voice = expert_tts && !request.relay; + let own_voice = reply_tts.is_some() && !request.relay; if !core .unknown_handoff_ids(&request.resolved_handoff_ids) .is_empty() @@ -5377,7 +5376,11 @@ fn submit_expert_prepare( let mut response = LiveResponse::new(Some(request.id), Some(speech_id)); response.expert_text = Some(message); response.reply = true; - response.voice = request.voice; + response.voice = Some( + request + .voice + .map_or_else(|| reply_tts.expect("own voice has a slot").lease(), Ok)?, + ); response.ready_by = Some(Instant::now() + Duration::from_secs(2)); response.audio_done = true; response.finish_sent = true; @@ -5538,7 +5541,6 @@ fn start_next_live_playback( audio_transport: &Arc, audio_control_tx: &mpsc::Sender, playback_tx: &mpsc::Sender, - reply_tts: Option<&ConfiguredTtsSlot>, reply_tx: &mpsc::Sender, core: &mut RealtimeExpertSpokespersonSession, runtime: &OpenAiSpokespersonRuntime, @@ -5555,7 +5557,6 @@ fn start_next_live_playback( continue; } if response.reply { - let tts = reply_tts.ok_or("an Expert reply has no voice of its own")?; *active = Some(spawn_reply_playback( response_id, response.speech_id.expect("an Expert reply has a speech ID"), @@ -5564,8 +5565,8 @@ fn start_next_live_playback( Arc::clone( response .voice - .clone() - .map_or_else(|| tts.lease(), Ok)? + .as_ref() + .expect("an Expert reply leases its voice when admitted") .backend(), ), Arc::clone(audio_transport), diff --git a/src-tauri/crates/berd-call/tests/session_protocol.rs b/src-tauri/crates/berd-call/tests/session_protocol.rs index c4f7fd828..6f0e9e460 100644 --- a/src-tauri/crates/berd-call/tests/session_protocol.rs +++ b/src-tauri/crates/berd-call/tests/session_protocol.rs @@ -3778,6 +3778,36 @@ fn expert_voice_reply_without_output_ready_fails_and_frees_the_slot() { session.shutdown(); } +#[test] +fn expert_voice_reply_keeps_the_voice_it_was_admitted_with() { + let (speech_url, speech_requests) = spawn_speech_server(); + let mut session = start_direct_speech_session(&speech_url); + let tts = session.ready["session"]["tts"].clone(); + session.send(json!({ + "type":"prepare_speak", + "id":2, + "acknowledgement":null, + "text":"I checked the repository." + })); + let admitted = session.recv(Duration::from_secs(2)); + assert_eq!(admitted["type"], "admitted", "{admitted}"); + session.send(json!({ + "type":"set_tts_settings", + "id":3, + "expected_revision":tts["revision"], + "settings":{"backend":"openai","model":tts["model"],"voice":"cedar","rate":1.0} + })); + let result = session.recv(Duration::from_secs(2)); + assert_eq!(result["outcome"], "applied", "{result}"); + session.send(json!({"type":"output_ready","id":2,"speech_id":admitted["speech_id"]})); + assert_eq!(session.recv(Duration::from_secs(2))["outcome"], "accepted"); + let request = speech_requests + .recv_timeout(Duration::from_secs(3)) + .unwrap(); + assert_eq!(request["voice"], tts["voice"], "{request}"); + session.shutdown(); +} + /// Stops the host's player once it has played `played_frames` of an Expert reply, as when the /// speakers change, returning how the reply ended and what the Spokesperson was told. fn stop_expert_reply_after(played_frames: u64) -> (Value, String) { From 531f12921c14dd0a59a650553cdd2902574f6084 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 20 Oct 2026 13:39:02 -0400 Subject: [PATCH 49/49] Drop the direct speech flag before merging switch arguments merge_arguments reads flag-value pairs, so a call started with --allow-direct-agent-speech before other options, such as --mode expert-spokesperson --allow-direct-agent-speech --voice X, misaligned every later pair when the Spokesperson was switched off, and the restart failed validation. Remove the valueless flag first; with_spokesperson already adds it back when the new mode wants it. Signed-off-by: John Tennant Co-Authored-By: Claude Opus 5.5 --- .../crates/berd-call/src/host_session.rs | 67 ++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/src-tauri/crates/berd-call/src/host_session.rs b/src-tauri/crates/berd-call/src/host_session.rs index 97537251f..c358edb38 100644 --- a/src-tauri/crates/berd-call/src/host_session.rs +++ b/src-tauri/crates/berd-call/src/host_session.rs @@ -809,8 +809,14 @@ impl HostControl for SessionControl { } fn set_spokesperson_mode(&self, on: bool) -> Result { - let arguments = &self.session_arguments[1..]; let realtime = crate::validate_session_arguments(&self.session_arguments)?; + // Without the one flag that takes no value, so the rest pair up. + let arguments: Vec<_> = self.session_arguments[1..] + .iter() + .filter(|argument| *argument != crate::DIRECT_AGENT_SPEECH) + .cloned() + .collect(); + let arguments = &arguments[..]; if on == realtime { return self.status(); } @@ -4522,6 +4528,65 @@ mod tests { assert!(crate::validate_session_arguments(&arguments).unwrap()); } + #[test] + fn turning_the_spokesperson_off_drops_direct_speech_wherever_it_was_given() { + let (commands, receiver) = mpsc::sync_channel(1); + let control = SessionControl { + commands, + running: Arc::new(AtomicBool::new(true)), + ready: ReadyState { + session: Arc::new(Mutex::new(test_snapshot())), + }, + non_blocking: Arc::new(AtomicBool::new(false)), + stop_requested: Arc::new(AtomicBool::new(false)), + muted: Arc::new(AtomicBool::new(false)), + transcript: Arc::new(Transcript::Stdout { live: true }), + speaks: true, + session_arguments: [ + "session", + "--mode", + "expert-spokesperson", + crate::DIRECT_AGENT_SPEECH, + "--voice", + "Samantha", + "--language", + "en-GB", + ] + .map(str::to_string) + .to_vec(), + }; + let worker = thread::spawn(move || match receiver.recv().unwrap() { + ControlCommand::Restart { + arguments, + expert_spokesperson, + response, + .. + } => { + response.send(Ok(json!({}))).unwrap(); + assert!(!expert_spokesperson); + arguments + } + _ => panic!("expected a restart"), + }); + control.set_spokesperson_mode(false).unwrap(); + assert_eq!( + worker.join().unwrap(), + [ + "session", + "--tts-backend", + "siri", + "--rate", + "1", + "--voice", + "Aaron", + "--language", + "en-US", + "--mode", + "conventional", + ] + ); + } + #[test] fn the_expert_speaks_only_through_the_spokesperson_unless_allowed() { let (commands, receiver) = mpsc::sync_channel(1);