From 8737ae2805eb99a2416fd5b06d53fd313ae69cf6 Mon Sep 17 00:00:00 2001 From: Raphael Barreiros Date: Sat, 3 Oct 2026 08:53:38 +0200 Subject: [PATCH 1/4] Fix: Speak failed cloud-TTS sentences with the Apple voice A Grok or OpenAI sentence whose request fails is now spoken in the Apple voice in its place in the reply instead of being skipped. The cloud queue pauses until the clips before it have played, so the two voices never overlap, and isSpeaking stays true throughout. Two failures in a row move the rest of the reply to the Apple voice. Stop and barge-in cancel the Apple sentence too. Voice previews in Settings keep showing the error instead. Closes #62 Co-Authored-By: Claude Opus 5.5 (1M context) --- OpenVision.xcodeproj/project.pbxproj | 4 + OpenVision/Services/TTS/CloudTTSService.swift | 118 ++++++++++++++++-- OpenVisionTests/CloudTTSFallbackTests.swift | 97 ++++++++++++++ README.md | 1 + 4 files changed, 210 insertions(+), 10 deletions(-) create mode 100644 OpenVisionTests/CloudTTSFallbackTests.swift diff --git a/OpenVision.xcodeproj/project.pbxproj b/OpenVision.xcodeproj/project.pbxproj index f1093430..3429dc29 100644 --- a/OpenVision.xcodeproj/project.pbxproj +++ b/OpenVision.xcodeproj/project.pbxproj @@ -16,6 +16,7 @@ 1D0AEF3C7A810FF410C8F2E5 /* SessionRecorder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 06B8E5499FB1BA51EF6F1741 /* SessionRecorder.swift */; }; 1EEA4B0E379A76881244FAB9 /* SettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3DE51DA812A9394A5064F60E /* SettingsView.swift */; }; 223474C328BCC3C81ADCB93D /* OpenAIService.swift in Sources */ = {isa = PBXBuildFile; fileRef = FFDF4A693D2CEB248757FBD3 /* OpenAIService.swift */; }; + 8A6467FB1A9E725FBEF8B262 /* CloudTTSFallbackTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5D73C5CCFE6382A4B4234D4A /* CloudTTSFallbackTests.swift */; }; 2241E9878F41D5D5856C9502 /* HermesTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 641FA355FB2C0B37F5575730 /* HermesTests.swift */; }; 23D894B8F06FA0DBE93C658B /* NeuralSpeech.swift in Sources */ = {isa = PBXBuildFile; fileRef = A82F52D647139EC398E043BF /* NeuralSpeech.swift */; }; 2947D6E5D6D5B7080C53D478 /* OnboardingView.swift in Sources */ = {isa = PBXBuildFile; fileRef = C27052ECD2FFC79652B5CDB3 /* OnboardingView.swift */; }; @@ -178,6 +179,7 @@ 60E2210A5A20177647BDB5AD /* Config.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Config.swift; sourceTree = ""; }; 6102B753E2737E698B780C8E /* HermesSettingsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HermesSettingsView.swift; sourceTree = ""; }; 6188DD26571BACB30096654D /* Conversation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Conversation.swift; sourceTree = ""; }; + 5D73C5CCFE6382A4B4234D4A /* CloudTTSFallbackTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudTTSFallbackTests.swift; sourceTree = ""; }; 641FA355FB2C0B37F5575730 /* HermesTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HermesTests.swift; sourceTree = ""; }; 669EB2820345D230AB0F9FAC /* MetricsCollector+Configuration.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "MetricsCollector+Configuration.swift"; sourceTree = ""; }; 6BD412B34D09394FE85F0181 /* NeuralVoiceListView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NeuralVoiceListView.swift; sourceTree = ""; }; @@ -662,6 +664,7 @@ 39552315579536D518404FED /* DocumentChunkingTests.swift */, B88E1A3CDAF6C7B3D7DF36FE /* FrameChangeTests.swift */, 3A5609ACD2D7FE78F322E775 /* GrokTests.swift */, + 5D73C5CCFE6382A4B4234D4A /* CloudTTSFallbackTests.swift */, 641FA355FB2C0B37F5575730 /* HermesTests.swift */, DD3DFE96152AAE9145F104D6 /* LocalAgentRouteTests.swift */, BE4D7E31C137C29246EB9281 /* NativeToolSupportTests.swift */, @@ -836,6 +839,7 @@ E9628B20FB4654FBBFC61EF9 /* DocumentChunkingTests.swift in Sources */, 9A7811CF026E8DF74F74C2A2 /* FrameChangeTests.swift in Sources */, 41DD47E61F53A95E861466C1 /* GrokTests.swift in Sources */, + 8A6467FB1A9E725FBEF8B262 /* CloudTTSFallbackTests.swift in Sources */, 2241E9878F41D5D5856C9502 /* HermesTests.swift in Sources */, DC6776A7CBFBB8E151FB6B85 /* LocalAgentRouteTests.swift in Sources */, 5F19226406CF5B705B57BACF /* NativeToolSupportTests.swift in Sources */, diff --git a/OpenVision/Services/TTS/CloudTTSService.swift b/OpenVision/Services/TTS/CloudTTSService.swift index c330e72d..74b95070 100644 --- a/OpenVision/Services/TTS/CloudTTSService.swift +++ b/OpenVision/Services/TTS/CloudTTSService.swift @@ -9,6 +9,10 @@ // // The engine is provider-agnostic; a CloudVoiceProvider only says whether it can speak, which // voice is selected, and how to turn one sentence into WAV bytes. +// +// A sentence whose request fails (a 429, a timeout, no network) is spoken in the Apple voice in +// its place in the queue, so the reply has no silent hole. After two failures in a row the rest +// of the reply goes to the Apple voice outright rather than alternating per sentence. import AVFoundation import Foundation @@ -45,7 +49,19 @@ final class CloudTTSService: ObservableObject { /// The format the player node is connected with (the first clip's, normally 24 kHz mono). private var playerFormat: AVAudioFormat? - private init(provider: CloudVoiceProvider) { + /// Speaks failed sentences. Its own synthesizer, not TTSService: the voice agent reads + /// TTSService finishing as the end of the reply. + private let fallback = AppleFallbackSpeaker() + /// Whether the current utterance falls back to the Apple voice (replies yes, voice previews + /// no: a failed preview shows `lastFailure` instead). + private var fallbackEnabled = true + /// Failed sentences in a row; at two, the rest of the reply uses the Apple voice. + private var consecutiveFailures = 0 + private var appleForRest = false + /// Test hook: each sentence as it starts playing, and whether it's in the Apple voice. + var onSentenceStarted: ((_ sentence: String, _ appleVoice: Bool) -> Void)? + + init(provider: CloudVoiceProvider) { self.provider = provider } @@ -53,15 +69,16 @@ final class CloudTTSService: ObservableObject { /// Speak a whole reply, replacing anything playing. func speak(_ text: String) async { - await speak(text, voice: provider.selectedVoice()) + await speak(text, voice: provider.selectedVoice(), appleFallback: true) } /// Speak in a specific voice — used to preview voices in Settings before choosing one. - func speak(_ text: String, voice: String) async { + func speak(_ text: String, voice: String, appleFallback: Bool = false) async { let clean = text.trimmingCharacters(in: .whitespacesAndNewlines) guard !clean.isEmpty, isReady else { return } stop() beginStreaming() + fallbackEnabled = appleFallback for sentence in TextChunking.sentences(clean) { enqueue(sentence, voice: voice) } endStreaming() } @@ -72,11 +89,12 @@ final class CloudTTSService: ObservableObject { streamStarted = false utteranceCancelled = true // in-flight synthesis discards its output pendingSentences.removeAll() - clipQueue.forEach { $0.cancel() } + clipQueue.forEach { $0.clip.cancel() } clipQueue.removeAll() pendingBuffers = 0 playerEpoch += 1 if audioReady { playerNode.stop() } + fallback.stop() } // MARK: - Streaming (speak sentences as the model produces them) @@ -88,7 +106,7 @@ final class CloudTTSService: ObservableObject { /// Synthesis tasks in sentence order, at most `maxInFlight` at a time (enough to stay ahead of /// playback without a 15-sentence reply firing 15 requests and tripping rate limits). ONE /// drainer awaits them in order, so playback order is structural. - private var clipQueue: [Task] = [] + private var clipQueue: [(text: String, clip: Task)] = [] /// Sentences waiting for a synthesis slot. private var pendingSentences: [(text: String, voice: String)] = [] private let maxInFlight = 3 @@ -123,6 +141,9 @@ final class CloudTTSService: ObservableObject { utteranceCancelled = false generationActive = true utteranceGeneration += 1 + fallbackEnabled = true + consecutiveFailures = 0 + appleForRest = false isSpeaking = true // pauses the recognizer for the whole utterance } @@ -142,7 +163,9 @@ final class CloudTTSService: ObservableObject { private func startSynthesis() { while clipQueue.count < maxInFlight, !pendingSentences.isEmpty { let next = pendingSentences.removeFirst() - clipQueue.append(Task { await self.synthesize(next.text, voice: next.voice) }) + // Once the reply has moved to the Apple voice, don't request it from the cloud. + let skip = appleForRest + clipQueue.append((next.text, Task { skip ? nil : await self.synthesize(next.text, voice: next.voice) })) } } @@ -155,16 +178,20 @@ final class CloudTTSService: ObservableObject { private func drainIfNeeded() { guard drainTask == nil else { return } drainTask = Task { [weak self] in - while let self, let task = self.clipQueue.first { - let clip = await task.value + while let self, let head = self.clipQueue.first { + let clip = await head.clip.value // stop() (and maybe a new reply) may have replaced the queue while we waited — // only play the clip if it's still the head of the CURRENT utterance. - guard self.clipQueue.first == task, !self.utteranceCancelled else { continue } + guard self.clipQueue.first?.clip == head.clip, !self.utteranceCancelled else { continue } self.clipQueue.removeFirst() self.startSynthesis() - if let clip { + if let clip, !self.appleForRest { + self.consecutiveFailures = 0 + self.onSentenceStarted?(head.text, false) self.schedule(clip, restartPlayer: !self.streamStarted) self.streamStarted = true + } else if self.fallbackEnabled { + await self.speakWithAppleVoice(head.text) } } guard let self else { return } @@ -173,6 +200,25 @@ final class CloudTTSService: ObservableObject { } } + /// Speak a sentence the cloud couldn't synthesize, in its place: after the clips queued before + /// it have played out, and before the next one. The drainer waits here, so the two audio + /// paths never overlap and `isSpeaking` stays true throughout. + private func speakWithAppleVoice(_ sentence: String) async { + consecutiveFailures += 1 + if consecutiveFailures >= 2 && !appleForRest { + // Probably offline: stop asking the cloud for the rest of this reply. + NSLog("[%@] two sentences failed in a row, using the Apple voice for the rest", provider.name) + appleForRest = true + } + while pendingBuffers > 0 && !utteranceCancelled { + try? await Task.sleep(for: .milliseconds(30)) + } + guard !utteranceCancelled else { return } + onSentenceStarted?(sentence, true) + MetricsCollector.shared.markFirstAudio() + await fallback.speak(sentence) + } + /// Ambient narration (watch loop): speaks only into silence and drops itself if a reply /// started while it was being synthesized. func speakAmbient(_ text: String) async { @@ -286,6 +332,58 @@ final class CloudTTSService: ObservableObject { extension CloudTTSService: NeuralSpeechEngine {} +// MARK: - Apple voice fallback + +/// Speaks one sentence in the user's Apple voice and returns when it has finished or been stopped. +@MainActor +final class AppleFallbackSpeaker: NSObject, AVSpeechSynthesizerDelegate { + private let synthesizer = AVSpeechSynthesizer() + /// The utterance being spoken and its waiter. A stopped utterance reports didCancel later, + /// so callbacks only finish the utterance they're about. + private var current: (id: ObjectIdentifier, finished: CheckedContinuation)? + + override init() { + super.init() + synthesizer.delegate = self + } + + func speak(_ sentence: String) async { + stop() + let utterance = AVSpeechUtterance(string: sentence) + if let identifier = SettingsManager.shared.settings.selectedVoiceIdentifier, + let voice = AVSpeechSynthesisVoice(identifier: identifier) { + utterance.voice = voice + } else { + utterance.voice = AVSpeechSynthesisVoice(language: "en-US") + } + await withCheckedContinuation { continuation in + current = (ObjectIdentifier(utterance), continuation) + synthesizer.speak(utterance) + } + } + + func stop() { + synthesizer.stopSpeaking(at: .immediate) + if let current { finish(current.id) } + } + + private func finish(_ id: ObjectIdentifier) { + guard let current, current.id == id else { return } + self.current = nil + current.finished.resume() + } + + nonisolated func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didFinish utterance: AVSpeechUtterance) { + let id = ObjectIdentifier(utterance) + Task { @MainActor in self.finish(id) } + } + + nonisolated func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didCancel utterance: AVSpeechUtterance) { + let id = ObjectIdentifier(utterance) + Task { @MainActor in self.finish(id) } + } +} + // MARK: - Voices struct CloudVoice: Identifiable, Hashable { diff --git a/OpenVisionTests/CloudTTSFallbackTests.swift b/OpenVisionTests/CloudTTSFallbackTests.swift new file mode 100644 index 00000000..df1382fa --- /dev/null +++ b/OpenVisionTests/CloudTTSFallbackTests.swift @@ -0,0 +1,97 @@ +// OpenVision - CloudTTSFallbackTests.swift +// A cloud-voice sentence that fails is spoken in the Apple voice, in its place in the reply (#62). +// Uses a fake provider that returns short silent WAV clips and fails chosen sentences; the audio +// really plays, so these run on a device or simulator with audio output. + +import XCTest +@testable import OpenVision + +@MainActor +final class CloudTTSFallbackTests: XCTestCase { + + private let reply = "Sentence one. Sentence two. Sentence three. Sentence four." + + func testFailedSentenceIsSpokenInTheAppleVoiceInOrder() async throws { + let events = try await play(reply, failing: ["two"]) + XCTAssertEqual(events.map(\.0), ["Sentence one.", "Sentence two.", "Sentence three.", "Sentence four."]) + XCTAssertEqual(events.map(\.1), [false, true, false, false], "only sentence two in the Apple voice") + } + + func testTwoFailuresInARowMoveTheRestToTheAppleVoice() async throws { + let events = try await play(reply, failing: ["two", "three"]) + XCTAssertEqual(events.map(\.0), ["Sentence one.", "Sentence two.", "Sentence three.", "Sentence four."]) + XCTAssertEqual(events.map(\.1), [false, true, true, true], "no alternating voices once offline") + } + + func testStopDuringTheFallbackStopsEverything() async throws { + let service = CloudTTSService(provider: Self.fakeProvider(failing: ["two"])) + var events: [(String, Bool)] = [] + service.onSentenceStarted = { sentence, apple in + events.append((sentence, apple)) + if apple { service.stop() } // barge-in while the Apple voice speaks sentence two + } + await service.speak(reply) + try await waitUntilQuiet(service) + try await Task.sleep(for: .seconds(1)) + XCTAssertEqual(events.map(\.0), ["Sentence one.", "Sentence two."]) + XCTAssertFalse(service.isSpeaking) + } + + func testVoicePreviewDoesNotFallBack() async throws { + let service = CloudTTSService(provider: Self.fakeProvider(failing: ["one"])) + var events: [(String, Bool)] = [] + service.onSentenceStarted = { events.append(($0, $1)) } + await service.speak("Sentence one.", voice: "test") + try await waitUntilQuiet(service) + XCTAssertTrue(events.isEmpty, "a failed preview shows the error instead") + XCTAssertNotNil(service.lastFailure) + } + + // MARK: - Helpers + + private func play(_ text: String, failing: Set) async throws -> [(String, Bool)] { + let service = CloudTTSService(provider: Self.fakeProvider(failing: failing)) + var events: [(String, Bool)] = [] + service.onSentenceStarted = { events.append(($0, $1)) } + await service.speak(text) + try await waitUntilQuiet(service) + return events + } + + private func waitUntilQuiet(_ service: CloudTTSService, timeout: TimeInterval = 30) async throws { + let deadline = Date().addingTimeInterval(timeout) + while service.isSpeaking { + guard Date() < deadline else { return XCTFail("still speaking after \(timeout)s") } + try await Task.sleep(for: .milliseconds(50)) + } + } + + /// Fails any sentence containing one of `failing` like a dropped connection; others get a + /// short silent clip. + private static func fakeProvider(failing: Set) -> CloudVoiceProvider { + CloudVoiceProvider( + name: "FakeTTS", + isReady: { true }, + selectedVoice: { "test" }, + synthesize: { text, _ in + if failing.contains(where: { text.lowercased().contains($0) }) { throw URLError(.timedOut) } + let response = HTTPURLResponse(url: URL(string: "https://tts.test")!, statusCode: 200, + httpVersion: nil, headerFields: nil)! + return (silentWAV(seconds: 0.15), response) + } + ) + } + + /// 16-bit mono PCM WAV at 24 kHz, like the providers return. + private static func silentWAV(seconds: Double, sampleRate: Int = 24_000) -> Data { + let samples = Int(Double(sampleRate) * seconds) + var data = Data() + func append(_ value: T) { withUnsafeBytes(of: value.littleEndian) { data.append(contentsOf: $0) } } + data.append(contentsOf: Array("RIFF".utf8)); append(UInt32(36 + samples * 2)) + data.append(contentsOf: Array("WAVEfmt ".utf8)); append(UInt32(16)); append(UInt16(1)); append(UInt16(1)) + append(UInt32(sampleRate)); append(UInt32(sampleRate * 2)); append(UInt16(2)); append(UInt16(16)) + data.append(contentsOf: Array("data".utf8)); append(UInt32(samples * 2)) + data.append(Data(count: samples * 2)) + return data + } +} diff --git a/README.md b/README.md index 9a05dc73..ec2e8bfd 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ With **SmolVLM2** selected as your local model, **"Ok Vision, take a photo and t ### Natural Voices — On-Device or Cloud - **Kokoro**: a **natural, offline, private voice** (Kokoro-82M) running on-device via MLX. - **Grok** (xAI, 28 multilingual voices) and **OpenAI** (`gpt-4o-mini-tts`, 13 voices) cloud voices. Grok works with your SuperGrok sign-in or an xAI key; OpenAI needs an API key with credits. +- If a cloud voice can't read a sentence (rate limit, timeout, no network), that sentence is spoken in the Apple voice instead of being skipped, and after two failures in a row the rest of the reply is too. - Pick one from the Speech Engine dropdown; every voice list has a ▶ button to hear a sample before choosing. - Apple's system voice stays the default (with Premium/Enhanced voice support). From 65f2979b020664b27bd61751c9d078294d3b5ba7 Mon Sep 17 00:00:00 2001 From: Raphael Barreiros Date: Sat, 3 Oct 2026 09:58:17 +0200 Subject: [PATCH 2/4] Fix: Tighten the Apple-voice fallback's ordering and cancellation - A failure is counted before the queue is refilled, so the sentence after a second failure in a row is never sent to the cloud; requests already out are cancelled. - A fallback sentence only plays for the reply it belongs to, even if a stop and a new reply happen while it waits for earlier clips. - Cancellation is checked again after the sentence-start hook, so a stop there can't start playback. Co-Authored-By: Claude Opus 5.5 (1M context) --- OpenVision/Services/TTS/CloudTTSService.swift | 35 +++++++++++++------ OpenVisionTests/CloudTTSFallbackTests.swift | 14 +++++++- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/OpenVision/Services/TTS/CloudTTSService.swift b/OpenVision/Services/TTS/CloudTTSService.swift index 74b95070..256ee828 100644 --- a/OpenVision/Services/TTS/CloudTTSService.swift +++ b/OpenVision/Services/TTS/CloudTTSService.swift @@ -60,6 +60,8 @@ final class CloudTTSService: ObservableObject { private var appleForRest = false /// Test hook: each sentence as it starts playing, and whether it's in the Apple voice. var onSentenceStarted: ((_ sentence: String, _ appleVoice: Bool) -> Void)? + /// Test hook: how many sentences the Apple voice has actually been asked to speak. + var fallbackUtterances: Int { fallback.utterances } init(provider: CloudVoiceProvider) { self.provider = provider @@ -184,10 +186,14 @@ final class CloudTTSService: ObservableObject { // only play the clip if it's still the head of the CURRENT utterance. guard self.clipQueue.first?.clip == head.clip, !self.utteranceCancelled else { continue } self.clipQueue.removeFirst() + let useCloud = clip != nil && !self.appleForRest + // Count the failure before refilling the queue, so a second failure in a row + // keeps the next sentence from being sent to the cloud. + if useCloud { self.consecutiveFailures = 0 } else if self.fallbackEnabled { self.noteFailure() } self.startSynthesis() - if let clip, !self.appleForRest { - self.consecutiveFailures = 0 + if let clip, useCloud { self.onSentenceStarted?(head.text, false) + guard !self.utteranceCancelled else { continue } self.schedule(clip, restartPlayer: !self.streamStarted) self.streamStarted = true } else if self.fallbackEnabled { @@ -204,21 +210,28 @@ final class CloudTTSService: ObservableObject { /// it have played out, and before the next one. The drainer waits here, so the two audio /// paths never overlap and `isSpeaking` stays true throughout. private func speakWithAppleVoice(_ sentence: String) async { - consecutiveFailures += 1 - if consecutiveFailures >= 2 && !appleForRest { - // Probably offline: stop asking the cloud for the rest of this reply. - NSLog("[%@] two sentences failed in a row, using the Apple voice for the rest", provider.name) - appleForRest = true - } - while pendingBuffers > 0 && !utteranceCancelled { + // A stop, then a new reply, can happen during the wait: only speak for this one. + let generation = utteranceGeneration + var current: Bool { !utteranceCancelled && utteranceGeneration == generation } + while pendingBuffers > 0 && current { try? await Task.sleep(for: .milliseconds(30)) } - guard !utteranceCancelled else { return } + guard current else { return } onSentenceStarted?(sentence, true) + guard current else { return } MetricsCollector.shared.markFirstAudio() await fallback.speak(sentence) } + private func noteFailure() { + consecutiveFailures += 1 + guard consecutiveFailures >= 2, !appleForRest else { return } + // Probably offline: stop asking the cloud for the rest of this reply. + NSLog("[%@] two sentences failed in a row, using the Apple voice for the rest", provider.name) + appleForRest = true + clipQueue.forEach { $0.clip.cancel() } // requests already out come back as failures + } + /// Ambient narration (watch loop): speaks only into silence and drops itself if a reply /// started while it was being synthesized. func speakAmbient(_ text: String) async { @@ -341,6 +354,7 @@ final class AppleFallbackSpeaker: NSObject, AVSpeechSynthesizerDelegate { /// The utterance being spoken and its waiter. A stopped utterance reports didCancel later, /// so callbacks only finish the utterance they're about. private var current: (id: ObjectIdentifier, finished: CheckedContinuation)? + private(set) var utterances = 0 override init() { super.init() @@ -356,6 +370,7 @@ final class AppleFallbackSpeaker: NSObject, AVSpeechSynthesizerDelegate { } else { utterance.voice = AVSpeechSynthesisVoice(language: "en-US") } + utterances += 1 await withCheckedContinuation { continuation in current = (ObjectIdentifier(utterance), continuation) synthesizer.speak(utterance) diff --git a/OpenVisionTests/CloudTTSFallbackTests.swift b/OpenVisionTests/CloudTTSFallbackTests.swift index df1382fa..10a582ce 100644 --- a/OpenVisionTests/CloudTTSFallbackTests.swift +++ b/OpenVisionTests/CloudTTSFallbackTests.swift @@ -23,6 +23,14 @@ final class CloudTTSFallbackTests: XCTestCase { XCTAssertEqual(events.map(\.1), [false, true, true, true], "no alternating voices once offline") } + func testAfterTwoFailuresTheCloudIsNotAskedAgain() async throws { + let requests = RequestLog() + let service = CloudTTSService(provider: Self.fakeProvider(failing: ["two", "three"], log: requests)) + await service.speak("One. Sentence two. Sentence three. Four. Five. Six.") + try await waitUntilQuiet(service) + XCTAssertFalse(requests.texts.contains("Six."), "sentence six was never sent: \(requests.texts)") + } + func testStopDuringTheFallbackStopsEverything() async throws { let service = CloudTTSService(provider: Self.fakeProvider(failing: ["two"])) var events: [(String, Bool)] = [] @@ -35,6 +43,7 @@ final class CloudTTSFallbackTests: XCTestCase { try await Task.sleep(for: .seconds(1)) XCTAssertEqual(events.map(\.0), ["Sentence one.", "Sentence two."]) XCTAssertFalse(service.isSpeaking) + XCTAssertEqual(service.fallbackUtterances, 0, "the Apple sentence must not start after the stop") } func testVoicePreviewDoesNotFallBack() async throws { @@ -68,12 +77,15 @@ final class CloudTTSFallbackTests: XCTestCase { /// Fails any sentence containing one of `failing` like a dropped connection; others get a /// short silent clip. - private static func fakeProvider(failing: Set) -> CloudVoiceProvider { + private final class RequestLog { var texts: [String] = [] } + + private static func fakeProvider(failing: Set, log: RequestLog? = nil) -> CloudVoiceProvider { CloudVoiceProvider( name: "FakeTTS", isReady: { true }, selectedVoice: { "test" }, synthesize: { text, _ in + log?.texts.append(text) if failing.contains(where: { text.lowercased().contains($0) }) { throw URLError(.timedOut) } let response = HTTPURLResponse(url: URL(string: "https://tts.test")!, statusCode: 200, httpVersion: nil, headerFields: nil)! From 67f938e278a636082356e04169b9f28497b3e7b1 Mon Sep 17 00:00:00 2001 From: Raphael Barreiros Date: Sun, 4 Oct 2026 06:48:17 +0200 Subject: [PATCH 3/4] Fix: Bound the Apple-voice fallback's wait If the synthesizer never reports a fallback sentence finished, a watchdog stops it after max(5 s, 120 ms per character) and the drainer moves on, so the recognizer can't stay paused. The Apple voice choice now lives in TTSService.userVoice, shared with the fallback. Co-Authored-By: Claude Opus 5.5 (1M context) --- OpenVision/Services/TTS/CloudTTSService.swift | 26 +++++++++++++------ OpenVision/Services/TTS/TTSService.swift | 6 ++++- OpenVisionTests/CloudTTSFallbackTests.swift | 6 +++++ 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/OpenVision/Services/TTS/CloudTTSService.swift b/OpenVision/Services/TTS/CloudTTSService.swift index 256ee828..b9b830f2 100644 --- a/OpenVision/Services/TTS/CloudTTSService.swift +++ b/OpenVision/Services/TTS/CloudTTSService.swift @@ -353,9 +353,15 @@ final class AppleFallbackSpeaker: NSObject, AVSpeechSynthesizerDelegate { private let synthesizer = AVSpeechSynthesizer() /// The utterance being spoken and its waiter. A stopped utterance reports didCancel later, /// so callbacks only finish the utterance they're about. - private var current: (id: ObjectIdentifier, finished: CheckedContinuation)? + private var current: (id: ObjectIdentifier, finished: CheckedContinuation, watchdog: Task)? private(set) var utterances = 0 + /// How long one sentence may take before we stop waiting for the synthesizer: the drainer + /// waits here with the recognizer paused, so a lost didFinish must not leave it deaf. + nonisolated static func deadline(for sentence: String) -> Duration { + .milliseconds(max(5_000, 120 * sentence.count)) + } + override init() { super.init() synthesizer.delegate = self @@ -364,15 +370,18 @@ final class AppleFallbackSpeaker: NSObject, AVSpeechSynthesizerDelegate { func speak(_ sentence: String) async { stop() let utterance = AVSpeechUtterance(string: sentence) - if let identifier = SettingsManager.shared.settings.selectedVoiceIdentifier, - let voice = AVSpeechSynthesisVoice(identifier: identifier) { - utterance.voice = voice - } else { - utterance.voice = AVSpeechSynthesisVoice(language: "en-US") - } + utterance.voice = TTSService.userVoice utterances += 1 + let id = ObjectIdentifier(utterance) await withCheckedContinuation { continuation in - current = (ObjectIdentifier(utterance), continuation) + let watchdog = Task { [weak self] in + try? await Task.sleep(for: Self.deadline(for: sentence)) + guard !Task.isCancelled, let self, self.current?.id == id else { return } + NSLog("[CloudTTS] Apple voice never finished a sentence; moving on") + self.synthesizer.stopSpeaking(at: .immediate) + self.finish(id) + } + current = (id, continuation, watchdog) synthesizer.speak(utterance) } } @@ -385,6 +394,7 @@ final class AppleFallbackSpeaker: NSObject, AVSpeechSynthesizerDelegate { private func finish(_ id: ObjectIdentifier) { guard let current, current.id == id else { return } self.current = nil + current.watchdog.cancel() current.finished.resume() } diff --git a/OpenVision/Services/TTS/TTSService.swift b/OpenVision/Services/TTS/TTSService.swift index e12c0865..b1e6a9a2 100644 --- a/OpenVision/Services/TTS/TTSService.swift +++ b/OpenVision/Services/TTS/TTSService.swift @@ -39,7 +39,11 @@ final class TTSService: NSObject, ObservableObject { // MARK: - Voice Selection - private var selectedVoice: AVSpeechSynthesisVoice? { + private var selectedVoice: AVSpeechSynthesisVoice? { Self.userVoice } + + /// The Apple voice chosen in Settings, or the default English voice. Also used by the cloud + /// voices' Apple fallback, so the choice lives in one place. + static var userVoice: AVSpeechSynthesisVoice? { // Check if user has selected a specific voice if let identifier = SettingsManager.shared.settings.selectedVoiceIdentifier, let voice = AVSpeechSynthesisVoice(identifier: identifier) { diff --git a/OpenVisionTests/CloudTTSFallbackTests.swift b/OpenVisionTests/CloudTTSFallbackTests.swift index 10a582ce..91443e5d 100644 --- a/OpenVisionTests/CloudTTSFallbackTests.swift +++ b/OpenVisionTests/CloudTTSFallbackTests.swift @@ -56,6 +56,12 @@ final class CloudTTSFallbackTests: XCTestCase { XCTAssertNotNil(service.lastFailure) } + func testAppleSentenceWaitIsBounded() { + XCTAssertEqual(AppleFallbackSpeaker.deadline(for: "Short."), .seconds(5)) + let long = String(repeating: "word ", count: 40) + XCTAssertEqual(AppleFallbackSpeaker.deadline(for: long), .milliseconds(120 * long.count)) + } + // MARK: - Helpers private func play(_ text: String, failing: Set) async throws -> [(String, Bool)] { From 8141fdb53bdb1bc42d82fab98375db03e581d4de Mon Sep 17 00:00:00 2001 From: Raphael Barreiros Date: Sun, 4 Oct 2026 07:17:55 +0200 Subject: [PATCH 4/4] Test: Check the Apple fallback waits for the cloud clip before it Co-Authored-By: Claude Opus 5.5 (1M context) --- OpenVisionTests/CloudTTSFallbackTests.swift | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/OpenVisionTests/CloudTTSFallbackTests.swift b/OpenVisionTests/CloudTTSFallbackTests.swift index 91443e5d..26d477b9 100644 --- a/OpenVisionTests/CloudTTSFallbackTests.swift +++ b/OpenVisionTests/CloudTTSFallbackTests.swift @@ -17,6 +17,18 @@ final class CloudTTSFallbackTests: XCTestCase { XCTAssertEqual(events.map(\.1), [false, true, false, false], "only sentence two in the Apple voice") } + func testAppleSentenceWaitsForTheCloudClipBeforeIt() async throws { + // Sentence one plays for a full second; the Apple voice must not start over it. + let service = CloudTTSService(provider: Self.fakeProvider(failing: ["two"], clipSeconds: 1.0)) + var starts: [(sentence: String, apple: Bool, at: Date)] = [] + service.onSentenceStarted = { starts.append(($0, $1, Date())) } + await service.speak("Sentence one. Sentence two.") + try await waitUntilQuiet(service) + XCTAssertEqual(starts.map(\.apple), [false, true]) + let gap = starts[1].at.timeIntervalSince(starts[0].at) + XCTAssertGreaterThanOrEqual(gap, 0.9, "the Apple voice started \(gap)s after a 1s cloud clip") + } + func testTwoFailuresInARowMoveTheRestToTheAppleVoice() async throws { let events = try await play(reply, failing: ["two", "three"]) XCTAssertEqual(events.map(\.0), ["Sentence one.", "Sentence two.", "Sentence three.", "Sentence four."]) @@ -85,7 +97,8 @@ final class CloudTTSFallbackTests: XCTestCase { /// short silent clip. private final class RequestLog { var texts: [String] = [] } - private static func fakeProvider(failing: Set, log: RequestLog? = nil) -> CloudVoiceProvider { + private static func fakeProvider(failing: Set, log: RequestLog? = nil, + clipSeconds: Double = 0.15) -> CloudVoiceProvider { CloudVoiceProvider( name: "FakeTTS", isReady: { true }, @@ -95,7 +108,7 @@ final class CloudTTSFallbackTests: XCTestCase { if failing.contains(where: { text.lowercased().contains($0) }) { throw URLError(.timedOut) } let response = HTTPURLResponse(url: URL(string: "https://tts.test")!, statusCode: 200, httpVersion: nil, headerFields: nil)! - return (silentWAV(seconds: 0.15), response) + return (silentWAV(seconds: clipSeconds), response) } ) }