diff --git a/OpenVision.xcodeproj/project.pbxproj b/OpenVision.xcodeproj/project.pbxproj index f1093430..3429dc29 100644 --- a/OpenVision.xcodeproj/project.pbxproj +++ b/OpenVision.xcodeproj/project.pbxproj @@ -16,6 +16,7 @@ 1D0AEF3C7A810FF410C8F2E5 /* SessionRecorder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 06B8E5499FB1BA51EF6F1741 /* SessionRecorder.swift */; }; 1EEA4B0E379A76881244FAB9 /* SettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3DE51DA812A9394A5064F60E /* SettingsView.swift */; }; 223474C328BCC3C81ADCB93D /* OpenAIService.swift in Sources */ = {isa = PBXBuildFile; fileRef = FFDF4A693D2CEB248757FBD3 /* OpenAIService.swift */; }; + 8A6467FB1A9E725FBEF8B262 /* CloudTTSFallbackTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5D73C5CCFE6382A4B4234D4A /* CloudTTSFallbackTests.swift */; }; 2241E9878F41D5D5856C9502 /* HermesTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 641FA355FB2C0B37F5575730 /* HermesTests.swift */; }; 23D894B8F06FA0DBE93C658B /* NeuralSpeech.swift in Sources */ = {isa = PBXBuildFile; fileRef = A82F52D647139EC398E043BF /* NeuralSpeech.swift */; }; 2947D6E5D6D5B7080C53D478 /* OnboardingView.swift in Sources */ = {isa = PBXBuildFile; fileRef = C27052ECD2FFC79652B5CDB3 /* OnboardingView.swift */; }; @@ -178,6 +179,7 @@ 60E2210A5A20177647BDB5AD /* Config.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Config.swift; sourceTree = ""; }; 6102B753E2737E698B780C8E /* HermesSettingsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HermesSettingsView.swift; sourceTree = ""; }; 6188DD26571BACB30096654D /* Conversation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Conversation.swift; sourceTree = ""; }; + 5D73C5CCFE6382A4B4234D4A /* CloudTTSFallbackTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudTTSFallbackTests.swift; sourceTree = ""; }; 641FA355FB2C0B37F5575730 /* HermesTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HermesTests.swift; sourceTree = ""; }; 669EB2820345D230AB0F9FAC /* MetricsCollector+Configuration.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "MetricsCollector+Configuration.swift"; sourceTree = ""; }; 6BD412B34D09394FE85F0181 /* NeuralVoiceListView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NeuralVoiceListView.swift; sourceTree = ""; }; @@ -662,6 +664,7 @@ 39552315579536D518404FED /* DocumentChunkingTests.swift */, B88E1A3CDAF6C7B3D7DF36FE /* FrameChangeTests.swift */, 3A5609ACD2D7FE78F322E775 /* GrokTests.swift */, + 5D73C5CCFE6382A4B4234D4A /* CloudTTSFallbackTests.swift */, 641FA355FB2C0B37F5575730 /* HermesTests.swift */, DD3DFE96152AAE9145F104D6 /* LocalAgentRouteTests.swift */, BE4D7E31C137C29246EB9281 /* NativeToolSupportTests.swift */, @@ -836,6 +839,7 @@ E9628B20FB4654FBBFC61EF9 /* DocumentChunkingTests.swift in Sources */, 9A7811CF026E8DF74F74C2A2 /* FrameChangeTests.swift in Sources */, 41DD47E61F53A95E861466C1 /* GrokTests.swift in Sources */, + 8A6467FB1A9E725FBEF8B262 /* CloudTTSFallbackTests.swift in Sources */, 2241E9878F41D5D5856C9502 /* HermesTests.swift in Sources */, DC6776A7CBFBB8E151FB6B85 /* LocalAgentRouteTests.swift in Sources */, 5F19226406CF5B705B57BACF /* NativeToolSupportTests.swift in Sources */, diff --git a/OpenVision/Services/TTS/CloudTTSService.swift b/OpenVision/Services/TTS/CloudTTSService.swift index c330e72d..b9b830f2 100644 --- a/OpenVision/Services/TTS/CloudTTSService.swift +++ b/OpenVision/Services/TTS/CloudTTSService.swift @@ -9,6 +9,10 @@ // // The engine is provider-agnostic; a CloudVoiceProvider only says whether it can speak, which // voice is selected, and how to turn one sentence into WAV bytes. +// +// A sentence whose request fails (a 429, a timeout, no network) is spoken in the Apple voice in +// its place in the queue, so the reply has no silent hole. After two failures in a row the rest +// of the reply goes to the Apple voice outright rather than alternating per sentence. import AVFoundation import Foundation @@ -45,7 +49,21 @@ final class CloudTTSService: ObservableObject { /// The format the player node is connected with (the first clip's, normally 24 kHz mono). private var playerFormat: AVAudioFormat? - private init(provider: CloudVoiceProvider) { + /// Speaks failed sentences. Its own synthesizer, not TTSService: the voice agent reads + /// TTSService finishing as the end of the reply. + private let fallback = AppleFallbackSpeaker() + /// Whether the current utterance falls back to the Apple voice (replies yes, voice previews + /// no: a failed preview shows `lastFailure` instead). + private var fallbackEnabled = true + /// Failed sentences in a row; at two, the rest of the reply uses the Apple voice. + private var consecutiveFailures = 0 + private var appleForRest = false + /// Test hook: each sentence as it starts playing, and whether it's in the Apple voice. + var onSentenceStarted: ((_ sentence: String, _ appleVoice: Bool) -> Void)? + /// Test hook: how many sentences the Apple voice has actually been asked to speak. + var fallbackUtterances: Int { fallback.utterances } + + init(provider: CloudVoiceProvider) { self.provider = provider } @@ -53,15 +71,16 @@ final class CloudTTSService: ObservableObject { /// Speak a whole reply, replacing anything playing. func speak(_ text: String) async { - await speak(text, voice: provider.selectedVoice()) + await speak(text, voice: provider.selectedVoice(), appleFallback: true) } /// Speak in a specific voice — used to preview voices in Settings before choosing one. - func speak(_ text: String, voice: String) async { + func speak(_ text: String, voice: String, appleFallback: Bool = false) async { let clean = text.trimmingCharacters(in: .whitespacesAndNewlines) guard !clean.isEmpty, isReady else { return } stop() beginStreaming() + fallbackEnabled = appleFallback for sentence in TextChunking.sentences(clean) { enqueue(sentence, voice: voice) } endStreaming() } @@ -72,11 +91,12 @@ final class CloudTTSService: ObservableObject { streamStarted = false utteranceCancelled = true // in-flight synthesis discards its output pendingSentences.removeAll() - clipQueue.forEach { $0.cancel() } + clipQueue.forEach { $0.clip.cancel() } clipQueue.removeAll() pendingBuffers = 0 playerEpoch += 1 if audioReady { playerNode.stop() } + fallback.stop() } // MARK: - Streaming (speak sentences as the model produces them) @@ -88,7 +108,7 @@ final class CloudTTSService: ObservableObject { /// Synthesis tasks in sentence order, at most `maxInFlight` at a time (enough to stay ahead of /// playback without a 15-sentence reply firing 15 requests and tripping rate limits). ONE /// drainer awaits them in order, so playback order is structural. - private var clipQueue: [Task] = [] + private var clipQueue: [(text: String, clip: Task)] = [] /// Sentences waiting for a synthesis slot. private var pendingSentences: [(text: String, voice: String)] = [] private let maxInFlight = 3 @@ -123,6 +143,9 @@ final class CloudTTSService: ObservableObject { utteranceCancelled = false generationActive = true utteranceGeneration += 1 + fallbackEnabled = true + consecutiveFailures = 0 + appleForRest = false isSpeaking = true // pauses the recognizer for the whole utterance } @@ -142,7 +165,9 @@ final class CloudTTSService: ObservableObject { private func startSynthesis() { while clipQueue.count < maxInFlight, !pendingSentences.isEmpty { let next = pendingSentences.removeFirst() - clipQueue.append(Task { await self.synthesize(next.text, voice: next.voice) }) + // Once the reply has moved to the Apple voice, don't request it from the cloud. + let skip = appleForRest + clipQueue.append((next.text, Task { skip ? nil : await self.synthesize(next.text, voice: next.voice) })) } } @@ -155,16 +180,24 @@ final class CloudTTSService: ObservableObject { private func drainIfNeeded() { guard drainTask == nil else { return } drainTask = Task { [weak self] in - while let self, let task = self.clipQueue.first { - let clip = await task.value + while let self, let head = self.clipQueue.first { + let clip = await head.clip.value // stop() (and maybe a new reply) may have replaced the queue while we waited — // only play the clip if it's still the head of the CURRENT utterance. - guard self.clipQueue.first == task, !self.utteranceCancelled else { continue } + guard self.clipQueue.first?.clip == head.clip, !self.utteranceCancelled else { continue } self.clipQueue.removeFirst() + let useCloud = clip != nil && !self.appleForRest + // Count the failure before refilling the queue, so a second failure in a row + // keeps the next sentence from being sent to the cloud. + if useCloud { self.consecutiveFailures = 0 } else if self.fallbackEnabled { self.noteFailure() } self.startSynthesis() - if let clip { + if let clip, useCloud { + self.onSentenceStarted?(head.text, false) + guard !self.utteranceCancelled else { continue } self.schedule(clip, restartPlayer: !self.streamStarted) self.streamStarted = true + } else if self.fallbackEnabled { + await self.speakWithAppleVoice(head.text) } } guard let self else { return } @@ -173,6 +206,32 @@ final class CloudTTSService: ObservableObject { } } + /// Speak a sentence the cloud couldn't synthesize, in its place: after the clips queued before + /// it have played out, and before the next one. The drainer waits here, so the two audio + /// paths never overlap and `isSpeaking` stays true throughout. + private func speakWithAppleVoice(_ sentence: String) async { + // A stop, then a new reply, can happen during the wait: only speak for this one. + let generation = utteranceGeneration + var current: Bool { !utteranceCancelled && utteranceGeneration == generation } + while pendingBuffers > 0 && current { + try? await Task.sleep(for: .milliseconds(30)) + } + guard current else { return } + onSentenceStarted?(sentence, true) + guard current else { return } + MetricsCollector.shared.markFirstAudio() + await fallback.speak(sentence) + } + + private func noteFailure() { + consecutiveFailures += 1 + guard consecutiveFailures >= 2, !appleForRest else { return } + // Probably offline: stop asking the cloud for the rest of this reply. + NSLog("[%@] two sentences failed in a row, using the Apple voice for the rest", provider.name) + appleForRest = true + clipQueue.forEach { $0.clip.cancel() } // requests already out come back as failures + } + /// Ambient narration (watch loop): speaks only into silence and drops itself if a reply /// started while it was being synthesized. func speakAmbient(_ text: String) async { @@ -286,6 +345,70 @@ final class CloudTTSService: ObservableObject { extension CloudTTSService: NeuralSpeechEngine {} +// MARK: - Apple voice fallback + +/// Speaks one sentence in the user's Apple voice and returns when it has finished or been stopped. +@MainActor +final class AppleFallbackSpeaker: NSObject, AVSpeechSynthesizerDelegate { + private let synthesizer = AVSpeechSynthesizer() + /// The utterance being spoken and its waiter. A stopped utterance reports didCancel later, + /// so callbacks only finish the utterance they're about. + private var current: (id: ObjectIdentifier, finished: CheckedContinuation, watchdog: Task)? + private(set) var utterances = 0 + + /// How long one sentence may take before we stop waiting for the synthesizer: the drainer + /// waits here with the recognizer paused, so a lost didFinish must not leave it deaf. + nonisolated static func deadline(for sentence: String) -> Duration { + .milliseconds(max(5_000, 120 * sentence.count)) + } + + override init() { + super.init() + synthesizer.delegate = self + } + + func speak(_ sentence: String) async { + stop() + let utterance = AVSpeechUtterance(string: sentence) + utterance.voice = TTSService.userVoice + utterances += 1 + let id = ObjectIdentifier(utterance) + await withCheckedContinuation { continuation in + let watchdog = Task { [weak self] in + try? await Task.sleep(for: Self.deadline(for: sentence)) + guard !Task.isCancelled, let self, self.current?.id == id else { return } + NSLog("[CloudTTS] Apple voice never finished a sentence; moving on") + self.synthesizer.stopSpeaking(at: .immediate) + self.finish(id) + } + current = (id, continuation, watchdog) + synthesizer.speak(utterance) + } + } + + func stop() { + synthesizer.stopSpeaking(at: .immediate) + if let current { finish(current.id) } + } + + private func finish(_ id: ObjectIdentifier) { + guard let current, current.id == id else { return } + self.current = nil + current.watchdog.cancel() + current.finished.resume() + } + + nonisolated func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didFinish utterance: AVSpeechUtterance) { + let id = ObjectIdentifier(utterance) + Task { @MainActor in self.finish(id) } + } + + nonisolated func speechSynthesizer(_ synthesizer: AVSpeechSynthesizer, didCancel utterance: AVSpeechUtterance) { + let id = ObjectIdentifier(utterance) + Task { @MainActor in self.finish(id) } + } +} + // MARK: - Voices struct CloudVoice: Identifiable, Hashable { diff --git a/OpenVision/Services/TTS/TTSService.swift b/OpenVision/Services/TTS/TTSService.swift index e12c0865..b1e6a9a2 100644 --- a/OpenVision/Services/TTS/TTSService.swift +++ b/OpenVision/Services/TTS/TTSService.swift @@ -39,7 +39,11 @@ final class TTSService: NSObject, ObservableObject { // MARK: - Voice Selection - private var selectedVoice: AVSpeechSynthesisVoice? { + private var selectedVoice: AVSpeechSynthesisVoice? { Self.userVoice } + + /// The Apple voice chosen in Settings, or the default English voice. Also used by the cloud + /// voices' Apple fallback, so the choice lives in one place. + static var userVoice: AVSpeechSynthesisVoice? { // Check if user has selected a specific voice if let identifier = SettingsManager.shared.settings.selectedVoiceIdentifier, let voice = AVSpeechSynthesisVoice(identifier: identifier) { diff --git a/OpenVisionTests/CloudTTSFallbackTests.swift b/OpenVisionTests/CloudTTSFallbackTests.swift new file mode 100644 index 00000000..26d477b9 --- /dev/null +++ b/OpenVisionTests/CloudTTSFallbackTests.swift @@ -0,0 +1,128 @@ +// OpenVision - CloudTTSFallbackTests.swift +// A cloud-voice sentence that fails is spoken in the Apple voice, in its place in the reply (#62). +// Uses a fake provider that returns short silent WAV clips and fails chosen sentences; the audio +// really plays, so these run on a device or simulator with audio output. + +import XCTest +@testable import OpenVision + +@MainActor +final class CloudTTSFallbackTests: XCTestCase { + + private let reply = "Sentence one. Sentence two. Sentence three. Sentence four." + + func testFailedSentenceIsSpokenInTheAppleVoiceInOrder() async throws { + let events = try await play(reply, failing: ["two"]) + XCTAssertEqual(events.map(\.0), ["Sentence one.", "Sentence two.", "Sentence three.", "Sentence four."]) + XCTAssertEqual(events.map(\.1), [false, true, false, false], "only sentence two in the Apple voice") + } + + func testAppleSentenceWaitsForTheCloudClipBeforeIt() async throws { + // Sentence one plays for a full second; the Apple voice must not start over it. + let service = CloudTTSService(provider: Self.fakeProvider(failing: ["two"], clipSeconds: 1.0)) + var starts: [(sentence: String, apple: Bool, at: Date)] = [] + service.onSentenceStarted = { starts.append(($0, $1, Date())) } + await service.speak("Sentence one. Sentence two.") + try await waitUntilQuiet(service) + XCTAssertEqual(starts.map(\.apple), [false, true]) + let gap = starts[1].at.timeIntervalSince(starts[0].at) + XCTAssertGreaterThanOrEqual(gap, 0.9, "the Apple voice started \(gap)s after a 1s cloud clip") + } + + func testTwoFailuresInARowMoveTheRestToTheAppleVoice() async throws { + let events = try await play(reply, failing: ["two", "three"]) + XCTAssertEqual(events.map(\.0), ["Sentence one.", "Sentence two.", "Sentence three.", "Sentence four."]) + XCTAssertEqual(events.map(\.1), [false, true, true, true], "no alternating voices once offline") + } + + func testAfterTwoFailuresTheCloudIsNotAskedAgain() async throws { + let requests = RequestLog() + let service = CloudTTSService(provider: Self.fakeProvider(failing: ["two", "three"], log: requests)) + await service.speak("One. Sentence two. Sentence three. Four. Five. Six.") + try await waitUntilQuiet(service) + XCTAssertFalse(requests.texts.contains("Six."), "sentence six was never sent: \(requests.texts)") + } + + func testStopDuringTheFallbackStopsEverything() async throws { + let service = CloudTTSService(provider: Self.fakeProvider(failing: ["two"])) + var events: [(String, Bool)] = [] + service.onSentenceStarted = { sentence, apple in + events.append((sentence, apple)) + if apple { service.stop() } // barge-in while the Apple voice speaks sentence two + } + await service.speak(reply) + try await waitUntilQuiet(service) + try await Task.sleep(for: .seconds(1)) + XCTAssertEqual(events.map(\.0), ["Sentence one.", "Sentence two."]) + XCTAssertFalse(service.isSpeaking) + XCTAssertEqual(service.fallbackUtterances, 0, "the Apple sentence must not start after the stop") + } + + func testVoicePreviewDoesNotFallBack() async throws { + let service = CloudTTSService(provider: Self.fakeProvider(failing: ["one"])) + var events: [(String, Bool)] = [] + service.onSentenceStarted = { events.append(($0, $1)) } + await service.speak("Sentence one.", voice: "test") + try await waitUntilQuiet(service) + XCTAssertTrue(events.isEmpty, "a failed preview shows the error instead") + XCTAssertNotNil(service.lastFailure) + } + + func testAppleSentenceWaitIsBounded() { + XCTAssertEqual(AppleFallbackSpeaker.deadline(for: "Short."), .seconds(5)) + let long = String(repeating: "word ", count: 40) + XCTAssertEqual(AppleFallbackSpeaker.deadline(for: long), .milliseconds(120 * long.count)) + } + + // MARK: - Helpers + + private func play(_ text: String, failing: Set) async throws -> [(String, Bool)] { + let service = CloudTTSService(provider: Self.fakeProvider(failing: failing)) + var events: [(String, Bool)] = [] + service.onSentenceStarted = { events.append(($0, $1)) } + await service.speak(text) + try await waitUntilQuiet(service) + return events + } + + private func waitUntilQuiet(_ service: CloudTTSService, timeout: TimeInterval = 30) async throws { + let deadline = Date().addingTimeInterval(timeout) + while service.isSpeaking { + guard Date() < deadline else { return XCTFail("still speaking after \(timeout)s") } + try await Task.sleep(for: .milliseconds(50)) + } + } + + /// Fails any sentence containing one of `failing` like a dropped connection; others get a + /// short silent clip. + private final class RequestLog { var texts: [String] = [] } + + private static func fakeProvider(failing: Set, log: RequestLog? = nil, + clipSeconds: Double = 0.15) -> CloudVoiceProvider { + CloudVoiceProvider( + name: "FakeTTS", + isReady: { true }, + selectedVoice: { "test" }, + synthesize: { text, _ in + log?.texts.append(text) + if failing.contains(where: { text.lowercased().contains($0) }) { throw URLError(.timedOut) } + let response = HTTPURLResponse(url: URL(string: "https://tts.test")!, statusCode: 200, + httpVersion: nil, headerFields: nil)! + return (silentWAV(seconds: clipSeconds), response) + } + ) + } + + /// 16-bit mono PCM WAV at 24 kHz, like the providers return. + private static func silentWAV(seconds: Double, sampleRate: Int = 24_000) -> Data { + let samples = Int(Double(sampleRate) * seconds) + var data = Data() + func append(_ value: T) { withUnsafeBytes(of: value.littleEndian) { data.append(contentsOf: $0) } } + data.append(contentsOf: Array("RIFF".utf8)); append(UInt32(36 + samples * 2)) + data.append(contentsOf: Array("WAVEfmt ".utf8)); append(UInt32(16)); append(UInt16(1)); append(UInt16(1)) + append(UInt32(sampleRate)); append(UInt32(sampleRate * 2)); append(UInt16(2)); append(UInt16(16)) + data.append(contentsOf: Array("data".utf8)); append(UInt32(samples * 2)) + data.append(Data(count: samples * 2)) + return data + } +} diff --git a/README.md b/README.md index fbd56f7a..2a5fc50a 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,7 @@ With **SmolVLM2** selected as your local model, **"Ok Vision, take a photo and t ### Natural Voices — On-Device or Cloud - **Kokoro**: a **natural, offline, private voice** (Kokoro-82M) running on-device via MLX. - **Grok** (xAI, 28 multilingual voices) and **OpenAI** (`gpt-4o-mini-tts`, 13 voices) cloud voices. Grok works with your SuperGrok sign-in or an xAI key; OpenAI needs an API key with credits. +- If a cloud voice can't read a sentence (rate limit, timeout, no network), that sentence is spoken in the Apple voice instead of being skipped, and after two failures in a row the rest of the reply is too. - Pick one from the Speech Engine dropdown; every voice list has a ▶ button to hear a sample before choosing. - Apple's system voice stays the default (with Premium/Enhanced voice support).