diff --git a/Documentation/ASR/Qwen3-ASR.md b/Documentation/ASR/Qwen3-ASR.md index bbcf66d4a..2396b941a 100644 --- a/Documentation/ASR/Qwen3-ASR.md +++ b/Documentation/ASR/Qwen3-ASR.md @@ -43,7 +43,8 @@ Qwen3-ASR supports 30 languages with automatic language detection: // Auto-detect language (default) let text = try await manager.transcribe(audioSamples: samples) -// Specify language for better accuracy +// Force the output language (official Qwen3-ASR `language X` prefix after the +// assistant header; the system turn stays empty) let text = try await manager.transcribe(audioSamples: samples, language: .chinese) let text = try await manager.transcribe(audioSamples: samples, language: .japanese) ``` diff --git a/Sources/FluidAudio/ASR/Qwen3/Qwen3AsrManager.swift b/Sources/FluidAudio/ASR/Qwen3/Qwen3AsrManager.swift index fdca84a0c..d08c518db 100644 --- a/Sources/FluidAudio/ASR/Qwen3/Qwen3AsrManager.swift +++ b/Sources/FluidAudio/ASR/Qwen3/Qwen3AsrManager.swift @@ -38,7 +38,8 @@ public actor Qwen3AsrManager { /// /// - Parameters: /// - audioSamples: 16kHz mono Float32 audio samples. - /// - language: Optional language hint (ISO code like "en", "zh", or English name like "English"). + /// - language: Optional language (ISO code like "en", "zh", or English name like "English"). + /// Forces the output language with the official `language X` prefix. /// Pass nil for automatic language detection. /// - maxNewTokens: Maximum number of tokens to generate. /// - Returns: Transcribed text. @@ -97,7 +98,7 @@ public actor Qwen3AsrManager { let audioEncodeTime = CFAbsoluteTimeGetCurrent() - t1 // Step 2: Build chat template with audio tokens - let promptTokens = buildPromptTokens(numAudioFrames: numAudioFrames, language: resolvedLanguage) + let promptTokens = Qwen3AsrPrompt.promptTokens(numAudioFrames: numAudioFrames, language: resolvedLanguage) // Step 3: Swift-side embedding + audio merge let t3 = CFAbsoluteTimeGetCurrent() @@ -208,76 +209,6 @@ public actor Qwen3AsrManager { ]) } - // MARK: - Token Building - - /// Task description token IDs for language-specific transcription. - /// These are tokenized versions of "Transcribe the audio to {Language} text." - private static let taskTokens: [Qwen3AsrConfig.Language: [Int32]] = [ - .english: [3246, 56541, 279, 7461, 311, 6364, 1467, 13], - .chinese: [3246, 56541, 279, 7461, 311, 8449, 1467, 13], - .cantonese: [3246, 56541, 279, 7461, 311, 56782, 26730, 1467, 13], - .japanese: [3246, 56541, 279, 7461, 311, 11411, 1467, 13], - .korean: [3246, 56541, 279, 7461, 311, 15791, 1467, 13], - .french: [3246, 56541, 279, 7461, 311, 8620, 1467, 13], - .german: [3246, 56541, 279, 7461, 311, 6581, 1467, 13], - .spanish: [3246, 56541, 279, 7461, 311, 14610, 1467, 13], - .portuguese: [3246, 56541, 279, 7461, 311, 42322, 1467, 13], - .italian: [3246, 56541, 279, 7461, 311, 15333, 1467, 13], - .russian: [3246, 56541, 279, 7461, 311, 10479, 1467, 13], - .arabic: [3246, 56541, 279, 7461, 311, 17900, 1467, 13], - .hindi: [3246, 56541, 279, 7461, 311, 43083, 1467, 13], - .thai: [3246, 56541, 279, 7461, 311, 40764, 1467, 13], - .vietnamese: [3246, 56541, 279, 7461, 311, 48416, 1467, 13], - .indonesian: [3246, 56541, 279, 7461, 311, 66986, 1467, 13], - .malay: [3246, 56541, 279, 7461, 311, 80985, 1467, 13], - .turkish: [3246, 56541, 279, 7461, 311, 38703, 1467, 13], - .dutch: [3246, 56541, 279, 7461, 311, 19227, 1467, 13], - .swedish: [3246, 56541, 279, 7461, 311, 54259, 1467, 13], - .danish: [3246, 56541, 279, 7461, 311, 39093, 1467, 13], - .finnish: [3246, 56541, 279, 7461, 311, 56391, 1467, 13], - .polish: [3246, 56541, 279, 7461, 311, 34827, 1467, 13], - .czech: [3246, 56541, 279, 7461, 311, 51728, 1467, 13], - .greek: [3246, 56541, 279, 7461, 311, 18173, 1467, 13], - .hungarian: [3246, 56541, 279, 7461, 311, 57751, 1467, 13], - .romanian: [3246, 56541, 279, 7461, 311, 56949, 1467, 13], - .persian: [3246, 56541, 279, 7461, 311, 59181, 1467, 13], - .filipino: [3246, 56541, 279, 7461, 311, 66847, 1467, 13], - .macedonian: [3246, 56541, 279, 7461, 311, 17067, 103881, 1467, 13], - ] - - private func buildPromptTokens(numAudioFrames: Int, language: Qwen3AsrConfig.Language?) -> [Int32] { - var tokens: [Int32] = [] - - // System message with optional task description - tokens.append(Int32(Qwen3AsrConfig.imStartTokenId)) - tokens.append(Int32(Qwen3AsrConfig.systemTokenId)) - tokens.append(Int32(Qwen3AsrConfig.newlineTokenId)) - if let lang = language, let taskToks = Self.taskTokens[lang] { - tokens.append(contentsOf: taskToks) - } - tokens.append(Int32(Qwen3AsrConfig.imEndTokenId)) - tokens.append(Int32(Qwen3AsrConfig.newlineTokenId)) - - // User message with audio - tokens.append(Int32(Qwen3AsrConfig.imStartTokenId)) - tokens.append(Int32(Qwen3AsrConfig.userTokenId)) - tokens.append(Int32(Qwen3AsrConfig.newlineTokenId)) - tokens.append(Int32(Qwen3AsrConfig.audioStartTokenId)) - for _ in 0..system\n{context}<|im_end|>\n +/// <|im_start|>user\n<|audio_start|><|audio_pad|>…<|audio_end|><|im_end|>\n +/// <|im_start|>assistant\n +/// ``` +/// The system turn carries optional *context* (biasing text), not a task instruction. +/// A language is forced by appending `language {Name}` after the assistant header +/// (`Qwen3ASRModel._build_text_prompt` → `force_language`); the model then emits the +/// transcript directly instead of predicting `language X` itself. +enum Qwen3AsrPrompt { + + /// Token ID of `language` (the text before the language name in the forced prefix). + static let languageWordTokenId: Int32 = 11_528 + + /// `language {englishName}` token IDs per language. + /// + /// Generated by tokenizing the official prompt with the Qwen/Qwen3-ASR-0.6B tokenizer + /// (`AutoTokenizer.from_pretrained(..., fix_mistral_regex=True)`, as `qwen_asr` loads it): + /// `tokenize(base + f"language {name}")[len(tokenize(base)):]`, where `base` is the + /// chat template above with an empty system turn. Every row decodes back to its exact string. + /// Replaces the former system-turn `taskTokens`, whose IDs decoded to + /// `agnıs the Oper to … text.` for all 30 languages. + static let forcedLanguagePrefixTokens: [Qwen3AsrConfig.Language: [Int32]] = [ + .chinese: [11_528, 8453, 151_704], // language Chinese + .english: [11_528, 6364, 151_704], // language English + .cantonese: [11_528, 72_366, 2367, 151_704], // language Cantonese + .arabic: [11_528, 34_117, 151_704], // language Arabic + .german: [11_528, 5938, 151_704], // language German + .french: [11_528, 8585, 151_704], // language French + .spanish: [11_528, 15_154, 151_704], // language Spanish + .portuguese: [11_528, 42_188, 151_704], // language Portuguese + .indonesian: [11_528, 58_829, 151_704], // language Indonesian + .italian: [11_528, 14_811, 151_704], // language Italian + .korean: [11_528, 16_134, 151_704], // language Korean + .russian: [11_528, 8522, 151_704], // language Russian + .thai: [11_528, 26_392, 151_704], // language Thai + .vietnamese: [11_528, 48_477, 151_704], // language Vietnamese + .japanese: [11_528, 10_769, 151_704], // language Japanese + .turkish: [11_528, 23_734, 151_704], // language Turkish + .hindi: [11_528, 43_980, 151_704], // language Hindi + .malay: [11_528, 79_140, 151_704], // language Malay + .dutch: [11_528, 23_234, 151_704], // language Dutch + .swedish: [11_528, 30_109, 151_704], // language Swedish + .danish: [11_528, 43_680, 151_704], // language Danish + .finnish: [11_528, 57_853, 151_704], // language Finnish + .polish: [11_528, 31_984, 151_704], // language Polish + .czech: [11_528, 33_150, 151_704], // language Czech + .filipino: [11_528, 62_417, 151_704], // language Filipino + .persian: [11_528, 49_861, 151_704], // language Persian + .greek: [11_528, 17_860, 151_704], // language Greek + .hungarian: [11_528, 56_769, 151_704], // language Hungarian + .macedonian: [11_528, 56_452, 75_491, 151_704], // language Macedonian + .romanian: [11_528, 73_597, 151_704], // language Romanian + ] + + /// Prompt tokens for one utterance: empty system turn, audio placeholders, assistant header, + /// and — when `language` is set — the official forced-language prefix. + static func promptTokens(numAudioFrames: Int, language: Qwen3AsrConfig.Language?) -> [Int32] { + var tokens: [Int32] = [] + + // System turn (empty context). + tokens.append(Int32(Qwen3AsrConfig.imStartTokenId)) + tokens.append(Int32(Qwen3AsrConfig.systemTokenId)) + tokens.append(Int32(Qwen3AsrConfig.newlineTokenId)) + tokens.append(Int32(Qwen3AsrConfig.imEndTokenId)) + tokens.append(Int32(Qwen3AsrConfig.newlineTokenId)) + + // User turn with audio. + tokens.append(Int32(Qwen3AsrConfig.imStartTokenId)) + tokens.append(Int32(Qwen3AsrConfig.userTokenId)) + tokens.append(Int32(Qwen3AsrConfig.newlineTokenId)) + tokens.append(Int32(Qwen3AsrConfig.audioStartTokenId)) + for _ in 0..`. + if let language, let prefix = forcedLanguagePrefixTokens[language] { + tokens.append(contentsOf: prefix) + } + + return tokens + } +} diff --git a/Tests/FluidAudioTests/ASR/Qwen3/Qwen3AsrPromptTests.swift b/Tests/FluidAudioTests/ASR/Qwen3/Qwen3AsrPromptTests.swift new file mode 100644 index 000000000..3e992b13c --- /dev/null +++ b/Tests/FluidAudioTests/ASR/Qwen3/Qwen3AsrPromptTests.swift @@ -0,0 +1,61 @@ +import Foundation +import XCTest + +@testable import FluidAudio + +final class Qwen3AsrPromptTests: XCTestCase { + + private let asrText = Int32(Qwen3AsrConfig.asrTextTokenId) + + /// Official template with an empty system turn, as tokenized by the Qwen/Qwen3-ASR-0.6B tokenizer: + /// `<|im_start|>system\n<|im_end|>\n<|im_start|>user\n<|audio_start|>{audio}<|audio_end|><|im_end|>\n` + /// `<|im_start|>assistant\n` + private func officialBase(numAudioFrames: Int) -> [Int32] { + [151_644, 8948, 198, 151_645, 198, 151_644, 872, 198, 151_669] + + Array(repeating: Int32(151_676), count: numAudioFrames) + + [151_670, 151_645, 198, 151_644, 77_091, 198] + } + + func testAutoLanguageMatchesOfficialTemplate() { + XCTAssertEqual(Qwen3AsrPrompt.promptTokens(numAudioFrames: 3, language: nil), officialBase(numAudioFrames: 3)) + } + + func testForcedLanguageAppendsOfficialPrefixAfterAssistantHeader() { + let tokens = Qwen3AsrPrompt.promptTokens(numAudioFrames: 2, language: .japanese) + // `language Japanese` + XCTAssertEqual(tokens, officialBase(numAudioFrames: 2) + [11_528, 10_769, 151_704]) + } + + func testSystemTurnStaysEmptyForEveryLanguage() { + let emptySystem: [Int32] = [151_644, 8948, 198, 151_645, 198] + for language in Qwen3AsrConfig.Language.allCases { + let tokens = Qwen3AsrPrompt.promptTokens(numAudioFrames: 1, language: language) + XCTAssertEqual(Array(tokens.prefix(emptySystem.count)), emptySystem, "\(language)") + } + } + + func testEveryLanguageHasForcedPrefix() { + for language in Qwen3AsrConfig.Language.allCases { + guard let prefix = Qwen3AsrPrompt.forcedLanguagePrefixTokens[language] else { + XCTFail("missing forced prefix for \(language)") + continue + } + XCTAssertEqual(prefix.first, Qwen3AsrPrompt.languageWordTokenId, "\(language) must start with `language`") + XCTAssertEqual(prefix.last, asrText, "\(language) must end with ") + XCTAssertGreaterThanOrEqual(prefix.count, 3, "\(language) needs at least one name token") + let nameTokens = prefix.dropFirst().dropLast() + XCTAssertFalse(nameTokens.contains(asrText), "\(language)") + XCTAssertTrue( + nameTokens.allSatisfy { $0 >= 0 && $0 < 151_643 }, "\(language) name must be ordinary BPE tokens") + } + } + + func testForcedPrefixesAreDistinct() { + let prefixes = Qwen3AsrConfig.Language.allCases.compactMap { Qwen3AsrPrompt.forcedLanguagePrefixTokens[$0] } + XCTAssertEqual(Set(prefixes).count, prefixes.count) + } + + func testAutoLanguagePromptHasNoAsrTextToken() { + XCTAssertFalse(Qwen3AsrPrompt.promptTokens(numAudioFrames: 4, language: nil).contains(asrText)) + } +}