Skip to content

fix(asr): force Qwen3-ASR language with the official prefix, not garbled task tokens #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
msandroid merged 1 commit into from
Sep 24, 2026
Merged

fix(asr): force Qwen3-ASR language with the official prefix, not garbled task tokens #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion Documentation/ASR/Qwen3-ASR.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,8 @@ Qwen3-ASR supports 30 languages with automatic language detection:
// Auto-detect language (default)
let text = try await manager.transcribe(audioSamples: samples)

// Specify language for better accuracy
// Force the output language (official Qwen3-ASR `language X<asr_text>` prefix after the
// assistant header; the system turn stays empty)
let text = try await manager.transcribe(audioSamples: samples, language: .chinese)
let text = try await manager.transcribe(audioSamples: samples, language: .japanese)
```
Expand Down
75 changes: 3 additions & 72 deletions Sources/FluidAudio/ASR/Qwen3/Qwen3AsrManager.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ public actor Qwen3AsrManager {
///
/// - Parameters:
/// - audioSamples: 16kHz mono Float32 audio samples.
/// - language: Optional language hint (ISO code like "en", "zh", or English name like "English").
/// - language: Optional language (ISO code like "en", "zh", or English name like "English").
/// Forces the output language with the official `language X<asr_text>` prefix.
/// Pass nil for automatic language detection.
/// - maxNewTokens: Maximum number of tokens to generate.
/// - Returns: Transcribed text.
Expand Down Expand Up @@ -97,7 +98,7 @@ public actor Qwen3AsrManager {
let audioEncodeTime = CFAbsoluteTimeGetCurrent() - t1

// Step 2: Build chat template with audio tokens
let promptTokens = buildPromptTokens(numAudioFrames: numAudioFrames, language: resolvedLanguage)
let promptTokens = Qwen3AsrPrompt.promptTokens(numAudioFrames: numAudioFrames, language: resolvedLanguage)

// Step 3: Swift-side embedding + audio merge
let t3 = CFAbsoluteTimeGetCurrent()
Expand Down Expand Up @@ -208,76 +209,6 @@ public actor Qwen3AsrManager {
])
}

// MARK: - Token Building

/// Task description token IDs for language-specific transcription.
/// These are tokenized versions of "Transcribe the audio to {Language} text."
private static let taskTokens: [Qwen3AsrConfig.Language: [Int32]] = [
.english: [3246, 56541, 279, 7461, 311, 6364, 1467, 13],
.chinese: [3246, 56541, 279, 7461, 311, 8449, 1467, 13],
.cantonese: [3246, 56541, 279, 7461, 311, 56782, 26730, 1467, 13],
.japanese: [3246, 56541, 279, 7461, 311, 11411, 1467, 13],
.korean: [3246, 56541, 279, 7461, 311, 15791, 1467, 13],
.french: [3246, 56541, 279, 7461, 311, 8620, 1467, 13],
.german: [3246, 56541, 279, 7461, 311, 6581, 1467, 13],
.spanish: [3246, 56541, 279, 7461, 311, 14610, 1467, 13],
.portuguese: [3246, 56541, 279, 7461, 311, 42322, 1467, 13],
.italian: [3246, 56541, 279, 7461, 311, 15333, 1467, 13],
.russian: [3246, 56541, 279, 7461, 311, 10479, 1467, 13],
.arabic: [3246, 56541, 279, 7461, 311, 17900, 1467, 13],
.hindi: [3246, 56541, 279, 7461, 311, 43083, 1467, 13],
.thai: [3246, 56541, 279, 7461, 311, 40764, 1467, 13],
.vietnamese: [3246, 56541, 279, 7461, 311, 48416, 1467, 13],
.indonesian: [3246, 56541, 279, 7461, 311, 66986, 1467, 13],
.malay: [3246, 56541, 279, 7461, 311, 80985, 1467, 13],
.turkish: [3246, 56541, 279, 7461, 311, 38703, 1467, 13],
.dutch: [3246, 56541, 279, 7461, 311, 19227, 1467, 13],
.swedish: [3246, 56541, 279, 7461, 311, 54259, 1467, 13],
.danish: [3246, 56541, 279, 7461, 311, 39093, 1467, 13],
.finnish: [3246, 56541, 279, 7461, 311, 56391, 1467, 13],
.polish: [3246, 56541, 279, 7461, 311, 34827, 1467, 13],
.czech: [3246, 56541, 279, 7461, 311, 51728, 1467, 13],
.greek: [3246, 56541, 279, 7461, 311, 18173, 1467, 13],
.hungarian: [3246, 56541, 279, 7461, 311, 57751, 1467, 13],
.romanian: [3246, 56541, 279, 7461, 311, 56949, 1467, 13],
.persian: [3246, 56541, 279, 7461, 311, 59181, 1467, 13],
.filipino: [3246, 56541, 279, 7461, 311, 66847, 1467, 13],
.macedonian: [3246, 56541, 279, 7461, 311, 17067, 103881, 1467, 13],
]

private func buildPromptTokens(numAudioFrames: Int, language: Qwen3AsrConfig.Language?) -> [Int32] {
var tokens: [Int32] = []

// System message with optional task description
tokens.append(Int32(Qwen3AsrConfig.imStartTokenId))
tokens.append(Int32(Qwen3AsrConfig.systemTokenId))
tokens.append(Int32(Qwen3AsrConfig.newlineTokenId))
if let lang = language, let taskToks = Self.taskTokens[lang] {
tokens.append(contentsOf: taskToks)
}
tokens.append(Int32(Qwen3AsrConfig.imEndTokenId))
tokens.append(Int32(Qwen3AsrConfig.newlineTokenId))

// User message with audio
tokens.append(Int32(Qwen3AsrConfig.imStartTokenId))
tokens.append(Int32(Qwen3AsrConfig.userTokenId))
tokens.append(Int32(Qwen3AsrConfig.newlineTokenId))
tokens.append(Int32(Qwen3AsrConfig.audioStartTokenId))
for _ in 0..<numAudioFrames {
tokens.append(Int32(Qwen3AsrConfig.audioTokenId))
}
tokens.append(Int32(Qwen3AsrConfig.audioEndTokenId))
tokens.append(Int32(Qwen3AsrConfig.imEndTokenId))
tokens.append(Int32(Qwen3AsrConfig.newlineTokenId))

// Assistant start
tokens.append(Int32(Qwen3AsrConfig.imStartTokenId))
tokens.append(Int32(Qwen3AsrConfig.assistantTokenId))
tokens.append(Int32(Qwen3AsrConfig.newlineTokenId))

return tokens
}

// MARK: - Swift-side Embedding & Audio Merge

private func embedAndMerge(
Expand Down
97 changes: 97 additions & 0 deletions Sources/FluidAudio/ASR/Qwen3/Qwen3AsrPrompt.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
import Foundation

/// Prompt token construction for Qwen3-ASR, matching the official `qwen_asr` package.
///
/// Official chat template (`Qwen/Qwen3-ASR-0.6B` `chat_template.json`):
/// ```
/// <|im_start|>system\n{context}<|im_end|>\n
/// <|im_start|>user\n<|audio_start|><|audio_pad|>…<|audio_end|><|im_end|>\n
/// <|im_start|>assistant\n
/// ```
/// The system turn carries optional *context* (biasing text), not a task instruction.
/// A language is forced by appending `language {Name}<asr_text>` after the assistant header
/// (`Qwen3ASRModel._build_text_prompt` → `force_language`); the model then emits the
/// transcript directly instead of predicting `language X<asr_text>` itself.
enum Qwen3AsrPrompt {

/// Token ID of `language` (the text before the language name in the forced prefix).
static let languageWordTokenId: Int32 = 11_528

/// `language {englishName}<asr_text>` token IDs per language.
///
/// Generated by tokenizing the official prompt with the Qwen/Qwen3-ASR-0.6B tokenizer
/// (`AutoTokenizer.from_pretrained(..., fix_mistral_regex=True)`, as `qwen_asr` loads it):
/// `tokenize(base + f"language {name}<asr_text>")[len(tokenize(base)):]`, where `base` is the
/// chat template above with an empty system turn. Every row decodes back to its exact string.
/// Replaces the former system-turn `taskTokens`, whose IDs decoded to
/// `agnıs the Oper to … text.` for all 30 languages.
static let forcedLanguagePrefixTokens: [Qwen3AsrConfig.Language: [Int32]] = [
.chinese: [11_528, 8453, 151_704], // language Chinese<asr_text>
.english: [11_528, 6364, 151_704], // language English<asr_text>
.cantonese: [11_528, 72_366, 2367, 151_704], // language Cantonese<asr_text>
.arabic: [11_528, 34_117, 151_704], // language Arabic<asr_text>
.german: [11_528, 5938, 151_704], // language German<asr_text>
.french: [11_528, 8585, 151_704], // language French<asr_text>
.spanish: [11_528, 15_154, 151_704], // language Spanish<asr_text>
.portuguese: [11_528, 42_188, 151_704], // language Portuguese<asr_text>
.indonesian: [11_528, 58_829, 151_704], // language Indonesian<asr_text>
.italian: [11_528, 14_811, 151_704], // language Italian<asr_text>
.korean: [11_528, 16_134, 151_704], // language Korean<asr_text>
.russian: [11_528, 8522, 151_704], // language Russian<asr_text>
.thai: [11_528, 26_392, 151_704], // language Thai<asr_text>
.vietnamese: [11_528, 48_477, 151_704], // language Vietnamese<asr_text>
.japanese: [11_528, 10_769, 151_704], // language Japanese<asr_text>
.turkish: [11_528, 23_734, 151_704], // language Turkish<asr_text>
.hindi: [11_528, 43_980, 151_704], // language Hindi<asr_text>
.malay: [11_528, 79_140, 151_704], // language Malay<asr_text>
.dutch: [11_528, 23_234, 151_704], // language Dutch<asr_text>
.swedish: [11_528, 30_109, 151_704], // language Swedish<asr_text>
.danish: [11_528, 43_680, 151_704], // language Danish<asr_text>
.finnish: [11_528, 57_853, 151_704], // language Finnish<asr_text>
.polish: [11_528, 31_984, 151_704], // language Polish<asr_text>
.czech: [11_528, 33_150, 151_704], // language Czech<asr_text>
.filipino: [11_528, 62_417, 151_704], // language Filipino<asr_text>
.persian: [11_528, 49_861, 151_704], // language Persian<asr_text>
.greek: [11_528, 17_860, 151_704], // language Greek<asr_text>
.hungarian: [11_528, 56_769, 151_704], // language Hungarian<asr_text>
.macedonian: [11_528, 56_452, 75_491, 151_704], // language Macedonian<asr_text>
.romanian: [11_528, 73_597, 151_704], // language Romanian<asr_text>
]

/// Prompt tokens for one utterance: empty system turn, audio placeholders, assistant header,
/// and — when `language` is set — the official forced-language prefix.
static func promptTokens(numAudioFrames: Int, language: Qwen3AsrConfig.Language?) -> [Int32] {
var tokens: [Int32] = []

// System turn (empty context).
tokens.append(Int32(Qwen3AsrConfig.imStartTokenId))
tokens.append(Int32(Qwen3AsrConfig.systemTokenId))
tokens.append(Int32(Qwen3AsrConfig.newlineTokenId))
tokens.append(Int32(Qwen3AsrConfig.imEndTokenId))
tokens.append(Int32(Qwen3AsrConfig.newlineTokenId))

// User turn with audio.
tokens.append(Int32(Qwen3AsrConfig.imStartTokenId))
tokens.append(Int32(Qwen3AsrConfig.userTokenId))
tokens.append(Int32(Qwen3AsrConfig.newlineTokenId))
tokens.append(Int32(Qwen3AsrConfig.audioStartTokenId))
for _ in 0..<numAudioFrames {
tokens.append(Int32(Qwen3AsrConfig.audioTokenId))
}
tokens.append(Int32(Qwen3AsrConfig.audioEndTokenId))
tokens.append(Int32(Qwen3AsrConfig.imEndTokenId))
tokens.append(Int32(Qwen3AsrConfig.newlineTokenId))

// Assistant header.
tokens.append(Int32(Qwen3AsrConfig.imStartTokenId))
tokens.append(Int32(Qwen3AsrConfig.assistantTokenId))
tokens.append(Int32(Qwen3AsrConfig.newlineTokenId))

// Forced language: `language {Name}<asr_text>`.
if let language, let prefix = forcedLanguagePrefixTokens[language] {
tokens.append(contentsOf: prefix)
}

return tokens
}
}
61 changes: 61 additions & 0 deletions Tests/FluidAudioTests/ASR/Qwen3/Qwen3AsrPromptTests.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
import Foundation
import XCTest

@testable import FluidAudio

final class Qwen3AsrPromptTests: XCTestCase {

private let asrText = Int32(Qwen3AsrConfig.asrTextTokenId)

/// Official template with an empty system turn, as tokenized by the Qwen/Qwen3-ASR-0.6B tokenizer:
/// `<|im_start|>system\n<|im_end|>\n<|im_start|>user\n<|audio_start|>{audio}<|audio_end|><|im_end|>\n`
/// `<|im_start|>assistant\n`
private func officialBase(numAudioFrames: Int) -> [Int32] {
[151_644, 8948, 198, 151_645, 198, 151_644, 872, 198, 151_669]
+ Array(repeating: Int32(151_676), count: numAudioFrames)
+ [151_670, 151_645, 198, 151_644, 77_091, 198]
}

func testAutoLanguageMatchesOfficialTemplate() {
XCTAssertEqual(Qwen3AsrPrompt.promptTokens(numAudioFrames: 3, language: nil), officialBase(numAudioFrames: 3))
}

func testForcedLanguageAppendsOfficialPrefixAfterAssistantHeader() {
let tokens = Qwen3AsrPrompt.promptTokens(numAudioFrames: 2, language: .japanese)
// `language Japanese<asr_text>`
XCTAssertEqual(tokens, officialBase(numAudioFrames: 2) + [11_528, 10_769, 151_704])
}

func testSystemTurnStaysEmptyForEveryLanguage() {
let emptySystem: [Int32] = [151_644, 8948, 198, 151_645, 198]
for language in Qwen3AsrConfig.Language.allCases {
let tokens = Qwen3AsrPrompt.promptTokens(numAudioFrames: 1, language: language)
XCTAssertEqual(Array(tokens.prefix(emptySystem.count)), emptySystem, "\(language)")
}
}

func testEveryLanguageHasForcedPrefix() {
for language in Qwen3AsrConfig.Language.allCases {
guard let prefix = Qwen3AsrPrompt.forcedLanguagePrefixTokens[language] else {
XCTFail("missing forced prefix for \(language)")
continue
}
XCTAssertEqual(prefix.first, Qwen3AsrPrompt.languageWordTokenId, "\(language) must start with `language`")
XCTAssertEqual(prefix.last, asrText, "\(language) must end with <asr_text>")
XCTAssertGreaterThanOrEqual(prefix.count, 3, "\(language) needs at least one name token")
let nameTokens = prefix.dropFirst().dropLast()
XCTAssertFalse(nameTokens.contains(asrText), "\(language)")
XCTAssertTrue(
nameTokens.allSatisfy { $0 >= 0 && $0 < 151_643 }, "\(language) name must be ordinary BPE tokens")
}
}

func testForcedPrefixesAreDistinct() {
let prefixes = Qwen3AsrConfig.Language.allCases.compactMap { Qwen3AsrPrompt.forcedLanguagePrefixTokens[$0] }
XCTAssertEqual(Set(prefixes).count, prefixes.count)
}

func testAutoLanguagePromptHasNoAsrTextToken() {
XCTAssertFalse(Qwen3AsrPrompt.promptTokens(numAudioFrames: 4, language: nil).contains(asrText))
}
}
Loading