diff --git a/.github/script/asr_test.py b/.github/script/asr_test.py index dca246db0..c6e5d8dc0 100644 --- a/.github/script/asr_test.py +++ b/.github/script/asr_test.py @@ -69,9 +69,10 @@ FEATURE_GROUPS = [ "verbose_json", "timestamp_rules", "window_tail", "language", "prompt", "quiet_log", ] +# verbose_json's language, as OpenAI reports it: openai-whisper's lower-case names. LANGUAGE_NAMES = { - "en": "English", "de": "German", "es": "Spanish", "fr": "French", - "ja": "Japanese", "zh": "Chinese", "ar": "Arabic", + "en": "english", "de": "german", "es": "spanish", "fr": "french", + "ja": "japanese", "zh": "chinese", "ar": "arabic", } CHAT_PROMPT = ( "Transcribe the speech in this audio exactly, in its original language. " diff --git a/.github/workflows/asr-test.yml b/.github/workflows/asr-test.yml index a4efc678b..4ad8be406 100644 --- a/.github/workflows/asr-test.yml +++ b/.github/workflows/asr-test.yml @@ -120,11 +120,11 @@ concurrency: env: # Feature groups main passes on every platform, so a pull request that # breaks one fails. Add a group in the pull request that makes main pass it: - # verbose_json, timestamp_rules (needs verbose_json), window_tail, language, - # quiet_log. The rest are measured and reported as warnings. window_tail is - # not here yet because main's Linux HRX build loops on the long-form file; - # a pull request that adds repeats is still caught by the baseline check. - ASR_REQUIRE: ${{ inputs.require || 'prompt' }} + # window_tail, language, quiet_log. The rest are measured and reported as + # warnings. window_tail is not here yet because main's Linux HRX build loops + # on the long-form file; a pull request that adds repeats is still caught by + # the baseline check. + ASR_REQUIRE: ${{ inputs.require || 'prompt verbose_json timestamp_rules' }} ASR_MAX_DRIFT: ${{ inputs.max_drift || '' }} jobs: diff --git a/src/common/whisper/modeling_whisper.cpp b/src/common/whisper/modeling_whisper.cpp index 7c9362aee..3348eda15 100644 --- a/src/common/whisper/modeling_whisper.cpp +++ b/src/common/whisper/modeling_whisper.cpp @@ -108,7 +108,8 @@ bool Whisper::load_audio(std::vector& audio_data) { return true; } -std::pair Whisper::generate(whisper_task_type_t task, bool enable_time_stamp, bool return_time_stamp, std::ostream& os) { +std::pair Whisper::generate(whisper_task_type_t task, bool enable_time_stamp, bool return_time_stamp, std::ostream& os, + const std::string& language, const std::string& prompt) { int length = this->audio_buffer.size(); int current_idx = 0; int overlapping_samples = 5 * FS; // @@ -122,8 +123,23 @@ std::pair Whisper::generate(whisper_task_type_t task, header_print("Error", "Return_time_stamp is true but timestamp is not enabled!"); return std::make_pair("", ""); } + const int requested_language = language.empty() ? -1 : this->_language_token(language); + std::vector prompt_tokens; + if (!prompt.empty()) { + // As openai-whisper: leading space, and at most half the context. + prompt_tokens = this->tokenizer->encode(" " + prompt); + std::erase_if(prompt_tokens, [](int token) { return token >= 50257; }); // text only + const size_t max_prompt_tokens = MAX_L / 2 - 1; + if (prompt_tokens.size() > max_prompt_tokens) { + prompt_tokens.erase(prompt_tokens.begin(), prompt_tokens.end() - max_prompt_tokens); + } + } + const int prefix_length = prompt_tokens.empty() ? 0 : 1 + (int)prompt_tokens.size(); + auto emit = [&](const std::string& text) { + os << text << std::flush; + result += text; + }; while (current_idx < length){ - bool allow_force_time_stamp = true; // std::cout << "Chunk " << _S2T_(current_idx) << "s to " << _S2T_(current_idx + l_this_round) << "s" << std::endl; float time_offset = _S2T_(current_idx); @@ -144,19 +160,24 @@ std::pair Whisper::generate(whisper_task_type_t task, this->whisper_engine->clear_context(); this->sampler->reset_penalties(); - last_idx = start_of_transcript; // the first token is fixed + if (!prompt_tokens.empty()) { + this->whisper_engine->decode_audio(start_of_prev_token); + for (int token : prompt_tokens) { + this->whisper_engine->decode_audio(token); + } + } + last_idx = start_of_transcript; buffer logits = this->whisper_engine->decode_audio(last_idx); - last_idx = this->_sample_in_language(logits); + last_idx = requested_language >= 0 ? requested_language : this->_sample_in_language(logits); // std::cout << "Language detected: " << this->tokenizer->run_time_decoder(last_idx) << "(" << langmap::to_language_name(this->tokenizer->run_time_decoder(last_idx)) << ")" << std::endl; language_detected = this->tokenizer->run_time_decoder(last_idx); + // The prompt is <|startoftranscript|><|lang|><|task|>. + this->whisper_engine->decode_audio(last_idx); if (task == e_translate) { - //header_print("info", "translate is not supported! Do transcribe instead!"); - //task = e_transcribe; - //buffer logits = this->whisper_engine->decode_audio(50259); // en - //logits = this->whisper_engine->decode_audio(translate_token); - //last_idx = this->_sample_in_time_stamp(logits); + buffer logits = this->whisper_engine->decode_audio(translate_token); + last_idx = this->_sample_in_time_stamp(logits); } else if (task == e_transcribe) { buffer logits = this->whisper_engine->decode_audio(transcribe_token); @@ -182,46 +203,50 @@ std::pair Whisper::generate(whisper_task_type_t task, os << token_str << std::flush; } - int watching_dog = 16; - for (int i = 0; i < 448 - 3; i++){ + // A full window is followed by one that starts at its last timestamp, + // so text after that timestamp would be decoded twice. Hold it, with + // the timestamp that opens it, until a timestamp closes it, and drop + // both if none does. + const bool full_window = enable_time_stamp && l_this_round == WINDOW_SAMPLES; + std::string pending; + last_time_stamp = this->token_time_map_offset; + int prev_idx = -1; + int window_time_stamp = enable_time_stamp ? last_idx : -1; + for (int i = 0; i < MAX_L - 4 - prefix_length; i++){ buffer logits = this->whisper_engine->decode_audio(last_idx); - if (watching_dog == 0 && allow_force_time_stamp){ - last_idx = this->_sample_in_time_stamp(logits); - watching_dog = 16; - } - else{ - last_idx = this->sampler->sample(logits); - if (watching_dog > 0){ - watching_dog--; - } + if (enable_time_stamp){ + this->_apply_time_stamp_rules(logits, prev_idx, last_idx, window_time_stamp); } + prev_idx = last_idx; + last_idx = this->sampler->sample(logits); std::string token_str = this->tokenizer->run_time_decoder(last_idx); if (_is_normal_token(last_idx)){ - os << token_str << std::flush; - result += token_str; - } - else if (return_time_stamp && _is_time_stemp(last_idx)){ - std::string offset_time_stamp = this->_offset_time_stamp(token_str, time_offset); - os << offset_time_stamp << std::flush; - result += offset_time_stamp; + if (full_window){ + pending += token_str; + } + else{ + emit(token_str); + } } if (enable_time_stamp && _is_time_stemp(last_idx)){ + std::string ts = return_time_stamp ? this->_offset_time_stamp(token_str, time_offset) : ""; + if (full_window && _is_time_stemp(prev_idx)){ + pending = ts; + } + else{ + emit(pending); + pending.clear(); + emit(ts); + } last_time_stamp = last_idx; - watching_dog = 16; + window_time_stamp = last_idx; } if (last_idx == 50257){ break; } - - if (!_is_valid_utf8(token_str)){ - allow_force_time_stamp = false; - } - else { - allow_force_time_stamp = true; - } } @@ -232,6 +257,8 @@ std::pair Whisper::generate(whisper_task_type_t task, if (enable_time_stamp){ float end_time = _get_time(last_time_stamp); if (end_time == 0){ + // No timestamp to resume from: keep the text, skip the window. + emit(pending); end_time = 30; } l_this_round = end_time * FS; @@ -252,8 +279,9 @@ void Whisper::_build_time_map(){ this->token_time_map.clear(); auto ids = this->tokenizer->encode("<|0.00|>"); this->token_time_map_offset = ids[0]; - for (float time = 0; time <= 30.00; time += 0.02){ - this->token_time_map.push_back(time); + // <|0.00|> to <|30.00|>: 1501 tokens. Summing 0.02f drifts and stops at 29.98. + for (int i = 0; i <= WINDOW_LENGTH * 50; i++){ + this->token_time_map.push_back(i / 50.0f); } this->total_time_stamps = this->token_time_map.size(); @@ -271,12 +299,38 @@ int Whisper::_sample_in_language(buffer& logits){ } +int Whisper::_language_token(const std::string& language){ + std::vector ids = this->tokenizer->encode("<|" + language + "|>"); + if (ids.size() != 1 || ids[0] < first_language_token || ids[0] > last_language_token){ + throw std::invalid_argument("unsupported language: " + language); + } + return ids[0]; +} + int Whisper::_sample_in_time_stamp(buffer& logits){ - for (int i = 0; i < this->token_time_map_offset; i++){ - logits[i] = -0x1.FEp127f; + // The first timestamp of a window, no later than <|1.00|>. + this->_mask(logits, 0, this->token_time_map_offset); + this->_mask(logits, this->token_time_map_offset + max_initial_time_stamp + 1, logits.size()); + return this->sampler->sample(logits); +} + +// openai-whisper's ApplyTimestampRules: timestamps come in pairs around +// text, a segment is not empty, and time never goes back. +void Whisper::_apply_time_stamp_rules(buffer& logits, int prev_idx, int last_idx, int last_time_stamp_idx){ + const int begin = this->token_time_map_offset; + const int end = begin + this->total_time_stamps; + const bool last_was_time_stamp = _is_time_stemp(last_idx); + const bool penultimate_was_time_stamp = prev_idx < 0 || _is_time_stemp(prev_idx); + if (last_was_time_stamp){ + if (penultimate_was_time_stamp){ + this->_mask(logits, begin, end); // a segment needs text + } + else{ + this->_mask(logits, 0, eot_token); // a closed segment: timestamp or EOT + } } - for (int i = this->total_time_stamps + this->token_time_map_offset; i < logits.size(); i++){ - logits[i] = -0x1.FEp127f; + if (last_time_stamp_idx >= 0){ + int lowest = (last_was_time_stamp && !penultimate_was_time_stamp) ? last_time_stamp_idx : last_time_stamp_idx + 1; + this->_mask(logits, begin, std::min(lowest, end)); } - return this->sampler->sample(logits); } \ No newline at end of file diff --git a/src/include/whisper/modeling_whisper.hpp b/src/include/whisper/modeling_whisper.hpp index 2a588a639..00b031cfa 100644 --- a/src/include/whisper/modeling_whisper.hpp +++ b/src/include/whisper/modeling_whisper.hpp @@ -78,11 +78,16 @@ class Whisper { static constexpr int FS = 16000; static constexpr int WINDOW_LENGTH = 30; // seconds static constexpr int WINDOW_SAMPLES = FS * WINDOW_LENGTH; + static constexpr int MAX_L = 448; static constexpr int start_of_transcript = 50258; + static constexpr int first_language_token = 50259; + static constexpr int last_language_token = 50358; static constexpr int start_of_prev_token = 50362; static constexpr int no_time_stamp_token = 50364; static constexpr int translate_token = 50359; static constexpr int transcribe_token = 50360; + static constexpr int eot_token = 50257; + static constexpr int max_initial_time_stamp = 50; // <|1.00|>, as openai-whisper typedef enum { PREFILL_TIME, DECODING_TIME, @@ -151,52 +156,16 @@ class Whisper { } int _sample_in_language(buffer& logits); + int _language_token(const std::string& language); int _sample_in_time_stamp(buffer& logits); - - inline bool _is_valid_utf8(const std::string& input) { - size_t i = 0; - while (i < input.size()) { - unsigned char c = input[i]; - size_t width; - - // --- Validate UTF-8 codepoint --- - if (c < 0x80) { - // Single byte ASCII character - width = 1; - } - else if ((c & 0xE0) == 0xC0) { - // 2-byte character - if (i + 1 >= input.size()) return false; - // Check that second byte is a valid continuation byte - if ((input[i+1] & 0xC0) != 0x80) return false; - width = 2; - } - else if ((c & 0xF0) == 0xE0) { - // 3-byte character - if (i + 2 >= input.size()) return false; - // Check that second and third bytes are valid continuation bytes - if ((input[i+1] & 0xC0) != 0x80 || (input[i+2] & 0xC0) != 0x80) return false; - width = 3; - } - else if ((c & 0xF8) == 0xF0) { - // 4-byte character - if (i + 3 >= input.size()) return false; - // Check that second, third, and fourth bytes are valid continuation bytes - if ((input[i+1] & 0xC0) != 0x80 || (input[i+2] & 0xC0) != 0x80 || (input[i+3] & 0xC0) != 0x80) return false; - width = 4; - } - else { - // Invalid UTF-8 byte - return false; - } - - i += width; + void _apply_time_stamp_rules(buffer& logits, int prev_idx, int last_idx, int last_time_stamp_idx); + inline void _mask(buffer& logits, int begin, int end){ + for (int i = begin; i < end; i++){ + logits[i] = -0x1.FEp127f; } - return true; } - std::string _offset_time_stamp(std::string& time_stamp, float offset){ float time; sscanf(time_stamp.c_str(), "<|%f|>", &time); @@ -217,6 +186,11 @@ class Whisper { //void toggle_enable_think() override; bool load_audio(std::string& audio_path); bool load_audio(std::vector& audio_data); - std::pair generate(whisper_task_type_t task, bool enable_time_stamp, bool return_time_stamp, std::ostream& os); + /// \return the loaded audio's length in seconds + double audio_duration() const { return (double)this->audio_buffer.size() / FS; } + /// \param language ISO 639-1 code (e.g. "en"); empty to detect it per window + /// \param prompt text to condition the decoder on, as OpenAI's `prompt` + std::pair generate(whisper_task_type_t task, bool enable_time_stamp, bool return_time_stamp, std::ostream& os, + const std::string& language = "", const std::string& prompt = ""); void setup_tokenizer(std::string model_path); }; \ No newline at end of file diff --git a/src/include/whisper/transcript_format.hpp b/src/include/whisper/transcript_format.hpp new file mode 100644 index 000000000..c0d5214f2 --- /dev/null +++ b/src/include/whisper/transcript_format.hpp @@ -0,0 +1,127 @@ +/// \file transcript_format.hpp +/// \brief Turns Whisper's timestamped output into OpenAI's transcription formats +/// \author FastFlowLM Team +/// \note verbose_json segments, srt and vtt, for /v1/audio/transcriptions +#pragma once +#include +#include +#include +#include +#include +#include + +namespace transcript { + +struct Segment { + double start; + double end; + std::string text; +}; + +/// The first "<|seconds|>" at or after pos, or npos. Sets its length and value. +inline size_t find_time_stamp(const std::string& output, size_t pos, size_t& length, double& seconds) { + for (size_t at = output.find("<|", pos); at != std::string::npos; at = output.find("<|", at + 1)) { + size_t close = output.find("|>", at + 2); + if (close == std::string::npos) { + break; + } + const char* begin = output.c_str() + at + 2; + char* end = nullptr; + seconds = std::strtod(begin, &end); + if (end != begin && end == output.c_str() + close) { + length = close + 2 - at; + return at; + } + } + return std::string::npos; +} + +/// The number of bytes at the end of text that begin a UTF-8 character it does not finish. +inline size_t incomplete_utf8_tail(const std::string& text) { + for (size_t back = 1; back <= 4 && back <= text.size(); back++) { + unsigned char c = text[text.size() - back]; + if ((c & 0xC0) == 0x80) { + continue; // a continuation byte + } + size_t width = c >= 0xF0 ? 4 : c >= 0xE0 ? 3 : c >= 0xC0 ? 2 : 1; + return width > back ? back : 0; + } + return 0; +} + +/// Splits Whisper::generate's output with return_time_stamp, as in +/// "<|0.00|> Hello.<|1.50|><|1.50|> Bye.<|3.00|>", into segments. Text starts +/// at the timestamp before it and ends at the one after it; text with no +/// timestamp after it, which a window cut off, ends with the clip. +inline std::vector split_segments(const std::string& output, double duration) { + std::vector segments; + double time = 0.0; + size_t pos = 0; + std::string carry; // tokens are bytes: a timestamp can split a character + while (true) { + size_t length = 0; + double seconds = 0.0; + size_t at = find_time_stamp(output, pos, length, seconds); + std::string text = carry + output.substr(pos, at == std::string::npos ? std::string::npos : at - pos); + carry.clear(); + if (at != std::string::npos) { + size_t tail = incomplete_utf8_tail(text); + // Only spaces before the split character: they move with it. + size_t keep = tail > 0 && text.find_first_not_of(" \t\r\n") >= text.size() - tail ? 0 : text.size() - tail; + carry = text.substr(keep); + text.resize(keep); + } + if (text.find_first_not_of(" \t\r\n") != std::string::npos) { + segments.push_back({time, at == std::string::npos ? duration : seconds, text}); + } + if (at == std::string::npos) { + break; + } + time = seconds; + pos = at + length; + } + // The last window is padded to 30 s, so its timestamps can pass the end of the clip. + for (Segment& segment : segments) { + segment.start = std::clamp(segment.start, 0.0, duration); + segment.end = std::clamp(segment.end, segment.start, duration); + } + return segments; +} + +inline std::string trim(const std::string& text) { + size_t begin = text.find_first_not_of(" \t\r\n"); + if (begin == std::string::npos) { + return ""; + } + return text.substr(begin, text.find_last_not_of(" \t\r\n") - begin + 1); +} + +/// HH:MM:SS followed by the separator and milliseconds: ',' for srt, '.' for vtt. +inline std::string clock(double seconds, char separator) { + long long ms = std::llround(seconds * 1000); + char buffer[32]; + std::snprintf(buffer, sizeof(buffer), "%02lld:%02lld:%02lld%c%03lld", + ms / 3600000, ms / 60000 % 60, ms / 1000 % 60, separator, ms % 1000); + return buffer; +} + +inline std::string to_srt(const std::vector& segments) { + std::string out; + for (size_t i = 0; i < segments.size(); i++) { + out += std::to_string(i + 1) + "\n" + + clock(segments[i].start, ',') + " --> " + clock(segments[i].end, ',') + "\n" + + trim(segments[i].text) + "\n\n"; + } + return out; +} + +inline std::string to_vtt(const std::vector& segments) { + std::string out = "WEBVTT\n\n"; + for (const Segment& segment : segments) { + out += clock(segment.start, '.') + " --> " + clock(segment.end, '.') + "\n" + + trim(segment.text) + "\n\n"; + } + return out; +} + +} // namespace transcript diff --git a/src/lib/hrx/libwhisper_npu.so b/src/lib/hrx/libwhisper_npu.so index 6456cf1a4..07c2b1171 100755 Binary files a/src/lib/hrx/libwhisper_npu.so and b/src/lib/hrx/libwhisper_npu.so differ diff --git a/src/lib/hrx/whisper_npu.dll b/src/lib/hrx/whisper_npu.dll index 9298b8381..4e52141cb 100644 Binary files a/src/lib/hrx/whisper_npu.dll and b/src/lib/hrx/whisper_npu.dll differ diff --git a/src/lib/hrx/whisper_npu.lib b/src/lib/hrx/whisper_npu.lib index 761313d42..6f12315a0 100644 Binary files a/src/lib/hrx/whisper_npu.lib and b/src/lib/hrx/whisper_npu.lib differ diff --git a/src/lib/xrt/libwhisper_npu.so b/src/lib/xrt/libwhisper_npu.so index 79303d17b..d56f536ab 100755 Binary files a/src/lib/xrt/libwhisper_npu.so and b/src/lib/xrt/libwhisper_npu.so differ diff --git a/src/lib/xrt/whisper_npu.dll b/src/lib/xrt/whisper_npu.dll index f16363303..fab7e24c1 100644 Binary files a/src/lib/xrt/whisper_npu.dll and b/src/lib/xrt/whisper_npu.dll differ diff --git a/src/lib/xrt/whisper_npu.lib b/src/lib/xrt/whisper_npu.lib index 6f5f3b939..e47e58698 100644 Binary files a/src/lib/xrt/whisper_npu.lib and b/src/lib/xrt/whisper_npu.lib differ diff --git a/src/server/rest_handler.cpp b/src/server/rest_handler.cpp index 582b3efa6..ecc2c5de2 100644 --- a/src/server/rest_handler.cpp +++ b/src/server/rest_handler.cpp @@ -11,6 +11,7 @@ #include "streaming_ostream.hpp" #include "streaming_ostream_openai.hpp" #include "image/image_reader.hpp" +#include "whisper/transcript_format.hpp" #include #include #include @@ -1426,38 +1427,91 @@ void RestHandler::handle_openai_audio_transcriptions(const json& request, std::string model = request["model"]; std::string file_content = request["file"].get(); std::vector audio_raw(file_content.begin(), file_content.end()); + std::string language = request.value("language", ""); + std::string prompt = request.value("prompt", ""); + std::string response_format = request.value("response_format", "json"); bool stream = request.value("stream", false); json response; + if (response_format != "json" && response_format != "text" && response_format != "verbose_json" && + response_format != "srt" && response_format != "vtt") { + throw std::invalid_argument("unsupported response_format: " + response_format + + " (json, text, verbose_json, srt or vtt)"); + } + // Segments need the timestamps; json and text are the same transcript without them. + const bool timed = response_format == "verbose_json" || response_format == "srt" || response_format == "vtt"; if (this->asr) { #ifndef FASTFLOWLM_LINUX_LIMITED_MODELS this->whisper_engine->load_audio(audio_raw); header_print("FLM", "Transforming audio to text..."); - // Show text - std::cout << "Audio content: " << std::flush; - std::pair audio_result = this->whisper_engine->generate(Whisper::whisper_task_type_t::e_transcribe, true, false, std::cout); + // Transcripts are user speech: log them only when FLM_LOG_RAW_OUTPUT asks. + const char* log_env = std::getenv("FLM_LOG_RAW_OUTPUT"); + const bool log_transcript = log_env != nullptr && log_env[0] != '\0' && std::strcmp(log_env, "0") != 0; + std::ostream no_log(nullptr); + std::ostream& transcript_log = log_transcript ? std::cout : no_log; + transcript_log << "Audio content: " << std::flush; + std::pair audio_result = this->whisper_engine->generate(Whisper::whisper_task_type_t::e_transcribe, true, timed, transcript_log, language, prompt); std::string audio_context = audio_result.first; - std::cout << std::endl; + std::string language_name = audio_result.second; + double duration = this->whisper_engine->audio_duration(); + transcript_log << std::endl; #else throw std::runtime_error("ASR models are not supported in this build"); std::string audio_context; + std::string language_name; + double duration = 0.0; #endif - response = { - {"model", model}, - {"text", audio_context} - //{"usage", { - // {"type", "tokens"}, - // {"input_tokens", 0}, - // {"input_tokens_details", json::array({ - // { - // {"text_tokens", 0}, - // {"audio_tokens", 0} - // } - // })}, - // {"output_tokens", 0}, - // {"total_tokens", 0} - //}} - }; + if (response_format == "json") { + response = { + {"model", model}, + {"text", audio_context} + }; + } + else if (response_format == "text") { + response = audio_context; // a string goes out as a text/plain body + } + else { + std::vector segments = transcript::split_segments(audio_context, duration); + if (response_format == "srt") { + response = transcript::to_srt(segments); + } + else if (response_format == "vtt") { + response = transcript::to_vtt(segments); + } + else { + // Whisper's decoder keeps no per-segment statistics: tokens, + // avg_logprob, compression_ratio and no_speech_prob are + // placeholders that no client filter drops a segment on. + json segments_json = json::array(); + std::string text; + for (const transcript::Segment& segment : segments) { + segments_json.push_back({ + {"id", (int)segments_json.size()}, + {"seek", 0}, + {"start", segment.start}, + {"end", segment.end}, + {"text", segment.text}, + {"tokens", json::array()}, + {"temperature", 0.0}, + {"avg_logprob", 0.0}, + {"compression_ratio", 0.0}, + {"no_speech_prob", 0.0} + }); + text += segment.text; + } + // OpenAI names the language in lower case, as openai-whisper's LANGUAGES. + std::transform(language_name.begin(), language_name.end(), language_name.begin(), + [](unsigned char c) { return (char)std::tolower(c); }); + response = { + {"task", "transcribe"}, + {"language", language_name}, + {"duration", duration}, + {"text", text}, + {"segments", segments_json}, + {"model", model} + }; + } + } } else { header_print("Warning", "No asr model loaded, cannot load audio file"); @@ -1465,6 +1519,17 @@ void RestHandler::handle_openai_audio_transcriptions(const json& request, send_response(response); //this->whisper_engine->clear_context(); } + catch (const std::invalid_argument& e) { + // A bad language or response_format. A 400, as OpenAI's: clients retry a 500. + json error_response = { + {"error", { + {"message", e.what()}, + {"type", "invalid_request_error"}, + {"code", 400} + }} + }; + send_response(error_response); + } catch (const std::exception& e) { json error_response = { {"error", { diff --git a/src/server/server.cpp b/src/server/server.cpp index 471010b6f..7c99881b2 100644 --- a/src/server/server.cpp +++ b/src/server/server.cpp @@ -707,8 +707,15 @@ bool WebServer::handle_request(http::request& req, } response_ref.result(status); - response_ref.body() = response_data.dump(); - response_ref.set(http::field::content_type, "application/json"); + if (response_data.is_string()) { + // A plain-text body: the text, srt and vtt transcription formats. + response_ref.body() = response_data.get(); + response_ref.set(http::field::content_type, "text/plain; charset=utf-8"); + } + else { + response_ref.body() = response_data.dump(); + response_ref.set(http::field::content_type, "application/json"); + } response_ref.prepare_payload(); cancellation_token->complete(); unregister_active_request(request_id); @@ -979,6 +986,12 @@ std::unique_ptr create_lm_server(model_list& models, ModelDownloader& json request_json; request_json["model"] = parts["model"].content; request_json["file"] = parts["file"].content; + for (const char* field : {"language", "prompt", "response_format"}) { + auto it = parts.find(field); + if (it != parts.end()) { + request_json[field] = it->second.content; + } + } rest_handler->handle_openai_audio_transcriptions(request_json, send_response, send_streaming_response, cancellation_token); }); diff --git a/src/xclbins/Whisper-V3-Turbo-NPU2/decoder_mv.xclbin b/src/xclbins/Whisper-V3-Turbo-NPU2/decoder_mv.xclbin index baaba122f..d1a888140 100644 Binary files a/src/xclbins/Whisper-V3-Turbo-NPU2/decoder_mv.xclbin and b/src/xclbins/Whisper-V3-Turbo-NPU2/decoder_mv.xclbin differ diff --git a/src/xclbins/Whisper-V3-Turbo-NPU2/encoder_attn.xclbin b/src/xclbins/Whisper-V3-Turbo-NPU2/encoder_attn.xclbin index b34f428d9..053b65995 100644 Binary files a/src/xclbins/Whisper-V3-Turbo-NPU2/encoder_attn.xclbin and b/src/xclbins/Whisper-V3-Turbo-NPU2/encoder_attn.xclbin differ diff --git a/src/xclbins/Whisper-V3-Turbo-NPU2/encoder_dequant.xclbin b/src/xclbins/Whisper-V3-Turbo-NPU2/encoder_dequant.xclbin index fcb59caf3..10b644a98 100644 Binary files a/src/xclbins/Whisper-V3-Turbo-NPU2/encoder_dequant.xclbin and b/src/xclbins/Whisper-V3-Turbo-NPU2/encoder_dequant.xclbin differ diff --git a/src/xclbins/Whisper-V3-Turbo-NPU2/encoder_mm.xclbin b/src/xclbins/Whisper-V3-Turbo-NPU2/encoder_mm.xclbin index fece7338d..787239efa 100644 Binary files a/src/xclbins/Whisper-V3-Turbo-NPU2/encoder_mm.xclbin and b/src/xclbins/Whisper-V3-Turbo-NPU2/encoder_mm.xclbin differ diff --git a/src/xclbins/Whisper-V3-Turbo-NPU2/whisper_head.xclbin b/src/xclbins/Whisper-V3-Turbo-NPU2/whisper_head.xclbin index cf6ff8684..26dea4329 100644 Binary files a/src/xclbins/Whisper-V3-Turbo-NPU2/whisper_head.xclbin and b/src/xclbins/Whisper-V3-Turbo-NPU2/whisper_head.xclbin differ