diff --git a/.github/script/asr_summary.py b/.github/script/asr_summary.py new file mode 100644 index 000000000..9b55203c3 --- /dev/null +++ b/.github/script/asr_summary.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python3 +"""Render ASR test results as one Markdown report, a section per model. + +Reads every *_results.json that asr_test.py wrote under the given directory +(only those in an asr-results folder when the artifacts also hold baseline +runs) and prints GitHub-flavoured Markdown to stdout, for $GITHUB_STEP_SUMMARY. +Each model gets its own section with the platforms side by side, so a reader +can see at once which model failed where and why. + +asr_test.py also calls render() for the report of a single run. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +PLATFORM_ORDER = ["linux", "linux-hrx", "windows", "windows-hrx", + "linux-models-from-source", "windows-models-from-source"] +PLATFORM_LABELS = { + "linux": "Linux", "linux-hrx": "Linux HRX", "windows": "Windows", + "windows-hrx": "Windows HRX", "linux-models-from-source": "Linux (models from source)", + "windows-models-from-source": "Windows (models from source)", +} +WHISPER_PREFIX = "whisper-v3" + + +def pct(value) -> str: + return "-" if value is None else f"{value * 100:.2f}%" + + +def points(current, base) -> str: + if current is None or base is None: + return "" + return f" ({(current - base) * 100:+.2f})" + + +def label(platform: str) -> str: + return PLATFORM_LABELS.get(platform, platform) + + +def cell(text: str) -> str: + return " ".join(str(text).split()).replace("|", "\\|") + + +def reference_name(result: dict) -> str: + base = result.get("baseline") or {} + if base.get("kind") == "build": + return "main" + if base.get("kind") == "expected": + return "expected" + return "" + + +def model_checks(result: dict, model: str) -> list[dict]: + return [c for c in result.get("checks", []) if c["model"] == model] + + +def verdict(result: dict | None, model: str) -> str: + if result is None: + return "no results" + checks = model_checks(result, model) + if not checks: + return "not run" + return "**FAIL**" if any(c["status"] == "FAIL" for c in checks) else "**PASS**" + + +def whisper_models(results: list[dict]) -> list[str]: + models = [] + for result in results: + for check in result.get("checks", []): + if check["model"].startswith(WHISPER_PREFIX) and check["model"] not in models: + models.append(check["model"]) + return models or [f"{WHISPER_PREFIX}:turbo"] + + +def chat_models(results: list[dict]) -> list[str]: + models = [] + for result in results: + for model in result.get("chat", {}): + if model not in models: + models.append(model) + return models + + +def problems(lines: list[str], results: list[dict], model: str) -> None: + """Failed checks and warnings for one model, grouped by platform.""" + lines += ["**Failed checks and warnings**", ""] + any_listed = False + for result in results: + rows = [c for c in model_checks(result, model) if c["status"] != "ok"] + if not rows: + continue + any_listed = True + lines.append(f"*{label(result['platform'])}*") + lines.append("") + for row in rows: + status = "FAIL" if row["status"] == "FAIL" else "warning" + group = "" if row["group"] == "core" else f" [{row['group']}]" + detail = f": {cell(row['detail'])}" if row["detail"] else "" + lines.append(f"- **{status}**{group} {cell(row['area'])}: " + f"{cell(row['check'])}{detail}") + lines.append("") + if not any_listed: + lines += ["_None._", ""] + + +def whisper_section(lines: list[str], results: list[dict], model: str) -> None: + columns = [label(r["platform"]) for r in results] + ref = next((reference_name(r) for r in results if reference_name(r)), "") + lines += [f"## {model}: speech-to-text (`/v1/audio/transcriptions`)", "", + "**Verdict:** " + " · ".join(f"{label(r['platform'])} {verdict(r, model)}" + for r in results), ""] + + lines += ["**Accuracy**", "", + "Error rate on each FLEURS set: WER, or CER for ja and zh. " + + (f"In brackets, the change in points against {ref}. " if ref else "") + + "Lower is better.", "", + "| Set | Metric | " + " | ".join(columns) + " |", + "|---|---|" + "---|" * len(columns)] + set_names: list[str] = [] + for result in results: + set_names += [s for s in result.get("sets", {}) if s not in set_names] + for set_name in set_names: + metric = next((r["sets"][set_name]["metric"].upper() for r in results + if set_name in r.get("sets", {})), "") + cells = [] + for result in results: + current = result.get("sets", {}).get(set_name) + base = ((result.get("baseline") or {}).get("sets", {}).get(set_name) + or {}).get("error_rate") + cells.append("-" if current is None else + pct(current["error_rate"]) + points(current["error_rate"], base)) + lines.append(f"| {set_name} | {metric} | " + " | ".join(cells) + " |") + cells = [] + for result in results: + current = (result.get("overall") or {}).get("error_rate") + base = ((result.get("baseline") or {}).get("overall") or {}).get("error_rate") + cells.append(f"**{pct(current)}**{points(current, base)}") + lines.append("| **pooled** | all | " + " | ".join(cells) + " |") + lines.append("") + + lines += ["**Behaviour**", "", + "| Measure | " + " | ".join(columns) + " |", + "|---|" + "---|" * len(columns)] + + def row(name: str, render) -> None: + lines.append(f"| {name} | " + " | ".join(render(r) for r in results) + " |") + + def runaways(result: dict) -> str: + current = (result.get("overall") or {}).get("runaways") + base = ((result.get("baseline") or {}).get("overall") or {}).get("runaways") + if current is None: + return "-" + return f"{current}" + (f" ({ref} {base})" if base is not None else "") + + def changed(result: dict) -> str: + sets = result.get("sets", {}).values() + if not any("changed" in s for s in sets): + return "-" + return f"{sum(s.get('changed', 0) for s in sets)}/{sum(s['files'] for s in sets)}" + + def speed(result: dict) -> str: + sets = result.get("sets", {}) + audio = sum(s.get("audio_s", 0) for s in sets.values()) + wall = sum(s.get("wall_s", 0) for s in sets.values()) + base_sets = (result.get("baseline") or {}).get("sets", {}) + base_audio = sum(s.get("audio_s", 0) for s in base_sets.values()) + base_wall = sum(s.get("wall_s", 0) for s in base_sets.values()) + text = f"{wall / audio:.3f}" if audio else "-" + if base_audio and ref == "main": + text += f" (main {base_wall / base_audio:.3f})" + return text + + row("Runaway transcripts (repetition loops)", runaways) + row("Files whose transcript changed vs main", changed) + row("Processing seconds per audio second", speed) + lines.append("") + + groups: list[str] = [] + for result in results: + groups += [g for g in result.get("features", {}) if g not in groups] + lines += ["**Checks by group** (passed/total; required groups fail the run)", "", + "| Group | Required | " + " | ".join(columns) + " |", + "|---|---|" + "---|" * len(columns)] + for group in groups: + required = "yes" if group == "core" or group in results[0].get("required", []) else "no" + cells = [] + for result in results: + checks = [c for c in model_checks(result, model) if c["group"] == group] + if not checks: + cells.append("-") + continue + passed = sum(c["status"] == "ok" for c in checks) + cells.append(f"{passed}/{len(checks)}") + lines.append(f"| {group} | {required} | " + " | ".join(cells) + " |") + lines.append("") + problems(lines, results, model) + + +def chat_section(lines: list[str], results: list[dict], model: str) -> None: + columns = [label(r["platform"]) for r in results] + lines += [f"## {model}: chat transcription (audio sent to the chat model)", "", + "**Verdict:** " + " · ".join(f"{label(r['platform'])} {verdict(r, model)}" + for r in results), "", + "| Measure | " + " | ".join(columns) + " |", + "|---|" + "---|" * len(columns)] + cells, clips = [], [] + for result in results: + current = result.get("chat", {}).get(model) + base = ((result.get("baseline") or {}).get("chat") or {}).get(model) + if current is None: + cells.append("-") + clips.append("-") + continue + text = pct(current["error_rate"]) + if base: + text += f" (main {pct(base['error_rate'])})" + cells.append(text) + clips.append(str(current["files"])) + lines.append("| WER on English clips | " + " | ".join(cells) + " |") + lines.append("| Clips | " + " | ".join(clips) + " |") + lines.append("") + problems(lines, results, model) + + +def render(results: list[dict], title: str = "ASR Test") -> str: + results = sorted(results, key=lambda r: (PLATFORM_ORDER.index(r["platform"]) + if r["platform"] in PLATFORM_ORDER else 99, + r["platform"])) + lines = [f"# {title}", ""] + if not results: + return "\n".join(lines + ["_No results were produced._", ""]) + + whisper = whisper_models(results) + chats = chat_models(results) + refs = {reference_name(r) for r in results} - {""} + required = results[0].get("required", []) + lines += [ + "Builds under test: flm " + ", ".join(sorted({str(r.get('version')) for r in results})) + + ". Compared against: " + (" and ".join( + "main's latest build, run first on the same machine" if r == "main" + else "the expected rates in asr_eval/expected.json" for r in sorted(refs)) + or "nothing") + ".", + "", + "Required feature groups: " + (", ".join(required) or "none") + + ". Other groups are measured and reported as warnings.", + "", + "| Model | " + " | ".join(label(r["platform"]) for r in results) + " |", + "|---|" + "---|" * len(results), + ] + for model in whisper + chats: + lines.append(f"| {model} | " + " | ".join(verdict(r, model) for r in results) + " |") + lines.append("") + + for model in whisper: + whisper_section(lines, results, model) + for model in chats: + chat_section(lines, results, model) + + other = [c for r in results for c in r.get("checks", []) + if c["status"] == "FAIL" and c["model"] not in whisper + chats] + if other: + lines += ["## Other failures", ""] + lines += [f"- {cell(c['model'])}: {cell(c['check'])} {cell(c['detail'])}" for c in other] + lines.append("") + return "\n".join(lines) + + +def load_results(root: Path) -> list[dict]: + paths = sorted(root.rglob("*_results.json")) + # Artifacts also carry the baseline's own run; report the builds under test. + if any("asr-results" in p.parts for p in paths): + paths = [p for p in paths if "asr-results" in p.parts] + results = [] + for path in paths: + try: + result = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + print(f"") + continue + if "platform" in result and "checks" in result: + results.append(result) + return results + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("directory", help="Directory holding the downloaded artifacts.") + args = parser.parse_args(argv) + print(render(load_results(Path(args.directory)))) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/script/asr_test.py b/.github/script/asr_test.py new file mode 100644 index 000000000..ccf7a4374 --- /dev/null +++ b/.github/script/asr_test.py @@ -0,0 +1,706 @@ +#!/usr/bin/env python3 +"""Measure and check ASR through /v1/audio/transcriptions and chat audio. + +sweep_audio.py only asks for a non-empty transcript. This runs the FLEURS +evaluation set in assets/asr_eval/ (see build_asr_eval.py) and measures: + + accuracy WER per set, CER for ja and zh, against human transcripts, + each file's errors capped at its reference length so one + runaway transcript cannot swamp a set; and a pooled rate + over every set + runaways transcripts over 1.5 times the reference length, which is + what a decoder stuck in a repetition loop produces + speed seconds of processing per second of audio + formats json, no response_format and verbose_json agree + chat audio WER of chat models that transcribe audio: Gemma 4 E2B, whose + audio front end E4B and the flash variants share, and 12B, + which has its own + +and checks these feature groups: + + verbose_json segments that are ordered, inside the clip, non-empty and + join into the text; duration equal to the clip length + timestamp_rules segments never overlap; the first one starts within 1 s + window_tail no text repeated across 30 s window boundaries + language the language field is honoured; unknown codes fail + prompt a prompt is accepted and helps with the words it names + quiet_log transcripts stay out of the server log + +A failed check in a group fails the run only if the group is listed in +--require; otherwise it is reported as "warn". Core checks always count. The +groups exist so a feature can be required once main has it, and measured +before then. + +With --baseline, the results of another build (normally main's, run on the +same machine just before) become the reference. The run fails if the pooled +error rate rises by more than --tolerance points, any one set's by more than +--set-tolerance, or the number of runaways grows. The report also shows how +many transcripts changed. Even a numerically equivalent change, such as a +different FFT, flips a few borderline decodes, so a few changed files and +small per-set moves are expected; --max-drift is for builds that should +reproduce the baseline exactly. Without a baseline, --expected gives fixed +error rates to stay under, with the same --tolerance on the pooled rate and +--set-tolerance per set; the workflow passes --tolerance 2 there, since those +rates come from one machine per platform. + +Every response body, and a results JSON that --baseline can read back, are +written to --output-dir. +""" + +from __future__ import annotations + +import base64 +import json +import re +import socket +import sys +import time +import unicodedata +from collections import Counter +from pathlib import Path + +from asr_summary import label as platform_label, render +from sweep_common import ASSETS_DIR, SweepTask, run_task + +FEATURE_GROUPS = [ + "verbose_json", "timestamp_rules", "window_tail", "language", "prompt", "quiet_log", +] +LANGUAGE_NAMES = { + "en": "English", "de": "German", "es": "Spanish", "fr": "French", + "ja": "Japanese", "zh": "Chinese", "ar": "Arabic", +} +CHAT_PROMPT = ( + "Transcribe the speech in this audio exactly, in its original language. " + "Reply with the transcript only." +) +# MP3 encoder padding and the server's own rounding. +DURATION_TOLERANCE_S = 0.1 +# The encoder is not bit-exact between passes, so two requests for the same +# clip can differ in a word or two (see the PR #794 notes). +SAME_TEXT_MAX_WER = 0.02 +REPEAT_NGRAM = 6 +RUNAWAY_RATIO = 1.5 +ARABIC_DIACRITICS = re.compile("[\u064b-\u065f\u0670\u06d6-\u06ed]") + + +# -- scoring ------------------------------------------------------------------ + +def normalize(text: str, language: str) -> str: + """Lowercase, no punctuation or symbols, single spaces, NFKC.""" + text = unicodedata.normalize("NFKC", text).lower() + if language == "ar": + text = ARABIC_DIACRITICS.sub("", text) + text = "".join(" " if unicodedata.category(c)[0] in "PS" else c for c in text) + return " ".join(text.split()) + + +def tokens(text: str, language: str, metric: str) -> list[str]: + text = normalize(text, language) + return list(text.replace(" ", "")) if metric == "cer" else text.split() + + +def edit_distance(ref: list[str], hyp: list[str]) -> int: + previous = list(range(len(hyp) + 1)) + for i, ref_token in enumerate(ref, 1): + current = [i] + for j, hyp_token in enumerate(hyp, 1): + current.append(min(previous[j] + 1, current[j - 1] + 1, + previous[j - 1] + (ref_token != hyp_token))) + previous = current + return previous[-1] + + +def error_rate(ref: str, hyp: str, language: str, metric: str) -> tuple[int, int]: + """(edits, reference length), so rates can be pooled over a set.""" + ref_tokens = tokens(ref, language, metric) + return edit_distance(ref_tokens, tokens(hyp, language, metric)), len(ref_tokens) + + +def pooled(pairs) -> float: + """Error rate over many files, each file's edits capped at its length.""" + edits = sum(min(e, n) for e, n in pairs) + length = sum(n for _, n in pairs) + return edits / length if length else 0.0 + + +def repeats(text: str, n: int = REPEAT_NGRAM) -> int: + """n-word sequences that occur more than once.""" + words = normalize(text, "en").split() + counts = Counter(tuple(words[i:i + n]) for i in range(len(words) - n + 1)) + return sum(c - 1 for c in counts.values() if c > 1) + + +def pct(value: float | None) -> str: + return "-" if value is None else f"{value * 100:.2f}%" + + +# -- test --------------------------------------------------------------------- + +class AsrTest(SweepTask): + name = "asr" + needs_asr = True + csv_header = ["Model", "Area", "Group", "Check", "Status", "Detail"] + + @staticmethod + def add_arguments(parser) -> None: + parser.add_argument("--eval-dir", default=str(ASSETS_DIR / "asr_eval"), + help="Directory holding manifest.json and the clips.") + parser.add_argument("--sets", nargs="+", default=None, metavar="SET", + help="Run only these sets (default: all).") + parser.add_argument("--require", nargs="*", default=[], metavar="GROUP", + choices=FEATURE_GROUPS, + help="Feature groups whose failures fail the run.") + parser.add_argument("--baseline", default=None, metavar="RESULTS_JSON", + help="Results of a reference build to compare against.") + parser.add_argument("--expected", default=None, metavar="JSON", + help="Error rates per platform and set to stay under when there " + "is no baseline. Builds decode differently per platform, " + "so only --platform's entry is used.") + parser.add_argument("--tolerance", type=float, default=1.0, + help="Allowed rise in the pooled error rate, in points (default: 1.0).") + parser.add_argument("--set-tolerance", type=float, default=5.0, + help="Allowed rise in any one set's error rate, in points " + "(default: 5.0). Sets are ten files, so one flipped " + "decode moves them by a point or more.") + parser.add_argument("--max-drift", type=float, default=None, + help="Fail a set whose transcripts differ from the baseline's " + "by more than this fraction, for builds meant to change " + "nothing. Unset: drift is only reported.") + parser.add_argument("--chat-models", nargs="*", + default=["gemma4-it:e2b", "gemma4-it:12b"], metavar="TAG", + help="Chat models to transcribe with (default: gemma4-it:e2b " + "gemma4-it:12b). Pass none to skip.") + parser.add_argument("--chat-clips", type=int, default=5, + help="en-short clips sent to each chat model (default: 5).") + parser.add_argument("--chat-tolerance", type=float, default=3.0, + help="Allowed rise in chat WER against the baseline, in points.") + + def __init__(self, args): + super().__init__(args) + self.rows: list[list[str]] = [] + self.check_failures: set[str] = set() + self.results: dict = {"sets": {}, "items": {}, "chat": {}, "features": {}, + "repeats": {}, "language": {}} + self.baseline = (json.loads(Path(args.baseline).read_text(encoding="utf-8")) + if args.baseline else None) + self.expected = None + if args.expected: + platforms = json.loads(Path(args.expected).read_text(encoding="utf-8"))["platforms"] + self.expected = platforms.get(args.platform) + if self.expected is None: + print(f"::warning::{args.expected} has no expected rates for " + f"{args.platform}; accuracy is reported but not checked.") + self.responses_dir = self.output_dir / "responses" + self.responses_dir.mkdir(exist_ok=True) + + def start_server(self, append_log: bool = False) -> None: + """Refuses to start when another server already holds the port. + + flm exits at once when it cannot bind, but the other server answers + /v1/version, so the run would carry on against a server without ASR, + or one that is running someone else's job. + """ + if not self.args.no_server: + try: + socket.create_connection((self.args.host, self.args.port), timeout=2).close() + except OSError: + pass + else: + raise RuntimeError(f"Port {self.args.port} on {self.args.host} is already in " + "use by another server; pass --port to use a free one.") + super().start_server(append_log) + + def select_models(self, catalog: list[dict]) -> list[str]: + models = [e["id"] for e in catalog if e["id"].startswith("whisper-v3")] + if not models: + print("[asr] No Whisper models found. The /v1/models fallback hides " + "Whisper, so pass --models explicitly if 'flm list' failed.") + return models + + # -- recording --------------------------------------------------------- + + def check(self, writer, model: str, area: str, name: str, ok: bool, + detail: str = "", group: str = "core") -> None: + required = group == "core" or group in self.args.require + status = "ok" if ok else ("FAIL" if required else "warn") + row = [model, area, group, name, status, detail] + self.rows.append(row) + error = None if ok or not required else f"{area}: {name} ({detail})" + if error: + self.check_failures.add(f"{model}: {error}") + self.record(writer, row, model=model, error=error) + feature = self.results["features"].setdefault(group, {"passed": 0, "total": 0}) + feature["total"] += 1 + feature["passed"] += ok + if not ok or group == "core": + print(f" {status:4} [{group}] {area}: {name}" + (f" ({detail})" if detail else "")) + + def save_body(self, model: str, label: str, text: str) -> None: + safe = re.sub(r"[^\w.-]", "_", f"{model}_{label}") + (self.responses_dir / f"{safe}.json").write_text(text, encoding="utf-8") + + # -- requests ---------------------------------------------------------- + + def transcribe(self, model: str, path: Path, label: str, **fields) -> tuple[int, dict | None, float]: + """(HTTP status, parsed body or None, seconds). The body is saved first.""" + kwargs = {key: value for key, value in fields.items() if value is not None} + started = time.monotonic() + with open(path, "rb") as handle: + raw = self.client.audio.transcriptions.with_raw_response.create( + model=model, file=handle, **kwargs) + seconds = time.monotonic() - started + response = raw.http_response + self.save_body(model, label, response.text) + try: + body = json.loads(response.text) + except json.JSONDecodeError: + body = None + return response.status_code, body, seconds + + def request(self, writer, model: str, path: Path, label: str, area: str, + **fields) -> tuple[dict | None, float]: + """A request that is expected to succeed; failures become core checks.""" + try: + status, body, seconds = self.transcribe(model, path, label, **fields) + except Exception as exc: + self.check(writer, model, area, f"{label}: request succeeds", False, str(exc)) + return None, 0.0 + if status >= 400 or body is None or "error" in body: + # The server answers with an empty body when ASR is not loaded. + self.check(writer, model, area, f"{label}: request succeeds", False, + f"HTTP {status}: {json.dumps(body)[:120] if body else 'not JSON'}") + return None, seconds + return body, seconds + + # -- formats ----------------------------------------------------------- + + def same_text(self, a: str, b: str) -> tuple[bool, str]: + edits, length = error_rate(a, b, "en", "wer") + rate = edits / length if length else float(bool(b.strip())) + return rate <= SAME_TEXT_MAX_WER, f"{rate:.1%} apart" if rate else "identical" + + def check_segments(self, writer, model: str, area: str, body: dict, duration: float, + failures: Counter) -> None: + """verbose_json and timestamp_rules checks, tallied into `failures`.""" + segments = body.get("segments") or [] + text = body.get("text") or "" + problems = { + "duration is the clip length": + not isinstance(body.get("duration"), (int, float)) + or abs(body["duration"] - duration) > DURATION_TOLERANCE_S, + "segment ids count up from 0": + [s.get("id") for s in segments] != list(range(len(segments))), + "segments lie inside the clip": + any(not 0 <= s["start"] <= s["end"] <= duration + DURATION_TOLERANCE_S + for s in segments), + "segments are in order": + any(b["start"] < a["start"] for a, b in zip(segments, segments[1:])), + "no empty or zero-length segments": + any(s["end"] - s["start"] < 0.01 or not s["text"].strip() for s in segments), + "segments join into the text": + "".join(s["text"] for s in segments).strip() != text.strip(), + } + for name, bad in problems.items(): + failures[("verbose_json", name)] += bad + rules = { + "segments never overlap": + any(b["start"] < a["end"] - 0.005 for a, b in zip(segments, segments[1:])), + "first segment starts within 1 s": + bool(segments) and segments[0]["start"] > 1.0, + } + for name, bad in rules.items(): + failures[("timestamp_rules", name)] += bad + + def run_formats(self, writer, model: str, item: dict, path: Path) -> None: + """json, default and verbose_json on one clip must agree.""" + area = f"formats ({item['file']})" + plain, _ = self.request(writer, model, path, "formats_json", area, response_format="json") + default, _ = self.request(writer, model, path, "formats_default", area) + verbose, _ = self.request(writer, model, path, "formats_verbose_json", area, + response_format="verbose_json") + if plain is None: + return + text = plain.get("text") or "" + self.check(writer, model, area, "json text is non-empty", bool(text.strip())) + if default is not None: + self.check(writer, model, area, "default returns only model and text", + set(default) == {"model", "text"}, ", ".join(sorted(default))) + ok, detail = self.same_text(text, default.get("text") or "") + self.check(writer, model, area, "default text matches json", ok, detail) + if verbose is not None: + has_segments = "segments" in verbose + self.check(writer, model, area, "verbose_json returns segments", has_segments, + "" if has_segments else "server ignores verbose_json", + group="verbose_json") + if has_segments: + ok, detail = self.same_text(text, verbose.get("text") or "") + self.check(writer, model, area, "verbose_json text matches json", ok, detail, + group="verbose_json") + + # -- evaluation -------------------------------------------------------- + + def run_eval(self, writer, model: str, items: list[dict], eval_dir: Path) -> None: + """One verbose_json request per clip: accuracy, speed and segments.""" + segment_failures: Counter = Counter() + verbose_items = 0 + by_set: dict[str, list[dict]] = {} + for item in items: + by_set.setdefault(item["set"], []).append(item) + + for set_name, set_items in by_set.items(): + print(f" set {set_name}: {len(set_items)} files") + pairs, audio_s, wall_s, detected, runaways = [], 0.0, 0.0, 0, 0 + for item in set_items: + body, seconds = self.request(writer, model, eval_dir / item["file"], + f"eval_{item['file']}", f"eval {set_name}", + response_format="verbose_json") + if body is None: + continue + text = body.get("text") or "" + edits, length = error_rate(item["reference"], text, item["language"], + item["metric"]) + pairs.append((edits, length)) + runaway = len(tokens(text, item["language"], item["metric"])) > RUNAWAY_RATIO * length + runaways += runaway + audio_s += item["duration"] + wall_s += seconds + self.results["items"][item["file"]] = { + "set": set_name, "language": item["language"], "metric": item["metric"], + "text": text, "seconds": seconds, "edits": edits, "length": length, + "runaway": runaway} + if body.get("language") == LANGUAGE_NAMES[item["language"]]: + detected += 1 + if "segments" in body: + verbose_items += 1 + self.check_segments(writer, model, f"eval {set_name}", body, + item["duration"], segment_failures) + if item["set"] in ("en-long", "en-longform"): + self.results["repeats"][item["file"]] = { + "reference": repeats(item["reference"]), "text": repeats(text)} + + rate = pooled(pairs) + self.results["sets"][set_name] = { + "metric": set_items[0]["metric"], "error_rate": rate, "files": len(pairs), + "runaways": runaways, + "audio_s": audio_s, "wall_s": wall_s, + "rtf": wall_s / audio_s if audio_s else None, + "language_detected": detected if verbose_items else None, + } + print(f" {set_items[0]['metric'].upper()} {pct(rate)}, {runaways} runaways, " + f"{audio_s:.0f} s of audio in {wall_s:.1f} s") + self.check(writer, model, f"eval {set_name}", "every file transcribed", + len(pairs) == len(set_items), f"{len(pairs)}/{len(set_items)}") + + all_items = self.results["items"].values() + self.results["overall"] = { + "error_rate": pooled([(r["edits"], r["length"]) for r in all_items]), + "runaways": sum(r["runaway"] for r in all_items), + "files": len(self.results["items"]), + } + print(f" pooled error rate {pct(self.results['overall']['error_rate'])}, " + f"{self.results['overall']['runaways']} runaways") + + # Without segments, run_formats has already reported verbose_json. + for (group, name), bad in sorted(segment_failures.items()): + self.check(writer, model, "eval", name, bad == 0, + f"{bad} of {verbose_items} files" if bad else f"{verbose_items} files", + group=group) + + for name, counts in self.results["repeats"].items(): + self.check(writer, model, f"eval {name}", + f"no {REPEAT_NGRAM}-word phrase repeated beyond the reference", + counts["text"] <= counts["reference"], + f"{counts['text']} repeats, reference {counts['reference']}", + group="window_tail") + + # -- feature probes ---------------------------------------------------- + + def run_language(self, writer, model: str, items: list[dict], eval_dir: Path) -> None: + """Forcing the language: same or better accuracy, and actually honoured.""" + group, area = "language", "language" + for language in sorted({i["language"] for i in items} - {"en"}): + probe = [i for i in items if i["language"] == language][:3] + pairs, reported = [], [] + for item in probe: + body, _ = self.request(writer, model, eval_dir / item["file"], + f"language_{item['file']}", area, + response_format="verbose_json", language=language) + if body is not None: + pairs.append(error_rate(item["reference"], body.get("text") or "", + language, item["metric"])) + if "segments" in body: + reported.append(body.get("language")) + auto = pooled([(self.results["items"][i["file"]]["edits"], + self.results["items"][i["file"]]["length"]) + for i in probe if i["file"] in self.results["items"]]) + forced = pooled(pairs) + self.results["language"][language] = {"auto": auto, "forced": forced} + self.check(writer, model, area, f"{language}: forcing it does not hurt", + len(pairs) == len(probe) + and forced <= auto + self.args.set_tolerance / 100, + f"{pct(forced)} forced, {pct(auto)} detected", group=group) + # Only verbose_json reports the language. Forcing a wrong language + # is no test: Whisper often keeps transcribing what it hears. + if reported: + self.check(writer, model, area, f"{language}: response reports it", + all(r == LANGUAGE_NAMES[language] for r in reported), + ", ".join(sorted({str(r) for r in reported})), group=group) + + english = next((i for i in items if i["set"] == "en-short"), None) + if english is None: + return + path = eval_dir / english["file"] + try: + status, bad_body, _ = self.transcribe(model, path, "language_invalid", language="xx") + rejected = status >= 400 or bad_body is None or "error" in bad_body + detail = f"HTTP {status}" + except Exception as exc: + rejected, detail = True, type(exc).__name__ + self.check(writer, model, area, "an unknown language code is rejected", rejected, + detail, group=group) + + def run_prompt(self, writer, model: str, items: list[dict], eval_dir: Path) -> None: + """A prompt naming the clip's proper nouns should not hurt, and should help them.""" + group, area = "prompt", "prompt" + probe = [i for i in items if i["set"] == "en-short"][:5] + hits_without = hits_with = names_total = 0 + pairs_without, pairs_with = [], [] + for item in probe: + names = sorted({w.strip(".,;:!?\"'()") for w in item["raw_reference"].split()[1:] + if w[:1].isupper()} - {""}) + if not names: + continue + body, _ = self.request(writer, model, eval_dir / item["file"], + f"prompt_{item['file']}", area, prompt=" ".join(names)) + before = self.results["items"].get(item["file"]) + if body is None or before is None: + continue + after = body.get("text") or "" + pairs_with.append(error_rate(item["reference"], after, "en", "wer")) + pairs_without.append((before["edits"], before["length"])) + names_total += len(names) + hits_without += sum(n in before["text"] for n in names) + hits_with += sum(n in after for n in names) + if not names_total: + return + with_rate, without_rate = pooled(pairs_with), pooled(pairs_without) + self.results["prompt"] = {"names": names_total, "hits_with": hits_with, + "hits_without": hits_without, + "wer_with": with_rate, "wer_without": without_rate} + self.check(writer, model, area, "a prompt does not hurt accuracy", + with_rate <= without_rate + self.args.set_tolerance / 100, + f"WER {pct(with_rate)} with, {pct(without_rate)} without", group=group) + self.check(writer, model, area, "names in the prompt are spelled as given", + hits_with >= hits_without, + f"{hits_with}/{names_total} with, {hits_without}/{names_total} without", + group=group) + + def run_quiet_log(self, writer, model: str) -> None: + # Streamed output interleaves timestamp markers with the text. + log = re.sub(r"<\|[\d.]+\|>", "", self.server_log_path.read_text( + encoding="utf-8", errors="replace")) + log = "".join(log.split()) + leaked = [f for f, r in self.results["items"].items() + if len("".join(r["text"].split())) >= 20 + and "".join(r["text"].split())[:20] in log] + self.check(writer, model, "server log", "transcripts are not logged", not leaked, + f"{len(leaked)} of {len(self.results['items'])} transcripts found" + if leaked else "", group="quiet_log") + + # -- chat audio -------------------------------------------------------- + + def run_chat(self, writer, items: list[dict], eval_dir: Path) -> None: + probe = [i for i in items if i["set"] == "en-short"][:self.args.chat_clips] + for model in self.args.chat_models: + print(f"\n--- Chat transcription: {model} ---") + pairs, failed = [], 0 + for item in probe: + encoded = base64.b64encode((eval_dir / item["file"]).read_bytes()).decode() + try: + response = self.client.chat.completions.create( + model=model, temperature=0.0, stream=False, + messages=[{"role": "user", "content": [ + {"type": "text", "text": CHAT_PROMPT}, + {"type": "input_audio", + "input_audio": {"data": encoded, "format": "mp3"}}, + ]}], + **self.completion_kwargs()) + text = response.choices[0].message.content or "" + except Exception as exc: + failed += 1 + text = "" + print(f" ERROR {item['file']}: {exc}") + self.save_body(model, f"chat_{item['file']}", json.dumps({"text": text})) + pairs.append(error_rate(item["reference"], text, "en", "wer")) + rate = pooled(pairs) + self.results["chat"][model] = {"error_rate": rate, "files": len(probe)} + print(f" WER {pct(rate)} over {len(probe)} clips") + self.check(writer, model, "chat", "every clip transcribed", failed == 0, + f"{len(probe) - failed}/{len(probe)}") + self.check(writer, model, "chat", "WER is under 50%", rate < 0.5, pct(rate)) + base = (self.baseline or {}).get("chat", {}).get(model) + if base: + self.check(writer, model, "chat", "WER no worse than the baseline", + rate - base["error_rate"] <= self.args.chat_tolerance / 100, + f"{pct(rate)} vs {pct(base['error_rate'])}") + + # -- comparison -------------------------------------------------------- + + def baseline_overall(self) -> dict | None: + """The baseline's pooled rate over the same files, so --sets compares like with like.""" + base_items = (self.baseline or {}).get("items", {}) + shared = [base_items[f] for f in self.results["items"] if f in base_items] + if not shared: + return None + return {"error_rate": pooled([(r["edits"], r["length"]) for r in shared]), + "runaways": sum(r.get("runaway", False) for r in shared)} + + def compare(self, writer, model: str) -> None: + """Error rates against the baseline build, or the expected values.""" + tolerance = self.args.tolerance / 100 + set_tolerance = self.args.set_tolerance / 100 + overall = self.results["overall"] + if self.baseline: + base_items = self.baseline.get("items", {}) + base_overall = self.baseline_overall() + if base_overall: + self.check(writer, model, "eval", "pooled error rate no worse than the baseline", + overall["error_rate"] - base_overall["error_rate"] <= tolerance, + f"{pct(overall['error_rate'])} vs {pct(base_overall['error_rate'])}") + self.check(writer, model, "eval", "no more runaways than the baseline", + overall["runaways"] <= base_overall["runaways"], + f"{overall['runaways']} vs {base_overall['runaways']}") + for set_name, current in self.results["sets"].items(): + base = self.baseline["sets"].get(set_name) + if not base: + continue + delta = current["error_rate"] - base["error_rate"] + self.check(writer, model, f"eval {set_name}", + "error rate no worse than the baseline", delta <= set_tolerance, + f"{pct(current['error_rate'])} vs {pct(base['error_rate'])}") + compared = [(f, r) for f, r in self.results["items"].items() + if r["set"] == set_name and f in base_items] + drift = pooled([error_rate(base_items[f]["text"], r["text"], r["language"], + r["metric"]) for f, r in compared]) + current["drift"] = drift + # Normalized, so whitespace and punctuation alone do not count. + current["changed"] = sum( + normalize(base_items[f]["text"], r["language"]) + != normalize(r["text"], r["language"]) for f, r in compared) + if self.args.max_drift is not None: + self.check(writer, model, f"eval {set_name}", + "transcripts match the baseline's", drift <= self.args.max_drift, + f"{pct(drift)} of words differ") + if base.get("rtf") and current.get("rtf") and current["rtf"] > base["rtf"] * 1.2: + print(f"::warning::{set_name}: {current['rtf']:.3f} s per audio second, " + f"baseline {base['rtf']:.3f}") + for name, counts in self.results["repeats"].items(): + base = self.baseline.get("repeats", {}).get(name) + if base: + self.check(writer, model, f"eval {name}", + "no more repeated phrases than the baseline", + counts["text"] <= base["text"], + f"{counts['text']} vs {base['text']}") + elif self.expected: + limit = self.expected.get("overall") + if limit is not None: + self.check(writer, model, "eval", "pooled error rate within the expected value", + overall["error_rate"] <= limit + tolerance, + f"{pct(overall['error_rate'])} vs expected {pct(limit)}") + for set_name, current in self.results["sets"].items(): + limit = self.expected.get("sets", {}).get(set_name) + if limit is not None: + self.check(writer, model, f"eval {set_name}", + "error rate within the expected value", + current["error_rate"] <= limit + set_tolerance, + f"{pct(current['error_rate'])} vs expected {pct(limit)}") + + # -- report ------------------------------------------------------------ + + @property + def report_path(self) -> Path: + return self.output_dir / f"{self.name}_{self.args.platform}_v{self.version}.md" + + @property + def results_path(self) -> Path: + return self.output_dir / f"{self.name}_{self.args.platform}_v{self.version}_results.json" + + def baseline_snapshot(self) -> dict | None: + """What this run was compared against, in one shape for asr_summary.py.""" + if self.baseline: + return { + "kind": "build", + "version": self.baseline.get("version"), + "overall": self.baseline_overall(), + "sets": {name: self.baseline["sets"][name] for name in self.results["sets"] + if name in self.baseline.get("sets", {})}, + "chat": self.baseline.get("chat", {}), + } + if self.expected: + return { + "kind": "expected", + "overall": {"error_rate": self.expected.get("overall")}, + "sets": {name: {"error_rate": rate} + for name, rate in self.expected.get("sets", {}).items()}, + } + return None + + def finish_results(self) -> None: + checks = [dict(zip(("model", "area", "group", "check", "status", "detail"), row)) + for row in self.rows] + # Failures that are not a check, such as the server crashing, still + # belong in the report. + checks += [{"model": "server", "area": "run", "group": "core", "check": failure, + "status": "FAIL", "detail": ""} + for failure in self.failures if failure not in self.check_failures] + self.results.update({ + "platform": self.args.platform, + "version": self.version, + "required": list(self.args.require), + "checks": checks, + "baseline": self.baseline_snapshot(), + }) + + def write_report(self) -> None: + self.report_path.write_text( + render([self.results], title=f"ASR test: {platform_label(self.args.platform)}") + + "\n", encoding="utf-8") + + # -- entry point ------------------------------------------------------- + + def run(self) -> None: + eval_dir = Path(self.args.eval_dir) + manifest = json.loads((eval_dir / "manifest.json").read_text(encoding="utf-8")) + items = [i for i in manifest["items"] + if not self.args.sets or i["set"] in self.args.sets] + print(f"[asr] {len(items)} files from {eval_dir}, " + f"required groups: {', '.join(self.args.require) or 'none'}") + self.results["version"] = self.version + + models = self.resolve_models() + if not models: + self.failures.append("no Whisper model to test") + handle, writer = self.open_csv() + try: + for model in self.iter_models(models): + print(f"\n--- ASR: {model} ---") + formats_item = next((i for i in items if i["set"] == "en-long"), items[0]) + self.run_formats(writer, model, formats_item, eval_dir / formats_item["file"]) + self.run_eval(writer, model, items, eval_dir) + self.run_language(writer, model, items, eval_dir) + self.run_prompt(writer, model, items, eval_dir) + self.run_quiet_log(writer, model) + self.compare(writer, model) + self.run_chat(writer, items, eval_dir) + finally: + handle.close() + self.finish_results() + self.results_path.write_text(json.dumps(self.results, indent=1, ensure_ascii=False), + encoding="utf-8") + self.write_report() + print(f"\nASR test complete. Report: {self.report_path}") + self.write_summary(models) + + +if __name__ == "__main__": + sys.exit(run_task(AsrTest)) diff --git a/.github/script/assets/asr_eval/README.md b/.github/script/assets/asr_eval/README.md new file mode 100644 index 000000000..5edc4629b --- /dev/null +++ b/.github/script/assets/asr_eval/README.md @@ -0,0 +1,41 @@ +# ASR evaluation set + +The audio and reference transcripts used by `.github/script/asr_test.py`. + +## Source and license + +Every clip and transcript here comes from the test split of **FLEURS** +(`google/fleurs` on Hugging Face), licensed under +[CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). The clips were +re-encoded to 16 kHz mono MP3, and the English ones in `en-long` and +`en-longform` were joined end to end with 0.6 s of silence between them. The +transcripts are unchanged. + +> Conneau, A., Ma, M., Khanuja, S., Zhang, Y., Axelrod, V., Dalmia, S., +> Riesa, J., Rivera, C., and Bapna, A. *FLEURS: Few-shot Learning Evaluation +> of Universal Representations of Speech.* IEEE SLT 2022. +> https://arxiv.org/abs/2205.12446 + +`manifest.json` records the FLEURS Parquet revision and, for every file, the +FLEURS sentence ids it was built from. + +## Contents + +| Set | Files | Audio | Metric | What it exercises | +|---|---|---|---|---| +| `en-short` | 20 | 194 s | WER | single 30 s window | +| `en-long` | 3 | 252 s | WER | ~83 s each, two window boundaries | +| `en-longform` | 1 | 308 s | WER | ten window boundaries | +| `de-fleurs`, `es-fleurs`, `fr-fleurs`, `ar-fleurs` | 10 each | 105-139 s each | WER | other languages | +| `ja-fleurs`, `zh-fleurs` | 10 each | 104-142 s each | CER | languages written without spaces | + +`expected.json` holds the error rates main reached, per platform and set, for +runs that have no baseline build to compare against. Platforms are kept apart +because their builds decode differently: Windows and Linux differ by up to +5 points on some sets with identical sources. + +## Rebuilding + +`.github/script/build_asr_eval.py` regenerates this directory from the FLEURS +Parquet files; its docstring says where to get them. The selection is +deterministic, so the same Parquet revision gives the same set. diff --git a/.github/script/assets/asr_eval/ar-fleurs-00.mp3 b/.github/script/assets/asr_eval/ar-fleurs-00.mp3 new file mode 100644 index 000000000..56967e37a Binary files /dev/null and b/.github/script/assets/asr_eval/ar-fleurs-00.mp3 differ diff --git a/.github/script/assets/asr_eval/ar-fleurs-01.mp3 b/.github/script/assets/asr_eval/ar-fleurs-01.mp3 new file mode 100644 index 000000000..06163e17b Binary files /dev/null and b/.github/script/assets/asr_eval/ar-fleurs-01.mp3 differ diff --git a/.github/script/assets/asr_eval/ar-fleurs-02.mp3 b/.github/script/assets/asr_eval/ar-fleurs-02.mp3 new file mode 100644 index 000000000..ff50a02a9 Binary files /dev/null and b/.github/script/assets/asr_eval/ar-fleurs-02.mp3 differ diff --git a/.github/script/assets/asr_eval/ar-fleurs-03.mp3 b/.github/script/assets/asr_eval/ar-fleurs-03.mp3 new file mode 100644 index 000000000..3dcb602db Binary files /dev/null and b/.github/script/assets/asr_eval/ar-fleurs-03.mp3 differ diff --git a/.github/script/assets/asr_eval/ar-fleurs-04.mp3 b/.github/script/assets/asr_eval/ar-fleurs-04.mp3 new file mode 100644 index 000000000..f4aae3f18 Binary files /dev/null and b/.github/script/assets/asr_eval/ar-fleurs-04.mp3 differ diff --git a/.github/script/assets/asr_eval/ar-fleurs-05.mp3 b/.github/script/assets/asr_eval/ar-fleurs-05.mp3 new file mode 100644 index 000000000..6323615eb Binary files /dev/null and b/.github/script/assets/asr_eval/ar-fleurs-05.mp3 differ diff --git a/.github/script/assets/asr_eval/ar-fleurs-06.mp3 b/.github/script/assets/asr_eval/ar-fleurs-06.mp3 new file mode 100644 index 000000000..a57d6648b Binary files /dev/null and b/.github/script/assets/asr_eval/ar-fleurs-06.mp3 differ diff --git a/.github/script/assets/asr_eval/ar-fleurs-07.mp3 b/.github/script/assets/asr_eval/ar-fleurs-07.mp3 new file mode 100644 index 000000000..a77774742 Binary files /dev/null and b/.github/script/assets/asr_eval/ar-fleurs-07.mp3 differ diff --git a/.github/script/assets/asr_eval/ar-fleurs-08.mp3 b/.github/script/assets/asr_eval/ar-fleurs-08.mp3 new file mode 100644 index 000000000..166506a98 Binary files /dev/null and b/.github/script/assets/asr_eval/ar-fleurs-08.mp3 differ diff --git a/.github/script/assets/asr_eval/ar-fleurs-09.mp3 b/.github/script/assets/asr_eval/ar-fleurs-09.mp3 new file mode 100644 index 000000000..838f55953 Binary files /dev/null and b/.github/script/assets/asr_eval/ar-fleurs-09.mp3 differ diff --git a/.github/script/assets/asr_eval/de-fleurs-00.mp3 b/.github/script/assets/asr_eval/de-fleurs-00.mp3 new file mode 100644 index 000000000..02715b9f5 Binary files /dev/null and b/.github/script/assets/asr_eval/de-fleurs-00.mp3 differ diff --git a/.github/script/assets/asr_eval/de-fleurs-01.mp3 b/.github/script/assets/asr_eval/de-fleurs-01.mp3 new file mode 100644 index 000000000..443b797db Binary files /dev/null and b/.github/script/assets/asr_eval/de-fleurs-01.mp3 differ diff --git a/.github/script/assets/asr_eval/de-fleurs-02.mp3 b/.github/script/assets/asr_eval/de-fleurs-02.mp3 new file mode 100644 index 000000000..da9baec1f Binary files /dev/null and b/.github/script/assets/asr_eval/de-fleurs-02.mp3 differ diff --git a/.github/script/assets/asr_eval/de-fleurs-03.mp3 b/.github/script/assets/asr_eval/de-fleurs-03.mp3 new file mode 100644 index 000000000..e405728ed Binary files /dev/null and b/.github/script/assets/asr_eval/de-fleurs-03.mp3 differ diff --git a/.github/script/assets/asr_eval/de-fleurs-04.mp3 b/.github/script/assets/asr_eval/de-fleurs-04.mp3 new file mode 100644 index 000000000..838dd215c Binary files /dev/null and b/.github/script/assets/asr_eval/de-fleurs-04.mp3 differ diff --git a/.github/script/assets/asr_eval/de-fleurs-05.mp3 b/.github/script/assets/asr_eval/de-fleurs-05.mp3 new file mode 100644 index 000000000..a094b0d78 Binary files /dev/null and b/.github/script/assets/asr_eval/de-fleurs-05.mp3 differ diff --git a/.github/script/assets/asr_eval/de-fleurs-06.mp3 b/.github/script/assets/asr_eval/de-fleurs-06.mp3 new file mode 100644 index 000000000..44a073ecf Binary files /dev/null and b/.github/script/assets/asr_eval/de-fleurs-06.mp3 differ diff --git a/.github/script/assets/asr_eval/de-fleurs-07.mp3 b/.github/script/assets/asr_eval/de-fleurs-07.mp3 new file mode 100644 index 000000000..edf85b39e Binary files /dev/null and b/.github/script/assets/asr_eval/de-fleurs-07.mp3 differ diff --git a/.github/script/assets/asr_eval/de-fleurs-08.mp3 b/.github/script/assets/asr_eval/de-fleurs-08.mp3 new file mode 100644 index 000000000..85f371cbf Binary files /dev/null and b/.github/script/assets/asr_eval/de-fleurs-08.mp3 differ diff --git a/.github/script/assets/asr_eval/de-fleurs-09.mp3 b/.github/script/assets/asr_eval/de-fleurs-09.mp3 new file mode 100644 index 000000000..865d9e15b Binary files /dev/null and b/.github/script/assets/asr_eval/de-fleurs-09.mp3 differ diff --git a/.github/script/assets/asr_eval/en-long-0.mp3 b/.github/script/assets/asr_eval/en-long-0.mp3 new file mode 100644 index 000000000..d1ba6957d Binary files /dev/null and b/.github/script/assets/asr_eval/en-long-0.mp3 differ diff --git a/.github/script/assets/asr_eval/en-long-1.mp3 b/.github/script/assets/asr_eval/en-long-1.mp3 new file mode 100644 index 000000000..1d952f724 Binary files /dev/null and b/.github/script/assets/asr_eval/en-long-1.mp3 differ diff --git a/.github/script/assets/asr_eval/en-long-2.mp3 b/.github/script/assets/asr_eval/en-long-2.mp3 new file mode 100644 index 000000000..737f5f5e9 Binary files /dev/null and b/.github/script/assets/asr_eval/en-long-2.mp3 differ diff --git a/.github/script/assets/asr_eval/en-longform-0.mp3 b/.github/script/assets/asr_eval/en-longform-0.mp3 new file mode 100644 index 000000000..a51ea947b Binary files /dev/null and b/.github/script/assets/asr_eval/en-longform-0.mp3 differ diff --git a/.github/script/assets/asr_eval/en-short-00.mp3 b/.github/script/assets/asr_eval/en-short-00.mp3 new file mode 100644 index 000000000..029d9b79f Binary files /dev/null and b/.github/script/assets/asr_eval/en-short-00.mp3 differ diff --git a/.github/script/assets/asr_eval/en-short-01.mp3 b/.github/script/assets/asr_eval/en-short-01.mp3 new file mode 100644 index 000000000..e082c14b0 Binary files /dev/null and b/.github/script/assets/asr_eval/en-short-01.mp3 differ diff --git a/.github/script/assets/asr_eval/en-short-02.mp3 b/.github/script/assets/asr_eval/en-short-02.mp3 new file mode 100644 index 000000000..0736e3274 Binary files /dev/null and b/.github/script/assets/asr_eval/en-short-02.mp3 differ diff --git a/.github/script/assets/asr_eval/en-short-03.mp3 b/.github/script/assets/asr_eval/en-short-03.mp3 new file mode 100644 index 000000000..f8afbd538 Binary files /dev/null and b/.github/script/assets/asr_eval/en-short-03.mp3 differ diff --git a/.github/script/assets/asr_eval/en-short-04.mp3 b/.github/script/assets/asr_eval/en-short-04.mp3 new file mode 100644 index 000000000..1e645c823 Binary files /dev/null and b/.github/script/assets/asr_eval/en-short-04.mp3 differ diff --git a/.github/script/assets/asr_eval/en-short-05.mp3 b/.github/script/assets/asr_eval/en-short-05.mp3 new file mode 100644 index 000000000..ba2071f2f Binary files /dev/null and b/.github/script/assets/asr_eval/en-short-05.mp3 differ diff --git a/.github/script/assets/asr_eval/en-short-06.mp3 b/.github/script/assets/asr_eval/en-short-06.mp3 new file mode 100644 index 000000000..bbf0d5abd Binary files /dev/null and b/.github/script/assets/asr_eval/en-short-06.mp3 differ diff --git a/.github/script/assets/asr_eval/en-short-07.mp3 b/.github/script/assets/asr_eval/en-short-07.mp3 new file mode 100644 index 000000000..68766d659 Binary files /dev/null and b/.github/script/assets/asr_eval/en-short-07.mp3 differ diff --git a/.github/script/assets/asr_eval/en-short-08.mp3 b/.github/script/assets/asr_eval/en-short-08.mp3 new file mode 100644 index 000000000..b715bfda5 Binary files /dev/null and b/.github/script/assets/asr_eval/en-short-08.mp3 differ diff --git a/.github/script/assets/asr_eval/en-short-09.mp3 b/.github/script/assets/asr_eval/en-short-09.mp3 new file mode 100644 index 000000000..424000bdb Binary files /dev/null and b/.github/script/assets/asr_eval/en-short-09.mp3 differ diff --git a/.github/script/assets/asr_eval/en-short-10.mp3 b/.github/script/assets/asr_eval/en-short-10.mp3 new file mode 100644 index 000000000..8e93eee35 Binary files /dev/null and b/.github/script/assets/asr_eval/en-short-10.mp3 differ diff --git a/.github/script/assets/asr_eval/en-short-11.mp3 b/.github/script/assets/asr_eval/en-short-11.mp3 new file mode 100644 index 000000000..93a6234bb Binary files /dev/null and b/.github/script/assets/asr_eval/en-short-11.mp3 differ diff --git a/.github/script/assets/asr_eval/en-short-12.mp3 b/.github/script/assets/asr_eval/en-short-12.mp3 new file mode 100644 index 000000000..3c955bb21 Binary files /dev/null and b/.github/script/assets/asr_eval/en-short-12.mp3 differ diff --git a/.github/script/assets/asr_eval/en-short-13.mp3 b/.github/script/assets/asr_eval/en-short-13.mp3 new file mode 100644 index 000000000..7de5fabe6 Binary files /dev/null and b/.github/script/assets/asr_eval/en-short-13.mp3 differ diff --git a/.github/script/assets/asr_eval/en-short-14.mp3 b/.github/script/assets/asr_eval/en-short-14.mp3 new file mode 100644 index 000000000..b74695b6c Binary files /dev/null and b/.github/script/assets/asr_eval/en-short-14.mp3 differ diff --git a/.github/script/assets/asr_eval/en-short-15.mp3 b/.github/script/assets/asr_eval/en-short-15.mp3 new file mode 100644 index 000000000..8afaa8e7a Binary files /dev/null and b/.github/script/assets/asr_eval/en-short-15.mp3 differ diff --git a/.github/script/assets/asr_eval/en-short-16.mp3 b/.github/script/assets/asr_eval/en-short-16.mp3 new file mode 100644 index 000000000..6a1d03824 Binary files /dev/null and b/.github/script/assets/asr_eval/en-short-16.mp3 differ diff --git a/.github/script/assets/asr_eval/en-short-17.mp3 b/.github/script/assets/asr_eval/en-short-17.mp3 new file mode 100644 index 000000000..60bd27d18 Binary files /dev/null and b/.github/script/assets/asr_eval/en-short-17.mp3 differ diff --git a/.github/script/assets/asr_eval/en-short-18.mp3 b/.github/script/assets/asr_eval/en-short-18.mp3 new file mode 100644 index 000000000..aba57e91f Binary files /dev/null and b/.github/script/assets/asr_eval/en-short-18.mp3 differ diff --git a/.github/script/assets/asr_eval/en-short-19.mp3 b/.github/script/assets/asr_eval/en-short-19.mp3 new file mode 100644 index 000000000..dccfe80a9 Binary files /dev/null and b/.github/script/assets/asr_eval/en-short-19.mp3 differ diff --git a/.github/script/assets/asr_eval/es-fleurs-00.mp3 b/.github/script/assets/asr_eval/es-fleurs-00.mp3 new file mode 100644 index 000000000..a9167b18f Binary files /dev/null and b/.github/script/assets/asr_eval/es-fleurs-00.mp3 differ diff --git a/.github/script/assets/asr_eval/es-fleurs-01.mp3 b/.github/script/assets/asr_eval/es-fleurs-01.mp3 new file mode 100644 index 000000000..5702ef7a1 Binary files /dev/null and b/.github/script/assets/asr_eval/es-fleurs-01.mp3 differ diff --git a/.github/script/assets/asr_eval/es-fleurs-02.mp3 b/.github/script/assets/asr_eval/es-fleurs-02.mp3 new file mode 100644 index 000000000..70a6c2b75 Binary files /dev/null and b/.github/script/assets/asr_eval/es-fleurs-02.mp3 differ diff --git a/.github/script/assets/asr_eval/es-fleurs-03.mp3 b/.github/script/assets/asr_eval/es-fleurs-03.mp3 new file mode 100644 index 000000000..c499e1361 Binary files /dev/null and b/.github/script/assets/asr_eval/es-fleurs-03.mp3 differ diff --git a/.github/script/assets/asr_eval/es-fleurs-04.mp3 b/.github/script/assets/asr_eval/es-fleurs-04.mp3 new file mode 100644 index 000000000..aabeee211 Binary files /dev/null and b/.github/script/assets/asr_eval/es-fleurs-04.mp3 differ diff --git a/.github/script/assets/asr_eval/es-fleurs-05.mp3 b/.github/script/assets/asr_eval/es-fleurs-05.mp3 new file mode 100644 index 000000000..f98ffe9b0 Binary files /dev/null and b/.github/script/assets/asr_eval/es-fleurs-05.mp3 differ diff --git a/.github/script/assets/asr_eval/es-fleurs-06.mp3 b/.github/script/assets/asr_eval/es-fleurs-06.mp3 new file mode 100644 index 000000000..7168e5d4b Binary files /dev/null and b/.github/script/assets/asr_eval/es-fleurs-06.mp3 differ diff --git a/.github/script/assets/asr_eval/es-fleurs-07.mp3 b/.github/script/assets/asr_eval/es-fleurs-07.mp3 new file mode 100644 index 000000000..8d67320ee Binary files /dev/null and b/.github/script/assets/asr_eval/es-fleurs-07.mp3 differ diff --git a/.github/script/assets/asr_eval/es-fleurs-08.mp3 b/.github/script/assets/asr_eval/es-fleurs-08.mp3 new file mode 100644 index 000000000..f8424d7be Binary files /dev/null and b/.github/script/assets/asr_eval/es-fleurs-08.mp3 differ diff --git a/.github/script/assets/asr_eval/es-fleurs-09.mp3 b/.github/script/assets/asr_eval/es-fleurs-09.mp3 new file mode 100644 index 000000000..15a8bbb34 Binary files /dev/null and b/.github/script/assets/asr_eval/es-fleurs-09.mp3 differ diff --git a/.github/script/assets/asr_eval/expected.json b/.github/script/assets/asr_eval/expected.json new file mode 100644 index 000000000..242037edf --- /dev/null +++ b/.github/script/assets/asr_eval/expected.json @@ -0,0 +1,65 @@ +{ + "_note": "Error rates of main (77bc6bdd, flm 1.0.9) per platform, Whisper-V3-Turbo-NPU2, 2026-10-16. Used only by runs with no baseline build. Update a platform's entry when main's accuracy on it changes on purpose.", + "platforms": { + "linux": { + "measured": "Strix NPU, local run of main's CI-built portable", + "overall": 0.1855, + "sets": { + "en-short": 0.0971, + "en-long": 0.1189, + "en-longform": 0.1653, + "de-fleurs": 0.3814, + "es-fleurs": 0.1542, + "fr-fleurs": 0.2769, + "ja-fleurs": 0.194, + "zh-fleurs": 0.1348, + "ar-fleurs": 0.4171 + } + }, + "linux-hrx": { + "measured": "Strix NPU, local run of main's CI-built portable", + "overall": 0.2263, + "sets": { + "en-short": 0.0971, + "en-long": 0.1189, + "en-longform": 0.3679, + "de-fleurs": 0.386, + "es-fleurs": 0.1542, + "fr-fleurs": 0.2851, + "ja-fleurs": 0.194, + "zh-fleurs": 0.1348, + "ar-fleurs": 0.4652 + } + }, + "windows": { + "measured": "ASR Test run 38853168399, main's portable as the baseline", + "overall": 0.1861, + "sets": { + "en-short": 0.0835, + "en-long": 0.1209, + "en-longform": 0.1167, + "de-fleurs": 0.4326, + "es-fleurs": 0.1542, + "fr-fleurs": 0.3347, + "ja-fleurs": 0.1902, + "zh-fleurs": 0.1509, + "ar-fleurs": 0.4599 + } + }, + "windows-hrx": { + "measured": "ASR Test run 38853168399, main's portable as the baseline", + "overall": 0.1944, + "sets": { + "en-short": 0.0971, + "en-long": 0.1189, + "en-longform": 0.1864, + "de-fleurs": 0.3907, + "es-fleurs": 0.1583, + "fr-fleurs": 0.2975, + "ja-fleurs": 0.1883, + "zh-fleurs": 0.1482, + "ar-fleurs": 0.4545 + } + } + } +} diff --git a/.github/script/assets/asr_eval/fr-fleurs-00.mp3 b/.github/script/assets/asr_eval/fr-fleurs-00.mp3 new file mode 100644 index 000000000..2c5729800 Binary files /dev/null and b/.github/script/assets/asr_eval/fr-fleurs-00.mp3 differ diff --git a/.github/script/assets/asr_eval/fr-fleurs-01.mp3 b/.github/script/assets/asr_eval/fr-fleurs-01.mp3 new file mode 100644 index 000000000..9b52287e7 Binary files /dev/null and b/.github/script/assets/asr_eval/fr-fleurs-01.mp3 differ diff --git a/.github/script/assets/asr_eval/fr-fleurs-02.mp3 b/.github/script/assets/asr_eval/fr-fleurs-02.mp3 new file mode 100644 index 000000000..b7709c593 Binary files /dev/null and b/.github/script/assets/asr_eval/fr-fleurs-02.mp3 differ diff --git a/.github/script/assets/asr_eval/fr-fleurs-03.mp3 b/.github/script/assets/asr_eval/fr-fleurs-03.mp3 new file mode 100644 index 000000000..88076391d Binary files /dev/null and b/.github/script/assets/asr_eval/fr-fleurs-03.mp3 differ diff --git a/.github/script/assets/asr_eval/fr-fleurs-04.mp3 b/.github/script/assets/asr_eval/fr-fleurs-04.mp3 new file mode 100644 index 000000000..920eed389 Binary files /dev/null and b/.github/script/assets/asr_eval/fr-fleurs-04.mp3 differ diff --git a/.github/script/assets/asr_eval/fr-fleurs-05.mp3 b/.github/script/assets/asr_eval/fr-fleurs-05.mp3 new file mode 100644 index 000000000..d4ef40c17 Binary files /dev/null and b/.github/script/assets/asr_eval/fr-fleurs-05.mp3 differ diff --git a/.github/script/assets/asr_eval/fr-fleurs-06.mp3 b/.github/script/assets/asr_eval/fr-fleurs-06.mp3 new file mode 100644 index 000000000..366f27c16 Binary files /dev/null and b/.github/script/assets/asr_eval/fr-fleurs-06.mp3 differ diff --git a/.github/script/assets/asr_eval/fr-fleurs-07.mp3 b/.github/script/assets/asr_eval/fr-fleurs-07.mp3 new file mode 100644 index 000000000..9bcbeaf97 Binary files /dev/null and b/.github/script/assets/asr_eval/fr-fleurs-07.mp3 differ diff --git a/.github/script/assets/asr_eval/fr-fleurs-08.mp3 b/.github/script/assets/asr_eval/fr-fleurs-08.mp3 new file mode 100644 index 000000000..1badcf09d Binary files /dev/null and b/.github/script/assets/asr_eval/fr-fleurs-08.mp3 differ diff --git a/.github/script/assets/asr_eval/fr-fleurs-09.mp3 b/.github/script/assets/asr_eval/fr-fleurs-09.mp3 new file mode 100644 index 000000000..d0d1abdbb Binary files /dev/null and b/.github/script/assets/asr_eval/fr-fleurs-09.mp3 differ diff --git a/.github/script/assets/asr_eval/ja-fleurs-00.mp3 b/.github/script/assets/asr_eval/ja-fleurs-00.mp3 new file mode 100644 index 000000000..8958656ed Binary files /dev/null and b/.github/script/assets/asr_eval/ja-fleurs-00.mp3 differ diff --git a/.github/script/assets/asr_eval/ja-fleurs-01.mp3 b/.github/script/assets/asr_eval/ja-fleurs-01.mp3 new file mode 100644 index 000000000..70cc5de5d Binary files /dev/null and b/.github/script/assets/asr_eval/ja-fleurs-01.mp3 differ diff --git a/.github/script/assets/asr_eval/ja-fleurs-02.mp3 b/.github/script/assets/asr_eval/ja-fleurs-02.mp3 new file mode 100644 index 000000000..e779a54c0 Binary files /dev/null and b/.github/script/assets/asr_eval/ja-fleurs-02.mp3 differ diff --git a/.github/script/assets/asr_eval/ja-fleurs-03.mp3 b/.github/script/assets/asr_eval/ja-fleurs-03.mp3 new file mode 100644 index 000000000..5b346f5bf Binary files /dev/null and b/.github/script/assets/asr_eval/ja-fleurs-03.mp3 differ diff --git a/.github/script/assets/asr_eval/ja-fleurs-04.mp3 b/.github/script/assets/asr_eval/ja-fleurs-04.mp3 new file mode 100644 index 000000000..0cc54aae0 Binary files /dev/null and b/.github/script/assets/asr_eval/ja-fleurs-04.mp3 differ diff --git a/.github/script/assets/asr_eval/ja-fleurs-05.mp3 b/.github/script/assets/asr_eval/ja-fleurs-05.mp3 new file mode 100644 index 000000000..3d3d3d152 Binary files /dev/null and b/.github/script/assets/asr_eval/ja-fleurs-05.mp3 differ diff --git a/.github/script/assets/asr_eval/ja-fleurs-06.mp3 b/.github/script/assets/asr_eval/ja-fleurs-06.mp3 new file mode 100644 index 000000000..c197bcbcb Binary files /dev/null and b/.github/script/assets/asr_eval/ja-fleurs-06.mp3 differ diff --git a/.github/script/assets/asr_eval/ja-fleurs-07.mp3 b/.github/script/assets/asr_eval/ja-fleurs-07.mp3 new file mode 100644 index 000000000..972261266 Binary files /dev/null and b/.github/script/assets/asr_eval/ja-fleurs-07.mp3 differ diff --git a/.github/script/assets/asr_eval/ja-fleurs-08.mp3 b/.github/script/assets/asr_eval/ja-fleurs-08.mp3 new file mode 100644 index 000000000..f54f571aa Binary files /dev/null and b/.github/script/assets/asr_eval/ja-fleurs-08.mp3 differ diff --git a/.github/script/assets/asr_eval/ja-fleurs-09.mp3 b/.github/script/assets/asr_eval/ja-fleurs-09.mp3 new file mode 100644 index 000000000..93aefad05 Binary files /dev/null and b/.github/script/assets/asr_eval/ja-fleurs-09.mp3 differ diff --git a/.github/script/assets/asr_eval/manifest.json b/.github/script/assets/asr_eval/manifest.json new file mode 100644 index 000000000..fbbe5f4cf --- /dev/null +++ b/.github/script/assets/asr_eval/manifest.json @@ -0,0 +1,1057 @@ +{ + "source": "FLEURS test split (google/fleurs), CC BY 4.0", + "parquet_revision": "168de341b3db6859a9bac1c50a2ef5e3b47647e0", + "sample_rate": 16000, + "items": [ + { + "file": "en-short-00.mp3", + "set": "en-short", + "language": "en", + "metric": "wer", + "duration": 10.68, + "fleurs_ids": [ + 1660 + ], + "reference": "romanticism had a large element of cultural determinism drawn from writers such as goethe fichte and schlegel", + "raw_reference": "Romanticism had a large element of cultural determinism, drawn from writers such as Goethe, Fichte, and Schlegel." + }, + { + "file": "en-short-01.mp3", + "set": "en-short", + "language": "en", + "metric": "wer", + "duration": 6.36, + "fleurs_ids": [ + 1661 + ], + "reference": "he did not set a figure for the cuts saying they will be made based on china's economic output", + "raw_reference": "He did not set a figure for the cuts, saying they will be made based on China's economic output." + }, + { + "file": "en-short-02.mp3", + "set": "en-short", + "language": "en", + "metric": "wer", + "duration": 8.38, + "fleurs_ids": [ + 1662 + ], + "reference": "alloys are basically a mixture of two or more metals don't forget that there are many elements on the periodic table", + "raw_reference": "Alloys are basically a mixture of two or more metals. Don't forget that there are many elements on the periodic table." + }, + { + "file": "en-short-03.mp3", + "set": "en-short", + "language": "en", + "metric": "wer", + "duration": 12.12, + "fleurs_ids": [ + 1663 + ], + "reference": "cochamó valley - chile's premier climbing destination known as the yosemite of south america with a variety of granite big walls and crags", + "raw_reference": "Cochamó Valley - Chile's premier climbing destination, known as the Yosemite of South America, with a variety of granite big walls and crags." + }, + { + "file": "en-short-04.mp3", + "set": "en-short", + "language": "en", + "metric": "wer", + "duration": 8.36, + "fleurs_ids": [ + 1664 + ], + "reference": "swirl the two dry powders together and then with clean wet hands squeeze them into a ball", + "raw_reference": "Swirl the two dry powders together and then, with clean wet hands, squeeze them into a ball." + }, + { + "file": "en-short-05.mp3", + "set": "en-short", + "language": "en", + "metric": "wer", + "duration": 8.28, + "fleurs_ids": [ + 1665 + ], + "reference": "the document according to the leak will refer to the borders dispute which palestine wants based on the borders before the 1967 mideast war", + "raw_reference": "The document, according to the leak, will refer to the borders dispute, which Palestine wants based on the borders before the 1967 Mideast War." + }, + { + "file": "en-short-06.mp3", + "set": "en-short", + "language": "en", + "metric": "wer", + "duration": 8.34, + "fleurs_ids": [ + 1666 + ], + "reference": "you may also wish to consult the advice of governments other than your own but their advice is designed for their citizens", + "raw_reference": "You may also wish to consult the advice of governments other than your own, but their advice is designed for their citizens." + }, + { + "file": "en-short-07.mp3", + "set": "en-short", + "language": "en", + "metric": "wer", + "duration": 14.22, + "fleurs_ids": [ + 1667 + ], + "reference": "generally speaking two behaviors can emerge as managers begin to lead their former peers. one end of the spectrum is trying to remain one of the guys” or gals", + "raw_reference": "Generally speaking, two behaviors can emerge as managers begin to lead their former peers. One end of the spectrum is trying to remain “one of the guys” (or gals)." + }, + { + "file": "en-short-08.mp3", + "set": "en-short", + "language": "en", + "metric": "wer", + "duration": 13.32, + "fleurs_ids": [ + 1668 + ], + "reference": "it may also be beneficial for one to buy a wild card which provides entry to either selections of parks in south africa or all of the south african national parks", + "raw_reference": "It may also be beneficial for one to buy a Wild Card, which provides entry to either selections of parks in South Africa or all of the South African National Parks." + }, + { + "file": "en-short-09.mp3", + "set": "en-short", + "language": "en", + "metric": "wer", + "duration": 7.2, + "fleurs_ids": [ + 1669 + ], + "reference": "police superintendent chandra shekhar solanki said the accused appeared in court with covered faces", + "raw_reference": "Police superintendent Chandra Shekhar Solanki said the accused appeared in court with covered faces." + }, + { + "file": "en-short-10.mp3", + "set": "en-short", + "language": "en", + "metric": "wer", + "duration": 19.8, + "fleurs_ids": [ + 1670 + ], + "reference": "their thermal behavior is not as steady as large caves on earth that often maintain a fairly constant temperature but it is consistent with these being deep holes in the ground, said glen cushing of the united states geological survey usgs astrogeology team and of northern arizona university located in flagstaff arizona", + "raw_reference": "Their thermal behavior is not as steady as large caves on Earth that often maintain a fairly constant temperature, but it is consistent with these being deep holes in the ground,\" said Glen Cushing of the United States Geological Survey (USGS) Astrogeology Team and of Northern Arizona University located in Flagstaff, Arizona." + }, + { + "file": "en-short-11.mp3", + "set": "en-short", + "language": "en", + "metric": "wer", + "duration": 10.02, + "fleurs_ids": [ + 1671 + ], + "reference": "congress began funding the obscenity initiative in fiscal 2005 and specified that the fbi must devote 10 agents to adult pornography", + "raw_reference": "Congress began funding the obscenity initiative in fiscal 2005 and specified that the FBI must devote 10 agents to adult pornography." + }, + { + "file": "en-short-12.mp3", + "set": "en-short", + "language": "en", + "metric": "wer", + "duration": 10.16, + "fleurs_ids": [ + 1672 + ], + "reference": "be careful not to allow fabric to become too hot which can cause shrinkage or in extreme cases scorch", + "raw_reference": "Be careful not to allow fabric to become too hot (which can cause shrinkage, or in extreme cases, scorch)." + }, + { + "file": "en-short-13.mp3", + "set": "en-short", + "language": "en", + "metric": "wer", + "duration": 10.08, + "fleurs_ids": [ + 1673 + ], + "reference": "during the revolutionary war the thirteen states first formed a weak central government-with the congress being its only component-under the articles of confederation", + "raw_reference": "During the Revolutionary War, the thirteen states first formed a weak central government—with the Congress being its only component—under the Articles of Confederation." + }, + { + "file": "en-short-14.mp3", + "set": "en-short", + "language": "en", + "metric": "wer", + "duration": 5.64, + "fleurs_ids": [ + 1674 + ], + "reference": "in some areas boiling water for a minute is enough in others several minutes are needed", + "raw_reference": "In some areas boiling water for a minute is enough, in others several minutes are needed." + }, + { + "file": "en-short-15.mp3", + "set": "en-short", + "language": "en", + "metric": "wer", + "duration": 8.76, + "fleurs_ids": [ + 1675 + ], + "reference": "all nouns alongside the word sie for you always begin with a capital letter even in the middle of a sentence", + "raw_reference": "All nouns, alongside the word Sie for you, always begin with a capital letter, even in the middle of a sentence." + }, + { + "file": "en-short-16.mp3", + "set": "en-short", + "language": "en", + "metric": "wer", + "duration": 9.42, + "fleurs_ids": [ + 1676 + ], + "reference": "as with all south african national parks there are daily conservation and entry fees for the park", + "raw_reference": "As with all South African National Parks, there are daily conservation and entry fees for the park." + }, + { + "file": "en-short-17.mp3", + "set": "en-short", + "language": "en", + "metric": "wer", + "duration": 5.94, + "fleurs_ids": [ + 1677 + ], + "reference": "today the only insects that cannot fold back their wings are dragon flies and mayflies", + "raw_reference": "Today, the only insects that cannot fold back their wings are dragon flies and mayflies." + }, + { + "file": "en-short-18.mp3", + "set": "en-short", + "language": "en", + "metric": "wer", + "duration": 12.78, + "fleurs_ids": [ + 1678 + ], + "reference": "oliver sacks in his paper the president's speech indicated how people who are unable to understand speech because of brain damage are nevertheless able to assess sincerity accurately", + "raw_reference": "Oliver Sacks, in his paper The President's Speech, indicated how people who are unable to understand speech because of brain damage are nevertheless able to assess sincerity accurately." + }, + { + "file": "en-short-19.mp3", + "set": "en-short", + "language": "en", + "metric": "wer", + "duration": 4.38, + "fleurs_ids": [ + 1679 + ], + "reference": "aerosmith have cancelled their remaining concerts on their tour", + "raw_reference": "Aerosmith have cancelled their remaining concerts on their tour." + }, + { + "file": "en-long-0.mp3", + "set": "en-long", + "language": "en", + "metric": "wer", + "duration": 84.8, + "fleurs_ids": [ + 1680, + 1681, + 1682, + 1683, + 1684, + 1685 + ], + "reference": "a well rounded athlete the tiger can climb though not well swim leap great distances and pull with five times the force of a strong human it isn't alone though experimentation and an experiment is a test that is used to eliminate one or more of the possible hypotheses asking questions and making observations also guide scientific research while no one knows for certain who wrote it it is known that early in its life the large parchment document it measures 29¾ inches by 24½ inches was rolled up for storage there are still many men and women alive who survived their time here and many more who had loved ones who were murdered or worked to death there jews and non-jews alike apia is the capital of samoa the town is on the island of upolu and has a population of just under 40,000 for example the most common still image photography format in the world is 35mm which was the dominant film size at the close of the analog film era", + "raw_reference": "A well rounded athlete, the tiger can climb (though not well), swim, leap great distances and pull with five times the force of a strong human. It isn’t alone though, experimentation, and an experiment is a test that is used to eliminate one or more of the possible hypotheses, asking questions, and making observations also guide scientific research. While no one knows for certain who wrote it, it is known that early in its life, the large parchment document (it measures 29¾ inches by 24½ inches) was rolled up for storage. There are still many men and women alive who survived their time here, and many more who had loved ones who were murdered or worked to death there, Jews and non-Jews alike. Apia is the capital of Samoa. The town is on the island of Upolu and has a population of just under 40,000. For example, the most common still image photography format in the world is 35mm, which was the dominant film size at the close of the analog film era." + }, + { + "file": "en-long-1.mp3", + "set": "en-long", + "language": "en", + "metric": "wer", + "duration": 83.9, + "fleurs_ids": [ + 1686, + 1687, + 1688, + 1689, + 1690 + ], + "reference": "as a result the process of an organization working together to overcome an obstacle can lead to a new innovative process to serve the customer's need segregation and recombination shuffle variation back and forth between the two pools with each generation the job of navies has traditionally been to ensure that your country maintains the ability to move your people and goods while at the same time interfering with your enemy's ability to move his people and goods europe is a continent that is relatively small but with many independent countries under normal circumstances travelling through multiple countries would mean having to go through visa applications and passport control multiple times we agree with the usoc's statement that the interests of our athletes and clubs and their sport may be better served by moving forward with meaningful change within our organization rather than decertification", + "raw_reference": "As a result, the process of an organization working together to overcome an obstacle can lead to a new innovative process to serve the customer's need. Segregation and recombination shuffle variation back and forth between the two pools with each generation. The job of navies has traditionally been to ensure that your country maintains the ability to move your people and goods, while at the same time, interfering with your enemy's ability to move his people and goods. Europe is a continent that is relatively small but with many independent countries. Under normal circumstances, travelling through multiple countries would mean having to go through visa applications and passport control multiple times. We agree with the USOC's statement that the interests of our athletes and clubs, and their sport, may be better served by moving forward with meaningful change within our organization, rather than decertification." + }, + { + "file": "en-long-2.mp3", + "set": "en-long", + "language": "en", + "metric": "wer", + "duration": 82.84, + "fleurs_ids": [ + 1691, + 1692, + 1693, + 1694, + 1695, + 1696, + 1697, + 1698, + 1699 + ], + "reference": "on some routes the larger companies have their own planes but for other routes and smaller firms there was a problem according to japan's nuclear agency radioactive caesium and iodine has been identified at the plant but after losing the captain's wicket india only made 36 runs loosing 7 wickets to end the innings children are placed in foster care for a wide variety of reasons that range from neglect to abuse and even to extortion a simple popular dinner especially during the summer is the pa amb oli bread with olive oil tomato and any available condiments such as cheese tunafish etc the two compounds react with one another to form crystals that may block kidney function researchers at the university said he has been unable to take the drugs needed to overcome his pain as they are banned from the games everything in the universe is made of matter all matter is made of tiny particles called atoms bishkek was described as sinking into a state of anarchy by one observer as gangs of people roamed the streets and plundered stores of consumer goods", + "raw_reference": "On some routes, the larger companies have their own planes, but for other routes and smaller firms there was a problem. According to Japan's nuclear agency, radioactive caesium and iodine has been identified at the plant. But, after losing the captain's wicket India only made 36 runs loosing 7 wickets to end the innings. Children are placed in Foster Care for a wide variety of reasons that range from neglect, to abuse, and even to extortion. A simple popular dinner, especially during the summer, is the Pa amb Oli: Bread with olive oil, tomato, and any available condiments such as cheese, tunafish, etc. The two compounds react with one another to form crystals that may block kidney function, researchers at the university said. He has been unable to take the drugs needed to overcome his pain as they are banned from the Games. Everything in the Universe is made of matter. All matter is made of tiny particles called atoms. Bishkek was described as sinking into a state of \"anarchy\" by one observer, as gangs of people roamed the streets and plundered stores of consumer goods." + }, + { + "file": "en-longform-0.mp3", + "set": "en-longform", + "language": "en", + "metric": "wer", + "duration": 307.8, + "fleurs_ids": [ + 1700, + 1701, + 1702, + 1703, + 1704, + 1705, + 1706, + 1707, + 1708, + 1709, + 1710, + 1711, + 1712, + 1713, + 1714, + 1715, + 1716, + 1717, + 1718, + 1719, + 1720, + 1721, + 1722, + 1723, + 1724, + 1725 + ], + "reference": "it is thinner under the maria and thicker under the highlands goats seem to have been first domesticated roughly 10,000 years ago in the zagros mountains of iran in just two weeks the americans and free french forces had liberated southern france and were turning towards germany martelly swore in a new provisional electoral council cep of nine members yesterday in 1990 it was added to the list of world heritage sites in danger due to the threat of desert sands on 15 august 1940 the allies invaded southern france the invasion was called operation dragoon more traditional churches often hold an easter vigil on saturday night during the easter weekend with the congregations often breaking into celebration at the stroke of midnight to celebrate christ's resurrection it is one of the main attractions of south africa and it is considered the flagship of south african national parks sanparks huhne has resigned and he will be replaced in the cabinet by ed davey mp norman lamb mp is expected to take the business minister job davey is vacating the report is highly critical of almost every aspect of the present policy of the executive towards iraq and it urges an immediate change of direction last week meti announced that apple had informed it of 34 additional overheating incidents which the company called non-serious these theories look at what it is about certain people that make them want the things that they do and what things in their environment will make them do or not do certain things since 1966 the sundarbans have been a wildlife sanctuary and it is estimated that there are now 400 royal bengal tigers and about 30,000 spotted deer in the area while goma is reasonably safe any visits outside of goma should be researched to understand the state of the fighting that persists in the north kivu province the satellite was sent into space by a rocket scientists use telescopes in space because the earth's atmosphere distorts some of our light and view regular announcements in the metro are made only in catalan but unplanned disruptions are announced by an automated system in a wide variety of languages including spanish english french arabic and japanese entertainment news website tmz understands the photographer stopped his vehicle on the other side of sepulveda boulevard and attempted to take pictures of the police stop before crossing the road and continuing prompting the california highway patrol police officer conducting the traffic stop to order him back across twice due to the underwater topology the return flow is concentrated at a few deeper sections and a fast current to deep water may form there there is no universal definition for which manufactured items are antiques some tax agencies define goods older than 100 years as antiques severe weather is the generic term for any dangerous weather phenomenon with the potential to cause damage serious social disruption or loss of human life the hot chocolate is up to belgian standards fruit juices are pricey but excellent after the dam was built in 1963 the seasonal floods that would spread sediment throughout the river were halted cuomo 53 began his governorship earlier this year and signed a bill last month legalizing same-sex marriage the harbor was the site of an infamous naval standoff in 1889 when seven ships from germany the us and britain refused to leave the harbor its renown for being an epicenter of luxury began in about 400 a.d. and lasted up until about 1100 a.d asus eee pc earlier launched world-wide for cost-saving and functionality factors became a hot topic in 2007 taipei it month", + "raw_reference": "It is thinner under the maria and thicker under the highlands. Goats seem to have been first domesticated roughly 10,000 years ago in the Zagros Mountains of Iran. In just two weeks the Americans and Free French forces had liberated southern France and were turning towards Germany. Martelly swore in a new Provisional Electoral Council (CEP) of nine members yesterday. In 1990, it was added to the list of world heritage sites in danger, due to the threat of desert sands. On 15 August 1940, the Allies invaded southern France, the invasion was called \"Operation Dragoon\". More traditional churches often hold an Easter Vigil on Saturday night during the Easter weekend, with the congregations often breaking into celebration at the stroke of midnight to celebrate Christ's resurrection. It is one of the main attractions of South Africa and it is considered the flagship of South African National Parks (SANParks). Huhne has resigned and he will be replaced in the Cabinet by Ed Davey MP. Norman Lamb MP is expected to take the Business Minister job Davey is vacating. The Report is highly critical of almost every aspect of the present policy of the Executive towards Iraq and it urges an immediate change of direction. Last week, METI announced that Apple had informed it of 34 additional overheating incidents, which the company called \"non-serious.\" These theories look at what it is about certain people that make them want the things that they do and what things in their environment will make them do or not do certain things. Since 1966 the Sundarbans have been a wildlife sanctuary, and it is estimated that there are now 400 Royal Bengal tigers and about 30,000 spotted deer in the area. While Goma is reasonably safe, any visits outside of Goma should be researched to understand the state of the fighting that persists in the North Kivu province. The satellite was sent into space by a rocket. Scientists use telescopes in space because the Earth’s atmosphere distorts some of our light and view. Regular announcements in the Metro are made only in Catalan, but unplanned disruptions are announced by an automated system in a wide variety of languages including Spanish, English, French, Arabic and Japanese. Entertainment news website TMZ understands the photographer stopped his vehicle on the other side of Sepulveda Boulevard and attempted to take pictures of the police stop before crossing the road and continuing, prompting the California Highway Patrol police officer conducting the traffic stop to order him back across, twice. Due to the underwater topology the return flow is concentrated at a few deeper sections, and a fast current to deep water may form there. There is no universal definition for which manufactured items are antiques. Some tax agencies define goods older than 100 years as antiques. Severe weather is the generic term for any dangerous weather phenomenon with the potential to cause damage, serious social disruption, or loss of human life. The hot chocolate is up to Belgian standards. Fruit juices are pricey but excellent. After the dam was built in 1963, the seasonal floods that would spread sediment throughout the river were halted. Cuomo, 53, began his governorship earlier this year and signed a bill last month legalizing same-sex marriage. The harbor was the site of an infamous naval standoff in 1889 when seven ships from Germany, the US, and Britain refused to leave the harbor. Its renown for being an epicenter of luxury began in about 400 A.D. and lasted up until about 1100 A.D. ASUS Eee PC, earlier launched world-wide for cost-saving and functionality factors, became a hot topic in 2007 Taipei IT Month." + }, + { + "file": "de-fleurs-00.mp3", + "set": "de-fleurs", + "language": "de", + "metric": "wer", + "duration": 10.38, + "fleurs_ids": [ + 1660 + ], + "reference": "die romantik hatte ein starkes kulturell deterministisches element dass sich aus schriftstellern wie goethe fichte und schlegel schöpfte", + "raw_reference": "Die Romantik hatte ein starkes, kulturell deterministisches Element, das sich aus Schriftstellern wie Goethe, Fichte und Schlegel schöpfte." + }, + { + "file": "de-fleurs-01.mp3", + "set": "de-fleurs", + "language": "de", + "metric": "wer", + "duration": 6.66, + "fleurs_ids": [ + 1661 + ], + "reference": "er nannte keine zahl für die kürzungen sondern sagte sie würden auf der grundlage der wirtschaftsleistung chinas vorgenommen", + "raw_reference": "Er nannte keine Zahl für die Kürzungen, sondern sagte, sie würden auf der Grundlage der Wirtschaftsleistung Chinas vorgenommen." + }, + { + "file": "de-fleurs-02.mp3", + "set": "de-fleurs", + "language": "de", + "metric": "wer", + "duration": 23.46, + "fleurs_ids": [ + 1662 + ], + "reference": "legierungen sind im grunde eine mischung aus zwei oder mehr metallen vergessen sie nicht dass es viele elemente im periodensystem gibt", + "raw_reference": "Legierungen sind im Grunde eine Mischung aus zwei oder mehr Metallen. Vergessen Sie nicht, dass es viele Elemente im Periodensystem gibt." + }, + { + "file": "de-fleurs-03.mp3", + "set": "de-fleurs", + "language": "de", + "metric": "wer", + "duration": 9.78, + "fleurs_ids": [ + 1663 + ], + "reference": "das cochamó-tal – chiles wichtigstes kletterziel bekannt als der yosemite-nationalpark von südamerika mit einer vielzahl von großen granitwänden und -klippen", + "raw_reference": "Das Cochamó-Tal – Chiles wichtigstes Kletterziel, bekannt als der Yosemite-Nationalpark von Südamerika, mit einer Vielzahl von großen Granitwänden und -klippen." + }, + { + "file": "de-fleurs-04.mp3", + "set": "de-fleurs", + "language": "de", + "metric": "wer", + "duration": 8.7, + "fleurs_ids": [ + 1664 + ], + "reference": "mischen sie die zwei trockenpulver zusammen und formen sie sie dann mit sauberen nassen händen zu einem ball", + "raw_reference": "Mischen Sie die zwei Trockenpulver zusammen und formen Sie sie dann mit sauberen, nassen Händen zu einem Ball." + }, + { + "file": "de-fleurs-05.mp3", + "set": "de-fleurs", + "language": "de", + "metric": "wer", + "duration": 10.68, + "fleurs_ids": [ + 1665 + ], + "reference": "dem leak zufolge bezieht sich das dokument auf den grenzstreit in dem die palästinenser ein zurücksetzen der grenzen in den zustand vor dem sechstagekrieg von 1967 fordern", + "raw_reference": "Dem Leak zufolge bezieht sich das Dokument auf den Grenzstreit, in dem die Palästinenser ein Zurücksetzen der Grenzen in den Zustand vor dem Sechstagekrieg von 1967 fordern." + }, + { + "file": "de-fleurs-06.mp3", + "set": "de-fleurs", + "language": "de", + "metric": "wer", + "duration": 17.76, + "fleurs_ids": [ + 1666 + ], + "reference": "sie können auch die ratschläge anderer regierungen als der ihren konsultieren deren hinweise sind jedoch auf ihre eigenen staatsbürger zugeschnitten", + "raw_reference": "Sie können auch die Ratschläge anderer Regierungen als der Ihren konsultieren. Deren Hinweise sind jedoch auf ihre eigenen Staatsbürger zugeschnitten." + }, + { + "file": "de-fleurs-07.mp3", + "set": "de-fleurs", + "language": "de", + "metric": "wer", + "duration": 24.06, + "fleurs_ids": [ + 1667 + ], + "reference": "im allgemeinen können sich zwei verhaltensweisen abzeichnen wenn manager beginnen ihre ehemaligen kollegen zu leiten. das eine ende des spektrums ist der versuch einer der jungs oder eines der mädels zu bleiben", + "raw_reference": "Im Allgemeinen können sich zwei Verhaltensweisen abzeichnen, wenn Manager beginnen, ihre ehemaligen Kollegen zu leiten. Das eine Ende des Spektrums ist der Versuch, „einer der Jungs“ (oder eines der Mädels) zu bleiben." + }, + { + "file": "de-fleurs-08.mp3", + "set": "de-fleurs", + "language": "de", + "metric": "wer", + "duration": 14.7, + "fleurs_ids": [ + 1668 + ], + "reference": "es kann sich auch lohnen eine wild card zu kaufen die zutritt entweder zu ausgewählten parks in südafrika oder zu allen südafrikanischen nationalparks gewährt", + "raw_reference": "Es kann sich auch lohnen, eine Wild Card zu kaufen, die Zutritt entweder zu ausgewählten Parks in Südafrika oder zu allen südafrikanischen Nationalparks gewährt." + }, + { + "file": "de-fleurs-09.mp3", + "set": "de-fleurs", + "language": "de", + "metric": "wer", + "duration": 12.78, + "fleurs_ids": [ + 1669 + ], + "reference": "die polizeipräsidentin chandra shekhar solanki gab an die angeklagten seien mit bedeckten gesichtern vor gericht erschienen", + "raw_reference": "Die Polizeipräsidentin Chandra Shekhar Solanki gab an, die Angeklagten seien mit bedeckten Gesichtern vor Gericht erschienen." + }, + { + "file": "es-fleurs-00.mp3", + "set": "es-fleurs", + "language": "es", + "metric": "wer", + "duration": 10.86, + "fleurs_ids": [ + 1660 + ], + "reference": "el elemento del determinismo cultural se encontraba muy presente en el romanticismo según estudiosos como goether fichte y schlegel", + "raw_reference": "El elemento del determinismo cultural se encontraba muy presente en el romanticismo, según estudiosos como Goether, Fichte y Schlegel." + }, + { + "file": "es-fleurs-01.mp3", + "set": "es-fleurs", + "language": "es", + "metric": "wer", + "duration": 7.8, + "fleurs_ids": [ + 1661 + ], + "reference": "no mencionó una cifra para los recortes y manifestó que se harán en base a la producción económica de china", + "raw_reference": "No mencionó una cifra para los recortes y manifestó que se harán en base a la producción económica de China." + }, + { + "file": "es-fleurs-02.mp3", + "set": "es-fleurs", + "language": "es", + "metric": "wer", + "duration": 8.82, + "fleurs_ids": [ + 1662 + ], + "reference": "las aleaciones son básicamente una combinación de dos o más metales recuerde que la tabla periódica tiene muchos elementos", + "raw_reference": "Las aleaciones son, básicamente, una combinación de dos o más metales. Recuerde que la tabla periódica tiene muchos elementos." + }, + { + "file": "es-fleurs-03.mp3", + "set": "es-fleurs", + "language": "es", + "metric": "wer", + "duration": 15.42, + "fleurs_ids": [ + 1663 + ], + "reference": "el valle de cochamó es el destino para escalar más popular de chile y es conocido como el yosemite de sudamérica con variedad de extensos muros y riscos de granito", + "raw_reference": "El Valle de Cochamó es el destino para escalar más popular de Chile y es conocido como el Yosemite de Sudamérica, con variedad de extensos muros y riscos de granito." + }, + { + "file": "es-fleurs-04.mp3", + "set": "es-fleurs", + "language": "es", + "metric": "wer", + "duration": 10.02, + "fleurs_ids": [ + 1664 + ], + "reference": "agite ambos polvos secos juntos y después de hacerlo con las manos higienizadas y húmedas exprímalos para formar una bola", + "raw_reference": "Agite ambos polvos secos juntos y, después de hacerlo, con las manos higienizadas y húmedas, exprímalos para formar una bola." + }, + { + "file": "es-fleurs-05.mp3", + "set": "es-fleurs", + "language": "es", + "metric": "wer", + "duration": 17.1, + "fleurs_ids": [ + 1665 + ], + "reference": "según la información que se filtró el documento tratará sobre el conflicto fronterizo que palestina pretende resolver basándose en los límites vigentes en forma previa a la guerra del medio oriente de 1967", + "raw_reference": "Según la información que se filtró, el documento tratará sobre el conflicto fronterizo, que Palestina pretende resolver basándose en los límites vigentes en forma previa a la Guerra del Medio Oriente de 1967." + }, + { + "file": "es-fleurs-06.mp3", + "set": "es-fleurs", + "language": "es", + "metric": "wer", + "duration": 10.14, + "fleurs_ids": [ + 1666 + ], + "reference": "es posible que quiera consultar las recomendaciones de otros gobiernos pero están pensadas para sus propios ciudadanos", + "raw_reference": "Es posible que quiera consultar las recomendaciones de otros gobiernos, pero están pensadas para sus propios ciudadanos." + }, + { + "file": "es-fleurs-07.mp3", + "set": "es-fleurs", + "language": "es", + "metric": "wer", + "duration": 18.3, + "fleurs_ids": [ + 1667 + ], + "reference": "en general pueden darse dos tipos de comportamiento cuando los gerentes comienzan a liderar a sus antiguos compañeros. de un lado del espectro está el intento de continuar siendo «uno de los muchachos» o muchachas", + "raw_reference": "En general, pueden darse dos tipos de comportamiento cuando los gerentes comienzan a liderar a sus antiguos compañeros. De un lado del espectro, está el intento de continuar siendo «uno de los muchachos» (o muchachas)." + }, + { + "file": "es-fleurs-08.mp3", + "set": "es-fleurs", + "language": "es", + "metric": "wer", + "duration": 12.6, + "fleurs_ids": [ + 1668 + ], + "reference": "también puede ser conveniente comprar una wild card que permite la entrada a selecciones de parques en sudáfrica o a todos los parques nacionales del país", + "raw_reference": "También puede ser conveniente comprar una Wild Card que permite la entrada a selecciones de parques en Sudáfrica o a todos los Parques Nacionales del país." + }, + { + "file": "es-fleurs-09.mp3", + "set": "es-fleurs", + "language": "es", + "metric": "wer", + "duration": 11.22, + "fleurs_ids": [ + 1669 + ], + "reference": "según manifestara el superintendente de la policía chandra shekhar solanki los acusados se presentaron ante el tribunal con sus rostros cubiertos", + "raw_reference": "Según manifestara el superintendente de la policía, Chandra Shekhar Solanki, los acusados se presentaron ante el tribunal con sus rostros cubiertos." + }, + { + "file": "fr-fleurs-00.mp3", + "set": "fr-fleurs", + "language": "fr", + "metric": "wer", + "duration": 7.86, + "fleurs_ids": [ + 1660 + ], + "reference": "le romantisme présentait un important élément de déterminisme culturel tiré d'écrivains tels que goethe fichte et schlegel", + "raw_reference": "Le romantisme présentait un important élément de déterminisme culturel, tiré d'écrivains tels que Goethe, Fichte et Schlegel." + }, + { + "file": "fr-fleurs-01.mp3", + "set": "fr-fleurs", + "language": "fr", + "metric": "wer", + "duration": 10.56, + "fleurs_ids": [ + 1661 + ], + "reference": "il n'a pas chiffré ces réductions disant qu'elles seront basées sur la production économique chinoise", + "raw_reference": "Il n'a pas chiffré ces réductions, disant qu'elles seront basées sur la production économique chinoise." + }, + { + "file": "fr-fleurs-02.mp3", + "set": "fr-fleurs", + "language": "fr", + "metric": "wer", + "duration": 7.14, + "fleurs_ids": [ + 1662 + ], + "reference": "les alliages sont essentiellement un mélange de deux métaux ou plus n'oubliez pas que le tableau périodique comporte de nombreux éléments", + "raw_reference": "Les alliages sont essentiellement un mélange de deux métaux ou plus. N'oubliez pas que le tableau périodique comporte de nombreux éléments." + }, + { + "file": "fr-fleurs-03.mp3", + "set": "fr-fleurs", + "language": "fr", + "metric": "wer", + "duration": 11.82, + "fleurs_ids": [ + 1663 + ], + "reference": "vallée de cochamó – la destination d'excellence pour l'escalade au chili connue comme le yosemite d'amérique du sud avec de nombreuses grandes parois et falaises granitiques différentes", + "raw_reference": "Vallée de Cochamó – La destination d'excellence pour l'escalade au Chili, connue comme le Yosemite d'Amérique du Sud, avec de nombreuses grandes parois et falaises granitiques différentes." + }, + { + "file": "fr-fleurs-04.mp3", + "set": "fr-fleurs", + "language": "fr", + "metric": "wer", + "duration": 7.8, + "fleurs_ids": [ + 1664 + ], + "reference": "mélangez les deux poudres sèches ensemble puis avec les mains propres et humides pressez-les en une boule", + "raw_reference": "Mélangez les deux poudres sèches ensemble, puis, avec les mains propres et humides, pressez-les en une boule." + }, + { + "file": "fr-fleurs-05.mp3", + "set": "fr-fleurs", + "language": "fr", + "metric": "wer", + "duration": 16.38, + "fleurs_ids": [ + 1665 + ], + "reference": "d'après la fuite le document portera sur le différend à la frontière que la palestine revendique sur la base des frontières existantes avant la guerre du moyen-orient de 1967", + "raw_reference": "D'après la fuite, le document portera sur le différend à la frontière que la Palestine revendique sur la base des frontières existantes avant la guerre du Moyen-Orient de 1967." + }, + { + "file": "fr-fleurs-06.mp3", + "set": "fr-fleurs", + "language": "fr", + "metric": "wer", + "duration": 11.64, + "fleurs_ids": [ + 1666 + ], + "reference": "vous pouvez également avoir recours aux conseils d'autres gouvernements que le vôtre mais leurs conseils sont adaptés à leurs citoyens", + "raw_reference": "Vous pouvez également avoir recours aux conseils d'autres gouvernements que le vôtre, mais leurs conseils sont adaptés à leurs citoyens." + }, + { + "file": "fr-fleurs-07.mp3", + "set": "fr-fleurs", + "language": "fr", + "metric": "wer", + "duration": 14.88, + "fleurs_ids": [ + 1667 + ], + "reference": "de manière générale deux comportements peuvent émerger lorsqu'un manager devient le chef d'anciens collègues. d'une des extrémités du spectre étant d'essayer de rester « le bon copain » ou la bonne copine", + "raw_reference": "De manière générale, deux comportements peuvent émerger lorsqu'un manager devient le chef d'anciens collègues. D'une des extrémités du spectre étant d'essayer de rester « le bon copain » (ou la bonne copine)." + }, + { + "file": "fr-fleurs-08.mp3", + "set": "fr-fleurs", + "language": "fr", + "metric": "wer", + "duration": 7.62, + "fleurs_ids": [ + 1668 + ], + "reference": "il peut être avantageux de se procurer une wild card qui donne accès à certains parcs d'afrique du sud ainsi qu'à tous les parcs nationaux sud-africains", + "raw_reference": "Il peut être avantageux de se procurer une Wild Card qui donne accès à certains parcs d'Afrique du Sud, ainsi qu’à tous les parcs nationaux sud-africains." + }, + { + "file": "fr-fleurs-09.mp3", + "set": "fr-fleurs", + "language": "fr", + "metric": "wer", + "duration": 9.6, + "fleurs_ids": [ + 1669 + ], + "reference": "chandra shekhar solanki commissaire de police a déclaré que les personnes accusées avaient le visage dissimulé lorsqu'elles ont comparu devant le tribunal", + "raw_reference": "Chandra Shekhar Solanki, commissaire de police, a déclaré que les personnes accusées avaient le visage dissimulé lorsqu'elles ont comparu devant le tribunal." + }, + { + "file": "ja-fleurs-00.mp3", + "set": "ja-fleurs", + "language": "ja", + "metric": "cer", + "duration": 15.42, + "fleurs_ids": [ + 1661 + ], + "reference": "胡氏は 削減する数値を定めませんでしたが 削減は中国の経済産出量に基づいて実施されるだろうと述べました", + "raw_reference": "胡氏は、削減する数値を定めませんでしたが、削減は中国の経済産出量に基づいて実施されるだろうと述べました。" + }, + { + "file": "ja-fleurs-01.mp3", + "set": "ja-fleurs", + "language": "ja", + "metric": "cer", + "duration": 14.58, + "fleurs_ids": [ + 1662 + ], + "reference": "合金とは 基本的には2種類以上の金属の混合物です 周期表に載っているたくさんの元素を思い出してください", + "raw_reference": "合金とは、基本的には2種類以上の金属の混合物です。周期表に載っているたくさんの元素を思い出してください。" + }, + { + "file": "ja-fleurs-02.mp3", + "set": "ja-fleurs", + "language": "ja", + "metric": "cer", + "duration": 9.9, + "fleurs_ids": [ + 1664 + ], + "reference": "乾いた粉を2つ合わせてくるくるとこね回し 清潔な濡れた手でぎゅっと握ります", + "raw_reference": "乾いた粉を2つ合わせてくるくるとこね回し、清潔な濡れた手でぎゅっと握ります。" + }, + { + "file": "ja-fleurs-03.mp3", + "set": "ja-fleurs", + "language": "ja", + "metric": "cer", + "duration": 16.74, + "fleurs_ids": [ + 1665 + ], + "reference": "漏れ聞いた情報によれば 当該文書は国境紛争に言及するもので パレスチナの希望は1967年の中東戦争前の国境に基づくものです", + "raw_reference": "漏れ聞いた情報によれば、当該文書は国境紛争に言及するもので、パレスチナの希望は1967年の中東戦争前の国境に基づくものです。" + }, + { + "file": "ja-fleurs-04.mp3", + "set": "ja-fleurs", + "language": "ja", + "metric": "cer", + "duration": 13.56, + "fleurs_ids": [ + 1666 + ], + "reference": "自国以外の政府からの助言を参考にしてもかいまいませんが そのような助言はその国の国民向けのものです", + "raw_reference": "自国以外の政府からの助言を参考にしてもかいまいませんが、そのような助言はその国の国民向けのものです。" + }, + { + "file": "ja-fleurs-05.mp3", + "set": "ja-fleurs", + "language": "ja", + "metric": "cer", + "duration": 17.82, + "fleurs_ids": [ + 1667 + ], + "reference": "一般的に言えば 管理職がかつての仲間を率いることになった場合 2とおりの行動が現れる可能性があります 1つは 仲間の1人 のままでいようとする行動です", + "raw_reference": "一般的に言えば、管理職がかつての仲間を率いることになった場合、2とおりの行動が現れる可能性があります。1つは「仲間の1人」のままでいようとする行動です。" + }, + { + "file": "ja-fleurs-06.mp3", + "set": "ja-fleurs", + "language": "ja", + "metric": "cer", + "duration": 13.38, + "fleurs_ids": [ + 1668 + ], + "reference": "ワイルドカードを購入するとお得な場合があります 南アフリカの主な公園または南アフリカのすべての国立公園に入場できます", + "raw_reference": "ワイルドカードを購入するとお得な場合があります。南アフリカの主な公園または南アフリカのすべての国立公園に入場できます。" + }, + { + "file": "ja-fleurs-07.mp3", + "set": "ja-fleurs", + "language": "ja", + "metric": "cer", + "duration": 10.08, + "fleurs_ids": [ + 1669 + ], + "reference": "警察のチャンドラ シェカール ソランキ総監によれば 被告人は顔を隠して法廷に出頭したそうです", + "raw_reference": "警察のチャンドラ・シェカール・ソランキ総監によれば、被告人は顔を隠して法廷に出頭したそうです。" + }, + { + "file": "ja-fleurs-08.mp3", + "set": "ja-fleurs", + "language": "ja", + "metric": "cer", + "duration": 18.42, + "fleurs_ids": [ + 1671 + ], + "reference": "連邦議会は2005年度から猥褻物取締法への資金提供を開始し fbiはアダルトポルノに10人の捜査員を投入しなければならないと規定しました", + "raw_reference": "連邦議会は2005年度から猥褻物取締法への資金提供を開始し、FBIはアダルトポルノに10人の捜査員を投入しなければならないと規定しました。" + }, + { + "file": "ja-fleurs-09.mp3", + "set": "ja-fleurs", + "language": "ja", + "metric": "cer", + "duration": 11.76, + "fleurs_ids": [ + 1672 + ], + "reference": "生地の温度が上がりすぎないように気を付けてください縮みの原因になったり 極端な場合は焦げ付きの原因になります", + "raw_reference": "生地の温度が上がりすぎないように気を付けてください(縮みの原因になったり、極端な場合は焦げ付きの原因になります)。" + }, + { + "file": "zh-fleurs-00.mp3", + "set": "zh-fleurs", + "language": "zh", + "metric": "cer", + "duration": 9.6, + "fleurs_ids": [ + 1660 + ], + "reference": "浪 漫 主 义 具 有 很 强 的 文 化 决 定 论 色 彩 这 源 于 歌 德 费 希 特 和 施 莱 格 尔 等 作 家", + "raw_reference": "浪漫主义具有很强的文化决定论色彩。这源于歌德、费希特和施莱格尔等作家。" + }, + { + "file": "zh-fleurs-01.mp3", + "set": "zh-fleurs", + "language": "zh", + "metric": "cer", + "duration": 7.74, + "fleurs_ids": [ + 1661 + ], + "reference": "他 没 有 设 定 削 减 数 额 只 是 表 示 将 根 据 中 国 的 经 济 产 出 来 决 定 数 额", + "raw_reference": "他没有设定削减数额,只是表示将根据中国的经济产出来决定数额。" + }, + { + "file": "zh-fleurs-02.mp3", + "set": "zh-fleurs", + "language": "zh", + "metric": "cer", + "duration": 7.5, + "fleurs_ids": [ + 1662 + ], + "reference": "合 金 基 本 上 由 两 种 或 以 上 金 属 混 合 而 成 不 要 忘 记 一 点 元 素 周 期 表 上 有 很 多 元 素", + "raw_reference": "合金基本上由两种或以上金属混合而成。不要忘记一点,元素周期表上有很多元素。" + }, + { + "file": "zh-fleurs-03.mp3", + "set": "zh-fleurs", + "language": "zh", + "metric": "cer", + "duration": 13.38, + "fleurs_ids": [ + 1663 + ], + "reference": "智利首选登山目的地科茶莫山谷 cochamó valley 被称为南美洲的优胜美地 yosemite这里有各式各样的花岗岩大石壁和峭壁", + "raw_reference": "智利首选登山目的地科茶莫山谷 (Cochamó Valley) 被称为南美洲的优胜美地 (Yosemite),这里有各式各样的花岗岩大石壁和峭壁。" + }, + { + "file": "zh-fleurs-04.mp3", + "set": "zh-fleurs", + "language": "zh", + "metric": "cer", + "duration": 11.4, + "fleurs_ids": [ + 1664 + ], + "reference": "搅 动 两 种 干 粉 使 其 混 合 然 后 用 干 净 的 湿 手 把 它 们 揉 成 一 个 球", + "raw_reference": "搅动两种干粉使其混合,然后用干净的湿手把它们揉成一个球。" + }, + { + "file": "zh-fleurs-05.mp3", + "set": "zh-fleurs", + "language": "zh", + "metric": "cer", + "duration": 15.1, + "fleurs_ids": [ + 1665 + ], + "reference": "据 透 露 这 份 文 件 将 提 及 边 界 争 端 巴 勒 斯 坦 希 望 以 1967 年 中 东 战 争 前 的 边 界 为 基 础", + "raw_reference": "据透露,这份文件将提及边界争端,巴勒斯坦希望以1967年中东战争前的边界为基础。" + }, + { + "file": "zh-fleurs-06.mp3", + "set": "zh-fleurs", + "language": "zh", + "metric": "cer", + "duration": 8.16, + "fleurs_ids": [ + 1666 + ], + "reference": "你 也 可 以 咨 询 除 贵 国 以 外 其 他 政 府 的 建 议 但 他 们 的 建 议 是 为 他 们 的 公 民 而 制 定 的", + "raw_reference": "你也可以咨询除贵国以外其他政府的建议,但他们的建议是为他们的公民而制定的。" + }, + { + "file": "zh-fleurs-07.mp3", + "set": "zh-fleurs", + "language": "zh", + "metric": "cer", + "duration": 15.64, + "fleurs_ids": [ + 1667 + ], + "reference": "一 般 来 说 当 管 理 者 开 始 领 导 过 去 的 同 事 时 会 出 现 两 种 行 为 一 种 情 况 是 他 们 会 努 力 与 团 队 成 员 保 持 亲 密 关 系", + "raw_reference": "一般来说,当管理者开始领导过去的同事时,会出现两种行为。一种情况是,他们会努力与团队成员保持“亲密关系”。" + }, + { + "file": "zh-fleurs-08.mp3", + "set": "zh-fleurs", + "language": "zh", + "metric": "cer", + "duration": 8.76, + "fleurs_ids": [ + 1668 + ], + "reference": "买 一 张 万 能 卡 wild card 也 有 不 少 好 处 带 着 这 张 卡 你 可 以 进 入 南 非 的 一 些 公 园 或 全 部 的 国 家 公 园", + "raw_reference": "买一张万能卡(Wild Card)也有不少好处。带着这张卡,你可以进入南非的一些公园或全部的国家公园。" + }, + { + "file": "zh-fleurs-09.mp3", + "set": "zh-fleurs", + "language": "zh", + "metric": "cer", + "duration": 6.84, + "fleurs_ids": [ + 1669 + ], + "reference": "警 察 局 长 钱 德 拉 谢 卡 尔 索 兰 基 说 被 告 人 出 庭 时 都 蒙 住 了 脸", + "raw_reference": "警察局长钱德拉·谢卡尔·索兰基说,被告人出庭时都蒙住了脸。" + }, + { + "file": "ar-fleurs-00.mp3", + "set": "ar-fleurs", + "language": "ar", + "metric": "wer", + "duration": 6.12, + "fleurs_ids": [ + 1661 + ], + "reference": "لم يحدد رقمًا للتخفيضات قائلًا إنها ستعتمد على الناتج الاقتصادي للصين", + "raw_reference": "لم يحدد رقمًا للتخفيضات، قائلًا إنها ستعتمد على الناتج الاقتصادي للصين." + }, + { + "file": "ar-fleurs-01.mp3", + "set": "ar-fleurs", + "language": "ar", + "metric": "wer", + "duration": 9.96, + "fleurs_ids": [ + 1662 + ], + "reference": "السبائك هي بشكل أساسي خليط من اثنين أو أكثر من المعادن لا تنسَ أن هناك العديد من العناصر في الجدول الدوري", + "raw_reference": "السبائك هي بشكل أساسي خليط من اثنين أو أكثر من المعادن. لا تنسَ أن هناك العديد من العناصر في الجدول الدوري." + }, + { + "file": "ar-fleurs-02.mp3", + "set": "ar-fleurs", + "language": "ar", + "metric": "wer", + "duration": 8.58, + "fleurs_ids": [ + 1664 + ], + "reference": "قلّب المسحوقين الجافّين سوية ثم اعصرهما بأيدٍ مبللة نظيفة حتى تصبح على شكل كرة", + "raw_reference": "قلّب المسحوقين الجافّين سوية ثم اعصرهما بأيدٍ مبللة نظيفة حتى تصبح على شكل كرة." + }, + { + "file": "ar-fleurs-03.mp3", + "set": "ar-fleurs", + "language": "ar", + "metric": "wer", + "duration": 13.32, + "fleurs_ids": [ + 1665 + ], + "reference": "وستشير الوثيقة بحسب التسريب إلى الخلاف على الحدود التي تريدها فلسطين مبنية على حدود ما قبل حرب عام 1967 في الشرق الأوسط", + "raw_reference": "وستشير الوثيقة، بحسب التسريب، إلى الخلاف على الحدود، التي تريدها فلسطين مبنية على حدود ما قبل حرب عام 1967 في الشرق الأوسط." + }, + { + "file": "ar-fleurs-04.mp3", + "set": "ar-fleurs", + "language": "ar", + "metric": "wer", + "duration": 10.2, + "fleurs_ids": [ + 1666 + ], + "reference": "ربما ترغب أيضاً في طلب المشورة من حكومات غير حكومتك ولكن نصيحة تلك الحكومات صادرة خصيصاً لمواطنيها", + "raw_reference": "ربما ترغب أيضاً في طلب المشورة من حكومات غير حكومتك، ولكن نصيحة تلك الحكومات صادرة خصيصاً لمواطنيها." + }, + { + "file": "ar-fleurs-05.mp3", + "set": "ar-fleurs", + "language": "ar", + "metric": "wer", + "duration": 12.24, + "fleurs_ids": [ + 1667 + ], + "reference": "بصفة عامة، يمكن أن يظهر سلوكان عندما يبدأ المديرون في قيادة أقرانهم السابقين أحدهما هو محاولة البقاء مثل الآخرين أو الأخريات", + "raw_reference": "بصفة عامة، يمكن أن يظهر سلوكان عندما يبدأ المديرون في قيادة أقرانهم السابقين أحدهما هو محاولة البقاء \"مثل الآخرين\" (أو الأخريات)." + }, + { + "file": "ar-fleurs-06.mp3", + "set": "ar-fleurs", + "language": "ar", + "metric": "wer", + "duration": 12.3, + "fleurs_ids": [ + 1668 + ], + "reference": "من الممكن أن يكون مفيداً أن تشتري بطاقة wild card فهي تسمح لك بالدخول إلى مجموعة مختارة من متنزهات جنوب إفريقيا أو جميع متنزهات جنوب إفريقيا الوطنية", + "raw_reference": "من الممكن أن يكون مفيداً أن تشتري بطاقة Wild Card، فهي تسمح لك بالدخول إلى مجموعة مختارة من متنزهات جنوب إفريقيا أو جميع متنزهات جنوب إفريقيا الوطنية." + }, + { + "file": "ar-fleurs-07.mp3", + "set": "ar-fleurs", + "language": "ar", + "metric": "wer", + "duration": 7.62, + "fleurs_ids": [ + 1669 + ], + "reference": "وقال مدير الشرطة شاندرا شيخار سولانكي إن المتهمين مثلوا أمام المحكمة بوجوه مغطاة", + "raw_reference": "وقال مدير الشرطة شاندرا شيخار سولانكي إن المتهمين مثلوا أمام المحكمة بوجوه مغطاة." + }, + { + "file": "ar-fleurs-08.mp3", + "set": "ar-fleurs", + "language": "ar", + "metric": "wer", + "duration": 15.78, + "fleurs_ids": [ + 1671 + ], + "reference": "بدأ الكونجرس في تمويل مبادرة الفحش في السنة المالية 2005 وحدد أن مكتب التحقيقات الفدرالي يجب أن يخصص 10 عملاء للمواد الإباحية للبالغين", + "raw_reference": "بدأ الكونجرس في تمويل مبادرة الفحش في السنة المالية 2005 وحدد أن مكتب التحقيقات الفدرالي يجب أن يخصص 10 عملاء للمواد الإباحية للبالغين." + }, + { + "file": "ar-fleurs-09.mp3", + "set": "ar-fleurs", + "language": "ar", + "metric": "wer", + "duration": 10.5, + "fleurs_ids": [ + 1672 + ], + "reference": "لا تجعل القماش يصبح ساخنًا أكثر من اللازم مما قد يؤدي إلى انكماشه أو احتراقه في أسوأ الحالات", + "raw_reference": "لا تجعل القماش يصبح ساخنًا أكثر من اللازم (مما قد يؤدي إلى انكماشه أو احتراقه في أسوأ الحالات)." + } + ] +} diff --git a/.github/script/assets/asr_eval/zh-fleurs-00.mp3 b/.github/script/assets/asr_eval/zh-fleurs-00.mp3 new file mode 100644 index 000000000..f2eceb624 Binary files /dev/null and b/.github/script/assets/asr_eval/zh-fleurs-00.mp3 differ diff --git a/.github/script/assets/asr_eval/zh-fleurs-01.mp3 b/.github/script/assets/asr_eval/zh-fleurs-01.mp3 new file mode 100644 index 000000000..0a04e7611 Binary files /dev/null and b/.github/script/assets/asr_eval/zh-fleurs-01.mp3 differ diff --git a/.github/script/assets/asr_eval/zh-fleurs-02.mp3 b/.github/script/assets/asr_eval/zh-fleurs-02.mp3 new file mode 100644 index 000000000..1cd877f08 Binary files /dev/null and b/.github/script/assets/asr_eval/zh-fleurs-02.mp3 differ diff --git a/.github/script/assets/asr_eval/zh-fleurs-03.mp3 b/.github/script/assets/asr_eval/zh-fleurs-03.mp3 new file mode 100644 index 000000000..1229156be Binary files /dev/null and b/.github/script/assets/asr_eval/zh-fleurs-03.mp3 differ diff --git a/.github/script/assets/asr_eval/zh-fleurs-04.mp3 b/.github/script/assets/asr_eval/zh-fleurs-04.mp3 new file mode 100644 index 000000000..ff5a0be37 Binary files /dev/null and b/.github/script/assets/asr_eval/zh-fleurs-04.mp3 differ diff --git a/.github/script/assets/asr_eval/zh-fleurs-05.mp3 b/.github/script/assets/asr_eval/zh-fleurs-05.mp3 new file mode 100644 index 000000000..99fe63bbf Binary files /dev/null and b/.github/script/assets/asr_eval/zh-fleurs-05.mp3 differ diff --git a/.github/script/assets/asr_eval/zh-fleurs-06.mp3 b/.github/script/assets/asr_eval/zh-fleurs-06.mp3 new file mode 100644 index 000000000..e4fbf2bc2 Binary files /dev/null and b/.github/script/assets/asr_eval/zh-fleurs-06.mp3 differ diff --git a/.github/script/assets/asr_eval/zh-fleurs-07.mp3 b/.github/script/assets/asr_eval/zh-fleurs-07.mp3 new file mode 100644 index 000000000..85642d982 Binary files /dev/null and b/.github/script/assets/asr_eval/zh-fleurs-07.mp3 differ diff --git a/.github/script/assets/asr_eval/zh-fleurs-08.mp3 b/.github/script/assets/asr_eval/zh-fleurs-08.mp3 new file mode 100644 index 000000000..60d2f035c Binary files /dev/null and b/.github/script/assets/asr_eval/zh-fleurs-08.mp3 differ diff --git a/.github/script/assets/asr_eval/zh-fleurs-09.mp3 b/.github/script/assets/asr_eval/zh-fleurs-09.mp3 new file mode 100644 index 000000000..8924fd346 Binary files /dev/null and b/.github/script/assets/asr_eval/zh-fleurs-09.mp3 differ diff --git a/.github/script/build_asr_eval.py b/.github/script/build_asr_eval.py new file mode 100644 index 000000000..9ba6b070d --- /dev/null +++ b/.github/script/build_asr_eval.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Build the ASR evaluation set in assets/asr_eval/ from FLEURS. + +Run this by hand, not in CI: it reads the FLEURS test splits from local +Parquet files and needs pyarrow, numpy and soundfile, none of which the test +itself uses. Download the splits from the dataset's Parquet conversion first: + + https://huggingface.co/api/datasets/google/fleurs/parquet//test + +The selection is deterministic: rows sorted by sentence id, one recording per +sentence, 4 to 25 s long, the first ones taken. So rerunning against the same +Parquet revision reproduces the same set. Sets: + + en-short 20 English clips, each inside one 30 s Whisper window + en-long 3 files of ~80 s, English clips joined with short gaps, so + each crosses two window boundaries + en-longform 1 file of ~300 s, built the same way + -fleurs 10 clips each for de, es, fr, ja, zh and ar + +Clips are written as 16 kHz mono MP3, the format the server already decodes +for the bundled test clip, to keep the set a few MB. +""" + +from __future__ import annotations + +import argparse +import io +import json +from pathlib import Path + +import numpy as np +import pyarrow.parquet as pq +import soundfile as sf + +SAMPLE_RATE = 16000 +GAP_SECONDS = 0.6 +MIN_SECONDS, MAX_SECONDS = 4.0, 25.0 + +# FLEURS config, ISO 639-1 code for the request's language field, and the +# metric: CER for languages written without spaces between words. +LANGUAGES = { + "en": ("en_us", "wer"), + "de": ("de_de", "wer"), + "es": ("es_419", "wer"), + "fr": ("fr_fr", "wer"), + "ja": ("ja_jp", "cer"), + "zh": ("cmn_hans_cn", "cer"), + "ar": ("ar_eg", "wer"), +} + + +def load_clips(parquet: Path) -> list[dict]: + """Rows usable as clips, sorted by sentence id, one per sentence.""" + table = pq.read_table( + parquet, columns=["id", "num_samples", "transcription", "raw_transcription", "audio"] + ) + rows = sorted(table.to_pylist(), key=lambda r: (r["id"], r["audio"]["path"])) + clips, seen = [], set() + for row in rows: + seconds = row["num_samples"] / SAMPLE_RATE + if row["id"] in seen or not MIN_SECONDS <= seconds <= MAX_SECONDS: + continue + seen.add(row["id"]) + audio, rate = sf.read(io.BytesIO(row["audio"]["bytes"]), dtype="float32") + if rate != SAMPLE_RATE: + raise ValueError(f"{parquet.name} id {row['id']}: {rate} Hz") + clips.append({ + "id": row["id"], + "audio": audio, + "reference": row["transcription"].strip(), + "raw_reference": row["raw_transcription"].strip(), + }) + return clips + + +def take_until(clips: list[dict], seconds: float) -> list[dict]: + """Pops clips off the front until their joined length reaches `seconds`.""" + taken, total = [], 0.0 + while clips and total < seconds: + clip = clips.pop(0) + taken.append(clip) + total += len(clip["audio"]) / SAMPLE_RATE + GAP_SECONDS + return taken + + +def write_item(out: Path, name: str, set_name: str, language: str, metric: str, + clips: list[dict]) -> dict: + gap = np.zeros(int(GAP_SECONDS * SAMPLE_RATE), dtype="float32") + parts = [] + for index, clip in enumerate(clips): + if index: + parts.append(gap) + parts.append(clip["audio"]) + audio = np.concatenate(parts) + path = out / f"{name}.mp3" + sf.write(path, audio, SAMPLE_RATE, format="MP3", subtype="MPEG_LAYER_III") + return { + "file": path.name, + "set": set_name, + "language": language, + "metric": metric, + "duration": round(len(audio) / SAMPLE_RATE, 3), + "fleurs_ids": [clip["id"] for clip in clips], + "reference": " ".join(clip["reference"] for clip in clips), + "raw_reference": " ".join(clip["raw_reference"] for clip in clips), + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("parquet_dir", type=Path, help="Directory of .parquet files") + parser.add_argument("--revision", required=True, + help="FLEURS Parquet revision the files came from, recorded in the manifest") + parser.add_argument("--out", type=Path, + default=Path(__file__).resolve().parent / "assets" / "asr_eval") + args = parser.parse_args() + args.out.mkdir(parents=True, exist_ok=True) + + items = [] + for language, (config, metric) in LANGUAGES.items(): + clips = load_clips(args.parquet_dir / f"{config}.parquet") + print(f"{config}: {len(clips)} usable clips") + if language == "en": + for index, clip in enumerate(clips[:20]): + items.append(write_item(args.out, f"en-short-{index:02d}", "en-short", + language, metric, [clip])) + rest = clips[20:] + for index in range(3): + items.append(write_item(args.out, f"en-long-{index}", "en-long", + language, metric, take_until(rest, 80))) + items.append(write_item(args.out, "en-longform-0", "en-longform", + language, metric, take_until(rest, 300))) + else: + for index, clip in enumerate(clips[:10]): + items.append(write_item(args.out, f"{language}-fleurs-{index:02d}", + f"{language}-fleurs", language, metric, [clip])) + + manifest = { + "source": "FLEURS test split (google/fleurs), CC BY 4.0", + "parquet_revision": args.revision, + "sample_rate": SAMPLE_RATE, + "items": items, + } + (args.out / "manifest.json").write_text( + json.dumps(manifest, indent=1, ensure_ascii=False) + "\n", encoding="utf-8") + total = sum(item["duration"] for item in items) + print(f"{len(items)} files, {total:.0f} s of audio, written to {args.out}") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/asr-test.yml b/.github/workflows/asr-test.yml new file mode 100644 index 000000000..a4efc678b --- /dev/null +++ b/.github/workflows/asr-test.yml @@ -0,0 +1,557 @@ +name: ASR Test + +# Measures speech recognition on real NPU hardware whenever ASR code changes. +# Nothing is rebuilt here: like the model sweep, this tests the portable builds +# that "Build Linux Portable" and "Build Windows Packages" produce for the same +# commit, default and HRX alike. .github/script/asr_test.py holds the checks +# and .github/script/assets/asr_eval/ the FLEURS evaluation set. +# +# On a pull request, main's latest build runs first on the same machine and +# becomes the baseline. The run fails if the pooled error rate rises by more +# than 1 point, any one set's by more than 5, or the number of runaway +# transcripts grows; the report also shows how far the transcripts moved. +# Pushes to main and the weekly run have no baseline build, so they compare +# against the per-platform rates in asr_eval/expected.json instead, with +# 2 points of slack on the pooled rate (those rates come from one machine per +# platform) and the same 5 points per set. +# +# The trigger paths are the Whisper engine, its NPU kernels and prebuilt NPU +# libraries, the FFmpeg, audio front end and FFT code, the audio front ends of +# the chat models that transcribe, and the server files that host +# /v1/audio/transcriptions. Those server files also hold every other endpoint, +# so any change to them runs this test too. The pull_request and push path +# lists must stay identical. Shared files such as src/CMakeLists.txt (the Linux +# FFmpeg version) and lm_config.hpp are left out, since every change touches +# them. +# +# The weekly run catches what no commit changes: Whisper weights republished +# on Hugging Face, and NPU driver or firmware updates on the runners. +# +# A manual dispatch tests the dispatched ref's commit, or a pull request's +# builds when given its number. + +on: + pull_request: + branches: [ main ] + paths: + - 'src/common/whisper/**' + - 'src/include/whisper/**' + - 'src/xclbins/Whisper-V3-Turbo-NPU2/**' + - 'src/lib/xrt/*whisper_npu*' + - 'src/lib/hrx/*whisper_npu*' + - 'src/common/audio_process_utils/**' + - 'src/include/audio_process_utils/**' + - 'src/common/audio/**' + - 'src/include/audio/**' + - 'src/common/AutoModel/*_audio.cpp' + - 'src/detail/gemma4e_npu/gemma4e_audio*' + - 'src/include/fftw3.h' + - 'src/include/pocketfft/**' + - 'src/lib/libfftw3*' + - 'src/lib/av*' + - 'src/lib/sw*' + - 'src/server/rest_handler.cpp' + - 'src/server/server.cpp' + - 'src/server/multipart.cpp' + - 'src/server/multipart.hpp' + - 'src/test/whisper_npu/**' + - '.github/script/asr_test.py' + - '.github/script/asr_summary.py' + - '.github/script/sweep_common.py' + - '.github/script/assets/asr_eval/**' + - '.github/workflows/asr-test.yml' + push: + branches: [ main ] + paths: + - 'src/common/whisper/**' + - 'src/include/whisper/**' + - 'src/xclbins/Whisper-V3-Turbo-NPU2/**' + - 'src/lib/xrt/*whisper_npu*' + - 'src/lib/hrx/*whisper_npu*' + - 'src/common/audio_process_utils/**' + - 'src/include/audio_process_utils/**' + - 'src/common/audio/**' + - 'src/include/audio/**' + - 'src/common/AutoModel/*_audio.cpp' + - 'src/detail/gemma4e_npu/gemma4e_audio*' + - 'src/include/fftw3.h' + - 'src/include/pocketfft/**' + - 'src/lib/libfftw3*' + - 'src/lib/av*' + - 'src/lib/sw*' + - 'src/server/rest_handler.cpp' + - 'src/server/server.cpp' + - 'src/server/multipart.cpp' + - 'src/server/multipart.hpp' + - 'src/test/whisper_npu/**' + - '.github/script/asr_test.py' + - '.github/script/asr_summary.py' + - '.github/script/sweep_common.py' + - '.github/script/assets/asr_eval/**' + - '.github/workflows/asr-test.yml' + schedule: + - cron: '23 4 * * 1' # Mondays 04:23 UTC + workflow_dispatch: + inputs: + pr: + description: "Pull request number whose builds to test (blank = the dispatched ref)" + type: string + default: "" + require: + description: "Feature groups that must pass, space-separated (blank = the default below)" + type: string + default: "" + max_drift: + description: "Fail a set whose transcripts differ from the baseline's by more than this fraction (blank = report only)" + type: string + default: "" + +permissions: + contents: read + # Needed to list and download artifacts from the build workflow runs. + actions: read + +# Only the newest commit on a ref is worth an NPU machine. The event name keeps +# the weekly run and a push to main from cancelling each other. +concurrency: + group: asr-test-${{ github.ref }}-${{ github.event_name }}-${{ inputs.pr }} + cancel-in-progress: true + +env: + # Feature groups main passes on every platform, so a pull request that + # breaks one fails. Add a group in the pull request that makes main pass it: + # verbose_json, timestamp_rules (needs verbose_json), window_tail, language, + # quiet_log. The rest are measured and reported as warnings. window_tail is + # not here yet because main's Linux HRX build loops on the long-form file; + # a pull request that adds repeats is still caught by the baseline check. + ASR_REQUIRE: ${{ inputs.require || 'prompt' }} + ASR_MAX_DRIFT: ${{ inputs.max_drift || '' }} + +jobs: + # Waits on a hosted runner for the build runs under test, the same way the + # model sweep's resolve-build does, and pins each to a run id so a stale + # artifact from an earlier push can never be tested. For a pull request it + # also finds main's latest successful build as the baseline. Outputs one + # JSON list of builds per platform, which the test jobs expand into their + # matrix. + resolve-build: + runs-on: ubuntu-latest + timeout-minutes: 90 + outputs: + linux_run_id: ${{ steps.resolve.outputs.linux_run_id }} + windows_run_id: ${{ steps.resolve.outputs.windows_run_id }} + linux_base_run_id: ${{ steps.resolve.outputs.linux_base_run_id }} + windows_base_run_id: ${{ steps.resolve.outputs.windows_base_run_id }} + linux_builds: ${{ steps.resolve.outputs.linux_builds }} + windows_builds: ${{ steps.resolve.outputs.windows_builds }} + + steps: + - name: Wait for the portable builds + id: resolve + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ inputs.pr }} + # A pull_request run records the PR head commit in head_sha, while + # github.sha is the synthetic merge commit. Try both. On a push, a + # dispatch or the weekly run only the second is set; for the weekly + # run it is main's latest commit, built by that commit's push. + CANDIDATE_SHAS: ${{ github.event.pull_request.head.sha }} ${{ github.sha }} + WANT_BASELINE: ${{ github.event_name == 'pull_request' || inputs.pr != '' }} + run: | + set -euo pipefail + + POLL_SECONDS=20 + APPEAR_DEADLINE=$(( SECONDS + 10 * 60 )) + FINISH_DEADLINE=$(( SECONDS + 80 * 60 )) + linux_builds='[]' + windows_builds='[]' + + if [ -n "${PR_NUMBER}" ]; then + CANDIDATE_SHAS=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq '.head.sha') + echo "Testing pull request #${PR_NUMBER} at ${CANDIDATE_SHAS}." + fi + + # Newest run for whichever candidate SHA matches, or empty. + find_run() { + local workflow="$1" sha run_id + for sha in ${CANDIDATE_SHAS}; do + run_id=$(gh api \ + "repos/${GITHUB_REPOSITORY}/actions/workflows/${workflow}/runs?head_sha=${sha}&per_page=100" \ + --jq 'if (.workflow_runs | length) == 0 then "" else (.workflow_runs | sort_by(.run_number) | last | .id) end') + if [ -n "${run_id}" ]; then + printf '%s' "${run_id}" + return 0 + fi + done + } + + list_artifacts() { + gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/$1/artifacts?per_page=100" \ + --jq '.artifacts[] | select(.expired | not) | .name' + } + + # main's newest successful push build that still has its artifacts. + # Sets base_run_id and base_artifacts, both empty when there is none. + # Not called in $(...), which would lose the assignments. + base_run_id="" + base_artifacts="" + find_base() { + local workflow="$1" run_id + base_run_id="" + base_artifacts="" + for run_id in $(gh api \ + "repos/${GITHUB_REPOSITORY}/actions/workflows/${workflow}/runs?branch=main&event=push&status=success&per_page=10" \ + --jq '.workflow_runs[].id'); do + base_artifacts=$(list_artifacts "${run_id}") + if [ -n "${base_artifacts}" ]; then + base_run_id="${run_id}" + return 0 + fi + done + } + + # Appends {artifact, base_artifact, platform} to the named list when + # the run published an artifact matching the pattern. An artifact + # exists only if its build job got all the way through, so a failed + # HRX build drops just the HRX test. base_artifact is empty when + # there is no baseline build of that kind. + add_build() { + local list="$1" artifacts="$2" pattern="$3" platform="$4" name base + name=$(printf '%s\n' "${artifacts}" | grep -E "${pattern}" | head -n 1 || true) + if [ -z "${name}" ]; then + echo "::warning::No ${platform} portable artifact was published; skipping that test." + return 0 + fi + base=$(printf '%s\n' "${base_artifacts}" | grep -E "${pattern}" | head -n 1 || true) + echo " ${platform}: ${name} (baseline: ${base:-none})" + printf -v "${list}" '%s' "$(jq -c --arg a "${name}" --arg b "${base}" --arg p "${platform}" \ + '. + [{artifact: $a, base_artifact: $b, platform: $p}]' <<< "${!list}")" + } + + wait_for_build() { + local workflow="$1" platform="$2" default_pattern="$3" hrx_pattern="$4" + local run_id="" status artifacts + + while :; do + run_id=$(find_run "${workflow}") + [ -n "${run_id}" ] && break + if [ "${SECONDS}" -ge "${APPEAR_DEADLINE}" ]; then + echo "::warning::No ${workflow} run appeared for this commit; skipping ${platform}." + return 1 + fi + echo "Waiting for a ${workflow} run to appear..." + sleep "${POLL_SECONDS}" + done + + while :; do + status=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}" --jq '.status') + [ "${status}" = "completed" ] && break + if [ "${SECONDS}" -ge "${FINISH_DEADLINE}" ]; then + echo "::warning::${workflow} run ${run_id} is still ${status}; skipping ${platform}." + return 1 + fi + echo "${workflow} run ${run_id} is ${status}; checking again in ${POLL_SECONDS}s." + sleep "${POLL_SECONDS}" + done + + artifacts=$(list_artifacts "${run_id}") + echo "${workflow} run ${run_id} published:" + printf '%s\n' "${artifacts}" | sed 's/^/ /' + echo "${platform}_run_id=${run_id}" >> "$GITHUB_OUTPUT" + + base_artifacts="" + if [ "${WANT_BASELINE}" = "true" ]; then + find_base "${workflow}" + if [ -n "${base_run_id}" ]; then + echo "Baseline: main's ${workflow} run ${base_run_id}." + echo "${platform}_base_run_id=${base_run_id}" >> "$GITHUB_OUTPUT" + else + echo "::warning::No ${workflow} build of main has artifacts left; ${platform} runs without a baseline." + fi + fi + + # Fully anchored, so neither pattern matches the other's artifact, + # the models-from-source one, or "fastflowlm-windows-msi-". + add_build "${platform}_builds" "${artifacts}" "${default_pattern}" "${platform}" + add_build "${platform}_builds" "${artifacts}" "${hrx_pattern}" "${platform}-hrx" + } + + wait_for_build debian-portable.yml linux \ + '^portable$' '^HRX-portable$' || true + wait_for_build windows-build.yml windows \ + '^fastflowlm-windows-[0-9a-f]+$' '^HRX-fastflowlm-windows-[0-9a-f]+$' || true + + { + echo "linux_builds=${linux_builds}" + echo "windows_builds=${windows_builds}" + } >> "$GITHUB_OUTPUT" + + # One missing platform is a warning above; nothing at all to test + # would otherwise leave a green check that tested nothing. + if [ "${linux_builds}" = "[]" ] && [ "${windows_builds}" = "[]" ]; then + # The Linux portable artifact is kept for 7 days, so a weekly run + # after a quiet week on main can find nothing left to download. + echo "::error::No portable build was found, so nothing was tested. Build artifacts expire, so for an older commit, rerun the build workflows first." + exit 1 + fi + + asr-linux: + name: asr (${{ matrix.build.platform }}) + needs: resolve-build + if: needs.resolve-build.outputs.linux_builds != '[]' + runs-on: [self-hosted, linux, x64, npu, sweep] + timeout-minutes: 120 + + strategy: + fail-fast: false + matrix: + build: ${{ fromJSON(needs.resolve-build.outputs.linux_builds) }} + + env: + # Weights are expected at $FLM_MODEL_PATH/models/ + FLM_MODEL_PATH: /scratch + + steps: + # Only the scripts and their test assets; this job compiles nothing. + - name: Checkout test scripts + uses: actions/checkout@v4 + with: + sparse-checkout: .github/script + filter: blob:none + + - name: Download ${{ matrix.build.platform }} portable build + uses: dawidd6/action-download-artifact@v25 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + run_id: ${{ needs.resolve-build.outputs.linux_run_id }} + name: ${{ matrix.build.artifact }} + path: build-artifact + if_no_artifact_found: fail + + - name: Download baseline build + if: matrix.build.base_artifact != '' + uses: dawidd6/action-download-artifact@v25 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + run_id: ${{ needs.resolve-build.outputs.linux_base_run_id }} + name: ${{ matrix.build.base_artifact }} + path: base-artifact + if_no_artifact_found: fail + + - name: Unpack portable builds + run: | + set -euo pipefail + for build in build base; do + [ -d "${build}-artifact" ] || continue + mkdir -p "flm-${build}" + tarball=$(ls "${build}"-artifact/*_linux.tar.gz) + echo "Using tarball: ${tarball}" + tar xzf "${tarball}" -C "flm-${build}" + chmod +x "flm-${build}/flm" "flm-${build}/flm-real" + "flm-${build}/flm" version + done + echo "FLM_BIN=${GITHUB_WORKSPACE}/flm-build/flm" >> "$GITHUB_ENV" + + # No-ops once the weights are on the runner. + - name: Pull models + run: | + set -euo pipefail + "${FLM_BIN}" pull whisper-v3:turbo + "${FLM_BIN}" pull gemma4-it:e2b + "${FLM_BIN}" pull gemma4-it:12b + + - name: Set up Python virtualenv + run: | + set -euo pipefail + python3 -m venv .venv + .venv/bin/python -m pip install --upgrade pip + for attempt in 1 2 3; do + .venv/bin/python -m pip install --no-cache-dir openai && break + if [ "${attempt}" -eq 3 ]; then + exit 1 + fi + done + + # The baseline's own failures are reported in its results, not here: + # what matters is how the build under test compares with it. + - name: Run ASR test on the baseline + if: matrix.build.base_artifact != '' + run: | + .venv/bin/python .github/script/asr_test.py \ + --flm-bin "${GITHUB_WORKSPACE}/flm-base/flm" \ + --platform "${{ matrix.build.platform }}" \ + --output-dir asr-baseline || true + + - name: Run ASR test + run: | + set -euo pipefail + args=(--flm-bin "${FLM_BIN}" --platform "${{ matrix.build.platform }}" + --output-dir asr-results) + baseline=$(ls asr-baseline/*_results.json 2>/dev/null || true) + if [ -n "${baseline}" ]; then + args+=(--baseline "${baseline}") + else + # Expected rates come from one machine, so the pooled rate gets + # 2 points of slack instead of 1; sets keep their 5 points. + args+=(--expected .github/script/assets/asr_eval/expected.json --tolerance 2) + fi + [ -n "${ASR_REQUIRE}" ] && args+=(--require ${ASR_REQUIRE}) + [ -n "${ASR_MAX_DRIFT}" ] && args+=(--max-drift "${ASR_MAX_DRIFT}") + .venv/bin/python .github/script/asr_test.py "${args[@]}" + + - name: Upload results + if: always() + uses: actions/upload-artifact@v4 + with: + name: asr-test-${{ matrix.build.platform }}-${{ github.run_number }} + path: | + asr-results/ + asr-baseline/ + if-no-files-found: warn + retention-days: 30 + + asr-windows: + name: asr (${{ matrix.build.platform }}) + needs: resolve-build + if: needs.resolve-build.outputs.windows_builds != '[]' + runs-on: [self-hosted, windows, x64, npu, sweep] + timeout-minutes: 120 + + strategy: + fail-fast: false + matrix: + build: ${{ fromJSON(needs.resolve-build.outputs.windows_builds) }} + + steps: + - name: Checkout test scripts + uses: actions/checkout@v4 + with: + sparse-checkout: .github/script + filter: blob:none + + - name: Resolve model path + shell: powershell + run: | + $modelPath = Join-Path $env:USERPROFILE ".flm" + New-Item -ItemType Directory -Force $modelPath | Out-Null + "FLM_MODEL_PATH=$modelPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + + # The artifacts hold flm.exe and its DLLs at the top level. + - name: Download ${{ matrix.build.platform }} portable build + uses: dawidd6/action-download-artifact@v25 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + run_id: ${{ needs.resolve-build.outputs.windows_run_id }} + name: ${{ matrix.build.artifact }} + path: flm-build + if_no_artifact_found: fail + + - name: Download baseline build + if: matrix.build.base_artifact != '' + uses: dawidd6/action-download-artifact@v25 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + run_id: ${{ needs.resolve-build.outputs.windows_base_run_id }} + name: ${{ matrix.build.base_artifact }} + path: flm-base + if_no_artifact_found: fail + + - name: Verify FLM binaries run + shell: powershell + run: | + $ErrorActionPreference = "Stop" + foreach ($build in "flm-build", "flm-base") { + $flm = Join-Path $env:GITHUB_WORKSPACE "$build\flm.exe" + if (Test-Path $flm) { & $flm version } + elseif ($build -eq "flm-build") { throw "flm.exe not found at $flm" } + } + "FLM_BIN=$(Join-Path $env:GITHUB_WORKSPACE 'flm-build\flm.exe')" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + + # No-ops once the weights are on the runner. + - name: Pull models + shell: powershell + run: | + foreach ($model in "whisper-v3:turbo", "gemma4-it:e2b", "gemma4-it:12b") { + & $env:FLM_BIN pull $model + # A native command's exit code does not trip ErrorActionPreference. + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + } + + - name: Set up Python virtualenv + shell: powershell + run: | + $ErrorActionPreference = "Stop" + python -m venv .venv + .\.venv\Scripts\python.exe -m pip install --upgrade pip + .\.venv\Scripts\python.exe -m pip install openai + + # See the Linux job: the baseline's own failures do not fail this job. + - name: Run ASR test on the baseline + if: matrix.build.base_artifact != '' + shell: powershell + run: | + & ".\.venv\Scripts\python.exe" .github\script\asr_test.py ` + --flm-bin (Join-Path $env:GITHUB_WORKSPACE "flm-base\flm.exe") ` + --platform "${{ matrix.build.platform }}" ` + --output-dir asr-baseline + exit 0 + + - name: Run ASR test + shell: powershell + run: | + $ErrorActionPreference = "Stop" + $cmdArgs = @( + ".github\script\asr_test.py", + "--flm-bin", $env:FLM_BIN, + "--platform", "${{ matrix.build.platform }}", + "--output-dir", "asr-results" + ) + $baseline = Get-ChildItem asr-baseline\*_results.json -ErrorAction SilentlyContinue | Select-Object -First 1 + if ($baseline) { + $cmdArgs += @("--baseline", $baseline.FullName) + } else { + # Expected rates come from one machine, so the pooled rate gets + # 2 points of slack instead of 1; sets keep their 5 points. + $cmdArgs += @("--expected", ".github\script\assets\asr_eval\expected.json", "--tolerance", "2") + } + if ($env:ASR_REQUIRE) { $cmdArgs += @("--require") + ($env:ASR_REQUIRE -split '\s+') } + if ($env:ASR_MAX_DRIFT) { $cmdArgs += @("--max-drift", $env:ASR_MAX_DRIFT) } + & ".\.venv\Scripts\python.exe" $cmdArgs + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + - name: Upload results + if: always() + uses: actions/upload-artifact@v4 + with: + name: asr-test-${{ matrix.build.platform }}-${{ github.run_number }} + path: | + asr-results/ + asr-baseline/ + if-no-files-found: warn + retention-days: 30 + + # One report for the whole run, a section per model with the platforms side + # by side, so it is clear which model failed where. Each job's own report is + # in its artifact. Reporting only, so it stays on a GitHub-hosted runner. + summary: + runs-on: ubuntu-latest + needs: [resolve-build, asr-linux, asr-windows] + if: always() && needs.resolve-build.result == 'success' + steps: + - name: Checkout test scripts + uses: actions/checkout@v4 + with: + sparse-checkout: .github/script + filter: blob:none + + - name: Download results + uses: actions/download-artifact@v4 + continue-on-error: true + with: + pattern: asr-test-*-${{ github.run_number }} + path: asr-artifacts + + - name: Write job summary + run: | + set -euo pipefail + python3 .github/script/asr_summary.py asr-artifacts >> "$GITHUB_STEP_SUMMARY"