From b1c8f4489b60f5f29c18977a5dc220079453d026 Mon Sep 17 00:00:00 2001 From: tawei-amd Date: Fri, 16 Oct 2026 12:44:34 -0700 Subject: [PATCH] fix(ci): stop the ASR test failing on decodes that got no worse The ASR test failed PR #798 on Windows HRX with "no more runaways than the baseline: 4 vs 3". The new runaway, de-fleurs-02, loops on its first half in both builds; #798's build also transcribed the second half that main's dropped. Same 12 errors, more of the real speech, but the extra words pushed it past 1.5 times the reference length. - A file now counts against a build only if it became a runaway, or repeats more phrases, and also has more errors than in the baseline. - The per-set limit applies to sets of ten or more files. A single long-form file moves by many points whenever one decode flips, which HRX builds do between identical runs (main's own Linux HRX build scored 18.5% and 36.8% on it); those files are left to the pooled rate and the two checks above. Replayed against every CI result so far (#798 and #799, four platforms each) and local runs of #510, #794 and #798: all pass, including #798 on Windows HRX. A synthetic new loop with more errors still fails. Co-Authored-By: Claude --- .github/script/asr_test.py | 46 ++++++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/.github/script/asr_test.py b/.github/script/asr_test.py index ccf7a4374..dca246db0 100644 --- a/.github/script/asr_test.py +++ b/.github/script/asr_test.py @@ -33,8 +33,12 @@ With --baseline, the results of another build (normally main's, run on the same machine just before) become the reference. The run fails if the pooled -error rate rises by more than --tolerance points, any one set's by more than ---set-tolerance, or the number of runaways grows. The report also shows how +error rate rises by more than --tolerance points, the rate of any set of ten +or more files by more than --set-tolerance, or a file turns into a runaway, or +repeats more phrases, while also getting more errors than in the baseline. +The long-form sets are left to the pooled rate and those last two checks: a +single file there moves by many points whenever one decode flips, which HRX +builds do between identical runs. The report also shows how many transcripts changed. Even a numerically equivalent change, such as a different FFT, flips a few borderline decodes, so a few changed files and small per-set moves are expected; --max-drift is for builds that should @@ -80,6 +84,8 @@ SAME_TEXT_MAX_WER = 0.02 REPEAT_NGRAM = 6 RUNAWAY_RATIO = 1.5 +# Smaller sets are judged through the pooled rate only; see the docstring. +SET_CHECK_MIN_FILES = 10 ARABIC_DIACRITICS = re.compile("[\u064b-\u065f\u0670\u06d6-\u06ed]") @@ -159,9 +165,10 @@ def add_arguments(parser) -> None: parser.add_argument("--tolerance", type=float, default=1.0, help="Allowed rise in the pooled error rate, in points (default: 1.0).") parser.add_argument("--set-tolerance", type=float, default=5.0, - help="Allowed rise in any one set's error rate, in points " - "(default: 5.0). Sets are ten files, so one flipped " - "decode moves them by a point or more.") + help="Allowed rise in the error rate of any set of " + f"{SET_CHECK_MIN_FILES} or more files, in points " + "(default: 5.0). One flipped decode moves a ten-file " + "set by a point or more.") parser.add_argument("--max-drift", type=float, default=None, help="Fail a set whose transcripts differ from the baseline's " "by more than this fraction, for builds meant to change " @@ -567,17 +574,27 @@ def compare(self, writer, model: str) -> None: self.check(writer, model, "eval", "pooled error rate no worse than the baseline", overall["error_rate"] - base_overall["error_rate"] <= tolerance, f"{pct(overall['error_rate'])} vs {pct(base_overall['error_rate'])}") - self.check(writer, model, "eval", "no more runaways than the baseline", - overall["runaways"] <= base_overall["runaways"], - f"{overall['runaways']} vs {base_overall['runaways']}") + # A file that crosses the runaway length while making no more + # errors, for instance by also transcribing what the baseline + # dropped, is not a regression. + new_runaways = sorted( + f for f, r in self.results["items"].items() + if r.get("runaway") and f in base_items + and not base_items[f].get("runaway") + and r["edits"] > base_items[f]["edits"]) + self.check(writer, model, "eval", "no new runaway transcripts", + not new_runaways, + ", ".join(new_runaways) if new_runaways else + f"{overall['runaways']} runaways, baseline {base_overall['runaways']}") for set_name, current in self.results["sets"].items(): base = self.baseline["sets"].get(set_name) if not base: continue delta = current["error_rate"] - base["error_rate"] - self.check(writer, model, f"eval {set_name}", - "error rate no worse than the baseline", delta <= set_tolerance, - f"{pct(current['error_rate'])} vs {pct(base['error_rate'])}") + if current["files"] >= SET_CHECK_MIN_FILES: + self.check(writer, model, f"eval {set_name}", + "error rate no worse than the baseline", delta <= set_tolerance, + f"{pct(current['error_rate'])} vs {pct(base['error_rate'])}") compared = [(f, r) for f, r in self.results["items"].items() if r["set"] == set_name and f in base_items] drift = pooled([error_rate(base_items[f]["text"], r["text"], r["language"], @@ -596,10 +613,11 @@ def compare(self, writer, model: str) -> None: f"baseline {base['rtf']:.3f}") for name, counts in self.results["repeats"].items(): base = self.baseline.get("repeats", {}).get(name) - if base: + if base and name in base_items: + more_errors = self.results["items"][name]["edits"] > base_items[name]["edits"] self.check(writer, model, f"eval {name}", "no more repeated phrases than the baseline", - counts["text"] <= base["text"], + counts["text"] <= base["text"] or not more_errors, f"{counts['text']} vs {base['text']}") elif self.expected: limit = self.expected.get("overall") @@ -609,7 +627,7 @@ def compare(self, writer, model: str) -> None: f"{pct(overall['error_rate'])} vs expected {pct(limit)}") for set_name, current in self.results["sets"].items(): limit = self.expected.get("sets", {}).get(set_name) - if limit is not None: + if limit is not None and current["files"] >= SET_CHECK_MIN_FILES: self.check(writer, model, f"eval {set_name}", "error rate within the expected value", current["error_rate"] <= limit + set_tolerance,