Skip to content

fix(ci): stop the ASR test failing on decodes that got no worse #803

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tawei-amd merged 1 commit into from
Oct 16, 2026
+32 −14
Merged

fix(ci): stop the ASR test failing on decodes that got no worse #803

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 32 additions & 14 deletions .github/script/asr_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,12 @@

With --baseline, the results of another build (normally main's, run on the
same machine just before) become the reference. The run fails if the pooled
error rate rises by more than --tolerance points, any one set's by more than
--set-tolerance, or the number of runaways grows. The report also shows how
error rate rises by more than --tolerance points, the rate of any set of ten
or more files by more than --set-tolerance, or a file turns into a runaway, or
repeats more phrases, while also getting more errors than in the baseline.
The long-form sets are left to the pooled rate and those last two checks: a
single file there moves by many points whenever one decode flips, which HRX
builds do between identical runs. The report also shows how
many transcripts changed. Even a numerically equivalent change, such as a
different FFT, flips a few borderline decodes, so a few changed files and
small per-set moves are expected; --max-drift is for builds that should
Expand Down Expand Up @@ -80,6 +84,8 @@
SAME_TEXT_MAX_WER = 0.02
REPEAT_NGRAM = 6
RUNAWAY_RATIO = 1.5
# Smaller sets are judged through the pooled rate only; see the docstring.
SET_CHECK_MIN_FILES = 10
ARABIC_DIACRITICS = re.compile("[\u064b-\u065f\u0670\u06d6-\u06ed]")


Expand Down Expand Up @@ -159,9 +165,10 @@ def add_arguments(parser) -> None:
parser.add_argument("--tolerance", type=float, default=1.0,
help="Allowed rise in the pooled error rate, in points (default: 1.0).")
parser.add_argument("--set-tolerance", type=float, default=5.0,
help="Allowed rise in any one set's error rate, in points "
"(default: 5.0). Sets are ten files, so one flipped "
"decode moves them by a point or more.")
help="Allowed rise in the error rate of any set of "
f"{SET_CHECK_MIN_FILES} or more files, in points "
"(default: 5.0). One flipped decode moves a ten-file "
"set by a point or more.")
parser.add_argument("--max-drift", type=float, default=None,
help="Fail a set whose transcripts differ from the baseline's "
"by more than this fraction, for builds meant to change "
Expand Down Expand Up @@ -567,17 +574,27 @@ def compare(self, writer, model: str) -> None:
self.check(writer, model, "eval", "pooled error rate no worse than the baseline",
overall["error_rate"] - base_overall["error_rate"] <= tolerance,
f"{pct(overall['error_rate'])} vs {pct(base_overall['error_rate'])}")
self.check(writer, model, "eval", "no more runaways than the baseline",
overall["runaways"] <= base_overall["runaways"],
f"{overall['runaways']} vs {base_overall['runaways']}")
# A file that crosses the runaway length while making no more
# errors, for instance by also transcribing what the baseline
# dropped, is not a regression.
new_runaways = sorted(
f for f, r in self.results["items"].items()
if r.get("runaway") and f in base_items
and not base_items[f].get("runaway")
and r["edits"] > base_items[f]["edits"])
self.check(writer, model, "eval", "no new runaway transcripts",
not new_runaways,
", ".join(new_runaways) if new_runaways else
f"{overall['runaways']} runaways, baseline {base_overall['runaways']}")
for set_name, current in self.results["sets"].items():
base = self.baseline["sets"].get(set_name)
if not base:
continue
delta = current["error_rate"] - base["error_rate"]
self.check(writer, model, f"eval {set_name}",
"error rate no worse than the baseline", delta <= set_tolerance,
f"{pct(current['error_rate'])} vs {pct(base['error_rate'])}")
if current["files"] >= SET_CHECK_MIN_FILES:
self.check(writer, model, f"eval {set_name}",
"error rate no worse than the baseline", delta <= set_tolerance,
f"{pct(current['error_rate'])} vs {pct(base['error_rate'])}")
compared = [(f, r) for f, r in self.results["items"].items()
if r["set"] == set_name and f in base_items]
drift = pooled([error_rate(base_items[f]["text"], r["text"], r["language"],
Expand All @@ -596,10 +613,11 @@ def compare(self, writer, model: str) -> None:
f"baseline {base['rtf']:.3f}")
for name, counts in self.results["repeats"].items():
base = self.baseline.get("repeats", {}).get(name)
if base:
if base and name in base_items:
more_errors = self.results["items"][name]["edits"] > base_items[name]["edits"]
self.check(writer, model, f"eval {name}",
"no more repeated phrases than the baseline",
counts["text"] <= base["text"],
counts["text"] <= base["text"] or not more_errors,
f"{counts['text']} vs {base['text']}")
elif self.expected:
limit = self.expected.get("overall")
Expand All @@ -609,7 +627,7 @@ def compare(self, writer, model: str) -> None:
f"{pct(overall['error_rate'])} vs expected {pct(limit)}")
for set_name, current in self.results["sets"].items():
limit = self.expected.get("sets", {}).get(set_name)
if limit is not None:
if limit is not None and current["files"] >= SET_CHECK_MIN_FILES:
self.check(writer, model, f"eval {set_name}",
"error rate within the expected value",
current["error_rate"] <= limit + set_tolerance,
Expand Down
Loading