diff --git a/model_validation/README.md b/model_validation/README.md index ae26a2c0..2a6bf353 100644 --- a/model_validation/README.md +++ b/model_validation/README.md @@ -532,6 +532,13 @@ label falls inside the audio output's timespan, a relationship between two outputs that no single-output rule can express, and it opts out when the model has no audio output or no label output. +The model-specific validators check MERIT's similarity scores and MuQ's ranked +candidates (`count`). Run their local regression checks with: + +```bash +python -m unittest discover -s model_validation/tests -v +``` + Add custom validators to [validators.py](src/validators.py), registered with the `@validator` decorator under the name test cases will reference: diff --git a/model_validation/config.yml b/model_validation/config.yml index ccde4b2a..dd171bce 100644 --- a/model_validation/config.yml +++ b/model_validation/config.yml @@ -218,4 +218,103 @@ include_extra: [] common_test_cases: [] -overrides: {} +overrides: + + # ---- Music similarity --------------------------------------------------- + + teamup-tech/MERIT: + # Both inputs are required to be between 5 and 10 seconds. + process_timeout: 150 + synthesized_inputs: + audio: + duration: 6.0 + test_cases: + - name: default + expect: + "Similarity Results": + ext: .json + validators: + merit_scores: {} + + teamup-tech/MuQ-MuLan: + # The Space accepts 10-60 second clips and up to eight descriptions. + process_timeout: 300 + synthesized_inputs: + audio: + duration: 12.0 + test_cases: + - name: default + controls: + "Candidate Descriptions": "a solo piano melody" + expect: + "Similarity Ranking": + ext: .json + validators: + muq_ranking: + count: 1 + + # ---- Symbolic music infilling ------------------------------------------ + + teamup-tech/MIDI-GPT: + # 20 × 0.5s = 5 bars at 120 BPM; the Space needs at least 4. + process_timeout: 150 + synthesized_inputs: + midi: + num_notes: 20 + note_duration: 0.5 + instrument: 0 + channel: 0 + test_cases: + - name: default + expect: + "Rewritten MIDI": + ext: .mid + min_notes: 1 + min_duration: 8.0 + + # ---- Audio dynamics / style transfer ------------------------------------ + + teamup-tech/DeepAFx-ST: + # The convolutional encoder needs at least ~5s of audio at 24kHz. + process_timeout: 150 + synthesized_inputs: + audio: + duration: 8.0 + test_cases: + - name: default + expect: + "Styled Output": + ext: .wav + num_channels: 1 + sample_rate: 24000 + min_duration: 7.0 + max_duration: 9.0 + min_rms_db: -60 + + # ---- Audio editing / restyling ----------------------------------------- + + teamup-tech/Stable-Audio-3: + # Restyle and Inpaint both preserve the six-second input duration. + process_timeout: 240 + synthesized_inputs: + audio: + duration: 6.0 + test_cases: + - name: restyle-default + controls: + "Prompt": "a warm synthwave groove" + expect: + "Edited Audio": + ext: .wav + bit_depth: 16 + min_duration: 5.8 + max_duration: 6.2 + min_rms_db: -60 + + # ---- Music structure analysis -------------------------------------------- + + teamup-tech/SongFormer: + # The model requires at least 5 seconds of audio. + synthesized_inputs: + audio: + duration: 6.0 diff --git a/model_validation/src/validators.py b/model_validation/src/validators.py index 1f146e92..7b26f6ed 100644 --- a/model_validation/src/validators.py +++ b/model_validation/src/validators.py @@ -31,6 +31,10 @@ the output at fault and say what was wrong with it. """ +import json +import math +from pathlib import Path + from audio import read_audio_props @@ -72,6 +76,49 @@ def register(fn): return register +def json_file(outputs, label): + """Read a model's JSON file output.""" + with Path(outputs[label]).open(encoding="utf-8") as stream: + result = json.load(stream) + assert isinstance(result, dict), f"'{label}' must contain a JSON object" + return result + + +def finite_number(value, label): + """Require a numeric prediction rather than null, a string, or NaN.""" + assert type(value) in (int, float) and math.isfinite(value), \ + f"'{label}' must be a finite number, got {value!r}" + + +@validator("merit_scores") +def merit_scores(outputs, controls, params): + """Check the three similarity scores in MERIT's result file.""" + result = json_file(outputs, "Similarity Results") + for name in ("melody", "rhythm", "timbre"): + value = result["scores"][name] + finite_number(value, name) + assert -1.000001 <= value <= 1.000001, \ + f"MERIT {name} cosine similarity is outside [-1, 1]: {value}" + + +@validator("muq_ranking") +def muq_ranking(outputs, controls, params): + """Check candidate count, ranks, and descending similarity scores.""" + result = json_file(outputs, "Similarity Ranking") + rows = result["results"] + assert isinstance(rows, list) and len(rows) == params["count"], \ + f"MuQ must return {params['count']} ranked candidates" + previous = math.inf + for rank, row in enumerate(rows, start=1): + assert row["rank"] == rank, f"MuQ rank must be {rank}" + assert isinstance(row["description"], str) and row["description"].strip(), \ + "MuQ candidate description must be nonempty" + score = row["similarity"] + finite_number(score, "similarity") + assert score <= previous, "MuQ candidates must be sorted by similarity" + previous = score + + @validator("labels_within_audio") def labels_within_audio(outputs, controls, params): """ diff --git a/model_validation/tests/test_model_outputs.py b/model_validation/tests/test_model_outputs.py new file mode 100644 index 00000000..712daf3a --- /dev/null +++ b/model_validation/tests/test_model_outputs.py @@ -0,0 +1,50 @@ +import json +from pathlib import Path +import sys +import tempfile +import unittest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) +from validators import VALIDATORS + + +class ModelOutputsTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.path = Path(self.temp.name) / "result.json" + + def check_file(self, name, label, data, params=None): + self.path.write_text(json.dumps(data), encoding="utf-8") + VALIDATORS[name]({label: str(self.path)}, {}, params or {}) + + def test_merit_scores(self): + self.check_file("merit_scores", "Similarity Results", + {"scores": {"melody": 1.0, "rhythm": 0.9, "timbre": -0.2}}) + with self.assertRaises(KeyError): + self.check_file("merit_scores", "Similarity Results", {"scores": {}}) + for value in (None, True, "0.9", float("nan"), 2.0): + with self.subTest(value=value), self.assertRaises(AssertionError): + self.check_file("merit_scores", "Similarity Results", + {"scores": {"melody": value}}) + + def test_invalid_json_is_rejected(self): + self.path.write_text("not json" * 30) + for name, label in (("merit_scores", "Similarity Results"), + ("muq_ranking", "Similarity Ranking")): + with self.subTest(name=name), self.assertRaises(json.JSONDecodeError): + VALIDATORS[name]({label: str(self.path)}, {}, {}) + + def test_muq_ranking(self): + rows = [{"rank": 1, "description": "piano", "similarity": 0.8}, + {"rank": 2, "description": "drums", "similarity": 0.2}] + self.check_file("muq_ranking", "Similarity Ranking", {"results": rows}, {"count": 2}) + with self.assertRaises(AssertionError): + self.check_file("muq_ranking", "Similarity Ranking", {"results": rows}, {"count": 3}) + rows[1]["similarity"] = 0.9 + with self.assertRaises(AssertionError): + self.check_file("muq_ranking", "Similarity Ranking", {"results": rows}, {"count": 2}) + + +if __name__ == "__main__": + unittest.main()