Added test configs and validators for CI-failing models

model_validation/README.md

@@ -532,6 +532,13 @@ label falls inside the audio output's timespan, a relationship between two
 outputs that no single-output rule can express, and it opts out when the model
 has no audio output or no label output.
 
+The model-specific validators check MERIT's similarity scores and MuQ's ranked
+candidates (`count`). Run their local regression checks with:
+
+```bash
+python -m unittest discover -s model_validation/tests -v
+```
+
 Add custom validators to [validators.py](src/validators.py), registered with the `@validator` decorator
 under the name test cases will reference:
 

model_validation/config.yml

@@ -218,4 +218,103 @@ include_extra: []
 
 common_test_cases: []
 
-overrides: {}
+overrides:
+
+  # ---- Music similarity ---------------------------------------------------
+
+  teamup-tech/MERIT:
+    # Both inputs are required to be between 5 and 10 seconds.
+    process_timeout: 150
+    synthesized_inputs:
+      audio:
+        duration: 6.0
+    test_cases:
+      - name: default
+        expect:
+          "Similarity Results":
+            ext: .json
+        validators:
+          merit_scores: {}
+
+  teamup-tech/MuQ-MuLan:
+    # The Space accepts 10-60 second clips and up to eight descriptions.
+    process_timeout: 300
+    synthesized_inputs:
+      audio:
+        duration: 12.0
+    test_cases:
+      - name: default
+        controls:
+          "Candidate Descriptions": "a solo piano melody"
+        expect:
+          "Similarity Ranking":
+            ext: .json
+        validators:
+          muq_ranking:
+            count: 1
+
+  # ---- Symbolic music infilling ------------------------------------------
+
+  teamup-tech/MIDI-GPT:
+    # 20 × 0.5s = 5 bars at 120 BPM; the Space needs at least 4.
+    process_timeout: 150
+    synthesized_inputs:
+      midi:
+        num_notes: 20
+        note_duration: 0.5
+        instrument: 0
+        channel: 0
+    test_cases:
+      - name: default
+        expect:
+          "Rewritten MIDI":
+            ext: .mid
+            min_notes: 1
+            min_duration: 8.0
+
+  # ---- Audio dynamics / style transfer ------------------------------------
+
+  teamup-tech/DeepAFx-ST:
+    # The convolutional encoder needs at least ~5s of audio at 24kHz.
+    process_timeout: 150
+    synthesized_inputs:
+      audio:
+        duration: 8.0
+    test_cases:
+      - name: default
+        expect:
+          "Styled Output":
+            ext: .wav
+            num_channels: 1
+            sample_rate: 24000
+            min_duration: 7.0
+            max_duration: 9.0
+            min_rms_db: -60
+
+  # ---- Audio editing / restyling -----------------------------------------
+
+  teamup-tech/Stable-Audio-3:
+    # Restyle and Inpaint both preserve the six-second input duration.
+    process_timeout: 240
+    synthesized_inputs:
+      audio:
+        duration: 6.0
+    test_cases:
+      - name: restyle-default
+        controls:
+          "Prompt": "a warm synthwave groove"
+        expect:
+          "Edited Audio":
+            ext: .wav
+            bit_depth: 16
+            min_duration: 5.8
+            max_duration: 6.2
+            min_rms_db: -60
+
+  # ---- Music structure analysis --------------------------------------------
+
+  teamup-tech/SongFormer:
+    # The model requires at least 5 seconds of audio.
+    synthesized_inputs:
+      audio:
+        duration: 6.0

model_validation/src/validators.py

@@ -31,6 +31,10 @@
 the output at fault and say what was wrong with it.
 """
 
+import json
+import math
+from pathlib import Path
+
 from audio import read_audio_props
 
 
@@ -72,6 +76,49 @@ def register(fn):
     return register
 
 
+def json_file(outputs, label):
+    """Read a model's JSON file output."""
+    with Path(outputs[label]).open(encoding="utf-8") as stream:
+        result = json.load(stream)
+    assert isinstance(result, dict), f"'{label}' must contain a JSON object"
+    return result
+
+
+def finite_number(value, label):
+    """Require a numeric prediction rather than null, a string, or NaN."""
+    assert type(value) in (int, float) and math.isfinite(value), \
+        f"'{label}' must be a finite number, got {value!r}"
+
+
+@validator("merit_scores")
+def merit_scores(outputs, controls, params):
+    """Check the three similarity scores in MERIT's result file."""
+    result = json_file(outputs, "Similarity Results")
+    for name in ("melody", "rhythm", "timbre"):
+        value = result["scores"][name]
+        finite_number(value, name)
+        assert -1.000001 <= value <= 1.000001, \
+            f"MERIT {name} cosine similarity is outside [-1, 1]: {value}"
+
+
+@validator("muq_ranking")
+def muq_ranking(outputs, controls, params):
+    """Check candidate count, ranks, and descending similarity scores."""
+    result = json_file(outputs, "Similarity Ranking")
+    rows = result["results"]
+    assert isinstance(rows, list) and len(rows) == params["count"], \
+        f"MuQ must return {params['count']} ranked candidates"
+    previous = math.inf
+    for rank, row in enumerate(rows, start=1):
+        assert row["rank"] == rank, f"MuQ rank must be {rank}"
+        assert isinstance(row["description"], str) and row["description"].strip(), \
+            "MuQ candidate description must be nonempty"
+        score = row["similarity"]
+        finite_number(score, "similarity")
+        assert score <= previous, "MuQ candidates must be sorted by similarity"
+        previous = score
+
+
 @validator("labels_within_audio")
 def labels_within_audio(outputs, controls, params):
     """

model_validation/tests/test_model_outputs.py

@@ -0,0 +1,50 @@
+import json
+from pathlib import Path
+import sys
+import tempfile
+import unittest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
+from validators import VALIDATORS
+
+
+class ModelOutputsTest(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.path = Path(self.temp.name) / "result.json"
+
+    def check_file(self, name, label, data, params=None):
+        self.path.write_text(json.dumps(data), encoding="utf-8")
+        VALIDATORS[name]({label: str(self.path)}, {}, params or {})
+
+    def test_merit_scores(self):
+        self.check_file("merit_scores", "Similarity Results",
+                        {"scores": {"melody": 1.0, "rhythm": 0.9, "timbre": -0.2}})
+        with self.assertRaises(KeyError):
+            self.check_file("merit_scores", "Similarity Results", {"scores": {}})
+        for value in (None, True, "0.9", float("nan"), 2.0):
+            with self.subTest(value=value), self.assertRaises(AssertionError):
+                self.check_file("merit_scores", "Similarity Results",
+                                {"scores": {"melody": value}})
+
+    def test_invalid_json_is_rejected(self):
+        self.path.write_text("not json" * 30)
+        for name, label in (("merit_scores", "Similarity Results"),
+                            ("muq_ranking", "Similarity Ranking")):
+            with self.subTest(name=name), self.assertRaises(json.JSONDecodeError):
+                VALIDATORS[name]({label: str(self.path)}, {}, {})
+
+    def test_muq_ranking(self):
+        rows = [{"rank": 1, "description": "piano", "similarity": 0.8},
+                {"rank": 2, "description": "drums", "similarity": 0.2}]
+        self.check_file("muq_ranking", "Similarity Ranking", {"results": rows}, {"count": 2})
+        with self.assertRaises(AssertionError):
+            self.check_file("muq_ranking", "Similarity Ranking", {"results": rows}, {"count": 3})
+        rows[1]["similarity"] = 0.9
+        with self.assertRaises(AssertionError):
+            self.check_file("muq_ranking", "Similarity Ranking", {"results": rows}, {"count": 2})
+
+
+if __name__ == "__main__":
+    unittest.main()