From ce1308631da613de61130d17955d33aafac2f6e4 Mon Sep 17 00:00:00 2001 From: LING Date: Wed, 16 Sep 2026 23:49:19 +0100 Subject: [PATCH 1/5] Added test cases and validators for 11 remote models. --- model_validation/README.md | 10 + model_validation/config.yml | 285 ++++++++++++++++++- model_validation/src/validators.py | 92 ++++++ model_validation/test_data/guitar_riff.mid | Bin 0 -> 164 bytes model_validation/tests/test_model_outputs.py | 105 +++++++ 5 files changed, 491 insertions(+), 1 deletion(-) create mode 100644 model_validation/test_data/guitar_riff.mid create mode 100644 model_validation/tests/test_model_outputs.py diff --git a/model_validation/README.md b/model_validation/README.md index ae26a2c0..c73f83ac 100644 --- a/model_validation/README.md +++ b/model_validation/README.md @@ -532,6 +532,16 @@ label falls inside the audio output's timespan, a relationship between two outputs that no single-output rule can express, and it opts out when the model has no audio output or no label output. +The model-specific validators check MERIT's similarity scores, MuQ's ranked +candidates (`count`), Music2Emo's mood probabilities (`threshold`), AudioBox's +four aesthetic scores, and Chord Extractor's ordered timestamps. +The file-based results are parsed as JSON. Empty mood and chord lists are valid. +Run their local regression checks with: + +```bash +python -m unittest discover -s model_validation/tests -v +``` + Add custom validators to [validators.py](src/validators.py), registered with the `@validator` decorator under the name test cases will reference: diff --git a/model_validation/config.yml b/model_validation/config.yml index ccde4b2a..8cf7bbce 100644 --- a/model_validation/config.yml +++ b/model_validation/config.yml @@ -218,4 +218,287 @@ include_extra: [] common_test_cases: [] -overrides: {} +overrides: + + # ---- Singing voice transcription ---------------------------------------- + + teamup-tech/GAME: + # Synthetic tones may produce an empty transcription. + process_timeout: 150 + test_cases: + - name: default + expect: + "Transcribed MIDI": + ext: .mid + min_notes: 0 + "Detailed Note Data": + ext: .csv + min_bytes: 20 + - name: english-four-step-8s + synthesized_inputs: + audio: + duration: 8.0 + controls: + "Language": "English" + "D3PM Steps": "4" + expect: + "Transcribed MIDI": + ext: .mid + min_notes: 0 + "Detailed Note Data": + ext: .csv + min_bytes: 20 + + # ---- Music similarity --------------------------------------------------- + + teamup-tech/MERIT: + # Both inputs are required to be between 5 and 10 seconds. + process_timeout: 150 + synthesized_inputs: + audio: + duration: 6.0 + test_cases: + - name: default + expect: + "Similarity Results": + ext: .json + validators: + merit_scores: {} + + teamup-tech/MuQ-MuLan: + # The Space accepts 10-60 second clips and up to eight descriptions. + process_timeout: 300 + synthesized_inputs: + audio: + duration: 12.0 + test_cases: + - name: default + controls: + "Candidate Descriptions": "a solo piano melody" + expect: + "Similarity Ranking": + ext: .json + validators: + muq_ranking: + count: 1 + - name: three-candidate-ranking + controls: + "Candidate Descriptions": "calm solo piano\nenergetic rock band\nambient synthesizer pad" + expect: + "Similarity Ranking": + ext: .json + validators: + muq_ranking: + count: 3 + + # ---- Music emotion recognition ----------------------------------------- + + teamup-tech/Music2Emo: + process_timeout: 150 + test_cases: + - name: default + expect: + "Emotion Analysis": + ext: .json + validators: + music2emo_analysis: + threshold: 0.5 + - name: minimum-mood-threshold + controls: + "Mood Threshold": 0.1 + expect: + "Emotion Analysis": + ext: .json + validators: + music2emo_analysis: + threshold: 0.1 + + # ---- Music generation --------------------------------------------------- + + teamup-tech/magenta-rt: + process_timeout: 75 + test_cases: + - name: default + expect: + "Generated Music": + ext: .wav + num_channels: 2 + sample_rate: 48000 + bit_depth: 16 + min_duration: 3.9 + max_duration: 4.1 + min_rms_db: -60 + - name: seeded-two-second-piano + controls: + "Music Prompt": "a calm solo piano melody" + "Duration (seconds)": "2" + "Temperature": 0.7 + "Top-k": 25 + "Seed": 42 + expect: + "Generated Music": + ext: .wav + num_channels: 2 + sample_rate: 48000 + bit_depth: 16 + min_duration: 1.9 + max_duration: 2.1 + min_rms_db: -60 + + # ---- Symbolic music infilling ------------------------------------------ + + teamup-tech/MIDI-GPT: + # 20 × 0.5s = 5 bars at 120 BPM; the Space needs at least 4. + process_timeout: 150 + synthesized_inputs: + midi: + num_notes: 20 + note_duration: 0.5 + instrument: 0 + channel: 0 + test_cases: + - name: default + expect: + "Rewritten MIDI": + ext: .mid + min_notes: 1 + min_duration: 8.0 + - name: seeded-four-bar-rewrite + controls: + "Number of Bars": "4" + "Temperature": 0.8 + "Top-p": 0.9 + "Seed": 42 + "Maximum Polyphony": 4 + expect: + "Rewritten MIDI": + ext: .mid + min_notes: 1 + min_duration: 8.0 + + # ---- MIDI-to-audio synthesis ------------------------------------------- + + teamup-tech/DDSP-Guitar: + # Program 24 is General MIDI nylon-string guitar. + process_timeout: 150 + synthesized_inputs: + midi: + num_notes: 12 + note_duration: 0.4 + instrument: 24 + channel: 0 + test_cases: + - name: synthesized-guitar-program + expect: + "Synthesized Guitar": + ext: .wav + min_duration: 4.0 + min_rms_db: -60 + - name: real-guitar-midi + files: + "MIDI File": test_data/guitar_riff.mid + expect: + "Synthesized Guitar": + ext: .wav + min_duration: 4.0 + min_rms_db: -60 + - name: real-guitar-five-second-cap + files: + "MIDI File": test_data/guitar_riff.mid + controls: + "Max Duration (seconds)": "5" + expect: + "Synthesized Guitar": + ext: .wav + min_duration: 4.0 + max_duration: 5.1 + min_rms_db: -60 + + # ---- Audio editing / restyling ----------------------------------------- + + teamup-tech/Stable-Audio-3: + # Restyle and Inpaint both preserve the six-second input duration. + process_timeout: 240 + synthesized_inputs: + audio: + duration: 6.0 + test_cases: + - name: restyle-default + controls: + "Prompt": "a warm synthwave groove" + expect: + "Edited Audio": + ext: .wav + bit_depth: 16 + min_duration: 5.8 + max_duration: 6.2 + min_rms_db: -60 + - name: inpaint-middle + controls: + "Prompt": "jazz piano with brushed drums" + "Edit Mode": "Inpaint" + "Edit Start (seconds)": 1.0 + "Edit End (seconds)": 3.0 + "Seed": 42 + expect: + "Edited Audio": + ext: .wav + bit_depth: 16 + min_duration: 5.8 + max_duration: 6.2 + min_rms_db: -60 + + # ---- CPU audio analysis and speech ------------------------------------- + + teamup-tech/audiobox-aesthetics: + process_timeout: 150 + synthesized_inputs: + audio: + duration: 4.0 + test_cases: + - name: default + validators: + aesthetics_scores: {} + + teamup-tech/pocket-tts: + process_timeout: 90 + test_cases: + - name: default-voice + controls: + "English Text": "The quick brown fox jumps over the lazy dog." + "Voice": "alba" + expect: + "Synthesized Speech": + ext: .wav + num_channels: 1 + sample_rate: 24000 + bit_depth: 16 + min_duration: 0.2 + min_rms_db: -60 + - name: alternate-voice + controls: + "English Text": "Hello from HARP." + "Voice": "george" + expect: + "Synthesized Speech": + ext: .wav + num_channels: 1 + sample_rate: 24000 + bit_depth: 16 + min_duration: 0.2 + min_rms_db: -60 + + teamup-tech/chord-extractor: + process_timeout: 90 + synthesized_inputs: + audio: + duration: 4.0 + test_cases: + - name: default-roll-on + validators: + chord_sequence: {} + - name: minimum-roll-on + controls: + "Spectral Roll-on": 0 + validators: + chord_sequence: {} diff --git a/model_validation/src/validators.py b/model_validation/src/validators.py index 1f146e92..26e91ba0 100644 --- a/model_validation/src/validators.py +++ b/model_validation/src/validators.py @@ -31,6 +31,10 @@ the output at fault and say what was wrong with it. """ +import json +import math +from pathlib import Path + from audio import read_audio_props @@ -72,6 +76,94 @@ def register(fn): return register +def json_file(outputs, label): + """Read a model's JSON file output.""" + with Path(outputs[label]).open(encoding="utf-8") as stream: + result = json.load(stream) + assert isinstance(result, dict), f"'{label}' must contain a JSON object" + return result + + +def finite_number(value, label): + """Require a numeric prediction rather than null, a string, or NaN.""" + assert type(value) in (int, float) and math.isfinite(value), \ + f"'{label}' must be a finite number, got {value!r}" + + +@validator("merit_scores") +def merit_scores(outputs, controls, params): + """Check the three similarity scores in MERIT's result file.""" + result = json_file(outputs, "Similarity Results") + for name in ("melody", "rhythm", "timbre"): + value = result["scores"][name] + finite_number(value, name) + assert -1.000001 <= value <= 1.000001, \ + f"MERIT {name} cosine similarity is outside [-1, 1]: {value}" + + +@validator("muq_ranking") +def muq_ranking(outputs, controls, params): + """Check candidate count, ranks, and descending similarity scores.""" + result = json_file(outputs, "Similarity Ranking") + rows = result["results"] + assert isinstance(rows, list) and len(rows) == params["count"], \ + f"MuQ must return {params['count']} ranked candidates" + previous = math.inf + for rank, row in enumerate(rows, start=1): + assert row["rank"] == rank, f"MuQ rank must be {rank}" + assert isinstance(row["description"], str) and row["description"].strip(), \ + "MuQ candidate description must be nonempty" + score = row["similarity"] + finite_number(score, "similarity") + assert score <= previous, "MuQ candidates must be sorted by similarity" + previous = score + + +@validator("music2emo_analysis") +def music2emo_analysis(outputs, controls, params): + """Check emotion dimensions and thresholded mood probabilities.""" + result = json_file(outputs, "Emotion Analysis") + for name in ("valence", "arousal"): + finite_number(result[name], name) + assert result["threshold"] == params["threshold"], "Unexpected mood threshold" + assert isinstance(result["moods"], list), "Music2Emo moods must be a list" + for mood in result["moods"]: + assert isinstance(mood["label"], str) and mood["label"], "Missing mood label" + score = mood["probability"] + finite_number(score, "mood probability") + # The Space rounds probabilities to four decimal places. + assert params["threshold"] - 0.0001 <= score <= 1, \ + f"Mood probability {score} does not meet the threshold" + + +@validator("aesthetics_scores") +def aesthetics_scores(outputs, controls, params): + """Require all four predicted aesthetic scores.""" + result = outputs["Aesthetic Scores"] + assert isinstance(result, dict), "Aesthetic Scores must be an object" + for name in ("content_enjoyment", "content_usefulness", + "production_complexity", "production_quality"): + finite_number(result["scores"][name], name) + + +@validator("chord_sequence") +def chord_sequence(outputs, controls, params): + """Check chord labels and nonnegative, ordered analysis-frame timestamps.""" + result = outputs["Chord Sequence"] + assert isinstance(result, dict), "Chord Sequence must be an object" + rows = result["chords"] + assert isinstance(rows, list), "Chord Sequence chords must be a list" + previous = 0.0 + for row in rows: + timestamp = row["timestamp_seconds"] + finite_number(timestamp, "chord timestamp") + # Chordino's final analysis frame can extend beyond the input audio. + assert previous <= timestamp, \ + f"Chord timestamp {timestamp} is negative or out of order" + assert isinstance(row["chord"], str) and row["chord"], "Missing chord label" + previous = timestamp + + @validator("labels_within_audio") def labels_within_audio(outputs, controls, params): """ diff --git a/model_validation/test_data/guitar_riff.mid b/model_validation/test_data/guitar_riff.mid new file mode 100644 index 0000000000000000000000000000000000000000..d848150c83c10c99d172569e52ed0136c953db56 GIT binary patch literal 164 zcmeYb$w*;fU|?flWME``;2Tnu4dnGP9FSy~pb^!a(4gVKFhM)2v7kX4$j}8bbb$ Date: Thu, 17 Sep 2026 18:57:34 +0100 Subject: [PATCH 2/5] Added test cases for DDSP-Piano, micro-TCN, DeepAFx-ST, and MelBandRoFormer. --- model_validation/config.yml | 119 +++++++++++++++++++ model_validation/src/validators.py | 8 ++ model_validation/test_data/piano_chords.mid | Bin 0 -> 156 bytes model_validation/tests/test_model_outputs.py | 10 ++ 4 files changed, 137 insertions(+) create mode 100644 model_validation/test_data/piano_chords.mid diff --git a/model_validation/config.yml b/model_validation/config.yml index 8cf7bbce..d93b2f1a 100644 --- a/model_validation/config.yml +++ b/model_validation/config.yml @@ -414,8 +414,102 @@ overrides: max_duration: 5.1 min_rms_db: -60 + teamup-tech/DDSP-Piano: + # Any MIDI input works; all notes are rendered as piano. + process_timeout: 300 + synthesized_inputs: + midi: + num_notes: 8 + note_duration: 0.5 + test_cases: + - name: default + expect: + "Synthesized Piano": + ext: .wav + sample_rate: 24000 + min_duration: 3.0 + min_rms_db: -60 + - name: real-piano-midi + files: + "MIDI File": test_data/piano_chords.mid + expect: + "Synthesized Piano": + ext: .wav + sample_rate: 24000 + min_duration: 4.0 + min_rms_db: -40 + - name: bright-timbre-five-seconds + files: + "MIDI File": test_data/piano_chords.mid + controls: + "Piano Timbre": "Piano 0 (Bright)" + "Max Duration (seconds)": "5" + expect: + "Synthesized Piano": + ext: .wav + sample_rate: 24000 + min_duration: 4.0 + max_duration: 5.5 + min_rms_db: -60 + - name: warm-timbre-long-duration + files: + "MIDI File": test_data/piano_chords.mid + controls: + "Piano Timbre": "Piano 9 (Warm)" + "Max Duration (seconds)": "10" + expect: + "Synthesized Piano": + ext: .wav + sample_rate: 24000 + min_duration: 7.0 + min_rms_db: -40 + + # ---- Audio dynamics processing ----------------------------------------- + + teamup-tech/micro-TCN: + # Neural LA-2A compressor/limiter emulation. Valid convolution shortens + # output by ~0.3s at 44.1kHz. + process_timeout: 90 + test_cases: + - name: default + expect: + "Processed Audio": + ext: .wav + min_duration: 1.5 + min_rms_db: -60 + - name: limiter-heavy-reduction + controls: + "Mode": "Limit" + "Peak Reduction": 0.8 + expect: + "Processed Audio": + ext: .wav + min_duration: 1.5 + min_rms_db: -60 + # ---- Audio editing / restyling ----------------------------------------- + teamup-tech/DeepAFx-ST: + process_timeout: 150 + synthesized_inputs: + audio: + duration: 8.0 + test_cases: + - name: default + expect: + "Styled Output": + ext: .wav + num_channels: 1 + sample_rate: 24000 + min_duration: 7.0 + max_duration: 9.0 + min_rms_db: -60 + "DSP Parameters": + ext: .json + min_bytes: 50 + validators: + deepafx_st_params: {} + teamup-tech/Stable-Audio-3: # Restyle and Inpaint both preserve the six-second input duration. process_timeout: 240 @@ -502,3 +596,28 @@ overrides: "Spectral Roll-on": 0 validators: chord_sequence: {} + + # ---- Music source separation --------------------------------------------- + + teamup-tech/MelBandRoFormer: + process_timeout: 300 + test_cases: + - name: default + expect: + "Vocals": + ext: .wav + "Instrumental": + ext: .wav + min_rms_db: -60 + - name: ten-second-chunked + synthesized_inputs: + audio: + duration: 10.0 + expect: + "Vocals": + min_duration: 9.5 + max_duration: 10.5 + "Instrumental": + min_duration: 9.5 + max_duration: 10.5 + min_rms_db: -60 diff --git a/model_validation/src/validators.py b/model_validation/src/validators.py index 26e91ba0..924c6eed 100644 --- a/model_validation/src/validators.py +++ b/model_validation/src/validators.py @@ -216,3 +216,11 @@ def labels_within_audio(outputs, controls, params): if not checked: raise ValidatorNotApplicable("no pyharp LabelList output to check") + + +@validator("deepafx_st_params") +def deepafx_st_params(outputs, controls, params): + """Check predicted EQ and compressor parameters.""" + result = json_file(outputs, "DSP Parameters") + for i, v in enumerate(result["raw_parameters"]): + finite_number(v, f"raw_parameters[{i}]") diff --git a/model_validation/test_data/piano_chords.mid b/model_validation/test_data/piano_chords.mid new file mode 100644 index 0000000000000000000000000000000000000000..54ed5a3dd3292e9aa87764bba1c6231b9a83ce0c GIT binary patch literal 156 zcmeYb$w*;fU|?flWME``;2Tnu4dk^k{10SiU#P(FpP4lvGchlp;Q#}}1e<{7ga#WR z!vVx_05Y6G3}=v36oW$)gL72714sc-3M6RY*3Zyj0Ax6ZGPs5^c!UCF9T^y085lf3 bvhfTy@eB^}49@Wk9`O*BaJ@hU|MeLF`c@>X literal 0 HcmV?d00001 diff --git a/model_validation/tests/test_model_outputs.py b/model_validation/tests/test_model_outputs.py index e5a2e5b6..2832fb6c 100644 --- a/model_validation/tests/test_model_outputs.py +++ b/model_validation/tests/test_model_outputs.py @@ -100,6 +100,16 @@ def test_game_midi_parses_without_requiring_notes(self): with self.assertRaises(AssertionError): check_expectations("Transcribed MIDI", "midi_track", str(path), {"min_notes": 0}) + def test_deepafx_st_params(self): + self.check_file("deepafx_st_params", "DSP Parameters", + {"raw_parameters": [0.5, 0.3, -0.1, 0.8]}) + with self.assertRaises(KeyError): + self.check_file("deepafx_st_params", "DSP Parameters", {}) + for value in (None, True, "0.9", float("nan")): + with self.subTest(value=value), self.assertRaises(AssertionError): + self.check_file("deepafx_st_params", "DSP Parameters", + {"raw_parameters": [value]}) + if __name__ == "__main__": unittest.main() From 9de9086aa7212730a7d6f2d54068aff53259c34e Mon Sep 17 00:00:00 2001 From: LING Date: Fri, 18 Sep 2026 03:39:08 +0100 Subject: [PATCH 3/5] Removed redundant test cases covered by the framework defaults. --- model_validation/README.md | 7 +- model_validation/config.yml | 330 ------------------- model_validation/src/validators.py | 53 --- model_validation/test_data/guitar_riff.mid | Bin 164 -> 0 bytes model_validation/test_data/piano_chords.mid | Bin 156 -> 0 bytes model_validation/tests/test_model_outputs.py | 67 +--- 6 files changed, 3 insertions(+), 454 deletions(-) delete mode 100644 model_validation/test_data/guitar_riff.mid delete mode 100644 model_validation/test_data/piano_chords.mid diff --git a/model_validation/README.md b/model_validation/README.md index c73f83ac..2a6bf353 100644 --- a/model_validation/README.md +++ b/model_validation/README.md @@ -532,11 +532,8 @@ label falls inside the audio output's timespan, a relationship between two outputs that no single-output rule can express, and it opts out when the model has no audio output or no label output. -The model-specific validators check MERIT's similarity scores, MuQ's ranked -candidates (`count`), Music2Emo's mood probabilities (`threshold`), AudioBox's -four aesthetic scores, and Chord Extractor's ordered timestamps. -The file-based results are parsed as JSON. Empty mood and chord lists are valid. -Run their local regression checks with: +The model-specific validators check MERIT's similarity scores and MuQ's ranked +candidates (`count`). Run their local regression checks with: ```bash python -m unittest discover -s model_validation/tests -v diff --git a/model_validation/config.yml b/model_validation/config.yml index d93b2f1a..107839de 100644 --- a/model_validation/config.yml +++ b/model_validation/config.yml @@ -220,35 +220,6 @@ common_test_cases: [] overrides: - # ---- Singing voice transcription ---------------------------------------- - - teamup-tech/GAME: - # Synthetic tones may produce an empty transcription. - process_timeout: 150 - test_cases: - - name: default - expect: - "Transcribed MIDI": - ext: .mid - min_notes: 0 - "Detailed Note Data": - ext: .csv - min_bytes: 20 - - name: english-four-step-8s - synthesized_inputs: - audio: - duration: 8.0 - controls: - "Language": "English" - "D3PM Steps": "4" - expect: - "Transcribed MIDI": - ext: .mid - min_notes: 0 - "Detailed Note Data": - ext: .csv - min_bytes: 20 - # ---- Music similarity --------------------------------------------------- teamup-tech/MERIT: @@ -281,69 +252,6 @@ overrides: validators: muq_ranking: count: 1 - - name: three-candidate-ranking - controls: - "Candidate Descriptions": "calm solo piano\nenergetic rock band\nambient synthesizer pad" - expect: - "Similarity Ranking": - ext: .json - validators: - muq_ranking: - count: 3 - - # ---- Music emotion recognition ----------------------------------------- - - teamup-tech/Music2Emo: - process_timeout: 150 - test_cases: - - name: default - expect: - "Emotion Analysis": - ext: .json - validators: - music2emo_analysis: - threshold: 0.5 - - name: minimum-mood-threshold - controls: - "Mood Threshold": 0.1 - expect: - "Emotion Analysis": - ext: .json - validators: - music2emo_analysis: - threshold: 0.1 - - # ---- Music generation --------------------------------------------------- - - teamup-tech/magenta-rt: - process_timeout: 75 - test_cases: - - name: default - expect: - "Generated Music": - ext: .wav - num_channels: 2 - sample_rate: 48000 - bit_depth: 16 - min_duration: 3.9 - max_duration: 4.1 - min_rms_db: -60 - - name: seeded-two-second-piano - controls: - "Music Prompt": "a calm solo piano melody" - "Duration (seconds)": "2" - "Temperature": 0.7 - "Top-k": 25 - "Seed": 42 - expect: - "Generated Music": - ext: .wav - num_channels: 2 - sample_rate: 48000 - bit_depth: 16 - min_duration: 1.9 - max_duration: 2.1 - min_rms_db: -60 # ---- Symbolic music infilling ------------------------------------------ @@ -363,153 +271,9 @@ overrides: ext: .mid min_notes: 1 min_duration: 8.0 - - name: seeded-four-bar-rewrite - controls: - "Number of Bars": "4" - "Temperature": 0.8 - "Top-p": 0.9 - "Seed": 42 - "Maximum Polyphony": 4 - expect: - "Rewritten MIDI": - ext: .mid - min_notes: 1 - min_duration: 8.0 - - # ---- MIDI-to-audio synthesis ------------------------------------------- - - teamup-tech/DDSP-Guitar: - # Program 24 is General MIDI nylon-string guitar. - process_timeout: 150 - synthesized_inputs: - midi: - num_notes: 12 - note_duration: 0.4 - instrument: 24 - channel: 0 - test_cases: - - name: synthesized-guitar-program - expect: - "Synthesized Guitar": - ext: .wav - min_duration: 4.0 - min_rms_db: -60 - - name: real-guitar-midi - files: - "MIDI File": test_data/guitar_riff.mid - expect: - "Synthesized Guitar": - ext: .wav - min_duration: 4.0 - min_rms_db: -60 - - name: real-guitar-five-second-cap - files: - "MIDI File": test_data/guitar_riff.mid - controls: - "Max Duration (seconds)": "5" - expect: - "Synthesized Guitar": - ext: .wav - min_duration: 4.0 - max_duration: 5.1 - min_rms_db: -60 - - teamup-tech/DDSP-Piano: - # Any MIDI input works; all notes are rendered as piano. - process_timeout: 300 - synthesized_inputs: - midi: - num_notes: 8 - note_duration: 0.5 - test_cases: - - name: default - expect: - "Synthesized Piano": - ext: .wav - sample_rate: 24000 - min_duration: 3.0 - min_rms_db: -60 - - name: real-piano-midi - files: - "MIDI File": test_data/piano_chords.mid - expect: - "Synthesized Piano": - ext: .wav - sample_rate: 24000 - min_duration: 4.0 - min_rms_db: -40 - - name: bright-timbre-five-seconds - files: - "MIDI File": test_data/piano_chords.mid - controls: - "Piano Timbre": "Piano 0 (Bright)" - "Max Duration (seconds)": "5" - expect: - "Synthesized Piano": - ext: .wav - sample_rate: 24000 - min_duration: 4.0 - max_duration: 5.5 - min_rms_db: -60 - - name: warm-timbre-long-duration - files: - "MIDI File": test_data/piano_chords.mid - controls: - "Piano Timbre": "Piano 9 (Warm)" - "Max Duration (seconds)": "10" - expect: - "Synthesized Piano": - ext: .wav - sample_rate: 24000 - min_duration: 7.0 - min_rms_db: -40 - - # ---- Audio dynamics processing ----------------------------------------- - - teamup-tech/micro-TCN: - # Neural LA-2A compressor/limiter emulation. Valid convolution shortens - # output by ~0.3s at 44.1kHz. - process_timeout: 90 - test_cases: - - name: default - expect: - "Processed Audio": - ext: .wav - min_duration: 1.5 - min_rms_db: -60 - - name: limiter-heavy-reduction - controls: - "Mode": "Limit" - "Peak Reduction": 0.8 - expect: - "Processed Audio": - ext: .wav - min_duration: 1.5 - min_rms_db: -60 # ---- Audio editing / restyling ----------------------------------------- - teamup-tech/DeepAFx-ST: - process_timeout: 150 - synthesized_inputs: - audio: - duration: 8.0 - test_cases: - - name: default - expect: - "Styled Output": - ext: .wav - num_channels: 1 - sample_rate: 24000 - min_duration: 7.0 - max_duration: 9.0 - min_rms_db: -60 - "DSP Parameters": - ext: .json - min_bytes: 50 - validators: - deepafx_st_params: {} - teamup-tech/Stable-Audio-3: # Restyle and Inpaint both preserve the six-second input duration. process_timeout: 240 @@ -527,97 +291,3 @@ overrides: min_duration: 5.8 max_duration: 6.2 min_rms_db: -60 - - name: inpaint-middle - controls: - "Prompt": "jazz piano with brushed drums" - "Edit Mode": "Inpaint" - "Edit Start (seconds)": 1.0 - "Edit End (seconds)": 3.0 - "Seed": 42 - expect: - "Edited Audio": - ext: .wav - bit_depth: 16 - min_duration: 5.8 - max_duration: 6.2 - min_rms_db: -60 - - # ---- CPU audio analysis and speech ------------------------------------- - - teamup-tech/audiobox-aesthetics: - process_timeout: 150 - synthesized_inputs: - audio: - duration: 4.0 - test_cases: - - name: default - validators: - aesthetics_scores: {} - - teamup-tech/pocket-tts: - process_timeout: 90 - test_cases: - - name: default-voice - controls: - "English Text": "The quick brown fox jumps over the lazy dog." - "Voice": "alba" - expect: - "Synthesized Speech": - ext: .wav - num_channels: 1 - sample_rate: 24000 - bit_depth: 16 - min_duration: 0.2 - min_rms_db: -60 - - name: alternate-voice - controls: - "English Text": "Hello from HARP." - "Voice": "george" - expect: - "Synthesized Speech": - ext: .wav - num_channels: 1 - sample_rate: 24000 - bit_depth: 16 - min_duration: 0.2 - min_rms_db: -60 - - teamup-tech/chord-extractor: - process_timeout: 90 - synthesized_inputs: - audio: - duration: 4.0 - test_cases: - - name: default-roll-on - validators: - chord_sequence: {} - - name: minimum-roll-on - controls: - "Spectral Roll-on": 0 - validators: - chord_sequence: {} - - # ---- Music source separation --------------------------------------------- - - teamup-tech/MelBandRoFormer: - process_timeout: 300 - test_cases: - - name: default - expect: - "Vocals": - ext: .wav - "Instrumental": - ext: .wav - min_rms_db: -60 - - name: ten-second-chunked - synthesized_inputs: - audio: - duration: 10.0 - expect: - "Vocals": - min_duration: 9.5 - max_duration: 10.5 - "Instrumental": - min_duration: 9.5 - max_duration: 10.5 - min_rms_db: -60 diff --git a/model_validation/src/validators.py b/model_validation/src/validators.py index 924c6eed..7b26f6ed 100644 --- a/model_validation/src/validators.py +++ b/model_validation/src/validators.py @@ -119,51 +119,6 @@ def muq_ranking(outputs, controls, params): previous = score -@validator("music2emo_analysis") -def music2emo_analysis(outputs, controls, params): - """Check emotion dimensions and thresholded mood probabilities.""" - result = json_file(outputs, "Emotion Analysis") - for name in ("valence", "arousal"): - finite_number(result[name], name) - assert result["threshold"] == params["threshold"], "Unexpected mood threshold" - assert isinstance(result["moods"], list), "Music2Emo moods must be a list" - for mood in result["moods"]: - assert isinstance(mood["label"], str) and mood["label"], "Missing mood label" - score = mood["probability"] - finite_number(score, "mood probability") - # The Space rounds probabilities to four decimal places. - assert params["threshold"] - 0.0001 <= score <= 1, \ - f"Mood probability {score} does not meet the threshold" - - -@validator("aesthetics_scores") -def aesthetics_scores(outputs, controls, params): - """Require all four predicted aesthetic scores.""" - result = outputs["Aesthetic Scores"] - assert isinstance(result, dict), "Aesthetic Scores must be an object" - for name in ("content_enjoyment", "content_usefulness", - "production_complexity", "production_quality"): - finite_number(result["scores"][name], name) - - -@validator("chord_sequence") -def chord_sequence(outputs, controls, params): - """Check chord labels and nonnegative, ordered analysis-frame timestamps.""" - result = outputs["Chord Sequence"] - assert isinstance(result, dict), "Chord Sequence must be an object" - rows = result["chords"] - assert isinstance(rows, list), "Chord Sequence chords must be a list" - previous = 0.0 - for row in rows: - timestamp = row["timestamp_seconds"] - finite_number(timestamp, "chord timestamp") - # Chordino's final analysis frame can extend beyond the input audio. - assert previous <= timestamp, \ - f"Chord timestamp {timestamp} is negative or out of order" - assert isinstance(row["chord"], str) and row["chord"], "Missing chord label" - previous = timestamp - - @validator("labels_within_audio") def labels_within_audio(outputs, controls, params): """ @@ -216,11 +171,3 @@ def labels_within_audio(outputs, controls, params): if not checked: raise ValidatorNotApplicable("no pyharp LabelList output to check") - - -@validator("deepafx_st_params") -def deepafx_st_params(outputs, controls, params): - """Check predicted EQ and compressor parameters.""" - result = json_file(outputs, "DSP Parameters") - for i, v in enumerate(result["raw_parameters"]): - finite_number(v, f"raw_parameters[{i}]") diff --git a/model_validation/test_data/guitar_riff.mid b/model_validation/test_data/guitar_riff.mid deleted file mode 100644 index d848150c83c10c99d172569e52ed0136c953db56..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 164 zcmeYb$w*;fU|?flWME``;2Tnu4dnGP9FSy~pb^!a(4gVKFhM)2v7kX4$j}8bbb$X diff --git a/model_validation/tests/test_model_outputs.py b/model_validation/tests/test_model_outputs.py index 2832fb6c..712daf3a 100644 --- a/model_validation/tests/test_model_outputs.py +++ b/model_validation/tests/test_model_outputs.py @@ -1,15 +1,10 @@ -"""Model output contracts, including valid empty predictions.""" - import json from pathlib import Path import sys import tempfile import unittest -import mido - sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) -from expectations import check_expectations from validators import VALIDATORS @@ -36,8 +31,7 @@ def test_merit_scores(self): def test_invalid_json_is_rejected(self): self.path.write_text("not json" * 30) for name, label in (("merit_scores", "Similarity Results"), - ("muq_ranking", "Similarity Ranking"), - ("music2emo_analysis", "Emotion Analysis")): + ("muq_ranking", "Similarity Ranking")): with self.subTest(name=name), self.assertRaises(json.JSONDecodeError): VALIDATORS[name]({label: str(self.path)}, {}, {}) @@ -51,65 +45,6 @@ def test_muq_ranking(self): with self.assertRaises(AssertionError): self.check_file("muq_ranking", "Similarity Ranking", {"results": rows}, {"count": 2}) - def test_music2emo_allows_no_moods(self): - result = {"valence": 4.0, "arousal": 5.0, "threshold": 0.5, "moods": []} - self.check_file("music2emo_analysis", "Emotion Analysis", result, {"threshold": 0.5}) - result["moods"] = [{"label": "calm", "probability": 0.6}] - self.check_file("music2emo_analysis", "Emotion Analysis", result, {"threshold": 0.5}) - result["moods"][0]["probability"] = 0.4 - with self.assertRaises(AssertionError): - self.check_file("music2emo_analysis", "Emotion Analysis", result, {"threshold": 0.5}) - - def test_required_json_outputs(self): - for name, label, params in (("aesthetics_scores", "Aesthetic Scores", {}), - ("chord_sequence", "Chord Sequence", {})): - for value in (None, []): - with self.subTest(name=name, value=value), self.assertRaises(AssertionError): - VALIDATORS[name]({label: value}, {}, params) - with self.subTest(name=name), self.assertRaises(KeyError): - VALIDATORS[name]({label: {}}, {}, params) - - def test_aesthetics_scores(self): - scores = dict.fromkeys(("content_enjoyment", "content_usefulness", - "production_complexity", "production_quality"), 5.0) - VALIDATORS["aesthetics_scores"]({"Aesthetic Scores": {"scores": scores}}, {}, {}) - scores["production_quality"] = float("inf") - with self.assertRaises(AssertionError): - VALIDATORS["aesthetics_scores"]({"Aesthetic Scores": {"scores": scores}}, {}, {}) - - def test_chords_allow_empty_but_validate_timestamps(self): - check = VALIDATORS["chord_sequence"] - check({"Chord Sequence": {"chords": []}}, {}, {}) - rows = [{"timestamp_seconds": 0, "chord": "N"}, {"timestamp_seconds": 2, "chord": "C"}] - check({"Chord Sequence": {"chords": rows}}, {}, {}) - rows.append({"timestamp_seconds": 4.365351, "chord": "N"}) - check({"Chord Sequence": {"chords": rows}}, {}, {}) - for timestamp in (-1, 1, float("nan")): - rows[0]["timestamp_seconds"] = 2 - rows[1]["timestamp_seconds"] = timestamp - with self.subTest(timestamp=timestamp), self.assertRaises(AssertionError): - check({"Chord Sequence": {"chords": rows}}, {}, {}) - - def test_game_midi_parses_without_requiring_notes(self): - path = Path(self.temp.name) / "result.mid" - midi = mido.MidiFile() - midi.tracks.append(mido.MidiTrack()) - midi.save(path) - check_expectations("Transcribed MIDI", "midi_track", str(path), {"min_notes": 0}) - path.write_bytes(b"not a midi file" * 10) - with self.assertRaises(AssertionError): - check_expectations("Transcribed MIDI", "midi_track", str(path), {"min_notes": 0}) - - def test_deepafx_st_params(self): - self.check_file("deepafx_st_params", "DSP Parameters", - {"raw_parameters": [0.5, 0.3, -0.1, 0.8]}) - with self.assertRaises(KeyError): - self.check_file("deepafx_st_params", "DSP Parameters", {}) - for value in (None, True, "0.9", float("nan")): - with self.subTest(value=value), self.assertRaises(AssertionError): - self.check_file("deepafx_st_params", "DSP Parameters", - {"raw_parameters": [value]}) - if __name__ == "__main__": unittest.main() From 113772cfae0dee1e3cdaed6b06b8451e9534ce00 Mon Sep 17 00:00:00 2001 From: LING Date: Fri, 18 Sep 2026 14:47:44 +0100 Subject: [PATCH 4/5] Added DeepAFx-ST to config for CI input length fix. --- model_validation/config.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/model_validation/config.yml b/model_validation/config.yml index 107839de..7ab5e435 100644 --- a/model_validation/config.yml +++ b/model_validation/config.yml @@ -272,6 +272,25 @@ overrides: min_notes: 1 min_duration: 8.0 + # ---- Audio dynamics / style transfer ------------------------------------ + + teamup-tech/DeepAFx-ST: + # The convolutional encoder needs at least ~5s of audio at 24kHz. + process_timeout: 150 + synthesized_inputs: + audio: + duration: 8.0 + test_cases: + - name: default + expect: + "Styled Output": + ext: .wav + num_channels: 1 + sample_rate: 24000 + min_duration: 7.0 + max_duration: 9.0 + min_rms_db: -60 + # ---- Audio editing / restyling ----------------------------------------- teamup-tech/Stable-Audio-3: From 75a3241f84f0fc2600c104034d2c936163baaefb Mon Sep 17 00:00:00 2001 From: LING Date: Wed, 30 Sep 2026 15:52:34 +0100 Subject: [PATCH 5/5] Added SongFormer to config for CI input length fix. --- model_validation/config.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/model_validation/config.yml b/model_validation/config.yml index 7ab5e435..dd171bce 100644 --- a/model_validation/config.yml +++ b/model_validation/config.yml @@ -310,3 +310,11 @@ overrides: min_duration: 5.8 max_duration: 6.2 min_rms_db: -60 + + # ---- Music structure analysis -------------------------------------------- + + teamup-tech/SongFormer: + # The model requires at least 5 seconds of audio. + synthesized_inputs: + audio: + duration: 6.0