From da4231ad36b6027235cb2b80a8092418aa6b2488 Mon Sep 17 00:00:00 2001 From: Mikkeli Matlock Date: Mon, 14 Sep 2026 10:16:52 +0900 Subject: [PATCH] Add parity tests for the browser port against metrics.py fixtures.py synthesises audio aimed at the metric code's branches: shorter than each window, digital silence, a passage below the relative gate, 96 kHz, every PCM layout the direct WAV/AIFF reader handles, and (with ffmpeg) MP3/Vorbis/Opus/AAC. reference.py loads each file as the desktop app does and saves the exact mono signal plus every compute() result; parity.mjs runs web/dsp.js on that same signal and compares every field. Current result: 323/323 fields in tolerance; float64 paths agree to ~1e-12, float32 paths (true peak, spectrogram dB, RMS) to ~1e-6. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_015UXA39Sh9coAx4FhRkd8R2 --- tests/parity/fixtures.py | 86 +++++++++++++++++++++++++++++++ tests/parity/parity.mjs | 103 ++++++++++++++++++++++++++++++++++++++ tests/parity/reference.py | 74 +++++++++++++++++++++++++++ 3 files changed, 263 insertions(+) create mode 100644 tests/parity/fixtures.py create mode 100644 tests/parity/parity.mjs create mode 100644 tests/parity/reference.py diff --git a/tests/parity/fixtures.py b/tests/parity/fixtures.py new file mode 100644 index 0000000..ab79acc --- /dev/null +++ b/tests/parity/fixtures.py @@ -0,0 +1,86 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Synthesise the audio the web-port parity tests run on. + +Aimed at the branches in the metric code rather than at sounding like a song: +shorter than each metric's window, digital silence (gated loudness -> -inf / +NaN), a passage below the relative gate, int16 / int24 / float WAV, 96 kHz FLAC +and stereo downmix. When ffmpeg is on PATH it also writes an ID3-tagged MP3 and +Ogg Vorbis / Opus / AAC files for the in-browser decoder checks. + + uv run --no-project --with numpy --with soundfile \ + python tests/parity/fixtures.py OUT_DIR +""" + +import shutil +import subprocess +import sys +from pathlib import Path + +import numpy as np +import soundfile as sf + + +def tone_mix(sr: int, seconds: float, seed: int) -> np.ndarray: + rng = np.random.default_rng(seed) + t = np.arange(int(sr * seconds)) / sr + y = (0.3 * np.sin(2 * np.pi * 110 * t) + + 0.15 * np.sin(2 * np.pi * 1760 * t + 0.3) + + 0.05 * rng.standard_normal(len(t))) + # Slow level movement so short-term loudness and LRA have something to show. + return (y * (0.6 + 0.4 * np.sin(2 * np.pi * 0.07 * t))).astype(np.float32) + + +def main() -> None: + out = Path(sys.argv[1] if len(sys.argv) > 1 else "/tmp/ujm-parity") + out.mkdir(parents=True, exist_ok=True) + + sr = 44100 + left, right = tone_mix(sr, 40, 1), tone_mix(sr, 40, 2) * 0.8 + for ch in (left, right): + ch[10 * sr:15 * sr] = 0.0 # below the absolute gate + ch[20 * sr:25 * sr] *= 10 ** (-45 / 20) # below the relative gate + left[30 * sr:30 * sr + 2000] = 0.99 # near-full-scale burst for true peak + music = out / "music_44k1_s16_stereo.wav" + sf.write(music, np.stack([left, right], 1), sr, subtype="PCM_16") + + y = tone_mix(48000, 120, 3) + sf.write(out / "long_48k_s24_stereo.wav", np.stack([y, np.roll(y, 480)], 1), 48000, subtype="PCM_24") + y = tone_mix(96000, 20, 4) + sf.write(out / "hires_96k_s24.flac", np.stack([y, -0.5 * y], 1), 96000, subtype="PCM_24") + sf.write(out / "short_1500ms_48k_float.wav", tone_mix(48000, 1.5, 5), 48000, subtype="FLOAT") + sf.write(out / "tiny_200ms_44k1_s16.wav", tone_mix(44100, 0.2, 6), 44100, subtype="PCM_16") + sf.write(out / "silence_5s_48k_s16.wav", np.zeros(5 * 48000, dtype=np.float32), 48000, subtype="PCM_16") + + # Every PCM layout the browser's direct WAV/AIFF reader handles. + y = tone_mix(48000, 4, 7) + stereo = np.stack([y, -0.7 * y], 1) + sf.write(out / "fmt_s24_48k.wav", stereo, 48000, subtype="PCM_24") + sf.write(out / "fmt_s32_48k.wav", stereo, 48000, subtype="PCM_32") + sf.write(out / "fmt_f64_48k_mono.wav", y, 48000, subtype="DOUBLE") + sf.write(out / "fmt_u8_22k05_mono.wav", tone_mix(22050, 4, 8), 22050, subtype="PCM_U8") + sf.write(out / "fmt_wavex_s24_96k.wav", np.stack([tone_mix(96000, 3, 9)] * 3, 1), 96000, + format="WAVEX", subtype="PCM_24") + sf.write(out / "fmt_s16_44k1.aiff", stereo[: 44100 * 4], 44100, format="AIFF", subtype="PCM_16") + sf.write(out / "fmt_f32_48k.aifc", stereo, 48000, format="AIFF", subtype="FLOAT") + + if shutil.which("ffmpeg"): + encodes = { + "tagged_44k1.mp3": ["-codec:a", "libmp3lame", "-b:a", "192k", "-id3v2_version", "3", + "-metadata", "artist=ウジ", "-metadata", "title=テスト曲"], + "vorbis_44k1.ogg": ["-codec:a", "libvorbis", "-q:a", "5"], + "opus_48k.opus": ["-codec:a", "libopus", "-b:a", "128k"], + "aac_44k1.m4a": ["-codec:a", "aac", "-b:a", "192k"], + } + for name, args in encodes.items(): + try: + subprocess.run(["ffmpeg", "-loglevel", "error", "-y", "-i", str(music), *args, str(out / name)], + check=True) + except subprocess.CalledProcessError as exc: + print(f"skipped {name}: ffmpeg failed ({exc.returncode})") + + for f in sorted(out.iterdir()): + print(f" {f.name:32s} {f.stat().st_size / 1024:8.0f} KiB") + + +if __name__ == "__main__": + main() diff --git a/tests/parity/parity.mjs b/tests/parity/parity.mjs new file mode 100644 index 0000000..6780570 --- /dev/null +++ b/tests/parity/parity.mjs @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Parity check: web/dsp.js against the Python metrics. +// +// reference.py runs metrics.py on each fixture and saves the exact mono signal +// it analysed. This feeds that same signal to dsp.js and compares every output +// field, so decoder differences are out of the picture. +// +// node tests/parity/parity.mjs REF_DIR + +import { existsSync, readdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +import { COMPUTE } from '../../web/dsp.js'; + +// Largest acceptable |js - python| per field. The float64 paths (loudness, +// crest, PSR) should agree to ~1e-12; float32 paths (RMS mean, 4x resampling, +// spectrogram dB) to ~1e-6. Waveform min/max are exact sample values. +const TOLERANCE = { + rms_power: { times: 1e-9, rms: 1e-6 }, + waveform: { times: 1e-9, lo: 0, hi: 0 }, + lufs: { times: 1e-9, lufs: 1e-6, integrated: 1e-6, lra: 1e-6 }, + crest_factor: { times: 1e-9, crest_db: 1e-6 }, + psr: { times: 1e-9, psr: 1e-6 }, + true_peak: { times: 1e-9, tp_db: 1e-4, integrated_tp_db: 1e-4 }, + spectrogram: { times: 1e-9, freqs: 1e-6, s_db: 1e-3 }, +}; + +function readF32(path) { + const buf = readFileSync(path); + return new Float32Array(buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength)); +} + +const isNum = (v) => typeof v === 'number' && Number.isFinite(v); + +function compare(js, py) { + const a = ArrayBuffer.isView(js) || Array.isArray(js) ? js : [js]; + const b = ArrayBuffer.isView(py) || Array.isArray(py) ? py : [py]; + if (a.length !== b.length) return { error: `length ${a.length} vs ${b.length}` }; + let max = 0; + let at = -1; + let nonFinite = 0; + for (let i = 0; i < a.length; i++) { + const fx = isNum(a[i]); + const fy = isNum(b[i]); + if (!fx || !fy) { + if (fx !== fy) nonFinite++; + continue; + } + const d = Math.abs(a[i] - b[i]); + if (d > max) { max = d; at = i; } + } + return { max, at, nonFinite }; +} + +const refDir = process.argv[2]; +if (!refDir) { + console.error('usage: node tests/parity/parity.mjs REF_DIR'); + process.exit(2); +} + +let failures = 0; +let checked = 0; +for (const name of readdirSync(refDir).sort()) { + const dir = join(refDir, name); + if (!existsSync(join(dir, 'reference.json'))) continue; + const ref = JSON.parse(readFileSync(join(dir, 'reference.json'), 'utf8')); + const mono = readF32(join(dir, 'mono.f32')); + const sr = ref.meta.sr; + const memo = {}; + console.log(`\n${name} (${sr} Hz, ${(mono.length / sr).toFixed(2)} s)`); + + for (const [metricId, expected] of Object.entries(ref.metrics)) { + const t0 = performance.now(); + const got = COMPUTE[metricId](mono, sr, memo); + const ms = performance.now() - t0; + + const fields = metricId === 'spectrogram' + ? { times: expected.times, freqs: expected.freqs, s_db: readF32(join(dir, 'spectrogram.f32')) } + : expected; + if (metricId === 'spectrogram' && (got.rows !== expected.rows || got.cols !== expected.cols)) { + failures++; + console.log(` FAIL spectrogram shape ${got.rows}x${got.cols} vs ${expected.rows}x${expected.cols}`); + } + + let first = true; + for (const [field, py] of Object.entries(fields)) { + const tol = TOLERANCE[metricId][field]; + if (tol === undefined) continue; + const r = compare(got[field], py); + const ok = !r.error && r.nonFinite === 0 && r.max <= tol; + checked++; + if (!ok) failures++; + const diff = r.error || `max|d| ${r.max.toExponential(2)}${r.nonFinite ? `, ${r.nonFinite} non-finite mismatches` : ''}`; + console.log(` ${ok ? 'ok ' : 'FAIL'} ${metricId.padEnd(13)} ${field.padEnd(17)} ${diff.padEnd(24)} tol ${tol}` + + (first ? ` ${ms.toFixed(0)} ms` : '')); + first = false; + } + } +} + +console.log(failures ? `\n${failures} of ${checked} fields out of tolerance` : `\nall ${checked} fields within tolerance`); +process.exit(failures ? 1 : 0); diff --git a/tests/parity/reference.py b/tests/parity/reference.py new file mode 100644 index 0000000..5ac8a5e --- /dev/null +++ b/tests/parity/reference.py @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Dump the Python metrics as the reference for the browser port (web/dsp.js). + +Each audio file is loaded exactly as the desktop app loads it +(`master_core.AudioFile`); the mono signal the metrics see is written out next +to every metric's `compute()` result. `parity.mjs` feeds that same signal to +dsp.js, so the comparison isolates the DSP port from decoder differences. + + uv run --no-project --with librosa --with pyloudnorm --with scipy --with mutagen \ + python tests/parity/reference.py OUT_DIR AUDIO... + +Writes OUT_DIR//{mono.f32, reference.json, spectrogram.f32}. +""" + +import json +import sys +from pathlib import Path + +import numpy as np + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from master_core import AudioFile # noqa: E402 +from metrics import METRICS # noqa: E402 + + +def _jsonable(value): + if isinstance(value, np.ndarray): + return [float(v) if np.isfinite(v) else None for v in value.astype(np.float64).ravel()] + if isinstance(value, (float, np.floating)): + return float(value) if np.isfinite(value) else None + return value + + +def main() -> None: + out = Path(sys.argv[1]) + out.mkdir(parents=True, exist_ok=True) + for path in sys.argv[2:]: + name = Path(path).name + try: + audio = AudioFile(path) + except Exception as exc: # librosa can't read every container the browser can + print(f"{name}: skipped ({exc})") + continue + + target = out / name + target.mkdir(exist_ok=True) + audio.y_mono.astype(np.float32).tofile(target / "mono.f32") + + results = {} + for metric_id, metric in METRICS.items(): + data = metric.compute(audio) + if metric_id == "spectrogram": + s_db = np.ascontiguousarray(data["s_db"], dtype=np.float32) + s_db.tofile(target / "spectrogram.f32") + results[metric_id] = { + "rows": int(s_db.shape[0]), "cols": int(s_db.shape[1]), + "times": _jsonable(data["times"]), "freqs": _jsonable(data["freqs"]), + "nyquist": float(data["nyquist"]), + } + else: + results[metric_id] = {key: _jsonable(val) for key, val in data.items()} + + meta = { + "file": name, "sr": int(audio.sr), "n": int(len(audio.y_mono)), + "song_name": audio.song_name, + "max_amplitude": float(audio.max_amplitude), "avg_amplitude": float(audio.avg_amplitude), + } + (target / "reference.json").write_text(json.dumps({"meta": meta, "metrics": results})) + print(f"{name}: {audio.sr} Hz, {len(audio.y_mono) / audio.sr:.2f} s, song_name={audio.song_name!r}") + + +if __name__ == "__main__": + main()