Add parity tests for the browser port against metrics.py
fixtures.py synthesises audio aimed at the metric code's branches: shorter than each window, digital silence, a passage below the relative gate, 96 kHz, every PCM layout the direct WAV/AIFF reader handles, and (with ffmpeg) MP3/Vorbis/Opus/AAC. reference.py loads each file as the desktop app does and saves the exact mono signal plus every compute() result; parity.mjs runs web/dsp.js on that same signal and compares every field. Current result: 323/323 fields in tolerance; float64 paths agree to ~1e-12, float32 paths (true peak, spectrogram dB, RMS) to ~1e-6. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_015UXA39Sh9coAx4FhRkd8R2
This commit is contained in:
@@ -0,0 +1,103 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
// Parity check: web/dsp.js against the Python metrics.
|
||||
//
|
||||
// reference.py runs metrics.py on each fixture and saves the exact mono signal
|
||||
// it analysed. This feeds that same signal to dsp.js and compares every output
|
||||
// field, so decoder differences are out of the picture.
|
||||
//
|
||||
// node tests/parity/parity.mjs REF_DIR
|
||||
|
||||
import { existsSync, readdirSync, readFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import { COMPUTE } from '../../web/dsp.js';
|
||||
|
||||
// Largest acceptable |js - python| per field. The float64 paths (loudness,
|
||||
// crest, PSR) should agree to ~1e-12; float32 paths (RMS mean, 4x resampling,
|
||||
// spectrogram dB) to ~1e-6. Waveform min/max are exact sample values.
|
||||
const TOLERANCE = {
|
||||
rms_power: { times: 1e-9, rms: 1e-6 },
|
||||
waveform: { times: 1e-9, lo: 0, hi: 0 },
|
||||
lufs: { times: 1e-9, lufs: 1e-6, integrated: 1e-6, lra: 1e-6 },
|
||||
crest_factor: { times: 1e-9, crest_db: 1e-6 },
|
||||
psr: { times: 1e-9, psr: 1e-6 },
|
||||
true_peak: { times: 1e-9, tp_db: 1e-4, integrated_tp_db: 1e-4 },
|
||||
spectrogram: { times: 1e-9, freqs: 1e-6, s_db: 1e-3 },
|
||||
};
|
||||
|
||||
function readF32(path) {
|
||||
const buf = readFileSync(path);
|
||||
return new Float32Array(buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength));
|
||||
}
|
||||
|
||||
const isNum = (v) => typeof v === 'number' && Number.isFinite(v);
|
||||
|
||||
function compare(js, py) {
|
||||
const a = ArrayBuffer.isView(js) || Array.isArray(js) ? js : [js];
|
||||
const b = ArrayBuffer.isView(py) || Array.isArray(py) ? py : [py];
|
||||
if (a.length !== b.length) return { error: `length ${a.length} vs ${b.length}` };
|
||||
let max = 0;
|
||||
let at = -1;
|
||||
let nonFinite = 0;
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
const fx = isNum(a[i]);
|
||||
const fy = isNum(b[i]);
|
||||
if (!fx || !fy) {
|
||||
if (fx !== fy) nonFinite++;
|
||||
continue;
|
||||
}
|
||||
const d = Math.abs(a[i] - b[i]);
|
||||
if (d > max) { max = d; at = i; }
|
||||
}
|
||||
return { max, at, nonFinite };
|
||||
}
|
||||
|
||||
const refDir = process.argv[2];
|
||||
if (!refDir) {
|
||||
console.error('usage: node tests/parity/parity.mjs REF_DIR');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
let failures = 0;
|
||||
let checked = 0;
|
||||
for (const name of readdirSync(refDir).sort()) {
|
||||
const dir = join(refDir, name);
|
||||
if (!existsSync(join(dir, 'reference.json'))) continue;
|
||||
const ref = JSON.parse(readFileSync(join(dir, 'reference.json'), 'utf8'));
|
||||
const mono = readF32(join(dir, 'mono.f32'));
|
||||
const sr = ref.meta.sr;
|
||||
const memo = {};
|
||||
console.log(`\n${name} (${sr} Hz, ${(mono.length / sr).toFixed(2)} s)`);
|
||||
|
||||
for (const [metricId, expected] of Object.entries(ref.metrics)) {
|
||||
const t0 = performance.now();
|
||||
const got = COMPUTE[metricId](mono, sr, memo);
|
||||
const ms = performance.now() - t0;
|
||||
|
||||
const fields = metricId === 'spectrogram'
|
||||
? { times: expected.times, freqs: expected.freqs, s_db: readF32(join(dir, 'spectrogram.f32')) }
|
||||
: expected;
|
||||
if (metricId === 'spectrogram' && (got.rows !== expected.rows || got.cols !== expected.cols)) {
|
||||
failures++;
|
||||
console.log(` FAIL spectrogram shape ${got.rows}x${got.cols} vs ${expected.rows}x${expected.cols}`);
|
||||
}
|
||||
|
||||
let first = true;
|
||||
for (const [field, py] of Object.entries(fields)) {
|
||||
const tol = TOLERANCE[metricId][field];
|
||||
if (tol === undefined) continue;
|
||||
const r = compare(got[field], py);
|
||||
const ok = !r.error && r.nonFinite === 0 && r.max <= tol;
|
||||
checked++;
|
||||
if (!ok) failures++;
|
||||
const diff = r.error || `max|d| ${r.max.toExponential(2)}${r.nonFinite ? `, ${r.nonFinite} non-finite mismatches` : ''}`;
|
||||
console.log(` ${ok ? 'ok ' : 'FAIL'} ${metricId.padEnd(13)} ${field.padEnd(17)} ${diff.padEnd(24)} tol ${tol}`
|
||||
+ (first ? ` ${ms.toFixed(0)} ms` : ''));
|
||||
first = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(failures ? `\n${failures} of ${checked} fields out of tolerance` : `\nall ${checked} fields within tolerance`);
|
||||
process.exit(failures ? 1 : 0);
|
||||
Reference in New Issue
Block a user