diff --git a/CHANGELOG.md b/CHANGELOG.md index 6026ea24..26900371 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ - feat(cli): add support for multi-objective pipeline optimization - feat(cli): add `--subfolder` option to `benchmark` command [@antoinelaurent](https://github.com/antoinelaurent) - fix(utils): fix `SpeakerDiarizationMixin.optimal_mapping` `reference` is a mapping [@Kayvan-Zahiri](https://github.com/Kayvan-Zahiri) +- fix(utils): fix `discrete_diarization_error_rate` returning 0.0 on long recordings (float16 frame counts overflowed) [@Kayvan-Zahiri](https://github.com/Kayvan-Zahiri) - fix(task): fix `balance` option crashing training with `KeyError: 'metadata'` [@MohammadHijjawi97](https://github.com/MohammadHijjawi97) ## Version 4.0.7 (2026-06-30) diff --git a/src/pyannote/audio/utils/metric.py b/src/pyannote/audio/utils/metric.py index 4f8326f6..2a3fd3bb 100644 --- a/src/pyannote/audio/utils/metric.py +++ b/src/pyannote/audio/utils/metric.py @@ -65,7 +65,10 @@ def discrete_diarization_error_rate(reference: np.ndarray, hypothesis: np.ndarra (hypothesis,), _ = permutate(reference[np.newaxis], hypothesis) # total speech duration (in number of frames) - total = 1.0 * np.sum(reference) + # NOTE: reference and hypothesis are float16, whose maximum is 65504. Frame + # counts pass that after about 18 minutes of one speaker, so the reduction + # has to accumulate in float64 or it saturates to inf and the rate is 0. + total = np.sum(reference, dtype=np.float64) # false alarm and missed detection (in number of frames) detection_error = np.sum(hypothesis, axis=1) - np.sum(reference, axis=1) @@ -75,9 +78,9 @@ def discrete_diarization_error_rate(reference: np.ndarray, hypothesis: np.ndarra # speaker confusion (in number of frames) confusion = np.sum((hypothesis != reference) * hypothesis, axis=1) - false_alarm - false_alarm = np.sum(false_alarm) - missed_detection = np.sum(missed_detection) - confusion = np.sum(confusion) + false_alarm = np.sum(false_alarm, dtype=np.float64) + missed_detection = np.sum(missed_detection, dtype=np.float64) + confusion = np.sum(confusion, dtype=np.float64) der = (false_alarm + missed_detection + confusion) / total diff --git a/tests/test_metrics.py b/tests/test_metrics.py index eb9a28f6..3d26720b 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -20,6 +20,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +import numpy as np import pytest import torch @@ -27,6 +28,7 @@ from pyannote.audio.torchmetrics.functional.audio.diarization_error_rate import _der_update, diarization_error_rate, ) +from pyannote.audio.utils.metric import discrete_diarization_error_rate @pytest.fixture @@ -143,3 +145,34 @@ def test_batch_der_with_components(target, prediction): def test_chunk_der(target, prediction): der = diarization_error_rate(prediction, target, reduce="chunk") torch.testing.assert_close(der, torch.Tensor([4.0 / 8.0, 2.0 / 4.0])) + + +@pytest.mark.parametrize( + "num_frames", + [ + pytest.param(1_000, id="short"), + pytest.param(65_000, id="just-below-float16-max"), + pytest.param(70_000, id="just-above-float16-max"), + pytest.param(450_000, id="two-hours"), + ], +) +def test_discrete_der_does_not_saturate(num_frames: int): + """Frame counts have to accumulate in float64. + + reference and hypothesis are cast to float16, whose maximum is 65504. If the + reduction stays in that dtype the total saturates to inf on any recording + past roughly 18 minutes of one speaker, and an obviously wrong hypothesis + scores a perfect 0.0. + """ + num_false_alarm = 100 + reference = np.zeros((num_frames, 2)) + reference[:, 0] = 1 + hypothesis = reference.copy() + hypothesis[:num_false_alarm, 1] = 1 + + der, components = discrete_diarization_error_rate(reference, hypothesis) + + assert np.isfinite(components["total"]) + assert components["total"] == pytest.approx(num_frames) + assert components["false alarm"] == pytest.approx(num_false_alarm) + assert der == pytest.approx(num_false_alarm / num_frames)