fix: accumulate discrete DER frame counts in float64 (#2049)

This commit is contained in:
Kayvan Zahiri authored and GitHub committed 2026-11-09 06:42:58 +01:00
1 parent eec9b7d665
commit 0d6f291328
3 files changed
+41 -4

No files matched your search

+1
View File
@@ -15,6 +15,7 @@
- feat(cli): add support for multi-objective pipeline optimization
- feat(cli): add `--subfolder` option to `benchmark` command [@antoinelaurent](https://github.com/antoinelaurent)
- fix(utils): fix `SpeakerDiarizationMixin.optimal_mapping` `reference` is a mapping [@Kayvan-Zahiri](https://github.com/Kayvan-Zahiri)
- fix(utils): fix `discrete_diarization_error_rate` returning 0.0 on long recordings (float16 frame counts overflowed) [@Kayvan-Zahiri](https://github.com/Kayvan-Zahiri)
- fix(task): fix `balance` option crashing training with `KeyError: 'metadata'` [@MohammadHijjawi97](https://github.com/MohammadHijjawi97)
## Version 4.0.7 (2026-06-30)
+7 -4
View File
@@ -65,7 +65,10 @@ def discrete_diarization_error_rate(reference: np.ndarray, hypothesis: np.ndarra
(hypothesis,), _ = permutate(reference[np.newaxis], hypothesis)
# total speech duration (in number of frames)
total = 1.0 * np.sum(reference)
# NOTE: reference and hypothesis are float16, whose maximum is 65504. Frame
# counts pass that after about 18 minutes of one speaker, so the reduction
# has to accumulate in float64 or it saturates to inf and the rate is 0.
total = np.sum(reference, dtype=np.float64)
# false alarm and missed detection (in number of frames)
detection_error = np.sum(hypothesis, axis=1) - np.sum(reference, axis=1)
@@ -75,9 +78,9 @@ def discrete_diarization_error_rate(reference: np.ndarray, hypothesis: np.ndarra
# speaker confusion (in number of frames)
confusion = np.sum((hypothesis != reference) * hypothesis, axis=1) - false_alarm
false_alarm = np.sum(false_alarm)
missed_detection = np.sum(missed_detection)
confusion = np.sum(confusion)
false_alarm = np.sum(false_alarm, dtype=np.float64)
missed_detection = np.sum(missed_detection, dtype=np.float64)
confusion = np.sum(confusion, dtype=np.float64)
der = (false_alarm + missed_detection + confusion) / total
+33
View File
@@ -20,6 +20,7 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import numpy as np
import pytest
import torch
@@ -27,6 +28,7 @@ from pyannote.audio.torchmetrics.functional.audio.diarization_error_rate import
_der_update,
diarization_error_rate,
)
from pyannote.audio.utils.metric import discrete_diarization_error_rate
@pytest.fixture
@@ -143,3 +145,34 @@ def test_batch_der_with_components(target, prediction):
def test_chunk_der(target, prediction):
der = diarization_error_rate(prediction, target, reduce="chunk")
torch.testing.assert_close(der, torch.Tensor([4.0 / 8.0, 2.0 / 4.0]))
@pytest.mark.parametrize(
"num_frames",
[
pytest.param(1_000, id="short"),
pytest.param(65_000, id="just-below-float16-max"),
pytest.param(70_000, id="just-above-float16-max"),
pytest.param(450_000, id="two-hours"),
],
)
def test_discrete_der_does_not_saturate(num_frames: int):
"""Frame counts have to accumulate in float64.
reference and hypothesis are cast to float16, whose maximum is 65504. If the
reduction stays in that dtype the total saturates to inf on any recording
past roughly 18 minutes of one speaker, and an obviously wrong hypothesis
scores a perfect 0.0.
"""
num_false_alarm = 100
reference = np.zeros((num_frames, 2))
reference[:, 0] = 1
hypothesis = reference.copy()
hypothesis[:num_false_alarm, 1] = 1
der, components = discrete_diarization_error_rate(reference, hypothesis)
assert np.isfinite(components["total"])
assert components["total"] == pytest.approx(num_frames)
assert components["false alarm"] == pytest.approx(num_false_alarm)
assert der == pytest.approx(num_false_alarm / num_frames)