mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
fix: accumulate discrete DER frame counts in float64 (#2049)
This commit is contained in:
1 parent
eec9b7d665
commit
0d6f291328
3 files changed
+41
-4
No files matched your search
@@ -15,6 +15,7 @@
|
||||
- feat(cli): add support for multi-objective pipeline optimization
|
||||
- feat(cli): add `--subfolder` option to `benchmark` command [@antoinelaurent](https://github.com/antoinelaurent)
|
||||
- fix(utils): fix `SpeakerDiarizationMixin.optimal_mapping` `reference` is a mapping [@Kayvan-Zahiri](https://github.com/Kayvan-Zahiri)
|
||||
- fix(utils): fix `discrete_diarization_error_rate` returning 0.0 on long recordings (float16 frame counts overflowed) [@Kayvan-Zahiri](https://github.com/Kayvan-Zahiri)
|
||||
- fix(task): fix `balance` option crashing training with `KeyError: 'metadata'` [@MohammadHijjawi97](https://github.com/MohammadHijjawi97)
|
||||
|
||||
## Version 4.0.7 (2026-06-30)
|
||||
|
||||
@@ -65,7 +65,10 @@ def discrete_diarization_error_rate(reference: np.ndarray, hypothesis: np.ndarra
|
||||
(hypothesis,), _ = permutate(reference[np.newaxis], hypothesis)
|
||||
|
||||
# total speech duration (in number of frames)
|
||||
total = 1.0 * np.sum(reference)
|
||||
# NOTE: reference and hypothesis are float16, whose maximum is 65504. Frame
|
||||
# counts pass that after about 18 minutes of one speaker, so the reduction
|
||||
# has to accumulate in float64 or it saturates to inf and the rate is 0.
|
||||
total = np.sum(reference, dtype=np.float64)
|
||||
|
||||
# false alarm and missed detection (in number of frames)
|
||||
detection_error = np.sum(hypothesis, axis=1) - np.sum(reference, axis=1)
|
||||
@@ -75,9 +78,9 @@ def discrete_diarization_error_rate(reference: np.ndarray, hypothesis: np.ndarra
|
||||
# speaker confusion (in number of frames)
|
||||
confusion = np.sum((hypothesis != reference) * hypothesis, axis=1) - false_alarm
|
||||
|
||||
false_alarm = np.sum(false_alarm)
|
||||
missed_detection = np.sum(missed_detection)
|
||||
confusion = np.sum(confusion)
|
||||
false_alarm = np.sum(false_alarm, dtype=np.float64)
|
||||
missed_detection = np.sum(missed_detection, dtype=np.float64)
|
||||
confusion = np.sum(confusion, dtype=np.float64)
|
||||
|
||||
der = (false_alarm + missed_detection + confusion) / total
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -27,6 +28,7 @@ from pyannote.audio.torchmetrics.functional.audio.diarization_error_rate import
|
||||
_der_update,
|
||||
diarization_error_rate,
|
||||
)
|
||||
from pyannote.audio.utils.metric import discrete_diarization_error_rate
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -143,3 +145,34 @@ def test_batch_der_with_components(target, prediction):
|
||||
def test_chunk_der(target, prediction):
|
||||
der = diarization_error_rate(prediction, target, reduce="chunk")
|
||||
torch.testing.assert_close(der, torch.Tensor([4.0 / 8.0, 2.0 / 4.0]))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_frames",
|
||||
[
|
||||
pytest.param(1_000, id="short"),
|
||||
pytest.param(65_000, id="just-below-float16-max"),
|
||||
pytest.param(70_000, id="just-above-float16-max"),
|
||||
pytest.param(450_000, id="two-hours"),
|
||||
],
|
||||
)
|
||||
def test_discrete_der_does_not_saturate(num_frames: int):
|
||||
"""Frame counts have to accumulate in float64.
|
||||
|
||||
reference and hypothesis are cast to float16, whose maximum is 65504. If the
|
||||
reduction stays in that dtype the total saturates to inf on any recording
|
||||
past roughly 18 minutes of one speaker, and an obviously wrong hypothesis
|
||||
scores a perfect 0.0.
|
||||
"""
|
||||
num_false_alarm = 100
|
||||
reference = np.zeros((num_frames, 2))
|
||||
reference[:, 0] = 1
|
||||
hypothesis = reference.copy()
|
||||
hypothesis[:num_false_alarm, 1] = 1
|
||||
|
||||
der, components = discrete_diarization_error_rate(reference, hypothesis)
|
||||
|
||||
assert np.isfinite(components["total"])
|
||||
assert components["total"] == pytest.approx(num_frames)
|
||||
assert components["false alarm"] == pytest.approx(num_false_alarm)
|
||||
assert der == pytest.approx(num_false_alarm / num_frames)
|
||||
Reference in new issue
Block a user