mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
BREAKING(task): deprecate a bunch of unused options
This commit is contained in:
1 parent
a73ded27f2
commit
8342e44cd8
1 file changed
+64
-298
@@ -23,7 +23,7 @@
|
||||
import math
|
||||
import warnings
|
||||
from collections import Counter
|
||||
from typing import Dict, Literal, Optional, Sequence, Text, Tuple, Union
|
||||
from typing import Dict, Literal, Optional, Sequence, Text, Union
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -43,14 +43,9 @@ from pyannote.audio.torchmetrics import (
|
||||
DiarizationErrorRate,
|
||||
FalseAlarmRate,
|
||||
MissedDetectionRate,
|
||||
OptimalDiarizationErrorRate,
|
||||
OptimalDiarizationErrorRateThreshold,
|
||||
OptimalFalseAlarmRate,
|
||||
OptimalMissedDetectionRate,
|
||||
OptimalSpeakerConfusionRate,
|
||||
SpeakerConfusionRate,
|
||||
)
|
||||
from pyannote.audio.utils.loss import binary_cross_entropy, mse_loss, nll_loss
|
||||
from pyannote.audio.utils.loss import nll_loss
|
||||
from pyannote.audio.utils.permutation import permutate
|
||||
from pyannote.audio.utils.powerset import Powerset
|
||||
|
||||
@@ -78,21 +73,7 @@ class SpeakerDiarization(SegmentationTask):
|
||||
Maximum number of speakers per chunk (must be at least 2).
|
||||
Defaults to estimating it from the training set.
|
||||
max_speakers_per_frame : int, optional
|
||||
Maximum number of (overlapping) speakers per frame.
|
||||
Setting this value to 1 or more enables `powerset multi-class` training.
|
||||
Default behavior is to use `multi-label` training.
|
||||
weigh_by_cardinality: bool, optional
|
||||
Weigh each powerset classes by the size of the corresponding speaker set.
|
||||
In other words, {0, 1} powerset class weight is 2x bigger than that of {0}
|
||||
or {1} powerset classes. Note that empty (non-speech) powerset class is
|
||||
assigned the same weight as mono-speaker classes. Defaults to False (i.e. use
|
||||
same weight for every class). Has no effect with `multi-label` training.
|
||||
warm_up : float or (float, float), optional
|
||||
Use that many seconds on the left- and rightmost parts of each chunk
|
||||
to warm up the model. While the model does process those left- and right-most
|
||||
parts, only the remaining central part of each chunk is used for computing the
|
||||
loss during training, and for aggregating scores during inference.
|
||||
Defaults to 0. (i.e. no warm-up).
|
||||
Maximum number of (overlapping) speakers per frame. Defaults to 2.
|
||||
balance: Sequence[Text], optional
|
||||
When provided, training samples are sampled uniformly with respect to these keys.
|
||||
For instance, setting `balance` to ["database","subset"] will make sure that each
|
||||
@@ -111,42 +92,34 @@ class SpeakerDiarization(SegmentationTask):
|
||||
augmentation : BaseWaveformTransform, optional
|
||||
torch_audiomentations waveform transform, used by dataloader
|
||||
during training.
|
||||
vad_loss : {"bce", "mse"}, optional
|
||||
Add voice activity detection loss.
|
||||
Cannot be used in conjunction with `max_speakers_per_frame`.
|
||||
metric : optional
|
||||
Validation metric(s). Can be anything supported by torchmetrics.MetricCollection.
|
||||
Defaults to AUROC (area under the ROC curve).
|
||||
|
||||
References
|
||||
----------
|
||||
Alexis Plaquet and Hervé Bredin
|
||||
"Powerset multi-class cross entropy loss for neural speaker diarization"
|
||||
Proc. Interspeech 2023
|
||||
|
||||
Hervé Bredin and Antoine Laurent
|
||||
"End-To-End Speaker Segmentation for Overlap-Aware Resegmentation."
|
||||
Proc. Interspeech 2021
|
||||
|
||||
Zhihao Du, Shiliang Zhang, Siqi Zheng, and Zhijie Yan
|
||||
"Speaker Embedding-aware Neural Diarization: an Efficient Framework for Overlapping
|
||||
Speech Diarization in Meeting Scenarios"
|
||||
https://arxiv.org/abs/2203.09767
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
protocol: SpeakerDiarizationProtocol,
|
||||
cache: Optional[Union[str, None]] = None,
|
||||
duration: float = 2.0,
|
||||
duration: float = 10.0,
|
||||
max_speakers_per_chunk: Optional[int] = None,
|
||||
max_speakers_per_frame: Optional[int] = None,
|
||||
weigh_by_cardinality: bool = False,
|
||||
warm_up: Union[float, Tuple[float, float]] = 0.0,
|
||||
max_speakers_per_frame: int = 2,
|
||||
balance: Optional[Sequence[Text]] = None,
|
||||
weight: Optional[Text] = None,
|
||||
batch_size: int = 32,
|
||||
num_workers: Optional[int] = None,
|
||||
pin_memory: bool = False,
|
||||
augmentation: Optional[BaseWaveformTransform] = None,
|
||||
vad_loss: Literal["bce", "mse"] = None,
|
||||
metric: Union[Metric, Sequence[Metric], Dict[str, Metric]] = None,
|
||||
max_num_speakers: Optional[
|
||||
int
|
||||
@@ -156,7 +129,6 @@ class SpeakerDiarization(SegmentationTask):
|
||||
super().__init__(
|
||||
protocol,
|
||||
duration=duration,
|
||||
warm_up=warm_up,
|
||||
batch_size=batch_size,
|
||||
num_workers=num_workers,
|
||||
pin_memory=pin_memory,
|
||||
@@ -180,22 +152,15 @@ class SpeakerDiarization(SegmentationTask):
|
||||
warnings.warn("`loss` has been deprecated and has no effect.")
|
||||
|
||||
# parameter validation
|
||||
if max_speakers_per_frame is not None:
|
||||
if max_speakers_per_frame < 1:
|
||||
raise ValueError(
|
||||
f"`max_speakers_per_frame` must be 1 or more (you used {max_speakers_per_frame})."
|
||||
)
|
||||
if vad_loss is not None:
|
||||
raise ValueError(
|
||||
"`vad_loss` cannot be used jointly with `max_speakers_per_frame`"
|
||||
)
|
||||
if max_speakers_per_frame < 1:
|
||||
raise ValueError(
|
||||
f"`max_speakers_per_frame` must be 1 or more (you used {max_speakers_per_frame})."
|
||||
)
|
||||
|
||||
self.max_speakers_per_chunk = max_speakers_per_chunk
|
||||
self.max_speakers_per_frame = max_speakers_per_frame
|
||||
self.weigh_by_cardinality = weigh_by_cardinality
|
||||
self.balance = balance
|
||||
self.weight = weight
|
||||
self.vad_loss = vad_loss
|
||||
|
||||
def setup(self, stage=None):
|
||||
super().setup(stage)
|
||||
@@ -276,10 +241,7 @@ class SpeakerDiarization(SegmentationTask):
|
||||
f"You can override this value (or avoid this estimation step) by passing `max_speakers_per_chunk={self.max_speakers_per_chunk}` to the task constructor."
|
||||
)
|
||||
|
||||
if (
|
||||
self.max_speakers_per_frame is not None
|
||||
and self.max_speakers_per_frame > self.max_speakers_per_chunk
|
||||
):
|
||||
if self.max_speakers_per_frame > self.max_speakers_per_chunk:
|
||||
raise ValueError(
|
||||
f"`max_speakers_per_frame` ({self.max_speakers_per_frame}) must be smaller "
|
||||
f"than `max_speakers_per_chunk` ({self.max_speakers_per_chunk})"
|
||||
@@ -288,24 +250,20 @@ class SpeakerDiarization(SegmentationTask):
|
||||
# now that we know about the number of speakers upper bound
|
||||
# we can set task specifications
|
||||
self.specifications = Specifications(
|
||||
problem=Problem.MULTI_LABEL_CLASSIFICATION
|
||||
if self.max_speakers_per_frame is None
|
||||
else Problem.MONO_LABEL_CLASSIFICATION,
|
||||
problem=Problem.MONO_LABEL_CLASSIFICATION,
|
||||
resolution=Resolution.FRAME,
|
||||
duration=self.duration,
|
||||
min_duration=self.min_duration,
|
||||
warm_up=self.warm_up,
|
||||
classes=[f"speaker#{i+1}" for i in range(self.max_speakers_per_chunk)],
|
||||
powerset_max_classes=self.max_speakers_per_frame,
|
||||
permutation_invariant=True,
|
||||
)
|
||||
|
||||
def setup_loss_func(self):
|
||||
if self.specifications.powerset:
|
||||
self.model.powerset = Powerset(
|
||||
len(self.specifications.classes),
|
||||
self.specifications.powerset_max_classes,
|
||||
)
|
||||
self.model.powerset = Powerset(
|
||||
len(self.specifications.classes),
|
||||
self.specifications.powerset_max_classes,
|
||||
)
|
||||
|
||||
def prepare_chunk(self, file_id: int, start_time: float, duration: float):
|
||||
"""Prepare chunk
|
||||
@@ -440,86 +398,6 @@ class SpeakerDiarization(SegmentationTask):
|
||||
|
||||
return torch.from_numpy(np.stack(collated_y))
|
||||
|
||||
def segmentation_loss(
|
||||
self,
|
||||
permutated_prediction: torch.Tensor,
|
||||
target: torch.Tensor,
|
||||
weight: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
"""Permutation-invariant segmentation loss
|
||||
|
||||
Parameters
|
||||
----------
|
||||
permutated_prediction : (batch_size, num_frames, num_classes) torch.Tensor
|
||||
Permutated speaker activity predictions.
|
||||
target : (batch_size, num_frames, num_speakers) torch.Tensor
|
||||
Speaker activity.
|
||||
weight : (batch_size, num_frames, 1) torch.Tensor, optional
|
||||
Frames weight.
|
||||
|
||||
Returns
|
||||
-------
|
||||
seg_loss : torch.Tensor
|
||||
Permutation-invariant segmentation loss
|
||||
"""
|
||||
|
||||
if self.specifications.powerset:
|
||||
# `clamp_min` is needed to set non-speech weight to 1.
|
||||
class_weight = (
|
||||
torch.clamp_min(self.model.powerset.cardinality, 1.0)
|
||||
if self.weigh_by_cardinality
|
||||
else None
|
||||
)
|
||||
seg_loss = nll_loss(
|
||||
permutated_prediction,
|
||||
torch.argmax(target, dim=-1),
|
||||
class_weight=class_weight,
|
||||
weight=weight,
|
||||
)
|
||||
else:
|
||||
seg_loss = binary_cross_entropy(
|
||||
permutated_prediction, target.float(), weight=weight
|
||||
)
|
||||
|
||||
return seg_loss
|
||||
|
||||
def voice_activity_detection_loss(
|
||||
self,
|
||||
permutated_prediction: torch.Tensor,
|
||||
target: torch.Tensor,
|
||||
weight: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
"""Voice activity detection loss
|
||||
|
||||
Parameters
|
||||
----------
|
||||
permutated_prediction : (batch_size, num_frames, num_classes) torch.Tensor
|
||||
Speaker activity predictions.
|
||||
target : (batch_size, num_frames, num_speakers) torch.Tensor
|
||||
Speaker activity.
|
||||
weight : (batch_size, num_frames, 1) torch.Tensor, optional
|
||||
Frames weight.
|
||||
|
||||
Returns
|
||||
-------
|
||||
vad_loss : torch.Tensor
|
||||
Voice activity detection loss.
|
||||
"""
|
||||
|
||||
vad_prediction, _ = torch.max(permutated_prediction, dim=2, keepdim=True)
|
||||
# (batch_size, num_frames, 1)
|
||||
|
||||
vad_target, _ = torch.max(target.float(), dim=2, keepdim=False)
|
||||
# (batch_size, num_frames)
|
||||
|
||||
if self.vad_loss == "bce":
|
||||
loss = binary_cross_entropy(vad_prediction, vad_target, weight=weight)
|
||||
|
||||
elif self.vad_loss == "mse":
|
||||
loss = mse_loss(vad_prediction, vad_target, weight=weight)
|
||||
|
||||
return loss
|
||||
|
||||
def training_step(self, batch, batch_idx: int):
|
||||
"""Compute permutation-invariant segmentation loss
|
||||
|
||||
@@ -566,67 +444,17 @@ class SpeakerDiarization(SegmentationTask):
|
||||
)
|
||||
# (batch_size, num_frames, 1)
|
||||
|
||||
# warm-up
|
||||
warm_up_left = round(self.warm_up[0] / self.duration * num_frames)
|
||||
weight[:, :warm_up_left] = 0.0
|
||||
warm_up_right = round(self.warm_up[1] / self.duration * num_frames)
|
||||
weight[:, num_frames - warm_up_right :] = 0.0
|
||||
|
||||
if self.specifications.powerset:
|
||||
multilabel = self.model.powerset.to_multilabel(prediction)
|
||||
permutated_target, _ = permutate(multilabel, target)
|
||||
permutated_target_powerset = self.model.powerset.to_powerset(
|
||||
permutated_target.float()
|
||||
)
|
||||
seg_loss = self.segmentation_loss(
|
||||
prediction, permutated_target_powerset, weight=weight
|
||||
)
|
||||
|
||||
else:
|
||||
permutated_prediction, _ = permutate(target, prediction)
|
||||
seg_loss = self.segmentation_loss(
|
||||
permutated_prediction, target, weight=weight
|
||||
)
|
||||
|
||||
self.model.log(
|
||||
"loss/train/segmentation",
|
||||
seg_loss,
|
||||
on_step=False,
|
||||
on_epoch=True,
|
||||
prog_bar=False,
|
||||
logger=True,
|
||||
multilabel = self.model.powerset.to_multilabel(prediction)
|
||||
permutated_target, _ = permutate(multilabel, target)
|
||||
permutated_target_powerset = self.model.powerset.to_powerset(
|
||||
permutated_target.float()
|
||||
)
|
||||
|
||||
if self.vad_loss is None:
|
||||
vad_loss = 0.0
|
||||
|
||||
else:
|
||||
# TODO: vad_loss probably does not make sense in powerset mode
|
||||
# because first class (empty set of labels) does exactly this...
|
||||
if self.specifications.powerset:
|
||||
vad_loss = self.voice_activity_detection_loss(
|
||||
prediction, permutated_target_powerset, weight=weight
|
||||
)
|
||||
|
||||
else:
|
||||
vad_loss = self.voice_activity_detection_loss(
|
||||
permutated_prediction, target, weight=weight
|
||||
)
|
||||
|
||||
self.model.log(
|
||||
"loss/train/vad",
|
||||
vad_loss,
|
||||
on_step=False,
|
||||
on_epoch=True,
|
||||
prog_bar=False,
|
||||
logger=True,
|
||||
)
|
||||
|
||||
loss = seg_loss + vad_loss
|
||||
|
||||
# skip batch if something went wrong for some reason
|
||||
if torch.isnan(loss):
|
||||
return None
|
||||
loss = nll_loss(
|
||||
prediction,
|
||||
torch.argmax(permutated_target_powerset, dim=-1),
|
||||
weight=weight,
|
||||
)
|
||||
|
||||
self.model.log(
|
||||
"loss/train",
|
||||
@@ -637,6 +465,21 @@ class SpeakerDiarization(SegmentationTask):
|
||||
logger=True,
|
||||
)
|
||||
|
||||
if not self.model.automatic_optimization:
|
||||
optimizers = self.model.optimizers()
|
||||
for optimizer in optimizers:
|
||||
optimizer.zero_grad()
|
||||
|
||||
self.model.manual_backward(loss)
|
||||
|
||||
for optimizer in optimizers:
|
||||
self.model.clip_gradients(
|
||||
optimizer,
|
||||
gradient_clip_val=5.0,
|
||||
gradient_clip_algorithm="norm",
|
||||
)
|
||||
optimizer.step()
|
||||
|
||||
return {"loss": loss}
|
||||
|
||||
def default_metric(
|
||||
@@ -644,20 +487,11 @@ class SpeakerDiarization(SegmentationTask):
|
||||
) -> Union[Metric, Sequence[Metric], Dict[str, Metric]]:
|
||||
"""Returns diarization error rate and its components"""
|
||||
|
||||
if self.specifications.powerset:
|
||||
return {
|
||||
"DiarizationErrorRate": DiarizationErrorRate(0.5),
|
||||
"DiarizationErrorRate/Confusion": SpeakerConfusionRate(0.5),
|
||||
"DiarizationErrorRate/Miss": MissedDetectionRate(0.5),
|
||||
"DiarizationErrorRate/FalseAlarm": FalseAlarmRate(0.5),
|
||||
}
|
||||
|
||||
return {
|
||||
"DiarizationErrorRate": OptimalDiarizationErrorRate(),
|
||||
"DiarizationErrorRate/Threshold": OptimalDiarizationErrorRateThreshold(),
|
||||
"DiarizationErrorRate/Confusion": OptimalSpeakerConfusionRate(),
|
||||
"DiarizationErrorRate/Miss": OptimalMissedDetectionRate(),
|
||||
"DiarizationErrorRate/FalseAlarm": OptimalFalseAlarmRate(),
|
||||
"DiarizationErrorRate": DiarizationErrorRate(0.5),
|
||||
"DiarizationErrorRate/Confusion": SpeakerConfusionRate(0.5),
|
||||
"DiarizationErrorRate/Miss": MissedDetectionRate(0.5),
|
||||
"DiarizationErrorRate/FalseAlarm": FalseAlarmRate(0.5),
|
||||
}
|
||||
|
||||
# TODO: no need to compute gradient in this method
|
||||
@@ -695,66 +529,20 @@ class SpeakerDiarization(SegmentationTask):
|
||||
)
|
||||
# (batch_size, num_frames, 1)
|
||||
|
||||
# warm-up
|
||||
warm_up_left = round(self.warm_up[0] / self.duration * num_frames)
|
||||
weight[:, :warm_up_left] = 0.0
|
||||
warm_up_right = round(self.warm_up[1] / self.duration * num_frames)
|
||||
weight[:, num_frames - warm_up_right :] = 0.0
|
||||
multilabel = self.model.powerset.to_multilabel(prediction)
|
||||
permutated_target, _ = permutate(multilabel, target)
|
||||
|
||||
if self.specifications.powerset:
|
||||
multilabel = self.model.powerset.to_multilabel(prediction)
|
||||
permutated_target, _ = permutate(multilabel, target)
|
||||
|
||||
# FIXME: handle case where target have too many speakers?
|
||||
# since we don't need
|
||||
permutated_target_powerset = self.model.powerset.to_powerset(
|
||||
permutated_target.float()
|
||||
)
|
||||
seg_loss = self.segmentation_loss(
|
||||
prediction, permutated_target_powerset, weight=weight
|
||||
)
|
||||
|
||||
else:
|
||||
permutated_prediction, _ = permutate(target, prediction)
|
||||
seg_loss = self.segmentation_loss(
|
||||
permutated_prediction, target, weight=weight
|
||||
)
|
||||
|
||||
self.model.log(
|
||||
"loss/val/segmentation",
|
||||
seg_loss,
|
||||
on_step=False,
|
||||
on_epoch=True,
|
||||
prog_bar=False,
|
||||
logger=True,
|
||||
# FIXME: handle case where target have too many speakers?
|
||||
# since we don't need
|
||||
permutated_target_powerset = self.model.powerset.to_powerset(
|
||||
permutated_target.float()
|
||||
)
|
||||
|
||||
if self.vad_loss is None:
|
||||
vad_loss = 0.0
|
||||
|
||||
else:
|
||||
# TODO: vad_loss probably does not make sense in powerset mode
|
||||
# because first class (empty set of labels) does exactly this...
|
||||
if self.specifications.powerset:
|
||||
vad_loss = self.voice_activity_detection_loss(
|
||||
prediction, permutated_target_powerset, weight=weight
|
||||
)
|
||||
|
||||
else:
|
||||
vad_loss = self.voice_activity_detection_loss(
|
||||
permutated_prediction, target, weight=weight
|
||||
)
|
||||
|
||||
self.model.log(
|
||||
"loss/val/vad",
|
||||
vad_loss,
|
||||
on_step=False,
|
||||
on_epoch=True,
|
||||
prog_bar=False,
|
||||
logger=True,
|
||||
)
|
||||
|
||||
loss = seg_loss + vad_loss
|
||||
loss = nll_loss(
|
||||
prediction,
|
||||
torch.argmax(permutated_target_powerset, dim=-1),
|
||||
weight=weight,
|
||||
)
|
||||
|
||||
self.model.log(
|
||||
"loss/val",
|
||||
@@ -765,24 +553,10 @@ class SpeakerDiarization(SegmentationTask):
|
||||
logger=True,
|
||||
)
|
||||
|
||||
if self.specifications.powerset:
|
||||
self.model.validation_metric(
|
||||
torch.transpose(
|
||||
multilabel[:, warm_up_left : num_frames - warm_up_right], 1, 2
|
||||
),
|
||||
torch.transpose(
|
||||
target[:, warm_up_left : num_frames - warm_up_right], 1, 2
|
||||
),
|
||||
)
|
||||
else:
|
||||
self.model.validation_metric(
|
||||
torch.transpose(
|
||||
prediction[:, warm_up_left : num_frames - warm_up_right], 1, 2
|
||||
),
|
||||
torch.transpose(
|
||||
target[:, warm_up_left : num_frames - warm_up_right], 1, 2
|
||||
),
|
||||
)
|
||||
self.model.validation_metric(
|
||||
torch.transpose(multilabel, 1, 2),
|
||||
torch.transpose(target, 1, 2),
|
||||
)
|
||||
|
||||
self.model.log_dict(
|
||||
self.model.validation_metric,
|
||||
@@ -802,12 +576,8 @@ class SpeakerDiarization(SegmentationTask):
|
||||
|
||||
# visualize first 9 validation samples of first batch in Tensorboard/MLflow
|
||||
|
||||
if self.specifications.powerset:
|
||||
y = permutated_target.float().cpu().numpy()
|
||||
y_pred = multilabel.cpu().numpy()
|
||||
else:
|
||||
y = target.float().cpu().numpy()
|
||||
y_pred = permutated_prediction.cpu().numpy()
|
||||
y = permutated_target.float().cpu().numpy()
|
||||
y_pred = multilabel.cpu().numpy()
|
||||
|
||||
# prepare 3 x 3 grid (or smaller if batch size is smaller)
|
||||
num_samples = min(self.batch_size, 9)
|
||||
@@ -841,10 +611,6 @@ class SpeakerDiarization(SegmentationTask):
|
||||
# plot predictions
|
||||
ax_hyp = axes[row_idx * 2 + 1, col_idx]
|
||||
sample_y_pred = y_pred[sample_idx]
|
||||
ax_hyp.axvspan(0, warm_up_left, color="k", alpha=0.5, lw=0)
|
||||
ax_hyp.axvspan(
|
||||
num_frames - warm_up_right, num_frames, color="k", alpha=0.5, lw=0
|
||||
)
|
||||
ax_hyp.plot(sample_y_pred)
|
||||
ax_hyp.set_ylim(-0.1, 1.1)
|
||||
ax_hyp.set_xlim(0, len(sample_y))
|
||||
|
||||
Reference in new issue
Block a user