diff --git a/pyannote/audio/cli/pretrained.py b/pyannote/audio/cli/pretrained.py index 95a439b3..8b8aae58 100644 --- a/pyannote/audio/cli/pretrained.py +++ b/pyannote/audio/cli/pretrained.py @@ -26,6 +26,4 @@ from pyannote.audio import Model def pretrained(checkpoint: Text): - return Model.from_pretrained( - checkpoint, map_location=lambda storage, loc: storage - ) + return Model.from_pretrained(checkpoint, map_location=lambda storage, loc: storage) diff --git a/pyannote/audio/core/inference.py b/pyannote/audio/core/inference.py index 16cdea65..cf92e2ee 100644 --- a/pyannote/audio/core/inference.py +++ b/pyannote/audio/core/inference.py @@ -35,6 +35,7 @@ from pyannote.audio.core.io import AudioFile from pyannote.audio.core.model import Model from pyannote.audio.core.task import Resolution from pyannote.audio.utils.permutation import mae_cost_func, permutate +from pyannote.audio.utils.powerset import Powerset TaskName = Union[Text, None] @@ -53,23 +54,26 @@ class Inference(BaseInference): window : {"sliding", "whole"}, optional Use a "sliding" window and aggregate the corresponding outputs (default) or just one (potentially long) window covering the "whole" file or chunk. - skip_aggregation : bool, optional - Do not aggregate outputs when using "sliding" window. Defaults to False. duration : float, optional Chunk duration, in seconds. Defaults to duration used for training the model. Has no effect when `window` is "whole". step : float, optional Step between consecutive chunks, in seconds. Defaults to warm-up duration when greater than 0s, otherwise 10% of duration. Has no effect when `window` is "whole". + pre_aggregation_hook : callable, optional + When a callable is provided, it is applied to the model output, just before aggregation. + Takes a (num_chunks, num_frames, dimension) numpy array as input and returns a modified + (num_chunks, num_frames, other_dimension) numpy array passed to overlap-add aggregation. + skip_aggregation : bool, optional + Do not aggregate outputs when using "sliding" window. Defaults to False. + skip_conversion: bool, optional + In case `model` has been trained with `powerset` mode, its output is automatically + converted to `multi-label`, unless `skip_conversion` is set to True. batch_size : int, optional Batch size. Larger values make inference faster. Defaults to 32. device : torch.device, optional Device used for inference. Defaults to `model.device`. In case `device` and `model.device` are different, model is sent to device. - pre_aggregation_hook : callable, optional - When a callable is provided, it is applied to the model output, just before aggregation. - Takes a (num_chunks, num_frames, dimension) numpy array as input and returns a modified - (num_chunks, num_frames, other_dimension) numpy array passed to overlap-add aggregation. use_auth_token : str, optional When loading a private huggingface.co model, set `use_auth_token` to True or to a string containing your hugginface.co authentication @@ -80,12 +84,13 @@ class Inference(BaseInference): self, model: Union[Model, Text, Path], window: Text = "sliding", - skip_aggregation: bool = False, - device: torch.device = None, duration: float = None, step: float = None, - batch_size: int = 32, pre_aggregation_hook: Callable[[np.ndarray], np.ndarray] = None, + skip_aggregation: bool = False, + skip_conversion: bool = False, + device: torch.device = None, + batch_size: int = 32, use_auth_token: Union[Text, None] = None, ): @@ -154,11 +159,19 @@ class Inference(BaseInference): self.step = step self.batch_size = batch_size + self.skip_conversion = skip_conversion + if specifications.powerset and not self.skip_conversion: + self._powerset = Powerset( + len(specifications.classes), specifications.powerset_max_classes + ) + self._powerset.to(self.device) def to(self, device: torch.device): """Send internal model to `device`""" self.model.to(device) + if self.model.specifications.powerset and not self.skip_conversion: + self._powerset.to(device) self.device = device return self @@ -190,6 +203,14 @@ class Inference(BaseInference): else: raise exception + # convert powerset to multi-label unless specifically requested not to + if self.model.specifications.powerset and not self.skip_conversion: + powerset = torch.nn.functional.one_hot( + torch.argmax(outputs, dim=-1), + self.model.specifications.num_powerset_classes, + ).float() + outputs = self._powerset.to_multilabel(powerset) + return outputs.cpu().numpy() def slide( diff --git a/pyannote/audio/core/task.py b/pyannote/audio/core/task.py index fb42fb7c..5b523176 100644 --- a/pyannote/audio/core/task.py +++ b/pyannote/audio/core/task.py @@ -25,6 +25,8 @@ from __future__ import annotations from functools import partial +import scipy.special + try: from functools import cached_property except ImportError: @@ -89,9 +91,38 @@ class Specifications: # (for classification tasks only) list of classes classes: Optional[List[Text]] = None + # (for powerset only) max number of simultaneous classes + # (n choose k with k <= powerset_max_classes) + powerset_max_classes: Optional[int] = None + # whether classes are permutation-invariant (e.g. diarization) permutation_invariant: bool = False + @cached_property + def powerset(self): + + if self.powerset_max_classes is None: + return False + + if self.problem != Problem.MONO_LABEL_CLASSIFICATION: + raise ValueError( + "`powerset_max_classes` only makes sense with multi-class classification problems." + ) + + return True + + @cached_property + def num_powerset_classes(self) -> int: + # compute number of subsets of size at most "powerset_max_classes" + # e.g. with len(classes) = 3 and powerset_max_classes = 2: + # {}, {0}, {1}, {2}, {0, 1}, {0, 2}, {1, 2} + return int( + sum( + scipy.special.binom(len(self.classes), i) + for i in range(0, self.powerset_max_classes + 1) + ) + ) + class TrainDataset(IterableDataset): def __init__(self, task: Task): @@ -314,7 +345,7 @@ class Task(pl.LightningDataModule): ]: return binary_cross_entropy(prediction, target, weight=weight) - elif specifications.problem == Problem.MONO_LABEL_CLASSIFICATION: + elif specifications.problem in [Problem.MONO_LABEL_CLASSIFICATION]: return nll_loss(prediction, target, weight=weight) else: diff --git a/pyannote/audio/models/segmentation/PyanNet.py b/pyannote/audio/models/segmentation/PyanNet.py index a39a8fa7..1b68a32a 100644 --- a/pyannote/audio/models/segmentation/PyanNet.py +++ b/pyannote/audio/models/segmentation/PyanNet.py @@ -27,12 +27,12 @@ import torch import torch.nn as nn import torch.nn.functional as F from einops import rearrange +from pyannote.core.utils.generators import pairwise from pyannote.audio.core.model import Model from pyannote.audio.core.task import Task from pyannote.audio.models.blocks.sincnet import SincNet from pyannote.audio.utils.params import merge_dict -from pyannote.core.utils.generators import pairwise class PyanNet(Model): @@ -130,7 +130,9 @@ class PyanNet(Model): [ nn.Linear(in_features, out_features) for in_features, out_features in pairwise( - [lstm_out_features,] + [ + lstm_out_features, + ] + [self.hparams.linear["hidden_size"]] * self.hparams.linear["num_layers"] ) @@ -146,7 +148,12 @@ class PyanNet(Model): 2 if self.hparams.lstm["bidirectional"] else 1 ) - self.classifier = nn.Linear(in_features, len(self.specifications.classes)) + if self.specifications.powerset: + out_features = self.specifications.num_powerset_classes + else: + out_features = len(self.specifications.classes) + + self.classifier = nn.Linear(in_features, out_features) self.activation = self.default_activation() def forward(self, waveforms: torch.Tensor) -> torch.Tensor: diff --git a/pyannote/audio/pipelines/speaker_diarization.py b/pyannote/audio/pipelines/speaker_diarization.py index dbe2686c..1e279b82 100644 --- a/pyannote/audio/pipelines/speaker_diarization.py +++ b/pyannote/audio/pipelines/speaker_diarization.py @@ -147,10 +147,16 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline): ) self._frames: SlidingWindow = self._segmentation.model.introspection.frames - self.segmentation = ParamDict( - threshold=Uniform(0.1, 0.9), - min_duration_off=Uniform(0.0, 1.0), - ) + if self._segmentation.model.specifications.powerset: + self.segmentation = ParamDict( + min_duration_off=Uniform(0.0, 1.0), + ) + + else: + self.segmentation = ParamDict( + threshold=Uniform(0.1, 0.9), + min_duration_off=Uniform(0.0, 1.0), + ) if self.klustering == "OracleClustering": metric = "not_applicable" @@ -260,18 +266,19 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline): embeddings : (num_chunks, num_speakers, dimension) array """ - # when optimizing the hyper-parameters of this pipeline with frozen "segmentation_onset", - # one can reuse the embeddings from the first trial, bringing a massive speed up to - # the optimization process (and hence allowing to use a larger search space). + # when optimizing the hyper-parameters of this pipeline with frozen + # "segmentation.threshold", one can reuse the embeddings from the first trial, + # bringing a massive speed up to the optimization process (and hence allowing to use + # a larger search space). if self.training: # we only re-use embeddings if they were extracted based on the same value of the - # "segmentation_onset" hyperparameter and "embedding_exclude_overlap" parameter. + # "segmentation.threshold" hyperparameter or if the segmentation model relies on + # `powerset` mode cache = file.get("training_cache/embeddings", dict()) - if ( - cache.get("segmentation.threshold", None) == self.segmentation.threshold - and cache.get("embedding_exclude_overlap", None) - == self.embedding_exclude_overlap + if ("embeddings" in cache) and ( + self._segmentation.model.specifications.powerset + or (cache["segmentation.threshold"] == self.segmentation.threshold) ): return cache["embeddings"] @@ -369,11 +376,16 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline): # caching embeddings for subsequent trials # (see comments at the top of this method for more details) if self.training: - file["training_cache/embeddings"] = { - "segmentation.threshold": self.segmentation.threshold, - "embedding_exclude_overlap": self.embedding_exclude_overlap, - "embeddings": embeddings, - } + + if self._segmentation.model.specifications.powerset: + file["training_cache/embeddings"] = { + "embeddings": embeddings, + } + else: + file["training_cache/embeddings"] = { + "segmentation.threshold": self.segmentation.threshold, + "embeddings": embeddings, + } return embeddings @@ -479,7 +491,9 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline): # estimate frame-level number of instantaneous speakers count = self.speaker_count( segmentations, - onset=self.segmentation.threshold, + onset=0.5 + if self._segmentation.model.specifications.powerset + else self.segmentation.threshold, frames=self._frames, ) hook("speaker_counting", count) @@ -491,11 +505,14 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline): return Annotation(uri=file["uri"]) # binarize segmentation - binarized_segmentations: SlidingWindowFeature = binarize( - segmentations, - onset=self.segmentation.threshold, - initial_state=False, - ) + if self._segmentation.model.specifications.powerset: + binarized_segmentations = segmentations + else: + binarized_segmentations: SlidingWindowFeature = binarize( + segmentations, + onset=self.segmentation.threshold, + initial_state=False, + ) if self.klustering == "OracleClustering": embeddings = None diff --git a/pyannote/audio/tasks/__init__.py b/pyannote/audio/tasks/__init__.py index e53600bc..eeaf19d1 100644 --- a/pyannote/audio/tasks/__init__.py +++ b/pyannote/audio/tasks/__init__.py @@ -25,9 +25,10 @@ from .segmentation.overlapped_speech_detection import ( # isort:skip OverlappedSpeechDetection, ) -from .segmentation.multilabel import MultiLabelSegmentation # isort:skip +from .segmentation.multilabel import MultiLabelSegmentation # isort:skip from .segmentation.segmentation import Segmentation # isort:skip from .embedding.arcface import SupervisedRepresentationLearningWithArcFace # isort:skip + SpeakerEmbedding = SupervisedRepresentationLearningWithArcFace __all__ = [ diff --git a/pyannote/audio/tasks/segmentation/mixins.py b/pyannote/audio/tasks/segmentation/mixins.py index beaa2bc6..d0f63d2c 100644 --- a/pyannote/audio/tasks/segmentation/mixins.py +++ b/pyannote/audio/tasks/segmentation/mixins.py @@ -29,14 +29,16 @@ from typing import Dict, Optional, Sequence, Text, Union import matplotlib.pyplot as plt import numpy as np import torch +import torch.nn.functional from pyannote.core import Segment, SlidingWindowFeature -from pytorch_lightning.loggers import TensorBoardLogger, MLFlowLogger +from pytorch_lightning.loggers import MLFlowLogger, TensorBoardLogger from torch.utils.data._utils.collate import default_collate from torchmetrics import Metric -from torchmetrics.classification import BinaryAUROC, MultilabelAUROC, MulticlassAUROC +from torchmetrics.classification import BinaryAUROC, MulticlassAUROC, MultilabelAUROC from pyannote.audio.core.io import AudioFile from pyannote.audio.core.task import Problem +from pyannote.audio.utils.powerset import Powerset from pyannote.audio.utils.random import create_rng_for_worker @@ -125,6 +127,13 @@ class SegmentationTaskMixin: random.shuffle(self._validation) + def setup_loss_func(self): + if self.specifications.powerset: + self.model.powerset = Powerset( + len(self.specifications.classes), + self.specifications.powerset_max_classes, + ) + def default_metric( self, ) -> Union[Metric, Sequence[Metric], Dict[str, Metric]]: @@ -372,10 +381,12 @@ class SegmentationTaskMixin: _, num_frames, _ = y_pred.shape # y_pred = (batch_size, num_frames, num_classes) - # - remove warm-up frames - # - downsample remaining frames + # compute warmup frames boundaries and weight warm_up_left = round(self.warm_up[0] / self.duration * num_frames) warm_up_right = round(self.warm_up[1] / self.duration * num_frames) + + # - remove warm-up frames + # - downsample remaining frames preds = y_pred[:, warm_up_left : num_frames - warm_up_right : 10] target = y[:, warm_up_left : num_frames - warm_up_right : 10] @@ -383,7 +394,6 @@ class SegmentationTaskMixin: # pyannote.audio is more explicit so we have to reshape target and preds for # torchmetrics to be happy... more details can be found here: # https://torchmetrics.readthedocs.io/en/latest/references/modules.html#input-types - if self.specifications.problem == Problem.BINARY_CLASSIFICATION: # target: shape (batch_size, num_frames), type binary # preds: shape (batch_size, num_frames, 1), type float diff --git a/pyannote/audio/tasks/segmentation/multilabel.py b/pyannote/audio/tasks/segmentation/multilabel.py index a501554b..20632b61 100644 --- a/pyannote/audio/tasks/segmentation/multilabel.py +++ b/pyannote/audio/tasks/segmentation/multilabel.py @@ -36,7 +36,7 @@ from pyannote.audio.tasks.segmentation.mixins import SegmentationTaskMixin class MultiLabelSegmentation(SegmentationTaskMixin, Task): """Generic multi-label segmentation - Multi-label segmentation is the process of detecting temporal intervals + Multi-label segmentation is the process of detecting temporal intervals when a specific audio class is active. Example use cases include speaker tracking, gender (male/female) @@ -80,18 +80,18 @@ class MultiLabelSegmentation(SegmentationTaskMixin, Task): """ def __init__( - self, - protocol: Protocol, - classes: Optional[List[str]] = None, - duration: float = 2.0, - warm_up: Union[float, Tuple[float, float]] = 0.0, - balance: Text = None, - weight: Text = None, - batch_size: int = 32, - num_workers: int = None, - pin_memory: bool = False, - augmentation: BaseWaveformTransform = None, - metric: Union[Metric, Sequence[Metric], Dict[str, Metric]] = None, + self, + protocol: Protocol, + classes: Optional[List[str]] = None, + duration: float = 2.0, + warm_up: Union[float, Tuple[float, float]] = 0.0, + balance: Text = None, + weight: Text = None, + batch_size: int = 32, + num_workers: int = None, + pin_memory: bool = False, + augmentation: BaseWaveformTransform = None, + metric: Union[Metric, Sequence[Metric], Dict[str, Metric]] = None, ): super().__init__( protocol, @@ -109,7 +109,7 @@ class MultiLabelSegmentation(SegmentationTaskMixin, Task): self.classes = classes # task specification depends on the data: we do not know in advance which - # classes should be detected. therefore, we postpone the definition of + # classes should be detected. therefore, we postpone the definition of # specifications to setup() def setup(self, stage: Optional[str] = None): diff --git a/pyannote/audio/tasks/segmentation/segmentation.py b/pyannote/audio/tasks/segmentation/segmentation.py index def43eb3..6dc68a44 100644 --- a/pyannote/audio/tasks/segmentation/segmentation.py +++ b/pyannote/audio/tasks/segmentation/segmentation.py @@ -20,13 +20,18 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +import math +import warnings from collections import Counter from typing import Dict, Optional, Sequence, Text, Tuple, Union import numpy as np import torch +import torch.nn.functional +from matplotlib import pyplot as plt from pyannote.core import SlidingWindow from pyannote.database import Protocol +from pytorch_lightning.loggers import MLFlowLogger, TensorBoardLogger from torch_audiomentations.core.transforms_interface import BaseWaveformTransform from torchmetrics import Metric from typing_extensions import Literal @@ -34,13 +39,17 @@ from typing_extensions import Literal from pyannote.audio.core.task import Problem, Resolution, Specifications, Task from pyannote.audio.tasks.segmentation.mixins import SegmentationTaskMixin from pyannote.audio.torchmetrics import ( + DiarizationErrorRate, + FalseAlarmRate, + MissedDetectionRate, OptimalDiarizationErrorRate, OptimalDiarizationErrorRateThreshold, OptimalFalseAlarmRate, OptimalMissedDetectionRate, OptimalSpeakerConfusionRate, + SpeakerConfusionRate, ) -from pyannote.audio.utils.loss import binary_cross_entropy, mse_loss +from pyannote.audio.utils.loss import binary_cross_entropy, mse_loss, nll_loss from pyannote.audio.utils.permutation import permutate @@ -53,9 +62,13 @@ class Segmentation(SegmentationTaskMixin, Task): pyannote.database protocol duration : float, optional Chunks duration. Defaults to 2s. - max_num_speakers : int, optional - Force maximum number of speakers per chunk (must be at least 2). + max_speakers_per_chunk : int, optional + Maximum number of speakers per chunk (must be at least 2). Defaults to estimating it from the training set. + max_speakers_per_frame : int, optional + Maximum number of (overlapping) speakers per frame. + Setting this value to 1 or more enables `powerset multi-class` training. + Default behavior is to use `multi-label` training. warm_up : float or (float, float), optional Use that many seconds on the left- and rightmost parts of each chunk to warm up the model. While the model does process those left- and right-most @@ -67,7 +80,7 @@ class Segmentation(SegmentationTaskMixin, Task): For instance, setting `balance` to "uri" will make sure that each file will be equally represented in the training samples. weight: str, optional - When provided, use this key to as frame-wise weight in loss function. + When provided, use this key as frame-wise weight in loss function. batch_size : int, optional Number of training samples per batch. Defaults to 32. num_workers : int, optional @@ -80,26 +93,32 @@ class Segmentation(SegmentationTaskMixin, Task): augmentation : BaseWaveformTransform, optional torch_audiomentations waveform transform, used by dataloader during training. - loss : {"bce", "mse"}, optional - Permutation-invariant segmentation loss. Defaults to "bce". vad_loss : {"bce", "mse"}, optional Add voice activity detection loss. + Cannot be used in conjunction with `max_speakers_per_frame`. metric : optional Validation metric(s). Can be anything supported by torchmetrics.MetricCollection. Defaults to AUROC (area under the ROC curve). - Reference + References ---------- Hervé Bredin and Antoine Laurent "End-To-End Speaker Segmentation for Overlap-Aware Resegmentation." Proc. Interspeech 2021 + + Zhihao Du, Shiliang Zhang, Siqi Zheng, and Zhijie Yan + "Speaker Embedding-aware Neural Diarization: an Efficient Framework for Overlapping + Speech Diarization in Meeting Scenarios" + https://arxiv.org/abs/2203.09767 + """ def __init__( self, protocol: Protocol, duration: float = 2.0, - max_num_speakers: int = None, + max_speakers_per_chunk: int = None, + max_speakers_per_frame: int = None, warm_up: Union[float, Tuple[float, float]] = 0.0, balance: Text = None, weight: Text = None, @@ -107,9 +126,10 @@ class Segmentation(SegmentationTaskMixin, Task): num_workers: int = None, pin_memory: bool = False, augmentation: BaseWaveformTransform = None, - loss: Literal["bce", "mse"] = "bce", vad_loss: Literal["bce", "mse"] = None, metric: Union[Metric, Sequence[Metric], Dict[str, Metric]] = None, + max_num_speakers: int = None, # deprecated in favor of `max_speakers_per_chunk`` + loss: Literal["bce", "mse"] = None, # deprecated ): super().__init__( @@ -123,20 +143,37 @@ class Segmentation(SegmentationTaskMixin, Task): metric=metric, ) - self.max_num_speakers = max_num_speakers + # deprecation warnings + if max_speakers_per_chunk is None and max_num_speakers is not None: + max_speakers_per_chunk = max_num_speakers + warnings.warn( + "`max_num_speakers` has been deprecated in favor of `max_speakers_per_chunk`." + ) + if loss is not None: + warnings.warn("`loss` has been deprecated and has no effect.") + + # parameter validation + if max_speakers_per_frame is not None: + if max_speakers_per_frame < 1: + raise ValueError( + f"`max_speakers_per_frame` must be 1 or more (you used {max_speakers_per_frame})." + ) + if vad_loss is not None: + raise ValueError( + "`vad_loss` cannot be used jointly with `max_speakers_per_frame`" + ) + + self.max_speakers_per_chunk = max_speakers_per_chunk + self.max_speakers_per_frame = max_speakers_per_frame self.balance = balance self.weight = weight - - if loss not in ["bce", "mse"]: - raise ValueError("'loss' must be one of {'bce', 'mse'}.") - self.loss = loss self.vad_loss = vad_loss def setup(self, stage: Optional[str] = None): super().setup(stage=stage) - if self.max_num_speakers is None: + if self.max_speakers_per_chunk is None: # TODO: optimize this @@ -165,19 +202,31 @@ class Segmentation(SegmentationTaskMixin, Task): num_speakers = num_speakers[sorting_indices] counts = counts[sorting_indices] - self.max_num_speakers = max( + self.max_speakers_per_chunk = max( 2, num_speakers[np.where(np.cumsum(counts) / np.sum(counts) > 0.99)[0][0]], ) + if ( + self.max_speakers_per_frame is not None + and self.max_speakers_per_frame > self.max_speakers_per_chunk + ): + raise ValueError( + f"`max_speakers_per_frame` ({self.max_speakers_per_frame}) must be smaller " + f"than `max_speakers_per_chunk` ({self.max_speakers_per_chunk})" + ) + # now that we know about the number of speakers upper bound # we can set task specifications self.specifications = Specifications( - problem=Problem.MULTI_LABEL_CLASSIFICATION, + problem=Problem.MULTI_LABEL_CLASSIFICATION + if self.max_speakers_per_frame is None + else Problem.MONO_LABEL_CLASSIFICATION, resolution=Resolution.FRAME, duration=self.duration, warm_up=self.warm_up, - classes=[f"speaker#{i+1}" for i in range(self.max_num_speakers)], + classes=[f"speaker#{i+1}" for i in range(self.max_speakers_per_chunk)], + powerset_max_classes=self.max_speakers_per_frame, permutation_invariant=True, ) @@ -193,27 +242,27 @@ class Segmentation(SegmentationTaskMixin, Task): Returns ------- - y : (batch_size, num_frames, max_num_speakers) tensor - Same as collated_y, except we only keep ``max_num_speakers`` most + y : (batch_size, num_frames, max_speakers_per_chunk) tensor + Same as collated_y, except we only keep ``max_speakers_per_chunk`` most talkative speakers (per sample). """ batch_size, num_frames, num_speakers = collated_y.shape # maximum number of active speakers in a chunk - max_num_speakers = torch.max( + max_speakers_per_chunk = torch.max( torch.sum(torch.sum(collated_y, dim=1) > 0.0, dim=1) ) # sort speakers in descending talkativeness order indices = torch.argsort(torch.sum(collated_y, dim=1), dim=1, descending=True) - # keep max_num_speakers most talkative speakers, for each chunk + # keep max_speakers_per_chunk most talkative speakers, for each chunk y = torch.zeros( - (batch_size, num_frames, max_num_speakers), dtype=collated_y.dtype + (batch_size, num_frames, max_speakers_per_chunk), dtype=collated_y.dtype ) for b, index in enumerate(indices): - for k, i in zip(range(max_num_speakers), index): + for k, i in zip(range(max_speakers_per_chunk), index): y[b, :, k] = collated_y[b, :, i.item()] return y @@ -241,14 +290,15 @@ class Segmentation(SegmentationTaskMixin, Task): Permutation-invariant segmentation loss """ - if self.loss == "bce": + if self.specifications.powerset: + seg_loss = nll_loss( + permutated_prediction, torch.argmax(target, dim=-1), weight=weight + ) + else: seg_loss = binary_cross_entropy( permutated_prediction, target.float(), weight=weight ) - elif self.loss == "mse": - seg_loss = mse_loss(permutated_prediction, target.float(), weight=weight) - return seg_loss def voice_activity_detection_loss( @@ -289,7 +339,7 @@ class Segmentation(SegmentationTaskMixin, Task): return loss def training_step(self, batch, batch_idx: int): - """Compute permutation-invariant binary cross-entropy + """Compute permutation-invariant segmentation loss Parameters ---------- @@ -313,7 +363,7 @@ class Segmentation(SegmentationTaskMixin, Task): # drop samples that contain too many speakers num_speakers: torch.Tensor = torch.sum(torch.any(target, dim=1), dim=1) - keep: torch.Tensor = num_speakers <= self.max_num_speakers + keep: torch.Tensor = num_speakers <= self.max_speakers_per_chunk target = target[keep] waveform = waveform[keep] @@ -337,9 +387,6 @@ class Segmentation(SegmentationTaskMixin, Task): batch_size, num_frames, _ = prediction.shape # (batch_size, num_frames, num_classes) - # find optimal permutation - permutated_prediction, _ = permutate(target, prediction) - # frames weight weight_key = getattr(self, "weight", None) weight = batch.get( @@ -354,7 +401,26 @@ class Segmentation(SegmentationTaskMixin, Task): warm_up_right = round(self.warm_up[1] / self.duration * num_frames) weight[:, num_frames - warm_up_right :] = 0.0 - seg_loss = self.segmentation_loss(permutated_prediction, target, weight=weight) + if self.specifications.powerset: + + powerset = torch.nn.functional.one_hot( + torch.argmax(prediction, dim=-1), + self.model.powerset.num_powerset_classes, + ).float() + multilabel = self.model.powerset.to_multilabel(powerset) + permutated_target, _ = permutate(multilabel, target) + permutated_target_powerset = self.model.powerset.to_powerset( + permutated_target.float() + ) + seg_loss = self.segmentation_loss( + prediction, permutated_target_powerset, weight=weight + ) + + else: + permutated_prediction, _ = permutate(target, prediction) + seg_loss = self.segmentation_loss( + permutated_prediction, target, weight=weight + ) self.model.log( f"{self.logging_prefix}TrainSegLoss", @@ -369,9 +435,18 @@ class Segmentation(SegmentationTaskMixin, Task): vad_loss = 0.0 else: - vad_loss = self.voice_activity_detection_loss( - permutated_prediction, target, weight=weight - ) + + # TODO: vad_loss probably does not make sense in powerset mode + # because first class (empty set of labels) does exactly this... + if self.specifications.powerset: + vad_loss = self.voice_activity_detection_loss( + prediction, permutated_target_powerset, weight=weight + ) + + else: + vad_loss = self.voice_activity_detection_loss( + permutated_prediction, target, weight=weight + ) self.model.log( f"{self.logging_prefix}TrainVADLoss", @@ -399,6 +474,15 @@ class Segmentation(SegmentationTaskMixin, Task): self, ) -> Union[Metric, Sequence[Metric], Dict[str, Metric]]: """Returns diarization error rate and its components""" + + if self.specifications.powerset: + return [ + DiarizationErrorRate(0.5), + SpeakerConfusionRate(0.5), + MissedDetectionRate(0.5), + FalseAlarmRate(0.5), + ] + return [ OptimalDiarizationErrorRate(), OptimalDiarizationErrorRateThreshold(), @@ -407,6 +491,227 @@ class Segmentation(SegmentationTaskMixin, Task): OptimalFalseAlarmRate(), ] + def train__iter__(self): + for chunk in super().train__iter__(): + # TODO: document why this filtering is needed + if self.specifications.powerset: + if len(chunk["y"].labels) <= self.max_speakers_per_chunk: + yield chunk + else: + yield chunk + + # TODO: no need to compute gradient in this method + def validation_step(self, batch, batch_idx: int): + """Compute validation loss and metric + + Parameters + ---------- + batch : dict of torch.Tensor + Current batch. + batch_idx: int + Batch index. + """ + + # target + target = batch["y"] + # (batch_size, num_frames, num_speakers) + + waveform = batch["X"] + # (batch_size, num_channels, num_samples) + + # TODO: should we handle validation samples with too many speakers + # waveform = waveform[keep] + # target = target[keep] + + # forward pass + prediction = self.model(waveform) + batch_size, num_frames, _ = prediction.shape + + # frames weight + weight_key = getattr(self, "weight", None) + weight = batch.get( + weight_key, + torch.ones(batch_size, num_frames, 1, device=self.model.device), + ) + # (batch_size, num_frames, 1) + + # warm-up + warm_up_left = round(self.warm_up[0] / self.duration * num_frames) + weight[:, :warm_up_left] = 0.0 + warm_up_right = round(self.warm_up[1] / self.duration * num_frames) + weight[:, num_frames - warm_up_right :] = 0.0 + + if self.specifications.powerset: + + powerset = torch.nn.functional.one_hot( + torch.argmax(prediction, dim=-1), + self.model.powerset.num_powerset_classes, + ).float() + multilabel = self.model.powerset.to_multilabel(powerset) + permutated_target, _ = permutate(multilabel, target) + + # FIXME: handle case where target have too many speakers? + # since we don't need + permutated_target_powerset = self.model.powerset.to_powerset( + permutated_target.float() + ) + seg_loss = self.segmentation_loss( + prediction, permutated_target_powerset, weight=weight + ) + + else: + permutated_prediction, _ = permutate(target, prediction) + seg_loss = self.segmentation_loss( + permutated_prediction, target, weight=weight + ) + + self.model.log( + f"{self.logging_prefix}ValSegLoss", + seg_loss, + on_step=False, + on_epoch=True, + prog_bar=False, + logger=True, + ) + + if self.vad_loss is None: + vad_loss = 0.0 + + else: + + # TODO: vad_loss probably does not make sense in powerset mode + # because first class (empty set of labels) does exactly this... + if self.specifications.powerset: + vad_loss = self.voice_activity_detection_loss( + prediction, permutated_target_powerset, weight=weight + ) + + else: + vad_loss = self.voice_activity_detection_loss( + permutated_prediction, target, weight=weight + ) + + self.model.log( + f"{self.logging_prefix}ValVADLoss", + vad_loss, + on_step=False, + on_epoch=True, + prog_bar=False, + logger=True, + ) + + loss = seg_loss + vad_loss + + self.model.log( + f"{self.logging_prefix}ValLoss", + loss, + on_step=False, + on_epoch=True, + prog_bar=False, + logger=True, + ) + + if self.specifications.powerset: + self.model.validation_metric( + torch.transpose( + multilabel[:, warm_up_left : num_frames - warm_up_right], 1, 2 + ), + torch.transpose( + target[:, warm_up_left : num_frames - warm_up_right], 1, 2 + ), + ) + else: + self.model.validation_metric( + torch.transpose( + prediction[:, warm_up_left : num_frames - warm_up_right], 1, 2 + ), + torch.transpose( + target[:, warm_up_left : num_frames - warm_up_right], 1, 2 + ), + ) + + self.model.log_dict( + self.model.validation_metric, + on_step=False, + on_epoch=True, + prog_bar=True, + logger=True, + ) + + # log first batch visualization every 2^n epochs. + if ( + self.model.current_epoch == 0 + or math.log2(self.model.current_epoch) % 1 > 0 + or batch_idx > 0 + ): + return + + # visualize first 9 validation samples of first batch in Tensorboard/MLflow + + if self.specifications.powerset: + y = permutated_target_powerset.float().cpu().numpy() + y_pred = multilabel.cpu().numpy() + else: + y = target.float().cpu().numpy() + y_pred = permutated_prediction.cpu().numpy() + + # prepare 3 x 3 grid (or smaller if batch size is smaller) + num_samples = min(self.batch_size, 9) + nrows = math.ceil(math.sqrt(num_samples)) + ncols = math.ceil(num_samples / nrows) + fig, axes = plt.subplots( + nrows=2 * nrows, ncols=ncols, figsize=(8, 5), squeeze=False + ) + + # reshape target so that there is one line per class when plotting it + y[y == 0] = np.NaN + if len(y.shape) == 2: + y = y[:, :, np.newaxis] + y *= np.arange(y.shape[2]) + + # plot each sample + for sample_idx in range(num_samples): + + # find where in the grid it should be plotted + row_idx = sample_idx // nrows + col_idx = sample_idx % ncols + + # plot target + ax_ref = axes[row_idx * 2 + 0, col_idx] + sample_y = y[sample_idx] + ax_ref.plot(sample_y) + ax_ref.set_xlim(0, len(sample_y)) + ax_ref.set_ylim(-1, sample_y.shape[1]) + ax_ref.get_xaxis().set_visible(False) + ax_ref.get_yaxis().set_visible(False) + + # plot predictions + ax_hyp = axes[row_idx * 2 + 1, col_idx] + sample_y_pred = y_pred[sample_idx] + ax_hyp.axvspan(0, warm_up_left, color="k", alpha=0.5, lw=0) + ax_hyp.axvspan( + num_frames - warm_up_right, num_frames, color="k", alpha=0.5, lw=0 + ) + ax_hyp.plot(sample_y_pred) + ax_hyp.set_ylim(-0.1, 1.1) + ax_hyp.set_xlim(0, len(sample_y)) + ax_hyp.get_xaxis().set_visible(False) + + plt.tight_layout() + + if isinstance(self.model.logger, TensorBoardLogger): + self.model.logger.experiment.add_figure( + f"{self.logging_prefix}ValSamples", fig, self.model.current_epoch + ) + elif isinstance(self.model.logger, MLFlowLogger): + self.model.logger.experiment.log_figure( + run_id=self.model.logger.run_id, + figure=fig, + artifact_file=f"{self.logging_prefix}ValSamples_epoch{self.model.current_epoch}.png", + ) + + plt.close(fig) + def main(protocol: str, subset: str = "test", model: str = "pyannote/segmentation"): """Evaluate a segmentation model""" diff --git a/pyannote/audio/utils/permutation.py b/pyannote/audio/utils/permutation.py index 689f4759..91337965 100644 --- a/pyannote/audio/utils/permutation.py +++ b/pyannote/audio/utils/permutation.py @@ -194,7 +194,7 @@ def build_permutation_graph( cost_func: Callable[[torch.Tensor, torch.Tensor], torch.Tensor] = mae_cost_func, ) -> nx.Graph: """Build permutation graph - + Parameters ---------- segmentations : (num_chunks, num_frames, local_num_speakers)-shaped SlidingWindowFeature @@ -203,7 +203,7 @@ def build_permutation_graph( Threshold above which a speaker is considered active. Defaults to 0.5 cost_func : callable Cost function used to find the optimal bijective mapping between speaker activations - of two overlapping chunks. Expects two (num_frames, num_classes) torch.tensor as input + of two overlapping chunks. Expects two (num_frames, num_classes) torch.tensor as input and returns cost as a (num_classes, ) torch.tensor. Defaults to mae_cost_func. Returns diff --git a/pyannote/audio/utils/powerset.py b/pyannote/audio/utils/powerset.py new file mode 100644 index 00000000..a41bf057 --- /dev/null +++ b/pyannote/audio/utils/powerset.py @@ -0,0 +1,153 @@ +# MIT License +# +# Copyright (c) 2023- CNRS +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# AUTHORS +# Hervé BREDIN - https://herve.niderb.fr +# Alexis PLAQUET + +try: + from functools import cached_property +except ImportError: + from backports.cached_property import cached_property + +from itertools import combinations + +import scipy.special +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class Powerset(nn.Module): + """Powerset to multilabel conversion, and back. + + Parameters + ---------- + num_classes : int + Number of regular classes. + max_set_size : int + Maximum number of classes in each set. + """ + + def __init__(self, num_classes: int, max_set_size: int): + super().__init__() + self.num_classes = num_classes + self.max_set_size = max_set_size + + mapping = self.build_mapping() + self.register_buffer("mapping", mapping, persistent=False) + + @cached_property + def num_powerset_classes(self) -> int: + # compute number of subsets of size at most "max_set_size" + # e.g. with num_classes = 3 and max_set_size = 2: + # {}, {0}, {1}, {2}, {0, 1}, {0, 2}, {1, 2} + return int( + sum( + scipy.special.binom(self.num_classes, i) + for i in range(0, self.max_set_size + 1) + ) + ) + + def build_mapping(self) -> torch.Tensor: + mapping = torch.zeros(self.num_powerset_classes, self.num_classes) + powerset_k = 0 + for set_size in range(0, self.max_set_size + 1): + for current_set in combinations(range(self.num_classes), set_size): + mapping[powerset_k, current_set] = 1 + powerset_k += 1 + + return mapping + + def to_multilabel(self, powerset: torch.Tensor) -> torch.Tensor: + """Convert (hard) predictions from powerset to multi-label + + Parameter + --------- + powerset : (batch_size, num_frames, num_powerset_classes) torch.Tensor + Hard predictions in "powerset" space. + + Returns + ------- + multi_label : (batch_size, num_frames, num_classes) torch.Tensor + Hard predictions in "multi-label" space. + + Note + ---- + This method will not complain if `powerset` is provided a soft predictions + (e.g. the output of a softmax-ed classifier). However, in that particular + case, the resulting soft multi-label output will not make much sense. + """ + return torch.matmul(powerset, self.mapping) + + def to_powerset(self, multilabel: torch.Tensor) -> torch.Tensor: + """Convert (hard) predictions from multi-label to powerset + + Parameter + --------- + multi_label : (batch_size, num_frames, num_classes) torch.Tensor + Prediction in "multi-label" space. + + Returns + ------- + powerset : (batch_size, num_frames, num_powerset_classes) torch.Tensor + Hard, one-hot prediction in "powerset" space. + + Note + ---- + This method will not complain if `multilabel` is provided a soft predictions + (e.g. the output of a sigmoid-ed classifier). However, in that particular + case, the resulting powerset output will most likely not make much sense. + """ + return F.one_hot( + torch.argmax(torch.matmul(multilabel, self.mapping.T), dim=-1), + num_classes=self.num_powerset_classes, + ) + + +if __name__ == "__main__": + + powerset = Powerset(3, 3) + print(powerset.mapping) + + # simulate a sequence where each frame is assigned to a different powerset class + one_sequence = [ + [0] * powerset.num_powerset_classes + for _ in range(powerset.num_powerset_classes) + ] + for i in range(powerset.num_powerset_classes): + one_sequence[i][i] = 1.0 + + # make a batch out of this sequence and the same sequence in reverse order + batch_powerset = torch.tensor([one_sequence, one_sequence[::-1]]) + print(batch_powerset) + print(batch_powerset.shape) + + # convert from powerset to multi-label + batch_multilabel = powerset.to_multilabel(batch_powerset) + print(batch_multilabel) + print(batch_multilabel.shape) + + # convert batch to powerset + reconstruction = powerset.to_powerset(batch_multilabel) + print(reconstruction) + print(reconstruction.shape) diff --git a/tests/io_test.py b/tests/io_test.py index 3729c37c..a07ce740 100644 --- a/tests/io_test.py +++ b/tests/io_test.py @@ -84,4 +84,3 @@ def test_can_crop_from_file_like(): assert isinstance(wav, Tensor) assert sr == 16000 assert wav.shape[1] == 0.5 * 16000 - diff --git a/tests/utils/test_powerset.py b/tests/utils/test_powerset.py new file mode 100644 index 00000000..dd12eed4 --- /dev/null +++ b/tests/utils/test_powerset.py @@ -0,0 +1,53 @@ +# MIT License +# +# Copyright (c) 2023- CNRS +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +import torch + +from pyannote.audio.utils.powerset import Powerset + + +def test_roundtrip(): + + for num_classes in range(2, 5): + for max_set_size in range(1, num_classes + 1): + + powerset = Powerset(num_classes, max_set_size) + + # simulate a sequence where each frame is assigned to a different powerset class + one_sequence = [ + [0] * powerset.num_powerset_classes + for _ in range(powerset.num_powerset_classes) + ] + for i in range(powerset.num_powerset_classes): + one_sequence[i][i] = 1.0 + + # make a batch out of this sequence and the same sequence in reverse order + batch_powerset = torch.tensor([one_sequence, one_sequence[::-1]]) + + # convert from powerset to multi-label + batch_multilabel = powerset.to_multilabel(batch_powerset) + + # convert batch back to powerset + reconstruction = powerset.to_powerset(batch_multilabel) + + assert torch.equal(batch_powerset, reconstruction)