feat: add support for "powerset" speaker segmentation training

Co-authored-by: Hervé BREDIN <hbredin@users.noreply.github.com>
This commit is contained in:
FrenchKrabandHervé BREDIN authored and GitHub committed 2023-01-20 11:22:18 +01:00
1 parent 0506ae38c3
commit 12dc20bde1
13 files changed
+694 -99

No files matched your search

+1 -3
View File
@@ -26,6 +26,4 @@ from pyannote.audio import Model
def pretrained(checkpoint: Text):
return Model.from_pretrained(
checkpoint, map_location=lambda storage, loc: storage
)
return Model.from_pretrained(checkpoint, map_location=lambda storage, loc: storage)
+30 -9
View File
@@ -35,6 +35,7 @@ from pyannote.audio.core.io import AudioFile
from pyannote.audio.core.model import Model
from pyannote.audio.core.task import Resolution
from pyannote.audio.utils.permutation import mae_cost_func, permutate
from pyannote.audio.utils.powerset import Powerset
TaskName = Union[Text, None]
@@ -53,23 +54,26 @@ class Inference(BaseInference):
window : {"sliding", "whole"}, optional
Use a "sliding" window and aggregate the corresponding outputs (default)
or just one (potentially long) window covering the "whole" file or chunk.
skip_aggregation : bool, optional
Do not aggregate outputs when using "sliding" window. Defaults to False.
duration : float, optional
Chunk duration, in seconds. Defaults to duration used for training the model.
Has no effect when `window` is "whole".
step : float, optional
Step between consecutive chunks, in seconds. Defaults to warm-up duration when
greater than 0s, otherwise 10% of duration. Has no effect when `window` is "whole".
pre_aggregation_hook : callable, optional
When a callable is provided, it is applied to the model output, just before aggregation.
Takes a (num_chunks, num_frames, dimension) numpy array as input and returns a modified
(num_chunks, num_frames, other_dimension) numpy array passed to overlap-add aggregation.
skip_aggregation : bool, optional
Do not aggregate outputs when using "sliding" window. Defaults to False.
skip_conversion: bool, optional
In case `model` has been trained with `powerset` mode, its output is automatically
converted to `multi-label`, unless `skip_conversion` is set to True.
batch_size : int, optional
Batch size. Larger values make inference faster. Defaults to 32.
device : torch.device, optional
Device used for inference. Defaults to `model.device`.
In case `device` and `model.device` are different, model is sent to device.
pre_aggregation_hook : callable, optional
When a callable is provided, it is applied to the model output, just before aggregation.
Takes a (num_chunks, num_frames, dimension) numpy array as input and returns a modified
(num_chunks, num_frames, other_dimension) numpy array passed to overlap-add aggregation.
use_auth_token : str, optional
When loading a private huggingface.co model, set `use_auth_token`
to True or to a string containing your hugginface.co authentication
@@ -80,12 +84,13 @@ class Inference(BaseInference):
self,
model: Union[Model, Text, Path],
window: Text = "sliding",
skip_aggregation: bool = False,
device: torch.device = None,
duration: float = None,
step: float = None,
batch_size: int = 32,
pre_aggregation_hook: Callable[[np.ndarray], np.ndarray] = None,
skip_aggregation: bool = False,
skip_conversion: bool = False,
device: torch.device = None,
batch_size: int = 32,
use_auth_token: Union[Text, None] = None,
):
@@ -154,11 +159,19 @@ class Inference(BaseInference):
self.step = step
self.batch_size = batch_size
self.skip_conversion = skip_conversion
if specifications.powerset and not self.skip_conversion:
self._powerset = Powerset(
len(specifications.classes), specifications.powerset_max_classes
)
self._powerset.to(self.device)
def to(self, device: torch.device):
"""Send internal model to `device`"""
self.model.to(device)
if self.model.specifications.powerset and not self.skip_conversion:
self._powerset.to(device)
self.device = device
return self
@@ -190,6 +203,14 @@ class Inference(BaseInference):
else:
raise exception
# convert powerset to multi-label unless specifically requested not to
if self.model.specifications.powerset and not self.skip_conversion:
powerset = torch.nn.functional.one_hot(
torch.argmax(outputs, dim=-1),
self.model.specifications.num_powerset_classes,
).float()
outputs = self._powerset.to_multilabel(powerset)
return outputs.cpu().numpy()
def slide(
+32 -1
View File
@@ -25,6 +25,8 @@ from __future__ import annotations
from functools import partial
import scipy.special
try:
from functools import cached_property
except ImportError:
@@ -89,9 +91,38 @@ class Specifications:
# (for classification tasks only) list of classes
classes: Optional[List[Text]] = None
# (for powerset only) max number of simultaneous classes
# (n choose k with k <= powerset_max_classes)
powerset_max_classes: Optional[int] = None
# whether classes are permutation-invariant (e.g. diarization)
permutation_invariant: bool = False
@cached_property
def powerset(self):
if self.powerset_max_classes is None:
return False
if self.problem != Problem.MONO_LABEL_CLASSIFICATION:
raise ValueError(
"`powerset_max_classes` only makes sense with multi-class classification problems."
)
return True
@cached_property
def num_powerset_classes(self) -> int:
# compute number of subsets of size at most "powerset_max_classes"
# e.g. with len(classes) = 3 and powerset_max_classes = 2:
# {}, {0}, {1}, {2}, {0, 1}, {0, 2}, {1, 2}
return int(
sum(
scipy.special.binom(len(self.classes), i)
for i in range(0, self.powerset_max_classes + 1)
)
)
class TrainDataset(IterableDataset):
def __init__(self, task: Task):
@@ -314,7 +345,7 @@ class Task(pl.LightningDataModule):
]:
return binary_cross_entropy(prediction, target, weight=weight)
elif specifications.problem == Problem.MONO_LABEL_CLASSIFICATION:
elif specifications.problem in [Problem.MONO_LABEL_CLASSIFICATION]:
return nll_loss(prediction, target, weight=weight)
else:
+10 -3
View File
@@ -27,12 +27,12 @@ import torch
import torch.nn as nn
import torch.nn.functional as F
from einops import rearrange
from pyannote.core.utils.generators import pairwise
from pyannote.audio.core.model import Model
from pyannote.audio.core.task import Task
from pyannote.audio.models.blocks.sincnet import SincNet
from pyannote.audio.utils.params import merge_dict
from pyannote.core.utils.generators import pairwise
class PyanNet(Model):
@@ -130,7 +130,9 @@ class PyanNet(Model):
[
nn.Linear(in_features, out_features)
for in_features, out_features in pairwise(
[lstm_out_features,]
[
lstm_out_features,
]
+ [self.hparams.linear["hidden_size"]]
* self.hparams.linear["num_layers"]
)
@@ -146,7 +148,12 @@ class PyanNet(Model):
2 if self.hparams.lstm["bidirectional"] else 1
)
self.classifier = nn.Linear(in_features, len(self.specifications.classes))
if self.specifications.powerset:
out_features = self.specifications.num_powerset_classes
else:
out_features = len(self.specifications.classes)
self.classifier = nn.Linear(in_features, out_features)
self.activation = self.default_activation()
def forward(self, waveforms: torch.Tensor) -> torch.Tensor:
+40 -23
View File
@@ -147,10 +147,16 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
)
self._frames: SlidingWindow = self._segmentation.model.introspection.frames
self.segmentation = ParamDict(
threshold=Uniform(0.1, 0.9),
min_duration_off=Uniform(0.0, 1.0),
)
if self._segmentation.model.specifications.powerset:
self.segmentation = ParamDict(
min_duration_off=Uniform(0.0, 1.0),
)
else:
self.segmentation = ParamDict(
threshold=Uniform(0.1, 0.9),
min_duration_off=Uniform(0.0, 1.0),
)
if self.klustering == "OracleClustering":
metric = "not_applicable"
@@ -260,18 +266,19 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
embeddings : (num_chunks, num_speakers, dimension) array
"""
# when optimizing the hyper-parameters of this pipeline with frozen "segmentation_onset",
# one can reuse the embeddings from the first trial, bringing a massive speed up to
# the optimization process (and hence allowing to use a larger search space).
# when optimizing the hyper-parameters of this pipeline with frozen
# "segmentation.threshold", one can reuse the embeddings from the first trial,
# bringing a massive speed up to the optimization process (and hence allowing to use
# a larger search space).
if self.training:
# we only re-use embeddings if they were extracted based on the same value of the
# "segmentation_onset" hyperparameter and "embedding_exclude_overlap" parameter.
# "segmentation.threshold" hyperparameter or if the segmentation model relies on
# `powerset` mode
cache = file.get("training_cache/embeddings", dict())
if (
cache.get("segmentation.threshold", None) == self.segmentation.threshold
and cache.get("embedding_exclude_overlap", None)
== self.embedding_exclude_overlap
if ("embeddings" in cache) and (
self._segmentation.model.specifications.powerset
or (cache["segmentation.threshold"] == self.segmentation.threshold)
):
return cache["embeddings"]
@@ -369,11 +376,16 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
# caching embeddings for subsequent trials
# (see comments at the top of this method for more details)
if self.training:
file["training_cache/embeddings"] = {
"segmentation.threshold": self.segmentation.threshold,
"embedding_exclude_overlap": self.embedding_exclude_overlap,
"embeddings": embeddings,
}
if self._segmentation.model.specifications.powerset:
file["training_cache/embeddings"] = {
"embeddings": embeddings,
}
else:
file["training_cache/embeddings"] = {
"segmentation.threshold": self.segmentation.threshold,
"embeddings": embeddings,
}
return embeddings
@@ -479,7 +491,9 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
# estimate frame-level number of instantaneous speakers
count = self.speaker_count(
segmentations,
onset=self.segmentation.threshold,
onset=0.5
if self._segmentation.model.specifications.powerset
else self.segmentation.threshold,
frames=self._frames,
)
hook("speaker_counting", count)
@@ -491,11 +505,14 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
return Annotation(uri=file["uri"])
# binarize segmentation
binarized_segmentations: SlidingWindowFeature = binarize(
segmentations,
onset=self.segmentation.threshold,
initial_state=False,
)
if self._segmentation.model.specifications.powerset:
binarized_segmentations = segmentations
else:
binarized_segmentations: SlidingWindowFeature = binarize(
segmentations,
onset=self.segmentation.threshold,
initial_state=False,
)
if self.klustering == "OracleClustering":
embeddings = None
+2 -1
View File
@@ -25,9 +25,10 @@ from .segmentation.overlapped_speech_detection import ( # isort:skip
OverlappedSpeechDetection,
)
from .segmentation.multilabel import MultiLabelSegmentation # isort:skip
from .segmentation.multilabel import MultiLabelSegmentation # isort:skip
from .segmentation.segmentation import Segmentation # isort:skip
from .embedding.arcface import SupervisedRepresentationLearningWithArcFace # isort:skip
SpeakerEmbedding = SupervisedRepresentationLearningWithArcFace
__all__ = [
+15 -5
View File
@@ -29,14 +29,16 @@ from typing import Dict, Optional, Sequence, Text, Union
import matplotlib.pyplot as plt
import numpy as np
import torch
import torch.nn.functional
from pyannote.core import Segment, SlidingWindowFeature
from pytorch_lightning.loggers import TensorBoardLogger, MLFlowLogger
from pytorch_lightning.loggers import MLFlowLogger, TensorBoardLogger
from torch.utils.data._utils.collate import default_collate
from torchmetrics import Metric
from torchmetrics.classification import BinaryAUROC, MultilabelAUROC, MulticlassAUROC
from torchmetrics.classification import BinaryAUROC, MulticlassAUROC, MultilabelAUROC
from pyannote.audio.core.io import AudioFile
from pyannote.audio.core.task import Problem
from pyannote.audio.utils.powerset import Powerset
from pyannote.audio.utils.random import create_rng_for_worker
@@ -125,6 +127,13 @@ class SegmentationTaskMixin:
random.shuffle(self._validation)
def setup_loss_func(self):
if self.specifications.powerset:
self.model.powerset = Powerset(
len(self.specifications.classes),
self.specifications.powerset_max_classes,
)
def default_metric(
self,
) -> Union[Metric, Sequence[Metric], Dict[str, Metric]]:
@@ -372,10 +381,12 @@ class SegmentationTaskMixin:
_, num_frames, _ = y_pred.shape
# y_pred = (batch_size, num_frames, num_classes)
# - remove warm-up frames
# - downsample remaining frames
# compute warmup frames boundaries and weight
warm_up_left = round(self.warm_up[0] / self.duration * num_frames)
warm_up_right = round(self.warm_up[1] / self.duration * num_frames)
# - remove warm-up frames
# - downsample remaining frames
preds = y_pred[:, warm_up_left : num_frames - warm_up_right : 10]
target = y[:, warm_up_left : num_frames - warm_up_right : 10]
@@ -383,7 +394,6 @@ class SegmentationTaskMixin:
# pyannote.audio is more explicit so we have to reshape target and preds for
# torchmetrics to be happy... more details can be found here:
# https://torchmetrics.readthedocs.io/en/latest/references/modules.html#input-types
if self.specifications.problem == Problem.BINARY_CLASSIFICATION:
# target: shape (batch_size, num_frames), type binary
# preds: shape (batch_size, num_frames, 1), type float
+14 -14
View File
@@ -36,7 +36,7 @@ from pyannote.audio.tasks.segmentation.mixins import SegmentationTaskMixin
class MultiLabelSegmentation(SegmentationTaskMixin, Task):
"""Generic multi-label segmentation
Multi-label segmentation is the process of detecting temporal intervals
Multi-label segmentation is the process of detecting temporal intervals
when a specific audio class is active.
Example use cases include speaker tracking, gender (male/female)
@@ -80,18 +80,18 @@ class MultiLabelSegmentation(SegmentationTaskMixin, Task):
"""
def __init__(
self,
protocol: Protocol,
classes: Optional[List[str]] = None,
duration: float = 2.0,
warm_up: Union[float, Tuple[float, float]] = 0.0,
balance: Text = None,
weight: Text = None,
batch_size: int = 32,
num_workers: int = None,
pin_memory: bool = False,
augmentation: BaseWaveformTransform = None,
metric: Union[Metric, Sequence[Metric], Dict[str, Metric]] = None,
self,
protocol: Protocol,
classes: Optional[List[str]] = None,
duration: float = 2.0,
warm_up: Union[float, Tuple[float, float]] = 0.0,
balance: Text = None,
weight: Text = None,
batch_size: int = 32,
num_workers: int = None,
pin_memory: bool = False,
augmentation: BaseWaveformTransform = None,
metric: Union[Metric, Sequence[Metric], Dict[str, Metric]] = None,
):
super().__init__(
protocol,
@@ -109,7 +109,7 @@ class MultiLabelSegmentation(SegmentationTaskMixin, Task):
self.classes = classes
# task specification depends on the data: we do not know in advance which
# classes should be detected. therefore, we postpone the definition of
# classes should be detected. therefore, we postpone the definition of
# specifications to setup()
def setup(self, stage: Optional[str] = None):
+342 -37
View File
@@ -20,13 +20,18 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import math
import warnings
from collections import Counter
from typing import Dict, Optional, Sequence, Text, Tuple, Union
import numpy as np
import torch
import torch.nn.functional
from matplotlib import pyplot as plt
from pyannote.core import SlidingWindow
from pyannote.database import Protocol
from pytorch_lightning.loggers import MLFlowLogger, TensorBoardLogger
from torch_audiomentations.core.transforms_interface import BaseWaveformTransform
from torchmetrics import Metric
from typing_extensions import Literal
@@ -34,13 +39,17 @@ from typing_extensions import Literal
from pyannote.audio.core.task import Problem, Resolution, Specifications, Task
from pyannote.audio.tasks.segmentation.mixins import SegmentationTaskMixin
from pyannote.audio.torchmetrics import (
DiarizationErrorRate,
FalseAlarmRate,
MissedDetectionRate,
OptimalDiarizationErrorRate,
OptimalDiarizationErrorRateThreshold,
OptimalFalseAlarmRate,
OptimalMissedDetectionRate,
OptimalSpeakerConfusionRate,
SpeakerConfusionRate,
)
from pyannote.audio.utils.loss import binary_cross_entropy, mse_loss
from pyannote.audio.utils.loss import binary_cross_entropy, mse_loss, nll_loss
from pyannote.audio.utils.permutation import permutate
@@ -53,9 +62,13 @@ class Segmentation(SegmentationTaskMixin, Task):
pyannote.database protocol
duration : float, optional
Chunks duration. Defaults to 2s.
max_num_speakers : int, optional
Force maximum number of speakers per chunk (must be at least 2).
max_speakers_per_chunk : int, optional
Maximum number of speakers per chunk (must be at least 2).
Defaults to estimating it from the training set.
max_speakers_per_frame : int, optional
Maximum number of (overlapping) speakers per frame.
Setting this value to 1 or more enables `powerset multi-class` training.
Default behavior is to use `multi-label` training.
warm_up : float or (float, float), optional
Use that many seconds on the left- and rightmost parts of each chunk
to warm up the model. While the model does process those left- and right-most
@@ -67,7 +80,7 @@ class Segmentation(SegmentationTaskMixin, Task):
For instance, setting `balance` to "uri" will make sure that each file will be
equally represented in the training samples.
weight: str, optional
When provided, use this key to as frame-wise weight in loss function.
When provided, use this key as frame-wise weight in loss function.
batch_size : int, optional
Number of training samples per batch. Defaults to 32.
num_workers : int, optional
@@ -80,26 +93,32 @@ class Segmentation(SegmentationTaskMixin, Task):
augmentation : BaseWaveformTransform, optional
torch_audiomentations waveform transform, used by dataloader
during training.
loss : {"bce", "mse"}, optional
Permutation-invariant segmentation loss. Defaults to "bce".
vad_loss : {"bce", "mse"}, optional
Add voice activity detection loss.
Cannot be used in conjunction with `max_speakers_per_frame`.
metric : optional
Validation metric(s). Can be anything supported by torchmetrics.MetricCollection.
Defaults to AUROC (area under the ROC curve).
Reference
References
----------
Hervé Bredin and Antoine Laurent
"End-To-End Speaker Segmentation for Overlap-Aware Resegmentation."
Proc. Interspeech 2021
Zhihao Du, Shiliang Zhang, Siqi Zheng, and Zhijie Yan
"Speaker Embedding-aware Neural Diarization: an Efficient Framework for Overlapping
Speech Diarization in Meeting Scenarios"
https://arxiv.org/abs/2203.09767
"""
def __init__(
self,
protocol: Protocol,
duration: float = 2.0,
max_num_speakers: int = None,
max_speakers_per_chunk: int = None,
max_speakers_per_frame: int = None,
warm_up: Union[float, Tuple[float, float]] = 0.0,
balance: Text = None,
weight: Text = None,
@@ -107,9 +126,10 @@ class Segmentation(SegmentationTaskMixin, Task):
num_workers: int = None,
pin_memory: bool = False,
augmentation: BaseWaveformTransform = None,
loss: Literal["bce", "mse"] = "bce",
vad_loss: Literal["bce", "mse"] = None,
metric: Union[Metric, Sequence[Metric], Dict[str, Metric]] = None,
max_num_speakers: int = None, # deprecated in favor of `max_speakers_per_chunk``
loss: Literal["bce", "mse"] = None, # deprecated
):
super().__init__(
@@ -123,20 +143,37 @@ class Segmentation(SegmentationTaskMixin, Task):
metric=metric,
)
self.max_num_speakers = max_num_speakers
# deprecation warnings
if max_speakers_per_chunk is None and max_num_speakers is not None:
max_speakers_per_chunk = max_num_speakers
warnings.warn(
"`max_num_speakers` has been deprecated in favor of `max_speakers_per_chunk`."
)
if loss is not None:
warnings.warn("`loss` has been deprecated and has no effect.")
# parameter validation
if max_speakers_per_frame is not None:
if max_speakers_per_frame < 1:
raise ValueError(
f"`max_speakers_per_frame` must be 1 or more (you used {max_speakers_per_frame})."
)
if vad_loss is not None:
raise ValueError(
"`vad_loss` cannot be used jointly with `max_speakers_per_frame`"
)
self.max_speakers_per_chunk = max_speakers_per_chunk
self.max_speakers_per_frame = max_speakers_per_frame
self.balance = balance
self.weight = weight
if loss not in ["bce", "mse"]:
raise ValueError("'loss' must be one of {'bce', 'mse'}.")
self.loss = loss
self.vad_loss = vad_loss
def setup(self, stage: Optional[str] = None):
super().setup(stage=stage)
if self.max_num_speakers is None:
if self.max_speakers_per_chunk is None:
# TODO: optimize this
@@ -165,19 +202,31 @@ class Segmentation(SegmentationTaskMixin, Task):
num_speakers = num_speakers[sorting_indices]
counts = counts[sorting_indices]
self.max_num_speakers = max(
self.max_speakers_per_chunk = max(
2,
num_speakers[np.where(np.cumsum(counts) / np.sum(counts) > 0.99)[0][0]],
)
if (
self.max_speakers_per_frame is not None
and self.max_speakers_per_frame > self.max_speakers_per_chunk
):
raise ValueError(
f"`max_speakers_per_frame` ({self.max_speakers_per_frame}) must be smaller "
f"than `max_speakers_per_chunk` ({self.max_speakers_per_chunk})"
)
# now that we know about the number of speakers upper bound
# we can set task specifications
self.specifications = Specifications(
problem=Problem.MULTI_LABEL_CLASSIFICATION,
problem=Problem.MULTI_LABEL_CLASSIFICATION
if self.max_speakers_per_frame is None
else Problem.MONO_LABEL_CLASSIFICATION,
resolution=Resolution.FRAME,
duration=self.duration,
warm_up=self.warm_up,
classes=[f"speaker#{i+1}" for i in range(self.max_num_speakers)],
classes=[f"speaker#{i+1}" for i in range(self.max_speakers_per_chunk)],
powerset_max_classes=self.max_speakers_per_frame,
permutation_invariant=True,
)
@@ -193,27 +242,27 @@ class Segmentation(SegmentationTaskMixin, Task):
Returns
-------
y : (batch_size, num_frames, max_num_speakers) tensor
Same as collated_y, except we only keep ``max_num_speakers`` most
y : (batch_size, num_frames, max_speakers_per_chunk) tensor
Same as collated_y, except we only keep ``max_speakers_per_chunk`` most
talkative speakers (per sample).
"""
batch_size, num_frames, num_speakers = collated_y.shape
# maximum number of active speakers in a chunk
max_num_speakers = torch.max(
max_speakers_per_chunk = torch.max(
torch.sum(torch.sum(collated_y, dim=1) > 0.0, dim=1)
)
# sort speakers in descending talkativeness order
indices = torch.argsort(torch.sum(collated_y, dim=1), dim=1, descending=True)
# keep max_num_speakers most talkative speakers, for each chunk
# keep max_speakers_per_chunk most talkative speakers, for each chunk
y = torch.zeros(
(batch_size, num_frames, max_num_speakers), dtype=collated_y.dtype
(batch_size, num_frames, max_speakers_per_chunk), dtype=collated_y.dtype
)
for b, index in enumerate(indices):
for k, i in zip(range(max_num_speakers), index):
for k, i in zip(range(max_speakers_per_chunk), index):
y[b, :, k] = collated_y[b, :, i.item()]
return y
@@ -241,14 +290,15 @@ class Segmentation(SegmentationTaskMixin, Task):
Permutation-invariant segmentation loss
"""
if self.loss == "bce":
if self.specifications.powerset:
seg_loss = nll_loss(
permutated_prediction, torch.argmax(target, dim=-1), weight=weight
)
else:
seg_loss = binary_cross_entropy(
permutated_prediction, target.float(), weight=weight
)
elif self.loss == "mse":
seg_loss = mse_loss(permutated_prediction, target.float(), weight=weight)
return seg_loss
def voice_activity_detection_loss(
@@ -289,7 +339,7 @@ class Segmentation(SegmentationTaskMixin, Task):
return loss
def training_step(self, batch, batch_idx: int):
"""Compute permutation-invariant binary cross-entropy
"""Compute permutation-invariant segmentation loss
Parameters
----------
@@ -313,7 +363,7 @@ class Segmentation(SegmentationTaskMixin, Task):
# drop samples that contain too many speakers
num_speakers: torch.Tensor = torch.sum(torch.any(target, dim=1), dim=1)
keep: torch.Tensor = num_speakers <= self.max_num_speakers
keep: torch.Tensor = num_speakers <= self.max_speakers_per_chunk
target = target[keep]
waveform = waveform[keep]
@@ -337,9 +387,6 @@ class Segmentation(SegmentationTaskMixin, Task):
batch_size, num_frames, _ = prediction.shape
# (batch_size, num_frames, num_classes)
# find optimal permutation
permutated_prediction, _ = permutate(target, prediction)
# frames weight
weight_key = getattr(self, "weight", None)
weight = batch.get(
@@ -354,7 +401,26 @@ class Segmentation(SegmentationTaskMixin, Task):
warm_up_right = round(self.warm_up[1] / self.duration * num_frames)
weight[:, num_frames - warm_up_right :] = 0.0
seg_loss = self.segmentation_loss(permutated_prediction, target, weight=weight)
if self.specifications.powerset:
powerset = torch.nn.functional.one_hot(
torch.argmax(prediction, dim=-1),
self.model.powerset.num_powerset_classes,
).float()
multilabel = self.model.powerset.to_multilabel(powerset)
permutated_target, _ = permutate(multilabel, target)
permutated_target_powerset = self.model.powerset.to_powerset(
permutated_target.float()
)
seg_loss = self.segmentation_loss(
prediction, permutated_target_powerset, weight=weight
)
else:
permutated_prediction, _ = permutate(target, prediction)
seg_loss = self.segmentation_loss(
permutated_prediction, target, weight=weight
)
self.model.log(
f"{self.logging_prefix}TrainSegLoss",
@@ -369,9 +435,18 @@ class Segmentation(SegmentationTaskMixin, Task):
vad_loss = 0.0
else:
vad_loss = self.voice_activity_detection_loss(
permutated_prediction, target, weight=weight
)
# TODO: vad_loss probably does not make sense in powerset mode
# because first class (empty set of labels) does exactly this...
if self.specifications.powerset:
vad_loss = self.voice_activity_detection_loss(
prediction, permutated_target_powerset, weight=weight
)
else:
vad_loss = self.voice_activity_detection_loss(
permutated_prediction, target, weight=weight
)
self.model.log(
f"{self.logging_prefix}TrainVADLoss",
@@ -399,6 +474,15 @@ class Segmentation(SegmentationTaskMixin, Task):
self,
) -> Union[Metric, Sequence[Metric], Dict[str, Metric]]:
"""Returns diarization error rate and its components"""
if self.specifications.powerset:
return [
DiarizationErrorRate(0.5),
SpeakerConfusionRate(0.5),
MissedDetectionRate(0.5),
FalseAlarmRate(0.5),
]
return [
OptimalDiarizationErrorRate(),
OptimalDiarizationErrorRateThreshold(),
@@ -407,6 +491,227 @@ class Segmentation(SegmentationTaskMixin, Task):
OptimalFalseAlarmRate(),
]
def train__iter__(self):
for chunk in super().train__iter__():
# TODO: document why this filtering is needed
if self.specifications.powerset:
if len(chunk["y"].labels) <= self.max_speakers_per_chunk:
yield chunk
else:
yield chunk
# TODO: no need to compute gradient in this method
def validation_step(self, batch, batch_idx: int):
"""Compute validation loss and metric
Parameters
----------
batch : dict of torch.Tensor
Current batch.
batch_idx: int
Batch index.
"""
# target
target = batch["y"]
# (batch_size, num_frames, num_speakers)
waveform = batch["X"]
# (batch_size, num_channels, num_samples)
# TODO: should we handle validation samples with too many speakers
# waveform = waveform[keep]
# target = target[keep]
# forward pass
prediction = self.model(waveform)
batch_size, num_frames, _ = prediction.shape
# frames weight
weight_key = getattr(self, "weight", None)
weight = batch.get(
weight_key,
torch.ones(batch_size, num_frames, 1, device=self.model.device),
)
# (batch_size, num_frames, 1)
# warm-up
warm_up_left = round(self.warm_up[0] / self.duration * num_frames)
weight[:, :warm_up_left] = 0.0
warm_up_right = round(self.warm_up[1] / self.duration * num_frames)
weight[:, num_frames - warm_up_right :] = 0.0
if self.specifications.powerset:
powerset = torch.nn.functional.one_hot(
torch.argmax(prediction, dim=-1),
self.model.powerset.num_powerset_classes,
).float()
multilabel = self.model.powerset.to_multilabel(powerset)
permutated_target, _ = permutate(multilabel, target)
# FIXME: handle case where target have too many speakers?
# since we don't need
permutated_target_powerset = self.model.powerset.to_powerset(
permutated_target.float()
)
seg_loss = self.segmentation_loss(
prediction, permutated_target_powerset, weight=weight
)
else:
permutated_prediction, _ = permutate(target, prediction)
seg_loss = self.segmentation_loss(
permutated_prediction, target, weight=weight
)
self.model.log(
f"{self.logging_prefix}ValSegLoss",
seg_loss,
on_step=False,
on_epoch=True,
prog_bar=False,
logger=True,
)
if self.vad_loss is None:
vad_loss = 0.0
else:
# TODO: vad_loss probably does not make sense in powerset mode
# because first class (empty set of labels) does exactly this...
if self.specifications.powerset:
vad_loss = self.voice_activity_detection_loss(
prediction, permutated_target_powerset, weight=weight
)
else:
vad_loss = self.voice_activity_detection_loss(
permutated_prediction, target, weight=weight
)
self.model.log(
f"{self.logging_prefix}ValVADLoss",
vad_loss,
on_step=False,
on_epoch=True,
prog_bar=False,
logger=True,
)
loss = seg_loss + vad_loss
self.model.log(
f"{self.logging_prefix}ValLoss",
loss,
on_step=False,
on_epoch=True,
prog_bar=False,
logger=True,
)
if self.specifications.powerset:
self.model.validation_metric(
torch.transpose(
multilabel[:, warm_up_left : num_frames - warm_up_right], 1, 2
),
torch.transpose(
target[:, warm_up_left : num_frames - warm_up_right], 1, 2
),
)
else:
self.model.validation_metric(
torch.transpose(
prediction[:, warm_up_left : num_frames - warm_up_right], 1, 2
),
torch.transpose(
target[:, warm_up_left : num_frames - warm_up_right], 1, 2
),
)
self.model.log_dict(
self.model.validation_metric,
on_step=False,
on_epoch=True,
prog_bar=True,
logger=True,
)
# log first batch visualization every 2^n epochs.
if (
self.model.current_epoch == 0
or math.log2(self.model.current_epoch) % 1 > 0
or batch_idx > 0
):
return
# visualize first 9 validation samples of first batch in Tensorboard/MLflow
if self.specifications.powerset:
y = permutated_target_powerset.float().cpu().numpy()
y_pred = multilabel.cpu().numpy()
else:
y = target.float().cpu().numpy()
y_pred = permutated_prediction.cpu().numpy()
# prepare 3 x 3 grid (or smaller if batch size is smaller)
num_samples = min(self.batch_size, 9)
nrows = math.ceil(math.sqrt(num_samples))
ncols = math.ceil(num_samples / nrows)
fig, axes = plt.subplots(
nrows=2 * nrows, ncols=ncols, figsize=(8, 5), squeeze=False
)
# reshape target so that there is one line per class when plotting it
y[y == 0] = np.NaN
if len(y.shape) == 2:
y = y[:, :, np.newaxis]
y *= np.arange(y.shape[2])
# plot each sample
for sample_idx in range(num_samples):
# find where in the grid it should be plotted
row_idx = sample_idx // nrows
col_idx = sample_idx % ncols
# plot target
ax_ref = axes[row_idx * 2 + 0, col_idx]
sample_y = y[sample_idx]
ax_ref.plot(sample_y)
ax_ref.set_xlim(0, len(sample_y))
ax_ref.set_ylim(-1, sample_y.shape[1])
ax_ref.get_xaxis().set_visible(False)
ax_ref.get_yaxis().set_visible(False)
# plot predictions
ax_hyp = axes[row_idx * 2 + 1, col_idx]
sample_y_pred = y_pred[sample_idx]
ax_hyp.axvspan(0, warm_up_left, color="k", alpha=0.5, lw=0)
ax_hyp.axvspan(
num_frames - warm_up_right, num_frames, color="k", alpha=0.5, lw=0
)
ax_hyp.plot(sample_y_pred)
ax_hyp.set_ylim(-0.1, 1.1)
ax_hyp.set_xlim(0, len(sample_y))
ax_hyp.get_xaxis().set_visible(False)
plt.tight_layout()
if isinstance(self.model.logger, TensorBoardLogger):
self.model.logger.experiment.add_figure(
f"{self.logging_prefix}ValSamples", fig, self.model.current_epoch
)
elif isinstance(self.model.logger, MLFlowLogger):
self.model.logger.experiment.log_figure(
run_id=self.model.logger.run_id,
figure=fig,
artifact_file=f"{self.logging_prefix}ValSamples_epoch{self.model.current_epoch}.png",
)
plt.close(fig)
def main(protocol: str, subset: str = "test", model: str = "pyannote/segmentation"):
"""Evaluate a segmentation model"""
+2 -2
View File
@@ -194,7 +194,7 @@ def build_permutation_graph(
cost_func: Callable[[torch.Tensor, torch.Tensor], torch.Tensor] = mae_cost_func,
) -> nx.Graph:
"""Build permutation graph
Parameters
----------
segmentations : (num_chunks, num_frames, local_num_speakers)-shaped SlidingWindowFeature
@@ -203,7 +203,7 @@ def build_permutation_graph(
Threshold above which a speaker is considered active. Defaults to 0.5
cost_func : callable
Cost function used to find the optimal bijective mapping between speaker activations
of two overlapping chunks. Expects two (num_frames, num_classes) torch.tensor as input
of two overlapping chunks. Expects two (num_frames, num_classes) torch.tensor as input
and returns cost as a (num_classes, ) torch.tensor. Defaults to mae_cost_func.
Returns
+153
View File
@@ -0,0 +1,153 @@
# MIT License
#
# Copyright (c) 2023- CNRS
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# AUTHORS
# Hervé BREDIN - https://herve.niderb.fr
# Alexis PLAQUET
try:
from functools import cached_property
except ImportError:
from backports.cached_property import cached_property
from itertools import combinations
import scipy.special
import torch
import torch.nn as nn
import torch.nn.functional as F
class Powerset(nn.Module):
"""Powerset to multilabel conversion, and back.
Parameters
----------
num_classes : int
Number of regular classes.
max_set_size : int
Maximum number of classes in each set.
"""
def __init__(self, num_classes: int, max_set_size: int):
super().__init__()
self.num_classes = num_classes
self.max_set_size = max_set_size
mapping = self.build_mapping()
self.register_buffer("mapping", mapping, persistent=False)
@cached_property
def num_powerset_classes(self) -> int:
# compute number of subsets of size at most "max_set_size"
# e.g. with num_classes = 3 and max_set_size = 2:
# {}, {0}, {1}, {2}, {0, 1}, {0, 2}, {1, 2}
return int(
sum(
scipy.special.binom(self.num_classes, i)
for i in range(0, self.max_set_size + 1)
)
)
def build_mapping(self) -> torch.Tensor:
mapping = torch.zeros(self.num_powerset_classes, self.num_classes)
powerset_k = 0
for set_size in range(0, self.max_set_size + 1):
for current_set in combinations(range(self.num_classes), set_size):
mapping[powerset_k, current_set] = 1
powerset_k += 1
return mapping
def to_multilabel(self, powerset: torch.Tensor) -> torch.Tensor:
"""Convert (hard) predictions from powerset to multi-label
Parameter
---------
powerset : (batch_size, num_frames, num_powerset_classes) torch.Tensor
Hard predictions in "powerset" space.
Returns
-------
multi_label : (batch_size, num_frames, num_classes) torch.Tensor
Hard predictions in "multi-label" space.
Note
----
This method will not complain if `powerset` is provided a soft predictions
(e.g. the output of a softmax-ed classifier). However, in that particular
case, the resulting soft multi-label output will not make much sense.
"""
return torch.matmul(powerset, self.mapping)
def to_powerset(self, multilabel: torch.Tensor) -> torch.Tensor:
"""Convert (hard) predictions from multi-label to powerset
Parameter
---------
multi_label : (batch_size, num_frames, num_classes) torch.Tensor
Prediction in "multi-label" space.
Returns
-------
powerset : (batch_size, num_frames, num_powerset_classes) torch.Tensor
Hard, one-hot prediction in "powerset" space.
Note
----
This method will not complain if `multilabel` is provided a soft predictions
(e.g. the output of a sigmoid-ed classifier). However, in that particular
case, the resulting powerset output will most likely not make much sense.
"""
return F.one_hot(
torch.argmax(torch.matmul(multilabel, self.mapping.T), dim=-1),
num_classes=self.num_powerset_classes,
)
if __name__ == "__main__":
powerset = Powerset(3, 3)
print(powerset.mapping)
# simulate a sequence where each frame is assigned to a different powerset class
one_sequence = [
[0] * powerset.num_powerset_classes
for _ in range(powerset.num_powerset_classes)
]
for i in range(powerset.num_powerset_classes):
one_sequence[i][i] = 1.0
# make a batch out of this sequence and the same sequence in reverse order
batch_powerset = torch.tensor([one_sequence, one_sequence[::-1]])
print(batch_powerset)
print(batch_powerset.shape)
# convert from powerset to multi-label
batch_multilabel = powerset.to_multilabel(batch_powerset)
print(batch_multilabel)
print(batch_multilabel.shape)
# convert batch to powerset
reconstruction = powerset.to_powerset(batch_multilabel)
print(reconstruction)
print(reconstruction.shape)
-1
View File
@@ -84,4 +84,3 @@ def test_can_crop_from_file_like():
assert isinstance(wav, Tensor)
assert sr == 16000
assert wav.shape[1] == 0.5 * 16000
+53
View File
@@ -0,0 +1,53 @@
# MIT License
#
# Copyright (c) 2023- CNRS
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import torch
from pyannote.audio.utils.powerset import Powerset
def test_roundtrip():
for num_classes in range(2, 5):
for max_set_size in range(1, num_classes + 1):
powerset = Powerset(num_classes, max_set_size)
# simulate a sequence where each frame is assigned to a different powerset class
one_sequence = [
[0] * powerset.num_powerset_classes
for _ in range(powerset.num_powerset_classes)
]
for i in range(powerset.num_powerset_classes):
one_sequence[i][i] = 1.0
# make a batch out of this sequence and the same sequence in reverse order
batch_powerset = torch.tensor([one_sequence, one_sequence[::-1]])
# convert from powerset to multi-label
batch_multilabel = powerset.to_multilabel(batch_powerset)
# convert batch back to powerset
reconstruction = powerset.to_powerset(batch_multilabel)
assert torch.equal(batch_powerset, reconstruction)