mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
feat: add support for "powerset" speaker segmentation training
Co-authored-by: Hervé BREDIN <hbredin@users.noreply.github.com>
This commit is contained in:
1 parent
0506ae38c3
commit
12dc20bde1
13 files changed
+694
-99
No files matched your search
@@ -26,6 +26,4 @@ from pyannote.audio import Model
|
||||
|
||||
|
||||
def pretrained(checkpoint: Text):
|
||||
return Model.from_pretrained(
|
||||
checkpoint, map_location=lambda storage, loc: storage
|
||||
)
|
||||
return Model.from_pretrained(checkpoint, map_location=lambda storage, loc: storage)
|
||||
@@ -35,6 +35,7 @@ from pyannote.audio.core.io import AudioFile
|
||||
from pyannote.audio.core.model import Model
|
||||
from pyannote.audio.core.task import Resolution
|
||||
from pyannote.audio.utils.permutation import mae_cost_func, permutate
|
||||
from pyannote.audio.utils.powerset import Powerset
|
||||
|
||||
TaskName = Union[Text, None]
|
||||
|
||||
@@ -53,23 +54,26 @@ class Inference(BaseInference):
|
||||
window : {"sliding", "whole"}, optional
|
||||
Use a "sliding" window and aggregate the corresponding outputs (default)
|
||||
or just one (potentially long) window covering the "whole" file or chunk.
|
||||
skip_aggregation : bool, optional
|
||||
Do not aggregate outputs when using "sliding" window. Defaults to False.
|
||||
duration : float, optional
|
||||
Chunk duration, in seconds. Defaults to duration used for training the model.
|
||||
Has no effect when `window` is "whole".
|
||||
step : float, optional
|
||||
Step between consecutive chunks, in seconds. Defaults to warm-up duration when
|
||||
greater than 0s, otherwise 10% of duration. Has no effect when `window` is "whole".
|
||||
pre_aggregation_hook : callable, optional
|
||||
When a callable is provided, it is applied to the model output, just before aggregation.
|
||||
Takes a (num_chunks, num_frames, dimension) numpy array as input and returns a modified
|
||||
(num_chunks, num_frames, other_dimension) numpy array passed to overlap-add aggregation.
|
||||
skip_aggregation : bool, optional
|
||||
Do not aggregate outputs when using "sliding" window. Defaults to False.
|
||||
skip_conversion: bool, optional
|
||||
In case `model` has been trained with `powerset` mode, its output is automatically
|
||||
converted to `multi-label`, unless `skip_conversion` is set to True.
|
||||
batch_size : int, optional
|
||||
Batch size. Larger values make inference faster. Defaults to 32.
|
||||
device : torch.device, optional
|
||||
Device used for inference. Defaults to `model.device`.
|
||||
In case `device` and `model.device` are different, model is sent to device.
|
||||
pre_aggregation_hook : callable, optional
|
||||
When a callable is provided, it is applied to the model output, just before aggregation.
|
||||
Takes a (num_chunks, num_frames, dimension) numpy array as input and returns a modified
|
||||
(num_chunks, num_frames, other_dimension) numpy array passed to overlap-add aggregation.
|
||||
use_auth_token : str, optional
|
||||
When loading a private huggingface.co model, set `use_auth_token`
|
||||
to True or to a string containing your hugginface.co authentication
|
||||
@@ -80,12 +84,13 @@ class Inference(BaseInference):
|
||||
self,
|
||||
model: Union[Model, Text, Path],
|
||||
window: Text = "sliding",
|
||||
skip_aggregation: bool = False,
|
||||
device: torch.device = None,
|
||||
duration: float = None,
|
||||
step: float = None,
|
||||
batch_size: int = 32,
|
||||
pre_aggregation_hook: Callable[[np.ndarray], np.ndarray] = None,
|
||||
skip_aggregation: bool = False,
|
||||
skip_conversion: bool = False,
|
||||
device: torch.device = None,
|
||||
batch_size: int = 32,
|
||||
use_auth_token: Union[Text, None] = None,
|
||||
):
|
||||
|
||||
@@ -154,11 +159,19 @@ class Inference(BaseInference):
|
||||
self.step = step
|
||||
|
||||
self.batch_size = batch_size
|
||||
self.skip_conversion = skip_conversion
|
||||
if specifications.powerset and not self.skip_conversion:
|
||||
self._powerset = Powerset(
|
||||
len(specifications.classes), specifications.powerset_max_classes
|
||||
)
|
||||
self._powerset.to(self.device)
|
||||
|
||||
def to(self, device: torch.device):
|
||||
"""Send internal model to `device`"""
|
||||
|
||||
self.model.to(device)
|
||||
if self.model.specifications.powerset and not self.skip_conversion:
|
||||
self._powerset.to(device)
|
||||
self.device = device
|
||||
return self
|
||||
|
||||
@@ -190,6 +203,14 @@ class Inference(BaseInference):
|
||||
else:
|
||||
raise exception
|
||||
|
||||
# convert powerset to multi-label unless specifically requested not to
|
||||
if self.model.specifications.powerset and not self.skip_conversion:
|
||||
powerset = torch.nn.functional.one_hot(
|
||||
torch.argmax(outputs, dim=-1),
|
||||
self.model.specifications.num_powerset_classes,
|
||||
).float()
|
||||
outputs = self._powerset.to_multilabel(powerset)
|
||||
|
||||
return outputs.cpu().numpy()
|
||||
|
||||
def slide(
|
||||
|
||||
@@ -25,6 +25,8 @@ from __future__ import annotations
|
||||
|
||||
from functools import partial
|
||||
|
||||
import scipy.special
|
||||
|
||||
try:
|
||||
from functools import cached_property
|
||||
except ImportError:
|
||||
@@ -89,9 +91,38 @@ class Specifications:
|
||||
# (for classification tasks only) list of classes
|
||||
classes: Optional[List[Text]] = None
|
||||
|
||||
# (for powerset only) max number of simultaneous classes
|
||||
# (n choose k with k <= powerset_max_classes)
|
||||
powerset_max_classes: Optional[int] = None
|
||||
|
||||
# whether classes are permutation-invariant (e.g. diarization)
|
||||
permutation_invariant: bool = False
|
||||
|
||||
@cached_property
|
||||
def powerset(self):
|
||||
|
||||
if self.powerset_max_classes is None:
|
||||
return False
|
||||
|
||||
if self.problem != Problem.MONO_LABEL_CLASSIFICATION:
|
||||
raise ValueError(
|
||||
"`powerset_max_classes` only makes sense with multi-class classification problems."
|
||||
)
|
||||
|
||||
return True
|
||||
|
||||
@cached_property
|
||||
def num_powerset_classes(self) -> int:
|
||||
# compute number of subsets of size at most "powerset_max_classes"
|
||||
# e.g. with len(classes) = 3 and powerset_max_classes = 2:
|
||||
# {}, {0}, {1}, {2}, {0, 1}, {0, 2}, {1, 2}
|
||||
return int(
|
||||
sum(
|
||||
scipy.special.binom(len(self.classes), i)
|
||||
for i in range(0, self.powerset_max_classes + 1)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class TrainDataset(IterableDataset):
|
||||
def __init__(self, task: Task):
|
||||
@@ -314,7 +345,7 @@ class Task(pl.LightningDataModule):
|
||||
]:
|
||||
return binary_cross_entropy(prediction, target, weight=weight)
|
||||
|
||||
elif specifications.problem == Problem.MONO_LABEL_CLASSIFICATION:
|
||||
elif specifications.problem in [Problem.MONO_LABEL_CLASSIFICATION]:
|
||||
return nll_loss(prediction, target, weight=weight)
|
||||
|
||||
else:
|
||||
|
||||
@@ -27,12 +27,12 @@ import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from einops import rearrange
|
||||
from pyannote.core.utils.generators import pairwise
|
||||
|
||||
from pyannote.audio.core.model import Model
|
||||
from pyannote.audio.core.task import Task
|
||||
from pyannote.audio.models.blocks.sincnet import SincNet
|
||||
from pyannote.audio.utils.params import merge_dict
|
||||
from pyannote.core.utils.generators import pairwise
|
||||
|
||||
|
||||
class PyanNet(Model):
|
||||
@@ -130,7 +130,9 @@ class PyanNet(Model):
|
||||
[
|
||||
nn.Linear(in_features, out_features)
|
||||
for in_features, out_features in pairwise(
|
||||
[lstm_out_features,]
|
||||
[
|
||||
lstm_out_features,
|
||||
]
|
||||
+ [self.hparams.linear["hidden_size"]]
|
||||
* self.hparams.linear["num_layers"]
|
||||
)
|
||||
@@ -146,7 +148,12 @@ class PyanNet(Model):
|
||||
2 if self.hparams.lstm["bidirectional"] else 1
|
||||
)
|
||||
|
||||
self.classifier = nn.Linear(in_features, len(self.specifications.classes))
|
||||
if self.specifications.powerset:
|
||||
out_features = self.specifications.num_powerset_classes
|
||||
else:
|
||||
out_features = len(self.specifications.classes)
|
||||
|
||||
self.classifier = nn.Linear(in_features, out_features)
|
||||
self.activation = self.default_activation()
|
||||
|
||||
def forward(self, waveforms: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
@@ -147,10 +147,16 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
|
||||
)
|
||||
self._frames: SlidingWindow = self._segmentation.model.introspection.frames
|
||||
|
||||
self.segmentation = ParamDict(
|
||||
threshold=Uniform(0.1, 0.9),
|
||||
min_duration_off=Uniform(0.0, 1.0),
|
||||
)
|
||||
if self._segmentation.model.specifications.powerset:
|
||||
self.segmentation = ParamDict(
|
||||
min_duration_off=Uniform(0.0, 1.0),
|
||||
)
|
||||
|
||||
else:
|
||||
self.segmentation = ParamDict(
|
||||
threshold=Uniform(0.1, 0.9),
|
||||
min_duration_off=Uniform(0.0, 1.0),
|
||||
)
|
||||
|
||||
if self.klustering == "OracleClustering":
|
||||
metric = "not_applicable"
|
||||
@@ -260,18 +266,19 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
|
||||
embeddings : (num_chunks, num_speakers, dimension) array
|
||||
"""
|
||||
|
||||
# when optimizing the hyper-parameters of this pipeline with frozen "segmentation_onset",
|
||||
# one can reuse the embeddings from the first trial, bringing a massive speed up to
|
||||
# the optimization process (and hence allowing to use a larger search space).
|
||||
# when optimizing the hyper-parameters of this pipeline with frozen
|
||||
# "segmentation.threshold", one can reuse the embeddings from the first trial,
|
||||
# bringing a massive speed up to the optimization process (and hence allowing to use
|
||||
# a larger search space).
|
||||
if self.training:
|
||||
|
||||
# we only re-use embeddings if they were extracted based on the same value of the
|
||||
# "segmentation_onset" hyperparameter and "embedding_exclude_overlap" parameter.
|
||||
# "segmentation.threshold" hyperparameter or if the segmentation model relies on
|
||||
# `powerset` mode
|
||||
cache = file.get("training_cache/embeddings", dict())
|
||||
if (
|
||||
cache.get("segmentation.threshold", None) == self.segmentation.threshold
|
||||
and cache.get("embedding_exclude_overlap", None)
|
||||
== self.embedding_exclude_overlap
|
||||
if ("embeddings" in cache) and (
|
||||
self._segmentation.model.specifications.powerset
|
||||
or (cache["segmentation.threshold"] == self.segmentation.threshold)
|
||||
):
|
||||
return cache["embeddings"]
|
||||
|
||||
@@ -369,11 +376,16 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
|
||||
# caching embeddings for subsequent trials
|
||||
# (see comments at the top of this method for more details)
|
||||
if self.training:
|
||||
file["training_cache/embeddings"] = {
|
||||
"segmentation.threshold": self.segmentation.threshold,
|
||||
"embedding_exclude_overlap": self.embedding_exclude_overlap,
|
||||
"embeddings": embeddings,
|
||||
}
|
||||
|
||||
if self._segmentation.model.specifications.powerset:
|
||||
file["training_cache/embeddings"] = {
|
||||
"embeddings": embeddings,
|
||||
}
|
||||
else:
|
||||
file["training_cache/embeddings"] = {
|
||||
"segmentation.threshold": self.segmentation.threshold,
|
||||
"embeddings": embeddings,
|
||||
}
|
||||
|
||||
return embeddings
|
||||
|
||||
@@ -479,7 +491,9 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
|
||||
# estimate frame-level number of instantaneous speakers
|
||||
count = self.speaker_count(
|
||||
segmentations,
|
||||
onset=self.segmentation.threshold,
|
||||
onset=0.5
|
||||
if self._segmentation.model.specifications.powerset
|
||||
else self.segmentation.threshold,
|
||||
frames=self._frames,
|
||||
)
|
||||
hook("speaker_counting", count)
|
||||
@@ -491,11 +505,14 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
|
||||
return Annotation(uri=file["uri"])
|
||||
|
||||
# binarize segmentation
|
||||
binarized_segmentations: SlidingWindowFeature = binarize(
|
||||
segmentations,
|
||||
onset=self.segmentation.threshold,
|
||||
initial_state=False,
|
||||
)
|
||||
if self._segmentation.model.specifications.powerset:
|
||||
binarized_segmentations = segmentations
|
||||
else:
|
||||
binarized_segmentations: SlidingWindowFeature = binarize(
|
||||
segmentations,
|
||||
onset=self.segmentation.threshold,
|
||||
initial_state=False,
|
||||
)
|
||||
|
||||
if self.klustering == "OracleClustering":
|
||||
embeddings = None
|
||||
|
||||
@@ -25,9 +25,10 @@ from .segmentation.overlapped_speech_detection import ( # isort:skip
|
||||
OverlappedSpeechDetection,
|
||||
)
|
||||
|
||||
from .segmentation.multilabel import MultiLabelSegmentation # isort:skip
|
||||
from .segmentation.multilabel import MultiLabelSegmentation # isort:skip
|
||||
from .segmentation.segmentation import Segmentation # isort:skip
|
||||
from .embedding.arcface import SupervisedRepresentationLearningWithArcFace # isort:skip
|
||||
|
||||
SpeakerEmbedding = SupervisedRepresentationLearningWithArcFace
|
||||
|
||||
__all__ = [
|
||||
|
||||
@@ -29,14 +29,16 @@ from typing import Dict, Optional, Sequence, Text, Union
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional
|
||||
from pyannote.core import Segment, SlidingWindowFeature
|
||||
from pytorch_lightning.loggers import TensorBoardLogger, MLFlowLogger
|
||||
from pytorch_lightning.loggers import MLFlowLogger, TensorBoardLogger
|
||||
from torch.utils.data._utils.collate import default_collate
|
||||
from torchmetrics import Metric
|
||||
from torchmetrics.classification import BinaryAUROC, MultilabelAUROC, MulticlassAUROC
|
||||
from torchmetrics.classification import BinaryAUROC, MulticlassAUROC, MultilabelAUROC
|
||||
|
||||
from pyannote.audio.core.io import AudioFile
|
||||
from pyannote.audio.core.task import Problem
|
||||
from pyannote.audio.utils.powerset import Powerset
|
||||
from pyannote.audio.utils.random import create_rng_for_worker
|
||||
|
||||
|
||||
@@ -125,6 +127,13 @@ class SegmentationTaskMixin:
|
||||
|
||||
random.shuffle(self._validation)
|
||||
|
||||
def setup_loss_func(self):
|
||||
if self.specifications.powerset:
|
||||
self.model.powerset = Powerset(
|
||||
len(self.specifications.classes),
|
||||
self.specifications.powerset_max_classes,
|
||||
)
|
||||
|
||||
def default_metric(
|
||||
self,
|
||||
) -> Union[Metric, Sequence[Metric], Dict[str, Metric]]:
|
||||
@@ -372,10 +381,12 @@ class SegmentationTaskMixin:
|
||||
_, num_frames, _ = y_pred.shape
|
||||
# y_pred = (batch_size, num_frames, num_classes)
|
||||
|
||||
# - remove warm-up frames
|
||||
# - downsample remaining frames
|
||||
# compute warmup frames boundaries and weight
|
||||
warm_up_left = round(self.warm_up[0] / self.duration * num_frames)
|
||||
warm_up_right = round(self.warm_up[1] / self.duration * num_frames)
|
||||
|
||||
# - remove warm-up frames
|
||||
# - downsample remaining frames
|
||||
preds = y_pred[:, warm_up_left : num_frames - warm_up_right : 10]
|
||||
target = y[:, warm_up_left : num_frames - warm_up_right : 10]
|
||||
|
||||
@@ -383,7 +394,6 @@ class SegmentationTaskMixin:
|
||||
# pyannote.audio is more explicit so we have to reshape target and preds for
|
||||
# torchmetrics to be happy... more details can be found here:
|
||||
# https://torchmetrics.readthedocs.io/en/latest/references/modules.html#input-types
|
||||
|
||||
if self.specifications.problem == Problem.BINARY_CLASSIFICATION:
|
||||
# target: shape (batch_size, num_frames), type binary
|
||||
# preds: shape (batch_size, num_frames, 1), type float
|
||||
|
||||
@@ -36,7 +36,7 @@ from pyannote.audio.tasks.segmentation.mixins import SegmentationTaskMixin
|
||||
class MultiLabelSegmentation(SegmentationTaskMixin, Task):
|
||||
"""Generic multi-label segmentation
|
||||
|
||||
Multi-label segmentation is the process of detecting temporal intervals
|
||||
Multi-label segmentation is the process of detecting temporal intervals
|
||||
when a specific audio class is active.
|
||||
|
||||
Example use cases include speaker tracking, gender (male/female)
|
||||
@@ -80,18 +80,18 @@ class MultiLabelSegmentation(SegmentationTaskMixin, Task):
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
protocol: Protocol,
|
||||
classes: Optional[List[str]] = None,
|
||||
duration: float = 2.0,
|
||||
warm_up: Union[float, Tuple[float, float]] = 0.0,
|
||||
balance: Text = None,
|
||||
weight: Text = None,
|
||||
batch_size: int = 32,
|
||||
num_workers: int = None,
|
||||
pin_memory: bool = False,
|
||||
augmentation: BaseWaveformTransform = None,
|
||||
metric: Union[Metric, Sequence[Metric], Dict[str, Metric]] = None,
|
||||
self,
|
||||
protocol: Protocol,
|
||||
classes: Optional[List[str]] = None,
|
||||
duration: float = 2.0,
|
||||
warm_up: Union[float, Tuple[float, float]] = 0.0,
|
||||
balance: Text = None,
|
||||
weight: Text = None,
|
||||
batch_size: int = 32,
|
||||
num_workers: int = None,
|
||||
pin_memory: bool = False,
|
||||
augmentation: BaseWaveformTransform = None,
|
||||
metric: Union[Metric, Sequence[Metric], Dict[str, Metric]] = None,
|
||||
):
|
||||
super().__init__(
|
||||
protocol,
|
||||
@@ -109,7 +109,7 @@ class MultiLabelSegmentation(SegmentationTaskMixin, Task):
|
||||
self.classes = classes
|
||||
|
||||
# task specification depends on the data: we do not know in advance which
|
||||
# classes should be detected. therefore, we postpone the definition of
|
||||
# classes should be detected. therefore, we postpone the definition of
|
||||
# specifications to setup()
|
||||
|
||||
def setup(self, stage: Optional[str] = None):
|
||||
|
||||
@@ -20,13 +20,18 @@
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
import math
|
||||
import warnings
|
||||
from collections import Counter
|
||||
from typing import Dict, Optional, Sequence, Text, Tuple, Union
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional
|
||||
from matplotlib import pyplot as plt
|
||||
from pyannote.core import SlidingWindow
|
||||
from pyannote.database import Protocol
|
||||
from pytorch_lightning.loggers import MLFlowLogger, TensorBoardLogger
|
||||
from torch_audiomentations.core.transforms_interface import BaseWaveformTransform
|
||||
from torchmetrics import Metric
|
||||
from typing_extensions import Literal
|
||||
@@ -34,13 +39,17 @@ from typing_extensions import Literal
|
||||
from pyannote.audio.core.task import Problem, Resolution, Specifications, Task
|
||||
from pyannote.audio.tasks.segmentation.mixins import SegmentationTaskMixin
|
||||
from pyannote.audio.torchmetrics import (
|
||||
DiarizationErrorRate,
|
||||
FalseAlarmRate,
|
||||
MissedDetectionRate,
|
||||
OptimalDiarizationErrorRate,
|
||||
OptimalDiarizationErrorRateThreshold,
|
||||
OptimalFalseAlarmRate,
|
||||
OptimalMissedDetectionRate,
|
||||
OptimalSpeakerConfusionRate,
|
||||
SpeakerConfusionRate,
|
||||
)
|
||||
from pyannote.audio.utils.loss import binary_cross_entropy, mse_loss
|
||||
from pyannote.audio.utils.loss import binary_cross_entropy, mse_loss, nll_loss
|
||||
from pyannote.audio.utils.permutation import permutate
|
||||
|
||||
|
||||
@@ -53,9 +62,13 @@ class Segmentation(SegmentationTaskMixin, Task):
|
||||
pyannote.database protocol
|
||||
duration : float, optional
|
||||
Chunks duration. Defaults to 2s.
|
||||
max_num_speakers : int, optional
|
||||
Force maximum number of speakers per chunk (must be at least 2).
|
||||
max_speakers_per_chunk : int, optional
|
||||
Maximum number of speakers per chunk (must be at least 2).
|
||||
Defaults to estimating it from the training set.
|
||||
max_speakers_per_frame : int, optional
|
||||
Maximum number of (overlapping) speakers per frame.
|
||||
Setting this value to 1 or more enables `powerset multi-class` training.
|
||||
Default behavior is to use `multi-label` training.
|
||||
warm_up : float or (float, float), optional
|
||||
Use that many seconds on the left- and rightmost parts of each chunk
|
||||
to warm up the model. While the model does process those left- and right-most
|
||||
@@ -67,7 +80,7 @@ class Segmentation(SegmentationTaskMixin, Task):
|
||||
For instance, setting `balance` to "uri" will make sure that each file will be
|
||||
equally represented in the training samples.
|
||||
weight: str, optional
|
||||
When provided, use this key to as frame-wise weight in loss function.
|
||||
When provided, use this key as frame-wise weight in loss function.
|
||||
batch_size : int, optional
|
||||
Number of training samples per batch. Defaults to 32.
|
||||
num_workers : int, optional
|
||||
@@ -80,26 +93,32 @@ class Segmentation(SegmentationTaskMixin, Task):
|
||||
augmentation : BaseWaveformTransform, optional
|
||||
torch_audiomentations waveform transform, used by dataloader
|
||||
during training.
|
||||
loss : {"bce", "mse"}, optional
|
||||
Permutation-invariant segmentation loss. Defaults to "bce".
|
||||
vad_loss : {"bce", "mse"}, optional
|
||||
Add voice activity detection loss.
|
||||
Cannot be used in conjunction with `max_speakers_per_frame`.
|
||||
metric : optional
|
||||
Validation metric(s). Can be anything supported by torchmetrics.MetricCollection.
|
||||
Defaults to AUROC (area under the ROC curve).
|
||||
|
||||
Reference
|
||||
References
|
||||
----------
|
||||
Hervé Bredin and Antoine Laurent
|
||||
"End-To-End Speaker Segmentation for Overlap-Aware Resegmentation."
|
||||
Proc. Interspeech 2021
|
||||
|
||||
Zhihao Du, Shiliang Zhang, Siqi Zheng, and Zhijie Yan
|
||||
"Speaker Embedding-aware Neural Diarization: an Efficient Framework for Overlapping
|
||||
Speech Diarization in Meeting Scenarios"
|
||||
https://arxiv.org/abs/2203.09767
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
protocol: Protocol,
|
||||
duration: float = 2.0,
|
||||
max_num_speakers: int = None,
|
||||
max_speakers_per_chunk: int = None,
|
||||
max_speakers_per_frame: int = None,
|
||||
warm_up: Union[float, Tuple[float, float]] = 0.0,
|
||||
balance: Text = None,
|
||||
weight: Text = None,
|
||||
@@ -107,9 +126,10 @@ class Segmentation(SegmentationTaskMixin, Task):
|
||||
num_workers: int = None,
|
||||
pin_memory: bool = False,
|
||||
augmentation: BaseWaveformTransform = None,
|
||||
loss: Literal["bce", "mse"] = "bce",
|
||||
vad_loss: Literal["bce", "mse"] = None,
|
||||
metric: Union[Metric, Sequence[Metric], Dict[str, Metric]] = None,
|
||||
max_num_speakers: int = None, # deprecated in favor of `max_speakers_per_chunk``
|
||||
loss: Literal["bce", "mse"] = None, # deprecated
|
||||
):
|
||||
|
||||
super().__init__(
|
||||
@@ -123,20 +143,37 @@ class Segmentation(SegmentationTaskMixin, Task):
|
||||
metric=metric,
|
||||
)
|
||||
|
||||
self.max_num_speakers = max_num_speakers
|
||||
# deprecation warnings
|
||||
if max_speakers_per_chunk is None and max_num_speakers is not None:
|
||||
max_speakers_per_chunk = max_num_speakers
|
||||
warnings.warn(
|
||||
"`max_num_speakers` has been deprecated in favor of `max_speakers_per_chunk`."
|
||||
)
|
||||
if loss is not None:
|
||||
warnings.warn("`loss` has been deprecated and has no effect.")
|
||||
|
||||
# parameter validation
|
||||
if max_speakers_per_frame is not None:
|
||||
if max_speakers_per_frame < 1:
|
||||
raise ValueError(
|
||||
f"`max_speakers_per_frame` must be 1 or more (you used {max_speakers_per_frame})."
|
||||
)
|
||||
if vad_loss is not None:
|
||||
raise ValueError(
|
||||
"`vad_loss` cannot be used jointly with `max_speakers_per_frame`"
|
||||
)
|
||||
|
||||
self.max_speakers_per_chunk = max_speakers_per_chunk
|
||||
self.max_speakers_per_frame = max_speakers_per_frame
|
||||
self.balance = balance
|
||||
self.weight = weight
|
||||
|
||||
if loss not in ["bce", "mse"]:
|
||||
raise ValueError("'loss' must be one of {'bce', 'mse'}.")
|
||||
self.loss = loss
|
||||
self.vad_loss = vad_loss
|
||||
|
||||
def setup(self, stage: Optional[str] = None):
|
||||
|
||||
super().setup(stage=stage)
|
||||
|
||||
if self.max_num_speakers is None:
|
||||
if self.max_speakers_per_chunk is None:
|
||||
|
||||
# TODO: optimize this
|
||||
|
||||
@@ -165,19 +202,31 @@ class Segmentation(SegmentationTaskMixin, Task):
|
||||
num_speakers = num_speakers[sorting_indices]
|
||||
counts = counts[sorting_indices]
|
||||
|
||||
self.max_num_speakers = max(
|
||||
self.max_speakers_per_chunk = max(
|
||||
2,
|
||||
num_speakers[np.where(np.cumsum(counts) / np.sum(counts) > 0.99)[0][0]],
|
||||
)
|
||||
|
||||
if (
|
||||
self.max_speakers_per_frame is not None
|
||||
and self.max_speakers_per_frame > self.max_speakers_per_chunk
|
||||
):
|
||||
raise ValueError(
|
||||
f"`max_speakers_per_frame` ({self.max_speakers_per_frame}) must be smaller "
|
||||
f"than `max_speakers_per_chunk` ({self.max_speakers_per_chunk})"
|
||||
)
|
||||
|
||||
# now that we know about the number of speakers upper bound
|
||||
# we can set task specifications
|
||||
self.specifications = Specifications(
|
||||
problem=Problem.MULTI_LABEL_CLASSIFICATION,
|
||||
problem=Problem.MULTI_LABEL_CLASSIFICATION
|
||||
if self.max_speakers_per_frame is None
|
||||
else Problem.MONO_LABEL_CLASSIFICATION,
|
||||
resolution=Resolution.FRAME,
|
||||
duration=self.duration,
|
||||
warm_up=self.warm_up,
|
||||
classes=[f"speaker#{i+1}" for i in range(self.max_num_speakers)],
|
||||
classes=[f"speaker#{i+1}" for i in range(self.max_speakers_per_chunk)],
|
||||
powerset_max_classes=self.max_speakers_per_frame,
|
||||
permutation_invariant=True,
|
||||
)
|
||||
|
||||
@@ -193,27 +242,27 @@ class Segmentation(SegmentationTaskMixin, Task):
|
||||
|
||||
Returns
|
||||
-------
|
||||
y : (batch_size, num_frames, max_num_speakers) tensor
|
||||
Same as collated_y, except we only keep ``max_num_speakers`` most
|
||||
y : (batch_size, num_frames, max_speakers_per_chunk) tensor
|
||||
Same as collated_y, except we only keep ``max_speakers_per_chunk`` most
|
||||
talkative speakers (per sample).
|
||||
"""
|
||||
|
||||
batch_size, num_frames, num_speakers = collated_y.shape
|
||||
|
||||
# maximum number of active speakers in a chunk
|
||||
max_num_speakers = torch.max(
|
||||
max_speakers_per_chunk = torch.max(
|
||||
torch.sum(torch.sum(collated_y, dim=1) > 0.0, dim=1)
|
||||
)
|
||||
|
||||
# sort speakers in descending talkativeness order
|
||||
indices = torch.argsort(torch.sum(collated_y, dim=1), dim=1, descending=True)
|
||||
|
||||
# keep max_num_speakers most talkative speakers, for each chunk
|
||||
# keep max_speakers_per_chunk most talkative speakers, for each chunk
|
||||
y = torch.zeros(
|
||||
(batch_size, num_frames, max_num_speakers), dtype=collated_y.dtype
|
||||
(batch_size, num_frames, max_speakers_per_chunk), dtype=collated_y.dtype
|
||||
)
|
||||
for b, index in enumerate(indices):
|
||||
for k, i in zip(range(max_num_speakers), index):
|
||||
for k, i in zip(range(max_speakers_per_chunk), index):
|
||||
y[b, :, k] = collated_y[b, :, i.item()]
|
||||
|
||||
return y
|
||||
@@ -241,14 +290,15 @@ class Segmentation(SegmentationTaskMixin, Task):
|
||||
Permutation-invariant segmentation loss
|
||||
"""
|
||||
|
||||
if self.loss == "bce":
|
||||
if self.specifications.powerset:
|
||||
seg_loss = nll_loss(
|
||||
permutated_prediction, torch.argmax(target, dim=-1), weight=weight
|
||||
)
|
||||
else:
|
||||
seg_loss = binary_cross_entropy(
|
||||
permutated_prediction, target.float(), weight=weight
|
||||
)
|
||||
|
||||
elif self.loss == "mse":
|
||||
seg_loss = mse_loss(permutated_prediction, target.float(), weight=weight)
|
||||
|
||||
return seg_loss
|
||||
|
||||
def voice_activity_detection_loss(
|
||||
@@ -289,7 +339,7 @@ class Segmentation(SegmentationTaskMixin, Task):
|
||||
return loss
|
||||
|
||||
def training_step(self, batch, batch_idx: int):
|
||||
"""Compute permutation-invariant binary cross-entropy
|
||||
"""Compute permutation-invariant segmentation loss
|
||||
|
||||
Parameters
|
||||
----------
|
||||
@@ -313,7 +363,7 @@ class Segmentation(SegmentationTaskMixin, Task):
|
||||
|
||||
# drop samples that contain too many speakers
|
||||
num_speakers: torch.Tensor = torch.sum(torch.any(target, dim=1), dim=1)
|
||||
keep: torch.Tensor = num_speakers <= self.max_num_speakers
|
||||
keep: torch.Tensor = num_speakers <= self.max_speakers_per_chunk
|
||||
target = target[keep]
|
||||
waveform = waveform[keep]
|
||||
|
||||
@@ -337,9 +387,6 @@ class Segmentation(SegmentationTaskMixin, Task):
|
||||
batch_size, num_frames, _ = prediction.shape
|
||||
# (batch_size, num_frames, num_classes)
|
||||
|
||||
# find optimal permutation
|
||||
permutated_prediction, _ = permutate(target, prediction)
|
||||
|
||||
# frames weight
|
||||
weight_key = getattr(self, "weight", None)
|
||||
weight = batch.get(
|
||||
@@ -354,7 +401,26 @@ class Segmentation(SegmentationTaskMixin, Task):
|
||||
warm_up_right = round(self.warm_up[1] / self.duration * num_frames)
|
||||
weight[:, num_frames - warm_up_right :] = 0.0
|
||||
|
||||
seg_loss = self.segmentation_loss(permutated_prediction, target, weight=weight)
|
||||
if self.specifications.powerset:
|
||||
|
||||
powerset = torch.nn.functional.one_hot(
|
||||
torch.argmax(prediction, dim=-1),
|
||||
self.model.powerset.num_powerset_classes,
|
||||
).float()
|
||||
multilabel = self.model.powerset.to_multilabel(powerset)
|
||||
permutated_target, _ = permutate(multilabel, target)
|
||||
permutated_target_powerset = self.model.powerset.to_powerset(
|
||||
permutated_target.float()
|
||||
)
|
||||
seg_loss = self.segmentation_loss(
|
||||
prediction, permutated_target_powerset, weight=weight
|
||||
)
|
||||
|
||||
else:
|
||||
permutated_prediction, _ = permutate(target, prediction)
|
||||
seg_loss = self.segmentation_loss(
|
||||
permutated_prediction, target, weight=weight
|
||||
)
|
||||
|
||||
self.model.log(
|
||||
f"{self.logging_prefix}TrainSegLoss",
|
||||
@@ -369,9 +435,18 @@ class Segmentation(SegmentationTaskMixin, Task):
|
||||
vad_loss = 0.0
|
||||
|
||||
else:
|
||||
vad_loss = self.voice_activity_detection_loss(
|
||||
permutated_prediction, target, weight=weight
|
||||
)
|
||||
|
||||
# TODO: vad_loss probably does not make sense in powerset mode
|
||||
# because first class (empty set of labels) does exactly this...
|
||||
if self.specifications.powerset:
|
||||
vad_loss = self.voice_activity_detection_loss(
|
||||
prediction, permutated_target_powerset, weight=weight
|
||||
)
|
||||
|
||||
else:
|
||||
vad_loss = self.voice_activity_detection_loss(
|
||||
permutated_prediction, target, weight=weight
|
||||
)
|
||||
|
||||
self.model.log(
|
||||
f"{self.logging_prefix}TrainVADLoss",
|
||||
@@ -399,6 +474,15 @@ class Segmentation(SegmentationTaskMixin, Task):
|
||||
self,
|
||||
) -> Union[Metric, Sequence[Metric], Dict[str, Metric]]:
|
||||
"""Returns diarization error rate and its components"""
|
||||
|
||||
if self.specifications.powerset:
|
||||
return [
|
||||
DiarizationErrorRate(0.5),
|
||||
SpeakerConfusionRate(0.5),
|
||||
MissedDetectionRate(0.5),
|
||||
FalseAlarmRate(0.5),
|
||||
]
|
||||
|
||||
return [
|
||||
OptimalDiarizationErrorRate(),
|
||||
OptimalDiarizationErrorRateThreshold(),
|
||||
@@ -407,6 +491,227 @@ class Segmentation(SegmentationTaskMixin, Task):
|
||||
OptimalFalseAlarmRate(),
|
||||
]
|
||||
|
||||
def train__iter__(self):
|
||||
for chunk in super().train__iter__():
|
||||
# TODO: document why this filtering is needed
|
||||
if self.specifications.powerset:
|
||||
if len(chunk["y"].labels) <= self.max_speakers_per_chunk:
|
||||
yield chunk
|
||||
else:
|
||||
yield chunk
|
||||
|
||||
# TODO: no need to compute gradient in this method
|
||||
def validation_step(self, batch, batch_idx: int):
|
||||
"""Compute validation loss and metric
|
||||
|
||||
Parameters
|
||||
----------
|
||||
batch : dict of torch.Tensor
|
||||
Current batch.
|
||||
batch_idx: int
|
||||
Batch index.
|
||||
"""
|
||||
|
||||
# target
|
||||
target = batch["y"]
|
||||
# (batch_size, num_frames, num_speakers)
|
||||
|
||||
waveform = batch["X"]
|
||||
# (batch_size, num_channels, num_samples)
|
||||
|
||||
# TODO: should we handle validation samples with too many speakers
|
||||
# waveform = waveform[keep]
|
||||
# target = target[keep]
|
||||
|
||||
# forward pass
|
||||
prediction = self.model(waveform)
|
||||
batch_size, num_frames, _ = prediction.shape
|
||||
|
||||
# frames weight
|
||||
weight_key = getattr(self, "weight", None)
|
||||
weight = batch.get(
|
||||
weight_key,
|
||||
torch.ones(batch_size, num_frames, 1, device=self.model.device),
|
||||
)
|
||||
# (batch_size, num_frames, 1)
|
||||
|
||||
# warm-up
|
||||
warm_up_left = round(self.warm_up[0] / self.duration * num_frames)
|
||||
weight[:, :warm_up_left] = 0.0
|
||||
warm_up_right = round(self.warm_up[1] / self.duration * num_frames)
|
||||
weight[:, num_frames - warm_up_right :] = 0.0
|
||||
|
||||
if self.specifications.powerset:
|
||||
|
||||
powerset = torch.nn.functional.one_hot(
|
||||
torch.argmax(prediction, dim=-1),
|
||||
self.model.powerset.num_powerset_classes,
|
||||
).float()
|
||||
multilabel = self.model.powerset.to_multilabel(powerset)
|
||||
permutated_target, _ = permutate(multilabel, target)
|
||||
|
||||
# FIXME: handle case where target have too many speakers?
|
||||
# since we don't need
|
||||
permutated_target_powerset = self.model.powerset.to_powerset(
|
||||
permutated_target.float()
|
||||
)
|
||||
seg_loss = self.segmentation_loss(
|
||||
prediction, permutated_target_powerset, weight=weight
|
||||
)
|
||||
|
||||
else:
|
||||
permutated_prediction, _ = permutate(target, prediction)
|
||||
seg_loss = self.segmentation_loss(
|
||||
permutated_prediction, target, weight=weight
|
||||
)
|
||||
|
||||
self.model.log(
|
||||
f"{self.logging_prefix}ValSegLoss",
|
||||
seg_loss,
|
||||
on_step=False,
|
||||
on_epoch=True,
|
||||
prog_bar=False,
|
||||
logger=True,
|
||||
)
|
||||
|
||||
if self.vad_loss is None:
|
||||
vad_loss = 0.0
|
||||
|
||||
else:
|
||||
|
||||
# TODO: vad_loss probably does not make sense in powerset mode
|
||||
# because first class (empty set of labels) does exactly this...
|
||||
if self.specifications.powerset:
|
||||
vad_loss = self.voice_activity_detection_loss(
|
||||
prediction, permutated_target_powerset, weight=weight
|
||||
)
|
||||
|
||||
else:
|
||||
vad_loss = self.voice_activity_detection_loss(
|
||||
permutated_prediction, target, weight=weight
|
||||
)
|
||||
|
||||
self.model.log(
|
||||
f"{self.logging_prefix}ValVADLoss",
|
||||
vad_loss,
|
||||
on_step=False,
|
||||
on_epoch=True,
|
||||
prog_bar=False,
|
||||
logger=True,
|
||||
)
|
||||
|
||||
loss = seg_loss + vad_loss
|
||||
|
||||
self.model.log(
|
||||
f"{self.logging_prefix}ValLoss",
|
||||
loss,
|
||||
on_step=False,
|
||||
on_epoch=True,
|
||||
prog_bar=False,
|
||||
logger=True,
|
||||
)
|
||||
|
||||
if self.specifications.powerset:
|
||||
self.model.validation_metric(
|
||||
torch.transpose(
|
||||
multilabel[:, warm_up_left : num_frames - warm_up_right], 1, 2
|
||||
),
|
||||
torch.transpose(
|
||||
target[:, warm_up_left : num_frames - warm_up_right], 1, 2
|
||||
),
|
||||
)
|
||||
else:
|
||||
self.model.validation_metric(
|
||||
torch.transpose(
|
||||
prediction[:, warm_up_left : num_frames - warm_up_right], 1, 2
|
||||
),
|
||||
torch.transpose(
|
||||
target[:, warm_up_left : num_frames - warm_up_right], 1, 2
|
||||
),
|
||||
)
|
||||
|
||||
self.model.log_dict(
|
||||
self.model.validation_metric,
|
||||
on_step=False,
|
||||
on_epoch=True,
|
||||
prog_bar=True,
|
||||
logger=True,
|
||||
)
|
||||
|
||||
# log first batch visualization every 2^n epochs.
|
||||
if (
|
||||
self.model.current_epoch == 0
|
||||
or math.log2(self.model.current_epoch) % 1 > 0
|
||||
or batch_idx > 0
|
||||
):
|
||||
return
|
||||
|
||||
# visualize first 9 validation samples of first batch in Tensorboard/MLflow
|
||||
|
||||
if self.specifications.powerset:
|
||||
y = permutated_target_powerset.float().cpu().numpy()
|
||||
y_pred = multilabel.cpu().numpy()
|
||||
else:
|
||||
y = target.float().cpu().numpy()
|
||||
y_pred = permutated_prediction.cpu().numpy()
|
||||
|
||||
# prepare 3 x 3 grid (or smaller if batch size is smaller)
|
||||
num_samples = min(self.batch_size, 9)
|
||||
nrows = math.ceil(math.sqrt(num_samples))
|
||||
ncols = math.ceil(num_samples / nrows)
|
||||
fig, axes = plt.subplots(
|
||||
nrows=2 * nrows, ncols=ncols, figsize=(8, 5), squeeze=False
|
||||
)
|
||||
|
||||
# reshape target so that there is one line per class when plotting it
|
||||
y[y == 0] = np.NaN
|
||||
if len(y.shape) == 2:
|
||||
y = y[:, :, np.newaxis]
|
||||
y *= np.arange(y.shape[2])
|
||||
|
||||
# plot each sample
|
||||
for sample_idx in range(num_samples):
|
||||
|
||||
# find where in the grid it should be plotted
|
||||
row_idx = sample_idx // nrows
|
||||
col_idx = sample_idx % ncols
|
||||
|
||||
# plot target
|
||||
ax_ref = axes[row_idx * 2 + 0, col_idx]
|
||||
sample_y = y[sample_idx]
|
||||
ax_ref.plot(sample_y)
|
||||
ax_ref.set_xlim(0, len(sample_y))
|
||||
ax_ref.set_ylim(-1, sample_y.shape[1])
|
||||
ax_ref.get_xaxis().set_visible(False)
|
||||
ax_ref.get_yaxis().set_visible(False)
|
||||
|
||||
# plot predictions
|
||||
ax_hyp = axes[row_idx * 2 + 1, col_idx]
|
||||
sample_y_pred = y_pred[sample_idx]
|
||||
ax_hyp.axvspan(0, warm_up_left, color="k", alpha=0.5, lw=0)
|
||||
ax_hyp.axvspan(
|
||||
num_frames - warm_up_right, num_frames, color="k", alpha=0.5, lw=0
|
||||
)
|
||||
ax_hyp.plot(sample_y_pred)
|
||||
ax_hyp.set_ylim(-0.1, 1.1)
|
||||
ax_hyp.set_xlim(0, len(sample_y))
|
||||
ax_hyp.get_xaxis().set_visible(False)
|
||||
|
||||
plt.tight_layout()
|
||||
|
||||
if isinstance(self.model.logger, TensorBoardLogger):
|
||||
self.model.logger.experiment.add_figure(
|
||||
f"{self.logging_prefix}ValSamples", fig, self.model.current_epoch
|
||||
)
|
||||
elif isinstance(self.model.logger, MLFlowLogger):
|
||||
self.model.logger.experiment.log_figure(
|
||||
run_id=self.model.logger.run_id,
|
||||
figure=fig,
|
||||
artifact_file=f"{self.logging_prefix}ValSamples_epoch{self.model.current_epoch}.png",
|
||||
)
|
||||
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def main(protocol: str, subset: str = "test", model: str = "pyannote/segmentation"):
|
||||
"""Evaluate a segmentation model"""
|
||||
|
||||
@@ -194,7 +194,7 @@ def build_permutation_graph(
|
||||
cost_func: Callable[[torch.Tensor, torch.Tensor], torch.Tensor] = mae_cost_func,
|
||||
) -> nx.Graph:
|
||||
"""Build permutation graph
|
||||
|
||||
|
||||
Parameters
|
||||
----------
|
||||
segmentations : (num_chunks, num_frames, local_num_speakers)-shaped SlidingWindowFeature
|
||||
@@ -203,7 +203,7 @@ def build_permutation_graph(
|
||||
Threshold above which a speaker is considered active. Defaults to 0.5
|
||||
cost_func : callable
|
||||
Cost function used to find the optimal bijective mapping between speaker activations
|
||||
of two overlapping chunks. Expects two (num_frames, num_classes) torch.tensor as input
|
||||
of two overlapping chunks. Expects two (num_frames, num_classes) torch.tensor as input
|
||||
and returns cost as a (num_classes, ) torch.tensor. Defaults to mae_cost_func.
|
||||
|
||||
Returns
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
# MIT License
|
||||
#
|
||||
# Copyright (c) 2023- CNRS
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
# AUTHORS
|
||||
# Hervé BREDIN - https://herve.niderb.fr
|
||||
# Alexis PLAQUET
|
||||
|
||||
try:
|
||||
from functools import cached_property
|
||||
except ImportError:
|
||||
from backports.cached_property import cached_property
|
||||
|
||||
from itertools import combinations
|
||||
|
||||
import scipy.special
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
|
||||
class Powerset(nn.Module):
|
||||
"""Powerset to multilabel conversion, and back.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
num_classes : int
|
||||
Number of regular classes.
|
||||
max_set_size : int
|
||||
Maximum number of classes in each set.
|
||||
"""
|
||||
|
||||
def __init__(self, num_classes: int, max_set_size: int):
|
||||
super().__init__()
|
||||
self.num_classes = num_classes
|
||||
self.max_set_size = max_set_size
|
||||
|
||||
mapping = self.build_mapping()
|
||||
self.register_buffer("mapping", mapping, persistent=False)
|
||||
|
||||
@cached_property
|
||||
def num_powerset_classes(self) -> int:
|
||||
# compute number of subsets of size at most "max_set_size"
|
||||
# e.g. with num_classes = 3 and max_set_size = 2:
|
||||
# {}, {0}, {1}, {2}, {0, 1}, {0, 2}, {1, 2}
|
||||
return int(
|
||||
sum(
|
||||
scipy.special.binom(self.num_classes, i)
|
||||
for i in range(0, self.max_set_size + 1)
|
||||
)
|
||||
)
|
||||
|
||||
def build_mapping(self) -> torch.Tensor:
|
||||
mapping = torch.zeros(self.num_powerset_classes, self.num_classes)
|
||||
powerset_k = 0
|
||||
for set_size in range(0, self.max_set_size + 1):
|
||||
for current_set in combinations(range(self.num_classes), set_size):
|
||||
mapping[powerset_k, current_set] = 1
|
||||
powerset_k += 1
|
||||
|
||||
return mapping
|
||||
|
||||
def to_multilabel(self, powerset: torch.Tensor) -> torch.Tensor:
|
||||
"""Convert (hard) predictions from powerset to multi-label
|
||||
|
||||
Parameter
|
||||
---------
|
||||
powerset : (batch_size, num_frames, num_powerset_classes) torch.Tensor
|
||||
Hard predictions in "powerset" space.
|
||||
|
||||
Returns
|
||||
-------
|
||||
multi_label : (batch_size, num_frames, num_classes) torch.Tensor
|
||||
Hard predictions in "multi-label" space.
|
||||
|
||||
Note
|
||||
----
|
||||
This method will not complain if `powerset` is provided a soft predictions
|
||||
(e.g. the output of a softmax-ed classifier). However, in that particular
|
||||
case, the resulting soft multi-label output will not make much sense.
|
||||
"""
|
||||
return torch.matmul(powerset, self.mapping)
|
||||
|
||||
def to_powerset(self, multilabel: torch.Tensor) -> torch.Tensor:
|
||||
"""Convert (hard) predictions from multi-label to powerset
|
||||
|
||||
Parameter
|
||||
---------
|
||||
multi_label : (batch_size, num_frames, num_classes) torch.Tensor
|
||||
Prediction in "multi-label" space.
|
||||
|
||||
Returns
|
||||
-------
|
||||
powerset : (batch_size, num_frames, num_powerset_classes) torch.Tensor
|
||||
Hard, one-hot prediction in "powerset" space.
|
||||
|
||||
Note
|
||||
----
|
||||
This method will not complain if `multilabel` is provided a soft predictions
|
||||
(e.g. the output of a sigmoid-ed classifier). However, in that particular
|
||||
case, the resulting powerset output will most likely not make much sense.
|
||||
"""
|
||||
return F.one_hot(
|
||||
torch.argmax(torch.matmul(multilabel, self.mapping.T), dim=-1),
|
||||
num_classes=self.num_powerset_classes,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
powerset = Powerset(3, 3)
|
||||
print(powerset.mapping)
|
||||
|
||||
# simulate a sequence where each frame is assigned to a different powerset class
|
||||
one_sequence = [
|
||||
[0] * powerset.num_powerset_classes
|
||||
for _ in range(powerset.num_powerset_classes)
|
||||
]
|
||||
for i in range(powerset.num_powerset_classes):
|
||||
one_sequence[i][i] = 1.0
|
||||
|
||||
# make a batch out of this sequence and the same sequence in reverse order
|
||||
batch_powerset = torch.tensor([one_sequence, one_sequence[::-1]])
|
||||
print(batch_powerset)
|
||||
print(batch_powerset.shape)
|
||||
|
||||
# convert from powerset to multi-label
|
||||
batch_multilabel = powerset.to_multilabel(batch_powerset)
|
||||
print(batch_multilabel)
|
||||
print(batch_multilabel.shape)
|
||||
|
||||
# convert batch to powerset
|
||||
reconstruction = powerset.to_powerset(batch_multilabel)
|
||||
print(reconstruction)
|
||||
print(reconstruction.shape)
|
||||
@@ -84,4 +84,3 @@ def test_can_crop_from_file_like():
|
||||
assert isinstance(wav, Tensor)
|
||||
assert sr == 16000
|
||||
assert wav.shape[1] == 0.5 * 16000
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
# MIT License
|
||||
#
|
||||
# Copyright (c) 2023- CNRS
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
|
||||
import torch
|
||||
|
||||
from pyannote.audio.utils.powerset import Powerset
|
||||
|
||||
|
||||
def test_roundtrip():
|
||||
|
||||
for num_classes in range(2, 5):
|
||||
for max_set_size in range(1, num_classes + 1):
|
||||
|
||||
powerset = Powerset(num_classes, max_set_size)
|
||||
|
||||
# simulate a sequence where each frame is assigned to a different powerset class
|
||||
one_sequence = [
|
||||
[0] * powerset.num_powerset_classes
|
||||
for _ in range(powerset.num_powerset_classes)
|
||||
]
|
||||
for i in range(powerset.num_powerset_classes):
|
||||
one_sequence[i][i] = 1.0
|
||||
|
||||
# make a batch out of this sequence and the same sequence in reverse order
|
||||
batch_powerset = torch.tensor([one_sequence, one_sequence[::-1]])
|
||||
|
||||
# convert from powerset to multi-label
|
||||
batch_multilabel = powerset.to_multilabel(batch_powerset)
|
||||
|
||||
# convert batch back to powerset
|
||||
reconstruction = powerset.to_powerset(batch_multilabel)
|
||||
|
||||
assert torch.equal(batch_powerset, reconstruction)
|
||||
Reference in new issue
Block a user