mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
feat: add generic "permutate" utility function
* feat: add "vad_loss" option to "Segmentation" * improve: improve permutation-invariant support in "Inference"
This commit is contained in:
1 parent
45f3c20b6b
commit
78a16a2ca3
5 files changed
+326
-79
No files matched your search
@@ -21,21 +21,20 @@
|
||||
# SOFTWARE.
|
||||
|
||||
import functools
|
||||
from typing import Iterable
|
||||
import os
|
||||
from typing import Iterable
|
||||
|
||||
import hydra
|
||||
from hydra.utils import instantiate
|
||||
from omegaconf import DictConfig, OmegaConf
|
||||
|
||||
from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
|
||||
from pytorch_lightning.loggers import TensorBoardLogger
|
||||
from pytorch_lightning.utilities.seed import seed_everything
|
||||
from torch.nn import Parameter
|
||||
from torch.optim import Optimizer
|
||||
from torch_audiomentations.utils.config import from_dict as get_augmentation
|
||||
|
||||
from pyannote.database import FileFinder, get_protocol
|
||||
from torch_audiomentations.utils.config import from_dict as get_augmentation
|
||||
|
||||
|
||||
def get_optimizer(
|
||||
@@ -51,7 +50,7 @@ def main(cfg: DictConfig) -> None:
|
||||
# so that each model initializes with the same weights when using DDP.
|
||||
seed = int(os.environ.get("PL_GLOBAL_SEED", "0"))
|
||||
seed_everything(seed=seed)
|
||||
|
||||
|
||||
protocol = get_protocol(cfg.protocol, preprocessors={"audio": FileFinder()})
|
||||
|
||||
# TODO: configure scheduler
|
||||
@@ -59,8 +58,12 @@ def main(cfg: DictConfig) -> None:
|
||||
|
||||
# TODO: remove this OmegaConf.to_container hack once bug is solved:
|
||||
# https://github.com/omry/omegaconf/pull/443
|
||||
augmentation = get_augmentation(OmegaConf.to_container(cfg.augmentation)) if "augmentation" in cfg else None
|
||||
|
||||
augmentation = (
|
||||
get_augmentation(OmegaConf.to_container(cfg.augmentation))
|
||||
if "augmentation" in cfg
|
||||
else None
|
||||
)
|
||||
|
||||
optimizer = functools.partial(get_optimizer, cfg=cfg)
|
||||
|
||||
task = instantiate(
|
||||
@@ -90,12 +93,12 @@ def main(cfg: DictConfig) -> None:
|
||||
save_last=True,
|
||||
save_weights_only=False,
|
||||
dirpath=".",
|
||||
filename=f"{{epoch}}-{{{monitor}:.3f}}",
|
||||
filename=f"{{epoch}}-{{{monitor}:.6f}}",
|
||||
verbose=cfg.verbose,
|
||||
)
|
||||
callbacks.append(model_checkpoint)
|
||||
|
||||
# TODO: add option to configure early stopping patience
|
||||
# TODO: add option to configure early stopping patience
|
||||
early_stopping = EarlyStopping(
|
||||
monitor=monitor,
|
||||
mode=mode,
|
||||
@@ -114,7 +117,6 @@ def main(cfg: DictConfig) -> None:
|
||||
# log_graph=True,
|
||||
)
|
||||
|
||||
|
||||
trainer = instantiate(
|
||||
cfg.trainer,
|
||||
callbacks=callbacks,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# MIT License
|
||||
#
|
||||
# Copyright (c) 2020 CNRS
|
||||
# Copyright (c) 2020-2021 CNRS
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
@@ -20,20 +20,22 @@
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
import math
|
||||
import warnings
|
||||
from collections import Counter, deque
|
||||
from functools import cached_property
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Text, Tuple, Union
|
||||
from typing import Deque, Dict, List, Optional, Text, Tuple, Union
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from einops import rearrange
|
||||
from pytorch_lightning.utilities.memory import is_oom_error
|
||||
from scipy.optimize import linear_sum_assignment
|
||||
|
||||
from pyannote.audio.core.io import AudioFile
|
||||
from pyannote.audio.core.model import Model, ModelIntrospection, load_from_checkpoint
|
||||
from pyannote.audio.core.task import Scale, TaskSpecification
|
||||
from pyannote.audio.utils.permutation import permutate
|
||||
from pyannote.core import Segment, SlidingWindow, SlidingWindowFeature
|
||||
|
||||
TaskName = Union[Text, None]
|
||||
@@ -87,8 +89,8 @@ class Inference:
|
||||
scale = task_specifications.scale
|
||||
if scale == Scale.FRAME and window == "whole":
|
||||
warnings.warn(
|
||||
'Using "whole" `window` inference with a frame-based model might lead to bad results '
|
||||
'and huge memory consumption: it is recommended to set `window` to "sliding".'
|
||||
'Using "whole" `window` inference with a frame-based model might lead to bad results '
|
||||
'and huge memory consumption: it is recommended to set `window` to "sliding".'
|
||||
)
|
||||
|
||||
self.window = window
|
||||
@@ -101,7 +103,7 @@ class Inference:
|
||||
self.model.to(self.device)
|
||||
|
||||
# chunk duration used during training. for multi-task,
|
||||
# we assume that the same duration was used for each task.
|
||||
# we assume that the same duration was used for each task.
|
||||
training_duration = self.task_specifications[0][1].duration
|
||||
|
||||
if duration is None:
|
||||
@@ -113,7 +115,7 @@ class Inference:
|
||||
)
|
||||
self.duration = duration
|
||||
|
||||
# step between consecutive chunks
|
||||
# step between consecutive chunks
|
||||
if step is None:
|
||||
step = 0.1 * self.duration
|
||||
if step > self.duration:
|
||||
@@ -153,7 +155,7 @@ class Inference:
|
||||
|
||||
Notes
|
||||
-----
|
||||
If model is mono-task, `task_name` is set to None.
|
||||
If model is mono-task, `task_name` is set to None.
|
||||
"""
|
||||
|
||||
with torch.no_grad():
|
||||
@@ -276,7 +278,7 @@ class Inference:
|
||||
|
||||
for t, (task_name, task_specifications) in enumerate(self.task_specifications):
|
||||
# if model outputs just one vector per chunk, return the outputs as they are
|
||||
# (i.e. do not aggregate them)
|
||||
# (i.e. do not aggregate them)
|
||||
if task_specifications.scale == Scale.CHUNK:
|
||||
frames = SlidingWindow(
|
||||
start=0.0, duration=self.duration, step=self.step
|
||||
@@ -291,7 +293,7 @@ class Inference:
|
||||
for task_name, output in self.infer(last_chunk[None]).items()
|
||||
}
|
||||
|
||||
# use model introspection to estimate the total number of frames
|
||||
# use model introspection to estimate the total number of frames
|
||||
_, model_introspection = self.model_introspection[t]
|
||||
num_frames, dimension = model_introspection(num_samples)
|
||||
num_frames_per_chunk, _ = model_introspection(window_size)
|
||||
@@ -314,10 +316,12 @@ class Inference:
|
||||
(num_frames, 1), dtype=np.float32
|
||||
)
|
||||
|
||||
# loop on the outputs of sliding chunks
|
||||
if task_specifications.permutation_invariant:
|
||||
previous_output = None
|
||||
# previous outputs that overlap with current output by at least 50%
|
||||
maxlen = max(1, math.floor(0.5 * self.duration / self.step))
|
||||
previous_outputs: Deque[np.ndarray] = deque([], maxlen=maxlen)
|
||||
|
||||
# loop on the outputs of sliding chunks
|
||||
for c, output in enumerate(outputs[task_name]):
|
||||
start_sample = c * step_size
|
||||
start_frame, _ = model_introspection(start_sample)
|
||||
@@ -325,9 +329,9 @@ class Inference:
|
||||
if task_specifications.permutation_invariant:
|
||||
if c > 0:
|
||||
output = self.permutate(
|
||||
previous_output, output, num_frames_per_step
|
||||
np.stack(previous_outputs), output, num_frames_per_step
|
||||
)
|
||||
previous_output = output
|
||||
previous_outputs.append(output)
|
||||
|
||||
aggregated_output[start_frame : start_frame + num_frames_per_chunk] += (
|
||||
output * hamming
|
||||
@@ -340,12 +344,12 @@ class Inference:
|
||||
# process last (right-aligned) chunk separately
|
||||
if has_last_chunk:
|
||||
|
||||
if (
|
||||
task_specifications.permutation_invariant
|
||||
and previous_output is not None
|
||||
):
|
||||
if task_specifications.permutation_invariant and previous_outputs:
|
||||
# FIXME
|
||||
last_output[task_name] = self.permutate(
|
||||
previous_output, last_output[task_name], num_frames_last_step
|
||||
previous_outputs[-1][np.newaxis],
|
||||
last_output[task_name],
|
||||
num_frames_last_step,
|
||||
)
|
||||
|
||||
aggregated_output[-num_frames_per_chunk:] += (
|
||||
@@ -368,41 +372,43 @@ class Inference:
|
||||
return results[None]
|
||||
|
||||
def permutate(
|
||||
self, output: np.ndarray, next_output: np.ndarray, step_size: int
|
||||
self, past_outputs: Deque[np.ndarray], output: np.ndarray, step_size: int
|
||||
) -> np.ndarray:
|
||||
"""Find correlation-maximizing permutation between two consecutive outputs
|
||||
"""Find optimal permutation between past outputs and current output
|
||||
|
||||
Parameters
|
||||
----------
|
||||
past_outputs : deque of(num_frames, num_classes) np.ndarray
|
||||
Previous output
|
||||
output : (num_frames, num_classes) np.ndarray
|
||||
Output
|
||||
next_output : (num_frames, num_classes) np.ndarray
|
||||
Next output
|
||||
Current output
|
||||
step_size : int
|
||||
Step between output and next_output. Should be smaller than num_frames.
|
||||
Step between previous and current outputs.
|
||||
Should be smaller than num_frames.
|
||||
|
||||
Returns
|
||||
-------
|
||||
perm_output : (num_frames, num_classes) np.ndarray
|
||||
Permutated next_output.
|
||||
Permutated current output.
|
||||
"""
|
||||
|
||||
num_frames, num_classes = output.shape
|
||||
hamming = np.hamming(num_frames)
|
||||
weights = np.sqrt(hamming[step_size:] * hamming[: num_frames - step_size])
|
||||
|
||||
cost = np.zeros((num_classes, num_classes))
|
||||
for o in range(num_classes):
|
||||
for n in range(num_classes):
|
||||
cost[o, n] = np.average(
|
||||
(output[step_size:, o] - next_output[: num_frames - step_size, n])
|
||||
** 2,
|
||||
weights=weights,
|
||||
)
|
||||
permutations = []
|
||||
for o, past_output in enumerate(reversed(past_outputs)):
|
||||
permutation = permutate(
|
||||
past_output[np.newaxis, (o + 1) * step_size :],
|
||||
output[: num_frames - (o + 1) * step_size],
|
||||
)[1][0]
|
||||
permutations.append(permutation)
|
||||
|
||||
mapping = linear_sum_assignment(cost, maximize=False)[1]
|
||||
# TODO: track regions where more than one permutation is selected
|
||||
# as those regions should probably not be trusted too much
|
||||
# TODO: be even smarter and re-initialize tracking at those regions
|
||||
|
||||
return next_output[:, mapping]
|
||||
# choose most frequent permutation
|
||||
((permutation, _),) = Counter(permutations).most_common(1)
|
||||
return output[:, permutation]
|
||||
|
||||
def __call__(
|
||||
self, file: AudioFile
|
||||
@@ -472,7 +478,7 @@ class Inference:
|
||||
errors that may result in a different number of audio samples for two
|
||||
chunks of the same duration.
|
||||
|
||||
# TODO: document "fixed" better in pyannote.audio.core.io.Audio
|
||||
# TODO: document "fixed" better in pyannote.audio.core.io.Audio
|
||||
|
||||
Returns
|
||||
-------
|
||||
|
||||
@@ -28,7 +28,6 @@ import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from pytorch_lightning.metrics.functional.classification import auroc
|
||||
from scipy.optimize import linear_sum_assignment
|
||||
from torch.nn import Parameter
|
||||
from torch.optim import Optimizer
|
||||
from torch_audiomentations.core.transforms_interface import BaseWaveformTransform
|
||||
@@ -37,6 +36,7 @@ from pyannote.audio.core.io import Audio
|
||||
from pyannote.audio.core.model import Model
|
||||
from pyannote.audio.core.task import Problem, Scale, Task, TaskSpecification
|
||||
from pyannote.audio.tasks.segmentation.mixins import SegmentationTaskMixin
|
||||
from pyannote.audio.utils.permutation import permutate
|
||||
from pyannote.audio.utils.random import create_rng_for_worker
|
||||
from pyannote.core import Segment
|
||||
from pyannote.database import Protocol
|
||||
@@ -85,6 +85,8 @@ class Segmentation(SegmentationTaskMixin, Task):
|
||||
augmentation : BaseWaveformTransform, optional
|
||||
torch_audiomentations waveform transform, used by dataloader
|
||||
during training.
|
||||
vad_loss : {"bce", "mse"}, optional
|
||||
Add voice activity detection loss.
|
||||
"""
|
||||
|
||||
ACRONYM = "seg"
|
||||
@@ -105,6 +107,7 @@ class Segmentation(SegmentationTaskMixin, Task):
|
||||
learning_rate: float = 1e-3,
|
||||
augmentation: BaseWaveformTransform = None,
|
||||
loss: Literal["bce", "mse"] = "bce",
|
||||
vad_loss: Literal["bce", "mse"] = None,
|
||||
):
|
||||
|
||||
super().__init__(
|
||||
@@ -128,6 +131,7 @@ class Segmentation(SegmentationTaskMixin, Task):
|
||||
if loss not in ["bce", "mse"]:
|
||||
raise ValueError("'loss' must be one of {'bce', 'mse'}.")
|
||||
self.loss = loss
|
||||
self.vad_loss = vad_loss
|
||||
|
||||
self.specifications = TaskSpecification(
|
||||
problem=Problem.MULTI_LABEL_CLASSIFICATION,
|
||||
@@ -138,20 +142,65 @@ class Segmentation(SegmentationTaskMixin, Task):
|
||||
)
|
||||
|
||||
def setup(self, stage=None):
|
||||
|
||||
super().setup(stage=stage)
|
||||
|
||||
if stage == "fit":
|
||||
|
||||
# loop over the training set, remove annotated regions shorter than
|
||||
# chunk duration, and keep track of the reference annotations.
|
||||
self.train = []
|
||||
for f in self.protocol.train():
|
||||
segments = [
|
||||
segment
|
||||
for segment in f["annotated"]
|
||||
if segment.duration > self.duration
|
||||
]
|
||||
duration = sum(segment.duration for segment in segments)
|
||||
self.train.append(
|
||||
{
|
||||
"uri": f["uri"],
|
||||
"annotated": segments,
|
||||
"annotation": f["annotation"],
|
||||
"duration": duration,
|
||||
"audio": f["audio"],
|
||||
}
|
||||
)
|
||||
|
||||
# build the list of domains
|
||||
if self.domain is not None:
|
||||
for f in self.train:
|
||||
f["domain"] = f[self.domain]
|
||||
self.domains = list(set(f["domain"] for f in self.train))
|
||||
|
||||
# loop over the validation set, remove annotated regions shorter than
|
||||
# chunk duration, and keep track of the reference annotations.
|
||||
self.validation = []
|
||||
|
||||
for f in self.protocol.development():
|
||||
segments = [
|
||||
segment
|
||||
for segment in f["annotated"]
|
||||
if segment.duration > self.duration
|
||||
]
|
||||
num_chunks = sum(
|
||||
round(segment.duration // self.duration) for segment in segments
|
||||
)
|
||||
self.validation.append(
|
||||
{
|
||||
"uri": f["uri"],
|
||||
"annotated": segments,
|
||||
"annotation": f["annotation"],
|
||||
"num_chunks": num_chunks,
|
||||
"audio": f["audio"],
|
||||
}
|
||||
)
|
||||
|
||||
def setup_loss_func(self, model: Model):
|
||||
|
||||
batch_size, num_frames, num_speakers = model(self.example_input_array).shape
|
||||
example_input_array = self.example_input_array
|
||||
_, _, num_samples = example_input_array.shape
|
||||
self.num_samples = num_samples
|
||||
|
||||
batch_size, num_frames, num_speakers = model(example_input_array).shape
|
||||
self.num_frames = num_frames
|
||||
hamming_window = torch.hamming_window(num_frames, periodic=False).reshape(-1, 1)
|
||||
model.register_buffer("hamming_window", hamming_window)
|
||||
|
||||
@@ -294,26 +343,54 @@ class Segmentation(SegmentationTaskMixin, Task):
|
||||
|
||||
yield {"X": X, "y": self.prepare_y(combined_y)}
|
||||
|
||||
def permutate(self, y: torch.Tensor, y_pred: torch.Tensor) -> torch.Tensor:
|
||||
"""Find permutation that minimizes average pairwise mean squared error"""
|
||||
# def segmentation_loss(self, model, y, y_pred):
|
||||
def segmentation_loss(self, y: torch.Tensor, y_pred: torch.Tensor) -> torch.Tensor:
|
||||
"""Permutation-invariant segmentation loss
|
||||
|
||||
# TODO use Asteroid's PIT wrapper
|
||||
Parameters
|
||||
----------
|
||||
y : (batch_size, num_frames, num_speakers) torch.Tensor
|
||||
Speaker activity.
|
||||
y_pred : torch.Tensor
|
||||
Speaker activations
|
||||
|
||||
batch_size, num_samples, num_speakers = y_pred.shape
|
||||
Returns
|
||||
-------
|
||||
seg_loss : torch.Tensor
|
||||
Permutation-invariant segmentation loss
|
||||
"""
|
||||
|
||||
cost = np.zeros((num_speakers, num_speakers))
|
||||
mapping = []
|
||||
for b in range(batch_size):
|
||||
for r in range(num_speakers):
|
||||
for h in range(num_speakers):
|
||||
cost[r, h] = (
|
||||
F.mse_loss(y[b, :, r], y_pred[b, :, h], reduction="mean")
|
||||
.detach()
|
||||
.cpu()
|
||||
)
|
||||
mapping.append(linear_sum_assignment(cost, maximize=False)[1])
|
||||
permutated_y_pred, _ = permutate(y, y_pred)
|
||||
|
||||
return torch.stack([y_pred[b, :, mapping[b]] for b in range(batch_size)])
|
||||
if self.loss == "bce":
|
||||
seg_losses = F.binary_cross_entropy(
|
||||
permutated_y_pred, y.float(), reduction="none"
|
||||
)
|
||||
|
||||
elif self.loss == "mse":
|
||||
seg_losses = F.mse_loss(permutated_y_pred, y.float(), reduction="none")
|
||||
|
||||
# seg_loss = torch.mean(seg_losses * model.hamming_window)
|
||||
seg_loss = torch.mean(seg_losses)
|
||||
|
||||
return seg_loss
|
||||
|
||||
def voice_activity_detection_loss(
|
||||
self, y: torch.Tensor, y_pred: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
|
||||
vad_y_pred, _ = torch.max(y_pred, dim=2, keepdim=False)
|
||||
vad_y, _ = torch.max(y.float(), dim=2, keepdim=False)
|
||||
|
||||
if self.vad_loss == "bce":
|
||||
vad_losses = F.binary_cross_entropy(vad_y_pred, vad_y, reduction="none")
|
||||
|
||||
elif self.vad_loss == "mse":
|
||||
vad_losses = F.mse_loss(vad_y_pred, vad_y, reduction="none")
|
||||
|
||||
vad_loss = torch.mean(vad_losses)
|
||||
|
||||
return vad_loss
|
||||
|
||||
def training_step(self, model: Model, batch, batch_idx: int):
|
||||
"""Compute permutation-invariant binary cross-entropy
|
||||
@@ -334,18 +411,36 @@ class Segmentation(SegmentationTaskMixin, Task):
|
||||
"""
|
||||
|
||||
X, y = batch["X"], batch["y"]
|
||||
y_pred = self.permutate(y, model(X))
|
||||
|
||||
batch_size, num_frames, num_speakers = y_pred.shape
|
||||
y_pred = model(X)
|
||||
# loss = self.segmentation_loss(model, y, y_pred)
|
||||
seg_loss = self.segmentation_loss(y, y_pred)
|
||||
|
||||
if self.loss == "bce":
|
||||
losses = F.binary_cross_entropy(y_pred, y.float(), reduction="none")
|
||||
elif self.loss == "mse":
|
||||
losses = F.mse_loss(y_pred, y.float(), reduction="none")
|
||||
model.log(
|
||||
f"{self.ACRONYM}@train_seg_loss",
|
||||
seg_loss,
|
||||
on_step=True,
|
||||
on_epoch=True,
|
||||
prog_bar=False,
|
||||
logger=True,
|
||||
)
|
||||
|
||||
# give less importance to start and end of chunks
|
||||
# using the same (Hamming) window as inference.
|
||||
loss = torch.mean(losses * model.hamming_window)
|
||||
if self.vad_loss is None:
|
||||
vad_loss = 0.0
|
||||
|
||||
else:
|
||||
vad_loss = self.voice_activity_detection_loss(y, y_pred)
|
||||
|
||||
model.log(
|
||||
f"{self.ACRONYM}@train_vad_loss",
|
||||
vad_loss,
|
||||
on_step=True,
|
||||
on_epoch=True,
|
||||
prog_bar=False,
|
||||
logger=True,
|
||||
)
|
||||
|
||||
loss = seg_loss + vad_loss
|
||||
|
||||
model.log(
|
||||
f"{self.ACRONYM}@train_loss",
|
||||
@@ -371,7 +466,7 @@ class Segmentation(SegmentationTaskMixin, Task):
|
||||
"""
|
||||
|
||||
X, y = batch["X"], batch["y"]
|
||||
y_pred = self.permutate(y, model(X))
|
||||
y_pred, _ = permutate(y, model(X))
|
||||
|
||||
try:
|
||||
auc = auroc(
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
from functools import singledispatch
|
||||
from typing import Callable, List, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from scipy.optimize import linear_sum_assignment
|
||||
|
||||
|
||||
@singledispatch
|
||||
def permutate(y1, y2, cost_func: Optional[Callable] = None):
|
||||
"""Find cost-minimizing permutation
|
||||
|
||||
Parameters
|
||||
----------
|
||||
y1 : np.ndarray or torch.Tensor
|
||||
(batch_size, num_samples, num_speakers)
|
||||
y2 : np.ndarray or torch.Tensor
|
||||
(num_samples, num_speakers) or (batch_size, num_samples, num_speakers)
|
||||
cost_func : callable
|
||||
Takes two (num_samples, num_speakers) sequences and returns (num_speakers, ) pairwise cost.
|
||||
Defaults to computing mean squared error.
|
||||
|
||||
Returns
|
||||
-------
|
||||
permutated_y2 : np.ndarray or torch.Tensor
|
||||
(batch_size, num_samples, num_speakers)
|
||||
permutations : list of tuple
|
||||
List of permutations
|
||||
"""
|
||||
raise TypeError()
|
||||
|
||||
|
||||
def mse_cost_func(Y, y):
|
||||
return torch.mean(F.mse_loss(Y, y, reduction="none"), axis=0)
|
||||
|
||||
|
||||
@permutate.register
|
||||
def permutate_torch(
|
||||
y1: torch.Tensor, y2: torch.Tensor, cost_func: Optional[Callable] = None
|
||||
) -> Tuple[torch.Tensor, List[Tuple[int]]]:
|
||||
|
||||
batch_size, num_samples, num_speakers = y1.shape
|
||||
|
||||
if len(y2.shape) == 2:
|
||||
y2 = y2.expand(batch_size, -1, -1)
|
||||
|
||||
if len(y2.shape) != 3:
|
||||
msg = "Incorrect shape: should be (batch_size, num_frames, num_speakers)."
|
||||
raise ValueError(msg)
|
||||
|
||||
batch_size_, num_samples_, num_speakers_ = y2.shape
|
||||
if (
|
||||
batch_size != batch_size_
|
||||
or num_samples != num_samples_
|
||||
or num_speakers != num_speakers_
|
||||
):
|
||||
msg = f"Shape mismatch: {tuple(y1.shape)} vs. {tuple(y2.shape)}."
|
||||
raise ValueError(msg)
|
||||
|
||||
if cost_func is None:
|
||||
cost_func = mse_cost_func
|
||||
|
||||
permutations = []
|
||||
permutated_y2 = []
|
||||
|
||||
for y1_, y2_ in zip(y1, y2):
|
||||
with torch.no_grad():
|
||||
cost = torch.stack(
|
||||
[
|
||||
cost_func(y2_, y1_[:, i : i + 1].expand(-1, num_speakers))
|
||||
for i in range(num_speakers)
|
||||
],
|
||||
)
|
||||
permutation = tuple(linear_sum_assignment(cost.cpu())[1])
|
||||
permutations.append(permutation)
|
||||
|
||||
permutated_y2.append(y2_[:, permutation])
|
||||
|
||||
return torch.stack(permutated_y2), permutations
|
||||
|
||||
|
||||
@permutate.register
|
||||
def permutate_numpy(
|
||||
y1: np.ndarray, y2: np.ndarray, cost_func: Optional[Callable] = None
|
||||
) -> Tuple[np.ndarray, List[Tuple[int]]]:
|
||||
|
||||
permutated_y2, permutations = permutate(
|
||||
torch.from_numpy(y1), torch.from_numpy(y2), cost_func=cost_func
|
||||
)
|
||||
|
||||
return permutated_y2.numpy(), permutations
|
||||
@@ -0,0 +1,52 @@
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from pyannote.audio.utils.permutation import permutate
|
||||
|
||||
|
||||
def test_permutate_torch():
|
||||
|
||||
num_frames, num_speakers = 10, 3
|
||||
|
||||
actual_permutations = [
|
||||
(0, 1, 2),
|
||||
(0, 2, 1),
|
||||
(1, 0, 2),
|
||||
(1, 2, 0),
|
||||
(2, 0, 1),
|
||||
(2, 1, 0),
|
||||
]
|
||||
batch_size = len(actual_permutations)
|
||||
|
||||
y2 = torch.randn((num_frames, num_speakers))
|
||||
y1 = torch.zeros((batch_size, num_frames, num_speakers))
|
||||
|
||||
for p, permutation in enumerate(actual_permutations):
|
||||
y1[p] = y2[:, permutation]
|
||||
|
||||
permutated_y2, permutations = permutate(y1, y2)
|
||||
assert actual_permutations == permutations
|
||||
|
||||
|
||||
def test_permutate_numpy():
|
||||
|
||||
num_frames, num_speakers = 10, 3
|
||||
|
||||
actual_permutations = [
|
||||
(0, 1, 2),
|
||||
(0, 2, 1),
|
||||
(1, 0, 2),
|
||||
(1, 2, 0),
|
||||
(2, 0, 1),
|
||||
(2, 1, 0),
|
||||
]
|
||||
batch_size = len(actual_permutations)
|
||||
|
||||
y2 = np.random.randn(num_frames, num_speakers)
|
||||
y1 = np.zeros((batch_size, num_frames, num_speakers))
|
||||
|
||||
for p, permutation in enumerate(actual_permutations):
|
||||
y1[p] = y2[:, permutation]
|
||||
|
||||
permutated_y2, permutations = permutate(y1, y2)
|
||||
assert actual_permutations == permutations
|
||||
Reference in new issue
Block a user