feat: add generic "permutate" utility function

* feat: add "vad_loss" option to "Segmentation"
* improve: improve permutation-invariant support in "Inference"
This commit is contained in:
Hervé BREDIN authored and GitHub committed 2021-01-04 16:08:59 +01:00
1 parent 45f3c20b6b
commit 78a16a2ca3
5 files changed
+326 -79

No files matched your search

+11 -9
View File
@@ -21,21 +21,20 @@
# SOFTWARE.
import functools
from typing import Iterable
import os
from typing import Iterable
import hydra
from hydra.utils import instantiate
from omegaconf import DictConfig, OmegaConf
from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
from pytorch_lightning.loggers import TensorBoardLogger
from pytorch_lightning.utilities.seed import seed_everything
from torch.nn import Parameter
from torch.optim import Optimizer
from torch_audiomentations.utils.config import from_dict as get_augmentation
from pyannote.database import FileFinder, get_protocol
from torch_audiomentations.utils.config import from_dict as get_augmentation
def get_optimizer(
@@ -51,7 +50,7 @@ def main(cfg: DictConfig) -> None:
# so that each model initializes with the same weights when using DDP.
seed = int(os.environ.get("PL_GLOBAL_SEED", "0"))
seed_everything(seed=seed)
protocol = get_protocol(cfg.protocol, preprocessors={"audio": FileFinder()})
# TODO: configure scheduler
@@ -59,8 +58,12 @@ def main(cfg: DictConfig) -> None:
# TODO: remove this OmegaConf.to_container hack once bug is solved:
# https://github.com/omry/omegaconf/pull/443
augmentation = get_augmentation(OmegaConf.to_container(cfg.augmentation)) if "augmentation" in cfg else None
augmentation = (
get_augmentation(OmegaConf.to_container(cfg.augmentation))
if "augmentation" in cfg
else None
)
optimizer = functools.partial(get_optimizer, cfg=cfg)
task = instantiate(
@@ -90,12 +93,12 @@ def main(cfg: DictConfig) -> None:
save_last=True,
save_weights_only=False,
dirpath=".",
filename=f"{{epoch}}-{{{monitor}:.3f}}",
filename=f"{{epoch}}-{{{monitor}:.6f}}",
verbose=cfg.verbose,
)
callbacks.append(model_checkpoint)
# TODO: add option to configure early stopping patience
# TODO: add option to configure early stopping patience
early_stopping = EarlyStopping(
monitor=monitor,
mode=mode,
@@ -114,7 +117,6 @@ def main(cfg: DictConfig) -> None:
# log_graph=True,
)
trainer = instantiate(
cfg.trainer,
callbacks=callbacks,
+45 -39
View File
@@ -1,6 +1,6 @@
# MIT License
#
# Copyright (c) 2020 CNRS
# Copyright (c) 2020-2021 CNRS
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
@@ -20,20 +20,22 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import math
import warnings
from collections import Counter, deque
from functools import cached_property
from pathlib import Path
from typing import Dict, List, Optional, Text, Tuple, Union
from typing import Deque, Dict, List, Optional, Text, Tuple, Union
import numpy as np
import torch
from einops import rearrange
from pytorch_lightning.utilities.memory import is_oom_error
from scipy.optimize import linear_sum_assignment
from pyannote.audio.core.io import AudioFile
from pyannote.audio.core.model import Model, ModelIntrospection, load_from_checkpoint
from pyannote.audio.core.task import Scale, TaskSpecification
from pyannote.audio.utils.permutation import permutate
from pyannote.core import Segment, SlidingWindow, SlidingWindowFeature
TaskName = Union[Text, None]
@@ -87,8 +89,8 @@ class Inference:
scale = task_specifications.scale
if scale == Scale.FRAME and window == "whole":
warnings.warn(
'Using "whole" `window` inference with a frame-based model might lead to bad results '
'and huge memory consumption: it is recommended to set `window` to "sliding".'
'Using "whole" `window` inference with a frame-based model might lead to bad results '
'and huge memory consumption: it is recommended to set `window` to "sliding".'
)
self.window = window
@@ -101,7 +103,7 @@ class Inference:
self.model.to(self.device)
# chunk duration used during training. for multi-task,
#  we assume that the same duration was used for each task.
# we assume that the same duration was used for each task.
training_duration = self.task_specifications[0][1].duration
if duration is None:
@@ -113,7 +115,7 @@ class Inference:
)
self.duration = duration
#  step between consecutive chunks
# step between consecutive chunks
if step is None:
step = 0.1 * self.duration
if step > self.duration:
@@ -153,7 +155,7 @@ class Inference:
Notes
-----
If model is mono-task, `task_name` is set to None.
If model is mono-task, `task_name` is set to None.
"""
with torch.no_grad():
@@ -276,7 +278,7 @@ class Inference:
for t, (task_name, task_specifications) in enumerate(self.task_specifications):
# if model outputs just one vector per chunk, return the outputs as they are
#  (i.e. do not aggregate them)
# (i.e. do not aggregate them)
if task_specifications.scale == Scale.CHUNK:
frames = SlidingWindow(
start=0.0, duration=self.duration, step=self.step
@@ -291,7 +293,7 @@ class Inference:
for task_name, output in self.infer(last_chunk[None]).items()
}
#  use model introspection to estimate the total number of frames
# use model introspection to estimate the total number of frames
_, model_introspection = self.model_introspection[t]
num_frames, dimension = model_introspection(num_samples)
num_frames_per_chunk, _ = model_introspection(window_size)
@@ -314,10 +316,12 @@ class Inference:
(num_frames, 1), dtype=np.float32
)
# loop on the outputs of sliding chunks
if task_specifications.permutation_invariant:
previous_output = None
# previous outputs that overlap with current output by at least 50%
maxlen = max(1, math.floor(0.5 * self.duration / self.step))
previous_outputs: Deque[np.ndarray] = deque([], maxlen=maxlen)
# loop on the outputs of sliding chunks
for c, output in enumerate(outputs[task_name]):
start_sample = c * step_size
start_frame, _ = model_introspection(start_sample)
@@ -325,9 +329,9 @@ class Inference:
if task_specifications.permutation_invariant:
if c > 0:
output = self.permutate(
previous_output, output, num_frames_per_step
np.stack(previous_outputs), output, num_frames_per_step
)
previous_output = output
previous_outputs.append(output)
aggregated_output[start_frame : start_frame + num_frames_per_chunk] += (
output * hamming
@@ -340,12 +344,12 @@ class Inference:
# process last (right-aligned) chunk separately
if has_last_chunk:
if (
task_specifications.permutation_invariant
and previous_output is not None
):
if task_specifications.permutation_invariant and previous_outputs:
# FIXME
last_output[task_name] = self.permutate(
previous_output, last_output[task_name], num_frames_last_step
previous_outputs[-1][np.newaxis],
last_output[task_name],
num_frames_last_step,
)
aggregated_output[-num_frames_per_chunk:] += (
@@ -368,41 +372,43 @@ class Inference:
return results[None]
def permutate(
self, output: np.ndarray, next_output: np.ndarray, step_size: int
self, past_outputs: Deque[np.ndarray], output: np.ndarray, step_size: int
) -> np.ndarray:
"""Find correlation-maximizing permutation between two consecutive outputs
"""Find optimal permutation between past outputs and current output
Parameters
----------
past_outputs : deque of(num_frames, num_classes) np.ndarray
Previous output
output : (num_frames, num_classes) np.ndarray
Output
next_output : (num_frames, num_classes) np.ndarray
Next output
Current output
step_size : int
Step between output and next_output. Should be smaller than num_frames.
Step between previous and current outputs.
Should be smaller than num_frames.
Returns
-------
perm_output : (num_frames, num_classes) np.ndarray
Permutated next_output.
Permutated current output.
"""
num_frames, num_classes = output.shape
hamming = np.hamming(num_frames)
weights = np.sqrt(hamming[step_size:] * hamming[: num_frames - step_size])
cost = np.zeros((num_classes, num_classes))
for o in range(num_classes):
for n in range(num_classes):
cost[o, n] = np.average(
(output[step_size:, o] - next_output[: num_frames - step_size, n])
** 2,
weights=weights,
)
permutations = []
for o, past_output in enumerate(reversed(past_outputs)):
permutation = permutate(
past_output[np.newaxis, (o + 1) * step_size :],
output[: num_frames - (o + 1) * step_size],
)[1][0]
permutations.append(permutation)
mapping = linear_sum_assignment(cost, maximize=False)[1]
# TODO: track regions where more than one permutation is selected
# as those regions should probably not be trusted too much
# TODO: be even smarter and re-initialize tracking at those regions
return next_output[:, mapping]
# choose most frequent permutation
((permutation, _),) = Counter(permutations).most_common(1)
return output[:, permutation]
def __call__(
self, file: AudioFile
@@ -472,7 +478,7 @@ class Inference:
errors that may result in a different number of audio samples for two
chunks of the same duration.
# TODO: document "fixed" better in pyannote.audio.core.io.Audio
# TODO: document "fixed" better in pyannote.audio.core.io.Audio
Returns
-------
+126 -31
View File
@@ -28,7 +28,6 @@ import numpy as np
import torch
import torch.nn.functional as F
from pytorch_lightning.metrics.functional.classification import auroc
from scipy.optimize import linear_sum_assignment
from torch.nn import Parameter
from torch.optim import Optimizer
from torch_audiomentations.core.transforms_interface import BaseWaveformTransform
@@ -37,6 +36,7 @@ from pyannote.audio.core.io import Audio
from pyannote.audio.core.model import Model
from pyannote.audio.core.task import Problem, Scale, Task, TaskSpecification
from pyannote.audio.tasks.segmentation.mixins import SegmentationTaskMixin
from pyannote.audio.utils.permutation import permutate
from pyannote.audio.utils.random import create_rng_for_worker
from pyannote.core import Segment
from pyannote.database import Protocol
@@ -85,6 +85,8 @@ class Segmentation(SegmentationTaskMixin, Task):
augmentation : BaseWaveformTransform, optional
torch_audiomentations waveform transform, used by dataloader
during training.
vad_loss : {"bce", "mse"}, optional
Add voice activity detection loss.
"""
ACRONYM = "seg"
@@ -105,6 +107,7 @@ class Segmentation(SegmentationTaskMixin, Task):
learning_rate: float = 1e-3,
augmentation: BaseWaveformTransform = None,
loss: Literal["bce", "mse"] = "bce",
vad_loss: Literal["bce", "mse"] = None,
):
super().__init__(
@@ -128,6 +131,7 @@ class Segmentation(SegmentationTaskMixin, Task):
if loss not in ["bce", "mse"]:
raise ValueError("'loss' must be one of {'bce', 'mse'}.")
self.loss = loss
self.vad_loss = vad_loss
self.specifications = TaskSpecification(
problem=Problem.MULTI_LABEL_CLASSIFICATION,
@@ -138,20 +142,65 @@ class Segmentation(SegmentationTaskMixin, Task):
)
def setup(self, stage=None):
super().setup(stage=stage)
if stage == "fit":
# loop over the training set, remove annotated regions shorter than
# chunk duration, and keep track of the reference annotations.
self.train = []
for f in self.protocol.train():
segments = [
segment
for segment in f["annotated"]
if segment.duration > self.duration
]
duration = sum(segment.duration for segment in segments)
self.train.append(
{
"uri": f["uri"],
"annotated": segments,
"annotation": f["annotation"],
"duration": duration,
"audio": f["audio"],
}
)
# build the list of domains
if self.domain is not None:
for f in self.train:
f["domain"] = f[self.domain]
self.domains = list(set(f["domain"] for f in self.train))
# loop over the validation set, remove annotated regions shorter than
# chunk duration, and keep track of the reference annotations.
self.validation = []
for f in self.protocol.development():
segments = [
segment
for segment in f["annotated"]
if segment.duration > self.duration
]
num_chunks = sum(
round(segment.duration // self.duration) for segment in segments
)
self.validation.append(
{
"uri": f["uri"],
"annotated": segments,
"annotation": f["annotation"],
"num_chunks": num_chunks,
"audio": f["audio"],
}
)
def setup_loss_func(self, model: Model):
batch_size, num_frames, num_speakers = model(self.example_input_array).shape
example_input_array = self.example_input_array
_, _, num_samples = example_input_array.shape
self.num_samples = num_samples
batch_size, num_frames, num_speakers = model(example_input_array).shape
self.num_frames = num_frames
hamming_window = torch.hamming_window(num_frames, periodic=False).reshape(-1, 1)
model.register_buffer("hamming_window", hamming_window)
@@ -294,26 +343,54 @@ class Segmentation(SegmentationTaskMixin, Task):
yield {"X": X, "y": self.prepare_y(combined_y)}
def permutate(self, y: torch.Tensor, y_pred: torch.Tensor) -> torch.Tensor:
"""Find permutation that minimizes average pairwise mean squared error"""
# def segmentation_loss(self, model, y, y_pred):
def segmentation_loss(self, y: torch.Tensor, y_pred: torch.Tensor) -> torch.Tensor:
"""Permutation-invariant segmentation loss
# TODO use Asteroid's PIT wrapper
Parameters
----------
y : (batch_size, num_frames, num_speakers) torch.Tensor
Speaker activity.
y_pred : torch.Tensor
Speaker activations
batch_size, num_samples, num_speakers = y_pred.shape
Returns
-------
seg_loss : torch.Tensor
Permutation-invariant segmentation loss
"""
cost = np.zeros((num_speakers, num_speakers))
mapping = []
for b in range(batch_size):
for r in range(num_speakers):
for h in range(num_speakers):
cost[r, h] = (
F.mse_loss(y[b, :, r], y_pred[b, :, h], reduction="mean")
.detach()
.cpu()
)
mapping.append(linear_sum_assignment(cost, maximize=False)[1])
permutated_y_pred, _ = permutate(y, y_pred)
return torch.stack([y_pred[b, :, mapping[b]] for b in range(batch_size)])
if self.loss == "bce":
seg_losses = F.binary_cross_entropy(
permutated_y_pred, y.float(), reduction="none"
)
elif self.loss == "mse":
seg_losses = F.mse_loss(permutated_y_pred, y.float(), reduction="none")
# seg_loss = torch.mean(seg_losses * model.hamming_window)
seg_loss = torch.mean(seg_losses)
return seg_loss
def voice_activity_detection_loss(
self, y: torch.Tensor, y_pred: torch.Tensor
) -> torch.Tensor:
vad_y_pred, _ = torch.max(y_pred, dim=2, keepdim=False)
vad_y, _ = torch.max(y.float(), dim=2, keepdim=False)
if self.vad_loss == "bce":
vad_losses = F.binary_cross_entropy(vad_y_pred, vad_y, reduction="none")
elif self.vad_loss == "mse":
vad_losses = F.mse_loss(vad_y_pred, vad_y, reduction="none")
vad_loss = torch.mean(vad_losses)
return vad_loss
def training_step(self, model: Model, batch, batch_idx: int):
"""Compute permutation-invariant binary cross-entropy
@@ -334,18 +411,36 @@ class Segmentation(SegmentationTaskMixin, Task):
"""
X, y = batch["X"], batch["y"]
y_pred = self.permutate(y, model(X))
batch_size, num_frames, num_speakers = y_pred.shape
y_pred = model(X)
# loss = self.segmentation_loss(model, y, y_pred)
seg_loss = self.segmentation_loss(y, y_pred)
if self.loss == "bce":
losses = F.binary_cross_entropy(y_pred, y.float(), reduction="none")
elif self.loss == "mse":
losses = F.mse_loss(y_pred, y.float(), reduction="none")
model.log(
f"{self.ACRONYM}@train_seg_loss",
seg_loss,
on_step=True,
on_epoch=True,
prog_bar=False,
logger=True,
)
# give less importance to start and end of chunks
# using the same (Hamming) window as inference.
loss = torch.mean(losses * model.hamming_window)
if self.vad_loss is None:
vad_loss = 0.0
else:
vad_loss = self.voice_activity_detection_loss(y, y_pred)
model.log(
f"{self.ACRONYM}@train_vad_loss",
vad_loss,
on_step=True,
on_epoch=True,
prog_bar=False,
logger=True,
)
loss = seg_loss + vad_loss
model.log(
f"{self.ACRONYM}@train_loss",
@@ -371,7 +466,7 @@ class Segmentation(SegmentationTaskMixin, Task):
"""
X, y = batch["X"], batch["y"]
y_pred = self.permutate(y, model(X))
y_pred, _ = permutate(y, model(X))
try:
auc = auroc(
+92
View File
@@ -0,0 +1,92 @@
from functools import singledispatch
from typing import Callable, List, Optional, Tuple
import numpy as np
import torch
import torch.nn.functional as F
from scipy.optimize import linear_sum_assignment
@singledispatch
def permutate(y1, y2, cost_func: Optional[Callable] = None):
"""Find cost-minimizing permutation
Parameters
----------
y1 : np.ndarray or torch.Tensor
(batch_size, num_samples, num_speakers)
y2 : np.ndarray or torch.Tensor
(num_samples, num_speakers) or (batch_size, num_samples, num_speakers)
cost_func : callable
Takes two (num_samples, num_speakers) sequences and returns (num_speakers, ) pairwise cost.
Defaults to computing mean squared error.
Returns
-------
permutated_y2 : np.ndarray or torch.Tensor
(batch_size, num_samples, num_speakers)
permutations : list of tuple
List of permutations
"""
raise TypeError()
def mse_cost_func(Y, y):
return torch.mean(F.mse_loss(Y, y, reduction="none"), axis=0)
@permutate.register
def permutate_torch(
y1: torch.Tensor, y2: torch.Tensor, cost_func: Optional[Callable] = None
) -> Tuple[torch.Tensor, List[Tuple[int]]]:
batch_size, num_samples, num_speakers = y1.shape
if len(y2.shape) == 2:
y2 = y2.expand(batch_size, -1, -1)
if len(y2.shape) != 3:
msg = "Incorrect shape: should be (batch_size, num_frames, num_speakers)."
raise ValueError(msg)
batch_size_, num_samples_, num_speakers_ = y2.shape
if (
batch_size != batch_size_
or num_samples != num_samples_
or num_speakers != num_speakers_
):
msg = f"Shape mismatch: {tuple(y1.shape)} vs. {tuple(y2.shape)}."
raise ValueError(msg)
if cost_func is None:
cost_func = mse_cost_func
permutations = []
permutated_y2 = []
for y1_, y2_ in zip(y1, y2):
with torch.no_grad():
cost = torch.stack(
[
cost_func(y2_, y1_[:, i : i + 1].expand(-1, num_speakers))
for i in range(num_speakers)
],
)
permutation = tuple(linear_sum_assignment(cost.cpu())[1])
permutations.append(permutation)
permutated_y2.append(y2_[:, permutation])
return torch.stack(permutated_y2), permutations
@permutate.register
def permutate_numpy(
y1: np.ndarray, y2: np.ndarray, cost_func: Optional[Callable] = None
) -> Tuple[np.ndarray, List[Tuple[int]]]:
permutated_y2, permutations = permutate(
torch.from_numpy(y1), torch.from_numpy(y2), cost_func=cost_func
)
return permutated_y2.numpy(), permutations
+52
View File
@@ -0,0 +1,52 @@
import numpy as np
import torch
from pyannote.audio.utils.permutation import permutate
def test_permutate_torch():
num_frames, num_speakers = 10, 3
actual_permutations = [
(0, 1, 2),
(0, 2, 1),
(1, 0, 2),
(1, 2, 0),
(2, 0, 1),
(2, 1, 0),
]
batch_size = len(actual_permutations)
y2 = torch.randn((num_frames, num_speakers))
y1 = torch.zeros((batch_size, num_frames, num_speakers))
for p, permutation in enumerate(actual_permutations):
y1[p] = y2[:, permutation]
permutated_y2, permutations = permutate(y1, y2)
assert actual_permutations == permutations
def test_permutate_numpy():
num_frames, num_speakers = 10, 3
actual_permutations = [
(0, 1, 2),
(0, 2, 1),
(1, 0, 2),
(1, 2, 0),
(2, 0, 1),
(2, 1, 0),
]
batch_size = len(actual_permutations)
y2 = np.random.randn(num_frames, num_speakers)
y1 = np.zeros((batch_size, num_frames, num_speakers))
for p, permutation in enumerate(actual_permutations):
y1[p] = y2[:, permutation]
permutated_y2, permutations = permutate(y1, y2)
assert actual_permutations == permutations