diff --git a/pyannote/audio/cli/train.py b/pyannote/audio/cli/train.py index 4e418ff6..0fb66b4c 100644 --- a/pyannote/audio/cli/train.py +++ b/pyannote/audio/cli/train.py @@ -21,21 +21,20 @@ # SOFTWARE. import functools -from typing import Iterable import os +from typing import Iterable import hydra from hydra.utils import instantiate from omegaconf import DictConfig, OmegaConf - from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint from pytorch_lightning.loggers import TensorBoardLogger from pytorch_lightning.utilities.seed import seed_everything from torch.nn import Parameter from torch.optim import Optimizer +from torch_audiomentations.utils.config import from_dict as get_augmentation from pyannote.database import FileFinder, get_protocol -from torch_audiomentations.utils.config import from_dict as get_augmentation def get_optimizer( @@ -51,7 +50,7 @@ def main(cfg: DictConfig) -> None: # so that each model initializes with the same weights when using DDP. seed = int(os.environ.get("PL_GLOBAL_SEED", "0")) seed_everything(seed=seed) - + protocol = get_protocol(cfg.protocol, preprocessors={"audio": FileFinder()}) # TODO: configure scheduler @@ -59,8 +58,12 @@ def main(cfg: DictConfig) -> None: # TODO: remove this OmegaConf.to_container hack once bug is solved: # https://github.com/omry/omegaconf/pull/443 - augmentation = get_augmentation(OmegaConf.to_container(cfg.augmentation)) if "augmentation" in cfg else None - + augmentation = ( + get_augmentation(OmegaConf.to_container(cfg.augmentation)) + if "augmentation" in cfg + else None + ) + optimizer = functools.partial(get_optimizer, cfg=cfg) task = instantiate( @@ -90,12 +93,12 @@ def main(cfg: DictConfig) -> None: save_last=True, save_weights_only=False, dirpath=".", - filename=f"{{epoch}}-{{{monitor}:.3f}}", + filename=f"{{epoch}}-{{{monitor}:.6f}}", verbose=cfg.verbose, ) callbacks.append(model_checkpoint) - # TODO: add option to configure early stopping patience + # TODO: add option to configure early stopping patience early_stopping = EarlyStopping( monitor=monitor, mode=mode, @@ -114,7 +117,6 @@ def main(cfg: DictConfig) -> None: # log_graph=True, ) - trainer = instantiate( cfg.trainer, callbacks=callbacks, diff --git a/pyannote/audio/core/inference.py b/pyannote/audio/core/inference.py index b02f86f2..d4135f35 100644 --- a/pyannote/audio/core/inference.py +++ b/pyannote/audio/core/inference.py @@ -1,6 +1,6 @@ # MIT License # -# Copyright (c) 2020 CNRS +# Copyright (c) 2020-2021 CNRS # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -20,20 +20,22 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +import math import warnings +from collections import Counter, deque from functools import cached_property from pathlib import Path -from typing import Dict, List, Optional, Text, Tuple, Union +from typing import Deque, Dict, List, Optional, Text, Tuple, Union import numpy as np import torch from einops import rearrange from pytorch_lightning.utilities.memory import is_oom_error -from scipy.optimize import linear_sum_assignment from pyannote.audio.core.io import AudioFile from pyannote.audio.core.model import Model, ModelIntrospection, load_from_checkpoint from pyannote.audio.core.task import Scale, TaskSpecification +from pyannote.audio.utils.permutation import permutate from pyannote.core import Segment, SlidingWindow, SlidingWindowFeature TaskName = Union[Text, None] @@ -87,8 +89,8 @@ class Inference: scale = task_specifications.scale if scale == Scale.FRAME and window == "whole": warnings.warn( - 'Using "whole" `window` inference with a frame-based model might lead to bad results ' - 'and huge memory consumption: it is recommended to set `window` to "sliding".' + 'Using "whole" `window` inference with a frame-based model might lead to bad results ' + 'and huge memory consumption: it is recommended to set `window` to "sliding".' ) self.window = window @@ -101,7 +103,7 @@ class Inference: self.model.to(self.device) # chunk duration used during training. for multi-task, - #  we assume that the same duration was used for each task. + # we assume that the same duration was used for each task. training_duration = self.task_specifications[0][1].duration if duration is None: @@ -113,7 +115,7 @@ class Inference: ) self.duration = duration - #  step between consecutive chunks + # step between consecutive chunks if step is None: step = 0.1 * self.duration if step > self.duration: @@ -153,7 +155,7 @@ class Inference: Notes ----- - If model is mono-task, `task_name` is set to None. + If model is mono-task, `task_name` is set to None. """ with torch.no_grad(): @@ -276,7 +278,7 @@ class Inference: for t, (task_name, task_specifications) in enumerate(self.task_specifications): # if model outputs just one vector per chunk, return the outputs as they are - #  (i.e. do not aggregate them) + # (i.e. do not aggregate them) if task_specifications.scale == Scale.CHUNK: frames = SlidingWindow( start=0.0, duration=self.duration, step=self.step @@ -291,7 +293,7 @@ class Inference: for task_name, output in self.infer(last_chunk[None]).items() } - #  use model introspection to estimate the total number of frames + # use model introspection to estimate the total number of frames _, model_introspection = self.model_introspection[t] num_frames, dimension = model_introspection(num_samples) num_frames_per_chunk, _ = model_introspection(window_size) @@ -314,10 +316,12 @@ class Inference: (num_frames, 1), dtype=np.float32 ) - # loop on the outputs of sliding chunks if task_specifications.permutation_invariant: - previous_output = None + # previous outputs that overlap with current output by at least 50% + maxlen = max(1, math.floor(0.5 * self.duration / self.step)) + previous_outputs: Deque[np.ndarray] = deque([], maxlen=maxlen) + # loop on the outputs of sliding chunks for c, output in enumerate(outputs[task_name]): start_sample = c * step_size start_frame, _ = model_introspection(start_sample) @@ -325,9 +329,9 @@ class Inference: if task_specifications.permutation_invariant: if c > 0: output = self.permutate( - previous_output, output, num_frames_per_step + np.stack(previous_outputs), output, num_frames_per_step ) - previous_output = output + previous_outputs.append(output) aggregated_output[start_frame : start_frame + num_frames_per_chunk] += ( output * hamming @@ -340,12 +344,12 @@ class Inference: # process last (right-aligned) chunk separately if has_last_chunk: - if ( - task_specifications.permutation_invariant - and previous_output is not None - ): + if task_specifications.permutation_invariant and previous_outputs: + # FIXME last_output[task_name] = self.permutate( - previous_output, last_output[task_name], num_frames_last_step + previous_outputs[-1][np.newaxis], + last_output[task_name], + num_frames_last_step, ) aggregated_output[-num_frames_per_chunk:] += ( @@ -368,41 +372,43 @@ class Inference: return results[None] def permutate( - self, output: np.ndarray, next_output: np.ndarray, step_size: int + self, past_outputs: Deque[np.ndarray], output: np.ndarray, step_size: int ) -> np.ndarray: - """Find correlation-maximizing permutation between two consecutive outputs + """Find optimal permutation between past outputs and current output Parameters ---------- + past_outputs : deque of(num_frames, num_classes) np.ndarray + Previous output output : (num_frames, num_classes) np.ndarray - Output - next_output : (num_frames, num_classes) np.ndarray - Next output + Current output step_size : int - Step between output and next_output. Should be smaller than num_frames. + Step between previous and current outputs. + Should be smaller than num_frames. Returns ------- perm_output : (num_frames, num_classes) np.ndarray - Permutated next_output. + Permutated current output. """ num_frames, num_classes = output.shape - hamming = np.hamming(num_frames) - weights = np.sqrt(hamming[step_size:] * hamming[: num_frames - step_size]) - cost = np.zeros((num_classes, num_classes)) - for o in range(num_classes): - for n in range(num_classes): - cost[o, n] = np.average( - (output[step_size:, o] - next_output[: num_frames - step_size, n]) - ** 2, - weights=weights, - ) + permutations = [] + for o, past_output in enumerate(reversed(past_outputs)): + permutation = permutate( + past_output[np.newaxis, (o + 1) * step_size :], + output[: num_frames - (o + 1) * step_size], + )[1][0] + permutations.append(permutation) - mapping = linear_sum_assignment(cost, maximize=False)[1] + # TODO: track regions where more than one permutation is selected + # as those regions should probably not be trusted too much + # TODO: be even smarter and re-initialize tracking at those regions - return next_output[:, mapping] + # choose most frequent permutation + ((permutation, _),) = Counter(permutations).most_common(1) + return output[:, permutation] def __call__( self, file: AudioFile @@ -472,7 +478,7 @@ class Inference: errors that may result in a different number of audio samples for two chunks of the same duration. - # TODO: document "fixed" better in pyannote.audio.core.io.Audio + # TODO: document "fixed" better in pyannote.audio.core.io.Audio Returns ------- diff --git a/pyannote/audio/tasks/segmentation/segmentation.py b/pyannote/audio/tasks/segmentation/segmentation.py index 02951704..9e66ad5b 100644 --- a/pyannote/audio/tasks/segmentation/segmentation.py +++ b/pyannote/audio/tasks/segmentation/segmentation.py @@ -28,7 +28,6 @@ import numpy as np import torch import torch.nn.functional as F from pytorch_lightning.metrics.functional.classification import auroc -from scipy.optimize import linear_sum_assignment from torch.nn import Parameter from torch.optim import Optimizer from torch_audiomentations.core.transforms_interface import BaseWaveformTransform @@ -37,6 +36,7 @@ from pyannote.audio.core.io import Audio from pyannote.audio.core.model import Model from pyannote.audio.core.task import Problem, Scale, Task, TaskSpecification from pyannote.audio.tasks.segmentation.mixins import SegmentationTaskMixin +from pyannote.audio.utils.permutation import permutate from pyannote.audio.utils.random import create_rng_for_worker from pyannote.core import Segment from pyannote.database import Protocol @@ -85,6 +85,8 @@ class Segmentation(SegmentationTaskMixin, Task): augmentation : BaseWaveformTransform, optional torch_audiomentations waveform transform, used by dataloader during training. + vad_loss : {"bce", "mse"}, optional + Add voice activity detection loss. """ ACRONYM = "seg" @@ -105,6 +107,7 @@ class Segmentation(SegmentationTaskMixin, Task): learning_rate: float = 1e-3, augmentation: BaseWaveformTransform = None, loss: Literal["bce", "mse"] = "bce", + vad_loss: Literal["bce", "mse"] = None, ): super().__init__( @@ -128,6 +131,7 @@ class Segmentation(SegmentationTaskMixin, Task): if loss not in ["bce", "mse"]: raise ValueError("'loss' must be one of {'bce', 'mse'}.") self.loss = loss + self.vad_loss = vad_loss self.specifications = TaskSpecification( problem=Problem.MULTI_LABEL_CLASSIFICATION, @@ -138,20 +142,65 @@ class Segmentation(SegmentationTaskMixin, Task): ) def setup(self, stage=None): - - super().setup(stage=stage) - if stage == "fit": + # loop over the training set, remove annotated regions shorter than + # chunk duration, and keep track of the reference annotations. + self.train = [] + for f in self.protocol.train(): + segments = [ + segment + for segment in f["annotated"] + if segment.duration > self.duration + ] + duration = sum(segment.duration for segment in segments) + self.train.append( + { + "uri": f["uri"], + "annotated": segments, + "annotation": f["annotation"], + "duration": duration, + "audio": f["audio"], + } + ) + # build the list of domains if self.domain is not None: for f in self.train: f["domain"] = f[self.domain] self.domains = list(set(f["domain"] for f in self.train)) + # loop over the validation set, remove annotated regions shorter than + # chunk duration, and keep track of the reference annotations. + self.validation = [] + + for f in self.protocol.development(): + segments = [ + segment + for segment in f["annotated"] + if segment.duration > self.duration + ] + num_chunks = sum( + round(segment.duration // self.duration) for segment in segments + ) + self.validation.append( + { + "uri": f["uri"], + "annotated": segments, + "annotation": f["annotation"], + "num_chunks": num_chunks, + "audio": f["audio"], + } + ) + def setup_loss_func(self, model: Model): - batch_size, num_frames, num_speakers = model(self.example_input_array).shape + example_input_array = self.example_input_array + _, _, num_samples = example_input_array.shape + self.num_samples = num_samples + + batch_size, num_frames, num_speakers = model(example_input_array).shape + self.num_frames = num_frames hamming_window = torch.hamming_window(num_frames, periodic=False).reshape(-1, 1) model.register_buffer("hamming_window", hamming_window) @@ -294,26 +343,54 @@ class Segmentation(SegmentationTaskMixin, Task): yield {"X": X, "y": self.prepare_y(combined_y)} - def permutate(self, y: torch.Tensor, y_pred: torch.Tensor) -> torch.Tensor: - """Find permutation that minimizes average pairwise mean squared error""" + # def segmentation_loss(self, model, y, y_pred): + def segmentation_loss(self, y: torch.Tensor, y_pred: torch.Tensor) -> torch.Tensor: + """Permutation-invariant segmentation loss - # TODO use Asteroid's PIT wrapper + Parameters + ---------- + y : (batch_size, num_frames, num_speakers) torch.Tensor + Speaker activity. + y_pred : torch.Tensor + Speaker activations - batch_size, num_samples, num_speakers = y_pred.shape + Returns + ------- + seg_loss : torch.Tensor + Permutation-invariant segmentation loss + """ - cost = np.zeros((num_speakers, num_speakers)) - mapping = [] - for b in range(batch_size): - for r in range(num_speakers): - for h in range(num_speakers): - cost[r, h] = ( - F.mse_loss(y[b, :, r], y_pred[b, :, h], reduction="mean") - .detach() - .cpu() - ) - mapping.append(linear_sum_assignment(cost, maximize=False)[1]) + permutated_y_pred, _ = permutate(y, y_pred) - return torch.stack([y_pred[b, :, mapping[b]] for b in range(batch_size)]) + if self.loss == "bce": + seg_losses = F.binary_cross_entropy( + permutated_y_pred, y.float(), reduction="none" + ) + + elif self.loss == "mse": + seg_losses = F.mse_loss(permutated_y_pred, y.float(), reduction="none") + + # seg_loss = torch.mean(seg_losses * model.hamming_window) + seg_loss = torch.mean(seg_losses) + + return seg_loss + + def voice_activity_detection_loss( + self, y: torch.Tensor, y_pred: torch.Tensor + ) -> torch.Tensor: + + vad_y_pred, _ = torch.max(y_pred, dim=2, keepdim=False) + vad_y, _ = torch.max(y.float(), dim=2, keepdim=False) + + if self.vad_loss == "bce": + vad_losses = F.binary_cross_entropy(vad_y_pred, vad_y, reduction="none") + + elif self.vad_loss == "mse": + vad_losses = F.mse_loss(vad_y_pred, vad_y, reduction="none") + + vad_loss = torch.mean(vad_losses) + + return vad_loss def training_step(self, model: Model, batch, batch_idx: int): """Compute permutation-invariant binary cross-entropy @@ -334,18 +411,36 @@ class Segmentation(SegmentationTaskMixin, Task): """ X, y = batch["X"], batch["y"] - y_pred = self.permutate(y, model(X)) - batch_size, num_frames, num_speakers = y_pred.shape + y_pred = model(X) + # loss = self.segmentation_loss(model, y, y_pred) + seg_loss = self.segmentation_loss(y, y_pred) - if self.loss == "bce": - losses = F.binary_cross_entropy(y_pred, y.float(), reduction="none") - elif self.loss == "mse": - losses = F.mse_loss(y_pred, y.float(), reduction="none") + model.log( + f"{self.ACRONYM}@train_seg_loss", + seg_loss, + on_step=True, + on_epoch=True, + prog_bar=False, + logger=True, + ) - # give less importance to start and end of chunks - # using the same (Hamming) window as inference. - loss = torch.mean(losses * model.hamming_window) + if self.vad_loss is None: + vad_loss = 0.0 + + else: + vad_loss = self.voice_activity_detection_loss(y, y_pred) + + model.log( + f"{self.ACRONYM}@train_vad_loss", + vad_loss, + on_step=True, + on_epoch=True, + prog_bar=False, + logger=True, + ) + + loss = seg_loss + vad_loss model.log( f"{self.ACRONYM}@train_loss", @@ -371,7 +466,7 @@ class Segmentation(SegmentationTaskMixin, Task): """ X, y = batch["X"], batch["y"] - y_pred = self.permutate(y, model(X)) + y_pred, _ = permutate(y, model(X)) try: auc = auroc( diff --git a/pyannote/audio/utils/permutation.py b/pyannote/audio/utils/permutation.py new file mode 100644 index 00000000..6a56e7e4 --- /dev/null +++ b/pyannote/audio/utils/permutation.py @@ -0,0 +1,92 @@ +from functools import singledispatch +from typing import Callable, List, Optional, Tuple + +import numpy as np +import torch +import torch.nn.functional as F +from scipy.optimize import linear_sum_assignment + + +@singledispatch +def permutate(y1, y2, cost_func: Optional[Callable] = None): + """Find cost-minimizing permutation + + Parameters + ---------- + y1 : np.ndarray or torch.Tensor + (batch_size, num_samples, num_speakers) + y2 : np.ndarray or torch.Tensor + (num_samples, num_speakers) or (batch_size, num_samples, num_speakers) + cost_func : callable + Takes two (num_samples, num_speakers) sequences and returns (num_speakers, ) pairwise cost. + Defaults to computing mean squared error. + + Returns + ------- + permutated_y2 : np.ndarray or torch.Tensor + (batch_size, num_samples, num_speakers) + permutations : list of tuple + List of permutations + """ + raise TypeError() + + +def mse_cost_func(Y, y): + return torch.mean(F.mse_loss(Y, y, reduction="none"), axis=0) + + +@permutate.register +def permutate_torch( + y1: torch.Tensor, y2: torch.Tensor, cost_func: Optional[Callable] = None +) -> Tuple[torch.Tensor, List[Tuple[int]]]: + + batch_size, num_samples, num_speakers = y1.shape + + if len(y2.shape) == 2: + y2 = y2.expand(batch_size, -1, -1) + + if len(y2.shape) != 3: + msg = "Incorrect shape: should be (batch_size, num_frames, num_speakers)." + raise ValueError(msg) + + batch_size_, num_samples_, num_speakers_ = y2.shape + if ( + batch_size != batch_size_ + or num_samples != num_samples_ + or num_speakers != num_speakers_ + ): + msg = f"Shape mismatch: {tuple(y1.shape)} vs. {tuple(y2.shape)}." + raise ValueError(msg) + + if cost_func is None: + cost_func = mse_cost_func + + permutations = [] + permutated_y2 = [] + + for y1_, y2_ in zip(y1, y2): + with torch.no_grad(): + cost = torch.stack( + [ + cost_func(y2_, y1_[:, i : i + 1].expand(-1, num_speakers)) + for i in range(num_speakers) + ], + ) + permutation = tuple(linear_sum_assignment(cost.cpu())[1]) + permutations.append(permutation) + + permutated_y2.append(y2_[:, permutation]) + + return torch.stack(permutated_y2), permutations + + +@permutate.register +def permutate_numpy( + y1: np.ndarray, y2: np.ndarray, cost_func: Optional[Callable] = None +) -> Tuple[np.ndarray, List[Tuple[int]]]: + + permutated_y2, permutations = permutate( + torch.from_numpy(y1), torch.from_numpy(y2), cost_func=cost_func + ) + + return permutated_y2.numpy(), permutations diff --git a/tests/utils/permutation.py b/tests/utils/permutation.py new file mode 100644 index 00000000..8704735c --- /dev/null +++ b/tests/utils/permutation.py @@ -0,0 +1,52 @@ +import numpy as np +import torch + +from pyannote.audio.utils.permutation import permutate + + +def test_permutate_torch(): + + num_frames, num_speakers = 10, 3 + + actual_permutations = [ + (0, 1, 2), + (0, 2, 1), + (1, 0, 2), + (1, 2, 0), + (2, 0, 1), + (2, 1, 0), + ] + batch_size = len(actual_permutations) + + y2 = torch.randn((num_frames, num_speakers)) + y1 = torch.zeros((batch_size, num_frames, num_speakers)) + + for p, permutation in enumerate(actual_permutations): + y1[p] = y2[:, permutation] + + permutated_y2, permutations = permutate(y1, y2) + assert actual_permutations == permutations + + +def test_permutate_numpy(): + + num_frames, num_speakers = 10, 3 + + actual_permutations = [ + (0, 1, 2), + (0, 2, 1), + (1, 0, 2), + (1, 2, 0), + (2, 0, 1), + (2, 1, 0), + ] + batch_size = len(actual_permutations) + + y2 = np.random.randn(num_frames, num_speakers) + y1 = np.zeros((batch_size, num_frames, num_speakers)) + + for p, permutation in enumerate(actual_permutations): + y1[p] = y2[:, permutation] + + permutated_y2, permutations = permutate(y1, y2) + assert actual_permutations == permutations