diff --git a/README.md b/README.md index 8ea284fb..2b07cfe2 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Neural speaker diarization with pyannote.audio +# Neural speaker diarization This is the development branch of upcoming `pyannote.audio` 2.0 for which it has been decided to rewrite almost everything from scratch. Highlights of this upcoming release will be: @@ -8,7 +8,7 @@ This is the development branch of upcoming `pyannote.audio` 2.0 for which it has - data augmentation with [torch-audiomentations](https://github.com/asteroid-team/torch-audiomentations) - [huggingface](https://huggingface.co/pyannote) model hosting - [prodigy](https://prodi.gy) recipes for audio annotations -- online [demo](https://share.streamlit.io/pyannote/pyannote-audio-demo/main) based on [streamlit](https://www.streamlit.io) +- online [demo](https://huggingface.co/spaces/pyannote/segmentation) based on [streamlit](https://www.streamlit.io) ## Installation diff --git a/pyannote/audio/cli/train.py b/pyannote/audio/cli/train.py index 957a27a8..e816ac0b 100644 --- a/pyannote/audio/cli/train.py +++ b/pyannote/audio/cli/train.py @@ -81,7 +81,6 @@ def main(cfg: DictConfig) -> Optional[float]: # add task-dependent layers so that later call to model.parameters() # does return all layers (even task-dependent ones). this is already # done for pretrained models (TODO: check that this is true) - task.setup(stage="fit") model.setup(stage="fit") def configure_optimizers(self): @@ -127,7 +126,7 @@ def main(cfg: DictConfig) -> Optional[float]: monitor=monitor, mode=direction, save_top_k=None if monitor is None else 5, - period=1, + every_n_epochs=1, save_last=True, save_weights_only=False, dirpath=".", diff --git a/pyannote/audio/core/model.py b/pyannote/audio/core/model.py index 6ce7e9ab..5a07768b 100644 --- a/pyannote/audio/core/model.py +++ b/pyannote/audio/core/model.py @@ -37,6 +37,7 @@ from huggingface_hub import cached_download, hf_hub_url from pytorch_lightning.core.memory import ModelSummary from pytorch_lightning.utilities.cloud_io import load as pl_load from semver import VersionInfo +from torch.utils.data import DataLoader from pyannote.audio import __version__ from pyannote.audio.core.io import Audio @@ -274,19 +275,16 @@ class Model(pl.LightningModule): sample_rate=self.hparams.sample_rate, mono=self.hparams.num_channels == 1 ) - @property - def datamodule(self): - return self.task - @property def example_input_array(self) -> torch.Tensor: batch_size = 3 if self.task is None else self.task.batch_size + duration = 2.0 if self.task is None else self.task.duration return torch.randn( ( batch_size, self.hparams.num_channels, - int(self.hparams.sample_rate * self.specifications.duration), + int(self.hparams.sample_rate * duration), ), device=self.device, ) @@ -347,6 +345,9 @@ class Model(pl.LightningModule): def setup(self, stage=None): + if stage == "fit": + self.task.setup() + # list of layers before adding task-dependent layers before = set((name, id(module)) for name, module in self.named_modules()) @@ -365,7 +366,11 @@ class Model(pl.LightningModule): # setup custom loss function self.task.setup_loss_func() # setup custom validation metrics - self.task.setup_validation_metric() + validation_metric = self.task.setup_validation_metric() + if validation_metric is not None: + self.validation_metric = validation_metric + self.validation_metric.to(self.device) + # this is to make sure introspection is performed here, once and for all _ = self.introspection @@ -477,11 +482,21 @@ class Model(pl.LightningModule): """ return self.helper_default_activation(self.specifications) + # training data logic is delegated to the task because the + # model does not really need to know how it is being used. + def train_dataloader(self) -> DataLoader: + return self.task.train_dataloader() + # training step logic is delegated to the task because the # model does not really need to know how it is being used. def training_step(self, batch, batch_idx): return self.task.training_step(batch, batch_idx) + # validation data logic is delegated to the task because the + # model does not really need to know how it is being used. + def val_dataloader(self) -> DataLoader: + return self.task.val_dataloader() + # validation logic is delegated to the task because the # model does not really need to know how it is being used. def validation_step(self, batch, batch_idx): @@ -822,7 +837,6 @@ class Model(pl.LightningModule): if task is not None: model.task = task - task.setup(stage="fit") model.setup(stage="fit") try: diff --git a/pyannote/audio/core/task.py b/pyannote/audio/core/task.py index 8aa88684..b42c9c90 100644 --- a/pyannote/audio/core/task.py +++ b/pyannote/audio/core/task.py @@ -218,13 +218,8 @@ class Task(pl.LightningDataModule): """ pass - def setup(self, stage=None): - """Called at the beginning of fit and test just before Model.setup() - - Parameters - ---------- - stage : "fit" or "test" - Whether model is being trained ("fit") or used for inference ("test"). + def setup(self, stage: Optional[str] = None): + """Called at the beginning of training at the very beginning of Model.setup(stage="fit") Notes ----- diff --git a/pyannote/audio/pipelines/speaker_verification.py b/pyannote/audio/pipelines/speaker_verification.py new file mode 100644 index 00000000..530dc531 --- /dev/null +++ b/pyannote/audio/pipelines/speaker_verification.py @@ -0,0 +1,157 @@ +# MIT License +# +# Copyright (c) 2021 CNRS +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import numpy as np +import torch + +from pyannote.audio import Inference, Model, Pipeline +from pyannote.audio.core.io import AudioFile +from pyannote.audio.pipelines.utils import PipelineModel, get_devices, get_model + + +class SpeakerEmbedding(Pipeline): + """Speaker embedding pipeline + + This pipeline assumes that each file contains exactly one speaker + and extracts one single embedding from the whole file. + + Parameters + ---------- + embedding : Model, str, or dict, optional + Pretrained embedding model. Defaults to "pyannote/embedding". + See pyannote.audio.pipelines.utils.get_model for supported format. + segmentation : Model, str, or dict, optional + Pretrained segmentation (or voice activity detection) model. + See pyannote.audio.pipelines.utils.get_model for supported format. + Defaults to no voice activity detection. + + Usage + ----- + >>> from pyannote.audio.pipelines import SpeakerEmbedding + >>> pipeline = SpeakerEmbedding() + >>> emb1 = pipeline("speaker1.wav") + >>> emb2 = pipeline("speaker2.wav") + >>> from scipy.spatial.distance import cdist + >>> distance = cdist(emb1, emb2, metric="cosine")[0,0] + """ + + def __init__( + self, + embedding: PipelineModel = "pyannote/embedding", + segmentation: PipelineModel = None, + ): + super().__init__() + + self.embedding = embedding + self.segmentation = segmentation + + self.embedding_model_: Model = get_model(embedding) + + if self.segmentation is None: + models = [self.embedding_model_] + else: + segmentation_model: Model = get_model(self.segmentation) + models = [self.embedding_model_, segmentation_model] + + # send models to GPU (when GPUs are available and model is not already on GPU) + cpu_models = [model for model in models if model.device.type == "cpu"] + for cpu_model, gpu_device in zip( + cpu_models, get_devices(needs=len(cpu_models)) + ): + cpu_model.to(gpu_device) + + if self.segmentation is not None: + self.voice_activity_ = Inference( + segmentation_model, + pre_aggregation_hook=lambda scores: np.max( + scores, axis=-1, keepdims=True + ), + ) + + def apply(self, file: AudioFile) -> np.ndarray: + + device = self.embedding_model_.device + + # read audio file and send it to GPU + waveform = self.embedding_model_.audio(file)[0][None].to(device) + + if self.segmentation is None: + weights = None + else: + # obtain voice activity scores + weights = self.voice_activity_(file).data + # HACK -- this should be fixed upstream + weights[np.isnan(weights)] = 0.0 + weights = torch.from_numpy(weights ** 3)[None, :, 0].to(device) + + # extract speaker embedding on parts of + with torch.no_grad(): + return self.embedding_model_(waveform, weights=weights).cpu().numpy() + + +def main( + protocol: str = "VoxCeleb.SpeakerVerification.VoxCeleb1", + subset: str = "test", + embedding: str = "pyannote/embedding", + segmentation: str = None, +): + + import typer + from scipy.spatial.distance import cdist + from tqdm import tqdm + + from pyannote.database import FileFinder, get_protocol + from pyannote.metrics.binary_classification import det_curve + + pipeline = SpeakerEmbedding(embedding=embedding, segmentation=segmentation) + + protocol = get_protocol(protocol, preprocessors={"audio": FileFinder()}) + + y_true, y_pred = [], [] + + emb = dict() + + trials = getattr(protocol, f"{subset}_trial")() + + for t, trial in enumerate(tqdm(trials)): + + audio1 = trial["file1"]["audio"] + if audio1 not in emb: + emb[audio1] = pipeline(audio1) + + audio2 = trial["file2"]["audio"] + if audio2 not in emb: + emb[audio2] = pipeline(audio2) + + y_pred.append(cdist(emb[audio1], emb[audio2], metric="cosine")[0][0]) + y_true.append(trial["reference"]) + + _, _, _, eer = det_curve(y_true, np.array(y_pred), distances=True) + typer.echo( + f"{protocol.name} | {subset} | {embedding} | {segmentation} | EER = {100 * eer:.3f}%" + ) + + +if __name__ == "__main__": + import typer + + typer.run(main) diff --git a/pyannote/audio/tasks/embedding/mixins.py b/pyannote/audio/tasks/embedding/mixins.py index 18f0ac68..4bb7b41c 100644 --- a/pyannote/audio/tasks/embedding/mixins.py +++ b/pyannote/audio/tasks/embedding/mixins.py @@ -1,6 +1,6 @@ # MIT License # -# Copyright (c) 2020 CNRS +# Copyright (c) 2020-2021 CNRS # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -23,20 +23,18 @@ import math from typing import Optional -import numpy as np import torch -from torch.utils.data import DataLoader +import torch.nn.functional as F +from torchmetrics import AUROC from tqdm import tqdm -from pyannote.audio.core.task import Problem, Resolution, Specifications, ValDataset +from pyannote.audio.core.task import Problem, Resolution, Specifications from pyannote.audio.utils.random import create_rng_for_worker from pyannote.core import Segment -from pyannote.core.utils.distance import cdist from pyannote.database.protocol import ( SpeakerDiarizationProtocol, SpeakerVerificationProtocol, ) -from pyannote.metrics.binary_classification import det_curve class SupervisedRepresentationLearningTaskMixin: @@ -74,65 +72,60 @@ class SupervisedRepresentationLearningTaskMixin: def batch_size(self, batch_size: int): self.batch_size_ = batch_size - def setup(self, stage=None): + def setup(self, stage: Optional[str] = None): - if stage == "fit": + # loop over the training set, remove annotated regions shorter than + # chunk duration, and keep track of the reference annotations, per class. - # loop over the training set, remove annotated regions shorter than - # chunk duration, and keep track of the reference annotations, per class. + self._train = dict() - self._train = dict() + desc = f"Loading {self.protocol.name} training labels" + for f in tqdm(iterable=self.protocol.train(), desc=desc, unit="file"): - desc = f"Loading {self.protocol.name} training labels" - for f in tqdm(iterable=self.protocol.train(), desc=desc, unit="file"): + for klass in f["annotation"].labels(): - for klass in f["annotation"].labels(): + # keep class's (long enough) speech turns... + speech_turns = [ + segment + for segment in f["annotation"].label_timeline(klass) + if segment.duration > self.min_duration + ] - # keep class's (long enough) speech turns... - speech_turns = [ - segment - for segment in f["annotation"].label_timeline(klass) - if segment.duration > self.duration - ] + # skip if there is no speech turns left + if not speech_turns: + continue - # skip if there is no speech turns left - if not speech_turns: - continue + # ... and their total duration + duration = sum(segment.duration for segment in speech_turns) - # ... and their total duration - duration = sum(segment.duration for segment in speech_turns) + # add class to the list of classes + if klass not in self._train: + self._train[klass] = list() - # add class to the list of classes - if klass not in self._train: - self._train[klass] = list() + self._train[klass].append( + { + "uri": f["uri"], + "audio": f["audio"], + "duration": duration, + "speech_turns": speech_turns, + } + ) - self._train[klass].append( - { - "uri": f["uri"], - "audio": f["audio"], - "duration": duration, - "speech_turns": speech_turns, - } - ) + self.specifications = Specifications( + problem=Problem.REPRESENTATION, + resolution=Resolution.CHUNK, + duration=self.duration, + classes=sorted(self._train), + ) - self.specifications = Specifications( - problem=Problem.REPRESENTATION, - resolution=Resolution.CHUNK, - duration=self.duration, - classes=sorted(self._train), - ) + if not self.has_validation: + return - if not self.has_validation: - return + if isinstance(self.protocol, SpeakerVerificationProtocol): + self._validation = list(self.protocol.development_trial()) - if isinstance(self.protocol, SpeakerVerificationProtocol): - sessions = dict() - for trial in self.protocol.development_trial(): - for session in ["file1", "file2"]: - session_hash = self.helper_trial_hash(trial[session]) - if session_hash not in sessions: - sessions[session_hash] = trial[session] - self._validation = list(sessions.items()) + def setup_validation_metric(self): + return AUROC(compute_on_step=False) def train__iter__(self): """Iterate over training samples @@ -183,18 +176,29 @@ class SupervisedRepresentationLearningTaskMixin: k=1, ) - # select one chunk at random (with uniform distribution) - start_time = rng.uniform( - speech_turn.start, speech_turn.end - batch_duration - ) - chunk = Segment(start_time, start_time + batch_duration) + # if speech turn is too short, pad with zeros + if speech_turn.duration < batch_duration: + X, _ = self.model.audio.crop(file, speech_turn, mode="center") + num_missing_frames = ( + math.floor(batch_duration * self.model.audio.sample_rate) + - X.shape[1] + ) + left_pad = rng.randint(0, num_missing_frames) + X = F.pad(X, (left_pad, num_missing_frames - left_pad)) - X, _ = self.model.audio.crop( - file, - chunk, - mode="center", - fixed=batch_duration, - ) + # if it is long enough, select chunk at random + else: + start_time = rng.uniform( + speech_turn.start, speech_turn.end - batch_duration + ) + chunk = Segment(start_time, start_time + batch_duration) + + X, _ = self.model.audio.crop( + file, + chunk, + mode="center", + fixed=batch_duration, + ) yield {"X": X, "y": y} @@ -225,21 +229,33 @@ class SupervisedRepresentationLearningTaskMixin: ) return {"loss": loss} - @staticmethod - def helper_trial_hash(file) -> int: - return hash((file["database"], file["uri"], tuple(file["try_with"]))) - def val__getitem__(self, idx): if isinstance(self.protocol, SpeakerVerificationProtocol): - session_hash, session = self._validation[idx] - X = np.concatenate( - [ - self.model.audio.crop(session, segment, mode="center")[0] - for segment in session["try_with"] - ], - axis=0, - ) - return {"session_hash": session_hash, "X": X} + trial = self._validation[idx] + + data = dict() + for idx in [1, 2]: + file = trial[f"file{idx:d}"] + duration = self.model.audio.get_duration(file) + if duration > self.duration: + middle = Segment( + 0.5 * duration - 0.5 * self.duration, + 0.5 * duration + 0.5 * self.duration, + ) + X, _ = self.model.audio.crop( + file, middle, mode="center", fixed=self.duration + ) + else: + X, _ = self.model.audio(file) + num_missing_frames = ( + math.floor(self.duration * self.model.audio.sample_rate) + - X.shape[1] + ) + X = F.pad(X, (0, num_missing_frames)) + data[f"X{idx:d}"] = X + data["y"] = trial["reference"] + + return data elif isinstance(self.protocol, SpeakerDiarizationProtocol): pass @@ -255,80 +271,34 @@ class SupervisedRepresentationLearningTaskMixin: def validation_step(self, batch, batch_idx: int): if isinstance(self.protocol, SpeakerVerificationProtocol): - return ( - batch["session_hash"][0].detach().cpu().numpy().item(), - self.model(batch["X"]).detach().cpu().numpy(), + + with torch.no_grad(): + emb1 = self.model(batch["X1"]).detach() + emb2 = self.model(batch["X2"]).detach() + y_pred = F.cosine_similarity(emb1, emb2) + + y_true = batch["y"] + self.model.validation_metric(y_pred, y_true) + + self.model.log( + f"{self.ACRONYM}@val_auroc", + self.model.validation_metric, + on_step=False, + on_epoch=True, + prog_bar=True, + logger=True, ) elif isinstance(self.protocol, SpeakerDiarizationProtocol): pass - def validation_epoch_end(self, outputs): - - if isinstance(self.protocol, SpeakerVerificationProtocol): - - embeddings = dict(outputs) - - y_true, y_pred = [], [] - for trial in self.protocol.development_trial(): - - session1_hash = self.helper_trial_hash(trial["file1"]) - session2_hash = self.helper_trial_hash(trial["file2"]) - - try: - emb1 = embeddings[session1_hash] - emb2 = embeddings[session2_hash] - except KeyError: - return - - y_pred.append(cdist(emb1, emb2, metric="cosine").item()) - y_true.append(trial["reference"]) - - y_pred = np.array(y_pred) - y_true = np.array(y_true) - - num_target_trials = np.sum(y_true) - num_non_target_trials = np.sum(1.0 - y_true) - if num_target_trials > 2 and num_non_target_trials > 2: - fpr, fnr, thresholds, eer = det_curve(y_true, y_pred, distances=True) - - self.model.log( - f"{self.ACRONYM}@val_eer", - torch.tensor(eer, device=self.model.device), - logger=True, - on_epoch=True, - prog_bar=True, - sync_dist=True, - ) - - elif isinstance(self.protocol, SpeakerDiarizationProtocol): - pass - - def val_dataloader(self) -> Optional[DataLoader]: - - if self.has_validation: - - if isinstance(self.protocol, SpeakerVerificationProtocol): - return DataLoader( - ValDataset(self), - batch_size=1, - pin_memory=self.pin_memory, - drop_last=False, - ) - - elif isinstance(self.protocol, SpeakerDiarizationProtocol): - return None - - else: - return None - @property def val_monitor(self): if self.has_validation: if isinstance(self.protocol, SpeakerVerificationProtocol): - return f"{self.ACRONYM}@val_eer", "min" + return f"{self.ACRONYM}@val_auroc", "max" elif isinstance(self.protocol, SpeakerDiarizationProtocol): return None, "min" diff --git a/pyannote/audio/tasks/segmentation/mixins.py b/pyannote/audio/tasks/segmentation/mixins.py index 0f4cc544..f30f16fc 100644 --- a/pyannote/audio/tasks/segmentation/mixins.py +++ b/pyannote/audio/tasks/segmentation/mixins.py @@ -27,7 +27,8 @@ from typing import List, Optional, Text, Tuple import matplotlib.pyplot as plt import numpy as np -from pytorch_lightning.metrics.classification import FBeta +from einops import rearrange +from torchmetrics import FBeta from typing_extensions import Literal from pyannote.audio.core.io import Audio, AudioFile @@ -39,92 +40,86 @@ from pyannote.core import Annotation, Segment, SlidingWindow, SlidingWindowFeatu class SegmentationTaskMixin: """Methods common to most segmentation tasks""" - def setup(self, stage=None): + def setup(self, stage: Optional[str] = None): - if stage == "fit": + # ================================================================== + # PREPARE TRAINING DATA + # ================================================================== - # ================================================================== - # PREPARE TRAINING DATA - # ================================================================== + self._train = [] + self._train_metadata = dict() - self._train = [] - self._train_metadata = dict() + for f in self.protocol.train(): - for f in self.protocol.train(): + file = dict() - file = dict() + for key, value in f.items(): - for key, value in f.items(): + # keep track of unique labels in self._train_metadata["annotation"] + if key == "annotation": + for label in value.labels(): + self._train_metadata.setdefault("annotation", set()).add(label) - # keep track of unique labels in self._train_metadata["annotation"] - if key == "annotation": - for label in value.labels(): - self._train_metadata.setdefault("annotation", set()).add( - label - ) + # pass "audio" entry as it is + elif key == "audio": + pass - # pass "audio" entry as it is - elif key == "audio": - pass + # remove segments shorter than chunks from "annotated" entry + elif key == "annotated": + value = [ + segment for segment in value if segment.duration > self.duration + ] + file["_annotated_duration"] = sum( + segment.duration for segment in value + ) - # remove segments shorter than chunks from "annotated" entry - elif key == "annotated": - value = [ - segment - for segment in value - if segment.duration > self.duration - ] - file["_annotated_duration"] = sum( - segment.duration for segment in value - ) + # keey track of unique text-like entries (incl. "uri" and "database") + # and pass them as they are + elif isinstance(value, Text): + self._train_metadata.setdefault(key, set()).add(value) - # keey track of unique text-like entries (incl. "uri" and "database") - # and pass them as they are - elif isinstance(value, Text): - self._train_metadata.setdefault(key, set()).add(value) + # pass score-like entries as they are + elif isinstance(value, SlidingWindowFeature): + pass - # pass score-like entries as they are - elif isinstance(value, SlidingWindowFeature): - pass + else: + msg = ( + f"Protocol '{self.protocol.name}' defines a '{key}' entry of type {type(value)} " + f"which we do not know how to handle." + ) + warnings.warn(msg) + file[key] = value - else: - msg = ( - f"Protocol '{self.protocol.name}' defines a '{key}' entry of type {type(value)} " - f"which we do not know how to handle." - ) - warnings.warn(msg) - file[key] = value + self._train.append(file) - self._train.append(file) + self._train_metadata = { + key: sorted(values) for key, values in self._train_metadata.items() + } - self._train_metadata = { - key: sorted(values) for key, values in self._train_metadata.items() - } + # ================================================================== + # PREPARE VALIDATION DATA + # ================================================================== - # ================================================================== - # PREPARE VALIDATION DATA - # ================================================================== + if not self.has_validation: + return - if not self.has_validation: - return + self._validation = [] - self._validation = [] + for f in self.protocol.development(): - for f in self.protocol.development(): + for segment in f["annotated"]: - for segment in f["annotated"]: + if segment.duration < self.duration: + continue - if segment.duration < self.duration: - continue + num_chunks = round(segment.duration // self.duration) - num_chunks = round(segment.duration // self.duration) + for c in range(num_chunks): + start_time = segment.start + c * self.duration + chunk = Segment(start_time, start_time + self.duration) + self._validation.append((f, chunk)) - for c in range(num_chunks): - start_time = segment.start + c * self.duration - chunk = Segment(start_time, start_time + self.duration) - self._validation.append((f, chunk)) - - random.shuffle(self._validation) + random.shuffle(self._validation) def setup_validation_metric(self): """Setup default validation metric @@ -132,14 +127,9 @@ class SegmentationTaskMixin: Use macro-average of F-score with a 0.5 threshold """ - self.val_fbeta = FBeta( - len(self.specifications.classes), - beta=1.0, - threshold=0.5, - multilabel=( - self.specifications.problem == Problem.MULTI_LABEL_CLASSIFICATION - ), - average="macro", + num_classes = len(self.specifications.classes) + return FBeta( + num_classes, beta=1.0, threshold=0.5, average="macro", compute_on_step=False ) def prepare_y(self, one_hot_y: np.ndarray) -> np.ndarray: @@ -457,8 +447,11 @@ class SegmentationTaskMixin: def val__len__(self): return len(self._validation) + def validation_postprocess(self, y, y_pred): + return y_pred + def validation_step(self, batch, batch_idx: int): - """Compute area under ROC curve + """Compute validation F-score Parameters ---------- @@ -468,27 +461,66 @@ class SegmentationTaskMixin: Batch index. """ - # move metric to model device - self.val_fbeta.to(self.model.device) - X, y = batch["X"], batch["y"] # X = (batch_size, num_channels, num_samples) - # y = (batch_size, num_frames, num_classes) + # y = (batch_size, num_frames, num_classes) or (batch_size, num_frames) y_pred = self.model(X) _, num_frames, _ = y_pred.shape # y_pred = (batch_size, num_frames, num_classes) + # postprocess + y_pred = self.validation_postprocess(y, y_pred) + + # - remove warm-up frames + # - downsample remaining frames warm_up_left = round(self.warm_up[0] / self.duration * num_frames) warm_up_right = round(self.warm_up[1] / self.duration * num_frames) + preds = y_pred[:, warm_up_left : num_frames - warm_up_right : 10] + target = y[:, warm_up_left : num_frames - warm_up_right : 10] + + # torchmetrics tries to be smart about the type of machine learning problem + # pyannote.audio is more explicit so we have to reshape target and preds for + # torchmetrics to be happy... more details can be found here: + # https://torchmetrics.readthedocs.io/en/latest/references/modules.html#input-types + + if self.specifications.problem == Problem.BINARY_CLASSIFICATION: + # target: shape (batch_size, num_frames), type binary + # preds: shape (batch_size, num_frames, 1), type float + + # torchmetrics expects: + # target: shape (N,), type binary + # preds: shape (N,), type float + + self.model.validation_metric(preds.reshape(-1), target.reshape(-1)) + + elif self.specifications.problem == Problem.MULTI_LABEL_CLASSIFICATION: + # target: shape (batch_size, num_frames, num_classes), type binary + # preds: shape (batch_size, num_frames, num_classes), type float + + # torchmetrics expects + # target: shape (N, num_classes), type binary + # preds: shape (N, num_classes), type float + + self.model.validation_metric( + rearrange(preds, "b f c -> (b f) c"), + rearrange(target, "b f c -> (b f) c"), + ) + + elif self.specifications.problem == Problem.MONO_LABEL_CLASSIFICATION: + # target: shape (batch_size, num_frames, num_classes), type binary + # preds: shape (batch_size, num_frames, num_classes), type float + + # torchmetrics expects: + # target: shape (N, ), type int + # preds: shape (N, num_classes), type float + + # TODO: implement when pyannote.audio gets its first mono-label segmentation task + raise NotImplementedError() - val_fbeta = self.val_fbeta( - y_pred[:, warm_up_left : num_frames - warm_up_right : 10].squeeze(), - y[:, warm_up_left : num_frames - warm_up_right : 10].squeeze(), - ) self.model.log( f"{self.ACRONYM}@val_fbeta", - val_fbeta, + self.model.validation_metric, on_step=False, on_epoch=True, prog_bar=True, diff --git a/pyannote/audio/tasks/segmentation/segmentation.py b/pyannote/audio/tasks/segmentation/segmentation.py index 7befb054..486a847f 100644 --- a/pyannote/audio/tasks/segmentation/segmentation.py +++ b/pyannote/audio/tasks/segmentation/segmentation.py @@ -20,11 +20,9 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -import math from collections import Counter -from typing import Text, Tuple, Union +from typing import Optional, Text, Tuple, Union -import matplotlib.pyplot as plt import numpy as np import torch from torch_audiomentations.core.transforms_interface import BaseWaveformTransform @@ -126,55 +124,53 @@ class Segmentation(SegmentationTaskMixin, Task): self.loss = loss self.vad_loss = vad_loss - def setup(self, stage=None): + def setup(self, stage: Optional[str] = None): super().setup(stage=stage) - if stage == "fit": - - # slide a window (with 1s step) over the whole training set - # and keep track of the number of speakers in each location - num_speakers = [] - for file in self._train: - start = file["annotated"][0].start - end = file["annotated"][-1].end - window = SlidingWindow( - start=start, - end=end, - duration=self.duration, - step=1.0, - ) - for chunk in window: - num_speakers.append(len(file["annotation"].crop(chunk).labels())) - - # because there might a few outliers, estimate the upper bound for the - # number of speakers as the 99th percentile - - num_speakers, counts = zip(*list(Counter(num_speakers).items())) - num_speakers, counts = np.array(num_speakers), np.array(counts) - - sorting_indices = np.argsort(num_speakers) - num_speakers = num_speakers[sorting_indices] - counts = counts[sorting_indices] - - self.num_speakers = num_speakers[ - np.where(np.cumsum(counts) / np.sum(counts) > 0.99)[0][0] - ] - - # TODO: add a few more speakers to make sure we don't skip - # too many artificial chunks (which might result in less - # overlap that we think we have) - - # now that we know about the number of speakers upper bound - # we can set task specifications - self.specifications = Specifications( - problem=Problem.MULTI_LABEL_CLASSIFICATION, - resolution=Resolution.FRAME, + # slide a window (with 1s step) over the whole training set + # and keep track of the number of speakers in each location + num_speakers = [] + for file in self._train: + start = file["annotated"][0].start + end = file["annotated"][-1].end + window = SlidingWindow( + start=start, + end=end, duration=self.duration, - warm_up=self.warm_up, - classes=[f"speaker#{i+1}" for i in range(self.num_speakers)], - permutation_invariant=True, + step=1.0, ) + for chunk in window: + num_speakers.append(len(file["annotation"].crop(chunk).labels())) + + # because there might a few outliers, estimate the upper bound for the + # number of speakers as the 99th percentile + + num_speakers, counts = zip(*list(Counter(num_speakers).items())) + num_speakers, counts = np.array(num_speakers), np.array(counts) + + sorting_indices = np.argsort(num_speakers) + num_speakers = num_speakers[sorting_indices] + counts = counts[sorting_indices] + + self.num_speakers = num_speakers[ + np.where(np.cumsum(counts) / np.sum(counts) > 0.99)[0][0] + ] + + # TODO: add a few more speakers to make sure we don't skip + # too many artificial chunks (which might result in less + # overlap that we think we have) + + # now that we know about the number of speakers upper bound + # we can set task specifications + self.specifications = Specifications( + problem=Problem.MULTI_LABEL_CLASSIFICATION, + resolution=Resolution.FRAME, + duration=self.duration, + warm_up=self.warm_up, + classes=[f"speaker#{i+1}" for i in range(self.num_speakers)], + permutation_invariant=True, + ) def prepare_y(self, one_hot_y: np.ndarray): """Zero-pad segmentation targets @@ -372,127 +368,6 @@ class Segmentation(SegmentationTaskMixin, Task): ) return {"loss": loss} - def validation_step(self, batch, batch_idx: int): - """Compute validation F-score - - Parameters - ---------- - batch : dict of torch.Tensor - Current batch. - batch_idx: int - Batch index. - """ - - # move metric to model device - self.val_fbeta.to(self.model.device) - - X, y = batch["X"], batch["y"] - # X = (batch_size, num_channels, num_samples) - # y = (batch_size, num_frames, num_classes) - - y_pred = self.model(X) - _, num_frames, _ = y_pred.shape - # y_pred = (batch_size, num_frames, num_classes) - + def validation_postprocess(self, y, y_pred): permutated_y_pred, _ = permutate(y, y_pred) - - warm_up_left = round(self.warm_up[0] / self.duration * num_frames) - warm_up_right = round(self.warm_up[1] / self.duration * num_frames) - - val_fbeta = self.val_fbeta( - permutated_y_pred[ - :, warm_up_left : num_frames - warm_up_right : 10 - ].squeeze(), - y[:, warm_up_left : num_frames - warm_up_right : 10].squeeze(), - ) - self.model.log( - f"{self.ACRONYM}@val_fbeta", - val_fbeta, - on_step=False, - on_epoch=True, - prog_bar=True, - logger=True, - ) - - # log first batch visualization every 2^n epochs. - if ( - self.model.current_epoch == 0 - or math.log2(self.model.current_epoch) % 1 > 0 - or batch_idx > 0 - ): - return - - # visualize first 9 validation samples of first batch in Tensorboard - X = X.cpu().numpy() - y = y.float().cpu().numpy() - y_pred = y_pred.cpu().numpy() - permutated_y_pred = permutated_y_pred.cpu().numpy() - - # prepare 3 x 3 grid (or smaller if batch size is smaller) - num_samples = min(self.batch_size, 9) - nrows = math.ceil(math.sqrt(num_samples)) - ncols = math.ceil(num_samples / nrows) - fig, axes = plt.subplots( - nrows=4 * nrows, - ncols=ncols, - figsize=(15, 10), - ) - - # reshape target so that there is one line per class when plottingit - y[y == 0] = np.NaN - y *= np.arange(y.shape[2]) - - # plot each sample - for sample_idx in range(num_samples): - - # find where in the grid it should be plotted - row_idx = sample_idx // nrows - col_idx = sample_idx % ncols - - # plot waveform - ax_wav = axes[row_idx * 4 + 0, col_idx] - sample_X = np.mean(X[sample_idx], axis=0) - ax_wav.plot(sample_X) - ax_wav.set_xlim(0, len(sample_X)) - ax_wav.get_xaxis().set_visible(False) - ax_wav.get_yaxis().set_visible(False) - - # plot target - ax_ref = axes[row_idx * 4 + 1, col_idx] - sample_y = y[sample_idx] - ax_ref.plot(sample_y) - ax_ref.set_xlim(0, len(sample_y)) - ax_ref.set_ylim(-1, sample_y.shape[1]) - ax_ref.get_xaxis().set_visible(False) - ax_ref.get_yaxis().set_visible(False) - - # plot prediction - ax_hyp = axes[row_idx * 4 + 2, col_idx] - sample_y_pred = y_pred[sample_idx] - ax_hyp.axvspan(0, warm_up_left, color="k", alpha=0.5, lw=0) - ax_hyp.axvspan( - num_frames - warm_up_right, num_frames, color="k", alpha=0.5, lw=0 - ) - ax_hyp.plot(sample_y_pred) - ax_hyp.set_ylim(-0.1, 1.1) - ax_hyp.set_xlim(0, len(sample_y)) - ax_hyp.get_xaxis().set_visible(False) - - # plot permutated prediction - ax_map = axes[row_idx * 4 + 3, col_idx] - sample_y_pred_map = permutated_y_pred[sample_idx] - ax_map.axvspan(0, warm_up_left, color="k", alpha=0.5, lw=0) - ax_map.axvspan( - num_frames - warm_up_right, num_frames, color="k", alpha=0.5, lw=0 - ) - ax_map.plot(sample_y_pred_map) - ax_map.set_ylim(-0.1, 1.1) - ax_map.set_xlim(0, len(sample_y)) - - plt.tight_layout() - - self.model.logger.experiment.add_figure( - f"{self.ACRONYM}@val_samples", fig, self.model.current_epoch - ) - - plt.close(fig) + return permutated_y_pred diff --git a/pyannote/audio/tasks/segmentation/speaker_tracking.py b/pyannote/audio/tasks/segmentation/speaker_tracking.py index 1852de4a..c726f03d 100644 --- a/pyannote/audio/tasks/segmentation/speaker_tracking.py +++ b/pyannote/audio/tasks/segmentation/speaker_tracking.py @@ -20,7 +20,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -from typing import List, Text, Tuple, Union +from typing import List, Optional, Text, Tuple, Union import numpy as np from torch_audiomentations.core.transforms_interface import BaseWaveformTransform @@ -105,21 +105,19 @@ class SpeakerTracking(SegmentationTaskMixin, Task): # speakers should be tracked. therefore, we postpone # the definition of specifications. - def setup(self, stage=None): + def setup(self, stage: Optional[str] = None): super().setup(stage=stage) - if stage == "fit": - - self.specifications = Specifications( - # one class per speaker - classes=sorted(self._train_metadata["annotation"]), - # multiple speakers can be active at once - problem=Problem.MULTI_LABEL_CLASSIFICATION, - resolution=Resolution.FRAME, - duration=self.duration, - warm_up=self.warm_up, - ) + self.specifications = Specifications( + # one class per speaker + classes=sorted(self._train_metadata["annotation"]), + # multiple speakers can be active at once + problem=Problem.MULTI_LABEL_CLASSIFICATION, + resolution=Resolution.FRAME, + duration=self.duration, + warm_up=self.warm_up, + ) @property def chunk_labels(self) -> List[Text]: diff --git a/requirements.txt b/requirements.txt index 146ffa56..c18992c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,11 +7,12 @@ pyannote.core >=4.1,<5.0 pyannote.database >=4.1,<5.0 pyannote.metrics >=3.0.1,<4.0 pyannote.pipeline >=2.0,<3.0 -pytorch_lightning >= 1.2,<1.3 +pytorch_lightning >=1.4.1,<1.5 pytorch_metric_learning >=0.9.98 semver >=2.10.2,<3.0 soundfile >=0.10.2,<0.11 torch >=1.8.1,<1.9 torch-audiomentations >=0.6.0 torchaudio >=0.8.1,<0.9 +torchmetrics >=0.4.1,<0.5 typing_extensions >=3.7.4.3 diff --git a/tests/augmentation_tests.py b/tests/augmentation_tests.py deleted file mode 100644 index def501da..00000000 --- a/tests/augmentation_tests.py +++ /dev/null @@ -1,38 +0,0 @@ -import pytest -import torch -from torch.nn import Module - -from pyannote.audio.augmentation.registry import ( - register_augmentation, - unregister_augmentation, -) - - -class RandomAugmentation(Module): - "Fake Noise tranform" - - def forward(self, waveforms): - if not self.training: - return waveforms - return torch.ones_like(waveforms) - - -def test_can_register_augmentation(): - for when in ["output", "input"]: - net = Module() - aug = RandomAugmentation() - register_augmentation(aug, net, when=when) - assert hasattr(net, "__augmentation") - assert net.__augmentation[when] == aug - - -def test_can_unregistrer_augmentation(): - net = Module() - register_augmentation(RandomAugmentation(), net, when="output") - unregister_augmentation(net, when="output") - assert not hasattr(net.__augmentation, "output") - - -def test_fail_unregister_augmentation(): - with pytest.raises(ValueError): - unregister_augmentation(RandomAugmentation(), when="output") diff --git a/tests/tasks/test_reproducibility.py b/tests/tasks/test_reproducibility.py index abb6db88..407b0860 100644 --- a/tests/tasks/test_reproducibility.py +++ b/tests/tasks/test_reproducibility.py @@ -16,7 +16,6 @@ def setup_tasks(task): def create_dl(model, task): m = model(task=task) - task.setup("fit") m.setup("fit") return task.train_dataloader() diff --git a/tests/test_train.py b/tests/test_train.py new file mode 100644 index 00000000..e75a79ea --- /dev/null +++ b/tests/test_train.py @@ -0,0 +1,144 @@ +import pytest +from pytorch_lightning import Trainer + +from pyannote.audio.models.segmentation.debug import SimpleSegmentationModel +from pyannote.audio.tasks import ( + OverlappedSpeechDetection, + Segmentation, + VoiceActivityDetection, +) +from pyannote.database import FileFinder, get_protocol + + +@pytest.fixture() +def protocol(): + return get_protocol( + "Debug.SpeakerDiarization.Debug", preprocessors={"audio": FileFinder()} + ) + + +def test_train_segmentation(protocol): + segmentation = Segmentation(protocol) + model = SimpleSegmentationModel(task=segmentation) + trainer = Trainer(fast_dev_run=True) + trainer.fit(model) + + +def test_train_voice_activity_detection(protocol): + voice_activity_detection = VoiceActivityDetection(protocol) + model = SimpleSegmentationModel(task=voice_activity_detection) + trainer = Trainer(fast_dev_run=True) + trainer.fit(model) + + +def test_train_overlapped_speech_detection(protocol): + overlapped_speech_detection = OverlappedSpeechDetection(protocol) + model = SimpleSegmentationModel(task=overlapped_speech_detection) + trainer = Trainer(fast_dev_run=True) + trainer.fit(model) + + +def test_finetune_with_task_that_does_not_need_setup_for_specs(protocol): + voice_activity_detection = VoiceActivityDetection(protocol) + model = SimpleSegmentationModel(task=voice_activity_detection) + trainer = Trainer(fast_dev_run=True) + trainer.fit(model) + + voice_activity_detection = VoiceActivityDetection(protocol) + model.task = voice_activity_detection + trainer = Trainer(fast_dev_run=True) + trainer.fit(model) + + +def test_finetune_with_task_that_needs_setup_for_specs(protocol): + segmentation = Segmentation(protocol) + model = SimpleSegmentationModel(task=segmentation) + trainer = Trainer(fast_dev_run=True) + trainer.fit(model) + + segmentation = Segmentation(protocol) + model.task = segmentation + trainer = Trainer(fast_dev_run=True) + trainer.fit(model) + + +def test_transfer_with_task_that_does_not_need_setup_for_specs(protocol): + + segmentation = Segmentation(protocol) + model = SimpleSegmentationModel(task=segmentation) + trainer = Trainer(fast_dev_run=True) + trainer.fit(model) + + voice_activity_detection = VoiceActivityDetection(protocol) + model.task = voice_activity_detection + trainer = Trainer(fast_dev_run=True) + trainer.fit(model) + + +def test_transfer_with_task_that_needs_setup_for_specs(protocol): + + voice_activity_detection = VoiceActivityDetection(protocol) + model = SimpleSegmentationModel(task=voice_activity_detection) + trainer = Trainer(fast_dev_run=True) + trainer.fit(model) + + segmentation = Segmentation(protocol) + model.task = segmentation + trainer = Trainer(fast_dev_run=True) + trainer.fit(model) + + +def test_finetune_freeze_with_task_that_needs_setup_for_specs(protocol): + + segmentation = Segmentation(protocol) + model = SimpleSegmentationModel(task=segmentation) + trainer = Trainer(fast_dev_run=True) + trainer.fit(model) + + segmentation = Segmentation(protocol) + model.task = segmentation + model.freeze_up_to("mfcc") + trainer = Trainer(fast_dev_run=True) + trainer.fit(model) + + +def test_finetune_freeze_with_task_that_does_not_need_setup_for_specs(protocol): + + vad = VoiceActivityDetection(protocol) + model = SimpleSegmentationModel(task=vad) + trainer = Trainer(fast_dev_run=True) + trainer.fit(model) + + vad = VoiceActivityDetection(protocol) + model.task = vad + model.freeze_up_to("mfcc") + trainer = Trainer(fast_dev_run=True) + trainer.fit(model) + + +def test_transfer_freeze_with_task_that_does_not_need_setup_for_specs(protocol): + + segmentation = Segmentation(protocol) + model = SimpleSegmentationModel(task=segmentation) + trainer = Trainer(fast_dev_run=True) + trainer.fit(model) + + voice_activity_detection = VoiceActivityDetection(protocol) + model.task = voice_activity_detection + model.freeze_up_to("mfcc") + trainer = Trainer(fast_dev_run=True) + trainer.fit(model) + + +def test_transfer_freeze_with_task_that_needs_setup_for_specs(protocol): + + voice_activity_detection = VoiceActivityDetection(protocol) + model = SimpleSegmentationModel(task=voice_activity_detection) + trainer = Trainer(fast_dev_run=True) + trainer.fit(model) + + segmentation = Segmentation(protocol) + model.task = segmentation + model.freeze_up_to("mfcc") + trainer = Trainer(fast_dev_run=True) + trainer.fit(model)