diff --git a/README.md b/README.md index f0aa5b7a..783b8899 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,98 @@ -# pyannote-audio-v2 -Rewriting pyannote.audio from scratch +# Rewriting pyannote.audio from scratch +This is alpha release of upcoming `pyannote.audio` 2.0 for which it has been decided to rewrite almost everything from scratch to benefit from `pytorch-lightning` framework. -## CLI +## Installation + +Until a proper release is available on PyPI, install from the `develop` branch: ```bash -pyannote-audio-train task=vad model=debug protocol=Debug.SpeakerDiarization.Debug +git clone https://github.com/pyannote/pyannote-audio.git +cd pyannote-audio +git checkout develop +pip install . +``` + +## pyannote.audio 101 + +*For now, this is the closest you can get to an actual documentation.* + +Experimental protocol is reproducible thanks to [`pyannote.database`](https://github.com/pyannote/pyannote-database). +*Here, we use the [AMI](https://github.com/BUTSpeechFIT/AMI-diarization-setup) "only_words" speaker diarization protocol.* + +```python +from pyannote.database import get_protocol +ami = get_protocol('AMI.SpeakerDiarization.only_words') +``` + +Data augmentation is supported via [`torch-audiomentations`](https://github.com/asteroid-team/torch-audiomentations). + +```python +from torch_audiomentations import Compose, ApplyImpulseResponse, AddBackgroundNoise +augmentation = Compose(transforms=[ApplyImpulseResponse(...), + AddBackgroundNoise(...)]) +``` + +A growing collection of tasks can be addressed. +*Here, we address speaker segmentation.* + +```python +from pyannote.audio.tasks import Segmentation +seg = Segmentation(ami, augmentation=augmentation) +``` + +A growing collection of model architecture can be used. +*Here, we use the PyanNet (sincnet + LSTM) architecture.* + +```python +from pyannote.audio.models.segmentation import PyanNet +model = PyanNet(task=seg) +``` + +We benefit from all the nice things that [`pytorch-lightning`](https://www.pytorchlightning.ai/) has to offer: distributed (GPU & TPU) training, model checkpointing, logging, etc. +*In this example, we don't really use any of this...* + +```python +from pytorch_lightning import Trainer +trainer = Trainer() +trainer.fit(model) +``` + +Predictions are obtained by wrapping the model into the `Inference` engine. + +```python +from pyannote.audio.core.inference import Inference +inference = Inference(model) +predictions = inference('audio.wav') +``` + +Pretrained models can be shared on [Huggingface.co](https://huggingface.co) model hub. +*Here, we download and use a [pretrained](https://huggingface.co/hbredin/VoiceActivityDetection-PyanNet-DIHARD) voice activity detection model.* + +```python +inference = Inference('hbredin/VoiceActivityDetection-PyanNet-DIHARD') +predictions = inference('audio.wav') +``` + +Fine-tuning is as easy as setting the `task` attribute, freezing early layers and training. +*Here, we fine-tune on AMI dataset a voice activity detection model pretrained on DIHARD dataset.* + +```python +from pyannote.audio.core.model import load_from_checkpoint +model = load_from_checkpoint('hbredin/VoiceActivityDetection-PyanNet-DIHARD') +model.task = VoiceActivityDetection(ami) +model.freeze_up_to('sincnet') +trainer.fit(model) +``` + +Transfer learning is also supported out of the box. +*Here, we do transfer learning from voice activity detection to overlapped speech detection.* + +```python +from pyannote.audio.tasks import OverlappedSpeechDetection +osd = OverlappedSpeechDetection(ami) +model.task = osd +trainer.fit(model) ``` diff --git a/notebook/example.ipynb b/notebook/example.ipynb index d39bc142..8ec632f4 100644 --- a/notebook/example.ipynb +++ b/notebook/example.ipynb @@ -50,7 +50,7 @@ "vad = VoiceActivityDetection(protocol, duration=2., batch_size=32, num_workers=4)\n", "model = SimpleSegmentationModel(task=vad)\n", "trainer = pl.Trainer(max_epochs=1)\n", - "_ = trainer.fit(model, vad)" + "_ = trainer.fit(model)" ] }, { @@ -70,7 +70,7 @@ "scd = SpeakerChangeDetection(protocol, duration=2., batch_size=32, num_workers=4)\n", "model = SimpleSegmentationModel(task=scd)\n", "trainer = pl.Trainer(max_epochs=1)\n", - "_ = trainer.fit(model, scd)" + "_ = trainer.fit(model)" ] }, { @@ -90,7 +90,7 @@ "ovl = OverlappedSpeechDetection(protocol, duration=2., batch_size=32, num_workers=4)\n", "model = SimpleSegmentationModel(task=ovl)\n", "trainer = pl.Trainer(max_epochs=1)\n", - "_ = trainer.fit(model, ovl)" + "_ = trainer.fit(model)" ] }, { @@ -110,7 +110,7 @@ "seg = Segmentation(protocol, duration=2., batch_size=32, num_workers=4)\n", "model = SimpleSegmentationModel(task=seg)\n", "trainer = pl.Trainer(max_epochs=1)\n", - "_ = trainer.fit(model, seg)" + "_ = trainer.fit(model)" ] }, { @@ -130,7 +130,7 @@ "seg = MultiTaskSegmentation(protocol, vad=True, scd=True, osd=True, duration=2., batch_size=32, num_workers=4)\n", "model = MultiTaskSegmentationModel(task=seg)\n", "trainer = pl.Trainer(max_epochs=1)\n", - "_ = trainer.fit(model, seg)" + "_ = trainer.fit(model)" ] }, { @@ -150,7 +150,7 @@ "spk = SpeakerTracking(protocol, duration=2., batch_size=32, num_workers=4)\n", "model = SimpleSegmentationModel(task=spk)\n", "trainer = pl.Trainer(max_epochs=1)\n", - "_ = trainer.fit(model, spk)" + "_ = trainer.fit(model)" ] }, { @@ -170,7 +170,7 @@ "emb = SpeakerEmbedding(protocol, duration=2., num_workers=4)\n", "model = SimpleEmbeddingModel(task=emb)\n", "trainer = pl.Trainer(max_epochs=1)\n", - "_ = trainer.fit(model, emb)" + "_ = trainer.fit(model)" ] } ], diff --git a/notebook/freeze.ipynb b/notebook/freeze.ipynb index 3ce13033..8973751a 100644 --- a/notebook/freeze.ipynb +++ b/notebook/freeze.ipynb @@ -31,7 +31,7 @@ "vad = VoiceActivityDetection(protocol, duration=2., batch_size=16, num_workers=4)\n", "model = SimpleSegmentationModel(task=vad)\n", "trainer = pl.Trainer(max_epochs=1)\n", - "_ = trainer.fit(model, vad)" + "_ = trainer.fit(model)" ] }, { @@ -87,7 +87,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.9" + "version": "3.8.5" } }, "nbformat": 4, diff --git a/notebook/inference.ipynb b/notebook/inference.ipynb index 0a47d82d..31372fa8 100644 --- a/notebook/inference.ipynb +++ b/notebook/inference.ipynb @@ -1,14 +1,5 @@ { "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -54,7 +45,7 @@ "vad = VoiceActivityDetection(protocol, duration=2., batch_size=16, num_workers=4)\n", "model = SimpleSegmentationModel(task=vad)\n", "trainer = pl.Trainer(max_epochs=1, default_root_dir='inference/vad')\n", - "_ = trainer.fit(model, vad)" + "_ = trainer.fit(model)" ] }, { @@ -163,7 +154,7 @@ "from pyannote.audio.models.embedding.debug import SimpleEmbeddingModel\n", "model = SimpleEmbeddingModel(task=emb)\n", "trainer = pl.Trainer(max_epochs=1, default_root_dir='inference/emb')\n", - "_ = trainer.fit(model, emb)" + "_ = trainer.fit(model)" ] }, { @@ -261,7 +252,7 @@ "from pyannote.audio.models.segmentation.debug import MultiTaskSegmentationModel\n", "model = MultiTaskSegmentationModel(task=xseg)\n", "trainer = pl.Trainer(max_epochs=1, default_root_dir='inference/xseg')\n", - "_ = trainer.fit(model, xseg)" + "_ = trainer.fit(model)" ] }, { @@ -283,7 +274,7 @@ "metadata": {}, "outputs": [], "source": [ - "np.exp(scores['vad'])" + "scores['vad']" ] }, { @@ -292,7 +283,7 @@ "metadata": {}, "outputs": [], "source": [ - "np.exp(scores['scd'])" + "scores['scd']" ] }, { @@ -301,7 +292,7 @@ "metadata": {}, "outputs": [], "source": [ - "np.exp(scores['osd'])" + "scores['osd']" ] }, { diff --git a/notebook/sharing.ipynb b/notebook/sharing.ipynb index f52f01aa..0222b93a 100644 --- a/notebook/sharing.ipynb +++ b/notebook/sharing.ipynb @@ -31,7 +31,7 @@ "vad = VoiceActivityDetection(protocol, duration=2., batch_size=32, num_workers=4)\n", "model = SimpleSegmentationModel(task=vad)\n", "trainer = pl.Trainer(max_epochs=1, default_root_dir='sharing/')\n", - "_ = trainer.fit(model, vad)" + "_ = trainer.fit(model)" ] }, { @@ -71,7 +71,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.9" + "version": "3.8.5" } }, "nbformat": 4, diff --git a/pyannote/audio/cli/train.py b/pyannote/audio/cli/train.py index 0fb66b4c..da660803 100644 --- a/pyannote/audio/cli/train.py +++ b/pyannote/audio/cli/train.py @@ -76,38 +76,33 @@ def main(cfg: DictConfig) -> None: callbacks = [] - val_callback = task.val_callback() - if val_callback is None: - monitor, mode = task.val_monitor - else: - callbacks.append(val_callback) - monitor, mode = val_callback.val_monitor - model = instantiate(cfg.model, task=task) - model_checkpoint = ModelCheckpoint( + monitor, direction = task.val_monitor + checkpoint = ModelCheckpoint( monitor=monitor, - mode=mode, - save_top_k=10, + mode=direction, + save_top_k=None if monitor is None else 3, period=1, save_last=True, save_weights_only=False, dirpath=".", - filename=f"{{epoch}}-{{{monitor}:.6f}}", + filename="{epoch}" if monitor is None else f"{{epoch}}-{{{monitor}:.6f}}", verbose=cfg.verbose, ) - callbacks.append(model_checkpoint) + callbacks.append(checkpoint) - # TODO: add option to configure early stopping patience - early_stopping = EarlyStopping( - monitor=monitor, - mode=mode, - min_delta=0.0, - patience=50, - strict=True, - verbose=cfg.verbose, - ) - callbacks.append(early_stopping) + if monitor is not None: + # TODO: add option to configure early stopping patience + early_stopping = EarlyStopping( + monitor=monitor, + mode=direction, + min_delta=0.0, + patience=50, + strict=True, + verbose=cfg.verbose, + ) + callbacks.append(early_stopping) # TODO: fail safely when log_graph=True raises an onnx error logger = TensorBoardLogger( @@ -117,35 +112,18 @@ def main(cfg: DictConfig) -> None: # log_graph=True, ) - trainer = instantiate( - cfg.trainer, - callbacks=callbacks, - logger=logger, - ) + trainer = instantiate(cfg.trainer, callbacks=callbacks, logger=logger) + trainer.fit(model) - if cfg.trainer.get("auto_lr_find", False): - # HACK: everything but trainer.tune(...) should be removed - # once the corresponding bug is fixed in pytorch-lighting. - # https://github.com/pyannote/pyannote-audio/issues/514 - task.setup(stage="fit") - model.setup(stage="fit") - trainer.tune(model, task) - lr = model.hparams.learning_rate - task = instantiate( - cfg.task, - protocol, - optimizer=optimizer, - learning_rate=lr, - ) - model = instantiate(cfg.model, task=task) + # save paths to best models + checkpoint.to_yaml() - trainer.fit(model, task) - - best_monitor = float(model_checkpoint.best_model_score) - if mode == "min": - return best_monitor - else: - return -best_monitor + if monitor is not None: + best_monitor = float(checkpoint.best_model_score) + if direction == "min": + return best_monitor + else: + return -best_monitor if __name__ == "__main__": diff --git a/pyannote/audio/cli/train_config/hydra/train.yaml b/pyannote/audio/cli/train_config/hydra/train.yaml index 777a3f66..ad916ea2 100644 --- a/pyannote/audio/cli/train_config/hydra/train.yaml +++ b/pyannote/audio/cli/train_config/hydra/train.yaml @@ -1,7 +1,6 @@ # @package _group_ run: - # dir: train/${now:%Y-%m-%d}/${now:%H-%M-%S} dir: ${protocol}/${task._target_}/${now:%Y-%m-%d}/${now:%H-%M-%S} sweep: diff --git a/pyannote/audio/cli/train_config/trainer/default.yaml b/pyannote/audio/cli/train_config/trainer/default.yaml index 33679a73..d6c15c1e 100644 --- a/pyannote/audio/cli/train_config/trainer/default.yaml +++ b/pyannote/audio/cli/train_config/trainer/default.yaml @@ -7,7 +7,7 @@ amp_level: 'O2' auto_lr_find: False auto_scale_batch_size: False auto_select_gpus: True -benchmark: True +benchmark: False check_val_every_n_epoch: 1 checkpoint_callback: True deterministic: False diff --git a/pyannote/audio/core/inference.py b/pyannote/audio/core/inference.py index 63744d5d..61144098 100644 --- a/pyannote/audio/core/inference.py +++ b/pyannote/audio/core/inference.py @@ -23,9 +23,8 @@ import math import warnings from collections import Counter, deque -from functools import cached_property from pathlib import Path -from typing import Any, Callable, Deque, Dict, List, Optional, Text, Tuple, Union +from typing import Any, Callable, Deque, Dict, List, Optional, Text, Union import numpy as np import torch @@ -33,8 +32,8 @@ from einops import rearrange from pytorch_lightning.utilities.memory import is_oom_error from pyannote.audio.core.io import AudioFile -from pyannote.audio.core.model import Model, ModelIntrospection, load_from_checkpoint -from pyannote.audio.core.task import Scale, TaskSpecification +from pyannote.audio.core.model import Model, load_from_checkpoint +from pyannote.audio.core.task import Scale from pyannote.audio.utils.permutation import permutate from pyannote.audio.utils.progress import InferenceProgressHook from pyannote.core import Segment, SlidingWindow, SlidingWindowFeature @@ -93,9 +92,8 @@ class Inference: if window not in ["sliding", "whole"]: raise ValueError('`window` must be "sliding" or "whole".') - for task_name, task_specifications in self.task_specifications: - scale = task_specifications.scale - if scale == Scale.FRAME and window == "whole": + for task_name, specifications in self.model.specifications.items(): + if specifications.scale == Scale.FRAME and window == "whole": warnings.warn( 'Using "whole" `window` inference with a frame-based model might lead to bad results ' 'and huge memory consumption: it is recommended to set `window` to "sliding".' @@ -112,7 +110,11 @@ class Inference: # chunk duration used during training. for multi-task, # we assume that the same duration was used for each task. - training_duration = self.task_specifications[0][1].duration + if self.model.is_multi_task: + _, specifications = next(iter(self.model.specifications.items())) + else: + specifications = self.model.specifications + training_duration = specifications.duration if duration is None: duration = training_duration @@ -146,18 +148,6 @@ class Inference: progress_hook = None self.progress_hook = progress_hook - @cached_property - def is_multi_task(self) -> bool: - return self.model.is_multi_task - - @cached_property - def task_specifications(self) -> List[Tuple[TaskName, TaskSpecification]]: - return list(self.model.hparams.task_specifications.items()) - - @cached_property - def model_introspection(self) -> List[Tuple[TaskName, ModelIntrospection]]: - return list(self.model.hparams.model_introspection.items()) - def infer(self, chunks: torch.Tensor) -> Dict[TaskName, np.ndarray]: """Forward pass @@ -188,7 +178,7 @@ class Inference: else: raise exception - if self.is_multi_task: + if self.model.is_multi_task: return { task_name: output.cpu().numpy() for task_name, output in outputs.items() } @@ -236,8 +226,8 @@ class Inference: one_output = self.infer(waveform[None, :]) - for task_name, task_specifications in self.task_specifications: - if task_specifications.scale == Scale.CHUNK: + for task_name, specifications in self.model.specifications.items(): + if specifications.scale == Scale.CHUNK: frames = SlidingWindow( start=0.0, duration=self.duration, step=self.step ) @@ -256,9 +246,10 @@ class Inference: one_output[task_name][0], frames ) - if self.is_multi_task: + if self.model.is_multi_task: return results - return results[None] + else: + return results.popitem()[1] # prepare (and count) sliding audio chunks step_size: int = round(self.step * sample_rate) @@ -278,7 +269,7 @@ class Inference: has_last_chunk = False outputs: Dict[TaskName, Union[List[np.ndarray], np.ndarray]] = { - task_name: list() for task_name, _ in self.task_specifications + task_name: list() for task_name, _ in self.model.specifications.items() } if self.progress_hook is not None: @@ -301,10 +292,10 @@ class Inference: for task_name, task_outputs in outputs.items() } - for t, (task_name, task_specifications) in enumerate(self.task_specifications): + for task_name, specifications in self.model.specifications.items(): # if model outputs just one vector per chunk, return the outputs as they are # (i.e. do not aggregate them) - if task_specifications.scale == Scale.CHUNK: + if specifications.scale == Scale.CHUNK: frames = SlidingWindow( start=0.0, duration=self.duration, step=self.step ) @@ -323,12 +314,12 @@ class Inference: ) # use model introspection to estimate the total number of frames - _, model_introspection = self.model_introspection[t] - num_frames, dimension = model_introspection(num_samples) - num_frames_per_chunk, _ = model_introspection(window_size) - num_frames_per_step, _ = model_introspection(step_size) + introspection = self.model.introspection[task_name] + num_frames, dimension = introspection(num_samples) + num_frames_per_chunk, _ = introspection(window_size) + num_frames_per_step, _ = introspection(step_size) if has_last_chunk: - num_frames_last_step, _ = model_introspection(last_step_size) + num_frames_last_step, _ = introspection(last_step_size) # Hamming window used for overlap-add aggregation hamming = np.hamming(num_frames_per_chunk).reshape(-1, 1) @@ -345,7 +336,7 @@ class Inference: (num_frames, 1), dtype=np.float32 ) - if task_specifications.permutation_invariant: + if specifications.permutation_invariant: # previous outputs that overlap with current output by at least 50% maxlen = max(1, math.floor(0.5 * self.duration / self.step)) previous_outputs: Deque[np.ndarray] = deque([], maxlen=maxlen) @@ -353,9 +344,9 @@ class Inference: # loop on the outputs of sliding chunks for c, output in enumerate(outputs[task_name]): start_sample = c * step_size - start_frame, _ = model_introspection(start_sample) + start_frame, _ = introspection(start_sample) - if task_specifications.permutation_invariant: + if specifications.permutation_invariant: if c > 0: output = self.permutate( np.stack(previous_outputs), output, num_frames_per_step @@ -373,7 +364,7 @@ class Inference: # process last (right-aligned) chunk separately if has_last_chunk: - if task_specifications.permutation_invariant and previous_outputs: + if specifications.permutation_invariant and previous_outputs: # FIXME last_output[task_name] = self.permutate( previous_outputs[-1][np.newaxis], @@ -396,9 +387,10 @@ class Inference: results[task_name] = SlidingWindowFeature(aggregated_output, frames) - if self.is_multi_task: + if self.model.is_multi_task: return results - return results[None] + else: + return results.popitem()[1] def permutate( self, past_outputs: Deque[np.ndarray], output: np.ndarray, step_size: int @@ -475,10 +467,10 @@ class Inference: task_name: task_output[0] for task_name, task_output in self.infer(waveform[None]).items() } - if self.is_multi_task: + if self.model.is_multi_task: return outputs - - return outputs[None] + else: + return outputs.popitem()[1] def crop( self, @@ -532,7 +524,7 @@ class Inference: waveform, sample_rate = self.model.audio.crop(file, chunk, fixed=fixed) output = self.slide(waveform, sample_rate) - if self.is_multi_task: + if self.model.is_multi_task: shifted_output = dict() for task_name, task_output in output.items(): frames = task_output.sliding_window @@ -563,9 +555,11 @@ class Inference: task_name: task_output[0] for task_name, task_output in self.infer(waveform[None]).items() } - if self.is_multi_task: + + if self.model.is_multi_task: return outputs - return outputs[None] + else: + return outputs.popitem()[1] else: raise NotImplementedError( diff --git a/pyannote/audio/core/model.py b/pyannote/audio/core/model.py index 26231878..8212bd1f 100644 --- a/pyannote/audio/core/model.py +++ b/pyannote/audio/core/model.py @@ -1,6 +1,6 @@ # MIT License # -# Copyright (c) 2020 CNRS +# Copyright (c) 2020-2021 CNRS # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -23,7 +23,6 @@ import os import warnings from dataclasses import dataclass -from functools import cached_property from importlib import import_module from pathlib import Path from typing import Any, Dict, List, Optional, Text, Tuple, Union @@ -38,7 +37,7 @@ from semver import VersionInfo from pyannote.audio import __version__ from pyannote.audio.core.io import Audio -from pyannote.audio.core.task import Problem, Scale, Task, TaskSpecification +from pyannote.audio.core.task import Problem, Scale, Specifications, Task CACHE_DIR = os.getenv( "PYANNOTE_CACHE", @@ -48,7 +47,11 @@ HF_PYTORCH_WEIGHTS_NAME = "pytorch_model.bin" @dataclass -class ModelIntrospection: +class Introspection: + + # TODO: make it a regular class + # TODO: add from_model class method that will replace Model.introspection property + # minimum number of input samples min_num_samples: int # corresponding minimum number of output frames @@ -89,13 +92,24 @@ class ModelIntrospection: def __len__(self): # makes it possible to do something like: - # multi_task = len(model_introspection) > 1 + # multi_task = len(introspection) > 1 # because multi-task introspections are stored as {task_name: introspection} dict return 1 + def __getitem__(self, key): + if key is not None: + raise KeyError + return self + def items(self): yield None, self + def keys(self): + yield None + + def __iter__(self): + yield None + class Model(pl.LightningModule): """Base model @@ -107,9 +121,7 @@ class Model(pl.LightningModule): num_channels : int, optional Number of channels. Defaults to mono (1). task : Task, optional - Task addressed by the model. Only provided when training the model. - A model should be `load_from_checkpoint`-able without a task as - `on_load_checkpoint` hook takes care of calling `setup`. + Task addressed by the model. """ def __init__( @@ -120,56 +132,96 @@ class Model(pl.LightningModule): ): super().__init__() - # set-up audio IO assert ( num_channels == 1 ), "Only mono audio is supported for now (num_channels = 1)" - self.hparams.sample_rate = sample_rate - self.hparams.num_channels = num_channels - self.audio = Audio(sample_rate=self.hparams.sample_rate, mono=True) - # set task attribute when available (i.e. at training time) - # and also tell the task what kind of audio is expected from - # the model - if task is not None: - self.task = task - self.task.audio = self.audio + self.save_hyperparameters("sample_rate", "num_channels") - @cached_property - def is_multi_task(self) -> bool: - if hasattr(self, "task"): - return self.task.is_multi_task - return len(self.hparams.task_specifications) > 1 + self.task = task + self.audio = Audio( + sample_rate=self.hparams.sample_rate, mono=self.hparams.num_channels == 1 + ) + + @property + def datamodule(self): + return self.task + + @property + def example_input_array(self) -> torch.Tensor: + batch_size = 3 if self.task is None else self.task.batch_size + + if self.is_multi_task: + # this assumes that all tasks share the same duration + _, specifications = next(iter(self.specifications.items())) + else: + specifications = self.specifications + duration = specifications.duration + + return torch.randn( + ( + batch_size, + self.hparams.num_channels, + int(self.hparams.sample_rate * duration), + ), + device=self.device, + ) + + @property + def task(self): + return self._task + + @task.setter + def task(self, task): + self._task = task + del self.introspection + del self.specifications + + @property + def specifications(self): + if self.task is not None: + return self.task.specifications + return self._specifications + + @specifications.setter + def specifications(self, specifications): + self._specifications = specifications + + @specifications.deleter + def specifications(self): + if hasattr(self, "_specifications"): + del self._specifications + + @property + def is_multi_task(self): + return len(self.specifications) > 1 def build(self): # use this method to add task-dependent layers to the model # (e.g. the final classification and activation layers) pass - #  used by Tensorboard logger to log model graph - @cached_property - def example_input_array(self) -> torch.Tensor: - return self.task.example_input_array - - def helper_introspect( + def helper_introspection( self, - specifications: TaskSpecification, + specifications: Specifications, task: str = None, - ) -> ModelIntrospection: + ) -> Introspection: """Helper function for model introspection Parameters ---------- - specifications : TaskSpecification + specifications : Specifications + Task specifications. task : str, optional Task name. Returns ------- - introspection : ModelIntrospection + introspection : Introspection Model introspection. """ - example_input_array = self.task.example_input_array + + example_input_array = self.example_input_array batch_size, num_channels, num_samples = example_input_array.shape example_input_array = torch.randn( (1, num_channels, num_samples), @@ -177,7 +229,7 @@ class Model(pl.LightningModule): layout=example_input_array.layout, device=example_input_array.device, requires_grad=False, - ).to(self.device) + ) # dichotomic search of "min_num_samples" lower, upper, min_num_samples = 1, num_samples, None @@ -214,7 +266,7 @@ class Model(pl.LightningModule): # corner case for chunk-scale tasks if specifications.scale == Scale.CHUNK: - return ModelIntrospection( + return Introspection( min_num_samples=min_num_samples, min_num_frames=1, inc_num_samples=0, @@ -266,7 +318,7 @@ class Model(pl.LightningModule): if lower + 1 == upper: break - return ModelIntrospection( + return Introspection( min_num_samples=min_num_samples, min_num_frames=min_num_frames, inc_num_samples=inc_num_samples, @@ -274,55 +326,49 @@ class Model(pl.LightningModule): dimension=dimension, ) - def introspect(self) -> Union[ModelIntrospection, Dict[Text, ModelIntrospection]]: - """Perform model introspection + @property + def introspection(self) -> Union[Introspection, Dict[Text, Introspection]]: + """Introspection Returns ------- - introspection: ModelIntrospection or {str: ModelIntrospection} dict + introspection: Introspection or {str: Introspection} dict Model introspection or {task_name: introspection} dictionary for multi-task models. """ - task_specifications = self.hparams.task_specifications + if not hasattr(self, "_introspection"): - if self.is_multi_task: - return { - name: self.helper_introspect(specs, task=name) - for name, specs in task_specifications.items() - } + if self.is_multi_task: + self._introspection = { + name: self.helper_introspection(specs, task=name) + for name, specs in self.specifications.items() + } + # TODO: raises an error in case of multiple tasks with different introspections + else: + self._introspection = self.helper_introspection(self.specifications) - return self.helper_introspect(task_specifications) + return self._introspection + + @introspection.setter + def introspection(self, introspection): + self._introspection = introspection + + @introspection.deleter + def introspection(self): + if hasattr(self, "_introspection"): + del self._introspection def setup(self, stage=None): - if stage == "fit": - #  keep track of task specifications - self.hparams.task_specifications = self.task.specifications - - # add task-dependent layers to the model - # (e.g. the final classification and activation layers) self.build() if stage == "fit": - # model introspection - self.hparams.model_introspection = self.introspect() - - # TODO: raises an error in case of multiple tasks with different introspections - - # let task know about model introspection - # so that its dataloader knows how to generate targets - self.task.model_introspection = self.hparams.model_introspection - - # this is needed to support pytorch-lightning auto_lr_find feature - # as it expects to find a "learning_rate" entry in model.hparams - self.hparams.learning_rate = self.task.learning_rate - - # some tasks use loss functions with learnable parameters - # setup_loss_func will take care of adding them to the model - self.task.setup_loss_func(self) - - self.task.setup_validation_metric(self) + self.task.model = self + self.task.setup_loss_func() + self.task.setup_validation_metric() + # this is to make sure introspection is performed here, once and for all + _ = self.introspection def on_save_checkpoint(self, checkpoint): @@ -333,10 +379,12 @@ class Model(pl.LightningModule): "torch": torch.__version__, "pyannote.audio": __version__, }, - "model": { + "architecture": { "module": self.__class__.__module__, "class": self.__class__.__name__, }, + "introspection": self.introspection, + "specifications": self.specifications, } @staticmethod @@ -367,32 +415,27 @@ class Model(pl.LightningModule): checkpoint["pyannote.audio"]["versions"]["torch"], torch.__version__, ) + self.check_version( "pytorch-lightning", checkpoint["pytorch-lightning_version"], pl.__version__ ) - self.hparams.task_specifications = checkpoint["hyper_parameters"][ - "task_specifications" - ] + self.specifications = checkpoint["pyannote.audio"]["specifications"] - self.hparams.model_introspection = checkpoint["hyper_parameters"][ - "model_introspection" - ] + self.build() - # now that setup()-defined hyper-parameters are available, - # we can actually setup() the model. - self.setup() + self.introspection = checkpoint["pyannote.audio"]["introspection"] def forward(self, waveforms: torch.Tensor) -> torch.Tensor: msg = "Class {self.__class__.__name__} should define a `forward` method." raise NotImplementedError(msg) - def helper_default_activation(self, specifications: TaskSpecification) -> nn.Module: + def helper_default_activation(self, specifications: Specifications) -> nn.Module: """Helper function for default_activation Parameters ---------- - specifications: TaskSpecification + specifications: Specifications Task specification. Returns @@ -428,37 +471,32 @@ class Model(pl.LightningModule): Activation or {task_name: activation} dictionary for multi-task models. """ - task_specifications = self.hparams.task_specifications - if self.is_multi_task: return nn.ModuleDict( { name: self.helper_default_activation(specs) - for name, specs in task_specifications.items() + for name, specs in self.specifications.items() } ) - return self.helper_default_activation(task_specifications) - - def on_epoch_start(self): - self.task.current_epoch = self.current_epoch + return self.helper_default_activation(self.specifications) # training step logic is delegated to the task because the # model does not really need to know how it is being used. def training_step(self, batch, batch_idx): - return self.task.training_step(self, batch, batch_idx) + return self.task.training_step(batch, batch_idx) # validation logic is delegated to the task because the # model does not really need to know how it is being used. def validation_step(self, batch, batch_idx): - return self.task.validation_step(self, batch, batch_idx) + return self.task.validation_step(batch, batch_idx) def validation_epoch_end(self, outputs): - return self.task.validation_epoch_end(self, outputs) + return self.task.validation_epoch_end(outputs) # optimizer is delegated to the task for the same reason as above def configure_optimizers(self): - return self.task.configure_optimizers(self) + return self.task.configure_optimizers() def _helper_up_to( self, module_name: Text, requires_grad: bool = False @@ -703,10 +741,10 @@ def load_from_checkpoint( # obtain model class from the checkpoint checkpoint = pl_load(path_for_pl, map_location=map_location) - module_name: str = checkpoint["pyannote.audio"]["model"]["module"] + module_name: str = checkpoint["pyannote.audio"]["architecture"]["module"] module = import_module(module_name) - class_name: str = checkpoint["pyannote.audio"]["model"]["class"] + class_name: str = checkpoint["pyannote.audio"]["architecture"]["class"] Klass: Model = getattr(module, class_name) return Klass.load_from_checkpoint( diff --git a/pyannote/audio/core/task.py b/pyannote/audio/core/task.py index aa58a58a..54177017 100644 --- a/pyannote/audio/core/task.py +++ b/pyannote/audio/core/task.py @@ -27,8 +27,7 @@ import sys import warnings from dataclasses import dataclass from enum import Enum -from functools import cached_property -from typing import TYPE_CHECKING, Callable, Iterable, List, Optional, Text +from typing import Callable, Iterable, List, Optional, Text import pytorch_lightning as pl import torch @@ -36,15 +35,11 @@ import torch.nn.functional as F from torch.nn import Parameter from torch.optim import Adam, Optimizer from torch.utils.data import DataLoader, Dataset, IterableDataset - -from pyannote.database import Protocol - -if TYPE_CHECKING: - from pyannote.audio.core.model import Model - from torch.utils.data._utils.collate import default_collate from torch_audiomentations.core.transforms_interface import BaseWaveformTransform +from pyannote.database import Protocol + # Type of machine learning problem class Problem(Enum): @@ -65,7 +60,7 @@ class Scale(Enum): @dataclass -class TaskSpecification: +class Specifications: problem: Problem scale: Scale @@ -81,13 +76,24 @@ class TaskSpecification: def __len__(self): # makes it possible to do something like: - # multi_task = len(task_specifications) > 1 + # multi_task = len(specifications) > 1 # because multi-task specifications are stored as {task_name: specifications} dict return 1 + def __getitem__(self, key): + if key is not None: + raise KeyError + return self + def items(self): yield None, self + def keys(self): + yield None + + def __iter__(self): + yield None + class TrainDataset(IterableDataset): def __init__(self, task: Task): @@ -155,10 +161,10 @@ class Task(pl.LightningDataModule): Attributes ---------- - specifications : TaskSpecification or dict of TaskSpecification + specifications : Specifications or dict of Specifications Task specifications (available after `Task.setup` has been called.) For multi-task learning, this should be a dictionary where keys are - task names and values are corresponding TaskSpecification instances. + task names and values are corresponding Specifications instances. """ def __init__( @@ -178,6 +184,9 @@ class Task(pl.LightningDataModule): # dataset self.protocol = protocol + # TODO: check that protocol files come with required fields: audio, annotation, annotated, uri, what else? + # if not, complain and explain :) + # batching self.duration = duration self.min_duration = duration if min_duration is None else min_duration @@ -235,29 +244,20 @@ class Task(pl.LightningDataModule): If `specifications` attribute has not been set in `__init__`, `setup` is your last chance to set it. - """ pass - def setup_loss_func(self, model: Model): + def setup_loss_func(self): pass - def setup_validation_metric(self, model: Model): + def setup_validation_metric(self): pass - @cached_property + @property def is_multi_task(self) -> bool: """"Check whether multiple tasks are addressed at once""" return len(self.specifications) > 1 - @property - def current_epoch(self) -> int: - return getattr(self, "current_epoch_", 0) - - @current_epoch.setter - def current_epoch(self, current_epoch: int): - self.current_epoch_ = current_epoch - def train__iter__(self): # will become train_dataset.__iter__ method msg = f"Missing '{self.__class__.__name__}.train__iter__' method." @@ -272,7 +272,7 @@ class Task(pl.LightningDataModule): collated_batch = default_collate(batch) if self.augmentation is not None: collated_batch["X"] = self.augmentation( - collated_batch["X"], sample_rate=self.audio.sample_rate + collated_batch["X"], sample_rate=self.model.hparams.sample_rate ) return collated_batch @@ -286,34 +286,7 @@ class Task(pl.LightningDataModule): collate_fn=self.collate_fn, ) - @cached_property - def example_input_array(self): - # this method is called in Model.introspect where it is used - # to automagically infer the temporal resolution of the - # model output, and hence allow the dataloader to shape - # its targets correctly. - - # since we plan to have the feature extraction step done - # on GPU as part of the model, the example input array is - # basically always a chunk of audio - - if self.audio.mono: - num_channels = 1 - else: - msg = "Only 'mono' audio is supported." - raise NotImplementedError(msg) - - return torch.randn( - ( - self.batch_size, - num_channels, - int(self.audio.sample_rate * self.duration), - ) - ) - - def default_loss( - self, specifications: TaskSpecification, y, y_pred - ) -> torch.Tensor: + def default_loss(self, specifications: Specifications, y, y_pred) -> torch.Tensor: """Guess and compute default loss according to task specification""" if specifications.problem == Problem.BINARY_CLASSIFICATION: @@ -333,7 +306,7 @@ class Task(pl.LightningDataModule): # default training_step provided for convenience # can obviously be overriden for each task - def training_step(self, model: Model, batch, batch_idx: int): + def training_step(self, batch, batch_idx: int): """Default training_step according to task specification * binary cross-entropy loss for binary or multi-label classification @@ -343,8 +316,6 @@ class Task(pl.LightningDataModule): Parameters ---------- - model : Model - Model currently being trained. batch : (usually) dict of torch.Tensor Current batch. batch_idx: int @@ -357,7 +328,7 @@ class Task(pl.LightningDataModule): """ X, y = batch["X"], batch["y"] - y_pred = model(X) + y_pred = self.model(X) if self.is_multi_task: loss = dict() @@ -365,7 +336,7 @@ class Task(pl.LightningDataModule): loss[task_name] = self.default_loss( specifications, y[task_name], y_pred[task_name] ) - model.log( + self.model.log( f"{task_name}@train_loss", loss[task_name], on_step=True, @@ -375,7 +346,7 @@ class Task(pl.LightningDataModule): ) loss["loss"] = sum(loss.values()) - model.log( + self.model.log( f"{self.ACRONYM}@train_loss", loss["loss"], on_step=True, @@ -386,7 +357,7 @@ class Task(pl.LightningDataModule): return loss loss = self.default_loss(self.specifications, y, y_pred) - model.log( + self.model.log( f"{self.ACRONYM}@train_loss", loss, on_step=True, @@ -407,21 +378,17 @@ class Task(pl.LightningDataModule): raise NotImplementedError(msg) def val_dataloader(self) -> Optional[DataLoader]: - val_callback = self.val_callback() - if val_callback is None: - return DataLoader( - ValDataset(self), - batch_size=self.batch_size, - num_workers=self.num_workers, - pin_memory=self.pin_memory, - drop_last=False, - ) - else: - return None + return DataLoader( + ValDataset(self), + batch_size=self.batch_size, + num_workers=self.num_workers, + pin_memory=self.pin_memory, + drop_last=False, + ) # default validation_step provided for convenience # can obviously be overriden for each task - def validation_step(self, model: Model, batch, batch_idx: int): + def validation_step(self, batch, batch_idx: int): """Guess default validation_step according to task specification * binary cross-entropy loss for binary or multi-label classification @@ -431,8 +398,6 @@ class Task(pl.LightningDataModule): Parameters ---------- - model : Model - Model currently being validated. batch : (usually) dict of torch.Tensor Current batch. batch_idx: int @@ -445,7 +410,7 @@ class Task(pl.LightningDataModule): """ X, y = batch["X"], batch["y"] - y_pred = model(X) + y_pred = self.model(X) if self.is_multi_task: loss = dict() @@ -453,10 +418,10 @@ class Task(pl.LightningDataModule): loss[task_name] = self.default_loss( specifications, y[task_name], y_pred[task_name] ) - model.log(f"{task_name}@val_loss", loss[task_name]) + self.model.log(f"{task_name}@val_loss", loss[task_name]) loss["loss"] = sum(loss.values()) - model.log( + self.model.log( f"{self.ACRONYM}@val_loss", loss["loss"], on_step=False, @@ -466,7 +431,7 @@ class Task(pl.LightningDataModule): return loss loss = self.default_loss(self.specifications, y, y_pred) - model.log( + self.model.log( f"{self.ACRONYM}@val_loss", loss, on_step=False, @@ -475,24 +440,13 @@ class Task(pl.LightningDataModule): ) return {"loss": loss} - def validation_epoch_end(self, model: Model, outputs): + def validation_epoch_end(self, outputs): pass - def val_callback(self): - return None - - def parameters(self, model: Model) -> Iterable[Parameter]: - return model.parameters() - # default configure_optimizers provided for convenience # can obviously be overriden for each task - def configure_optimizers(self, model: Model): - # this is needed to support pytorch-lightning auto_lr_find feature - # as it modifies model.hparams.learning_rate and not task.learning_rate. - # in case one does not use auto_lr_find, Model.setup() takes care of - # setting model.hparams.learning_rate to task.learning_rate so we are safe. - lr = model.hparams.learning_rate - return self.optimizer(self.parameters(model), lr=lr) + def configure_optimizers(self): + return self.optimizer(self.model.parameters(), lr=self.learning_rate) @property def val_monitor(self): @@ -512,4 +466,5 @@ class Task(pl.LightningDataModule): pytorch_lightning.callbacks.ModelCheckpoint pytorch_lightning.callbacks.EarlyStopping """ + return f"{self.ACRONYM}@val_loss", "min" diff --git a/pyannote/audio/models/segmentation/MultiPyanNet.py b/pyannote/audio/models/segmentation/MultiPyanNet.py index 221747d0..83273be2 100644 --- a/pyannote/audio/models/segmentation/MultiPyanNet.py +++ b/pyannote/audio/models/segmentation/MultiPyanNet.py @@ -35,6 +35,8 @@ from pyannote.audio.models.blocks.sincnet import SincNet from pyannote.audio.utils.probe import probe from pyannote.core.utils.generators import pairwise +# from pyannote.core.utils.params import merge_dict + class Branch(nn.Module): """Branch""" @@ -192,7 +194,7 @@ class MultiPyanNet(Model): probes = dict() self.branches = nn.ModuleDict() self.classifiers = nn.ModuleDict() - for task_name, specifications in self.hparams.task_specifications.items(): + for task_name, specifications in self.specifications.items(): branch_hparams = self.hparams.branches[task_name] probes[task_name] = f"lstm.{branch_hparams['head']:d}" branch = Branch(self.trunk.out_features, **branch_hparams) @@ -230,5 +232,5 @@ class MultiPyanNet(Model): self.branches[task_name](outputs[task_name][0]) ) ) - for task_name in self.hparams.task_specifications + for task_name in self.specifications } diff --git a/pyannote/audio/models/segmentation/PyanNet.py b/pyannote/audio/models/segmentation/PyanNet.py index 1b81d905..7cc31276 100644 --- a/pyannote/audio/models/segmentation/PyanNet.py +++ b/pyannote/audio/models/segmentation/PyanNet.py @@ -31,6 +31,7 @@ from einops import rearrange from pyannote.audio.core.model import Model from pyannote.audio.core.task import Task from pyannote.audio.models.blocks.sincnet import SincNet +from pyannote.audio.utils.params import merge_dict from pyannote.core.utils.generators import pairwise @@ -64,62 +65,55 @@ class PyanNet(Model): def __init__( self, - sample_rate: int = 16000, - num_channels: int = 1, sincnet: dict = None, lstm: dict = None, linear: dict = None, + sample_rate: int = 16000, + num_channels: int = 1, task: Optional[Task] = None, ): super().__init__(sample_rate=sample_rate, num_channels=num_channels, task=task) - sincnet_hparams = dict(**self.SINCNET_DEFAULTS) - if sincnet is not None: - sincnet_hparams.update(**sincnet) - sincnet_hparams["sample_rate"] = sample_rate - self.hparams.sincnet = sincnet_hparams - self.sincnet = SincNet(**self.hparams.sincnet) + sincnet = merge_dict(self.SINCNET_DEFAULTS, sincnet) + sincnet["sample_rate"] = sample_rate + lstm = merge_dict(self.LSTM_DEFAULTS, lstm) + lstm["batch_first"] = True + linear = merge_dict(self.LINEAR_DEFAULTS, linear) + self.save_hyperparameters("sincnet", "lstm", "linear") - lstm_hparams = dict(**self.LSTM_DEFAULTS) - if lstm is not None: - lstm_hparams.update(**lstm) - lstm_hparams["batch_first"] = True # this is not negotiable - self.hparams.lstm = lstm_hparams + self.sincnet = SincNet(**self.hparams.sincnet) self.lstm = nn.LSTM(60, **self.hparams.lstm) - lstm_out_features: int = self.lstm.hidden_size * ( - 2 if self.lstm.bidirectional else 1 - ) + if linear["num_layers"] < 1: + return - linear_hparams = dict(**self.LINEAR_DEFAULTS) - if linear is not None: - linear_hparams.update(**linear) - self.hparams.linear = linear_hparams - if self.hparams.linear["num_layers"] > 0: - self.linear = nn.ModuleList( - [ - nn.Linear(in_features, out_features) - for in_features, out_features in pairwise( - [ - lstm_out_features, - ] - + [self.hparams.linear["hidden_size"]] - * self.hparams.linear["num_layers"] - ) - ] - ) + lstm_out_features: int = self.hparams.lstm["hidden_size"] * ( + 2 if self.hparams.lstm["bidirectional"] else 1 + ) + self.linear = nn.ModuleList( + [ + nn.Linear(in_features, out_features) + for in_features, out_features in pairwise( + [ + lstm_out_features, + ] + + [self.hparams.linear["hidden_size"]] + * self.hparams.linear["num_layers"] + ) + ] + ) def build(self): if self.hparams.linear["num_layers"] > 0: in_features = self.hparams.linear["hidden_size"] else: - in_features = self.lstm.hidden_size * (2 if self.lstm.bidirectional else 1) + in_features = self.hparams.lstm["hidden_size"] * ( + 2 if self.hparams.lstm["bidirectional"] else 1 + ) - self.classifier = nn.Linear( - in_features, len(self.hparams.task_specifications.classes) - ) + self.classifier = nn.Linear(in_features, len(self.specifications.classes)) self.activation = self.default_activation() def forward(self, waveforms: torch.Tensor) -> torch.Tensor: diff --git a/pyannote/audio/models/segmentation/debug.py b/pyannote/audio/models/segmentation/debug.py index fcc7440f..2953a884 100644 --- a/pyannote/audio/models/segmentation/debug.py +++ b/pyannote/audio/models/segmentation/debug.py @@ -60,9 +60,7 @@ class SimpleSegmentationModel(Model): def build(self): # define task-dependent layers - self.classifier = nn.Linear( - 32 * 2, len(self.hparams.task_specifications.classes) - ) + self.classifier = nn.Linear(32 * 2, len(self.specifications.classes)) self.activation = self.default_activation() def forward(self, waveforms: torch.Tensor) -> torch.Tensor: @@ -115,7 +113,7 @@ class MultiTaskSegmentationModel(Model): self.classifier = nn.ModuleDict( { name: nn.Linear(32 * 2, len(specifications.classes)) - for name, specifications in self.hparams.task_specifications.items() + for name, specifications in self.specifications.items() } ) @@ -129,5 +127,5 @@ class MultiTaskSegmentationModel(Model): return { name: self.activation[name](self.classifier[name](output)) - for name in self.hparams.task_specifications + for name in self.specifications } diff --git a/pyannote/audio/tasks/embedding/arcface.py b/pyannote/audio/tasks/embedding/arcface.py index f0c37818..83ba86cc 100644 --- a/pyannote/audio/tasks/embedding/arcface.py +++ b/pyannote/audio/tasks/embedding/arcface.py @@ -30,7 +30,6 @@ from torch.nn import Parameter from torch.optim import Optimizer from torch_audiomentations.core.transforms_interface import BaseWaveformTransform -from pyannote.audio.core.model import Model from pyannote.audio.core.task import Task from pyannote.database import Protocol @@ -118,11 +117,11 @@ class SupervisedRepresentationLearningWithArcFace( augmentation=augmentation, ) - def setup_loss_func(self, model: Model): + def setup_loss_func(self): - _, embedding_size = model(self.example_input_array).shape + _, embedding_size = self.model(self.model.example_input_array).shape - model.loss_func = pytorch_metric_learning.losses.ArcFaceLoss( + self.model.loss_func = pytorch_metric_learning.losses.ArcFaceLoss( len(self.specifications.classes), embedding_size, margin=self.margin, diff --git a/pyannote/audio/tasks/embedding/mixins.py b/pyannote/audio/tasks/embedding/mixins.py index 299b3430..a4c433a9 100644 --- a/pyannote/audio/tasks/embedding/mixins.py +++ b/pyannote/audio/tasks/embedding/mixins.py @@ -25,16 +25,13 @@ from typing import Optional import numpy as np import torch -from pytorch_lightning import Callback, Trainer from torch.utils.data import DataLoader from tqdm import tqdm -from pyannote.audio.core.inference import Inference -from pyannote.audio.core.model import Model -from pyannote.audio.core.task import Problem, Scale, Task, TaskSpecification, ValDataset +from pyannote.audio.core.task import Problem, Scale, Specifications, ValDataset from pyannote.audio.utils.random import create_rng_for_worker from pyannote.core import Segment -from pyannote.core.utils.distance import cdist, pdist +from pyannote.core.utils.distance import cdist from pyannote.database.protocol import ( SpeakerDiarizationProtocol, SpeakerVerificationProtocol, @@ -84,7 +81,7 @@ class SupervisedRepresentationLearningTaskMixin: # loop over the training set, remove annotated regions shorter than # chunk duration, and keep track of the reference annotations, per class. - self.train = dict() + self._train = dict() desc = f"Loading {self.protocol.name} training labels" for f in tqdm(iterable=self.protocol.train(), desc=desc, unit="file"): @@ -106,10 +103,10 @@ class SupervisedRepresentationLearningTaskMixin: duration = sum(segment.duration for segment in speech_turns) # add class to the list of classes - if klass not in self.train: - self.train[klass] = list() + if klass not in self._train: + self._train[klass] = list() - self.train[klass].append( + self._train[klass].append( { "uri": f["uri"], "audio": f["audio"], @@ -118,13 +115,11 @@ class SupervisedRepresentationLearningTaskMixin: } ) - # there is no such thing as a "class" in representation - # learning, so we do not need to define it here. - self.specifications = TaskSpecification( + self.specifications = Specifications( problem=Problem.REPRESENTATION, scale=Scale.CHUNK, duration=self.duration, - classes=sorted(self.train), + classes=sorted(self._train), ) if isinstance(self.protocol, SpeakerVerificationProtocol): @@ -134,7 +129,7 @@ class SupervisedRepresentationLearningTaskMixin: session_hash = self.helper_trial_hash(trial[session]) if session_hash not in sessions: sessions[session_hash] = trial[session] - self.validation = list(sessions.items()) + self._validation = list(sessions.items()) def train__iter__(self): """Iterate over training samples @@ -148,7 +143,7 @@ class SupervisedRepresentationLearningTaskMixin: """ # create worker-specific random number generator - rng = create_rng_for_worker(self.current_epoch) + rng = create_rng_for_worker(self.model.current_epoch) classes = list(self.specifications.classes) @@ -173,8 +168,8 @@ class SupervisedRepresentationLearningTaskMixin: # select one file at random (with probability proportional to its class duration) file, *_ = rng.choices( - self.train[klass], - weights=[f["duration"] for f in self.train[klass]], + self._train[klass], + weights=[f["duration"] for f in self._train[klass]], k=1, ) @@ -191,7 +186,7 @@ class SupervisedRepresentationLearningTaskMixin: ) chunk = Segment(start_time, start_time + batch_duration) - X, _ = self.audio.crop( + X, _ = self.model.audio.crop( file, chunk, mode="center", @@ -207,17 +202,17 @@ class SupervisedRepresentationLearningTaskMixin: def train__len__(self): duration = sum( - datum["duration"] for data in self.train.values() for datum in data + datum["duration"] for data in self._train.values() for datum in data ) avg_chunk_duration = 0.5 * (self.min_duration + self.duration) return math.ceil(duration / avg_chunk_duration) - def training_step(self, model: "Model", batch, batch_idx: int): + def training_step(self, batch, batch_idx: int): X, y = batch["X"], batch["y"] - loss = model.loss_func(model(X), y) + loss = self.model.loss_func(self.model(X), y) - model.log( + self.model.log( f"{self.ACRONYM}@train_loss", loss, on_step=True, @@ -233,10 +228,10 @@ class SupervisedRepresentationLearningTaskMixin: def val__getitem__(self, idx): if isinstance(self.protocol, SpeakerVerificationProtocol): - session_hash, session = self.validation[idx] + session_hash, session = self._validation[idx] X = np.concatenate( [ - self.audio.crop(session, segment, mode="center")[0] + self.model.audio.crop(session, segment, mode="center")[0] for segment in session["try_with"] ], axis=0, @@ -249,23 +244,23 @@ class SupervisedRepresentationLearningTaskMixin: def val__len__(self): if isinstance(self.protocol, SpeakerVerificationProtocol): - return len(self.validation) + return len(self._validation) elif isinstance(self.protocol, SpeakerDiarizationProtocol): return 0 - def validation_step(self, model: "Model", batch, batch_idx: int): + def validation_step(self, batch, batch_idx: int): if isinstance(self.protocol, SpeakerVerificationProtocol): return ( batch["session_hash"][0].detach().cpu().numpy().item(), - model(batch["X"]).detach().cpu().numpy(), + self.model(batch["X"]).detach().cpu().numpy(), ) elif isinstance(self.protocol, SpeakerDiarizationProtocol): pass - def validation_epoch_end(self, model: "Model", outputs): + def validation_epoch_end(self, outputs): if isinstance(self.protocol, SpeakerVerificationProtocol): @@ -294,9 +289,9 @@ class SupervisedRepresentationLearningTaskMixin: if num_target_trials > 2 and num_non_target_trials > 2: fpr, fnr, thresholds, eer = det_curve(y_true, y_pred, distances=True) - model.log( + self.model.log( f"{self.ACRONYM}@val_eer", - torch.tensor(eer, device=model.device), + torch.tensor(eer, device=self.model.device), logger=True, on_epoch=True, prog_bar=True, @@ -321,16 +316,6 @@ class SupervisedRepresentationLearningTaskMixin: return None - def val_callback(self) -> Optional[Callback]: - - if isinstance(self.protocol, SpeakerVerificationProtocol): - return None - - elif isinstance(self.protocol, SpeakerDiarizationProtocol): - return _SpeakerDiarizationValidationCallback(self) - - return None - @property def val_monitor(self): @@ -338,50 +323,4 @@ class SupervisedRepresentationLearningTaskMixin: return f"{self.ACRONYM}@val_eer", "min" elif isinstance(self.protocol, SpeakerDiarizationProtocol): - pass - - -class _SpeakerDiarizationValidationCallback(Callback): - def __init__(self, task: Task): - super().__init__() - self.task = task - - def on_epoch_end(self, trainer: Trainer, model: Model): - - inference = Inference(model, window="whole") - - y_true, y_pred = [], [] - for file in tqdm(self.task.protocol.development()): - X, y = zip( - *[ - (inference.crop(file, chunk), label) - for chunk, _, label in file["annotation"].itertracks( - yield_label=True - ) - ] - ) - - y_true.append(pdist(np.array(y), metric="equal")) - y_pred.append(pdist(np.stack(X), metric="cosine")) - y_true = np.concatenate(y_true) - y_pred = np.concatenate(y_pred) - - fpr, fnr, thresholds, eer = det_curve(y_true, y_pred, distances=True) - - model.log( - f"{self.task.ACRONYM}@val_eer", - torch.tensor(eer, device=model.device), - logger=True, - on_epoch=True, - prog_bar=True, - sync_dist=True, - ) - - # TODO: log det curve - - # set model back to "train" mode - model.train() - - @property - def val_monitor(self): - return f"{self.task.ACRONYM}@val_eer", "min" + return None, "min" diff --git a/pyannote/audio/tasks/segmentation/mixins.py b/pyannote/audio/tasks/segmentation/mixins.py index bab7e802..a77fd723 100644 --- a/pyannote/audio/tasks/segmentation/mixins.py +++ b/pyannote/audio/tasks/segmentation/mixins.py @@ -30,7 +30,6 @@ import numpy as np from pytorch_lightning.metrics.classification import FBeta from pyannote.audio.core.io import AudioFile -from pyannote.audio.core.model import Model from pyannote.audio.core.task import Problem from pyannote.audio.utils.random import create_rng_for_worker from pyannote.core import Segment, SlidingWindow @@ -43,7 +42,7 @@ class SegmentationTaskMixin: if stage == "fit": # loop over the training set, remove annotated regions shorter than # chunk duration, and keep track of the reference annotations. - self.train = [] + self._train = [] for f in self.protocol.train(): segments = [ segment @@ -51,7 +50,7 @@ class SegmentationTaskMixin: if segment.duration > self.duration ] duration = sum(segment.duration for segment in segments) - self.train.append( + self._train.append( { "uri": f["uri"], "annotated": segments, @@ -63,7 +62,7 @@ class SegmentationTaskMixin: # loop over the validation set, remove annotated regions shorter than # chunk duration, and keep track of the reference annotations. - self.validation = [] + self._validation = [] for f in self.protocol.development(): for segment in f["annotated"]: @@ -76,11 +75,11 @@ class SegmentationTaskMixin: for c in range(num_chunks): start_time = segment.start + c * self.duration chunk = Segment(start_time, start_time + self.duration) - self.validation.append((f, chunk)) + self._validation.append((f, chunk)) - random.shuffle(self.validation) + random.shuffle(self._validation) - def setup_validation_metric(self, model): + def setup_validation_metric(self): self.val_fbeta = FBeta( len(self.specifications.classes), @@ -133,20 +132,22 @@ class SegmentationTaskMixin: Ordered labels such that y[:, k] corresponds to activity of labels[k]. """ - X, _ = self.audio.crop( + X, _ = self.model.audio.crop( file, chunk, mode="center", fixed=self.duration if duration is None else duration, ) + introspection = self.model.introspection + if self.is_multi_task: # this assumes that all tasks share the same model introspection. # this is a reasonable assumption for now. - any_task = next(iter(self.model_introspection.keys())) - num_frames, _ = self.model_introspection[any_task](X.shape[1]) + any_task = next(iter(introspection.keys())) + num_frames, _ = introspection[any_task](X.shape[1]) else: - num_frames, _ = self.model_introspection(X.shape[1]) + num_frames, _ = introspection(X.shape[1]) annotation = file["annotation"].crop(chunk) labels = annotation.labels() if self.chunk_labels is None else self.chunk_labels @@ -189,14 +190,14 @@ class SegmentationTaskMixin: """ # create worker-specific random number generator - rng = create_rng_for_worker(self.current_epoch) + rng = create_rng_for_worker(self.model.current_epoch) while True: # select one file at random (with probability proportional to its annotated duration) file, *_ = rng.choices( - self.train, - weights=[f["duration"] for f in self.train], + self._train, + weights=[f["duration"] for f in self._train], k=1, ) @@ -219,25 +220,23 @@ class SegmentationTaskMixin: def train__len__(self): # Number of training samples in one epoch - duration = sum(file["duration"] for file in self.train) + duration = sum(file["duration"] for file in self._train) return math.ceil(duration / self.duration) def val__getitem__(self, idx): - f, chunk = self.validation[idx] + f, chunk = self._validation[idx] X, one_hot_y, _ = self.prepare_chunk(f, chunk, duration=self.duration) y = self.prepare_y(one_hot_y) return {"X": X, "y": y} def val__len__(self): - return len(self.validation) + return len(self._validation) - def validation_step(self, model: Model, batch, batch_idx: int): + def validation_step(self, batch, batch_idx: int): """Compute area under ROC curve Parameters ---------- - model : Model - Model currently being validated. batch : dict of torch.Tensor Current batch. batch_idx: int @@ -245,17 +244,17 @@ class SegmentationTaskMixin: """ # move metric to model device - self.val_fbeta.to(model.device) + self.val_fbeta.to(self.model.device) X, y = batch["X"], batch["y"] # X = (batch_size, num_channels, num_samples) # y = (batch_size, num_frames, num_classes) - y_pred = model(X) + y_pred = self.model(X) # y_pred = (batch_size, num_frames, num_classes) val_fbeta = self.val_fbeta(y_pred[:, ::10].squeeze(), y[:, ::10].squeeze()) - model.log( + self.model.log( f"{self.ACRONYM}@val_fbeta", val_fbeta, on_step=False, @@ -322,8 +321,8 @@ class SegmentationTaskMixin: plt.tight_layout() - model.logger.experiment.add_figure( - f"{self.ACRONYM}@val_samples", fig, model.current_epoch + self.model.logger.experiment.add_figure( + f"{self.ACRONYM}@val_samples", fig, self.model.current_epoch ) @property diff --git a/pyannote/audio/tasks/segmentation/multi_task_segmentation.py b/pyannote/audio/tasks/segmentation/multi_task_segmentation.py index 09a3e6e9..7528a85d 100644 --- a/pyannote/audio/tasks/segmentation/multi_task_segmentation.py +++ b/pyannote/audio/tasks/segmentation/multi_task_segmentation.py @@ -31,7 +31,6 @@ from torch.optim import Optimizer from torch_audiomentations.core.transforms_interface import BaseWaveformTransform from pyannote.audio.core.io import Audio -from pyannote.audio.core.model import Model from pyannote.audio.core.task import Problem, Task from pyannote.audio.tasks import ( OverlappedSpeechDetection, @@ -180,9 +179,9 @@ class MultiTaskSegmentation(SegmentationTaskMixin, Task): if stage == "fit": if self.osd and self.tasks["osd"].domain is not None: - for f in self.train: + for f in self._train: f["domain"] = f[self.tasks["osd"].domain] - self.domains = list(set(f["domain"] for f in self.train)) + self.domains = list(set(f["domain"] for f in self._train)) self.specifications = { name: task.specifications for name, task in self.tasks.items() @@ -208,7 +207,7 @@ class MultiTaskSegmentation(SegmentationTaskMixin, Task): def train__iter__helper(self, rng: random.Random, domain: str = None): - train = self.train + train = self._train if domain is not None: train = [f for f in train if f["domain"] == domain] @@ -249,7 +248,7 @@ class MultiTaskSegmentation(SegmentationTaskMixin, Task): """ # create worker-specific random number generator - rng = create_rng_for_worker(self.current_epoch) + rng = create_rng_for_worker(self.model.current_epoch) if self.osd and self.tasks["osd"].domain is not None: chunks_by_domain = { @@ -307,7 +306,7 @@ class MultiTaskSegmentation(SegmentationTaskMixin, Task): yield {"X": X, "y": self.prepare_y(combined_y)} - def setup_validation_metric(self, model): + def setup_validation_metric(self): self.val_fbeta = { task_name: FBeta( @@ -322,13 +321,11 @@ class MultiTaskSegmentation(SegmentationTaskMixin, Task): for task_name, specifications in self.specifications.items() } - def validation_step(self, model: Model, batch, batch_idx: int): + def validation_step(self, batch, batch_idx: int): """Compute areas under ROC curve Parameters ---------- - model : Model - Model currently being validated. batch : dict of torch.Tensor Current batch. batch_idx: int @@ -336,20 +333,20 @@ class MultiTaskSegmentation(SegmentationTaskMixin, Task): """ X, y = batch["X"], batch["y"] - y_pred = model(X) + y_pred = self.model(X) val_fbeta = dict() for task_name in self.specifications: # move metric to model device - self.val_fbeta[task_name].to(model.device) + self.val_fbeta[task_name].to(self.model.device) val_fbeta[task_name] = self.val_fbeta[task_name]( y_pred[task_name][:, ::10].squeeze(), y[task_name][:, ::10].squeeze() ) - model.log( + self.model.log( f"{task_name}@val_fbeta", val_fbeta[task_name], on_step=False, @@ -358,7 +355,7 @@ class MultiTaskSegmentation(SegmentationTaskMixin, Task): logger=True, ) - model.log( + self.model.log( f"{self.ACRONYM}@val_fbeta", sum(val_fbeta.values()) / len(val_fbeta), on_step=False, diff --git a/pyannote/audio/tasks/segmentation/overlapped_speech_detection.py b/pyannote/audio/tasks/segmentation/overlapped_speech_detection.py index 25b3b07d..e65026f4 100644 --- a/pyannote/audio/tasks/segmentation/overlapped_speech_detection.py +++ b/pyannote/audio/tasks/segmentation/overlapped_speech_detection.py @@ -30,7 +30,7 @@ from torch.optim import Optimizer from torch_audiomentations.core.transforms_interface import BaseWaveformTransform from pyannote.audio.core.io import Audio -from pyannote.audio.core.task import Problem, Scale, Task, TaskSpecification +from pyannote.audio.core.task import Problem, Scale, Specifications, Task from pyannote.audio.tasks.segmentation.mixins import SegmentationTaskMixin from pyannote.audio.utils.random import create_rng_for_worker from pyannote.core import Segment @@ -115,7 +115,7 @@ class OverlappedSpeechDetection(SegmentationTaskMixin, Task): augmentation=augmentation, ) - self.specifications = TaskSpecification( + self.specifications = Specifications( problem=Problem.BINARY_CLASSIFICATION, scale=Scale.FRAME, duration=self.duration, @@ -137,9 +137,9 @@ class OverlappedSpeechDetection(SegmentationTaskMixin, Task): # build the list of domains if self.domain is not None: - for f in self.train: + for f in self._train: f["domain"] = f[self.domain] - self.domains = list(set(f["domain"] for f in self.train)) + self.domains = list(set(f["domain"] for f in self._train)) def prepare_y(self, one_hot_y: np.ndarray): """Get overlapped speech detection targets @@ -161,7 +161,7 @@ class OverlappedSpeechDetection(SegmentationTaskMixin, Task): def train__iter__helper(self, rng: random.Random, domain: str = None): - train = self.train + train = self._train if domain is not None: train = [f for f in train if f["domain"] == domain] @@ -202,7 +202,7 @@ class OverlappedSpeechDetection(SegmentationTaskMixin, Task): """ # create worker-specific random number generator - rng = create_rng_for_worker(self.current_epoch) + rng = create_rng_for_worker(self.model.current_epoch) if self.domain is None: chunks = self.train__iter__helper(rng) diff --git a/pyannote/audio/tasks/segmentation/segmentation.py b/pyannote/audio/tasks/segmentation/segmentation.py index 2c3d7b8e..86dbb947 100644 --- a/pyannote/audio/tasks/segmentation/segmentation.py +++ b/pyannote/audio/tasks/segmentation/segmentation.py @@ -34,8 +34,7 @@ from torch.optim import Optimizer from torch_audiomentations.core.transforms_interface import BaseWaveformTransform from pyannote.audio.core.io import Audio -from pyannote.audio.core.model import Model -from pyannote.audio.core.task import Problem, Scale, Task, TaskSpecification +from pyannote.audio.core.task import Problem, Scale, Specifications, Task from pyannote.audio.tasks.segmentation.mixins import SegmentationTaskMixin from pyannote.audio.utils.permutation import permutate from pyannote.audio.utils.random import create_rng_for_worker @@ -134,16 +133,18 @@ class Segmentation(SegmentationTaskMixin, Task): if stage == "fit": + # TODO: handle this domain thing in SegmentationTaskMixin + # build the list of domains if self.domain is not None: - for f in self.train: + for f in self._train: f["domain"] = f[self.domain] - self.domains = list(set(f["domain"] for f in self.train)) + self.domains = list(set(f["domain"] for f in self._train)) # slide a window (with 1s step) over the whole training set # and keep track of the number of speakers in each location num_speakers = [] - for file in self.train: + for file in self._train: start = file["annotated"][0].start end = file["annotated"][-1].end window = SlidingWindow( @@ -169,9 +170,13 @@ class Segmentation(SegmentationTaskMixin, Task): np.where(np.cumsum(counts) / np.sum(counts) > 0.99)[0][0] ] + # TODO: add a few more speakers to make sure we don't skip + # too many artificial chunks (which might result in less + # overlap that we think we have) + # now that we know about the number of speakers upper bound # we can set task specifications - self.specifications = TaskSpecification( + self.specifications = Specifications( problem=Problem.MULTI_LABEL_CLASSIFICATION, scale=Scale.FRAME, duration=self.duration, @@ -179,22 +184,22 @@ class Segmentation(SegmentationTaskMixin, Task): permutation_invariant=True, ) - def setup_loss_func(self, model: Model): + def setup_loss_func(self): - example_input_array = self.example_input_array + example_input_array = self.model.example_input_array _, _, num_samples = example_input_array.shape self.num_samples = num_samples - batch_size, num_frames, num_speakers = model(example_input_array).shape + batch_size, num_frames, num_speakers = self.model(example_input_array).shape self.num_frames = num_frames hamming_window = torch.hamming_window(num_frames, periodic=False).reshape(-1, 1) - model.register_buffer("hamming_window", hamming_window) + self.model.register_buffer("hamming_window", hamming_window) val_sample_weight = hamming_window.expand( batch_size, num_frames, num_speakers ).flatten() - model.register_buffer("val_sample_weight", val_sample_weight) + self.model.register_buffer("val_sample_weight", val_sample_weight) def prepare_y(self, one_hot_y: np.ndarray): """Zero-pad segmentation targets @@ -225,7 +230,7 @@ class Segmentation(SegmentationTaskMixin, Task): def train__iter__helper(self, rng: random.Random, domain: str = None): - train = self.train + train = self._train if domain is not None: train = [f for f in train if f["domain"] == domain] @@ -266,7 +271,7 @@ class Segmentation(SegmentationTaskMixin, Task): """ # create worker-specific random number generator - rng = create_rng_for_worker(self.current_epoch) + rng = create_rng_for_worker(self.model.current_epoch) if self.domain is None: chunks = self.train__iter__helper(rng) @@ -332,7 +337,7 @@ class Segmentation(SegmentationTaskMixin, Task): yield {"X": X, "y": self.prepare_y(combined_y)} def val__getitem__(self, idx): - f, chunk = self.validation[idx] + f, chunk = self._validation[idx] X, one_hot_y, _ = self.prepare_chunk(f, chunk, duration=self.duration) # since number of speakers is estimated from the training set, @@ -345,7 +350,6 @@ class Segmentation(SegmentationTaskMixin, Task): return {"X": X, "y": y} - # def segmentation_loss(self, model, y, y_pred): def segmentation_loss(self, y: torch.Tensor, y_pred: torch.Tensor) -> torch.Tensor: """Permutation-invariant segmentation loss @@ -394,13 +398,11 @@ class Segmentation(SegmentationTaskMixin, Task): return vad_loss - def training_step(self, model: Model, batch, batch_idx: int): + def training_step(self, batch, batch_idx: int): """Compute permutation-invariant binary cross-entropy Parameters ---------- - model : Model - Model currently being trained. batch : (usually) dict of torch.Tensor Current batch. batch_idx: int @@ -414,11 +416,11 @@ class Segmentation(SegmentationTaskMixin, Task): X, y = batch["X"], batch["y"] - y_pred = model(X) + y_pred = self.model(X) # loss = self.segmentation_loss(model, y, y_pred) seg_loss = self.segmentation_loss(y, y_pred) - model.log( + self.model.log( f"{self.ACRONYM}@train_seg_loss", seg_loss, on_step=True, @@ -433,7 +435,7 @@ class Segmentation(SegmentationTaskMixin, Task): else: vad_loss = self.voice_activity_detection_loss(y, y_pred) - model.log( + self.model.log( f"{self.ACRONYM}@train_vad_loss", vad_loss, on_step=True, @@ -444,7 +446,7 @@ class Segmentation(SegmentationTaskMixin, Task): loss = seg_loss + vad_loss - model.log( + self.model.log( f"{self.ACRONYM}@train_loss", loss, on_step=True, @@ -454,13 +456,11 @@ class Segmentation(SegmentationTaskMixin, Task): ) return {"loss": loss} - def validation_step(self, model: Model, batch, batch_idx: int): - """Compute area under ROC curve + def validation_step(self, batch, batch_idx: int): + """Compute validation F-score Parameters ---------- - model : Model - Model currently being validated. batch : dict of torch.Tensor Current batch. batch_idx: int @@ -468,13 +468,13 @@ class Segmentation(SegmentationTaskMixin, Task): """ # move metric to model device - self.val_fbeta.to(model.device) + self.val_fbeta.to(self.model.device) X, y = batch["X"], batch["y"] # X = (batch_size, num_channels, num_samples) # y = (batch_size, num_frames, num_classes) - y_pred = model(X) + y_pred = self.model(X) permutated_y_pred, _ = permutate(y, y_pred) @@ -483,7 +483,7 @@ class Segmentation(SegmentationTaskMixin, Task): val_fbeta = self.val_fbeta( permutated_y_pred[:, ::10].squeeze(), y[:, ::10].squeeze() ) - model.log( + self.model.log( f"{self.ACRONYM}@val_fbeta", val_fbeta, on_step=False, @@ -556,6 +556,6 @@ class Segmentation(SegmentationTaskMixin, Task): plt.tight_layout() - model.logger.experiment.add_figure( - f"{self.ACRONYM}@val_samples", fig, model.current_epoch + self.model.logger.experiment.add_figure( + f"{self.ACRONYM}@val_samples", fig, self.model.current_epoch ) diff --git a/pyannote/audio/tasks/segmentation/speaker_change_detection.py b/pyannote/audio/tasks/segmentation/speaker_change_detection.py index a96c8545..1a432767 100644 --- a/pyannote/audio/tasks/segmentation/speaker_change_detection.py +++ b/pyannote/audio/tasks/segmentation/speaker_change_detection.py @@ -28,7 +28,7 @@ from torch.nn import Parameter from torch.optim import Optimizer from torch_audiomentations.core.transforms_interface import BaseWaveformTransform -from pyannote.audio.core.task import Problem, Scale, Task, TaskSpecification +from pyannote.audio.core.task import Problem, Scale, Specifications, Task from pyannote.audio.tasks.segmentation.mixins import SegmentationTaskMixin from pyannote.database import Protocol @@ -98,7 +98,7 @@ class SpeakerChangeDetection(SegmentationTaskMixin, Task): augmentation=augmentation, ) - self.specifications = TaskSpecification( + self.specifications = Specifications( problem=Problem.BINARY_CLASSIFICATION, scale=Scale.FRAME, duration=self.duration, diff --git a/pyannote/audio/tasks/segmentation/speaker_tracking.py b/pyannote/audio/tasks/segmentation/speaker_tracking.py index 3acbf305..c193496f 100644 --- a/pyannote/audio/tasks/segmentation/speaker_tracking.py +++ b/pyannote/audio/tasks/segmentation/speaker_tracking.py @@ -26,7 +26,7 @@ from torch.nn import Parameter from torch.optim import Optimizer from torch_audiomentations.core.transforms_interface import BaseWaveformTransform -from pyannote.audio.core.task import Problem, Scale, Task, TaskSpecification +from pyannote.audio.core.task import Problem, Scale, Specifications, Task from pyannote.audio.tasks.segmentation.mixins import SegmentationTaskMixin from pyannote.database import Protocol @@ -103,7 +103,7 @@ class SpeakerTracking(SegmentationTaskMixin, Task): # build the list of speakers to be tracked. speakers = set() - for f in self.train: + for f in self._train: speakers.update(f["annotation"].labels()) # now that we now who the speakers are, we can @@ -111,7 +111,7 @@ class SpeakerTracking(SegmentationTaskMixin, Task): # note that, since multiple speakers can be active # at once, the problem is multi-label classification. - self.specifications = TaskSpecification( + self.specifications = Specifications( problem=Problem.MULTI_LABEL_CLASSIFICATION, scale=Scale.FRAME, duration=self.duration, diff --git a/pyannote/audio/tasks/segmentation/voice_activity_detection.py b/pyannote/audio/tasks/segmentation/voice_activity_detection.py index f804ce10..bb1c576f 100644 --- a/pyannote/audio/tasks/segmentation/voice_activity_detection.py +++ b/pyannote/audio/tasks/segmentation/voice_activity_detection.py @@ -27,7 +27,7 @@ from torch.nn import Parameter from torch.optim import Optimizer from torch_audiomentations.core.transforms_interface import BaseWaveformTransform -from pyannote.audio.core.task import Problem, Scale, Task, TaskSpecification +from pyannote.audio.core.task import Problem, Scale, Specifications, Task from pyannote.audio.tasks.segmentation.mixins import SegmentationTaskMixin from pyannote.database import Protocol @@ -90,7 +90,7 @@ class VoiceActivityDetection(SegmentationTaskMixin, Task): augmentation=augmentation, ) - self.specifications = TaskSpecification( + self.specifications = Specifications( problem=Problem.BINARY_CLASSIFICATION, scale=Scale.FRAME, duration=self.duration, diff --git a/pyannote/audio/utils/params.py b/pyannote/audio/utils/params.py new file mode 100644 index 00000000..685e0165 --- /dev/null +++ b/pyannote/audio/utils/params.py @@ -0,0 +1,9 @@ +# TODO - make it depth-recursive +# TODO - switch to Omegaconf maybe? + + +def merge_dict(defaults: dict, custom: dict = None): + params = dict(defaults) + if custom is not None: + params.update(custom) + return params diff --git a/tests/inference_test.py b/tests/inference_test.py index e2c4c85d..fe3b5b23 100644 --- a/tests/inference_test.py +++ b/tests/inference_test.py @@ -12,7 +12,8 @@ from pyannote.audio.tasks import MultiTaskSegmentation, VoiceActivityDetection from pyannote.core import SlidingWindowFeature from pyannote.database import FileFinder, get_protocol -HF_SAMPLE_MODEL_ID = "julien-c/voice-activity-detection" +# TODO: upload a very light model just for unit tests +# HF_SAMPLE_MODEL_ID = "julien-c/voice-activity-detection" @pytest.fixture() @@ -30,7 +31,7 @@ def trained(): def test_duration_warning(trained): protocol, model = trained with pytest.warns(UserWarning): - duration = model.hparams.task_specifications.duration + duration = model.specifications.duration new_duration = duration + 1 Inference(model, duration=new_duration, step=0.1, batch_size=128) @@ -38,7 +39,7 @@ def test_duration_warning(trained): def test_step_check_warning(trained): protocol, model = trained with pytest.raises(ValueError): - duration = model.hparams.task_specifications.duration + duration = model.specifications.duration Inference(model, step=duration + 1, batch_size=128) @@ -51,7 +52,7 @@ def test_invalid_window_fails(trained): def test_invalid_scale_fails(trained): protocol, model = trained with pytest.warns(UserWarning): - model.hparams.task_specifications.scale = Scale.FRAME + model.specifications.scale = Scale.FRAME Inference(model, window="whole", batch_size=128) @@ -97,6 +98,6 @@ def test_multi_seg_infer(): assert isinstance(scores[attr], SlidingWindowFeature) -def test_hf_download(): - inference = Inference(HF_SAMPLE_MODEL_ID, device="cpu") - assert isinstance(inference, Inference) +# def test_hf_download(): +# inference = Inference(HF_SAMPLE_MODEL_ID, device="cpu") +# assert isinstance(inference, Inference)