diff --git a/notebook/example.ipynb b/notebook/example.ipynb index ca555196..468f30b9 100644 --- a/notebook/example.ipynb +++ b/notebook/example.ipynb @@ -18,7 +18,6 @@ "outputs": [], "source": [ "from pyannote.audio.models.segmentation.debug import SimpleSegmentationModel\n", - "from pyannote.audio.models.segmentation.debug import MultiTaskSegmentationModel\n", "from pyannote.audio.models.embedding.debug import SimpleEmbeddingModel\n", "import pytorch_lightning as pl" ] @@ -113,26 +112,6 @@ "_ = trainer.fit(model)" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Multi-task segmentation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from pyannote.audio.tasks.segmentation.multi_task_segmentation import MultiTaskSegmentation\n", - "seg = MultiTaskSegmentation(protocol, vad=True, scd=True, osd=True, duration=2., batch_size=32, num_workers=4)\n", - "model = MultiTaskSegmentationModel(task=seg)\n", - "trainer = pl.Trainer(max_epochs=1)\n", - "_ = trainer.fit(model)" - ] - }, { "cell_type": "markdown", "metadata": {}, diff --git a/notebook/inference.ipynb b/notebook/inference.ipynb index fab9766d..77e02a27 100644 --- a/notebook/inference.ipynb +++ b/notebook/inference.ipynb @@ -234,135 +234,6 @@ "\n", "embeddings.shape" ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Train and apply a multi-task segmentation model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from pyannote.audio.tasks.segmentation.multi_task_segmentation import MultiTaskSegmentation\n", - "xseg = MultiTaskSegmentation(protocol, duration=2., vad=True, scd=True, osd=True, batch_size=32, num_workers=4)\n", - "from pyannote.audio.models.segmentation.debug import MultiTaskSegmentationModel\n", - "model = MultiTaskSegmentationModel(task=xseg)\n", - "trainer = pl.Trainer(max_epochs=1, default_root_dir='inference/xseg')\n", - "_ = trainer.fit(model)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# inference using a sliding window\n", - "inference = Inference(model, duration=2., step=0.5)\n", - "dev_file = next(protocol.development())\n", - "scores = inference(dev_file)\n", - "scores" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "scores['vad']" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "scores['scd']" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "scores['osd']" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# inference on an excerpt \n", - "scores = inference.crop(dev_file, Segment(10, 15))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "scores['vad']" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "scores['scd']" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "scores['osd']" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# inference on an excerpt shorter than sliding window duration\n", - "scores = inference.crop(dev_file, Segment(10, 11.5))\n", - "scores['vad']" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# inference on a whole chunk\n", - "inference = Inference(model, window=\"whole\")\n", - "scores = inference.crop(dev_file, Segment(10, 15))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "scores['vad'].shape" - ] } ], "metadata": { diff --git a/pyannote/audio/cli/train_config/hydra/train.yaml b/pyannote/audio/cli/train_config/hydra/train.yaml index ad916ea2..87488f41 100644 --- a/pyannote/audio/cli/train_config/hydra/train.yaml +++ b/pyannote/audio/cli/train_config/hydra/train.yaml @@ -28,7 +28,6 @@ help: * vad (default) = voice activity detection * scd = speaker change detection * osd = overlapped speech detection - * xseg = multi-task segmentation {model} can be any of the following: * debug (default) = simple segmentation model for debugging purposes diff --git a/pyannote/audio/core/inference.py b/pyannote/audio/core/inference.py index 7c9322f0..ec7aad7b 100644 --- a/pyannote/audio/core/inference.py +++ b/pyannote/audio/core/inference.py @@ -24,7 +24,7 @@ import math import warnings from collections import Counter, deque from pathlib import Path -from typing import Any, Callable, Deque, Dict, List, Optional, Text, Union +from typing import Any, Callable, Deque, List, Optional, Text, Union import numpy as np import torch @@ -105,12 +105,12 @@ class Inference: if window not in ["sliding", "whole"]: raise ValueError('`window` must be "sliding" or "whole".') - for task_name, specifications in self.model.specifications.items(): - if specifications.resolution == Resolution.FRAME and window == "whole": - warnings.warn( - 'Using "whole" `window` inference with a frame-based model might lead to bad results ' - 'and huge memory consumption: it is recommended to set `window` to "sliding".' - ) + specifications = self.model.specifications + if specifications.resolution == Resolution.FRAME and window == "whole": + warnings.warn( + 'Using "whole" `window` inference with a frame-based model might lead to bad results ' + 'and huge memory consumption: it is recommended to set `window` to "sliding".' + ) self.window = window self.skip_aggregation = skip_aggregation @@ -122,12 +122,8 @@ class Inference: self.model.eval() self.model.to(self.device) - # chunk duration used during training. for multi-task, - # we assume that the same duration was used for each task. - if self.model.is_multi_task: - _, specifications = next(iter(self.model.specifications.items())) - else: - specifications = self.model.specifications + # chunk duration used during training + specifications = self.model.specifications training_duration = specifications.duration if duration is None: @@ -169,7 +165,7 @@ class Inference: progress_hook = None self.progress_hook = progress_hook - def infer(self, chunks: torch.Tensor) -> Dict[TaskName, np.ndarray]: + def infer(self, chunks: torch.Tensor) -> np.ndarray: """Forward pass Parameters @@ -179,12 +175,8 @@ class Inference: Returns ------- - outputs : {task_name: np.ndarray} dict - Model outputs. - - Notes - ----- - If model is mono-task, `task_name` is set to None. + outputs : np.ndarray + Model output. """ with torch.no_grad(): @@ -199,16 +191,9 @@ class Inference: else: raise exception - if self.model.is_multi_task: - return { - task_name: output.cpu().numpy() for task_name, output in outputs.items() - } + return outputs.cpu().numpy() - return {None: outputs.cpu().numpy()} - - def slide( - self, waveform: torch.Tensor, sample_rate: int - ) -> Union[SlidingWindowFeature, Dict[Text, SlidingWindowFeature]]: + def slide(self, waveform: torch.Tensor, sample_rate: int) -> SlidingWindowFeature: """Slide model on a waveform Parameters @@ -223,11 +208,6 @@ class Inference: output : SlidingWindowFeature Model output. Shape is (num_chunks, dimension) for chunk-level tasks, and (num_frames, dimension) for frame-level tasks. - - Notes - ----- - If model has several outputs (multi-task), those will be returned as a - {task_name: output} dictionary. """ # prepare sliding audio chunks @@ -235,8 +215,6 @@ class Inference: file_duration = num_samples / sample_rate window_size: int = round(self.duration * sample_rate) - results: Dict[Text, SlidingWindowFeature] = dict() - # corner case: waveform is shorter than chunk duration if num_samples < window_size: @@ -247,30 +225,22 @@ class Inference: one_output = self.infer(waveform[None, :]) - for task_name, specifications in self.model.specifications.items(): - if specifications.resolution == Resolution.CHUNK: - frames = SlidingWindow( - start=0.0, duration=self.duration, step=self.step - ) - results[task_name] = SlidingWindowFeature( - one_output[task_name], frames - ) + if self.model.specifications.resolution == Resolution.CHUNK: + frames = SlidingWindow( + start=0.0, duration=self.duration, step=self.step + ) + results = SlidingWindowFeature(one_output, frames) - else: - _, num_frames, dimension = one_output[task_name].shape - frames = SlidingWindow( - start=0, - duration=file_duration / num_frames, - step=file_duration / num_frames, - ) - results[task_name] = SlidingWindowFeature( - one_output[task_name][0], frames - ) - - if self.model.is_multi_task: - return results else: - return results.popitem()[1] + _, num_frames, dimension = one_output.shape + frames = SlidingWindow( + start=0, + duration=file_duration / num_frames, + step=file_duration / num_frames, + ) + results = SlidingWindowFeature(one_output[0], frames) + + return results # prepare (and count) sliding audio chunks step_size: int = round(self.step * sample_rate) @@ -289,205 +259,182 @@ class Inference: else: has_last_chunk = False - outputs: Dict[TaskName, Union[List[np.ndarray], np.ndarray]] = { - task_name: list() for task_name, _ in self.model.specifications.items() - } + outputs: Union[List[np.ndarray], np.ndarray] = list() if self.progress_hook is not None: self.progress_hook(0, num_chunks + has_last_chunk) # slide over audio chunks in batch for c in np.arange(0, num_chunks, self.batch_size): - batch: torch.Tensor = chunks[c : c + self.batch_size] - - output = self.infer(batch) - for task_name, task_output in output.items(): - outputs[task_name].append(task_output) - + outputs.append(self.infer(batch)) if self.progress_hook is not None: self.progress_hook(c + 1, num_chunks + has_last_chunk) - outputs = { - task_name: np.vstack(task_outputs) - for task_name, task_outputs in outputs.items() - } + outputs = np.vstack(outputs) - for task_name, specifications in self.model.specifications.items(): - # skip aggregation when requested - # or when model outputs just one vector per chunk - if self.skip_aggregation or specifications.resolution == Resolution.CHUNK: - frames = SlidingWindow( - start=0.0, duration=self.duration, step=self.step - ) - results[task_name] = SlidingWindowFeature(outputs[task_name], frames) - continue + specifications = self.model.specifications - # process orphan last chunk - if has_last_chunk: - last_output = { - task_name: output[0] - for task_name, output in self.infer(last_chunk[None]).items() - } - if self.progress_hook is not None: - self.progress_hook( - num_chunks + has_last_chunk, num_chunks + has_last_chunk - ) + # skip aggregation when requested + # or when model outputs just one vector per chunk + if self.skip_aggregation or specifications.resolution == Resolution.CHUNK: + frames = SlidingWindow(start=0.0, duration=self.duration, step=self.step) + return SlidingWindowFeature(outputs, frames) - # use model introspection to estimate the total number of frames - introspection = self.model.introspection[task_name] - num_frames, dimension = introspection(num_samples) - num_frames_per_chunk, _ = introspection(window_size) - num_frames_per_step, _ = introspection(step_size) - if has_last_chunk: - num_frames_last_step, _ = introspection(last_step_size) - - # warm-up window used for overlap-add aggregation - warm_up = np.ones((num_frames_per_chunk, 1)) - # ... for very first chunk - warm_up_first = np.ones((num_frames_per_chunk, 1)) - # ... forvery last chunk - warm_up_last = np.ones((num_frames_per_chunk, 1)) - - if not specifications.permutation_invariant: - - # anything before warm_up_left (and after num_frames_per_chunk - warm_up_right) - # will not be used in the final aggregation - warm_up_left = round( - self.warm_up[0] / self.duration * num_frames_per_chunk - ) - warm_up_right = round( - self.warm_up[1] / self.duration * num_frames_per_chunk + # process orphan last chunk + if has_last_chunk: + last_output = self.infer(last_chunk[None])[0] + if self.progress_hook is not None: + self.progress_hook( + num_chunks + has_last_chunk, num_chunks + has_last_chunk ) - # Hamming window used for overlap-add aggregation - window = np.hamming(num_frames_per_chunk).reshape(-1, 1) + # use model introspection to estimate the total number of frames + introspection = self.model.introspection + num_frames, dimension = introspection(num_samples) + num_frames_per_chunk, _ = introspection(window_size) + num_frames_per_step, _ = introspection(step_size) + if has_last_chunk: + num_frames_last_step, _ = introspection(last_step_size) - else: - # Regular overlap-add aggregation cannot be used directly for - # permutation-invariant models. Why? Because two consecutive - # chunks may disagree on part of their output (even after optimal - # permutation). + # warm-up window used for overlap-add aggregation + warm_up = np.ones((num_frames_per_chunk, 1)) + # ... for very first chunk + warm_up_first = np.ones((num_frames_per_chunk, 1)) + # ... forvery last chunk + warm_up_last = np.ones((num_frames_per_chunk, 1)) - # These two lines make inference use only the (step-long) central - # part of each chunk in the final aggregation... essentially - # switching to a simple concatenation. + if not specifications.permutation_invariant: - warm_up_left = math.floor( - 0.5 - * (self.duration + self.warm_up[0] - self.warm_up[1] - self.step) - / self.duration - * num_frames_per_chunk - ) - warm_up_right = num_frames_per_chunk - math.ceil( - 0.5 - * (self.duration + self.warm_up[0] - self.warm_up[1] + self.step) - / self.duration - * num_frames_per_chunk - ) - - # we do not need a window here because there is no overlap - # between concatenated chunk central parts. - window = 1.0 - - warm_up[:warm_up_left] = 0.0 - warm_up_last[:warm_up_left] = 0.0 - warm_up[num_frames_per_chunk - warm_up_right :] = 0.0 - warm_up_first[num_frames_per_chunk - warm_up_right :] = 0.0 - - # aggregated_output[i] will be used to store the sum of all predictions - # for frame #i - aggregated_output: np.ndarray = np.zeros( - (num_frames, dimension), dtype=np.float32 + # anything before warm_up_left (and after num_frames_per_chunk - warm_up_right) + # will not be used in the final aggregation + warm_up_left = round(self.warm_up[0] / self.duration * num_frames_per_chunk) + warm_up_right = round( + self.warm_up[1] / self.duration * num_frames_per_chunk ) - # overlapping_chunk_count[i] will be used to store the number of chunks - # that contributed to frame #i - overlapping_chunk_count: np.ndarray = np.zeros( - (num_frames, 1), dtype=np.float32 - ) + # Hamming window used for overlap-add aggregation + window = np.hamming(num_frames_per_chunk).reshape(-1, 1) - if specifications.permutation_invariant: - # number of previous outputs that overlap with current one by at least 50% of their warmed up region - num_overlap = math.floor( - 0.5 - * (self.duration - self.warm_up[0] - self.warm_up[1]) - / self.step - ) - # keep track of those previous outputs in a "deque" - if num_overlap > 0: - previous_outputs: Deque[np.ndarray] = deque([], maxlen=num_overlap) - - # loop on the outputs of sliding chunks - for c, output in enumerate(outputs[task_name]): - start_sample = c * step_size - start_frame, _ = introspection(start_sample) - - if specifications.permutation_invariant and num_overlap > 0: - if c > 0: - output = self.permutate( - np.stack(previous_outputs), output, num_frames_per_step - ) - previous_outputs.append(output) - - # when processing first chunk, do not weigh-down its left-most side - if c == 0: - # unless there is just one chunk, where we do not weigh-down any side - if not has_last_chunk and num_chunks == 1: - warm_up_ = 1.0 - else: - warm_up_ = warm_up_first - - # when processing last chunk, make sure to not weigh-down its right-most side - elif not has_last_chunk and c + 1 == num_chunks: - warm_up_ = warm_up_last - - # when processing an internal chunk, weigh-down both sides - else: - warm_up_ = warm_up - - aggregated_output[start_frame : start_frame + num_frames_per_chunk] += ( - output * window * warm_up_ - ) - - overlapping_chunk_count[ - start_frame : start_frame + num_frames_per_chunk - ] += (window * warm_up_) - - # process last (right-aligned) chunk separately - if has_last_chunk: - - if ( - specifications.permutation_invariant - and num_overlap > 0 - and previous_outputs - ): - # FIXME - last_output[task_name] = self.permutate( - previous_outputs[-1][np.newaxis], - last_output[task_name], - num_frames_last_step, - ) - - aggregated_output[-num_frames_per_chunk:] += ( - last_output[task_name] * window * warm_up_last - ) - overlapping_chunk_count[-num_frames_per_chunk:] += window * warm_up_last - - aggregated_output /= np.maximum(overlapping_chunk_count, 1e-12) - - frames = SlidingWindow( - start=0, - duration=file_duration / num_frames, - step=file_duration / num_frames, - ) - - results[task_name] = SlidingWindowFeature(aggregated_output, frames) - - if self.model.is_multi_task: - return results else: - return results.popitem()[1] + # Regular overlap-add aggregation cannot be used directly for + # permutation-invariant models. Why? Because two consecutive + # chunks may disagree on part of their output (even after optimal + # permutation). + + # These two lines make inference use only the (step-long) central + # part of each chunk in the final aggregation... essentially + # switching to a simple concatenation. + + warm_up_left = math.floor( + 0.5 + * (self.duration + self.warm_up[0] - self.warm_up[1] - self.step) + / self.duration + * num_frames_per_chunk + ) + warm_up_right = num_frames_per_chunk - math.ceil( + 0.5 + * (self.duration + self.warm_up[0] - self.warm_up[1] + self.step) + / self.duration + * num_frames_per_chunk + ) + + # we do not need a window here because there is no overlap + # between concatenated chunk central parts. + window = 1.0 + + warm_up[:warm_up_left] = 0.0 + warm_up_last[:warm_up_left] = 0.0 + warm_up[num_frames_per_chunk - warm_up_right :] = 0.0 + warm_up_first[num_frames_per_chunk - warm_up_right :] = 0.0 + + # aggregated_output[i] will be used to store the sum of all predictions + # for frame #i + aggregated_output: np.ndarray = np.zeros( + (num_frames, dimension), dtype=np.float32 + ) + + # overlapping_chunk_count[i] will be used to store the number of chunks + # that contributed to frame #i + overlapping_chunk_count: np.ndarray = np.zeros( + (num_frames, 1), dtype=np.float32 + ) + + if specifications.permutation_invariant: + # number of previous outputs that overlap with current one by at least 50% of their warmed up region + num_overlap = math.floor( + 0.5 * (self.duration - self.warm_up[0] - self.warm_up[1]) / self.step + ) + # keep track of those previous outputs in a "deque" + if num_overlap > 0: + previous_outputs: Deque[np.ndarray] = deque([], maxlen=num_overlap) + + # loop on the outputs of sliding chunks + for c, output in enumerate(outputs): + start_sample = c * step_size + start_frame, _ = introspection(start_sample) + + if specifications.permutation_invariant and num_overlap > 0: + if c > 0: + output = self.permutate( + np.stack(previous_outputs), output, num_frames_per_step + ) + previous_outputs.append(output) + + # when processing first chunk, do not weigh-down its left-most side + if c == 0: + # unless there is just one chunk, where we do not weigh-down any side + if not has_last_chunk and num_chunks == 1: + warm_up_ = 1.0 + else: + warm_up_ = warm_up_first + + # when processing last chunk, make sure to not weigh-down its right-most side + elif not has_last_chunk and c + 1 == num_chunks: + warm_up_ = warm_up_last + + # when processing an internal chunk, weigh-down both sides + else: + warm_up_ = warm_up + + aggregated_output[start_frame : start_frame + num_frames_per_chunk] += ( + output * window * warm_up_ + ) + + overlapping_chunk_count[ + start_frame : start_frame + num_frames_per_chunk + ] += (window * warm_up_) + + # process last (right-aligned) chunk separately + if has_last_chunk: + + if ( + specifications.permutation_invariant + and num_overlap > 0 + and previous_outputs + ): + # FIXME + last_output = self.permutate( + previous_outputs[-1][np.newaxis], + last_output, + num_frames_last_step, + ) + + aggregated_output[-num_frames_per_chunk:] += ( + last_output * window * warm_up_last + ) + overlapping_chunk_count[-num_frames_per_chunk:] += window * warm_up_last + + aggregated_output /= np.maximum(overlapping_chunk_count, 1e-12) + + # FIXME -- get this from introspection + frames = SlidingWindow( + start=0, + duration=file_duration / num_frames, + step=file_duration / num_frames, + ) + + return SlidingWindowFeature(aggregated_output, frames) def permutate( self, past_outputs: Deque[np.ndarray], output: np.ndarray, step_size: int @@ -533,14 +480,7 @@ class Inference: ((permutation, _),) = Counter(permutations).most_common(1) return output[:, permutation] - def __call__( - self, file: AudioFile - ) -> Union[ - SlidingWindowFeature, - Dict[Text, SlidingWindowFeature], - np.ndarray, - Dict[Text, np.ndarray], - ]: + def __call__(self, file: AudioFile) -> Union[SlidingWindowFeature, np.ndarray]: """Run inference on a whole file Parameters @@ -554,10 +494,6 @@ class Inference: Model output, as `SlidingWindowFeature` if `window` is set to "sliding" and `np.ndarray` if is set to "whole". - Notes - ----- - If model has several outputs (multi-task), those will be returned as a - {task_name: output} dictionary. """ waveform, sample_rate = self.model.audio(file) @@ -565,26 +501,14 @@ class Inference: if self.window == "sliding": return self.slide(waveform, sample_rate) - outputs = { - task_name: task_output[0] - for task_name, task_output in self.infer(waveform[None]).items() - } - if self.model.is_multi_task: - return outputs - else: - return outputs.popitem()[1] + return self.infer(waveform[None])[0] def crop( self, file: AudioFile, chunk: Union[Segment, List[Segment]], fixed: Optional[float] = None, - ) -> Union[ - SlidingWindowFeature, - Dict[Text, SlidingWindowFeature], - np.ndarray, - Dict[Text, np.ndarray], - ]: + ) -> Union[SlidingWindowFeature, np.ndarray]: """Run inference on a chunk or a list of chunks Parameters @@ -612,9 +536,6 @@ class Inference: Notes ----- - If model has several outputs (multi-task), those will be returned as a - {task_name: output} dictionary. - If model needs to be warmed up, remember to extend the requested chunk with the corresponding amount of time so that it is actually warmed up when processing the chunk of interest: @@ -633,23 +554,11 @@ class Inference: waveform, sample_rate = self.model.audio.crop(file, chunk, fixed=fixed) output = self.slide(waveform, sample_rate) - if self.model.is_multi_task: - shifted_output = dict() - for task_name, task_output in output.items(): - frames = task_output.sliding_window - shifted_frames = SlidingWindow( - start=chunk.start, duration=frames.duration, step=frames.step - ) - shifted_output[task_name] = SlidingWindowFeature( - task_output.data, shifted_frames - ) - return shifted_output - else: - frames = output.sliding_window - shifted_frames = SlidingWindow( - start=chunk.start, duration=frames.duration, step=frames.step - ) - return SlidingWindowFeature(output.data, shifted_frames) + frames = output.sliding_window + shifted_frames = SlidingWindow( + start=chunk.start, duration=frames.duration, step=frames.step + ) + return SlidingWindowFeature(output.data, shifted_frames) elif self.window == "whole": @@ -660,15 +569,7 @@ class Inference: [self.model.audio.crop(file, c)[0] for c in chunk], dim=1 ) - outputs = { - task_name: task_output[0] - for task_name, task_output in self.infer(waveform[None]).items() - } - - if self.model.is_multi_task: - return outputs - else: - return outputs.popitem()[1] + return self.infer(waveform[None])[0] else: raise NotImplementedError( diff --git a/pyannote/audio/core/model.py b/pyannote/audio/core/model.py index 4c3f6e16..7afbfad3 100644 --- a/pyannote/audio/core/model.py +++ b/pyannote/audio/core/model.py @@ -240,26 +240,6 @@ class Introspection: step = (self.inc_num_samples / self.inc_num_frames) / sample_rate return SlidingWindow(start=0.0, step=step, duration=step) - def __len__(self): - # makes it possible to do something like: - # multi_task = len(introspection) > 1 - # because multi-task introspections are stored as {task_name: introspection} dict - return 1 - - def __getitem__(self, key): - if key is not None: - raise KeyError - return self - - def items(self): - yield None, self - - def keys(self): - yield None - - def __iter__(self): - yield None - class Model(pl.LightningModule): """Base model @@ -301,18 +281,11 @@ class Model(pl.LightningModule): def example_input_array(self) -> torch.Tensor: batch_size = 3 if self.task is None else self.task.batch_size - if self.is_multi_task: - # this assumes that all tasks share the same duration - _, specifications = next(iter(self.specifications.items())) - else: - specifications = self.specifications - duration = specifications.duration - return torch.randn( ( batch_size, self.hparams.num_channels, - int(self.hparams.sample_rate * duration), + int(self.hparams.sample_rate * self.specifications.duration), ), device=self.device, ) @@ -342,36 +315,23 @@ class Model(pl.LightningModule): if hasattr(self, "_specifications"): del self._specifications - @property - def is_multi_task(self): - return len(self.specifications) > 1 - def build(self): # use this method to add task-dependent layers to the model # (e.g. the final classification and activation layers) pass @property - def introspection(self) -> Union[Introspection, Dict[Text, Introspection]]: + def introspection(self) -> Introspection: """Introspection Returns ------- - introspection: Introspection or {str: Introspection} dict - Model introspection or {task_name: introspection} dictionary for - multi-task models. + introspection: Introspection + Model introspection """ if not hasattr(self, "_introspection"): - - if self.is_multi_task: - self._introspection = { - name: Introspection.from_model(self, task=name) - for name in self.specifications - } - # TODO: raises an error in case of multiple tasks with different introspections - else: - self._introspection = Introspection.from_model(self) + self._introspection = Introspection.from_model(self) return self._introspection @@ -502,7 +462,7 @@ class Model(pl.LightningModule): raise NotImplementedError(msg) # convenience function to automate the choice of the final activation function - def default_activation(self) -> Union[nn.Module, Dict[str, nn.Module]]: + def default_activation(self) -> nn.Module: """Guess default activation function according to task specification * sigmoid for binary classification @@ -511,18 +471,9 @@ class Model(pl.LightningModule): Returns ------- - activation : nn.Module or {str: nn.Module} - Activation or {task_name: activation} dictionary for multi-task models. + activation : nn.Module + Activation. """ - - if self.is_multi_task: - return nn.ModuleDict( - { - name: self.helper_default_activation(specs) - for name, specs in self.specifications.items() - } - ) - return self.helper_default_activation(self.specifications) # training step logic is delegated to the task because the diff --git a/pyannote/audio/core/task.py b/pyannote/audio/core/task.py index c01b8b3c..8aa88684 100644 --- a/pyannote/audio/core/task.py +++ b/pyannote/audio/core/task.py @@ -84,26 +84,6 @@ class Specifications: # whether classes are permutation-invariant (e.g. diarization) permutation_invariant: bool = False - def __len__(self): - # makes it possible to do something like: - # multi_task = len(specifications) > 1 - # because multi-task specifications are stored as {task_name: specifications} dict - return 1 - - def __getitem__(self, key): - if key is not None: - raise KeyError - return self - - def items(self): - yield None, self - - def keys(self): - yield None - - def __iter__(self): - yield None - class TrainDataset(IterableDataset): def __init__(self, task: Task): @@ -176,8 +156,6 @@ class Task(pl.LightningDataModule): ---------- specifications : Specifications or dict of Specifications Task specifications (available after `Task.setup` has been called.) - For multi-task learning, this should be a dictionary where keys are - task names and values are corresponding Specifications instances. """ def __init__( @@ -263,11 +241,6 @@ class Task(pl.LightningDataModule): def setup_validation_metric(self): pass - @property - def is_multi_task(self) -> bool: - """"Check whether multiple tasks are addressed at once""" - return len(self.specifications) > 1 - def train__iter__(self): # will become train_dataset.__iter__ method msg = f"Missing '{self.__class__.__name__}.train__iter__' method." @@ -341,8 +314,6 @@ class Task(pl.LightningDataModule): * binary cross-entropy loss for binary or multi-label classification * negative log-likelihood loss for regular classification - In case of multi-tasking, it will default to summing loss of each task. - If "weight" attribute exists, batch[self.weight] is also passed to the loss function during training (but has no effect in validation). @@ -358,18 +329,14 @@ class Task(pl.LightningDataModule): Returns ------- loss : {str: torch.tensor} - {"loss": loss} with additional "loss_{task_name}" keys for multi-task models. + {"loss": loss} """ # forward pass y_pred = self.model(batch["X"]) - if self.is_multi_task: - _, any_y_pred = next(iter(y_pred.items())) - batch_size, num_frames, _ = any_y_pred.shape - else: - batch_size, num_frames, _ = y_pred.shape - # (batch_size, num_frames, num_classes) + batch_size, num_frames, _ = y_pred.shape + # (batch_size, num_frames, num_classes) # target y = batch["y"] @@ -388,34 +355,7 @@ class Task(pl.LightningDataModule): warm_up_right = round(self.warm_up[1] / self.duration * num_frames) weight[:, num_frames - warm_up_right :] = 0.0 - # compute multi-task loss as the sum of loss of each task - if self.is_multi_task: - loss = dict() - for task_name, specifications in self.specifications.items(): - loss[task_name] = self.default_loss( - specifications, y[task_name], y_pred[task_name], weight=weight - ) - self.model.log( - f"{task_name}@{stage}_loss", - loss[task_name], - on_step=False, - on_epoch=True, - prog_bar=False, - logger=False, - ) - - loss["loss"] = sum(loss.values()) - self.model.log( - f"{self.ACRONYM}@{stage}_loss", - loss["loss"], - on_step=False, - on_epoch=True, - prog_bar=True, - logger=True, - ) - return loss - - # compute mono-task loss + # compute loss loss = self.default_loss(self.specifications, y, y_pred, weight=weight) self.model.log( f"{self.ACRONYM}@{stage}_loss", diff --git a/pyannote/audio/models/segmentation/MultiPyanNet.py b/pyannote/audio/models/segmentation/MultiPyanNet.py deleted file mode 100644 index 83273be2..00000000 --- a/pyannote/audio/models/segmentation/MultiPyanNet.py +++ /dev/null @@ -1,236 +0,0 @@ -# MIT License -# -# Copyright (c) 2020 CNRS -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -from functools import cached_property -from typing import Optional - -import torch -import torch.nn as nn -import torch.nn.functional as F -from einops import rearrange - -from pyannote.audio.core.model import Model -from pyannote.audio.core.task import Task -from pyannote.audio.models.blocks.sincnet import SincNet -from pyannote.audio.utils.probe import probe -from pyannote.core.utils.generators import pairwise - -# from pyannote.core.utils.params import merge_dict - - -class Branch(nn.Module): - """Branch""" - - LSTM_DEFAULTS = {"hidden_size": 128, "num_layers": 1, "bidirectional": True} - LINEAR_DEFAULTS = {"hidden_size": 128, "num_layers": 1} - - def __init__( - self, - in_features, - lstm: dict = None, - linear: dict = None, - head: int = None, - ): - - super().__init__() - - self.lstm_hparams = dict(**self.LSTM_DEFAULTS) - if lstm is not None: - self.lstm_hparams.update(**lstm) - self.lstm_hparams["batch_first"] = True # this is not negotiable - - self.linear_hparams = dict(**self.LINEAR_DEFAULTS) - if linear is not None: - self.linear_hparams.update(**linear) - - self.lstm = nn.ModuleList() - one_lstm_hparams = dict(self.lstm_hparams) - one_lstm_hparams["num_layers"] = 1 - for i in range(self.lstm_hparams["num_layers"]): - self.lstm.append(nn.LSTM(in_features, **one_lstm_hparams)) - in_features = self.lstm_hparams["hidden_size"] * ( - 2 if self.lstm_hparams["bidirectional"] else 1 - ) - - lstm_out_features = in_features - self.linear = nn.ModuleList( - [ - nn.Linear(in_features, out_features) - for in_features, out_features in pairwise( - [ - lstm_out_features, - ] - + [self.linear_hparams["hidden_size"]] - * self.linear_hparams["num_layers"] - ) - ] - ) - - def forward(self, frames: torch.Tensor) -> torch.Tensor: - """Pass forward - - Parameters - ---------- - frames : (batch, ???, ???) - """ - - outputs = frames - - for i, lstm in enumerate(self.lstm): - if i == 0: - outputs, hidden = lstm(outputs) - else: - outputs, hidden = lstm(outputs, hidden) - - if len(self.linear) > 0: - for linear in self.linear: - outputs = F.leaky_relu(linear(outputs)) - - return outputs - - @cached_property - def out_features(self) -> int: - - if len(self.linear) > 0: - return self.linear_hparams["hidden_size"] - - return self.lstm_hparams["hidden_size"] * ( - 2 if self.lstm_hparams["bidirectional"] else 1 - ) - - -class Trunk(Branch): - def __init__(self, in_features, lstm: dict = None): - no_linear = {"num_layers": 0} - super().__init__(in_features, lstm=lstm, linear=no_linear) - - -class MultiPyanNet(Model): - """PyanNet segmentation model - - SincFilterbank > Conv1 > Conv1d > LSTM > Feed forward > Classifier - - Parameters - ---------- - sample_rate : int, optional - Audio sample rate. Defaults to 16kHz (16000). - num_channels : int, optional - Number of channels. Defaults to mono (1). - sincnet : dict, optional - Keyword arugments passed to the SincNet block. - Defaults to {"stride": 1}. - trunk : dict, optional - branches : dict, optional - """ - - SINCNET_DEFAULTS = {"stride": 1} - - TRUNK_DEFAULTS = { - "lstm": {"num_layers": 3, "hidden_size": 128, "bidirectional": True}, - } - - BRANCH_DEFAULTS = { - "lstm": {"num_layers": 1, "hidden_size": 128, "bidirectional": True}, - "linear": {"num_layers": 1, "hidden_size": 128}, - } - - BRANCHES_DEFAULTS = { - "vad": {"head": 0, **BRANCH_DEFAULTS}, - "scd": {"head": 1, **BRANCH_DEFAULTS}, - "osd": {"head": 2, **BRANCH_DEFAULTS}, - } - - def __init__( - self, - sample_rate: int = 16000, - num_channels: int = 1, - sincnet: dict = None, - trunk: dict = None, - branches: dict = None, - task: Optional[Task] = None, - ): - - super().__init__(sample_rate=sample_rate, num_channels=num_channels, task=task) - - sincnet_hparams = dict(**self.SINCNET_DEFAULTS) - if sincnet is not None: - sincnet_hparams.update(**sincnet) - sincnet_hparams["sample_rate"] = sample_rate - self.hparams.sincnet = sincnet_hparams - self.sincnet = SincNet(**self.hparams.sincnet) - - if trunk is None: - trunk = self.TRUNK_DEFAULTS - self.hparams.trunk = trunk - - if branches is None: - branches = self.BRANCHES_DEFAULTS - self.hparams.branches = branches - - self.trunk = Trunk(60, **self.hparams.trunk) - - def build(self): - - probes = dict() - self.branches = nn.ModuleDict() - self.classifiers = nn.ModuleDict() - for task_name, specifications in self.specifications.items(): - branch_hparams = self.hparams.branches[task_name] - probes[task_name] = f"lstm.{branch_hparams['head']:d}" - branch = Branch(self.trunk.out_features, **branch_hparams) - self.branches[task_name] = branch - self.classifiers[task_name] = nn.Linear( - branch.out_features, len(specifications.classes) - ) - - # probe trunk - _ = probe(self.trunk, probes) - - self.activation: dict = self.default_activation() - - def forward(self, waveforms: torch.Tensor) -> torch.Tensor: - """Pass forward - - Parameters - ---------- - waveforms : (batch, channel, sample) - - Returns - ------- - scores : {task: (batch, frame, classes) scores} dict - """ - - frames = self.sincnet(waveforms) - - outputs: dict = self.trunk( - rearrange(frames, "batch feature frame -> batch frame feature") - ) - - return { - task_name: self.activation[task_name]( - self.classifiers[task_name]( - self.branches[task_name](outputs[task_name][0]) - ) - ) - for task_name in self.specifications - } diff --git a/pyannote/audio/models/segmentation/debug.py b/pyannote/audio/models/segmentation/debug.py index 2953a884..498faee2 100644 --- a/pyannote/audio/models/segmentation/debug.py +++ b/pyannote/audio/models/segmentation/debug.py @@ -80,52 +80,3 @@ class SimpleSegmentationModel(Model): output, hidden = self.lstm(rearrange(mfcc, "b c f t -> b t (c f)")) # apply the final classifier to get logits return self.activation(self.classifier(output)) - - -class MultiTaskSegmentationModel(Model): - def __init__( - self, - sample_rate: int = 16000, - num_channels: int = 1, - task: Optional[Task] = None, - ): - - super().__init__(sample_rate=sample_rate, num_channels=num_channels, task=task) - - self.mfcc = MFCC( - sample_rate=self.hparams.sample_rate, - n_mfcc=40, - dct_type=2, - norm="ortho", - log_mels=False, - ) - - self.lstm = nn.LSTM( - self.mfcc.n_mfcc * self.hparams.num_channels, - 32, - num_layers=1, - batch_first=True, - bidirectional=True, - ) - - def build(self): - - self.classifier = nn.ModuleDict( - { - name: nn.Linear(32 * 2, len(specifications.classes)) - for name, specifications in self.specifications.items() - } - ) - - self.activation = self.default_activation() - - def forward(self, waveforms: torch.Tensor) -> torch.Tensor: - # extract MFCC - mfcc = self.mfcc(waveforms) - # pass MFCC sequence into the recurrent layer - output, hidden = self.lstm(rearrange(mfcc, "b c f t -> b t (c f)")) - - return { - name: self.activation[name](self.classifier[name](output)) - for name in self.specifications - } diff --git a/pyannote/audio/tasks/segmentation/mixins.py b/pyannote/audio/tasks/segmentation/mixins.py index 13d228fe..63358434 100644 --- a/pyannote/audio/tasks/segmentation/mixins.py +++ b/pyannote/audio/tasks/segmentation/mixins.py @@ -208,14 +208,7 @@ class SegmentationTaskMixin: # use model introspection to predict how many frames it will output num_samples = sample["X"].shape[1] - introspection = self.model.introspection - if self.is_multi_task: - # this assumes that all tasks share the same model introspection. - # this is a reasonable assumption for now. - any_task = next(iter(introspection.keys())) - num_frames, _ = introspection[any_task](num_samples) - else: - num_frames, _ = introspection(num_samples) + num_frames, _ = self.model.introspection(num_samples) # crop "annotation" and keep track of corresponding list of labels if needed annotation: Annotation = file["annotation"].crop(chunk) diff --git a/pyannote/audio/tasks/segmentation/multi_task_segmentation.py b/pyannote/audio/tasks/segmentation/multi_task_segmentation.py deleted file mode 100644 index eaa80b6c..00000000 --- a/pyannote/audio/tasks/segmentation/multi_task_segmentation.py +++ /dev/null @@ -1,264 +0,0 @@ -# MIT License -# -# Copyright (c) 2020-2021 CNRS -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -from typing import Mapping, Text - -import numpy as np -from pytorch_lightning.metrics.classification import FBeta -from torch_audiomentations.core.transforms_interface import BaseWaveformTransform - -from pyannote.audio.core.task import Problem, Task -from pyannote.audio.tasks import ( - OverlappedSpeechDetection, - SpeakerChangeDetection, - VoiceActivityDetection, -) -from pyannote.audio.tasks.segmentation.mixins import SegmentationTaskMixin -from pyannote.database import Protocol - - -class MultiTaskSegmentation(SegmentationTaskMixin, Task): - """Multi-task segmentation - - Multi-task training of segmentation tasks, including: - - vad: voice activity detection - - scd: speaker change detection - - osd: overlapped speech detection - - Note that, when "osd" is one of the tasks considered, data augmentation is used - to artificially increase the proportion of "overlap". This is achieved by - generating chunks made out of the (weighted) sum of two random chunks. - - Parameters - ---------- - protocol : Protocol - pyannote.database protocol - duration : float, optional - Chunks duration. Defaults to 2s. - balance: str, optional - When provided, training samples are sampled uniformly with respect to that key. - For instance, setting `balance` to "uri" will make sure that each file will be - equally represented in the training samples. - overlap: dict, optional - Controls how artificial chunks with overlapping speech are generated: - - "probability" key is the probability of artificial overlapping chunks. Setting - "probability" to 0.6 means that, on average, 40% of training chunks are "real" - chunks, while 60% are artifical chunks made out of the (weighted) sum of two - chunks. Defaults to 0.5. - - "snr_min" and "snr_max" keys control the minimum and maximum signal-to-noise - ratio between summed chunks, in dB. Default to 0.0 and 10. - weight: str, optional - When provided, use this key to as frame-wise weight in loss function. - vad : bool, optional - Add voice activity detection in the pool of tasks. - Defaults to False. - vad_params : dict, optional - Additional vad-specific parameters. Has no effect when `vad` is False. - See VoiceActivityDetection docstring for details and default value. - scd : bool, optional - Add speaker change detection to the pool of tasks. - Defaults to False. - scd_params : dict, optional - Additional scd-specific parameters. Has no effect when `scd` is False. - See SpeakerChangeDetection docstring for details and default value. - osd : bool, optional - Add overlapped speech detection to the pool of tasks. - Defaults to False. - osd_params : dict, optional - Additional osd-specific parameters. Has no effect when `osd` is False. - See OverlappedSpeechDetection docstring for details and default value. - batch_size : int, optional - Number of training samples per batch. Defaults to 32. - num_workers : int, optional - Number of workers used for generating training samples. - Defaults to multiprocessing.cpu_count() // 2. - pin_memory : bool, optional - If True, data loaders will copy tensors into CUDA pinned - memory before returning them. See pytorch documentation - for more details. Defaults to False. - augmentation : BaseWaveformTransform, optional - torch_audiomentations waveform transform, used by dataloader - during training. - - """ - - ACRONYM = "xseg" - - OVERLAP_DEFAULTS = {"probability": 0.5, "snr_min": 0.0, "snr_max": 10.0} - - def __init__( - self, - protocol: Protocol, - duration: float = 2.0, - overlap: dict = OVERLAP_DEFAULTS, - balance: Text = None, - weight: Text = None, - vad: bool = False, - vad_params: Mapping = None, - scd: bool = False, - scd_params: Mapping = None, - osd: bool = False, - osd_params: Mapping = None, - batch_size: int = 32, - num_workers: int = None, - pin_memory: bool = False, - augmentation: BaseWaveformTransform = None, - ): - - super().__init__( - protocol, - duration=duration, - batch_size=batch_size, - num_workers=num_workers, - pin_memory=pin_memory, - augmentation=augmentation, - ) - - self.vad = vad - self.vad_params = dict() if vad_params is None else vad_params - - self.scd = scd - self.scd_params = dict() if scd_params is None else scd_params - - self.osd = osd - self.osd_params = dict() if osd_params is None else osd_params - - if sum([vad, scd, osd]) < 2: - raise ValueError( - "You must activate at least two tasks among 'vad', 'scd', and 'osd'." - ) - - self.tasks = dict() - if self.vad: - self.tasks["vad"] = VoiceActivityDetection( - protocol, - duration=duration, - batch_size=batch_size, - num_workers=num_workers, - pin_memory=pin_memory, - **self.vad_params, - ) - if self.scd: - self.tasks["scd"] = SpeakerChangeDetection( - protocol, - duration=duration, - batch_size=batch_size, - num_workers=num_workers, - pin_memory=pin_memory, - **self.scd_params, - ) - if self.osd: - self.tasks["osd"] = OverlappedSpeechDetection( - protocol, - duration=duration, - batch_size=batch_size, - num_workers=num_workers, - pin_memory=pin_memory, - **self.osd_params, - ) - - def setup(self, stage=None): - - super().setup(stage=stage) - - if stage == "fit": - - self.specifications = { - name: task.specifications for name, task in self.tasks.items() - } - - def prepare_y(self, one_hot_y: np.ndarray): - """Get multi-task targets - - Parameters - ---------- - one_hot_y : (num_frames, num_speakers) np.ndarray - One-hot-encoding of current chunk speaker activity: - * one_hot_y[t, k] = 1 if kth speaker is active at tth frame - * one_hot_y[t, k] = 0 otherwise. - - Returns - ------- - y : (num_frames, ) np.ndarray - y[t] = 1 if there is two or more active speakers at tth frame, 0 otherwise. - """ - - return {name: task.prepare_y(one_hot_y) for name, task in self.tasks.items()} - - def setup_validation_metric(self): - - self.val_fbeta = { - task_name: FBeta( - len(specifications.classes), - beta=1.0, - threshold=0.5, - multilabel=( - specifications.problem == Problem.MULTI_LABEL_CLASSIFICATION - ), - average="macro", - ) - for task_name, specifications in self.specifications.items() - } - - def validation_step(self, batch, batch_idx: int): - """Compute areas under ROC curve - - Parameters - ---------- - batch : dict of torch.Tensor - Current batch. - batch_idx: int - Batch index. - """ - - X, y = batch["X"], batch["y"] - y_pred = self.model(X) - - val_fbeta = dict() - - for task_name in self.specifications: - - # move metric to model device - self.val_fbeta[task_name].to(self.model.device) - - val_fbeta[task_name] = self.val_fbeta[task_name]( - y_pred[task_name][:, ::10].squeeze(), y[task_name][:, ::10].squeeze() - ) - - self.model.log( - f"{task_name}@val_fbeta", - val_fbeta[task_name], - on_step=False, - on_epoch=True, - prog_bar=True, - logger=True, - ) - - self.model.log( - f"{self.ACRONYM}@val_fbeta", - sum(val_fbeta.values()) / len(val_fbeta), - on_step=False, - on_epoch=True, - prog_bar=True, - logger=True, - ) diff --git a/pyannote/audio/tasks/segmentation/segmentation.py b/pyannote/audio/tasks/segmentation/segmentation.py index 4cbf56c7..fca6b62f 100644 --- a/pyannote/audio/tasks/segmentation/segmentation.py +++ b/pyannote/audio/tasks/segmentation/segmentation.py @@ -305,7 +305,7 @@ class Segmentation(SegmentationTaskMixin, Task): Returns ------- loss : {str: torch.tensor} - {"loss": loss} with additional "loss_{task_name}" keys for multi-task models. + {"loss": loss} """ # forward pass diff --git a/tests/inference_test.py b/tests/inference_test.py index 5c99550b..807f94cc 100644 --- a/tests/inference_test.py +++ b/tests/inference_test.py @@ -4,14 +4,8 @@ import pytorch_lightning as pl from pyannote.audio import Inference, Model from pyannote.audio.core.task import Resolution -from pyannote.audio.models.segmentation.debug import ( - MultiTaskSegmentationModel, - SimpleSegmentationModel, -) +from pyannote.audio.models.segmentation.debug import SimpleSegmentationModel from pyannote.audio.tasks import VoiceActivityDetection -from pyannote.audio.tasks.segmentation.multi_task_segmentation import ( - MultiTaskSegmentation, -) from pyannote.core import SlidingWindowFeature from pyannote.database import FileFinder, get_protocol @@ -96,33 +90,6 @@ def test_on_file_path(trained): assert isinstance(output, SlidingWindowFeature) -def test_multi_seg_infer(): - protocol = get_protocol( - "Debug.SpeakerDiarization.Debug", preprocessors={"audio": FileFinder()} - ) - xseg = MultiTaskSegmentation( - protocol, - duration=2.0, - vad=True, - scd=True, - osd=True, - batch_size=32, - num_workers=4, - ) - model = MultiTaskSegmentationModel(task=xseg) - trainer = pl.Trainer(max_epochs=1, fast_dev_run=True) - _ = trainer.fit(model, xseg) - inference = Inference(model, duration=2.0, step=0.5) - dev_file = next(protocol.development()) - scores = inference(dev_file) - - assert isinstance(scores, dict) - - for attr in ["vad", "scd", "osd"]: - assert attr in scores - assert isinstance(scores[attr], SlidingWindowFeature) - - def test_skip_aggregation(pretrained_model, dev_file): inference = Inference(pretrained_model, skip_aggregation=True)