From f735ba843da96e5d1f97bde0b7ae3df5e60aaa4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20BREDIN?= Date: Fri, 22 Aug 2025 15:05:26 +0200 Subject: [PATCH] BREAKING(task): remove `OverlappedSpeechDetection` task (#1903) --- CHANGELOG.md | 1 + src/pyannote/audio/tasks/__init__.py | 11 +- .../overlapped_speech_detection.py | 212 ------------------ tests/test_train.py | 28 +-- 4 files changed, 6 insertions(+), 246 deletions(-) delete mode 100644 src/pyannote/audio/tasks/segmentation/overlapped_speech_detection.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c9ecd06..611f7f1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,7 @@ With the optional telemetry feature in `pyannote.audio`, you can choose to send - BREAKING(task): drop support for `warm_up` option in `SpeakerDiarization` task - BREAKING(task): drop support for `weigh_by_cardinality` option in `SpeakerDiarization` task - BREAKING(task): drop support for `vad_loss` option in `SpeakerDiarization` task +- BREAKING(task): remove `OverlappedSpeechDetection` task (part of `SpeakerDiarization` task) - BREAKING(cli): remove deprecated `pyannote-audio-train` CLI ### New features diff --git a/src/pyannote/audio/tasks/__init__.py b/src/pyannote/audio/tasks/__init__.py index 517c6dd5..5ba6b4fa 100644 --- a/src/pyannote/audio/tasks/__init__.py +++ b/src/pyannote/audio/tasks/__init__.py @@ -1,6 +1,7 @@ # MIT License # -# Copyright (c) 2020-2021 CNRS +# Copyright (c) 2020-2025 CNRS +# Copyright (c) 2025- pyannoteAI # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -24,23 +25,15 @@ from .segmentation.multilabel import MultiLabelSegmentation # isort:skip from .segmentation.speaker_diarization import SpeakerDiarization # isort:skip from .separation.PixIT import PixIT # isort:skip from .segmentation.voice_activity_detection import VoiceActivityDetection # isort:skip -from .segmentation.overlapped_speech_detection import ( # isort:skip - OverlappedSpeechDetection, -) from .embedding.arcface import SupervisedRepresentationLearningWithArcFace # isort:skip -# Segmentation has been renamed to SpeakerDiarization but we keep Segmentation here for backward compatibility -Segmentation = SpeakerDiarization - # SpeakerEmbedding is more human-friendly SpeakerEmbedding = SupervisedRepresentationLearningWithArcFace __all__ = [ "SpeakerDiarization", "VoiceActivityDetection", - "OverlappedSpeechDetection", "MultiLabelSegmentation", "SpeakerEmbedding", - "Segmentation", "PixIT", ] diff --git a/src/pyannote/audio/tasks/segmentation/overlapped_speech_detection.py b/src/pyannote/audio/tasks/segmentation/overlapped_speech_detection.py deleted file mode 100644 index 89d299a8..00000000 --- a/src/pyannote/audio/tasks/segmentation/overlapped_speech_detection.py +++ /dev/null @@ -1,212 +0,0 @@ -# MIT License -# -# Copyright (c) 2020- CNRS -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -from typing import Dict, Optional, Sequence, Text, Tuple, Union - -import numpy as np -from pyannote.core import Segment, SlidingWindowFeature -from pyannote.database import Protocol -from torch_audiomentations.core.transforms_interface import BaseWaveformTransform -from torchmetrics import Metric - -from pyannote.audio.core.task import Problem, Resolution, Specifications -from pyannote.audio.tasks.segmentation.mixins import SegmentationTask - - -class OverlappedSpeechDetection(SegmentationTask): - """Overlapped speech detection - - Overlapped speech detection is the task of detecting regions where at least - two speakers are speaking at the same time. - - Here, it is addressed with the same approach as voice activity detection, - except "speech" class is replaced by "overlap", where a frame is marked as - "overlap" if two speakers or more are active. - - Note that data augmentation is used to increase the proporition of "overlap". - This is achieved by generating chunks made out of the (weighted) sum of two - random chunks. - - Parameters - ---------- - protocol : Protocol - pyannote.database protocol - cache : str, optional - As (meta-)data preparation might take a very long time for large datasets, - it can be cached to disk for later (and faster!) re-use. - When `cache` does not exist, `Task.prepare_data()` generates training - and validation metadata from `protocol` and save them to disk. - When `cache` exists, `Task.prepare_data()` is skipped and (meta)-data - are loaded from disk. Defaults to a temporary path. - duration : float, optional - Chunks duration. Defaults to 2s. - warm_up : float or (float, float), optional - Use that many seconds on the left- and rightmost parts of each chunk - to warm up the model. While the model does process those left- and right-most - parts, only the remaining central part of each chunk is used for computing the - loss during training, and for aggregating scores during inference. - Defaults to 0. (i.e. no warm-up). - balance: Sequence[Text], optional - When provided, training samples are sampled uniformly with respect to these keys. - For instance, setting `balance` to ["database","subset"] will make sure that each - database & subset combination will be equally represented in the training samples. - overlap: dict, optional - Controls how artificial chunks with overlapping speech are generated: - - "probability" key is the probability of artificial overlapping chunks. Setting - "probability" to 0.6 means that, on average, 40% of training chunks are "real" - chunks, while 60% are artifical chunks made out of the (weighted) sum of two - chunks. Defaults to 0.5. - - "snr_min" and "snr_max" keys control the minimum and maximum signal-to-noise - ratio between summed chunks, in dB. Default to 0.0 and 10. - - weight: str, optional - When provided, use this key to as frame-wise weight in loss function. - batch_size : int, optional - Number of training samples per batch. Defaults to 32. - num_workers : int, optional - Number of workers used for generating training samples. - Defaults to multiprocessing.cpu_count() // 2. - pin_memory : bool, optional - If True, data loaders will copy tensors into CUDA pinned - memory before returning them. See pytorch documentation - for more details. Defaults to False. - augmentation : BaseWaveformTransform, optional - torch_audiomentations waveform transform, used by dataloader - during training. - metric : optional - Validation metric(s). Can be anything supported by torchmetrics.MetricCollection. - Defaults to AUROC (area under the ROC curve). - """ - - OVERLAP_DEFAULTS = {"probability": 0.5, "snr_min": 0.0, "snr_max": 10.0} - - def __init__( - self, - protocol: Protocol, - duration: float = 2.0, - warm_up: Union[float, Tuple[float, float]] = 0.0, - overlap: dict = OVERLAP_DEFAULTS, - balance: Optional[Sequence[Text]] = None, - weight: Optional[Text] = None, - batch_size: int = 32, - num_workers: Optional[int] = None, - pin_memory: bool = False, - augmentation: Optional[BaseWaveformTransform] = None, - metric: Union[Metric, Sequence[Metric], Dict[str, Metric]] = None, - cache: Optional[Union[str, None]] = None, - ): - super().__init__( - protocol, - duration=duration, - warm_up=warm_up, - batch_size=batch_size, - num_workers=num_workers, - pin_memory=pin_memory, - augmentation=augmentation, - metric=metric, - cache=cache, - ) - - self.specifications = Specifications( - problem=Problem.BINARY_CLASSIFICATION, - resolution=Resolution.FRAME, - duration=self.duration, - min_duration=self.min_duration, - warm_up=self.warm_up, - classes=[ - "overlap", - ], - ) - - self.overlap = overlap - self.balance = balance - self.weight = weight - - def prepare_chunk(self, file_id: int, start_time: float, duration: float): - """Prepare chunk for overlapped speech detection - - Parameters - ---------- - file_id : int - File index - start_time : float - Chunk start time - duration : float - Chunk duration. - - Returns - ------- - sample : dict - Dictionary containing the chunk data with the following keys: - - `X`: waveform - - `y`: target as a SlidingWindowFeature instance - - `meta`: - - `database`: database index - - `file`: file index - """ - - file = self.get_file(file_id) - - chunk = Segment(start_time, start_time + duration) - - sample = dict() - sample["X"], _ = self.model.audio.crop(file, chunk, duration=duration) - - # gather all annotations of current file - annotations = self.prepared_data["annotations-segments"][ - self.prepared_data["annotations-segments"]["file_id"] == file_id - ] - - # gather all annotations with non-empty intersection with current chunk - chunk_annotations = annotations[ - (annotations["start"] < chunk.end) & (annotations["end"] > chunk.start) - ] - - # discretize chunk annotations at model output resolution - step = self.model.receptive_field.step - half = 0.5 * self.model.receptive_field.duration - - start = np.maximum(chunk_annotations["start"], chunk.start) - chunk.start - half - start_idx = np.maximum(0, np.round(start / step)).astype(int) - - end = np.minimum(chunk_annotations["end"], chunk.end) - chunk.start - half - end_idx = np.round(end / step).astype(int) - - # frame-level targets - num_frames = self.model.num_frames( - round(duration * self.model.hparams.sample_rate) - ) - y = np.zeros((num_frames, 1), dtype=np.uint8) - for start, end in zip(start_idx, end_idx): - y[start : end + 1, 0] += 1 - y = 1 * (y > 1) - - sample["y"] = SlidingWindowFeature( - y, self.model.receptive_field, labels=["speech"] - ) - - metadata = self.prepared_data["audio-metadata"][file_id] - sample["meta"] = {key: metadata[key] for key in metadata.dtype.names} - sample["meta"]["file"] = file_id - - return sample diff --git a/tests/test_train.py b/tests/test_train.py index 0b868d16..ccb4b678 100644 --- a/tests/test_train.py +++ b/tests/test_train.py @@ -1,6 +1,7 @@ # The MIT License (MIT) # -# Copyright (c) 2024- CNRS +# Copyright (c) 2024-2025 CNRS +# Copyright (c) 2025- pyannoteAI # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -25,17 +26,15 @@ from tempfile import mkstemp import pytest from lightning import Trainer -from pyannote.database import FileFinder, registry - from pyannote.audio.models.embedding.debug import SimpleEmbeddingModel from pyannote.audio.models.segmentation.debug import SimpleSegmentationModel from pyannote.audio.tasks import ( MultiLabelSegmentation, - OverlappedSpeechDetection, SpeakerDiarization, SupervisedRepresentationLearningWithArcFace, VoiceActivityDetection, ) +from pyannote.database import FileFinder, registry @pytest.fixture() @@ -138,27 +137,6 @@ def test_train_voice_activity_detection_with_cached_data_mono_device(protocol, c second_trainer.fit(second_model) -def test_train_overlapped_speech_detection(protocol): - overlapped_speech_detection = OverlappedSpeechDetection(protocol) - model = SimpleSegmentationModel(task=overlapped_speech_detection) - trainer = Trainer(fast_dev_run=True, accelerator="cpu") - trainer.fit(model) - - -def test_train_overlapped_speech_detection_with_cached_data_mono_device( - protocol, cache -): - first_task = OverlappedSpeechDetection(protocol, cache=cache) - first_model = SimpleSegmentationModel(task=first_task) - first_trainer = Trainer(fast_dev_run=True, accelerator="cpu", devices=1) - first_trainer.fit(first_model) - - second_task = OverlappedSpeechDetection(protocol, cache=cache) - second_model = SimpleSegmentationModel(task=second_task) - second_trainer = Trainer(fast_dev_run=True, accelerator="cpu", devices=1) - second_trainer.fit(second_model) - - def test_finetune_with_task_that_does_not_need_setup_for_specs(protocol): voice_activity_detection = VoiceActivityDetection(protocol) model = SimpleSegmentationModel(task=voice_activity_detection)