diff --git a/CHANGELOG.md b/CHANGELOG.md index 2dbd9a96..a767b539 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # CHANGELOG +## Version 4.0.2 (2025-11-19) + +- BREAKING(util): make `Binarize.__call__` return `string` tracks (instead of `int`) [@benniekiss](https://github.com/benniekiss/) +- fix(torch): pin `torch`, `torchcodec`, and `torchaudio` versions to [avoid segmentation fault](https://github.com/meta-pytorch/torchcodec/issues/995) +- fix(pyannoteAI): update pyannoteAI wrapper to return both regular and exclusive diarization +- feat(pipeline): add `Pipeline.cuda()` convenience method [@tkanarsky](https://github.com/tkanarsky/) +- feat(pipeline): add `preload` option to base `Pipeline.__call__` to force preloading audio in memory ([@antoinelaurent](https://github.com/antoinelaurent/)) +- feat(cli): add option to apply pipeline on a directory of audio files +- improve(util): make `permutate` faster thanks to vectorized cost function + ## Version 4.0.1 (2025-10-10) - feat: allow passing preloaded pipeline config to `get_pipeline` diff --git a/pyproject.toml b/pyproject.toml index 39087f04..7f86184b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,9 +23,9 @@ dependencies = [ "rich>=13.9.4", "safetensors>=0.5.2", "torch-audiomentations>=0.12.0", - "torch>=2.8.0", - "torchaudio>=2.8.0", - "torchcodec>=0.6.0", + "torch==2.8.0", + "torchaudio==2.8.0", + "torchcodec==0.7.0", "torchmetrics>=1.6.1", "soundfile>=0.13.1", "matplotlib>=3.10.0", @@ -36,7 +36,7 @@ dependencies = [ pyannote-audio = "pyannote.audio.__main__:app" [project.optional-dependencies] -test = ["papermill>=2.6.0", "pytest>=8.3.4", "speechbrain>=1.0.2"] +test = ["papermill>=2.6.0", "pytest>=8.3.4"] doc = ["sphinx-rtd-theme>=3.0.2", "sphinx>=8.1.3"] cli = ["typer>=0.15.1"] separation = ["asteroid>=0.5.2", "transformers>=4.48.3"] diff --git a/src/pyannote/audio/__main__.py b/src/pyannote/audio/__main__.py index e809c3c2..c828c7c7 100644 --- a/src/pyannote/audio/__main__.py +++ b/src/pyannote/audio/__main__.py @@ -327,18 +327,19 @@ def apply( audio: Annotated[ Path, typer.Argument( - help="Path to audio file", + help="Path to audio file or directory", exists=True, file_okay=True, + dir_okay=True, readable=True, ), ], into: Annotated[ Path, typer.Option( - help="Path to file where results are saved.", + help="Path to file or directory where results are saved.", exists=False, - dir_okay=False, + dir_okay=True, file_okay=True, writable=True, resolve_path=True, @@ -360,7 +361,7 @@ def apply( ] = None, ): """ - Apply a pretrained PIPELINE to an AUDIO file + Apply a pretrained PIPELINE to an AUDIO file or directory """ # load pretrained pipeline @@ -373,14 +374,39 @@ def apply( torch_device = parse_device(device) pretrained_pipeline.to(torch_device) - # apply pipeline to audio file - prediction: Annotation = pretrained_pipeline(audio) + if audio.is_dir(): - speaker_diarization = get_diarization(prediction) + if into is None or not into.is_dir(): + typer.echo("When AUDIO is a directory, INTO must also be a directory.") + raise typer.exit(code=1) - # save (or print) results - with open(into, "w") if into else nullcontext(sys.stdout) as rttm: - speaker_diarization.write_rttm(rttm) + inputs: list[Path] = sorted(path for path in audio.iterdir() if path.is_file()) + rttms: list[Path | None] = [into / (path.stem + ".rttm") for path in inputs] + jsons: list[Path | None] = [into / (path.stem + ".json") for path in inputs] + + else: + + if not (into is None or into.is_file()): + typer.echo("When AUDIO is a file, INTO must also be a file.") + raise typer.exit(code=1) + + inputs = [audio] + rttms: list[Path | None] = [into] + jsons: list[Path | None] = [into.with_suffix(".json") if into else None] + + for current_input, current_rttm, current_json in zip(inputs, rttms, jsons): + + prediction = pretrained_pipeline(current_input) + + speaker_diarization = get_diarization(prediction) + + with open(current_rttm, "w") if current_rttm else nullcontext(sys.stdout) as r: + speaker_diarization.write_rttm(r) + + if hasattr(prediction, "serialize") and current_json: + serialized: dict = prediction.serialize() + with open(current_json, "w") as j: + json.dump(serialized, j, indent=2) class MinDurationOffOptimizer: diff --git a/src/pyannote/audio/core/pipeline.py b/src/pyannote/audio/core/pipeline.py index 079aba5c..d082858a 100644 --- a/src/pyannote/audio/core/pipeline.py +++ b/src/pyannote/audio/core/pipeline.py @@ -21,6 +21,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from __future__ import annotations import os import warnings from collections import OrderedDict @@ -406,7 +407,23 @@ class Pipeline(_Pipeline): """ raise NotImplementedError() - def __call__(self, file: AudioFile, **kwargs): + def __call__(self, file: AudioFile, preload: bool = False, **kwargs): + """Validate file, (optionally) load it in memory, then process it + + Parameters + ---------- + file : AudioFile + File to process + preload : bool, optional + Whether to preload waveform before applying the pipeline. + kwargs : keyword arguments, optional + Additional keyword arguments passed to `self.apply(...)` + + Returns + ------- + output : Any + Whatever `self.apply(...)` returns + """ fix_reproducibility(getattr(self, "device", torch.device("cpu"))) if not self.instantiated: @@ -432,16 +449,35 @@ class Pipeline(_Pipeline): file = Audio.validate_file(file) + # check if the instance has preprocessors and wrap the file if so if hasattr(self, "preprocessors"): file = ProtocolFile(file, lazy=self.preprocessors) + # pre-load the audio in memory if requested + if preload: + # raise error if `waveform`` is already in memory (or will be via a preprocessor) + if ( + "waveform" in getattr(self, "preprocessors", dict()) + or "waveform" in file + ): + raise ValueError( + "Cannot preload audio: `waveform` key is already available or will be via a preprocessor." + ) + + # load waveform in memory (and keep track of its original sample rate) + file["waveform"], file["sample_rate"] = Audio()(file) + + # the above line already took care of channel selection, + # therefore we remove the `channel` key from the file + file.pop("channel", None) + # send file duration to telemetry as well as # requested number of speakers in case of diarization track_pipeline_apply(self, file, **kwargs) return self.apply(file, **kwargs) - def to(self, device: torch.device): + def to(self, device: torch.device) -> Pipeline: """Send pipeline to `device`""" if not isinstance(device, torch.device): @@ -462,3 +498,14 @@ class Pipeline(_Pipeline): self.device = device return self + + def cuda(self, device: torch.device | int | None = None) -> Pipeline: + """Send pipeline to (optionally specified) cuda device""" + if device is None: + return self.to(torch.device("cuda")) + elif isinstance(device, int): + return self.to(torch.device("cuda", device)) + else: + if device.type != "cuda": + raise ValueError("Expected CUDA device. Use `Pipeline.to(device)` for other devices.") + return self.to(device) diff --git a/src/pyannote/audio/pipelines/pyannoteai/local.py b/src/pyannote/audio/pipelines/pyannoteai/local.py index 75e6be14..56a82af0 100644 --- a/src/pyannote/audio/pipelines/pyannoteai/local.py +++ b/src/pyannote/audio/pipelines/pyannoteai/local.py @@ -21,11 +21,13 @@ # SOFTWARE. import os -from pathlib import Path from pyannote.audio import Pipeline +from pyannote.audio.core.io import AudioFile from pyannote.core import Annotation, Segment +from ..speaker_diarization import DiarizeOutput + class Local(Pipeline): """Wrapper around official pyannoteAI on-premise package @@ -51,29 +53,24 @@ class Local(Pipeline): self.token = token or os.environ.get("PYANNOTEAI_API_KEY", None) self._pipeline = _LocalPipeline(self.token) - def _to_annotation(self, completed_job: dict) -> Annotation: - """Deserialize job output into pyannote.core Annotation""" - - output = completed_job["output"]["diarization"] - job_id = completed_job["jobId"] - - annotation = Annotation(uri=job_id) - for t, turn in enumerate(output): + def _deserialize(self, diarization: list[dict]) -> Annotation: + # deserialize the output into a good-old Annotation instance + annotation = Annotation() + for t, turn in enumerate(diarization): segment = Segment(start=turn["start"], end=turn["end"]) speaker = turn["speaker"] annotation[segment, t] = speaker - return annotation.rename_tracks("string") def apply( self, - file: Path, + file: AudioFile, num_speakers: int | None = None, min_speakers: int | None = None, max_speakers: int | None = None, - exclusive: bool = False, - ) -> Annotation: - """Speaker diarization using pyannoteAI on-premise package + **kwargs, + ) -> DiarizeOutput: + """Speaker diarization using on-premise pyannoteAI package. Parameters ---------- @@ -86,41 +83,45 @@ class Local(Pipeline): Not supported yet. Minimum number of speakers. Has no effect when `num_speakers` is provided. max_speakers : int, optional Not supported yet. Maximum number of speakers. Has no effect when `num_speakers` is provided. - exclusive : bool, optional - Enable exclusive diarization. Returns ------- - speaker_diarization : Annotation - Speaker diarization result (when successful) - - Raises - ------ - PyannoteAIFailedJob - If the job failed - PyannoteAICanceledJob - If the job was canceled - HTTPError - If something else went wrong + output : DiarizeOutput + DiarizeOutput object containing both regular and exclusive speaker diarization results. """ - predictions = self._pipeline.diarize( - file["audio"], - num_speakers=num_speakers, - min_speakers=min_speakers, - max_speakers=max_speakers, + # if file provides "audio" path + if "audio" in file: + predictions = self._pipeline.diarize( + file["audio"], + num_speakers=num_speakers, + min_speakers=min_speakers, + max_speakers=max_speakers, + **kwargs, + ) + + # if file provides "waveform", make sure it is numpy (and not torch) array + elif "waveform" in file: + waveform = file["waveform"] + if hasattr(waveform, "numpy"): + waveform = waveform.numpy(force=True) + + predictions = self._pipeline.diarize( + {"waveform": waveform, "sample_rate": file["sample_rate"]}, + num_speakers=num_speakers, + min_speakers=min_speakers, + max_speakers=max_speakers, + **kwargs, + ) + else: + raise ValueError("AudioFile must provide either 'audio' or 'waveform' key") + + speaker_diarization: Annotation = self._deserialize(predictions["diarization"]) + exclusive_speaker_diarization: Annotation = self._deserialize( + predictions["exclusive_diarization"] ) - # use exclusive diarization whenever requested - if exclusive: - diarization = predictions["exclusive_diarization"] - else: - diarization = predictions["diarization"] - - # deserialize the output into a good-old Annotation instance - annotation = Annotation() - for t, turn in enumerate(diarization): - segment = Segment(start=turn["start"], end=turn["end"]) - speaker = turn["speaker"] - annotation[segment, t] = speaker - return annotation.rename_tracks("string") + return DiarizeOutput( + speaker_diarization=speaker_diarization, + exclusive_speaker_diarization=exclusive_speaker_diarization, + ) diff --git a/src/pyannote/audio/pipelines/pyannoteai/sdk.py b/src/pyannote/audio/pipelines/pyannoteai/sdk.py index 979b4d3e..a3c16376 100644 --- a/src/pyannote/audio/pipelines/pyannoteai/sdk.py +++ b/src/pyannote/audio/pipelines/pyannoteai/sdk.py @@ -21,9 +21,9 @@ # SOFTWARE. import os -from pathlib import Path from pyannote.audio import Pipeline +from pyannote.audio.core.io import AudioFile from pyannote.core import Annotation, Segment from pyannoteai.sdk import Client @@ -68,7 +68,7 @@ class SDK(Pipeline): def apply( self, - file: Path, + file: AudioFile, num_speakers: int | None = None, min_speakers: int | None = None, max_speakers: int | None = None, diff --git a/src/pyannote/audio/pipelines/utils/diarization.py b/src/pyannote/audio/pipelines/utils/diarization.py index 2130fb96..288da319 100644 --- a/src/pyannote/audio/pipelines/utils/diarization.py +++ b/src/pyannote/audio/pipelines/utils/diarization.py @@ -215,7 +215,7 @@ class SpeakerDiarizationMixin: min_duration_off=min_duration_off, ) - return binarize(discrete_diarization).rename_tracks(generator="string") + return binarize(discrete_diarization) @staticmethod def to_diarization( diff --git a/src/pyannote/audio/utils/permutation.py b/src/pyannote/audio/utils/permutation.py index 3179e0f2..885dd76c 100644 --- a/src/pyannote/audio/utils/permutation.py +++ b/src/pyannote/audio/utils/permutation.py @@ -1,6 +1,7 @@ # MIT License # -# Copyright (c) 2020-2022 CNRS +# Copyright (c) 2020-2025 CNRS +# Copyright (c) 2025- pyannoteAI # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -23,7 +24,7 @@ import math from functools import partial, singledispatch -from typing import Callable, List, Optional, Tuple +from typing import Callable, List, Literal, Tuple import networkx as nx import numpy as np @@ -34,7 +35,7 @@ from scipy.optimize import linear_sum_assignment @singledispatch -def permutate(y1, y2, cost_func: Optional[Callable] = None, return_cost: bool = False): +def permutate(y1, y2, cost_func: Callable | Literal["mse", "mae"] = "mse", return_cost: bool = False): """Find cost-minimizing permutation Parameters @@ -43,9 +44,11 @@ def permutate(y1, y2, cost_func: Optional[Callable] = None, return_cost: bool = (batch_size, num_samples, num_classes_1) y2 : np.ndarray or torch.Tensor (num_samples, num_classes_2) or (batch_size, num_samples, num_classes_2) - cost_func : callable - Takes two (num_samples, num_classes) sequences and returns (num_classes, ) pairwise cost. - Defaults to computing mean squared error. + cost_func : callable or {"mse", "mae"}, optional + Can be either "mse" (mean squared error) or "mae" (mean absolute error) or a callable. + When callable, takes two (num_samples, num_classes) sequences + and returns (num_classes, ) pairwise cost. + Defaults to computing mean squared error ("mse"). return_cost : bool, optional Whether to return cost matrix. Defaults to False. @@ -97,7 +100,7 @@ def mae_cost_func(Y, y, **kwargs): def permutate_torch( y1: torch.Tensor, y2: torch.Tensor, - cost_func: Optional[Callable] = None, + cost_func: Callable | Literal["mse", "mae"] = "mse", return_cost: bool = False, ) -> Tuple[torch.Tensor, List[Tuple[int]]]: @@ -116,7 +119,7 @@ def permutate_torch( raise ValueError(msg) if cost_func is None: - cost_func = mse_cost_func + cost_func = "mse" permutations = [] permutated_y2 = [] @@ -130,12 +133,19 @@ def permutate_torch( # y1_ is (num_samples, num_classes_1)-shaped # y2_ is (num_samples, num_classes_2)-shaped with torch.no_grad(): - cost = torch.stack( - [ - cost_func(y2_, y1_[:, i : i + 1].expand(-1, num_classes_2)) - for i in range(num_classes_1) - ], - ) + if cost_func == "mse": + diff = y1_.unsqueeze(2) - y2_.unsqueeze(1) + cost = torch.mean(diff * diff, dim=0) # (C1, C2) + elif cost_func == "mae": + diff = y1_.unsqueeze(2) - y2_.unsqueeze(1) + cost = torch.mean(torch.abs(diff), dim=0) # (C1, C2) + else: + cost = torch.stack( + [ + cost_func(y2_, y1_[:, i : i + 1].expand(-1, num_classes_2)) + for i in range(num_classes_1) + ], + ) if num_classes_2 > num_classes_1: padded_cost = F.pad( @@ -167,7 +177,7 @@ def permutate_torch( def permutate_numpy( y1: np.ndarray, y2: np.ndarray, - cost_func: Optional[Callable] = None, + cost_func: Callable | Literal["mse", "mae"] = "mse", return_cost: bool = False, ) -> Tuple[np.ndarray, List[Tuple[int]]]: diff --git a/src/pyannote/audio/utils/signal.py b/src/pyannote/audio/utils/signal.py index c02bf01a..1b372ce2 100644 --- a/src/pyannote/audio/utils/signal.py +++ b/src/pyannote/audio/utils/signal.py @@ -38,7 +38,7 @@ import einops import numpy as np import scipy.signal from pyannote.core import Annotation, Segment, SlidingWindowFeature, Timeline -from pyannote.core.utils.generators import pairwise +from pyannote.core.utils.generators import pairwise, string_generator @singledispatch @@ -271,10 +271,11 @@ class Binarize: # annotation meant to store 'active' regions active = Annotation() + track_generator = string_generator() for k, k_scores in enumerate(scores.data.T): - label = k if scores.labels is None else scores.labels[k] + track = next(track_generator) # initial state start = timestamps[0] @@ -287,7 +288,7 @@ class Binarize: # switching from active to inactive if y < self.offset: region = Segment(start - self.pad_onset, t + self.pad_offset) - active[region, k] = label + active[region, track] = label start = t is_active = False @@ -301,7 +302,7 @@ class Binarize: # if active at the end, add final region if is_active: region = Segment(start - self.pad_onset, t + self.pad_offset) - active[region, k] = label + active[region, track] = label # because of padding, some active regions might be overlapping: merge them. # also: fill same speaker gaps shorter than min_duration_off diff --git a/tests/io_test.py b/tests/io_test.py index 4a582b79..3c542bef 100644 --- a/tests/io_test.py +++ b/tests/io_test.py @@ -1,5 +1,6 @@ import torch -import torchaudio +from torchcodec.decoders import AudioDecoder, AudioStreamMetadata + from pyannote.core import Segment from torch import Tensor @@ -9,8 +10,8 @@ from pyannote.audio.core.io import Audio def test_audio_resample(): "Audio is correctly resampled when it isn't the correct sample rate" test_file = "tests/data/dev00.wav" - info = torchaudio.info(test_file) - old_sr = info.sample_rate + metadata: AudioStreamMetadata = AudioDecoder(test_file).metadata + old_sr = metadata.sample_rate loader = Audio(sample_rate=old_sr // 2, mono="downmix") wav, sr = loader(test_file) assert isinstance(wav, Tensor) diff --git a/tests/test_speechbrain.py b/tests/test_speechbrain.py deleted file mode 100644 index 0ac9b60c..00000000 --- a/tests/test_speechbrain.py +++ /dev/null @@ -1,20 +0,0 @@ -import tempfile -import pytest -from speechbrain.inference import EncoderClassifier - - -@pytest.fixture() -def cache(): - return tempfile.mkdtemp() - -def test_import_speechbrain_encoder_classifier(cache): - """This is a simple test that check if speechbrain - EncoderClassifier can be imported. It does not check - if the model is working properly. - """ - - model = EncoderClassifier.from_hparams( - source="speechbrain/spkrec-ecapa-voxceleb", - savedir=cache, - ) - assert isinstance(model, EncoderClassifier)