From 29c883fad6dba6957e158c569d38876d4de7a232 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20BREDIN?= Date: Tue, 12 Aug 2025 17:39:10 +0200 Subject: [PATCH] PYA-849 feat: add support for VBx clustering (#1894) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Petr Pálka <82486052+Selesnyan@users.noreply.github.com> --- CHANGELOG.md | 7 +- README.md | 46 ++-- src/pyannote/audio/core/plda.py | 120 ++++++++++ src/pyannote/audio/pipelines/clustering.py | 151 ++++++++++-- .../audio/pipelines/speaker_diarization.py | 29 ++- .../audio/pipelines/utils/__init__.py | 4 + src/pyannote/audio/pipelines/utils/getter.py | 80 ++++++- src/pyannote/audio/utils/hf_hub.py | 4 +- src/pyannote/audio/utils/vbx.py | 218 ++++++++++++++++++ 9 files changed, 597 insertions(+), 62 deletions(-) create mode 100644 src/pyannote/audio/core/plda.py create mode 100644 src/pyannote/audio/utils/vbx.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 30d7dc7b..ba42f246 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,13 @@ # CHANGELOG -## develop +## develop ### TL;DR +#### Better speaker assignment + +`pyannote/speaker-diarization-4.0` pretrained pipeline relies on VBx clustering instead of agglomerative hierarchical clustering (as suggested by [BUT Speech@FIT](https://speech.fit.vut.cz/) researchers [Petr Pálka](https://github.com/Selesnyan) and [Jiangyu Han](https://github.com/jyhan03)). + #### Faster training Metadata caching and optimized dataloaders make training on large scale datasets much faster. @@ -45,6 +49,7 @@ Models can now be stored alongside their pipelines in the same repository, strea ### New features +- feat(pipeline): add support for VBx clustering ([@Selesnyan](https://github.com/Selesnyan) and [jyhan03](https://github.com/jyhan03)) - feat(pyannoteAI): add wrapper around pyannoteAI SDK - improve(hub): add support for pipeline repos that also include underlying models - feat(clustering): add support for `k-means` clustering diff --git a/README.md b/README.md index 6ace8319..43b7f8ae 100644 --- a/README.md +++ b/README.md @@ -21,9 +21,8 @@ Consider switching to [pyannoteAI](https://www.pyannote.ai) for better and faste ## Open-source speaker diarization pipeline 1. Install [`pyannote.audio`](https://github.com/pyannote/pyannote-audio) with `pip install pyannote.audio` -2. Accept [`pyannote/segmentation-3.0`](https://hf.co/pyannote/segmentation-3.0) user conditions -3. Accept [`pyannote/speaker-diarization-3.1`](https://hf.co/pyannote/speaker-diarization-3.1) user conditions -4. Create Huggingface access token at [`hf.co/settings/tokens`](https://hf.co/settings/tokens). +2. Accept [`pyannote/speaker-diarization-4.0`](https://hf.co/pyannote/speaker-diarization-4.0) user conditions +3. Create Huggingface access token at [`hf.co/settings/tokens`](https://hf.co/settings/tokens). ```python import torch @@ -32,7 +31,7 @@ from pyannote.audio.pipelines.utils.hook import ProgressHook # Open-source pyannote speaker diarization pipeline pipeline = Pipeline.from_pretrained( - "pyannote/speaker-diarization-3.1", + "pyannote/speaker-diarization-4.0", token="HUGGINGFACE_ACCESS_TOKEN") # send pipeline to GPU (when available) @@ -54,8 +53,8 @@ for turn, _, speaker in diarization.itertracks(yield_label=True): ## Premium pyannoteAI speaker diarization pipeline -1. Install [`pyannote.audio`](https://github.com/pyannote/pyannote-audio) with `pip install pyannote.audio` -2. Create pyannoteAI API key at [`dashboard.pyannote.ai`](https://dashboard.pyannote.ai) +1. Create pyannoteAI API key at [`dashboard.pyannote.ai`](https://dashboard.pyannote.ai) +2. Enjoy 150 hours of free credits! ```python from pyannote.audio import Pipeline @@ -78,27 +77,26 @@ for turn, _, speaker in diarization.itertracks(yield_label=True): Visit [`docs.pyannote.ai`](https://docs.pyannote.ai) to learn about other pyannoteAI features (voiceprinting, confidence scores, ...) ## Benchmark +Out of the box, `pyannote.audio` speaker diarization [pipeline v4.0](https://hf.co/pyannote/speaker-diarization-4.0) is expected to be much better than v3.1. -Out of the box, `pyannote.audio` speaker diarization [pipeline v3.1](https://hf.co/pyannote/speaker-diarization-3.1) is expected to be much better (and faster) than v2.x. [`pyannoteAI`](https://www.pyannote.ai) premium model goes one step further. Those numbers are diarization error rates (in %) - the lower the better. + `pyannoteAI` premium model are even better (and also faster). -| Benchmark (2025-03) | [v2.1](https://hf.co/pyannote/speaker-diarization-2.1) | [v3.1](https://hf.co/pyannote/speaker-diarization-3.1) | | -| --------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------ | ------------------------------------------------------ | ------------------------------------------------ | -| [AISHELL-4](https://arxiv.org/abs/2104.03603) | 14.1 | 12.2 | 11.9 | -| [AliMeeting](https://www.openslr.org/119/) (channel 1) | 27.4 | 24.5 | 16.6 | -| [AMI](https://groups.inf.ed.ac.uk/ami/corpus/) (IHM) | 18.9 | 18.8 | 13.2 | -| [AMI](https://groups.inf.ed.ac.uk/ami/corpus/) (SDM) | 27.1 | 22.7 | 15.8 | -| [AVA-AVD](https://arxiv.org/abs/2111.14448) | 66.3 | 49.7 | 39.9 | -| [CALLHOME](https://catalog.ldc.upenn.edu/LDC2001S97) ([part 2](https://github.com/BUTSpeechFIT/CALLHOME_sublists/issues/1)) | 31.6 | 28.4 | 17.8 | -| [DIHARD 3](https://catalog.ldc.upenn.edu/LDC2022S14) ([full](https://arxiv.org/abs/2012.01477)) | 26.9 | 21.4 | 15.7 | -| [Earnings21](https://github.com/revdotcom/speech-datasets) | 17.0 | 9.4 | 9.1 | -| [Ego4D](https://arxiv.org/abs/2110.07058) (dev.) | 61.5 | 51.2 | 42.8 | -| [MSDWild](https://github.com/X-LANCE/MSDWILD) | 32.8 | 25.4 | 17.8 | -| [RAMC](https://www.openslr.org/123/) | 22.5 | 22.2 | 10.6 | -| [REPERE](https://www.islrn.org/resources/360-758-359-485-0/) (phase2) | 8.2 | 7.9 | 7.3 | -| [VoxConverse](https://github.com/joonson/voxconverse) (v0.3) | 11.2 | 11.2 | 8.9 | - -[Diarization error rate](http://pyannote.github.io/pyannote-metrics/reference.html#diarization) (in %) +| Benchmark (2025-08) |
v3.1
|
v4.0
|
API
|
labs
| +| --------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------ | -------------------------------------------------| ------------------------------------------------ | --- | +| [AISHELL-4](https://arxiv.org/abs/2104.03603) | 12.2 | 11.6 | 11.9 | 11.4 | +| [AliMeeting](https://www.openslr.org/119/) (channel 1) | 24.5 | 20.7 | 16.6 | 15.2 | +| [AMI](https://groups.inf.ed.ac.uk/ami/corpus/) (IHM) | 18.8 | 17.4 | 13.2 | 12.9 | +| [AMI](https://groups.inf.ed.ac.uk/ami/corpus/) (SDM) | 22.7 | 19.8 | 15.8 | 15.6 | +| [AVA-AVD](https://arxiv.org/abs/2111.14448) | 49.7 | 45.3 | 39.9 | 37.1 | +| [CALLHOME](https://catalog.ldc.upenn.edu/LDC2001S97) ([part 2](https://github.com/BUTSpeechFIT/CALLHOME_sublists/issues/1)) | 28.4 | 27.5 | 17.8 | 16.6 | +| [DIHARD 3](https://catalog.ldc.upenn.edu/LDC2022S14) ([full](https://arxiv.org/abs/2012.01477)) | 21.4 | 20.5 | 15.7 | 14.7 | +| [Ego4D](https://arxiv.org/abs/2110.07058) (dev.) | 51.2 | 46.5 | 42.8 | 39.0 | +| [MSDWild](https://github.com/X-LANCE/MSDWILD) | 25.4 | 23.0 | 17.8 | 17.3 | +| [RAMC](https://www.openslr.org/123/) | 22.2 | 20.9 | 10.6 | 10.5 | +| [REPERE](https://www.islrn.org/resources/360-758-359-485-0/) (phase2) | 7.9 | 8.9 | 7.3 | 7.4 | +| [VoxConverse](https://github.com/joonson/voxconverse) (v0.3) | 11.2 | 11.2 | 8.9 | 8.5 | +[Diarization error rate](http://pyannote.github.io/pyannote-metrics/reference.html#diarization) (in %, the lower, the better) ## Documentation diff --git a/src/pyannote/audio/core/plda.py b/src/pyannote/audio/core/plda.py new file mode 100644 index 00000000..24d2b93d --- /dev/null +++ b/src/pyannote/audio/core/plda.py @@ -0,0 +1,120 @@ +# MIT License +# +# Copyright (c) 2024- CNRS +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import os +from pathlib import Path +from typing import Optional + +import numpy as np +from pyannote.audio.utils.hf_hub import download_from_hf_hub +from pyannote.audio.utils.vbx import vbx_setup + + +class PLDA: + """PLDA""" + + def __init__( + self, transform_npz: str | Path, plda_npz: str | Path, lda_dimension: int = 128 + ): + self._xvec_tf, self._plda_tf, self._plda_psi = vbx_setup( + transform_npz, plda_npz + ) + + self.lda_dimension = lda_dimension + + @property + def phi(self): + """Between-class covariance in the PLDA space.""" + return self._plda_psi[: self.lda_dimension] + + def __call__(self, embeddings: np.ndarray): + """ + + Parameters + ---------- + embeddings : (num_embeddings, embedding_dimension) ndarray + Embeddings to be transformed into the PLDA space. + + Returns + ------- + fea : (num_embeddings, lda_dimension) ndarray + Embeddings transformed into the PLDA space. + """ + return self._plda_tf(self._xvec_tf(embeddings), lda_dim=self.lda_dimension) + + @classmethod + def from_pretrained( + cls, + checkpoint: Path | str, + subfolder: str | None = None, + token: str | None = None, + cache_dir: str | Path | None = None, + **kwargs, + ) -> Optional["PLDA"]: + """Load PLDA from disk or Huggingface Hub + + Parameters + ---------- + checkpoint : Path or str + Path to checkpoint or a model identifier from the hf.co model hub. + subfolder : str, optional + Folder inside the hf.co model repo. + token : str, optional + When loading a private hf.co model, set `token` + to True or to a string containing your hugginface.co authentication + token that can be obtained by running `huggingface-cli login` + cache_dir: Path or str, optional + Path to model cache directory. + """ + + # if checkpoint is a directory, look for the PLDA checkpoint + # inside this directory (or inside a subfolder if specified) + if os.path.isdir(checkpoint): + if subfolder: + path_to_transform = Path(checkpoint) / subfolder / "xvec_transform.npz" + path_to_plda = Path(checkpoint) / subfolder / "plda.npz" + else: + path_to_transform = Path(checkpoint) / "xvec_transform.npz" + path_to_plda = Path(checkpoint) / "plda.npz" + + # otherwise, assume that the checkpoint is hosted on HF model hub + else: + _, _, path_to_transform = download_from_hf_hub( + str(checkpoint), + "xvec_transform.npz", + subfolder=subfolder, + cache_dir=cache_dir, + token=token, + ) + + _, _, path_to_plda = download_from_hf_hub( + str(checkpoint), + "plda.npz", + subfolder=subfolder, + cache_dir=cache_dir, + token=token, + ) + + if path_to_transform is None or path_to_plda is None: + return None + + return cls(path_to_transform, path_to_plda) diff --git a/src/pyannote/audio/pipelines/clustering.py b/src/pyannote/audio/pipelines/clustering.py index 18b3b527..c6a6a8f3 100644 --- a/src/pyannote/audio/pipelines/clustering.py +++ b/src/pyannote/audio/pipelines/clustering.py @@ -22,12 +22,16 @@ """Clustering pipelines""" -import random from enum import Enum from typing import Optional, Tuple import numpy as np from einops import rearrange +from pyannote.audio.core.io import AudioFile +from pyannote.audio.core.plda import PLDA +from pyannote.audio.pipelines.utils import oracle_segmentation +from pyannote.audio.utils.permutation import permutate +from pyannote.audio.utils.vbx import cluster_vbx from pyannote.core import SlidingWindow, SlidingWindowFeature from pyannote.pipeline import Pipeline from pyannote.pipeline.parameter import Categorical, Integer, Uniform @@ -36,21 +40,15 @@ from scipy.optimize import linear_sum_assignment from scipy.spatial.distance import cdist from sklearn.cluster import KMeans -from pyannote.audio.core.io import AudioFile -from pyannote.audio.pipelines.utils import oracle_segmentation -from pyannote.audio.utils.permutation import permutate - class BaseClustering(Pipeline): def __init__( self, metric: str = "cosine", - max_num_embeddings: int = 1000, constrained_assignment: bool = False, ): super().__init__() self.metric = metric - self.max_num_embeddings = max_num_embeddings self.constrained_assignment = constrained_assignment def set_num_clusters( @@ -80,8 +78,13 @@ class BaseClustering(Pipeline): self, embeddings: np.ndarray, segmentations: Optional[SlidingWindowFeature] = None, + min_active_ratio: float = 0.2, ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: - """Filter NaN embeddings and downsample embeddings + """Filter embeddings before clustering + + Embeddings that are removed: + * NaN embeddings + * embeddings speaking less than `min_active_ratio` times the chunk duration Parameters ---------- @@ -89,6 +92,9 @@ class BaseClustering(Pipeline): Sequence of embeddings. segmentations : (num_chunks, num_frames, num_speakers) array Binary segmentations. + min_active_ratio : float, optional + Minimum active ratio for a speaker to be considered active + during clustering. Returns ------- @@ -97,26 +103,29 @@ class BaseClustering(Pipeline): speaker_idx : (num_embeddings, ) array """ - # whether speaker is active - active = np.sum(segmentations.data, axis=1) > 0 + _, num_frames, _ = segmentations.data.shape + + # frames where only one speaker is active + single_active_mask = (np.sum(segmentations.data, axis=2, keepdims=True) == 1) + + # count number of clean frames per chunk and speaker + num_clean_frames = np.sum(segmentations.data * single_active_mask, axis=1) + # num_chunks, num_speakers + + # whether speaker is active enough on their own + active = num_clean_frames >= min_active_ratio * num_frames + # num_chunks, num_speakers + # whether speaker embedding extraction went fine valid = ~np.any(np.isnan(embeddings), axis=2) - # indices of embeddings that are both active and valid + # indices of embeddings that are both active enough and valid chunk_idx, speaker_idx = np.where(active * valid) - # sample max_num_embeddings embeddings - num_embeddings = len(chunk_idx) - if num_embeddings > self.max_num_embeddings: - indices = list(range(num_embeddings)) - random.shuffle(indices) - indices = sorted(indices[: self.max_num_embeddings]) - chunk_idx = chunk_idx[indices] - speaker_idx = speaker_idx[indices] - return embeddings[chunk_idx, speaker_idx], chunk_idx, speaker_idx def constrained_argmax(self, soft_clusters: np.ndarray) -> np.ndarray: + soft_clusters = np.nan_to_num(soft_clusters, nan=np.nanmin(soft_clusters)) num_chunks, num_speakers, num_clusters = soft_clusters.shape # num_chunks, num_speakers, num_clusters @@ -303,12 +312,10 @@ class AgglomerativeClustering(BaseClustering): def __init__( self, metric: str = "cosine", - max_num_embeddings: int = np.inf, constrained_assignment: bool = False, ): super().__init__( metric=metric, - max_num_embeddings=max_num_embeddings, constrained_assignment=constrained_assignment, ) @@ -540,6 +547,105 @@ class KMeansClustering(BaseClustering): ).fit_predict(embeddings) +class VBxClustering(BaseClustering): + + expects_num_clusters: bool = False + + + def __init__( + self, + plda: PLDA, + metric: str = "cosine", + constrained_assignment: bool = True, + ): + super().__init__( + metric=metric, + constrained_assignment=constrained_assignment, + ) + + self.plda = plda + + self.threshold = Uniform(0.5, 0.8) # assume unit-normalized embeddings + self.Fa = Uniform(0.01, 0.5) + self.Fb = Uniform(0.01, 15.0) + + def __call__( + self, + embeddings: np.ndarray, + segmentations: Optional[SlidingWindowFeature] = None, + num_clusters: Optional[int] = None, # not used but kept for compatibility + min_clusters: Optional[int] = None, # not used but kept for compatibility + max_clusters: Optional[int] = None, # not used but kept for compatibility + **kwargs, + ) -> np.ndarray: + train_embeddings, _, _ = self.filter_embeddings( + embeddings, segmentations=segmentations + ) + + if train_embeddings.shape[0] < 2: + # do NOT apply clustering when the number of training embeddings is less than 2 + num_chunks, num_speakers, _ = embeddings.shape + hard_clusters = np.zeros((num_chunks, num_speakers), dtype=np.int8) + soft_clusters = np.ones((num_chunks, num_speakers, 1)) + centroids = np.mean(train_embeddings, axis=0, keepdims=True) + return hard_clusters, soft_clusters, centroids + + # AHC + train_embeddings_normed = train_embeddings / np.linalg.norm( + train_embeddings, axis=1, keepdims=True + ) + dendrogram = linkage( + train_embeddings_normed, method="centroid", metric="euclidean" + ) + ahc_clusters = fcluster(dendrogram, self.threshold, criterion="distance") - 1 + _, ahc_clusters = np.unique(ahc_clusters, return_inverse=True) + + # VBx + + fea = self.plda(train_embeddings) + q, sp = cluster_vbx( + ahc_clusters, + fea, + self.plda.phi, + Fa=self.Fa, + Fb=self.Fb, + maxIters=20, + ) + + # calculate distance + num_chunks, num_speakers, dimension = embeddings.shape + W = q[:, sp > 1e-7] # responsibilities of speakers that VBx kept + centroids = W.T @ train_embeddings.reshape(-1, dimension) / W.sum(0, keepdims=True).T + + e2k_distance = rearrange( + cdist( + rearrange(embeddings, "c s d -> (c s) d"), + centroids, + metric=self.metric, + ), + "(c s) k -> c s k", + c=num_chunks, + s=num_speakers, + ) + soft_clusters = 2 - e2k_distance + + # assign each embedding to the cluster with the most similar centroid + if self.constrained_assignment: + const = soft_clusters.min() - 1. # const < any_valid_score + soft_clusters[segmentations.data.sum(1) == 0] = const + hard_clusters = self.constrained_argmax( + soft_clusters, + ) + else: + hard_clusters = np.argmax(soft_clusters, axis=2) + + # re-number clusters from 0 to num_large_clusters + _, hard_clusters = np.unique(hard_clusters, return_inverse=True) + hard_clusters = hard_clusters.reshape(num_chunks, num_speakers) + + return hard_clusters, soft_clusters, centroids + + class OracleClustering(BaseClustering): """Oracle clustering""" @@ -630,4 +736,5 @@ class OracleClustering(BaseClustering): class Clustering(Enum): AgglomerativeClustering = AgglomerativeClustering KMeansClustering = KMeansClustering + VBxClustering = VBxClustering OracleClustering = OracleClustering diff --git a/src/pyannote/audio/pipelines/speaker_diarization.py b/src/pyannote/audio/pipelines/speaker_diarization.py index 98d23fb3..edcd17b2 100644 --- a/src/pyannote/audio/pipelines/speaker_diarization.py +++ b/src/pyannote/audio/pipelines/speaker_diarization.py @@ -1,6 +1,7 @@ # The MIT License (MIT) # -# Copyright (c) 2021- CNRS +# Copyright (c) 2021-2025 CNRS +# Copyright (c) 2025- pyannoteAI # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -43,8 +44,10 @@ from pyannote.audio.pipelines.clustering import Clustering from pyannote.audio.pipelines.speaker_verification import PretrainedSpeakerEmbedding from pyannote.audio.pipelines.utils import ( PipelineModel, + PipelinePLDA, SpeakerDiarizationMixin, get_model, + get_plda, ) from pyannote.audio.pipelines.utils.diarization import set_num_speakers from pyannote.audio.utils.signal import binarize @@ -116,11 +119,15 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline): def __init__( self, - segmentation: PipelineModel = "pyannote/segmentation@2022.07", + segmentation: PipelineModel = "pyannote/segmentation-3.0", segmentation_step: float = 0.1, - embedding: PipelineModel = "speechbrain/spkrec-ecapa-voxceleb@5c0be3875fda05e81f3c004ed8c7c06be308de1e", + embedding: PipelineModel = "pyannote/wespeaker-voxceleb-resnet34-LM", embedding_exclude_overlap: bool = False, - clustering: str = "AgglomerativeClustering", + plda: PipelinePLDA = { + "checkpoint": "BUT-FIT/diarizen-wavlm-large-s80-md", + "subfolder": "plda", + }, + clustering: str = "VBxClustering", embedding_batch_size: int = 1, segmentation_batch_size: int = 1, der_variant: Optional[dict] = None, @@ -138,6 +145,9 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline): self.embedding_batch_size = embedding_batch_size self.embedding_exclude_overlap = embedding_exclude_overlap + self.plda = plda + self._plda = get_plda(plda, token=token, cache_dir=cache_dir) + self.klustering = clustering self.der_variant = der_variant or {"collar": 0.0, "skip_overlap": False} @@ -178,7 +188,11 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline): raise ValueError( f'clustering must be one of [{", ".join(list(Clustering.__members__))}]' ) - self.clustering = Klustering.value(metric=metric) + + if self.klustering == "VBxClustering": + self.clustering = Klustering.value(self._plda, metric=metric) + else: + self.clustering = Klustering.value(metric=metric) self._expects_num_speakers = self.clustering.expects_num_clusters @@ -191,7 +205,10 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline): self._segmentation.batch_size = batch_size def default_parameters(self): - raise NotImplementedError() + return { + 'segmentation': {'min_duration_off': 0.0}, + 'clustering': {'threshold': 0.6, 'Fa': 0.07, 'Fb': 0.8} + } def classes(self): speaker = 0 diff --git a/src/pyannote/audio/pipelines/utils/__init__.py b/src/pyannote/audio/pipelines/utils/__init__.py index 5bfa186a..2fdfcdcc 100644 --- a/src/pyannote/audio/pipelines/utils/__init__.py +++ b/src/pyannote/audio/pipelines/utils/__init__.py @@ -25,10 +25,12 @@ from .getter import ( PipelineAugmentation, PipelineCalibration, PipelineModel, + PipelinePLDA, get_augmentation, get_calibration, get_devices, get_model, + get_plda, ) from .oracle import oracle_segmentation @@ -40,6 +42,8 @@ __all__ = [ "get_devices", "get_model", "PipelineModel", + "get_plda", + "PipelinePLDA", "get_calibration", "PipelineCalibration", ] diff --git a/src/pyannote/audio/pipelines/utils/getter.py b/src/pyannote/audio/pipelines/utils/getter.py index 08216fcb..ed92fe0c 100644 --- a/src/pyannote/audio/pipelines/utils/getter.py +++ b/src/pyannote/audio/pipelines/utils/getter.py @@ -25,11 +25,11 @@ from pathlib import Path from typing import Dict, Mapping, Optional, Text, Union import torch -from torch_audiomentations.core.transforms_interface import BaseWaveformTransform -from torch_audiomentations.utils.config import from_dict as augmentation_from_dict - from pyannote.audio.core.calibration import Calibration from pyannote.audio.core.model import Model +from pyannote.audio.core.plda import PLDA +from torch_audiomentations.core.transforms_interface import BaseWaveformTransform +from torch_audiomentations.utils.config import from_dict as augmentation_from_dict PipelineModel = Union[Model, Text, Mapping] @@ -54,7 +54,7 @@ def get_model( cache_dir: Path or str, optional Path to the folder where files downloaded from Huggingface hub are stored. skip_dependencies : bool, optional - If True, skip dependency check. Defaults to False. + If True, skip dependency check. Defaults to False. Use at your own risk, as this may lead to unexpected behavior. Returns @@ -80,7 +80,11 @@ def get_model( elif isinstance(model, Text): _model = Model.from_pretrained( - model, token=token, cache_dir=cache_dir, strict=False, skip_dependencies=skip_dependencies + model, + token=token, + cache_dir=cache_dir, + strict=False, + skip_dependencies=skip_dependencies, ) if _model: model = _model @@ -111,7 +115,7 @@ def get_calibration( calibration: PipelineCalibration, token: Union[Text, None] = None, cache_dir: Union[Path, Text, None] = None, - skip_dependencies: bool = False, + skip_dependencies: bool = False, ) -> Optional[Calibration]: """Load pretrained calibration @@ -145,7 +149,10 @@ def get_calibration( elif isinstance(calibration, Text): loaded_calibration = Calibration.from_pretrained( - calibration, token=token, cache_dir=cache_dir, skip_dependencies=skip_dependencies + calibration, + token=token, + cache_dir=cache_dir, + skip_dependencies=skip_dependencies, ) elif isinstance(calibration, Dict): @@ -163,6 +170,65 @@ def get_calibration( return loaded_calibration +PipelinePLDA = Union[PLDA, Text, Dict] + + +def get_plda( + plda: PipelinePLDA, + token: Union[Text, None] = None, + cache_dir: Union[Path, Text, None] = None, + skip_dependencies: bool = False, +) -> Optional[PLDA]: + """Load pretrained calibration + + Parameters + ---------- + plda : PLDA, str, or dict + When `PLDA`, returns `plda` as is. + When `str`, assumes that this is either the path to a checkpoint or the name of a + pretrained PLDA on Huggingface.co and loads with `PLDA.from_pretrained(PLDA)`. + When `dict`, loads with `PLDA.from_pretrained(**plda)`. + token : str or bool, optional + Huggingface token to be used for downloading from Huggingface hub. + cache_dir: Path or str, optional + Path to the folder where files downloaded from Huggingface hub are stored. + skip_dependencies : bool, optional + If True, skip dependency check. Defaults to False. + Use at your own risk, as this may lead to unexpected behavior. + + Returns + ------- + plda : PLDA + PLDA. + + See also + -------- + pyannote.audio.core.plda.PLDA.from_pretrained + """ + + if isinstance(plda, PLDA): + loaded_plda = plda + + elif isinstance(plda, Text): + loaded_plda = PLDA.from_pretrained( + plda, token=token, cache_dir=cache_dir, skip_dependencies=skip_dependencies + ) + + elif isinstance(plda, Dict): + plda.setdefault("token", token) + plda.setdefault("cache_dir", cache_dir) + plda.setdefault("skip_dependencies", skip_dependencies) + loaded_plda = PLDA.from_pretrained(**plda) + + else: + raise TypeError( + f"Unsupported type ({type(plda)}) for loading PLDA: " + f"expected `str` or `dict`." + ) + + return loaded_plda + + def get_augmentation(augmentation: PipelineAugmentation) -> BaseWaveformTransform: """Load augmentation diff --git a/src/pyannote/audio/utils/hf_hub.py b/src/pyannote/audio/utils/hf_hub.py index c0da9d14..1a5ceb41 100644 --- a/src/pyannote/audio/utils/hf_hub.py +++ b/src/pyannote/audio/utils/hf_hub.py @@ -40,7 +40,7 @@ class AssetFileName(Enum): def download_from_hf_hub( checkpoint: str, - asset_file: AssetFileName, + asset_file: AssetFileName | str, subfolder: Optional[str] = None, cache_dir: Optional[Union[str, Path]] = None, token: Union[bool, str, None] = None, @@ -81,7 +81,7 @@ def download_from_hf_hub( revision, hf_hub_download( model_id, - asset_file.value, + asset_file.value if isinstance(asset_file, AssetFileName) else asset_file, subfolder=subfolder, repo_type="model", revision=revision, diff --git a/src/pyannote/audio/utils/vbx.py b/src/pyannote/audio/utils/vbx.py new file mode 100644 index 00000000..01c0ea05 --- /dev/null +++ b/src/pyannote/audio/utils/vbx.py @@ -0,0 +1,218 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Revision History +# L. Burget 20/1/2021 1:00AM - original version derived from the more +# P. Pálka 28/4/2025 17:00PM - original version derived from the more +# complex VB_diarization.py avaiable at +# https://github.com/BUTSpeechFIT/VBx/blob/e39af548bb41143a7136d08310765746192e34da/VBx/VB_diarization.py +# H. Bredin 12/8/2025 2:00PM - update vbx_setup signature to expect two paths +# drop HMM support +# format with ruff + +import numpy as np +from scipy.linalg import eigh +from scipy.special import logsumexp, softmax + + +def VBx( + X, + Phi, + Fa=1.0, + Fb=1.0, + pi=10, + gamma=None, + maxIters=10, + epsilon=1e-4, + alphaQInit=1.0, + ref=None, + plot=False, + return_model=False, + alpha=None, + invL=None, +): + """ + Inputs: + X - T x D array, where columns are D dimensional feature vectors + (e.g. x-vectors) for T frames + Phi - D array with across-class covariance matrix diagonal. + The model assumes zero mean, diagonal across-class and + identity within-class covariance matrix. + Fa - Scale sufficient statistics + Fb - Speaker regularization coefficient Fb controls the final number of speakers + pi - If integer value, it sets the maximum number of speakers + that can be found in the utterance. + If vector, it is the initialization for speaker priors (see Outputs: pi) + gamma - An initialization for the matrix of responsibilities (see Outputs: gamma) + maxIters - The maximum number of VB iterations + epsilon - Stop iterating, if the obj. fun. improvement is less than epsilon + alphaQInit - Dirichlet concentraion parameter for initializing gamma + ref - T dim. integer vector with per frame reference speaker IDs (0:maxSpeakers) + plot - If set to True, plot per-frame marginal speaker posteriors 'gamma' + return_model- Return also speaker model parameter + alpha, invL - If provided, these are speaker model parameters used in the first iteration + + Outputs: + gamma - S x T matrix of responsibilities (marginal posteriors) + attributing each frame to one of S possible speakers + (S is defined by input parameter pi) + pi - S dimensional column vector of ML learned speaker priors. + This allows us to estimate the number of speaker in the + utterance as the probabilities of the redundant speaker + converge to zero. + Li - Values of auxiliary function (and DER and frame cross-entropy + between gamma and reference, if 'ref' is provided) over iterations. + alpha, invL - Speaker model parameters returned only if return_model=True + + Reference: + Landini F., Profant J., Diez M., Burget L.: Bayesian HMM clustering of + x-vector sequences (VBx) in speaker diarization: theory, implementation + and analysis on standard tasks + """ + """ + The comments in the code refers to the equations from the paper above. Also + the names of variables try to be consistent with the symbols in the paper. + """ + D = X.shape[1] # feature (e.g. x-vector) dimensionality + + if type(pi) is int: + pi = np.ones(pi) / pi + + if gamma is None: + # initialize gamma from flat Dirichlet prior with + # concentration parameter alphaQInit + gamma = np.random.gamma(alphaQInit, size=(X.shape[0], len(pi))) + gamma = gamma / gamma.sum(1, keepdims=True) + + assert gamma.shape[1] == len(pi) and gamma.shape[0] == X.shape[0] + + G = -0.5 * ( + np.sum(X**2, axis=1, keepdims=True) + D * np.log(2 * np.pi) + ) # per-frame constant term in (23) + V = np.sqrt(Phi) # between (5) and (6) + rho = X * V # (18) + Li = [] + ELBO = None ## + for ii in range(maxIters): + # Do not start with estimating speaker models if those are provided + # in the argument + if ii > 0 or alpha is None or invL is None: + invL = 1.0 / ( + 1 + Fa / Fb * gamma.sum(axis=0, keepdims=True).T * Phi + ) # (17) for all speakers + alpha = Fa / Fb * invL * gamma.T.dot(rho) # (16) for all speakers + log_p_ = Fa * ( + rho.dot(alpha.T) - 0.5 * (invL + alpha**2).dot(Phi) + G + ) # (23) for all speakers + + # use GMM update + eps = 1e-8 + lpi = np.log(pi + eps) + log_p_x = logsumexp(log_p_ + lpi, axis=-1) # marginal LLH of each data point + log_pX_ = np.sum( + log_p_x, axis=0 + ) # total LLH over all data points (to monitor ELBO) + + gamma = np.exp(log_p_ + lpi - log_p_x[:, None]) # responsibilities + pi = np.sum(gamma, axis=0) + + pi = pi / pi.sum() + + ELBO = log_pX_ + Fb * 0.5 * np.sum(np.log(invL) - invL - alpha**2 + 1) # (25) + Li.append([ELBO]) + + if ii > 0 and ELBO - Li[-2][0] < epsilon: + if ELBO - Li[-2][0] < 0: + print("WARNING: Value of auxiliary function has decreased!") + break + return (gamma, pi, Li) + ((alpha, invL) if return_model else ()) + + +def cluster_vbx(ahc_init, fea, Phi, Fa, Fb, maxIters=20, init_smoothing=7.0): + """ahc_init (T x N_clusters)""" + qinit = np.zeros((len(ahc_init), ahc_init.max() + 1)) + qinit[range(len(ahc_init)), ahc_init.astype(int)] = 1.0 + qinit = qinit if init_smoothing < 0 else softmax(qinit * init_smoothing, axis=1) + gamma, pi, _, _, _ = VBx( + fea, + Phi, + Fa=Fa, + Fb=Fb, + pi=qinit.shape[1], + gamma=qinit, + maxIters=maxIters, + return_model=True, + ) + return gamma, pi + + +def l2_norm(vec_or_matrix): + """L2 normalization of vector array. + + Args: + vec_or_matrix (np.array): one vector or array of vectors + + Returns: + np.array: normalized vector or array of normalized vectors + """ + if len(vec_or_matrix.shape) == 1: + # linear vector + return vec_or_matrix / np.linalg.norm(vec_or_matrix) + elif len(vec_or_matrix.shape) == 2: + return ( + vec_or_matrix / np.linalg.norm(vec_or_matrix, axis=1, ord=2)[:, np.newaxis] + ) + else: + raise ValueError( + "Wrong number of dimensions, 1 or 2 is supported, not %i." + % len(vec_or_matrix.shape) + ) + + +def vbx_setup(transform_npz, plda_npz): + """ + Loads the transformation pipeline for x-vectors into the PLDA space for VBx. + + Args: + transform_npz (str | Path): Path to the x-vector transformation file. + plda_npz (str | Path): Path to the PLDA file. + + Returns: + xvec_tf (function): Transformation function to preprocess x-vectors (centering, whitening, LDA). + plda_tf (function): Transformation function to map x-vectors into the PLDA latent space. + plda_psi (np.ndarray): Eigenvalues of the between-class covariance in the PLDA space (reordered). + """ + + x = np.load(transform_npz) + mean1, mean2, lda = x["mean1"], x["mean2"], x["lda"] + + p = np.load(plda_npz) + plda_mu, plda_tr, plda_psi = p["mu"], p["tr"], p["psi"] + + # within-class, between-class matrices (W, B) + W = np.linalg.inv(plda_tr.T.dot(plda_tr)) + B = np.linalg.inv((plda_tr.T / plda_psi).dot(plda_tr)) + + # Solve the generalized eigenvalue problem for whitening and sort eigenvalues + acvar, wccn = eigh(B, W) + plda_psi = acvar[::-1] + plda_tr = wccn.T[::-1] + + # tf for preprocessing x-vectors (centering, whitening, LDA) + xvec_tf = lambda x: np.sqrt(lda.shape[1]) * l2_norm( + lda.T.dot(np.sqrt(lda.shape[0]) * l2_norm(x - mean1).T).T - mean2 + ) + # tf to map x-vectors to the PLDA latent space (center, apply transform, optional truncate to 'lda_dim') + plda_tf = lambda x0, lda_dim=lda.shape[1]: (x0 - plda_mu).dot(plda_tr.T)[ + :, :lda_dim + ] + return xvec_tf, plda_tf, plda_psi