BREAKING: remove exclusive and return_embeddings flag, add legacy flag (#1917)

This commit is contained in:
Hervé BREDIN authored and GitHub committed 2025-09-13 21:57:32 +02:00
1 parent 1fcd256e4c
commit 5859a71a50
3 files changed
+185 -77

No files matched your search

+39 -9
View File
@@ -4,10 +4,24 @@
### TL;DR
#### Better speaker assignment
#### Improved speaker assignment and counting
`pyannote/speaker-diarization-4.0` pretrained pipeline relies on VBx clustering instead of agglomerative hierarchical clustering (as suggested by [BUT Speech@FIT](https://speech.fit.vut.cz/) researchers [Petr Pálka](https://github.com/Selesnyan) and [Jiangyu Han](https://github.com/jyhan03)).
#### *Exclusive* speaker diarization
`pyannote/speaker-diarization-4.0` pretrained pipeline returns a new *exclusive* speaker diarization, on top of the regular speaker diarization.
This is a feature which is [backported from our latest commercial model](https://www.pyannote.ai/blog/precision-2) that simplifies the reconciliation between fine-grained speaker diarization timestamps and (sometimes not so precise) transcription timestamps.
```python
from pyannote.audio import Pipeline
pipeline = Pipeline.from_pretrained(
"pyannote/speaker-diarization-4.0", token="huggingface-access-token")
output = pipeline("/path/to/conversation.wav")
print(output.speaker_diarization) # regular speaker diarization
print(output.exclusive_speaker_diarization) # exclusive speaker diarization
```
#### Faster training
Metadata caching and optimized dataloaders make training on large scale datasets much faster.
@@ -20,18 +34,34 @@ Change one line of code to use [pyannoteAI](https://docs.pyannote.ai) premium mo
```diff
from pyannote.audio import Pipeline
pipeline = Pipeline.from_pretrained(
- "pyannote/speaker-diarization-3.1", token="huggingface-access-token")
+ "pyannoteAI/speaker-diarization-precision, token="pyannoteAI-api-key")
- "pyannote/speaker-diarization-4.0", token="huggingface-access-token")
+ "pyannote/speaker-diarization-precision-2, token="pyannoteAI-api-key")
diarization = pipeline("/path/to/conversation.wav")
```
#### Quality-of-Life improvements
#### Offline (air-gapped) use
Pipelines can now be stored alongside their internal models in the same repository, streamlining fully offline use.
1. Accept `pyannote/speaker-diarization-4.0` pipeline [user agreement](https://hf.co/pyannote/speaker-diarization-4.0)
2. Clone the pipeline repository from Huggingface (if prompted for a password, use a Huggingface access token with correct permissions)
```bash
$ git lfs install
$ git clone https://hf.co/pyannote/speaker-diarization-4.0 /path/to/directory/pyannote-speaker-diarization-4.0
```
3. Enjoy!
```python
# load pipeline from disk (works without internet connection)
from pyannote.audio import Pipeline
pipeline = Pipeline.from_pretrained('/path/to/directory/pyannote-speaker-diarization-4.0')
# run the pipeline locally on your computer
diarization = pipeline("audio.wav")
```
Models can now be stored alongside their pipelines in the same repository, streamlining offline use and gating mechanism:
- accept `pyannote/speaker-diarization-x.x` pipeline user agreement
- ~~accept `pyannote/segmentation-3.0` model user agreement~~
- ~~accept `pyannote/wespeaker-voxceleb-resnet34-LM` model user agreement~~
- load pipeline with `Pipeline.from_pretrained("pyannote/speaker-diarization-3.1", token=True)`
#### Telemetry
+3 -3
View File
@@ -39,10 +39,10 @@ pipeline.to(torch.device("cuda"))
# apply pretrained pipeline (with optional progress hook)
with ProgressHook() as hook:
diarization = pipeline("audio.wav", hook=hook) # runs locally
output = pipeline("audio.wav", hook=hook) # runs locally
# print the result
for turn, _, speaker in diarization.itertracks(yield_label=True):
for turn, _, speaker in output.speaker_diarization.itertracks(yield_label=True):
print(f"start={turn.start:.1f}s stop={turn.end:.1f}s speaker_{speaker}")
# start=0.2s stop=1.5s speaker_0
# start=1.8s stop=3.9s speaker_1
@@ -61,7 +61,7 @@ from pyannote.audio import Pipeline
# Premium pyannoteAI speaker diarization service
pipeline = Pipeline.from_pretrained(
"pyannoteAI/speaker-diarization-precision", token="PYANNOTEAI_API_KEY")
"pyannote/speaker-diarization-precision-2", token="PYANNOTEAI_API_KEY")
diarization = pipeline("audio.wav") # runs on pyannoteAI servers
@@ -29,7 +29,8 @@ import math
import textwrap
import warnings
from pathlib import Path
from typing import Callable, Mapping, Optional, Text, Union
from typing import Callable, Mapping, Optional, Text, Union, Any
from dataclasses import dataclass
import numpy as np
import torch
@@ -59,12 +60,75 @@ def batchify(iterable, batch_size: int = 32, fillvalue=None):
return itertools.zip_longest(*args, fillvalue=fillvalue)
@dataclass
class DiarizeOutput:
# speaker diarization
speaker_diarization: Annotation
# one speaker embedding per speaker
# as (num_speakers, dimension) array
# sorted in speaker_diarization.labels() order
speaker_embeddings: np.ndarray
# speaker diarization adapted to downstream transcription
# (does not contain overlapping speech turns)
exclusive_speaker_diarization: Annotation
def serialize(self) -> dict[str, Any]:
"""Serialize diarization output
Example
-------
{
'diarization': [{
'start': 6.665,
'end': 7.165,
'speaker': 'SPEAKER_00'},
...],
'exclusive_diarization': [{
'start': 6.665,
'end': 7.165,
'speaker': 'SPEAKER_00'},
...],
}
"""
diarization = []
for speech_turn, _, speaker in self.speaker_diarization.itertracks(
yield_label=True
):
diarization.append(
{
"start": round(speech_turn.start, 3),
"end": round(speech_turn.end, 3),
"speaker": speaker,
}
)
exclusive_diarization = []
for speech_turn, _, speaker in self.exclusive_speaker_diarization.itertracks(
yield_label=True
):
exclusive_diarization.append(
{
"start": round(speech_turn.start, 3),
"end": round(speech_turn.end, 3),
"speaker": speaker,
}
)
return {
"diarization": diarization,
"exclusive_diarization": exclusive_diarization,
}
class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
"""Speaker diarization pipeline
Parameters
----------
segmentation : Model, str, or dict, optional
segmentation : Model, str, or dict, optional
Pretrained segmentation model. Defaults to "pyannote/segmentation-3.0".
See pyannote.audio.pipelines.utils.get_model for supported format.
segmentation_step: float, optional
@@ -92,7 +156,10 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
Huggingface token to be used for downloading from Huggingface hub.
cache_dir: Path or str, optional
Path to the folder where files downloaded from Huggingface hub are stored.
legacy : bool, optional
Return only the diarization output. Defaults to return the full output
with diarization, exclusive diarization, and speaker embeddings.
Usage
-----
# perform (unconstrained) diarization
@@ -118,6 +185,7 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
def __init__(
self,
legacy: bool = False,
segmentation: PipelineModel = "pyannote/segmentation-3.0",
segmentation_step: float = 0.1,
embedding: PipelineModel = "pyannote/wespeaker-voxceleb-resnet34-LM",
@@ -135,6 +203,8 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
):
super().__init__()
self.legacy = legacy
self.segmentation_model = segmentation
model: Model = get_model(segmentation, token=token, cache_dir=cache_dir)
@@ -450,10 +520,8 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
num_speakers: Optional[int] = None,
min_speakers: Optional[int] = None,
max_speakers: Optional[int] = None,
exclusive: bool = False,
return_embeddings: bool = False,
hook: Optional[Callable] = None,
) -> Annotation:
) -> DiarizeOutput | Annotation:
"""Apply speaker diarization
Parameters
@@ -466,11 +534,6 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
Minimum number of speakers. Has no effect when `num_speakers` is provided.
max_speakers : int, optional
Maximum number of speakers. Has no effect when `num_speakers` is provided.
exclusive : bool, optional
Enforce exclusive diarization, i.e. only one speaker can be active at a time.
Defaults to False, i.e. overlapping speech is allowed.
return_embeddings : bool, optional
Return representative speaker embeddings.
hook : callable, optional
Callback called after each major steps of the pipeline as follows:
hook(step_name, # human-readable name of current step
@@ -482,12 +545,7 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
Returns
-------
diarization : Annotation
Speaker diarization
embeddings : np.array, optional
Representative speaker embeddings such that `embeddings[i]` is the
speaker embedding for i-th speaker in diarization.labels().
Only returned when `return_embeddings` is True.
output : DiarizeOutput (or Annotation if `self.legacy` is True)
"""
# setup hook (e.g. for debugging purposes)
@@ -537,49 +595,39 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
# shape: (num_frames, 1)
# dtype: int
if exclusive:
count.data = np.minimum(count.data, 1).astype(np.int8)
# exit early when no speaker is ever active
if np.nanmax(count.data) == 0.0:
diarization = Annotation(uri=file["uri"])
if return_embeddings:
return diarization, np.zeros((0, self._embedding.dimension))
return diarization
# skip speaker embedding extraction and clustering when only one speaker
if not return_embeddings and max_speakers < 2:
hard_clusters = np.zeros((num_chunks, local_num_speakers), dtype=np.int8)
embeddings = None
centroids = None
else:
# skip speaker embedding extraction with oracle clustering
if self.klustering == "OracleClustering" and not return_embeddings:
embeddings = None
else:
embeddings = self.get_embeddings(
file,
binarized_segmentations,
exclude_overlap=self.embedding_exclude_overlap,
hook=hook,
)
hook("embeddings", embeddings)
# shape: (num_chunks, local_num_speakers, dimension)
hard_clusters, _, centroids = self.clustering(
embeddings=embeddings,
segmentations=binarized_segmentations,
num_clusters=num_speakers,
min_clusters=min_speakers,
max_clusters=max_speakers,
file=file, # <== for oracle clustering
frames=self._segmentation.model.receptive_field, # <== for oracle clustering
output = DiarizeOutput(
speaker_diarization=Annotation(uri=file["uri"]),
exclusive_speaker_diarization=Annotation(uri=file["uri"]),
speaker_embeddings=np.zeros((0, self._embedding.dimension)),
)
# hard_clusters: (num_chunks, num_speakers)
# centroids: (num_speakers, dimension)
if self.legacy:
return output.speaker_diarization
return output
embeddings = self.get_embeddings(
file,
binarized_segmentations,
exclude_overlap=self.embedding_exclude_overlap,
hook=hook,
)
hook("embeddings", embeddings)
# shape: (num_chunks, local_num_speakers, dimension)
hard_clusters, _, centroids = self.clustering(
embeddings=embeddings,
segmentations=binarized_segmentations,
num_clusters=num_speakers,
min_clusters=min_speakers,
max_clusters=max_speakers,
file=file, # <== for oracle clustering
frames=self._segmentation.model.receptive_field, # <== for oracle clustering
)
# hard_clusters: (num_chunks, num_speakers)
# centroids: (num_speakers, dimension)
# number of detected clusters is the number of different speakers
num_different_speakers = np.max(hard_clusters) + 1
@@ -613,15 +661,16 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
inactive_speakers = np.sum(binarized_segmentations.data, axis=1) == 0
# shape: (num_chunks, num_speakers)
# force-assign them to throw-away cluster
hard_clusters[inactive_speakers] = -2
# convert to continuous diarization
discrete_diarization = self.reconstruct(
segmentations,
hard_clusters,
count,
)
hook("discrete_diarization", discrete_diarization)
# convert to continuous diarization
diarization = self.to_annotation(
discrete_diarization,
min_duration_on=0.0,
@@ -629,6 +678,20 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
)
diarization.uri = file["uri"]
# convert to continuous exclusive diarization
count.data = np.minimum(count.data, 1).astype(np.int8)
exclusive_discrete_diarization = self.reconstruct(
segmentations,
hard_clusters,
count,
)
exclusive_diarization = self.to_annotation(
exclusive_discrete_diarization,
min_duration_on=0.0,
min_duration_off=self.segmentation.min_duration_off,
)
exclusive_diarization.uri = file["uri"]
# at this point, `diarization` speaker labels are integers
# from 0 to `num_speakers - 1`, aligned with `centroids` rows.
@@ -654,17 +717,23 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
}
diarization = diarization.rename_labels(mapping=mapping)
exclusive_diarization = exclusive_diarization.rename_labels(mapping=mapping)
# at this point, `diarization` speaker labels are strings (or mix of
# strings and integers when reference is available and some hypothesis
# speakers are not present in the reference)
if not return_embeddings:
return diarization
# this can happen when we use OracleClustering
# centroids may be None when we use OracleClustering
if centroids is None:
return diarization, None
output = DiarizeOutput(
speaker_diarization=diarization,
exclusive_speaker_diarization=exclusive_diarization,
speaker_embeddings=centroids,
)
if self.legacy:
return output.speaker_diarization
return output
# The number of centroids may be smaller than the number of speakers
# in the annotation. This can happen if the number of active speakers
@@ -683,7 +752,16 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
[inverse_mapping[label] for label in diarization.labels()]
]
return diarization, centroids
output = DiarizeOutput(
speaker_diarization=diarization,
exclusive_speaker_diarization=exclusive_diarization,
speaker_embeddings=centroids,
)
if self.legacy:
return output.speaker_diarization
return output
def get_metric(self) -> GreedyDiarizationErrorRate:
return GreedyDiarizationErrorRate(**self.der_variant)