mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
BREAKING: remove exclusive and return_embeddings flag, add legacy flag (#1917)
This commit is contained in:
1 parent
1fcd256e4c
commit
5859a71a50
3 files changed
+185
-77
No files matched your search
+39
-9
@@ -4,10 +4,24 @@
|
||||
|
||||
### TL;DR
|
||||
|
||||
#### Better speaker assignment
|
||||
#### Improved speaker assignment and counting
|
||||
|
||||
`pyannote/speaker-diarization-4.0` pretrained pipeline relies on VBx clustering instead of agglomerative hierarchical clustering (as suggested by [BUT Speech@FIT](https://speech.fit.vut.cz/) researchers [Petr Pálka](https://github.com/Selesnyan) and [Jiangyu Han](https://github.com/jyhan03)).
|
||||
|
||||
#### *Exclusive* speaker diarization
|
||||
|
||||
`pyannote/speaker-diarization-4.0` pretrained pipeline returns a new *exclusive* speaker diarization, on top of the regular speaker diarization.
|
||||
This is a feature which is [backported from our latest commercial model](https://www.pyannote.ai/blog/precision-2) that simplifies the reconciliation between fine-grained speaker diarization timestamps and (sometimes not so precise) transcription timestamps.
|
||||
|
||||
```python
|
||||
from pyannote.audio import Pipeline
|
||||
pipeline = Pipeline.from_pretrained(
|
||||
"pyannote/speaker-diarization-4.0", token="huggingface-access-token")
|
||||
output = pipeline("/path/to/conversation.wav")
|
||||
print(output.speaker_diarization) # regular speaker diarization
|
||||
print(output.exclusive_speaker_diarization) # exclusive speaker diarization
|
||||
```
|
||||
|
||||
#### Faster training
|
||||
|
||||
Metadata caching and optimized dataloaders make training on large scale datasets much faster.
|
||||
@@ -20,18 +34,34 @@ Change one line of code to use [pyannoteAI](https://docs.pyannote.ai) premium mo
|
||||
```diff
|
||||
from pyannote.audio import Pipeline
|
||||
pipeline = Pipeline.from_pretrained(
|
||||
- "pyannote/speaker-diarization-3.1", token="huggingface-access-token")
|
||||
+ "pyannoteAI/speaker-diarization-precision, token="pyannoteAI-api-key")
|
||||
- "pyannote/speaker-diarization-4.0", token="huggingface-access-token")
|
||||
+ "pyannote/speaker-diarization-precision-2, token="pyannoteAI-api-key")
|
||||
diarization = pipeline("/path/to/conversation.wav")
|
||||
```
|
||||
|
||||
#### Quality-of-Life improvements
|
||||
#### Offline (air-gapped) use
|
||||
|
||||
Pipelines can now be stored alongside their internal models in the same repository, streamlining fully offline use.
|
||||
|
||||
1. Accept `pyannote/speaker-diarization-4.0` pipeline [user agreement](https://hf.co/pyannote/speaker-diarization-4.0)
|
||||
2. Clone the pipeline repository from Huggingface (if prompted for a password, use a Huggingface access token with correct permissions)
|
||||
|
||||
```bash
|
||||
$ git lfs install
|
||||
$ git clone https://hf.co/pyannote/speaker-diarization-4.0 /path/to/directory/pyannote-speaker-diarization-4.0
|
||||
```
|
||||
|
||||
3. Enjoy!
|
||||
|
||||
```python
|
||||
# load pipeline from disk (works without internet connection)
|
||||
from pyannote.audio import Pipeline
|
||||
pipeline = Pipeline.from_pretrained('/path/to/directory/pyannote-speaker-diarization-4.0')
|
||||
|
||||
# run the pipeline locally on your computer
|
||||
diarization = pipeline("audio.wav")
|
||||
```
|
||||
|
||||
Models can now be stored alongside their pipelines in the same repository, streamlining offline use and gating mechanism:
|
||||
- accept `pyannote/speaker-diarization-x.x` pipeline user agreement
|
||||
- ~~accept `pyannote/segmentation-3.0` model user agreement~~
|
||||
- ~~accept `pyannote/wespeaker-voxceleb-resnet34-LM` model user agreement~~
|
||||
- load pipeline with `Pipeline.from_pretrained("pyannote/speaker-diarization-3.1", token=True)`
|
||||
|
||||
#### Telemetry
|
||||
|
||||
|
||||
@@ -39,10 +39,10 @@ pipeline.to(torch.device("cuda"))
|
||||
|
||||
# apply pretrained pipeline (with optional progress hook)
|
||||
with ProgressHook() as hook:
|
||||
diarization = pipeline("audio.wav", hook=hook) # runs locally
|
||||
output = pipeline("audio.wav", hook=hook) # runs locally
|
||||
|
||||
# print the result
|
||||
for turn, _, speaker in diarization.itertracks(yield_label=True):
|
||||
for turn, _, speaker in output.speaker_diarization.itertracks(yield_label=True):
|
||||
print(f"start={turn.start:.1f}s stop={turn.end:.1f}s speaker_{speaker}")
|
||||
# start=0.2s stop=1.5s speaker_0
|
||||
# start=1.8s stop=3.9s speaker_1
|
||||
@@ -61,7 +61,7 @@ from pyannote.audio import Pipeline
|
||||
|
||||
# Premium pyannoteAI speaker diarization service
|
||||
pipeline = Pipeline.from_pretrained(
|
||||
"pyannoteAI/speaker-diarization-precision", token="PYANNOTEAI_API_KEY")
|
||||
"pyannote/speaker-diarization-precision-2", token="PYANNOTEAI_API_KEY")
|
||||
|
||||
diarization = pipeline("audio.wav") # runs on pyannoteAI servers
|
||||
|
||||
|
||||
@@ -29,7 +29,8 @@ import math
|
||||
import textwrap
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
from typing import Callable, Mapping, Optional, Text, Union
|
||||
from typing import Callable, Mapping, Optional, Text, Union, Any
|
||||
from dataclasses import dataclass
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -59,12 +60,75 @@ def batchify(iterable, batch_size: int = 32, fillvalue=None):
|
||||
return itertools.zip_longest(*args, fillvalue=fillvalue)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DiarizeOutput:
|
||||
# speaker diarization
|
||||
speaker_diarization: Annotation
|
||||
|
||||
# one speaker embedding per speaker
|
||||
# as (num_speakers, dimension) array
|
||||
# sorted in speaker_diarization.labels() order
|
||||
speaker_embeddings: np.ndarray
|
||||
|
||||
# speaker diarization adapted to downstream transcription
|
||||
# (does not contain overlapping speech turns)
|
||||
exclusive_speaker_diarization: Annotation
|
||||
|
||||
def serialize(self) -> dict[str, Any]:
|
||||
"""Serialize diarization output
|
||||
|
||||
Example
|
||||
-------
|
||||
{
|
||||
'diarization': [{
|
||||
'start': 6.665,
|
||||
'end': 7.165,
|
||||
'speaker': 'SPEAKER_00'},
|
||||
...],
|
||||
'exclusive_diarization': [{
|
||||
'start': 6.665,
|
||||
'end': 7.165,
|
||||
'speaker': 'SPEAKER_00'},
|
||||
...],
|
||||
}
|
||||
"""
|
||||
|
||||
diarization = []
|
||||
for speech_turn, _, speaker in self.speaker_diarization.itertracks(
|
||||
yield_label=True
|
||||
):
|
||||
diarization.append(
|
||||
{
|
||||
"start": round(speech_turn.start, 3),
|
||||
"end": round(speech_turn.end, 3),
|
||||
"speaker": speaker,
|
||||
}
|
||||
)
|
||||
|
||||
exclusive_diarization = []
|
||||
for speech_turn, _, speaker in self.exclusive_speaker_diarization.itertracks(
|
||||
yield_label=True
|
||||
):
|
||||
exclusive_diarization.append(
|
||||
{
|
||||
"start": round(speech_turn.start, 3),
|
||||
"end": round(speech_turn.end, 3),
|
||||
"speaker": speaker,
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"diarization": diarization,
|
||||
"exclusive_diarization": exclusive_diarization,
|
||||
}
|
||||
|
||||
|
||||
class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
|
||||
"""Speaker diarization pipeline
|
||||
|
||||
Parameters
|
||||
----------
|
||||
segmentation : Model, str, or dict, optional
|
||||
segmentation : Model, str, or dict, optional
|
||||
Pretrained segmentation model. Defaults to "pyannote/segmentation-3.0".
|
||||
See pyannote.audio.pipelines.utils.get_model for supported format.
|
||||
segmentation_step: float, optional
|
||||
@@ -92,7 +156,10 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
|
||||
Huggingface token to be used for downloading from Huggingface hub.
|
||||
cache_dir: Path or str, optional
|
||||
Path to the folder where files downloaded from Huggingface hub are stored.
|
||||
|
||||
legacy : bool, optional
|
||||
Return only the diarization output. Defaults to return the full output
|
||||
with diarization, exclusive diarization, and speaker embeddings.
|
||||
|
||||
Usage
|
||||
-----
|
||||
# perform (unconstrained) diarization
|
||||
@@ -118,6 +185,7 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
legacy: bool = False,
|
||||
segmentation: PipelineModel = "pyannote/segmentation-3.0",
|
||||
segmentation_step: float = 0.1,
|
||||
embedding: PipelineModel = "pyannote/wespeaker-voxceleb-resnet34-LM",
|
||||
@@ -135,6 +203,8 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
self.legacy = legacy
|
||||
|
||||
self.segmentation_model = segmentation
|
||||
model: Model = get_model(segmentation, token=token, cache_dir=cache_dir)
|
||||
|
||||
@@ -450,10 +520,8 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
|
||||
num_speakers: Optional[int] = None,
|
||||
min_speakers: Optional[int] = None,
|
||||
max_speakers: Optional[int] = None,
|
||||
exclusive: bool = False,
|
||||
return_embeddings: bool = False,
|
||||
hook: Optional[Callable] = None,
|
||||
) -> Annotation:
|
||||
) -> DiarizeOutput | Annotation:
|
||||
"""Apply speaker diarization
|
||||
|
||||
Parameters
|
||||
@@ -466,11 +534,6 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
|
||||
Minimum number of speakers. Has no effect when `num_speakers` is provided.
|
||||
max_speakers : int, optional
|
||||
Maximum number of speakers. Has no effect when `num_speakers` is provided.
|
||||
exclusive : bool, optional
|
||||
Enforce exclusive diarization, i.e. only one speaker can be active at a time.
|
||||
Defaults to False, i.e. overlapping speech is allowed.
|
||||
return_embeddings : bool, optional
|
||||
Return representative speaker embeddings.
|
||||
hook : callable, optional
|
||||
Callback called after each major steps of the pipeline as follows:
|
||||
hook(step_name, # human-readable name of current step
|
||||
@@ -482,12 +545,7 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
|
||||
|
||||
Returns
|
||||
-------
|
||||
diarization : Annotation
|
||||
Speaker diarization
|
||||
embeddings : np.array, optional
|
||||
Representative speaker embeddings such that `embeddings[i]` is the
|
||||
speaker embedding for i-th speaker in diarization.labels().
|
||||
Only returned when `return_embeddings` is True.
|
||||
output : DiarizeOutput (or Annotation if `self.legacy` is True)
|
||||
"""
|
||||
|
||||
# setup hook (e.g. for debugging purposes)
|
||||
@@ -537,49 +595,39 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
|
||||
# shape: (num_frames, 1)
|
||||
# dtype: int
|
||||
|
||||
if exclusive:
|
||||
count.data = np.minimum(count.data, 1).astype(np.int8)
|
||||
|
||||
# exit early when no speaker is ever active
|
||||
if np.nanmax(count.data) == 0.0:
|
||||
diarization = Annotation(uri=file["uri"])
|
||||
if return_embeddings:
|
||||
return diarization, np.zeros((0, self._embedding.dimension))
|
||||
|
||||
return diarization
|
||||
|
||||
# skip speaker embedding extraction and clustering when only one speaker
|
||||
if not return_embeddings and max_speakers < 2:
|
||||
hard_clusters = np.zeros((num_chunks, local_num_speakers), dtype=np.int8)
|
||||
embeddings = None
|
||||
centroids = None
|
||||
|
||||
else:
|
||||
# skip speaker embedding extraction with oracle clustering
|
||||
if self.klustering == "OracleClustering" and not return_embeddings:
|
||||
embeddings = None
|
||||
|
||||
else:
|
||||
embeddings = self.get_embeddings(
|
||||
file,
|
||||
binarized_segmentations,
|
||||
exclude_overlap=self.embedding_exclude_overlap,
|
||||
hook=hook,
|
||||
)
|
||||
hook("embeddings", embeddings)
|
||||
# shape: (num_chunks, local_num_speakers, dimension)
|
||||
|
||||
hard_clusters, _, centroids = self.clustering(
|
||||
embeddings=embeddings,
|
||||
segmentations=binarized_segmentations,
|
||||
num_clusters=num_speakers,
|
||||
min_clusters=min_speakers,
|
||||
max_clusters=max_speakers,
|
||||
file=file, # <== for oracle clustering
|
||||
frames=self._segmentation.model.receptive_field, # <== for oracle clustering
|
||||
output = DiarizeOutput(
|
||||
speaker_diarization=Annotation(uri=file["uri"]),
|
||||
exclusive_speaker_diarization=Annotation(uri=file["uri"]),
|
||||
speaker_embeddings=np.zeros((0, self._embedding.dimension)),
|
||||
)
|
||||
# hard_clusters: (num_chunks, num_speakers)
|
||||
# centroids: (num_speakers, dimension)
|
||||
|
||||
if self.legacy:
|
||||
return output.speaker_diarization
|
||||
|
||||
return output
|
||||
|
||||
embeddings = self.get_embeddings(
|
||||
file,
|
||||
binarized_segmentations,
|
||||
exclude_overlap=self.embedding_exclude_overlap,
|
||||
hook=hook,
|
||||
)
|
||||
hook("embeddings", embeddings)
|
||||
# shape: (num_chunks, local_num_speakers, dimension)
|
||||
|
||||
hard_clusters, _, centroids = self.clustering(
|
||||
embeddings=embeddings,
|
||||
segmentations=binarized_segmentations,
|
||||
num_clusters=num_speakers,
|
||||
min_clusters=min_speakers,
|
||||
max_clusters=max_speakers,
|
||||
file=file, # <== for oracle clustering
|
||||
frames=self._segmentation.model.receptive_field, # <== for oracle clustering
|
||||
)
|
||||
# hard_clusters: (num_chunks, num_speakers)
|
||||
# centroids: (num_speakers, dimension)
|
||||
|
||||
# number of detected clusters is the number of different speakers
|
||||
num_different_speakers = np.max(hard_clusters) + 1
|
||||
@@ -613,15 +661,16 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
|
||||
inactive_speakers = np.sum(binarized_segmentations.data, axis=1) == 0
|
||||
# shape: (num_chunks, num_speakers)
|
||||
|
||||
# force-assign them to throw-away cluster
|
||||
hard_clusters[inactive_speakers] = -2
|
||||
|
||||
# convert to continuous diarization
|
||||
discrete_diarization = self.reconstruct(
|
||||
segmentations,
|
||||
hard_clusters,
|
||||
count,
|
||||
)
|
||||
hook("discrete_diarization", discrete_diarization)
|
||||
|
||||
# convert to continuous diarization
|
||||
diarization = self.to_annotation(
|
||||
discrete_diarization,
|
||||
min_duration_on=0.0,
|
||||
@@ -629,6 +678,20 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
|
||||
)
|
||||
diarization.uri = file["uri"]
|
||||
|
||||
# convert to continuous exclusive diarization
|
||||
count.data = np.minimum(count.data, 1).astype(np.int8)
|
||||
exclusive_discrete_diarization = self.reconstruct(
|
||||
segmentations,
|
||||
hard_clusters,
|
||||
count,
|
||||
)
|
||||
exclusive_diarization = self.to_annotation(
|
||||
exclusive_discrete_diarization,
|
||||
min_duration_on=0.0,
|
||||
min_duration_off=self.segmentation.min_duration_off,
|
||||
)
|
||||
exclusive_diarization.uri = file["uri"]
|
||||
|
||||
# at this point, `diarization` speaker labels are integers
|
||||
# from 0 to `num_speakers - 1`, aligned with `centroids` rows.
|
||||
|
||||
@@ -654,17 +717,23 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
|
||||
}
|
||||
|
||||
diarization = diarization.rename_labels(mapping=mapping)
|
||||
exclusive_diarization = exclusive_diarization.rename_labels(mapping=mapping)
|
||||
|
||||
# at this point, `diarization` speaker labels are strings (or mix of
|
||||
# strings and integers when reference is available and some hypothesis
|
||||
# speakers are not present in the reference)
|
||||
|
||||
if not return_embeddings:
|
||||
return diarization
|
||||
|
||||
# this can happen when we use OracleClustering
|
||||
# centroids may be None when we use OracleClustering
|
||||
if centroids is None:
|
||||
return diarization, None
|
||||
output = DiarizeOutput(
|
||||
speaker_diarization=diarization,
|
||||
exclusive_speaker_diarization=exclusive_diarization,
|
||||
speaker_embeddings=centroids,
|
||||
)
|
||||
if self.legacy:
|
||||
return output.speaker_diarization
|
||||
|
||||
return output
|
||||
|
||||
# The number of centroids may be smaller than the number of speakers
|
||||
# in the annotation. This can happen if the number of active speakers
|
||||
@@ -683,7 +752,16 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
|
||||
[inverse_mapping[label] for label in diarization.labels()]
|
||||
]
|
||||
|
||||
return diarization, centroids
|
||||
output = DiarizeOutput(
|
||||
speaker_diarization=diarization,
|
||||
exclusive_speaker_diarization=exclusive_diarization,
|
||||
speaker_embeddings=centroids,
|
||||
)
|
||||
|
||||
if self.legacy:
|
||||
return output.speaker_diarization
|
||||
|
||||
return output
|
||||
|
||||
def get_metric(self) -> GreedyDiarizationErrorRate:
|
||||
return GreedyDiarizationErrorRate(**self.der_variant)
|
||||
Reference in new issue
Block a user