wip: add extra output to pipeline

This commit is contained in:
Hervé BREDIN committed 2025-04-28 21:47:14 +02:00
1 parent a9d8183a72
commit 985ebf4ad6
1 file changed
+73 -65
@@ -1,6 +1,7 @@
# The MIT License (MIT)
#
# Copyright (c) 2021- CNRS
# Copyright (c) 2021-2025 CNRS
# Copyright (c) 2025- pyannoteAI
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
@@ -27,16 +28,13 @@ import itertools
import math
import textwrap
import warnings
from dataclasses import dataclass
from pathlib import Path
from typing import Callable, Mapping, Optional, Text, Union
import numpy as np
import torch
from einops import rearrange
from pyannote.core import Annotation, SlidingWindowFeature
from pyannote.metrics.diarization import GreedyDiarizationErrorRate
from pyannote.pipeline.parameter import ParamDict, Uniform
from pyannote.audio import Audio, Inference, Model, Pipeline
from pyannote.audio.core.io import AudioFile
from pyannote.audio.pipelines.clustering import Clustering
@@ -48,6 +46,9 @@ from pyannote.audio.pipelines.utils import (
)
from pyannote.audio.pipelines.utils.diarization import set_num_speakers
from pyannote.audio.utils.signal import binarize
from pyannote.core import Annotation, SlidingWindowFeature
from pyannote.metrics.diarization import GreedyDiarizationErrorRate
from pyannote.pipeline.parameter import ParamDict, Uniform
def batchify(iterable, batch_size: int = 32, fillvalue=None):
@@ -57,6 +58,20 @@ def batchify(iterable, batch_size: int = 32, fillvalue=None):
return itertools.zip_longest(*args, fillvalue=fillvalue)
@dataclass
class SpeakerDiarizationOutput:
# speaker diarization
speaker_diarization: Annotation
# frame-wise speech probability
# as (num_frames, ) probability array
speech_probability: SlidingWindowFeature
# one speaker embedding per speaker
# as (num_speakers, dimension) array
speaker_embeddings: np.ndarray
class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
"""Speaker diarization pipeline
@@ -82,6 +97,9 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
Batch size used for speaker segmentation. Defaults to 1.
embedding_batch_size : int, optional
Batch size used for speaker embedding. Defaults to 1.
diarization_only : bool, optional
Return only the diarization output. Defaults to return the full output
with speech probability and speaker embeddings.
der_variant : dict, optional
Optimize for a variant of diarization error rate.
Defaults to {"collar": 0.0, "skip_overlap": False}. This is used in `get_metric`
@@ -103,9 +121,10 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
>>> diarization = pipeline("/path/to/audio.wav", min_speakers=2, max_speakers=10)
# perform diarization and get one representative embedding per speaker
>>> diarization, embeddings = pipeline("/path/to/audio.wav", return_embeddings=True)
>>> for s, speaker in enumerate(diarization.labels()):
... # embeddings[s] is the embedding of speaker `speaker`
>>> pipeline.diarization_only = False
>>> result = pipeline("/path/to/audio.wav")
>>> for s, speaker in enumerate(result.speaker_diarization.labels()):
... # result.speaker_embeddings[s] is the embedding of speaker `speaker`
Hyper-parameters
----------------
@@ -123,6 +142,7 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
clustering: str = "AgglomerativeClustering",
embedding_batch_size: int = 1,
segmentation_batch_size: int = 1,
diarization_only: bool = True,
der_variant: Optional[dict] = None,
token: Union[Text, None] = None,
cache_dir: Union[Path, Text, None] = None,
@@ -132,6 +152,10 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
self.segmentation_model = segmentation
model: Model = get_model(segmentation, token=token, cache_dir=cache_dir)
# since pyannote.audio 4.x, only powerset segmentation models are supported
if not model.specifications.powerset:
raise ValueError("Segmentation model must be a powerset model.")
self.segmentation_step = segmentation_step
self.embedding = embedding
@@ -151,16 +175,9 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
batch_size=segmentation_batch_size,
)
if self._segmentation.model.specifications.powerset:
self.segmentation = ParamDict(
min_duration_off=Uniform(0.0, 1.0),
)
else:
self.segmentation = ParamDict(
threshold=Uniform(0.1, 0.9),
min_duration_off=Uniform(0.0, 1.0),
)
self.segmentation = ParamDict(
min_duration_off=Uniform(0.0, 1.0),
)
if self.klustering == "OracleClustering":
metric = "not_applicable"
@@ -176,12 +193,14 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
Klustering = Clustering[clustering]
except KeyError:
raise ValueError(
f'clustering must be one of [{", ".join(list(Clustering.__members__))}]'
f"clustering must be one of [{', '.join(list(Clustering.__members__))}]"
)
self.clustering = Klustering.value(metric=metric)
self._expects_num_speakers = self.clustering.expects_num_clusters
self.diarization_only = diarization_only
@property
def segmentation_batch_size(self) -> int:
return self._segmentation.batch_size
@@ -255,19 +274,11 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
embeddings : (num_chunks, num_speakers, dimension) array
"""
# when optimizing the hyper-parameters of this pipeline with frozen
# "segmentation.threshold", one can reuse the embeddings from the first trial,
# bringing a massive speed up to the optimization process (and hence allowing to use
# a larger search space).
# when optimizing the hyper-parameters of this pipeline, one can reuse the embeddings from the first trial,
# bringing a massive speed up to the optimization process (and hence allowing to use a larger search space).
if self.training:
# we only re-use embeddings if they were extracted based on the same value of the
# "segmentation.threshold" hyperparameter or if the segmentation model relies on
# `powerset` mode
cache = file.get("training_cache/embeddings", dict())
if ("embeddings" in cache) and (
self._segmentation.model.specifications.powerset
or (cache["segmentation.threshold"] == self.segmentation.threshold)
):
if "embeddings" in cache:
return cache["embeddings"]
duration = binary_segmentations.sliding_window.duration
@@ -367,15 +378,9 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
# caching embeddings for subsequent trials
# (see comments at the top of this method for more details)
if self.training:
if self._segmentation.model.specifications.powerset:
file["training_cache/embeddings"] = {
"embeddings": embeddings,
}
else:
file["training_cache/embeddings"] = {
"segmentation.threshold": self.segmentation.threshold,
"embeddings": embeddings,
}
file["training_cache/embeddings"] = {
"embeddings": embeddings,
}
return embeddings
@@ -435,9 +440,8 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
num_speakers: Optional[int] = None,
min_speakers: Optional[int] = None,
max_speakers: Optional[int] = None,
return_embeddings: bool = False,
hook: Optional[Callable] = None,
) -> Annotation:
) -> Annotation | SpeakerDiarizationOutput:
"""Apply speaker diarization
Parameters
@@ -450,8 +454,6 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
Minimum number of speakers. Has no effect when `num_speakers` is provided.
max_speakers : int, optional
Maximum number of speakers. Has no effect when `num_speakers` is provided.
return_embeddings : bool, optional
Return representative speaker embeddings.
hook : callable, optional
Callback called after each major steps of the pipeline as follows:
hook(step_name, # human-readable name of current step
@@ -463,12 +465,9 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
Returns
-------
diarization : Annotation
Speaker diarization
embeddings : np.array, optional
Representative speaker embeddings such that `embeddings[i]` is the
speaker embedding for i-th speaker in diarization.labels().
Only returned when `return_embeddings` is True.
diarization : Annotation or SpeakerDiarizationOutput
Speaker diarization if `diarization_only` is True.
Speaker diarization, speech probability and speaker embeddings otherwise.
"""
# setup hook (e.g. for debugging purposes)
@@ -499,14 +498,8 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
num_chunks, num_frames, local_num_speakers = segmentations.data.shape
# binarize segmentation
if self._segmentation.model.specifications.powerset:
binarized_segmentations = segmentations
else:
binarized_segmentations: SlidingWindowFeature = binarize(
segmentations,
onset=self.segmentation.threshold,
initial_state=False,
)
# TODO: update get_segmentation to reeturn speech prob as well
binarized_segmentations = segmentations
# estimate frame-level number of instantaneous speakers
count = self.speaker_count(
@@ -521,20 +514,26 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
# exit early when no speaker is ever active
if np.nanmax(count.data) == 0.0:
diarization = Annotation(uri=file["uri"])
if return_embeddings:
return diarization, np.zeros((0, self._embedding.dimension))
if self.diarization_only:
return diarization
return SpeakerDiarizationOutput(
speaker_diarization=diarization,
speech_probability=...,
speech_probability_resolution=...,
speaker_embeddings=np.zeros((0, self._embedding.dimension)),
)
return diarization
# skip speaker embedding extraction and clustering when only one speaker
if not return_embeddings and max_speakers < 2:
if self.diarization_only and max_speakers < 2:
hard_clusters = np.zeros((num_chunks, local_num_speakers), dtype=np.int8)
embeddings = None
centroids = None
else:
# skip speaker embedding extraction with oracle clustering
if self.klustering == "OracleClustering" and not return_embeddings:
if self.klustering == "OracleClustering" and self.diarization_only:
embeddings = None
else:
@@ -637,12 +636,16 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
# strings and integers when reference is available and some hypothesis
# speakers are not present in the reference)
if not return_embeddings:
if self.diarization_only:
return diarization
# this can happen when we use OracleClustering
if centroids is None:
return diarization, None
return SpeakerDiarizationOutput(
speaker_diarization=diarization,
speech_probability=None,
speaker_embeddings=None,
)
# The number of centroids may be smaller than the number of speakers
# in the annotation. This can happen if the number of active speakers
@@ -661,7 +664,12 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
[inverse_mapping[label] for label in diarization.labels()]
]
return diarization, centroids
return SpeakerDiarizationOutput(
speaker_diarization=diarization,
speech_probability=None,
speech_probability_resolution=None,
speaker_embeddings=centroids,
)
def get_metric(self) -> GreedyDiarizationErrorRate:
return GreedyDiarizationErrorRate(**self.der_variant)