feat(pipeline): add return_embeddings option to SpeakerDiarization pipeline

Co-authored-by: Hervé BREDIN <hbredin@users.noreply.github.com>
This commit is contained in:
Dmitry MukhutdinovandHervé BREDIN authored and Hervé BREDIN committed 2023-09-20 17:39:30 +02:00
1 parent e9a92740a0
commit ef029bccd2
4 files changed
+129 -47

No files matched your search

+2 -2
View File
@@ -30,13 +30,13 @@
If, for some weird reason, you wrote some custom code based on that,
you should instead rely on `Model.example_output`.
### Features and improvements
- feat(task): add support for multi-task models
- feat(pipeline): send pipeline to device with `pipeline.to(device)`
- feat(pipeline): make `segmentation_batch_size` and `embedding_batch_size` mutable in `SpeakerDiarization` pipeline (they now default to `1`)
- feat(task): add [powerset](https://arxiv.org/PLACEHOLDER) support to `SpeakerDiarization` task
- feat(pipeline): add `return_embeddings` option to `SpeakerDiarization` pipeline
- feat(pipeline): add progress hook to pipelines
- feat(pipeline): check version compatibility at load time
- feat(task): add support for label scope in speaker diarization task
@@ -88,7 +88,7 @@
- last release before complete rewriting
## Version 1.0.1 (2018--07-19)
## Version 1.0.1 (2018-07-19)
- fix: fix regression in Precomputed.__call__ (#110, #105)
+37 -24
View File
@@ -48,7 +48,6 @@ class BaseClustering(Pipeline):
max_num_embeddings: int = 1000,
constrained_assignment: bool = False,
):
super().__init__()
self.metric = metric
self.max_num_embeddings = max_num_embeddings
@@ -61,7 +60,6 @@ class BaseClustering(Pipeline):
min_clusters: int = None,
max_clusters: int = None,
):
min_clusters = num_clusters or min_clusters or 1
min_clusters = max(1, min(num_embeddings, min_clusters))
max_clusters = num_clusters or max_clusters or num_embeddings
@@ -113,7 +111,6 @@ class BaseClustering(Pipeline):
return embeddings[chunk_idx, speaker_idx], chunk_idx, speaker_idx
def constrained_argmax(self, soft_clusters: np.ndarray) -> np.ndarray:
soft_clusters = np.nan_to_num(soft_clusters, nan=np.nanmin(soft_clusters))
num_chunks, num_speakers, num_clusters = soft_clusters.shape
# num_chunks, num_speakers, num_clusters
@@ -156,6 +153,8 @@ class BaseClustering(Pipeline):
-------
soft_clusters : (num_chunks, num_speakers, num_clusters)-shaped array
hard_clusters : (num_chunks, num_speakers)-shaped array
centroids : (num_clusters, dimension)-shaped array
Clusters centroids
"""
# TODO: option to add a new (dummy) cluster in case num_clusters < max(frame_speaker_count)
@@ -194,7 +193,7 @@ class BaseClustering(Pipeline):
# TODO: add a flag to revert argmax for trainign subset
# hard_clusters[train_chunk_idx, train_speaker_idx] = train_clusters
return hard_clusters, soft_clusters
return hard_clusters, soft_clusters, centroids
def __call__(
self,
@@ -230,6 +229,8 @@ class BaseClustering(Pipeline):
soft_clusters : (num_chunks, num_speakers, num_clusters) array
Soft cluster assignment (the higher soft_clusters[c, s, k], the most likely
the sth speaker of cth chunk belongs to kth cluster)
centroids : (num_clusters, dimension) array
Centroid vectors of each cluster
"""
train_embeddings, train_chunk_idx, train_speaker_idx = self.filter_embeddings(
@@ -250,7 +251,9 @@ class BaseClustering(Pipeline):
num_chunks, num_speakers, _ = embeddings.shape
hard_clusters = np.zeros((num_chunks, num_speakers), dtype=np.int8)
soft_clusters = np.ones((num_chunks, num_speakers, 1))
return hard_clusters, soft_clusters
centroids = np.mean(train_embeddings, axis=0, keepdims=True)
return hard_clusters, soft_clusters, centroids
train_clusters = self.cluster(
train_embeddings,
@@ -259,7 +262,7 @@ class BaseClustering(Pipeline):
num_clusters=num_clusters,
)
hard_clusters, soft_clusters = self.assign_embeddings(
hard_clusters, soft_clusters, centroids = self.assign_embeddings(
embeddings,
train_chunk_idx,
train_speaker_idx,
@@ -267,7 +270,7 @@ class BaseClustering(Pipeline):
constrained=self.constrained_assignment,
)
return hard_clusters, soft_clusters
return hard_clusters, soft_clusters, centroids
class AgglomerativeClustering(BaseClustering):
@@ -286,19 +289,6 @@ class AgglomerativeClustering(BaseClustering):
Clustering threshold.
min_cluster_size : int in range [1, 20]
Minimum cluster size
Usage
-----
>>> clustering = AgglomerativeClustering(metric="cosine")
>>> clustering.instantiate({"method": "average",
... "threshold": 1.0,
... "min_cluster_size": 1})
>>> clusters, _ = clustering(embeddings, # shape
... num_clusters=None,
... min_clusters=None,
... max_clusters=None)
where `embeddings` is a np.ndarray with shape (num_embeddings, embedding_dimension)
and `clusters` is a np.ndarray with shape (num_embeddings, )
"""
def __init__(
@@ -307,7 +297,6 @@ class AgglomerativeClustering(BaseClustering):
max_num_embeddings: int = np.inf,
constrained_assignment: bool = False,
):
super().__init__(
metric=metric,
max_num_embeddings=max_num_embeddings,
@@ -397,7 +386,6 @@ class AgglomerativeClustering(BaseClustering):
num_clusters = max_clusters
if num_clusters is not None:
# switch stopping criterion from "inter-cluster distance" stopping to "iteration index"
_dendrogram = np.copy(dendrogram)
_dendrogram[:, 2] = np.arange(num_embeddings - 1)
@@ -409,7 +397,6 @@ class AgglomerativeClustering(BaseClustering):
# from the "optimal" threshold
for iteration in np.argsort(np.abs(dendrogram[:, 2] - self.threshold)):
# only consider iterations that might have resulted
# in changing the number of (large) clusters
new_cluster_size = _dendrogram[iteration, 3]
@@ -481,6 +468,7 @@ class OracleClustering(BaseClustering):
def __call__(
self,
embeddings: np.ndarray = None,
segmentations: SlidingWindowFeature = None,
file: AudioFile = None,
frames: SlidingWindow = None,
@@ -490,6 +478,9 @@ class OracleClustering(BaseClustering):
Parameters
----------
embeddings : (num_chunks, num_speakers, dimension) array, optional
Sequence of embeddings. When provided, compute speaker centroids
based on these embeddings.
segmentations : (num_chunks, num_frames, num_speakers) array
Binary segmentations.
file : AudioFile
@@ -503,6 +494,8 @@ class OracleClustering(BaseClustering):
soft_clusters : (num_chunks, num_speakers, num_clusters) array
Soft cluster assignment (the higher soft_clusters[c, s, k], the most likely
the sth speaker of cth chunk belongs to kth cluster)
centroids : (num_clusters, dimension), optional
Clusters centroids if `embeddings` is provided, None otherwise.
"""
num_chunks, num_frames, num_speakers = segmentations.data.shape
@@ -532,7 +525,27 @@ class OracleClustering(BaseClustering):
hard_clusters[c, i] = j
soft_clusters[c, i, j] = 1.0
return hard_clusters, soft_clusters
if embeddings is None:
return hard_clusters, soft_clusters, None
(
train_embeddings,
train_chunk_idx,
train_speaker_idx,
) = self.filter_embeddings(
embeddings,
segmentations=segmentations,
)
train_clusters = hard_clusters[train_chunk_idx, train_speaker_idx]
centroids = np.vstack(
[
np.mean(train_embeddings[train_clusters == k], axis=0)
for k in range(num_clusters)
]
)
return hard_clusters, soft_clusters, centroids
class Clustering(Enum):
+68 -15
View File
@@ -89,11 +89,20 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
Usage
-----
>>> pipeline = SpeakerDiarization()
# perform (unconstrained) diarization
>>> diarization = pipeline("/path/to/audio.wav")
# perform diarization, targetting exactly 4 speakers
>>> diarization = pipeline("/path/to/audio.wav", num_speakers=4)
# perform diarization, with at least 2 speakers and at most 10 speakers
>>> diarization = pipeline("/path/to/audio.wav", min_speakers=2, max_speakers=10)
# perform diarization and get one representative embedding per speaker
>>> diarization, embeddings = pipeline("/path/to/audio.wav", return_embedding=True)
>>> for s, speaker in enumerate(diarization.labels()):
... # embeddings[s] is the embedding of speaker `speaker`
Hyper-parameters
----------------
segmentation.threshold
@@ -417,6 +426,7 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
num_speakers: int = None,
min_speakers: int = None,
max_speakers: int = None,
return_embeddings: bool = False,
hook: Optional[Callable] = None,
) -> Annotation:
"""Apply speaker diarization
@@ -431,6 +441,8 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
Minimum number of speakers. Has no effect when `num_speakers` is provided.
max_speakers : int, optional
Maximum number of speakers. Has no effect when `num_speakers` is provided.
return_embeddings : bool, optional
Return representative speaker embeddings.
hook : callable, optional
Callback called after each major steps of the pipeline as follows:
hook(step_name, # human-readable name of current step
@@ -444,6 +456,10 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
-------
diarization : Annotation
Speaker diarization
embeddings : np.array, optional
Representative speaker embeddings such that `embeddings[i]` is the
speaker embedding for i-th speaker in diarization.labels().
Only returned when `return_embeddings` is True.
"""
# setup hook (e.g. for debugging purposes)
@@ -473,7 +489,11 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
# exit early when no speaker is ever active
if np.nanmax(count.data) == 0.0:
return Annotation(uri=file["uri"])
diarization = Annotation(uri=file["uri"])
if return_embeddings:
return diarization, np.zeros((0, self._embedding.dimension))
return diarization
# binarize segmentation
if self._segmentation.model.specifications.powerset:
@@ -485,7 +505,7 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
initial_state=False,
)
if self.klustering == "OracleClustering":
if self.klustering == "OracleClustering" and not return_embeddings:
embeddings = None
else:
embeddings = self.get_embeddings(
@@ -497,7 +517,7 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
hook("embeddings", embeddings)
# shape: (num_chunks, local_num_speakers, dimension)
hard_clusters, _ = self.clustering(
hard_clusters, _, centroids = self.clustering(
embeddings=embeddings,
segmentations=binarized_segmentations,
num_clusters=num_speakers,
@@ -506,7 +526,8 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
file=file, # <== for oracle clustering
frames=self._frames, # <== for oracle clustering
)
# hard_clusters: (num_chunks, num_speakers)
# hard_clusters: (num_chunks, num_speakers)
# centroids: (num_speakers, dimension)
# reconstruct discrete diarization from raw hard clusters
@@ -530,20 +551,52 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
)
diarization.uri = file["uri"]
# when reference is available, use it to map hypothesized speakers
# to reference speakers (this makes later error analysis easier
# but does not modify the actual output of the diarization pipeline)
if "annotation" in file and file["annotation"]:
return self.optimal_mapping(file["annotation"], diarization)
# at this point, `diarization` speaker labels are integers
# from 0 to `num_speakers - 1`, aligned with `centroids` rows.
# when reference is not available, rename hypothesized speakers
# to human-readable SPEAKER_00, SPEAKER_01, ...
return diarization.rename_labels(
{
if "annotation" in file and file["annotation"]:
# when reference is available, use it to map hypothesized speakers
# to reference speakers (this makes later error analysis easier
# but does not modify the actual output of the diarization pipeline)
_, mapping = self.optimal_mapping(
file["annotation"], diarization, return_mapping=True
)
# in case there are more speakers in the hypothesis than in
# the reference, those extra speakers are missing from `mapping`.
# we add them back here
mapping = {key: mapping.get(key, key) for key in diarization.labels()}
else:
# when reference is not available, rename hypothesized speakers
# to human-readable SPEAKER_00, SPEAKER_01, ...
mapping = {
label: expected_label
for label, expected_label in zip(diarization.labels(), self.classes())
}
)
diarization = diarization.rename_labels(mapping=mapping)
# at this point, `diarization` speaker labels are strings (or mix of
# strings and integers when reference is available and some hypothesis
# speakers are not present in the reference)
if not return_embeddings:
return diarization
# re-order centroids so that they match
# the order given by diarization.labels()
inverse_mapping = {label: index for index, label in mapping.items()}
centroids = centroids[
[inverse_mapping[label] for label in diarization.labels()]
]
# FIXME: the number of centroids may be smaller than the number of speakers
# in the annotation. This can happen if the number of active speakers
# obtained from `speaker_count` for some frames is larger than the number
# of clusters obtained from `clustering`. Will be fixed in the future
return diarization, centroids
def get_metric(self) -> GreedyDiarizationErrorRate:
return GreedyDiarizationErrorRate(**self.der_variant)
+22 -6
View File
@@ -20,10 +20,11 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from typing import Mapping, Tuple, Union
from typing import Dict, Mapping, Tuple, Union
import numpy as np
from pyannote.core import Annotation, SlidingWindow, SlidingWindowFeature
from pyannote.core.utils.types import Label
from pyannote.metrics.diarization import DiarizationErrorRate
from pyannote.audio.core.inference import Inference
@@ -74,8 +75,10 @@ class SpeakerDiarizationMixin:
@staticmethod
def optimal_mapping(
reference: Union[Mapping, Annotation], hypothesis: Annotation
) -> Annotation:
reference: Union[Mapping, Annotation],
hypothesis: Annotation,
return_mapping: bool = False,
) -> Union[Annotation, Tuple[Annotation, Dict[Label, Label]]]:
"""Find the optimal bijective mapping between reference and hypothesis labels
Parameters
@@ -84,13 +87,19 @@ class SpeakerDiarizationMixin:
Reference annotation. Can be an Annotation instance or
a mapping with an "annotation" key.
hypothesis : Annotation
Hypothesized annotation.
return_mapping : bool, optional
Return the label mapping itself along with the mapped annotation. Defaults to False.
Returns
-------
mapped : Annotation
Hypothesis mapped to reference speakers.
mapping : dict, optional
Mapping between hypothesis (key) and reference (value) labels
Only returned if `return_mapping` is True.
"""
if isinstance(reference, Mapping):
reference = reference["annotation"]
annotated = reference["annotated"] if "annotated" in reference else None
@@ -100,7 +109,13 @@ class SpeakerDiarizationMixin:
mapping = DiarizationErrorRate().optimal_mapping(
reference, hypothesis, uem=annotated
)
return hypothesis.rename_labels(mapping=mapping)
mapped_hypothesis = hypothesis.rename_labels(mapping=mapping)
if return_mapping:
return mapped_hypothesis, mapping
else:
return mapped_hypothesis
# TODO: get rid of onset/offset (binarization should be applied before calling speaker_count)
# TODO: get rid of warm-up parameter (trimming should be applied before calling speaker_count)
@@ -171,7 +186,8 @@ class SpeakerDiarizationMixin:
Returns
-------
continuous_diarization : Annotation
Continuous diarization
Continuous diarization, with speaker labels as integers,
corresponding to the speaker indices in the discrete diarization.
"""
binarize = Binarize(