mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
feat(pipeline): add return_embeddings option to SpeakerDiarization pipeline
Co-authored-by: Hervé BREDIN <hbredin@users.noreply.github.com>
This commit is contained in:
1 parent
e9a92740a0
commit
ef029bccd2
4 files changed
+129
-47
No files matched your search
+2
-2
@@ -30,13 +30,13 @@
|
||||
If, for some weird reason, you wrote some custom code based on that,
|
||||
you should instead rely on `Model.example_output`.
|
||||
|
||||
|
||||
### Features and improvements
|
||||
|
||||
- feat(task): add support for multi-task models
|
||||
- feat(pipeline): send pipeline to device with `pipeline.to(device)`
|
||||
- feat(pipeline): make `segmentation_batch_size` and `embedding_batch_size` mutable in `SpeakerDiarization` pipeline (they now default to `1`)
|
||||
- feat(task): add [powerset](https://arxiv.org/PLACEHOLDER) support to `SpeakerDiarization` task
|
||||
- feat(pipeline): add `return_embeddings` option to `SpeakerDiarization` pipeline
|
||||
- feat(pipeline): add progress hook to pipelines
|
||||
- feat(pipeline): check version compatibility at load time
|
||||
- feat(task): add support for label scope in speaker diarization task
|
||||
@@ -88,7 +88,7 @@
|
||||
|
||||
- last release before complete rewriting
|
||||
|
||||
## Version 1.0.1 (2018--07-19)
|
||||
## Version 1.0.1 (2018-07-19)
|
||||
|
||||
- fix: fix regression in Precomputed.__call__ (#110, #105)
|
||||
|
||||
|
||||
@@ -48,7 +48,6 @@ class BaseClustering(Pipeline):
|
||||
max_num_embeddings: int = 1000,
|
||||
constrained_assignment: bool = False,
|
||||
):
|
||||
|
||||
super().__init__()
|
||||
self.metric = metric
|
||||
self.max_num_embeddings = max_num_embeddings
|
||||
@@ -61,7 +60,6 @@ class BaseClustering(Pipeline):
|
||||
min_clusters: int = None,
|
||||
max_clusters: int = None,
|
||||
):
|
||||
|
||||
min_clusters = num_clusters or min_clusters or 1
|
||||
min_clusters = max(1, min(num_embeddings, min_clusters))
|
||||
max_clusters = num_clusters or max_clusters or num_embeddings
|
||||
@@ -113,7 +111,6 @@ class BaseClustering(Pipeline):
|
||||
return embeddings[chunk_idx, speaker_idx], chunk_idx, speaker_idx
|
||||
|
||||
def constrained_argmax(self, soft_clusters: np.ndarray) -> np.ndarray:
|
||||
|
||||
soft_clusters = np.nan_to_num(soft_clusters, nan=np.nanmin(soft_clusters))
|
||||
num_chunks, num_speakers, num_clusters = soft_clusters.shape
|
||||
# num_chunks, num_speakers, num_clusters
|
||||
@@ -156,6 +153,8 @@ class BaseClustering(Pipeline):
|
||||
-------
|
||||
soft_clusters : (num_chunks, num_speakers, num_clusters)-shaped array
|
||||
hard_clusters : (num_chunks, num_speakers)-shaped array
|
||||
centroids : (num_clusters, dimension)-shaped array
|
||||
Clusters centroids
|
||||
"""
|
||||
|
||||
# TODO: option to add a new (dummy) cluster in case num_clusters < max(frame_speaker_count)
|
||||
@@ -194,7 +193,7 @@ class BaseClustering(Pipeline):
|
||||
# TODO: add a flag to revert argmax for trainign subset
|
||||
# hard_clusters[train_chunk_idx, train_speaker_idx] = train_clusters
|
||||
|
||||
return hard_clusters, soft_clusters
|
||||
return hard_clusters, soft_clusters, centroids
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
@@ -230,6 +229,8 @@ class BaseClustering(Pipeline):
|
||||
soft_clusters : (num_chunks, num_speakers, num_clusters) array
|
||||
Soft cluster assignment (the higher soft_clusters[c, s, k], the most likely
|
||||
the sth speaker of cth chunk belongs to kth cluster)
|
||||
centroids : (num_clusters, dimension) array
|
||||
Centroid vectors of each cluster
|
||||
"""
|
||||
|
||||
train_embeddings, train_chunk_idx, train_speaker_idx = self.filter_embeddings(
|
||||
@@ -250,7 +251,9 @@ class BaseClustering(Pipeline):
|
||||
num_chunks, num_speakers, _ = embeddings.shape
|
||||
hard_clusters = np.zeros((num_chunks, num_speakers), dtype=np.int8)
|
||||
soft_clusters = np.ones((num_chunks, num_speakers, 1))
|
||||
return hard_clusters, soft_clusters
|
||||
centroids = np.mean(train_embeddings, axis=0, keepdims=True)
|
||||
|
||||
return hard_clusters, soft_clusters, centroids
|
||||
|
||||
train_clusters = self.cluster(
|
||||
train_embeddings,
|
||||
@@ -259,7 +262,7 @@ class BaseClustering(Pipeline):
|
||||
num_clusters=num_clusters,
|
||||
)
|
||||
|
||||
hard_clusters, soft_clusters = self.assign_embeddings(
|
||||
hard_clusters, soft_clusters, centroids = self.assign_embeddings(
|
||||
embeddings,
|
||||
train_chunk_idx,
|
||||
train_speaker_idx,
|
||||
@@ -267,7 +270,7 @@ class BaseClustering(Pipeline):
|
||||
constrained=self.constrained_assignment,
|
||||
)
|
||||
|
||||
return hard_clusters, soft_clusters
|
||||
return hard_clusters, soft_clusters, centroids
|
||||
|
||||
|
||||
class AgglomerativeClustering(BaseClustering):
|
||||
@@ -286,19 +289,6 @@ class AgglomerativeClustering(BaseClustering):
|
||||
Clustering threshold.
|
||||
min_cluster_size : int in range [1, 20]
|
||||
Minimum cluster size
|
||||
|
||||
Usage
|
||||
-----
|
||||
>>> clustering = AgglomerativeClustering(metric="cosine")
|
||||
>>> clustering.instantiate({"method": "average",
|
||||
... "threshold": 1.0,
|
||||
... "min_cluster_size": 1})
|
||||
>>> clusters, _ = clustering(embeddings, # shape
|
||||
... num_clusters=None,
|
||||
... min_clusters=None,
|
||||
... max_clusters=None)
|
||||
where `embeddings` is a np.ndarray with shape (num_embeddings, embedding_dimension)
|
||||
and `clusters` is a np.ndarray with shape (num_embeddings, )
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -307,7 +297,6 @@ class AgglomerativeClustering(BaseClustering):
|
||||
max_num_embeddings: int = np.inf,
|
||||
constrained_assignment: bool = False,
|
||||
):
|
||||
|
||||
super().__init__(
|
||||
metric=metric,
|
||||
max_num_embeddings=max_num_embeddings,
|
||||
@@ -397,7 +386,6 @@ class AgglomerativeClustering(BaseClustering):
|
||||
num_clusters = max_clusters
|
||||
|
||||
if num_clusters is not None:
|
||||
|
||||
# switch stopping criterion from "inter-cluster distance" stopping to "iteration index"
|
||||
_dendrogram = np.copy(dendrogram)
|
||||
_dendrogram[:, 2] = np.arange(num_embeddings - 1)
|
||||
@@ -409,7 +397,6 @@ class AgglomerativeClustering(BaseClustering):
|
||||
# from the "optimal" threshold
|
||||
|
||||
for iteration in np.argsort(np.abs(dendrogram[:, 2] - self.threshold)):
|
||||
|
||||
# only consider iterations that might have resulted
|
||||
# in changing the number of (large) clusters
|
||||
new_cluster_size = _dendrogram[iteration, 3]
|
||||
@@ -481,6 +468,7 @@ class OracleClustering(BaseClustering):
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
embeddings: np.ndarray = None,
|
||||
segmentations: SlidingWindowFeature = None,
|
||||
file: AudioFile = None,
|
||||
frames: SlidingWindow = None,
|
||||
@@ -490,6 +478,9 @@ class OracleClustering(BaseClustering):
|
||||
|
||||
Parameters
|
||||
----------
|
||||
embeddings : (num_chunks, num_speakers, dimension) array, optional
|
||||
Sequence of embeddings. When provided, compute speaker centroids
|
||||
based on these embeddings.
|
||||
segmentations : (num_chunks, num_frames, num_speakers) array
|
||||
Binary segmentations.
|
||||
file : AudioFile
|
||||
@@ -503,6 +494,8 @@ class OracleClustering(BaseClustering):
|
||||
soft_clusters : (num_chunks, num_speakers, num_clusters) array
|
||||
Soft cluster assignment (the higher soft_clusters[c, s, k], the most likely
|
||||
the sth speaker of cth chunk belongs to kth cluster)
|
||||
centroids : (num_clusters, dimension), optional
|
||||
Clusters centroids if `embeddings` is provided, None otherwise.
|
||||
"""
|
||||
|
||||
num_chunks, num_frames, num_speakers = segmentations.data.shape
|
||||
@@ -532,7 +525,27 @@ class OracleClustering(BaseClustering):
|
||||
hard_clusters[c, i] = j
|
||||
soft_clusters[c, i, j] = 1.0
|
||||
|
||||
return hard_clusters, soft_clusters
|
||||
if embeddings is None:
|
||||
return hard_clusters, soft_clusters, None
|
||||
|
||||
(
|
||||
train_embeddings,
|
||||
train_chunk_idx,
|
||||
train_speaker_idx,
|
||||
) = self.filter_embeddings(
|
||||
embeddings,
|
||||
segmentations=segmentations,
|
||||
)
|
||||
|
||||
train_clusters = hard_clusters[train_chunk_idx, train_speaker_idx]
|
||||
centroids = np.vstack(
|
||||
[
|
||||
np.mean(train_embeddings[train_clusters == k], axis=0)
|
||||
for k in range(num_clusters)
|
||||
]
|
||||
)
|
||||
|
||||
return hard_clusters, soft_clusters, centroids
|
||||
|
||||
|
||||
class Clustering(Enum):
|
||||
|
||||
@@ -89,11 +89,20 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
|
||||
|
||||
Usage
|
||||
-----
|
||||
>>> pipeline = SpeakerDiarization()
|
||||
# perform (unconstrained) diarization
|
||||
>>> diarization = pipeline("/path/to/audio.wav")
|
||||
|
||||
# perform diarization, targetting exactly 4 speakers
|
||||
>>> diarization = pipeline("/path/to/audio.wav", num_speakers=4)
|
||||
|
||||
# perform diarization, with at least 2 speakers and at most 10 speakers
|
||||
>>> diarization = pipeline("/path/to/audio.wav", min_speakers=2, max_speakers=10)
|
||||
|
||||
# perform diarization and get one representative embedding per speaker
|
||||
>>> diarization, embeddings = pipeline("/path/to/audio.wav", return_embedding=True)
|
||||
>>> for s, speaker in enumerate(diarization.labels()):
|
||||
... # embeddings[s] is the embedding of speaker `speaker`
|
||||
|
||||
Hyper-parameters
|
||||
----------------
|
||||
segmentation.threshold
|
||||
@@ -417,6 +426,7 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
|
||||
num_speakers: int = None,
|
||||
min_speakers: int = None,
|
||||
max_speakers: int = None,
|
||||
return_embeddings: bool = False,
|
||||
hook: Optional[Callable] = None,
|
||||
) -> Annotation:
|
||||
"""Apply speaker diarization
|
||||
@@ -431,6 +441,8 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
|
||||
Minimum number of speakers. Has no effect when `num_speakers` is provided.
|
||||
max_speakers : int, optional
|
||||
Maximum number of speakers. Has no effect when `num_speakers` is provided.
|
||||
return_embeddings : bool, optional
|
||||
Return representative speaker embeddings.
|
||||
hook : callable, optional
|
||||
Callback called after each major steps of the pipeline as follows:
|
||||
hook(step_name, # human-readable name of current step
|
||||
@@ -444,6 +456,10 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
|
||||
-------
|
||||
diarization : Annotation
|
||||
Speaker diarization
|
||||
embeddings : np.array, optional
|
||||
Representative speaker embeddings such that `embeddings[i]` is the
|
||||
speaker embedding for i-th speaker in diarization.labels().
|
||||
Only returned when `return_embeddings` is True.
|
||||
"""
|
||||
|
||||
# setup hook (e.g. for debugging purposes)
|
||||
@@ -473,7 +489,11 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
|
||||
|
||||
# exit early when no speaker is ever active
|
||||
if np.nanmax(count.data) == 0.0:
|
||||
return Annotation(uri=file["uri"])
|
||||
diarization = Annotation(uri=file["uri"])
|
||||
if return_embeddings:
|
||||
return diarization, np.zeros((0, self._embedding.dimension))
|
||||
|
||||
return diarization
|
||||
|
||||
# binarize segmentation
|
||||
if self._segmentation.model.specifications.powerset:
|
||||
@@ -485,7 +505,7 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
|
||||
initial_state=False,
|
||||
)
|
||||
|
||||
if self.klustering == "OracleClustering":
|
||||
if self.klustering == "OracleClustering" and not return_embeddings:
|
||||
embeddings = None
|
||||
else:
|
||||
embeddings = self.get_embeddings(
|
||||
@@ -497,7 +517,7 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
|
||||
hook("embeddings", embeddings)
|
||||
# shape: (num_chunks, local_num_speakers, dimension)
|
||||
|
||||
hard_clusters, _ = self.clustering(
|
||||
hard_clusters, _, centroids = self.clustering(
|
||||
embeddings=embeddings,
|
||||
segmentations=binarized_segmentations,
|
||||
num_clusters=num_speakers,
|
||||
@@ -506,7 +526,8 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
|
||||
file=file, # <== for oracle clustering
|
||||
frames=self._frames, # <== for oracle clustering
|
||||
)
|
||||
# hard_clusters: (num_chunks, num_speakers)
|
||||
# hard_clusters: (num_chunks, num_speakers)
|
||||
# centroids: (num_speakers, dimension)
|
||||
|
||||
# reconstruct discrete diarization from raw hard clusters
|
||||
|
||||
@@ -530,20 +551,52 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline):
|
||||
)
|
||||
diarization.uri = file["uri"]
|
||||
|
||||
# when reference is available, use it to map hypothesized speakers
|
||||
# to reference speakers (this makes later error analysis easier
|
||||
# but does not modify the actual output of the diarization pipeline)
|
||||
if "annotation" in file and file["annotation"]:
|
||||
return self.optimal_mapping(file["annotation"], diarization)
|
||||
# at this point, `diarization` speaker labels are integers
|
||||
# from 0 to `num_speakers - 1`, aligned with `centroids` rows.
|
||||
|
||||
# when reference is not available, rename hypothesized speakers
|
||||
# to human-readable SPEAKER_00, SPEAKER_01, ...
|
||||
return diarization.rename_labels(
|
||||
{
|
||||
if "annotation" in file and file["annotation"]:
|
||||
# when reference is available, use it to map hypothesized speakers
|
||||
# to reference speakers (this makes later error analysis easier
|
||||
# but does not modify the actual output of the diarization pipeline)
|
||||
_, mapping = self.optimal_mapping(
|
||||
file["annotation"], diarization, return_mapping=True
|
||||
)
|
||||
|
||||
# in case there are more speakers in the hypothesis than in
|
||||
# the reference, those extra speakers are missing from `mapping`.
|
||||
# we add them back here
|
||||
mapping = {key: mapping.get(key, key) for key in diarization.labels()}
|
||||
|
||||
else:
|
||||
# when reference is not available, rename hypothesized speakers
|
||||
# to human-readable SPEAKER_00, SPEAKER_01, ...
|
||||
mapping = {
|
||||
label: expected_label
|
||||
for label, expected_label in zip(diarization.labels(), self.classes())
|
||||
}
|
||||
)
|
||||
|
||||
diarization = diarization.rename_labels(mapping=mapping)
|
||||
|
||||
# at this point, `diarization` speaker labels are strings (or mix of
|
||||
# strings and integers when reference is available and some hypothesis
|
||||
# speakers are not present in the reference)
|
||||
|
||||
if not return_embeddings:
|
||||
return diarization
|
||||
|
||||
# re-order centroids so that they match
|
||||
# the order given by diarization.labels()
|
||||
inverse_mapping = {label: index for index, label in mapping.items()}
|
||||
centroids = centroids[
|
||||
[inverse_mapping[label] for label in diarization.labels()]
|
||||
]
|
||||
|
||||
# FIXME: the number of centroids may be smaller than the number of speakers
|
||||
# in the annotation. This can happen if the number of active speakers
|
||||
# obtained from `speaker_count` for some frames is larger than the number
|
||||
# of clusters obtained from `clustering`. Will be fixed in the future
|
||||
|
||||
return diarization, centroids
|
||||
|
||||
def get_metric(self) -> GreedyDiarizationErrorRate:
|
||||
return GreedyDiarizationErrorRate(**self.der_variant)
|
||||
@@ -20,10 +20,11 @@
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
from typing import Mapping, Tuple, Union
|
||||
from typing import Dict, Mapping, Tuple, Union
|
||||
|
||||
import numpy as np
|
||||
from pyannote.core import Annotation, SlidingWindow, SlidingWindowFeature
|
||||
from pyannote.core.utils.types import Label
|
||||
from pyannote.metrics.diarization import DiarizationErrorRate
|
||||
|
||||
from pyannote.audio.core.inference import Inference
|
||||
@@ -74,8 +75,10 @@ class SpeakerDiarizationMixin:
|
||||
|
||||
@staticmethod
|
||||
def optimal_mapping(
|
||||
reference: Union[Mapping, Annotation], hypothesis: Annotation
|
||||
) -> Annotation:
|
||||
reference: Union[Mapping, Annotation],
|
||||
hypothesis: Annotation,
|
||||
return_mapping: bool = False,
|
||||
) -> Union[Annotation, Tuple[Annotation, Dict[Label, Label]]]:
|
||||
"""Find the optimal bijective mapping between reference and hypothesis labels
|
||||
|
||||
Parameters
|
||||
@@ -84,13 +87,19 @@ class SpeakerDiarizationMixin:
|
||||
Reference annotation. Can be an Annotation instance or
|
||||
a mapping with an "annotation" key.
|
||||
hypothesis : Annotation
|
||||
Hypothesized annotation.
|
||||
return_mapping : bool, optional
|
||||
Return the label mapping itself along with the mapped annotation. Defaults to False.
|
||||
|
||||
Returns
|
||||
-------
|
||||
mapped : Annotation
|
||||
Hypothesis mapped to reference speakers.
|
||||
|
||||
mapping : dict, optional
|
||||
Mapping between hypothesis (key) and reference (value) labels
|
||||
Only returned if `return_mapping` is True.
|
||||
"""
|
||||
|
||||
if isinstance(reference, Mapping):
|
||||
reference = reference["annotation"]
|
||||
annotated = reference["annotated"] if "annotated" in reference else None
|
||||
@@ -100,7 +109,13 @@ class SpeakerDiarizationMixin:
|
||||
mapping = DiarizationErrorRate().optimal_mapping(
|
||||
reference, hypothesis, uem=annotated
|
||||
)
|
||||
return hypothesis.rename_labels(mapping=mapping)
|
||||
mapped_hypothesis = hypothesis.rename_labels(mapping=mapping)
|
||||
|
||||
if return_mapping:
|
||||
return mapped_hypothesis, mapping
|
||||
|
||||
else:
|
||||
return mapped_hypothesis
|
||||
|
||||
# TODO: get rid of onset/offset (binarization should be applied before calling speaker_count)
|
||||
# TODO: get rid of warm-up parameter (trimming should be applied before calling speaker_count)
|
||||
@@ -171,7 +186,8 @@ class SpeakerDiarizationMixin:
|
||||
Returns
|
||||
-------
|
||||
continuous_diarization : Annotation
|
||||
Continuous diarization
|
||||
Continuous diarization, with speaker labels as integers,
|
||||
corresponding to the speaker indices in the discrete diarization.
|
||||
"""
|
||||
|
||||
binarize = Binarize(
|
||||
|
||||
Reference in new issue
Block a user