From c7581cfa18552e8d45136d403cf84dcfe0e54c03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20BREDIN?= Date: Fri, 8 Nov 2024 10:25:54 +0100 Subject: [PATCH] wip: use hooks in clustering step --- pyannote/audio/pipelines/clustering.py | 10 +++++++++- pyannote/audio/pipelines/speaker_diarization.py | 4 ++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/pyannote/audio/pipelines/clustering.py b/pyannote/audio/pipelines/clustering.py index 18b3b527..6460b6d3 100644 --- a/pyannote/audio/pipelines/clustering.py +++ b/pyannote/audio/pipelines/clustering.py @@ -24,7 +24,7 @@ import random from enum import Enum -from typing import Optional, Tuple +from typing import Callable, Optional, Tuple import numpy as np from einops import rearrange @@ -209,6 +209,7 @@ class BaseClustering(Pipeline): num_clusters: Optional[int] = None, min_clusters: Optional[int] = None, max_clusters: Optional[int] = None, + hook: Optional[Callable] = None, **kwargs, ) -> np.ndarray: """Apply clustering @@ -267,6 +268,7 @@ class BaseClustering(Pipeline): min_clusters=min_clusters, max_clusters=max_clusters, num_clusters=num_clusters, + hook=hook, ) hard_clusters, soft_clusters, centroids = self.assign_embeddings( @@ -326,6 +328,7 @@ class AgglomerativeClustering(BaseClustering): min_clusters: Optional[int] = None, max_clusters: Optional[int] = None, num_clusters: Optional[int] = None, + hook: Optional[Callable] = None, ): """ @@ -374,9 +377,13 @@ class AgglomerativeClustering(BaseClustering): embeddings, method=self.method, metric=self.metric ) + hook("clustering.dendrogram", dendrogram) + # apply the predefined threshold clusters = fcluster(dendrogram, self.threshold, criterion="distance") - 1 + hook("clustering.fcluster", clusters) + # split clusters into two categories based on their number of items: # large clusters vs. small clusters cluster_unique, cluster_counts = np.unique( @@ -470,6 +477,7 @@ class AgglomerativeClustering(BaseClustering): # re-number clusters from 0 to num_large_clusters _, clusters = np.unique(clusters, return_inverse=True) + return clusters diff --git a/pyannote/audio/pipelines/speaker_diarization.py b/pyannote/audio/pipelines/speaker_diarization.py index e0d43e30..8e8e6180 100644 --- a/pyannote/audio/pipelines/speaker_diarization.py +++ b/pyannote/audio/pipelines/speaker_diarization.py @@ -553,6 +553,7 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline): max_clusters=max_speakers, file=file, # <== for oracle clustering frames=self._segmentation.model.receptive_field, # <== for oracle clustering + hook=hook, ) # hard_clusters: (num_chunks, num_speakers) # centroids: (num_speakers, dimension) @@ -590,6 +591,9 @@ class SpeakerDiarization(SpeakerDiarizationMixin, Pipeline): # shape: (num_chunks, num_speakers) hard_clusters[inactive_speakers] = -2 + + hook("hard_clusters", hard_clusters) + discrete_diarization = self.reconstruct( segmentations, hard_clusters,