mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
wip: add constrained HAC
This commit is contained in:
1 parent
bc0920fc4e
commit
fc8736128f
1 file changed
+313
-1
@@ -25,16 +25,20 @@
|
||||
|
||||
import random
|
||||
from enum import Enum
|
||||
from functools import lru_cache
|
||||
from typing import Tuple
|
||||
|
||||
import numpy as np
|
||||
from einops import rearrange
|
||||
from pyannote.core import SlidingWindow, SlidingWindowFeature
|
||||
from pyannote.core.utils.distance import pdist, to_condensed, to_squared
|
||||
from pyannote.core.utils.hierarchy import _average_pooling_func
|
||||
from pyannote.pipeline import Pipeline
|
||||
from pyannote.pipeline.parameter import Categorical, Integer, Uniform
|
||||
from scipy.cluster.hierarchy import fcluster, linkage
|
||||
from scipy.optimize import linear_sum_assignment
|
||||
from scipy.spatial.distance import cdist
|
||||
from scipy.spatial.distance import cdist, squareform
|
||||
from scipy.stats import ttest_rel
|
||||
|
||||
from pyannote.audio.core.io import AudioFile
|
||||
from pyannote.audio.pipelines.utils import oracle_segmentation
|
||||
@@ -464,6 +468,313 @@ class AgglomerativeClustering(BaseClustering):
|
||||
return clusters
|
||||
|
||||
|
||||
class ConstrainedAgglomerativeClustering(BaseClustering):
|
||||
"""Agglomerative clustering
|
||||
|
||||
Parameters
|
||||
----------
|
||||
metric : {"cosine", "euclidean", ...}, optional
|
||||
Distance metric to use. Defaults to "cosine".
|
||||
|
||||
Hyper-parameters
|
||||
----------------
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
metric: str = "cosine",
|
||||
max_num_embeddings: int = np.inf,
|
||||
constrained_assignment: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
metric=metric,
|
||||
max_num_embeddings=max_num_embeddings,
|
||||
constrained_assignment=constrained_assignment,
|
||||
)
|
||||
|
||||
self.distance_threshold = Uniform(0.6, 1.0)
|
||||
self.constraint_threshold = Uniform(0.0, 0.2)
|
||||
self.min_cluster_size = Integer(1, 20)
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def pvalue(
|
||||
self,
|
||||
breakable: int,
|
||||
broken_before: int,
|
||||
broken_after: int,
|
||||
) -> float:
|
||||
"""
|
||||
Compute p-value for the null hypothesis that the number of broken constraints increases
|
||||
|
||||
Parameters
|
||||
----------
|
||||
breakable : int
|
||||
Number of constraints
|
||||
broken_before : int
|
||||
Number of broken constraints before merge
|
||||
broken_after: int
|
||||
Number of broken constraints after merge
|
||||
|
||||
Returns
|
||||
-------
|
||||
p_value : float
|
||||
p-value for the null hypothesis that the number of broken constraints increases
|
||||
"""
|
||||
|
||||
if broken_before == broken_after:
|
||||
return 1.0
|
||||
|
||||
return ttest_rel(
|
||||
[0] * (breakable - broken_before) + [1] * broken_before,
|
||||
[0] * (breakable - broken_after) + [1] * broken_after,
|
||||
alternative="less",
|
||||
).pvalue
|
||||
|
||||
def cluster(
|
||||
self,
|
||||
embeddings: np.ndarray,
|
||||
min_clusters: int,
|
||||
max_clusters: int,
|
||||
num_clusters: int = None,
|
||||
chunk_idx: np.ndarray = None,
|
||||
**kwargs,
|
||||
):
|
||||
"""
|
||||
|
||||
Parameters
|
||||
----------
|
||||
embeddings : (num_embeddings, dimension) array
|
||||
Embeddings
|
||||
min_clusters : int
|
||||
Minimum number of clusters
|
||||
max_clusters : int
|
||||
Maximum number of clusters
|
||||
num_clusters : int, optional
|
||||
Actual number of clusters. Default behavior is to estimate it based
|
||||
on values provided for `min_clusters`, `max_clusters`, and `threshold`.
|
||||
|
||||
Returns
|
||||
-------
|
||||
clusters : (num_embeddings, ) array
|
||||
0-indexed cluster indices.
|
||||
"""
|
||||
|
||||
self.pvalue.cache_clear()
|
||||
|
||||
# obtain number of original observations
|
||||
num_embeddings, dimension = embeddings.shape
|
||||
|
||||
if num_embeddings == 1:
|
||||
return np.zeros((1,), dtype=np.uint8)
|
||||
|
||||
# heuristic to reduce self.min_cluster_size when num_embeddings is very small
|
||||
# (0.1 value is kind of arbitrary, though)
|
||||
min_cluster_size = min(
|
||||
self.min_cluster_size, max(1, round(0.1 * num_embeddings))
|
||||
)
|
||||
|
||||
n = num_embeddings
|
||||
X = embeddings
|
||||
|
||||
# compute similarity matrix
|
||||
d = pdist(X, metric=self.metric)
|
||||
|
||||
# convert condensed pdist matrix for the `n` original observation to a
|
||||
# condensed pdist matrix for the `2n-1` clusters (including the `n`
|
||||
# original observations) that will exist at some point during the process.
|
||||
|
||||
distance = np.infty * np.ones((2 * n - 1) * (2 * n - 2) // 2)
|
||||
distance[
|
||||
to_condensed(2 * n - 1, *to_squared(n, np.arange(n * (n - 1) // 2)))
|
||||
] = d
|
||||
|
||||
# clusters[j] contains the index of the cluster to which
|
||||
# the jth observation is currently assigned
|
||||
clusters = np.arange(n)
|
||||
|
||||
# size[k] contains the current size of kth cluster
|
||||
size = np.zeros(2 * n - 1, dtype=np.int16)
|
||||
size[:n] = 1
|
||||
|
||||
# centroids[k] contains the centroid of kth cluster
|
||||
centroids = np.zeros((2 * n - 1, dimension))
|
||||
# at the beginning, each observation is assigned to its own cluster
|
||||
centroids[:n, :] = X
|
||||
|
||||
# clustering tree (aka dendrogram)
|
||||
# dendrogram[i, 0] and dendrogram[i, 1] are merged at ith iteration
|
||||
# dendrogram[i, 2] is the distance between dendrogram[i, 0] and dendrogram[i, 1]
|
||||
# dendrogram[i, 3] is the total number of original observation in the newly formed cluster
|
||||
dendrogram = np.zeros((n - 1, 4))
|
||||
|
||||
# cannot-link constraints (False/0: free to link, True/1: cannot link)
|
||||
constraints = pdist(chunk_idx, metric="equal")
|
||||
breakable = np.sum(constraints)
|
||||
|
||||
# breaking[u, v] is the number of "cannot-link" constraints
|
||||
# that merging clusters u and v would break
|
||||
breaking = np.zeros((2 * n - 1, 2 * n - 1), dtype=int)
|
||||
breaking[:n, :n] = squareform(constraints)
|
||||
|
||||
# current number of broken constraints
|
||||
broken = np.zeros((1,), dtype=int)
|
||||
|
||||
def merge(u, v, iteration):
|
||||
"""Merge two clusters
|
||||
|
||||
Parameters
|
||||
----------
|
||||
u, v : int
|
||||
Indices of clusters to merge.
|
||||
iteration : int
|
||||
Current clustering iteration.
|
||||
|
||||
Returns
|
||||
-------
|
||||
uv : int
|
||||
Indices of resulting cluster.
|
||||
|
||||
"""
|
||||
# global broken
|
||||
# print(broken)
|
||||
|
||||
# merge clusters index in condensed matrices
|
||||
uv = to_condensed(2 * n - 1, u, v)
|
||||
|
||||
# keep track of ...
|
||||
# ... which clusters are merged at this iteration
|
||||
dendrogram[iteration, 0] = v if size[v] > size[u] else v
|
||||
dendrogram[iteration, 1] = u if dendrogram[iteration, 0] == v else v
|
||||
|
||||
# ... their distance
|
||||
dendrogram[iteration, 2] = distance[uv]
|
||||
|
||||
# ... the size of the newly formed cluster
|
||||
dendrogram[iteration, 3] = size[u] + size[v]
|
||||
size[n + iteration] = size[u] + size[v]
|
||||
|
||||
# ... the number of broken constraints
|
||||
broken[0] += breaking[u, v]
|
||||
|
||||
# compute "representation" of newly formed cluster
|
||||
centroids[n + iteration] = _average_pooling_func(u, v, S=size, C=centroids)
|
||||
|
||||
# merged clusters are now empty...
|
||||
size[u] = 0
|
||||
size[v] = 0
|
||||
|
||||
# move observations of merged clusters into the newly formed cluster
|
||||
clusters[clusters == u] = n + iteration
|
||||
clusters[clusters == v] = n + iteration
|
||||
|
||||
breaking_by_row = breaking[u, :] + breaking[v, :]
|
||||
breaking_by_col = breaking[:, u] + breaking[:, v]
|
||||
breaking[n + iteration, :] = breaking_by_row
|
||||
breaking[:, n + iteration] = breaking_by_col
|
||||
breaking[u, :] = 0
|
||||
breaking[v, :] = 0
|
||||
breaking[:, u] = 0
|
||||
breaking[:, v] = 0
|
||||
|
||||
# compute distance to newly formed cluster
|
||||
# (only for clusters that still exists, i.e. those that are not empty)
|
||||
empty = size[: n + iteration] == 0
|
||||
i_ = to_condensed(
|
||||
2 * n - 1, n + iteration, np.arange(n + iteration)[~empty]
|
||||
)
|
||||
distance[i_] = cdist(
|
||||
centroids[np.newaxis, n + iteration, :],
|
||||
centroids[: n + iteration, :][~empty, :],
|
||||
metric=self.metric,
|
||||
)
|
||||
|
||||
# condensed indices of all (u, _) and (v, _) pairs
|
||||
_u = to_condensed(2 * n - 1, u, np.arange(u))
|
||||
u_ = to_condensed(2 * n - 1, u, np.arange(u + 1, n + iteration))
|
||||
_v = to_condensed(2 * n - 1, v, np.arange(v))
|
||||
v_ = to_condensed(2 * n - 1, v, np.arange(v + 1, n + iteration))
|
||||
|
||||
# distance to merged clusters u and v no longer exist
|
||||
distance[_u] = distance[u_] = distance[_v] = distance[v_] = np.infty
|
||||
|
||||
# is this needed
|
||||
k = to_condensed(2 * n - 1, n + iteration, np.arange(n + iteration)[empty])
|
||||
distance[k] = np.infty
|
||||
|
||||
return np.array(clusters)
|
||||
|
||||
iteration = 0
|
||||
|
||||
# iterate until one cluster remains
|
||||
current_clusters = np.array(clusters)
|
||||
|
||||
while iteration < n - 1:
|
||||
while True:
|
||||
# find two most similar clusters
|
||||
k = np.argmin(distance)
|
||||
u, v = to_squared(2 * n - 1, k)
|
||||
|
||||
# if there are too far apart, stop
|
||||
if distance[k] > self.distance_threshold:
|
||||
break
|
||||
|
||||
pvalue = self.pvalue(breakable, broken[0], broken[0] + breaking[u, v])
|
||||
|
||||
# if they are breaking too many constraints, try next best pair
|
||||
if pvalue < self.constraint_threshold:
|
||||
distance[k] = np.infty
|
||||
else:
|
||||
break
|
||||
|
||||
if distance[k] > self.distance_threshold:
|
||||
break
|
||||
|
||||
current_clusters = merge(u, v, iteration)
|
||||
|
||||
iteration += 1
|
||||
|
||||
# convert clusters to 0-based indices
|
||||
_, clusters = np.unique(current_clusters, return_inverse=True)
|
||||
|
||||
# split clusters into two categories based on their number of items:
|
||||
# large clusters vs. small clusters
|
||||
cluster_unique, cluster_counts = np.unique(
|
||||
clusters,
|
||||
return_counts=True,
|
||||
)
|
||||
large_clusters = cluster_unique[cluster_counts >= min_cluster_size]
|
||||
num_large_clusters = len(large_clusters)
|
||||
|
||||
if num_large_clusters == 0:
|
||||
clusters[:] = 0
|
||||
return clusters
|
||||
|
||||
small_clusters = cluster_unique[cluster_counts < min_cluster_size]
|
||||
if len(small_clusters) == 0:
|
||||
return clusters
|
||||
|
||||
# re-assign each small cluster to the most similar large cluster based on their respective centroids
|
||||
large_centroids = np.vstack(
|
||||
[
|
||||
np.mean(embeddings[clusters == large_k], axis=0)
|
||||
for large_k in large_clusters
|
||||
]
|
||||
)
|
||||
small_centroids = np.vstack(
|
||||
[
|
||||
np.mean(embeddings[clusters == small_k], axis=0)
|
||||
for small_k in small_clusters
|
||||
]
|
||||
)
|
||||
centroids_cdist = cdist(large_centroids, small_centroids, metric=self.metric)
|
||||
for small_k, large_k in enumerate(np.argmin(centroids_cdist, axis=0)):
|
||||
clusters[clusters == small_clusters[small_k]] = large_clusters[large_k]
|
||||
|
||||
# re-number clusters from 0 to num_large_clusters
|
||||
_, clusters = np.unique(clusters, return_inverse=True)
|
||||
return clusters
|
||||
|
||||
|
||||
class OracleClustering(BaseClustering):
|
||||
"""Oracle clustering"""
|
||||
|
||||
@@ -551,4 +862,5 @@ class OracleClustering(BaseClustering):
|
||||
|
||||
class Clustering(Enum):
|
||||
AgglomerativeClustering = AgglomerativeClustering
|
||||
ConstrainedAgglomerativeClustering = ConstrainedAgglomerativeClustering
|
||||
OracleClustering = OracleClustering
|
||||
Reference in new issue
Block a user