mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
feat: add resegmentation pipeline (#652)
* feat: add Introspection.frames property * feat: add support for mismatched number of classes in `permutate` utility function
This commit is contained in:
1 parent
9311a79541
commit
f8786e2b5d
7 files changed
+506
-214
No files matched your search
+163
-140
@@ -20,9 +20,10 @@
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import warnings
|
||||
from dataclasses import dataclass
|
||||
from importlib import import_module
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Text, Tuple, Union
|
||||
@@ -40,6 +41,7 @@ from semver import VersionInfo
|
||||
from pyannote.audio import __version__
|
||||
from pyannote.audio.core.io import Audio
|
||||
from pyannote.audio.core.task import Problem, Resolution, Specifications, Task
|
||||
from pyannote.core import SlidingWindow
|
||||
|
||||
CACHE_DIR = os.getenv(
|
||||
"PYANNOTE_CACHE",
|
||||
@@ -48,25 +50,165 @@ CACHE_DIR = os.getenv(
|
||||
HF_PYTORCH_WEIGHTS_NAME = "pytorch_model.bin"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Introspection:
|
||||
"""Model introspection
|
||||
|
||||
# TODO: make it a regular class
|
||||
# TODO: add from_model class method that will replace Model.introspection property
|
||||
|
||||
# minimum number of input samples
|
||||
Parameters
|
||||
----------
|
||||
min_num_samples: int
|
||||
# corresponding minimum number of output frames
|
||||
Minimum number of input samples
|
||||
min_num_frames: int
|
||||
# number of input samples leading to an increase of number of output frames
|
||||
Corresponding minimum number of output frames
|
||||
inc_num_samples: int
|
||||
# corresponding increase in number of output frames
|
||||
Number of input samples leading to an increase of number of output frames
|
||||
inc_num_frames: int
|
||||
# output dimension
|
||||
Corresponding increase in number of output frames
|
||||
dimension: int
|
||||
Output dimension
|
||||
sample_rate: int
|
||||
Expected input sample rate
|
||||
|
||||
Usage
|
||||
-----
|
||||
>>> introspection = Introspection.from_model(model)
|
||||
>>> isinstance(introspection.frames, SlidingWindow)
|
||||
>>> num_samples = 16000 # 1s at 16kHz
|
||||
>>> num_frames, dimension = introspection(num_samples)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
min_num_samples: int,
|
||||
min_num_frames: int,
|
||||
inc_num_samples: int,
|
||||
inc_num_frames: int,
|
||||
dimension: int,
|
||||
sample_rate: int,
|
||||
):
|
||||
super().__init__()
|
||||
self.min_num_samples = min_num_samples
|
||||
self.min_num_frames = min_num_frames
|
||||
self.inc_num_samples = inc_num_samples
|
||||
self.inc_num_frames = inc_num_frames
|
||||
self.dimension = dimension
|
||||
self.sample_rate = sample_rate
|
||||
|
||||
@classmethod
|
||||
def from_model(cls, model: "Model", task: str = None) -> Introspection:
|
||||
|
||||
specifications = model.specifications
|
||||
if task is not None:
|
||||
specifications = specifications[task]
|
||||
|
||||
example_input_array = model.example_input_array
|
||||
batch_size, num_channels, num_samples = example_input_array.shape
|
||||
example_input_array = torch.randn(
|
||||
(1, num_channels, num_samples),
|
||||
dtype=example_input_array.dtype,
|
||||
layout=example_input_array.layout,
|
||||
device=example_input_array.device,
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
# dichotomic search of "min_num_samples"
|
||||
lower, upper, min_num_samples = 1, num_samples, None
|
||||
while True:
|
||||
num_samples = (lower + upper) // 2
|
||||
try:
|
||||
with torch.no_grad():
|
||||
frames = model(example_input_array[:, :, :num_samples])
|
||||
if task is not None:
|
||||
frames = frames[task]
|
||||
except Exception:
|
||||
lower = num_samples
|
||||
else:
|
||||
min_num_samples = num_samples
|
||||
if specifications.resolution == Resolution.FRAME:
|
||||
_, min_num_frames, dimension = frames.shape
|
||||
elif specifications.resolution == Resolution.CHUNK:
|
||||
min_num_frames, dimension = frames.shape
|
||||
else:
|
||||
# should never happen
|
||||
pass
|
||||
upper = num_samples
|
||||
|
||||
if lower + 1 == upper:
|
||||
break
|
||||
|
||||
# if "min_num_samples" is still None at this point, it means that
|
||||
# the forward pass always failed and raised an exception. most likely,
|
||||
# it means that there is a problem with the model definition.
|
||||
# we try again without catching the exception to help the end user debug
|
||||
# their model
|
||||
if min_num_samples is None:
|
||||
frames = model(example_input_array)
|
||||
|
||||
# corner case for chunk-level tasks
|
||||
if specifications.resolution == Resolution.CHUNK:
|
||||
return cls(
|
||||
min_num_samples=min_num_samples,
|
||||
min_num_frames=1,
|
||||
inc_num_samples=0,
|
||||
inc_num_frames=0,
|
||||
dimension=dimension,
|
||||
sample_rate=model.hparams.sample_rate,
|
||||
)
|
||||
|
||||
# search reasonable upper bound for "inc_num_samples"
|
||||
while True:
|
||||
num_samples = 2 * min_num_samples
|
||||
example_input_array = torch.randn(
|
||||
(1, num_channels, num_samples),
|
||||
dtype=example_input_array.dtype,
|
||||
layout=example_input_array.layout,
|
||||
device=example_input_array.device,
|
||||
requires_grad=False,
|
||||
)
|
||||
with torch.no_grad():
|
||||
frames = model(example_input_array)
|
||||
if task is not None:
|
||||
frames = frames[task]
|
||||
num_frames = frames.shape[1]
|
||||
if num_frames > min_num_frames:
|
||||
break
|
||||
|
||||
# dichotomic search of "inc_num_samples"
|
||||
lower, upper = min_num_samples, num_samples
|
||||
while True:
|
||||
num_samples = (lower + upper) // 2
|
||||
example_input_array = torch.randn(
|
||||
(1, num_channels, num_samples),
|
||||
dtype=example_input_array.dtype,
|
||||
layout=example_input_array.layout,
|
||||
device=example_input_array.device,
|
||||
requires_grad=False,
|
||||
)
|
||||
with torch.no_grad():
|
||||
frames = model(example_input_array)
|
||||
if task is not None:
|
||||
frames = frames[task]
|
||||
num_frames = frames.shape[1]
|
||||
if num_frames > min_num_frames:
|
||||
inc_num_frames = num_frames - min_num_frames
|
||||
inc_num_samples = num_samples - min_num_samples
|
||||
upper = num_samples
|
||||
else:
|
||||
lower = num_samples
|
||||
|
||||
if lower + 1 == upper:
|
||||
break
|
||||
|
||||
return cls(
|
||||
min_num_samples=min_num_samples,
|
||||
min_num_frames=min_num_frames,
|
||||
inc_num_samples=inc_num_samples,
|
||||
inc_num_frames=inc_num_frames,
|
||||
dimension=dimension,
|
||||
sample_rate=model.hparams.sample_rate,
|
||||
)
|
||||
|
||||
def __call__(self, num_samples: int) -> Tuple[int, int]:
|
||||
"""Estimate output shape
|
||||
"""Predict output shape, given number of input samples
|
||||
|
||||
Parameters
|
||||
----------
|
||||
@@ -79,7 +221,6 @@ class Introspection:
|
||||
Number of output frames
|
||||
dimension : int
|
||||
Dimension of output frames
|
||||
|
||||
"""
|
||||
|
||||
if num_samples < self.min_num_samples:
|
||||
@@ -92,6 +233,13 @@ class Introspection:
|
||||
self.dimension,
|
||||
)
|
||||
|
||||
@property
|
||||
def frames(self) -> SlidingWindow:
|
||||
# HACK to support model trained before 'sample_rate' was an Introspection attribute
|
||||
sample_rate = getattr(self, "sample_rate", 16000)
|
||||
step = (self.inc_num_samples / self.inc_num_frames) / sample_rate
|
||||
return SlidingWindow(start=0.0, step=step, duration=step)
|
||||
|
||||
def __len__(self):
|
||||
# makes it possible to do something like:
|
||||
# multi_task = len(introspection) > 1
|
||||
@@ -203,131 +351,6 @@ class Model(pl.LightningModule):
|
||||
# (e.g. the final classification and activation layers)
|
||||
pass
|
||||
|
||||
def helper_introspection(
|
||||
self,
|
||||
specifications: Specifications,
|
||||
task: str = None,
|
||||
) -> Introspection:
|
||||
"""Helper function for model introspection
|
||||
|
||||
Parameters
|
||||
----------
|
||||
specifications : Specifications
|
||||
Task specifications.
|
||||
task : str, optional
|
||||
Task name.
|
||||
|
||||
Returns
|
||||
-------
|
||||
introspection : Introspection
|
||||
Model introspection.
|
||||
"""
|
||||
|
||||
example_input_array = self.example_input_array
|
||||
batch_size, num_channels, num_samples = example_input_array.shape
|
||||
example_input_array = torch.randn(
|
||||
(1, num_channels, num_samples),
|
||||
dtype=example_input_array.dtype,
|
||||
layout=example_input_array.layout,
|
||||
device=example_input_array.device,
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
# dichotomic search of "min_num_samples"
|
||||
lower, upper, min_num_samples = 1, num_samples, None
|
||||
while True:
|
||||
num_samples = (lower + upper) // 2
|
||||
try:
|
||||
with torch.no_grad():
|
||||
frames = self(example_input_array[:, :, :num_samples])
|
||||
if task is not None:
|
||||
frames = frames[task]
|
||||
except Exception:
|
||||
lower = num_samples
|
||||
else:
|
||||
min_num_samples = num_samples
|
||||
if specifications.resolution == Resolution.FRAME:
|
||||
_, min_num_frames, dimension = frames.shape
|
||||
elif specifications.resolution == Resolution.CHUNK:
|
||||
min_num_frames, dimension = frames.shape
|
||||
else:
|
||||
# should never happen
|
||||
pass
|
||||
upper = num_samples
|
||||
|
||||
if lower + 1 == upper:
|
||||
break
|
||||
|
||||
# if "min_num_samples" is still None at this point, it means that
|
||||
# the forward pass always failed and raised an exception. most likely,
|
||||
# it means that there is a problem with the model definition.
|
||||
# we try again without catching the exception to help the end user debug
|
||||
# their model
|
||||
if min_num_samples is None:
|
||||
frames = self(example_input_array)
|
||||
|
||||
# corner case for chunk-level tasks
|
||||
if specifications.resolution == Resolution.CHUNK:
|
||||
return Introspection(
|
||||
min_num_samples=min_num_samples,
|
||||
min_num_frames=1,
|
||||
inc_num_samples=0,
|
||||
inc_num_frames=0,
|
||||
dimension=dimension,
|
||||
)
|
||||
|
||||
# search reasonable upper bound for "inc_num_samples"
|
||||
while True:
|
||||
num_samples = 2 * min_num_samples
|
||||
example_input_array = torch.randn(
|
||||
(1, num_channels, num_samples),
|
||||
dtype=example_input_array.dtype,
|
||||
layout=example_input_array.layout,
|
||||
device=example_input_array.device,
|
||||
requires_grad=False,
|
||||
)
|
||||
with torch.no_grad():
|
||||
frames = self(example_input_array)
|
||||
if task is not None:
|
||||
frames = frames[task]
|
||||
num_frames = frames.shape[1]
|
||||
if num_frames > min_num_frames:
|
||||
break
|
||||
|
||||
# dichotomic search of "inc_num_samples"
|
||||
lower, upper = min_num_samples, num_samples
|
||||
while True:
|
||||
num_samples = (lower + upper) // 2
|
||||
example_input_array = torch.randn(
|
||||
(1, num_channels, num_samples),
|
||||
dtype=example_input_array.dtype,
|
||||
layout=example_input_array.layout,
|
||||
device=example_input_array.device,
|
||||
requires_grad=False,
|
||||
)
|
||||
with torch.no_grad():
|
||||
frames = self(example_input_array)
|
||||
if task is not None:
|
||||
frames = frames[task]
|
||||
num_frames = frames.shape[1]
|
||||
if num_frames > min_num_frames:
|
||||
inc_num_frames = num_frames - min_num_frames
|
||||
inc_num_samples = num_samples - min_num_samples
|
||||
upper = num_samples
|
||||
else:
|
||||
lower = num_samples
|
||||
|
||||
if lower + 1 == upper:
|
||||
break
|
||||
|
||||
return Introspection(
|
||||
min_num_samples=min_num_samples,
|
||||
min_num_frames=min_num_frames,
|
||||
inc_num_samples=inc_num_samples,
|
||||
inc_num_frames=inc_num_frames,
|
||||
dimension=dimension,
|
||||
)
|
||||
|
||||
@property
|
||||
def introspection(self) -> Union[Introspection, Dict[Text, Introspection]]:
|
||||
"""Introspection
|
||||
@@ -343,12 +366,12 @@ class Model(pl.LightningModule):
|
||||
|
||||
if self.is_multi_task:
|
||||
self._introspection = {
|
||||
name: self.helper_introspection(specs, task=name)
|
||||
for name, specs in self.specifications.items()
|
||||
name: Introspection.from_model(self, task=name)
|
||||
for name in self.specifications
|
||||
}
|
||||
# TODO: raises an error in case of multiple tasks with different introspections
|
||||
else:
|
||||
self._introspection = self.helper_introspection(self.specifications)
|
||||
self._introspection = Introspection.from_model(self)
|
||||
|
||||
return self._introspection
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
# SOFTWARE.
|
||||
|
||||
from .overlapped_speech_detection import OverlappedSpeechDetection
|
||||
from .resegmentation import Resegmentation
|
||||
from .segmentation import Segmentation
|
||||
from .speaker_diarization import SpeakerDiarization
|
||||
from .voice_activity_detection import VoiceActivityDetection
|
||||
@@ -33,4 +34,5 @@ __all__ = [
|
||||
"Segmentation",
|
||||
"OverlappedSpeechDetection",
|
||||
"SpeakerDiarization",
|
||||
"Resegmentation",
|
||||
]
|
||||
@@ -0,0 +1,194 @@
|
||||
# MIT License
|
||||
#
|
||||
# Copyright (c) 2018-2021 CNRS
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
"""Resegmentation pipeline"""
|
||||
|
||||
from typing import Text
|
||||
|
||||
import numpy as np
|
||||
|
||||
from pyannote.audio import Inference
|
||||
from pyannote.audio.core.io import AudioFile
|
||||
from pyannote.audio.core.pipeline import Pipeline
|
||||
from pyannote.audio.pipelines.utils import PipelineModel, get_devices, get_model
|
||||
from pyannote.audio.utils.permutation import permutate
|
||||
from pyannote.audio.utils.signal import Binarize
|
||||
from pyannote.core import Annotation, SlidingWindowFeature
|
||||
from pyannote.metrics.diarization import GreedyDiarizationErrorRate
|
||||
from pyannote.pipeline.parameter import Uniform
|
||||
|
||||
|
||||
class Resegmentation(Pipeline):
|
||||
"""Resegmentation pipeline
|
||||
|
||||
This pipeline relies on a pretrained segmentation model to improve an existing diarization
|
||||
hypothesis. Resegmentation is done locally by sliding the segmentation model over the whole
|
||||
file. For each position of the sliding window, we find the optimal mapping between the input
|
||||
diarization and the output of the segmentation model and permutate the latter accordingly.
|
||||
Permutated local segmentations scores are then aggregated over time and postprocessed using
|
||||
hysteresis thresholding.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
segmentation : Model, str, or dict, optional
|
||||
Pretrained segmentation model. Defaults to "pyannote/Segmentation-PyanNet-DIHARD".
|
||||
See pyannote.audio.pipelines.utils.get_model for supported format.
|
||||
diarization : str, optional
|
||||
File key to use as input diarization. Defaults to "diarization".
|
||||
|
||||
Hyper-parameters
|
||||
----------------
|
||||
onset, offset : float
|
||||
Onset/offset detection thresholds
|
||||
min_duration_on : float
|
||||
Remove speaker turn shorter than that many seconds.
|
||||
min_duration_off : float
|
||||
Fill same-speaker gaps shorter than that many seconds.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
segmentation: PipelineModel = "pyannote/Segmentation-PyanNet-DIHARD",
|
||||
batch_size: int = 32,
|
||||
diarization: Text = "diarization",
|
||||
):
|
||||
|
||||
super().__init__()
|
||||
|
||||
self.segmentation = segmentation
|
||||
self.diarization = diarization
|
||||
self.batch_size = batch_size
|
||||
|
||||
# load model and send it to GPU (when available and not already on GPU)
|
||||
model = get_model(segmentation)
|
||||
if model.device.type == "cpu":
|
||||
(segmentation_device,) = get_devices(needs=1)
|
||||
model.to(segmentation_device)
|
||||
|
||||
self.audio_ = model.audio
|
||||
|
||||
# number of speakers in output of segmentation model
|
||||
self.num_frames_in_chunk_, self.seg_num_speakers_ = model.introspection(
|
||||
round(model.specifications.duration * model.hparams.sample_rate)
|
||||
)
|
||||
|
||||
# output frames as SlidingWindow instances
|
||||
self.seg_frames_ = model.introspection.frames
|
||||
|
||||
# prepare segmentation model for inference
|
||||
self.seg_inference_ = Inference(
|
||||
model,
|
||||
window="sliding",
|
||||
skip_aggregation=True,
|
||||
duration=model.specifications.duration,
|
||||
batch_size=32,
|
||||
)
|
||||
|
||||
# hyper-parameters used for hysteresis thresholding
|
||||
self.onset = Uniform(0.0, 1.0)
|
||||
self.offset = Uniform(0.0, 1.0)
|
||||
|
||||
# hyper-parameters used for post-processing i.e. removing short speech turns
|
||||
# or filling short gaps between speech turns of one speaker
|
||||
self.min_duration_on = Uniform(0.0, 1.0)
|
||||
self.min_duration_off = Uniform(0.0, 1.0)
|
||||
|
||||
def initialize(self):
|
||||
"""Initialize pipeline with current set of parameters"""
|
||||
|
||||
self._binarize = Binarize(
|
||||
onset=self.onset,
|
||||
offset=self.offset,
|
||||
min_duration_on=self.min_duration_on,
|
||||
min_duration_off=self.min_duration_off,
|
||||
)
|
||||
|
||||
def apply(self, file: AudioFile) -> Annotation:
|
||||
"""Apply speaker diarization
|
||||
|
||||
Parameters
|
||||
----------
|
||||
file : AudioFile
|
||||
Processed file.
|
||||
|
||||
Returns
|
||||
-------
|
||||
diarization : Annotation
|
||||
Speaker diarization
|
||||
"""
|
||||
|
||||
# output of segmentation model on each chunk
|
||||
segmentations: SlidingWindowFeature = self.seg_inference_(file)
|
||||
|
||||
# number of frames in the whole file
|
||||
num_frames_in_file = self.seg_frames_.samples(
|
||||
self.audio_.get_duration(file), mode="center"
|
||||
)
|
||||
|
||||
# turn input diarization into binary (0 or 1) activations
|
||||
labels = file[self.diarization].labels()
|
||||
num_clusters = len(labels)
|
||||
y_original = np.zeros(
|
||||
(num_frames_in_file, len(labels)), dtype=segmentations.data.dtype
|
||||
)
|
||||
for k, label in enumerate(labels):
|
||||
segments = file[self.diarization].label_timeline(label)
|
||||
for start, stop in self.seg_frames_.crop(
|
||||
segments, mode="center", return_ranges=True
|
||||
):
|
||||
y_original[start:stop, k] += 1
|
||||
y_original = np.minimum(y_original, 1, out=y_original)
|
||||
diarization = SlidingWindowFeature(y_original, self.seg_frames_)
|
||||
file["@resegmentation/diarization"] = diarization
|
||||
|
||||
aggregated = np.zeros((num_frames_in_file, num_clusters))
|
||||
overlapped = np.zeros((num_frames_in_file, num_clusters))
|
||||
|
||||
for chunk, segmentation in segmentations:
|
||||
|
||||
# only consider active speakers in `segmentation`
|
||||
active = np.max(segmentation, axis=0) > self.onset
|
||||
if np.sum(active) == 0:
|
||||
continue
|
||||
segmentation = segmentation[:, active]
|
||||
|
||||
local_diarization = diarization.crop(chunk)[
|
||||
np.newaxis, : self.num_frames_in_chunk_
|
||||
]
|
||||
(permutated_segmentation,), _ = permutate(local_diarization, segmentation)
|
||||
|
||||
start_frame = round(chunk.start / self.seg_frames_.duration)
|
||||
aggregated[
|
||||
start_frame : start_frame + self.num_frames_in_chunk_
|
||||
] += permutated_segmentation
|
||||
overlapped[start_frame : start_frame + self.num_frames_in_chunk_] += 1.0
|
||||
|
||||
speaker_activations = SlidingWindowFeature(
|
||||
aggregated / overlapped, self.seg_frames_, labels=labels
|
||||
)
|
||||
|
||||
file["@resegmentation/activations"] = speaker_activations
|
||||
|
||||
return self._binarize(speaker_activations)
|
||||
|
||||
def get_metric(self) -> GreedyDiarizationErrorRate:
|
||||
return GreedyDiarizationErrorRate(collar=0.0, skip_overlap=False)
|
||||
@@ -128,7 +128,7 @@ class Segmentation(Pipeline):
|
||||
"""
|
||||
|
||||
speaker_activations = self.segmentation_inference_(file)
|
||||
file["@segmentation/speaker_activations"] = speaker_activations
|
||||
file["@segmentation/activations"] = speaker_activations
|
||||
segmentation = self._binarize(speaker_activations)
|
||||
segmentation.uri = file["uri"]
|
||||
return segmentation
|
||||
|
||||
@@ -14,11 +14,11 @@ def permutate(y1, y2, cost_func: Optional[Callable] = None, returns_cost: bool =
|
||||
Parameters
|
||||
----------
|
||||
y1 : np.ndarray or torch.Tensor
|
||||
(batch_size, num_samples, num_speakers)
|
||||
(batch_size, num_samples, num_classes_1)
|
||||
y2 : np.ndarray or torch.Tensor
|
||||
(num_samples, num_speakers) or (batch_size, num_samples, num_speakers)
|
||||
(num_samples, num_classes_2) or (batch_size, num_samples, num_classes_2)
|
||||
cost_func : callable
|
||||
Takes two (num_samples, num_speakers) sequences and returns (num_speakers, ) pairwise cost.
|
||||
Takes two (num_samples, num_classes) sequences and returns (num_classes, ) pairwise cost.
|
||||
Defaults to computing mean squared error.
|
||||
returns_cost : bool, optional
|
||||
Whether to return cost matrix. Defaults to False.
|
||||
@@ -26,12 +26,12 @@ def permutate(y1, y2, cost_func: Optional[Callable] = None, returns_cost: bool =
|
||||
Returns
|
||||
-------
|
||||
permutated_y2 : np.ndarray or torch.Tensor
|
||||
(batch_size, num_samples, num_speakers)
|
||||
(batch_size, num_samples, num_classes_1)
|
||||
permutations : list of tuple
|
||||
List of permutations so that permutation[i] == j indicates that jth speaker of y2
|
||||
should be mapped to ith speaker of y1.
|
||||
cost : np.ndarray or torch.Tensor, optional
|
||||
(batch_size, num_speakers, num_speakers)
|
||||
(batch_size, num_classes_1, num_classes_2)
|
||||
"""
|
||||
raise TypeError()
|
||||
|
||||
@@ -48,21 +48,17 @@ def permutate_torch(
|
||||
returns_cost: bool = False,
|
||||
) -> Tuple[torch.Tensor, List[Tuple[int]]]:
|
||||
|
||||
batch_size, num_samples, num_speakers = y1.shape
|
||||
batch_size, num_samples, num_classes_1 = y1.shape
|
||||
|
||||
if len(y2.shape) == 2:
|
||||
y2 = y2.expand(batch_size, -1, -1)
|
||||
|
||||
if len(y2.shape) != 3:
|
||||
msg = "Incorrect shape: should be (batch_size, num_frames, num_speakers)."
|
||||
msg = "Incorrect shape: should be (batch_size, num_frames, num_classes_1)."
|
||||
raise ValueError(msg)
|
||||
|
||||
batch_size_, num_samples_, num_speakers_ = y2.shape
|
||||
if (
|
||||
batch_size != batch_size_
|
||||
or num_samples != num_samples_
|
||||
or num_speakers != num_speakers_
|
||||
):
|
||||
batch_size_, num_samples_, num_classes_2 = y2.shape
|
||||
if batch_size != batch_size_ or num_samples != num_samples_:
|
||||
msg = f"Shape mismatch: {tuple(y1.shape)} vs. {tuple(y2.shape)}."
|
||||
raise ValueError(msg)
|
||||
|
||||
@@ -75,24 +71,41 @@ def permutate_torch(
|
||||
if returns_cost:
|
||||
costs = []
|
||||
|
||||
for y1_, y2_ in zip(y1, y2):
|
||||
permutated_y2 = torch.zeros(y1.shape, device=y2.device, dtype=y2.dtype)
|
||||
|
||||
for b, (y1_, y2_) in enumerate(zip(y1, y2)):
|
||||
with torch.no_grad():
|
||||
cost = torch.stack(
|
||||
[
|
||||
cost_func(y2_, y1_[:, i : i + 1].expand(-1, num_speakers))
|
||||
for i in range(num_speakers)
|
||||
cost_func(y2_, y1_[:, i : i + 1].expand(-1, num_classes_2))
|
||||
for i in range(num_classes_1)
|
||||
],
|
||||
)
|
||||
permutation = tuple(linear_sum_assignment(cost.cpu())[1])
|
||||
permutations.append(permutation)
|
||||
permutated_y2.append(y2_[:, permutation])
|
||||
|
||||
if num_classes_2 > num_classes_1:
|
||||
padded_cost = F.pad(
|
||||
cost,
|
||||
(0, 0, 0, num_classes_2 - num_classes_1),
|
||||
"constant",
|
||||
torch.max(cost) + 1,
|
||||
)
|
||||
else:
|
||||
padded_cost = cost
|
||||
|
||||
permutation = [None] * num_classes_1
|
||||
for k1, k2 in zip(*linear_sum_assignment(padded_cost.cpu())):
|
||||
if k1 < num_classes_1:
|
||||
permutation[k1] = k2
|
||||
permutated_y2[b, :, k1] = y2_[:, k2]
|
||||
permutations.append(tuple(permutation))
|
||||
|
||||
if returns_cost:
|
||||
costs.append(cost)
|
||||
|
||||
if returns_cost:
|
||||
return torch.stack(permutated_y2), permutations, torch.stack(costs)
|
||||
return permutated_y2, permutations, torch.stack(costs)
|
||||
|
||||
return torch.stack(permutated_y2), permutations
|
||||
return permutated_y2, permutations
|
||||
|
||||
|
||||
@permutate.register
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from pyannote.audio.utils.permutation import permutate
|
||||
|
||||
|
||||
def test_permutate_torch():
|
||||
|
||||
num_frames, num_speakers = 10, 3
|
||||
|
||||
actual_permutations = [
|
||||
(0, 1, 2),
|
||||
(0, 2, 1),
|
||||
(1, 0, 2),
|
||||
(1, 2, 0),
|
||||
(2, 0, 1),
|
||||
(2, 1, 0),
|
||||
]
|
||||
batch_size = len(actual_permutations)
|
||||
|
||||
y2 = torch.randn((num_frames, num_speakers))
|
||||
y1 = torch.zeros((batch_size, num_frames, num_speakers))
|
||||
|
||||
for p, permutation in enumerate(actual_permutations):
|
||||
y1[p] = y2[:, permutation]
|
||||
|
||||
permutated_y2, permutations = permutate(y1, y2)
|
||||
assert actual_permutations == permutations
|
||||
|
||||
|
||||
def test_permutate_numpy():
|
||||
|
||||
num_frames, num_speakers = 10, 3
|
||||
|
||||
actual_permutations = [
|
||||
(0, 1, 2),
|
||||
(0, 2, 1),
|
||||
(1, 0, 2),
|
||||
(1, 2, 0),
|
||||
(2, 0, 1),
|
||||
(2, 1, 0),
|
||||
]
|
||||
batch_size = len(actual_permutations)
|
||||
|
||||
y2 = np.random.randn(num_frames, num_speakers)
|
||||
y1 = np.zeros((batch_size, num_frames, num_speakers))
|
||||
|
||||
for p, permutation in enumerate(actual_permutations):
|
||||
y1[p] = y2[:, permutation]
|
||||
|
||||
permutated_y2, permutations = permutate(y1, y2)
|
||||
assert actual_permutations == permutations
|
||||
@@ -0,0 +1,112 @@
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from pyannote.audio.utils.permutation import permutate
|
||||
|
||||
|
||||
def test_permutate_torch():
|
||||
|
||||
num_frames, num_speakers = 10, 3
|
||||
|
||||
actual_permutations = [
|
||||
(0, 1, 2),
|
||||
(0, 2, 1),
|
||||
(1, 0, 2),
|
||||
(1, 2, 0),
|
||||
(2, 0, 1),
|
||||
(2, 1, 0),
|
||||
]
|
||||
batch_size = len(actual_permutations)
|
||||
|
||||
y2 = torch.randn((num_frames, num_speakers))
|
||||
y1 = torch.zeros((batch_size, num_frames, num_speakers))
|
||||
|
||||
for p, permutation in enumerate(actual_permutations):
|
||||
y1[p] = y2[:, permutation]
|
||||
|
||||
permutated_y2, permutations = permutate(y1, y2)
|
||||
assert actual_permutations == permutations
|
||||
|
||||
for p, permutation in enumerate(actual_permutations):
|
||||
np.testing.assert_allclose(permutated_y2[p], y2[:, permutation])
|
||||
|
||||
|
||||
def test_permutate_numpy():
|
||||
|
||||
num_frames, num_speakers = 10, 3
|
||||
|
||||
actual_permutations = [
|
||||
(0, 1, 2),
|
||||
(0, 2, 1),
|
||||
(1, 0, 2),
|
||||
(1, 2, 0),
|
||||
(2, 0, 1),
|
||||
(2, 1, 0),
|
||||
]
|
||||
batch_size = len(actual_permutations)
|
||||
|
||||
y2 = np.random.randn(num_frames, num_speakers)
|
||||
y1 = np.zeros((batch_size, num_frames, num_speakers))
|
||||
|
||||
for p, permutation in enumerate(actual_permutations):
|
||||
y1[p] = y2[:, permutation]
|
||||
|
||||
permutated_y2, permutations = permutate(y1, y2)
|
||||
assert actual_permutations == permutations
|
||||
|
||||
for p, permutation in enumerate(actual_permutations):
|
||||
np.testing.assert_allclose(permutated_y2[p], y2[:, permutation])
|
||||
|
||||
|
||||
def test_permutate_less_speakers():
|
||||
|
||||
num_frames = 10
|
||||
|
||||
actual_permutations = [
|
||||
(0, 1, None),
|
||||
(0, None, 1),
|
||||
(1, 0, None),
|
||||
(1, None, 0),
|
||||
(None, 0, 1),
|
||||
(None, 1, 0),
|
||||
]
|
||||
batch_size = len(actual_permutations)
|
||||
|
||||
y2 = np.random.randn(num_frames, 2)
|
||||
y1 = np.zeros((batch_size, num_frames, 3))
|
||||
|
||||
for p, permutation in enumerate(actual_permutations):
|
||||
for i, j in enumerate(permutation):
|
||||
if j is not None:
|
||||
y1[p, :, i] = y2[:, j]
|
||||
|
||||
permutated_y2, permutations = permutate(y1, y2)
|
||||
|
||||
assert permutations == actual_permutations
|
||||
|
||||
|
||||
def test_permutate_more_speakers():
|
||||
|
||||
num_frames = 10
|
||||
|
||||
actual_permutations = [
|
||||
(0, 1),
|
||||
(0, 2),
|
||||
(1, 0),
|
||||
(1, 2),
|
||||
(2, 0),
|
||||
(2, 1),
|
||||
]
|
||||
batch_size = len(actual_permutations)
|
||||
|
||||
y2 = np.random.randn(num_frames, 3)
|
||||
y1 = np.zeros((batch_size, num_frames, 2))
|
||||
|
||||
for p, permutation in enumerate(actual_permutations):
|
||||
for i, j in enumerate(permutation):
|
||||
y1[p, :, i] = y2[:, j]
|
||||
|
||||
permutated_y2, permutations = permutate(y1, y2)
|
||||
|
||||
assert permutations == actual_permutations
|
||||
np.testing.assert_allclose(permutated_y2, y1)
|
||||
Reference in new issue
Block a user