Merge branch 'develop' of https://github.com/pyannote/pyannote-audio into develop

This commit is contained in:
Hervé Bredin committed 2021-09-08 16:06:12 +02:00
commit 23c4eaaed5
13 files changed
+609 -463

No files matched your search

+2 -2
View File
@@ -1,4 +1,4 @@
# Neural speaker diarization with pyannote.audio
# Neural speaker diarization
This is the development branch of upcoming `pyannote.audio` 2.0 for which it has been decided to rewrite almost everything from scratch. Highlights of this upcoming release will be:
@@ -8,7 +8,7 @@ This is the development branch of upcoming `pyannote.audio` 2.0 for which it has
- data augmentation with [torch-audiomentations](https://github.com/asteroid-team/torch-audiomentations)
- [huggingface](https://huggingface.co/pyannote) model hosting
- [prodigy](https://prodi.gy) recipes for audio annotations
- online [demo](https://share.streamlit.io/pyannote/pyannote-audio-demo/main) based on [streamlit](https://www.streamlit.io)
- online [demo](https://huggingface.co/spaces/pyannote/segmentation) based on [streamlit](https://www.streamlit.io)
## Installation
+1 -2
View File
@@ -81,7 +81,6 @@ def main(cfg: DictConfig) -> Optional[float]:
# add task-dependent layers so that later call to model.parameters()
# does return all layers (even task-dependent ones). this is already
# done for pretrained models (TODO: check that this is true)
task.setup(stage="fit")
model.setup(stage="fit")
def configure_optimizers(self):
@@ -127,7 +126,7 @@ def main(cfg: DictConfig) -> Optional[float]:
monitor=monitor,
mode=direction,
save_top_k=None if monitor is None else 5,
period=1,
every_n_epochs=1,
save_last=True,
save_weights_only=False,
dirpath=".",
+21 -7
View File
@@ -37,6 +37,7 @@ from huggingface_hub import cached_download, hf_hub_url
from pytorch_lightning.core.memory import ModelSummary
from pytorch_lightning.utilities.cloud_io import load as pl_load
from semver import VersionInfo
from torch.utils.data import DataLoader
from pyannote.audio import __version__
from pyannote.audio.core.io import Audio
@@ -274,19 +275,16 @@ class Model(pl.LightningModule):
sample_rate=self.hparams.sample_rate, mono=self.hparams.num_channels == 1
)
@property
def datamodule(self):
return self.task
@property
def example_input_array(self) -> torch.Tensor:
batch_size = 3 if self.task is None else self.task.batch_size
duration = 2.0 if self.task is None else self.task.duration
return torch.randn(
(
batch_size,
self.hparams.num_channels,
int(self.hparams.sample_rate * self.specifications.duration),
int(self.hparams.sample_rate * duration),
),
device=self.device,
)
@@ -347,6 +345,9 @@ class Model(pl.LightningModule):
def setup(self, stage=None):
if stage == "fit":
self.task.setup()
# list of layers before adding task-dependent layers
before = set((name, id(module)) for name, module in self.named_modules())
@@ -365,7 +366,11 @@ class Model(pl.LightningModule):
# setup custom loss function
self.task.setup_loss_func()
# setup custom validation metrics
self.task.setup_validation_metric()
validation_metric = self.task.setup_validation_metric()
if validation_metric is not None:
self.validation_metric = validation_metric
self.validation_metric.to(self.device)
# this is to make sure introspection is performed here, once and for all
_ = self.introspection
@@ -477,11 +482,21 @@ class Model(pl.LightningModule):
"""
return self.helper_default_activation(self.specifications)
# training data logic is delegated to the task because the
# model does not really need to know how it is being used.
def train_dataloader(self) -> DataLoader:
return self.task.train_dataloader()
# training step logic is delegated to the task because the
# model does not really need to know how it is being used.
def training_step(self, batch, batch_idx):
return self.task.training_step(batch, batch_idx)
# validation data logic is delegated to the task because the
# model does not really need to know how it is being used.
def val_dataloader(self) -> DataLoader:
return self.task.val_dataloader()
# validation logic is delegated to the task because the
# model does not really need to know how it is being used.
def validation_step(self, batch, batch_idx):
@@ -822,7 +837,6 @@ class Model(pl.LightningModule):
if task is not None:
model.task = task
task.setup(stage="fit")
model.setup(stage="fit")
try:
+2 -7
View File
@@ -218,13 +218,8 @@ class Task(pl.LightningDataModule):
"""
pass
def setup(self, stage=None):
"""Called at the beginning of fit and test just before Model.setup()
Parameters
----------
stage : "fit" or "test"
Whether model is being trained ("fit") or used for inference ("test").
def setup(self, stage: Optional[str] = None):
"""Called at the beginning of training at the very beginning of Model.setup(stage="fit")
Notes
-----
@@ -0,0 +1,157 @@
# MIT License
#
# Copyright (c) 2021 CNRS
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import numpy as np
import torch
from pyannote.audio import Inference, Model, Pipeline
from pyannote.audio.core.io import AudioFile
from pyannote.audio.pipelines.utils import PipelineModel, get_devices, get_model
class SpeakerEmbedding(Pipeline):
"""Speaker embedding pipeline
This pipeline assumes that each file contains exactly one speaker
and extracts one single embedding from the whole file.
Parameters
----------
embedding : Model, str, or dict, optional
Pretrained embedding model. Defaults to "pyannote/embedding".
See pyannote.audio.pipelines.utils.get_model for supported format.
segmentation : Model, str, or dict, optional
Pretrained segmentation (or voice activity detection) model.
See pyannote.audio.pipelines.utils.get_model for supported format.
Defaults to no voice activity detection.
Usage
-----
>>> from pyannote.audio.pipelines import SpeakerEmbedding
>>> pipeline = SpeakerEmbedding()
>>> emb1 = pipeline("speaker1.wav")
>>> emb2 = pipeline("speaker2.wav")
>>> from scipy.spatial.distance import cdist
>>> distance = cdist(emb1, emb2, metric="cosine")[0,0]
"""
def __init__(
self,
embedding: PipelineModel = "pyannote/embedding",
segmentation: PipelineModel = None,
):
super().__init__()
self.embedding = embedding
self.segmentation = segmentation
self.embedding_model_: Model = get_model(embedding)
if self.segmentation is None:
models = [self.embedding_model_]
else:
segmentation_model: Model = get_model(self.segmentation)
models = [self.embedding_model_, segmentation_model]
# send models to GPU (when GPUs are available and model is not already on GPU)
cpu_models = [model for model in models if model.device.type == "cpu"]
for cpu_model, gpu_device in zip(
cpu_models, get_devices(needs=len(cpu_models))
):
cpu_model.to(gpu_device)
if self.segmentation is not None:
self.voice_activity_ = Inference(
segmentation_model,
pre_aggregation_hook=lambda scores: np.max(
scores, axis=-1, keepdims=True
),
)
def apply(self, file: AudioFile) -> np.ndarray:
device = self.embedding_model_.device
# read audio file and send it to GPU
waveform = self.embedding_model_.audio(file)[0][None].to(device)
if self.segmentation is None:
weights = None
else:
# obtain voice activity scores
weights = self.voice_activity_(file).data
# HACK -- this should be fixed upstream
weights[np.isnan(weights)] = 0.0
weights = torch.from_numpy(weights ** 3)[None, :, 0].to(device)
# extract speaker embedding on parts of
with torch.no_grad():
return self.embedding_model_(waveform, weights=weights).cpu().numpy()
def main(
protocol: str = "VoxCeleb.SpeakerVerification.VoxCeleb1",
subset: str = "test",
embedding: str = "pyannote/embedding",
segmentation: str = None,
):
import typer
from scipy.spatial.distance import cdist
from tqdm import tqdm
from pyannote.database import FileFinder, get_protocol
from pyannote.metrics.binary_classification import det_curve
pipeline = SpeakerEmbedding(embedding=embedding, segmentation=segmentation)
protocol = get_protocol(protocol, preprocessors={"audio": FileFinder()})
y_true, y_pred = [], []
emb = dict()
trials = getattr(protocol, f"{subset}_trial")()
for t, trial in enumerate(tqdm(trials)):
audio1 = trial["file1"]["audio"]
if audio1 not in emb:
emb[audio1] = pipeline(audio1)
audio2 = trial["file2"]["audio"]
if audio2 not in emb:
emb[audio2] = pipeline(audio2)
y_pred.append(cdist(emb[audio1], emb[audio2], metric="cosine")[0][0])
y_true.append(trial["reference"])
_, _, _, eer = det_curve(y_true, np.array(y_pred), distances=True)
typer.echo(
f"{protocol.name} | {subset} | {embedding} | {segmentation} | EER = {100 * eer:.3f}%"
)
if __name__ == "__main__":
import typer
typer.run(main)
+109 -139
View File
@@ -1,6 +1,6 @@
# MIT License
#
# Copyright (c) 2020 CNRS
# Copyright (c) 2020-2021 CNRS
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
@@ -23,20 +23,18 @@
import math
from typing import Optional
import numpy as np
import torch
from torch.utils.data import DataLoader
import torch.nn.functional as F
from torchmetrics import AUROC
from tqdm import tqdm
from pyannote.audio.core.task import Problem, Resolution, Specifications, ValDataset
from pyannote.audio.core.task import Problem, Resolution, Specifications
from pyannote.audio.utils.random import create_rng_for_worker
from pyannote.core import Segment
from pyannote.core.utils.distance import cdist
from pyannote.database.protocol import (
SpeakerDiarizationProtocol,
SpeakerVerificationProtocol,
)
from pyannote.metrics.binary_classification import det_curve
class SupervisedRepresentationLearningTaskMixin:
@@ -74,65 +72,60 @@ class SupervisedRepresentationLearningTaskMixin:
def batch_size(self, batch_size: int):
self.batch_size_ = batch_size
def setup(self, stage=None):
def setup(self, stage: Optional[str] = None):
if stage == "fit":
# loop over the training set, remove annotated regions shorter than
# chunk duration, and keep track of the reference annotations, per class.
# loop over the training set, remove annotated regions shorter than
# chunk duration, and keep track of the reference annotations, per class.
self._train = dict()
self._train = dict()
desc = f"Loading {self.protocol.name} training labels"
for f in tqdm(iterable=self.protocol.train(), desc=desc, unit="file"):
desc = f"Loading {self.protocol.name} training labels"
for f in tqdm(iterable=self.protocol.train(), desc=desc, unit="file"):
for klass in f["annotation"].labels():
for klass in f["annotation"].labels():
# keep class's (long enough) speech turns...
speech_turns = [
segment
for segment in f["annotation"].label_timeline(klass)
if segment.duration > self.min_duration
]
# keep class's (long enough) speech turns...
speech_turns = [
segment
for segment in f["annotation"].label_timeline(klass)
if segment.duration > self.duration
]
# skip if there is no speech turns left
if not speech_turns:
continue
# skip if there is no speech turns left
if not speech_turns:
continue
# ... and their total duration
duration = sum(segment.duration for segment in speech_turns)
# ... and their total duration
duration = sum(segment.duration for segment in speech_turns)
# add class to the list of classes
if klass not in self._train:
self._train[klass] = list()
# add class to the list of classes
if klass not in self._train:
self._train[klass] = list()
self._train[klass].append(
{
"uri": f["uri"],
"audio": f["audio"],
"duration": duration,
"speech_turns": speech_turns,
}
)
self._train[klass].append(
{
"uri": f["uri"],
"audio": f["audio"],
"duration": duration,
"speech_turns": speech_turns,
}
)
self.specifications = Specifications(
problem=Problem.REPRESENTATION,
resolution=Resolution.CHUNK,
duration=self.duration,
classes=sorted(self._train),
)
self.specifications = Specifications(
problem=Problem.REPRESENTATION,
resolution=Resolution.CHUNK,
duration=self.duration,
classes=sorted(self._train),
)
if not self.has_validation:
return
if not self.has_validation:
return
if isinstance(self.protocol, SpeakerVerificationProtocol):
self._validation = list(self.protocol.development_trial())
if isinstance(self.protocol, SpeakerVerificationProtocol):
sessions = dict()
for trial in self.protocol.development_trial():
for session in ["file1", "file2"]:
session_hash = self.helper_trial_hash(trial[session])
if session_hash not in sessions:
sessions[session_hash] = trial[session]
self._validation = list(sessions.items())
def setup_validation_metric(self):
return AUROC(compute_on_step=False)
def train__iter__(self):
"""Iterate over training samples
@@ -183,18 +176,29 @@ class SupervisedRepresentationLearningTaskMixin:
k=1,
)
# select one chunk at random (with uniform distribution)
start_time = rng.uniform(
speech_turn.start, speech_turn.end - batch_duration
)
chunk = Segment(start_time, start_time + batch_duration)
# if speech turn is too short, pad with zeros
if speech_turn.duration < batch_duration:
X, _ = self.model.audio.crop(file, speech_turn, mode="center")
num_missing_frames = (
math.floor(batch_duration * self.model.audio.sample_rate)
- X.shape[1]
)
left_pad = rng.randint(0, num_missing_frames)
X = F.pad(X, (left_pad, num_missing_frames - left_pad))
X, _ = self.model.audio.crop(
file,
chunk,
mode="center",
fixed=batch_duration,
)
# if it is long enough, select chunk at random
else:
start_time = rng.uniform(
speech_turn.start, speech_turn.end - batch_duration
)
chunk = Segment(start_time, start_time + batch_duration)
X, _ = self.model.audio.crop(
file,
chunk,
mode="center",
fixed=batch_duration,
)
yield {"X": X, "y": y}
@@ -225,21 +229,33 @@ class SupervisedRepresentationLearningTaskMixin:
)
return {"loss": loss}
@staticmethod
def helper_trial_hash(file) -> int:
return hash((file["database"], file["uri"], tuple(file["try_with"])))
def val__getitem__(self, idx):
if isinstance(self.protocol, SpeakerVerificationProtocol):
session_hash, session = self._validation[idx]
X = np.concatenate(
[
self.model.audio.crop(session, segment, mode="center")[0]
for segment in session["try_with"]
],
axis=0,
)
return {"session_hash": session_hash, "X": X}
trial = self._validation[idx]
data = dict()
for idx in [1, 2]:
file = trial[f"file{idx:d}"]
duration = self.model.audio.get_duration(file)
if duration > self.duration:
middle = Segment(
0.5 * duration - 0.5 * self.duration,
0.5 * duration + 0.5 * self.duration,
)
X, _ = self.model.audio.crop(
file, middle, mode="center", fixed=self.duration
)
else:
X, _ = self.model.audio(file)
num_missing_frames = (
math.floor(self.duration * self.model.audio.sample_rate)
- X.shape[1]
)
X = F.pad(X, (0, num_missing_frames))
data[f"X{idx:d}"] = X
data["y"] = trial["reference"]
return data
elif isinstance(self.protocol, SpeakerDiarizationProtocol):
pass
@@ -255,80 +271,34 @@ class SupervisedRepresentationLearningTaskMixin:
def validation_step(self, batch, batch_idx: int):
if isinstance(self.protocol, SpeakerVerificationProtocol):
return (
batch["session_hash"][0].detach().cpu().numpy().item(),
self.model(batch["X"]).detach().cpu().numpy(),
with torch.no_grad():
emb1 = self.model(batch["X1"]).detach()
emb2 = self.model(batch["X2"]).detach()
y_pred = F.cosine_similarity(emb1, emb2)
y_true = batch["y"]
self.model.validation_metric(y_pred, y_true)
self.model.log(
f"{self.ACRONYM}@val_auroc",
self.model.validation_metric,
on_step=False,
on_epoch=True,
prog_bar=True,
logger=True,
)
elif isinstance(self.protocol, SpeakerDiarizationProtocol):
pass
def validation_epoch_end(self, outputs):
if isinstance(self.protocol, SpeakerVerificationProtocol):
embeddings = dict(outputs)
y_true, y_pred = [], []
for trial in self.protocol.development_trial():
session1_hash = self.helper_trial_hash(trial["file1"])
session2_hash = self.helper_trial_hash(trial["file2"])
try:
emb1 = embeddings[session1_hash]
emb2 = embeddings[session2_hash]
except KeyError:
return
y_pred.append(cdist(emb1, emb2, metric="cosine").item())
y_true.append(trial["reference"])
y_pred = np.array(y_pred)
y_true = np.array(y_true)
num_target_trials = np.sum(y_true)
num_non_target_trials = np.sum(1.0 - y_true)
if num_target_trials > 2 and num_non_target_trials > 2:
fpr, fnr, thresholds, eer = det_curve(y_true, y_pred, distances=True)
self.model.log(
f"{self.ACRONYM}@val_eer",
torch.tensor(eer, device=self.model.device),
logger=True,
on_epoch=True,
prog_bar=True,
sync_dist=True,
)
elif isinstance(self.protocol, SpeakerDiarizationProtocol):
pass
def val_dataloader(self) -> Optional[DataLoader]:
if self.has_validation:
if isinstance(self.protocol, SpeakerVerificationProtocol):
return DataLoader(
ValDataset(self),
batch_size=1,
pin_memory=self.pin_memory,
drop_last=False,
)
elif isinstance(self.protocol, SpeakerDiarizationProtocol):
return None
else:
return None
@property
def val_monitor(self):
if self.has_validation:
if isinstance(self.protocol, SpeakerVerificationProtocol):
return f"{self.ACRONYM}@val_eer", "min"
return f"{self.ACRONYM}@val_auroc", "max"
elif isinstance(self.protocol, SpeakerDiarizationProtocol):
return None, "min"
+115 -83
View File
@@ -27,7 +27,8 @@ from typing import List, Optional, Text, Tuple
import matplotlib.pyplot as plt
import numpy as np
from pytorch_lightning.metrics.classification import FBeta
from einops import rearrange
from torchmetrics import FBeta
from typing_extensions import Literal
from pyannote.audio.core.io import Audio, AudioFile
@@ -39,92 +40,86 @@ from pyannote.core import Annotation, Segment, SlidingWindow, SlidingWindowFeatu
class SegmentationTaskMixin:
"""Methods common to most segmentation tasks"""
def setup(self, stage=None):
def setup(self, stage: Optional[str] = None):
if stage == "fit":
# ==================================================================
# PREPARE TRAINING DATA
# ==================================================================
# ==================================================================
# PREPARE TRAINING DATA
# ==================================================================
self._train = []
self._train_metadata = dict()
self._train = []
self._train_metadata = dict()
for f in self.protocol.train():
for f in self.protocol.train():
file = dict()
file = dict()
for key, value in f.items():
for key, value in f.items():
# keep track of unique labels in self._train_metadata["annotation"]
if key == "annotation":
for label in value.labels():
self._train_metadata.setdefault("annotation", set()).add(label)
# keep track of unique labels in self._train_metadata["annotation"]
if key == "annotation":
for label in value.labels():
self._train_metadata.setdefault("annotation", set()).add(
label
)
# pass "audio" entry as it is
elif key == "audio":
pass
# pass "audio" entry as it is
elif key == "audio":
pass
# remove segments shorter than chunks from "annotated" entry
elif key == "annotated":
value = [
segment for segment in value if segment.duration > self.duration
]
file["_annotated_duration"] = sum(
segment.duration for segment in value
)
# remove segments shorter than chunks from "annotated" entry
elif key == "annotated":
value = [
segment
for segment in value
if segment.duration > self.duration
]
file["_annotated_duration"] = sum(
segment.duration for segment in value
)
# keey track of unique text-like entries (incl. "uri" and "database")
# and pass them as they are
elif isinstance(value, Text):
self._train_metadata.setdefault(key, set()).add(value)
# keey track of unique text-like entries (incl. "uri" and "database")
# and pass them as they are
elif isinstance(value, Text):
self._train_metadata.setdefault(key, set()).add(value)
# pass score-like entries as they are
elif isinstance(value, SlidingWindowFeature):
pass
# pass score-like entries as they are
elif isinstance(value, SlidingWindowFeature):
pass
else:
msg = (
f"Protocol '{self.protocol.name}' defines a '{key}' entry of type {type(value)} "
f"which we do not know how to handle."
)
warnings.warn(msg)
file[key] = value
else:
msg = (
f"Protocol '{self.protocol.name}' defines a '{key}' entry of type {type(value)} "
f"which we do not know how to handle."
)
warnings.warn(msg)
file[key] = value
self._train.append(file)
self._train.append(file)
self._train_metadata = {
key: sorted(values) for key, values in self._train_metadata.items()
}
self._train_metadata = {
key: sorted(values) for key, values in self._train_metadata.items()
}
# ==================================================================
# PREPARE VALIDATION DATA
# ==================================================================
# ==================================================================
# PREPARE VALIDATION DATA
# ==================================================================
if not self.has_validation:
return
if not self.has_validation:
return
self._validation = []
self._validation = []
for f in self.protocol.development():
for f in self.protocol.development():
for segment in f["annotated"]:
for segment in f["annotated"]:
if segment.duration < self.duration:
continue
if segment.duration < self.duration:
continue
num_chunks = round(segment.duration // self.duration)
num_chunks = round(segment.duration // self.duration)
for c in range(num_chunks):
start_time = segment.start + c * self.duration
chunk = Segment(start_time, start_time + self.duration)
self._validation.append((f, chunk))
for c in range(num_chunks):
start_time = segment.start + c * self.duration
chunk = Segment(start_time, start_time + self.duration)
self._validation.append((f, chunk))
random.shuffle(self._validation)
random.shuffle(self._validation)
def setup_validation_metric(self):
"""Setup default validation metric
@@ -132,14 +127,9 @@ class SegmentationTaskMixin:
Use macro-average of F-score with a 0.5 threshold
"""
self.val_fbeta = FBeta(
len(self.specifications.classes),
beta=1.0,
threshold=0.5,
multilabel=(
self.specifications.problem == Problem.MULTI_LABEL_CLASSIFICATION
),
average="macro",
num_classes = len(self.specifications.classes)
return FBeta(
num_classes, beta=1.0, threshold=0.5, average="macro", compute_on_step=False
)
def prepare_y(self, one_hot_y: np.ndarray) -> np.ndarray:
@@ -457,8 +447,11 @@ class SegmentationTaskMixin:
def val__len__(self):
return len(self._validation)
def validation_postprocess(self, y, y_pred):
return y_pred
def validation_step(self, batch, batch_idx: int):
"""Compute area under ROC curve
"""Compute validation F-score
Parameters
----------
@@ -468,27 +461,66 @@ class SegmentationTaskMixin:
Batch index.
"""
# move metric to model device
self.val_fbeta.to(self.model.device)
X, y = batch["X"], batch["y"]
# X = (batch_size, num_channels, num_samples)
# y = (batch_size, num_frames, num_classes)
# y = (batch_size, num_frames, num_classes) or (batch_size, num_frames)
y_pred = self.model(X)
_, num_frames, _ = y_pred.shape
# y_pred = (batch_size, num_frames, num_classes)
# postprocess
y_pred = self.validation_postprocess(y, y_pred)
# - remove warm-up frames
# - downsample remaining frames
warm_up_left = round(self.warm_up[0] / self.duration * num_frames)
warm_up_right = round(self.warm_up[1] / self.duration * num_frames)
preds = y_pred[:, warm_up_left : num_frames - warm_up_right : 10]
target = y[:, warm_up_left : num_frames - warm_up_right : 10]
# torchmetrics tries to be smart about the type of machine learning problem
# pyannote.audio is more explicit so we have to reshape target and preds for
# torchmetrics to be happy... more details can be found here:
# https://torchmetrics.readthedocs.io/en/latest/references/modules.html#input-types
if self.specifications.problem == Problem.BINARY_CLASSIFICATION:
# target: shape (batch_size, num_frames), type binary
# preds: shape (batch_size, num_frames, 1), type float
# torchmetrics expects:
# target: shape (N,), type binary
# preds: shape (N,), type float
self.model.validation_metric(preds.reshape(-1), target.reshape(-1))
elif self.specifications.problem == Problem.MULTI_LABEL_CLASSIFICATION:
# target: shape (batch_size, num_frames, num_classes), type binary
# preds: shape (batch_size, num_frames, num_classes), type float
# torchmetrics expects
# target: shape (N, num_classes), type binary
# preds: shape (N, num_classes), type float
self.model.validation_metric(
rearrange(preds, "b f c -> (b f) c"),
rearrange(target, "b f c -> (b f) c"),
)
elif self.specifications.problem == Problem.MONO_LABEL_CLASSIFICATION:
# target: shape (batch_size, num_frames, num_classes), type binary
# preds: shape (batch_size, num_frames, num_classes), type float
# torchmetrics expects:
# target: shape (N, ), type int
# preds: shape (N, num_classes), type float
# TODO: implement when pyannote.audio gets its first mono-label segmentation task
raise NotImplementedError()
val_fbeta = self.val_fbeta(
y_pred[:, warm_up_left : num_frames - warm_up_right : 10].squeeze(),
y[:, warm_up_left : num_frames - warm_up_right : 10].squeeze(),
)
self.model.log(
f"{self.ACRONYM}@val_fbeta",
val_fbeta,
self.model.validation_metric,
on_step=False,
on_epoch=True,
prog_bar=True,
+45 -170
View File
@@ -20,11 +20,9 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import math
from collections import Counter
from typing import Text, Tuple, Union
from typing import Optional, Text, Tuple, Union
import matplotlib.pyplot as plt
import numpy as np
import torch
from torch_audiomentations.core.transforms_interface import BaseWaveformTransform
@@ -126,55 +124,53 @@ class Segmentation(SegmentationTaskMixin, Task):
self.loss = loss
self.vad_loss = vad_loss
def setup(self, stage=None):
def setup(self, stage: Optional[str] = None):
super().setup(stage=stage)
if stage == "fit":
# slide a window (with 1s step) over the whole training set
# and keep track of the number of speakers in each location
num_speakers = []
for file in self._train:
start = file["annotated"][0].start
end = file["annotated"][-1].end
window = SlidingWindow(
start=start,
end=end,
duration=self.duration,
step=1.0,
)
for chunk in window:
num_speakers.append(len(file["annotation"].crop(chunk).labels()))
# because there might a few outliers, estimate the upper bound for the
# number of speakers as the 99th percentile
num_speakers, counts = zip(*list(Counter(num_speakers).items()))
num_speakers, counts = np.array(num_speakers), np.array(counts)
sorting_indices = np.argsort(num_speakers)
num_speakers = num_speakers[sorting_indices]
counts = counts[sorting_indices]
self.num_speakers = num_speakers[
np.where(np.cumsum(counts) / np.sum(counts) > 0.99)[0][0]
]
# TODO: add a few more speakers to make sure we don't skip
# too many artificial chunks (which might result in less
# overlap that we think we have)
# now that we know about the number of speakers upper bound
# we can set task specifications
self.specifications = Specifications(
problem=Problem.MULTI_LABEL_CLASSIFICATION,
resolution=Resolution.FRAME,
# slide a window (with 1s step) over the whole training set
# and keep track of the number of speakers in each location
num_speakers = []
for file in self._train:
start = file["annotated"][0].start
end = file["annotated"][-1].end
window = SlidingWindow(
start=start,
end=end,
duration=self.duration,
warm_up=self.warm_up,
classes=[f"speaker#{i+1}" for i in range(self.num_speakers)],
permutation_invariant=True,
step=1.0,
)
for chunk in window:
num_speakers.append(len(file["annotation"].crop(chunk).labels()))
# because there might a few outliers, estimate the upper bound for the
# number of speakers as the 99th percentile
num_speakers, counts = zip(*list(Counter(num_speakers).items()))
num_speakers, counts = np.array(num_speakers), np.array(counts)
sorting_indices = np.argsort(num_speakers)
num_speakers = num_speakers[sorting_indices]
counts = counts[sorting_indices]
self.num_speakers = num_speakers[
np.where(np.cumsum(counts) / np.sum(counts) > 0.99)[0][0]
]
# TODO: add a few more speakers to make sure we don't skip
# too many artificial chunks (which might result in less
# overlap that we think we have)
# now that we know about the number of speakers upper bound
# we can set task specifications
self.specifications = Specifications(
problem=Problem.MULTI_LABEL_CLASSIFICATION,
resolution=Resolution.FRAME,
duration=self.duration,
warm_up=self.warm_up,
classes=[f"speaker#{i+1}" for i in range(self.num_speakers)],
permutation_invariant=True,
)
def prepare_y(self, one_hot_y: np.ndarray):
"""Zero-pad segmentation targets
@@ -372,127 +368,6 @@ class Segmentation(SegmentationTaskMixin, Task):
)
return {"loss": loss}
def validation_step(self, batch, batch_idx: int):
"""Compute validation F-score
Parameters
----------
batch : dict of torch.Tensor
Current batch.
batch_idx: int
Batch index.
"""
# move metric to model device
self.val_fbeta.to(self.model.device)
X, y = batch["X"], batch["y"]
# X = (batch_size, num_channels, num_samples)
# y = (batch_size, num_frames, num_classes)
y_pred = self.model(X)
_, num_frames, _ = y_pred.shape
# y_pred = (batch_size, num_frames, num_classes)
def validation_postprocess(self, y, y_pred):
permutated_y_pred, _ = permutate(y, y_pred)
warm_up_left = round(self.warm_up[0] / self.duration * num_frames)
warm_up_right = round(self.warm_up[1] / self.duration * num_frames)
val_fbeta = self.val_fbeta(
permutated_y_pred[
:, warm_up_left : num_frames - warm_up_right : 10
].squeeze(),
y[:, warm_up_left : num_frames - warm_up_right : 10].squeeze(),
)
self.model.log(
f"{self.ACRONYM}@val_fbeta",
val_fbeta,
on_step=False,
on_epoch=True,
prog_bar=True,
logger=True,
)
# log first batch visualization every 2^n epochs.
if (
self.model.current_epoch == 0
or math.log2(self.model.current_epoch) % 1 > 0
or batch_idx > 0
):
return
# visualize first 9 validation samples of first batch in Tensorboard
X = X.cpu().numpy()
y = y.float().cpu().numpy()
y_pred = y_pred.cpu().numpy()
permutated_y_pred = permutated_y_pred.cpu().numpy()
# prepare 3 x 3 grid (or smaller if batch size is smaller)
num_samples = min(self.batch_size, 9)
nrows = math.ceil(math.sqrt(num_samples))
ncols = math.ceil(num_samples / nrows)
fig, axes = plt.subplots(
nrows=4 * nrows,
ncols=ncols,
figsize=(15, 10),
)
# reshape target so that there is one line per class when plottingit
y[y == 0] = np.NaN
y *= np.arange(y.shape[2])
# plot each sample
for sample_idx in range(num_samples):
# find where in the grid it should be plotted
row_idx = sample_idx // nrows
col_idx = sample_idx % ncols
# plot waveform
ax_wav = axes[row_idx * 4 + 0, col_idx]
sample_X = np.mean(X[sample_idx], axis=0)
ax_wav.plot(sample_X)
ax_wav.set_xlim(0, len(sample_X))
ax_wav.get_xaxis().set_visible(False)
ax_wav.get_yaxis().set_visible(False)
# plot target
ax_ref = axes[row_idx * 4 + 1, col_idx]
sample_y = y[sample_idx]
ax_ref.plot(sample_y)
ax_ref.set_xlim(0, len(sample_y))
ax_ref.set_ylim(-1, sample_y.shape[1])
ax_ref.get_xaxis().set_visible(False)
ax_ref.get_yaxis().set_visible(False)
# plot prediction
ax_hyp = axes[row_idx * 4 + 2, col_idx]
sample_y_pred = y_pred[sample_idx]
ax_hyp.axvspan(0, warm_up_left, color="k", alpha=0.5, lw=0)
ax_hyp.axvspan(
num_frames - warm_up_right, num_frames, color="k", alpha=0.5, lw=0
)
ax_hyp.plot(sample_y_pred)
ax_hyp.set_ylim(-0.1, 1.1)
ax_hyp.set_xlim(0, len(sample_y))
ax_hyp.get_xaxis().set_visible(False)
# plot permutated prediction
ax_map = axes[row_idx * 4 + 3, col_idx]
sample_y_pred_map = permutated_y_pred[sample_idx]
ax_map.axvspan(0, warm_up_left, color="k", alpha=0.5, lw=0)
ax_map.axvspan(
num_frames - warm_up_right, num_frames, color="k", alpha=0.5, lw=0
)
ax_map.plot(sample_y_pred_map)
ax_map.set_ylim(-0.1, 1.1)
ax_map.set_xlim(0, len(sample_y))
plt.tight_layout()
self.model.logger.experiment.add_figure(
f"{self.ACRONYM}@val_samples", fig, self.model.current_epoch
)
plt.close(fig)
return permutated_y_pred
@@ -20,7 +20,7 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from typing import List, Text, Tuple, Union
from typing import List, Optional, Text, Tuple, Union
import numpy as np
from torch_audiomentations.core.transforms_interface import BaseWaveformTransform
@@ -105,21 +105,19 @@ class SpeakerTracking(SegmentationTaskMixin, Task):
# speakers should be tracked. therefore, we postpone
# the definition of specifications.
def setup(self, stage=None):
def setup(self, stage: Optional[str] = None):
super().setup(stage=stage)
if stage == "fit":
self.specifications = Specifications(
# one class per speaker
classes=sorted(self._train_metadata["annotation"]),
# multiple speakers can be active at once
problem=Problem.MULTI_LABEL_CLASSIFICATION,
resolution=Resolution.FRAME,
duration=self.duration,
warm_up=self.warm_up,
)
self.specifications = Specifications(
# one class per speaker
classes=sorted(self._train_metadata["annotation"]),
# multiple speakers can be active at once
problem=Problem.MULTI_LABEL_CLASSIFICATION,
resolution=Resolution.FRAME,
duration=self.duration,
warm_up=self.warm_up,
)
@property
def chunk_labels(self) -> List[Text]:
+2 -1
View File
@@ -7,11 +7,12 @@ pyannote.core >=4.1,<5.0
pyannote.database >=4.1,<5.0
pyannote.metrics >=3.0.1,<4.0
pyannote.pipeline >=2.0,<3.0
pytorch_lightning >= 1.2,<1.3
pytorch_lightning >=1.4.1,<1.5
pytorch_metric_learning >=0.9.98
semver >=2.10.2,<3.0
soundfile >=0.10.2,<0.11
torch >=1.8.1,<1.9
torch-audiomentations >=0.6.0
torchaudio >=0.8.1,<0.9
torchmetrics >=0.4.1,<0.5
typing_extensions >=3.7.4.3
-38
View File
@@ -1,38 +0,0 @@
import pytest
import torch
from torch.nn import Module
from pyannote.audio.augmentation.registry import (
register_augmentation,
unregister_augmentation,
)
class RandomAugmentation(Module):
"Fake Noise tranform"
def forward(self, waveforms):
if not self.training:
return waveforms
return torch.ones_like(waveforms)
def test_can_register_augmentation():
for when in ["output", "input"]:
net = Module()
aug = RandomAugmentation()
register_augmentation(aug, net, when=when)
assert hasattr(net, "__augmentation")
assert net.__augmentation[when] == aug
def test_can_unregistrer_augmentation():
net = Module()
register_augmentation(RandomAugmentation(), net, when="output")
unregister_augmentation(net, when="output")
assert not hasattr(net.__augmentation, "output")
def test_fail_unregister_augmentation():
with pytest.raises(ValueError):
unregister_augmentation(RandomAugmentation(), when="output")
-1
View File
@@ -16,7 +16,6 @@ def setup_tasks(task):
def create_dl(model, task):
m = model(task=task)
task.setup("fit")
m.setup("fit")
return task.train_dataloader()
+144
View File
@@ -0,0 +1,144 @@
import pytest
from pytorch_lightning import Trainer
from pyannote.audio.models.segmentation.debug import SimpleSegmentationModel
from pyannote.audio.tasks import (
OverlappedSpeechDetection,
Segmentation,
VoiceActivityDetection,
)
from pyannote.database import FileFinder, get_protocol
@pytest.fixture()
def protocol():
return get_protocol(
"Debug.SpeakerDiarization.Debug", preprocessors={"audio": FileFinder()}
)
def test_train_segmentation(protocol):
segmentation = Segmentation(protocol)
model = SimpleSegmentationModel(task=segmentation)
trainer = Trainer(fast_dev_run=True)
trainer.fit(model)
def test_train_voice_activity_detection(protocol):
voice_activity_detection = VoiceActivityDetection(protocol)
model = SimpleSegmentationModel(task=voice_activity_detection)
trainer = Trainer(fast_dev_run=True)
trainer.fit(model)
def test_train_overlapped_speech_detection(protocol):
overlapped_speech_detection = OverlappedSpeechDetection(protocol)
model = SimpleSegmentationModel(task=overlapped_speech_detection)
trainer = Trainer(fast_dev_run=True)
trainer.fit(model)
def test_finetune_with_task_that_does_not_need_setup_for_specs(protocol):
voice_activity_detection = VoiceActivityDetection(protocol)
model = SimpleSegmentationModel(task=voice_activity_detection)
trainer = Trainer(fast_dev_run=True)
trainer.fit(model)
voice_activity_detection = VoiceActivityDetection(protocol)
model.task = voice_activity_detection
trainer = Trainer(fast_dev_run=True)
trainer.fit(model)
def test_finetune_with_task_that_needs_setup_for_specs(protocol):
segmentation = Segmentation(protocol)
model = SimpleSegmentationModel(task=segmentation)
trainer = Trainer(fast_dev_run=True)
trainer.fit(model)
segmentation = Segmentation(protocol)
model.task = segmentation
trainer = Trainer(fast_dev_run=True)
trainer.fit(model)
def test_transfer_with_task_that_does_not_need_setup_for_specs(protocol):
segmentation = Segmentation(protocol)
model = SimpleSegmentationModel(task=segmentation)
trainer = Trainer(fast_dev_run=True)
trainer.fit(model)
voice_activity_detection = VoiceActivityDetection(protocol)
model.task = voice_activity_detection
trainer = Trainer(fast_dev_run=True)
trainer.fit(model)
def test_transfer_with_task_that_needs_setup_for_specs(protocol):
voice_activity_detection = VoiceActivityDetection(protocol)
model = SimpleSegmentationModel(task=voice_activity_detection)
trainer = Trainer(fast_dev_run=True)
trainer.fit(model)
segmentation = Segmentation(protocol)
model.task = segmentation
trainer = Trainer(fast_dev_run=True)
trainer.fit(model)
def test_finetune_freeze_with_task_that_needs_setup_for_specs(protocol):
segmentation = Segmentation(protocol)
model = SimpleSegmentationModel(task=segmentation)
trainer = Trainer(fast_dev_run=True)
trainer.fit(model)
segmentation = Segmentation(protocol)
model.task = segmentation
model.freeze_up_to("mfcc")
trainer = Trainer(fast_dev_run=True)
trainer.fit(model)
def test_finetune_freeze_with_task_that_does_not_need_setup_for_specs(protocol):
vad = VoiceActivityDetection(protocol)
model = SimpleSegmentationModel(task=vad)
trainer = Trainer(fast_dev_run=True)
trainer.fit(model)
vad = VoiceActivityDetection(protocol)
model.task = vad
model.freeze_up_to("mfcc")
trainer = Trainer(fast_dev_run=True)
trainer.fit(model)
def test_transfer_freeze_with_task_that_does_not_need_setup_for_specs(protocol):
segmentation = Segmentation(protocol)
model = SimpleSegmentationModel(task=segmentation)
trainer = Trainer(fast_dev_run=True)
trainer.fit(model)
voice_activity_detection = VoiceActivityDetection(protocol)
model.task = voice_activity_detection
model.freeze_up_to("mfcc")
trainer = Trainer(fast_dev_run=True)
trainer.fit(model)
def test_transfer_freeze_with_task_that_needs_setup_for_specs(protocol):
voice_activity_detection = VoiceActivityDetection(protocol)
model = SimpleSegmentationModel(task=voice_activity_detection)
trainer = Trainer(fast_dev_run=True)
trainer.fit(model)
segmentation = Segmentation(protocol)
model.task = segmentation
model.freeze_up_to("mfcc")
trainer = Trainer(fast_dev_run=True)
trainer.fit(model)