mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
Merge branch 'develop' of https://github.com/pyannote/pyannote-audio into develop
This commit is contained in:
commit
23c4eaaed5
13 files changed
+609
-463
No files matched your search
@@ -1,4 +1,4 @@
|
||||
# Neural speaker diarization with pyannote.audio
|
||||
# Neural speaker diarization
|
||||
|
||||
This is the development branch of upcoming `pyannote.audio` 2.0 for which it has been decided to rewrite almost everything from scratch. Highlights of this upcoming release will be:
|
||||
|
||||
@@ -8,7 +8,7 @@ This is the development branch of upcoming `pyannote.audio` 2.0 for which it has
|
||||
- data augmentation with [torch-audiomentations](https://github.com/asteroid-team/torch-audiomentations)
|
||||
- [huggingface](https://huggingface.co/pyannote) model hosting
|
||||
- [prodigy](https://prodi.gy) recipes for audio annotations
|
||||
- online [demo](https://share.streamlit.io/pyannote/pyannote-audio-demo/main) based on [streamlit](https://www.streamlit.io)
|
||||
- online [demo](https://huggingface.co/spaces/pyannote/segmentation) based on [streamlit](https://www.streamlit.io)
|
||||
|
||||
## Installation
|
||||
|
||||
|
||||
@@ -81,7 +81,6 @@ def main(cfg: DictConfig) -> Optional[float]:
|
||||
# add task-dependent layers so that later call to model.parameters()
|
||||
# does return all layers (even task-dependent ones). this is already
|
||||
# done for pretrained models (TODO: check that this is true)
|
||||
task.setup(stage="fit")
|
||||
model.setup(stage="fit")
|
||||
|
||||
def configure_optimizers(self):
|
||||
@@ -127,7 +126,7 @@ def main(cfg: DictConfig) -> Optional[float]:
|
||||
monitor=monitor,
|
||||
mode=direction,
|
||||
save_top_k=None if monitor is None else 5,
|
||||
period=1,
|
||||
every_n_epochs=1,
|
||||
save_last=True,
|
||||
save_weights_only=False,
|
||||
dirpath=".",
|
||||
|
||||
@@ -37,6 +37,7 @@ from huggingface_hub import cached_download, hf_hub_url
|
||||
from pytorch_lightning.core.memory import ModelSummary
|
||||
from pytorch_lightning.utilities.cloud_io import load as pl_load
|
||||
from semver import VersionInfo
|
||||
from torch.utils.data import DataLoader
|
||||
|
||||
from pyannote.audio import __version__
|
||||
from pyannote.audio.core.io import Audio
|
||||
@@ -274,19 +275,16 @@ class Model(pl.LightningModule):
|
||||
sample_rate=self.hparams.sample_rate, mono=self.hparams.num_channels == 1
|
||||
)
|
||||
|
||||
@property
|
||||
def datamodule(self):
|
||||
return self.task
|
||||
|
||||
@property
|
||||
def example_input_array(self) -> torch.Tensor:
|
||||
batch_size = 3 if self.task is None else self.task.batch_size
|
||||
duration = 2.0 if self.task is None else self.task.duration
|
||||
|
||||
return torch.randn(
|
||||
(
|
||||
batch_size,
|
||||
self.hparams.num_channels,
|
||||
int(self.hparams.sample_rate * self.specifications.duration),
|
||||
int(self.hparams.sample_rate * duration),
|
||||
),
|
||||
device=self.device,
|
||||
)
|
||||
@@ -347,6 +345,9 @@ class Model(pl.LightningModule):
|
||||
|
||||
def setup(self, stage=None):
|
||||
|
||||
if stage == "fit":
|
||||
self.task.setup()
|
||||
|
||||
# list of layers before adding task-dependent layers
|
||||
before = set((name, id(module)) for name, module in self.named_modules())
|
||||
|
||||
@@ -365,7 +366,11 @@ class Model(pl.LightningModule):
|
||||
# setup custom loss function
|
||||
self.task.setup_loss_func()
|
||||
# setup custom validation metrics
|
||||
self.task.setup_validation_metric()
|
||||
validation_metric = self.task.setup_validation_metric()
|
||||
if validation_metric is not None:
|
||||
self.validation_metric = validation_metric
|
||||
self.validation_metric.to(self.device)
|
||||
|
||||
# this is to make sure introspection is performed here, once and for all
|
||||
_ = self.introspection
|
||||
|
||||
@@ -477,11 +482,21 @@ class Model(pl.LightningModule):
|
||||
"""
|
||||
return self.helper_default_activation(self.specifications)
|
||||
|
||||
# training data logic is delegated to the task because the
|
||||
# model does not really need to know how it is being used.
|
||||
def train_dataloader(self) -> DataLoader:
|
||||
return self.task.train_dataloader()
|
||||
|
||||
# training step logic is delegated to the task because the
|
||||
# model does not really need to know how it is being used.
|
||||
def training_step(self, batch, batch_idx):
|
||||
return self.task.training_step(batch, batch_idx)
|
||||
|
||||
# validation data logic is delegated to the task because the
|
||||
# model does not really need to know how it is being used.
|
||||
def val_dataloader(self) -> DataLoader:
|
||||
return self.task.val_dataloader()
|
||||
|
||||
# validation logic is delegated to the task because the
|
||||
# model does not really need to know how it is being used.
|
||||
def validation_step(self, batch, batch_idx):
|
||||
@@ -822,7 +837,6 @@ class Model(pl.LightningModule):
|
||||
|
||||
if task is not None:
|
||||
model.task = task
|
||||
task.setup(stage="fit")
|
||||
model.setup(stage="fit")
|
||||
|
||||
try:
|
||||
|
||||
@@ -218,13 +218,8 @@ class Task(pl.LightningDataModule):
|
||||
"""
|
||||
pass
|
||||
|
||||
def setup(self, stage=None):
|
||||
"""Called at the beginning of fit and test just before Model.setup()
|
||||
|
||||
Parameters
|
||||
----------
|
||||
stage : "fit" or "test"
|
||||
Whether model is being trained ("fit") or used for inference ("test").
|
||||
def setup(self, stage: Optional[str] = None):
|
||||
"""Called at the beginning of training at the very beginning of Model.setup(stage="fit")
|
||||
|
||||
Notes
|
||||
-----
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
# MIT License
|
||||
#
|
||||
# Copyright (c) 2021 CNRS
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from pyannote.audio import Inference, Model, Pipeline
|
||||
from pyannote.audio.core.io import AudioFile
|
||||
from pyannote.audio.pipelines.utils import PipelineModel, get_devices, get_model
|
||||
|
||||
|
||||
class SpeakerEmbedding(Pipeline):
|
||||
"""Speaker embedding pipeline
|
||||
|
||||
This pipeline assumes that each file contains exactly one speaker
|
||||
and extracts one single embedding from the whole file.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
embedding : Model, str, or dict, optional
|
||||
Pretrained embedding model. Defaults to "pyannote/embedding".
|
||||
See pyannote.audio.pipelines.utils.get_model for supported format.
|
||||
segmentation : Model, str, or dict, optional
|
||||
Pretrained segmentation (or voice activity detection) model.
|
||||
See pyannote.audio.pipelines.utils.get_model for supported format.
|
||||
Defaults to no voice activity detection.
|
||||
|
||||
Usage
|
||||
-----
|
||||
>>> from pyannote.audio.pipelines import SpeakerEmbedding
|
||||
>>> pipeline = SpeakerEmbedding()
|
||||
>>> emb1 = pipeline("speaker1.wav")
|
||||
>>> emb2 = pipeline("speaker2.wav")
|
||||
>>> from scipy.spatial.distance import cdist
|
||||
>>> distance = cdist(emb1, emb2, metric="cosine")[0,0]
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
embedding: PipelineModel = "pyannote/embedding",
|
||||
segmentation: PipelineModel = None,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
self.embedding = embedding
|
||||
self.segmentation = segmentation
|
||||
|
||||
self.embedding_model_: Model = get_model(embedding)
|
||||
|
||||
if self.segmentation is None:
|
||||
models = [self.embedding_model_]
|
||||
else:
|
||||
segmentation_model: Model = get_model(self.segmentation)
|
||||
models = [self.embedding_model_, segmentation_model]
|
||||
|
||||
# send models to GPU (when GPUs are available and model is not already on GPU)
|
||||
cpu_models = [model for model in models if model.device.type == "cpu"]
|
||||
for cpu_model, gpu_device in zip(
|
||||
cpu_models, get_devices(needs=len(cpu_models))
|
||||
):
|
||||
cpu_model.to(gpu_device)
|
||||
|
||||
if self.segmentation is not None:
|
||||
self.voice_activity_ = Inference(
|
||||
segmentation_model,
|
||||
pre_aggregation_hook=lambda scores: np.max(
|
||||
scores, axis=-1, keepdims=True
|
||||
),
|
||||
)
|
||||
|
||||
def apply(self, file: AudioFile) -> np.ndarray:
|
||||
|
||||
device = self.embedding_model_.device
|
||||
|
||||
# read audio file and send it to GPU
|
||||
waveform = self.embedding_model_.audio(file)[0][None].to(device)
|
||||
|
||||
if self.segmentation is None:
|
||||
weights = None
|
||||
else:
|
||||
# obtain voice activity scores
|
||||
weights = self.voice_activity_(file).data
|
||||
# HACK -- this should be fixed upstream
|
||||
weights[np.isnan(weights)] = 0.0
|
||||
weights = torch.from_numpy(weights ** 3)[None, :, 0].to(device)
|
||||
|
||||
# extract speaker embedding on parts of
|
||||
with torch.no_grad():
|
||||
return self.embedding_model_(waveform, weights=weights).cpu().numpy()
|
||||
|
||||
|
||||
def main(
|
||||
protocol: str = "VoxCeleb.SpeakerVerification.VoxCeleb1",
|
||||
subset: str = "test",
|
||||
embedding: str = "pyannote/embedding",
|
||||
segmentation: str = None,
|
||||
):
|
||||
|
||||
import typer
|
||||
from scipy.spatial.distance import cdist
|
||||
from tqdm import tqdm
|
||||
|
||||
from pyannote.database import FileFinder, get_protocol
|
||||
from pyannote.metrics.binary_classification import det_curve
|
||||
|
||||
pipeline = SpeakerEmbedding(embedding=embedding, segmentation=segmentation)
|
||||
|
||||
protocol = get_protocol(protocol, preprocessors={"audio": FileFinder()})
|
||||
|
||||
y_true, y_pred = [], []
|
||||
|
||||
emb = dict()
|
||||
|
||||
trials = getattr(protocol, f"{subset}_trial")()
|
||||
|
||||
for t, trial in enumerate(tqdm(trials)):
|
||||
|
||||
audio1 = trial["file1"]["audio"]
|
||||
if audio1 not in emb:
|
||||
emb[audio1] = pipeline(audio1)
|
||||
|
||||
audio2 = trial["file2"]["audio"]
|
||||
if audio2 not in emb:
|
||||
emb[audio2] = pipeline(audio2)
|
||||
|
||||
y_pred.append(cdist(emb[audio1], emb[audio2], metric="cosine")[0][0])
|
||||
y_true.append(trial["reference"])
|
||||
|
||||
_, _, _, eer = det_curve(y_true, np.array(y_pred), distances=True)
|
||||
typer.echo(
|
||||
f"{protocol.name} | {subset} | {embedding} | {segmentation} | EER = {100 * eer:.3f}%"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import typer
|
||||
|
||||
typer.run(main)
|
||||
@@ -1,6 +1,6 @@
|
||||
# MIT License
|
||||
#
|
||||
# Copyright (c) 2020 CNRS
|
||||
# Copyright (c) 2020-2021 CNRS
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
@@ -23,20 +23,18 @@
|
||||
import math
|
||||
from typing import Optional
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from torch.utils.data import DataLoader
|
||||
import torch.nn.functional as F
|
||||
from torchmetrics import AUROC
|
||||
from tqdm import tqdm
|
||||
|
||||
from pyannote.audio.core.task import Problem, Resolution, Specifications, ValDataset
|
||||
from pyannote.audio.core.task import Problem, Resolution, Specifications
|
||||
from pyannote.audio.utils.random import create_rng_for_worker
|
||||
from pyannote.core import Segment
|
||||
from pyannote.core.utils.distance import cdist
|
||||
from pyannote.database.protocol import (
|
||||
SpeakerDiarizationProtocol,
|
||||
SpeakerVerificationProtocol,
|
||||
)
|
||||
from pyannote.metrics.binary_classification import det_curve
|
||||
|
||||
|
||||
class SupervisedRepresentationLearningTaskMixin:
|
||||
@@ -74,65 +72,60 @@ class SupervisedRepresentationLearningTaskMixin:
|
||||
def batch_size(self, batch_size: int):
|
||||
self.batch_size_ = batch_size
|
||||
|
||||
def setup(self, stage=None):
|
||||
def setup(self, stage: Optional[str] = None):
|
||||
|
||||
if stage == "fit":
|
||||
# loop over the training set, remove annotated regions shorter than
|
||||
# chunk duration, and keep track of the reference annotations, per class.
|
||||
|
||||
# loop over the training set, remove annotated regions shorter than
|
||||
# chunk duration, and keep track of the reference annotations, per class.
|
||||
self._train = dict()
|
||||
|
||||
self._train = dict()
|
||||
desc = f"Loading {self.protocol.name} training labels"
|
||||
for f in tqdm(iterable=self.protocol.train(), desc=desc, unit="file"):
|
||||
|
||||
desc = f"Loading {self.protocol.name} training labels"
|
||||
for f in tqdm(iterable=self.protocol.train(), desc=desc, unit="file"):
|
||||
for klass in f["annotation"].labels():
|
||||
|
||||
for klass in f["annotation"].labels():
|
||||
# keep class's (long enough) speech turns...
|
||||
speech_turns = [
|
||||
segment
|
||||
for segment in f["annotation"].label_timeline(klass)
|
||||
if segment.duration > self.min_duration
|
||||
]
|
||||
|
||||
# keep class's (long enough) speech turns...
|
||||
speech_turns = [
|
||||
segment
|
||||
for segment in f["annotation"].label_timeline(klass)
|
||||
if segment.duration > self.duration
|
||||
]
|
||||
# skip if there is no speech turns left
|
||||
if not speech_turns:
|
||||
continue
|
||||
|
||||
# skip if there is no speech turns left
|
||||
if not speech_turns:
|
||||
continue
|
||||
# ... and their total duration
|
||||
duration = sum(segment.duration for segment in speech_turns)
|
||||
|
||||
# ... and their total duration
|
||||
duration = sum(segment.duration for segment in speech_turns)
|
||||
# add class to the list of classes
|
||||
if klass not in self._train:
|
||||
self._train[klass] = list()
|
||||
|
||||
# add class to the list of classes
|
||||
if klass not in self._train:
|
||||
self._train[klass] = list()
|
||||
self._train[klass].append(
|
||||
{
|
||||
"uri": f["uri"],
|
||||
"audio": f["audio"],
|
||||
"duration": duration,
|
||||
"speech_turns": speech_turns,
|
||||
}
|
||||
)
|
||||
|
||||
self._train[klass].append(
|
||||
{
|
||||
"uri": f["uri"],
|
||||
"audio": f["audio"],
|
||||
"duration": duration,
|
||||
"speech_turns": speech_turns,
|
||||
}
|
||||
)
|
||||
self.specifications = Specifications(
|
||||
problem=Problem.REPRESENTATION,
|
||||
resolution=Resolution.CHUNK,
|
||||
duration=self.duration,
|
||||
classes=sorted(self._train),
|
||||
)
|
||||
|
||||
self.specifications = Specifications(
|
||||
problem=Problem.REPRESENTATION,
|
||||
resolution=Resolution.CHUNK,
|
||||
duration=self.duration,
|
||||
classes=sorted(self._train),
|
||||
)
|
||||
if not self.has_validation:
|
||||
return
|
||||
|
||||
if not self.has_validation:
|
||||
return
|
||||
if isinstance(self.protocol, SpeakerVerificationProtocol):
|
||||
self._validation = list(self.protocol.development_trial())
|
||||
|
||||
if isinstance(self.protocol, SpeakerVerificationProtocol):
|
||||
sessions = dict()
|
||||
for trial in self.protocol.development_trial():
|
||||
for session in ["file1", "file2"]:
|
||||
session_hash = self.helper_trial_hash(trial[session])
|
||||
if session_hash not in sessions:
|
||||
sessions[session_hash] = trial[session]
|
||||
self._validation = list(sessions.items())
|
||||
def setup_validation_metric(self):
|
||||
return AUROC(compute_on_step=False)
|
||||
|
||||
def train__iter__(self):
|
||||
"""Iterate over training samples
|
||||
@@ -183,18 +176,29 @@ class SupervisedRepresentationLearningTaskMixin:
|
||||
k=1,
|
||||
)
|
||||
|
||||
# select one chunk at random (with uniform distribution)
|
||||
start_time = rng.uniform(
|
||||
speech_turn.start, speech_turn.end - batch_duration
|
||||
)
|
||||
chunk = Segment(start_time, start_time + batch_duration)
|
||||
# if speech turn is too short, pad with zeros
|
||||
if speech_turn.duration < batch_duration:
|
||||
X, _ = self.model.audio.crop(file, speech_turn, mode="center")
|
||||
num_missing_frames = (
|
||||
math.floor(batch_duration * self.model.audio.sample_rate)
|
||||
- X.shape[1]
|
||||
)
|
||||
left_pad = rng.randint(0, num_missing_frames)
|
||||
X = F.pad(X, (left_pad, num_missing_frames - left_pad))
|
||||
|
||||
X, _ = self.model.audio.crop(
|
||||
file,
|
||||
chunk,
|
||||
mode="center",
|
||||
fixed=batch_duration,
|
||||
)
|
||||
# if it is long enough, select chunk at random
|
||||
else:
|
||||
start_time = rng.uniform(
|
||||
speech_turn.start, speech_turn.end - batch_duration
|
||||
)
|
||||
chunk = Segment(start_time, start_time + batch_duration)
|
||||
|
||||
X, _ = self.model.audio.crop(
|
||||
file,
|
||||
chunk,
|
||||
mode="center",
|
||||
fixed=batch_duration,
|
||||
)
|
||||
|
||||
yield {"X": X, "y": y}
|
||||
|
||||
@@ -225,21 +229,33 @@ class SupervisedRepresentationLearningTaskMixin:
|
||||
)
|
||||
return {"loss": loss}
|
||||
|
||||
@staticmethod
|
||||
def helper_trial_hash(file) -> int:
|
||||
return hash((file["database"], file["uri"], tuple(file["try_with"])))
|
||||
|
||||
def val__getitem__(self, idx):
|
||||
if isinstance(self.protocol, SpeakerVerificationProtocol):
|
||||
session_hash, session = self._validation[idx]
|
||||
X = np.concatenate(
|
||||
[
|
||||
self.model.audio.crop(session, segment, mode="center")[0]
|
||||
for segment in session["try_with"]
|
||||
],
|
||||
axis=0,
|
||||
)
|
||||
return {"session_hash": session_hash, "X": X}
|
||||
trial = self._validation[idx]
|
||||
|
||||
data = dict()
|
||||
for idx in [1, 2]:
|
||||
file = trial[f"file{idx:d}"]
|
||||
duration = self.model.audio.get_duration(file)
|
||||
if duration > self.duration:
|
||||
middle = Segment(
|
||||
0.5 * duration - 0.5 * self.duration,
|
||||
0.5 * duration + 0.5 * self.duration,
|
||||
)
|
||||
X, _ = self.model.audio.crop(
|
||||
file, middle, mode="center", fixed=self.duration
|
||||
)
|
||||
else:
|
||||
X, _ = self.model.audio(file)
|
||||
num_missing_frames = (
|
||||
math.floor(self.duration * self.model.audio.sample_rate)
|
||||
- X.shape[1]
|
||||
)
|
||||
X = F.pad(X, (0, num_missing_frames))
|
||||
data[f"X{idx:d}"] = X
|
||||
data["y"] = trial["reference"]
|
||||
|
||||
return data
|
||||
|
||||
elif isinstance(self.protocol, SpeakerDiarizationProtocol):
|
||||
pass
|
||||
@@ -255,80 +271,34 @@ class SupervisedRepresentationLearningTaskMixin:
|
||||
def validation_step(self, batch, batch_idx: int):
|
||||
|
||||
if isinstance(self.protocol, SpeakerVerificationProtocol):
|
||||
return (
|
||||
batch["session_hash"][0].detach().cpu().numpy().item(),
|
||||
self.model(batch["X"]).detach().cpu().numpy(),
|
||||
|
||||
with torch.no_grad():
|
||||
emb1 = self.model(batch["X1"]).detach()
|
||||
emb2 = self.model(batch["X2"]).detach()
|
||||
y_pred = F.cosine_similarity(emb1, emb2)
|
||||
|
||||
y_true = batch["y"]
|
||||
self.model.validation_metric(y_pred, y_true)
|
||||
|
||||
self.model.log(
|
||||
f"{self.ACRONYM}@val_auroc",
|
||||
self.model.validation_metric,
|
||||
on_step=False,
|
||||
on_epoch=True,
|
||||
prog_bar=True,
|
||||
logger=True,
|
||||
)
|
||||
|
||||
elif isinstance(self.protocol, SpeakerDiarizationProtocol):
|
||||
pass
|
||||
|
||||
def validation_epoch_end(self, outputs):
|
||||
|
||||
if isinstance(self.protocol, SpeakerVerificationProtocol):
|
||||
|
||||
embeddings = dict(outputs)
|
||||
|
||||
y_true, y_pred = [], []
|
||||
for trial in self.protocol.development_trial():
|
||||
|
||||
session1_hash = self.helper_trial_hash(trial["file1"])
|
||||
session2_hash = self.helper_trial_hash(trial["file2"])
|
||||
|
||||
try:
|
||||
emb1 = embeddings[session1_hash]
|
||||
emb2 = embeddings[session2_hash]
|
||||
except KeyError:
|
||||
return
|
||||
|
||||
y_pred.append(cdist(emb1, emb2, metric="cosine").item())
|
||||
y_true.append(trial["reference"])
|
||||
|
||||
y_pred = np.array(y_pred)
|
||||
y_true = np.array(y_true)
|
||||
|
||||
num_target_trials = np.sum(y_true)
|
||||
num_non_target_trials = np.sum(1.0 - y_true)
|
||||
if num_target_trials > 2 and num_non_target_trials > 2:
|
||||
fpr, fnr, thresholds, eer = det_curve(y_true, y_pred, distances=True)
|
||||
|
||||
self.model.log(
|
||||
f"{self.ACRONYM}@val_eer",
|
||||
torch.tensor(eer, device=self.model.device),
|
||||
logger=True,
|
||||
on_epoch=True,
|
||||
prog_bar=True,
|
||||
sync_dist=True,
|
||||
)
|
||||
|
||||
elif isinstance(self.protocol, SpeakerDiarizationProtocol):
|
||||
pass
|
||||
|
||||
def val_dataloader(self) -> Optional[DataLoader]:
|
||||
|
||||
if self.has_validation:
|
||||
|
||||
if isinstance(self.protocol, SpeakerVerificationProtocol):
|
||||
return DataLoader(
|
||||
ValDataset(self),
|
||||
batch_size=1,
|
||||
pin_memory=self.pin_memory,
|
||||
drop_last=False,
|
||||
)
|
||||
|
||||
elif isinstance(self.protocol, SpeakerDiarizationProtocol):
|
||||
return None
|
||||
|
||||
else:
|
||||
return None
|
||||
|
||||
@property
|
||||
def val_monitor(self):
|
||||
|
||||
if self.has_validation:
|
||||
|
||||
if isinstance(self.protocol, SpeakerVerificationProtocol):
|
||||
return f"{self.ACRONYM}@val_eer", "min"
|
||||
return f"{self.ACRONYM}@val_auroc", "max"
|
||||
|
||||
elif isinstance(self.protocol, SpeakerDiarizationProtocol):
|
||||
return None, "min"
|
||||
|
||||
@@ -27,7 +27,8 @@ from typing import List, Optional, Text, Tuple
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
from pytorch_lightning.metrics.classification import FBeta
|
||||
from einops import rearrange
|
||||
from torchmetrics import FBeta
|
||||
from typing_extensions import Literal
|
||||
|
||||
from pyannote.audio.core.io import Audio, AudioFile
|
||||
@@ -39,92 +40,86 @@ from pyannote.core import Annotation, Segment, SlidingWindow, SlidingWindowFeatu
|
||||
class SegmentationTaskMixin:
|
||||
"""Methods common to most segmentation tasks"""
|
||||
|
||||
def setup(self, stage=None):
|
||||
def setup(self, stage: Optional[str] = None):
|
||||
|
||||
if stage == "fit":
|
||||
# ==================================================================
|
||||
# PREPARE TRAINING DATA
|
||||
# ==================================================================
|
||||
|
||||
# ==================================================================
|
||||
# PREPARE TRAINING DATA
|
||||
# ==================================================================
|
||||
self._train = []
|
||||
self._train_metadata = dict()
|
||||
|
||||
self._train = []
|
||||
self._train_metadata = dict()
|
||||
for f in self.protocol.train():
|
||||
|
||||
for f in self.protocol.train():
|
||||
file = dict()
|
||||
|
||||
file = dict()
|
||||
for key, value in f.items():
|
||||
|
||||
for key, value in f.items():
|
||||
# keep track of unique labels in self._train_metadata["annotation"]
|
||||
if key == "annotation":
|
||||
for label in value.labels():
|
||||
self._train_metadata.setdefault("annotation", set()).add(label)
|
||||
|
||||
# keep track of unique labels in self._train_metadata["annotation"]
|
||||
if key == "annotation":
|
||||
for label in value.labels():
|
||||
self._train_metadata.setdefault("annotation", set()).add(
|
||||
label
|
||||
)
|
||||
# pass "audio" entry as it is
|
||||
elif key == "audio":
|
||||
pass
|
||||
|
||||
# pass "audio" entry as it is
|
||||
elif key == "audio":
|
||||
pass
|
||||
# remove segments shorter than chunks from "annotated" entry
|
||||
elif key == "annotated":
|
||||
value = [
|
||||
segment for segment in value if segment.duration > self.duration
|
||||
]
|
||||
file["_annotated_duration"] = sum(
|
||||
segment.duration for segment in value
|
||||
)
|
||||
|
||||
# remove segments shorter than chunks from "annotated" entry
|
||||
elif key == "annotated":
|
||||
value = [
|
||||
segment
|
||||
for segment in value
|
||||
if segment.duration > self.duration
|
||||
]
|
||||
file["_annotated_duration"] = sum(
|
||||
segment.duration for segment in value
|
||||
)
|
||||
# keey track of unique text-like entries (incl. "uri" and "database")
|
||||
# and pass them as they are
|
||||
elif isinstance(value, Text):
|
||||
self._train_metadata.setdefault(key, set()).add(value)
|
||||
|
||||
# keey track of unique text-like entries (incl. "uri" and "database")
|
||||
# and pass them as they are
|
||||
elif isinstance(value, Text):
|
||||
self._train_metadata.setdefault(key, set()).add(value)
|
||||
# pass score-like entries as they are
|
||||
elif isinstance(value, SlidingWindowFeature):
|
||||
pass
|
||||
|
||||
# pass score-like entries as they are
|
||||
elif isinstance(value, SlidingWindowFeature):
|
||||
pass
|
||||
else:
|
||||
msg = (
|
||||
f"Protocol '{self.protocol.name}' defines a '{key}' entry of type {type(value)} "
|
||||
f"which we do not know how to handle."
|
||||
)
|
||||
warnings.warn(msg)
|
||||
file[key] = value
|
||||
|
||||
else:
|
||||
msg = (
|
||||
f"Protocol '{self.protocol.name}' defines a '{key}' entry of type {type(value)} "
|
||||
f"which we do not know how to handle."
|
||||
)
|
||||
warnings.warn(msg)
|
||||
file[key] = value
|
||||
self._train.append(file)
|
||||
|
||||
self._train.append(file)
|
||||
self._train_metadata = {
|
||||
key: sorted(values) for key, values in self._train_metadata.items()
|
||||
}
|
||||
|
||||
self._train_metadata = {
|
||||
key: sorted(values) for key, values in self._train_metadata.items()
|
||||
}
|
||||
# ==================================================================
|
||||
# PREPARE VALIDATION DATA
|
||||
# ==================================================================
|
||||
|
||||
# ==================================================================
|
||||
# PREPARE VALIDATION DATA
|
||||
# ==================================================================
|
||||
if not self.has_validation:
|
||||
return
|
||||
|
||||
if not self.has_validation:
|
||||
return
|
||||
self._validation = []
|
||||
|
||||
self._validation = []
|
||||
for f in self.protocol.development():
|
||||
|
||||
for f in self.protocol.development():
|
||||
for segment in f["annotated"]:
|
||||
|
||||
for segment in f["annotated"]:
|
||||
if segment.duration < self.duration:
|
||||
continue
|
||||
|
||||
if segment.duration < self.duration:
|
||||
continue
|
||||
num_chunks = round(segment.duration // self.duration)
|
||||
|
||||
num_chunks = round(segment.duration // self.duration)
|
||||
for c in range(num_chunks):
|
||||
start_time = segment.start + c * self.duration
|
||||
chunk = Segment(start_time, start_time + self.duration)
|
||||
self._validation.append((f, chunk))
|
||||
|
||||
for c in range(num_chunks):
|
||||
start_time = segment.start + c * self.duration
|
||||
chunk = Segment(start_time, start_time + self.duration)
|
||||
self._validation.append((f, chunk))
|
||||
|
||||
random.shuffle(self._validation)
|
||||
random.shuffle(self._validation)
|
||||
|
||||
def setup_validation_metric(self):
|
||||
"""Setup default validation metric
|
||||
@@ -132,14 +127,9 @@ class SegmentationTaskMixin:
|
||||
Use macro-average of F-score with a 0.5 threshold
|
||||
"""
|
||||
|
||||
self.val_fbeta = FBeta(
|
||||
len(self.specifications.classes),
|
||||
beta=1.0,
|
||||
threshold=0.5,
|
||||
multilabel=(
|
||||
self.specifications.problem == Problem.MULTI_LABEL_CLASSIFICATION
|
||||
),
|
||||
average="macro",
|
||||
num_classes = len(self.specifications.classes)
|
||||
return FBeta(
|
||||
num_classes, beta=1.0, threshold=0.5, average="macro", compute_on_step=False
|
||||
)
|
||||
|
||||
def prepare_y(self, one_hot_y: np.ndarray) -> np.ndarray:
|
||||
@@ -457,8 +447,11 @@ class SegmentationTaskMixin:
|
||||
def val__len__(self):
|
||||
return len(self._validation)
|
||||
|
||||
def validation_postprocess(self, y, y_pred):
|
||||
return y_pred
|
||||
|
||||
def validation_step(self, batch, batch_idx: int):
|
||||
"""Compute area under ROC curve
|
||||
"""Compute validation F-score
|
||||
|
||||
Parameters
|
||||
----------
|
||||
@@ -468,27 +461,66 @@ class SegmentationTaskMixin:
|
||||
Batch index.
|
||||
"""
|
||||
|
||||
# move metric to model device
|
||||
self.val_fbeta.to(self.model.device)
|
||||
|
||||
X, y = batch["X"], batch["y"]
|
||||
# X = (batch_size, num_channels, num_samples)
|
||||
# y = (batch_size, num_frames, num_classes)
|
||||
# y = (batch_size, num_frames, num_classes) or (batch_size, num_frames)
|
||||
|
||||
y_pred = self.model(X)
|
||||
_, num_frames, _ = y_pred.shape
|
||||
# y_pred = (batch_size, num_frames, num_classes)
|
||||
|
||||
# postprocess
|
||||
y_pred = self.validation_postprocess(y, y_pred)
|
||||
|
||||
# - remove warm-up frames
|
||||
# - downsample remaining frames
|
||||
warm_up_left = round(self.warm_up[0] / self.duration * num_frames)
|
||||
warm_up_right = round(self.warm_up[1] / self.duration * num_frames)
|
||||
preds = y_pred[:, warm_up_left : num_frames - warm_up_right : 10]
|
||||
target = y[:, warm_up_left : num_frames - warm_up_right : 10]
|
||||
|
||||
# torchmetrics tries to be smart about the type of machine learning problem
|
||||
# pyannote.audio is more explicit so we have to reshape target and preds for
|
||||
# torchmetrics to be happy... more details can be found here:
|
||||
# https://torchmetrics.readthedocs.io/en/latest/references/modules.html#input-types
|
||||
|
||||
if self.specifications.problem == Problem.BINARY_CLASSIFICATION:
|
||||
# target: shape (batch_size, num_frames), type binary
|
||||
# preds: shape (batch_size, num_frames, 1), type float
|
||||
|
||||
# torchmetrics expects:
|
||||
# target: shape (N,), type binary
|
||||
# preds: shape (N,), type float
|
||||
|
||||
self.model.validation_metric(preds.reshape(-1), target.reshape(-1))
|
||||
|
||||
elif self.specifications.problem == Problem.MULTI_LABEL_CLASSIFICATION:
|
||||
# target: shape (batch_size, num_frames, num_classes), type binary
|
||||
# preds: shape (batch_size, num_frames, num_classes), type float
|
||||
|
||||
# torchmetrics expects
|
||||
# target: shape (N, num_classes), type binary
|
||||
# preds: shape (N, num_classes), type float
|
||||
|
||||
self.model.validation_metric(
|
||||
rearrange(preds, "b f c -> (b f) c"),
|
||||
rearrange(target, "b f c -> (b f) c"),
|
||||
)
|
||||
|
||||
elif self.specifications.problem == Problem.MONO_LABEL_CLASSIFICATION:
|
||||
# target: shape (batch_size, num_frames, num_classes), type binary
|
||||
# preds: shape (batch_size, num_frames, num_classes), type float
|
||||
|
||||
# torchmetrics expects:
|
||||
# target: shape (N, ), type int
|
||||
# preds: shape (N, num_classes), type float
|
||||
|
||||
# TODO: implement when pyannote.audio gets its first mono-label segmentation task
|
||||
raise NotImplementedError()
|
||||
|
||||
val_fbeta = self.val_fbeta(
|
||||
y_pred[:, warm_up_left : num_frames - warm_up_right : 10].squeeze(),
|
||||
y[:, warm_up_left : num_frames - warm_up_right : 10].squeeze(),
|
||||
)
|
||||
self.model.log(
|
||||
f"{self.ACRONYM}@val_fbeta",
|
||||
val_fbeta,
|
||||
self.model.validation_metric,
|
||||
on_step=False,
|
||||
on_epoch=True,
|
||||
prog_bar=True,
|
||||
|
||||
@@ -20,11 +20,9 @@
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
import math
|
||||
from collections import Counter
|
||||
from typing import Text, Tuple, Union
|
||||
from typing import Optional, Text, Tuple, Union
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import torch
|
||||
from torch_audiomentations.core.transforms_interface import BaseWaveformTransform
|
||||
@@ -126,55 +124,53 @@ class Segmentation(SegmentationTaskMixin, Task):
|
||||
self.loss = loss
|
||||
self.vad_loss = vad_loss
|
||||
|
||||
def setup(self, stage=None):
|
||||
def setup(self, stage: Optional[str] = None):
|
||||
|
||||
super().setup(stage=stage)
|
||||
|
||||
if stage == "fit":
|
||||
|
||||
# slide a window (with 1s step) over the whole training set
|
||||
# and keep track of the number of speakers in each location
|
||||
num_speakers = []
|
||||
for file in self._train:
|
||||
start = file["annotated"][0].start
|
||||
end = file["annotated"][-1].end
|
||||
window = SlidingWindow(
|
||||
start=start,
|
||||
end=end,
|
||||
duration=self.duration,
|
||||
step=1.0,
|
||||
)
|
||||
for chunk in window:
|
||||
num_speakers.append(len(file["annotation"].crop(chunk).labels()))
|
||||
|
||||
# because there might a few outliers, estimate the upper bound for the
|
||||
# number of speakers as the 99th percentile
|
||||
|
||||
num_speakers, counts = zip(*list(Counter(num_speakers).items()))
|
||||
num_speakers, counts = np.array(num_speakers), np.array(counts)
|
||||
|
||||
sorting_indices = np.argsort(num_speakers)
|
||||
num_speakers = num_speakers[sorting_indices]
|
||||
counts = counts[sorting_indices]
|
||||
|
||||
self.num_speakers = num_speakers[
|
||||
np.where(np.cumsum(counts) / np.sum(counts) > 0.99)[0][0]
|
||||
]
|
||||
|
||||
# TODO: add a few more speakers to make sure we don't skip
|
||||
# too many artificial chunks (which might result in less
|
||||
# overlap that we think we have)
|
||||
|
||||
# now that we know about the number of speakers upper bound
|
||||
# we can set task specifications
|
||||
self.specifications = Specifications(
|
||||
problem=Problem.MULTI_LABEL_CLASSIFICATION,
|
||||
resolution=Resolution.FRAME,
|
||||
# slide a window (with 1s step) over the whole training set
|
||||
# and keep track of the number of speakers in each location
|
||||
num_speakers = []
|
||||
for file in self._train:
|
||||
start = file["annotated"][0].start
|
||||
end = file["annotated"][-1].end
|
||||
window = SlidingWindow(
|
||||
start=start,
|
||||
end=end,
|
||||
duration=self.duration,
|
||||
warm_up=self.warm_up,
|
||||
classes=[f"speaker#{i+1}" for i in range(self.num_speakers)],
|
||||
permutation_invariant=True,
|
||||
step=1.0,
|
||||
)
|
||||
for chunk in window:
|
||||
num_speakers.append(len(file["annotation"].crop(chunk).labels()))
|
||||
|
||||
# because there might a few outliers, estimate the upper bound for the
|
||||
# number of speakers as the 99th percentile
|
||||
|
||||
num_speakers, counts = zip(*list(Counter(num_speakers).items()))
|
||||
num_speakers, counts = np.array(num_speakers), np.array(counts)
|
||||
|
||||
sorting_indices = np.argsort(num_speakers)
|
||||
num_speakers = num_speakers[sorting_indices]
|
||||
counts = counts[sorting_indices]
|
||||
|
||||
self.num_speakers = num_speakers[
|
||||
np.where(np.cumsum(counts) / np.sum(counts) > 0.99)[0][0]
|
||||
]
|
||||
|
||||
# TODO: add a few more speakers to make sure we don't skip
|
||||
# too many artificial chunks (which might result in less
|
||||
# overlap that we think we have)
|
||||
|
||||
# now that we know about the number of speakers upper bound
|
||||
# we can set task specifications
|
||||
self.specifications = Specifications(
|
||||
problem=Problem.MULTI_LABEL_CLASSIFICATION,
|
||||
resolution=Resolution.FRAME,
|
||||
duration=self.duration,
|
||||
warm_up=self.warm_up,
|
||||
classes=[f"speaker#{i+1}" for i in range(self.num_speakers)],
|
||||
permutation_invariant=True,
|
||||
)
|
||||
|
||||
def prepare_y(self, one_hot_y: np.ndarray):
|
||||
"""Zero-pad segmentation targets
|
||||
@@ -372,127 +368,6 @@ class Segmentation(SegmentationTaskMixin, Task):
|
||||
)
|
||||
return {"loss": loss}
|
||||
|
||||
def validation_step(self, batch, batch_idx: int):
|
||||
"""Compute validation F-score
|
||||
|
||||
Parameters
|
||||
----------
|
||||
batch : dict of torch.Tensor
|
||||
Current batch.
|
||||
batch_idx: int
|
||||
Batch index.
|
||||
"""
|
||||
|
||||
# move metric to model device
|
||||
self.val_fbeta.to(self.model.device)
|
||||
|
||||
X, y = batch["X"], batch["y"]
|
||||
# X = (batch_size, num_channels, num_samples)
|
||||
# y = (batch_size, num_frames, num_classes)
|
||||
|
||||
y_pred = self.model(X)
|
||||
_, num_frames, _ = y_pred.shape
|
||||
# y_pred = (batch_size, num_frames, num_classes)
|
||||
|
||||
def validation_postprocess(self, y, y_pred):
|
||||
permutated_y_pred, _ = permutate(y, y_pred)
|
||||
|
||||
warm_up_left = round(self.warm_up[0] / self.duration * num_frames)
|
||||
warm_up_right = round(self.warm_up[1] / self.duration * num_frames)
|
||||
|
||||
val_fbeta = self.val_fbeta(
|
||||
permutated_y_pred[
|
||||
:, warm_up_left : num_frames - warm_up_right : 10
|
||||
].squeeze(),
|
||||
y[:, warm_up_left : num_frames - warm_up_right : 10].squeeze(),
|
||||
)
|
||||
self.model.log(
|
||||
f"{self.ACRONYM}@val_fbeta",
|
||||
val_fbeta,
|
||||
on_step=False,
|
||||
on_epoch=True,
|
||||
prog_bar=True,
|
||||
logger=True,
|
||||
)
|
||||
|
||||
# log first batch visualization every 2^n epochs.
|
||||
if (
|
||||
self.model.current_epoch == 0
|
||||
or math.log2(self.model.current_epoch) % 1 > 0
|
||||
or batch_idx > 0
|
||||
):
|
||||
return
|
||||
|
||||
# visualize first 9 validation samples of first batch in Tensorboard
|
||||
X = X.cpu().numpy()
|
||||
y = y.float().cpu().numpy()
|
||||
y_pred = y_pred.cpu().numpy()
|
||||
permutated_y_pred = permutated_y_pred.cpu().numpy()
|
||||
|
||||
# prepare 3 x 3 grid (or smaller if batch size is smaller)
|
||||
num_samples = min(self.batch_size, 9)
|
||||
nrows = math.ceil(math.sqrt(num_samples))
|
||||
ncols = math.ceil(num_samples / nrows)
|
||||
fig, axes = plt.subplots(
|
||||
nrows=4 * nrows,
|
||||
ncols=ncols,
|
||||
figsize=(15, 10),
|
||||
)
|
||||
|
||||
# reshape target so that there is one line per class when plottingit
|
||||
y[y == 0] = np.NaN
|
||||
y *= np.arange(y.shape[2])
|
||||
|
||||
# plot each sample
|
||||
for sample_idx in range(num_samples):
|
||||
|
||||
# find where in the grid it should be plotted
|
||||
row_idx = sample_idx // nrows
|
||||
col_idx = sample_idx % ncols
|
||||
|
||||
# plot waveform
|
||||
ax_wav = axes[row_idx * 4 + 0, col_idx]
|
||||
sample_X = np.mean(X[sample_idx], axis=0)
|
||||
ax_wav.plot(sample_X)
|
||||
ax_wav.set_xlim(0, len(sample_X))
|
||||
ax_wav.get_xaxis().set_visible(False)
|
||||
ax_wav.get_yaxis().set_visible(False)
|
||||
|
||||
# plot target
|
||||
ax_ref = axes[row_idx * 4 + 1, col_idx]
|
||||
sample_y = y[sample_idx]
|
||||
ax_ref.plot(sample_y)
|
||||
ax_ref.set_xlim(0, len(sample_y))
|
||||
ax_ref.set_ylim(-1, sample_y.shape[1])
|
||||
ax_ref.get_xaxis().set_visible(False)
|
||||
ax_ref.get_yaxis().set_visible(False)
|
||||
|
||||
# plot prediction
|
||||
ax_hyp = axes[row_idx * 4 + 2, col_idx]
|
||||
sample_y_pred = y_pred[sample_idx]
|
||||
ax_hyp.axvspan(0, warm_up_left, color="k", alpha=0.5, lw=0)
|
||||
ax_hyp.axvspan(
|
||||
num_frames - warm_up_right, num_frames, color="k", alpha=0.5, lw=0
|
||||
)
|
||||
ax_hyp.plot(sample_y_pred)
|
||||
ax_hyp.set_ylim(-0.1, 1.1)
|
||||
ax_hyp.set_xlim(0, len(sample_y))
|
||||
ax_hyp.get_xaxis().set_visible(False)
|
||||
|
||||
# plot permutated prediction
|
||||
ax_map = axes[row_idx * 4 + 3, col_idx]
|
||||
sample_y_pred_map = permutated_y_pred[sample_idx]
|
||||
ax_map.axvspan(0, warm_up_left, color="k", alpha=0.5, lw=0)
|
||||
ax_map.axvspan(
|
||||
num_frames - warm_up_right, num_frames, color="k", alpha=0.5, lw=0
|
||||
)
|
||||
ax_map.plot(sample_y_pred_map)
|
||||
ax_map.set_ylim(-0.1, 1.1)
|
||||
ax_map.set_xlim(0, len(sample_y))
|
||||
|
||||
plt.tight_layout()
|
||||
|
||||
self.model.logger.experiment.add_figure(
|
||||
f"{self.ACRONYM}@val_samples", fig, self.model.current_epoch
|
||||
)
|
||||
|
||||
plt.close(fig)
|
||||
return permutated_y_pred
|
||||
@@ -20,7 +20,7 @@
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
from typing import List, Text, Tuple, Union
|
||||
from typing import List, Optional, Text, Tuple, Union
|
||||
|
||||
import numpy as np
|
||||
from torch_audiomentations.core.transforms_interface import BaseWaveformTransform
|
||||
@@ -105,21 +105,19 @@ class SpeakerTracking(SegmentationTaskMixin, Task):
|
||||
# speakers should be tracked. therefore, we postpone
|
||||
# the definition of specifications.
|
||||
|
||||
def setup(self, stage=None):
|
||||
def setup(self, stage: Optional[str] = None):
|
||||
|
||||
super().setup(stage=stage)
|
||||
|
||||
if stage == "fit":
|
||||
|
||||
self.specifications = Specifications(
|
||||
# one class per speaker
|
||||
classes=sorted(self._train_metadata["annotation"]),
|
||||
# multiple speakers can be active at once
|
||||
problem=Problem.MULTI_LABEL_CLASSIFICATION,
|
||||
resolution=Resolution.FRAME,
|
||||
duration=self.duration,
|
||||
warm_up=self.warm_up,
|
||||
)
|
||||
self.specifications = Specifications(
|
||||
# one class per speaker
|
||||
classes=sorted(self._train_metadata["annotation"]),
|
||||
# multiple speakers can be active at once
|
||||
problem=Problem.MULTI_LABEL_CLASSIFICATION,
|
||||
resolution=Resolution.FRAME,
|
||||
duration=self.duration,
|
||||
warm_up=self.warm_up,
|
||||
)
|
||||
|
||||
@property
|
||||
def chunk_labels(self) -> List[Text]:
|
||||
|
||||
+2
-1
@@ -7,11 +7,12 @@ pyannote.core >=4.1,<5.0
|
||||
pyannote.database >=4.1,<5.0
|
||||
pyannote.metrics >=3.0.1,<4.0
|
||||
pyannote.pipeline >=2.0,<3.0
|
||||
pytorch_lightning >= 1.2,<1.3
|
||||
pytorch_lightning >=1.4.1,<1.5
|
||||
pytorch_metric_learning >=0.9.98
|
||||
semver >=2.10.2,<3.0
|
||||
soundfile >=0.10.2,<0.11
|
||||
torch >=1.8.1,<1.9
|
||||
torch-audiomentations >=0.6.0
|
||||
torchaudio >=0.8.1,<0.9
|
||||
torchmetrics >=0.4.1,<0.5
|
||||
typing_extensions >=3.7.4.3
|
||||
@@ -1,38 +0,0 @@
|
||||
import pytest
|
||||
import torch
|
||||
from torch.nn import Module
|
||||
|
||||
from pyannote.audio.augmentation.registry import (
|
||||
register_augmentation,
|
||||
unregister_augmentation,
|
||||
)
|
||||
|
||||
|
||||
class RandomAugmentation(Module):
|
||||
"Fake Noise tranform"
|
||||
|
||||
def forward(self, waveforms):
|
||||
if not self.training:
|
||||
return waveforms
|
||||
return torch.ones_like(waveforms)
|
||||
|
||||
|
||||
def test_can_register_augmentation():
|
||||
for when in ["output", "input"]:
|
||||
net = Module()
|
||||
aug = RandomAugmentation()
|
||||
register_augmentation(aug, net, when=when)
|
||||
assert hasattr(net, "__augmentation")
|
||||
assert net.__augmentation[when] == aug
|
||||
|
||||
|
||||
def test_can_unregistrer_augmentation():
|
||||
net = Module()
|
||||
register_augmentation(RandomAugmentation(), net, when="output")
|
||||
unregister_augmentation(net, when="output")
|
||||
assert not hasattr(net.__augmentation, "output")
|
||||
|
||||
|
||||
def test_fail_unregister_augmentation():
|
||||
with pytest.raises(ValueError):
|
||||
unregister_augmentation(RandomAugmentation(), when="output")
|
||||
@@ -16,7 +16,6 @@ def setup_tasks(task):
|
||||
|
||||
def create_dl(model, task):
|
||||
m = model(task=task)
|
||||
task.setup("fit")
|
||||
m.setup("fit")
|
||||
return task.train_dataloader()
|
||||
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
import pytest
|
||||
from pytorch_lightning import Trainer
|
||||
|
||||
from pyannote.audio.models.segmentation.debug import SimpleSegmentationModel
|
||||
from pyannote.audio.tasks import (
|
||||
OverlappedSpeechDetection,
|
||||
Segmentation,
|
||||
VoiceActivityDetection,
|
||||
)
|
||||
from pyannote.database import FileFinder, get_protocol
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def protocol():
|
||||
return get_protocol(
|
||||
"Debug.SpeakerDiarization.Debug", preprocessors={"audio": FileFinder()}
|
||||
)
|
||||
|
||||
|
||||
def test_train_segmentation(protocol):
|
||||
segmentation = Segmentation(protocol)
|
||||
model = SimpleSegmentationModel(task=segmentation)
|
||||
trainer = Trainer(fast_dev_run=True)
|
||||
trainer.fit(model)
|
||||
|
||||
|
||||
def test_train_voice_activity_detection(protocol):
|
||||
voice_activity_detection = VoiceActivityDetection(protocol)
|
||||
model = SimpleSegmentationModel(task=voice_activity_detection)
|
||||
trainer = Trainer(fast_dev_run=True)
|
||||
trainer.fit(model)
|
||||
|
||||
|
||||
def test_train_overlapped_speech_detection(protocol):
|
||||
overlapped_speech_detection = OverlappedSpeechDetection(protocol)
|
||||
model = SimpleSegmentationModel(task=overlapped_speech_detection)
|
||||
trainer = Trainer(fast_dev_run=True)
|
||||
trainer.fit(model)
|
||||
|
||||
|
||||
def test_finetune_with_task_that_does_not_need_setup_for_specs(protocol):
|
||||
voice_activity_detection = VoiceActivityDetection(protocol)
|
||||
model = SimpleSegmentationModel(task=voice_activity_detection)
|
||||
trainer = Trainer(fast_dev_run=True)
|
||||
trainer.fit(model)
|
||||
|
||||
voice_activity_detection = VoiceActivityDetection(protocol)
|
||||
model.task = voice_activity_detection
|
||||
trainer = Trainer(fast_dev_run=True)
|
||||
trainer.fit(model)
|
||||
|
||||
|
||||
def test_finetune_with_task_that_needs_setup_for_specs(protocol):
|
||||
segmentation = Segmentation(protocol)
|
||||
model = SimpleSegmentationModel(task=segmentation)
|
||||
trainer = Trainer(fast_dev_run=True)
|
||||
trainer.fit(model)
|
||||
|
||||
segmentation = Segmentation(protocol)
|
||||
model.task = segmentation
|
||||
trainer = Trainer(fast_dev_run=True)
|
||||
trainer.fit(model)
|
||||
|
||||
|
||||
def test_transfer_with_task_that_does_not_need_setup_for_specs(protocol):
|
||||
|
||||
segmentation = Segmentation(protocol)
|
||||
model = SimpleSegmentationModel(task=segmentation)
|
||||
trainer = Trainer(fast_dev_run=True)
|
||||
trainer.fit(model)
|
||||
|
||||
voice_activity_detection = VoiceActivityDetection(protocol)
|
||||
model.task = voice_activity_detection
|
||||
trainer = Trainer(fast_dev_run=True)
|
||||
trainer.fit(model)
|
||||
|
||||
|
||||
def test_transfer_with_task_that_needs_setup_for_specs(protocol):
|
||||
|
||||
voice_activity_detection = VoiceActivityDetection(protocol)
|
||||
model = SimpleSegmentationModel(task=voice_activity_detection)
|
||||
trainer = Trainer(fast_dev_run=True)
|
||||
trainer.fit(model)
|
||||
|
||||
segmentation = Segmentation(protocol)
|
||||
model.task = segmentation
|
||||
trainer = Trainer(fast_dev_run=True)
|
||||
trainer.fit(model)
|
||||
|
||||
|
||||
def test_finetune_freeze_with_task_that_needs_setup_for_specs(protocol):
|
||||
|
||||
segmentation = Segmentation(protocol)
|
||||
model = SimpleSegmentationModel(task=segmentation)
|
||||
trainer = Trainer(fast_dev_run=True)
|
||||
trainer.fit(model)
|
||||
|
||||
segmentation = Segmentation(protocol)
|
||||
model.task = segmentation
|
||||
model.freeze_up_to("mfcc")
|
||||
trainer = Trainer(fast_dev_run=True)
|
||||
trainer.fit(model)
|
||||
|
||||
|
||||
def test_finetune_freeze_with_task_that_does_not_need_setup_for_specs(protocol):
|
||||
|
||||
vad = VoiceActivityDetection(protocol)
|
||||
model = SimpleSegmentationModel(task=vad)
|
||||
trainer = Trainer(fast_dev_run=True)
|
||||
trainer.fit(model)
|
||||
|
||||
vad = VoiceActivityDetection(protocol)
|
||||
model.task = vad
|
||||
model.freeze_up_to("mfcc")
|
||||
trainer = Trainer(fast_dev_run=True)
|
||||
trainer.fit(model)
|
||||
|
||||
|
||||
def test_transfer_freeze_with_task_that_does_not_need_setup_for_specs(protocol):
|
||||
|
||||
segmentation = Segmentation(protocol)
|
||||
model = SimpleSegmentationModel(task=segmentation)
|
||||
trainer = Trainer(fast_dev_run=True)
|
||||
trainer.fit(model)
|
||||
|
||||
voice_activity_detection = VoiceActivityDetection(protocol)
|
||||
model.task = voice_activity_detection
|
||||
model.freeze_up_to("mfcc")
|
||||
trainer = Trainer(fast_dev_run=True)
|
||||
trainer.fit(model)
|
||||
|
||||
|
||||
def test_transfer_freeze_with_task_that_needs_setup_for_specs(protocol):
|
||||
|
||||
voice_activity_detection = VoiceActivityDetection(protocol)
|
||||
model = SimpleSegmentationModel(task=voice_activity_detection)
|
||||
trainer = Trainer(fast_dev_run=True)
|
||||
trainer.fit(model)
|
||||
|
||||
segmentation = Segmentation(protocol)
|
||||
model.task = segmentation
|
||||
model.freeze_up_to("mfcc")
|
||||
trainer = Trainer(fast_dev_run=True)
|
||||
trainer.fit(model)
|
||||
Reference in new issue
Block a user