mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
chore: remove multi-task support (#675)
This commit is contained in:
1 parent
062bd7bd9a
commit
c69e92df77
12 files changed
+206
-1154
No files matched your search
@@ -18,7 +18,6 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from pyannote.audio.models.segmentation.debug import SimpleSegmentationModel\n",
|
||||
"from pyannote.audio.models.segmentation.debug import MultiTaskSegmentationModel\n",
|
||||
"from pyannote.audio.models.embedding.debug import SimpleEmbeddingModel\n",
|
||||
"import pytorch_lightning as pl"
|
||||
]
|
||||
@@ -113,26 +112,6 @@
|
||||
"_ = trainer.fit(model)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Multi-task segmentation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from pyannote.audio.tasks.segmentation.multi_task_segmentation import MultiTaskSegmentation\n",
|
||||
"seg = MultiTaskSegmentation(protocol, vad=True, scd=True, osd=True, duration=2., batch_size=32, num_workers=4)\n",
|
||||
"model = MultiTaskSegmentationModel(task=seg)\n",
|
||||
"trainer = pl.Trainer(max_epochs=1)\n",
|
||||
"_ = trainer.fit(model)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
|
||||
@@ -234,135 +234,6 @@
|
||||
"\n",
|
||||
"embeddings.shape"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Train and apply a multi-task segmentation model"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from pyannote.audio.tasks.segmentation.multi_task_segmentation import MultiTaskSegmentation\n",
|
||||
"xseg = MultiTaskSegmentation(protocol, duration=2., vad=True, scd=True, osd=True, batch_size=32, num_workers=4)\n",
|
||||
"from pyannote.audio.models.segmentation.debug import MultiTaskSegmentationModel\n",
|
||||
"model = MultiTaskSegmentationModel(task=xseg)\n",
|
||||
"trainer = pl.Trainer(max_epochs=1, default_root_dir='inference/xseg')\n",
|
||||
"_ = trainer.fit(model)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# inference using a sliding window\n",
|
||||
"inference = Inference(model, duration=2., step=0.5)\n",
|
||||
"dev_file = next(protocol.development())\n",
|
||||
"scores = inference(dev_file)\n",
|
||||
"scores"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"scores['vad']"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"scores['scd']"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"scores['osd']"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# inference on an excerpt \n",
|
||||
"scores = inference.crop(dev_file, Segment(10, 15))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"scores['vad']"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"scores['scd']"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"scores['osd']"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# inference on an excerpt shorter than sliding window duration\n",
|
||||
"scores = inference.crop(dev_file, Segment(10, 11.5))\n",
|
||||
"scores['vad']"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# inference on a whole chunk\n",
|
||||
"inference = Inference(model, window=\"whole\")\n",
|
||||
"scores = inference.crop(dev_file, Segment(10, 15))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"scores['vad'].shape"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
@@ -28,7 +28,6 @@ help:
|
||||
* vad (default) = voice activity detection
|
||||
* scd = speaker change detection
|
||||
* osd = overlapped speech detection
|
||||
* xseg = multi-task segmentation
|
||||
|
||||
{model} can be any of the following:
|
||||
* debug (default) = simple segmentation model for debugging purposes
|
||||
|
||||
+191
-290
@@ -24,7 +24,7 @@ import math
|
||||
import warnings
|
||||
from collections import Counter, deque
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Deque, Dict, List, Optional, Text, Union
|
||||
from typing import Any, Callable, Deque, List, Optional, Text, Union
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -105,12 +105,12 @@ class Inference:
|
||||
if window not in ["sliding", "whole"]:
|
||||
raise ValueError('`window` must be "sliding" or "whole".')
|
||||
|
||||
for task_name, specifications in self.model.specifications.items():
|
||||
if specifications.resolution == Resolution.FRAME and window == "whole":
|
||||
warnings.warn(
|
||||
'Using "whole" `window` inference with a frame-based model might lead to bad results '
|
||||
'and huge memory consumption: it is recommended to set `window` to "sliding".'
|
||||
)
|
||||
specifications = self.model.specifications
|
||||
if specifications.resolution == Resolution.FRAME and window == "whole":
|
||||
warnings.warn(
|
||||
'Using "whole" `window` inference with a frame-based model might lead to bad results '
|
||||
'and huge memory consumption: it is recommended to set `window` to "sliding".'
|
||||
)
|
||||
|
||||
self.window = window
|
||||
self.skip_aggregation = skip_aggregation
|
||||
@@ -122,12 +122,8 @@ class Inference:
|
||||
self.model.eval()
|
||||
self.model.to(self.device)
|
||||
|
||||
# chunk duration used during training. for multi-task,
|
||||
# we assume that the same duration was used for each task.
|
||||
if self.model.is_multi_task:
|
||||
_, specifications = next(iter(self.model.specifications.items()))
|
||||
else:
|
||||
specifications = self.model.specifications
|
||||
# chunk duration used during training
|
||||
specifications = self.model.specifications
|
||||
training_duration = specifications.duration
|
||||
|
||||
if duration is None:
|
||||
@@ -169,7 +165,7 @@ class Inference:
|
||||
progress_hook = None
|
||||
self.progress_hook = progress_hook
|
||||
|
||||
def infer(self, chunks: torch.Tensor) -> Dict[TaskName, np.ndarray]:
|
||||
def infer(self, chunks: torch.Tensor) -> np.ndarray:
|
||||
"""Forward pass
|
||||
|
||||
Parameters
|
||||
@@ -179,12 +175,8 @@ class Inference:
|
||||
|
||||
Returns
|
||||
-------
|
||||
outputs : {task_name: np.ndarray} dict
|
||||
Model outputs.
|
||||
|
||||
Notes
|
||||
-----
|
||||
If model is mono-task, `task_name` is set to None.
|
||||
outputs : np.ndarray
|
||||
Model output.
|
||||
"""
|
||||
|
||||
with torch.no_grad():
|
||||
@@ -199,16 +191,9 @@ class Inference:
|
||||
else:
|
||||
raise exception
|
||||
|
||||
if self.model.is_multi_task:
|
||||
return {
|
||||
task_name: output.cpu().numpy() for task_name, output in outputs.items()
|
||||
}
|
||||
return outputs.cpu().numpy()
|
||||
|
||||
return {None: outputs.cpu().numpy()}
|
||||
|
||||
def slide(
|
||||
self, waveform: torch.Tensor, sample_rate: int
|
||||
) -> Union[SlidingWindowFeature, Dict[Text, SlidingWindowFeature]]:
|
||||
def slide(self, waveform: torch.Tensor, sample_rate: int) -> SlidingWindowFeature:
|
||||
"""Slide model on a waveform
|
||||
|
||||
Parameters
|
||||
@@ -223,11 +208,6 @@ class Inference:
|
||||
output : SlidingWindowFeature
|
||||
Model output. Shape is (num_chunks, dimension) for chunk-level tasks,
|
||||
and (num_frames, dimension) for frame-level tasks.
|
||||
|
||||
Notes
|
||||
-----
|
||||
If model has several outputs (multi-task), those will be returned as a
|
||||
{task_name: output} dictionary.
|
||||
"""
|
||||
|
||||
# prepare sliding audio chunks
|
||||
@@ -235,8 +215,6 @@ class Inference:
|
||||
file_duration = num_samples / sample_rate
|
||||
window_size: int = round(self.duration * sample_rate)
|
||||
|
||||
results: Dict[Text, SlidingWindowFeature] = dict()
|
||||
|
||||
# corner case: waveform is shorter than chunk duration
|
||||
if num_samples < window_size:
|
||||
|
||||
@@ -247,30 +225,22 @@ class Inference:
|
||||
|
||||
one_output = self.infer(waveform[None, :])
|
||||
|
||||
for task_name, specifications in self.model.specifications.items():
|
||||
if specifications.resolution == Resolution.CHUNK:
|
||||
frames = SlidingWindow(
|
||||
start=0.0, duration=self.duration, step=self.step
|
||||
)
|
||||
results[task_name] = SlidingWindowFeature(
|
||||
one_output[task_name], frames
|
||||
)
|
||||
if self.model.specifications.resolution == Resolution.CHUNK:
|
||||
frames = SlidingWindow(
|
||||
start=0.0, duration=self.duration, step=self.step
|
||||
)
|
||||
results = SlidingWindowFeature(one_output, frames)
|
||||
|
||||
else:
|
||||
_, num_frames, dimension = one_output[task_name].shape
|
||||
frames = SlidingWindow(
|
||||
start=0,
|
||||
duration=file_duration / num_frames,
|
||||
step=file_duration / num_frames,
|
||||
)
|
||||
results[task_name] = SlidingWindowFeature(
|
||||
one_output[task_name][0], frames
|
||||
)
|
||||
|
||||
if self.model.is_multi_task:
|
||||
return results
|
||||
else:
|
||||
return results.popitem()[1]
|
||||
_, num_frames, dimension = one_output.shape
|
||||
frames = SlidingWindow(
|
||||
start=0,
|
||||
duration=file_duration / num_frames,
|
||||
step=file_duration / num_frames,
|
||||
)
|
||||
results = SlidingWindowFeature(one_output[0], frames)
|
||||
|
||||
return results
|
||||
|
||||
# prepare (and count) sliding audio chunks
|
||||
step_size: int = round(self.step * sample_rate)
|
||||
@@ -289,205 +259,182 @@ class Inference:
|
||||
else:
|
||||
has_last_chunk = False
|
||||
|
||||
outputs: Dict[TaskName, Union[List[np.ndarray], np.ndarray]] = {
|
||||
task_name: list() for task_name, _ in self.model.specifications.items()
|
||||
}
|
||||
outputs: Union[List[np.ndarray], np.ndarray] = list()
|
||||
|
||||
if self.progress_hook is not None:
|
||||
self.progress_hook(0, num_chunks + has_last_chunk)
|
||||
|
||||
# slide over audio chunks in batch
|
||||
for c in np.arange(0, num_chunks, self.batch_size):
|
||||
|
||||
batch: torch.Tensor = chunks[c : c + self.batch_size]
|
||||
|
||||
output = self.infer(batch)
|
||||
for task_name, task_output in output.items():
|
||||
outputs[task_name].append(task_output)
|
||||
|
||||
outputs.append(self.infer(batch))
|
||||
if self.progress_hook is not None:
|
||||
self.progress_hook(c + 1, num_chunks + has_last_chunk)
|
||||
|
||||
outputs = {
|
||||
task_name: np.vstack(task_outputs)
|
||||
for task_name, task_outputs in outputs.items()
|
||||
}
|
||||
outputs = np.vstack(outputs)
|
||||
|
||||
for task_name, specifications in self.model.specifications.items():
|
||||
# skip aggregation when requested
|
||||
# or when model outputs just one vector per chunk
|
||||
if self.skip_aggregation or specifications.resolution == Resolution.CHUNK:
|
||||
frames = SlidingWindow(
|
||||
start=0.0, duration=self.duration, step=self.step
|
||||
)
|
||||
results[task_name] = SlidingWindowFeature(outputs[task_name], frames)
|
||||
continue
|
||||
specifications = self.model.specifications
|
||||
|
||||
# process orphan last chunk
|
||||
if has_last_chunk:
|
||||
last_output = {
|
||||
task_name: output[0]
|
||||
for task_name, output in self.infer(last_chunk[None]).items()
|
||||
}
|
||||
if self.progress_hook is not None:
|
||||
self.progress_hook(
|
||||
num_chunks + has_last_chunk, num_chunks + has_last_chunk
|
||||
)
|
||||
# skip aggregation when requested
|
||||
# or when model outputs just one vector per chunk
|
||||
if self.skip_aggregation or specifications.resolution == Resolution.CHUNK:
|
||||
frames = SlidingWindow(start=0.0, duration=self.duration, step=self.step)
|
||||
return SlidingWindowFeature(outputs, frames)
|
||||
|
||||
# use model introspection to estimate the total number of frames
|
||||
introspection = self.model.introspection[task_name]
|
||||
num_frames, dimension = introspection(num_samples)
|
||||
num_frames_per_chunk, _ = introspection(window_size)
|
||||
num_frames_per_step, _ = introspection(step_size)
|
||||
if has_last_chunk:
|
||||
num_frames_last_step, _ = introspection(last_step_size)
|
||||
|
||||
# warm-up window used for overlap-add aggregation
|
||||
warm_up = np.ones((num_frames_per_chunk, 1))
|
||||
# ... for very first chunk
|
||||
warm_up_first = np.ones((num_frames_per_chunk, 1))
|
||||
# ... forvery last chunk
|
||||
warm_up_last = np.ones((num_frames_per_chunk, 1))
|
||||
|
||||
if not specifications.permutation_invariant:
|
||||
|
||||
# anything before warm_up_left (and after num_frames_per_chunk - warm_up_right)
|
||||
# will not be used in the final aggregation
|
||||
warm_up_left = round(
|
||||
self.warm_up[0] / self.duration * num_frames_per_chunk
|
||||
)
|
||||
warm_up_right = round(
|
||||
self.warm_up[1] / self.duration * num_frames_per_chunk
|
||||
# process orphan last chunk
|
||||
if has_last_chunk:
|
||||
last_output = self.infer(last_chunk[None])[0]
|
||||
if self.progress_hook is not None:
|
||||
self.progress_hook(
|
||||
num_chunks + has_last_chunk, num_chunks + has_last_chunk
|
||||
)
|
||||
|
||||
# Hamming window used for overlap-add aggregation
|
||||
window = np.hamming(num_frames_per_chunk).reshape(-1, 1)
|
||||
# use model introspection to estimate the total number of frames
|
||||
introspection = self.model.introspection
|
||||
num_frames, dimension = introspection(num_samples)
|
||||
num_frames_per_chunk, _ = introspection(window_size)
|
||||
num_frames_per_step, _ = introspection(step_size)
|
||||
if has_last_chunk:
|
||||
num_frames_last_step, _ = introspection(last_step_size)
|
||||
|
||||
else:
|
||||
# Regular overlap-add aggregation cannot be used directly for
|
||||
# permutation-invariant models. Why? Because two consecutive
|
||||
# chunks may disagree on part of their output (even after optimal
|
||||
# permutation).
|
||||
# warm-up window used for overlap-add aggregation
|
||||
warm_up = np.ones((num_frames_per_chunk, 1))
|
||||
# ... for very first chunk
|
||||
warm_up_first = np.ones((num_frames_per_chunk, 1))
|
||||
# ... forvery last chunk
|
||||
warm_up_last = np.ones((num_frames_per_chunk, 1))
|
||||
|
||||
# These two lines make inference use only the (step-long) central
|
||||
# part of each chunk in the final aggregation... essentially
|
||||
# switching to a simple concatenation.
|
||||
if not specifications.permutation_invariant:
|
||||
|
||||
warm_up_left = math.floor(
|
||||
0.5
|
||||
* (self.duration + self.warm_up[0] - self.warm_up[1] - self.step)
|
||||
/ self.duration
|
||||
* num_frames_per_chunk
|
||||
)
|
||||
warm_up_right = num_frames_per_chunk - math.ceil(
|
||||
0.5
|
||||
* (self.duration + self.warm_up[0] - self.warm_up[1] + self.step)
|
||||
/ self.duration
|
||||
* num_frames_per_chunk
|
||||
)
|
||||
|
||||
# we do not need a window here because there is no overlap
|
||||
# between concatenated chunk central parts.
|
||||
window = 1.0
|
||||
|
||||
warm_up[:warm_up_left] = 0.0
|
||||
warm_up_last[:warm_up_left] = 0.0
|
||||
warm_up[num_frames_per_chunk - warm_up_right :] = 0.0
|
||||
warm_up_first[num_frames_per_chunk - warm_up_right :] = 0.0
|
||||
|
||||
# aggregated_output[i] will be used to store the sum of all predictions
|
||||
# for frame #i
|
||||
aggregated_output: np.ndarray = np.zeros(
|
||||
(num_frames, dimension), dtype=np.float32
|
||||
# anything before warm_up_left (and after num_frames_per_chunk - warm_up_right)
|
||||
# will not be used in the final aggregation
|
||||
warm_up_left = round(self.warm_up[0] / self.duration * num_frames_per_chunk)
|
||||
warm_up_right = round(
|
||||
self.warm_up[1] / self.duration * num_frames_per_chunk
|
||||
)
|
||||
|
||||
# overlapping_chunk_count[i] will be used to store the number of chunks
|
||||
# that contributed to frame #i
|
||||
overlapping_chunk_count: np.ndarray = np.zeros(
|
||||
(num_frames, 1), dtype=np.float32
|
||||
)
|
||||
# Hamming window used for overlap-add aggregation
|
||||
window = np.hamming(num_frames_per_chunk).reshape(-1, 1)
|
||||
|
||||
if specifications.permutation_invariant:
|
||||
# number of previous outputs that overlap with current one by at least 50% of their warmed up region
|
||||
num_overlap = math.floor(
|
||||
0.5
|
||||
* (self.duration - self.warm_up[0] - self.warm_up[1])
|
||||
/ self.step
|
||||
)
|
||||
# keep track of those previous outputs in a "deque"
|
||||
if num_overlap > 0:
|
||||
previous_outputs: Deque[np.ndarray] = deque([], maxlen=num_overlap)
|
||||
|
||||
# loop on the outputs of sliding chunks
|
||||
for c, output in enumerate(outputs[task_name]):
|
||||
start_sample = c * step_size
|
||||
start_frame, _ = introspection(start_sample)
|
||||
|
||||
if specifications.permutation_invariant and num_overlap > 0:
|
||||
if c > 0:
|
||||
output = self.permutate(
|
||||
np.stack(previous_outputs), output, num_frames_per_step
|
||||
)
|
||||
previous_outputs.append(output)
|
||||
|
||||
# when processing first chunk, do not weigh-down its left-most side
|
||||
if c == 0:
|
||||
# unless there is just one chunk, where we do not weigh-down any side
|
||||
if not has_last_chunk and num_chunks == 1:
|
||||
warm_up_ = 1.0
|
||||
else:
|
||||
warm_up_ = warm_up_first
|
||||
|
||||
# when processing last chunk, make sure to not weigh-down its right-most side
|
||||
elif not has_last_chunk and c + 1 == num_chunks:
|
||||
warm_up_ = warm_up_last
|
||||
|
||||
# when processing an internal chunk, weigh-down both sides
|
||||
else:
|
||||
warm_up_ = warm_up
|
||||
|
||||
aggregated_output[start_frame : start_frame + num_frames_per_chunk] += (
|
||||
output * window * warm_up_
|
||||
)
|
||||
|
||||
overlapping_chunk_count[
|
||||
start_frame : start_frame + num_frames_per_chunk
|
||||
] += (window * warm_up_)
|
||||
|
||||
# process last (right-aligned) chunk separately
|
||||
if has_last_chunk:
|
||||
|
||||
if (
|
||||
specifications.permutation_invariant
|
||||
and num_overlap > 0
|
||||
and previous_outputs
|
||||
):
|
||||
# FIXME
|
||||
last_output[task_name] = self.permutate(
|
||||
previous_outputs[-1][np.newaxis],
|
||||
last_output[task_name],
|
||||
num_frames_last_step,
|
||||
)
|
||||
|
||||
aggregated_output[-num_frames_per_chunk:] += (
|
||||
last_output[task_name] * window * warm_up_last
|
||||
)
|
||||
overlapping_chunk_count[-num_frames_per_chunk:] += window * warm_up_last
|
||||
|
||||
aggregated_output /= np.maximum(overlapping_chunk_count, 1e-12)
|
||||
|
||||
frames = SlidingWindow(
|
||||
start=0,
|
||||
duration=file_duration / num_frames,
|
||||
step=file_duration / num_frames,
|
||||
)
|
||||
|
||||
results[task_name] = SlidingWindowFeature(aggregated_output, frames)
|
||||
|
||||
if self.model.is_multi_task:
|
||||
return results
|
||||
else:
|
||||
return results.popitem()[1]
|
||||
# Regular overlap-add aggregation cannot be used directly for
|
||||
# permutation-invariant models. Why? Because two consecutive
|
||||
# chunks may disagree on part of their output (even after optimal
|
||||
# permutation).
|
||||
|
||||
# These two lines make inference use only the (step-long) central
|
||||
# part of each chunk in the final aggregation... essentially
|
||||
# switching to a simple concatenation.
|
||||
|
||||
warm_up_left = math.floor(
|
||||
0.5
|
||||
* (self.duration + self.warm_up[0] - self.warm_up[1] - self.step)
|
||||
/ self.duration
|
||||
* num_frames_per_chunk
|
||||
)
|
||||
warm_up_right = num_frames_per_chunk - math.ceil(
|
||||
0.5
|
||||
* (self.duration + self.warm_up[0] - self.warm_up[1] + self.step)
|
||||
/ self.duration
|
||||
* num_frames_per_chunk
|
||||
)
|
||||
|
||||
# we do not need a window here because there is no overlap
|
||||
# between concatenated chunk central parts.
|
||||
window = 1.0
|
||||
|
||||
warm_up[:warm_up_left] = 0.0
|
||||
warm_up_last[:warm_up_left] = 0.0
|
||||
warm_up[num_frames_per_chunk - warm_up_right :] = 0.0
|
||||
warm_up_first[num_frames_per_chunk - warm_up_right :] = 0.0
|
||||
|
||||
# aggregated_output[i] will be used to store the sum of all predictions
|
||||
# for frame #i
|
||||
aggregated_output: np.ndarray = np.zeros(
|
||||
(num_frames, dimension), dtype=np.float32
|
||||
)
|
||||
|
||||
# overlapping_chunk_count[i] will be used to store the number of chunks
|
||||
# that contributed to frame #i
|
||||
overlapping_chunk_count: np.ndarray = np.zeros(
|
||||
(num_frames, 1), dtype=np.float32
|
||||
)
|
||||
|
||||
if specifications.permutation_invariant:
|
||||
# number of previous outputs that overlap with current one by at least 50% of their warmed up region
|
||||
num_overlap = math.floor(
|
||||
0.5 * (self.duration - self.warm_up[0] - self.warm_up[1]) / self.step
|
||||
)
|
||||
# keep track of those previous outputs in a "deque"
|
||||
if num_overlap > 0:
|
||||
previous_outputs: Deque[np.ndarray] = deque([], maxlen=num_overlap)
|
||||
|
||||
# loop on the outputs of sliding chunks
|
||||
for c, output in enumerate(outputs):
|
||||
start_sample = c * step_size
|
||||
start_frame, _ = introspection(start_sample)
|
||||
|
||||
if specifications.permutation_invariant and num_overlap > 0:
|
||||
if c > 0:
|
||||
output = self.permutate(
|
||||
np.stack(previous_outputs), output, num_frames_per_step
|
||||
)
|
||||
previous_outputs.append(output)
|
||||
|
||||
# when processing first chunk, do not weigh-down its left-most side
|
||||
if c == 0:
|
||||
# unless there is just one chunk, where we do not weigh-down any side
|
||||
if not has_last_chunk and num_chunks == 1:
|
||||
warm_up_ = 1.0
|
||||
else:
|
||||
warm_up_ = warm_up_first
|
||||
|
||||
# when processing last chunk, make sure to not weigh-down its right-most side
|
||||
elif not has_last_chunk and c + 1 == num_chunks:
|
||||
warm_up_ = warm_up_last
|
||||
|
||||
# when processing an internal chunk, weigh-down both sides
|
||||
else:
|
||||
warm_up_ = warm_up
|
||||
|
||||
aggregated_output[start_frame : start_frame + num_frames_per_chunk] += (
|
||||
output * window * warm_up_
|
||||
)
|
||||
|
||||
overlapping_chunk_count[
|
||||
start_frame : start_frame + num_frames_per_chunk
|
||||
] += (window * warm_up_)
|
||||
|
||||
# process last (right-aligned) chunk separately
|
||||
if has_last_chunk:
|
||||
|
||||
if (
|
||||
specifications.permutation_invariant
|
||||
and num_overlap > 0
|
||||
and previous_outputs
|
||||
):
|
||||
# FIXME
|
||||
last_output = self.permutate(
|
||||
previous_outputs[-1][np.newaxis],
|
||||
last_output,
|
||||
num_frames_last_step,
|
||||
)
|
||||
|
||||
aggregated_output[-num_frames_per_chunk:] += (
|
||||
last_output * window * warm_up_last
|
||||
)
|
||||
overlapping_chunk_count[-num_frames_per_chunk:] += window * warm_up_last
|
||||
|
||||
aggregated_output /= np.maximum(overlapping_chunk_count, 1e-12)
|
||||
|
||||
# FIXME -- get this from introspection
|
||||
frames = SlidingWindow(
|
||||
start=0,
|
||||
duration=file_duration / num_frames,
|
||||
step=file_duration / num_frames,
|
||||
)
|
||||
|
||||
return SlidingWindowFeature(aggregated_output, frames)
|
||||
|
||||
def permutate(
|
||||
self, past_outputs: Deque[np.ndarray], output: np.ndarray, step_size: int
|
||||
@@ -533,14 +480,7 @@ class Inference:
|
||||
((permutation, _),) = Counter(permutations).most_common(1)
|
||||
return output[:, permutation]
|
||||
|
||||
def __call__(
|
||||
self, file: AudioFile
|
||||
) -> Union[
|
||||
SlidingWindowFeature,
|
||||
Dict[Text, SlidingWindowFeature],
|
||||
np.ndarray,
|
||||
Dict[Text, np.ndarray],
|
||||
]:
|
||||
def __call__(self, file: AudioFile) -> Union[SlidingWindowFeature, np.ndarray]:
|
||||
"""Run inference on a whole file
|
||||
|
||||
Parameters
|
||||
@@ -554,10 +494,6 @@ class Inference:
|
||||
Model output, as `SlidingWindowFeature` if `window` is set to "sliding"
|
||||
and `np.ndarray` if is set to "whole".
|
||||
|
||||
Notes
|
||||
-----
|
||||
If model has several outputs (multi-task), those will be returned as a
|
||||
{task_name: output} dictionary.
|
||||
"""
|
||||
|
||||
waveform, sample_rate = self.model.audio(file)
|
||||
@@ -565,26 +501,14 @@ class Inference:
|
||||
if self.window == "sliding":
|
||||
return self.slide(waveform, sample_rate)
|
||||
|
||||
outputs = {
|
||||
task_name: task_output[0]
|
||||
for task_name, task_output in self.infer(waveform[None]).items()
|
||||
}
|
||||
if self.model.is_multi_task:
|
||||
return outputs
|
||||
else:
|
||||
return outputs.popitem()[1]
|
||||
return self.infer(waveform[None])[0]
|
||||
|
||||
def crop(
|
||||
self,
|
||||
file: AudioFile,
|
||||
chunk: Union[Segment, List[Segment]],
|
||||
fixed: Optional[float] = None,
|
||||
) -> Union[
|
||||
SlidingWindowFeature,
|
||||
Dict[Text, SlidingWindowFeature],
|
||||
np.ndarray,
|
||||
Dict[Text, np.ndarray],
|
||||
]:
|
||||
) -> Union[SlidingWindowFeature, np.ndarray]:
|
||||
"""Run inference on a chunk or a list of chunks
|
||||
|
||||
Parameters
|
||||
@@ -612,9 +536,6 @@ class Inference:
|
||||
|
||||
Notes
|
||||
-----
|
||||
If model has several outputs (multi-task), those will be returned as a
|
||||
{task_name: output} dictionary.
|
||||
|
||||
If model needs to be warmed up, remember to extend the requested chunk with the
|
||||
corresponding amount of time so that it is actually warmed up when processing the
|
||||
chunk of interest:
|
||||
@@ -633,23 +554,11 @@ class Inference:
|
||||
waveform, sample_rate = self.model.audio.crop(file, chunk, fixed=fixed)
|
||||
output = self.slide(waveform, sample_rate)
|
||||
|
||||
if self.model.is_multi_task:
|
||||
shifted_output = dict()
|
||||
for task_name, task_output in output.items():
|
||||
frames = task_output.sliding_window
|
||||
shifted_frames = SlidingWindow(
|
||||
start=chunk.start, duration=frames.duration, step=frames.step
|
||||
)
|
||||
shifted_output[task_name] = SlidingWindowFeature(
|
||||
task_output.data, shifted_frames
|
||||
)
|
||||
return shifted_output
|
||||
else:
|
||||
frames = output.sliding_window
|
||||
shifted_frames = SlidingWindow(
|
||||
start=chunk.start, duration=frames.duration, step=frames.step
|
||||
)
|
||||
return SlidingWindowFeature(output.data, shifted_frames)
|
||||
frames = output.sliding_window
|
||||
shifted_frames = SlidingWindow(
|
||||
start=chunk.start, duration=frames.duration, step=frames.step
|
||||
)
|
||||
return SlidingWindowFeature(output.data, shifted_frames)
|
||||
|
||||
elif self.window == "whole":
|
||||
|
||||
@@ -660,15 +569,7 @@ class Inference:
|
||||
[self.model.audio.crop(file, c)[0] for c in chunk], dim=1
|
||||
)
|
||||
|
||||
outputs = {
|
||||
task_name: task_output[0]
|
||||
for task_name, task_output in self.infer(waveform[None]).items()
|
||||
}
|
||||
|
||||
if self.model.is_multi_task:
|
||||
return outputs
|
||||
else:
|
||||
return outputs.popitem()[1]
|
||||
return self.infer(waveform[None])[0]
|
||||
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
|
||||
@@ -240,26 +240,6 @@ class Introspection:
|
||||
step = (self.inc_num_samples / self.inc_num_frames) / sample_rate
|
||||
return SlidingWindow(start=0.0, step=step, duration=step)
|
||||
|
||||
def __len__(self):
|
||||
# makes it possible to do something like:
|
||||
# multi_task = len(introspection) > 1
|
||||
# because multi-task introspections are stored as {task_name: introspection} dict
|
||||
return 1
|
||||
|
||||
def __getitem__(self, key):
|
||||
if key is not None:
|
||||
raise KeyError
|
||||
return self
|
||||
|
||||
def items(self):
|
||||
yield None, self
|
||||
|
||||
def keys(self):
|
||||
yield None
|
||||
|
||||
def __iter__(self):
|
||||
yield None
|
||||
|
||||
|
||||
class Model(pl.LightningModule):
|
||||
"""Base model
|
||||
@@ -301,18 +281,11 @@ class Model(pl.LightningModule):
|
||||
def example_input_array(self) -> torch.Tensor:
|
||||
batch_size = 3 if self.task is None else self.task.batch_size
|
||||
|
||||
if self.is_multi_task:
|
||||
# this assumes that all tasks share the same duration
|
||||
_, specifications = next(iter(self.specifications.items()))
|
||||
else:
|
||||
specifications = self.specifications
|
||||
duration = specifications.duration
|
||||
|
||||
return torch.randn(
|
||||
(
|
||||
batch_size,
|
||||
self.hparams.num_channels,
|
||||
int(self.hparams.sample_rate * duration),
|
||||
int(self.hparams.sample_rate * self.specifications.duration),
|
||||
),
|
||||
device=self.device,
|
||||
)
|
||||
@@ -342,36 +315,23 @@ class Model(pl.LightningModule):
|
||||
if hasattr(self, "_specifications"):
|
||||
del self._specifications
|
||||
|
||||
@property
|
||||
def is_multi_task(self):
|
||||
return len(self.specifications) > 1
|
||||
|
||||
def build(self):
|
||||
# use this method to add task-dependent layers to the model
|
||||
# (e.g. the final classification and activation layers)
|
||||
pass
|
||||
|
||||
@property
|
||||
def introspection(self) -> Union[Introspection, Dict[Text, Introspection]]:
|
||||
def introspection(self) -> Introspection:
|
||||
"""Introspection
|
||||
|
||||
Returns
|
||||
-------
|
||||
introspection: Introspection or {str: Introspection} dict
|
||||
Model introspection or {task_name: introspection} dictionary for
|
||||
multi-task models.
|
||||
introspection: Introspection
|
||||
Model introspection
|
||||
"""
|
||||
|
||||
if not hasattr(self, "_introspection"):
|
||||
|
||||
if self.is_multi_task:
|
||||
self._introspection = {
|
||||
name: Introspection.from_model(self, task=name)
|
||||
for name in self.specifications
|
||||
}
|
||||
# TODO: raises an error in case of multiple tasks with different introspections
|
||||
else:
|
||||
self._introspection = Introspection.from_model(self)
|
||||
self._introspection = Introspection.from_model(self)
|
||||
|
||||
return self._introspection
|
||||
|
||||
@@ -502,7 +462,7 @@ class Model(pl.LightningModule):
|
||||
raise NotImplementedError(msg)
|
||||
|
||||
# convenience function to automate the choice of the final activation function
|
||||
def default_activation(self) -> Union[nn.Module, Dict[str, nn.Module]]:
|
||||
def default_activation(self) -> nn.Module:
|
||||
"""Guess default activation function according to task specification
|
||||
|
||||
* sigmoid for binary classification
|
||||
@@ -511,18 +471,9 @@ class Model(pl.LightningModule):
|
||||
|
||||
Returns
|
||||
-------
|
||||
activation : nn.Module or {str: nn.Module}
|
||||
Activation or {task_name: activation} dictionary for multi-task models.
|
||||
activation : nn.Module
|
||||
Activation.
|
||||
"""
|
||||
|
||||
if self.is_multi_task:
|
||||
return nn.ModuleDict(
|
||||
{
|
||||
name: self.helper_default_activation(specs)
|
||||
for name, specs in self.specifications.items()
|
||||
}
|
||||
)
|
||||
|
||||
return self.helper_default_activation(self.specifications)
|
||||
|
||||
# training step logic is delegated to the task because the
|
||||
|
||||
@@ -84,26 +84,6 @@ class Specifications:
|
||||
# whether classes are permutation-invariant (e.g. diarization)
|
||||
permutation_invariant: bool = False
|
||||
|
||||
def __len__(self):
|
||||
# makes it possible to do something like:
|
||||
# multi_task = len(specifications) > 1
|
||||
# because multi-task specifications are stored as {task_name: specifications} dict
|
||||
return 1
|
||||
|
||||
def __getitem__(self, key):
|
||||
if key is not None:
|
||||
raise KeyError
|
||||
return self
|
||||
|
||||
def items(self):
|
||||
yield None, self
|
||||
|
||||
def keys(self):
|
||||
yield None
|
||||
|
||||
def __iter__(self):
|
||||
yield None
|
||||
|
||||
|
||||
class TrainDataset(IterableDataset):
|
||||
def __init__(self, task: Task):
|
||||
@@ -176,8 +156,6 @@ class Task(pl.LightningDataModule):
|
||||
----------
|
||||
specifications : Specifications or dict of Specifications
|
||||
Task specifications (available after `Task.setup` has been called.)
|
||||
For multi-task learning, this should be a dictionary where keys are
|
||||
task names and values are corresponding Specifications instances.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -263,11 +241,6 @@ class Task(pl.LightningDataModule):
|
||||
def setup_validation_metric(self):
|
||||
pass
|
||||
|
||||
@property
|
||||
def is_multi_task(self) -> bool:
|
||||
""""Check whether multiple tasks are addressed at once"""
|
||||
return len(self.specifications) > 1
|
||||
|
||||
def train__iter__(self):
|
||||
# will become train_dataset.__iter__ method
|
||||
msg = f"Missing '{self.__class__.__name__}.train__iter__' method."
|
||||
@@ -341,8 +314,6 @@ class Task(pl.LightningDataModule):
|
||||
* binary cross-entropy loss for binary or multi-label classification
|
||||
* negative log-likelihood loss for regular classification
|
||||
|
||||
In case of multi-tasking, it will default to summing loss of each task.
|
||||
|
||||
If "weight" attribute exists, batch[self.weight] is also passed to the loss function
|
||||
during training (but has no effect in validation).
|
||||
|
||||
@@ -358,18 +329,14 @@ class Task(pl.LightningDataModule):
|
||||
Returns
|
||||
-------
|
||||
loss : {str: torch.tensor}
|
||||
{"loss": loss} with additional "loss_{task_name}" keys for multi-task models.
|
||||
{"loss": loss}
|
||||
"""
|
||||
|
||||
# forward pass
|
||||
y_pred = self.model(batch["X"])
|
||||
|
||||
if self.is_multi_task:
|
||||
_, any_y_pred = next(iter(y_pred.items()))
|
||||
batch_size, num_frames, _ = any_y_pred.shape
|
||||
else:
|
||||
batch_size, num_frames, _ = y_pred.shape
|
||||
# (batch_size, num_frames, num_classes)
|
||||
batch_size, num_frames, _ = y_pred.shape
|
||||
# (batch_size, num_frames, num_classes)
|
||||
|
||||
# target
|
||||
y = batch["y"]
|
||||
@@ -388,34 +355,7 @@ class Task(pl.LightningDataModule):
|
||||
warm_up_right = round(self.warm_up[1] / self.duration * num_frames)
|
||||
weight[:, num_frames - warm_up_right :] = 0.0
|
||||
|
||||
# compute multi-task loss as the sum of loss of each task
|
||||
if self.is_multi_task:
|
||||
loss = dict()
|
||||
for task_name, specifications in self.specifications.items():
|
||||
loss[task_name] = self.default_loss(
|
||||
specifications, y[task_name], y_pred[task_name], weight=weight
|
||||
)
|
||||
self.model.log(
|
||||
f"{task_name}@{stage}_loss",
|
||||
loss[task_name],
|
||||
on_step=False,
|
||||
on_epoch=True,
|
||||
prog_bar=False,
|
||||
logger=False,
|
||||
)
|
||||
|
||||
loss["loss"] = sum(loss.values())
|
||||
self.model.log(
|
||||
f"{self.ACRONYM}@{stage}_loss",
|
||||
loss["loss"],
|
||||
on_step=False,
|
||||
on_epoch=True,
|
||||
prog_bar=True,
|
||||
logger=True,
|
||||
)
|
||||
return loss
|
||||
|
||||
# compute mono-task loss
|
||||
# compute loss
|
||||
loss = self.default_loss(self.specifications, y, y_pred, weight=weight)
|
||||
self.model.log(
|
||||
f"{self.ACRONYM}@{stage}_loss",
|
||||
|
||||
@@ -1,236 +0,0 @@
|
||||
# MIT License
|
||||
#
|
||||
# Copyright (c) 2020 CNRS
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
|
||||
from functools import cached_property
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from einops import rearrange
|
||||
|
||||
from pyannote.audio.core.model import Model
|
||||
from pyannote.audio.core.task import Task
|
||||
from pyannote.audio.models.blocks.sincnet import SincNet
|
||||
from pyannote.audio.utils.probe import probe
|
||||
from pyannote.core.utils.generators import pairwise
|
||||
|
||||
# from pyannote.core.utils.params import merge_dict
|
||||
|
||||
|
||||
class Branch(nn.Module):
|
||||
"""Branch"""
|
||||
|
||||
LSTM_DEFAULTS = {"hidden_size": 128, "num_layers": 1, "bidirectional": True}
|
||||
LINEAR_DEFAULTS = {"hidden_size": 128, "num_layers": 1}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
in_features,
|
||||
lstm: dict = None,
|
||||
linear: dict = None,
|
||||
head: int = None,
|
||||
):
|
||||
|
||||
super().__init__()
|
||||
|
||||
self.lstm_hparams = dict(**self.LSTM_DEFAULTS)
|
||||
if lstm is not None:
|
||||
self.lstm_hparams.update(**lstm)
|
||||
self.lstm_hparams["batch_first"] = True # this is not negotiable
|
||||
|
||||
self.linear_hparams = dict(**self.LINEAR_DEFAULTS)
|
||||
if linear is not None:
|
||||
self.linear_hparams.update(**linear)
|
||||
|
||||
self.lstm = nn.ModuleList()
|
||||
one_lstm_hparams = dict(self.lstm_hparams)
|
||||
one_lstm_hparams["num_layers"] = 1
|
||||
for i in range(self.lstm_hparams["num_layers"]):
|
||||
self.lstm.append(nn.LSTM(in_features, **one_lstm_hparams))
|
||||
in_features = self.lstm_hparams["hidden_size"] * (
|
||||
2 if self.lstm_hparams["bidirectional"] else 1
|
||||
)
|
||||
|
||||
lstm_out_features = in_features
|
||||
self.linear = nn.ModuleList(
|
||||
[
|
||||
nn.Linear(in_features, out_features)
|
||||
for in_features, out_features in pairwise(
|
||||
[
|
||||
lstm_out_features,
|
||||
]
|
||||
+ [self.linear_hparams["hidden_size"]]
|
||||
* self.linear_hparams["num_layers"]
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
def forward(self, frames: torch.Tensor) -> torch.Tensor:
|
||||
"""Pass forward
|
||||
|
||||
Parameters
|
||||
----------
|
||||
frames : (batch, ???, ???)
|
||||
"""
|
||||
|
||||
outputs = frames
|
||||
|
||||
for i, lstm in enumerate(self.lstm):
|
||||
if i == 0:
|
||||
outputs, hidden = lstm(outputs)
|
||||
else:
|
||||
outputs, hidden = lstm(outputs, hidden)
|
||||
|
||||
if len(self.linear) > 0:
|
||||
for linear in self.linear:
|
||||
outputs = F.leaky_relu(linear(outputs))
|
||||
|
||||
return outputs
|
||||
|
||||
@cached_property
|
||||
def out_features(self) -> int:
|
||||
|
||||
if len(self.linear) > 0:
|
||||
return self.linear_hparams["hidden_size"]
|
||||
|
||||
return self.lstm_hparams["hidden_size"] * (
|
||||
2 if self.lstm_hparams["bidirectional"] else 1
|
||||
)
|
||||
|
||||
|
||||
class Trunk(Branch):
|
||||
def __init__(self, in_features, lstm: dict = None):
|
||||
no_linear = {"num_layers": 0}
|
||||
super().__init__(in_features, lstm=lstm, linear=no_linear)
|
||||
|
||||
|
||||
class MultiPyanNet(Model):
|
||||
"""PyanNet segmentation model
|
||||
|
||||
SincFilterbank > Conv1 > Conv1d > LSTM > Feed forward > Classifier
|
||||
|
||||
Parameters
|
||||
----------
|
||||
sample_rate : int, optional
|
||||
Audio sample rate. Defaults to 16kHz (16000).
|
||||
num_channels : int, optional
|
||||
Number of channels. Defaults to mono (1).
|
||||
sincnet : dict, optional
|
||||
Keyword arugments passed to the SincNet block.
|
||||
Defaults to {"stride": 1}.
|
||||
trunk : dict, optional
|
||||
branches : dict, optional
|
||||
"""
|
||||
|
||||
SINCNET_DEFAULTS = {"stride": 1}
|
||||
|
||||
TRUNK_DEFAULTS = {
|
||||
"lstm": {"num_layers": 3, "hidden_size": 128, "bidirectional": True},
|
||||
}
|
||||
|
||||
BRANCH_DEFAULTS = {
|
||||
"lstm": {"num_layers": 1, "hidden_size": 128, "bidirectional": True},
|
||||
"linear": {"num_layers": 1, "hidden_size": 128},
|
||||
}
|
||||
|
||||
BRANCHES_DEFAULTS = {
|
||||
"vad": {"head": 0, **BRANCH_DEFAULTS},
|
||||
"scd": {"head": 1, **BRANCH_DEFAULTS},
|
||||
"osd": {"head": 2, **BRANCH_DEFAULTS},
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
sample_rate: int = 16000,
|
||||
num_channels: int = 1,
|
||||
sincnet: dict = None,
|
||||
trunk: dict = None,
|
||||
branches: dict = None,
|
||||
task: Optional[Task] = None,
|
||||
):
|
||||
|
||||
super().__init__(sample_rate=sample_rate, num_channels=num_channels, task=task)
|
||||
|
||||
sincnet_hparams = dict(**self.SINCNET_DEFAULTS)
|
||||
if sincnet is not None:
|
||||
sincnet_hparams.update(**sincnet)
|
||||
sincnet_hparams["sample_rate"] = sample_rate
|
||||
self.hparams.sincnet = sincnet_hparams
|
||||
self.sincnet = SincNet(**self.hparams.sincnet)
|
||||
|
||||
if trunk is None:
|
||||
trunk = self.TRUNK_DEFAULTS
|
||||
self.hparams.trunk = trunk
|
||||
|
||||
if branches is None:
|
||||
branches = self.BRANCHES_DEFAULTS
|
||||
self.hparams.branches = branches
|
||||
|
||||
self.trunk = Trunk(60, **self.hparams.trunk)
|
||||
|
||||
def build(self):
|
||||
|
||||
probes = dict()
|
||||
self.branches = nn.ModuleDict()
|
||||
self.classifiers = nn.ModuleDict()
|
||||
for task_name, specifications in self.specifications.items():
|
||||
branch_hparams = self.hparams.branches[task_name]
|
||||
probes[task_name] = f"lstm.{branch_hparams['head']:d}"
|
||||
branch = Branch(self.trunk.out_features, **branch_hparams)
|
||||
self.branches[task_name] = branch
|
||||
self.classifiers[task_name] = nn.Linear(
|
||||
branch.out_features, len(specifications.classes)
|
||||
)
|
||||
|
||||
# probe trunk
|
||||
_ = probe(self.trunk, probes)
|
||||
|
||||
self.activation: dict = self.default_activation()
|
||||
|
||||
def forward(self, waveforms: torch.Tensor) -> torch.Tensor:
|
||||
"""Pass forward
|
||||
|
||||
Parameters
|
||||
----------
|
||||
waveforms : (batch, channel, sample)
|
||||
|
||||
Returns
|
||||
-------
|
||||
scores : {task: (batch, frame, classes) scores} dict
|
||||
"""
|
||||
|
||||
frames = self.sincnet(waveforms)
|
||||
|
||||
outputs: dict = self.trunk(
|
||||
rearrange(frames, "batch feature frame -> batch frame feature")
|
||||
)
|
||||
|
||||
return {
|
||||
task_name: self.activation[task_name](
|
||||
self.classifiers[task_name](
|
||||
self.branches[task_name](outputs[task_name][0])
|
||||
)
|
||||
)
|
||||
for task_name in self.specifications
|
||||
}
|
||||
@@ -80,52 +80,3 @@ class SimpleSegmentationModel(Model):
|
||||
output, hidden = self.lstm(rearrange(mfcc, "b c f t -> b t (c f)"))
|
||||
# apply the final classifier to get logits
|
||||
return self.activation(self.classifier(output))
|
||||
|
||||
|
||||
class MultiTaskSegmentationModel(Model):
|
||||
def __init__(
|
||||
self,
|
||||
sample_rate: int = 16000,
|
||||
num_channels: int = 1,
|
||||
task: Optional[Task] = None,
|
||||
):
|
||||
|
||||
super().__init__(sample_rate=sample_rate, num_channels=num_channels, task=task)
|
||||
|
||||
self.mfcc = MFCC(
|
||||
sample_rate=self.hparams.sample_rate,
|
||||
n_mfcc=40,
|
||||
dct_type=2,
|
||||
norm="ortho",
|
||||
log_mels=False,
|
||||
)
|
||||
|
||||
self.lstm = nn.LSTM(
|
||||
self.mfcc.n_mfcc * self.hparams.num_channels,
|
||||
32,
|
||||
num_layers=1,
|
||||
batch_first=True,
|
||||
bidirectional=True,
|
||||
)
|
||||
|
||||
def build(self):
|
||||
|
||||
self.classifier = nn.ModuleDict(
|
||||
{
|
||||
name: nn.Linear(32 * 2, len(specifications.classes))
|
||||
for name, specifications in self.specifications.items()
|
||||
}
|
||||
)
|
||||
|
||||
self.activation = self.default_activation()
|
||||
|
||||
def forward(self, waveforms: torch.Tensor) -> torch.Tensor:
|
||||
# extract MFCC
|
||||
mfcc = self.mfcc(waveforms)
|
||||
# pass MFCC sequence into the recurrent layer
|
||||
output, hidden = self.lstm(rearrange(mfcc, "b c f t -> b t (c f)"))
|
||||
|
||||
return {
|
||||
name: self.activation[name](self.classifier[name](output))
|
||||
for name in self.specifications
|
||||
}
|
||||
@@ -208,14 +208,7 @@ class SegmentationTaskMixin:
|
||||
|
||||
# use model introspection to predict how many frames it will output
|
||||
num_samples = sample["X"].shape[1]
|
||||
introspection = self.model.introspection
|
||||
if self.is_multi_task:
|
||||
# this assumes that all tasks share the same model introspection.
|
||||
# this is a reasonable assumption for now.
|
||||
any_task = next(iter(introspection.keys()))
|
||||
num_frames, _ = introspection[any_task](num_samples)
|
||||
else:
|
||||
num_frames, _ = introspection(num_samples)
|
||||
num_frames, _ = self.model.introspection(num_samples)
|
||||
|
||||
# crop "annotation" and keep track of corresponding list of labels if needed
|
||||
annotation: Annotation = file["annotation"].crop(chunk)
|
||||
|
||||
@@ -1,264 +0,0 @@
|
||||
# MIT License
|
||||
#
|
||||
# Copyright (c) 2020-2021 CNRS
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
|
||||
from typing import Mapping, Text
|
||||
|
||||
import numpy as np
|
||||
from pytorch_lightning.metrics.classification import FBeta
|
||||
from torch_audiomentations.core.transforms_interface import BaseWaveformTransform
|
||||
|
||||
from pyannote.audio.core.task import Problem, Task
|
||||
from pyannote.audio.tasks import (
|
||||
OverlappedSpeechDetection,
|
||||
SpeakerChangeDetection,
|
||||
VoiceActivityDetection,
|
||||
)
|
||||
from pyannote.audio.tasks.segmentation.mixins import SegmentationTaskMixin
|
||||
from pyannote.database import Protocol
|
||||
|
||||
|
||||
class MultiTaskSegmentation(SegmentationTaskMixin, Task):
|
||||
"""Multi-task segmentation
|
||||
|
||||
Multi-task training of segmentation tasks, including:
|
||||
- vad: voice activity detection
|
||||
- scd: speaker change detection
|
||||
- osd: overlapped speech detection
|
||||
|
||||
Note that, when "osd" is one of the tasks considered, data augmentation is used
|
||||
to artificially increase the proportion of "overlap". This is achieved by
|
||||
generating chunks made out of the (weighted) sum of two random chunks.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
protocol : Protocol
|
||||
pyannote.database protocol
|
||||
duration : float, optional
|
||||
Chunks duration. Defaults to 2s.
|
||||
balance: str, optional
|
||||
When provided, training samples are sampled uniformly with respect to that key.
|
||||
For instance, setting `balance` to "uri" will make sure that each file will be
|
||||
equally represented in the training samples.
|
||||
overlap: dict, optional
|
||||
Controls how artificial chunks with overlapping speech are generated:
|
||||
- "probability" key is the probability of artificial overlapping chunks. Setting
|
||||
"probability" to 0.6 means that, on average, 40% of training chunks are "real"
|
||||
chunks, while 60% are artifical chunks made out of the (weighted) sum of two
|
||||
chunks. Defaults to 0.5.
|
||||
- "snr_min" and "snr_max" keys control the minimum and maximum signal-to-noise
|
||||
ratio between summed chunks, in dB. Default to 0.0 and 10.
|
||||
weight: str, optional
|
||||
When provided, use this key to as frame-wise weight in loss function.
|
||||
vad : bool, optional
|
||||
Add voice activity detection in the pool of tasks.
|
||||
Defaults to False.
|
||||
vad_params : dict, optional
|
||||
Additional vad-specific parameters. Has no effect when `vad` is False.
|
||||
See VoiceActivityDetection docstring for details and default value.
|
||||
scd : bool, optional
|
||||
Add speaker change detection to the pool of tasks.
|
||||
Defaults to False.
|
||||
scd_params : dict, optional
|
||||
Additional scd-specific parameters. Has no effect when `scd` is False.
|
||||
See SpeakerChangeDetection docstring for details and default value.
|
||||
osd : bool, optional
|
||||
Add overlapped speech detection to the pool of tasks.
|
||||
Defaults to False.
|
||||
osd_params : dict, optional
|
||||
Additional osd-specific parameters. Has no effect when `osd` is False.
|
||||
See OverlappedSpeechDetection docstring for details and default value.
|
||||
batch_size : int, optional
|
||||
Number of training samples per batch. Defaults to 32.
|
||||
num_workers : int, optional
|
||||
Number of workers used for generating training samples.
|
||||
Defaults to multiprocessing.cpu_count() // 2.
|
||||
pin_memory : bool, optional
|
||||
If True, data loaders will copy tensors into CUDA pinned
|
||||
memory before returning them. See pytorch documentation
|
||||
for more details. Defaults to False.
|
||||
augmentation : BaseWaveformTransform, optional
|
||||
torch_audiomentations waveform transform, used by dataloader
|
||||
during training.
|
||||
|
||||
"""
|
||||
|
||||
ACRONYM = "xseg"
|
||||
|
||||
OVERLAP_DEFAULTS = {"probability": 0.5, "snr_min": 0.0, "snr_max": 10.0}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
protocol: Protocol,
|
||||
duration: float = 2.0,
|
||||
overlap: dict = OVERLAP_DEFAULTS,
|
||||
balance: Text = None,
|
||||
weight: Text = None,
|
||||
vad: bool = False,
|
||||
vad_params: Mapping = None,
|
||||
scd: bool = False,
|
||||
scd_params: Mapping = None,
|
||||
osd: bool = False,
|
||||
osd_params: Mapping = None,
|
||||
batch_size: int = 32,
|
||||
num_workers: int = None,
|
||||
pin_memory: bool = False,
|
||||
augmentation: BaseWaveformTransform = None,
|
||||
):
|
||||
|
||||
super().__init__(
|
||||
protocol,
|
||||
duration=duration,
|
||||
batch_size=batch_size,
|
||||
num_workers=num_workers,
|
||||
pin_memory=pin_memory,
|
||||
augmentation=augmentation,
|
||||
)
|
||||
|
||||
self.vad = vad
|
||||
self.vad_params = dict() if vad_params is None else vad_params
|
||||
|
||||
self.scd = scd
|
||||
self.scd_params = dict() if scd_params is None else scd_params
|
||||
|
||||
self.osd = osd
|
||||
self.osd_params = dict() if osd_params is None else osd_params
|
||||
|
||||
if sum([vad, scd, osd]) < 2:
|
||||
raise ValueError(
|
||||
"You must activate at least two tasks among 'vad', 'scd', and 'osd'."
|
||||
)
|
||||
|
||||
self.tasks = dict()
|
||||
if self.vad:
|
||||
self.tasks["vad"] = VoiceActivityDetection(
|
||||
protocol,
|
||||
duration=duration,
|
||||
batch_size=batch_size,
|
||||
num_workers=num_workers,
|
||||
pin_memory=pin_memory,
|
||||
**self.vad_params,
|
||||
)
|
||||
if self.scd:
|
||||
self.tasks["scd"] = SpeakerChangeDetection(
|
||||
protocol,
|
||||
duration=duration,
|
||||
batch_size=batch_size,
|
||||
num_workers=num_workers,
|
||||
pin_memory=pin_memory,
|
||||
**self.scd_params,
|
||||
)
|
||||
if self.osd:
|
||||
self.tasks["osd"] = OverlappedSpeechDetection(
|
||||
protocol,
|
||||
duration=duration,
|
||||
batch_size=batch_size,
|
||||
num_workers=num_workers,
|
||||
pin_memory=pin_memory,
|
||||
**self.osd_params,
|
||||
)
|
||||
|
||||
def setup(self, stage=None):
|
||||
|
||||
super().setup(stage=stage)
|
||||
|
||||
if stage == "fit":
|
||||
|
||||
self.specifications = {
|
||||
name: task.specifications for name, task in self.tasks.items()
|
||||
}
|
||||
|
||||
def prepare_y(self, one_hot_y: np.ndarray):
|
||||
"""Get multi-task targets
|
||||
|
||||
Parameters
|
||||
----------
|
||||
one_hot_y : (num_frames, num_speakers) np.ndarray
|
||||
One-hot-encoding of current chunk speaker activity:
|
||||
* one_hot_y[t, k] = 1 if kth speaker is active at tth frame
|
||||
* one_hot_y[t, k] = 0 otherwise.
|
||||
|
||||
Returns
|
||||
-------
|
||||
y : (num_frames, ) np.ndarray
|
||||
y[t] = 1 if there is two or more active speakers at tth frame, 0 otherwise.
|
||||
"""
|
||||
|
||||
return {name: task.prepare_y(one_hot_y) for name, task in self.tasks.items()}
|
||||
|
||||
def setup_validation_metric(self):
|
||||
|
||||
self.val_fbeta = {
|
||||
task_name: FBeta(
|
||||
len(specifications.classes),
|
||||
beta=1.0,
|
||||
threshold=0.5,
|
||||
multilabel=(
|
||||
specifications.problem == Problem.MULTI_LABEL_CLASSIFICATION
|
||||
),
|
||||
average="macro",
|
||||
)
|
||||
for task_name, specifications in self.specifications.items()
|
||||
}
|
||||
|
||||
def validation_step(self, batch, batch_idx: int):
|
||||
"""Compute areas under ROC curve
|
||||
|
||||
Parameters
|
||||
----------
|
||||
batch : dict of torch.Tensor
|
||||
Current batch.
|
||||
batch_idx: int
|
||||
Batch index.
|
||||
"""
|
||||
|
||||
X, y = batch["X"], batch["y"]
|
||||
y_pred = self.model(X)
|
||||
|
||||
val_fbeta = dict()
|
||||
|
||||
for task_name in self.specifications:
|
||||
|
||||
# move metric to model device
|
||||
self.val_fbeta[task_name].to(self.model.device)
|
||||
|
||||
val_fbeta[task_name] = self.val_fbeta[task_name](
|
||||
y_pred[task_name][:, ::10].squeeze(), y[task_name][:, ::10].squeeze()
|
||||
)
|
||||
|
||||
self.model.log(
|
||||
f"{task_name}@val_fbeta",
|
||||
val_fbeta[task_name],
|
||||
on_step=False,
|
||||
on_epoch=True,
|
||||
prog_bar=True,
|
||||
logger=True,
|
||||
)
|
||||
|
||||
self.model.log(
|
||||
f"{self.ACRONYM}@val_fbeta",
|
||||
sum(val_fbeta.values()) / len(val_fbeta),
|
||||
on_step=False,
|
||||
on_epoch=True,
|
||||
prog_bar=True,
|
||||
logger=True,
|
||||
)
|
||||
@@ -305,7 +305,7 @@ class Segmentation(SegmentationTaskMixin, Task):
|
||||
Returns
|
||||
-------
|
||||
loss : {str: torch.tensor}
|
||||
{"loss": loss} with additional "loss_{task_name}" keys for multi-task models.
|
||||
{"loss": loss}
|
||||
"""
|
||||
|
||||
# forward pass
|
||||
|
||||
+1
-34
@@ -4,14 +4,8 @@ import pytorch_lightning as pl
|
||||
|
||||
from pyannote.audio import Inference, Model
|
||||
from pyannote.audio.core.task import Resolution
|
||||
from pyannote.audio.models.segmentation.debug import (
|
||||
MultiTaskSegmentationModel,
|
||||
SimpleSegmentationModel,
|
||||
)
|
||||
from pyannote.audio.models.segmentation.debug import SimpleSegmentationModel
|
||||
from pyannote.audio.tasks import VoiceActivityDetection
|
||||
from pyannote.audio.tasks.segmentation.multi_task_segmentation import (
|
||||
MultiTaskSegmentation,
|
||||
)
|
||||
from pyannote.core import SlidingWindowFeature
|
||||
from pyannote.database import FileFinder, get_protocol
|
||||
|
||||
@@ -96,33 +90,6 @@ def test_on_file_path(trained):
|
||||
assert isinstance(output, SlidingWindowFeature)
|
||||
|
||||
|
||||
def test_multi_seg_infer():
|
||||
protocol = get_protocol(
|
||||
"Debug.SpeakerDiarization.Debug", preprocessors={"audio": FileFinder()}
|
||||
)
|
||||
xseg = MultiTaskSegmentation(
|
||||
protocol,
|
||||
duration=2.0,
|
||||
vad=True,
|
||||
scd=True,
|
||||
osd=True,
|
||||
batch_size=32,
|
||||
num_workers=4,
|
||||
)
|
||||
model = MultiTaskSegmentationModel(task=xseg)
|
||||
trainer = pl.Trainer(max_epochs=1, fast_dev_run=True)
|
||||
_ = trainer.fit(model, xseg)
|
||||
inference = Inference(model, duration=2.0, step=0.5)
|
||||
dev_file = next(protocol.development())
|
||||
scores = inference(dev_file)
|
||||
|
||||
assert isinstance(scores, dict)
|
||||
|
||||
for attr in ["vad", "scd", "osd"]:
|
||||
assert attr in scores
|
||||
assert isinstance(scores[attr], SlidingWindowFeature)
|
||||
|
||||
|
||||
def test_skip_aggregation(pretrained_model, dev_file):
|
||||
|
||||
inference = Inference(pretrained_model, skip_aggregation=True)
|
||||
|
||||
Reference in new issue
Block a user