chore: remove multi-task support (#675)

This commit is contained in:
Hervé BREDIN authored and GitHub committed 2021-06-04 16:35:32 +02:00
1 parent 062bd7bd9a
commit c69e92df77
12 files changed
+206 -1154

No files matched your search

-21
View File
@@ -18,7 +18,6 @@
"outputs": [],
"source": [
"from pyannote.audio.models.segmentation.debug import SimpleSegmentationModel\n",
"from pyannote.audio.models.segmentation.debug import MultiTaskSegmentationModel\n",
"from pyannote.audio.models.embedding.debug import SimpleEmbeddingModel\n",
"import pytorch_lightning as pl"
]
@@ -113,26 +112,6 @@
"_ = trainer.fit(model)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Multi-task segmentation"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from pyannote.audio.tasks.segmentation.multi_task_segmentation import MultiTaskSegmentation\n",
"seg = MultiTaskSegmentation(protocol, vad=True, scd=True, osd=True, duration=2., batch_size=32, num_workers=4)\n",
"model = MultiTaskSegmentationModel(task=seg)\n",
"trainer = pl.Trainer(max_epochs=1)\n",
"_ = trainer.fit(model)"
]
},
{
"cell_type": "markdown",
"metadata": {},
-129
View File
@@ -234,135 +234,6 @@
"\n",
"embeddings.shape"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Train and apply a multi-task segmentation model"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from pyannote.audio.tasks.segmentation.multi_task_segmentation import MultiTaskSegmentation\n",
"xseg = MultiTaskSegmentation(protocol, duration=2., vad=True, scd=True, osd=True, batch_size=32, num_workers=4)\n",
"from pyannote.audio.models.segmentation.debug import MultiTaskSegmentationModel\n",
"model = MultiTaskSegmentationModel(task=xseg)\n",
"trainer = pl.Trainer(max_epochs=1, default_root_dir='inference/xseg')\n",
"_ = trainer.fit(model)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# inference using a sliding window\n",
"inference = Inference(model, duration=2., step=0.5)\n",
"dev_file = next(protocol.development())\n",
"scores = inference(dev_file)\n",
"scores"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"scores['vad']"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"scores['scd']"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"scores['osd']"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# inference on an excerpt \n",
"scores = inference.crop(dev_file, Segment(10, 15))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"scores['vad']"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"scores['scd']"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"scores['osd']"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# inference on an excerpt shorter than sliding window duration\n",
"scores = inference.crop(dev_file, Segment(10, 11.5))\n",
"scores['vad']"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# inference on a whole chunk\n",
"inference = Inference(model, window=\"whole\")\n",
"scores = inference.crop(dev_file, Segment(10, 15))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"scores['vad'].shape"
]
}
],
"metadata": {
@@ -28,7 +28,6 @@ help:
* vad (default) = voice activity detection
* scd = speaker change detection
* osd = overlapped speech detection
* xseg = multi-task segmentation
{model} can be any of the following:
* debug (default) = simple segmentation model for debugging purposes
+191 -290
View File
@@ -24,7 +24,7 @@ import math
import warnings
from collections import Counter, deque
from pathlib import Path
from typing import Any, Callable, Deque, Dict, List, Optional, Text, Union
from typing import Any, Callable, Deque, List, Optional, Text, Union
import numpy as np
import torch
@@ -105,12 +105,12 @@ class Inference:
if window not in ["sliding", "whole"]:
raise ValueError('`window` must be "sliding" or "whole".')
for task_name, specifications in self.model.specifications.items():
if specifications.resolution == Resolution.FRAME and window == "whole":
warnings.warn(
'Using "whole" `window` inference with a frame-based model might lead to bad results '
'and huge memory consumption: it is recommended to set `window` to "sliding".'
)
specifications = self.model.specifications
if specifications.resolution == Resolution.FRAME and window == "whole":
warnings.warn(
'Using "whole" `window` inference with a frame-based model might lead to bad results '
'and huge memory consumption: it is recommended to set `window` to "sliding".'
)
self.window = window
self.skip_aggregation = skip_aggregation
@@ -122,12 +122,8 @@ class Inference:
self.model.eval()
self.model.to(self.device)
# chunk duration used during training. for multi-task,
# we assume that the same duration was used for each task.
if self.model.is_multi_task:
_, specifications = next(iter(self.model.specifications.items()))
else:
specifications = self.model.specifications
# chunk duration used during training
specifications = self.model.specifications
training_duration = specifications.duration
if duration is None:
@@ -169,7 +165,7 @@ class Inference:
progress_hook = None
self.progress_hook = progress_hook
def infer(self, chunks: torch.Tensor) -> Dict[TaskName, np.ndarray]:
def infer(self, chunks: torch.Tensor) -> np.ndarray:
"""Forward pass
Parameters
@@ -179,12 +175,8 @@ class Inference:
Returns
-------
outputs : {task_name: np.ndarray} dict
Model outputs.
Notes
-----
If model is mono-task, `task_name` is set to None.
outputs : np.ndarray
Model output.
"""
with torch.no_grad():
@@ -199,16 +191,9 @@ class Inference:
else:
raise exception
if self.model.is_multi_task:
return {
task_name: output.cpu().numpy() for task_name, output in outputs.items()
}
return outputs.cpu().numpy()
return {None: outputs.cpu().numpy()}
def slide(
self, waveform: torch.Tensor, sample_rate: int
) -> Union[SlidingWindowFeature, Dict[Text, SlidingWindowFeature]]:
def slide(self, waveform: torch.Tensor, sample_rate: int) -> SlidingWindowFeature:
"""Slide model on a waveform
Parameters
@@ -223,11 +208,6 @@ class Inference:
output : SlidingWindowFeature
Model output. Shape is (num_chunks, dimension) for chunk-level tasks,
and (num_frames, dimension) for frame-level tasks.
Notes
-----
If model has several outputs (multi-task), those will be returned as a
{task_name: output} dictionary.
"""
# prepare sliding audio chunks
@@ -235,8 +215,6 @@ class Inference:
file_duration = num_samples / sample_rate
window_size: int = round(self.duration * sample_rate)
results: Dict[Text, SlidingWindowFeature] = dict()
# corner case: waveform is shorter than chunk duration
if num_samples < window_size:
@@ -247,30 +225,22 @@ class Inference:
one_output = self.infer(waveform[None, :])
for task_name, specifications in self.model.specifications.items():
if specifications.resolution == Resolution.CHUNK:
frames = SlidingWindow(
start=0.0, duration=self.duration, step=self.step
)
results[task_name] = SlidingWindowFeature(
one_output[task_name], frames
)
if self.model.specifications.resolution == Resolution.CHUNK:
frames = SlidingWindow(
start=0.0, duration=self.duration, step=self.step
)
results = SlidingWindowFeature(one_output, frames)
else:
_, num_frames, dimension = one_output[task_name].shape
frames = SlidingWindow(
start=0,
duration=file_duration / num_frames,
step=file_duration / num_frames,
)
results[task_name] = SlidingWindowFeature(
one_output[task_name][0], frames
)
if self.model.is_multi_task:
return results
else:
return results.popitem()[1]
_, num_frames, dimension = one_output.shape
frames = SlidingWindow(
start=0,
duration=file_duration / num_frames,
step=file_duration / num_frames,
)
results = SlidingWindowFeature(one_output[0], frames)
return results
# prepare (and count) sliding audio chunks
step_size: int = round(self.step * sample_rate)
@@ -289,205 +259,182 @@ class Inference:
else:
has_last_chunk = False
outputs: Dict[TaskName, Union[List[np.ndarray], np.ndarray]] = {
task_name: list() for task_name, _ in self.model.specifications.items()
}
outputs: Union[List[np.ndarray], np.ndarray] = list()
if self.progress_hook is not None:
self.progress_hook(0, num_chunks + has_last_chunk)
# slide over audio chunks in batch
for c in np.arange(0, num_chunks, self.batch_size):
batch: torch.Tensor = chunks[c : c + self.batch_size]
output = self.infer(batch)
for task_name, task_output in output.items():
outputs[task_name].append(task_output)
outputs.append(self.infer(batch))
if self.progress_hook is not None:
self.progress_hook(c + 1, num_chunks + has_last_chunk)
outputs = {
task_name: np.vstack(task_outputs)
for task_name, task_outputs in outputs.items()
}
outputs = np.vstack(outputs)
for task_name, specifications in self.model.specifications.items():
# skip aggregation when requested
# or when model outputs just one vector per chunk
if self.skip_aggregation or specifications.resolution == Resolution.CHUNK:
frames = SlidingWindow(
start=0.0, duration=self.duration, step=self.step
)
results[task_name] = SlidingWindowFeature(outputs[task_name], frames)
continue
specifications = self.model.specifications
# process orphan last chunk
if has_last_chunk:
last_output = {
task_name: output[0]
for task_name, output in self.infer(last_chunk[None]).items()
}
if self.progress_hook is not None:
self.progress_hook(
num_chunks + has_last_chunk, num_chunks + has_last_chunk
)
# skip aggregation when requested
# or when model outputs just one vector per chunk
if self.skip_aggregation or specifications.resolution == Resolution.CHUNK:
frames = SlidingWindow(start=0.0, duration=self.duration, step=self.step)
return SlidingWindowFeature(outputs, frames)
# use model introspection to estimate the total number of frames
introspection = self.model.introspection[task_name]
num_frames, dimension = introspection(num_samples)
num_frames_per_chunk, _ = introspection(window_size)
num_frames_per_step, _ = introspection(step_size)
if has_last_chunk:
num_frames_last_step, _ = introspection(last_step_size)
# warm-up window used for overlap-add aggregation
warm_up = np.ones((num_frames_per_chunk, 1))
# ... for very first chunk
warm_up_first = np.ones((num_frames_per_chunk, 1))
# ... forvery last chunk
warm_up_last = np.ones((num_frames_per_chunk, 1))
if not specifications.permutation_invariant:
# anything before warm_up_left (and after num_frames_per_chunk - warm_up_right)
# will not be used in the final aggregation
warm_up_left = round(
self.warm_up[0] / self.duration * num_frames_per_chunk
)
warm_up_right = round(
self.warm_up[1] / self.duration * num_frames_per_chunk
# process orphan last chunk
if has_last_chunk:
last_output = self.infer(last_chunk[None])[0]
if self.progress_hook is not None:
self.progress_hook(
num_chunks + has_last_chunk, num_chunks + has_last_chunk
)
# Hamming window used for overlap-add aggregation
window = np.hamming(num_frames_per_chunk).reshape(-1, 1)
# use model introspection to estimate the total number of frames
introspection = self.model.introspection
num_frames, dimension = introspection(num_samples)
num_frames_per_chunk, _ = introspection(window_size)
num_frames_per_step, _ = introspection(step_size)
if has_last_chunk:
num_frames_last_step, _ = introspection(last_step_size)
else:
# Regular overlap-add aggregation cannot be used directly for
# permutation-invariant models. Why? Because two consecutive
# chunks may disagree on part of their output (even after optimal
# permutation).
# warm-up window used for overlap-add aggregation
warm_up = np.ones((num_frames_per_chunk, 1))
# ... for very first chunk
warm_up_first = np.ones((num_frames_per_chunk, 1))
# ... forvery last chunk
warm_up_last = np.ones((num_frames_per_chunk, 1))
# These two lines make inference use only the (step-long) central
# part of each chunk in the final aggregation... essentially
# switching to a simple concatenation.
if not specifications.permutation_invariant:
warm_up_left = math.floor(
0.5
* (self.duration + self.warm_up[0] - self.warm_up[1] - self.step)
/ self.duration
* num_frames_per_chunk
)
warm_up_right = num_frames_per_chunk - math.ceil(
0.5
* (self.duration + self.warm_up[0] - self.warm_up[1] + self.step)
/ self.duration
* num_frames_per_chunk
)
# we do not need a window here because there is no overlap
# between concatenated chunk central parts.
window = 1.0
warm_up[:warm_up_left] = 0.0
warm_up_last[:warm_up_left] = 0.0
warm_up[num_frames_per_chunk - warm_up_right :] = 0.0
warm_up_first[num_frames_per_chunk - warm_up_right :] = 0.0
# aggregated_output[i] will be used to store the sum of all predictions
# for frame #i
aggregated_output: np.ndarray = np.zeros(
(num_frames, dimension), dtype=np.float32
# anything before warm_up_left (and after num_frames_per_chunk - warm_up_right)
# will not be used in the final aggregation
warm_up_left = round(self.warm_up[0] / self.duration * num_frames_per_chunk)
warm_up_right = round(
self.warm_up[1] / self.duration * num_frames_per_chunk
)
# overlapping_chunk_count[i] will be used to store the number of chunks
# that contributed to frame #i
overlapping_chunk_count: np.ndarray = np.zeros(
(num_frames, 1), dtype=np.float32
)
# Hamming window used for overlap-add aggregation
window = np.hamming(num_frames_per_chunk).reshape(-1, 1)
if specifications.permutation_invariant:
# number of previous outputs that overlap with current one by at least 50% of their warmed up region
num_overlap = math.floor(
0.5
* (self.duration - self.warm_up[0] - self.warm_up[1])
/ self.step
)
# keep track of those previous outputs in a "deque"
if num_overlap > 0:
previous_outputs: Deque[np.ndarray] = deque([], maxlen=num_overlap)
# loop on the outputs of sliding chunks
for c, output in enumerate(outputs[task_name]):
start_sample = c * step_size
start_frame, _ = introspection(start_sample)
if specifications.permutation_invariant and num_overlap > 0:
if c > 0:
output = self.permutate(
np.stack(previous_outputs), output, num_frames_per_step
)
previous_outputs.append(output)
# when processing first chunk, do not weigh-down its left-most side
if c == 0:
# unless there is just one chunk, where we do not weigh-down any side
if not has_last_chunk and num_chunks == 1:
warm_up_ = 1.0
else:
warm_up_ = warm_up_first
# when processing last chunk, make sure to not weigh-down its right-most side
elif not has_last_chunk and c + 1 == num_chunks:
warm_up_ = warm_up_last
# when processing an internal chunk, weigh-down both sides
else:
warm_up_ = warm_up
aggregated_output[start_frame : start_frame + num_frames_per_chunk] += (
output * window * warm_up_
)
overlapping_chunk_count[
start_frame : start_frame + num_frames_per_chunk
] += (window * warm_up_)
# process last (right-aligned) chunk separately
if has_last_chunk:
if (
specifications.permutation_invariant
and num_overlap > 0
and previous_outputs
):
# FIXME
last_output[task_name] = self.permutate(
previous_outputs[-1][np.newaxis],
last_output[task_name],
num_frames_last_step,
)
aggregated_output[-num_frames_per_chunk:] += (
last_output[task_name] * window * warm_up_last
)
overlapping_chunk_count[-num_frames_per_chunk:] += window * warm_up_last
aggregated_output /= np.maximum(overlapping_chunk_count, 1e-12)
frames = SlidingWindow(
start=0,
duration=file_duration / num_frames,
step=file_duration / num_frames,
)
results[task_name] = SlidingWindowFeature(aggregated_output, frames)
if self.model.is_multi_task:
return results
else:
return results.popitem()[1]
# Regular overlap-add aggregation cannot be used directly for
# permutation-invariant models. Why? Because two consecutive
# chunks may disagree on part of their output (even after optimal
# permutation).
# These two lines make inference use only the (step-long) central
# part of each chunk in the final aggregation... essentially
# switching to a simple concatenation.
warm_up_left = math.floor(
0.5
* (self.duration + self.warm_up[0] - self.warm_up[1] - self.step)
/ self.duration
* num_frames_per_chunk
)
warm_up_right = num_frames_per_chunk - math.ceil(
0.5
* (self.duration + self.warm_up[0] - self.warm_up[1] + self.step)
/ self.duration
* num_frames_per_chunk
)
# we do not need a window here because there is no overlap
# between concatenated chunk central parts.
window = 1.0
warm_up[:warm_up_left] = 0.0
warm_up_last[:warm_up_left] = 0.0
warm_up[num_frames_per_chunk - warm_up_right :] = 0.0
warm_up_first[num_frames_per_chunk - warm_up_right :] = 0.0
# aggregated_output[i] will be used to store the sum of all predictions
# for frame #i
aggregated_output: np.ndarray = np.zeros(
(num_frames, dimension), dtype=np.float32
)
# overlapping_chunk_count[i] will be used to store the number of chunks
# that contributed to frame #i
overlapping_chunk_count: np.ndarray = np.zeros(
(num_frames, 1), dtype=np.float32
)
if specifications.permutation_invariant:
# number of previous outputs that overlap with current one by at least 50% of their warmed up region
num_overlap = math.floor(
0.5 * (self.duration - self.warm_up[0] - self.warm_up[1]) / self.step
)
# keep track of those previous outputs in a "deque"
if num_overlap > 0:
previous_outputs: Deque[np.ndarray] = deque([], maxlen=num_overlap)
# loop on the outputs of sliding chunks
for c, output in enumerate(outputs):
start_sample = c * step_size
start_frame, _ = introspection(start_sample)
if specifications.permutation_invariant and num_overlap > 0:
if c > 0:
output = self.permutate(
np.stack(previous_outputs), output, num_frames_per_step
)
previous_outputs.append(output)
# when processing first chunk, do not weigh-down its left-most side
if c == 0:
# unless there is just one chunk, where we do not weigh-down any side
if not has_last_chunk and num_chunks == 1:
warm_up_ = 1.0
else:
warm_up_ = warm_up_first
# when processing last chunk, make sure to not weigh-down its right-most side
elif not has_last_chunk and c + 1 == num_chunks:
warm_up_ = warm_up_last
# when processing an internal chunk, weigh-down both sides
else:
warm_up_ = warm_up
aggregated_output[start_frame : start_frame + num_frames_per_chunk] += (
output * window * warm_up_
)
overlapping_chunk_count[
start_frame : start_frame + num_frames_per_chunk
] += (window * warm_up_)
# process last (right-aligned) chunk separately
if has_last_chunk:
if (
specifications.permutation_invariant
and num_overlap > 0
and previous_outputs
):
# FIXME
last_output = self.permutate(
previous_outputs[-1][np.newaxis],
last_output,
num_frames_last_step,
)
aggregated_output[-num_frames_per_chunk:] += (
last_output * window * warm_up_last
)
overlapping_chunk_count[-num_frames_per_chunk:] += window * warm_up_last
aggregated_output /= np.maximum(overlapping_chunk_count, 1e-12)
# FIXME -- get this from introspection
frames = SlidingWindow(
start=0,
duration=file_duration / num_frames,
step=file_duration / num_frames,
)
return SlidingWindowFeature(aggregated_output, frames)
def permutate(
self, past_outputs: Deque[np.ndarray], output: np.ndarray, step_size: int
@@ -533,14 +480,7 @@ class Inference:
((permutation, _),) = Counter(permutations).most_common(1)
return output[:, permutation]
def __call__(
self, file: AudioFile
) -> Union[
SlidingWindowFeature,
Dict[Text, SlidingWindowFeature],
np.ndarray,
Dict[Text, np.ndarray],
]:
def __call__(self, file: AudioFile) -> Union[SlidingWindowFeature, np.ndarray]:
"""Run inference on a whole file
Parameters
@@ -554,10 +494,6 @@ class Inference:
Model output, as `SlidingWindowFeature` if `window` is set to "sliding"
and `np.ndarray` if is set to "whole".
Notes
-----
If model has several outputs (multi-task), those will be returned as a
{task_name: output} dictionary.
"""
waveform, sample_rate = self.model.audio(file)
@@ -565,26 +501,14 @@ class Inference:
if self.window == "sliding":
return self.slide(waveform, sample_rate)
outputs = {
task_name: task_output[0]
for task_name, task_output in self.infer(waveform[None]).items()
}
if self.model.is_multi_task:
return outputs
else:
return outputs.popitem()[1]
return self.infer(waveform[None])[0]
def crop(
self,
file: AudioFile,
chunk: Union[Segment, List[Segment]],
fixed: Optional[float] = None,
) -> Union[
SlidingWindowFeature,
Dict[Text, SlidingWindowFeature],
np.ndarray,
Dict[Text, np.ndarray],
]:
) -> Union[SlidingWindowFeature, np.ndarray]:
"""Run inference on a chunk or a list of chunks
Parameters
@@ -612,9 +536,6 @@ class Inference:
Notes
-----
If model has several outputs (multi-task), those will be returned as a
{task_name: output} dictionary.
If model needs to be warmed up, remember to extend the requested chunk with the
corresponding amount of time so that it is actually warmed up when processing the
chunk of interest:
@@ -633,23 +554,11 @@ class Inference:
waveform, sample_rate = self.model.audio.crop(file, chunk, fixed=fixed)
output = self.slide(waveform, sample_rate)
if self.model.is_multi_task:
shifted_output = dict()
for task_name, task_output in output.items():
frames = task_output.sliding_window
shifted_frames = SlidingWindow(
start=chunk.start, duration=frames.duration, step=frames.step
)
shifted_output[task_name] = SlidingWindowFeature(
task_output.data, shifted_frames
)
return shifted_output
else:
frames = output.sliding_window
shifted_frames = SlidingWindow(
start=chunk.start, duration=frames.duration, step=frames.step
)
return SlidingWindowFeature(output.data, shifted_frames)
frames = output.sliding_window
shifted_frames = SlidingWindow(
start=chunk.start, duration=frames.duration, step=frames.step
)
return SlidingWindowFeature(output.data, shifted_frames)
elif self.window == "whole":
@@ -660,15 +569,7 @@ class Inference:
[self.model.audio.crop(file, c)[0] for c in chunk], dim=1
)
outputs = {
task_name: task_output[0]
for task_name, task_output in self.infer(waveform[None]).items()
}
if self.model.is_multi_task:
return outputs
else:
return outputs.popitem()[1]
return self.infer(waveform[None])[0]
else:
raise NotImplementedError(
+8 -57
View File
@@ -240,26 +240,6 @@ class Introspection:
step = (self.inc_num_samples / self.inc_num_frames) / sample_rate
return SlidingWindow(start=0.0, step=step, duration=step)
def __len__(self):
# makes it possible to do something like:
# multi_task = len(introspection) > 1
# because multi-task introspections are stored as {task_name: introspection} dict
return 1
def __getitem__(self, key):
if key is not None:
raise KeyError
return self
def items(self):
yield None, self
def keys(self):
yield None
def __iter__(self):
yield None
class Model(pl.LightningModule):
"""Base model
@@ -301,18 +281,11 @@ class Model(pl.LightningModule):
def example_input_array(self) -> torch.Tensor:
batch_size = 3 if self.task is None else self.task.batch_size
if self.is_multi_task:
# this assumes that all tasks share the same duration
_, specifications = next(iter(self.specifications.items()))
else:
specifications = self.specifications
duration = specifications.duration
return torch.randn(
(
batch_size,
self.hparams.num_channels,
int(self.hparams.sample_rate * duration),
int(self.hparams.sample_rate * self.specifications.duration),
),
device=self.device,
)
@@ -342,36 +315,23 @@ class Model(pl.LightningModule):
if hasattr(self, "_specifications"):
del self._specifications
@property
def is_multi_task(self):
return len(self.specifications) > 1
def build(self):
# use this method to add task-dependent layers to the model
# (e.g. the final classification and activation layers)
pass
@property
def introspection(self) -> Union[Introspection, Dict[Text, Introspection]]:
def introspection(self) -> Introspection:
"""Introspection
Returns
-------
introspection: Introspection or {str: Introspection} dict
Model introspection or {task_name: introspection} dictionary for
multi-task models.
introspection: Introspection
Model introspection
"""
if not hasattr(self, "_introspection"):
if self.is_multi_task:
self._introspection = {
name: Introspection.from_model(self, task=name)
for name in self.specifications
}
# TODO: raises an error in case of multiple tasks with different introspections
else:
self._introspection = Introspection.from_model(self)
self._introspection = Introspection.from_model(self)
return self._introspection
@@ -502,7 +462,7 @@ class Model(pl.LightningModule):
raise NotImplementedError(msg)
# convenience function to automate the choice of the final activation function
def default_activation(self) -> Union[nn.Module, Dict[str, nn.Module]]:
def default_activation(self) -> nn.Module:
"""Guess default activation function according to task specification
* sigmoid for binary classification
@@ -511,18 +471,9 @@ class Model(pl.LightningModule):
Returns
-------
activation : nn.Module or {str: nn.Module}
Activation or {task_name: activation} dictionary for multi-task models.
activation : nn.Module
Activation.
"""
if self.is_multi_task:
return nn.ModuleDict(
{
name: self.helper_default_activation(specs)
for name, specs in self.specifications.items()
}
)
return self.helper_default_activation(self.specifications)
# training step logic is delegated to the task because the
+4 -64
View File
@@ -84,26 +84,6 @@ class Specifications:
# whether classes are permutation-invariant (e.g. diarization)
permutation_invariant: bool = False
def __len__(self):
# makes it possible to do something like:
# multi_task = len(specifications) > 1
# because multi-task specifications are stored as {task_name: specifications} dict
return 1
def __getitem__(self, key):
if key is not None:
raise KeyError
return self
def items(self):
yield None, self
def keys(self):
yield None
def __iter__(self):
yield None
class TrainDataset(IterableDataset):
def __init__(self, task: Task):
@@ -176,8 +156,6 @@ class Task(pl.LightningDataModule):
----------
specifications : Specifications or dict of Specifications
Task specifications (available after `Task.setup` has been called.)
For multi-task learning, this should be a dictionary where keys are
task names and values are corresponding Specifications instances.
"""
def __init__(
@@ -263,11 +241,6 @@ class Task(pl.LightningDataModule):
def setup_validation_metric(self):
pass
@property
def is_multi_task(self) -> bool:
""""Check whether multiple tasks are addressed at once"""
return len(self.specifications) > 1
def train__iter__(self):
# will become train_dataset.__iter__ method
msg = f"Missing '{self.__class__.__name__}.train__iter__' method."
@@ -341,8 +314,6 @@ class Task(pl.LightningDataModule):
* binary cross-entropy loss for binary or multi-label classification
* negative log-likelihood loss for regular classification
In case of multi-tasking, it will default to summing loss of each task.
If "weight" attribute exists, batch[self.weight] is also passed to the loss function
during training (but has no effect in validation).
@@ -358,18 +329,14 @@ class Task(pl.LightningDataModule):
Returns
-------
loss : {str: torch.tensor}
{"loss": loss} with additional "loss_{task_name}" keys for multi-task models.
{"loss": loss}
"""
# forward pass
y_pred = self.model(batch["X"])
if self.is_multi_task:
_, any_y_pred = next(iter(y_pred.items()))
batch_size, num_frames, _ = any_y_pred.shape
else:
batch_size, num_frames, _ = y_pred.shape
# (batch_size, num_frames, num_classes)
batch_size, num_frames, _ = y_pred.shape
# (batch_size, num_frames, num_classes)
# target
y = batch["y"]
@@ -388,34 +355,7 @@ class Task(pl.LightningDataModule):
warm_up_right = round(self.warm_up[1] / self.duration * num_frames)
weight[:, num_frames - warm_up_right :] = 0.0
# compute multi-task loss as the sum of loss of each task
if self.is_multi_task:
loss = dict()
for task_name, specifications in self.specifications.items():
loss[task_name] = self.default_loss(
specifications, y[task_name], y_pred[task_name], weight=weight
)
self.model.log(
f"{task_name}@{stage}_loss",
loss[task_name],
on_step=False,
on_epoch=True,
prog_bar=False,
logger=False,
)
loss["loss"] = sum(loss.values())
self.model.log(
f"{self.ACRONYM}@{stage}_loss",
loss["loss"],
on_step=False,
on_epoch=True,
prog_bar=True,
logger=True,
)
return loss
# compute mono-task loss
# compute loss
loss = self.default_loss(self.specifications, y, y_pred, weight=weight)
self.model.log(
f"{self.ACRONYM}@{stage}_loss",
@@ -1,236 +0,0 @@
# MIT License
#
# Copyright (c) 2020 CNRS
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from functools import cached_property
from typing import Optional
import torch
import torch.nn as nn
import torch.nn.functional as F
from einops import rearrange
from pyannote.audio.core.model import Model
from pyannote.audio.core.task import Task
from pyannote.audio.models.blocks.sincnet import SincNet
from pyannote.audio.utils.probe import probe
from pyannote.core.utils.generators import pairwise
# from pyannote.core.utils.params import merge_dict
class Branch(nn.Module):
"""Branch"""
LSTM_DEFAULTS = {"hidden_size": 128, "num_layers": 1, "bidirectional": True}
LINEAR_DEFAULTS = {"hidden_size": 128, "num_layers": 1}
def __init__(
self,
in_features,
lstm: dict = None,
linear: dict = None,
head: int = None,
):
super().__init__()
self.lstm_hparams = dict(**self.LSTM_DEFAULTS)
if lstm is not None:
self.lstm_hparams.update(**lstm)
self.lstm_hparams["batch_first"] = True # this is not negotiable
self.linear_hparams = dict(**self.LINEAR_DEFAULTS)
if linear is not None:
self.linear_hparams.update(**linear)
self.lstm = nn.ModuleList()
one_lstm_hparams = dict(self.lstm_hparams)
one_lstm_hparams["num_layers"] = 1
for i in range(self.lstm_hparams["num_layers"]):
self.lstm.append(nn.LSTM(in_features, **one_lstm_hparams))
in_features = self.lstm_hparams["hidden_size"] * (
2 if self.lstm_hparams["bidirectional"] else 1
)
lstm_out_features = in_features
self.linear = nn.ModuleList(
[
nn.Linear(in_features, out_features)
for in_features, out_features in pairwise(
[
lstm_out_features,
]
+ [self.linear_hparams["hidden_size"]]
* self.linear_hparams["num_layers"]
)
]
)
def forward(self, frames: torch.Tensor) -> torch.Tensor:
"""Pass forward
Parameters
----------
frames : (batch, ???, ???)
"""
outputs = frames
for i, lstm in enumerate(self.lstm):
if i == 0:
outputs, hidden = lstm(outputs)
else:
outputs, hidden = lstm(outputs, hidden)
if len(self.linear) > 0:
for linear in self.linear:
outputs = F.leaky_relu(linear(outputs))
return outputs
@cached_property
def out_features(self) -> int:
if len(self.linear) > 0:
return self.linear_hparams["hidden_size"]
return self.lstm_hparams["hidden_size"] * (
2 if self.lstm_hparams["bidirectional"] else 1
)
class Trunk(Branch):
def __init__(self, in_features, lstm: dict = None):
no_linear = {"num_layers": 0}
super().__init__(in_features, lstm=lstm, linear=no_linear)
class MultiPyanNet(Model):
"""PyanNet segmentation model
SincFilterbank > Conv1 > Conv1d > LSTM > Feed forward > Classifier
Parameters
----------
sample_rate : int, optional
Audio sample rate. Defaults to 16kHz (16000).
num_channels : int, optional
Number of channels. Defaults to mono (1).
sincnet : dict, optional
Keyword arugments passed to the SincNet block.
Defaults to {"stride": 1}.
trunk : dict, optional
branches : dict, optional
"""
SINCNET_DEFAULTS = {"stride": 1}
TRUNK_DEFAULTS = {
"lstm": {"num_layers": 3, "hidden_size": 128, "bidirectional": True},
}
BRANCH_DEFAULTS = {
"lstm": {"num_layers": 1, "hidden_size": 128, "bidirectional": True},
"linear": {"num_layers": 1, "hidden_size": 128},
}
BRANCHES_DEFAULTS = {
"vad": {"head": 0, **BRANCH_DEFAULTS},
"scd": {"head": 1, **BRANCH_DEFAULTS},
"osd": {"head": 2, **BRANCH_DEFAULTS},
}
def __init__(
self,
sample_rate: int = 16000,
num_channels: int = 1,
sincnet: dict = None,
trunk: dict = None,
branches: dict = None,
task: Optional[Task] = None,
):
super().__init__(sample_rate=sample_rate, num_channels=num_channels, task=task)
sincnet_hparams = dict(**self.SINCNET_DEFAULTS)
if sincnet is not None:
sincnet_hparams.update(**sincnet)
sincnet_hparams["sample_rate"] = sample_rate
self.hparams.sincnet = sincnet_hparams
self.sincnet = SincNet(**self.hparams.sincnet)
if trunk is None:
trunk = self.TRUNK_DEFAULTS
self.hparams.trunk = trunk
if branches is None:
branches = self.BRANCHES_DEFAULTS
self.hparams.branches = branches
self.trunk = Trunk(60, **self.hparams.trunk)
def build(self):
probes = dict()
self.branches = nn.ModuleDict()
self.classifiers = nn.ModuleDict()
for task_name, specifications in self.specifications.items():
branch_hparams = self.hparams.branches[task_name]
probes[task_name] = f"lstm.{branch_hparams['head']:d}"
branch = Branch(self.trunk.out_features, **branch_hparams)
self.branches[task_name] = branch
self.classifiers[task_name] = nn.Linear(
branch.out_features, len(specifications.classes)
)
# probe trunk
_ = probe(self.trunk, probes)
self.activation: dict = self.default_activation()
def forward(self, waveforms: torch.Tensor) -> torch.Tensor:
"""Pass forward
Parameters
----------
waveforms : (batch, channel, sample)
Returns
-------
scores : {task: (batch, frame, classes) scores} dict
"""
frames = self.sincnet(waveforms)
outputs: dict = self.trunk(
rearrange(frames, "batch feature frame -> batch frame feature")
)
return {
task_name: self.activation[task_name](
self.classifiers[task_name](
self.branches[task_name](outputs[task_name][0])
)
)
for task_name in self.specifications
}
@@ -80,52 +80,3 @@ class SimpleSegmentationModel(Model):
output, hidden = self.lstm(rearrange(mfcc, "b c f t -> b t (c f)"))
# apply the final classifier to get logits
return self.activation(self.classifier(output))
class MultiTaskSegmentationModel(Model):
def __init__(
self,
sample_rate: int = 16000,
num_channels: int = 1,
task: Optional[Task] = None,
):
super().__init__(sample_rate=sample_rate, num_channels=num_channels, task=task)
self.mfcc = MFCC(
sample_rate=self.hparams.sample_rate,
n_mfcc=40,
dct_type=2,
norm="ortho",
log_mels=False,
)
self.lstm = nn.LSTM(
self.mfcc.n_mfcc * self.hparams.num_channels,
32,
num_layers=1,
batch_first=True,
bidirectional=True,
)
def build(self):
self.classifier = nn.ModuleDict(
{
name: nn.Linear(32 * 2, len(specifications.classes))
for name, specifications in self.specifications.items()
}
)
self.activation = self.default_activation()
def forward(self, waveforms: torch.Tensor) -> torch.Tensor:
# extract MFCC
mfcc = self.mfcc(waveforms)
# pass MFCC sequence into the recurrent layer
output, hidden = self.lstm(rearrange(mfcc, "b c f t -> b t (c f)"))
return {
name: self.activation[name](self.classifier[name](output))
for name in self.specifications
}
+1 -8
View File
@@ -208,14 +208,7 @@ class SegmentationTaskMixin:
# use model introspection to predict how many frames it will output
num_samples = sample["X"].shape[1]
introspection = self.model.introspection
if self.is_multi_task:
# this assumes that all tasks share the same model introspection.
# this is a reasonable assumption for now.
any_task = next(iter(introspection.keys()))
num_frames, _ = introspection[any_task](num_samples)
else:
num_frames, _ = introspection(num_samples)
num_frames, _ = self.model.introspection(num_samples)
# crop "annotation" and keep track of corresponding list of labels if needed
annotation: Annotation = file["annotation"].crop(chunk)
@@ -1,264 +0,0 @@
# MIT License
#
# Copyright (c) 2020-2021 CNRS
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from typing import Mapping, Text
import numpy as np
from pytorch_lightning.metrics.classification import FBeta
from torch_audiomentations.core.transforms_interface import BaseWaveformTransform
from pyannote.audio.core.task import Problem, Task
from pyannote.audio.tasks import (
OverlappedSpeechDetection,
SpeakerChangeDetection,
VoiceActivityDetection,
)
from pyannote.audio.tasks.segmentation.mixins import SegmentationTaskMixin
from pyannote.database import Protocol
class MultiTaskSegmentation(SegmentationTaskMixin, Task):
"""Multi-task segmentation
Multi-task training of segmentation tasks, including:
- vad: voice activity detection
- scd: speaker change detection
- osd: overlapped speech detection
Note that, when "osd" is one of the tasks considered, data augmentation is used
to artificially increase the proportion of "overlap". This is achieved by
generating chunks made out of the (weighted) sum of two random chunks.
Parameters
----------
protocol : Protocol
pyannote.database protocol
duration : float, optional
Chunks duration. Defaults to 2s.
balance: str, optional
When provided, training samples are sampled uniformly with respect to that key.
For instance, setting `balance` to "uri" will make sure that each file will be
equally represented in the training samples.
overlap: dict, optional
Controls how artificial chunks with overlapping speech are generated:
- "probability" key is the probability of artificial overlapping chunks. Setting
"probability" to 0.6 means that, on average, 40% of training chunks are "real"
chunks, while 60% are artifical chunks made out of the (weighted) sum of two
chunks. Defaults to 0.5.
- "snr_min" and "snr_max" keys control the minimum and maximum signal-to-noise
ratio between summed chunks, in dB. Default to 0.0 and 10.
weight: str, optional
When provided, use this key to as frame-wise weight in loss function.
vad : bool, optional
Add voice activity detection in the pool of tasks.
Defaults to False.
vad_params : dict, optional
Additional vad-specific parameters. Has no effect when `vad` is False.
See VoiceActivityDetection docstring for details and default value.
scd : bool, optional
Add speaker change detection to the pool of tasks.
Defaults to False.
scd_params : dict, optional
Additional scd-specific parameters. Has no effect when `scd` is False.
See SpeakerChangeDetection docstring for details and default value.
osd : bool, optional
Add overlapped speech detection to the pool of tasks.
Defaults to False.
osd_params : dict, optional
Additional osd-specific parameters. Has no effect when `osd` is False.
See OverlappedSpeechDetection docstring for details and default value.
batch_size : int, optional
Number of training samples per batch. Defaults to 32.
num_workers : int, optional
Number of workers used for generating training samples.
Defaults to multiprocessing.cpu_count() // 2.
pin_memory : bool, optional
If True, data loaders will copy tensors into CUDA pinned
memory before returning them. See pytorch documentation
for more details. Defaults to False.
augmentation : BaseWaveformTransform, optional
torch_audiomentations waveform transform, used by dataloader
during training.
"""
ACRONYM = "xseg"
OVERLAP_DEFAULTS = {"probability": 0.5, "snr_min": 0.0, "snr_max": 10.0}
def __init__(
self,
protocol: Protocol,
duration: float = 2.0,
overlap: dict = OVERLAP_DEFAULTS,
balance: Text = None,
weight: Text = None,
vad: bool = False,
vad_params: Mapping = None,
scd: bool = False,
scd_params: Mapping = None,
osd: bool = False,
osd_params: Mapping = None,
batch_size: int = 32,
num_workers: int = None,
pin_memory: bool = False,
augmentation: BaseWaveformTransform = None,
):
super().__init__(
protocol,
duration=duration,
batch_size=batch_size,
num_workers=num_workers,
pin_memory=pin_memory,
augmentation=augmentation,
)
self.vad = vad
self.vad_params = dict() if vad_params is None else vad_params
self.scd = scd
self.scd_params = dict() if scd_params is None else scd_params
self.osd = osd
self.osd_params = dict() if osd_params is None else osd_params
if sum([vad, scd, osd]) < 2:
raise ValueError(
"You must activate at least two tasks among 'vad', 'scd', and 'osd'."
)
self.tasks = dict()
if self.vad:
self.tasks["vad"] = VoiceActivityDetection(
protocol,
duration=duration,
batch_size=batch_size,
num_workers=num_workers,
pin_memory=pin_memory,
**self.vad_params,
)
if self.scd:
self.tasks["scd"] = SpeakerChangeDetection(
protocol,
duration=duration,
batch_size=batch_size,
num_workers=num_workers,
pin_memory=pin_memory,
**self.scd_params,
)
if self.osd:
self.tasks["osd"] = OverlappedSpeechDetection(
protocol,
duration=duration,
batch_size=batch_size,
num_workers=num_workers,
pin_memory=pin_memory,
**self.osd_params,
)
def setup(self, stage=None):
super().setup(stage=stage)
if stage == "fit":
self.specifications = {
name: task.specifications for name, task in self.tasks.items()
}
def prepare_y(self, one_hot_y: np.ndarray):
"""Get multi-task targets
Parameters
----------
one_hot_y : (num_frames, num_speakers) np.ndarray
One-hot-encoding of current chunk speaker activity:
* one_hot_y[t, k] = 1 if kth speaker is active at tth frame
* one_hot_y[t, k] = 0 otherwise.
Returns
-------
y : (num_frames, ) np.ndarray
y[t] = 1 if there is two or more active speakers at tth frame, 0 otherwise.
"""
return {name: task.prepare_y(one_hot_y) for name, task in self.tasks.items()}
def setup_validation_metric(self):
self.val_fbeta = {
task_name: FBeta(
len(specifications.classes),
beta=1.0,
threshold=0.5,
multilabel=(
specifications.problem == Problem.MULTI_LABEL_CLASSIFICATION
),
average="macro",
)
for task_name, specifications in self.specifications.items()
}
def validation_step(self, batch, batch_idx: int):
"""Compute areas under ROC curve
Parameters
----------
batch : dict of torch.Tensor
Current batch.
batch_idx: int
Batch index.
"""
X, y = batch["X"], batch["y"]
y_pred = self.model(X)
val_fbeta = dict()
for task_name in self.specifications:
# move metric to model device
self.val_fbeta[task_name].to(self.model.device)
val_fbeta[task_name] = self.val_fbeta[task_name](
y_pred[task_name][:, ::10].squeeze(), y[task_name][:, ::10].squeeze()
)
self.model.log(
f"{task_name}@val_fbeta",
val_fbeta[task_name],
on_step=False,
on_epoch=True,
prog_bar=True,
logger=True,
)
self.model.log(
f"{self.ACRONYM}@val_fbeta",
sum(val_fbeta.values()) / len(val_fbeta),
on_step=False,
on_epoch=True,
prog_bar=True,
logger=True,
)
@@ -305,7 +305,7 @@ class Segmentation(SegmentationTaskMixin, Task):
Returns
-------
loss : {str: torch.tensor}
{"loss": loss} with additional "loss_{task_name}" keys for multi-task models.
{"loss": loss}
"""
# forward pass
+1 -34
View File
@@ -4,14 +4,8 @@ import pytorch_lightning as pl
from pyannote.audio import Inference, Model
from pyannote.audio.core.task import Resolution
from pyannote.audio.models.segmentation.debug import (
MultiTaskSegmentationModel,
SimpleSegmentationModel,
)
from pyannote.audio.models.segmentation.debug import SimpleSegmentationModel
from pyannote.audio.tasks import VoiceActivityDetection
from pyannote.audio.tasks.segmentation.multi_task_segmentation import (
MultiTaskSegmentation,
)
from pyannote.core import SlidingWindowFeature
from pyannote.database import FileFinder, get_protocol
@@ -96,33 +90,6 @@ def test_on_file_path(trained):
assert isinstance(output, SlidingWindowFeature)
def test_multi_seg_infer():
protocol = get_protocol(
"Debug.SpeakerDiarization.Debug", preprocessors={"audio": FileFinder()}
)
xseg = MultiTaskSegmentation(
protocol,
duration=2.0,
vad=True,
scd=True,
osd=True,
batch_size=32,
num_workers=4,
)
model = MultiTaskSegmentationModel(task=xseg)
trainer = pl.Trainer(max_epochs=1, fast_dev_run=True)
_ = trainer.fit(model, xseg)
inference = Inference(model, duration=2.0, step=0.5)
dev_file = next(protocol.development())
scores = inference(dev_file)
assert isinstance(scores, dict)
for attr in ["vad", "scd", "osd"]:
assert attr in scores
assert isinstance(scores[attr], SlidingWindowFeature)
def test_skip_aggregation(pretrained_model, dev_file):
inference = Inference(pretrained_model, skip_aggregation=True)