mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
881 lines
32 KiB
Python
881 lines
32 KiB
Python
# MIT License
|
||
#
|
||
# Copyright (c) 2020- CNRS
|
||
#
|
||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||
# of this software and associated documentation files (the "Software"), to deal
|
||
# in the Software without restriction, including without limitation the rights
|
||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||
# copies of the Software, and to permit persons to whom the Software is
|
||
# furnished to do so, subject to the following conditions:
|
||
#
|
||
# The above copyright notice and this permission notice shall be included in all
|
||
# copies or substantial portions of the Software.
|
||
#
|
||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||
# SOFTWARE.
|
||
|
||
|
||
from __future__ import annotations
|
||
|
||
import itertools
|
||
import multiprocessing
|
||
import sys
|
||
import warnings
|
||
from collections import defaultdict
|
||
from dataclasses import dataclass
|
||
from enum import Enum
|
||
from functools import cached_property, partial
|
||
from numbers import Number
|
||
from pathlib import Path
|
||
from tempfile import mkstemp
|
||
from typing import Dict, List, Literal, Optional, Sequence, Text, Tuple, Union
|
||
|
||
import lightning.pytorch as pl
|
||
import numpy as np
|
||
import scipy.special
|
||
import torch
|
||
from pyannote.database import Protocol
|
||
from pyannote.database.protocol.protocol import Scope, Subset
|
||
from torch.utils.data import DataLoader, Dataset, IterableDataset
|
||
from torch_audiomentations import Identity
|
||
from torch_audiomentations.core.transforms_interface import BaseWaveformTransform
|
||
from torchmetrics import Metric, MetricCollection
|
||
|
||
from pyannote.audio.utils.loss import binary_cross_entropy, nll_loss
|
||
from pyannote.audio.utils.protocol import check_protocol
|
||
|
||
Subsets = list(Subset.__args__)
|
||
Scopes = list(Scope.__args__)
|
||
|
||
|
||
# Type of machine learning problem
|
||
class Problem(Enum):
|
||
BINARY_CLASSIFICATION = 0
|
||
MONO_LABEL_CLASSIFICATION = 1
|
||
MULTI_LABEL_CLASSIFICATION = 2
|
||
REPRESENTATION = 3
|
||
REGRESSION = 4
|
||
# any other we could think of?
|
||
|
||
|
||
# A task takes an audio chunk as input and returns
|
||
# either a temporal sequence of predictions
|
||
# or just one prediction for the whole audio chunk
|
||
class Resolution(Enum):
|
||
FRAME = 1 # model outputs a sequence of frames
|
||
CHUNK = 2 # model outputs just one vector for the whole chunk
|
||
|
||
|
||
class UnknownSpecificationsError(Exception):
|
||
pass
|
||
|
||
|
||
@dataclass
|
||
class Specifications:
|
||
problem: Problem
|
||
resolution: Resolution
|
||
|
||
# (maximum) chunk duration in seconds
|
||
duration: float
|
||
|
||
# (for variable-duration tasks only) minimum chunk duration in seconds
|
||
min_duration: Optional[float] = None
|
||
|
||
# use that many seconds on the left- and rightmost parts of each chunk
|
||
# to warm up the model. This is mostly useful for segmentation tasks.
|
||
# While the model does process those left- and right-most parts, only
|
||
# the remaining central part of each chunk is used for computing the
|
||
# loss during training, and for aggregating scores during inference.
|
||
# Defaults to 0. (i.e. no warm-up).
|
||
warm_up: Optional[Tuple[float, float]] = (0.0, 0.0)
|
||
|
||
# (for classification tasks only) list of classes
|
||
classes: Optional[List[Text]] = None
|
||
|
||
# (for powerset only) max number of simultaneous classes
|
||
# (n choose k with k <= powerset_max_classes)
|
||
powerset_max_classes: Optional[int] = None
|
||
|
||
# whether classes are permutation-invariant (e.g. diarization)
|
||
permutation_invariant: bool = False
|
||
|
||
@cached_property
|
||
def powerset(self) -> bool:
|
||
if self.powerset_max_classes is None:
|
||
return False
|
||
|
||
if self.problem != Problem.MONO_LABEL_CLASSIFICATION:
|
||
raise ValueError(
|
||
"`powerset_max_classes` only makes sense with multi-class classification problems."
|
||
)
|
||
|
||
return True
|
||
|
||
@cached_property
|
||
def num_powerset_classes(self) -> int:
|
||
# compute number of subsets of size at most "powerset_max_classes"
|
||
# e.g. with len(classes) = 3 and powerset_max_classes = 2:
|
||
# {}, {0}, {1}, {2}, {0, 1}, {0, 2}, {1, 2}
|
||
return int(
|
||
sum(
|
||
scipy.special.binom(len(self.classes), i)
|
||
for i in range(0, self.powerset_max_classes + 1)
|
||
)
|
||
)
|
||
|
||
def __len__(self):
|
||
return 1
|
||
|
||
def __iter__(self):
|
||
yield self
|
||
|
||
|
||
class TrainDataset(IterableDataset):
|
||
def __init__(self, task: Task):
|
||
super().__init__()
|
||
self.task = task
|
||
|
||
def __iter__(self):
|
||
return self.task.train__iter__()
|
||
|
||
def __len__(self):
|
||
return self.task.train__len__()
|
||
|
||
|
||
class ValDataset(Dataset):
|
||
def __init__(self, task: Task):
|
||
super().__init__()
|
||
self.task = task
|
||
|
||
def __getitem__(self, idx):
|
||
return self.task.val__getitem__(idx)
|
||
|
||
def __len__(self):
|
||
return self.task.val__len__()
|
||
|
||
|
||
def get_dtype(value: int) -> str:
|
||
"""Return the most suitable type for storing the
|
||
value passed in parameter in memory.
|
||
|
||
Parameters
|
||
----------
|
||
value: int
|
||
value whose type is best suited to storage in memory
|
||
|
||
Returns
|
||
-------
|
||
str:
|
||
numpy formatted type
|
||
(see https://numpy.org/doc/stable/reference/arrays.dtypes.html)
|
||
"""
|
||
# signe byte (8 bits), signed short (16 bits), signed int (32 bits):
|
||
types_list = [(127, "b"), (32_768, "i2"), (2_147_483_648, "i")]
|
||
filtered_list = [
|
||
(max_val, type) for max_val, type in types_list if max_val > abs(value)
|
||
]
|
||
if not filtered_list:
|
||
return "i8" # signed long (64 bits)
|
||
return filtered_list[0][1]
|
||
|
||
|
||
class Task(pl.LightningDataModule):
|
||
"""Base task class
|
||
|
||
A task is the combination of a "problem" and a "dataset".
|
||
For example, here are a few tasks:
|
||
- voice activity detection on the AMI corpus
|
||
- speaker embedding on the VoxCeleb corpus
|
||
- end-to-end speaker diarization on the VoxConverse corpus
|
||
|
||
A task is expected to be solved by a "model" that takes an
|
||
audio chunk as input and returns the solution. Hence, the
|
||
task is in charge of generating (input, expected_output)
|
||
samples used for training the model.
|
||
|
||
Parameters
|
||
----------
|
||
protocol : Protocol
|
||
pyannote.database protocol
|
||
cache : str, optional
|
||
As (meta-)data preparation might take a very long time for large datasets,
|
||
it can be cached to disk for later (and faster!) re-use.
|
||
When `cache` does not exist, `Task.prepare_data()` generates training
|
||
and validation metadata from `protocol` and save them to disk.
|
||
When `cache` exists, `Task.prepare_data()` is skipped and (meta)-data
|
||
are loaded from disk. Defaults to a temporary path.
|
||
duration : float, optional
|
||
Chunks duration in seconds. Defaults to two seconds (2.).
|
||
min_duration : float, optional
|
||
Sample training chunks duration uniformely between `min_duration`
|
||
and `duration`. Defaults to `duration` (i.e. fixed length chunks).
|
||
warm_up : float or (float, float), optional
|
||
Use that many seconds on the left- and rightmost parts of each chunk
|
||
to warm up the model. This is mostly useful for segmentation tasks.
|
||
While the model does process those left- and right-most parts, only
|
||
the remaining central part of each chunk is used for computing the
|
||
loss during training, and for aggregating scores during inference.
|
||
Defaults to 0. (i.e. no warm-up).
|
||
batch_size : int, optional
|
||
Number of training samples per batch. Defaults to 32.
|
||
num_workers : int, optional
|
||
Number of workers used for generating training samples.
|
||
Defaults to multiprocessing.cpu_count() // 2.
|
||
pin_memory : bool, optional
|
||
If True, data loaders will copy tensors into CUDA pinned
|
||
memory before returning them. See pytorch documentation
|
||
for more details. Defaults to False.
|
||
augmentation : BaseWaveformTransform, optional
|
||
torch_audiomentations waveform transform, used by dataloader
|
||
during training.
|
||
metric : optional
|
||
Validation metric(s). Can be anything supported by torchmetrics.MetricCollection.
|
||
Defaults to value returned by `default_metric` method.
|
||
|
||
Attributes
|
||
----------
|
||
specifications : Specifications or tuple of Specifications
|
||
Task specifications (available after `Task.setup` has been called.)
|
||
|
||
"""
|
||
|
||
def __init__(
|
||
self,
|
||
protocol: Protocol,
|
||
cache: Optional[Union[str, None]] = None,
|
||
duration: float = 2.0,
|
||
min_duration: Optional[float] = None,
|
||
warm_up: Union[float, Tuple[float, float]] = 0.0,
|
||
batch_size: int = 32,
|
||
num_workers: Optional[int] = None,
|
||
pin_memory: bool = False,
|
||
augmentation: Optional[BaseWaveformTransform] = None,
|
||
metric: Union[Metric, Sequence[Metric], Dict[str, Metric]] = None,
|
||
):
|
||
super().__init__()
|
||
|
||
# dataset
|
||
self.protocol, checks = check_protocol(protocol)
|
||
self.has_validation = checks["has_validation"]
|
||
self.has_scope = checks["has_scope"]
|
||
if not self.has_scope:
|
||
raise ValueError(
|
||
"Protocol must provide 'scope' information (e.g. 'file', 'database', or 'global')."
|
||
)
|
||
|
||
self.has_classes = checks["has_classes"]
|
||
|
||
# metadata cache
|
||
self.cache = Path(cache) if cache else cache
|
||
|
||
# batching
|
||
self.duration = duration
|
||
self.min_duration = duration if min_duration is None else min_duration
|
||
self.batch_size = batch_size
|
||
|
||
# training
|
||
if isinstance(warm_up, Number):
|
||
warm_up = (warm_up, warm_up)
|
||
self.warm_up = warm_up
|
||
|
||
# multi-processing
|
||
if num_workers is None:
|
||
num_workers = multiprocessing.cpu_count() // 2
|
||
|
||
if (
|
||
num_workers > 0
|
||
and sys.platform == "darwin"
|
||
and sys.version_info[0] >= 3
|
||
and sys.version_info[1] >= 8
|
||
):
|
||
warnings.warn(
|
||
"num_workers > 0 is not supported with macOS and Python 3.8+: "
|
||
"setting num_workers = 0."
|
||
)
|
||
num_workers = 0
|
||
|
||
self.num_workers = num_workers
|
||
self.pin_memory = pin_memory
|
||
self.augmentation = augmentation or Identity(output_type="dict")
|
||
self._metric = metric
|
||
|
||
def prepare_data(self):
|
||
"""Use this to prepare data from task protocol
|
||
|
||
Notes
|
||
-----
|
||
Called only once on the main process (and only on it), for global_rank 0.
|
||
|
||
After this method is called, the task should have a `prepared_data` attribute
|
||
with the following dictionary structure:
|
||
|
||
prepared_data = {
|
||
'protocol': name of the protocol
|
||
'audio-path': array of N paths to audio
|
||
'audio-metadata': array of N audio infos such as audio subset, scope and database
|
||
'audio-info': array of N audio torchaudio.info struct
|
||
'audio-encoding': array of N audio encodings
|
||
'audio-annotated': array of N annotated duration (usually equals file duration but might be shorter if file is not fully annotated)
|
||
'annotations-regions': array of M annotated regions
|
||
'annotations-segments': array of M' annotated segments
|
||
'metadata-values': dict of lists of values for subset, scope and database
|
||
'metadata-`database-name`-labels': array of `database-name` labels. Each database with "database" scope labels has it own array.
|
||
'metadata-labels': array of global scope labels
|
||
}
|
||
|
||
"""
|
||
|
||
if self.cache:
|
||
# check if cache exists and is not empty:
|
||
if self.cache.exists() and self.cache.stat().st_size > 0:
|
||
# data was already created, nothing to do
|
||
return
|
||
# create parent directory if needed
|
||
self.cache.parent.mkdir(parents=True, exist_ok=True)
|
||
else:
|
||
# if no cache was provided by user, create a temporary file
|
||
# in system directory used for temp files
|
||
self.cache = Path(mkstemp()[1])
|
||
|
||
# list of possible values for each metadata key
|
||
# (will become .prepared_data[""])
|
||
metadata_unique_values = defaultdict(list)
|
||
metadata_unique_values["subset"] = Subsets
|
||
metadata_unique_values["scope"] = Scopes
|
||
|
||
audios = list() # list of path to audio files
|
||
audio_infos = list()
|
||
audio_encodings = list()
|
||
metadata = list() # list of metadata
|
||
|
||
annotated_duration = list() # total duration of annotated regions (per file)
|
||
annotated_regions = list() # annotated regions
|
||
annotations = list() # actual annotations
|
||
unique_labels = list()
|
||
database_unique_labels = {}
|
||
|
||
if self.has_validation:
|
||
files_iter = itertools.chain(
|
||
zip(itertools.repeat("train"), self.protocol.train()),
|
||
zip(itertools.repeat("development"), self.protocol.development()),
|
||
)
|
||
else:
|
||
files_iter = zip(itertools.repeat("train"), self.protocol.train())
|
||
|
||
for file_id, (subset, file) in enumerate(files_iter):
|
||
# gather metadata and update metadata_unique_values so that each metadatum
|
||
# (e.g. source database or label) is represented by an integer.
|
||
metadatum = dict()
|
||
|
||
# keep track of source database and subset (train, development, or test)
|
||
if file["database"] not in metadata_unique_values["database"]:
|
||
metadata_unique_values["database"].append(file["database"])
|
||
metadatum["database"] = metadata_unique_values["database"].index(
|
||
file["database"]
|
||
)
|
||
|
||
metadatum["subset"] = Subsets.index(subset)
|
||
|
||
# keep track of label scope (file, database, or global)
|
||
metadatum["scope"] = Scopes.index(file["scope"])
|
||
|
||
remaining_metadata_keys = set(file) - set(
|
||
[
|
||
"uri",
|
||
"database",
|
||
"subset",
|
||
"audio",
|
||
"torchaudio.info",
|
||
"scope",
|
||
"classes",
|
||
"annotation",
|
||
"annotated",
|
||
]
|
||
)
|
||
|
||
# keep track of any other (integer or string) metadata provided by the protocol
|
||
# (e.g. a "domain" key for domain-adversarial training)
|
||
for key in remaining_metadata_keys:
|
||
value = file[key]
|
||
|
||
if isinstance(value, str):
|
||
if value not in metadata_unique_values[key]:
|
||
metadata_unique_values[key].append(value)
|
||
metadatum[key] = metadata_unique_values[key].index(value)
|
||
|
||
elif isinstance(value, int):
|
||
metadatum[key] = value
|
||
|
||
else:
|
||
warnings.warn(
|
||
f"Ignoring '{key}' metadata because of its type ({type(value)}). Only str and int are supported for now.",
|
||
category=UserWarning,
|
||
)
|
||
|
||
metadata.append(metadatum)
|
||
|
||
# reset list of file-scoped labels
|
||
file_unique_labels = list()
|
||
|
||
# path to audio file
|
||
audios.append(str(file["audio"]))
|
||
|
||
# audio info
|
||
audio_info = file["torchaudio.info"]
|
||
audio_infos.append(
|
||
(
|
||
audio_info.sample_rate, # sample rate
|
||
audio_info.num_frames, # number of frames
|
||
audio_info.num_channels, # number of channels
|
||
audio_info.bits_per_sample, # bits per sample
|
||
)
|
||
)
|
||
audio_encodings.append(audio_info.encoding) # encoding
|
||
|
||
# annotated regions and duration
|
||
_annotated_duration = 0.0
|
||
for segment in file["annotated"]:
|
||
# skip annotated regions that are shorter than training chunk duration
|
||
if segment.duration < self.duration:
|
||
continue
|
||
|
||
# append annotated region
|
||
annotated_region = (
|
||
file_id,
|
||
segment.duration,
|
||
segment.start,
|
||
)
|
||
annotated_regions.append(annotated_region)
|
||
|
||
# increment annotated duration
|
||
_annotated_duration += segment.duration
|
||
|
||
# append annotated duration
|
||
annotated_duration.append(_annotated_duration)
|
||
|
||
# annotations
|
||
for segment, _, label in file["annotation"].itertracks(yield_label=True):
|
||
# "scope" is provided by speaker diarization protocols to indicate
|
||
# whether speaker labels are local to the file ('file'), consistent across
|
||
# all files in a database ('database'), or globally consistent ('global')
|
||
|
||
# 0 = 'file' / 1 = 'database' / 2 = 'global'
|
||
scope = Scopes.index(file["scope"])
|
||
|
||
# update list of file-scope labels
|
||
if label not in file_unique_labels:
|
||
file_unique_labels.append(label)
|
||
# and convert label to its (file-scope) index
|
||
file_label_idx = file_unique_labels.index(label)
|
||
|
||
database_label_idx = global_label_idx = -1
|
||
|
||
if scope > 0: # 'database' or 'global'
|
||
# update list of database-scope labels
|
||
database = file["database"]
|
||
if database not in database_unique_labels:
|
||
database_unique_labels[database] = []
|
||
if label not in database_unique_labels[database]:
|
||
database_unique_labels[database].append(label)
|
||
|
||
# and convert label to its (database-scope) index
|
||
database_label_idx = database_unique_labels[database].index(label)
|
||
|
||
if scope > 1: # 'global'
|
||
# update list of global-scope labels
|
||
if label not in unique_labels:
|
||
unique_labels.append(label)
|
||
# and convert label to its (global-scope) index
|
||
global_label_idx = unique_labels.index(label)
|
||
|
||
annotations.append(
|
||
(
|
||
file_id, # index of file
|
||
segment.start, # start time
|
||
segment.end, # end time
|
||
file_label_idx, # file-scope label index
|
||
database_label_idx, # database-scope label index
|
||
global_label_idx, # global-scope index
|
||
)
|
||
)
|
||
|
||
# since not all metadata keys are present in all files, fallback to -1 when a key is missing
|
||
metadata = [
|
||
tuple(metadatum.get(key, -1) for key in metadata_unique_values)
|
||
for metadatum in metadata
|
||
]
|
||
metadata_dtype = [
|
||
(key, get_dtype(max(m[i] for m in metadata)))
|
||
for i, key in enumerate(metadata_unique_values)
|
||
]
|
||
|
||
# turn list of files metadata into a single numpy array
|
||
# TODO: improve using https://github.com/pytorch/pytorch/issues/13246#issuecomment-617140519
|
||
info_dtype = [
|
||
(
|
||
"sample_rate",
|
||
get_dtype(max(ai[0] for ai in audio_infos)),
|
||
),
|
||
(
|
||
"num_frames",
|
||
get_dtype(max(ai[1] for ai in audio_infos)),
|
||
),
|
||
("num_channels", "B"),
|
||
("bits_per_sample", "B"),
|
||
]
|
||
|
||
# turn list of annotated regions into a single numpy array
|
||
region_dtype = [
|
||
(
|
||
"file_id",
|
||
get_dtype(max(ar[0] for ar in annotated_regions)),
|
||
),
|
||
("duration", "f"),
|
||
("start", "f"),
|
||
]
|
||
|
||
# turn list of annotations into a single numpy array
|
||
segment_dtype = [
|
||
(
|
||
"file_id",
|
||
get_dtype(max(a[0] for a in annotations)),
|
||
),
|
||
("start", "f"),
|
||
("end", "f"),
|
||
("file_label_idx", get_dtype(max(a[3] for a in annotations))),
|
||
("database_label_idx", get_dtype(max(a[4] for a in annotations))),
|
||
("global_label_idx", get_dtype(max(a[5] for a in annotations))),
|
||
]
|
||
|
||
# save all protocol data in a dict
|
||
prepared_data = {}
|
||
|
||
# keep track of protocol name
|
||
prepared_data["protocol"] = self.protocol.name
|
||
|
||
prepared_data["audio-path"] = np.array(audios, dtype=np.str_)
|
||
audios.clear()
|
||
|
||
prepared_data["audio-metadata"] = np.array(metadata, dtype=metadata_dtype)
|
||
metadata.clear()
|
||
|
||
prepared_data["audio-info"] = np.array(audio_infos, dtype=info_dtype)
|
||
audio_infos.clear()
|
||
|
||
prepared_data["audio-encoding"] = np.array(audio_encodings, dtype=np.str_)
|
||
audio_encodings.clear()
|
||
|
||
prepared_data["audio-annotated"] = np.array(annotated_duration)
|
||
annotated_duration.clear()
|
||
|
||
prepared_data["annotations-regions"] = np.array(
|
||
annotated_regions, dtype=region_dtype
|
||
)
|
||
annotated_regions.clear()
|
||
|
||
prepared_data["annotations-segments"] = np.array(
|
||
annotations, dtype=segment_dtype
|
||
)
|
||
annotations.clear()
|
||
|
||
prepared_data["metadata-values"] = metadata_unique_values
|
||
|
||
for database, labels in database_unique_labels.items():
|
||
prepared_data[f"metadata-{database}-labels"] = np.array(
|
||
labels, dtype=np.str_
|
||
)
|
||
database_unique_labels.clear()
|
||
|
||
prepared_data["metadata-labels"] = np.array(unique_labels, dtype=np.str_)
|
||
unique_labels.clear()
|
||
|
||
if self.has_validation:
|
||
self.prepare_validation(prepared_data)
|
||
|
||
self.post_prepare_data(prepared_data)
|
||
|
||
# save prepared data on the disk
|
||
with open(self.cache, "wb") as cache_file:
|
||
np.savez_compressed(cache_file, **prepared_data)
|
||
|
||
def post_prepare_data(self, prepared_data: Dict):
|
||
"""Method for completing `prepared_data` with task-specific data.
|
||
For instance, for a classification task, this could be a list of
|
||
possible classes.
|
||
|
||
Parameters
|
||
----------
|
||
prepared_data: dict
|
||
dictionnary containing protocol data prepared by
|
||
`prepare_data()`
|
||
Note
|
||
----
|
||
This method does not return anything. Thus, user have to directly modify
|
||
`prepared_data`, for updates to be taken into account
|
||
"""
|
||
pass
|
||
|
||
def setup(self, stage=None):
|
||
"""Setup data cached by prepare_data into the task on each device"""
|
||
|
||
# send cache path on all processes used for the training,
|
||
# allowing them to access the cache generated by prepare_data
|
||
if stage == "fit":
|
||
self.cache = self.trainer.strategy.broadcast(self.cache)
|
||
|
||
try:
|
||
with open(self.cache, "rb") as cache_file:
|
||
self.prepared_data = dict(np.load(cache_file, allow_pickle=True))
|
||
except FileNotFoundError:
|
||
print(
|
||
"Cached data for protocol not found. Ensure that prepare_data() was called",
|
||
" and executed correctly or/and that the path to the task cache is correct.",
|
||
)
|
||
raise
|
||
|
||
# checks that the task current protocol matches the cached protocol
|
||
if self.protocol.name != self.prepared_data["protocol"]:
|
||
raise ValueError(
|
||
f"Protocol specified for the task ({self.protocol.name}) "
|
||
f"does not correspond to the cached one ({self.prepared_data['protocol']})"
|
||
)
|
||
|
||
@property
|
||
def automatic_optimization(self) -> bool:
|
||
return self.model.automatic_optimization
|
||
|
||
@automatic_optimization.setter
|
||
def automatic_optimization(self, automatic_optimisation: bool) -> None:
|
||
self.model.automatic_optimization = automatic_optimisation
|
||
|
||
@property
|
||
def specifications(self) -> Union[Specifications, Tuple[Specifications]]:
|
||
# setup metadata on-demand the first time specifications are requested and missing
|
||
if not hasattr(self, "_specifications"):
|
||
raise UnknownSpecificationsError(
|
||
"Task specifications are not available. This is most likely because they depend on "
|
||
"the content of the training subset. Use `task.prepare_data()` and `task.setup()` "
|
||
"to go over the training subset and fix this, or let lightning trainer do that for you in `trainer.fit(model)`."
|
||
)
|
||
return self._specifications
|
||
|
||
@specifications.setter
|
||
def specifications(
|
||
self, specifications: Union[Specifications, Tuple[Specifications]]
|
||
):
|
||
self._specifications = specifications
|
||
|
||
def setup_loss_func(self):
|
||
pass
|
||
|
||
def train__iter__(self):
|
||
# will become train_dataset.__iter__ method
|
||
msg = f"Missing '{self.__class__.__name__}.train__iter__' method."
|
||
raise NotImplementedError(msg)
|
||
|
||
def train__len__(self):
|
||
# will become train_dataset.__len__ method
|
||
msg = f"Missing '{self.__class__.__name__}.train__len__' method."
|
||
raise NotImplementedError(msg)
|
||
|
||
def collate_fn(self, batch, stage="train"):
|
||
msg = f"Missing '{self.__class__.__name__}.collate_fn' method."
|
||
raise NotImplementedError(msg)
|
||
|
||
def train_dataloader(self) -> DataLoader:
|
||
return DataLoader(
|
||
TrainDataset(self),
|
||
batch_size=self.batch_size,
|
||
num_workers=self.num_workers,
|
||
pin_memory=self.pin_memory,
|
||
drop_last=True,
|
||
collate_fn=partial(self.collate_fn, stage="train"),
|
||
)
|
||
|
||
def default_loss(
|
||
self, specifications: Specifications, target, prediction, weight=None
|
||
) -> torch.Tensor:
|
||
"""Guess and compute default loss according to task specification
|
||
|
||
Parameters
|
||
----------
|
||
specifications : Specifications
|
||
Task specifications
|
||
target : torch.Tensor
|
||
* (batch_size, num_frames) for binary classification
|
||
* (batch_size, num_frames) for multi-class classification
|
||
* (batch_size, num_frames, num_classes) for multi-label classification
|
||
prediction : torch.Tensor
|
||
(batch_size, num_frames, num_classes)
|
||
weight : torch.Tensor, optional
|
||
(batch_size, num_frames, 1)
|
||
|
||
Returns
|
||
-------
|
||
loss : torch.Tensor
|
||
Binary cross-entropy loss in case of binary and multi-label classification,
|
||
Negative log-likelihood loss in case of multi-class classification.
|
||
|
||
"""
|
||
|
||
if specifications.problem in [
|
||
Problem.BINARY_CLASSIFICATION,
|
||
Problem.MULTI_LABEL_CLASSIFICATION,
|
||
]:
|
||
return binary_cross_entropy(prediction, target, weight=weight)
|
||
|
||
elif specifications.problem in [Problem.MONO_LABEL_CLASSIFICATION]:
|
||
return nll_loss(prediction, target, weight=weight)
|
||
|
||
else:
|
||
msg = "TODO: implement for other types of problems"
|
||
raise NotImplementedError(msg)
|
||
|
||
def common_step(self, batch, batch_idx: int, stage: Literal["train", "val"]):
|
||
"""Default training or validation step according to task specification
|
||
|
||
* binary cross-entropy loss for binary or multi-label classification
|
||
* negative log-likelihood loss for regular classification
|
||
|
||
If "weight" attribute exists, batch[self.weight] is also passed to the loss function
|
||
during training (but has no effect in validation).
|
||
|
||
Parameters
|
||
----------
|
||
batch : (usually) dict of torch.Tensor
|
||
Current batch.
|
||
batch_idx: int
|
||
Batch index.
|
||
stage : {"train", "val"}
|
||
"train" for training step, "val" for validation step
|
||
|
||
Returns
|
||
-------
|
||
loss : {str: torch.tensor}
|
||
{"loss": loss}
|
||
"""
|
||
|
||
if isinstance(self.specifications, tuple):
|
||
raise NotImplementedError(
|
||
"Default training/validation step is not implemented for multi-task."
|
||
)
|
||
|
||
# forward pass
|
||
y_pred = self.model(batch["X"])
|
||
|
||
batch_size, num_frames, _ = y_pred.shape
|
||
# (batch_size, num_frames, num_classes)
|
||
|
||
# target
|
||
y = batch["y"]
|
||
|
||
# frames weight
|
||
weight_key = getattr(self, "weight", None) if stage == "train" else None
|
||
weight = batch.get(
|
||
weight_key,
|
||
torch.ones(batch_size, num_frames, 1, device=self.model.device),
|
||
)
|
||
# (batch_size, num_frames, 1)
|
||
|
||
# warm-up
|
||
warm_up_left = round(self.warm_up[0] / self.duration * num_frames)
|
||
weight[:, :warm_up_left] = 0.0
|
||
warm_up_right = round(self.warm_up[1] / self.duration * num_frames)
|
||
weight[:, num_frames - warm_up_right :] = 0.0
|
||
|
||
# compute loss
|
||
loss = self.default_loss(self.specifications, y, y_pred, weight=weight)
|
||
|
||
# skip batch if something went wrong for some reason
|
||
if torch.isnan(loss):
|
||
return None
|
||
|
||
self.model.log(
|
||
f"loss/{stage}",
|
||
loss,
|
||
on_step=False,
|
||
on_epoch=True,
|
||
prog_bar=False,
|
||
logger=True,
|
||
)
|
||
return {"loss": loss}
|
||
|
||
# default training_step provided for convenience
|
||
# can obviously be overriden for each task
|
||
def training_step(self, batch, batch_idx: int):
|
||
return self.common_step(batch, batch_idx, "train")
|
||
|
||
def val__getitem__(self, idx):
|
||
# will become val_dataset.__getitem__ method
|
||
msg = f"Missing '{self.__class__.__name__}.val__getitem__' method."
|
||
raise NotImplementedError(msg)
|
||
|
||
def val__len__(self):
|
||
# will become val_dataset.__len__ method
|
||
msg = f"Missing '{self.__class__.__name__}.val__len__' method."
|
||
raise NotImplementedError(msg)
|
||
|
||
def val_dataloader(self) -> Optional[DataLoader]:
|
||
if self.has_validation:
|
||
return DataLoader(
|
||
ValDataset(self),
|
||
batch_size=self.batch_size,
|
||
num_workers=self.num_workers,
|
||
pin_memory=self.pin_memory,
|
||
drop_last=False,
|
||
collate_fn=partial(self.collate_fn, stage="val"),
|
||
)
|
||
else:
|
||
return None
|
||
|
||
# default validation_step provided for convenience
|
||
# can obviously be overriden for each task
|
||
def validation_step(self, batch, batch_idx: int):
|
||
return self.common_step(batch, batch_idx, "val")
|
||
|
||
def default_metric(self) -> Union[Metric, Sequence[Metric], Dict[str, Metric]]:
|
||
"""Default validation metric"""
|
||
msg = f"Missing '{self.__class__.__name__}.default_metric' method."
|
||
raise NotImplementedError(msg)
|
||
|
||
@cached_property
|
||
def metric(self) -> MetricCollection:
|
||
if self._metric is None:
|
||
self._metric = self.default_metric()
|
||
|
||
return MetricCollection(self._metric)
|
||
|
||
def setup_validation_metric(self):
|
||
metric = self.metric
|
||
if metric is not None:
|
||
self.model.validation_metric = metric
|
||
self.model.validation_metric.to(self.model.device)
|
||
|
||
@property
|
||
def val_monitor(self):
|
||
"""Quantity (and direction) to monitor
|
||
|
||
Useful for model checkpointing or early stopping.
|
||
|
||
Returns
|
||
-------
|
||
monitor : str
|
||
Name of quantity to monitor.
|
||
mode : {'min', 'max}
|
||
Minimize
|
||
|
||
See also
|
||
--------
|
||
lightning.pytorch.callbacks.ModelCheckpoint
|
||
lightning.pytorch.callbacks.EarlyStopping
|
||
"""
|
||
|
||
name, metric = next(iter(self.metric.items()))
|
||
return name, "max" if metric.higher_is_better else "min"
|