mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
10 KiB
10 KiB
In [ ]:
from typing import Optional
import torch
import torch.nn as nn
from pyannote.audio import Model
from pyannote.audio.core.task import Task, Resolution
from torchaudio.transforms import MFCC
# Your custom model must be a subclass of `pyannote.audio.Model`,
# which is a subclass of `pytorch_lightning.LightningModule`,
# which is a subclass of `torch.nn.Module`.
class MyCustomModel(Model):
"""My custom model"""
def __init__(
self,
sample_rate: int = 16000,
num_channels: int = 1,
task: Optional[Task] = None,
param1: int = 32,
param2: int = 16,
):
# First three parameters (sample_rate, num_channels, and task)
# must be there and passed to super().__init__()
super().__init__(sample_rate=sample_rate,
num_channels=num_channels,
task=task)
# Mark param1 and param2 as hyper-parameters.
self.save_hyperparameters("param1", "param2")
# They will be saved automatically into checkpoints.
# They are now also available in self.hparams:
# - param1 == self.hparams.param1
# - param2 == self.hparams.param2
# Layers that do not depend on the addressed task should be defined in '__init__'.
self.mfcc = MFCC()
self.linear1 = nn.Linear(self.mfcc.n_mfcc, self.hparams.param1)
self.linear2 = nn.Linear(self.hparams.param1, self.hparams.param2)
def build(self):
# Add layers that depend on the specifications of the task addressed
# by this model.
# For instance, this simple model could be used for "speech vs. non-speech"
# or "speech vs. music vs. other" classification and the only difference
# would lie in the number of classes (2 or 3) in the final classifier.
# Since task specifications are not available at the time '__init__' is called,
# task-dependent layers can only be added a 'build' time (where task specifications
# are available in 'specifications' attribute)
num_classes = len(self.specifications.classes)
self.classifier = nn.Linear(self.hparams.param2, num_classes)
# 'specifications' has several attributes describing what the task is:
# - classes: the list of classes
# - problem: the type of machine learning problem (e.g. binary
# classification or representation learning)
# - duration: the duration of input audio chunks, in seconds
# - resolution: the resolution of the output (e.g. frame-wise scores
# for voice activity detection or chunk-wise vector for speaker
# embedding)
# - permutation_invariant : whether classes are permutation-invariant
# (e.g. in the case of speaker diarization)
# Depending on the type of 'problem', 'default_activation' can be used
# to automatically guess what the final activation should be (e.g. softmax
# for multi-class classification or sigmoid for multi-label classification).
self.activation = self.default_activation()
# You obviously do not _have_ to use 'default_activation' and can choose to
# use any activation you see fit (or even not use any activation layer). But
# note that pyannote.audio tasks also define default loss functions that are
# consistent with `default_activation` (e.g. binary cross entropy with softmax
# for binary classification tasks)
def forward(self, waveforms: torch.Tensor) -> torch.Tensor:
# Models are expected to work on batches of audio chunks provided as tensors
# with shape (batch_size, num_channels, num_samples) and using the sample rate
# passed to __init__. Resampling will be done automatically for you so you do
# not have to bother about that when preparing the data.
# Extract sequence of MFCCs and passed them through two linear layers
mfcc = self.mfcc(waveforms).squeeze(dim=1).transpose(1, 2)
output = self.linear1(mfcc)
output = self.linear2(output)
# Apply temporal pooling for tasks which need an output at chunk-level.
if self.specifications.resolution == Resolution.CHUNK:
output = torch.mean(output, dim=-1)
# Keep 'mfcc' frame resolution for frame-level tasks.
elif self.specifications.resolution == Resolution.FRAME:
pass
# Apply final classifier and activation function
output = self.classifier(output)
return self.activation(output) In [ ]:
# initialize your experimental protocol
from pyannote.database import get_protocol
protocol = get_protocol('Debug.SpeakerDiarization.Debug')
# initialize the task you want to address
from pyannote.audio.tasks import VoiceActivityDetection
task = VoiceActivityDetection(protocol)
# initialize the model
model = MyCustomModel(task=task)
# train the model
from pytorch_lightning import Trainer
trainer = Trainer(max_epochs=1)
trainer.fit(model)