mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
feat: add x-vector architecture
This commit is contained in:
1 parent
ce256cd694
commit
20e7f35cf4
3 files changed
+314
-1
No files matched your search
@@ -26,4 +26,4 @@
|
||||
# AUTHORS
|
||||
# Hervé BREDIN - http://herve.niderb.fr
|
||||
|
||||
from .models import PyanNet
|
||||
from .models import PyanNet, SincTDNN
|
||||
@@ -34,6 +34,7 @@ import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from .sincnet import SincNet
|
||||
from .tdnn import XVectorNet
|
||||
from .pooling import TemporalPooling
|
||||
from pyannote.audio.train.model import Model
|
||||
from pyannote.audio.train.model import Resolution
|
||||
@@ -526,3 +527,129 @@ class PyanNet(Model):
|
||||
if layer == 0:
|
||||
return self.sincnet_.dimension
|
||||
return self.rnn_.intermediate_dimension(layer-1)
|
||||
|
||||
|
||||
class SincTDNN(Model):
|
||||
"""waveform -> SincNet -> XVectorNet (TDNN -> FC) -> output
|
||||
|
||||
Parameters
|
||||
----------
|
||||
sincnet : `dict`, optional
|
||||
SincNet parameters. Defaults to `pyannote.audio.models.sincnet.SincNet`
|
||||
default parameters.
|
||||
xvector : `dict`, optional
|
||||
X-Vector Time-Delay neural network parameters.
|
||||
Defaults to `pyannote.audio.models.tdnn.XVectorNet` default parameters.
|
||||
embedding : `dict`, optional
|
||||
Embedding parameters. Defaults to `Embedding` default parameters. This
|
||||
only has effect when model is used for representation learning.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def get_alignment(sincnet=None, **kwargs):
|
||||
"""
|
||||
"""
|
||||
|
||||
if sincnet is None:
|
||||
sincnet = dict()
|
||||
|
||||
return SincNet.get_alignment(**sincnet)
|
||||
|
||||
supports_packed = False
|
||||
|
||||
@staticmethod
|
||||
def get_resolution(sincnet : Optional[dict] = None, **kwargs) -> Resolution:
|
||||
"""Get sliding window used for feature extraction
|
||||
|
||||
Parameters
|
||||
----------
|
||||
sincnet : dict, optional
|
||||
|
||||
Returns
|
||||
-------
|
||||
sliding_window : `pyannote.core.SlidingWindow` or {`window`, `frame`}
|
||||
"""
|
||||
|
||||
if sincnet is None:
|
||||
sincnet = dict()
|
||||
|
||||
return SincNet.get_resolution(**sincnet)
|
||||
|
||||
def init(self,
|
||||
sincnet : Optional[dict] = None,
|
||||
xvector : Optional[dict] = None,
|
||||
embedding : Optional[dict] = None):
|
||||
"""waveform -> SincNet -> XVectorNet (TDNN -> FC) -> output
|
||||
|
||||
Parameters
|
||||
----------
|
||||
sincnet : `dict`, optional
|
||||
SincNet parameters. Defaults to `pyannote.audio.models.sincnet.SincNet`
|
||||
default parameters.
|
||||
xvector : `dict`, optional
|
||||
X-Vector Time-Delay neural network parameters.
|
||||
Defaults to `pyannote.audio.models.tdnn.XVectorNet` default parameters.
|
||||
embedding : `dict`, optional
|
||||
Embedding parameters. Defaults to `Embedding` default parameters. This
|
||||
only has effect when model is used for representation learning.
|
||||
"""
|
||||
|
||||
n_features = self.n_features
|
||||
|
||||
if sincnet is None:
|
||||
sincnet = dict()
|
||||
self.sincnet = sincnet
|
||||
|
||||
if n_features != 1:
|
||||
raise ValueError('SincNet only supports mono waveforms. '
|
||||
f'Here, waveform has {n_features} channels.')
|
||||
self.sincnet_ = SincNet(**sincnet)
|
||||
n_features = self.sincnet_.dimension
|
||||
|
||||
if xvector is None:
|
||||
xvector = dict()
|
||||
self.xvector = xvector
|
||||
self.xvector_ = XVectorNet(n_features, **xvector)
|
||||
n_features = self.xvector_.dimension
|
||||
|
||||
if self.task.is_representation_learning:
|
||||
if embedding is None:
|
||||
embedding = dict()
|
||||
self.embedding = embedding
|
||||
self.embedding_ = Embedding(n_features, **embedding)
|
||||
else:
|
||||
self.linear_ = nn.Linear(n_features, len(self.classes), bias=True)
|
||||
self.activation_ = self.task.default_activation
|
||||
|
||||
def forward(self, waveforms: torch.Tensor, **kwargs) -> torch.Tensor:
|
||||
"""Forward pass
|
||||
|
||||
Parameters
|
||||
----------
|
||||
waveforms : (batch_size, n_samples, 1) `torch.Tensor`
|
||||
Batch of waveforms
|
||||
|
||||
Returns
|
||||
-------
|
||||
output : `torch.Tensor`
|
||||
Final network output or intermediate network output
|
||||
(only when `return_intermediate` is provided).
|
||||
"""
|
||||
|
||||
output = self.sincnet_(waveforms)
|
||||
|
||||
return_intermediate = 'segment6' if self.task.is_representation_learning else None
|
||||
output = self.xvector_(output, return_intermediate=return_intermediate)
|
||||
|
||||
if self.task.is_representation_learning:
|
||||
return self.embedding_(output)
|
||||
|
||||
return self.activation_(self.linear_(output))
|
||||
|
||||
|
||||
@property
|
||||
def dimension(self):
|
||||
if self.task.is_representation_learning:
|
||||
return self.embedding_.dimension
|
||||
|
||||
return Model.dimension.fget(self)
|
||||
@@ -0,0 +1,186 @@
|
||||
#!/usr/bin/env python
|
||||
# encoding: utf-8
|
||||
|
||||
# The MIT License (MIT)
|
||||
|
||||
# Copyright (c) 2019 CNRS
|
||||
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
# AUTHORS
|
||||
# Juan Manuel Coria
|
||||
|
||||
# The TDNN class defined here was taken from https://github.com/jonasvdd/TDNN
|
||||
|
||||
# Please give proper credit to the authors if you are using TDNN based or X-Vector based
|
||||
# models by citing their papers:
|
||||
|
||||
# Waibel, Alexander H., Toshiyuki Hanazawa, Geoffrey E. Hinton, Kiyohiro Shikano and Kevin J. Lang.
|
||||
# "Phoneme recognition using time-delay neural networks."
|
||||
# IEEE Trans. Acoustics, Speech, and Signal Processing 37 (1989): 328-339.
|
||||
# https://pdfs.semanticscholar.org/cd62/c9976534a6a2096a38244f6cbb03635a127e.pdf?_ga=2.86820248.1800960571.1579515113-23298545.1575886658
|
||||
|
||||
# Peddinti, Vijayaditya, Daniel Povey and Sanjeev Khudanpur.
|
||||
# "A time delay neural network architecture for efficient modeling of long temporal contexts."
|
||||
# INTERSPEECH (2015).
|
||||
# https://www.danielpovey.com/files/2015_interspeech_multisplice.pdf
|
||||
|
||||
# Snyder, David, Daniel Garcia-Romero, Gregory Sell, Daniel Povey and Sanjeev Khudanpur.
|
||||
# "X-Vectors: Robust DNN Embeddings for Speaker Recognition."
|
||||
# 2018 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP) (2018): 5329-5333.
|
||||
# https://www.danielpovey.com/files/2018_icassp_xvectors.pdf
|
||||
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.nn.utils import weight_norm
|
||||
import torch.nn.functional as F
|
||||
|
||||
from .pooling import StatsPool
|
||||
|
||||
|
||||
class TDNN(nn.Module):
|
||||
def __init__(self, context: list, input_channels: int, output_channels: int, full_context: bool = True):
|
||||
"""
|
||||
Implementation of a 'Fast' TDNN layer by exploiting the dilation argument of the PyTorch Conv1d class
|
||||
|
||||
Due to its fastness the context has gained two constraints:
|
||||
* The context must be symmetric
|
||||
* The context must have equal spacing between each consecutive element
|
||||
|
||||
For example: the non-full and symmetric context {-3, -2, 0, +2, +3} is not valid since it doesn't have
|
||||
equal spacing; The non-full context {-6, -3, 0, 3, 6} is both symmetric and has an equal spacing, this is
|
||||
considered valid.
|
||||
|
||||
:param context: The temporal context
|
||||
:param input_channels: The number of input channels
|
||||
:param output_channels: The number of channels produced by the temporal convolution
|
||||
:param full_context: Indicates whether a full context needs to be used
|
||||
"""
|
||||
super(TDNN, self).__init__()
|
||||
self.full_context = full_context
|
||||
self.input_dim = input_channels
|
||||
self.output_dim = output_channels
|
||||
|
||||
context = sorted(context)
|
||||
self.check_valid_context(context, full_context)
|
||||
|
||||
if full_context:
|
||||
kernel_size = context[-1] - context[0] + 1 if len(context) > 1 else 1
|
||||
self.temporal_conv = weight_norm(nn.Conv1d(input_channels, output_channels, kernel_size))
|
||||
else:
|
||||
# use dilation
|
||||
delta = context[1] - context[0]
|
||||
self.temporal_conv = weight_norm(
|
||||
nn.Conv1d(input_channels, output_channels, kernel_size=len(context), dilation=delta))
|
||||
|
||||
def forward(self, x):
|
||||
"""
|
||||
:param x: is one batch of data, x.size(): [batch_size, sequence_length, input_channels]
|
||||
sequence length is the dimension of the arbitrary length data
|
||||
:return: [batch_size, len(valid_steps), output_dim]
|
||||
"""
|
||||
x = self.temporal_conv(torch.transpose(x, 1, 2))
|
||||
return F.relu(torch.transpose(x, 1, 2))
|
||||
|
||||
@staticmethod
|
||||
def check_valid_context(context: list, full_context: bool) -> None:
|
||||
"""
|
||||
Check whether the context is symmetrical and whether and whether the passed
|
||||
context can be used for creating a convolution kernel with dil
|
||||
|
||||
:param full_context: indicates whether the full context (dilation=1) will be used
|
||||
:param context: The context of the model, must be symmetric if no full context and have an equal spacing.
|
||||
"""
|
||||
if full_context:
|
||||
assert len(context) <= 2, "If the full context is given one must only define the smallest and largest"
|
||||
if len(context) == 2:
|
||||
assert context[0] + context[-1] == 0, "The context must be symmetric"
|
||||
else:
|
||||
assert len(context) % 2 != 0, "The context size must be odd"
|
||||
assert context[len(context) // 2] == 0, "The context contain 0 in the center"
|
||||
if len(context) > 1:
|
||||
delta = [context[i] - context[i - 1] for i in range(1, len(context))]
|
||||
assert all(delta[0] == delta[i] for i in range(1, len(delta))), "Intra context spacing must be equal!"
|
||||
|
||||
|
||||
class XVectorNet(nn.Module):
|
||||
"""
|
||||
X-Vector neural network architecture as defined by https://www.danielpovey.com/files/2018_icassp_xvectors.pdf
|
||||
|
||||
Parameters
|
||||
----------
|
||||
input_dim : int, default 24
|
||||
dimension of the input frames
|
||||
embedding_dim : int, default 512
|
||||
dimension of latent embeddings
|
||||
"""
|
||||
|
||||
@property
|
||||
def dimension(self):
|
||||
return self.embedding_dim
|
||||
|
||||
def __init__(self, input_dim: int = 24, embedding_dim: int = 512):
|
||||
super(XVectorNet, self).__init__()
|
||||
frame1 = TDNN(context=[-2, 2], input_channels=input_dim, output_channels=512, full_context=True)
|
||||
frame2 = TDNN(context=[-2, 0, 2], input_channels=512, output_channels=512, full_context=False)
|
||||
frame3 = TDNN(context=[-3, 0, 3], input_channels=512, output_channels=512, full_context=False)
|
||||
frame4 = TDNN(context=[0], input_channels=512, output_channels=512, full_context=True)
|
||||
frame5 = TDNN(context=[0], input_channels=512, output_channels=1500, full_context=True)
|
||||
self.tdnn = nn.Sequential(frame1, frame2, frame3, frame4, frame5, StatsPool())
|
||||
self.segment6 = nn.Linear(3000, embedding_dim)
|
||||
self.segment7 = nn.Linear(embedding_dim, embedding_dim)
|
||||
self.embedding_dim = embedding_dim
|
||||
|
||||
def forward(self, x: torch.Tensor, return_intermediate: Optional[str] = None):
|
||||
"""Calculate X-Vector network activations.
|
||||
Return the requested intermediate layer without computing unnecessary activations.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
x : (batch_size, n_frames, out_channels)
|
||||
Batch of frames
|
||||
return_intermediate : 'stats_pool' | 'segment6' | 'segment7' | None
|
||||
If specified, return the activation of this specific layer.
|
||||
segment6 and segment7 activations are returned before the application of non linearity.
|
||||
|
||||
Returns
|
||||
-------
|
||||
activations :
|
||||
(batch_size, 3000) if return_intermediate == 'stats_pool'
|
||||
(batch_size, embedding_dim) if return_intermediate == 'segment6' | 'segment7' | None
|
||||
"""
|
||||
|
||||
x = self.tdnn(x)
|
||||
|
||||
if return_intermediate == 'stats_pool':
|
||||
return x
|
||||
|
||||
x = self.segment6(x)
|
||||
|
||||
if return_intermediate == 'segment6':
|
||||
return x
|
||||
|
||||
x = self.segment7(F.relu(x))
|
||||
|
||||
if return_intermediate == 'segment7':
|
||||
return x
|
||||
|
||||
return F.relu(x)
|
||||
Reference in new issue
Block a user