mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
Feat/rawnet (#1081)
This commit is contained in:
1 parent
993c74805d
commit
652c944c96
7 files changed
+774
-8
No files matched your search
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2021 Jee-weon Jung
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
+159
@@ -0,0 +1,159 @@
|
||||
# MIT License
|
||||
|
||||
# Copyright (c) 2021 Jee-weon Jung
|
||||
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from asteroid_filterbanks import Encoder, ParamSincFB
|
||||
|
||||
from .RawNetBasicBlock import Bottle2neck, PreEmphasis
|
||||
|
||||
|
||||
class RawNet3(nn.Module):
|
||||
def __init__(self, block, model_scale, context, summed, C=1024, **kwargs):
|
||||
super().__init__()
|
||||
|
||||
nOut = kwargs["nOut"]
|
||||
|
||||
self.context = context
|
||||
self.encoder_type = kwargs["encoder_type"]
|
||||
self.log_sinc = kwargs["log_sinc"]
|
||||
self.norm_sinc = kwargs["norm_sinc"]
|
||||
self.out_bn = kwargs["out_bn"]
|
||||
self.summed = summed
|
||||
|
||||
self.preprocess = nn.Sequential(
|
||||
PreEmphasis(), nn.InstanceNorm1d(1, eps=1e-4, affine=True)
|
||||
)
|
||||
self.conv1 = Encoder(
|
||||
ParamSincFB(
|
||||
C // 4,
|
||||
251,
|
||||
stride=kwargs["sinc_stride"],
|
||||
)
|
||||
)
|
||||
self.relu = nn.ReLU()
|
||||
self.bn1 = nn.BatchNorm1d(C // 4)
|
||||
|
||||
self.layer1 = block(
|
||||
C // 4, C, kernel_size=3, dilation=2, scale=model_scale, pool=5
|
||||
)
|
||||
self.layer2 = block(C, C, kernel_size=3, dilation=3, scale=model_scale, pool=3)
|
||||
self.layer3 = block(C, C, kernel_size=3, dilation=4, scale=model_scale)
|
||||
self.layer4 = nn.Conv1d(3 * C, 1536, kernel_size=1)
|
||||
|
||||
if self.context:
|
||||
attn_input = 1536 * 3
|
||||
else:
|
||||
attn_input = 1536
|
||||
print("self.encoder_type", self.encoder_type)
|
||||
if self.encoder_type == "ECA":
|
||||
attn_output = 1536
|
||||
elif self.encoder_type == "ASP":
|
||||
attn_output = 1
|
||||
else:
|
||||
raise ValueError("Undefined encoder")
|
||||
|
||||
self.attention = nn.Sequential(
|
||||
nn.Conv1d(attn_input, 128, kernel_size=1),
|
||||
nn.ReLU(),
|
||||
nn.BatchNorm1d(128),
|
||||
nn.Conv1d(128, attn_output, kernel_size=1),
|
||||
nn.Softmax(dim=2),
|
||||
)
|
||||
|
||||
self.bn5 = nn.BatchNorm1d(3072)
|
||||
|
||||
self.fc6 = nn.Linear(3072, nOut)
|
||||
self.bn6 = nn.BatchNorm1d(nOut)
|
||||
|
||||
self.mp3 = nn.MaxPool1d(3)
|
||||
|
||||
def forward(self, x):
|
||||
"""
|
||||
:param x: input mini-batch (bs, samp)
|
||||
"""
|
||||
|
||||
with torch.cuda.amp.autocast(enabled=False):
|
||||
x = self.preprocess(x)
|
||||
x = torch.abs(self.conv1(x))
|
||||
if self.log_sinc:
|
||||
x = torch.log(x + 1e-6)
|
||||
if self.norm_sinc == "mean":
|
||||
x = x - torch.mean(x, dim=-1, keepdim=True)
|
||||
elif self.norm_sinc == "mean_std":
|
||||
m = torch.mean(x, dim=-1, keepdim=True)
|
||||
s = torch.std(x, dim=-1, keepdim=True)
|
||||
s[s < 0.001] = 0.001
|
||||
x = (x - m) / s
|
||||
|
||||
if self.summed:
|
||||
x1 = self.layer1(x)
|
||||
x2 = self.layer2(x1)
|
||||
x3 = self.layer3(self.mp3(x1) + x2)
|
||||
else:
|
||||
x1 = self.layer1(x)
|
||||
x2 = self.layer2(x1)
|
||||
x3 = self.layer3(x2)
|
||||
|
||||
x = self.layer4(torch.cat((self.mp3(x1), x2, x3), dim=1))
|
||||
x = self.relu(x)
|
||||
|
||||
t = x.size()[-1]
|
||||
|
||||
if self.context:
|
||||
global_x = torch.cat(
|
||||
(
|
||||
x,
|
||||
torch.mean(x, dim=2, keepdim=True).repeat(1, 1, t),
|
||||
torch.sqrt(
|
||||
torch.var(x, dim=2, keepdim=True).clamp(min=1e-4, max=1e4)
|
||||
).repeat(1, 1, t),
|
||||
),
|
||||
dim=1,
|
||||
)
|
||||
else:
|
||||
global_x = x
|
||||
|
||||
w = self.attention(global_x)
|
||||
|
||||
mu = torch.sum(x * w, dim=2)
|
||||
sg = torch.sqrt(
|
||||
(torch.sum((x**2) * w, dim=2) - mu**2).clamp(min=1e-4, max=1e4)
|
||||
)
|
||||
|
||||
x = torch.cat((mu, sg), 1)
|
||||
|
||||
x = self.bn5(x)
|
||||
|
||||
x = self.fc6(x)
|
||||
|
||||
if self.out_bn:
|
||||
x = self.bn6(x)
|
||||
|
||||
return x
|
||||
|
||||
|
||||
def MainModel(**kwargs):
|
||||
|
||||
model = RawNet3(Bottle2neck, model_scale=8, context=True, summed=True, **kwargs)
|
||||
return model
|
||||
+165
@@ -0,0 +1,165 @@
|
||||
# MIT License
|
||||
|
||||
# Copyright (c) 2021 Jee-weon Jung
|
||||
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
|
||||
import math
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
|
||||
class PreEmphasis(torch.nn.Module):
|
||||
def __init__(self, coef: float = 0.97) -> None:
|
||||
super().__init__()
|
||||
self.coef = coef
|
||||
# make kernel
|
||||
# In pytorch, the convolution operation uses cross-correlation. So, filter is flipped.
|
||||
self.register_buffer(
|
||||
"flipped_filter",
|
||||
torch.FloatTensor([-self.coef, 1.0]).unsqueeze(0).unsqueeze(0),
|
||||
)
|
||||
|
||||
def forward(self, input: torch.tensor) -> torch.tensor:
|
||||
assert (
|
||||
len(input.size()) == 2
|
||||
), "The number of dimensions of input tensor must be 2!"
|
||||
# reflect padding to match lengths of in/out
|
||||
input = input.unsqueeze(1)
|
||||
input = F.pad(input, (1, 0), "reflect")
|
||||
return F.conv1d(input, self.flipped_filter)
|
||||
|
||||
|
||||
class AFMS(nn.Module):
|
||||
"""
|
||||
Alpha-Feature map scaling, added to the output of each residual block[1,2].
|
||||
|
||||
Reference:
|
||||
[1] RawNet2 : https://www.isca-speech.org/archive/Interspeech_2020/pdfs/1011.pdf
|
||||
[2] AMFS : https://www.koreascience.or.kr/article/JAKO202029757857763.page
|
||||
"""
|
||||
|
||||
def __init__(self, nb_dim: int) -> None:
|
||||
super().__init__()
|
||||
self.alpha = nn.Parameter(torch.ones((nb_dim, 1)))
|
||||
self.fc = nn.Linear(nb_dim, nb_dim)
|
||||
self.sig = nn.Sigmoid()
|
||||
|
||||
def forward(self, x):
|
||||
y = F.adaptive_avg_pool1d(x, 1).view(x.size(0), -1)
|
||||
y = self.sig(self.fc(y)).view(x.size(0), x.size(1), -1)
|
||||
|
||||
x = x + self.alpha
|
||||
x = x * y
|
||||
return x
|
||||
|
||||
|
||||
class Bottle2neck(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
inplanes,
|
||||
planes,
|
||||
kernel_size=None,
|
||||
dilation=None,
|
||||
scale=4,
|
||||
pool=False,
|
||||
):
|
||||
|
||||
super().__init__()
|
||||
|
||||
width = int(math.floor(planes / scale))
|
||||
|
||||
self.conv1 = nn.Conv1d(inplanes, width * scale, kernel_size=1)
|
||||
self.bn1 = nn.BatchNorm1d(width * scale)
|
||||
|
||||
self.nums = scale - 1
|
||||
|
||||
convs = []
|
||||
bns = []
|
||||
|
||||
num_pad = math.floor(kernel_size / 2) * dilation
|
||||
|
||||
for i in range(self.nums):
|
||||
convs.append(
|
||||
nn.Conv1d(
|
||||
width,
|
||||
width,
|
||||
kernel_size=kernel_size,
|
||||
dilation=dilation,
|
||||
padding=num_pad,
|
||||
)
|
||||
)
|
||||
bns.append(nn.BatchNorm1d(width))
|
||||
|
||||
self.convs = nn.ModuleList(convs)
|
||||
self.bns = nn.ModuleList(bns)
|
||||
|
||||
self.conv3 = nn.Conv1d(width * scale, planes, kernel_size=1)
|
||||
self.bn3 = nn.BatchNorm1d(planes)
|
||||
|
||||
self.relu = nn.ReLU()
|
||||
|
||||
self.width = width
|
||||
|
||||
self.mp = nn.MaxPool1d(pool) if pool else False
|
||||
self.afms = AFMS(planes)
|
||||
|
||||
if inplanes != planes: # if change in number of filters
|
||||
self.residual = nn.Sequential(
|
||||
nn.Conv1d(inplanes, planes, kernel_size=1, stride=1, bias=False)
|
||||
)
|
||||
else:
|
||||
self.residual = nn.Identity()
|
||||
|
||||
def forward(self, x):
|
||||
residual = self.residual(x)
|
||||
|
||||
out = self.conv1(x)
|
||||
out = self.relu(out)
|
||||
out = self.bn1(out)
|
||||
|
||||
spx = torch.split(out, self.width, 1)
|
||||
for i in range(self.nums):
|
||||
if i == 0:
|
||||
sp = spx[i]
|
||||
else:
|
||||
sp = sp + spx[i]
|
||||
sp = self.convs[i](sp)
|
||||
sp = self.relu(sp)
|
||||
sp = self.bns[i](sp)
|
||||
if i == 0:
|
||||
out = sp
|
||||
else:
|
||||
out = torch.cat((out, sp), 1)
|
||||
|
||||
out = torch.cat((out, spx[self.nums]), 1)
|
||||
|
||||
out = self.conv3(out)
|
||||
out = self.relu(out)
|
||||
out = self.bn3(out)
|
||||
|
||||
out += residual
|
||||
if self.mp:
|
||||
out = self.mp(out)
|
||||
out = self.afms(out)
|
||||
|
||||
return out
|
||||
@@ -0,0 +1,21 @@
|
||||
# MIT License
|
||||
|
||||
# Copyright (c) 2021 Jee-weon Jung
|
||||
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
@@ -48,6 +48,14 @@ from pyannote.audio.core.io import AudioFile
|
||||
from pyannote.audio.pipelines.utils import oracle_segmentation
|
||||
from pyannote.audio.utils.permutation import permutate
|
||||
|
||||
try:
|
||||
from finch import FINCH
|
||||
|
||||
FINCH_IS_AVAILABLE = True
|
||||
|
||||
except ImportError:
|
||||
FINCH_IS_AVAILABLE = False
|
||||
|
||||
|
||||
class BaseClustering(Pipeline):
|
||||
def __init__(
|
||||
@@ -278,6 +286,106 @@ class BaseClustering(Pipeline):
|
||||
return hard_clusters, soft_clusters
|
||||
|
||||
|
||||
class FINCHClustering(BaseClustering):
|
||||
"""FINCH clustering
|
||||
|
||||
Parameters
|
||||
----------
|
||||
metric : {"cosine", "euclidean", ...}, optional
|
||||
Distance metric to use. Defaults to "cosine".
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
metric: str = "cosine",
|
||||
max_num_embeddings: int = np.inf,
|
||||
constrained_assignment: bool = False,
|
||||
):
|
||||
|
||||
if not FINCH_IS_AVAILABLE:
|
||||
raise ImportError(
|
||||
"'finch-clust' must be installed to use FINCH clustering. "
|
||||
"Visit https://pypi.org/project/finch-clust/ for installation instructions."
|
||||
)
|
||||
|
||||
super().__init__(
|
||||
metric=metric,
|
||||
max_num_embeddings=max_num_embeddings,
|
||||
constrained_assignment=constrained_assignment,
|
||||
)
|
||||
|
||||
self.threshold = Uniform(0.0, 2.0) # assume unit-normalized embeddings
|
||||
self.method = Categorical(["average", "complete", "single"])
|
||||
|
||||
def cluster(
|
||||
self,
|
||||
embeddings: np.ndarray,
|
||||
min_clusters: int,
|
||||
max_clusters: int,
|
||||
num_clusters: int = None,
|
||||
):
|
||||
"""
|
||||
|
||||
Parameters
|
||||
----------
|
||||
embeddings : (num_embeddings, dimension) array
|
||||
Embeddings
|
||||
min_clusters : int
|
||||
Minimum number of clusters
|
||||
max_clusters : int
|
||||
Maximum number of clusters
|
||||
num_clusters : int, optional
|
||||
Actual number of clusters. Default behavior is to estimate it based
|
||||
on values provided for `min_clusters`, `max_clusters`, and `threshold`.
|
||||
|
||||
Returns
|
||||
-------
|
||||
clusters : (num_embeddings, ) array
|
||||
0-indexed cluster indices.
|
||||
"""
|
||||
|
||||
num_embeddings, _ = embeddings.shape
|
||||
if num_embeddings == 1:
|
||||
return np.zeros((1,), dtype=np.uint8)
|
||||
|
||||
# apply FINCH clustering and keep (supposedly pure) penultimate partition
|
||||
clusters, _, _ = FINCH(
|
||||
embeddings,
|
||||
initial_rank=None,
|
||||
req_clust=None,
|
||||
distance=self.metric,
|
||||
ensure_early_exit=True,
|
||||
verbose=False,
|
||||
)
|
||||
|
||||
_, num_partitions = clusters.shape
|
||||
if num_partitions < 2:
|
||||
clusters = clusters[:, 0]
|
||||
else:
|
||||
clusters = clusters[:, -2]
|
||||
num_clusters = np.max(clusters) + 1
|
||||
|
||||
# compute centroids
|
||||
centroids = np.vstack(
|
||||
[np.mean(embeddings[clusters == k], axis=0) for k in range(num_clusters)]
|
||||
)
|
||||
|
||||
# perform agglomerative clustering on centroids
|
||||
dendrogram = linkage(centroids, metric=self.metric, method=self.method)
|
||||
klusters = fcluster(dendrogram, self.threshold, criterion="distance") - 1
|
||||
|
||||
# update clusters
|
||||
clusters = -clusters
|
||||
for i, k in enumerate(klusters):
|
||||
clusters[clusters == -i] = k
|
||||
|
||||
# TODO: handle min/max/num_clusters
|
||||
# TODO: handle min_cluster_size
|
||||
|
||||
return clusters
|
||||
|
||||
|
||||
class AgglomerativeClustering(BaseClustering):
|
||||
"""Agglomerative clustering
|
||||
|
||||
@@ -297,7 +405,7 @@ class AgglomerativeClustering(BaseClustering):
|
||||
def __init__(
|
||||
self,
|
||||
metric: str = "cosine",
|
||||
max_num_embeddings: int = 1000,
|
||||
max_num_embeddings: int = np.inf,
|
||||
constrained_assignment: bool = False,
|
||||
):
|
||||
|
||||
@@ -697,5 +805,6 @@ class HiddenMarkovModelClustering(BaseClustering):
|
||||
|
||||
class Clustering(Enum):
|
||||
AgglomerativeClustering = AgglomerativeClustering
|
||||
FINCHClustering = FINCHClustering
|
||||
HiddenMarkovModelClustering = HiddenMarkovModelClustering
|
||||
OracleClustering = OracleClustering
|
||||
@@ -229,18 +229,21 @@ class Resegmentation(SpeakerDiarizationMixin, Pipeline):
|
||||
discrete_diarization = self.to_diarization(permutated_segmentations, count)
|
||||
|
||||
# convert to continuous diarization
|
||||
diarization = self.to_annotation(
|
||||
resegmentation = self.to_annotation(
|
||||
discrete_diarization,
|
||||
min_duration_on=self.min_duration_on,
|
||||
min_duration_off=self.min_duration_off,
|
||||
)
|
||||
|
||||
diarization.uri = file["uri"]
|
||||
resegmentation.uri = file["uri"]
|
||||
|
||||
if "annotation" in file:
|
||||
diarization = self.optimal_mapping(file["annotation"], diarization)
|
||||
# when reference is available, use it to map hypothesized speakers
|
||||
# to reference speakers (this makes later error analysis easier
|
||||
# but does not modify the actual output of the resegmentation pipeline)
|
||||
if "annotation" in file and file["annotation"]:
|
||||
resegmentation = self.optimal_mapping(file["annotation"], resegmentation)
|
||||
|
||||
return diarization
|
||||
return resegmentation
|
||||
|
||||
def get_metric(self) -> GreedyDiarizationErrorRate:
|
||||
return GreedyDiarizationErrorRate(**self.der_variant)
|
||||
@@ -42,7 +42,9 @@ from pyannote.audio.pipelines.utils import PipelineModel, get_devices, get_model
|
||||
|
||||
backend = torchaudio.get_audio_backend()
|
||||
try:
|
||||
from speechbrain.pretrained import EncoderClassifier
|
||||
from speechbrain.pretrained import (
|
||||
EncoderClassifier as SpeechBrain_EncoderClassifier,
|
||||
)
|
||||
|
||||
SPEECHBRAIN_IS_AVAILABLE = True
|
||||
except ImportError:
|
||||
@@ -50,6 +52,283 @@ except ImportError:
|
||||
finally:
|
||||
torchaudio.set_audio_backend(backend)
|
||||
|
||||
try:
|
||||
from nemo.collections.asr.models import (
|
||||
EncDecSpeakerLabelModel as NeMo_EncDecSpeakerLabelModel,
|
||||
)
|
||||
|
||||
NEMO_IS_AVAILABLE = True
|
||||
except ImportError:
|
||||
NEMO_IS_AVAILABLE = False
|
||||
|
||||
|
||||
class NeMoPretrainedSpeakerEmbedding:
|
||||
def __init__(
|
||||
self,
|
||||
embedding: Text = "nvidia/speakerverification_en_titanet_large",
|
||||
device: torch.device = None,
|
||||
):
|
||||
|
||||
if not NEMO_IS_AVAILABLE:
|
||||
raise ImportError(
|
||||
f"'NeMo' must be installed to use '{embedding}' embeddings. "
|
||||
"Visit https://nvidia.github.io/NeMo/ for installation instructions."
|
||||
)
|
||||
|
||||
super().__init__()
|
||||
self.embedding = embedding
|
||||
self.device = device
|
||||
|
||||
self.model_ = NeMo_EncDecSpeakerLabelModel.from_pretrained(self.embedding)
|
||||
self.model_.freeze()
|
||||
self.model_.to(self.device)
|
||||
|
||||
@cached_property
|
||||
def sample_rate(self) -> int:
|
||||
return self.model_._cfg.train_ds.get("sample_rate", 16000)
|
||||
|
||||
@cached_property
|
||||
def dimension(self) -> int:
|
||||
|
||||
input_signal = torch.rand(1, self.sample_rate).to(self.device)
|
||||
input_signal_length = torch.tensor([self.sample_rate]).to(self.device)
|
||||
_, embeddings = self.model_(
|
||||
input_signal=input_signal, input_signal_length=input_signal_length
|
||||
)
|
||||
_, dimension = embeddings.shape
|
||||
return dimension
|
||||
|
||||
@cached_property
|
||||
def metric(self) -> str:
|
||||
return "cosine"
|
||||
|
||||
@cached_property
|
||||
def min_num_samples(self) -> int:
|
||||
|
||||
lower, upper = 2, round(0.5 * self.sample_rate)
|
||||
middle = (lower + upper) // 2
|
||||
while lower + 1 < upper:
|
||||
try:
|
||||
input_signal = torch.rand(1, middle).to(self.device)
|
||||
input_signal_length = torch.tensor([middle]).to(self.device)
|
||||
|
||||
_ = self.model_(
|
||||
input_signal=input_signal, input_signal_length=input_signal_length
|
||||
)
|
||||
|
||||
upper = middle
|
||||
except RuntimeError:
|
||||
lower = middle
|
||||
|
||||
middle = (lower + upper) // 2
|
||||
|
||||
return upper
|
||||
|
||||
def __call__(
|
||||
self, waveforms: torch.Tensor, masks: torch.Tensor = None
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
|
||||
Parameters
|
||||
----------
|
||||
waveforms : (batch_size, num_channels, num_samples)
|
||||
Only num_channels == 1 is supported.
|
||||
masks : (batch_size, num_samples), optional
|
||||
|
||||
Returns
|
||||
-------
|
||||
embeddings : (batch_size, dimension)
|
||||
|
||||
"""
|
||||
|
||||
batch_size, num_channels, num_samples = waveforms.shape
|
||||
assert num_channels == 1
|
||||
|
||||
waveforms = waveforms.squeeze(dim=1)
|
||||
|
||||
if masks is None:
|
||||
signals = waveforms.squeeze(dim=1)
|
||||
wav_lens = signals.shape[1] * torch.ones(batch_size)
|
||||
|
||||
else:
|
||||
|
||||
batch_size_masks, _ = masks.shape
|
||||
assert batch_size == batch_size_masks
|
||||
|
||||
# TODO: speed up the creation of "signals"
|
||||
# preliminary profiling experiments show
|
||||
# that it accounts for 15% of __call__
|
||||
# (the remaining 85% being the actual forward pass)
|
||||
|
||||
imasks = F.interpolate(
|
||||
masks.unsqueeze(dim=1), size=num_samples, mode="nearest"
|
||||
).squeeze(dim=1)
|
||||
|
||||
imasks = imasks > 0.5
|
||||
|
||||
signals = pad_sequence(
|
||||
[waveform[imask] for waveform, imask in zip(waveforms, imasks)],
|
||||
batch_first=True,
|
||||
)
|
||||
|
||||
wav_lens = imasks.sum(dim=1)
|
||||
|
||||
max_len = wav_lens.max()
|
||||
|
||||
# corner case: every signal is too short
|
||||
if max_len < self.min_num_samples:
|
||||
return np.NAN * np.zeros((batch_size, self.dimension))
|
||||
|
||||
too_short = wav_lens < self.min_num_samples
|
||||
wav_lens[too_short] = max_len
|
||||
|
||||
_, embeddings = self.model_(
|
||||
input_signal=waveforms.to(self.device),
|
||||
input_signal_length=wav_lens.to(self.device),
|
||||
)
|
||||
|
||||
embeddings = embeddings.cpu().numpy()
|
||||
embeddings[too_short.cpu().numpy()] = np.NAN
|
||||
|
||||
return embeddings
|
||||
|
||||
|
||||
class RawNetPretrainedSpeakerEmbedding:
|
||||
def __init__(
|
||||
self,
|
||||
embedding: Text = "jungjee/RawNet3",
|
||||
device: torch.device = None,
|
||||
):
|
||||
|
||||
super().__init__()
|
||||
self.embedding = embedding
|
||||
self.device = device
|
||||
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
from pyannote.audio.models.embedding.rawnet.RawNet3 import RawNet3
|
||||
from pyannote.audio.models.embedding.rawnet.RawNetBasicBlock import Bottle2neck
|
||||
|
||||
self.model_ = RawNet3(
|
||||
Bottle2neck,
|
||||
model_scale=8,
|
||||
context=True,
|
||||
summed=True,
|
||||
encoder_type="ECA",
|
||||
nOut=256,
|
||||
out_bn=False,
|
||||
sinc_stride=10,
|
||||
log_sinc=True,
|
||||
norm_sinc="mean",
|
||||
grad_mult=1,
|
||||
)
|
||||
|
||||
model_pt = hf_hub_download(repo_id="jungjee/RawNet3", filename="model.pt")
|
||||
self.model_.load_state_dict(
|
||||
torch.load(
|
||||
model_pt,
|
||||
map_location=lambda storage, loc: storage,
|
||||
)["model"]
|
||||
)
|
||||
self.model_.eval()
|
||||
self.model_.to(self.device)
|
||||
|
||||
@cached_property
|
||||
def sample_rate(self) -> int:
|
||||
return 16000
|
||||
|
||||
@cached_property
|
||||
def dimension(self) -> int:
|
||||
input_signal = torch.rand(1, self.sample_rate).to(self.device)
|
||||
embeddings = self.model_(input_signal)
|
||||
_, dimension = embeddings.shape
|
||||
return dimension
|
||||
|
||||
@cached_property
|
||||
def metric(self) -> str:
|
||||
return "cosine"
|
||||
|
||||
@cached_property
|
||||
def min_num_samples(self) -> int:
|
||||
|
||||
lower, upper = 2, round(0.5 * self.sample_rate)
|
||||
middle = (lower + upper) // 2
|
||||
while lower + 1 < upper:
|
||||
try:
|
||||
input_signal = torch.rand(1, middle).to(self.device)
|
||||
_ = self.model_(input_signal)
|
||||
upper = middle
|
||||
except RuntimeError:
|
||||
lower = middle
|
||||
|
||||
middle = (lower + upper) // 2
|
||||
|
||||
return upper
|
||||
|
||||
def __call__(
|
||||
self, waveforms: torch.Tensor, masks: torch.Tensor = None
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
|
||||
Parameters
|
||||
----------
|
||||
waveforms : (batch_size, num_channels, num_samples)
|
||||
Only num_channels == 1 is supported.
|
||||
masks : (batch_size, num_samples), optional
|
||||
|
||||
Returns
|
||||
-------
|
||||
embeddings : (batch_size, dimension)
|
||||
|
||||
"""
|
||||
|
||||
batch_size, num_channels, num_samples = waveforms.shape
|
||||
assert num_channels == 1
|
||||
|
||||
waveforms = waveforms.squeeze(dim=1)
|
||||
|
||||
if masks is None:
|
||||
signals = waveforms.squeeze(dim=1)
|
||||
wav_lens = signals.shape[1] * torch.ones(batch_size)
|
||||
|
||||
else:
|
||||
|
||||
batch_size_masks, _ = masks.shape
|
||||
assert batch_size == batch_size_masks
|
||||
|
||||
# TODO: speed up the creation of "signals"
|
||||
# preliminary profiling experiments show
|
||||
# that it accounts for 15% of __call__
|
||||
# (the remaining 85% being the actual forward pass)
|
||||
|
||||
imasks = F.interpolate(
|
||||
masks.unsqueeze(dim=1), size=num_samples, mode="nearest"
|
||||
).squeeze(dim=1)
|
||||
|
||||
imasks = imasks > 0.5
|
||||
wav_lens = imasks.sum(dim=1)
|
||||
max_len = max(wav_lens.max(), 3 * 16000)
|
||||
|
||||
signals = np.stack(
|
||||
[
|
||||
np.pad(waveform[imask], (0, max_len - wav_len), mode="wrap")
|
||||
if wav_len > self.min_num_samples
|
||||
else np.zeros((max_len,))
|
||||
for waveform, imask, wav_len in zip(waveforms, imasks, wav_lens)
|
||||
]
|
||||
)
|
||||
|
||||
with torch.no_grad():
|
||||
embeddings = self.model_(
|
||||
torch.from_numpy(signals.astype(np.float32)).to(self.device)
|
||||
)
|
||||
embeddings = embeddings.cpu().numpy()
|
||||
|
||||
too_short = wav_lens < self.min_num_samples
|
||||
embeddings[too_short.cpu().numpy()] = np.NAN
|
||||
|
||||
return embeddings
|
||||
|
||||
|
||||
class SpeechBrainPretrainedSpeakerEmbedding:
|
||||
"""Pretrained SpeechBrain speaker embedding
|
||||
@@ -92,7 +371,7 @@ class SpeechBrainPretrainedSpeakerEmbedding:
|
||||
self.embedding = embedding
|
||||
self.device = device
|
||||
|
||||
self.classifier_ = EncoderClassifier.from_hparams(
|
||||
self.classifier_ = SpeechBrain_EncoderClassifier.from_hparams(
|
||||
source=self.embedding,
|
||||
savedir=f"{CACHE_DIR}/speechbrain",
|
||||
run_opts={"device": self.device},
|
||||
@@ -285,6 +564,8 @@ def PretrainedSpeakerEmbedding(embedding: PipelineModel, device: torch.device =
|
||||
-----
|
||||
>>> get_embedding = PretrainedSpeakerEmbedding("pyannote/embedding")
|
||||
>>> get_embedding = PretrainedSpeakerEmbedding("speechbrain/spkrec-ecapa-voxceleb")
|
||||
>>> get_embedding = PretrainedSpeakerEmbedding("nvidia/speakerverification_en_titanet_large")
|
||||
>>> get_embedding = PretrainedSpeakerEmbedding("jungjee/RawNet3")
|
||||
>>> assert waveforms.ndim == 3
|
||||
>>> batch_size, num_channels, num_samples = waveforms.shape
|
||||
>>> assert num_channels == 1
|
||||
@@ -299,6 +580,13 @@ def PretrainedSpeakerEmbedding(embedding: PipelineModel, device: torch.device =
|
||||
|
||||
if isinstance(embedding, str) and "speechbrain" in embedding:
|
||||
return SpeechBrainPretrainedSpeakerEmbedding(embedding, device=device)
|
||||
|
||||
elif isinstance(embedding, str) and "nvidia" in embedding:
|
||||
return NeMoPretrainedSpeakerEmbedding(embedding, device=device)
|
||||
|
||||
elif isinstance(embedding, str) and "jungjee" in embedding:
|
||||
return RawNetPretrainedSpeakerEmbedding(embedding, device=device)
|
||||
|
||||
else:
|
||||
return PyannoteAudioPretrainedSpeakerEmbedding(embedding, device=device)
|
||||
|
||||
|
||||
Reference in new issue
Block a user