Feat/rawnet (#1081)

This commit is contained in:
Hervé BREDIN authored and GitHub committed 2022-09-12 09:31:01 +02:00
1 parent 993c74805d
commit 652c944c96
7 files changed
+774 -8

No files matched your search

@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2021 Jee-weon Jung
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
+159
View File
@@ -0,0 +1,159 @@
# MIT License
# Copyright (c) 2021 Jee-weon Jung
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import torch
import torch.nn as nn
from asteroid_filterbanks import Encoder, ParamSincFB
from .RawNetBasicBlock import Bottle2neck, PreEmphasis
class RawNet3(nn.Module):
def __init__(self, block, model_scale, context, summed, C=1024, **kwargs):
super().__init__()
nOut = kwargs["nOut"]
self.context = context
self.encoder_type = kwargs["encoder_type"]
self.log_sinc = kwargs["log_sinc"]
self.norm_sinc = kwargs["norm_sinc"]
self.out_bn = kwargs["out_bn"]
self.summed = summed
self.preprocess = nn.Sequential(
PreEmphasis(), nn.InstanceNorm1d(1, eps=1e-4, affine=True)
)
self.conv1 = Encoder(
ParamSincFB(
C // 4,
251,
stride=kwargs["sinc_stride"],
)
)
self.relu = nn.ReLU()
self.bn1 = nn.BatchNorm1d(C // 4)
self.layer1 = block(
C // 4, C, kernel_size=3, dilation=2, scale=model_scale, pool=5
)
self.layer2 = block(C, C, kernel_size=3, dilation=3, scale=model_scale, pool=3)
self.layer3 = block(C, C, kernel_size=3, dilation=4, scale=model_scale)
self.layer4 = nn.Conv1d(3 * C, 1536, kernel_size=1)
if self.context:
attn_input = 1536 * 3
else:
attn_input = 1536
print("self.encoder_type", self.encoder_type)
if self.encoder_type == "ECA":
attn_output = 1536
elif self.encoder_type == "ASP":
attn_output = 1
else:
raise ValueError("Undefined encoder")
self.attention = nn.Sequential(
nn.Conv1d(attn_input, 128, kernel_size=1),
nn.ReLU(),
nn.BatchNorm1d(128),
nn.Conv1d(128, attn_output, kernel_size=1),
nn.Softmax(dim=2),
)
self.bn5 = nn.BatchNorm1d(3072)
self.fc6 = nn.Linear(3072, nOut)
self.bn6 = nn.BatchNorm1d(nOut)
self.mp3 = nn.MaxPool1d(3)
def forward(self, x):
"""
:param x: input mini-batch (bs, samp)
"""
with torch.cuda.amp.autocast(enabled=False):
x = self.preprocess(x)
x = torch.abs(self.conv1(x))
if self.log_sinc:
x = torch.log(x + 1e-6)
if self.norm_sinc == "mean":
x = x - torch.mean(x, dim=-1, keepdim=True)
elif self.norm_sinc == "mean_std":
m = torch.mean(x, dim=-1, keepdim=True)
s = torch.std(x, dim=-1, keepdim=True)
s[s < 0.001] = 0.001
x = (x - m) / s
if self.summed:
x1 = self.layer1(x)
x2 = self.layer2(x1)
x3 = self.layer3(self.mp3(x1) + x2)
else:
x1 = self.layer1(x)
x2 = self.layer2(x1)
x3 = self.layer3(x2)
x = self.layer4(torch.cat((self.mp3(x1), x2, x3), dim=1))
x = self.relu(x)
t = x.size()[-1]
if self.context:
global_x = torch.cat(
(
x,
torch.mean(x, dim=2, keepdim=True).repeat(1, 1, t),
torch.sqrt(
torch.var(x, dim=2, keepdim=True).clamp(min=1e-4, max=1e4)
).repeat(1, 1, t),
),
dim=1,
)
else:
global_x = x
w = self.attention(global_x)
mu = torch.sum(x * w, dim=2)
sg = torch.sqrt(
(torch.sum((x**2) * w, dim=2) - mu**2).clamp(min=1e-4, max=1e4)
)
x = torch.cat((mu, sg), 1)
x = self.bn5(x)
x = self.fc6(x)
if self.out_bn:
x = self.bn6(x)
return x
def MainModel(**kwargs):
model = RawNet3(Bottle2neck, model_scale=8, context=True, summed=True, **kwargs)
return model
+165
View File
@@ -0,0 +1,165 @@
# MIT License
# Copyright (c) 2021 Jee-weon Jung
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
class PreEmphasis(torch.nn.Module):
def __init__(self, coef: float = 0.97) -> None:
super().__init__()
self.coef = coef
# make kernel
# In pytorch, the convolution operation uses cross-correlation. So, filter is flipped.
self.register_buffer(
"flipped_filter",
torch.FloatTensor([-self.coef, 1.0]).unsqueeze(0).unsqueeze(0),
)
def forward(self, input: torch.tensor) -> torch.tensor:
assert (
len(input.size()) == 2
), "The number of dimensions of input tensor must be 2!"
# reflect padding to match lengths of in/out
input = input.unsqueeze(1)
input = F.pad(input, (1, 0), "reflect")
return F.conv1d(input, self.flipped_filter)
class AFMS(nn.Module):
"""
Alpha-Feature map scaling, added to the output of each residual block[1,2].
Reference:
[1] RawNet2 : https://www.isca-speech.org/archive/Interspeech_2020/pdfs/1011.pdf
[2] AMFS : https://www.koreascience.or.kr/article/JAKO202029757857763.page
"""
def __init__(self, nb_dim: int) -> None:
super().__init__()
self.alpha = nn.Parameter(torch.ones((nb_dim, 1)))
self.fc = nn.Linear(nb_dim, nb_dim)
self.sig = nn.Sigmoid()
def forward(self, x):
y = F.adaptive_avg_pool1d(x, 1).view(x.size(0), -1)
y = self.sig(self.fc(y)).view(x.size(0), x.size(1), -1)
x = x + self.alpha
x = x * y
return x
class Bottle2neck(nn.Module):
def __init__(
self,
inplanes,
planes,
kernel_size=None,
dilation=None,
scale=4,
pool=False,
):
super().__init__()
width = int(math.floor(planes / scale))
self.conv1 = nn.Conv1d(inplanes, width * scale, kernel_size=1)
self.bn1 = nn.BatchNorm1d(width * scale)
self.nums = scale - 1
convs = []
bns = []
num_pad = math.floor(kernel_size / 2) * dilation
for i in range(self.nums):
convs.append(
nn.Conv1d(
width,
width,
kernel_size=kernel_size,
dilation=dilation,
padding=num_pad,
)
)
bns.append(nn.BatchNorm1d(width))
self.convs = nn.ModuleList(convs)
self.bns = nn.ModuleList(bns)
self.conv3 = nn.Conv1d(width * scale, planes, kernel_size=1)
self.bn3 = nn.BatchNorm1d(planes)
self.relu = nn.ReLU()
self.width = width
self.mp = nn.MaxPool1d(pool) if pool else False
self.afms = AFMS(planes)
if inplanes != planes: # if change in number of filters
self.residual = nn.Sequential(
nn.Conv1d(inplanes, planes, kernel_size=1, stride=1, bias=False)
)
else:
self.residual = nn.Identity()
def forward(self, x):
residual = self.residual(x)
out = self.conv1(x)
out = self.relu(out)
out = self.bn1(out)
spx = torch.split(out, self.width, 1)
for i in range(self.nums):
if i == 0:
sp = spx[i]
else:
sp = sp + spx[i]
sp = self.convs[i](sp)
sp = self.relu(sp)
sp = self.bns[i](sp)
if i == 0:
out = sp
else:
out = torch.cat((out, sp), 1)
out = torch.cat((out, spx[self.nums]), 1)
out = self.conv3(out)
out = self.relu(out)
out = self.bn3(out)
out += residual
if self.mp:
out = self.mp(out)
out = self.afms(out)
return out
@@ -0,0 +1,21 @@
# MIT License
# Copyright (c) 2021 Jee-weon Jung
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
+110 -1
View File
@@ -48,6 +48,14 @@ from pyannote.audio.core.io import AudioFile
from pyannote.audio.pipelines.utils import oracle_segmentation
from pyannote.audio.utils.permutation import permutate
try:
from finch import FINCH
FINCH_IS_AVAILABLE = True
except ImportError:
FINCH_IS_AVAILABLE = False
class BaseClustering(Pipeline):
def __init__(
@@ -278,6 +286,106 @@ class BaseClustering(Pipeline):
return hard_clusters, soft_clusters
class FINCHClustering(BaseClustering):
"""FINCH clustering
Parameters
----------
metric : {"cosine", "euclidean", ...}, optional
Distance metric to use. Defaults to "cosine".
"""
def __init__(
self,
metric: str = "cosine",
max_num_embeddings: int = np.inf,
constrained_assignment: bool = False,
):
if not FINCH_IS_AVAILABLE:
raise ImportError(
"'finch-clust' must be installed to use FINCH clustering. "
"Visit https://pypi.org/project/finch-clust/ for installation instructions."
)
super().__init__(
metric=metric,
max_num_embeddings=max_num_embeddings,
constrained_assignment=constrained_assignment,
)
self.threshold = Uniform(0.0, 2.0) # assume unit-normalized embeddings
self.method = Categorical(["average", "complete", "single"])
def cluster(
self,
embeddings: np.ndarray,
min_clusters: int,
max_clusters: int,
num_clusters: int = None,
):
"""
Parameters
----------
embeddings : (num_embeddings, dimension) array
Embeddings
min_clusters : int
Minimum number of clusters
max_clusters : int
Maximum number of clusters
num_clusters : int, optional
Actual number of clusters. Default behavior is to estimate it based
on values provided for `min_clusters`, `max_clusters`, and `threshold`.
Returns
-------
clusters : (num_embeddings, ) array
0-indexed cluster indices.
"""
num_embeddings, _ = embeddings.shape
if num_embeddings == 1:
return np.zeros((1,), dtype=np.uint8)
# apply FINCH clustering and keep (supposedly pure) penultimate partition
clusters, _, _ = FINCH(
embeddings,
initial_rank=None,
req_clust=None,
distance=self.metric,
ensure_early_exit=True,
verbose=False,
)
_, num_partitions = clusters.shape
if num_partitions < 2:
clusters = clusters[:, 0]
else:
clusters = clusters[:, -2]
num_clusters = np.max(clusters) + 1
# compute centroids
centroids = np.vstack(
[np.mean(embeddings[clusters == k], axis=0) for k in range(num_clusters)]
)
# perform agglomerative clustering on centroids
dendrogram = linkage(centroids, metric=self.metric, method=self.method)
klusters = fcluster(dendrogram, self.threshold, criterion="distance") - 1
# update clusters
clusters = -clusters
for i, k in enumerate(klusters):
clusters[clusters == -i] = k
# TODO: handle min/max/num_clusters
# TODO: handle min_cluster_size
return clusters
class AgglomerativeClustering(BaseClustering):
"""Agglomerative clustering
@@ -297,7 +405,7 @@ class AgglomerativeClustering(BaseClustering):
def __init__(
self,
metric: str = "cosine",
max_num_embeddings: int = 1000,
max_num_embeddings: int = np.inf,
constrained_assignment: bool = False,
):
@@ -697,5 +805,6 @@ class HiddenMarkovModelClustering(BaseClustering):
class Clustering(Enum):
AgglomerativeClustering = AgglomerativeClustering
FINCHClustering = FINCHClustering
HiddenMarkovModelClustering = HiddenMarkovModelClustering
OracleClustering = OracleClustering
+8 -5
View File
@@ -229,18 +229,21 @@ class Resegmentation(SpeakerDiarizationMixin, Pipeline):
discrete_diarization = self.to_diarization(permutated_segmentations, count)
# convert to continuous diarization
diarization = self.to_annotation(
resegmentation = self.to_annotation(
discrete_diarization,
min_duration_on=self.min_duration_on,
min_duration_off=self.min_duration_off,
)
diarization.uri = file["uri"]
resegmentation.uri = file["uri"]
if "annotation" in file:
diarization = self.optimal_mapping(file["annotation"], diarization)
# when reference is available, use it to map hypothesized speakers
# to reference speakers (this makes later error analysis easier
# but does not modify the actual output of the resegmentation pipeline)
if "annotation" in file and file["annotation"]:
resegmentation = self.optimal_mapping(file["annotation"], resegmentation)
return diarization
return resegmentation
def get_metric(self) -> GreedyDiarizationErrorRate:
return GreedyDiarizationErrorRate(**self.der_variant)
@@ -42,7 +42,9 @@ from pyannote.audio.pipelines.utils import PipelineModel, get_devices, get_model
backend = torchaudio.get_audio_backend()
try:
from speechbrain.pretrained import EncoderClassifier
from speechbrain.pretrained import (
EncoderClassifier as SpeechBrain_EncoderClassifier,
)
SPEECHBRAIN_IS_AVAILABLE = True
except ImportError:
@@ -50,6 +52,283 @@ except ImportError:
finally:
torchaudio.set_audio_backend(backend)
try:
from nemo.collections.asr.models import (
EncDecSpeakerLabelModel as NeMo_EncDecSpeakerLabelModel,
)
NEMO_IS_AVAILABLE = True
except ImportError:
NEMO_IS_AVAILABLE = False
class NeMoPretrainedSpeakerEmbedding:
def __init__(
self,
embedding: Text = "nvidia/speakerverification_en_titanet_large",
device: torch.device = None,
):
if not NEMO_IS_AVAILABLE:
raise ImportError(
f"'NeMo' must be installed to use '{embedding}' embeddings. "
"Visit https://nvidia.github.io/NeMo/ for installation instructions."
)
super().__init__()
self.embedding = embedding
self.device = device
self.model_ = NeMo_EncDecSpeakerLabelModel.from_pretrained(self.embedding)
self.model_.freeze()
self.model_.to(self.device)
@cached_property
def sample_rate(self) -> int:
return self.model_._cfg.train_ds.get("sample_rate", 16000)
@cached_property
def dimension(self) -> int:
input_signal = torch.rand(1, self.sample_rate).to(self.device)
input_signal_length = torch.tensor([self.sample_rate]).to(self.device)
_, embeddings = self.model_(
input_signal=input_signal, input_signal_length=input_signal_length
)
_, dimension = embeddings.shape
return dimension
@cached_property
def metric(self) -> str:
return "cosine"
@cached_property
def min_num_samples(self) -> int:
lower, upper = 2, round(0.5 * self.sample_rate)
middle = (lower + upper) // 2
while lower + 1 < upper:
try:
input_signal = torch.rand(1, middle).to(self.device)
input_signal_length = torch.tensor([middle]).to(self.device)
_ = self.model_(
input_signal=input_signal, input_signal_length=input_signal_length
)
upper = middle
except RuntimeError:
lower = middle
middle = (lower + upper) // 2
return upper
def __call__(
self, waveforms: torch.Tensor, masks: torch.Tensor = None
) -> np.ndarray:
"""
Parameters
----------
waveforms : (batch_size, num_channels, num_samples)
Only num_channels == 1 is supported.
masks : (batch_size, num_samples), optional
Returns
-------
embeddings : (batch_size, dimension)
"""
batch_size, num_channels, num_samples = waveforms.shape
assert num_channels == 1
waveforms = waveforms.squeeze(dim=1)
if masks is None:
signals = waveforms.squeeze(dim=1)
wav_lens = signals.shape[1] * torch.ones(batch_size)
else:
batch_size_masks, _ = masks.shape
assert batch_size == batch_size_masks
# TODO: speed up the creation of "signals"
# preliminary profiling experiments show
# that it accounts for 15% of __call__
# (the remaining 85% being the actual forward pass)
imasks = F.interpolate(
masks.unsqueeze(dim=1), size=num_samples, mode="nearest"
).squeeze(dim=1)
imasks = imasks > 0.5
signals = pad_sequence(
[waveform[imask] for waveform, imask in zip(waveforms, imasks)],
batch_first=True,
)
wav_lens = imasks.sum(dim=1)
max_len = wav_lens.max()
# corner case: every signal is too short
if max_len < self.min_num_samples:
return np.NAN * np.zeros((batch_size, self.dimension))
too_short = wav_lens < self.min_num_samples
wav_lens[too_short] = max_len
_, embeddings = self.model_(
input_signal=waveforms.to(self.device),
input_signal_length=wav_lens.to(self.device),
)
embeddings = embeddings.cpu().numpy()
embeddings[too_short.cpu().numpy()] = np.NAN
return embeddings
class RawNetPretrainedSpeakerEmbedding:
def __init__(
self,
embedding: Text = "jungjee/RawNet3",
device: torch.device = None,
):
super().__init__()
self.embedding = embedding
self.device = device
from huggingface_hub import hf_hub_download
from pyannote.audio.models.embedding.rawnet.RawNet3 import RawNet3
from pyannote.audio.models.embedding.rawnet.RawNetBasicBlock import Bottle2neck
self.model_ = RawNet3(
Bottle2neck,
model_scale=8,
context=True,
summed=True,
encoder_type="ECA",
nOut=256,
out_bn=False,
sinc_stride=10,
log_sinc=True,
norm_sinc="mean",
grad_mult=1,
)
model_pt = hf_hub_download(repo_id="jungjee/RawNet3", filename="model.pt")
self.model_.load_state_dict(
torch.load(
model_pt,
map_location=lambda storage, loc: storage,
)["model"]
)
self.model_.eval()
self.model_.to(self.device)
@cached_property
def sample_rate(self) -> int:
return 16000
@cached_property
def dimension(self) -> int:
input_signal = torch.rand(1, self.sample_rate).to(self.device)
embeddings = self.model_(input_signal)
_, dimension = embeddings.shape
return dimension
@cached_property
def metric(self) -> str:
return "cosine"
@cached_property
def min_num_samples(self) -> int:
lower, upper = 2, round(0.5 * self.sample_rate)
middle = (lower + upper) // 2
while lower + 1 < upper:
try:
input_signal = torch.rand(1, middle).to(self.device)
_ = self.model_(input_signal)
upper = middle
except RuntimeError:
lower = middle
middle = (lower + upper) // 2
return upper
def __call__(
self, waveforms: torch.Tensor, masks: torch.Tensor = None
) -> np.ndarray:
"""
Parameters
----------
waveforms : (batch_size, num_channels, num_samples)
Only num_channels == 1 is supported.
masks : (batch_size, num_samples), optional
Returns
-------
embeddings : (batch_size, dimension)
"""
batch_size, num_channels, num_samples = waveforms.shape
assert num_channels == 1
waveforms = waveforms.squeeze(dim=1)
if masks is None:
signals = waveforms.squeeze(dim=1)
wav_lens = signals.shape[1] * torch.ones(batch_size)
else:
batch_size_masks, _ = masks.shape
assert batch_size == batch_size_masks
# TODO: speed up the creation of "signals"
# preliminary profiling experiments show
# that it accounts for 15% of __call__
# (the remaining 85% being the actual forward pass)
imasks = F.interpolate(
masks.unsqueeze(dim=1), size=num_samples, mode="nearest"
).squeeze(dim=1)
imasks = imasks > 0.5
wav_lens = imasks.sum(dim=1)
max_len = max(wav_lens.max(), 3 * 16000)
signals = np.stack(
[
np.pad(waveform[imask], (0, max_len - wav_len), mode="wrap")
if wav_len > self.min_num_samples
else np.zeros((max_len,))
for waveform, imask, wav_len in zip(waveforms, imasks, wav_lens)
]
)
with torch.no_grad():
embeddings = self.model_(
torch.from_numpy(signals.astype(np.float32)).to(self.device)
)
embeddings = embeddings.cpu().numpy()
too_short = wav_lens < self.min_num_samples
embeddings[too_short.cpu().numpy()] = np.NAN
return embeddings
class SpeechBrainPretrainedSpeakerEmbedding:
"""Pretrained SpeechBrain speaker embedding
@@ -92,7 +371,7 @@ class SpeechBrainPretrainedSpeakerEmbedding:
self.embedding = embedding
self.device = device
self.classifier_ = EncoderClassifier.from_hparams(
self.classifier_ = SpeechBrain_EncoderClassifier.from_hparams(
source=self.embedding,
savedir=f"{CACHE_DIR}/speechbrain",
run_opts={"device": self.device},
@@ -285,6 +564,8 @@ def PretrainedSpeakerEmbedding(embedding: PipelineModel, device: torch.device =
-----
>>> get_embedding = PretrainedSpeakerEmbedding("pyannote/embedding")
>>> get_embedding = PretrainedSpeakerEmbedding("speechbrain/spkrec-ecapa-voxceleb")
>>> get_embedding = PretrainedSpeakerEmbedding("nvidia/speakerverification_en_titanet_large")
>>> get_embedding = PretrainedSpeakerEmbedding("jungjee/RawNet3")
>>> assert waveforms.ndim == 3
>>> batch_size, num_channels, num_samples = waveforms.shape
>>> assert num_channels == 1
@@ -299,6 +580,13 @@ def PretrainedSpeakerEmbedding(embedding: PipelineModel, device: torch.device =
if isinstance(embedding, str) and "speechbrain" in embedding:
return SpeechBrainPretrainedSpeakerEmbedding(embedding, device=device)
elif isinstance(embedding, str) and "nvidia" in embedding:
return NeMoPretrainedSpeakerEmbedding(embedding, device=device)
elif isinstance(embedding, str) and "jungjee" in embedding:
return RawNetPretrainedSpeakerEmbedding(embedding, device=device)
else:
return PyannoteAudioPretrainedSpeakerEmbedding(embedding, device=device)