diff --git a/pyannote/audio/models/embedding/rawnet/LICENSE b/pyannote/audio/models/embedding/rawnet/LICENSE new file mode 100644 index 00000000..d69b5d8d --- /dev/null +++ b/pyannote/audio/models/embedding/rawnet/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Jee-weon Jung + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/pyannote/audio/models/embedding/rawnet/RawNet3.py b/pyannote/audio/models/embedding/rawnet/RawNet3.py new file mode 100755 index 00000000..3b505a15 --- /dev/null +++ b/pyannote/audio/models/embedding/rawnet/RawNet3.py @@ -0,0 +1,159 @@ +# MIT License + +# Copyright (c) 2021 Jee-weon Jung + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +import torch +import torch.nn as nn +from asteroid_filterbanks import Encoder, ParamSincFB + +from .RawNetBasicBlock import Bottle2neck, PreEmphasis + + +class RawNet3(nn.Module): + def __init__(self, block, model_scale, context, summed, C=1024, **kwargs): + super().__init__() + + nOut = kwargs["nOut"] + + self.context = context + self.encoder_type = kwargs["encoder_type"] + self.log_sinc = kwargs["log_sinc"] + self.norm_sinc = kwargs["norm_sinc"] + self.out_bn = kwargs["out_bn"] + self.summed = summed + + self.preprocess = nn.Sequential( + PreEmphasis(), nn.InstanceNorm1d(1, eps=1e-4, affine=True) + ) + self.conv1 = Encoder( + ParamSincFB( + C // 4, + 251, + stride=kwargs["sinc_stride"], + ) + ) + self.relu = nn.ReLU() + self.bn1 = nn.BatchNorm1d(C // 4) + + self.layer1 = block( + C // 4, C, kernel_size=3, dilation=2, scale=model_scale, pool=5 + ) + self.layer2 = block(C, C, kernel_size=3, dilation=3, scale=model_scale, pool=3) + self.layer3 = block(C, C, kernel_size=3, dilation=4, scale=model_scale) + self.layer4 = nn.Conv1d(3 * C, 1536, kernel_size=1) + + if self.context: + attn_input = 1536 * 3 + else: + attn_input = 1536 + print("self.encoder_type", self.encoder_type) + if self.encoder_type == "ECA": + attn_output = 1536 + elif self.encoder_type == "ASP": + attn_output = 1 + else: + raise ValueError("Undefined encoder") + + self.attention = nn.Sequential( + nn.Conv1d(attn_input, 128, kernel_size=1), + nn.ReLU(), + nn.BatchNorm1d(128), + nn.Conv1d(128, attn_output, kernel_size=1), + nn.Softmax(dim=2), + ) + + self.bn5 = nn.BatchNorm1d(3072) + + self.fc6 = nn.Linear(3072, nOut) + self.bn6 = nn.BatchNorm1d(nOut) + + self.mp3 = nn.MaxPool1d(3) + + def forward(self, x): + """ + :param x: input mini-batch (bs, samp) + """ + + with torch.cuda.amp.autocast(enabled=False): + x = self.preprocess(x) + x = torch.abs(self.conv1(x)) + if self.log_sinc: + x = torch.log(x + 1e-6) + if self.norm_sinc == "mean": + x = x - torch.mean(x, dim=-1, keepdim=True) + elif self.norm_sinc == "mean_std": + m = torch.mean(x, dim=-1, keepdim=True) + s = torch.std(x, dim=-1, keepdim=True) + s[s < 0.001] = 0.001 + x = (x - m) / s + + if self.summed: + x1 = self.layer1(x) + x2 = self.layer2(x1) + x3 = self.layer3(self.mp3(x1) + x2) + else: + x1 = self.layer1(x) + x2 = self.layer2(x1) + x3 = self.layer3(x2) + + x = self.layer4(torch.cat((self.mp3(x1), x2, x3), dim=1)) + x = self.relu(x) + + t = x.size()[-1] + + if self.context: + global_x = torch.cat( + ( + x, + torch.mean(x, dim=2, keepdim=True).repeat(1, 1, t), + torch.sqrt( + torch.var(x, dim=2, keepdim=True).clamp(min=1e-4, max=1e4) + ).repeat(1, 1, t), + ), + dim=1, + ) + else: + global_x = x + + w = self.attention(global_x) + + mu = torch.sum(x * w, dim=2) + sg = torch.sqrt( + (torch.sum((x**2) * w, dim=2) - mu**2).clamp(min=1e-4, max=1e4) + ) + + x = torch.cat((mu, sg), 1) + + x = self.bn5(x) + + x = self.fc6(x) + + if self.out_bn: + x = self.bn6(x) + + return x + + +def MainModel(**kwargs): + + model = RawNet3(Bottle2neck, model_scale=8, context=True, summed=True, **kwargs) + return model diff --git a/pyannote/audio/models/embedding/rawnet/RawNetBasicBlock.py b/pyannote/audio/models/embedding/rawnet/RawNetBasicBlock.py new file mode 100755 index 00000000..ea81ab6a --- /dev/null +++ b/pyannote/audio/models/embedding/rawnet/RawNetBasicBlock.py @@ -0,0 +1,165 @@ +# MIT License + +# Copyright (c) 2021 Jee-weon Jung + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class PreEmphasis(torch.nn.Module): + def __init__(self, coef: float = 0.97) -> None: + super().__init__() + self.coef = coef + # make kernel + # In pytorch, the convolution operation uses cross-correlation. So, filter is flipped. + self.register_buffer( + "flipped_filter", + torch.FloatTensor([-self.coef, 1.0]).unsqueeze(0).unsqueeze(0), + ) + + def forward(self, input: torch.tensor) -> torch.tensor: + assert ( + len(input.size()) == 2 + ), "The number of dimensions of input tensor must be 2!" + # reflect padding to match lengths of in/out + input = input.unsqueeze(1) + input = F.pad(input, (1, 0), "reflect") + return F.conv1d(input, self.flipped_filter) + + +class AFMS(nn.Module): + """ + Alpha-Feature map scaling, added to the output of each residual block[1,2]. + + Reference: + [1] RawNet2 : https://www.isca-speech.org/archive/Interspeech_2020/pdfs/1011.pdf + [2] AMFS : https://www.koreascience.or.kr/article/JAKO202029757857763.page + """ + + def __init__(self, nb_dim: int) -> None: + super().__init__() + self.alpha = nn.Parameter(torch.ones((nb_dim, 1))) + self.fc = nn.Linear(nb_dim, nb_dim) + self.sig = nn.Sigmoid() + + def forward(self, x): + y = F.adaptive_avg_pool1d(x, 1).view(x.size(0), -1) + y = self.sig(self.fc(y)).view(x.size(0), x.size(1), -1) + + x = x + self.alpha + x = x * y + return x + + +class Bottle2neck(nn.Module): + def __init__( + self, + inplanes, + planes, + kernel_size=None, + dilation=None, + scale=4, + pool=False, + ): + + super().__init__() + + width = int(math.floor(planes / scale)) + + self.conv1 = nn.Conv1d(inplanes, width * scale, kernel_size=1) + self.bn1 = nn.BatchNorm1d(width * scale) + + self.nums = scale - 1 + + convs = [] + bns = [] + + num_pad = math.floor(kernel_size / 2) * dilation + + for i in range(self.nums): + convs.append( + nn.Conv1d( + width, + width, + kernel_size=kernel_size, + dilation=dilation, + padding=num_pad, + ) + ) + bns.append(nn.BatchNorm1d(width)) + + self.convs = nn.ModuleList(convs) + self.bns = nn.ModuleList(bns) + + self.conv3 = nn.Conv1d(width * scale, planes, kernel_size=1) + self.bn3 = nn.BatchNorm1d(planes) + + self.relu = nn.ReLU() + + self.width = width + + self.mp = nn.MaxPool1d(pool) if pool else False + self.afms = AFMS(planes) + + if inplanes != planes: # if change in number of filters + self.residual = nn.Sequential( + nn.Conv1d(inplanes, planes, kernel_size=1, stride=1, bias=False) + ) + else: + self.residual = nn.Identity() + + def forward(self, x): + residual = self.residual(x) + + out = self.conv1(x) + out = self.relu(out) + out = self.bn1(out) + + spx = torch.split(out, self.width, 1) + for i in range(self.nums): + if i == 0: + sp = spx[i] + else: + sp = sp + spx[i] + sp = self.convs[i](sp) + sp = self.relu(sp) + sp = self.bns[i](sp) + if i == 0: + out = sp + else: + out = torch.cat((out, sp), 1) + + out = torch.cat((out, spx[self.nums]), 1) + + out = self.conv3(out) + out = self.relu(out) + out = self.bn3(out) + + out += residual + if self.mp: + out = self.mp(out) + out = self.afms(out) + + return out diff --git a/pyannote/audio/models/embedding/rawnet/__init__.py b/pyannote/audio/models/embedding/rawnet/__init__.py new file mode 100644 index 00000000..0f9ebfa3 --- /dev/null +++ b/pyannote/audio/models/embedding/rawnet/__init__.py @@ -0,0 +1,21 @@ +# MIT License + +# Copyright (c) 2021 Jee-weon Jung + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. diff --git a/pyannote/audio/pipelines/clustering.py b/pyannote/audio/pipelines/clustering.py index ee84d92e..d61cd385 100644 --- a/pyannote/audio/pipelines/clustering.py +++ b/pyannote/audio/pipelines/clustering.py @@ -48,6 +48,14 @@ from pyannote.audio.core.io import AudioFile from pyannote.audio.pipelines.utils import oracle_segmentation from pyannote.audio.utils.permutation import permutate +try: + from finch import FINCH + + FINCH_IS_AVAILABLE = True + +except ImportError: + FINCH_IS_AVAILABLE = False + class BaseClustering(Pipeline): def __init__( @@ -278,6 +286,106 @@ class BaseClustering(Pipeline): return hard_clusters, soft_clusters +class FINCHClustering(BaseClustering): + """FINCH clustering + + Parameters + ---------- + metric : {"cosine", "euclidean", ...}, optional + Distance metric to use. Defaults to "cosine". + + """ + + def __init__( + self, + metric: str = "cosine", + max_num_embeddings: int = np.inf, + constrained_assignment: bool = False, + ): + + if not FINCH_IS_AVAILABLE: + raise ImportError( + "'finch-clust' must be installed to use FINCH clustering. " + "Visit https://pypi.org/project/finch-clust/ for installation instructions." + ) + + super().__init__( + metric=metric, + max_num_embeddings=max_num_embeddings, + constrained_assignment=constrained_assignment, + ) + + self.threshold = Uniform(0.0, 2.0) # assume unit-normalized embeddings + self.method = Categorical(["average", "complete", "single"]) + + def cluster( + self, + embeddings: np.ndarray, + min_clusters: int, + max_clusters: int, + num_clusters: int = None, + ): + """ + + Parameters + ---------- + embeddings : (num_embeddings, dimension) array + Embeddings + min_clusters : int + Minimum number of clusters + max_clusters : int + Maximum number of clusters + num_clusters : int, optional + Actual number of clusters. Default behavior is to estimate it based + on values provided for `min_clusters`, `max_clusters`, and `threshold`. + + Returns + ------- + clusters : (num_embeddings, ) array + 0-indexed cluster indices. + """ + + num_embeddings, _ = embeddings.shape + if num_embeddings == 1: + return np.zeros((1,), dtype=np.uint8) + + # apply FINCH clustering and keep (supposedly pure) penultimate partition + clusters, _, _ = FINCH( + embeddings, + initial_rank=None, + req_clust=None, + distance=self.metric, + ensure_early_exit=True, + verbose=False, + ) + + _, num_partitions = clusters.shape + if num_partitions < 2: + clusters = clusters[:, 0] + else: + clusters = clusters[:, -2] + num_clusters = np.max(clusters) + 1 + + # compute centroids + centroids = np.vstack( + [np.mean(embeddings[clusters == k], axis=0) for k in range(num_clusters)] + ) + + # perform agglomerative clustering on centroids + dendrogram = linkage(centroids, metric=self.metric, method=self.method) + klusters = fcluster(dendrogram, self.threshold, criterion="distance") - 1 + + # update clusters + clusters = -clusters + for i, k in enumerate(klusters): + clusters[clusters == -i] = k + + # TODO: handle min/max/num_clusters + # TODO: handle min_cluster_size + + return clusters + + class AgglomerativeClustering(BaseClustering): """Agglomerative clustering @@ -297,7 +405,7 @@ class AgglomerativeClustering(BaseClustering): def __init__( self, metric: str = "cosine", - max_num_embeddings: int = 1000, + max_num_embeddings: int = np.inf, constrained_assignment: bool = False, ): @@ -697,5 +805,6 @@ class HiddenMarkovModelClustering(BaseClustering): class Clustering(Enum): AgglomerativeClustering = AgglomerativeClustering + FINCHClustering = FINCHClustering HiddenMarkovModelClustering = HiddenMarkovModelClustering OracleClustering = OracleClustering diff --git a/pyannote/audio/pipelines/resegmentation.py b/pyannote/audio/pipelines/resegmentation.py index 307f6a33..659a18fe 100644 --- a/pyannote/audio/pipelines/resegmentation.py +++ b/pyannote/audio/pipelines/resegmentation.py @@ -229,18 +229,21 @@ class Resegmentation(SpeakerDiarizationMixin, Pipeline): discrete_diarization = self.to_diarization(permutated_segmentations, count) # convert to continuous diarization - diarization = self.to_annotation( + resegmentation = self.to_annotation( discrete_diarization, min_duration_on=self.min_duration_on, min_duration_off=self.min_duration_off, ) - diarization.uri = file["uri"] + resegmentation.uri = file["uri"] - if "annotation" in file: - diarization = self.optimal_mapping(file["annotation"], diarization) + # when reference is available, use it to map hypothesized speakers + # to reference speakers (this makes later error analysis easier + # but does not modify the actual output of the resegmentation pipeline) + if "annotation" in file and file["annotation"]: + resegmentation = self.optimal_mapping(file["annotation"], resegmentation) - return diarization + return resegmentation def get_metric(self) -> GreedyDiarizationErrorRate: return GreedyDiarizationErrorRate(**self.der_variant) diff --git a/pyannote/audio/pipelines/speaker_verification.py b/pyannote/audio/pipelines/speaker_verification.py index a982dbb4..adc9a3e3 100644 --- a/pyannote/audio/pipelines/speaker_verification.py +++ b/pyannote/audio/pipelines/speaker_verification.py @@ -42,7 +42,9 @@ from pyannote.audio.pipelines.utils import PipelineModel, get_devices, get_model backend = torchaudio.get_audio_backend() try: - from speechbrain.pretrained import EncoderClassifier + from speechbrain.pretrained import ( + EncoderClassifier as SpeechBrain_EncoderClassifier, + ) SPEECHBRAIN_IS_AVAILABLE = True except ImportError: @@ -50,6 +52,283 @@ except ImportError: finally: torchaudio.set_audio_backend(backend) +try: + from nemo.collections.asr.models import ( + EncDecSpeakerLabelModel as NeMo_EncDecSpeakerLabelModel, + ) + + NEMO_IS_AVAILABLE = True +except ImportError: + NEMO_IS_AVAILABLE = False + + +class NeMoPretrainedSpeakerEmbedding: + def __init__( + self, + embedding: Text = "nvidia/speakerverification_en_titanet_large", + device: torch.device = None, + ): + + if not NEMO_IS_AVAILABLE: + raise ImportError( + f"'NeMo' must be installed to use '{embedding}' embeddings. " + "Visit https://nvidia.github.io/NeMo/ for installation instructions." + ) + + super().__init__() + self.embedding = embedding + self.device = device + + self.model_ = NeMo_EncDecSpeakerLabelModel.from_pretrained(self.embedding) + self.model_.freeze() + self.model_.to(self.device) + + @cached_property + def sample_rate(self) -> int: + return self.model_._cfg.train_ds.get("sample_rate", 16000) + + @cached_property + def dimension(self) -> int: + + input_signal = torch.rand(1, self.sample_rate).to(self.device) + input_signal_length = torch.tensor([self.sample_rate]).to(self.device) + _, embeddings = self.model_( + input_signal=input_signal, input_signal_length=input_signal_length + ) + _, dimension = embeddings.shape + return dimension + + @cached_property + def metric(self) -> str: + return "cosine" + + @cached_property + def min_num_samples(self) -> int: + + lower, upper = 2, round(0.5 * self.sample_rate) + middle = (lower + upper) // 2 + while lower + 1 < upper: + try: + input_signal = torch.rand(1, middle).to(self.device) + input_signal_length = torch.tensor([middle]).to(self.device) + + _ = self.model_( + input_signal=input_signal, input_signal_length=input_signal_length + ) + + upper = middle + except RuntimeError: + lower = middle + + middle = (lower + upper) // 2 + + return upper + + def __call__( + self, waveforms: torch.Tensor, masks: torch.Tensor = None + ) -> np.ndarray: + """ + + Parameters + ---------- + waveforms : (batch_size, num_channels, num_samples) + Only num_channels == 1 is supported. + masks : (batch_size, num_samples), optional + + Returns + ------- + embeddings : (batch_size, dimension) + + """ + + batch_size, num_channels, num_samples = waveforms.shape + assert num_channels == 1 + + waveforms = waveforms.squeeze(dim=1) + + if masks is None: + signals = waveforms.squeeze(dim=1) + wav_lens = signals.shape[1] * torch.ones(batch_size) + + else: + + batch_size_masks, _ = masks.shape + assert batch_size == batch_size_masks + + # TODO: speed up the creation of "signals" + # preliminary profiling experiments show + # that it accounts for 15% of __call__ + # (the remaining 85% being the actual forward pass) + + imasks = F.interpolate( + masks.unsqueeze(dim=1), size=num_samples, mode="nearest" + ).squeeze(dim=1) + + imasks = imasks > 0.5 + + signals = pad_sequence( + [waveform[imask] for waveform, imask in zip(waveforms, imasks)], + batch_first=True, + ) + + wav_lens = imasks.sum(dim=1) + + max_len = wav_lens.max() + + # corner case: every signal is too short + if max_len < self.min_num_samples: + return np.NAN * np.zeros((batch_size, self.dimension)) + + too_short = wav_lens < self.min_num_samples + wav_lens[too_short] = max_len + + _, embeddings = self.model_( + input_signal=waveforms.to(self.device), + input_signal_length=wav_lens.to(self.device), + ) + + embeddings = embeddings.cpu().numpy() + embeddings[too_short.cpu().numpy()] = np.NAN + + return embeddings + + +class RawNetPretrainedSpeakerEmbedding: + def __init__( + self, + embedding: Text = "jungjee/RawNet3", + device: torch.device = None, + ): + + super().__init__() + self.embedding = embedding + self.device = device + + from huggingface_hub import hf_hub_download + + from pyannote.audio.models.embedding.rawnet.RawNet3 import RawNet3 + from pyannote.audio.models.embedding.rawnet.RawNetBasicBlock import Bottle2neck + + self.model_ = RawNet3( + Bottle2neck, + model_scale=8, + context=True, + summed=True, + encoder_type="ECA", + nOut=256, + out_bn=False, + sinc_stride=10, + log_sinc=True, + norm_sinc="mean", + grad_mult=1, + ) + + model_pt = hf_hub_download(repo_id="jungjee/RawNet3", filename="model.pt") + self.model_.load_state_dict( + torch.load( + model_pt, + map_location=lambda storage, loc: storage, + )["model"] + ) + self.model_.eval() + self.model_.to(self.device) + + @cached_property + def sample_rate(self) -> int: + return 16000 + + @cached_property + def dimension(self) -> int: + input_signal = torch.rand(1, self.sample_rate).to(self.device) + embeddings = self.model_(input_signal) + _, dimension = embeddings.shape + return dimension + + @cached_property + def metric(self) -> str: + return "cosine" + + @cached_property + def min_num_samples(self) -> int: + + lower, upper = 2, round(0.5 * self.sample_rate) + middle = (lower + upper) // 2 + while lower + 1 < upper: + try: + input_signal = torch.rand(1, middle).to(self.device) + _ = self.model_(input_signal) + upper = middle + except RuntimeError: + lower = middle + + middle = (lower + upper) // 2 + + return upper + + def __call__( + self, waveforms: torch.Tensor, masks: torch.Tensor = None + ) -> np.ndarray: + """ + + Parameters + ---------- + waveforms : (batch_size, num_channels, num_samples) + Only num_channels == 1 is supported. + masks : (batch_size, num_samples), optional + + Returns + ------- + embeddings : (batch_size, dimension) + + """ + + batch_size, num_channels, num_samples = waveforms.shape + assert num_channels == 1 + + waveforms = waveforms.squeeze(dim=1) + + if masks is None: + signals = waveforms.squeeze(dim=1) + wav_lens = signals.shape[1] * torch.ones(batch_size) + + else: + + batch_size_masks, _ = masks.shape + assert batch_size == batch_size_masks + + # TODO: speed up the creation of "signals" + # preliminary profiling experiments show + # that it accounts for 15% of __call__ + # (the remaining 85% being the actual forward pass) + + imasks = F.interpolate( + masks.unsqueeze(dim=1), size=num_samples, mode="nearest" + ).squeeze(dim=1) + + imasks = imasks > 0.5 + wav_lens = imasks.sum(dim=1) + max_len = max(wav_lens.max(), 3 * 16000) + + signals = np.stack( + [ + np.pad(waveform[imask], (0, max_len - wav_len), mode="wrap") + if wav_len > self.min_num_samples + else np.zeros((max_len,)) + for waveform, imask, wav_len in zip(waveforms, imasks, wav_lens) + ] + ) + + with torch.no_grad(): + embeddings = self.model_( + torch.from_numpy(signals.astype(np.float32)).to(self.device) + ) + embeddings = embeddings.cpu().numpy() + + too_short = wav_lens < self.min_num_samples + embeddings[too_short.cpu().numpy()] = np.NAN + + return embeddings + class SpeechBrainPretrainedSpeakerEmbedding: """Pretrained SpeechBrain speaker embedding @@ -92,7 +371,7 @@ class SpeechBrainPretrainedSpeakerEmbedding: self.embedding = embedding self.device = device - self.classifier_ = EncoderClassifier.from_hparams( + self.classifier_ = SpeechBrain_EncoderClassifier.from_hparams( source=self.embedding, savedir=f"{CACHE_DIR}/speechbrain", run_opts={"device": self.device}, @@ -285,6 +564,8 @@ def PretrainedSpeakerEmbedding(embedding: PipelineModel, device: torch.device = ----- >>> get_embedding = PretrainedSpeakerEmbedding("pyannote/embedding") >>> get_embedding = PretrainedSpeakerEmbedding("speechbrain/spkrec-ecapa-voxceleb") + >>> get_embedding = PretrainedSpeakerEmbedding("nvidia/speakerverification_en_titanet_large") + >>> get_embedding = PretrainedSpeakerEmbedding("jungjee/RawNet3") >>> assert waveforms.ndim == 3 >>> batch_size, num_channels, num_samples = waveforms.shape >>> assert num_channels == 1 @@ -299,6 +580,13 @@ def PretrainedSpeakerEmbedding(embedding: PipelineModel, device: torch.device = if isinstance(embedding, str) and "speechbrain" in embedding: return SpeechBrainPretrainedSpeakerEmbedding(embedding, device=device) + + elif isinstance(embedding, str) and "nvidia" in embedding: + return NeMoPretrainedSpeakerEmbedding(embedding, device=device) + + elif isinstance(embedding, str) and "jungjee" in embedding: + return RawNetPretrainedSpeakerEmbedding(embedding, device=device) + else: return PyannoteAudioPretrainedSpeakerEmbedding(embedding, device=device)