diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cd56ad3..3c8a4d8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - BREAKING(pipeline): rename `WeSpeakerPretrainedSpeakerEmbedding` to `ONNXWeSpeakerPretrainedSpeakerEmbedding` - BREAKING(setup): remove `onnxruntime` dependency. You can still use ONNX `hbredin/wespeaker-voxceleb-resnet34-LM` but you will have to install `onnxruntime` yourself. +- feat(model): add support for multi-speaker statistics pooling ## Version 3.0.1 (2023-09-28) diff --git a/pyannote/audio/models/blocks/pooling.py b/pyannote/audio/models/blocks/pooling.py index debb05d1..b02f6f69 100644 --- a/pyannote/audio/models/blocks/pooling.py +++ b/pyannote/audio/models/blocks/pooling.py @@ -1,6 +1,6 @@ # MIT License # -# Copyright (c) 2020 CNRS +# Copyright (c) 2020- CNRS # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -26,6 +26,7 @@ from typing import Optional import torch import torch.nn as nn import torch.nn.functional as F +from einops import rearrange class StatsPool(nn.Module): @@ -40,6 +41,39 @@ class StatsPool(nn.Module): """ + def _pool(self, sequences: torch.Tensor, weights: torch.Tensor) -> torch.Tensor: + """Helper function to compute statistics pooling + + Assumes that weights are already interpolated to match the number of frames + in sequences and that they encode the activation of only one speaker. + + Parameters + ---------- + sequences : (batch, features, frames) torch.Tensor + Sequences of features. + weights : (batch, frames) torch.Tensor + (Already interpolated) weights. + + Returns + ------- + output : (batch, 2 * features) torch.Tensor + Concatenation of mean and (unbiased) standard deviation. + """ + + weights = weights.unsqueeze(dim=1) + # (batch, 1, frames) + + v1 = weights.sum(dim=2) + 1e-8 + mean = torch.sum(sequences * weights, dim=2) / v1 + + dx2 = torch.square(sequences - mean.unsqueeze(2)) + v2 = torch.square(weights).sum(dim=2) + + var = torch.sum(dx2 * weights, dim=2) / (v1 - v2 / v1) + std = torch.sqrt(var) + + return torch.cat([mean, std], dim=1) + def forward( self, sequences: torch.Tensor, weights: Optional[torch.Tensor] = None ) -> torch.Tensor: @@ -47,42 +81,53 @@ class StatsPool(nn.Module): Parameters ---------- - sequences : (batch, channel, frames) torch.Tensor - Sequences. - weights : (batch, frames) torch.Tensor, optional - When provided, compute weighted mean and standard deviation. + sequences : (batch, features, frames) torch.Tensor + Sequences of features. + weights : (batch, frames) or (batch, speakers, frames) torch.Tensor, optional + Compute weighted mean and standard deviation, using provided `weights`. + + Note + ---- + `sequences` and `weights` might use a different number of frames, in which case `weights` + are interpolated linearly to reach the number of frames in `sequences`. Returns ------- - output : (batch, 2 * channel) torch.Tensor - Concatenation of mean and (unbiased) standard deviation. + output : (batch, 2 * features) or (batch, speakers, 2 * features) torch.Tensor + Concatenation of mean and (unbiased) standard deviation. When `weights` are + provided with the `speakers` dimension, `output` is computed for each speaker + separately and returned as (batch, speakers, 2 * channel)-shaped tensor. """ if weights is None: - mean = sequences.mean(dim=2) - std = sequences.std(dim=2, unbiased=True) + mean = sequences.mean(dim=-1) + std = sequences.std(dim=-1, correction=1) + return torch.cat([mean, std], dim=-1) - else: + if weights.dim() == 2: + has_speaker_dimension = False weights = weights.unsqueeze(dim=1) - # (batch, 1, frames) + # (batch, frames) -> (batch, 1, frames) + else: + has_speaker_dimension = True - num_frames = sequences.shape[2] - num_weights = weights.shape[2] - if num_frames != num_weights: - warnings.warn( - f"Mismatch between frames ({num_frames}) and weights ({num_weights}) numbers." - ) - weights = F.interpolate( - weights, size=num_frames, mode="linear", align_corners=False - ) + # interpolate weights if needed + _, _, num_frames = sequences.shape + _, _, num_weights = weights.shape + if num_frames != num_weights: + warnings.warn( + f"Mismatch between frames ({num_frames}) and weights ({num_weights}) numbers." + ) + weights = F.interpolate( + weights, size=num_frames, mode="linear", align_corners=False + ) - v1 = weights.sum(dim=2) - mean = torch.sum(sequences * weights, dim=2) / v1 + output = rearrange( + torch.vmap(self._pool, in_dims=(None, 1))(sequences, weights), + "speakers batch features -> batch speakers features", + ) - dx2 = torch.square(sequences - mean.unsqueeze(2)) - v2 = torch.square(weights).sum(dim=2) + if not has_speaker_dimension: + return output.squeeze(dim=1) - var = torch.sum(dx2 * weights, dim=2) / (v1 - v2 / v1) - std = torch.sqrt(var) - - return torch.cat([mean, std], dim=1) + return output diff --git a/tests/test_stats_pool.py b/tests/test_stats_pool.py new file mode 100644 index 00000000..e30262ed --- /dev/null +++ b/tests/test_stats_pool.py @@ -0,0 +1,131 @@ +# MIT License +# +# Copyright (c) 2023- CNRS +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import torch + +from pyannote.audio.models.blocks.pooling import StatsPool + + +def test_stats_pool_weightless(): + x = torch.Tensor([[[2.0, 4.0], [2.0, 4.0]], [[1.0, 1.0], [1.0, 1.0]]]) + # (batch = 2, features = 2, frames = 2) + + stats_pool = StatsPool() + + y = stats_pool(x) + # (batch = 2, features = 4) + + assert torch.equal( + torch.round(y, decimals=4), + torch.Tensor([[3.0, 3.0, 1.4142, 1.4142], [1.0, 1.0, 0.0, 0.0]]), + ) + + +def test_stats_pool_one_speaker(): + x = torch.Tensor([[[2.0, 4.0], [2.0, 4.0]], [[1.0, 1.0], [1.0, 1.0]]]) + # (batch = 2, features = 2, frames = 2) + + w = torch.Tensor( + [ + [0.5, 0.01], + [0.2, 0.1], + ] + ) + # (batch = 2, frames = 2) + + stats_pool = StatsPool() + + y = stats_pool(x, weights=w) + # (batch = 2, features = 4) + + assert torch.equal( + torch.round(y, decimals=4), + torch.Tensor([[2.0392, 2.0392, 1.4142, 1.4142], [1.0, 1.0, 0.0, 0.0]]), + ) + + +def test_stats_pool_multi_speaker(): + x = torch.Tensor([[[2.0, 4.0], [2.0, 4.0]], [[1.0, 1.0], [1.0, 1.0]]]) + # (batch = 2, features = 2, frames = 2) + + w = torch.Tensor([[[0.1, 0.2], [0.2, 0.3]], [[0.001, 0.001], [0.2, 0.3]]]) + # (batch = 2, speakers = 2, frames = 2) + + stats_pool = StatsPool() + + y = stats_pool(x, weights=w) + # (batch = 2, speakers = 2, features = 4) + + assert torch.equal( + torch.round(y, decimals=4), + torch.Tensor( + [ + [[3.3333, 3.3333, 1.4142, 1.4142], [3.2, 3.2, 1.4142, 1.4142]], + [[1.0, 1.0, 0.0, 0.0], [1.0, 1.0, 0.0, 0.0]], + ] + ), + ) + + +def test_stats_pool_frame_mismatch(): + x = torch.Tensor([[[2.0, 2.0], [2.0, 2.0]], [[1.0, 1.0], [1.0, 1.0]]]) + # (batch = 2, features = 2, frames = 2) + + stats_pool = StatsPool() + w = torch.Tensor( + [ + [0.5, 0.5, 0.0], + [0.0, 0.5, 0.5], + ] + ) + # (batch = 2, frames = 3) + + y = stats_pool(x, weights=w) + # (batch = 2, features = 4) + + assert torch.equal( + torch.round(y, decimals=4), + torch.Tensor([[2.0, 2.0, 0.0, 0.0], [1.0, 1.0, 0.0, 0.0]]), + ) + + +def test_stats_pool_all_zero_weights(): + x = torch.Tensor([[[2.0, 4.0], [2.0, 4.0]], [[1.0, 1.0], [1.0, 1.0]]]) + # (batch = 2, features = 2, frames = 2) + + w = torch.Tensor( + [ + [0.5, 0.01], + [0.0, 0.0], # all zero weights + ] + ) + # (batch = 2, frames = 2) + + stats_pool = StatsPool() + + y = stats_pool(x, weights=w) + # (batch = 2, features = 4) + + assert torch.equal( + torch.round(y, decimals=4), + torch.Tensor([[2.0392, 2.0392, 1.4142, 1.4142], [0.0, 0.0, 0.0, 0.0]]), + )