mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
feat(model): add support for multi-speaker statistics pooling (#1386)
Co-authored-by: Hervé BREDIN <hbredin@users.noreply.github.com>
This commit is contained in:
1 parent
2787f123cc
commit
884913e1ac
3 files changed
+205
-28
No files matched your search
@@ -13,6 +13,7 @@
|
||||
- BREAKING(pipeline): rename `WeSpeakerPretrainedSpeakerEmbedding` to `ONNXWeSpeakerPretrainedSpeakerEmbedding`
|
||||
- BREAKING(setup): remove `onnxruntime` dependency.
|
||||
You can still use ONNX `hbredin/wespeaker-voxceleb-resnet34-LM` but you will have to install `onnxruntime` yourself.
|
||||
- feat(model): add support for multi-speaker statistics pooling
|
||||
|
||||
## Version 3.0.1 (2023-09-28)
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# MIT License
|
||||
#
|
||||
# Copyright (c) 2020 CNRS
|
||||
# Copyright (c) 2020- CNRS
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
@@ -26,6 +26,7 @@ from typing import Optional
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from einops import rearrange
|
||||
|
||||
|
||||
class StatsPool(nn.Module):
|
||||
@@ -40,6 +41,39 @@ class StatsPool(nn.Module):
|
||||
|
||||
"""
|
||||
|
||||
def _pool(self, sequences: torch.Tensor, weights: torch.Tensor) -> torch.Tensor:
|
||||
"""Helper function to compute statistics pooling
|
||||
|
||||
Assumes that weights are already interpolated to match the number of frames
|
||||
in sequences and that they encode the activation of only one speaker.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
sequences : (batch, features, frames) torch.Tensor
|
||||
Sequences of features.
|
||||
weights : (batch, frames) torch.Tensor
|
||||
(Already interpolated) weights.
|
||||
|
||||
Returns
|
||||
-------
|
||||
output : (batch, 2 * features) torch.Tensor
|
||||
Concatenation of mean and (unbiased) standard deviation.
|
||||
"""
|
||||
|
||||
weights = weights.unsqueeze(dim=1)
|
||||
# (batch, 1, frames)
|
||||
|
||||
v1 = weights.sum(dim=2) + 1e-8
|
||||
mean = torch.sum(sequences * weights, dim=2) / v1
|
||||
|
||||
dx2 = torch.square(sequences - mean.unsqueeze(2))
|
||||
v2 = torch.square(weights).sum(dim=2)
|
||||
|
||||
var = torch.sum(dx2 * weights, dim=2) / (v1 - v2 / v1)
|
||||
std = torch.sqrt(var)
|
||||
|
||||
return torch.cat([mean, std], dim=1)
|
||||
|
||||
def forward(
|
||||
self, sequences: torch.Tensor, weights: Optional[torch.Tensor] = None
|
||||
) -> torch.Tensor:
|
||||
@@ -47,42 +81,53 @@ class StatsPool(nn.Module):
|
||||
|
||||
Parameters
|
||||
----------
|
||||
sequences : (batch, channel, frames) torch.Tensor
|
||||
Sequences.
|
||||
weights : (batch, frames) torch.Tensor, optional
|
||||
When provided, compute weighted mean and standard deviation.
|
||||
sequences : (batch, features, frames) torch.Tensor
|
||||
Sequences of features.
|
||||
weights : (batch, frames) or (batch, speakers, frames) torch.Tensor, optional
|
||||
Compute weighted mean and standard deviation, using provided `weights`.
|
||||
|
||||
Note
|
||||
----
|
||||
`sequences` and `weights` might use a different number of frames, in which case `weights`
|
||||
are interpolated linearly to reach the number of frames in `sequences`.
|
||||
|
||||
Returns
|
||||
-------
|
||||
output : (batch, 2 * channel) torch.Tensor
|
||||
Concatenation of mean and (unbiased) standard deviation.
|
||||
output : (batch, 2 * features) or (batch, speakers, 2 * features) torch.Tensor
|
||||
Concatenation of mean and (unbiased) standard deviation. When `weights` are
|
||||
provided with the `speakers` dimension, `output` is computed for each speaker
|
||||
separately and returned as (batch, speakers, 2 * channel)-shaped tensor.
|
||||
"""
|
||||
|
||||
if weights is None:
|
||||
mean = sequences.mean(dim=2)
|
||||
std = sequences.std(dim=2, unbiased=True)
|
||||
mean = sequences.mean(dim=-1)
|
||||
std = sequences.std(dim=-1, correction=1)
|
||||
return torch.cat([mean, std], dim=-1)
|
||||
|
||||
else:
|
||||
if weights.dim() == 2:
|
||||
has_speaker_dimension = False
|
||||
weights = weights.unsqueeze(dim=1)
|
||||
# (batch, 1, frames)
|
||||
# (batch, frames) -> (batch, 1, frames)
|
||||
else:
|
||||
has_speaker_dimension = True
|
||||
|
||||
num_frames = sequences.shape[2]
|
||||
num_weights = weights.shape[2]
|
||||
if num_frames != num_weights:
|
||||
warnings.warn(
|
||||
f"Mismatch between frames ({num_frames}) and weights ({num_weights}) numbers."
|
||||
)
|
||||
weights = F.interpolate(
|
||||
weights, size=num_frames, mode="linear", align_corners=False
|
||||
)
|
||||
# interpolate weights if needed
|
||||
_, _, num_frames = sequences.shape
|
||||
_, _, num_weights = weights.shape
|
||||
if num_frames != num_weights:
|
||||
warnings.warn(
|
||||
f"Mismatch between frames ({num_frames}) and weights ({num_weights}) numbers."
|
||||
)
|
||||
weights = F.interpolate(
|
||||
weights, size=num_frames, mode="linear", align_corners=False
|
||||
)
|
||||
|
||||
v1 = weights.sum(dim=2)
|
||||
mean = torch.sum(sequences * weights, dim=2) / v1
|
||||
output = rearrange(
|
||||
torch.vmap(self._pool, in_dims=(None, 1))(sequences, weights),
|
||||
"speakers batch features -> batch speakers features",
|
||||
)
|
||||
|
||||
dx2 = torch.square(sequences - mean.unsqueeze(2))
|
||||
v2 = torch.square(weights).sum(dim=2)
|
||||
if not has_speaker_dimension:
|
||||
return output.squeeze(dim=1)
|
||||
|
||||
var = torch.sum(dx2 * weights, dim=2) / (v1 - v2 / v1)
|
||||
std = torch.sqrt(var)
|
||||
|
||||
return torch.cat([mean, std], dim=1)
|
||||
return output
|
||||
@@ -0,0 +1,131 @@
|
||||
# MIT License
|
||||
#
|
||||
# Copyright (c) 2023- CNRS
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
import torch
|
||||
|
||||
from pyannote.audio.models.blocks.pooling import StatsPool
|
||||
|
||||
|
||||
def test_stats_pool_weightless():
|
||||
x = torch.Tensor([[[2.0, 4.0], [2.0, 4.0]], [[1.0, 1.0], [1.0, 1.0]]])
|
||||
# (batch = 2, features = 2, frames = 2)
|
||||
|
||||
stats_pool = StatsPool()
|
||||
|
||||
y = stats_pool(x)
|
||||
# (batch = 2, features = 4)
|
||||
|
||||
assert torch.equal(
|
||||
torch.round(y, decimals=4),
|
||||
torch.Tensor([[3.0, 3.0, 1.4142, 1.4142], [1.0, 1.0, 0.0, 0.0]]),
|
||||
)
|
||||
|
||||
|
||||
def test_stats_pool_one_speaker():
|
||||
x = torch.Tensor([[[2.0, 4.0], [2.0, 4.0]], [[1.0, 1.0], [1.0, 1.0]]])
|
||||
# (batch = 2, features = 2, frames = 2)
|
||||
|
||||
w = torch.Tensor(
|
||||
[
|
||||
[0.5, 0.01],
|
||||
[0.2, 0.1],
|
||||
]
|
||||
)
|
||||
# (batch = 2, frames = 2)
|
||||
|
||||
stats_pool = StatsPool()
|
||||
|
||||
y = stats_pool(x, weights=w)
|
||||
# (batch = 2, features = 4)
|
||||
|
||||
assert torch.equal(
|
||||
torch.round(y, decimals=4),
|
||||
torch.Tensor([[2.0392, 2.0392, 1.4142, 1.4142], [1.0, 1.0, 0.0, 0.0]]),
|
||||
)
|
||||
|
||||
|
||||
def test_stats_pool_multi_speaker():
|
||||
x = torch.Tensor([[[2.0, 4.0], [2.0, 4.0]], [[1.0, 1.0], [1.0, 1.0]]])
|
||||
# (batch = 2, features = 2, frames = 2)
|
||||
|
||||
w = torch.Tensor([[[0.1, 0.2], [0.2, 0.3]], [[0.001, 0.001], [0.2, 0.3]]])
|
||||
# (batch = 2, speakers = 2, frames = 2)
|
||||
|
||||
stats_pool = StatsPool()
|
||||
|
||||
y = stats_pool(x, weights=w)
|
||||
# (batch = 2, speakers = 2, features = 4)
|
||||
|
||||
assert torch.equal(
|
||||
torch.round(y, decimals=4),
|
||||
torch.Tensor(
|
||||
[
|
||||
[[3.3333, 3.3333, 1.4142, 1.4142], [3.2, 3.2, 1.4142, 1.4142]],
|
||||
[[1.0, 1.0, 0.0, 0.0], [1.0, 1.0, 0.0, 0.0]],
|
||||
]
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_stats_pool_frame_mismatch():
|
||||
x = torch.Tensor([[[2.0, 2.0], [2.0, 2.0]], [[1.0, 1.0], [1.0, 1.0]]])
|
||||
# (batch = 2, features = 2, frames = 2)
|
||||
|
||||
stats_pool = StatsPool()
|
||||
w = torch.Tensor(
|
||||
[
|
||||
[0.5, 0.5, 0.0],
|
||||
[0.0, 0.5, 0.5],
|
||||
]
|
||||
)
|
||||
# (batch = 2, frames = 3)
|
||||
|
||||
y = stats_pool(x, weights=w)
|
||||
# (batch = 2, features = 4)
|
||||
|
||||
assert torch.equal(
|
||||
torch.round(y, decimals=4),
|
||||
torch.Tensor([[2.0, 2.0, 0.0, 0.0], [1.0, 1.0, 0.0, 0.0]]),
|
||||
)
|
||||
|
||||
|
||||
def test_stats_pool_all_zero_weights():
|
||||
x = torch.Tensor([[[2.0, 4.0], [2.0, 4.0]], [[1.0, 1.0], [1.0, 1.0]]])
|
||||
# (batch = 2, features = 2, frames = 2)
|
||||
|
||||
w = torch.Tensor(
|
||||
[
|
||||
[0.5, 0.01],
|
||||
[0.0, 0.0], # all zero weights
|
||||
]
|
||||
)
|
||||
# (batch = 2, frames = 2)
|
||||
|
||||
stats_pool = StatsPool()
|
||||
|
||||
y = stats_pool(x, weights=w)
|
||||
# (batch = 2, features = 4)
|
||||
|
||||
assert torch.equal(
|
||||
torch.round(y, decimals=4),
|
||||
torch.Tensor([[2.0392, 2.0392, 1.4142, 1.4142], [0.0, 0.0, 0.0, 0.0]]),
|
||||
)
|
||||
Reference in new issue
Block a user