mirror of
https://github.com/pyannote/pyannote-audio.git
synced 2026-11-20 13:07:51 +09:00
wip: add ReDimNet2 wrapper
This commit is contained in:
1 parent
c3795f2266
commit
fc1e540c08
16 files changed
+2801
No files matched your search
@@ -21,6 +21,7 @@
|
||||
# SOFTWARE.
|
||||
|
||||
|
||||
from .redimnet2 import BaseReDimNet2, ReDimNet2
|
||||
from .wespeaker import (
|
||||
WeSpeakerResNet34,
|
||||
WeSpeakerResNet152,
|
||||
@@ -36,4 +37,6 @@ __all__ = [
|
||||
"WeSpeakerResNet152",
|
||||
"WeSpeakerResNet221",
|
||||
"WeSpeakerResNet293",
|
||||
"BaseReDimNet2",
|
||||
"ReDimNet2",
|
||||
]
|
||||
@@ -0,0 +1,132 @@
|
||||
# ReDimNet2: Scaling Speaker Verification via Time-Pooled Dimension Reshaping
|
||||
|
||||
> **Anonymous submission to Interspeech 2026**
|
||||
>
|
||||
> This repository contains the official implementation and pretrained weights for the paper *"ReDimNet2: Scaling Speaker Verification via Time-Pooled Dimension Reshaping"*.
|
||||
|
||||
## Overview
|
||||
|
||||
ReDimNet2 extends the [ReDimNet](https://github.com/IDRnD/redimnet) dimension-reshaping framework by introducing pooling over the time dimension in the 1D processing pathway. This allows significantly more aggressive channel scaling without proportional compute increase, yielding a strictly better accuracy–efficiency Pareto front at every scale point.
|
||||
|
||||
<p align="center">
|
||||
<img src="assets/pareto.png" width="600" height="427" alt="Computational Cost vs. Average EER"/>
|
||||
</p>
|
||||
|
||||
## Results
|
||||
|
||||
### Trained on VoxCeleb2-dev
|
||||
|
||||
EER (%) on VoxCeleb1 cleaned protocols:
|
||||
|
||||
| Model | Params | GMACs | Vox1-O | Vox1-E | Vox1-H |
|
||||
|:---|:---:|:---:|:---:|:---:|:---:|
|
||||
| ReDimNet2-B0 | 1.1M | 0.33 | 1.04 | 1.16 | 1.97 |
|
||||
| ReDimNet2-B1 | 2.1M | 0.56 | 0.78 | 0.96 | 1.72 |
|
||||
| ReDimNet2-B2 | 3.6M | 0.95 | 0.57 | 0.76 | 1.41 |
|
||||
| ReDimNet2-B3 | 4.1M | 2.70 | 0.42 | 0.66 | 1.22 |
|
||||
| ReDimNet2-B4 | 6.6M | 4.62 | 0.37 | 0.58 | 1.07 |
|
||||
| ReDimNet2-B5 | 8.9M | 9.62 | 0.33 | 0.56 | 1.07 |
|
||||
| ReDimNet2-B6 | 12.3M | 13.05 | 0.29 | 0.52 | 0.99 |
|
||||
|
||||
### Trained on VoxBlink2 + VoxCeleb2
|
||||
|
||||
EER (%) on VoxCeleb1 cleaned protocols:
|
||||
|
||||
| Model | Params | GMACs | Vox1-O | Vox1-E | Vox1-H | Vox1-B |
|
||||
|:---|:---:|:---:|:---:|:---:|:---:|:---:|
|
||||
| ReDimNet2-B6 | 12.5M | 13.05 | 0.17 | 0.34 | 0.68 | 0.95 |
|
||||
|
||||
EER (%) on out-of-domain cross-dataset benchmarks (`train_type="lm"`, `dataset="vb2+vox2_v0"`):
|
||||
|
||||
| Model | SITW (eval-core) | VOICES (eval) | CN-Celeb (multi-enroll) |
|
||||
|:---|:---:|:---:|:---:|
|
||||
| ReDimNet2-B6 | 0.66 | 2.59 | 7.39 |
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Via torch.hub
|
||||
|
||||
```python
|
||||
import torch
|
||||
import torchaudio
|
||||
|
||||
# VoxCeleb2-only variant (default)
|
||||
model = torch.hub.load("PalabraAI/redimnet2", "redimnet2",
|
||||
model_name="b6", train_type="lm", pretrained=True)
|
||||
|
||||
# VoxBlink2 + VoxCeleb2 variant (stronger, B6 only)
|
||||
model = torch.hub.load("PalabraAI/redimnet2", "redimnet2",
|
||||
model_name="b6", train_type="lm",
|
||||
dataset="vb2+vox2_v0", pretrained=True)
|
||||
model.eval()
|
||||
|
||||
waveform, sr = torchaudio.load("audio.wav")
|
||||
# waveform: (batch, samples), 16 kHz
|
||||
with torch.no_grad():
|
||||
embedding = model(waveform) # (batch, emb_dim)
|
||||
```
|
||||
|
||||
Available model names: `b0`, `b1`, `b2`, `b3`, `b4`, `b5`, `b6`.
|
||||
Available train types: `ptn` (pretraining), `lm` (large-margin fine-tuning, recommended).
|
||||
Available datasets: `vox2` (default), `vb2+vox2_v0` (B6 only).
|
||||
|
||||
### Local installation
|
||||
|
||||
```bash
|
||||
git clone https://github.com/PalabraAI/redimnet2.git
|
||||
cd redimnet2
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
```python
|
||||
from redimnet2 import ReDimNet2
|
||||
|
||||
model = ReDimNet2.from_pretrained("b3", train_type="lm")
|
||||
```
|
||||
|
||||
## Model Zoo
|
||||
|
||||
Pretrained weights are available as GitHub Release assets.
|
||||
|
||||
### VoxCeleb2-dev
|
||||
|
||||
| Config | Pretraining (ptn) | Large-Margin (lm) |
|
||||
|:---|:---|:---|
|
||||
| B0 | [b0-vox2-ptn.pt](https://github.com/PalabraAI/redimnet2/releases/download/v1.0.0/b0-vox2-ptn.pt) | [b0-vox2-lm.pt](https://github.com/PalabraAI/redimnet2/releases/download/v1.0.0/b0-vox2-lm.pt) |
|
||||
| B1 | [b1-vox2-ptn.pt](https://github.com/PalabraAI/redimnet2/releases/download/v1.0.0/b1-vox2-ptn.pt) | [b1-vox2-lm.pt](https://github.com/PalabraAI/redimnet2/releases/download/v1.0.0/b1-vox2-lm.pt) |
|
||||
| B2 | [b2-vox2-ptn.pt](https://github.com/PalabraAI/redimnet2/releases/download/v1.0.0/b2-vox2-ptn.pt) | [b2-vox2-lm.pt](https://github.com/PalabraAI/redimnet2/releases/download/v1.0.0/b2-vox2-lm.pt) |
|
||||
| B3 | [b3-vox2-ptn.pt](https://github.com/PalabraAI/redimnet2/releases/download/v1.0.0/b3-vox2-ptn.pt) | [b3-vox2-lm.pt](https://github.com/PalabraAI/redimnet2/releases/download/v1.0.0/b3-vox2-lm.pt) |
|
||||
| B4 | [b4-vox2-ptn.pt](https://github.com/PalabraAI/redimnet2/releases/download/v1.0.0/b4-vox2-ptn.pt) | [b4-vox2-lm.pt](https://github.com/PalabraAI/redimnet2/releases/download/v1.0.0/b4-vox2-lm.pt) |
|
||||
| B5 | [b5-vox2-ptn.pt](https://github.com/PalabraAI/redimnet2/releases/download/v1.0.0/b5-vox2-ptn.pt) | [b5-vox2-lm.pt](https://github.com/PalabraAI/redimnet2/releases/download/v1.0.0/b5-vox2-lm.pt) |
|
||||
| B6 | [b6-vox2-ptn.pt](https://github.com/PalabraAI/redimnet2/releases/download/v1.0.0/b6-vox2-ptn.pt) | [b6-vox2-lm.pt](https://github.com/PalabraAI/redimnet2/releases/download/v1.0.0/b6-vox2-lm.pt) |
|
||||
|
||||
### VoxBlink2 + VoxCeleb2
|
||||
|
||||
| Config | Pretraining (ptn) | Large-Margin (lm) |
|
||||
|:---|:---|:---|
|
||||
| B6 | [b6-vb2+vox2_v0-ptn.pt](https://github.com/PalabraAI/redimnet2/releases/download/v1.0.0/b6-vb2+vox2_v0-ptn.pt) | [b6-vb2+vox2_v0-lm.pt](https://github.com/PalabraAI/redimnet2/releases/download/v1.0.0/b6-vb2+vox2_v0-lm.pt) |
|
||||
|
||||
## Roadmap
|
||||
|
||||
We are actively working on improving model stability and expanding the release. Planned updates:
|
||||
|
||||
- [x] Release a new version of ReDimNet2 with improved normalization unified for both 1D and 2D feature map representations (`agg_gnorm` + `dual_agg` options, first used by the `vb2+vox2_v0` B6 release)
|
||||
- [ ] Release the training pipeline (VoxCeleb2)
|
||||
- [x] Release VoxBlink2 + VoxCeleb2 pretrained model (B6)
|
||||
- [ ] Release VoxBlink2 + VoxCeleb2 pretrained models (B3, distilled B6 → B3)
|
||||
- [ ] Release training recipes for VoxBlink2 + VoxCeleb2 training
|
||||
|
||||
## Citation
|
||||
|
||||
```bibtex
|
||||
@inproceedings{redimnet2_2026,
|
||||
title={ReDimNet2: Scaling Speaker Verification via Time-Pooled Dimension Reshaping},
|
||||
author={Anonymous},
|
||||
booktitle={Submitted to Interspeech 2026},
|
||||
year={2026}
|
||||
}
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
MIT
|
||||
@@ -0,0 +1,311 @@
|
||||
# MIT License
|
||||
#
|
||||
# Copyright (c) 2025 CNRS
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
|
||||
from functools import lru_cache
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
from pyannote.audio.core.model import Model
|
||||
from pyannote.audio.core.task import Task
|
||||
from pyannote.audio.utils.receptive_field import (
|
||||
conv1d_receptive_field_center,
|
||||
conv1d_receptive_field_size,
|
||||
)
|
||||
|
||||
from .redimnet2 import ReDimNet2Wrap
|
||||
|
||||
|
||||
class BaseReDimNet2(Model):
|
||||
"""Base class for ReDimNet2 speaker embedding models
|
||||
|
||||
ReDimNet2 ("Scaling Speaker Verification via Time-Pooled Dimension Reshaping")
|
||||
extracts its own (log-mel) features from the waveform. As with the WeSpeaker
|
||||
models, these features are exposed through `compute_fbank` and, when
|
||||
`fbank_only` is set, returned directly by `forward`.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
config : dict, optional
|
||||
Architecture configuration passed to `ReDimNet2Wrap`. This is exactly the
|
||||
`model_config` dictionary shipped alongside the official pretrained weights.
|
||||
Defaults (None) to an empty dictionary, i.e. `ReDimNet2Wrap` defaults.
|
||||
sample_rate : int, optional
|
||||
Audio sample rate. Defaults to 16kHz (16000).
|
||||
num_channels : int, optional
|
||||
Number of channels. Defaults to mono (1).
|
||||
task : Task, optional
|
||||
Task addressed by the model.
|
||||
|
||||
See also
|
||||
--------
|
||||
pyannote.audio.models.embedding.redimnet2.redimnet2.ReDimNet2Wrap
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Optional[dict] = None,
|
||||
sample_rate: int = 16000,
|
||||
num_channels: int = 1,
|
||||
task: Optional[Task] = None,
|
||||
):
|
||||
super().__init__(sample_rate=sample_rate, num_channels=num_channels, task=task)
|
||||
|
||||
config = dict(config or {})
|
||||
self.save_hyperparameters("config")
|
||||
|
||||
self.redimnet = ReDimNet2Wrap(**config)
|
||||
|
||||
# (mel) feature extraction parameters, needed to map samples to frames.
|
||||
# `hop_length` is a `ReDimNet2Wrap` argument while `win_length` lives in
|
||||
# `spec_params` (passed through to the feature extractor). As in the
|
||||
# WeSpeaker models, the centering padding is ignored, so `num_frames` is a
|
||||
# (close) approximation -- this does not affect the `weights` path, which
|
||||
# interpolates weights to the actual number of feature frames.
|
||||
spec_params = config.get("spec_params", {})
|
||||
self._hop_length = config.get("hop_length", 160)
|
||||
self._win_length = spec_params.get("win_length", 400)
|
||||
# number of mel bins is the `F` (frequency) dimension fed to the backbone
|
||||
self._num_mel_bins = config.get("F", 72)
|
||||
|
||||
@property
|
||||
def resnet(self):
|
||||
"""Temporary alias for `redimnet` (WeSpeaker-style compatibility)
|
||||
|
||||
Some code expects a WeSpeaker-style model exposing its inner network as
|
||||
`model.resnet` (e.g. to call `forward_frames` / `forward_embedding` on it).
|
||||
This alias lets such code run against ReDimNet2 unchanged. It is a thin
|
||||
pointer to `self.redimnet`, so the underlying `state_dict` is unaffected
|
||||
(weights stay under `redimnet.*`).
|
||||
"""
|
||||
return self.redimnet
|
||||
|
||||
@property
|
||||
def fbank_only(self) -> bool:
|
||||
"""Whether to only extract (log-mel) features"""
|
||||
return getattr(self, "_fbank_only", False)
|
||||
|
||||
@fbank_only.setter
|
||||
def fbank_only(self, value: bool):
|
||||
if hasattr(self, "receptive_field"):
|
||||
del self.receptive_field
|
||||
|
||||
# `num_frames` is cached and depends on `fbank_only`
|
||||
self.num_frames.cache_clear()
|
||||
|
||||
self._fbank_only = value
|
||||
|
||||
def compute_fbank(self, waveforms: torch.Tensor) -> torch.Tensor:
|
||||
"""Extract (log-mel) features
|
||||
|
||||
Parameters
|
||||
----------
|
||||
waveforms : (batch_size, num_channels, num_samples) torch.Tensor
|
||||
|
||||
Returns
|
||||
-------
|
||||
fbank : (batch_size, num_frames, num_mel_bins) torch.Tensor
|
||||
(log-mel) features, using the same layout as the WeSpeaker models.
|
||||
|
||||
Note
|
||||
----
|
||||
These are exactly the features consumed internally by `forward` and
|
||||
`forward_frames`.
|
||||
"""
|
||||
return self.redimnet.compute_fbank(waveforms)
|
||||
|
||||
@property
|
||||
def dimension(self) -> int:
|
||||
"""Dimension of output"""
|
||||
|
||||
if self.fbank_only:
|
||||
return self._num_mel_bins
|
||||
|
||||
return self.redimnet.embed_dim
|
||||
|
||||
@lru_cache
|
||||
def num_frames(self, num_samples: int) -> int:
|
||||
"""Compute number of output frames
|
||||
|
||||
Parameters
|
||||
----------
|
||||
num_samples : int
|
||||
Number of input samples.
|
||||
|
||||
Returns
|
||||
-------
|
||||
num_frames : int
|
||||
Number of output frames.
|
||||
"""
|
||||
|
||||
# torchaudio's MelSpectrogram is centered (it pads the signal by `n_fft // 2`
|
||||
# on each side), so the number of (mel) frames is exactly `1 + samples // hop`.
|
||||
num_frames = 1 + num_samples // self._hop_length
|
||||
|
||||
if self.fbank_only:
|
||||
return num_frames
|
||||
|
||||
# the 1D pathway pools then up-samples the time axis back to the (mel) frame
|
||||
# rate, but the input is first truncated to a multiple of `time_stride`.
|
||||
time_stride = self.redimnet.backbone.time_stride
|
||||
return (num_frames // time_stride) * time_stride
|
||||
|
||||
def receptive_field_size(self, num_frames: int = 1) -> int:
|
||||
"""Compute size of receptive field
|
||||
|
||||
Parameters
|
||||
----------
|
||||
num_frames : int, optional
|
||||
Number of frames in the output signal
|
||||
|
||||
Returns
|
||||
-------
|
||||
receptive_field_size : int
|
||||
Receptive field size.
|
||||
"""
|
||||
return conv1d_receptive_field_size(
|
||||
num_frames=num_frames,
|
||||
kernel_size=self._win_length,
|
||||
stride=self._hop_length,
|
||||
padding=0,
|
||||
dilation=1,
|
||||
)
|
||||
|
||||
def receptive_field_center(self, frame: int = 0) -> int:
|
||||
"""Compute center of receptive field
|
||||
|
||||
Parameters
|
||||
----------
|
||||
frame : int, optional
|
||||
Frame index
|
||||
|
||||
Returns
|
||||
-------
|
||||
receptive_field_center : int
|
||||
Index of receptive field center.
|
||||
"""
|
||||
return conv1d_receptive_field_center(
|
||||
frame=frame,
|
||||
kernel_size=self._win_length,
|
||||
stride=self._hop_length,
|
||||
padding=0,
|
||||
dilation=1,
|
||||
)
|
||||
|
||||
def forward_frames(self, waveforms: torch.Tensor) -> torch.Tensor:
|
||||
"""Extract frame-wise embeddings
|
||||
|
||||
Parameters
|
||||
----------
|
||||
waveforms : torch.Tensor
|
||||
Batch of waveforms with shape (batch, channel, sample)
|
||||
|
||||
Returns
|
||||
-------
|
||||
embeddings : (batch, dimension, embedding_frames) torch.Tensor
|
||||
Batch of frame-wise embeddings.
|
||||
"""
|
||||
fbank = self.compute_fbank(waveforms)
|
||||
return self.redimnet.forward_frames(fbank)
|
||||
|
||||
def forward_embedding(
|
||||
self, frames: torch.Tensor, weights: Optional[torch.Tensor] = None
|
||||
) -> torch.Tensor:
|
||||
"""Extract speaker embeddings from frame-wise embeddings
|
||||
|
||||
Parameters
|
||||
----------
|
||||
frames : torch.Tensor
|
||||
Batch of frames with shape (batch, dimension, embedding_frames).
|
||||
weights : (batch, frames) or (batch, speakers, frames) torch.Tensor, optional
|
||||
Batch of weights passed to the pooling layer.
|
||||
|
||||
Returns
|
||||
-------
|
||||
embeddings : (batch, dimension) or (batch, speakers, dimension) torch.Tensor
|
||||
Batch of embeddings.
|
||||
"""
|
||||
return self.redimnet.forward_embedding(frames, weights=weights)
|
||||
|
||||
def forward(
|
||||
self, waveforms: torch.Tensor, weights: Optional[torch.Tensor] = None
|
||||
) -> torch.Tensor:
|
||||
"""Extract speaker embeddings
|
||||
|
||||
Parameters
|
||||
----------
|
||||
waveforms : torch.Tensor
|
||||
Batch of waveforms with shape (batch, channel, sample)
|
||||
weights : (batch, frames) or (batch, speakers, frames) torch.Tensor, optional
|
||||
Batch of weights giving more weight to some frames than others (e.g. the
|
||||
activation of the speaker of interest). They are passed to the pooling
|
||||
layer, possibly after being interpolated to match the number of frames.
|
||||
When a `speakers` dimension is provided, one embedding is computed per
|
||||
speaker.
|
||||
|
||||
Returns
|
||||
-------
|
||||
embeddings : (batch, dimension) or (batch, speakers, dimension) torch.Tensor
|
||||
Batch of embeddings. When `fbank_only` is set, the (log-mel) features
|
||||
with shape (batch, num_frames, num_mel_bins) are returned instead.
|
||||
"""
|
||||
if self.fbank_only:
|
||||
return self.compute_fbank(waveforms)
|
||||
|
||||
return self.redimnet(waveforms, weights=weights)
|
||||
|
||||
|
||||
class ReDimNet2(BaseReDimNet2):
|
||||
"""ReDimNet2 speaker embedding model
|
||||
|
||||
Instantiates a ReDimNet2 model from a configuration dictionary. To load one of
|
||||
the official pretrained variants (`b0` ... `b6`), pass the `model_config`
|
||||
dictionary shipped alongside its weights, e.g.:
|
||||
|
||||
>>> ckpt = torch.hub.load_state_dict_from_url(url)
|
||||
>>> model = ReDimNet2(config=ckpt["model_config"])
|
||||
>>> model.redimnet.load_state_dict(ckpt["state_dict"])
|
||||
|
||||
When `config` is not provided, the default `ReDimNet2Wrap` architecture is used.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Optional[dict] = None,
|
||||
sample_rate: int = 16000,
|
||||
num_channels: int = 1,
|
||||
task: Optional[Task] = None,
|
||||
):
|
||||
# `ReDimNet2Wrap` expects `causal` to be one of {"none", "full", "only_1d"}
|
||||
# but defaults to the (invalid) boolean `False`, so make sure it is set.
|
||||
config = dict(config or {})
|
||||
config.setdefault("causal", "none")
|
||||
super().__init__(
|
||||
config=config,
|
||||
sample_rate=sample_rate,
|
||||
num_channels=num_channels,
|
||||
task=task,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["BaseReDimNet2", "ReDimNet2"]
|
||||
@@ -0,0 +1,40 @@
|
||||
import os
|
||||
import sys
|
||||
import torch
|
||||
|
||||
sys.path.append(os.path.dirname(__file__))
|
||||
from redimnet2.redimnet2 import ReDimNet2Wrap
|
||||
|
||||
dependencies = ['torch', 'torchaudio']
|
||||
|
||||
URL_TEMPLATE = "https://github.com/PalabraAI/redimnet2/releases/download/v1.0.0/{model_name}"
|
||||
|
||||
|
||||
def load_custom(model_name='b0', train_type='lm', dataset='vox2'):
|
||||
model_name = f'{model_name}-{dataset}-{train_type}.pt'
|
||||
url = URL_TEMPLATE.format(model_name=model_name)
|
||||
full_state_dict = torch.hub.load_state_dict_from_url(url, progress=True)
|
||||
|
||||
model_config = full_state_dict['model_config']
|
||||
state_dict = full_state_dict['state_dict']
|
||||
model = ReDimNet2Wrap(**model_config)
|
||||
load_res = model.load_state_dict(state_dict)
|
||||
assert len(load_res.missing_keys) == 0 and len(load_res.unexpected_keys) == 0
|
||||
return model
|
||||
|
||||
|
||||
def redimnet2(model_name='b0', train_type='lm', dataset='vox2', pretrained=True):
|
||||
"""Load a ReDimNet2 speaker embedding model.
|
||||
|
||||
Args:
|
||||
model_name: One of 'b0', 'b1', 'b2', 'b3', 'b4', 'b5', 'b6'.
|
||||
train_type: 'ptn' (pretraining) or 'lm' (large-margin fine-tuning).
|
||||
dataset: Training dataset, default 'vox2'.
|
||||
pretrained: If True, load pretrained weights.
|
||||
|
||||
Returns:
|
||||
ReDimNet2Wrap model.
|
||||
"""
|
||||
if pretrained:
|
||||
return load_custom(model_name, train_type=train_type, dataset=dataset)
|
||||
raise ValueError("Only pretrained=True is supported. Use ReDimNet2Wrap directly for custom models.")
|
||||
@@ -0,0 +1,32 @@
|
||||
from .redimnet2 import ReDimNet2Wrap, ReDimNet2Custom
|
||||
|
||||
import os
|
||||
import torch
|
||||
|
||||
|
||||
URL_TEMPLATE = "https://github.com/PalabraAI/redimnet2/releases/download/v1.0.0/{model_name}-vox2-{train_type}.pt"
|
||||
|
||||
|
||||
def from_pretrained(model_name='b0', train_type='lm'):
|
||||
"""Load a pretrained ReDimNet2 model.
|
||||
|
||||
Args:
|
||||
model_name: One of 'b0', 'b1', 'b2', 'b3', 'b4', 'b5', 'b6'.
|
||||
train_type: 'ptn' (pretraining) or 'lm' (large-margin fine-tuning).
|
||||
|
||||
Returns:
|
||||
ReDimNet2Wrap model with loaded weights.
|
||||
"""
|
||||
url = URL_TEMPLATE.format(model_name=model_name, train_type=train_type)
|
||||
full_state_dict = torch.hub.load_state_dict_from_url(url, progress=True)
|
||||
model_config = full_state_dict['model_config']
|
||||
state_dict = full_state_dict['state_dict']
|
||||
model = ReDimNet2Wrap(**model_config)
|
||||
load_res = model.load_state_dict(state_dict)
|
||||
assert len(load_res.missing_keys) == 0 and len(load_res.unexpected_keys) == 0
|
||||
return model
|
||||
|
||||
|
||||
class ReDimNet2:
|
||||
"""Namespace for from_pretrained access."""
|
||||
from_pretrained = staticmethod(from_pretrained)
|
||||
Whitespace-only changes.
+258
@@ -0,0 +1,258 @@
|
||||
import math
|
||||
import torch
|
||||
import functools
|
||||
import numpy as np
|
||||
import torch.nn as nn
|
||||
from typing import List
|
||||
from torch import Tensor
|
||||
import torch.nn.functional as F
|
||||
from collections import OrderedDict
|
||||
from typing import Iterable, Optional
|
||||
|
||||
#-------------------------------------------------------------
|
||||
# Copy multi-head attention module from hugginface wav2vec2
|
||||
#-------------------------------------------------------------
|
||||
|
||||
# Copied from https://github.com/huggingface/transformers/blob/v4.26.1/src/transformers/models/wav2vec2/modeling_wav2vec2.py
|
||||
# Copied from transformers.models.bart.modeling_bart.BartAttention with Bart->Wav2Vec2
|
||||
class MultiHeadAttention(nn.Module):
|
||||
"""Multi-headed attention from 'Attention Is All You Need' paper"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
embed_dim: int,
|
||||
num_heads: int,
|
||||
dropout: float = 0.0,
|
||||
bias: bool = True,
|
||||
):
|
||||
super().__init__()
|
||||
self.embed_dim = embed_dim
|
||||
self.num_heads = num_heads
|
||||
self.dropout = dropout
|
||||
self.head_dim = embed_dim // num_heads
|
||||
|
||||
if (self.head_dim * num_heads) != self.embed_dim:
|
||||
raise ValueError(
|
||||
f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}"
|
||||
f" and `num_heads`: {num_heads})."
|
||||
)
|
||||
self.scaling = self.head_dim**-0.5
|
||||
|
||||
self.k_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
|
||||
self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
|
||||
self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
|
||||
self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
|
||||
|
||||
def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
|
||||
return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous()
|
||||
|
||||
def forward(self,hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""Input shape: Batch x Time x Channel"""
|
||||
bsz, tgt_len, _ = hidden_states.size()
|
||||
|
||||
# get query proj
|
||||
query_states = self.q_proj(hidden_states) * self.scaling
|
||||
# self_attention
|
||||
key_states = self._shape(self.k_proj(hidden_states), -1, bsz)
|
||||
value_states = self._shape(self.v_proj(hidden_states), -1, bsz)
|
||||
|
||||
proj_shape = (bsz * self.num_heads, -1, self.head_dim)
|
||||
query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape)
|
||||
key_states = key_states.view(*proj_shape)
|
||||
value_states = value_states.view(*proj_shape)
|
||||
|
||||
src_len = key_states.size(1)
|
||||
attn_weights = torch.bmm(query_states, key_states.transpose(1, 2))
|
||||
attn_weights = F.softmax(attn_weights, dim=-1)
|
||||
|
||||
attn_probs = F.dropout(attn_weights, p=self.dropout, training=self.training)
|
||||
|
||||
attn_output = torch.bmm(attn_probs, value_states)
|
||||
|
||||
attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim)
|
||||
attn_output = attn_output.transpose(1, 2)
|
||||
|
||||
# Use the `embed_dim` from the config (stored in the class) rather than `hidden_state` because `attn_output` can be
|
||||
# partitioned aross GPUs when using tensor-parallelism.
|
||||
attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim)
|
||||
|
||||
attn_output = self.out_proj(attn_output)
|
||||
return attn_output
|
||||
|
||||
#-------------------------------------------------------------
|
||||
# Import activations from hugginface
|
||||
#-------------------------------------------------------------
|
||||
# Copied from https://github.com/huggingface/transformers/blob/v4.26.1/src/transformers/activations.py
|
||||
|
||||
def gelu(x):
|
||||
"""This is the gelu implementation from the original ESM repo. Using F.gelu yields subtly wrong results."""
|
||||
return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))
|
||||
|
||||
class NewGELUActivation(nn.Module):
|
||||
def forward(self, input: Tensor) -> Tensor:
|
||||
return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi) * (input + 0.044715 * torch.pow(input, 3.0))))
|
||||
|
||||
class GELUActivation(nn.Module):
|
||||
def __init__(self, use_gelu_python: bool = False):
|
||||
super().__init__()
|
||||
if use_gelu_python:
|
||||
self.act = self._gelu_python
|
||||
else:
|
||||
self.act = F.gelu
|
||||
|
||||
def _gelu_python(self, input: Tensor) -> Tensor:
|
||||
return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0)))
|
||||
|
||||
def forward(self, input: Tensor) -> Tensor:
|
||||
return self.act(input)
|
||||
|
||||
class ClippedGELUActivation(nn.Module):
|
||||
def __init__(self, min: float, max: float):
|
||||
if min > max:
|
||||
raise ValueError(f"min should be < max (got min: {min}, max: {max})")
|
||||
|
||||
super().__init__()
|
||||
self.min = min
|
||||
self.max = max
|
||||
|
||||
def forward(self, x: Tensor) -> Tensor:
|
||||
return torch.clip(gelu(x), self.min, self.max)
|
||||
|
||||
# Copied from https://github.com/huggingface/transformers/blob/v4.26.1/src/transformers/activations.py
|
||||
class FastGELUActivation(nn.Module):
|
||||
def forward(self, input: Tensor) -> Tensor:
|
||||
return 0.5 * input * (1.0 + torch.tanh(input * 0.7978845608 * (1.0 + 0.044715 * input * input)))
|
||||
|
||||
# Copied from https://github.com/huggingface/transformers/blob/v4.26.1/src/transformers/activations.py
|
||||
class QuickGELUActivation(nn.Module):
|
||||
def forward(self, input: Tensor) -> Tensor:
|
||||
return input * torch.sigmoid(1.702 * input)
|
||||
|
||||
# Copied from https://github.com/huggingface/transformers/blob/v4.26.1/src/transformers/activations.py
|
||||
class SiLUActivation(nn.Module):
|
||||
def forward(self, input: Tensor) -> Tensor:
|
||||
return F.silu(input)
|
||||
|
||||
class MishActivation(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
if version.parse(torch.__version__) < version.parse("1.9.0"):
|
||||
self.act = self._mish_python
|
||||
else:
|
||||
self.act = F.mish
|
||||
|
||||
def _mish_python(self, input: Tensor) -> Tensor:
|
||||
return input * torch.tanh(F.softplus(input))
|
||||
|
||||
def forward(self, input: Tensor) -> Tensor:
|
||||
return self.act(input)
|
||||
|
||||
class LinearActivation(nn.Module):
|
||||
def forward(self, input: Tensor) -> Tensor:
|
||||
return input
|
||||
|
||||
class ClassInstantier(OrderedDict):
|
||||
def __getitem__(self, key):
|
||||
content = super().__getitem__(key)
|
||||
cls, kwargs = content if isinstance(content, tuple) else (content, {})
|
||||
return cls(**kwargs)
|
||||
|
||||
ACT2CLS = {
|
||||
"gelu": GELUActivation,
|
||||
"gelu_10": (ClippedGELUActivation, {"min": -10, "max": 10}),
|
||||
"gelu_fast": FastGELUActivation,
|
||||
"gelu_new": NewGELUActivation,
|
||||
"gelu_python": (GELUActivation, {"use_gelu_python": True}),
|
||||
"linear": LinearActivation,
|
||||
"mish": MishActivation,
|
||||
"quick_gelu": QuickGELUActivation,
|
||||
"relu": nn.ReLU,
|
||||
"relu6": nn.ReLU6,
|
||||
"sigmoid": nn.Sigmoid,
|
||||
"silu": SiLUActivation,
|
||||
"swish": SiLUActivation,
|
||||
"tanh": nn.Tanh,
|
||||
}
|
||||
ACT2FN = ClassInstantier(ACT2CLS)
|
||||
|
||||
#-------------------------------------------------------------
|
||||
# Copy FeedForward & Encoder from hugginface wav2vec2
|
||||
#-------------------------------------------------------------
|
||||
|
||||
class FeedForward(nn.Module):
|
||||
def __init__(self,
|
||||
hidden_size:int,
|
||||
intermediate_size:int,
|
||||
hidden_act:str='gelu_new',
|
||||
activation_dropout:float=0.0,
|
||||
hidden_dropout:float=0.0
|
||||
):
|
||||
super().__init__()
|
||||
self.intermediate_dropout = nn.Dropout(activation_dropout)
|
||||
self.intermediate_dense = nn.Linear(hidden_size, intermediate_size)
|
||||
self.intermediate_act_fn = ACT2FN[hidden_act]
|
||||
self.output_dense = nn.Linear(intermediate_size, hidden_size)
|
||||
self.output_dropout = nn.Dropout(hidden_dropout)
|
||||
|
||||
def forward(self, hidden_states):
|
||||
hidden_states = self.intermediate_dense(hidden_states)
|
||||
hidden_states = self.intermediate_act_fn(hidden_states)
|
||||
hidden_states = self.intermediate_dropout(hidden_states)
|
||||
hidden_states = self.output_dense(hidden_states)
|
||||
hidden_states = self.output_dropout(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
class TransformerEncoderLayer(nn.Module):
|
||||
def __init__(self,
|
||||
n_state: int,
|
||||
n_mlp : int,
|
||||
n_head: int,
|
||||
channel_last: bool = False,
|
||||
act: str = 'gelu_new',
|
||||
act_do:float = 0.0,
|
||||
att_do:float = 0.0,
|
||||
hid_do:float = 0.0,
|
||||
ln_eps:float=1e-6
|
||||
):
|
||||
|
||||
hidden_size=n_state
|
||||
num_attention_heads=n_head
|
||||
intermediate_size=n_mlp
|
||||
hidden_act=act
|
||||
activation_dropout=act_do
|
||||
attention_dropout=att_do
|
||||
hidden_dropout=hid_do
|
||||
layer_norm_eps=ln_eps
|
||||
|
||||
super().__init__()
|
||||
self.channel_last = channel_last
|
||||
self.attention = MultiHeadAttention(
|
||||
embed_dim=hidden_size,
|
||||
num_heads=num_attention_heads,
|
||||
dropout=attention_dropout
|
||||
)
|
||||
self.layer_norm = nn.LayerNorm(hidden_size, eps=layer_norm_eps)
|
||||
self.feed_forward = FeedForward(
|
||||
hidden_size=hidden_size,
|
||||
hidden_act=hidden_act,
|
||||
intermediate_size=intermediate_size,
|
||||
activation_dropout=activation_dropout,
|
||||
hidden_dropout=hidden_dropout
|
||||
)
|
||||
self.final_layer_norm = nn.LayerNorm(hidden_size, eps=layer_norm_eps)
|
||||
|
||||
def forward(self, hidden_states):
|
||||
if not self.channel_last:
|
||||
hidden_states = hidden_states.permute(0, 2, 1)
|
||||
attn_residual = hidden_states
|
||||
hidden_states = self.attention(hidden_states)
|
||||
hidden_states = attn_residual + hidden_states
|
||||
|
||||
hidden_states = self.layer_norm(hidden_states)
|
||||
hidden_states = hidden_states + self.feed_forward(hidden_states)
|
||||
hidden_states = self.final_layer_norm(hidden_states)
|
||||
|
||||
outputs = hidden_states
|
||||
if not self.channel_last:
|
||||
outputs = outputs.permute(0, 2, 1)
|
||||
return outputs
|
||||
@@ -0,0 +1,110 @@
|
||||
import torch.nn as nn
|
||||
from .layernorm import LayerNorm
|
||||
from .resblocks import ResBasicBlock
|
||||
from .convnext import ConvNeXtLikeBlock
|
||||
from .attention import TransformerEncoderLayer
|
||||
|
||||
#------------------------------------------
|
||||
# Main blocks
|
||||
#------------------------------------------
|
||||
|
||||
class ConvBlock2d(nn.Module):
|
||||
def __init__(self, c, f, block_type="convnext_like", Gdiv=1, kernel_sizes=None):
|
||||
super().__init__()
|
||||
if kernel_sizes is None:
|
||||
kernel_sizes = [(3, 3)]
|
||||
if block_type == "convnext_like":
|
||||
self.conv_block = ConvNeXtLikeBlock(c, dim=2, kernel_sizes=kernel_sizes,
|
||||
Gdiv=Gdiv, padding='same', activation='gelu')
|
||||
elif block_type == "convnext_like_relu":
|
||||
self.conv_block = ConvNeXtLikeBlock(c, dim=2, kernel_sizes=kernel_sizes,
|
||||
Gdiv=Gdiv, padding='same', activation='relu')
|
||||
elif block_type == "basic_resnet":
|
||||
self.conv_block = ResBasicBlock(c, c, f, stride=1, se_channels=min(64,max(c,32)), Gdiv=Gdiv, use_fwSE=False)
|
||||
elif block_type == "basic_resnet_fwse":
|
||||
self.conv_block = ResBasicBlock(c, c, f, stride=1, se_channels=min(64,max(c,32)), Gdiv=Gdiv, use_fwSE=True)
|
||||
else:
|
||||
raise NotImplemented()
|
||||
|
||||
def forward(self, x):
|
||||
return self.conv_block(x)
|
||||
|
||||
#------------------------------------------
|
||||
# 1D block
|
||||
#------------------------------------------
|
||||
|
||||
class PosEncConv(nn.Module):
|
||||
def __init__(self, C, ks, groups=None):
|
||||
super().__init__()
|
||||
assert ks % 2 == 1
|
||||
self.conv = nn.Conv1d(C,C,ks,
|
||||
padding=ks//2,
|
||||
groups=C if groups is None else groups)
|
||||
self.norm = LayerNorm(C, eps=1e-6, data_format="channels_first")
|
||||
|
||||
def forward(self,x):
|
||||
return x + self.norm(self.conv(x))
|
||||
|
||||
class TimeContextBlock1d(nn.Module):
|
||||
def __init__(self,
|
||||
C,
|
||||
hC,
|
||||
pos_ker_sz = 59,
|
||||
block_type = 'att',
|
||||
red_dim_conv = None,
|
||||
exp_dim_conv = None
|
||||
):
|
||||
super().__init__()
|
||||
assert pos_ker_sz
|
||||
|
||||
self.red_dim_conv = nn.Sequential(
|
||||
nn.Conv1d(C,hC,1),
|
||||
LayerNorm(hC, eps=1e-6, data_format="channels_first")
|
||||
)
|
||||
if block_type == 'fc':
|
||||
self.tcm = nn.Sequential(
|
||||
nn.Conv1d(hC,hC*2,1),
|
||||
LayerNorm(hC*2, eps=1e-6,
|
||||
data_format="channels_first"),
|
||||
nn.GELU(),
|
||||
nn.Conv1d(hC*2,hC,1)
|
||||
)
|
||||
elif block_type == 'conv':
|
||||
# Just large kernel size conv like in convformer
|
||||
self.tcm = nn.Sequential(*[ConvNeXtLikeBlock(
|
||||
hC, dim=1, kernel_sizes=[7, 15, 31], Gdiv=1, padding='same'
|
||||
) for i in range(4)])
|
||||
elif block_type == 'att':
|
||||
# Basic Transformer self-attention encoder block
|
||||
self.tcm = nn.Sequential(
|
||||
PosEncConv(hC, ks=pos_ker_sz, groups=hC),
|
||||
TransformerEncoderLayer(
|
||||
n_state=hC,
|
||||
n_mlp=hC*2,
|
||||
n_head=4
|
||||
)
|
||||
)
|
||||
elif block_type == 'conv+att':
|
||||
# Basic Transformer self-attention encoder block
|
||||
self.tcm = nn.Sequential(
|
||||
ConvNeXtLikeBlock(hC, dim=1, kernel_sizes=[7], Gdiv=1, padding='same'),
|
||||
ConvNeXtLikeBlock(hC, dim=1, kernel_sizes=[19], Gdiv=1, padding='same'),
|
||||
ConvNeXtLikeBlock(hC, dim=1, kernel_sizes=[31], Gdiv=1, padding='same'),
|
||||
ConvNeXtLikeBlock(hC, dim=1, kernel_sizes=[59], Gdiv=1, padding='same'),
|
||||
TransformerEncoderLayer(
|
||||
n_state=hC,
|
||||
n_mlp=hC,
|
||||
n_head=4
|
||||
)
|
||||
)
|
||||
else:
|
||||
raise NotImplemented()
|
||||
|
||||
self.exp_dim_conv = nn.Conv1d(hC,C,1)
|
||||
|
||||
def forward(self,x):
|
||||
skip = x
|
||||
x = self.red_dim_conv(x)
|
||||
x = self.tcm(x)
|
||||
x = self.exp_dim_conv(x)
|
||||
return skip + x
|
||||
@@ -0,0 +1,78 @@
|
||||
# MIT License
|
||||
#
|
||||
# Copyright (c) 2024 ID R&D, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
import math
|
||||
import torch
|
||||
import functools
|
||||
import numpy as np
|
||||
import torch.nn as nn
|
||||
from typing import List
|
||||
from torch import Tensor
|
||||
import torch.nn.functional as F
|
||||
from collections import OrderedDict
|
||||
from typing import Iterable, Optional
|
||||
|
||||
#------------------------------------------
|
||||
# ConvNeXtV2 block
|
||||
#------------------------------------------
|
||||
|
||||
MaxPoolNd = {
|
||||
1 : nn.MaxPool1d,
|
||||
2 : nn.MaxPool2d
|
||||
}
|
||||
|
||||
ConvNd = {
|
||||
1 : nn.Conv1d,
|
||||
2 : nn.Conv2d
|
||||
}
|
||||
|
||||
BatchNormNd = {
|
||||
1 : nn.BatchNorm1d,
|
||||
2 : nn.BatchNorm2d
|
||||
}
|
||||
|
||||
# https://github.com/facebookresearch/ConvNeXt/blob/main/models/convnext.py
|
||||
# https://github.com/facebookresearch/ConvNeXt-V2/blob/main/models/convnextv2.py
|
||||
class ConvNeXtLikeBlock(nn.Module):
|
||||
def __init__(self, C, dim=2, kernel_sizes=[(3,3),], Gdiv=1, padding='same', activation='gelu'):
|
||||
super().__init__()
|
||||
# if C//Gdiv==0:
|
||||
# Gdiv = C
|
||||
self.dwconvs = nn.ModuleList(modules=[
|
||||
ConvNd[dim](C, C, kernel_size=ks,
|
||||
padding=padding, groups=C//Gdiv if Gdiv is not None else 1) for ks in kernel_sizes
|
||||
])
|
||||
self.norm = BatchNormNd[dim](C * len(kernel_sizes))
|
||||
if activation == 'gelu':
|
||||
self.act = nn.GELU()
|
||||
elif activation == 'relu':
|
||||
self.act = nn.ReLU()
|
||||
self.pwconv1 = ConvNd[dim](C * len(kernel_sizes), C, 1) # pointwise/1x1 convs, implemented with linear layers
|
||||
|
||||
|
||||
def forward(self, x):
|
||||
skip = x
|
||||
x = torch.cat([dwconv(x) for dwconv in self.dwconvs],dim=1)
|
||||
x = self.act(self.norm(x))
|
||||
x = self.pwconv1(x)
|
||||
x = skip + x
|
||||
return x
|
||||
+145
@@ -0,0 +1,145 @@
|
||||
# MIT License
|
||||
#
|
||||
# Copyright (c) 2024 ID R&D, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
import math, torch, torchaudio
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
class NormalizeAudio(nn.Module):
|
||||
def __init__(self,eps:float=1e-6):
|
||||
super().__init__()
|
||||
self.eps = eps
|
||||
|
||||
def forward(self,x):
|
||||
if x.ndim == 2:
|
||||
x = x.unsqueeze(1)
|
||||
return (x - x.mean(dim=2, keepdims=True)) /\
|
||||
(x.std(dim=2, keepdims=True, unbiased=False) + self.eps)
|
||||
|
||||
class PreEmphasis(torch.nn.Module):
|
||||
|
||||
def __init__(self, coef: float = 0.97):
|
||||
super().__init__()
|
||||
self.coef = coef
|
||||
self.register_buffer(
|
||||
'flipped_filter', torch.FloatTensor([-self.coef, 1.]).unsqueeze(0).unsqueeze(0)
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
if x.ndim == 2:
|
||||
x = x.unsqueeze(1)
|
||||
x = F.pad(x, (1, 0), 'reflect')
|
||||
return F.conv1d(x, self.flipped_filter).squeeze(1)
|
||||
|
||||
class FbankAug(nn.Module):
|
||||
def __init__(self, freq_mask_width = (0, 8), time_mask_width = (0, 10), freq_start_bin=0):
|
||||
self.time_mask_width = time_mask_width
|
||||
self.freq_mask_width = freq_mask_width
|
||||
self.freq_start_bin = freq_start_bin
|
||||
super().__init__()
|
||||
|
||||
def mask_along_axis(self, x, dim):
|
||||
original_size = x.shape
|
||||
batch, fea, time = x.shape
|
||||
if dim == 1:
|
||||
D = fea
|
||||
width_range = self.freq_mask_width
|
||||
else:
|
||||
D = time
|
||||
width_range = self.time_mask_width
|
||||
|
||||
mask_len = torch.randint(width_range[0], width_range[1], (batch, 1), device=x.device).unsqueeze(2)
|
||||
mask_pos = torch.randint(self.freq_start_bin, max(1, D - mask_len.max()), (batch, 1), device=x.device).unsqueeze(2)
|
||||
arange = torch.arange(D, device=x.device).view(1, 1, -1)
|
||||
mask = (mask_pos <= arange) * (arange < (mask_pos + mask_len))
|
||||
mask = mask.any(dim=1)
|
||||
|
||||
if dim == 1:
|
||||
mask = mask.unsqueeze(2)
|
||||
else:
|
||||
mask = mask.unsqueeze(1)
|
||||
|
||||
x = x.masked_fill_(mask, 0.0)
|
||||
return x.view(*original_size)
|
||||
|
||||
def forward(self, x):
|
||||
x = self.mask_along_axis(x, dim=2)
|
||||
x = self.mask_along_axis(x, dim=1)
|
||||
return x
|
||||
|
||||
class MelBanks(nn.Module):
|
||||
def __init__(self,
|
||||
sample_rate=16000,
|
||||
n_fft=512,
|
||||
win_length=400,
|
||||
hop_length=160,
|
||||
f_min = 20,
|
||||
f_max = 7600,
|
||||
n_mels=80,
|
||||
do_spec_aug=False,
|
||||
norm_signal=False,
|
||||
do_preemph=True,
|
||||
spec_norm='mn',
|
||||
freq_start_bin = 0,
|
||||
num_apply_spec_aug = 1,
|
||||
freq_mask_width = (0, 8),
|
||||
time_mask_width = (0, 10),
|
||||
):
|
||||
super(MelBanks, self).__init__()
|
||||
self.num_apply_spec_aug = num_apply_spec_aug
|
||||
self.torchfbank = torch.nn.Sequential(
|
||||
NormalizeAudio() if norm_signal else nn.Identity(),
|
||||
PreEmphasis() if do_preemph else nn.Identity(),
|
||||
torchaudio.transforms.MelSpectrogram(sample_rate=sample_rate, n_fft=n_fft, win_length=win_length, hop_length=hop_length, \
|
||||
f_min = f_min, f_max = f_max, n_mels=n_mels, window_fn=torch.hamming_window),
|
||||
)
|
||||
self.spec_norm = spec_norm
|
||||
if spec_norm == 'mn':
|
||||
self.spec_norm = lambda x : x - torch.mean(x, dim=-1, keepdim=True)
|
||||
elif spec_norm == 'mvn':
|
||||
self.spec_norm = lambda x : (x - torch.mean(x, dim=-1, keepdim=True)) / (torch.std(x, dim=-1, keepdim=True)+1e-8)
|
||||
elif spec_norm == 'bn':
|
||||
self.spec_norm = nn.BatchNorm1d(n_mels)
|
||||
else:
|
||||
pass
|
||||
if do_spec_aug:
|
||||
self.specaug = FbankAug(
|
||||
freq_start_bin=freq_start_bin,
|
||||
freq_mask_width=freq_mask_width,
|
||||
time_mask_width=time_mask_width) # Spec augmentation
|
||||
else:
|
||||
self.specaug = nn.Identity()
|
||||
|
||||
|
||||
def forward(self, x):
|
||||
xdtype = x.dtype
|
||||
x = x.float()
|
||||
with torch.no_grad(), torch.amp.autocast('cuda', enabled=False):
|
||||
x = self.torchfbank(x)+1e-6
|
||||
x = x.log()
|
||||
# print(f"spec : {x.size()}")
|
||||
# x = x - torch.mean(x, dim=-1, keepdim=True)
|
||||
x = self.spec_norm(x)
|
||||
if self.training:
|
||||
for _ in range(self.num_apply_spec_aug):
|
||||
x = self.specaug(x)
|
||||
return x.to(xdtype)
|
||||
+458
@@ -0,0 +1,458 @@
|
||||
# MIT License
|
||||
#
|
||||
# Copyright (c) 2024 ID R&D, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
import math
|
||||
import torch
|
||||
import numpy as np
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from scipy.signal import windows
|
||||
|
||||
def hz2mel(hz):
|
||||
"""Convert a value in Hertz to Mels
|
||||
:param hz: a value in Hz. This can also be a numpy array, conversion proceeds element-wise.
|
||||
:returns: a value in Mels. If an array was passed in, an identical sized array is returned.
|
||||
"""
|
||||
return 2595 * np.log10(1 + hz / 700.)
|
||||
|
||||
def mel2hz(mel):
|
||||
"""Convert a value in Mels to Hertz
|
||||
:param mel: a value in Mels. This can also be a numpy array, conversion proceeds element-wise.
|
||||
:returns: a value in Hertz. If an array was passed in, an identical sized array is returned.
|
||||
"""
|
||||
return 700 * (10 ** (mel / 2595.0) - 1)
|
||||
|
||||
def get_filterbanks(low_freq: int = 20,
|
||||
high_freq: int = 7600,
|
||||
nfilt: int = 80,
|
||||
nfft: int = 512,
|
||||
samplerate: int = 16000):
|
||||
"""Compute a Mel-filterbank. The filters are stored in the rows, the columns correspond
|
||||
to fft bins. The filters are returned as an array of size nfilt * (nfft/2 + 1)
|
||||
:param nfilt: the number of filters in the filterbank, default 20.
|
||||
:param nfft: the FFT size. Default is 512.
|
||||
:param samplerate: the samplerate of the signal we are working with. Affects mel spacing.
|
||||
:param low_freq: lowest band edge of mel filters, default 0 Hz
|
||||
:param high_freq: highest band edge of mel filters, default samplerate/2
|
||||
:returns: A numpy array of size nfilt * (nfft/2 + 1) containing filterbank. Each row holds 1 filter.
|
||||
"""
|
||||
|
||||
# compute points evenly spaced in mels
|
||||
lowmel = hz2mel(low_freq)
|
||||
highmel = hz2mel(high_freq)
|
||||
melpoints = np.linspace(lowmel, highmel, nfilt + 2)
|
||||
|
||||
lower_edge_mel = melpoints[:-2].reshape(1, -1)
|
||||
center_mel = melpoints[1:-1].reshape(1, -1)
|
||||
upper_edge_mel = melpoints[2:].reshape(1, -1)
|
||||
|
||||
spectrogram_bins_mel = hz2mel(np.linspace(0, samplerate // 2, nfft))[1:].reshape(-1, 1)
|
||||
|
||||
lower_slopes = (spectrogram_bins_mel - lower_edge_mel) / (
|
||||
center_mel - lower_edge_mel)
|
||||
upper_slopes = (upper_edge_mel - spectrogram_bins_mel) / (
|
||||
upper_edge_mel - center_mel)
|
||||
|
||||
mel_weights_matrix = np.maximum(0.0, np.minimum(lower_slopes, upper_slopes))
|
||||
return np.vstack([np.zeros((1, nfilt)), mel_weights_matrix])[:, :].astype('float32')
|
||||
|
||||
class SpectralFeaturesTF(nn.Module):
|
||||
def __init__(self,
|
||||
frame_length: int = 400,
|
||||
frame_step: int = 160,
|
||||
fft_length: int = 512,
|
||||
sample_rate: int = 16000,
|
||||
window: str = 'hann',
|
||||
normalize_spectrogram: bool = False,
|
||||
normalize_signal: bool = False,
|
||||
eps: float = 1e-8,
|
||||
mode: str = 'melbanks',
|
||||
low_freq: int = 20,
|
||||
high_freq: int = 7600,
|
||||
num_bins: int = 80,
|
||||
log_mels: bool = True,
|
||||
fft_mode: str = 'abs',
|
||||
sqrt_real_imag: bool = False,
|
||||
return_img: bool = False,
|
||||
**kwargs):
|
||||
"""
|
||||
Requirements
|
||||
------------
|
||||
input shape must meet the conditions: mod((input.shape[0] - length), shift) == 0
|
||||
fft_length >= frame_length
|
||||
|
||||
Parameters
|
||||
------------
|
||||
:param frame_length: Length of each segment in # of samples
|
||||
:param frame_step: Shift between segments in # of samples
|
||||
:param fft_length: number of dft points, if None => fft_length == frame_length
|
||||
:param fft_mode: "abs" - amplitude spectrum; "real" - only real part, "imag" - only imag part,
|
||||
"complex" - concatenate real and imag part.
|
||||
:param kwargs: unuse
|
||||
|
||||
Input
|
||||
-----
|
||||
input mut have shape: [n_batch, signal_length, 1]
|
||||
|
||||
Returns
|
||||
-------
|
||||
A keras model that has output shape of
|
||||
(None, nfft / 2, n_time) (if type == "abs" || "real" || "imag") or
|
||||
(None, nfft / 2, n_frame, 2) (if type = "abs" & `img_dim_ordering() == 'tf').
|
||||
(None, nfft / 2, n_frame, 2) (if type = "complex" & `img_dim_ordering() == 'tf').
|
||||
|
||||
number of time point of output spectrogram: n_time = (input.shape[0] - length) / shift + 1
|
||||
"""
|
||||
super().__init__()
|
||||
|
||||
assert mode in ['fft', 'melbanks', 'mfcc', 'complex']
|
||||
assert isinstance(frame_length, int) and isinstance(frame_step, int) and isinstance(fft_length, int)
|
||||
|
||||
self.length = frame_length
|
||||
self.shift = frame_step
|
||||
self.sqrt_real_imag = sqrt_real_imag
|
||||
self.normalize_spectrogram = normalize_spectrogram
|
||||
self.normalize_signal = normalize_signal
|
||||
self.window = window
|
||||
self.eps = eps
|
||||
if fft_length is None:
|
||||
self.nfft = frame_length
|
||||
else:
|
||||
self.nfft = fft_length
|
||||
|
||||
self.samplerate = sample_rate
|
||||
self.features = mode
|
||||
self.low_freq = low_freq
|
||||
self.high_freq = high_freq
|
||||
self.num_bins = num_bins
|
||||
|
||||
self.return_img = return_img
|
||||
|
||||
if mode in ['melbanks', 'mfcc']:
|
||||
fft_mode = 'abs'
|
||||
self.fft_mode = fft_mode
|
||||
self.log_mels = log_mels
|
||||
self.build()
|
||||
|
||||
def build(self):
|
||||
assert self.nfft >= self.length
|
||||
|
||||
if self.window:
|
||||
if self.window == 'hamming':
|
||||
self.window = windows.hamming(self.length)
|
||||
elif self.window in ['hann', 'hanning']:
|
||||
self.window = np.array([0.5 - 0.5 * (np.cos((2 * np.pi * l) / (self.length - 1) )) \
|
||||
for l in range(self.length)])
|
||||
elif self.window == 'sqrt_hann':
|
||||
self.window = np.array([0.5 - 0.5 * (np.cos((2 * np.pi * l) / (self.length - 1) )) \
|
||||
for l in range(self.length)]) ** 0.5
|
||||
elif self.window == 'kaiser':
|
||||
self.window = windows.kaiser(self.length)
|
||||
else:
|
||||
self.window = np.ones(self.length)
|
||||
self.window = self.window.astype("float32")
|
||||
|
||||
# real kernel
|
||||
real_kernel = np.asarray([np.cos(2 * np.pi * np.arange(0, self.nfft) * n / self.nfft)
|
||||
for n in range(self.nfft)]).astype("float32").T
|
||||
self.real_kernel = real_kernel[:self.length, :self.nfft // 2]
|
||||
if self.window is not None:
|
||||
self.real_kernel *= self.window[:, None]
|
||||
self.real_kernel = self.real_kernel[:,None,:]
|
||||
|
||||
# imag kernel
|
||||
image_kernel = np.asarray([np.sin(2 * np.pi * np.arange(0, self.nfft) * n / self.nfft)
|
||||
for n in range(self.nfft)]).astype("float32").T
|
||||
self.image_kernel = image_kernel[:self.length, :self.nfft // 2]
|
||||
if self.window is not None:
|
||||
self.image_kernel *= self.window[:, None]
|
||||
self.image_kernel = self.image_kernel[:,None,:]
|
||||
|
||||
self.register_buffer('real_kernel_pt',
|
||||
torch.from_numpy(self.real_kernel).permute(2,1,0).float())
|
||||
self.register_buffer('image_kernel_pt',
|
||||
torch.from_numpy(self.image_kernel).permute(2,1,0).float())
|
||||
|
||||
if self.features in ['melbanks']:
|
||||
linear_to_mel_weight_matrix = get_filterbanks(
|
||||
nfilt=self.num_bins,
|
||||
nfft=self.nfft // 2,
|
||||
samplerate=self.samplerate,
|
||||
low_freq=self.low_freq,
|
||||
high_freq=self.high_freq)
|
||||
linear_to_mel_weight_matrix = linear_to_mel_weight_matrix[:,:,None]
|
||||
self.register_buffer('melbanks_pt',
|
||||
torch.from_numpy(linear_to_mel_weight_matrix).permute(1,0,2).float())
|
||||
|
||||
def forward(self, inputs):
|
||||
# inputs.size() : (bs,1,T)
|
||||
dtype = inputs.dtype
|
||||
inputs = inputs.float()
|
||||
if inputs.ndim == 2:
|
||||
inputs = inputs.unsqueeze(1)
|
||||
|
||||
if self.normalize_signal:
|
||||
inputs = (inputs - inputs.mean(dim=2, keepdims=True)) /\
|
||||
(inputs.std(dim=2, keepdims=True, unbiased=False) + self.eps)
|
||||
|
||||
real_part = F.conv1d(inputs, self.real_kernel_pt, stride=self.shift, padding=self.shift//2)
|
||||
imag_part = F.conv1d(inputs, self.image_kernel_pt, stride=self.shift, padding=self.shift//2)
|
||||
|
||||
if self.features == 'complex':
|
||||
return [real_part, imag_part]
|
||||
|
||||
fft = torch.square(real_part) + torch.square(imag_part)
|
||||
if self.sqrt_real_imag:
|
||||
fft = torch.sqrt(fft)
|
||||
|
||||
feat = fft.clip(self.eps, 1/self.eps)
|
||||
|
||||
if self.fft_mode == 'log':
|
||||
feat = torch.log(feat)
|
||||
|
||||
if self.features in ['melbanks']:
|
||||
mel_spectrograms = F.conv1d(feat, self.melbanks_pt, stride=1, padding=0)
|
||||
mel_spectrograms = mel_spectrograms.clip(self.eps, 1/self.eps)
|
||||
if self.log_mels:
|
||||
feat = torch.log(mel_spectrograms)
|
||||
else:
|
||||
feat = mel_spectrograms
|
||||
|
||||
if self.normalize_spectrogram:
|
||||
feat = (feat - feat.mean(dim=(1, 2), keepdims=True)) /\
|
||||
(feat.std(dim=(1, 2), keepdims=True, unbiased=False) + self.eps)
|
||||
if self.return_img:
|
||||
feat = feat[:,None,:,:]
|
||||
return feat.to(dtype)
|
||||
|
||||
class LogSpec(nn.Module):
|
||||
def __init__(self,eps:float=1e-10):
|
||||
super().__init__()
|
||||
self.eps = eps
|
||||
|
||||
def forward(self,x):
|
||||
return x.clip(self.eps,1e+8).log()
|
||||
|
||||
class NormalizeAudio(nn.Module):
|
||||
def __init__(self,eps:float=1e-10):
|
||||
super().__init__()
|
||||
self.eps = eps
|
||||
|
||||
def forward(self,x):
|
||||
if x.ndim == 2:
|
||||
x = x.unsqueeze(1)
|
||||
return ((x - x.mean(dim=2, keepdims=True)) /\
|
||||
(x.std(dim=2, keepdims=True, unbiased=False) + self.eps)).squeeze(1)
|
||||
|
||||
class PreEmphasis(torch.nn.Module):
|
||||
def __init__(self, coef: float = 0.97):
|
||||
super().__init__()
|
||||
self.coef = coef
|
||||
self.register_buffer(
|
||||
'flipped_filter', torch.FloatTensor([-self.coef, 1.]).unsqueeze(0).unsqueeze(0)
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
if x.ndim == 2:
|
||||
x = x.unsqueeze(1)
|
||||
x = F.pad(x, (1, 0), 'reflect')
|
||||
return F.conv1d(x, self.flipped_filter).squeeze(1)
|
||||
|
||||
class FbankAug(nn.Module):
|
||||
def __init__(self, freq_mask_width = (0, 8), time_mask_width = (0, 10), freq_start_bin=0):
|
||||
self.time_mask_width = time_mask_width
|
||||
self.freq_mask_width = freq_mask_width
|
||||
self.freq_start_bin = freq_start_bin
|
||||
super().__init__()
|
||||
|
||||
def mask_along_axis(self, x, dim):
|
||||
original_size = x.shape
|
||||
batch, fea, time = x.shape
|
||||
if dim == 1:
|
||||
D = fea
|
||||
width_range = self.freq_mask_width
|
||||
else:
|
||||
D = time
|
||||
width_range = self.time_mask_width
|
||||
|
||||
mask_len = torch.randint(width_range[0], width_range[1], (batch, 1), device=x.device).unsqueeze(2)
|
||||
mask_pos = torch.randint(self.freq_start_bin, max(1, D - mask_len.max()), (batch, 1), device=x.device).unsqueeze(2)
|
||||
arange = torch.arange(D, device=x.device).view(1, 1, -1)
|
||||
mask = (mask_pos <= arange) * (arange < (mask_pos + mask_len))
|
||||
mask = mask.any(dim=1)
|
||||
|
||||
if dim == 1:
|
||||
mask = mask.unsqueeze(2)
|
||||
else:
|
||||
mask = mask.unsqueeze(1)
|
||||
|
||||
x = x.masked_fill_(mask, 0.0)
|
||||
return x.view(*original_size)
|
||||
|
||||
def forward(self, x):
|
||||
x = self.mask_along_axis(x, dim=2)
|
||||
x = self.mask_along_axis(x, dim=1)
|
||||
return x
|
||||
|
||||
class TFMelBanks(nn.Module):
|
||||
def __init__(self,
|
||||
sample_rate=16000,
|
||||
n_fft=512,
|
||||
win_length=400,
|
||||
hop_length=160,
|
||||
f_min = 20,
|
||||
f_max = 7600,
|
||||
n_mels = 80,
|
||||
do_spec_aug=False,
|
||||
norm_signal=False,
|
||||
do_preemph=True,
|
||||
freq_start_bin = 0,
|
||||
freq_mask_width = (0, 8),
|
||||
time_mask_width = (0, 10),
|
||||
eps = 1e-8
|
||||
):
|
||||
super(TFMelBanks, self).__init__()
|
||||
self.torchfbank = torch.nn.Sequential(
|
||||
NormalizeAudio(eps) if norm_signal else nn.Identity(),
|
||||
PreEmphasis() if do_preemph else nn.Identity(),
|
||||
|
||||
SpectralFeaturesTF(
|
||||
frame_length = win_length,
|
||||
frame_step = hop_length,
|
||||
fft_length = n_fft,
|
||||
sample_rate = sample_rate,
|
||||
window = 'hamming',
|
||||
normalize_spectrogram = False,
|
||||
normalize_signal = False,
|
||||
eps = eps,
|
||||
mode = 'melbanks',
|
||||
low_freq = f_min,
|
||||
high_freq = f_max,
|
||||
num_bins = n_mels,
|
||||
log_mels = False,
|
||||
fft_mode = 'abs',
|
||||
sqrt_real_imag = False,
|
||||
return_img = False,
|
||||
)
|
||||
)
|
||||
self.eps = eps
|
||||
if do_spec_aug:
|
||||
self.specaug = FbankAug(
|
||||
freq_start_bin=freq_start_bin,
|
||||
freq_mask_width=freq_mask_width,
|
||||
time_mask_width=time_mask_width) # Spec augmentation
|
||||
else:
|
||||
self.specaug = nn.Identity()
|
||||
|
||||
def forward(self, x):
|
||||
xdtype = x.dtype
|
||||
x = x.float()
|
||||
with torch.no_grad():
|
||||
with torch.cuda.amp.autocast(enabled=False):
|
||||
x = self.torchfbank(x)+self.eps
|
||||
x = x.log()
|
||||
x = x - torch.mean(x, dim=-1, keepdim=True)
|
||||
if self.training:
|
||||
x = self.specaug(x)
|
||||
return x.to(xdtype)
|
||||
|
||||
class TFSpectrogram(nn.Module):
|
||||
def __init__(self,
|
||||
sample_rate=16000,
|
||||
n_fft=512,
|
||||
win_length=400,
|
||||
hop_length=160,
|
||||
f_min = 20,
|
||||
f_max = 7600,
|
||||
n_mels = 80,
|
||||
window = 'hamming',
|
||||
normalize_spectrogram = False,
|
||||
normalize_signal = False,
|
||||
mode = 'fft',
|
||||
fft_mode = 'abs',
|
||||
pool_freqs = (2,1),
|
||||
|
||||
do_spec_aug=False,
|
||||
norm_signal=False,
|
||||
do_preemph=True,
|
||||
|
||||
freq_start_bin = 0,
|
||||
num_apply_spec_aug = 1,
|
||||
freq_mask_width = (0, 8),
|
||||
time_mask_width = (0, 10),
|
||||
eps = 1e-8
|
||||
):
|
||||
super(TFSpectrogram, self).__init__()
|
||||
self.num_apply_spec_aug = num_apply_spec_aug
|
||||
self.spectrogram = torch.nn.Sequential(
|
||||
NormalizeAudio() if norm_signal else nn.Identity(),
|
||||
PreEmphasis() if do_preemph else nn.Identity(),
|
||||
|
||||
SpectralFeaturesTF(
|
||||
frame_length = win_length,
|
||||
frame_step = hop_length,
|
||||
fft_length = n_fft,
|
||||
sample_rate = sample_rate,
|
||||
window = window,
|
||||
eps = eps,
|
||||
mode = mode,
|
||||
low_freq = f_min,
|
||||
high_freq = f_max,
|
||||
num_bins = n_mels,
|
||||
|
||||
normalize_spectrogram = False,
|
||||
normalize_signal = False,
|
||||
|
||||
fft_mode = 'abs',
|
||||
log_mels = False,
|
||||
sqrt_real_imag = False,
|
||||
return_img = False,
|
||||
)
|
||||
)
|
||||
|
||||
if pool_freqs is not None:
|
||||
self.pool_freq = nn.AvgPool2d(pool_freqs, stride=pool_freqs)
|
||||
else:
|
||||
self.pool_freq = nn.Identity()
|
||||
|
||||
self.eps = eps
|
||||
if do_spec_aug:
|
||||
self.specaug = FbankAug(
|
||||
freq_start_bin=freq_start_bin,
|
||||
freq_mask_width=freq_mask_width,
|
||||
time_mask_width=time_mask_width) # Spec augmentation
|
||||
else:
|
||||
self.specaug = nn.Identity()
|
||||
|
||||
def forward(self, x):
|
||||
xdtype = x.dtype
|
||||
x = x.float()
|
||||
with torch.no_grad():
|
||||
with torch.cuda.amp.autocast(enabled=False):
|
||||
x = self.spectrogram(x)+self.eps
|
||||
x = x.log()
|
||||
x = x - torch.mean(x, dim=-1, keepdim=True)
|
||||
if self.training:
|
||||
for _ in range(self.num_apply_spec_aug):
|
||||
x = self.specaug(x)
|
||||
x = self.pool_freq(x.unsqueeze(1))
|
||||
return x.to(xdtype)
|
||||
+67
@@ -0,0 +1,67 @@
|
||||
# MIT License
|
||||
#
|
||||
# Copyright (c) 2024 ID R&D, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
import math
|
||||
import torch
|
||||
import functools
|
||||
import numpy as np
|
||||
import torch.nn as nn
|
||||
from typing import List
|
||||
from torch import Tensor
|
||||
import torch.nn.functional as F
|
||||
from collections import OrderedDict
|
||||
from typing import Iterable, Optional
|
||||
|
||||
class LayerNorm(nn.Module): # ⚡
|
||||
""" LayerNorm that supports two data formats: channels_last (default) or channels_first.
|
||||
The ordering of the dimensions in the inputs. channels_last corresponds to inputs with
|
||||
shape (batch_size, T, channels) while channels_first corresponds to inputs
|
||||
with shape (batch_size, channels, T).
|
||||
"""
|
||||
def __init__(self, C, eps=1e-6, data_format="channels_last"):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(C))
|
||||
self.bias = nn.Parameter(torch.zeros(C))
|
||||
self.eps = eps
|
||||
self.data_format = data_format
|
||||
if self.data_format not in ["channels_last", "channels_first"]:
|
||||
raise NotImplementedError
|
||||
self.C = (C, )
|
||||
|
||||
def forward(self, x):
|
||||
if self.data_format == "channels_last":
|
||||
return F.layer_norm(x, self.C, self.weight, self.bias, self.eps)
|
||||
elif self.data_format == "channels_first":
|
||||
u = x.mean(1, keepdim=True)
|
||||
s = (x - u).pow(2).mean(1, keepdim=True)
|
||||
x = (x - u) / torch.sqrt(s + self.eps)
|
||||
|
||||
w = self.weight
|
||||
b = self.bias
|
||||
for _ in range(x.ndim-2):
|
||||
w = w.unsqueeze(-1)
|
||||
b = b.unsqueeze(-1)
|
||||
x = w * x + b # ⚡
|
||||
return x
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
return ", ".join([f"{k}={v}" for k,v in {"C" : self.C, "data_format" : self.data_format, "eps" : self.eps}.items()])
|
||||
+426
@@ -0,0 +1,426 @@
|
||||
# Copyright (c) 2021 Shuai Wang (wsstriving@gmail.com)
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Pooling functions to aggregate frame-level deep features
|
||||
into segment-level speaker embeddings
|
||||
|
||||
High-order statistics are surprisingly effective, TSDP acts similarly as TSTP,
|
||||
even though we remove the mean statistic, on Voxceleb.
|
||||
"""
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
|
||||
def _prepare_weights(weights: torch.Tensor, num_frames: int) -> torch.Tensor:
|
||||
"""Reshape and (linearly) interpolate frame weights to match features.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
weights : (batch, frames) torch.Tensor
|
||||
Per-frame weights, e.g. the activation of the speaker of interest.
|
||||
num_frames : int
|
||||
Number of frames in the (temporal) feature axis the weights should match.
|
||||
|
||||
Returns
|
||||
-------
|
||||
weights : (batch, 1, num_frames) torch.Tensor
|
||||
Weights interpolated to `num_frames` and ready to broadcast over channels.
|
||||
|
||||
Note
|
||||
----
|
||||
`weights` and the features they apply to might use a different number of frames,
|
||||
in which case `weights` are interpolated to reach the number of feature frames.
|
||||
"""
|
||||
weights = weights.unsqueeze(dim=1)
|
||||
# (batch, frames) -> (batch, 1, frames)
|
||||
|
||||
if weights.shape[-1] != num_frames:
|
||||
weights = F.interpolate(weights, size=num_frames, mode="nearest")
|
||||
|
||||
return weights
|
||||
|
||||
|
||||
class TAP(nn.Module):
|
||||
"""
|
||||
Temporal average pooling, only first-order mean is considered
|
||||
"""
|
||||
|
||||
def __init__(self, in_dim=0, **kwargs):
|
||||
super(TAP, self).__init__()
|
||||
self.in_dim = in_dim
|
||||
|
||||
def forward(self, x, weights=None):
|
||||
if weights is not None:
|
||||
# weighted temporal average over the (last) time axis
|
||||
if x.ndim == 4:
|
||||
x = x.reshape(x.shape[0], x.shape[1] * x.shape[2], x.shape[3])
|
||||
w = _prepare_weights(weights, x.shape[-1])
|
||||
v1 = w.sum(dim=-1) + 1e-8
|
||||
pooling_mean = torch.sum(x * w, dim=-1) / v1
|
||||
return pooling_mean.flatten(start_dim=1)
|
||||
if x.ndim == 3:
|
||||
pooling_mean = x.mean(dim=-1)
|
||||
elif x.ndim == 4:
|
||||
pooling_mean = x.mean(dim=(-1,-2))
|
||||
# To be compatable with 2D input
|
||||
pooling_mean = pooling_mean.flatten(start_dim=1)
|
||||
return pooling_mean
|
||||
|
||||
def get_out_dim(self):
|
||||
self.out_dim = self.in_dim
|
||||
return self.out_dim
|
||||
|
||||
|
||||
class TMAP(nn.Module):
|
||||
"""
|
||||
Temporal max-average pooling, only first-order mean is considered
|
||||
"""
|
||||
|
||||
def __init__(self, in_dim=0, **kwargs):
|
||||
super(TMAP, self).__init__()
|
||||
self.in_dim = in_dim
|
||||
|
||||
def forward(self, x):
|
||||
if x.ndim == 3:
|
||||
(x1, _) = torch.max(x, dim=-1)
|
||||
x2 = torch.mean(x, dim=-1)
|
||||
elif x.ndim == 4:
|
||||
# (x1, _) = torch.max(x, dim=(-1,-2))
|
||||
x1 = torch.max(
|
||||
torch.max(x, dim=-1)[0],
|
||||
dim=-1)[0]
|
||||
x2 = torch.mean(x, dim=(-1,-2))
|
||||
max_avg = x1 + x2
|
||||
return max_avg
|
||||
|
||||
def get_out_dim(self):
|
||||
self.out_dim = self.in_dim
|
||||
return self.out_dim
|
||||
|
||||
class TSDP(nn.Module):
|
||||
"""
|
||||
Temporal standard deviation pooling, only second-order std is considered
|
||||
"""
|
||||
|
||||
def __init__(self, in_dim=0, **kwargs):
|
||||
super(TSDP, self).__init__()
|
||||
self.in_dim = in_dim
|
||||
|
||||
def forward(self, x, weights=None):
|
||||
if weights is not None:
|
||||
if x.ndim == 4:
|
||||
x = x.reshape(x.shape[0], x.shape[1] * x.shape[2], x.shape[3])
|
||||
w = _prepare_weights(weights, x.shape[-1])
|
||||
v1 = w.sum(dim=-1) + 1e-8
|
||||
mean = torch.sum(x * w, dim=-1) / v1
|
||||
v2 = torch.square(w).sum(dim=-1)
|
||||
var = torch.sum(torch.square(x - mean.unsqueeze(-1)) * w, dim=-1) / (
|
||||
v1 - v2 / v1 + 1e-8
|
||||
)
|
||||
pooling_std = torch.sqrt(var.clamp(min=1e-7))
|
||||
return pooling_std.flatten(start_dim=1)
|
||||
# The last dimension is the temporal axis
|
||||
pooling_std = torch.sqrt(torch.var(x, dim=-1) + 1e-7)
|
||||
pooling_std = pooling_std.flatten(start_dim=1)
|
||||
return pooling_std
|
||||
|
||||
def get_out_dim(self):
|
||||
self.out_dim = self.in_dim
|
||||
return self.out_dim
|
||||
|
||||
|
||||
class TSTP(nn.Module):
|
||||
"""
|
||||
Temporal statistics pooling, concatenate mean and std, which is used in
|
||||
x-vector
|
||||
Comment: simple concatenation can not make full use of both statistics
|
||||
"""
|
||||
|
||||
def __init__(self, in_dim=0, **kwargs):
|
||||
super(TSTP, self).__init__()
|
||||
self.in_dim = in_dim
|
||||
|
||||
def forward(self, x, weights=None):
|
||||
if weights is not None:
|
||||
if x.ndim == 4:
|
||||
x = x.reshape(x.shape[0], x.shape[1] * x.shape[2], x.shape[3])
|
||||
w = _prepare_weights(weights, x.shape[-1])
|
||||
v1 = w.sum(dim=-1) + 1e-8
|
||||
pooling_mean = torch.sum(x * w, dim=-1) / v1
|
||||
v2 = torch.square(w).sum(dim=-1)
|
||||
var = torch.sum(
|
||||
torch.square(x - pooling_mean.unsqueeze(-1)) * w, dim=-1
|
||||
) / (v1 - v2 / v1 + 1e-8)
|
||||
pooling_std = torch.sqrt(var.clamp(min=1e-7))
|
||||
return torch.cat(
|
||||
(pooling_mean.flatten(start_dim=1), pooling_std.flatten(start_dim=1)), 1
|
||||
)
|
||||
# The last dimension is the temporal axis
|
||||
pooling_mean = x.mean(dim=-1)
|
||||
pooling_std = torch.sqrt(torch.var(x, dim=-1) + 1e-7)
|
||||
pooling_mean = pooling_mean.flatten(start_dim=1)
|
||||
pooling_std = pooling_std.flatten(start_dim=1)
|
||||
stats = torch.cat((pooling_mean, pooling_std), 1)
|
||||
return stats
|
||||
|
||||
def get_out_dim(self):
|
||||
self.out_dim = self.in_dim * 2
|
||||
return self.out_dim
|
||||
|
||||
class TSTP_var(nn.Module):
|
||||
"""
|
||||
Temporal statistics pooling, concatenate mean and std, which is used in
|
||||
x-vector
|
||||
Comment: simple concatenation can not make full use of both statistics
|
||||
"""
|
||||
|
||||
def __init__(self, in_dim=0, **kwargs):
|
||||
super(TSTP, self).__init__()
|
||||
self.in_dim = in_dim
|
||||
|
||||
def forward(self, x):
|
||||
# The last dimension is the temporal axis
|
||||
pooling_mean = x.mean(dim=-1)
|
||||
pooling_var = torch.var(x, dim=-1)
|
||||
pooling_mean = pooling_mean.flatten(start_dim=1)
|
||||
pooling_var = pooling_var.flatten(start_dim=1)
|
||||
stats = torch.cat((pooling_mean, pooling_var), 1)
|
||||
return stats
|
||||
|
||||
def get_out_dim(self):
|
||||
self.out_dim = self.in_dim * 2
|
||||
return self.out_dim
|
||||
|
||||
class ASTP(nn.Module):
|
||||
""" Attentive statistics pooling: Channel- and context-dependent
|
||||
statistics pooling, first used in ECAPA_TDNN.
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
in_dim,
|
||||
bottleneck_dim=128,
|
||||
global_context_att=False,
|
||||
**kwargs):
|
||||
super(ASTP, self).__init__()
|
||||
self.in_dim = in_dim
|
||||
self.global_context_att = global_context_att
|
||||
|
||||
# Use Conv1d with stride == 1 rather than Linear, then we don't
|
||||
# need to transpose inputs.
|
||||
if global_context_att:
|
||||
self.linear1 = nn.Conv1d(
|
||||
in_dim * 3, bottleneck_dim,
|
||||
kernel_size=1) # equals W and b in the paper
|
||||
else:
|
||||
self.linear1 = nn.Conv1d(
|
||||
in_dim, bottleneck_dim,
|
||||
kernel_size=1) # equals W and b in the paper
|
||||
self.linear2 = nn.Conv1d(bottleneck_dim, in_dim,
|
||||
kernel_size=1) # equals V and k in the paper
|
||||
|
||||
def forward(self, x, weights=None):
|
||||
"""
|
||||
x: a 3-dimensional tensor in tdnn-based architecture (B,F,T)
|
||||
or a 4-dimensional tensor in resnet architecture (B,C,F,T)
|
||||
0-dim: batch-dimension, last-dim: time-dimension (frame-dimension)
|
||||
weights: an optional (B,T) tensor giving more weight to some frames than
|
||||
others (e.g. the activation of the speaker of interest). The attention
|
||||
weights are multiplied by `weights` and renormalized over time, so that
|
||||
statistics focus on the highly-weighted frames. When `weights` is None,
|
||||
this is the standard attentive statistics pooling.
|
||||
"""
|
||||
if len(x.shape) == 4:
|
||||
x = x.reshape(x.shape[0], x.shape[1] * x.shape[2], x.shape[3])
|
||||
assert len(x.shape) == 3
|
||||
|
||||
if self.global_context_att:
|
||||
context_mean = torch.mean(x, dim=-1, keepdim=True).expand_as(x)
|
||||
context_std = torch.sqrt(
|
||||
torch.var(x, dim=-1, keepdim=True) + 1e-7).expand_as(x)
|
||||
x_in = torch.cat((x, context_mean, context_std), dim=1)
|
||||
else:
|
||||
x_in = x
|
||||
|
||||
# DON'T use ReLU here! ReLU may be hard to converge.
|
||||
alpha = torch.tanh(
|
||||
self.linear1(x_in)) # alpha = F.relu(self.linear1(x_in))
|
||||
alpha = torch.softmax(self.linear2(alpha), dim=2)
|
||||
|
||||
if weights is not None:
|
||||
# give more weight to some frames than others by modulating the
|
||||
# attention weights, then renormalize so they still sum to one
|
||||
alpha = alpha * _prepare_weights(weights, alpha.shape[-1])
|
||||
alpha = alpha / (torch.sum(alpha, dim=2, keepdim=True) + 1e-8)
|
||||
|
||||
mean = torch.sum(alpha * x, dim=2)
|
||||
var = torch.sum(alpha * (x**2), dim=2) - mean**2
|
||||
std = torch.sqrt(var.clamp(min=1e-7))
|
||||
return torch.cat([mean, std], dim=1)
|
||||
|
||||
def get_out_dim(self):
|
||||
self.out_dim = 2 * self.in_dim
|
||||
return self.out_dim
|
||||
|
||||
|
||||
class MHASTP(torch.nn.Module):
|
||||
""" Multi head attentive statistics pooling
|
||||
Reference:
|
||||
Self Multi-Head Attention for Speaker Recognition
|
||||
https://arxiv.org/pdf/1906.09890.pdf
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
in_dim,
|
||||
layer_num=2,
|
||||
head_num=2,
|
||||
d_s=1,
|
||||
bottleneck_dim=64,
|
||||
**kwargs):
|
||||
super(MHASTP, self).__init__()
|
||||
assert (in_dim % head_num
|
||||
) == 0 # make sure that head num can be divided by input_dim
|
||||
self.in_dim = in_dim
|
||||
self.head_num = head_num
|
||||
d_model = int(in_dim / head_num)
|
||||
channel_dims = [bottleneck_dim for i in range(layer_num + 1)]
|
||||
if d_s > 1:
|
||||
d_s = d_model
|
||||
else:
|
||||
d_s = 1
|
||||
self.d_s = d_s
|
||||
channel_dims[0], channel_dims[-1] = d_model, d_s
|
||||
heads_att_trans = []
|
||||
for i in range(self.head_num):
|
||||
att_trans = nn.Sequential()
|
||||
for i in range(layer_num - 1):
|
||||
att_trans.add_module(
|
||||
'att_' + str(i),
|
||||
nn.Conv1d(channel_dims[i], channel_dims[i + 1], 1, 1))
|
||||
att_trans.add_module('tanh' + str(i), nn.Tanh())
|
||||
att_trans.add_module(
|
||||
'att_' + str(layer_num - 1),
|
||||
nn.Conv1d(channel_dims[layer_num - 1], channel_dims[layer_num],
|
||||
1, 1))
|
||||
heads_att_trans.append(att_trans)
|
||||
self.heads_att_trans = nn.ModuleList(heads_att_trans)
|
||||
|
||||
def forward(self, input):
|
||||
"""
|
||||
input: a 3-dimensional tensor in xvector architecture
|
||||
or a 4-dimensional tensor in resnet architecture
|
||||
0-dim: batch-dimension, last-dim: time-dimension (frame-dimension)
|
||||
"""
|
||||
if len(input.shape) == 4: # B x F x T
|
||||
input = input.reshape(input.shape[0],
|
||||
input.shape[1] * input.shape[2],
|
||||
input.shape[3])
|
||||
assert len(input.shape) == 3
|
||||
bs, f_dim, t_dim = input.shape
|
||||
chunks = torch.chunk(input, self.head_num, 1)
|
||||
# split
|
||||
chunks_out = []
|
||||
# for i in range(self.head_num):
|
||||
# att_score = self.heads_att_trans[i](chunks[i])
|
||||
for i, layer in enumerate(self.heads_att_trans):
|
||||
att_score = layer(chunks[i])
|
||||
alpha = F.softmax(att_score, dim=-1)
|
||||
mean = torch.sum(alpha * chunks[i], dim=2)
|
||||
var = torch.sum(alpha * chunks[i]**2, dim=2) - mean**2
|
||||
std = torch.sqrt(var.clamp(min=1e-7))
|
||||
chunks_out.append(torch.cat((mean, std), dim=1))
|
||||
out = torch.cat(chunks_out, dim=1)
|
||||
return out
|
||||
|
||||
def get_out_dim(self):
|
||||
self.out_dim = 2 * self.in_dim
|
||||
return self.out_dim
|
||||
|
||||
|
||||
class MQMHASTP(torch.nn.Module):
|
||||
""" An attentive pooling
|
||||
Reference:
|
||||
multi query multi head attentive statistics pooling
|
||||
https://arxiv.org/pdf/2110.05042.pdf
|
||||
Args:
|
||||
in_dim: the feature dimension of input
|
||||
layer_num: the number of layer in the pooling layer
|
||||
query_num: the number of querys
|
||||
head_num: the number of heads
|
||||
bottleneck_dim: the bottleneck dimension
|
||||
|
||||
SA (H = 1, Q = 1, n = 2, d_s = 1) ref:
|
||||
https://www.danielpovey.com/files/2018_interspeech_xvector_attention.pdf
|
||||
MHA (H > 1, Q = 1, n = 1, d_s = 1) ref:
|
||||
https://arxiv.org/pdf/1906.09890.pdf
|
||||
AS (H = 1, Q > 1, n = 2, d_s = 1) ref:
|
||||
https://arxiv.org/pdf/1803.10963.pdf
|
||||
VSA (H = 1, Q > 1, n = 2, d_s = d_h) ref:
|
||||
http://www.interspeech2020.org/uploadfile/pdf/Mon-2-10-5.pdf
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
in_dim,
|
||||
layer_num=2,
|
||||
query_num=2,
|
||||
head_num=8,
|
||||
d_s=2,
|
||||
bottleneck_dim=64,
|
||||
**kwargs):
|
||||
super(MQMHASTP, self).__init__()
|
||||
self.n_query = nn.ModuleList([
|
||||
MHASTP(in_dim,
|
||||
layer_num=layer_num,
|
||||
head_num=head_num,
|
||||
d_s=d_s,
|
||||
bottleneck_dim=bottleneck_dim) for i in range(query_num)
|
||||
])
|
||||
self.query_num = query_num
|
||||
self.in_dim = in_dim
|
||||
|
||||
def forward(self, input):
|
||||
"""
|
||||
input: a 3-dimensional tensor in xvector architecture
|
||||
or a 4-dimensional tensor in resnet architecture
|
||||
0-dim: batch-dimension, last-dim: time-dimension (frame-dimension)
|
||||
"""
|
||||
if len(input.shape) == 4: # B x F x T
|
||||
input = input.reshape(input.shape[0],
|
||||
input.shape[1] * input.shape[2],
|
||||
input.shape[3])
|
||||
assert len(input.shape) == 3
|
||||
res = []
|
||||
for i, layer in enumerate(self.n_query):
|
||||
res.append(layer(input))
|
||||
out = torch.cat(res, dim=-1)
|
||||
return out
|
||||
|
||||
def get_out_dim(self):
|
||||
self.out_dim = self.in_dim * 2 * self.query_num
|
||||
return self.out_dim
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
data = torch.randn(16, 512, 10, 35)
|
||||
# model = StatisticsPooling()
|
||||
model = MQMHASTP(512 * 10)
|
||||
model = MHASTP(512 * 10)
|
||||
model = MQMHASTP(512 * 10, context=False)
|
||||
print(model)
|
||||
|
||||
out = model(data)
|
||||
print(out.shape)
|
||||
print(model.get_out_dim())
|
||||
+99
@@ -0,0 +1,99 @@
|
||||
# MIT License
|
||||
#
|
||||
# Copyright (c) 2024 ID R&D, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
#------------------------------------------
|
||||
# ReDimNet main structural modules
|
||||
#------------------------------------------
|
||||
|
||||
class to1d(nn.Module):
|
||||
def forward(self,x):
|
||||
size = x.size()
|
||||
# bs,c,f,t = tuple(size)
|
||||
bs,c,f,t = size
|
||||
return x.permute((0,2,1,3)).reshape((bs,c*f,t))
|
||||
|
||||
class to2d(nn.Module):
|
||||
def __init__(self, f, c):
|
||||
super().__init__()
|
||||
self.f = f
|
||||
self.c = c
|
||||
|
||||
def forward(self,x):
|
||||
size = x.size()
|
||||
# bs,cf,t = tuple(size)
|
||||
bs,cf,t = size
|
||||
out = x.reshape((bs,self.f,self.c,t)).permute((0,2,1,3))
|
||||
# print(f"to2d : {out.size()}")
|
||||
return out
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
return f"f={self.f},c={self.c}"
|
||||
|
||||
class to1d_tfopt(nn.Module):
|
||||
def forward(self,x):
|
||||
bs,c,t,f = x.size()
|
||||
return x.permute((0,3,1,2)).reshape((bs,c*f,t))
|
||||
|
||||
class to2d_tfopt(nn.Module):
|
||||
def __init__(self, f, c):
|
||||
super().__init__()
|
||||
self.f = f
|
||||
self.c = c
|
||||
|
||||
def forward(self,x):
|
||||
bs,cf,t = x.size()
|
||||
out = x.reshape((bs, self.f, self.c, t)) # bs,f,c,t
|
||||
out = out.permute((0,2,3,1)) # bs,c,t,f
|
||||
return out
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
return f"f={self.f},c={self.c}"
|
||||
|
||||
class weigth1d(nn.Module):
|
||||
def __init__(self, N, C, sequential=False,
|
||||
requires_grad=True):
|
||||
super().__init__()
|
||||
self.N = N
|
||||
self.sequential = sequential
|
||||
self.w = nn.Parameter(torch.zeros(1,N,C,1),
|
||||
requires_grad=requires_grad)
|
||||
|
||||
def forward(self, xs):
|
||||
w = F.softmax(self.w,dim=1)
|
||||
if not self.sequential:
|
||||
xs = torch.cat([t.unsqueeze(1) for t in xs],dim=1)
|
||||
x = (w*xs).sum(dim=1)
|
||||
# print(f"weigth1d : {x.size()}")
|
||||
else:
|
||||
s = torch.zeros_like(xs[0])
|
||||
for i,t in enumerate(xs):
|
||||
s += t*w[:,i,:,:]
|
||||
x = s
|
||||
# x = sum([t*w[:,i,:,:] for i,t in enumerate(xs)])
|
||||
return x
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
return f"w={tuple(self.w.size())},sequential={self.sequential}"
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
# MIT License
|
||||
#
|
||||
# Copyright (c) 2024 ID R&D, Inc.
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in all
|
||||
# copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
import math
|
||||
import torch
|
||||
import functools
|
||||
import numpy as np
|
||||
import torch.nn as nn
|
||||
from typing import List
|
||||
from torch import Tensor
|
||||
import torch.nn.functional as F
|
||||
from collections import OrderedDict
|
||||
from typing import Iterable, Optional
|
||||
|
||||
#------------------------------------------
|
||||
# ResNet basice block (+fwSE)
|
||||
#------------------------------------------
|
||||
|
||||
class fwSEBlock(nn.Module):
|
||||
"""
|
||||
Squeeze-and-Excitation block
|
||||
link: https://arxiv.org/pdf/1709.01507.pdf
|
||||
PyTorch implementation
|
||||
"""
|
||||
def __init__(self, num_freq, num_feats=64):
|
||||
super(fwSEBlock, self).__init__()
|
||||
self.squeeze = nn.Linear(num_freq, num_feats)
|
||||
self.exitation = nn.Linear(num_feats, num_freq)
|
||||
|
||||
self.activation = nn.ReLU() # Assuming ReLU, modify as needed
|
||||
|
||||
def forward(self, inputs):
|
||||
# [bs, C, F, T]
|
||||
x = torch.mean(inputs, dim=[1, 3])
|
||||
x = self.squeeze(x)
|
||||
x = self.activation(x)
|
||||
x = self.exitation(x)
|
||||
x = torch.sigmoid(x)
|
||||
# Reshape and apply excitation
|
||||
x = x[:,None,:,None]
|
||||
x = inputs * x
|
||||
return x
|
||||
|
||||
class ResBasicBlock(nn.Module):
|
||||
def __init__(self, inc, outc, num_freq, stride=1, se_channels=64, Gdiv=4, use_fwSE=False):
|
||||
super().__init__()
|
||||
# if inc//Gdiv==0:
|
||||
# Gdiv = inc
|
||||
self.conv1 = nn.Conv2d(inc, inc if Gdiv is not None else outc, kernel_size=3, stride=stride, padding=1, bias=False,
|
||||
groups=inc//Gdiv if Gdiv is not None else 1)
|
||||
if Gdiv is not None:
|
||||
self.conv1pw = nn.Conv2d(inc, outc, 1)
|
||||
else:
|
||||
self.conv1pw = nn.Identity()
|
||||
|
||||
self.bn1 = nn.BatchNorm2d(outc)
|
||||
self.conv2 = nn.Conv2d(outc, outc, kernel_size=3, padding=1, bias=False, groups=outc//Gdiv if Gdiv is not None else 1)
|
||||
|
||||
if Gdiv is not None:
|
||||
self.conv2pw = nn.Conv2d(outc, outc, 1)
|
||||
else:
|
||||
self.conv2pw = nn.Identity()
|
||||
|
||||
self.bn2 = nn.BatchNorm2d(outc)
|
||||
self.relu = nn.ReLU(inplace=True)
|
||||
|
||||
if use_fwSE:
|
||||
self.se = fwSEBlock(num_freq, se_channels)
|
||||
else:
|
||||
self.se = nn.Identity()
|
||||
|
||||
if outc != inc:
|
||||
self.downsample = nn.Sequential(
|
||||
nn.Conv2d(inc, outc, kernel_size=1, stride=stride, bias=False),
|
||||
nn.BatchNorm2d(outc),
|
||||
)
|
||||
else:
|
||||
self.downsample = nn.Identity()
|
||||
|
||||
def forward(self, x):
|
||||
residual = x
|
||||
|
||||
out = self.conv1pw(self.conv1(x))
|
||||
out = self.relu(out)
|
||||
out = self.bn1(out)
|
||||
|
||||
out = self.conv2pw(self.conv2(out))
|
||||
out = self.bn2(out)
|
||||
out = self.se(out)
|
||||
|
||||
out += self.downsample(residual)
|
||||
out = self.relu(out)
|
||||
# print(out.size())
|
||||
return out
|
||||
@@ -0,0 +1,529 @@
|
||||
import math
|
||||
import torch
|
||||
import functools
|
||||
import numpy as np
|
||||
import torch.nn as nn
|
||||
|
||||
from .layers import features as features
|
||||
from .layers import features_tf as features_tf
|
||||
from .layers import poolings as pooling_layers
|
||||
from .layers.layernorm import LayerNorm
|
||||
from .layers.blocks import ConvBlock2d, TimeContextBlock1d
|
||||
from .layers.redim_structural import to1d, to2d, to1d_tfopt, to2d_tfopt, weigth1d
|
||||
|
||||
# class ShapeLogger(nn.Module):
|
||||
# def __init__(self, module):
|
||||
# super().__init__()
|
||||
# self.module = module
|
||||
|
||||
# def forward(self,x):
|
||||
# cls_name = self.module.__class__.__name__
|
||||
# in_shape = tuple(x.size())
|
||||
# h = self.module(x)
|
||||
# out_shape = tuple(h.size())
|
||||
# print(f"{cls_name} : {in_shape} -> {out_shape}")
|
||||
# return h
|
||||
|
||||
ShapeLogger = lambda x : x
|
||||
|
||||
#------------------------------------------
|
||||
|
||||
class FreqEncoder(nn.Module):
|
||||
def __init__(self,c,bins):
|
||||
super().__init__()
|
||||
self.freq_embedder = nn.Embedding(
|
||||
num_embeddings=bins,
|
||||
embedding_dim=c)
|
||||
|
||||
def forward(self,x):
|
||||
b, c, f, t = x.size()
|
||||
freqs = torch.range(start=0,end=f-1, step=1, dtype=torch.long)
|
||||
freqs = freqs.unsqueeze(0).repeat(b,1).to(x.device) # [bs,f]
|
||||
fe = self.freq_embedder(freqs).permute(0,2,1).unsqueeze(-1) # [bs, freq_emb_dim, f, 1]
|
||||
fe = fe.repeat(1,1,1,t)
|
||||
x = x + fe
|
||||
return x
|
||||
|
||||
import collections
|
||||
from itertools import repeat
|
||||
from typing import Any
|
||||
# https://github.com/pytorch/pytorch/blob/dc8692b0eb093d5af150ae0f3a29a0957c3e4c0d/torch/nn/modules/utils.py#L10
|
||||
def _ntuple(n, name="parse"):
|
||||
def parse(x):
|
||||
if isinstance(x, collections.abc.Iterable):
|
||||
return tuple(x)
|
||||
return tuple(repeat(x, n))
|
||||
|
||||
parse.__name__ = name
|
||||
return parse
|
||||
|
||||
_pair = _ntuple(2, "_pair")
|
||||
|
||||
#------------------------------------------
|
||||
# UReDimNet
|
||||
#------------------------------------------
|
||||
class ReDimNet2(nn.Module):
|
||||
# UNet-like ReDimNet
|
||||
def __init__(self,
|
||||
F = 72,
|
||||
C = 24,
|
||||
spec_in_channels = 1, # Phase + Magnitude
|
||||
causal = 'none',
|
||||
out_channels = None,
|
||||
block_1d_type = 'tf-att',
|
||||
block_2d_type = "basic_resnet",
|
||||
return_2d_output = False,
|
||||
fm_weigthing_type = 'NC',
|
||||
use_freq_pos_enc = False,
|
||||
compress_tconvs = True,
|
||||
stages_setup = [
|
||||
# Encoder part:
|
||||
((1,1),2,4,[(3,3)],None), # 16
|
||||
((2,1),3,3,[(3,3)],None), # 32
|
||||
|
||||
((1,2),4,2,[(3,3)],None), # 64,
|
||||
((2,1),5,1,[(3,3)],48), # 128
|
||||
|
||||
((1,2),4,1,[(3,3)],64), # 128
|
||||
((2,1),3,1,[(3,3)],96), # 128
|
||||
],
|
||||
group_divisor = 1,
|
||||
dual_agg = False,
|
||||
agg_gnorm = False,
|
||||
#-----------------------
|
||||
# Subnet stuff
|
||||
#-----------------------
|
||||
return_all_outputs = False,
|
||||
offset_fm_weights = 0,
|
||||
is_subnet = False,
|
||||
):
|
||||
super().__init__()
|
||||
self.F = F
|
||||
self.C = C
|
||||
|
||||
if causal == 'full':
|
||||
block_1d_type = block_1d_type + '-causal'
|
||||
block_2d_type = block_2d_type + '-causal'
|
||||
self.causal = True
|
||||
elif causal == 'only_1d':
|
||||
block_1d_type = block_1d_type + '-causal'
|
||||
self.causal = True
|
||||
elif causal == 'none':
|
||||
self.causal = False
|
||||
else:
|
||||
raise NotImplementedError()
|
||||
|
||||
self.block_1d_type = block_1d_type
|
||||
self.block_2d_type = block_2d_type
|
||||
|
||||
self.stages_setup = stages_setup
|
||||
self.fm_weigthing_type = fm_weigthing_type
|
||||
self.dual_agg = dual_agg
|
||||
self.agg_gnorm = agg_gnorm
|
||||
|
||||
# Subnet stuff
|
||||
self.is_subnet = is_subnet
|
||||
self.offset_fm_weights = offset_fm_weights
|
||||
self.return_all_outputs = return_all_outputs
|
||||
|
||||
self.build(F,C,spec_in_channels,out_channels,stages_setup,group_divisor,
|
||||
compress_tconvs,return_2d_output,use_freq_pos_enc)
|
||||
|
||||
def build(self,F,C,spec_in_channels,out_channels,stages_setup,group_divisor,
|
||||
compress_tconvs,return_2d_output,use_freq_pos_enc):
|
||||
self.F = F
|
||||
self.C = C
|
||||
|
||||
c = C
|
||||
f = F
|
||||
|
||||
stt = 1
|
||||
sft = 1
|
||||
|
||||
max_stt = stt
|
||||
|
||||
self.num_stages = len(stages_setup)
|
||||
|
||||
append_to1d_before_tcm = True
|
||||
Block1d = functools.partial(TimeContextBlock1d,block_type=self.block_1d_type)
|
||||
Block2d = functools.partial(ConvBlock2d,block_type=self.block_2d_type)
|
||||
|
||||
if self.fm_weigthing_type == 'NC':
|
||||
agg1d = functools.partial(weigth1d,C=F*C)
|
||||
elif self.fm_weigthing_type == 'N':
|
||||
agg1d = functools.partial(weigth1d,C=None)
|
||||
else:
|
||||
raise NotImplementedError()
|
||||
|
||||
if not self.is_subnet:
|
||||
self.stem = nn.Sequential(
|
||||
nn.Conv2d(spec_in_channels, int(c), kernel_size=3, stride=1, padding='same'),
|
||||
LayerNorm(int(c), eps=1e-6, data_format="channels_first"),
|
||||
to1d()
|
||||
)
|
||||
else:
|
||||
# Subnet stem: aggregate offset_fm_weights incoming 1D feature maps,
|
||||
# reshape to 2D, then apply a standard conv+norm stem before to1d().
|
||||
assert self.offset_fm_weights > 0, \
|
||||
"offset_fm_weights must be > 0 when is_subnet=True"
|
||||
self.stem = nn.Sequential(
|
||||
agg1d(N=self.offset_fm_weights,
|
||||
requires_grad=self.offset_fm_weights>1),
|
||||
to2d(f=F, c=C),
|
||||
nn.Conv2d(int(c), int(c), kernel_size=3, stride=1, padding='same'),
|
||||
LayerNorm(int(c), eps=1e-6, data_format="channels_first"),
|
||||
to1d()
|
||||
)
|
||||
|
||||
if self.agg_gnorm:
|
||||
self.stem_gnorm = nn.GroupNorm(num_groups=C, num_channels=C*F)
|
||||
|
||||
# Track accumulated feature-map count for the weigth1d N parameter.
|
||||
# Starts at offset_fm_weights+1 to account for the subnet offset + stem output.
|
||||
feat_count = self.offset_fm_weights + 1
|
||||
self._stage_has_dual = []
|
||||
|
||||
for stage_ind, (stride, num_blocks, conv_exp, kernel_sizes, att_block_red) in enumerate(stages_setup):
|
||||
(sf, st) = stride
|
||||
tot_stride = np.prod((sf, st))
|
||||
num_feats_to_weight = feat_count
|
||||
# if tot_stride > 1:
|
||||
layers = []
|
||||
sft = sft * sf
|
||||
stt = stt * st
|
||||
layers.append(agg1d(N=num_feats_to_weight, requires_grad=num_feats_to_weight>1))
|
||||
layers.append(to2d(f=f, c=c))
|
||||
if use_freq_pos_enc:
|
||||
layers.append(FreqEncoder(c=c,bins=f))
|
||||
|
||||
layers.append(ShapeLogger(nn.Conv2d(int(c), int(sf*c*conv_exp),
|
||||
kernel_size=(sf,stt),
|
||||
stride=(sf,stt),
|
||||
padding=0, groups=1 if not compress_tconvs else
|
||||
math.gcd(int(c),int(sf*c*conv_exp)))))
|
||||
|
||||
c = sf * c
|
||||
assert f % sf == 0
|
||||
f = f // sf
|
||||
|
||||
if stt >= max_stt:
|
||||
max_stt = stt
|
||||
|
||||
for block_ind in range(num_blocks):
|
||||
layers.append(Block2d(c=int(c*conv_exp), f=f,
|
||||
kernel_sizes=kernel_sizes, Gdiv=group_divisor))
|
||||
|
||||
if conv_exp != 1:
|
||||
_group_divisor = group_divisor
|
||||
layers.append(nn.Sequential(
|
||||
nn.Conv2d(int(c*conv_exp), c, kernel_size=1, stride=1, padding='same'),
|
||||
nn.BatchNorm2d(c, eps=1e-6)
|
||||
))
|
||||
|
||||
has_dual = self.dual_agg and att_block_red is not None
|
||||
|
||||
if has_dual:
|
||||
# Split the stage so the 1D-attention branch runs in parallel with
|
||||
# a plain 2D->1D reshape branch; both are upsampled (+gnorm) and
|
||||
# aggregated alongside prior feature maps.
|
||||
if append_to1d_before_tcm:
|
||||
layers.append(to1d())
|
||||
setattr(self, f'stage{stage_ind}_pre', nn.Sequential(*layers))
|
||||
|
||||
blk_1d = Block1d(C*F, hC=(C*F)//att_block_red)
|
||||
setattr(self, f'stage{stage_ind}_1d', blk_1d)
|
||||
|
||||
up_2d = [ShapeLogger(nn.Upsample(scale_factor=stt, mode='nearest'))]
|
||||
if self.agg_gnorm:
|
||||
up_2d.append(nn.GroupNorm(num_groups=C, num_channels=C*F))
|
||||
setattr(self, f'stage{stage_ind}_up_2d', nn.Sequential(*up_2d))
|
||||
|
||||
up_1d = [ShapeLogger(nn.Upsample(scale_factor=stt, mode='nearest'))]
|
||||
if self.agg_gnorm:
|
||||
up_1d.append(nn.GroupNorm(num_groups=C, num_channels=C*F))
|
||||
setattr(self, f'stage{stage_ind}_up_1d', nn.Sequential(*up_1d))
|
||||
|
||||
self._stage_has_dual.append(True)
|
||||
feat_count += 2
|
||||
else:
|
||||
if append_to1d_before_tcm:
|
||||
layers.append(to1d())
|
||||
if att_block_red is not None:
|
||||
if append_to1d_before_tcm:
|
||||
layers.append(Block1d(C*F,hC=(C*F)//att_block_red))
|
||||
else:
|
||||
layers.append(Block1d(C=c,F=f,hC=att_block_red))
|
||||
if not append_to1d_before_tcm:
|
||||
layers.append(to1d())
|
||||
layers.append(ShapeLogger(nn.Upsample(scale_factor=stt, mode='nearest')))
|
||||
if self.agg_gnorm:
|
||||
layers.append(nn.GroupNorm(num_groups=C, num_channels=C*F))
|
||||
setattr(self,f'stage{stage_ind}',nn.Sequential(*layers))
|
||||
|
||||
self._stage_has_dual.append(False)
|
||||
feat_count += 1
|
||||
|
||||
self.fin_wght1d = agg1d(N=feat_count, requires_grad=feat_count>1)
|
||||
|
||||
self.time_stride = max_stt
|
||||
self.freq_stride = sft
|
||||
self.head = nn.Identity()
|
||||
print(f"out_channels : {out_channels}")
|
||||
if return_2d_output:
|
||||
self.fin_to2d = to2d(f=f,c=c)
|
||||
if out_channels is not None:
|
||||
self.head = nn.Conv2d(c, out_channels, 1)
|
||||
else:
|
||||
self.fin_to2d = nn.Identity()
|
||||
if out_channels is not None:
|
||||
self.head = nn.Conv1d(C*F, out_channels, 1)
|
||||
|
||||
def run_stage(self,prev_outs_1d, stage_ind):
|
||||
if self._stage_has_dual[stage_ind]:
|
||||
pre = getattr(self, f'stage{stage_ind}_pre')
|
||||
blk_1d = getattr(self, f'stage{stage_ind}_1d')
|
||||
up_2d = getattr(self, f'stage{stage_ind}_up_2d')
|
||||
up_1d = getattr(self, f'stage{stage_ind}_up_1d')
|
||||
x_pre = pre(prev_outs_1d)
|
||||
x_2d = up_2d(x_pre)
|
||||
x_1d = up_1d(blk_1d(x_pre))
|
||||
return [x_2d, x_1d]
|
||||
stage = getattr(self,f'stage{stage_ind}')
|
||||
return [stage(prev_outs_1d)]
|
||||
|
||||
def forward(self,inp):
|
||||
if not self.is_subnet:
|
||||
bs, _, _, T = inp.size()
|
||||
inp = inp[:,:,:,:(T//self.time_stride)*self.time_stride] # Needed for right reshape operations
|
||||
# print(f"T = {T} -> T = {(T//self.time_stride)*self.time_stride}")
|
||||
x = self.stem(inp)
|
||||
if self.agg_gnorm:
|
||||
x = self.stem_gnorm(x)
|
||||
outputs_1d = [x]
|
||||
else:
|
||||
assert isinstance(inp, list), \
|
||||
"Subnet-mode ReDimNet2 expects a list of 1D feature maps as input"
|
||||
outputs_1d = list(inp)
|
||||
x = self.stem(inp)
|
||||
if self.agg_gnorm:
|
||||
x = self.stem_gnorm(x)
|
||||
outputs_1d.append(x)
|
||||
|
||||
for stage_ind in range(self.num_stages):
|
||||
outputs_1d.extend(self.run_stage(outputs_1d,stage_ind))
|
||||
x = self.fin_wght1d(outputs_1d)
|
||||
outputs_1d.append(x)
|
||||
x = self.fin_to2d(x)
|
||||
x = self.head(x)
|
||||
|
||||
if self.return_all_outputs:
|
||||
return x, outputs_1d
|
||||
return x
|
||||
|
||||
class ReDimNet2Wrap(nn.Module):
|
||||
def __init__(self,
|
||||
F = 72,
|
||||
C = 24,
|
||||
causal = False,
|
||||
spec_in_channels = 1, # Phase + Magnitude
|
||||
out_channels = None,
|
||||
# block_1d_type = 'tf-att',
|
||||
block_1d_type = 'conv+att',
|
||||
block_2d_type = "basic_resnet",
|
||||
compress_tconvs = True,
|
||||
return_2d_output = False,
|
||||
use_freq_pos_enc = False,
|
||||
fm_weigthing_type = 'NC',
|
||||
stages_setup = [
|
||||
# Encoder part:
|
||||
((1,1),2,4,[(3,3)],24), # 16
|
||||
((2,1),3,3,[(3,3)],24), # 32
|
||||
|
||||
((1,2),4,2,[(3,3)],24), # 64,
|
||||
((2,1),5,1,[(3,3)],24), # 128
|
||||
|
||||
((1,2),4,1,[(3,3)],24), # 128
|
||||
((2,1),3,1,[(3,3)],24), # 128
|
||||
],
|
||||
group_divisor = 1,
|
||||
dual_agg = False,
|
||||
agg_gnorm = False,
|
||||
#-------------------------
|
||||
embed_dim=192,
|
||||
num_classes=None,
|
||||
feat_agg_dropout=0.0,
|
||||
head_activation=None,
|
||||
hop_length=160,
|
||||
pooling_func='ASTP',
|
||||
pad_right_samples=None,
|
||||
before_pool_offset=None,
|
||||
feat_type='pt',
|
||||
global_context_att=True,
|
||||
emb_bn=False,
|
||||
#-------------------------
|
||||
spec_params = dict(
|
||||
do_spec_aug=False,
|
||||
freq_mask_width = (0, 6),
|
||||
time_mask_width = (0, 8),
|
||||
),
|
||||
#-------------------------
|
||||
return_all_outputs = False,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
self.return_all_outputs = return_all_outputs
|
||||
|
||||
self.backbone = ReDimNet2(
|
||||
F = F, C = C,
|
||||
causal = causal,
|
||||
spec_in_channels = spec_in_channels, # Phase + Magnitude
|
||||
out_channels = out_channels,
|
||||
return_2d_output = return_2d_output,
|
||||
block_1d_type = block_1d_type,
|
||||
block_2d_type = block_2d_type,
|
||||
compress_tconvs = compress_tconvs,
|
||||
fm_weigthing_type = fm_weigthing_type,
|
||||
use_freq_pos_enc = use_freq_pos_enc,
|
||||
stages_setup = stages_setup,
|
||||
group_divisor = group_divisor,
|
||||
dual_agg = dual_agg,
|
||||
agg_gnorm = agg_gnorm,
|
||||
return_all_outputs = return_all_outputs,
|
||||
)
|
||||
if feat_type in ['pt','pt_mel']:
|
||||
self.spec = features.MelBanks(n_mels=F,hop_length=hop_length,**spec_params)
|
||||
elif feat_type in ['tf','tf_mel']:
|
||||
self.spec = features_tf.TFMelBanks(n_mels=F,hop_length=hop_length,**spec_params)
|
||||
elif feat_type == 'tf_spec':
|
||||
self.spec = features_tf.TFSpectrogram(**spec_params)
|
||||
elif feat_type == 'pt_stft':
|
||||
self.spec = features.STFT(**spec_params)
|
||||
|
||||
if out_channels is None:
|
||||
out_channels = C*F
|
||||
else:
|
||||
if return_2d_output:
|
||||
out_channels = (F//self.backbone.freq_stride) * out_channels
|
||||
else:
|
||||
out_channels = out_channels
|
||||
|
||||
self.pool = getattr(pooling_layers, pooling_func)(
|
||||
in_dim=out_channels, global_context_att=global_context_att)
|
||||
|
||||
self.pad_right_samples = pad_right_samples
|
||||
self.before_pool_offset = before_pool_offset
|
||||
self.pool_out_dim = self.pool.get_out_dim()
|
||||
self.bn = nn.BatchNorm1d(self.pool_out_dim)
|
||||
self.linear = nn.Linear(self.pool_out_dim, embed_dim)
|
||||
self.embed_dim = embed_dim
|
||||
self.emb_bn = emb_bn
|
||||
if emb_bn: # better in SSL for SV
|
||||
self.bn2 = nn.BatchNorm1d(embed_dim)
|
||||
else:
|
||||
self.bn2 = None
|
||||
|
||||
def compute_fbank(self, x):
|
||||
"""Extract (log-mel) features from waveforms
|
||||
|
||||
Parameters
|
||||
----------
|
||||
x : (batch, samples) or (batch, 1, samples) torch.Tensor
|
||||
Batch of waveforms.
|
||||
|
||||
Returns
|
||||
-------
|
||||
fbank : (batch, frames, num_mel_bins) torch.Tensor
|
||||
Batch of (log-mel) features.
|
||||
"""
|
||||
if self.pad_right_samples is not None:
|
||||
x = torch.nn.functional.pad(x, (0, self.pad_right_samples), mode='constant', value=None)
|
||||
fbank = self.spec(x)
|
||||
# print(f"spec : {fbank.size()}")
|
||||
if fbank.ndim == 4:
|
||||
# some feature extractors keep the (singleton) channel dimension
|
||||
fbank = fbank.squeeze(1)
|
||||
# (batch, num_mel_bins, frames) -> (batch, frames, num_mel_bins)
|
||||
return fbank.transpose(1, 2)
|
||||
|
||||
def _backbone(self, fbank):
|
||||
"""Run the backbone on (log-mel) features, returning frame-level features"""
|
||||
# (batch, frames, num_mel_bins) -> (batch, 1, num_mel_bins, frames)
|
||||
return self.backbone(fbank.transpose(1, 2).unsqueeze(1))
|
||||
|
||||
def forward_frames(self, fbank):
|
||||
"""Extract frame-level features (before pooling) from (log-mel) features
|
||||
|
||||
Parameters
|
||||
----------
|
||||
fbank : (batch, frames, num_mel_bins) torch.Tensor
|
||||
Batch of (log-mel) features (as returned by `compute_fbank`).
|
||||
|
||||
Returns
|
||||
-------
|
||||
out : (batch, C * F, frames) torch.Tensor
|
||||
Batch of frame-level features.
|
||||
"""
|
||||
out = self._backbone(fbank)
|
||||
# print(f"pre pool : {out.size()}")
|
||||
if out.ndim == 4:
|
||||
bs, C, F, T = out.size()
|
||||
out = out.reshape(bs, C*F, T)
|
||||
if self.before_pool_offset is not None:
|
||||
out = out[:,:,self.before_pool_offset:]
|
||||
return out
|
||||
|
||||
def forward_embedding(self, out, weights=None):
|
||||
"""Pool frame-level features into speaker embedding(s)
|
||||
|
||||
Parameters
|
||||
----------
|
||||
out : (batch, C * F, frames) torch.Tensor
|
||||
Batch of frame-level features (as returned by `forward_frames`).
|
||||
weights : (batch, frames) or (batch, speakers, frames) torch.Tensor, optional
|
||||
Batch of weights giving more weight to some frames than others (e.g. the
|
||||
activation of the speaker of interest), passed to the pooling layer. When
|
||||
a `speakers` dimension is provided, one embedding is computed per speaker.
|
||||
|
||||
Returns
|
||||
-------
|
||||
embedding : (batch, embed_dim) or (batch, speakers, embed_dim) torch.Tensor
|
||||
Batch of speaker embeddings.
|
||||
"""
|
||||
if weights is not None and weights.dim() == 3:
|
||||
# one embedding per speaker
|
||||
return torch.stack(
|
||||
[
|
||||
self.forward_embedding(out, weights=weights[:, speaker, :])
|
||||
for speaker in range(weights.shape[1])
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
|
||||
if weights is None:
|
||||
pooled = self.pool(out)
|
||||
else:
|
||||
pooled = self.pool(out, weights=weights)
|
||||
embedding = self.linear(self.bn(pooled))
|
||||
|
||||
if self.bn2 is not None:
|
||||
embedding = self.bn2(embedding)
|
||||
|
||||
return embedding
|
||||
|
||||
def forward(self, x, weights=None):
|
||||
fbank = self.compute_fbank(x)
|
||||
|
||||
if self.return_all_outputs:
|
||||
out, all_outs_1d = self._backbone(fbank)
|
||||
if out.ndim == 4:
|
||||
bs, C, F, T = out.size()
|
||||
out = out.reshape(bs, C*F, T)
|
||||
if self.before_pool_offset is not None:
|
||||
out = out[:,:,self.before_pool_offset:]
|
||||
return self.forward_embedding(out, weights=weights), all_outs_1d
|
||||
|
||||
out = self.forward_frames(fbank)
|
||||
return self.forward_embedding(out, weights=weights)
|
||||
|
||||
def ReDimNet2Custom(**kwargs):
|
||||
return ReDimNet2Wrap(**kwargs)
|
||||
Reference in new issue
Block a user