322 lines
11 KiB
Python
Executable File
322 lines
11 KiB
Python
Executable File
"""
|
|
SCNet model architecture — vendored from:
|
|
https://github.com/ZFTurbo/Music-Source-Separation-Training (MIT license)
|
|
Paper: https://arxiv.org/abs/2401.13276
|
|
|
|
Contains SeparationNet and SCNet classes only; no training or I/O code.
|
|
"""
|
|
import math
|
|
from collections import deque
|
|
|
|
import torch
|
|
import torch.nn as nn
|
|
import torch.nn.functional as F
|
|
from torch.nn.modules.rnn import LSTM
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# SeparationNet (dual-path RNN + FFT feature conversion)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class FeatureConversion(nn.Module):
|
|
def __init__(self, channels, inverse):
|
|
super().__init__()
|
|
self.inverse = inverse
|
|
self.channels = channels
|
|
|
|
def forward(self, x):
|
|
if self.inverse:
|
|
x = x.float()
|
|
x_r = x[:, :self.channels // 2, :, :]
|
|
x_i = x[:, self.channels // 2:, :, :]
|
|
x = torch.complex(x_r, x_i)
|
|
x = torch.fft.irfft(x, dim=3, norm="ortho")
|
|
else:
|
|
x = x.float()
|
|
x = torch.fft.rfft(x, dim=3, norm="ortho")
|
|
x = torch.cat([x.real, x.imag], dim=1)
|
|
return x
|
|
|
|
|
|
class DualPathRNN(nn.Module):
|
|
def __init__(self, d_model, expand, bidirectional=True):
|
|
super().__init__()
|
|
self.d_model = d_model
|
|
self.hidden_size = d_model * expand
|
|
self.lstm_layers = nn.ModuleList([
|
|
LSTM(d_model, self.hidden_size, num_layers=1, bidirectional=bidirectional, batch_first=True)
|
|
for _ in range(2)
|
|
])
|
|
self.linear_layers = nn.ModuleList([nn.Linear(self.hidden_size * 2, d_model) for _ in range(2)])
|
|
self.norm_layers = nn.ModuleList([nn.GroupNorm(1, d_model) for _ in range(2)])
|
|
|
|
def forward(self, x):
|
|
B, C, F, T = x.shape
|
|
original_x = x
|
|
x = self.norm_layers[0](x)
|
|
x = x.transpose(1, 3).contiguous().view(B * T, F, C)
|
|
x, _ = self.lstm_layers[0](x)
|
|
x = self.linear_layers[0](x)
|
|
x = x.view(B, T, F, C).transpose(1, 3) + original_x
|
|
|
|
original_x = x
|
|
x = self.norm_layers[1](x)
|
|
x = x.transpose(1, 2).contiguous().view(B * F, C, T).transpose(1, 2)
|
|
x, _ = self.lstm_layers[1](x)
|
|
x = self.linear_layers[1](x)
|
|
x = x.transpose(1, 2).contiguous().view(B, F, C, T).transpose(1, 2) + original_x
|
|
return x
|
|
|
|
|
|
class SeparationNet(nn.Module):
|
|
def __init__(self, channels, expand=1, num_layers=6):
|
|
super().__init__()
|
|
self.num_layers = num_layers
|
|
self.dp_modules = nn.ModuleList([
|
|
DualPathRNN(channels * (2 if i % 2 == 1 else 1), expand)
|
|
for i in range(num_layers)
|
|
])
|
|
self.feature_conversion = nn.ModuleList([
|
|
FeatureConversion(channels * 2, inverse=(i % 2 == 1))
|
|
for i in range(num_layers)
|
|
])
|
|
|
|
def forward(self, x):
|
|
for dp, fc in zip(self.dp_modules, self.feature_conversion):
|
|
x = dp(x)
|
|
x = fc(x)
|
|
return x
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# SCNet building blocks
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class Swish(nn.Module):
|
|
def forward(self, x):
|
|
return x * x.sigmoid()
|
|
|
|
|
|
class ConvolutionModule(nn.Module):
|
|
def __init__(self, channels, depth=2, compress=4, kernel=3):
|
|
super().__init__()
|
|
assert kernel % 2 == 1
|
|
hidden = int(channels / compress)
|
|
norm = lambda d: nn.GroupNorm(1, d)
|
|
self.layers = nn.ModuleList()
|
|
for _ in range(abs(depth)):
|
|
p = kernel // 2
|
|
self.layers.append(nn.Sequential(
|
|
norm(channels),
|
|
nn.Conv1d(channels, hidden * 2, kernel, padding=p),
|
|
nn.GLU(1),
|
|
nn.Conv1d(hidden, hidden, kernel, padding=p, groups=hidden),
|
|
norm(hidden),
|
|
Swish(),
|
|
nn.Conv1d(hidden, channels, 1),
|
|
))
|
|
|
|
def forward(self, x):
|
|
for layer in self.layers:
|
|
x = x + layer(x)
|
|
return x
|
|
|
|
|
|
class FusionLayer(nn.Module):
|
|
def __init__(self, channels, kernel_size=3, stride=1, padding=1):
|
|
super().__init__()
|
|
self.conv = nn.Conv2d(channels * 2, channels * 2, kernel_size, stride=stride, padding=padding)
|
|
|
|
def forward(self, x, skip=None):
|
|
if skip is not None:
|
|
x = x + skip
|
|
x = self.conv(x.repeat(1, 2, 1, 1))
|
|
return F.glu(x, dim=1)
|
|
|
|
|
|
class SDlayer(nn.Module):
|
|
def __init__(self, channels_in, channels_out, band_configs):
|
|
super().__init__()
|
|
self.convs = nn.ModuleList()
|
|
self.strides, self.kernels = [], []
|
|
for cfg in band_configs.values():
|
|
self.convs.append(
|
|
nn.Conv2d(channels_in, channels_out, (cfg['kernel'], 1), (cfg['stride'], 1))
|
|
)
|
|
self.strides.append(cfg['stride'])
|
|
self.kernels.append(cfg['kernel'])
|
|
self.SR_low = band_configs['low']['SR']
|
|
self.SR_mid = band_configs['mid']['SR']
|
|
|
|
def forward(self, x):
|
|
B, C, Fr, T = x.shape
|
|
splits = [
|
|
(0, math.ceil(Fr * self.SR_low)),
|
|
(math.ceil(Fr * self.SR_low), math.ceil(Fr * (self.SR_low + self.SR_mid))),
|
|
(math.ceil(Fr * (self.SR_low + self.SR_mid)), Fr),
|
|
]
|
|
outputs, original_lengths = [], []
|
|
for conv, stride, kernel, (start, end) in zip(self.convs, self.strides, self.kernels, splits):
|
|
extracted = x[:, :, start:end, :]
|
|
original_lengths.append(end - start)
|
|
L = extracted.shape[2]
|
|
total_pad = (kernel - stride) if stride == 1 else (stride - L % stride) % stride
|
|
pad_l = total_pad // 2
|
|
outputs.append(conv(F.pad(extracted, (0, 0, pad_l, total_pad - pad_l))))
|
|
return outputs, original_lengths
|
|
|
|
|
|
class SUlayer(nn.Module):
|
|
def __init__(self, channels_in, channels_out, band_configs):
|
|
super().__init__()
|
|
self.convtrs = nn.ModuleList([
|
|
nn.ConvTranspose2d(channels_in, channels_out, [cfg['kernel'], 1], [cfg['stride'], 1])
|
|
for cfg in band_configs.values()
|
|
])
|
|
|
|
def forward(self, x, lengths, origin_lengths):
|
|
splits = [
|
|
(0, lengths[0]),
|
|
(lengths[0], lengths[0] + lengths[1]),
|
|
(lengths[0] + lengths[1], None),
|
|
]
|
|
outputs = []
|
|
for convtr, (start, end), orig_len in zip(self.convtrs, splits, origin_lengths):
|
|
out = convtr(x[:, :, start:end, :])
|
|
dist = abs(orig_len - out.shape[2]) // 2
|
|
outputs.append(out[:, :, dist:dist + orig_len, :])
|
|
return torch.cat(outputs, dim=2)
|
|
|
|
|
|
class SDblock(nn.Module):
|
|
def __init__(self, channels_in, channels_out, band_configs={}, conv_config={}, depths=[3, 2, 1], kernel_size=3):
|
|
super().__init__()
|
|
self.SDlayer = SDlayer(channels_in, channels_out, band_configs)
|
|
self.conv_modules = nn.ModuleList([
|
|
ConvolutionModule(channels_out, depth, **conv_config) for depth in depths
|
|
])
|
|
self.globalconv = nn.Conv2d(channels_out, channels_out, kernel_size, 1, (kernel_size - 1) // 2)
|
|
|
|
def forward(self, x):
|
|
bands, original_lengths = self.SDlayer(x)
|
|
bands = [
|
|
F.gelu(
|
|
conv(band.permute(0, 2, 1, 3).reshape(-1, band.shape[1], band.shape[3]))
|
|
.view(band.shape[0], band.shape[2], band.shape[1], band.shape[3])
|
|
.permute(0, 2, 1, 3)
|
|
)
|
|
for conv, band in zip(self.conv_modules, bands)
|
|
]
|
|
lengths = [b.size(-2) for b in bands]
|
|
full_band = torch.cat(bands, dim=2)
|
|
return self.globalconv(full_band), full_band, lengths, original_lengths
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# SCNet
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class SCNet(nn.Module):
|
|
"""
|
|
Sparse Compression Network for music source separation.
|
|
Paper: https://arxiv.org/abs/2401.13276
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
sources=['drums', 'bass', 'other', 'vocals'],
|
|
audio_channels=2,
|
|
dims=[4, 32, 64, 128],
|
|
nfft=4096,
|
|
hop_size=1024,
|
|
win_size=4096,
|
|
normalized=True,
|
|
band_SR=[0.175, 0.392, 0.433],
|
|
band_stride=[1, 4, 16],
|
|
band_kernel=[3, 4, 16],
|
|
conv_depths=[3, 2, 1],
|
|
compress=4,
|
|
conv_kernel=3,
|
|
num_dplayer=6,
|
|
expand=1,
|
|
):
|
|
super().__init__()
|
|
self.sources = sources
|
|
self.audio_channels = audio_channels
|
|
self.dims = dims
|
|
band_keys = ['low', 'mid', 'high']
|
|
self.band_configs = {
|
|
band_keys[i]: {'SR': band_SR[i], 'stride': band_stride[i], 'kernel': band_kernel[i]}
|
|
for i in range(3)
|
|
}
|
|
self.hop_length = hop_size
|
|
self.conv_config = {'compress': compress, 'kernel': conv_kernel}
|
|
self.stft_config = {
|
|
'n_fft': nfft, 'hop_length': hop_size, 'win_length': win_size,
|
|
'center': True, 'normalized': normalized,
|
|
}
|
|
|
|
self.encoder = nn.ModuleList()
|
|
self.decoder = nn.ModuleList()
|
|
|
|
for i in range(len(dims) - 1):
|
|
self.encoder.append(SDblock(
|
|
channels_in=dims[i], channels_out=dims[i + 1],
|
|
band_configs=self.band_configs, conv_config=self.conv_config,
|
|
depths=conv_depths,
|
|
))
|
|
self.decoder.insert(0, nn.Sequential(
|
|
FusionLayer(channels=dims[i + 1]),
|
|
SUlayer(
|
|
channels_in=dims[i + 1],
|
|
channels_out=dims[i] if i != 0 else dims[i] * len(sources),
|
|
band_configs=self.band_configs,
|
|
),
|
|
))
|
|
|
|
self.separation_net = SeparationNet(
|
|
channels=dims[-1], expand=expand, num_layers=num_dplayer,
|
|
)
|
|
|
|
def forward(self, x):
|
|
B = x.shape[0]
|
|
padding = self.hop_length - x.shape[-1] % self.hop_length
|
|
if (x.shape[-1] + padding) // self.hop_length % 2 == 0:
|
|
padding += self.hop_length
|
|
x = F.pad(x, (0, padding))
|
|
|
|
L = x.shape[-1]
|
|
x = x.reshape(-1, L)
|
|
x = torch.stft(x, **self.stft_config, return_complex=True)
|
|
x = torch.view_as_real(x)
|
|
x = x.permute(0, 3, 1, 2).reshape(
|
|
x.shape[0] // self.audio_channels,
|
|
x.shape[3] * self.audio_channels,
|
|
x.shape[1], x.shape[2],
|
|
)
|
|
B, C, Fr, T = x.shape
|
|
|
|
save_skip = deque()
|
|
save_lengths = deque()
|
|
save_orig = deque()
|
|
|
|
for sd_layer in self.encoder:
|
|
x, skip, lengths, orig_len = sd_layer(x)
|
|
save_skip.append(skip)
|
|
save_lengths.append(lengths)
|
|
save_orig.append(orig_len)
|
|
|
|
x = self.separation_net(x)
|
|
|
|
for fusion_layer, su_layer in self.decoder:
|
|
x = fusion_layer(x, save_skip.pop())
|
|
x = su_layer(x, save_lengths.pop(), save_orig.pop())
|
|
|
|
n = self.dims[0]
|
|
x = x.view(B, n, -1, Fr, T)
|
|
x = x.reshape(-1, 2, Fr, T).permute(0, 2, 3, 1)
|
|
x = torch.view_as_complex(x.contiguous())
|
|
x = torch.istft(x, **self.stft_config)
|
|
x = x.reshape(B, len(self.sources), self.audio_channels, -1)
|
|
return x[:, :, :, :-padding]
|