Files
stemengine_old/backend/services/scnet_arch.py
T
2026-03-03 14:52:19 +09:00

322 lines
11 KiB
Python
Executable File

"""
SCNet model architecture — vendored from:
https://github.com/ZFTurbo/Music-Source-Separation-Training (MIT license)
Paper: https://arxiv.org/abs/2401.13276
Contains SeparationNet and SCNet classes only; no training or I/O code.
"""
import math
from collections import deque
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.nn.modules.rnn import LSTM
# ---------------------------------------------------------------------------
# SeparationNet (dual-path RNN + FFT feature conversion)
# ---------------------------------------------------------------------------
class FeatureConversion(nn.Module):
def __init__(self, channels, inverse):
super().__init__()
self.inverse = inverse
self.channels = channels
def forward(self, x):
if self.inverse:
x = x.float()
x_r = x[:, :self.channels // 2, :, :]
x_i = x[:, self.channels // 2:, :, :]
x = torch.complex(x_r, x_i)
x = torch.fft.irfft(x, dim=3, norm="ortho")
else:
x = x.float()
x = torch.fft.rfft(x, dim=3, norm="ortho")
x = torch.cat([x.real, x.imag], dim=1)
return x
class DualPathRNN(nn.Module):
def __init__(self, d_model, expand, bidirectional=True):
super().__init__()
self.d_model = d_model
self.hidden_size = d_model * expand
self.lstm_layers = nn.ModuleList([
LSTM(d_model, self.hidden_size, num_layers=1, bidirectional=bidirectional, batch_first=True)
for _ in range(2)
])
self.linear_layers = nn.ModuleList([nn.Linear(self.hidden_size * 2, d_model) for _ in range(2)])
self.norm_layers = nn.ModuleList([nn.GroupNorm(1, d_model) for _ in range(2)])
def forward(self, x):
B, C, F, T = x.shape
original_x = x
x = self.norm_layers[0](x)
x = x.transpose(1, 3).contiguous().view(B * T, F, C)
x, _ = self.lstm_layers[0](x)
x = self.linear_layers[0](x)
x = x.view(B, T, F, C).transpose(1, 3) + original_x
original_x = x
x = self.norm_layers[1](x)
x = x.transpose(1, 2).contiguous().view(B * F, C, T).transpose(1, 2)
x, _ = self.lstm_layers[1](x)
x = self.linear_layers[1](x)
x = x.transpose(1, 2).contiguous().view(B, F, C, T).transpose(1, 2) + original_x
return x
class SeparationNet(nn.Module):
def __init__(self, channels, expand=1, num_layers=6):
super().__init__()
self.num_layers = num_layers
self.dp_modules = nn.ModuleList([
DualPathRNN(channels * (2 if i % 2 == 1 else 1), expand)
for i in range(num_layers)
])
self.feature_conversion = nn.ModuleList([
FeatureConversion(channels * 2, inverse=(i % 2 == 1))
for i in range(num_layers)
])
def forward(self, x):
for dp, fc in zip(self.dp_modules, self.feature_conversion):
x = dp(x)
x = fc(x)
return x
# ---------------------------------------------------------------------------
# SCNet building blocks
# ---------------------------------------------------------------------------
class Swish(nn.Module):
def forward(self, x):
return x * x.sigmoid()
class ConvolutionModule(nn.Module):
def __init__(self, channels, depth=2, compress=4, kernel=3):
super().__init__()
assert kernel % 2 == 1
hidden = int(channels / compress)
norm = lambda d: nn.GroupNorm(1, d)
self.layers = nn.ModuleList()
for _ in range(abs(depth)):
p = kernel // 2
self.layers.append(nn.Sequential(
norm(channels),
nn.Conv1d(channels, hidden * 2, kernel, padding=p),
nn.GLU(1),
nn.Conv1d(hidden, hidden, kernel, padding=p, groups=hidden),
norm(hidden),
Swish(),
nn.Conv1d(hidden, channels, 1),
))
def forward(self, x):
for layer in self.layers:
x = x + layer(x)
return x
class FusionLayer(nn.Module):
def __init__(self, channels, kernel_size=3, stride=1, padding=1):
super().__init__()
self.conv = nn.Conv2d(channels * 2, channels * 2, kernel_size, stride=stride, padding=padding)
def forward(self, x, skip=None):
if skip is not None:
x = x + skip
x = self.conv(x.repeat(1, 2, 1, 1))
return F.glu(x, dim=1)
class SDlayer(nn.Module):
def __init__(self, channels_in, channels_out, band_configs):
super().__init__()
self.convs = nn.ModuleList()
self.strides, self.kernels = [], []
for cfg in band_configs.values():
self.convs.append(
nn.Conv2d(channels_in, channels_out, (cfg['kernel'], 1), (cfg['stride'], 1))
)
self.strides.append(cfg['stride'])
self.kernels.append(cfg['kernel'])
self.SR_low = band_configs['low']['SR']
self.SR_mid = band_configs['mid']['SR']
def forward(self, x):
B, C, Fr, T = x.shape
splits = [
(0, math.ceil(Fr * self.SR_low)),
(math.ceil(Fr * self.SR_low), math.ceil(Fr * (self.SR_low + self.SR_mid))),
(math.ceil(Fr * (self.SR_low + self.SR_mid)), Fr),
]
outputs, original_lengths = [], []
for conv, stride, kernel, (start, end) in zip(self.convs, self.strides, self.kernels, splits):
extracted = x[:, :, start:end, :]
original_lengths.append(end - start)
L = extracted.shape[2]
total_pad = (kernel - stride) if stride == 1 else (stride - L % stride) % stride
pad_l = total_pad // 2
outputs.append(conv(F.pad(extracted, (0, 0, pad_l, total_pad - pad_l))))
return outputs, original_lengths
class SUlayer(nn.Module):
def __init__(self, channels_in, channels_out, band_configs):
super().__init__()
self.convtrs = nn.ModuleList([
nn.ConvTranspose2d(channels_in, channels_out, [cfg['kernel'], 1], [cfg['stride'], 1])
for cfg in band_configs.values()
])
def forward(self, x, lengths, origin_lengths):
splits = [
(0, lengths[0]),
(lengths[0], lengths[0] + lengths[1]),
(lengths[0] + lengths[1], None),
]
outputs = []
for convtr, (start, end), orig_len in zip(self.convtrs, splits, origin_lengths):
out = convtr(x[:, :, start:end, :])
dist = abs(orig_len - out.shape[2]) // 2
outputs.append(out[:, :, dist:dist + orig_len, :])
return torch.cat(outputs, dim=2)
class SDblock(nn.Module):
def __init__(self, channels_in, channels_out, band_configs={}, conv_config={}, depths=[3, 2, 1], kernel_size=3):
super().__init__()
self.SDlayer = SDlayer(channels_in, channels_out, band_configs)
self.conv_modules = nn.ModuleList([
ConvolutionModule(channels_out, depth, **conv_config) for depth in depths
])
self.globalconv = nn.Conv2d(channels_out, channels_out, kernel_size, 1, (kernel_size - 1) // 2)
def forward(self, x):
bands, original_lengths = self.SDlayer(x)
bands = [
F.gelu(
conv(band.permute(0, 2, 1, 3).reshape(-1, band.shape[1], band.shape[3]))
.view(band.shape[0], band.shape[2], band.shape[1], band.shape[3])
.permute(0, 2, 1, 3)
)
for conv, band in zip(self.conv_modules, bands)
]
lengths = [b.size(-2) for b in bands]
full_band = torch.cat(bands, dim=2)
return self.globalconv(full_band), full_band, lengths, original_lengths
# ---------------------------------------------------------------------------
# SCNet
# ---------------------------------------------------------------------------
class SCNet(nn.Module):
"""
Sparse Compression Network for music source separation.
Paper: https://arxiv.org/abs/2401.13276
"""
def __init__(
self,
sources=['drums', 'bass', 'other', 'vocals'],
audio_channels=2,
dims=[4, 32, 64, 128],
nfft=4096,
hop_size=1024,
win_size=4096,
normalized=True,
band_SR=[0.175, 0.392, 0.433],
band_stride=[1, 4, 16],
band_kernel=[3, 4, 16],
conv_depths=[3, 2, 1],
compress=4,
conv_kernel=3,
num_dplayer=6,
expand=1,
):
super().__init__()
self.sources = sources
self.audio_channels = audio_channels
self.dims = dims
band_keys = ['low', 'mid', 'high']
self.band_configs = {
band_keys[i]: {'SR': band_SR[i], 'stride': band_stride[i], 'kernel': band_kernel[i]}
for i in range(3)
}
self.hop_length = hop_size
self.conv_config = {'compress': compress, 'kernel': conv_kernel}
self.stft_config = {
'n_fft': nfft, 'hop_length': hop_size, 'win_length': win_size,
'center': True, 'normalized': normalized,
}
self.encoder = nn.ModuleList()
self.decoder = nn.ModuleList()
for i in range(len(dims) - 1):
self.encoder.append(SDblock(
channels_in=dims[i], channels_out=dims[i + 1],
band_configs=self.band_configs, conv_config=self.conv_config,
depths=conv_depths,
))
self.decoder.insert(0, nn.Sequential(
FusionLayer(channels=dims[i + 1]),
SUlayer(
channels_in=dims[i + 1],
channels_out=dims[i] if i != 0 else dims[i] * len(sources),
band_configs=self.band_configs,
),
))
self.separation_net = SeparationNet(
channels=dims[-1], expand=expand, num_layers=num_dplayer,
)
def forward(self, x):
B = x.shape[0]
padding = self.hop_length - x.shape[-1] % self.hop_length
if (x.shape[-1] + padding) // self.hop_length % 2 == 0:
padding += self.hop_length
x = F.pad(x, (0, padding))
L = x.shape[-1]
x = x.reshape(-1, L)
x = torch.stft(x, **self.stft_config, return_complex=True)
x = torch.view_as_real(x)
x = x.permute(0, 3, 1, 2).reshape(
x.shape[0] // self.audio_channels,
x.shape[3] * self.audio_channels,
x.shape[1], x.shape[2],
)
B, C, Fr, T = x.shape
save_skip = deque()
save_lengths = deque()
save_orig = deque()
for sd_layer in self.encoder:
x, skip, lengths, orig_len = sd_layer(x)
save_skip.append(skip)
save_lengths.append(lengths)
save_orig.append(orig_len)
x = self.separation_net(x)
for fusion_layer, su_layer in self.decoder:
x = fusion_layer(x, save_skip.pop())
x = su_layer(x, save_lengths.pop(), save_orig.pop())
n = self.dims[0]
x = x.view(B, n, -1, Fr, T)
x = x.reshape(-1, 2, Fr, T).permute(0, 2, 3, 1)
x = torch.view_as_complex(x.contiguous())
x = torch.istft(x, **self.stft_config)
x = x.reshape(B, len(self.sources), self.audio_channels, -1)
return x[:, :, :, :-padding]