""" SCNet model architecture — vendored from: https://github.com/ZFTurbo/Music-Source-Separation-Training (MIT license) Paper: https://arxiv.org/abs/2401.13276 Contains SeparationNet and SCNet classes only; no training or I/O code. """ import math from collections import deque import torch import torch.nn as nn import torch.nn.functional as F from torch.nn.modules.rnn import LSTM # --------------------------------------------------------------------------- # SeparationNet (dual-path RNN + FFT feature conversion) # --------------------------------------------------------------------------- class FeatureConversion(nn.Module): def __init__(self, channels, inverse): super().__init__() self.inverse = inverse self.channels = channels def forward(self, x): if self.inverse: x = x.float() x_r = x[:, :self.channels // 2, :, :] x_i = x[:, self.channels // 2:, :, :] x = torch.complex(x_r, x_i) x = torch.fft.irfft(x, dim=3, norm="ortho") else: x = x.float() x = torch.fft.rfft(x, dim=3, norm="ortho") x = torch.cat([x.real, x.imag], dim=1) return x class DualPathRNN(nn.Module): def __init__(self, d_model, expand, bidirectional=True): super().__init__() self.d_model = d_model self.hidden_size = d_model * expand self.lstm_layers = nn.ModuleList([ LSTM(d_model, self.hidden_size, num_layers=1, bidirectional=bidirectional, batch_first=True) for _ in range(2) ]) self.linear_layers = nn.ModuleList([nn.Linear(self.hidden_size * 2, d_model) for _ in range(2)]) self.norm_layers = nn.ModuleList([nn.GroupNorm(1, d_model) for _ in range(2)]) def forward(self, x): B, C, F, T = x.shape original_x = x x = self.norm_layers[0](x) x = x.transpose(1, 3).contiguous().view(B * T, F, C) x, _ = self.lstm_layers[0](x) x = self.linear_layers[0](x) x = x.view(B, T, F, C).transpose(1, 3) + original_x original_x = x x = self.norm_layers[1](x) x = x.transpose(1, 2).contiguous().view(B * F, C, T).transpose(1, 2) x, _ = self.lstm_layers[1](x) x = self.linear_layers[1](x) x = x.transpose(1, 2).contiguous().view(B, F, C, T).transpose(1, 2) + original_x return x class SeparationNet(nn.Module): def __init__(self, channels, expand=1, num_layers=6): super().__init__() self.num_layers = num_layers self.dp_modules = nn.ModuleList([ DualPathRNN(channels * (2 if i % 2 == 1 else 1), expand) for i in range(num_layers) ]) self.feature_conversion = nn.ModuleList([ FeatureConversion(channels * 2, inverse=(i % 2 == 1)) for i in range(num_layers) ]) def forward(self, x): for dp, fc in zip(self.dp_modules, self.feature_conversion): x = dp(x) x = fc(x) return x # --------------------------------------------------------------------------- # SCNet building blocks # --------------------------------------------------------------------------- class Swish(nn.Module): def forward(self, x): return x * x.sigmoid() class ConvolutionModule(nn.Module): def __init__(self, channels, depth=2, compress=4, kernel=3): super().__init__() assert kernel % 2 == 1 hidden = int(channels / compress) norm = lambda d: nn.GroupNorm(1, d) self.layers = nn.ModuleList() for _ in range(abs(depth)): p = kernel // 2 self.layers.append(nn.Sequential( norm(channels), nn.Conv1d(channels, hidden * 2, kernel, padding=p), nn.GLU(1), nn.Conv1d(hidden, hidden, kernel, padding=p, groups=hidden), norm(hidden), Swish(), nn.Conv1d(hidden, channels, 1), )) def forward(self, x): for layer in self.layers: x = x + layer(x) return x class FusionLayer(nn.Module): def __init__(self, channels, kernel_size=3, stride=1, padding=1): super().__init__() self.conv = nn.Conv2d(channels * 2, channels * 2, kernel_size, stride=stride, padding=padding) def forward(self, x, skip=None): if skip is not None: x = x + skip x = self.conv(x.repeat(1, 2, 1, 1)) return F.glu(x, dim=1) class SDlayer(nn.Module): def __init__(self, channels_in, channels_out, band_configs): super().__init__() self.convs = nn.ModuleList() self.strides, self.kernels = [], [] for cfg in band_configs.values(): self.convs.append( nn.Conv2d(channels_in, channels_out, (cfg['kernel'], 1), (cfg['stride'], 1)) ) self.strides.append(cfg['stride']) self.kernels.append(cfg['kernel']) self.SR_low = band_configs['low']['SR'] self.SR_mid = band_configs['mid']['SR'] def forward(self, x): B, C, Fr, T = x.shape splits = [ (0, math.ceil(Fr * self.SR_low)), (math.ceil(Fr * self.SR_low), math.ceil(Fr * (self.SR_low + self.SR_mid))), (math.ceil(Fr * (self.SR_low + self.SR_mid)), Fr), ] outputs, original_lengths = [], [] for conv, stride, kernel, (start, end) in zip(self.convs, self.strides, self.kernels, splits): extracted = x[:, :, start:end, :] original_lengths.append(end - start) L = extracted.shape[2] total_pad = (kernel - stride) if stride == 1 else (stride - L % stride) % stride pad_l = total_pad // 2 outputs.append(conv(F.pad(extracted, (0, 0, pad_l, total_pad - pad_l)))) return outputs, original_lengths class SUlayer(nn.Module): def __init__(self, channels_in, channels_out, band_configs): super().__init__() self.convtrs = nn.ModuleList([ nn.ConvTranspose2d(channels_in, channels_out, [cfg['kernel'], 1], [cfg['stride'], 1]) for cfg in band_configs.values() ]) def forward(self, x, lengths, origin_lengths): splits = [ (0, lengths[0]), (lengths[0], lengths[0] + lengths[1]), (lengths[0] + lengths[1], None), ] outputs = [] for convtr, (start, end), orig_len in zip(self.convtrs, splits, origin_lengths): out = convtr(x[:, :, start:end, :]) dist = abs(orig_len - out.shape[2]) // 2 outputs.append(out[:, :, dist:dist + orig_len, :]) return torch.cat(outputs, dim=2) class SDblock(nn.Module): def __init__(self, channels_in, channels_out, band_configs={}, conv_config={}, depths=[3, 2, 1], kernel_size=3): super().__init__() self.SDlayer = SDlayer(channels_in, channels_out, band_configs) self.conv_modules = nn.ModuleList([ ConvolutionModule(channels_out, depth, **conv_config) for depth in depths ]) self.globalconv = nn.Conv2d(channels_out, channels_out, kernel_size, 1, (kernel_size - 1) // 2) def forward(self, x): bands, original_lengths = self.SDlayer(x) bands = [ F.gelu( conv(band.permute(0, 2, 1, 3).reshape(-1, band.shape[1], band.shape[3])) .view(band.shape[0], band.shape[2], band.shape[1], band.shape[3]) .permute(0, 2, 1, 3) ) for conv, band in zip(self.conv_modules, bands) ] lengths = [b.size(-2) for b in bands] full_band = torch.cat(bands, dim=2) return self.globalconv(full_band), full_band, lengths, original_lengths # --------------------------------------------------------------------------- # SCNet # --------------------------------------------------------------------------- class SCNet(nn.Module): """ Sparse Compression Network for music source separation. Paper: https://arxiv.org/abs/2401.13276 """ def __init__( self, sources=['drums', 'bass', 'other', 'vocals'], audio_channels=2, dims=[4, 32, 64, 128], nfft=4096, hop_size=1024, win_size=4096, normalized=True, band_SR=[0.175, 0.392, 0.433], band_stride=[1, 4, 16], band_kernel=[3, 4, 16], conv_depths=[3, 2, 1], compress=4, conv_kernel=3, num_dplayer=6, expand=1, ): super().__init__() self.sources = sources self.audio_channels = audio_channels self.dims = dims band_keys = ['low', 'mid', 'high'] self.band_configs = { band_keys[i]: {'SR': band_SR[i], 'stride': band_stride[i], 'kernel': band_kernel[i]} for i in range(3) } self.hop_length = hop_size self.conv_config = {'compress': compress, 'kernel': conv_kernel} self.stft_config = { 'n_fft': nfft, 'hop_length': hop_size, 'win_length': win_size, 'center': True, 'normalized': normalized, } self.encoder = nn.ModuleList() self.decoder = nn.ModuleList() for i in range(len(dims) - 1): self.encoder.append(SDblock( channels_in=dims[i], channels_out=dims[i + 1], band_configs=self.band_configs, conv_config=self.conv_config, depths=conv_depths, )) self.decoder.insert(0, nn.Sequential( FusionLayer(channels=dims[i + 1]), SUlayer( channels_in=dims[i + 1], channels_out=dims[i] if i != 0 else dims[i] * len(sources), band_configs=self.band_configs, ), )) self.separation_net = SeparationNet( channels=dims[-1], expand=expand, num_layers=num_dplayer, ) def forward(self, x): B = x.shape[0] padding = self.hop_length - x.shape[-1] % self.hop_length if (x.shape[-1] + padding) // self.hop_length % 2 == 0: padding += self.hop_length x = F.pad(x, (0, padding)) L = x.shape[-1] x = x.reshape(-1, L) x = torch.stft(x, **self.stft_config, return_complex=True) x = torch.view_as_real(x) x = x.permute(0, 3, 1, 2).reshape( x.shape[0] // self.audio_channels, x.shape[3] * self.audio_channels, x.shape[1], x.shape[2], ) B, C, Fr, T = x.shape save_skip = deque() save_lengths = deque() save_orig = deque() for sd_layer in self.encoder: x, skip, lengths, orig_len = sd_layer(x) save_skip.append(skip) save_lengths.append(lengths) save_orig.append(orig_len) x = self.separation_net(x) for fusion_layer, su_layer in self.decoder: x = fusion_layer(x, save_skip.pop()) x = su_layer(x, save_lengths.pop(), save_orig.pop()) n = self.dims[0] x = x.view(B, n, -1, Fr, T) x = x.reshape(-1, 2, Fr, T).permute(0, 2, 3, 1) x = torch.view_as_complex(x.contiguous()) x = torch.istft(x, **self.stft_config) x = x.reshape(B, len(self.sources), self.audio_channels, -1) return x[:, :, :, :-padding]