445 lines
17 KiB
Python
Executable File
445 lines
17 KiB
Python
Executable File
"""
|
||
MedleyVox separator - Multi-singer vocal separation
|
||
HuggingFace model: Cyru5/MedleyVox
|
||
Backup: miercolesv/MVSEP-Central-Backup
|
||
|
||
The MVSEP checkpoints are flat EMA state-dicts (OrderedDict with an
|
||
'ema_model.module.' key prefix). This file contains an inline Conv-TasNet
|
||
implementation whose architecture was reverse-engineered from those weights:
|
||
N=2050 filters, L=2048 kernel, S=1024 stride,
|
||
B=256 bottleneck, H=1024 hidden, P=3 TCN kernel,
|
||
X=8 blocks/repeat, R=3 repeats, n_src=2 (vocals + instrumental).
|
||
"""
|
||
import torch
|
||
import torch.nn as nn
|
||
import torch.nn.functional as F
|
||
import torchaudio
|
||
from pathlib import Path
|
||
from typing import Dict
|
||
|
||
from ..config import MEDLEYVOX_MODEL_DIR, CPU_SEPARATION
|
||
|
||
|
||
# ── Model configurations ───────────────────────────────────────────────────────
|
||
|
||
MEDLEYVOX_MODELS = {
|
||
"medleyvox": {
|
||
"model_path": "vocals 100.pth",
|
||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||
"hf_subfolder": "MedleyVox-MultiSinger",
|
||
"stems": ["vocals", "instrumental"],
|
||
},
|
||
"medleyvox_55ep": {
|
||
"model_path": "vocals 55 epochs.pth",
|
||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||
"hf_subfolder": "MedleyVox-MultiSinger",
|
||
"stems": ["vocals", "instrumental"],
|
||
},
|
||
"medleyvox_66ep": {
|
||
"model_path": "vocals 66.pth",
|
||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||
"hf_subfolder": "MedleyVox-MultiSinger",
|
||
"stems": ["vocals", "instrumental"],
|
||
},
|
||
"medleyvox_88ep": {
|
||
"model_path": "vocals 88.pth",
|
||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||
"hf_subfolder": "MedleyVox-MultiSinger",
|
||
"stems": ["vocals", "instrumental"],
|
||
},
|
||
"medleyvox_100ep": {
|
||
"model_path": "vocals 100.pth",
|
||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||
"hf_subfolder": "MedleyVox-MultiSinger",
|
||
"stems": ["vocals", "instrumental"],
|
||
},
|
||
}
|
||
|
||
# Sample rate expected by all MedleyVox checkpoints
|
||
_TARGET_SR = 44100
|
||
# Chunk length for streaming inference (seconds) — keeps VRAM bounded
|
||
_CHUNK_SECS = 30
|
||
|
||
|
||
# ── Inline Conv-TasNet (architecture reverse-engineered from checkpoint) ────────
|
||
|
||
class _GlobalLayerNorm(nn.Module):
|
||
"""Global Layer Norm over (channel × time) with learned gamma/beta."""
|
||
|
||
def __init__(self, n_feats: int, eps: float = 1e-8):
|
||
super().__init__()
|
||
self.gamma = nn.Parameter(torch.ones(n_feats))
|
||
self.beta = nn.Parameter(torch.zeros(n_feats))
|
||
self.eps = eps
|
||
|
||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||
# x: (B, C, T) — normalise over C and T jointly
|
||
mean = x.mean(dim=(1, 2), keepdim=True)
|
||
var = ((x - mean) ** 2).mean(dim=(1, 2), keepdim=True)
|
||
return (
|
||
self.gamma[None, :, None] * (x - mean) / (var + self.eps).sqrt()
|
||
+ self.beta[None, :, None]
|
||
)
|
||
|
||
|
||
class _Conv1DBlock(nn.Module):
|
||
"""One TCN block: pointwise expand → depthwise → residual + skip."""
|
||
|
||
def __init__(self, in_chan: int, hid_chan: int,
|
||
kernel_size: int, dilation: int):
|
||
super().__init__()
|
||
pad = dilation * (kernel_size - 1) // 2
|
||
self.shared_block = nn.Sequential(
|
||
nn.Conv1d(in_chan, hid_chan, 1),
|
||
nn.PReLU(),
|
||
_GlobalLayerNorm(hid_chan),
|
||
nn.Conv1d(hid_chan, hid_chan, kernel_size,
|
||
dilation=dilation, groups=hid_chan, padding=pad),
|
||
nn.PReLU(),
|
||
_GlobalLayerNorm(hid_chan),
|
||
)
|
||
self.res_conv = nn.Conv1d(hid_chan, in_chan, 1)
|
||
self.skip_conv = nn.Conv1d(hid_chan, in_chan, 1)
|
||
|
||
def forward(self, x: torch.Tensor):
|
||
y = self.shared_block(x)
|
||
return x + self.res_conv(y), self.skip_conv(y)
|
||
|
||
|
||
class _Filterbank(nn.Module):
|
||
"""Learnable windowed filterbank stored as _filters + torch_window buffer."""
|
||
|
||
def __init__(self, n_filters: int, kernel_size: int):
|
||
super().__init__()
|
||
self._filters = nn.Parameter(torch.empty(n_filters, 1, kernel_size))
|
||
self.register_buffer("torch_window", torch.hann_window(kernel_size))
|
||
|
||
@property
|
||
def filters(self) -> torch.Tensor:
|
||
"""Windowed filters ready for conv1d / conv_transpose1d."""
|
||
return self._filters * self.torch_window
|
||
|
||
|
||
class _Encoder(nn.Module):
|
||
def __init__(self, n_filters: int, kernel_size: int, stride: int):
|
||
super().__init__()
|
||
self.filterbank = _Filterbank(n_filters, kernel_size)
|
||
self._stride = stride
|
||
self._pad = (kernel_size - stride) // 2
|
||
|
||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||
# x: (B, 1, T) → (B, N, T')
|
||
out = F.conv1d(x, self.filterbank.filters,
|
||
stride=self._stride, padding=self._pad)
|
||
return F.relu(out)
|
||
|
||
|
||
class _Masker(nn.Module):
|
||
def __init__(self, n_filters: int, n_feats: int, n_hidden: int,
|
||
tcn_kernel: int, n_blocks: int, n_repeats: int, n_src: int):
|
||
super().__init__()
|
||
self.bottleneck = nn.Sequential(
|
||
_GlobalLayerNorm(n_filters),
|
||
nn.Conv1d(n_filters, n_feats, 1),
|
||
)
|
||
blocks = []
|
||
for _ in range(n_repeats):
|
||
for b in range(n_blocks):
|
||
blocks.append(
|
||
_Conv1DBlock(n_feats, n_hidden, tcn_kernel, dilation=2 ** b)
|
||
)
|
||
self.TCN = nn.ModuleList(blocks)
|
||
self.mask_net = nn.Sequential(
|
||
nn.PReLU(),
|
||
nn.Conv1d(n_feats, n_filters * n_src, 1),
|
||
)
|
||
self._n_src = n_src
|
||
self._n_filters = n_filters
|
||
|
||
def forward(self, enc: torch.Tensor) -> torch.Tensor:
|
||
# enc: (B, N, T') → masks: (B, n_src, N, T')
|
||
feat = self.bottleneck(enc)
|
||
skip = torch.zeros_like(feat)
|
||
for block in self.TCN:
|
||
feat, s = block(feat)
|
||
skip = skip + s
|
||
masks = self.mask_net(skip) # (B, N*n_src, T')
|
||
B = enc.shape[0]
|
||
masks = masks.view(B, self._n_src, self._n_filters, -1)
|
||
return torch.sigmoid(masks)
|
||
|
||
|
||
class _Decoder(nn.Module):
|
||
def __init__(self, n_filters: int, kernel_size: int, stride: int):
|
||
super().__init__()
|
||
self.filterbank = _Filterbank(n_filters, kernel_size)
|
||
self._stride = stride
|
||
self._pad = (kernel_size - stride) // 2
|
||
|
||
def forward(self, x: torch.Tensor, orig_len: int) -> torch.Tensor:
|
||
# x: (B*n_src, N, T') → (B*n_src, orig_len)
|
||
out = F.conv_transpose1d(x, self.filterbank.filters,
|
||
stride=self._stride, padding=self._pad)
|
||
return out.squeeze(1)[..., :orig_len]
|
||
|
||
|
||
class _ConvTasNet(nn.Module):
|
||
"""
|
||
Conv-TasNet for MedleyVox (2-source: vocals + instrumental).
|
||
Hyperparameters inferred from 'vocals 100.pth' weight shapes:
|
||
N=2050, L=2048, S=1024, B=256, H=1024, P=3, X=8, R=3, n_src=2
|
||
"""
|
||
|
||
def __init__(self):
|
||
super().__init__()
|
||
N, L, S = 2050, 2048, 1024 # encoder/decoder params
|
||
B, H, P, X, R, Ns = 256, 1024, 3, 8, 3, 2
|
||
|
||
self.encoder = _Encoder(N, L, S)
|
||
self.masker = _Masker(N, B, H, P, X, R, Ns)
|
||
self.decoder = _Decoder(N, L, S)
|
||
self._n_src = Ns
|
||
self._n_filters = N
|
||
|
||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||
# x: (B, 1, T) or (B, T) → (B, n_src, T)
|
||
if x.dim() == 2:
|
||
x = x.unsqueeze(1)
|
||
orig_len = x.shape[-1]
|
||
B = x.shape[0]
|
||
|
||
enc = self.encoder(x) # (B, N, T')
|
||
masks = self.masker(enc) # (B, n_src, N, T')
|
||
|
||
masked = (enc.unsqueeze(1) * masks).view(
|
||
B * self._n_src, self._n_filters, -1
|
||
) # (B*n_src, N, T')
|
||
|
||
sources = self.decoder(masked, orig_len) # (B*n_src, T)
|
||
return sources.view(B, self._n_src, orig_len) # (B, n_src, T)
|
||
|
||
|
||
# ── Separator ──────────────────────────────────────────────────────────────────
|
||
|
||
class MedleyVoxSeparator:
|
||
"""
|
||
MedleyVox separator for multi-singer vocal separation.
|
||
Specialized for separating vocals when multiple singers are present.
|
||
"""
|
||
|
||
def __init__(
|
||
self,
|
||
model_id: str = "medleyvox",
|
||
cpu_separation: bool = CPU_SEPARATION,
|
||
output_format: str = "mp3",
|
||
mp3_bitrate: str = "256k",
|
||
):
|
||
if model_id not in MEDLEYVOX_MODELS:
|
||
raise ValueError(f"Unknown MedleyVox model: {model_id}")
|
||
|
||
self.model_id = model_id
|
||
self.model_config = MEDLEYVOX_MODELS[model_id]
|
||
self.cpu_separation = cpu_separation
|
||
self.output_format = output_format
|
||
self.mp3_bitrate = mp3_bitrate
|
||
self._model = None
|
||
self._device = None
|
||
|
||
# ── Model download ────────────────────────────────────────────────────────
|
||
|
||
def _ensure_model(self) -> Path:
|
||
"""Download checkpoint from HuggingFace if not already present."""
|
||
model_dir = MEDLEYVOX_MODEL_DIR / self.model_id
|
||
model_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
model_path = model_dir / self.model_config["model_path"]
|
||
|
||
if not model_path.exists():
|
||
try:
|
||
from huggingface_hub import hf_hub_download
|
||
print(f"Downloading MedleyVox model: {self.model_id}")
|
||
hf_hub_download(
|
||
repo_id=self.model_config["hf_repo"],
|
||
filename=f"{self.model_config['hf_subfolder']}/{self.model_config['model_path']}",
|
||
local_dir=model_dir,
|
||
local_dir_use_symlinks=False,
|
||
)
|
||
except Exception as e:
|
||
raise RuntimeError(f"Failed to download MedleyVox model: {e}")
|
||
|
||
return model_path
|
||
|
||
# ── Model loading ─────────────────────────────────────────────────────────
|
||
|
||
def _load_model(self):
|
||
"""
|
||
Load the checkpoint.
|
||
|
||
Tries four strategies in order:
|
||
1. Checkpoint IS an nn.Module (torch.save(model, path))
|
||
2. Dict with a model/net/network/module key holding an nn.Module
|
||
3. Dict with an EMA online/shadow key holding an nn.Module
|
||
4. Flat state dict (possibly prefixed with 'ema_model.module.' or
|
||
'module.') — loaded into an inline _ConvTasNet instance.
|
||
"""
|
||
if self._model is not None:
|
||
return self._model
|
||
|
||
model_path = self._ensure_model()
|
||
self._device = (
|
||
"cpu" if self.cpu_separation
|
||
else ("cuda" if torch.cuda.is_available() else "cpu")
|
||
)
|
||
|
||
# weights_only=False is required to deserialise a full model object.
|
||
# These checkpoints come from a trusted HuggingFace repository.
|
||
checkpoint = torch.load(str(model_path), map_location="cpu", weights_only=False)
|
||
|
||
# Case 1: checkpoint IS the model (torch.save(model, path))
|
||
if callable(getattr(checkpoint, "forward", None)):
|
||
self._model = checkpoint.to(self._device).eval()
|
||
return self._model
|
||
|
||
if isinstance(checkpoint, dict):
|
||
# Case 2: simple dict wrapper {"model": <nn.Module>, ...}
|
||
for key in ("model", "net", "network", "module"):
|
||
inner = checkpoint.get(key)
|
||
if inner is not None and callable(getattr(inner, "forward", None)):
|
||
self._model = inner.to(self._device).eval()
|
||
return self._model
|
||
|
||
# Case 3: EMA training wrapper — model stored under the online/shadow key
|
||
for key in ("online_model", "online", "ema_model", "ema",
|
||
"shadow_model", "shadow", "averaged_model"):
|
||
inner = checkpoint.get(key)
|
||
if inner is not None and callable(getattr(inner, "forward", None)):
|
||
self._model = inner.to(self._device).eval()
|
||
return self._model
|
||
|
||
# Case 4: flat EMA state dict — strip common prefix, load into _ConvTasNet.
|
||
# MVSEP checkpoints use ema-pytorch which saves a flat OrderedDict
|
||
# with keys prefixed by 'ema_model.module.'.
|
||
for prefix in ("ema_model.module.", "module.", ""):
|
||
if prefix and not any(
|
||
k.startswith(prefix) for k in checkpoint.keys()
|
||
):
|
||
continue
|
||
stripped = {
|
||
(k[len(prefix):] if k.startswith(prefix) else k): v
|
||
for k, v in checkpoint.items()
|
||
}
|
||
net = _ConvTasNet()
|
||
result = net.load_state_dict(stripped, strict=False)
|
||
if not result.missing_keys:
|
||
if result.unexpected_keys:
|
||
print(
|
||
f"[MedleyVox] Ignoring {len(result.unexpected_keys)} "
|
||
"unexpected checkpoint keys."
|
||
)
|
||
self._model = net.to(self._device).eval()
|
||
return self._model
|
||
|
||
keys = list(checkpoint.keys()) if isinstance(checkpoint, dict) else "N/A"
|
||
raise RuntimeError(
|
||
f"Cannot load MedleyVox checkpoint '{model_path.name}'. "
|
||
f"Expected a full torch.nn.Module or compatible state dict but got "
|
||
f"{type(checkpoint).__name__} (keys: {str(keys)[:300]}). "
|
||
"The checkpoint format may have changed — check the HuggingFace repo."
|
||
)
|
||
|
||
# ── Inference ─────────────────────────────────────────────────────────────
|
||
|
||
def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]:
|
||
"""
|
||
Separate audio into stems (vocals / instrumental).
|
||
|
||
Returns:
|
||
Dict mapping stem name → output file path.
|
||
"""
|
||
output_path = Path(output_dir)
|
||
output_path.mkdir(parents=True, exist_ok=True)
|
||
|
||
model = self._load_model()
|
||
|
||
# Load and normalise to mono at the model's expected sample rate
|
||
audio, sr = torchaudio.load(input_path)
|
||
if sr != _TARGET_SR:
|
||
audio = torchaudio.functional.resample(audio, sr, _TARGET_SR)
|
||
sr = _TARGET_SR
|
||
if audio.shape[0] > 1:
|
||
mix = audio.mean(dim=0, keepdim=True) # (1, T)
|
||
else:
|
||
mix = audio
|
||
|
||
mix = mix.to(self._device)
|
||
chunk_samples = sr * _CHUNK_SECS
|
||
|
||
# Process in chunks so GPU memory stays bounded for long tracks
|
||
source_chunks: list[torch.Tensor] = []
|
||
for start in range(0, mix.shape[-1], chunk_samples):
|
||
chunk = mix[:, start : start + chunk_samples] # (1, T')
|
||
with torch.no_grad():
|
||
out = model(chunk.unsqueeze(0)) # (1, n_src, T')
|
||
out = out.squeeze(0).cpu() # (n_src, T')
|
||
source_chunks.append(out)
|
||
|
||
# Concatenate along the time axis (last dim)
|
||
sources = torch.cat(source_chunks, dim=-1) # (n_src, T)
|
||
|
||
# Save each stem to the target format
|
||
stems = self.model_config.get("stems", ["vocals", "instrumental"])
|
||
output_paths: Dict[str, str] = {}
|
||
|
||
for i, stem_name in enumerate(stems):
|
||
if i >= sources.shape[0]:
|
||
break
|
||
stem_audio = sources[i]
|
||
# Ensure shape is (channels, T)
|
||
if stem_audio.dim() == 1:
|
||
stem_audio = stem_audio.unsqueeze(0)
|
||
elif stem_audio.dim() == 3:
|
||
# (channels, 1, T) → squeeze middle dim
|
||
stem_audio = stem_audio.squeeze(1)
|
||
|
||
self._save_stem(stem_audio, sr, stem_name, output_path, output_paths)
|
||
|
||
return output_paths
|
||
|
||
# ── Output helpers ────────────────────────────────────────────────────────
|
||
|
||
def _save_stem(
|
||
self,
|
||
audio: torch.Tensor,
|
||
sample_rate: int,
|
||
stem_name: str,
|
||
output_dir: Path,
|
||
output_paths: Dict[str, str],
|
||
) -> None:
|
||
"""Save a stem tensor in the configured output format."""
|
||
fmt = self.output_format.lower()
|
||
|
||
if fmt in ("mp3", "mp3_320"):
|
||
# Write to WAV first, then encode with pydub
|
||
wav_path = output_dir / f"{stem_name}_tmp.wav"
|
||
torchaudio.save(str(wav_path), audio, sample_rate)
|
||
final_path = output_dir / f"{stem_name}.mp3"
|
||
bitrate = self.mp3_bitrate or ("320k" if fmt == "mp3_320" else "256k")
|
||
from pydub import AudioSegment
|
||
AudioSegment.from_wav(str(wav_path)).export(
|
||
str(final_path), format="mp3", bitrate=bitrate
|
||
)
|
||
wav_path.unlink(missing_ok=True)
|
||
elif fmt == "flac":
|
||
wav_path = output_dir / f"{stem_name}_tmp.wav"
|
||
torchaudio.save(str(wav_path), audio, sample_rate)
|
||
final_path = output_dir / f"{stem_name}.flac"
|
||
from pydub import AudioSegment
|
||
AudioSegment.from_wav(str(wav_path)).export(str(final_path), format="flac")
|
||
wav_path.unlink(missing_ok=True)
|
||
else:
|
||
# Default: WAV
|
||
final_path = output_dir / f"{stem_name}.wav"
|
||
torchaudio.save(str(final_path), audio, sample_rate)
|
||
|
||
output_paths[stem_name] = str(final_path)
|