Files
stemengine_old/backend/services/medleyvox_separator.py
T
2026-03-03 14:52:19 +09:00

445 lines
17 KiB
Python
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
MedleyVox separator - Multi-singer vocal separation
HuggingFace model: Cyru5/MedleyVox
Backup: miercolesv/MVSEP-Central-Backup
The MVSEP checkpoints are flat EMA state-dicts (OrderedDict with an
'ema_model.module.' key prefix). This file contains an inline Conv-TasNet
implementation whose architecture was reverse-engineered from those weights:
N=2050 filters, L=2048 kernel, S=1024 stride,
B=256 bottleneck, H=1024 hidden, P=3 TCN kernel,
X=8 blocks/repeat, R=3 repeats, n_src=2 (vocals + instrumental).
"""
import torch
import torch.nn as nn
import torch.nn.functional as F
import torchaudio
from pathlib import Path
from typing import Dict
from ..config import MEDLEYVOX_MODEL_DIR, CPU_SEPARATION
# ── Model configurations ───────────────────────────────────────────────────────
MEDLEYVOX_MODELS = {
"medleyvox": {
"model_path": "vocals 100.pth",
"hf_repo": "miercolesv/MVSEP-Central-Backup",
"hf_subfolder": "MedleyVox-MultiSinger",
"stems": ["vocals", "instrumental"],
},
"medleyvox_55ep": {
"model_path": "vocals 55 epochs.pth",
"hf_repo": "miercolesv/MVSEP-Central-Backup",
"hf_subfolder": "MedleyVox-MultiSinger",
"stems": ["vocals", "instrumental"],
},
"medleyvox_66ep": {
"model_path": "vocals 66.pth",
"hf_repo": "miercolesv/MVSEP-Central-Backup",
"hf_subfolder": "MedleyVox-MultiSinger",
"stems": ["vocals", "instrumental"],
},
"medleyvox_88ep": {
"model_path": "vocals 88.pth",
"hf_repo": "miercolesv/MVSEP-Central-Backup",
"hf_subfolder": "MedleyVox-MultiSinger",
"stems": ["vocals", "instrumental"],
},
"medleyvox_100ep": {
"model_path": "vocals 100.pth",
"hf_repo": "miercolesv/MVSEP-Central-Backup",
"hf_subfolder": "MedleyVox-MultiSinger",
"stems": ["vocals", "instrumental"],
},
}
# Sample rate expected by all MedleyVox checkpoints
_TARGET_SR = 44100
# Chunk length for streaming inference (seconds) — keeps VRAM bounded
_CHUNK_SECS = 30
# ── Inline Conv-TasNet (architecture reverse-engineered from checkpoint) ────────
class _GlobalLayerNorm(nn.Module):
"""Global Layer Norm over (channel × time) with learned gamma/beta."""
def __init__(self, n_feats: int, eps: float = 1e-8):
super().__init__()
self.gamma = nn.Parameter(torch.ones(n_feats))
self.beta = nn.Parameter(torch.zeros(n_feats))
self.eps = eps
def forward(self, x: torch.Tensor) -> torch.Tensor:
# x: (B, C, T) — normalise over C and T jointly
mean = x.mean(dim=(1, 2), keepdim=True)
var = ((x - mean) ** 2).mean(dim=(1, 2), keepdim=True)
return (
self.gamma[None, :, None] * (x - mean) / (var + self.eps).sqrt()
+ self.beta[None, :, None]
)
class _Conv1DBlock(nn.Module):
"""One TCN block: pointwise expand → depthwise → residual + skip."""
def __init__(self, in_chan: int, hid_chan: int,
kernel_size: int, dilation: int):
super().__init__()
pad = dilation * (kernel_size - 1) // 2
self.shared_block = nn.Sequential(
nn.Conv1d(in_chan, hid_chan, 1),
nn.PReLU(),
_GlobalLayerNorm(hid_chan),
nn.Conv1d(hid_chan, hid_chan, kernel_size,
dilation=dilation, groups=hid_chan, padding=pad),
nn.PReLU(),
_GlobalLayerNorm(hid_chan),
)
self.res_conv = nn.Conv1d(hid_chan, in_chan, 1)
self.skip_conv = nn.Conv1d(hid_chan, in_chan, 1)
def forward(self, x: torch.Tensor):
y = self.shared_block(x)
return x + self.res_conv(y), self.skip_conv(y)
class _Filterbank(nn.Module):
"""Learnable windowed filterbank stored as _filters + torch_window buffer."""
def __init__(self, n_filters: int, kernel_size: int):
super().__init__()
self._filters = nn.Parameter(torch.empty(n_filters, 1, kernel_size))
self.register_buffer("torch_window", torch.hann_window(kernel_size))
@property
def filters(self) -> torch.Tensor:
"""Windowed filters ready for conv1d / conv_transpose1d."""
return self._filters * self.torch_window
class _Encoder(nn.Module):
def __init__(self, n_filters: int, kernel_size: int, stride: int):
super().__init__()
self.filterbank = _Filterbank(n_filters, kernel_size)
self._stride = stride
self._pad = (kernel_size - stride) // 2
def forward(self, x: torch.Tensor) -> torch.Tensor:
# x: (B, 1, T) → (B, N, T')
out = F.conv1d(x, self.filterbank.filters,
stride=self._stride, padding=self._pad)
return F.relu(out)
class _Masker(nn.Module):
def __init__(self, n_filters: int, n_feats: int, n_hidden: int,
tcn_kernel: int, n_blocks: int, n_repeats: int, n_src: int):
super().__init__()
self.bottleneck = nn.Sequential(
_GlobalLayerNorm(n_filters),
nn.Conv1d(n_filters, n_feats, 1),
)
blocks = []
for _ in range(n_repeats):
for b in range(n_blocks):
blocks.append(
_Conv1DBlock(n_feats, n_hidden, tcn_kernel, dilation=2 ** b)
)
self.TCN = nn.ModuleList(blocks)
self.mask_net = nn.Sequential(
nn.PReLU(),
nn.Conv1d(n_feats, n_filters * n_src, 1),
)
self._n_src = n_src
self._n_filters = n_filters
def forward(self, enc: torch.Tensor) -> torch.Tensor:
# enc: (B, N, T') → masks: (B, n_src, N, T')
feat = self.bottleneck(enc)
skip = torch.zeros_like(feat)
for block in self.TCN:
feat, s = block(feat)
skip = skip + s
masks = self.mask_net(skip) # (B, N*n_src, T')
B = enc.shape[0]
masks = masks.view(B, self._n_src, self._n_filters, -1)
return torch.sigmoid(masks)
class _Decoder(nn.Module):
def __init__(self, n_filters: int, kernel_size: int, stride: int):
super().__init__()
self.filterbank = _Filterbank(n_filters, kernel_size)
self._stride = stride
self._pad = (kernel_size - stride) // 2
def forward(self, x: torch.Tensor, orig_len: int) -> torch.Tensor:
# x: (B*n_src, N, T') → (B*n_src, orig_len)
out = F.conv_transpose1d(x, self.filterbank.filters,
stride=self._stride, padding=self._pad)
return out.squeeze(1)[..., :orig_len]
class _ConvTasNet(nn.Module):
"""
Conv-TasNet for MedleyVox (2-source: vocals + instrumental).
Hyperparameters inferred from 'vocals 100.pth' weight shapes:
N=2050, L=2048, S=1024, B=256, H=1024, P=3, X=8, R=3, n_src=2
"""
def __init__(self):
super().__init__()
N, L, S = 2050, 2048, 1024 # encoder/decoder params
B, H, P, X, R, Ns = 256, 1024, 3, 8, 3, 2
self.encoder = _Encoder(N, L, S)
self.masker = _Masker(N, B, H, P, X, R, Ns)
self.decoder = _Decoder(N, L, S)
self._n_src = Ns
self._n_filters = N
def forward(self, x: torch.Tensor) -> torch.Tensor:
# x: (B, 1, T) or (B, T) → (B, n_src, T)
if x.dim() == 2:
x = x.unsqueeze(1)
orig_len = x.shape[-1]
B = x.shape[0]
enc = self.encoder(x) # (B, N, T')
masks = self.masker(enc) # (B, n_src, N, T')
masked = (enc.unsqueeze(1) * masks).view(
B * self._n_src, self._n_filters, -1
) # (B*n_src, N, T')
sources = self.decoder(masked, orig_len) # (B*n_src, T)
return sources.view(B, self._n_src, orig_len) # (B, n_src, T)
# ── Separator ──────────────────────────────────────────────────────────────────
class MedleyVoxSeparator:
"""
MedleyVox separator for multi-singer vocal separation.
Specialized for separating vocals when multiple singers are present.
"""
def __init__(
self,
model_id: str = "medleyvox",
cpu_separation: bool = CPU_SEPARATION,
output_format: str = "mp3",
mp3_bitrate: str = "256k",
):
if model_id not in MEDLEYVOX_MODELS:
raise ValueError(f"Unknown MedleyVox model: {model_id}")
self.model_id = model_id
self.model_config = MEDLEYVOX_MODELS[model_id]
self.cpu_separation = cpu_separation
self.output_format = output_format
self.mp3_bitrate = mp3_bitrate
self._model = None
self._device = None
# ── Model download ────────────────────────────────────────────────────────
def _ensure_model(self) -> Path:
"""Download checkpoint from HuggingFace if not already present."""
model_dir = MEDLEYVOX_MODEL_DIR / self.model_id
model_dir.mkdir(parents=True, exist_ok=True)
model_path = model_dir / self.model_config["model_path"]
if not model_path.exists():
try:
from huggingface_hub import hf_hub_download
print(f"Downloading MedleyVox model: {self.model_id}")
hf_hub_download(
repo_id=self.model_config["hf_repo"],
filename=f"{self.model_config['hf_subfolder']}/{self.model_config['model_path']}",
local_dir=model_dir,
local_dir_use_symlinks=False,
)
except Exception as e:
raise RuntimeError(f"Failed to download MedleyVox model: {e}")
return model_path
# ── Model loading ─────────────────────────────────────────────────────────
def _load_model(self):
"""
Load the checkpoint.
Tries four strategies in order:
1. Checkpoint IS an nn.Module (torch.save(model, path))
2. Dict with a model/net/network/module key holding an nn.Module
3. Dict with an EMA online/shadow key holding an nn.Module
4. Flat state dict (possibly prefixed with 'ema_model.module.' or
'module.') — loaded into an inline _ConvTasNet instance.
"""
if self._model is not None:
return self._model
model_path = self._ensure_model()
self._device = (
"cpu" if self.cpu_separation
else ("cuda" if torch.cuda.is_available() else "cpu")
)
# weights_only=False is required to deserialise a full model object.
# These checkpoints come from a trusted HuggingFace repository.
checkpoint = torch.load(str(model_path), map_location="cpu", weights_only=False)
# Case 1: checkpoint IS the model (torch.save(model, path))
if callable(getattr(checkpoint, "forward", None)):
self._model = checkpoint.to(self._device).eval()
return self._model
if isinstance(checkpoint, dict):
# Case 2: simple dict wrapper {"model": <nn.Module>, ...}
for key in ("model", "net", "network", "module"):
inner = checkpoint.get(key)
if inner is not None and callable(getattr(inner, "forward", None)):
self._model = inner.to(self._device).eval()
return self._model
# Case 3: EMA training wrapper — model stored under the online/shadow key
for key in ("online_model", "online", "ema_model", "ema",
"shadow_model", "shadow", "averaged_model"):
inner = checkpoint.get(key)
if inner is not None and callable(getattr(inner, "forward", None)):
self._model = inner.to(self._device).eval()
return self._model
# Case 4: flat EMA state dict — strip common prefix, load into _ConvTasNet.
# MVSEP checkpoints use ema-pytorch which saves a flat OrderedDict
# with keys prefixed by 'ema_model.module.'.
for prefix in ("ema_model.module.", "module.", ""):
if prefix and not any(
k.startswith(prefix) for k in checkpoint.keys()
):
continue
stripped = {
(k[len(prefix):] if k.startswith(prefix) else k): v
for k, v in checkpoint.items()
}
net = _ConvTasNet()
result = net.load_state_dict(stripped, strict=False)
if not result.missing_keys:
if result.unexpected_keys:
print(
f"[MedleyVox] Ignoring {len(result.unexpected_keys)} "
"unexpected checkpoint keys."
)
self._model = net.to(self._device).eval()
return self._model
keys = list(checkpoint.keys()) if isinstance(checkpoint, dict) else "N/A"
raise RuntimeError(
f"Cannot load MedleyVox checkpoint '{model_path.name}'. "
f"Expected a full torch.nn.Module or compatible state dict but got "
f"{type(checkpoint).__name__} (keys: {str(keys)[:300]}). "
"The checkpoint format may have changed — check the HuggingFace repo."
)
# ── Inference ─────────────────────────────────────────────────────────────
def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]:
"""
Separate audio into stems (vocals / instrumental).
Returns:
Dict mapping stem name → output file path.
"""
output_path = Path(output_dir)
output_path.mkdir(parents=True, exist_ok=True)
model = self._load_model()
# Load and normalise to mono at the model's expected sample rate
audio, sr = torchaudio.load(input_path)
if sr != _TARGET_SR:
audio = torchaudio.functional.resample(audio, sr, _TARGET_SR)
sr = _TARGET_SR
if audio.shape[0] > 1:
mix = audio.mean(dim=0, keepdim=True) # (1, T)
else:
mix = audio
mix = mix.to(self._device)
chunk_samples = sr * _CHUNK_SECS
# Process in chunks so GPU memory stays bounded for long tracks
source_chunks: list[torch.Tensor] = []
for start in range(0, mix.shape[-1], chunk_samples):
chunk = mix[:, start : start + chunk_samples] # (1, T')
with torch.no_grad():
out = model(chunk.unsqueeze(0)) # (1, n_src, T')
out = out.squeeze(0).cpu() # (n_src, T')
source_chunks.append(out)
# Concatenate along the time axis (last dim)
sources = torch.cat(source_chunks, dim=-1) # (n_src, T)
# Save each stem to the target format
stems = self.model_config.get("stems", ["vocals", "instrumental"])
output_paths: Dict[str, str] = {}
for i, stem_name in enumerate(stems):
if i >= sources.shape[0]:
break
stem_audio = sources[i]
# Ensure shape is (channels, T)
if stem_audio.dim() == 1:
stem_audio = stem_audio.unsqueeze(0)
elif stem_audio.dim() == 3:
# (channels, 1, T) → squeeze middle dim
stem_audio = stem_audio.squeeze(1)
self._save_stem(stem_audio, sr, stem_name, output_path, output_paths)
return output_paths
# ── Output helpers ────────────────────────────────────────────────────────
def _save_stem(
self,
audio: torch.Tensor,
sample_rate: int,
stem_name: str,
output_dir: Path,
output_paths: Dict[str, str],
) -> None:
"""Save a stem tensor in the configured output format."""
fmt = self.output_format.lower()
if fmt in ("mp3", "mp3_320"):
# Write to WAV first, then encode with pydub
wav_path = output_dir / f"{stem_name}_tmp.wav"
torchaudio.save(str(wav_path), audio, sample_rate)
final_path = output_dir / f"{stem_name}.mp3"
bitrate = self.mp3_bitrate or ("320k" if fmt == "mp3_320" else "256k")
from pydub import AudioSegment
AudioSegment.from_wav(str(wav_path)).export(
str(final_path), format="mp3", bitrate=bitrate
)
wav_path.unlink(missing_ok=True)
elif fmt == "flac":
wav_path = output_dir / f"{stem_name}_tmp.wav"
torchaudio.save(str(wav_path), audio, sample_rate)
final_path = output_dir / f"{stem_name}.flac"
from pydub import AudioSegment
AudioSegment.from_wav(str(wav_path)).export(str(final_path), format="flac")
wav_path.unlink(missing_ok=True)
else:
# Default: WAV
final_path = output_dir / f"{stem_name}.wav"
torchaudio.save(str(final_path), audio, sample_rate)
output_paths[stem_name] = str(final_path)