Files
stemengine_old/backend/services/scnet_separator.py
T
2026-03-03 14:52:19 +09:00

300 lines
11 KiB
Python
Executable File

"""
SCNet separator — native PyTorch inference.
Models from MVSEP-Central-Backup (HuggingFace).
Architecture vendored from MSST (MIT license):
https://github.com/ZFTurbo/Music-Source-Separation-Training
"""
import shutil
from pathlib import Path
from typing import Dict, Optional
import numpy as np
import torch
import torchaudio
import yaml
from .scnet_arch import SCNet
from ..config import SCNET_MODEL_DIR, CPU_SEPARATION
# ---------------------------------------------------------------------------
# Model registry
# ---------------------------------------------------------------------------
SCNET_MODELS = {
"scnet_xl": {
"model_file": "scnet_checkpoint_musdb18.ckpt",
"config_file": "config_musdb18_scnet_xl.yaml",
"hf_repo": "miercolesv/MVSEP-Central-Backup",
"hf_model_path": "Politrees/UVR_resources/models/SCnet/scnet_checkpoint_musdb18.ckpt",
"hf_config_path": "Politrees/UVR_resources/models/SCnet/config_musdb18_scnet_xl.yaml",
},
"scnet_xl_ihf": {
"model_file": "scnet_checkpoint_musdb18.ckpt",
"config_file": "config_musdb18_scnet_xl.yaml",
"hf_repo": "miercolesv/MVSEP-Central-Backup",
"hf_model_path": "Politrees/UVR_resources/models/SCnet/scnet_checkpoint_musdb18.ckpt",
"hf_config_path": "Politrees/UVR_resources/models/SCnet/config_musdb18_scnet_xl.yaml",
},
"scnet_large": {
"model_file": "SCNet-large_starrytong_fixed.ckpt",
"config_file": "config_musdb18_scnet_large_starrytong.yaml",
"hf_repo": "miercolesv/MVSEP-Central-Backup",
"hf_model_path": "Politrees/UVR_resources/models/SCnet/SCNet-large_starrytong_fixed.ckpt",
"hf_config_path": "Politrees/UVR_resources/models/SCnet/config_musdb18_scnet_large_starrytong.yaml",
},
"scnet_large_starrytong": {
"model_file": "SCNet-large_starrytong_fixed.ckpt",
"config_file": "config_musdb18_scnet_large_starrytong.yaml",
"hf_repo": "miercolesv/MVSEP-Central-Backup",
"hf_model_path": "Politrees/UVR_resources/models/SCnet/SCNet-large_starrytong_fixed.ckpt",
"hf_config_path": "Politrees/UVR_resources/models/SCnet/config_musdb18_scnet_large_starrytong.yaml",
},
}
# ---------------------------------------------------------------------------
# Audio I/O helpers
# ---------------------------------------------------------------------------
def _load_audio(path: str, sample_rate: int) -> np.ndarray:
"""Load audio and return (2, samples) float32 array at target sample rate."""
waveform, sr = torchaudio.load(path)
if sr != sample_rate:
waveform = torchaudio.functional.resample(waveform, sr, sample_rate)
if waveform.shape[0] == 1:
waveform = waveform.repeat(2, 1)
elif waveform.shape[0] > 2:
waveform = waveform[:2]
return waveform.numpy().astype(np.float32)
def _save_audio(path: str, audio: np.ndarray, sample_rate: int, output_format: str):
"""Save (channels, samples) numpy array to file via torchaudio (ffmpeg backend)."""
tensor = torch.tensor(audio, dtype=torch.float32)
fmt = output_format.lower()
if fmt == "mp3":
torchaudio.save(path, tensor, sample_rate, format="mp3")
elif fmt == "flac":
torchaudio.save(path, tensor, sample_rate, format="flac")
else:
torchaudio.save(path, tensor, sample_rate)
# ---------------------------------------------------------------------------
# Overlapping-window inference (MSST style)
# ---------------------------------------------------------------------------
def _build_window(chunk_size: int, step: int) -> np.ndarray:
"""
Linear fade-in / flat / fade-out window of length chunk_size.
Consecutive windows (offset by step) partition of unity: they sum to a
constant, so the overlap-add correctly averages.
"""
fade = step
fade_in = np.linspace(0.0, 1.0, fade, endpoint=False, dtype=np.float32)
fade_out = np.linspace(1.0, 0.0, fade, endpoint=False, dtype=np.float32)
flat_len = chunk_size - 2 * fade
if flat_len > 0:
return np.concatenate([fade_in, np.ones(flat_len, dtype=np.float32), fade_out])
# num_overlap == 2: no flat region
return np.concatenate([fade_in, fade_out])
def _demix(
model: SCNet,
mix: np.ndarray,
chunk_size: int,
num_overlap: int,
device: torch.device,
) -> np.ndarray:
"""
MSST-style overlapping-window inference.
Args:
mix: (channels, samples) float32
Returns:
(num_sources, channels, samples) float32
"""
C, N = mix.shape
step = chunk_size // num_overlap
border = chunk_size - step # == (num_overlap-1)*step
# Reflect-pad both sides so edge frames get proper context.
mix_padded = np.pad(mix, ((0, 0), (border, border)), mode="reflect")
window = _build_window(chunk_size, step)
sources_sum: Optional[np.ndarray] = None
weight_sum = np.zeros(mix_padded.shape[-1], dtype=np.float32)
with torch.no_grad():
for start in range(0, N, step):
p_end = start + chunk_size
if p_end > mix_padded.shape[-1]:
break
chunk = mix_padded[:, start:p_end] # (C, chunk_size)
# Per-chunk loudness normalisation
mean = float(chunk.mean())
std = max(float(chunk.std()), 1e-8)
chunk_norm = (chunk - mean) / std
inp = torch.tensor(chunk_norm[np.newaxis], dtype=torch.float32, device=device)
out = model(inp) # (1, num_sources, C, chunk_size)
out_np = out.squeeze(0).cpu().numpy() # (num_sources, C, chunk_size)
out_np = out_np * std + mean # denormalise
if sources_sum is None:
num_sources = out_np.shape[0]
sources_sum = np.zeros(
(num_sources, C, mix_padded.shape[-1]), dtype=np.float32
)
sources_sum[:, :, start:p_end] += out_np * window[np.newaxis, np.newaxis, :]
weight_sum[start:p_end] += window
if sources_sum is None:
raise RuntimeError("Audio is too short for chunked SCNet inference.")
weight_sum = np.maximum(weight_sum, 1e-8)
sources = sources_sum / weight_sum[np.newaxis, np.newaxis, :]
return sources[:, :, border : border + N]
# ---------------------------------------------------------------------------
# Separator class
# ---------------------------------------------------------------------------
class SCNetSeparator:
"""Native PyTorch inference wrapper for MSST SCNet models."""
def __init__(
self,
model_id: str = "scnet_xl",
cpu_separation: bool = CPU_SEPARATION,
output_format: str = "mp3",
mp3_bitrate: str = "256k",
):
if model_id not in SCNET_MODELS:
raise ValueError(
f"Unknown SCNet model: {model_id}. "
f"Available: {list(SCNET_MODELS)}"
)
self.model_id = model_id
self.model_cfg = SCNET_MODELS[model_id]
self.cpu_separation = cpu_separation
self.output_format = output_format
self.mp3_bitrate = mp3_bitrate
self.device = torch.device(
"cpu" if cpu_separation else ("cuda" if torch.cuda.is_available() else "cpu")
)
self._model: Optional[SCNet] = None
self._config: Optional[dict] = None
# ------------------------------------------------------------------
# Model management
# ------------------------------------------------------------------
def _ensure_model(self) -> tuple[Path, Path]:
"""Download model and config from HuggingFace if not already cached."""
from huggingface_hub import hf_hub_download
model_dir = SCNET_MODEL_DIR / self.model_id
model_dir.mkdir(parents=True, exist_ok=True)
model_path = model_dir / self.model_cfg["model_file"]
config_path = model_dir / self.model_cfg["config_file"]
if not model_path.exists():
print(f"[SCNet] Downloading {self.model_cfg['model_file']} …")
cached = hf_hub_download(
repo_id=self.model_cfg["hf_repo"],
filename=self.model_cfg["hf_model_path"],
)
shutil.copy2(cached, model_path)
if not config_path.exists():
print(f"[SCNet] Downloading {self.model_cfg['config_file']} …")
cached = hf_hub_download(
repo_id=self.model_cfg["hf_repo"],
filename=self.model_cfg["hf_config_path"],
)
shutil.copy2(cached, config_path)
return model_path, config_path
def _load_model(self) -> None:
"""Load YAML config and PyTorch checkpoint; instantiate SCNet."""
model_path, config_path = self._ensure_model()
# Some config files use !!python/tuple (e.g. mixup_probs); add a
# constructor so safe_load can handle them without crashing.
loader = yaml.SafeLoader
loader.add_constructor(
"tag:yaml.org,2002:python/tuple",
lambda ldr, node: tuple(ldr.construct_sequence(node)),
)
with open(config_path) as f:
cfg = yaml.load(f, Loader=loader)
self._config = cfg
model_params = cfg.get("model", {})
model = SCNet(**model_params)
state = torch.load(str(model_path), map_location="cpu", weights_only=False)
if "state_dict" in state:
state = state["state_dict"]
model.load_state_dict(state)
model.eval()
self._model = model.to(self.device)
def _get_model(self) -> SCNet:
if self._model is None:
self._load_model()
return self._model
# ------------------------------------------------------------------
# Public API
# ------------------------------------------------------------------
def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]:
"""
Separate audio into stems.
Returns:
Dict mapping stem name → output file path.
"""
model = self._get_model()
cfg = self._config
sample_rate: int = cfg.get("audio", {}).get("sample_rate", 44100)
inf_cfg: dict = cfg.get("inference", {})
chunk_size: int = inf_cfg.get("chunk_size", 485100)
num_overlap: int = inf_cfg.get("num_overlap", 2)
# Stem order from training config; fall back to model attribute
instruments = cfg.get("training", {}).get("instruments") or list(model.sources)
mix = _load_audio(input_path, sample_rate) # (2, N)
print(f"[SCNet] Running {self.model_id} on {self.device} …")
sources_np = _demix(model, mix, chunk_size, num_overlap, self.device)
# sources_np: (num_sources, 2, N)
out_dir = Path(output_dir)
out_dir.mkdir(parents=True, exist_ok=True)
output_paths: Dict[str, str] = {}
for i, stem in enumerate(instruments):
if i >= sources_np.shape[0]:
break
ext = self.output_format.lower()
out_path = str(out_dir / f"{stem}.{ext}")
_save_audio(out_path, sources_np[i], sample_rate, self.output_format)
output_paths[stem] = out_path
print(f"[SCNet] Saved {stem} → {out_path}")
return output_paths