300 lines
11 KiB
Python
Executable File
300 lines
11 KiB
Python
Executable File
"""
|
|
SCNet separator — native PyTorch inference.
|
|
Models from MVSEP-Central-Backup (HuggingFace).
|
|
Architecture vendored from MSST (MIT license):
|
|
https://github.com/ZFTurbo/Music-Source-Separation-Training
|
|
"""
|
|
import shutil
|
|
from pathlib import Path
|
|
from typing import Dict, Optional
|
|
|
|
import numpy as np
|
|
import torch
|
|
import torchaudio
|
|
import yaml
|
|
|
|
from .scnet_arch import SCNet
|
|
from ..config import SCNET_MODEL_DIR, CPU_SEPARATION
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Model registry
|
|
# ---------------------------------------------------------------------------
|
|
|
|
SCNET_MODELS = {
|
|
"scnet_xl": {
|
|
"model_file": "scnet_checkpoint_musdb18.ckpt",
|
|
"config_file": "config_musdb18_scnet_xl.yaml",
|
|
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
|
"hf_model_path": "Politrees/UVR_resources/models/SCnet/scnet_checkpoint_musdb18.ckpt",
|
|
"hf_config_path": "Politrees/UVR_resources/models/SCnet/config_musdb18_scnet_xl.yaml",
|
|
},
|
|
"scnet_xl_ihf": {
|
|
"model_file": "scnet_checkpoint_musdb18.ckpt",
|
|
"config_file": "config_musdb18_scnet_xl.yaml",
|
|
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
|
"hf_model_path": "Politrees/UVR_resources/models/SCnet/scnet_checkpoint_musdb18.ckpt",
|
|
"hf_config_path": "Politrees/UVR_resources/models/SCnet/config_musdb18_scnet_xl.yaml",
|
|
},
|
|
"scnet_large": {
|
|
"model_file": "SCNet-large_starrytong_fixed.ckpt",
|
|
"config_file": "config_musdb18_scnet_large_starrytong.yaml",
|
|
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
|
"hf_model_path": "Politrees/UVR_resources/models/SCnet/SCNet-large_starrytong_fixed.ckpt",
|
|
"hf_config_path": "Politrees/UVR_resources/models/SCnet/config_musdb18_scnet_large_starrytong.yaml",
|
|
},
|
|
"scnet_large_starrytong": {
|
|
"model_file": "SCNet-large_starrytong_fixed.ckpt",
|
|
"config_file": "config_musdb18_scnet_large_starrytong.yaml",
|
|
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
|
"hf_model_path": "Politrees/UVR_resources/models/SCnet/SCNet-large_starrytong_fixed.ckpt",
|
|
"hf_config_path": "Politrees/UVR_resources/models/SCnet/config_musdb18_scnet_large_starrytong.yaml",
|
|
},
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Audio I/O helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _load_audio(path: str, sample_rate: int) -> np.ndarray:
|
|
"""Load audio and return (2, samples) float32 array at target sample rate."""
|
|
waveform, sr = torchaudio.load(path)
|
|
if sr != sample_rate:
|
|
waveform = torchaudio.functional.resample(waveform, sr, sample_rate)
|
|
if waveform.shape[0] == 1:
|
|
waveform = waveform.repeat(2, 1)
|
|
elif waveform.shape[0] > 2:
|
|
waveform = waveform[:2]
|
|
return waveform.numpy().astype(np.float32)
|
|
|
|
|
|
def _save_audio(path: str, audio: np.ndarray, sample_rate: int, output_format: str):
|
|
"""Save (channels, samples) numpy array to file via torchaudio (ffmpeg backend)."""
|
|
tensor = torch.tensor(audio, dtype=torch.float32)
|
|
fmt = output_format.lower()
|
|
if fmt == "mp3":
|
|
torchaudio.save(path, tensor, sample_rate, format="mp3")
|
|
elif fmt == "flac":
|
|
torchaudio.save(path, tensor, sample_rate, format="flac")
|
|
else:
|
|
torchaudio.save(path, tensor, sample_rate)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Overlapping-window inference (MSST style)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _build_window(chunk_size: int, step: int) -> np.ndarray:
|
|
"""
|
|
Linear fade-in / flat / fade-out window of length chunk_size.
|
|
Consecutive windows (offset by step) partition of unity: they sum to a
|
|
constant, so the overlap-add correctly averages.
|
|
"""
|
|
fade = step
|
|
fade_in = np.linspace(0.0, 1.0, fade, endpoint=False, dtype=np.float32)
|
|
fade_out = np.linspace(1.0, 0.0, fade, endpoint=False, dtype=np.float32)
|
|
flat_len = chunk_size - 2 * fade
|
|
if flat_len > 0:
|
|
return np.concatenate([fade_in, np.ones(flat_len, dtype=np.float32), fade_out])
|
|
# num_overlap == 2: no flat region
|
|
return np.concatenate([fade_in, fade_out])
|
|
|
|
|
|
def _demix(
|
|
model: SCNet,
|
|
mix: np.ndarray,
|
|
chunk_size: int,
|
|
num_overlap: int,
|
|
device: torch.device,
|
|
) -> np.ndarray:
|
|
"""
|
|
MSST-style overlapping-window inference.
|
|
|
|
Args:
|
|
mix: (channels, samples) float32
|
|
Returns:
|
|
(num_sources, channels, samples) float32
|
|
"""
|
|
C, N = mix.shape
|
|
step = chunk_size // num_overlap
|
|
border = chunk_size - step # == (num_overlap-1)*step
|
|
|
|
# Reflect-pad both sides so edge frames get proper context.
|
|
mix_padded = np.pad(mix, ((0, 0), (border, border)), mode="reflect")
|
|
|
|
window = _build_window(chunk_size, step)
|
|
sources_sum: Optional[np.ndarray] = None
|
|
weight_sum = np.zeros(mix_padded.shape[-1], dtype=np.float32)
|
|
|
|
with torch.no_grad():
|
|
for start in range(0, N, step):
|
|
p_end = start + chunk_size
|
|
if p_end > mix_padded.shape[-1]:
|
|
break
|
|
chunk = mix_padded[:, start:p_end] # (C, chunk_size)
|
|
|
|
# Per-chunk loudness normalisation
|
|
mean = float(chunk.mean())
|
|
std = max(float(chunk.std()), 1e-8)
|
|
chunk_norm = (chunk - mean) / std
|
|
|
|
inp = torch.tensor(chunk_norm[np.newaxis], dtype=torch.float32, device=device)
|
|
out = model(inp) # (1, num_sources, C, chunk_size)
|
|
out_np = out.squeeze(0).cpu().numpy() # (num_sources, C, chunk_size)
|
|
out_np = out_np * std + mean # denormalise
|
|
|
|
if sources_sum is None:
|
|
num_sources = out_np.shape[0]
|
|
sources_sum = np.zeros(
|
|
(num_sources, C, mix_padded.shape[-1]), dtype=np.float32
|
|
)
|
|
|
|
sources_sum[:, :, start:p_end] += out_np * window[np.newaxis, np.newaxis, :]
|
|
weight_sum[start:p_end] += window
|
|
|
|
if sources_sum is None:
|
|
raise RuntimeError("Audio is too short for chunked SCNet inference.")
|
|
|
|
weight_sum = np.maximum(weight_sum, 1e-8)
|
|
sources = sources_sum / weight_sum[np.newaxis, np.newaxis, :]
|
|
return sources[:, :, border : border + N]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Separator class
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class SCNetSeparator:
|
|
"""Native PyTorch inference wrapper for MSST SCNet models."""
|
|
|
|
def __init__(
|
|
self,
|
|
model_id: str = "scnet_xl",
|
|
cpu_separation: bool = CPU_SEPARATION,
|
|
output_format: str = "mp3",
|
|
mp3_bitrate: str = "256k",
|
|
):
|
|
if model_id not in SCNET_MODELS:
|
|
raise ValueError(
|
|
f"Unknown SCNet model: {model_id}. "
|
|
f"Available: {list(SCNET_MODELS)}"
|
|
)
|
|
|
|
self.model_id = model_id
|
|
self.model_cfg = SCNET_MODELS[model_id]
|
|
self.cpu_separation = cpu_separation
|
|
self.output_format = output_format
|
|
self.mp3_bitrate = mp3_bitrate
|
|
self.device = torch.device(
|
|
"cpu" if cpu_separation else ("cuda" if torch.cuda.is_available() else "cpu")
|
|
)
|
|
|
|
self._model: Optional[SCNet] = None
|
|
self._config: Optional[dict] = None
|
|
|
|
# ------------------------------------------------------------------
|
|
# Model management
|
|
# ------------------------------------------------------------------
|
|
|
|
def _ensure_model(self) -> tuple[Path, Path]:
|
|
"""Download model and config from HuggingFace if not already cached."""
|
|
from huggingface_hub import hf_hub_download
|
|
|
|
model_dir = SCNET_MODEL_DIR / self.model_id
|
|
model_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
model_path = model_dir / self.model_cfg["model_file"]
|
|
config_path = model_dir / self.model_cfg["config_file"]
|
|
|
|
if not model_path.exists():
|
|
print(f"[SCNet] Downloading {self.model_cfg['model_file']} …")
|
|
cached = hf_hub_download(
|
|
repo_id=self.model_cfg["hf_repo"],
|
|
filename=self.model_cfg["hf_model_path"],
|
|
)
|
|
shutil.copy2(cached, model_path)
|
|
|
|
if not config_path.exists():
|
|
print(f"[SCNet] Downloading {self.model_cfg['config_file']} …")
|
|
cached = hf_hub_download(
|
|
repo_id=self.model_cfg["hf_repo"],
|
|
filename=self.model_cfg["hf_config_path"],
|
|
)
|
|
shutil.copy2(cached, config_path)
|
|
|
|
return model_path, config_path
|
|
|
|
def _load_model(self) -> None:
|
|
"""Load YAML config and PyTorch checkpoint; instantiate SCNet."""
|
|
model_path, config_path = self._ensure_model()
|
|
|
|
# Some config files use !!python/tuple (e.g. mixup_probs); add a
|
|
# constructor so safe_load can handle them without crashing.
|
|
loader = yaml.SafeLoader
|
|
loader.add_constructor(
|
|
"tag:yaml.org,2002:python/tuple",
|
|
lambda ldr, node: tuple(ldr.construct_sequence(node)),
|
|
)
|
|
with open(config_path) as f:
|
|
cfg = yaml.load(f, Loader=loader)
|
|
self._config = cfg
|
|
|
|
model_params = cfg.get("model", {})
|
|
model = SCNet(**model_params)
|
|
|
|
state = torch.load(str(model_path), map_location="cpu", weights_only=False)
|
|
if "state_dict" in state:
|
|
state = state["state_dict"]
|
|
model.load_state_dict(state)
|
|
model.eval()
|
|
self._model = model.to(self.device)
|
|
|
|
def _get_model(self) -> SCNet:
|
|
if self._model is None:
|
|
self._load_model()
|
|
return self._model
|
|
|
|
# ------------------------------------------------------------------
|
|
# Public API
|
|
# ------------------------------------------------------------------
|
|
|
|
def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]:
|
|
"""
|
|
Separate audio into stems.
|
|
|
|
Returns:
|
|
Dict mapping stem name → output file path.
|
|
"""
|
|
model = self._get_model()
|
|
cfg = self._config
|
|
|
|
sample_rate: int = cfg.get("audio", {}).get("sample_rate", 44100)
|
|
inf_cfg: dict = cfg.get("inference", {})
|
|
chunk_size: int = inf_cfg.get("chunk_size", 485100)
|
|
num_overlap: int = inf_cfg.get("num_overlap", 2)
|
|
|
|
# Stem order from training config; fall back to model attribute
|
|
instruments = cfg.get("training", {}).get("instruments") or list(model.sources)
|
|
|
|
mix = _load_audio(input_path, sample_rate) # (2, N)
|
|
|
|
print(f"[SCNet] Running {self.model_id} on {self.device} …")
|
|
sources_np = _demix(model, mix, chunk_size, num_overlap, self.device)
|
|
# sources_np: (num_sources, 2, N)
|
|
|
|
out_dir = Path(output_dir)
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
output_paths: Dict[str, str] = {}
|
|
|
|
for i, stem in enumerate(instruments):
|
|
if i >= sources_np.shape[0]:
|
|
break
|
|
ext = self.output_format.lower()
|
|
out_path = str(out_dir / f"{stem}.{ext}")
|
|
_save_audio(out_path, sources_np[i], sample_rate, self.output_format)
|
|
output_paths[stem] = out_path
|
|
print(f"[SCNet] Saved {stem} → {out_path}")
|
|
|
|
return output_paths
|