387 lines
13 KiB
Python
387 lines
13 KiB
Python
"""
|
||
MDX23C (TFC-TDF U-Net) separator — native PyTorch inference.
|
||
Architecture vendored from ZFTurbo MSST (MIT licence):
|
||
https://github.com/ZFTurbo/Music-Source-Separation-Training
|
||
|
||
Handles the `mdxnet_drumsep` model (MDX23C-DrumSep-aufr33-jarredou) which
|
||
is a 6-stem drum separator: kick, snare, toms, hh, ride, crash.
|
||
"""
|
||
import shutil
|
||
import types
|
||
import logging
|
||
from pathlib import Path
|
||
from typing import Dict, Optional
|
||
|
||
import numpy as np
|
||
import torch
|
||
import torchaudio
|
||
import yaml
|
||
|
||
from .mdx23c_arch import TFC_TDF_net
|
||
from ..config import MDXNET_MODEL_DIR, CPU_SEPARATION
|
||
|
||
|
||
log = logging.getLogger(__name__)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Model registry
|
||
# ---------------------------------------------------------------------------
|
||
|
||
MDX23C_MODELS: Dict[str, dict] = {
|
||
"mdxnet_drumsep": {
|
||
"model_file": "MDX23C-DrumSep-aufr33-jarredou.ckpt",
|
||
"config_file": "config_drumsep_mdx23c.yaml",
|
||
"hf_repo": "miercolesv/MVSEP-Central-Backup",
|
||
"hf_model_path": "Politrees/UVR_resources/models/MDX23C/MDX23C-DrumSep-aufr33-jarredou.ckpt",
|
||
"hf_config_path": "Politrees/UVR_resources/models/MDX23C/config_drumsep_mdx23c.yaml",
|
||
},
|
||
}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Config helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _to_ns(d):
|
||
"""Recursively convert dicts to SimpleNamespace for attribute access."""
|
||
if isinstance(d, dict):
|
||
return types.SimpleNamespace(**{k: _to_ns(v) for k, v in d.items()})
|
||
if isinstance(d, list):
|
||
return [_to_ns(i) for i in d]
|
||
return d
|
||
|
||
|
||
def _load_config(config_path: Path) -> types.SimpleNamespace:
|
||
"""Load YAML config, tolerate !!python/tuple tags."""
|
||
loader = yaml.SafeLoader
|
||
loader.add_constructor(
|
||
"tag:yaml.org,2002:python/tuple",
|
||
lambda ldr, node: tuple(ldr.construct_sequence(node)),
|
||
)
|
||
with open(config_path) as f:
|
||
raw = yaml.load(f, Loader=loader)
|
||
return _to_ns(raw)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Audio I/O
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _load_audio(path: str, sample_rate: int) -> np.ndarray:
|
||
"""Load audio and return (2, samples) float32 array at target sample rate."""
|
||
waveform, sr = torchaudio.load(path)
|
||
if sr != sample_rate:
|
||
waveform = torchaudio.functional.resample(waveform, sr, sample_rate)
|
||
if waveform.shape[0] == 1:
|
||
waveform = waveform.repeat(2, 1)
|
||
elif waveform.shape[0] > 2:
|
||
waveform = waveform[:2]
|
||
return waveform.numpy().astype(np.float32)
|
||
|
||
|
||
def _save_audio(path: str, audio: np.ndarray, sample_rate: int, output_format: str):
|
||
"""Save (channels, samples) numpy array via torchaudio."""
|
||
tensor = torch.tensor(audio, dtype=torch.float32)
|
||
fmt = output_format.lower()
|
||
if fmt == "mp3":
|
||
torchaudio.save(path, tensor, sample_rate, format="mp3")
|
||
elif fmt == "flac":
|
||
torchaudio.save(path, tensor, sample_rate, format="flac")
|
||
else:
|
||
torchaudio.save(path, tensor, sample_rate)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Overlapping-window inference (MSST style)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _build_window(chunk_size: int, step: int) -> np.ndarray:
|
||
"""Linear fade-in/flat/fade-out window; consecutive windows sum to 1."""
|
||
fade = step
|
||
fade_in = np.linspace(0.0, 1.0, fade, endpoint=False, dtype=np.float32)
|
||
fade_out = np.linspace(1.0, 0.0, fade, endpoint=False, dtype=np.float32)
|
||
flat_len = chunk_size - 2 * fade
|
||
if flat_len > 0:
|
||
return np.concatenate([fade_in, np.ones(flat_len, dtype=np.float32), fade_out])
|
||
return np.concatenate([fade_in, fade_out])
|
||
|
||
|
||
def _demix(
|
||
model: TFC_TDF_net,
|
||
mix: np.ndarray,
|
||
chunk_size: int,
|
||
num_overlap: int,
|
||
device: torch.device,
|
||
) -> np.ndarray:
|
||
"""
|
||
MSST-style overlapping-window inference.
|
||
|
||
Args:
|
||
mix: (2, samples) float32 stereo audio
|
||
Returns:
|
||
(num_sources, 2, samples) float32
|
||
"""
|
||
C, N = mix.shape
|
||
step = chunk_size // num_overlap
|
||
border = chunk_size - step
|
||
|
||
mix_padded = np.pad(mix, ((0, 0), (border, border)), mode="reflect")
|
||
|
||
window = _build_window(chunk_size, step)
|
||
sources_sum: Optional[np.ndarray] = None
|
||
weight_sum = np.zeros(mix_padded.shape[-1], dtype=np.float32)
|
||
|
||
with torch.no_grad(), \
|
||
torch.amp.autocast("cuda", dtype=torch.float16, enabled=(device.type == "cuda")):
|
||
for start in range(0, N, step):
|
||
p_end = start + chunk_size
|
||
if p_end > mix_padded.shape[-1]:
|
||
break
|
||
chunk = mix_padded[:, start:p_end]
|
||
|
||
mean = float(chunk.mean())
|
||
std = max(float(chunk.std()), 1e-8)
|
||
chunk_norm = (chunk - mean) / std
|
||
|
||
inp = torch.tensor(chunk_norm[np.newaxis], dtype=torch.float32, device=device)
|
||
out = model(inp) # (1, num_sources, 2, chunk_size)
|
||
out_np = out.squeeze(0).cpu().numpy() # (num_sources, 2, chunk_size)
|
||
out_np = out_np * std + mean
|
||
|
||
if sources_sum is None:
|
||
num_sources = out_np.shape[0]
|
||
sources_sum = np.zeros(
|
||
(num_sources, C, mix_padded.shape[-1]), dtype=np.float32
|
||
)
|
||
|
||
sources_sum[:, :, start:p_end] += out_np * window[np.newaxis, np.newaxis, :]
|
||
weight_sum[start:p_end] += window
|
||
|
||
if sources_sum is None:
|
||
raise RuntimeError("Audio is too short for MDX23C chunked inference.")
|
||
|
||
weight_sum = np.maximum(weight_sum, 1e-8)
|
||
sources = sources_sum / weight_sum[np.newaxis, np.newaxis, :]
|
||
return sources[:, :, border : border + N]
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Separator class
|
||
# ---------------------------------------------------------------------------
|
||
|
||
class MDX23CSeparator:
|
||
"""Native PyTorch inference wrapper for MDX23C TFC-TDF U-Net models."""
|
||
|
||
def __init__(
|
||
self,
|
||
model_id: str = "mdxnet_drumsep",
|
||
cpu_separation: bool = CPU_SEPARATION,
|
||
output_format: str = "mp3",
|
||
mp3_bitrate: str = "256k",
|
||
):
|
||
if model_id not in MDX23C_MODELS:
|
||
raise ValueError(
|
||
f"Unknown MDX23C model: {model_id!r}. "
|
||
f"Available: {list(MDX23C_MODELS)}"
|
||
)
|
||
|
||
self.model_id = model_id
|
||
self.model_cfg = MDX23C_MODELS[model_id]
|
||
self.cpu_separation = cpu_separation
|
||
self.output_format = output_format
|
||
self.mp3_bitrate = mp3_bitrate
|
||
self.device = torch.device(
|
||
"cpu" if cpu_separation else ("cuda" if torch.cuda.is_available() else "cpu")
|
||
)
|
||
|
||
self._model: Optional[TFC_TDF_net] = None
|
||
self._config = None
|
||
self._instruments = None
|
||
|
||
# ------------------------------------------------------------------
|
||
# Model management
|
||
# ------------------------------------------------------------------
|
||
|
||
def _ensure_files(self) -> tuple[Path, Path]:
|
||
"""
|
||
Locate or download model checkpoint and config YAML.
|
||
|
||
The checkpoint may already exist at MDXNET_MODEL_DIR (downloaded by
|
||
the pre-existing ONNX download path). Config YAML is downloaded next
|
||
to it if absent.
|
||
"""
|
||
from urllib.request import urlretrieve
|
||
from huggingface_hub import hf_hub_download
|
||
|
||
model_path = MDXNET_MODEL_DIR / self.model_cfg["model_file"]
|
||
config_path = MDXNET_MODEL_DIR / self.model_cfg["config_file"]
|
||
|
||
if not model_path.exists():
|
||
log.info("[MDX23C] Downloading %s …", self.model_cfg['model_file'])
|
||
MDXNET_MODEL_DIR.mkdir(parents=True, exist_ok=True)
|
||
cached = hf_hub_download(
|
||
repo_id=self.model_cfg["hf_repo"],
|
||
filename=self.model_cfg["hf_model_path"],
|
||
)
|
||
shutil.copy2(cached, model_path)
|
||
|
||
if not config_path.exists():
|
||
log.info("[MDX23C] Downloading %s …", self.model_cfg['config_file'])
|
||
MDXNET_MODEL_DIR.mkdir(parents=True, exist_ok=True)
|
||
cached = hf_hub_download(
|
||
repo_id=self.model_cfg["hf_repo"],
|
||
filename=self.model_cfg["hf_config_path"],
|
||
)
|
||
shutil.copy2(cached, config_path)
|
||
|
||
return model_path, config_path
|
||
|
||
def _load_model(self) -> None:
|
||
"""Load config, instantiate TFC_TDF_net, load checkpoint weights."""
|
||
model_path, config_path = self._ensure_files()
|
||
|
||
cfg = _load_config(config_path)
|
||
self._config = cfg
|
||
self._instruments = cfg.training.instruments
|
||
|
||
model = TFC_TDF_net(cfg)
|
||
|
||
state = torch.load(str(model_path), map_location="cpu", weights_only=False)
|
||
if "state_dict" in state:
|
||
state = state["state_dict"]
|
||
result = model.load_state_dict(state, strict=True)
|
||
model.eval()
|
||
self._model = model.to(self.device)
|
||
log.info(
|
||
"[MDX23C] Loaded %s (%d stems: %s) on %s",
|
||
self.model_id,
|
||
len(self._instruments),
|
||
', '.join(self._instruments),
|
||
self.device,
|
||
)
|
||
|
||
def _get_model(self) -> TFC_TDF_net:
|
||
if self._model is None:
|
||
self._load_model()
|
||
return self._model
|
||
|
||
# ------------------------------------------------------------------
|
||
# Public API
|
||
# ------------------------------------------------------------------
|
||
|
||
def separate_into_parts(
|
||
self,
|
||
input_path: str,
|
||
output_dir: str,
|
||
chunk_size_override: Optional[int] = None,
|
||
num_overlap_override: Optional[int] = None,
|
||
) -> Dict[str, str]:
|
||
"""
|
||
Separate audio into per-drum-component stems.
|
||
|
||
Returns:
|
||
{stem_name: absolute_output_path}
|
||
"""
|
||
model = self._get_model()
|
||
cfg = self._config
|
||
|
||
sample_rate: int = cfg.audio.sample_rate
|
||
inf_cfg = cfg.inference
|
||
chunk_size: int = (
|
||
chunk_size_override
|
||
if chunk_size_override is not None
|
||
else getattr(cfg.audio, "chunk_size", 130560)
|
||
)
|
||
num_overlap: int = (
|
||
num_overlap_override
|
||
if num_overlap_override is not None
|
||
else getattr(inf_cfg, "num_overlap", 4)
|
||
)
|
||
|
||
mix = _load_audio(input_path, sample_rate) # (2, N)
|
||
|
||
log.info("[MDX23C] Running %s on %s …", self.model_id, self.device)
|
||
sources_np = _demix(model, mix, chunk_size, num_overlap, self.device)
|
||
# sources_np: (num_sources, 2, N)
|
||
|
||
out_dir = Path(output_dir)
|
||
out_dir.mkdir(parents=True, exist_ok=True)
|
||
output_paths: Dict[str, str] = {}
|
||
|
||
for i, stem in enumerate(self._instruments):
|
||
if i >= sources_np.shape[0]:
|
||
break
|
||
ext = self.output_format.lower()
|
||
out_path = str(out_dir / f"{stem}.{ext}")
|
||
_save_audio(out_path, sources_np[i], sample_rate, self.output_format)
|
||
output_paths[stem] = out_path
|
||
log.info("[MDX23C] Saved %s → %s", stem, out_path)
|
||
|
||
return output_paths
|
||
|
||
def separate_into_parts_chunked(
|
||
self,
|
||
input_path: str,
|
||
output_dir: str,
|
||
chunk_plan,
|
||
chunk_size_override: Optional[int] = None,
|
||
num_overlap_override: Optional[int] = None,
|
||
progress_cb=None,
|
||
) -> Dict[str, str]:
|
||
"""Chunked variant for long tracks (overlap-add reassembly)."""
|
||
from .separation.chunking import extract_chunk, overlap_add_reconstruct
|
||
|
||
model = self._get_model()
|
||
cfg = self._config
|
||
|
||
sample_rate: int = cfg.audio.sample_rate
|
||
inf_cfg = cfg.inference
|
||
chunk_size: int = (
|
||
chunk_size_override
|
||
if chunk_size_override is not None
|
||
else getattr(cfg.audio, "chunk_size", 130560)
|
||
)
|
||
num_overlap: int = (
|
||
num_overlap_override
|
||
if num_overlap_override is not None
|
||
else getattr(inf_cfg, "num_overlap", 4)
|
||
)
|
||
|
||
audio = _load_audio(input_path, sample_rate)
|
||
n_chunks = len(chunk_plan.chunks)
|
||
chunk_outputs = []
|
||
|
||
for i, (start, end) in enumerate(chunk_plan.chunks):
|
||
ts_start = start / chunk_plan.sample_rate
|
||
ts_end = min(end, chunk_plan.total_samples) / chunk_plan.sample_rate
|
||
msg = f"Chunk {i + 1}/{n_chunks} ({ts_start:.0f} s – {ts_end:.0f} s)..."
|
||
if progress_cb:
|
||
progress_cb(msg)
|
||
|
||
chunk_audio = extract_chunk(audio, start, end)
|
||
sources_chunk = _demix(model, chunk_audio, chunk_size, num_overlap, self.device)
|
||
stem_dict = {
|
||
self._instruments[j]: sources_chunk[j]
|
||
for j in range(min(len(self._instruments), sources_chunk.shape[0]))
|
||
}
|
||
chunk_outputs.append(stem_dict)
|
||
|
||
stem_names = list(chunk_outputs[0].keys())
|
||
full_sources = overlap_add_reconstruct(chunk_outputs, chunk_plan, stem_names)
|
||
|
||
out_dir = Path(output_dir)
|
||
out_dir.mkdir(parents=True, exist_ok=True)
|
||
output_paths: Dict[str, str] = {}
|
||
|
||
for stem in stem_names:
|
||
if stem not in full_sources:
|
||
continue
|
||
ext = self.output_format.lower()
|
||
out_path = str(out_dir / f"{stem}.{ext}")
|
||
_save_audio(out_path, full_sources[stem], sample_rate, self.output_format)
|
||
output_paths[stem] = out_path
|
||
log.info("[MDX23C chunked] Saved %s → %s", stem, out_path)
|
||
|
||
return output_paths
|