Files
2026-04-14 16:32:36 +09:00

387 lines
13 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
MDX23C (TFC-TDF U-Net) separator — native PyTorch inference.
Architecture vendored from ZFTurbo MSST (MIT licence):
https://github.com/ZFTurbo/Music-Source-Separation-Training
Handles the `mdxnet_drumsep` model (MDX23C-DrumSep-aufr33-jarredou) which
is a 6-stem drum separator: kick, snare, toms, hh, ride, crash.
"""
import shutil
import types
import logging
from pathlib import Path
from typing import Dict, Optional
import numpy as np
import torch
import torchaudio
import yaml
from .mdx23c_arch import TFC_TDF_net
from ..config import MDXNET_MODEL_DIR, CPU_SEPARATION
log = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Model registry
# ---------------------------------------------------------------------------
MDX23C_MODELS: Dict[str, dict] = {
"mdxnet_drumsep": {
"model_file": "MDX23C-DrumSep-aufr33-jarredou.ckpt",
"config_file": "config_drumsep_mdx23c.yaml",
"hf_repo": "miercolesv/MVSEP-Central-Backup",
"hf_model_path": "Politrees/UVR_resources/models/MDX23C/MDX23C-DrumSep-aufr33-jarredou.ckpt",
"hf_config_path": "Politrees/UVR_resources/models/MDX23C/config_drumsep_mdx23c.yaml",
},
}
# ---------------------------------------------------------------------------
# Config helpers
# ---------------------------------------------------------------------------
def _to_ns(d):
"""Recursively convert dicts to SimpleNamespace for attribute access."""
if isinstance(d, dict):
return types.SimpleNamespace(**{k: _to_ns(v) for k, v in d.items()})
if isinstance(d, list):
return [_to_ns(i) for i in d]
return d
def _load_config(config_path: Path) -> types.SimpleNamespace:
"""Load YAML config, tolerate !!python/tuple tags."""
loader = yaml.SafeLoader
loader.add_constructor(
"tag:yaml.org,2002:python/tuple",
lambda ldr, node: tuple(ldr.construct_sequence(node)),
)
with open(config_path) as f:
raw = yaml.load(f, Loader=loader)
return _to_ns(raw)
# ---------------------------------------------------------------------------
# Audio I/O
# ---------------------------------------------------------------------------
def _load_audio(path: str, sample_rate: int) -> np.ndarray:
"""Load audio and return (2, samples) float32 array at target sample rate."""
waveform, sr = torchaudio.load(path)
if sr != sample_rate:
waveform = torchaudio.functional.resample(waveform, sr, sample_rate)
if waveform.shape[0] == 1:
waveform = waveform.repeat(2, 1)
elif waveform.shape[0] > 2:
waveform = waveform[:2]
return waveform.numpy().astype(np.float32)
def _save_audio(path: str, audio: np.ndarray, sample_rate: int, output_format: str):
"""Save (channels, samples) numpy array via torchaudio."""
tensor = torch.tensor(audio, dtype=torch.float32)
fmt = output_format.lower()
if fmt == "mp3":
torchaudio.save(path, tensor, sample_rate, format="mp3")
elif fmt == "flac":
torchaudio.save(path, tensor, sample_rate, format="flac")
else:
torchaudio.save(path, tensor, sample_rate)
# ---------------------------------------------------------------------------
# Overlapping-window inference (MSST style)
# ---------------------------------------------------------------------------
def _build_window(chunk_size: int, step: int) -> np.ndarray:
"""Linear fade-in/flat/fade-out window; consecutive windows sum to 1."""
fade = step
fade_in = np.linspace(0.0, 1.0, fade, endpoint=False, dtype=np.float32)
fade_out = np.linspace(1.0, 0.0, fade, endpoint=False, dtype=np.float32)
flat_len = chunk_size - 2 * fade
if flat_len > 0:
return np.concatenate([fade_in, np.ones(flat_len, dtype=np.float32), fade_out])
return np.concatenate([fade_in, fade_out])
def _demix(
model: TFC_TDF_net,
mix: np.ndarray,
chunk_size: int,
num_overlap: int,
device: torch.device,
) -> np.ndarray:
"""
MSST-style overlapping-window inference.
Args:
mix: (2, samples) float32 stereo audio
Returns:
(num_sources, 2, samples) float32
"""
C, N = mix.shape
step = chunk_size // num_overlap
border = chunk_size - step
mix_padded = np.pad(mix, ((0, 0), (border, border)), mode="reflect")
window = _build_window(chunk_size, step)
sources_sum: Optional[np.ndarray] = None
weight_sum = np.zeros(mix_padded.shape[-1], dtype=np.float32)
with torch.no_grad(), \
torch.amp.autocast("cuda", dtype=torch.float16, enabled=(device.type == "cuda")):
for start in range(0, N, step):
p_end = start + chunk_size
if p_end > mix_padded.shape[-1]:
break
chunk = mix_padded[:, start:p_end]
mean = float(chunk.mean())
std = max(float(chunk.std()), 1e-8)
chunk_norm = (chunk - mean) / std
inp = torch.tensor(chunk_norm[np.newaxis], dtype=torch.float32, device=device)
out = model(inp) # (1, num_sources, 2, chunk_size)
out_np = out.squeeze(0).cpu().numpy() # (num_sources, 2, chunk_size)
out_np = out_np * std + mean
if sources_sum is None:
num_sources = out_np.shape[0]
sources_sum = np.zeros(
(num_sources, C, mix_padded.shape[-1]), dtype=np.float32
)
sources_sum[:, :, start:p_end] += out_np * window[np.newaxis, np.newaxis, :]
weight_sum[start:p_end] += window
if sources_sum is None:
raise RuntimeError("Audio is too short for MDX23C chunked inference.")
weight_sum = np.maximum(weight_sum, 1e-8)
sources = sources_sum / weight_sum[np.newaxis, np.newaxis, :]
return sources[:, :, border : border + N]
# ---------------------------------------------------------------------------
# Separator class
# ---------------------------------------------------------------------------
class MDX23CSeparator:
"""Native PyTorch inference wrapper for MDX23C TFC-TDF U-Net models."""
def __init__(
self,
model_id: str = "mdxnet_drumsep",
cpu_separation: bool = CPU_SEPARATION,
output_format: str = "mp3",
mp3_bitrate: str = "256k",
):
if model_id not in MDX23C_MODELS:
raise ValueError(
f"Unknown MDX23C model: {model_id!r}. "
f"Available: {list(MDX23C_MODELS)}"
)
self.model_id = model_id
self.model_cfg = MDX23C_MODELS[model_id]
self.cpu_separation = cpu_separation
self.output_format = output_format
self.mp3_bitrate = mp3_bitrate
self.device = torch.device(
"cpu" if cpu_separation else ("cuda" if torch.cuda.is_available() else "cpu")
)
self._model: Optional[TFC_TDF_net] = None
self._config = None
self._instruments = None
# ------------------------------------------------------------------
# Model management
# ------------------------------------------------------------------
def _ensure_files(self) -> tuple[Path, Path]:
"""
Locate or download model checkpoint and config YAML.
The checkpoint may already exist at MDXNET_MODEL_DIR (downloaded by
the pre-existing ONNX download path). Config YAML is downloaded next
to it if absent.
"""
from urllib.request import urlretrieve
from huggingface_hub import hf_hub_download
model_path = MDXNET_MODEL_DIR / self.model_cfg["model_file"]
config_path = MDXNET_MODEL_DIR / self.model_cfg["config_file"]
if not model_path.exists():
log.info("[MDX23C] Downloading %s …", self.model_cfg['model_file'])
MDXNET_MODEL_DIR.mkdir(parents=True, exist_ok=True)
cached = hf_hub_download(
repo_id=self.model_cfg["hf_repo"],
filename=self.model_cfg["hf_model_path"],
)
shutil.copy2(cached, model_path)
if not config_path.exists():
log.info("[MDX23C] Downloading %s …", self.model_cfg['config_file'])
MDXNET_MODEL_DIR.mkdir(parents=True, exist_ok=True)
cached = hf_hub_download(
repo_id=self.model_cfg["hf_repo"],
filename=self.model_cfg["hf_config_path"],
)
shutil.copy2(cached, config_path)
return model_path, config_path
def _load_model(self) -> None:
"""Load config, instantiate TFC_TDF_net, load checkpoint weights."""
model_path, config_path = self._ensure_files()
cfg = _load_config(config_path)
self._config = cfg
self._instruments = cfg.training.instruments
model = TFC_TDF_net(cfg)
state = torch.load(str(model_path), map_location="cpu", weights_only=False)
if "state_dict" in state:
state = state["state_dict"]
result = model.load_state_dict(state, strict=True)
model.eval()
self._model = model.to(self.device)
log.info(
"[MDX23C] Loaded %s (%d stems: %s) on %s",
self.model_id,
len(self._instruments),
', '.join(self._instruments),
self.device,
)
def _get_model(self) -> TFC_TDF_net:
if self._model is None:
self._load_model()
return self._model
# ------------------------------------------------------------------
# Public API
# ------------------------------------------------------------------
def separate_into_parts(
self,
input_path: str,
output_dir: str,
chunk_size_override: Optional[int] = None,
num_overlap_override: Optional[int] = None,
) -> Dict[str, str]:
"""
Separate audio into per-drum-component stems.
Returns:
{stem_name: absolute_output_path}
"""
model = self._get_model()
cfg = self._config
sample_rate: int = cfg.audio.sample_rate
inf_cfg = cfg.inference
chunk_size: int = (
chunk_size_override
if chunk_size_override is not None
else getattr(cfg.audio, "chunk_size", 130560)
)
num_overlap: int = (
num_overlap_override
if num_overlap_override is not None
else getattr(inf_cfg, "num_overlap", 4)
)
mix = _load_audio(input_path, sample_rate) # (2, N)
log.info("[MDX23C] Running %s on %s …", self.model_id, self.device)
sources_np = _demix(model, mix, chunk_size, num_overlap, self.device)
# sources_np: (num_sources, 2, N)
out_dir = Path(output_dir)
out_dir.mkdir(parents=True, exist_ok=True)
output_paths: Dict[str, str] = {}
for i, stem in enumerate(self._instruments):
if i >= sources_np.shape[0]:
break
ext = self.output_format.lower()
out_path = str(out_dir / f"{stem}.{ext}")
_save_audio(out_path, sources_np[i], sample_rate, self.output_format)
output_paths[stem] = out_path
log.info("[MDX23C] Saved %s → %s", stem, out_path)
return output_paths
def separate_into_parts_chunked(
self,
input_path: str,
output_dir: str,
chunk_plan,
chunk_size_override: Optional[int] = None,
num_overlap_override: Optional[int] = None,
progress_cb=None,
) -> Dict[str, str]:
"""Chunked variant for long tracks (overlap-add reassembly)."""
from .separation.chunking import extract_chunk, overlap_add_reconstruct
model = self._get_model()
cfg = self._config
sample_rate: int = cfg.audio.sample_rate
inf_cfg = cfg.inference
chunk_size: int = (
chunk_size_override
if chunk_size_override is not None
else getattr(cfg.audio, "chunk_size", 130560)
)
num_overlap: int = (
num_overlap_override
if num_overlap_override is not None
else getattr(inf_cfg, "num_overlap", 4)
)
audio = _load_audio(input_path, sample_rate)
n_chunks = len(chunk_plan.chunks)
chunk_outputs = []
for i, (start, end) in enumerate(chunk_plan.chunks):
ts_start = start / chunk_plan.sample_rate
ts_end = min(end, chunk_plan.total_samples) / chunk_plan.sample_rate
msg = f"Chunk {i + 1}/{n_chunks} ({ts_start:.0f} s – {ts_end:.0f} s)..."
if progress_cb:
progress_cb(msg)
chunk_audio = extract_chunk(audio, start, end)
sources_chunk = _demix(model, chunk_audio, chunk_size, num_overlap, self.device)
stem_dict = {
self._instruments[j]: sources_chunk[j]
for j in range(min(len(self._instruments), sources_chunk.shape[0]))
}
chunk_outputs.append(stem_dict)
stem_names = list(chunk_outputs[0].keys())
full_sources = overlap_add_reconstruct(chunk_outputs, chunk_plan, stem_names)
out_dir = Path(output_dir)
out_dir.mkdir(parents=True, exist_ok=True)
output_paths: Dict[str, str] = {}
for stem in stem_names:
if stem not in full_sources:
continue
ext = self.output_format.lower()
out_path = str(out_dir / f"{stem}.{ext}")
_save_audio(out_path, full_sources[stem], sample_rate, self.output_format)
output_paths[stem] = out_path
log.info("[MDX23C chunked] Saved %s → %s", stem, out_path)
return output_paths