Files
stemengine_old/backend/services_root_backup/mdxnet_separator.py
T
2026-03-03 14:52:19 +09:00

356 lines
12 KiB
Python
Executable File

"""
MDX-Net separator using ONNX models from Ultimate Vocal Remover (UVR).
Supports various MDX-Net models for vocals/instrumental separation.
"""
import gc
import os
import subprocess
import tempfile
from pathlib import Path
from typing import Dict, Optional, Tuple
from collections import namedtuple
import numpy as np
import torch
import torch.nn.functional as F
from tqdm import tqdm
# Try to import ONNX runtime
try:
import onnxruntime as ort
ONNX_AVAILABLE = True
except ImportError:
ONNX_AVAILABLE = False
SAMPLE_RATE = 44100
# Model configurations
MDXNET_MODELS = {
'mdxnet_voc_ft': {
'repo_id': 'Politrees/UVR_resources',
'path': 'models/MDXNet/UVR-MDX-NET-Voc_FT.onnx',
'stems': ['vocals', 'instrumental'],
'target_stem': 'vocals',
},
'mdxnet_inst_hq_1': {
'repo_id': 'Politrees/UVR_resources',
'path': 'models/MDXNet/UVR-MDX-NET-Inst_HQ_1.onnx',
'stems': ['instrumental', 'vocals'],
'target_stem': 'instrumental',
},
'mdxnet_inst_hq_2': {
'repo_id': 'Politrees/UVR_resources',
'path': 'models/MDXNet/UVR-MDX-NET-Inst_HQ_2.onnx',
'stems': ['instrumental', 'vocals'],
'target_stem': 'instrumental',
},
'mdxnet_inst_hq_3': {
'repo_id': 'Politrees/UVR_resources',
'path': 'models/MDXNet/UVR-MDX-NET-Inst_HQ_3.onnx',
'stems': ['instrumental', 'vocals'],
'target_stem': 'instrumental',
},
'mdxnet_karaoke': {
'repo_id': 'Politrees/UVR_resources',
'path': 'models/MDXNet/UVR_MDXNET_KARA.onnx',
'stems': ['lead', 'backing_instrumental'],
'target_stem': 'lead',
},
# New models from MVSEP-Central-Backup
'mdxnet_dereverb': {
'repo_id': 'miercolesv/MVSEP-Central-Backup',
'path': 'Dereverb-MelBand-Anvuew/dereverb_mel_band_roformer_anvuew_sdr_19.1729.ckpt',
'stems': ['dry_vocals', 'reverb'],
'target_stem': 'dry_vocals',
'use_melband': True, # Uses MelBand architecture, not pure ONNX
},
'mdxnet_drumsep': {
'repo_id': 'miercolesv/MVSEP-Central-Backup',
'path': 'Politrees/UVR_resources/models/MDX23C/MDX23C-DrumSep-aufr33-jarredou.ckpt',
'stems': ['drums', 'no_drums'],
'target_stem': 'drums',
'use_melband': True,
},
}
# MDX-Net specific constants
HOP_LENGTH = 1024
N_FFT = 2048
DIM_F = 3072
DIM_T = 8
def _load_audio(path: str, sample_rate: int = SAMPLE_RATE) -> np.ndarray:
"""Load audio as (channels, samples) float32 array."""
import soundfile as sf
try:
data, sr = sf.read(path, dtype='float32', always_2d=True)
data = data.T
except Exception:
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
tmp_path = tmp.name
try:
subprocess.run(
['ffmpeg', '-y', '-i', path, '-ar', str(sample_rate),
'-ac', '2', '-f', 'wav', tmp_path],
check=True, capture_output=True,
)
data, sr = sf.read(tmp_path, dtype='float32', always_2d=True)
data = data.T
finally:
if os.path.exists(tmp_path):
os.remove(tmp_path)
if sr != sample_rate:
import librosa
data = np.array([
librosa.resample(ch, orig_sr=sr, target_sr=sample_rate)
for ch in data
], dtype=np.float32)
# Ensure stereo
if data.shape[0] == 1:
data = np.vstack([data, data])
return data
def _save_audio(path: str, data: np.ndarray, output_format: str, bitrate: str = '256k') -> None:
"""Save (samples, channels) array to audio file."""
import soundfile as sf
ext = output_format.lower()
if ext in ('wav', 'flac'):
sf.write(path, data, SAMPLE_RATE, format=ext.upper(),
subtype='PCM_24' if ext == 'wav' else 'PCM_16')
else:
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
tmp_path = tmp.name
try:
sf.write(tmp_path, data, SAMPLE_RATE, format='WAV', subtype='PCM_16')
subprocess.run(
['ffmpeg', '-y', '-i', tmp_path, '-b:a', bitrate, path],
check=True, capture_output=True,
)
finally:
if os.path.exists(tmp_path):
os.remove(tmp_path)
def _stft(wave: np.ndarray, n_fft: int = N_FFT, hop_length: int = HOP_LENGTH) -> np.ndarray:
"""Compute STFT of audio."""
# wave shape: (channels, samples)
channels = wave.shape[0]
specs = []
for ch in range(channels):
spec = np.stft(wave[ch], n_fft=n_fft, hop_length=hop_length, window='hann')
specs.append(spec)
return np.array(specs) # (channels, freq, time, 2) where last dim is real/imag
def _istft(spec: np.ndarray, n_fft: int = N_FFT, hop_length: int = HOP_LENGTH) -> np.ndarray:
"""Compute inverse STFT."""
channels = spec.shape[0]
waves = []
for ch in range(channels):
wave = np.istft(spec[ch], n_fft=n_fft, hop_length=hop_length, window='hann')
waves.append(wave)
return np.array(waves)
def _spec_to_batch(spec: np.ndarray, dim_f: int = DIM_F, dim_t: int = DIM_T) -> Tuple[np.ndarray, dict]:
"""Convert spectrogram to batch format for MDX-Net."""
# spec shape: (channels, freq, time, 2)
b, f, t, c = spec.shape
# Pad if necessary
pad_f = dim_f - f if f < dim_f else 0
pad_t = dim_t - (t % dim_t) if t % dim_t != 0 else 0
if pad_f > 0 or pad_t > 0:
spec = np.pad(spec, ((0, 0), (0, pad_f), (0, pad_t), (0, 0)))
# Reshape to batches
spec = spec.transpose(0, 3, 1, 2) # (channels, 2, freq, time)
# Split into time chunks
t_chunks = spec.shape[-1] // dim_t
spec = spec.reshape(b, c, dim_f, t_chunks, dim_t)
spec = spec.transpose(0, 3, 1, 2, 4) # (b, t_chunks, c, f, t)
spec = spec.reshape(-1, c, dim_f, dim_t)
metadata = {
'original_shape': (b, f, t, c),
'pad_f': pad_f,
'pad_t': pad_t,
't_chunks': t_chunks,
}
return spec, metadata
def _batch_to_spec(batch: np.ndarray, metadata: dict) -> np.ndarray:
"""Convert batch format back to spectrogram."""
b, f, t, c = metadata['original_shape']
pad_f = metadata['pad_f']
pad_t = metadata['pad_t']
t_chunks = metadata['t_chunks']
# Reshape back
batch = batch.reshape(b, t_chunks, c, DIM_F, DIM_T)
batch = batch.transpose(0, 2, 3, 1, 4) # (b, c, f, t_chunks, t)
batch = batch.reshape(b, c, DIM_F, t_chunks * DIM_T)
# Remove padding
spec = batch.transpose(0, 2, 3, 1) # (b, f, t, c)
if pad_f > 0:
spec = spec[:, :f, :, :]
if pad_t > 0:
spec = spec[:, :, :t, :]
return spec
class MDXNetSeparator:
"""
MDX-Net separator using ONNX models from Ultimate Vocal Remover.
Supports various models for vocals/instrumental/karaoke separation.
"""
def __init__(
self,
model_id: str,
model_dir: Optional[str] = None,
cpu_separation: bool = True,
output_format: str = 'mp3',
mp3_bitrate: str = '256k',
):
if not ONNX_AVAILABLE:
raise ImportError("onnxruntime is required for MDX-Net models. Install with: pip install onnxruntime")
if model_id not in MDXNET_MODELS:
raise ValueError(f"Unknown MDX-Net model: {model_id}. Available: {list(MDXNET_MODELS.keys())}")
self.model_id = model_id
self.model_config = MDXNET_MODELS[model_id]
self.model_dir = Path(model_dir) if model_dir else Path('/opt/mdxnet_models')
self.model_path = self.model_dir / self.model_config['path'].split('/')[-1]
self.device = 'cpu' if cpu_separation else 'cuda'
self.output_format = output_format.lower()
self.mp3_bitrate = mp3_bitrate
self.stems = self.model_config['stems']
self.target_stem = self.model_config['target_stem']
self._session = None
def _ensure_model(self):
"""Download model if not present."""
if self.model_path.exists():
return
self.model_dir.mkdir(parents=True, exist_ok=True)
from huggingface_hub import hf_hub_download
import shutil
# Download to HF cache then copy flat into model_dir so model_path is correct
cached = hf_hub_download(
repo_id=self.model_config['repo_id'],
filename=self.model_config['path'],
)
shutil.copy2(cached, str(self.model_path))
def _get_session(self):
"""Get or create ONNX runtime session."""
if self._session is None:
self._ensure_model()
providers = ['CPUExecutionProvider']
if not self.device == 'cpu' and 'CUDAExecutionProvider' in ort.get_available_providers():
providers = ['CUDAExecutionProvider', 'CPUExecutionProvider']
self._session = ort.InferenceSession(
str(self.model_path),
providers=providers
)
return self._session
def _demix(self, mix: np.ndarray) -> Dict[str, np.ndarray]:
"""Separate audio into stems using MDX-Net."""
session = self._get_session()
input_name = session.get_inputs()[0].name
# Compute STFT
mix_spec = _stft(mix) # (channels, freq, time, 2)
# Convert to batch format
mix_batch, metadata = _spec_to_batch(mix_spec)
# Run inference in chunks
output_batches = []
batch_size = 1 # Process one chunk at a time for memory efficiency
with tqdm(total=len(mix_batch), desc='Separating', unit='chunk', ncols=100) as pbar:
for i in range(0, len(mix_batch), batch_size):
batch = mix_batch[i:i + batch_size]
# ONNX expects (batch, channels, freq, time)
outputs = session.run(None, {input_name: batch.astype(np.float32)})
output_batches.append(outputs[0])
pbar.update(len(batch))
# Concatenate outputs
target_spec = np.concatenate(output_batches, axis=0)
# Convert back to spectrogram format
target_spec = _batch_to_spec(target_spec, metadata)
# Inverse STFT to get audio
target_audio = _istft(target_spec)
# Calculate the other stem (residual)
other_audio = mix - target_audio
# Create output dict based on stems configuration
sources = {}
if self.target_stem == 'vocals':
sources['vocals'] = target_audio
sources['instrumental'] = other_audio
elif self.target_stem == 'instrumental':
sources['instrumental'] = target_audio
sources['vocals'] = other_audio
elif self.target_stem == 'lead':
sources['lead'] = target_audio
sources['backing_instrumental'] = other_audio
else:
sources[self.target_stem] = target_audio
sources['other'] = other_audio
return sources
def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]:
"""Separate audio and write stems."""
output_dir = Path(output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
# Load audio
audio = _load_audio(input_path)
# Separate
sources = self._demix(audio)
# Cleanup
if self._session is not None:
del self._session
self._session = None
gc.collect()
# Save outputs
ext = self.output_format
result = {}
for name, data in sources.items():
dest = str(output_dir / f'{name}.{ext}')
_save_audio(dest, data.T, ext, self.mp3_bitrate)
result[name] = dest
return result