""" MDX-Net separator using ONNX models from Ultimate Vocal Remover (UVR). Supports various MDX-Net models for vocals/instrumental separation. """ import gc import os import subprocess import tempfile from pathlib import Path from typing import Dict, Optional, Tuple from collections import namedtuple import numpy as np import torch import torch.nn.functional as F from tqdm import tqdm # Try to import ONNX runtime try: import onnxruntime as ort ONNX_AVAILABLE = True except ImportError: ONNX_AVAILABLE = False SAMPLE_RATE = 44100 # Model configurations MDXNET_MODELS = { 'mdxnet_voc_ft': { 'repo_id': 'Politrees/UVR_resources', 'path': 'models/MDXNet/UVR-MDX-NET-Voc_FT.onnx', 'stems': ['vocals', 'instrumental'], 'target_stem': 'vocals', }, 'mdxnet_inst_hq_1': { 'repo_id': 'Politrees/UVR_resources', 'path': 'models/MDXNet/UVR-MDX-NET-Inst_HQ_1.onnx', 'stems': ['instrumental', 'vocals'], 'target_stem': 'instrumental', }, 'mdxnet_inst_hq_2': { 'repo_id': 'Politrees/UVR_resources', 'path': 'models/MDXNet/UVR-MDX-NET-Inst_HQ_2.onnx', 'stems': ['instrumental', 'vocals'], 'target_stem': 'instrumental', }, 'mdxnet_inst_hq_3': { 'repo_id': 'Politrees/UVR_resources', 'path': 'models/MDXNet/UVR-MDX-NET-Inst_HQ_3.onnx', 'stems': ['instrumental', 'vocals'], 'target_stem': 'instrumental', }, 'mdxnet_karaoke': { 'repo_id': 'Politrees/UVR_resources', 'path': 'models/MDXNet/UVR_MDXNET_KARA.onnx', 'stems': ['lead', 'backing_instrumental'], 'target_stem': 'lead', }, # New models from MVSEP-Central-Backup 'mdxnet_dereverb': { 'repo_id': 'miercolesv/MVSEP-Central-Backup', 'path': 'Dereverb-MelBand-Anvuew/dereverb_mel_band_roformer_anvuew_sdr_19.1729.ckpt', 'stems': ['dry_vocals', 'reverb'], 'target_stem': 'dry_vocals', 'use_melband': True, # Uses MelBand architecture, not pure ONNX }, 'mdxnet_drumsep': { 'repo_id': 'miercolesv/MVSEP-Central-Backup', 'path': 'Politrees/UVR_resources/models/MDX23C/MDX23C-DrumSep-aufr33-jarredou.ckpt', 'stems': ['drums', 'no_drums'], 'target_stem': 'drums', 'use_melband': True, }, } # MDX-Net specific constants HOP_LENGTH = 1024 N_FFT = 2048 DIM_F = 3072 DIM_T = 8 def _load_audio(path: str, sample_rate: int = SAMPLE_RATE) -> np.ndarray: """Load audio as (channels, samples) float32 array.""" import soundfile as sf try: data, sr = sf.read(path, dtype='float32', always_2d=True) data = data.T except Exception: with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp: tmp_path = tmp.name try: subprocess.run( ['ffmpeg', '-y', '-i', path, '-ar', str(sample_rate), '-ac', '2', '-f', 'wav', tmp_path], check=True, capture_output=True, ) data, sr = sf.read(tmp_path, dtype='float32', always_2d=True) data = data.T finally: if os.path.exists(tmp_path): os.remove(tmp_path) if sr != sample_rate: import librosa data = np.array([ librosa.resample(ch, orig_sr=sr, target_sr=sample_rate) for ch in data ], dtype=np.float32) # Ensure stereo if data.shape[0] == 1: data = np.vstack([data, data]) return data def _save_audio(path: str, data: np.ndarray, output_format: str, bitrate: str = '256k') -> None: """Save (samples, channels) array to audio file.""" import soundfile as sf ext = output_format.lower() if ext in ('wav', 'flac'): sf.write(path, data, SAMPLE_RATE, format=ext.upper(), subtype='PCM_24' if ext == 'wav' else 'PCM_16') else: with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp: tmp_path = tmp.name try: sf.write(tmp_path, data, SAMPLE_RATE, format='WAV', subtype='PCM_16') subprocess.run( ['ffmpeg', '-y', '-i', tmp_path, '-b:a', bitrate, path], check=True, capture_output=True, ) finally: if os.path.exists(tmp_path): os.remove(tmp_path) def _stft(wave: np.ndarray, n_fft: int = N_FFT, hop_length: int = HOP_LENGTH) -> np.ndarray: """Compute STFT of audio.""" # wave shape: (channels, samples) channels = wave.shape[0] specs = [] for ch in range(channels): spec = np.stft(wave[ch], n_fft=n_fft, hop_length=hop_length, window='hann') specs.append(spec) return np.array(specs) # (channels, freq, time, 2) where last dim is real/imag def _istft(spec: np.ndarray, n_fft: int = N_FFT, hop_length: int = HOP_LENGTH) -> np.ndarray: """Compute inverse STFT.""" channels = spec.shape[0] waves = [] for ch in range(channels): wave = np.istft(spec[ch], n_fft=n_fft, hop_length=hop_length, window='hann') waves.append(wave) return np.array(waves) def _spec_to_batch(spec: np.ndarray, dim_f: int = DIM_F, dim_t: int = DIM_T) -> Tuple[np.ndarray, dict]: """Convert spectrogram to batch format for MDX-Net.""" # spec shape: (channels, freq, time, 2) b, f, t, c = spec.shape # Pad if necessary pad_f = dim_f - f if f < dim_f else 0 pad_t = dim_t - (t % dim_t) if t % dim_t != 0 else 0 if pad_f > 0 or pad_t > 0: spec = np.pad(spec, ((0, 0), (0, pad_f), (0, pad_t), (0, 0))) # Reshape to batches spec = spec.transpose(0, 3, 1, 2) # (channels, 2, freq, time) # Split into time chunks t_chunks = spec.shape[-1] // dim_t spec = spec.reshape(b, c, dim_f, t_chunks, dim_t) spec = spec.transpose(0, 3, 1, 2, 4) # (b, t_chunks, c, f, t) spec = spec.reshape(-1, c, dim_f, dim_t) metadata = { 'original_shape': (b, f, t, c), 'pad_f': pad_f, 'pad_t': pad_t, 't_chunks': t_chunks, } return spec, metadata def _batch_to_spec(batch: np.ndarray, metadata: dict) -> np.ndarray: """Convert batch format back to spectrogram.""" b, f, t, c = metadata['original_shape'] pad_f = metadata['pad_f'] pad_t = metadata['pad_t'] t_chunks = metadata['t_chunks'] # Reshape back batch = batch.reshape(b, t_chunks, c, DIM_F, DIM_T) batch = batch.transpose(0, 2, 3, 1, 4) # (b, c, f, t_chunks, t) batch = batch.reshape(b, c, DIM_F, t_chunks * DIM_T) # Remove padding spec = batch.transpose(0, 2, 3, 1) # (b, f, t, c) if pad_f > 0: spec = spec[:, :f, :, :] if pad_t > 0: spec = spec[:, :, :t, :] return spec class MDXNetSeparator: """ MDX-Net separator using ONNX models from Ultimate Vocal Remover. Supports various models for vocals/instrumental/karaoke separation. """ def __init__( self, model_id: str, model_dir: Optional[str] = None, cpu_separation: bool = True, output_format: str = 'mp3', mp3_bitrate: str = '256k', ): if not ONNX_AVAILABLE: raise ImportError("onnxruntime is required for MDX-Net models. Install with: pip install onnxruntime") if model_id not in MDXNET_MODELS: raise ValueError(f"Unknown MDX-Net model: {model_id}. Available: {list(MDXNET_MODELS.keys())}") self.model_id = model_id self.model_config = MDXNET_MODELS[model_id] self.model_dir = Path(model_dir) if model_dir else Path('/opt/mdxnet_models') self.model_path = self.model_dir / self.model_config['path'].split('/')[-1] self.device = 'cpu' if cpu_separation else 'cuda' self.output_format = output_format.lower() self.mp3_bitrate = mp3_bitrate self.stems = self.model_config['stems'] self.target_stem = self.model_config['target_stem'] self._session = None def _ensure_model(self): """Download model if not present.""" if self.model_path.exists(): return self.model_dir.mkdir(parents=True, exist_ok=True) from huggingface_hub import hf_hub_download import shutil # Download to HF cache then copy flat into model_dir so model_path is correct cached = hf_hub_download( repo_id=self.model_config['repo_id'], filename=self.model_config['path'], ) shutil.copy2(cached, str(self.model_path)) def _get_session(self): """Get or create ONNX runtime session.""" if self._session is None: self._ensure_model() providers = ['CPUExecutionProvider'] if not self.device == 'cpu' and 'CUDAExecutionProvider' in ort.get_available_providers(): providers = ['CUDAExecutionProvider', 'CPUExecutionProvider'] self._session = ort.InferenceSession( str(self.model_path), providers=providers ) return self._session def _demix(self, mix: np.ndarray) -> Dict[str, np.ndarray]: """Separate audio into stems using MDX-Net.""" session = self._get_session() input_name = session.get_inputs()[0].name # Compute STFT mix_spec = _stft(mix) # (channels, freq, time, 2) # Convert to batch format mix_batch, metadata = _spec_to_batch(mix_spec) # Run inference in chunks output_batches = [] batch_size = 1 # Process one chunk at a time for memory efficiency with tqdm(total=len(mix_batch), desc='Separating', unit='chunk', ncols=100) as pbar: for i in range(0, len(mix_batch), batch_size): batch = mix_batch[i:i + batch_size] # ONNX expects (batch, channels, freq, time) outputs = session.run(None, {input_name: batch.astype(np.float32)}) output_batches.append(outputs[0]) pbar.update(len(batch)) # Concatenate outputs target_spec = np.concatenate(output_batches, axis=0) # Convert back to spectrogram format target_spec = _batch_to_spec(target_spec, metadata) # Inverse STFT to get audio target_audio = _istft(target_spec) # Calculate the other stem (residual) other_audio = mix - target_audio # Create output dict based on stems configuration sources = {} if self.target_stem == 'vocals': sources['vocals'] = target_audio sources['instrumental'] = other_audio elif self.target_stem == 'instrumental': sources['instrumental'] = target_audio sources['vocals'] = other_audio elif self.target_stem == 'lead': sources['lead'] = target_audio sources['backing_instrumental'] = other_audio else: sources[self.target_stem] = target_audio sources['other'] = other_audio return sources def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]: """Separate audio and write stems.""" output_dir = Path(output_dir) output_dir.mkdir(parents=True, exist_ok=True) # Load audio audio = _load_audio(input_path) # Separate sources = self._demix(audio) # Cleanup if self._session is not None: del self._session self._session = None gc.collect() # Save outputs ext = self.output_format result = {} for name, data in sources.items(): dest = str(output_dir / f'{name}.{ext}') _save_audio(dest, data.T, ext, self.mp3_bitrate) result[name] = dest return result