356 lines
12 KiB
Python
Executable File
356 lines
12 KiB
Python
Executable File
"""
|
|
MDX-Net separator using ONNX models from Ultimate Vocal Remover (UVR).
|
|
Supports various MDX-Net models for vocals/instrumental separation.
|
|
"""
|
|
import gc
|
|
import os
|
|
import subprocess
|
|
import tempfile
|
|
from pathlib import Path
|
|
from typing import Dict, Optional, Tuple
|
|
from collections import namedtuple
|
|
|
|
import numpy as np
|
|
import torch
|
|
import torch.nn.functional as F
|
|
from tqdm import tqdm
|
|
|
|
# Try to import ONNX runtime
|
|
try:
|
|
import onnxruntime as ort
|
|
ONNX_AVAILABLE = True
|
|
except ImportError:
|
|
ONNX_AVAILABLE = False
|
|
|
|
SAMPLE_RATE = 44100
|
|
|
|
# Model configurations
|
|
MDXNET_MODELS = {
|
|
'mdxnet_voc_ft': {
|
|
'repo_id': 'Politrees/UVR_resources',
|
|
'path': 'models/MDXNet/UVR-MDX-NET-Voc_FT.onnx',
|
|
'stems': ['vocals', 'instrumental'],
|
|
'target_stem': 'vocals',
|
|
},
|
|
'mdxnet_inst_hq_1': {
|
|
'repo_id': 'Politrees/UVR_resources',
|
|
'path': 'models/MDXNet/UVR-MDX-NET-Inst_HQ_1.onnx',
|
|
'stems': ['instrumental', 'vocals'],
|
|
'target_stem': 'instrumental',
|
|
},
|
|
'mdxnet_inst_hq_2': {
|
|
'repo_id': 'Politrees/UVR_resources',
|
|
'path': 'models/MDXNet/UVR-MDX-NET-Inst_HQ_2.onnx',
|
|
'stems': ['instrumental', 'vocals'],
|
|
'target_stem': 'instrumental',
|
|
},
|
|
'mdxnet_inst_hq_3': {
|
|
'repo_id': 'Politrees/UVR_resources',
|
|
'path': 'models/MDXNet/UVR-MDX-NET-Inst_HQ_3.onnx',
|
|
'stems': ['instrumental', 'vocals'],
|
|
'target_stem': 'instrumental',
|
|
},
|
|
'mdxnet_karaoke': {
|
|
'repo_id': 'Politrees/UVR_resources',
|
|
'path': 'models/MDXNet/UVR_MDXNET_KARA.onnx',
|
|
'stems': ['lead', 'backing_instrumental'],
|
|
'target_stem': 'lead',
|
|
},
|
|
# New models from MVSEP-Central-Backup
|
|
'mdxnet_dereverb': {
|
|
'repo_id': 'miercolesv/MVSEP-Central-Backup',
|
|
'path': 'Dereverb-MelBand-Anvuew/dereverb_mel_band_roformer_anvuew_sdr_19.1729.ckpt',
|
|
'stems': ['dry_vocals', 'reverb'],
|
|
'target_stem': 'dry_vocals',
|
|
'use_melband': True, # Uses MelBand architecture, not pure ONNX
|
|
},
|
|
'mdxnet_drumsep': {
|
|
'repo_id': 'miercolesv/MVSEP-Central-Backup',
|
|
'path': 'Politrees/UVR_resources/models/MDX23C/MDX23C-DrumSep-aufr33-jarredou.ckpt',
|
|
'stems': ['drums', 'no_drums'],
|
|
'target_stem': 'drums',
|
|
'use_melband': True,
|
|
},
|
|
}
|
|
|
|
# MDX-Net specific constants
|
|
HOP_LENGTH = 1024
|
|
N_FFT = 2048
|
|
DIM_F = 3072
|
|
DIM_T = 8
|
|
|
|
|
|
def _load_audio(path: str, sample_rate: int = SAMPLE_RATE) -> np.ndarray:
|
|
"""Load audio as (channels, samples) float32 array."""
|
|
import soundfile as sf
|
|
|
|
try:
|
|
data, sr = sf.read(path, dtype='float32', always_2d=True)
|
|
data = data.T
|
|
except Exception:
|
|
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
|
|
tmp_path = tmp.name
|
|
try:
|
|
subprocess.run(
|
|
['ffmpeg', '-y', '-i', path, '-ar', str(sample_rate),
|
|
'-ac', '2', '-f', 'wav', tmp_path],
|
|
check=True, capture_output=True,
|
|
)
|
|
data, sr = sf.read(tmp_path, dtype='float32', always_2d=True)
|
|
data = data.T
|
|
finally:
|
|
if os.path.exists(tmp_path):
|
|
os.remove(tmp_path)
|
|
|
|
if sr != sample_rate:
|
|
import librosa
|
|
data = np.array([
|
|
librosa.resample(ch, orig_sr=sr, target_sr=sample_rate)
|
|
for ch in data
|
|
], dtype=np.float32)
|
|
|
|
# Ensure stereo
|
|
if data.shape[0] == 1:
|
|
data = np.vstack([data, data])
|
|
|
|
return data
|
|
|
|
|
|
def _save_audio(path: str, data: np.ndarray, output_format: str, bitrate: str = '256k') -> None:
|
|
"""Save (samples, channels) array to audio file."""
|
|
import soundfile as sf
|
|
|
|
ext = output_format.lower()
|
|
|
|
if ext in ('wav', 'flac'):
|
|
sf.write(path, data, SAMPLE_RATE, format=ext.upper(),
|
|
subtype='PCM_24' if ext == 'wav' else 'PCM_16')
|
|
else:
|
|
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
|
|
tmp_path = tmp.name
|
|
try:
|
|
sf.write(tmp_path, data, SAMPLE_RATE, format='WAV', subtype='PCM_16')
|
|
subprocess.run(
|
|
['ffmpeg', '-y', '-i', tmp_path, '-b:a', bitrate, path],
|
|
check=True, capture_output=True,
|
|
)
|
|
finally:
|
|
if os.path.exists(tmp_path):
|
|
os.remove(tmp_path)
|
|
|
|
|
|
def _stft(wave: np.ndarray, n_fft: int = N_FFT, hop_length: int = HOP_LENGTH) -> np.ndarray:
|
|
"""Compute STFT of audio."""
|
|
# wave shape: (channels, samples)
|
|
channels = wave.shape[0]
|
|
specs = []
|
|
for ch in range(channels):
|
|
spec = np.stft(wave[ch], n_fft=n_fft, hop_length=hop_length, window='hann')
|
|
specs.append(spec)
|
|
return np.array(specs) # (channels, freq, time, 2) where last dim is real/imag
|
|
|
|
|
|
def _istft(spec: np.ndarray, n_fft: int = N_FFT, hop_length: int = HOP_LENGTH) -> np.ndarray:
|
|
"""Compute inverse STFT."""
|
|
channels = spec.shape[0]
|
|
waves = []
|
|
for ch in range(channels):
|
|
wave = np.istft(spec[ch], n_fft=n_fft, hop_length=hop_length, window='hann')
|
|
waves.append(wave)
|
|
return np.array(waves)
|
|
|
|
|
|
def _spec_to_batch(spec: np.ndarray, dim_f: int = DIM_F, dim_t: int = DIM_T) -> Tuple[np.ndarray, dict]:
|
|
"""Convert spectrogram to batch format for MDX-Net."""
|
|
# spec shape: (channels, freq, time, 2)
|
|
b, f, t, c = spec.shape
|
|
|
|
# Pad if necessary
|
|
pad_f = dim_f - f if f < dim_f else 0
|
|
pad_t = dim_t - (t % dim_t) if t % dim_t != 0 else 0
|
|
|
|
if pad_f > 0 or pad_t > 0:
|
|
spec = np.pad(spec, ((0, 0), (0, pad_f), (0, pad_t), (0, 0)))
|
|
|
|
# Reshape to batches
|
|
spec = spec.transpose(0, 3, 1, 2) # (channels, 2, freq, time)
|
|
|
|
# Split into time chunks
|
|
t_chunks = spec.shape[-1] // dim_t
|
|
spec = spec.reshape(b, c, dim_f, t_chunks, dim_t)
|
|
spec = spec.transpose(0, 3, 1, 2, 4) # (b, t_chunks, c, f, t)
|
|
spec = spec.reshape(-1, c, dim_f, dim_t)
|
|
|
|
metadata = {
|
|
'original_shape': (b, f, t, c),
|
|
'pad_f': pad_f,
|
|
'pad_t': pad_t,
|
|
't_chunks': t_chunks,
|
|
}
|
|
|
|
return spec, metadata
|
|
|
|
|
|
def _batch_to_spec(batch: np.ndarray, metadata: dict) -> np.ndarray:
|
|
"""Convert batch format back to spectrogram."""
|
|
b, f, t, c = metadata['original_shape']
|
|
pad_f = metadata['pad_f']
|
|
pad_t = metadata['pad_t']
|
|
t_chunks = metadata['t_chunks']
|
|
|
|
# Reshape back
|
|
batch = batch.reshape(b, t_chunks, c, DIM_F, DIM_T)
|
|
batch = batch.transpose(0, 2, 3, 1, 4) # (b, c, f, t_chunks, t)
|
|
batch = batch.reshape(b, c, DIM_F, t_chunks * DIM_T)
|
|
|
|
# Remove padding
|
|
spec = batch.transpose(0, 2, 3, 1) # (b, f, t, c)
|
|
if pad_f > 0:
|
|
spec = spec[:, :f, :, :]
|
|
if pad_t > 0:
|
|
spec = spec[:, :, :t, :]
|
|
|
|
return spec
|
|
|
|
|
|
class MDXNetSeparator:
|
|
"""
|
|
MDX-Net separator using ONNX models from Ultimate Vocal Remover.
|
|
Supports various models for vocals/instrumental/karaoke separation.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
model_id: str,
|
|
model_dir: Optional[str] = None,
|
|
cpu_separation: bool = True,
|
|
output_format: str = 'mp3',
|
|
mp3_bitrate: str = '256k',
|
|
):
|
|
if not ONNX_AVAILABLE:
|
|
raise ImportError("onnxruntime is required for MDX-Net models. Install with: pip install onnxruntime")
|
|
|
|
if model_id not in MDXNET_MODELS:
|
|
raise ValueError(f"Unknown MDX-Net model: {model_id}. Available: {list(MDXNET_MODELS.keys())}")
|
|
|
|
self.model_id = model_id
|
|
self.model_config = MDXNET_MODELS[model_id]
|
|
self.model_dir = Path(model_dir) if model_dir else Path('/opt/mdxnet_models')
|
|
self.model_path = self.model_dir / self.model_config['path'].split('/')[-1]
|
|
self.device = 'cpu' if cpu_separation else 'cuda'
|
|
self.output_format = output_format.lower()
|
|
self.mp3_bitrate = mp3_bitrate
|
|
self.stems = self.model_config['stems']
|
|
self.target_stem = self.model_config['target_stem']
|
|
self._session = None
|
|
|
|
def _ensure_model(self):
|
|
"""Download model if not present."""
|
|
if self.model_path.exists():
|
|
return
|
|
|
|
self.model_dir.mkdir(parents=True, exist_ok=True)
|
|
from huggingface_hub import hf_hub_download
|
|
import shutil
|
|
|
|
# Download to HF cache then copy flat into model_dir so model_path is correct
|
|
cached = hf_hub_download(
|
|
repo_id=self.model_config['repo_id'],
|
|
filename=self.model_config['path'],
|
|
)
|
|
shutil.copy2(cached, str(self.model_path))
|
|
|
|
def _get_session(self):
|
|
"""Get or create ONNX runtime session."""
|
|
if self._session is None:
|
|
self._ensure_model()
|
|
|
|
providers = ['CPUExecutionProvider']
|
|
if not self.device == 'cpu' and 'CUDAExecutionProvider' in ort.get_available_providers():
|
|
providers = ['CUDAExecutionProvider', 'CPUExecutionProvider']
|
|
|
|
self._session = ort.InferenceSession(
|
|
str(self.model_path),
|
|
providers=providers
|
|
)
|
|
return self._session
|
|
|
|
def _demix(self, mix: np.ndarray) -> Dict[str, np.ndarray]:
|
|
"""Separate audio into stems using MDX-Net."""
|
|
session = self._get_session()
|
|
input_name = session.get_inputs()[0].name
|
|
|
|
# Compute STFT
|
|
mix_spec = _stft(mix) # (channels, freq, time, 2)
|
|
|
|
# Convert to batch format
|
|
mix_batch, metadata = _spec_to_batch(mix_spec)
|
|
|
|
# Run inference in chunks
|
|
output_batches = []
|
|
batch_size = 1 # Process one chunk at a time for memory efficiency
|
|
|
|
with tqdm(total=len(mix_batch), desc='Separating', unit='chunk', ncols=100) as pbar:
|
|
for i in range(0, len(mix_batch), batch_size):
|
|
batch = mix_batch[i:i + batch_size]
|
|
# ONNX expects (batch, channels, freq, time)
|
|
outputs = session.run(None, {input_name: batch.astype(np.float32)})
|
|
output_batches.append(outputs[0])
|
|
pbar.update(len(batch))
|
|
|
|
# Concatenate outputs
|
|
target_spec = np.concatenate(output_batches, axis=0)
|
|
|
|
# Convert back to spectrogram format
|
|
target_spec = _batch_to_spec(target_spec, metadata)
|
|
|
|
# Inverse STFT to get audio
|
|
target_audio = _istft(target_spec)
|
|
|
|
# Calculate the other stem (residual)
|
|
other_audio = mix - target_audio
|
|
|
|
# Create output dict based on stems configuration
|
|
sources = {}
|
|
if self.target_stem == 'vocals':
|
|
sources['vocals'] = target_audio
|
|
sources['instrumental'] = other_audio
|
|
elif self.target_stem == 'instrumental':
|
|
sources['instrumental'] = target_audio
|
|
sources['vocals'] = other_audio
|
|
elif self.target_stem == 'lead':
|
|
sources['lead'] = target_audio
|
|
sources['backing_instrumental'] = other_audio
|
|
else:
|
|
sources[self.target_stem] = target_audio
|
|
sources['other'] = other_audio
|
|
|
|
return sources
|
|
|
|
def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]:
|
|
"""Separate audio and write stems."""
|
|
output_dir = Path(output_dir)
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Load audio
|
|
audio = _load_audio(input_path)
|
|
|
|
# Separate
|
|
sources = self._demix(audio)
|
|
|
|
# Cleanup
|
|
if self._session is not None:
|
|
del self._session
|
|
self._session = None
|
|
gc.collect()
|
|
|
|
# Save outputs
|
|
ext = self.output_format
|
|
result = {}
|
|
for name, data in sources.items():
|
|
dest = str(output_dir / f'{name}.{ext}')
|
|
_save_audio(dest, data.T, ext, self.mp3_bitrate)
|
|
result[name] = dest
|
|
|
|
return result
|