467 lines
16 KiB
Python
Executable File
467 lines
16 KiB
Python
Executable File
"""
|
||
MDX-Net separator using ONNX models from Ultimate Vocal Remover (UVR).
|
||
Supports various MDX-Net models for vocals/instrumental separation.
|
||
"""
|
||
import gc
|
||
import os
|
||
import subprocess
|
||
import tempfile
|
||
from pathlib import Path
|
||
from typing import Dict, Optional, Tuple
|
||
from collections import namedtuple
|
||
|
||
import numpy as np
|
||
import torch
|
||
import torch.nn.functional as F
|
||
from tqdm import tqdm
|
||
|
||
# Try to import ONNX runtime
|
||
try:
|
||
import onnxruntime as ort
|
||
ONNX_AVAILABLE = True
|
||
except ImportError:
|
||
ONNX_AVAILABLE = False
|
||
|
||
SAMPLE_RATE = 44100
|
||
|
||
# Model configurations
|
||
MDXNET_MODELS = {
|
||
'mdxnet_voc_ft': {
|
||
'repo_id': 'Politrees/UVR_resources',
|
||
'path': 'models/MDXNet/UVR-MDX-NET-Voc_FT.onnx',
|
||
'stems': ['vocals', 'instrumental'],
|
||
'target_stem': 'vocals',
|
||
},
|
||
'mdxnet_inst_hq_1': {
|
||
'repo_id': 'Politrees/UVR_resources',
|
||
'path': 'models/MDXNet/UVR-MDX-NET-Inst_HQ_1.onnx',
|
||
'stems': ['instrumental', 'vocals'],
|
||
'target_stem': 'instrumental',
|
||
},
|
||
'mdxnet_inst_hq_2': {
|
||
'repo_id': 'Politrees/UVR_resources',
|
||
'path': 'models/MDXNet/UVR-MDX-NET-Inst_HQ_2.onnx',
|
||
'stems': ['instrumental', 'vocals'],
|
||
'target_stem': 'instrumental',
|
||
},
|
||
'mdxnet_inst_hq_3': {
|
||
'repo_id': 'Politrees/UVR_resources',
|
||
'path': 'models/MDXNet/UVR-MDX-NET-Inst_HQ_3.onnx',
|
||
'stems': ['instrumental', 'vocals'],
|
||
'target_stem': 'instrumental',
|
||
},
|
||
'mdxnet_karaoke': {
|
||
'repo_id': 'Politrees/UVR_resources',
|
||
'path': 'models/MDXNet/UVR_MDXNET_KARA.onnx',
|
||
'stems': ['lead', 'backing_instrumental'],
|
||
'target_stem': 'lead',
|
||
},
|
||
# New models from MVSEP-Central-Backup
|
||
'mdxnet_dereverb': {
|
||
'repo_id': 'miercolesv/MVSEP-Central-Backup',
|
||
'path': 'Dereverb-MelBand-Anvuew/dereverb_mel_band_roformer_anvuew_sdr_19.1729.ckpt',
|
||
'stems': ['dry_vocals', 'reverb'],
|
||
'target_stem': 'dry_vocals',
|
||
'use_melband': True, # Uses MelBand architecture, not pure ONNX
|
||
},
|
||
# mdxnet_drumsep is intentionally absent here — it is a PyTorch TFC-TDF
|
||
# checkpoint (MDX23C architecture), not an ONNX model. Loading it with
|
||
# ort.InferenceSession raises INVALID_PROTOBUF. It is handled instead
|
||
# by mdx23c_separator.MDX23CSeparator via _run_mdx23c() in
|
||
# separator_service.py.
|
||
}
|
||
|
||
# MDX-Net specific constants
|
||
HOP_LENGTH = 1024
|
||
# DIM_F and DIM_T are model-specific and are read from the ONNX input shape at runtime.
|
||
# N_FFT = DIM_F * 2 → produces DIM_F+1 freq bins; top bin is cropped before ONNX inference.
|
||
|
||
|
||
def _load_audio(path: str, sample_rate: int = SAMPLE_RATE) -> np.ndarray:
|
||
"""Load audio as (channels, samples) float32 array."""
|
||
import soundfile as sf
|
||
|
||
try:
|
||
data, sr = sf.read(path, dtype='float32', always_2d=True)
|
||
data = data.T
|
||
except Exception:
|
||
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
|
||
tmp_path = tmp.name
|
||
try:
|
||
subprocess.run(
|
||
['ffmpeg', '-y', '-i', path, '-ar', str(sample_rate),
|
||
'-ac', '2', '-f', 'wav', tmp_path],
|
||
check=True, capture_output=True,
|
||
)
|
||
data, sr = sf.read(tmp_path, dtype='float32', always_2d=True)
|
||
data = data.T
|
||
finally:
|
||
if os.path.exists(tmp_path):
|
||
os.remove(tmp_path)
|
||
|
||
if sr != sample_rate:
|
||
import librosa
|
||
data = np.array([
|
||
librosa.resample(ch, orig_sr=sr, target_sr=sample_rate)
|
||
for ch in data
|
||
], dtype=np.float32)
|
||
|
||
# Ensure stereo
|
||
if data.shape[0] == 1:
|
||
data = np.vstack([data, data])
|
||
|
||
return data
|
||
|
||
|
||
def _save_audio(path: str, data: np.ndarray, output_format: str, bitrate: str = '256k') -> None:
|
||
"""Save (samples, channels) array to audio file."""
|
||
import soundfile as sf
|
||
|
||
ext = output_format.lower()
|
||
|
||
if ext in ('wav', 'flac'):
|
||
sf.write(path, data, SAMPLE_RATE, format=ext.upper(),
|
||
subtype='PCM_24' if ext == 'wav' else 'PCM_16')
|
||
else:
|
||
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
|
||
tmp_path = tmp.name
|
||
try:
|
||
sf.write(tmp_path, data, SAMPLE_RATE, format='WAV', subtype='PCM_16')
|
||
subprocess.run(
|
||
['ffmpeg', '-y', '-i', tmp_path, '-b:a', bitrate, path],
|
||
check=True, capture_output=True,
|
||
)
|
||
finally:
|
||
if os.path.exists(tmp_path):
|
||
os.remove(tmp_path)
|
||
|
||
|
||
def _stft(wave: np.ndarray, n_fft: int = 4096, hop_length: int = HOP_LENGTH) -> np.ndarray:
|
||
"""Compute STFT of audio.
|
||
|
||
np.stft/np.istft never existed; use librosa which shares the same
|
||
n_fft/hop_length parameter names. librosa.stft returns a complex
|
||
(freq, time) array; we pack it as (freq, time, 2) real/imag to match
|
||
the shape contract expected by _spec_to_batch.
|
||
"""
|
||
import librosa
|
||
# wave shape: (channels, samples)
|
||
channels = wave.shape[0]
|
||
specs = []
|
||
for ch in range(channels):
|
||
spec = librosa.stft(wave[ch], n_fft=n_fft, hop_length=hop_length, window='hann')
|
||
# complex (freq, time) → (freq, time, 2) real/imag
|
||
specs.append(np.stack([spec.real, spec.imag], axis=-1))
|
||
return np.array(specs) # (channels, freq, time, 2) where last dim is real/imag
|
||
|
||
|
||
def _istft(spec: np.ndarray, n_fft: int = 4096, hop_length: int = HOP_LENGTH) -> np.ndarray:
|
||
"""Compute inverse STFT."""
|
||
import librosa
|
||
channels = spec.shape[0]
|
||
waves = []
|
||
for ch in range(channels):
|
||
# (freq, time, 2) real/imag → complex (freq, time)
|
||
complex_spec = spec[ch, :, :, 0] + 1j * spec[ch, :, :, 1]
|
||
wave = librosa.istft(complex_spec, n_fft=n_fft, hop_length=hop_length, window='hann')
|
||
waves.append(wave)
|
||
return np.array(waves)
|
||
|
||
|
||
def _spec_to_batch(spec: np.ndarray, dim_f: int = 2048, dim_t: int = 256) -> Tuple[np.ndarray, dict]:
|
||
"""Convert spectrogram to batch format for MDX-Net.
|
||
|
||
Input spec: (stereo=2, freq, time, complex=2)
|
||
Output batch: (t_chunks, 4, dim_f, dim_t)
|
||
where the 4 channels are (L_real, L_imag, R_real, R_imag); freq is
|
||
cropped to dim_f; time is padded to a multiple of dim_t.
|
||
"""
|
||
_, freq, time, _ = spec.shape # (2, freq, time, 2)
|
||
|
||
# Pad time to a multiple of dim_t
|
||
pad_t = (dim_t - time % dim_t) % dim_t
|
||
if pad_t > 0:
|
||
spec = np.pad(spec, ((0, 0), (0, 0), (0, pad_t), (0, 0)))
|
||
|
||
# Pad freq to dim_f if necessary (normally freq >= dim_f)
|
||
pad_f = max(0, dim_f - freq)
|
||
if pad_f > 0:
|
||
spec = np.pad(spec, ((0, 0), (0, pad_f), (0, 0), (0, 0)))
|
||
|
||
# Crop freq to exactly dim_f
|
||
spec = spec[:, :dim_f, :, :] # (2, dim_f, time_padded, 2)
|
||
time_padded = time + pad_t
|
||
t_chunks = time_padded // dim_t
|
||
|
||
# Merge stereo(2) × complex(2) → 4 channels
|
||
spec = spec.transpose(0, 3, 1, 2) # (2, 2, dim_f, time_padded)
|
||
spec = spec.reshape(4, dim_f, time_padded) # (4, dim_f, time_padded)
|
||
|
||
# Chunk time dimension
|
||
spec = spec.reshape(4, dim_f, t_chunks, dim_t)
|
||
spec = spec.transpose(2, 0, 1, 3) # (t_chunks, 4, dim_f, dim_t)
|
||
|
||
metadata = {
|
||
'freq': freq,
|
||
'time': time,
|
||
'pad_t': pad_t,
|
||
't_chunks': t_chunks,
|
||
'dim_f': dim_f,
|
||
'dim_t': dim_t,
|
||
}
|
||
|
||
return spec, metadata
|
||
|
||
|
||
def _batch_to_spec(batch: np.ndarray, metadata: dict) -> np.ndarray:
|
||
"""Convert batch format back to spectrogram.
|
||
|
||
Input batch: (t_chunks, 4, dim_f, dim_t)
|
||
Output spec: (stereo=2, freq, time, complex=2)
|
||
"""
|
||
freq = metadata['freq']
|
||
time = metadata['time']
|
||
pad_t = metadata['pad_t']
|
||
t_chunks = metadata['t_chunks']
|
||
dim_f = metadata['dim_f']
|
||
dim_t = metadata['dim_t']
|
||
|
||
# Reassemble time chunks
|
||
batch = batch.transpose(1, 2, 0, 3) # (4, dim_f, t_chunks, dim_t)
|
||
batch = batch.reshape(4, dim_f, t_chunks * dim_t) # (4, dim_f, time_padded)
|
||
|
||
# Remove time padding
|
||
if pad_t > 0:
|
||
batch = batch[:, :, :time] # (4, dim_f, time)
|
||
|
||
# Restore freq bins cropped before ONNX (pad back to original freq)
|
||
if freq > dim_f:
|
||
zeros = np.zeros((4, freq - dim_f, time), dtype=batch.dtype)
|
||
batch = np.concatenate([batch, zeros], axis=1) # (4, freq, time)
|
||
|
||
# Split 4 channels back to stereo(2) × complex(2)
|
||
batch = batch.reshape(2, 2, freq, time) # (stereo, complex, freq, time)
|
||
spec = batch.transpose(0, 2, 3, 1) # (2, freq, time, 2)
|
||
|
||
return spec
|
||
|
||
|
||
class MDXNetSeparator:
|
||
"""
|
||
MDX-Net separator using ONNX models from Ultimate Vocal Remover.
|
||
Supports various models for vocals/instrumental/karaoke separation.
|
||
"""
|
||
|
||
def __init__(
|
||
self,
|
||
model_id: str,
|
||
model_dir: Optional[str] = None,
|
||
cpu_separation: bool = True,
|
||
output_format: str = 'mp3',
|
||
mp3_bitrate: str = '256k',
|
||
):
|
||
if not ONNX_AVAILABLE:
|
||
raise ImportError("onnxruntime is required for MDX-Net models. Install with: pip install onnxruntime")
|
||
|
||
if model_id not in MDXNET_MODELS:
|
||
raise ValueError(f"Unknown MDX-Net model: {model_id}. Available: {list(MDXNET_MODELS.keys())}")
|
||
|
||
self.model_id = model_id
|
||
self.model_config = MDXNET_MODELS[model_id]
|
||
self.model_dir = Path(model_dir) if model_dir else Path('/opt/mdxnet_models')
|
||
self.model_path = self.model_dir / self.model_config['path'].split('/')[-1]
|
||
self.device = 'cpu' if cpu_separation else 'cuda'
|
||
self.output_format = output_format.lower()
|
||
self.mp3_bitrate = mp3_bitrate
|
||
self.stems = self.model_config['stems']
|
||
self.target_stem = self.model_config['target_stem']
|
||
self._session = None
|
||
self.dim_f = None
|
||
self.dim_t = None
|
||
self.n_fft = None
|
||
|
||
def _ensure_model(self):
|
||
"""Download model if not present."""
|
||
if self.model_path.exists():
|
||
return
|
||
|
||
self.model_dir.mkdir(parents=True, exist_ok=True)
|
||
from huggingface_hub import hf_hub_download
|
||
import shutil
|
||
|
||
# Download to HF cache then copy flat into model_dir so model_path is correct
|
||
cached = hf_hub_download(
|
||
repo_id=self.model_config['repo_id'],
|
||
filename=self.model_config['path'],
|
||
)
|
||
shutil.copy2(cached, str(self.model_path))
|
||
|
||
def _get_session(self):
|
||
"""Get or create ONNX runtime session."""
|
||
if self._session is None:
|
||
self._ensure_model()
|
||
|
||
providers = ['CPUExecutionProvider']
|
||
if not self.device == 'cpu' and 'CUDAExecutionProvider' in ort.get_available_providers():
|
||
providers = ['CUDAExecutionProvider', 'CPUExecutionProvider']
|
||
|
||
self._session = ort.InferenceSession(
|
||
str(self.model_path),
|
||
providers=providers
|
||
)
|
||
|
||
# Read model-specific dimensions from the ONNX input shape.
|
||
# Input shape: (batch_size, 4, dim_f, dim_t) — varies per model file.
|
||
input_shape = self._session.get_inputs()[0].shape
|
||
self.dim_f = input_shape[2]
|
||
self.dim_t = input_shape[3]
|
||
# n_fft * 2 gives dim_f+1 freq bins; the top bin is cropped before inference.
|
||
self.n_fft = self.dim_f * 2
|
||
|
||
return self._session
|
||
|
||
def _demix(self, mix: np.ndarray) -> Dict[str, np.ndarray]:
|
||
"""Separate audio into stems using MDX-Net."""
|
||
session = self._get_session()
|
||
input_name = session.get_inputs()[0].name
|
||
|
||
# Compute STFT using model-specific n_fft (= dim_f * 2)
|
||
mix_spec = _stft(mix, n_fft=self.n_fft) # (2, freq, time, 2)
|
||
|
||
# Convert to batch format using model-specific dim_f / dim_t
|
||
mix_batch, metadata = _spec_to_batch(mix_spec, dim_f=self.dim_f, dim_t=self.dim_t)
|
||
|
||
# Run inference in chunks
|
||
output_batches = []
|
||
batch_size = 1 # Process one chunk at a time for memory efficiency
|
||
|
||
with tqdm(total=len(mix_batch), desc='Separating', unit='chunk', ncols=100) as pbar:
|
||
for i in range(0, len(mix_batch), batch_size):
|
||
batch = mix_batch[i:i + batch_size]
|
||
outputs = session.run(None, {input_name: batch.astype(np.float32)})
|
||
output_batches.append(outputs[0])
|
||
pbar.update(len(batch))
|
||
|
||
# Concatenate outputs
|
||
target_spec = np.concatenate(output_batches, axis=0)
|
||
|
||
# Convert back to spectrogram format
|
||
target_spec = _batch_to_spec(target_spec, metadata)
|
||
|
||
# Inverse STFT to get audio
|
||
target_audio = _istft(target_spec, n_fft=self.n_fft)
|
||
|
||
# Calculate the other stem (residual).
|
||
# ISTFT may return a slightly different length than the original mix;
|
||
# trim or zero-pad to match before subtracting.
|
||
n = mix.shape[1]
|
||
if target_audio.shape[1] > n:
|
||
target_audio = target_audio[:, :n]
|
||
elif target_audio.shape[1] < n:
|
||
target_audio = np.pad(target_audio, ((0, 0), (0, n - target_audio.shape[1])))
|
||
other_audio = mix - target_audio
|
||
|
||
# Create output dict based on stems configuration
|
||
sources = {}
|
||
if self.target_stem == 'vocals':
|
||
sources['vocals'] = target_audio
|
||
sources['instrumental'] = other_audio
|
||
elif self.target_stem == 'instrumental':
|
||
sources['instrumental'] = target_audio
|
||
sources['vocals'] = other_audio
|
||
elif self.target_stem == 'lead':
|
||
sources['lead'] = target_audio
|
||
sources['backing_instrumental'] = other_audio
|
||
else:
|
||
sources[self.target_stem] = target_audio
|
||
sources['other'] = other_audio
|
||
|
||
return sources
|
||
|
||
def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]:
|
||
"""Separate audio and write stems."""
|
||
output_dir = Path(output_dir)
|
||
output_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
# Load audio
|
||
audio = _load_audio(input_path)
|
||
|
||
# Separate
|
||
sources = self._demix(audio)
|
||
|
||
# Cleanup
|
||
if self._session is not None:
|
||
del self._session
|
||
self._session = None
|
||
gc.collect()
|
||
|
||
# Save outputs
|
||
ext = self.output_format
|
||
result = {}
|
||
for name, data in sources.items():
|
||
dest = str(output_dir / f'{name}.{ext}')
|
||
_save_audio(dest, data.T, ext, self.mp3_bitrate)
|
||
result[name] = dest
|
||
|
||
return result
|
||
|
||
def separate_into_parts_chunked(
|
||
self,
|
||
input_path: str,
|
||
output_dir: str,
|
||
chunk_plan,
|
||
progress_cb=None,
|
||
) -> Dict[str, str]:
|
||
"""Chunked variant of separate_into_parts for long tracks.
|
||
|
||
Splits audio into overlapping time-domain chunks, runs _demix() on
|
||
each chunk sequentially, then reassembles via crossfade overlap-add.
|
||
|
||
Args:
|
||
input_path: Path to the source audio file.
|
||
output_dir: Directory to write separated stem files.
|
||
chunk_plan: ChunkPlan from separation.chunking.plan_chunks().
|
||
progress_cb: Optional callable(str) for per-chunk progress.
|
||
|
||
Returns:
|
||
{stem_name: absolute_path} for each written stem file.
|
||
"""
|
||
from .separation.chunking import extract_chunk, overlap_add_reconstruct
|
||
|
||
output_dir = Path(output_dir)
|
||
output_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
audio = _load_audio(input_path)
|
||
n_chunks = len(chunk_plan.chunks)
|
||
chunk_outputs = []
|
||
|
||
for i, (start, end) in enumerate(chunk_plan.chunks):
|
||
ts_start = start / chunk_plan.sample_rate
|
||
ts_end = min(end, chunk_plan.total_samples) / chunk_plan.sample_rate
|
||
msg = f"Chunk {i + 1}/{n_chunks} ({ts_start:.0f} s – {ts_end:.0f} s)..."
|
||
if progress_cb:
|
||
progress_cb(msg)
|
||
|
||
chunk_audio = extract_chunk(audio, start, end)
|
||
sources = self._demix(chunk_audio)
|
||
chunk_outputs.append(sources)
|
||
|
||
stem_names = list(chunk_outputs[0].keys())
|
||
full_sources = overlap_add_reconstruct(chunk_outputs, chunk_plan, stem_names)
|
||
|
||
# Cleanup session
|
||
if self._session is not None:
|
||
del self._session
|
||
self._session = None
|
||
gc.collect()
|
||
|
||
# Save outputs
|
||
ext = self.output_format
|
||
result = {}
|
||
for name, data in full_sources.items():
|
||
dest = str(output_dir / f'{name}.{ext}')
|
||
_save_audio(dest, data.T, ext, self.mp3_bitrate)
|
||
result[name] = dest
|
||
|
||
return result
|