Files
2026-03-31 17:59:33 +09:00

467 lines
16 KiB
Python
Executable File
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
MDX-Net separator using ONNX models from Ultimate Vocal Remover (UVR).
Supports various MDX-Net models for vocals/instrumental separation.
"""
import gc
import os
import subprocess
import tempfile
from pathlib import Path
from typing import Dict, Optional, Tuple
from collections import namedtuple
import numpy as np
import torch
import torch.nn.functional as F
from tqdm import tqdm
# Try to import ONNX runtime
try:
import onnxruntime as ort
ONNX_AVAILABLE = True
except ImportError:
ONNX_AVAILABLE = False
SAMPLE_RATE = 44100
# Model configurations
MDXNET_MODELS = {
'mdxnet_voc_ft': {
'repo_id': 'Politrees/UVR_resources',
'path': 'models/MDXNet/UVR-MDX-NET-Voc_FT.onnx',
'stems': ['vocals', 'instrumental'],
'target_stem': 'vocals',
},
'mdxnet_inst_hq_1': {
'repo_id': 'Politrees/UVR_resources',
'path': 'models/MDXNet/UVR-MDX-NET-Inst_HQ_1.onnx',
'stems': ['instrumental', 'vocals'],
'target_stem': 'instrumental',
},
'mdxnet_inst_hq_2': {
'repo_id': 'Politrees/UVR_resources',
'path': 'models/MDXNet/UVR-MDX-NET-Inst_HQ_2.onnx',
'stems': ['instrumental', 'vocals'],
'target_stem': 'instrumental',
},
'mdxnet_inst_hq_3': {
'repo_id': 'Politrees/UVR_resources',
'path': 'models/MDXNet/UVR-MDX-NET-Inst_HQ_3.onnx',
'stems': ['instrumental', 'vocals'],
'target_stem': 'instrumental',
},
'mdxnet_karaoke': {
'repo_id': 'Politrees/UVR_resources',
'path': 'models/MDXNet/UVR_MDXNET_KARA.onnx',
'stems': ['lead', 'backing_instrumental'],
'target_stem': 'lead',
},
# New models from MVSEP-Central-Backup
'mdxnet_dereverb': {
'repo_id': 'miercolesv/MVSEP-Central-Backup',
'path': 'Dereverb-MelBand-Anvuew/dereverb_mel_band_roformer_anvuew_sdr_19.1729.ckpt',
'stems': ['dry_vocals', 'reverb'],
'target_stem': 'dry_vocals',
'use_melband': True, # Uses MelBand architecture, not pure ONNX
},
# mdxnet_drumsep is intentionally absent here — it is a PyTorch TFC-TDF
# checkpoint (MDX23C architecture), not an ONNX model. Loading it with
# ort.InferenceSession raises INVALID_PROTOBUF. It is handled instead
# by mdx23c_separator.MDX23CSeparator via _run_mdx23c() in
# separator_service.py.
}
# MDX-Net specific constants
HOP_LENGTH = 1024
# DIM_F and DIM_T are model-specific and are read from the ONNX input shape at runtime.
# N_FFT = DIM_F * 2 → produces DIM_F+1 freq bins; top bin is cropped before ONNX inference.
def _load_audio(path: str, sample_rate: int = SAMPLE_RATE) -> np.ndarray:
"""Load audio as (channels, samples) float32 array."""
import soundfile as sf
try:
data, sr = sf.read(path, dtype='float32', always_2d=True)
data = data.T
except Exception:
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
tmp_path = tmp.name
try:
subprocess.run(
['ffmpeg', '-y', '-i', path, '-ar', str(sample_rate),
'-ac', '2', '-f', 'wav', tmp_path],
check=True, capture_output=True,
)
data, sr = sf.read(tmp_path, dtype='float32', always_2d=True)
data = data.T
finally:
if os.path.exists(tmp_path):
os.remove(tmp_path)
if sr != sample_rate:
import librosa
data = np.array([
librosa.resample(ch, orig_sr=sr, target_sr=sample_rate)
for ch in data
], dtype=np.float32)
# Ensure stereo
if data.shape[0] == 1:
data = np.vstack([data, data])
return data
def _save_audio(path: str, data: np.ndarray, output_format: str, bitrate: str = '256k') -> None:
"""Save (samples, channels) array to audio file."""
import soundfile as sf
ext = output_format.lower()
if ext in ('wav', 'flac'):
sf.write(path, data, SAMPLE_RATE, format=ext.upper(),
subtype='PCM_24' if ext == 'wav' else 'PCM_16')
else:
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
tmp_path = tmp.name
try:
sf.write(tmp_path, data, SAMPLE_RATE, format='WAV', subtype='PCM_16')
subprocess.run(
['ffmpeg', '-y', '-i', tmp_path, '-b:a', bitrate, path],
check=True, capture_output=True,
)
finally:
if os.path.exists(tmp_path):
os.remove(tmp_path)
def _stft(wave: np.ndarray, n_fft: int = 4096, hop_length: int = HOP_LENGTH) -> np.ndarray:
"""Compute STFT of audio.
np.stft/np.istft never existed; use librosa which shares the same
n_fft/hop_length parameter names. librosa.stft returns a complex
(freq, time) array; we pack it as (freq, time, 2) real/imag to match
the shape contract expected by _spec_to_batch.
"""
import librosa
# wave shape: (channels, samples)
channels = wave.shape[0]
specs = []
for ch in range(channels):
spec = librosa.stft(wave[ch], n_fft=n_fft, hop_length=hop_length, window='hann')
# complex (freq, time) → (freq, time, 2) real/imag
specs.append(np.stack([spec.real, spec.imag], axis=-1))
return np.array(specs) # (channels, freq, time, 2) where last dim is real/imag
def _istft(spec: np.ndarray, n_fft: int = 4096, hop_length: int = HOP_LENGTH) -> np.ndarray:
"""Compute inverse STFT."""
import librosa
channels = spec.shape[0]
waves = []
for ch in range(channels):
# (freq, time, 2) real/imag → complex (freq, time)
complex_spec = spec[ch, :, :, 0] + 1j * spec[ch, :, :, 1]
wave = librosa.istft(complex_spec, n_fft=n_fft, hop_length=hop_length, window='hann')
waves.append(wave)
return np.array(waves)
def _spec_to_batch(spec: np.ndarray, dim_f: int = 2048, dim_t: int = 256) -> Tuple[np.ndarray, dict]:
"""Convert spectrogram to batch format for MDX-Net.
Input spec: (stereo=2, freq, time, complex=2)
Output batch: (t_chunks, 4, dim_f, dim_t)
where the 4 channels are (L_real, L_imag, R_real, R_imag); freq is
cropped to dim_f; time is padded to a multiple of dim_t.
"""
_, freq, time, _ = spec.shape # (2, freq, time, 2)
# Pad time to a multiple of dim_t
pad_t = (dim_t - time % dim_t) % dim_t
if pad_t > 0:
spec = np.pad(spec, ((0, 0), (0, 0), (0, pad_t), (0, 0)))
# Pad freq to dim_f if necessary (normally freq >= dim_f)
pad_f = max(0, dim_f - freq)
if pad_f > 0:
spec = np.pad(spec, ((0, 0), (0, pad_f), (0, 0), (0, 0)))
# Crop freq to exactly dim_f
spec = spec[:, :dim_f, :, :] # (2, dim_f, time_padded, 2)
time_padded = time + pad_t
t_chunks = time_padded // dim_t
# Merge stereo(2) × complex(2) → 4 channels
spec = spec.transpose(0, 3, 1, 2) # (2, 2, dim_f, time_padded)
spec = spec.reshape(4, dim_f, time_padded) # (4, dim_f, time_padded)
# Chunk time dimension
spec = spec.reshape(4, dim_f, t_chunks, dim_t)
spec = spec.transpose(2, 0, 1, 3) # (t_chunks, 4, dim_f, dim_t)
metadata = {
'freq': freq,
'time': time,
'pad_t': pad_t,
't_chunks': t_chunks,
'dim_f': dim_f,
'dim_t': dim_t,
}
return spec, metadata
def _batch_to_spec(batch: np.ndarray, metadata: dict) -> np.ndarray:
"""Convert batch format back to spectrogram.
Input batch: (t_chunks, 4, dim_f, dim_t)
Output spec: (stereo=2, freq, time, complex=2)
"""
freq = metadata['freq']
time = metadata['time']
pad_t = metadata['pad_t']
t_chunks = metadata['t_chunks']
dim_f = metadata['dim_f']
dim_t = metadata['dim_t']
# Reassemble time chunks
batch = batch.transpose(1, 2, 0, 3) # (4, dim_f, t_chunks, dim_t)
batch = batch.reshape(4, dim_f, t_chunks * dim_t) # (4, dim_f, time_padded)
# Remove time padding
if pad_t > 0:
batch = batch[:, :, :time] # (4, dim_f, time)
# Restore freq bins cropped before ONNX (pad back to original freq)
if freq > dim_f:
zeros = np.zeros((4, freq - dim_f, time), dtype=batch.dtype)
batch = np.concatenate([batch, zeros], axis=1) # (4, freq, time)
# Split 4 channels back to stereo(2) × complex(2)
batch = batch.reshape(2, 2, freq, time) # (stereo, complex, freq, time)
spec = batch.transpose(0, 2, 3, 1) # (2, freq, time, 2)
return spec
class MDXNetSeparator:
"""
MDX-Net separator using ONNX models from Ultimate Vocal Remover.
Supports various models for vocals/instrumental/karaoke separation.
"""
def __init__(
self,
model_id: str,
model_dir: Optional[str] = None,
cpu_separation: bool = True,
output_format: str = 'mp3',
mp3_bitrate: str = '256k',
):
if not ONNX_AVAILABLE:
raise ImportError("onnxruntime is required for MDX-Net models. Install with: pip install onnxruntime")
if model_id not in MDXNET_MODELS:
raise ValueError(f"Unknown MDX-Net model: {model_id}. Available: {list(MDXNET_MODELS.keys())}")
self.model_id = model_id
self.model_config = MDXNET_MODELS[model_id]
self.model_dir = Path(model_dir) if model_dir else Path('/opt/mdxnet_models')
self.model_path = self.model_dir / self.model_config['path'].split('/')[-1]
self.device = 'cpu' if cpu_separation else 'cuda'
self.output_format = output_format.lower()
self.mp3_bitrate = mp3_bitrate
self.stems = self.model_config['stems']
self.target_stem = self.model_config['target_stem']
self._session = None
self.dim_f = None
self.dim_t = None
self.n_fft = None
def _ensure_model(self):
"""Download model if not present."""
if self.model_path.exists():
return
self.model_dir.mkdir(parents=True, exist_ok=True)
from huggingface_hub import hf_hub_download
import shutil
# Download to HF cache then copy flat into model_dir so model_path is correct
cached = hf_hub_download(
repo_id=self.model_config['repo_id'],
filename=self.model_config['path'],
)
shutil.copy2(cached, str(self.model_path))
def _get_session(self):
"""Get or create ONNX runtime session."""
if self._session is None:
self._ensure_model()
providers = ['CPUExecutionProvider']
if not self.device == 'cpu' and 'CUDAExecutionProvider' in ort.get_available_providers():
providers = ['CUDAExecutionProvider', 'CPUExecutionProvider']
self._session = ort.InferenceSession(
str(self.model_path),
providers=providers
)
# Read model-specific dimensions from the ONNX input shape.
# Input shape: (batch_size, 4, dim_f, dim_t) — varies per model file.
input_shape = self._session.get_inputs()[0].shape
self.dim_f = input_shape[2]
self.dim_t = input_shape[3]
# n_fft * 2 gives dim_f+1 freq bins; the top bin is cropped before inference.
self.n_fft = self.dim_f * 2
return self._session
def _demix(self, mix: np.ndarray) -> Dict[str, np.ndarray]:
"""Separate audio into stems using MDX-Net."""
session = self._get_session()
input_name = session.get_inputs()[0].name
# Compute STFT using model-specific n_fft (= dim_f * 2)
mix_spec = _stft(mix, n_fft=self.n_fft) # (2, freq, time, 2)
# Convert to batch format using model-specific dim_f / dim_t
mix_batch, metadata = _spec_to_batch(mix_spec, dim_f=self.dim_f, dim_t=self.dim_t)
# Run inference in chunks
output_batches = []
batch_size = 1 # Process one chunk at a time for memory efficiency
with tqdm(total=len(mix_batch), desc='Separating', unit='chunk', ncols=100) as pbar:
for i in range(0, len(mix_batch), batch_size):
batch = mix_batch[i:i + batch_size]
outputs = session.run(None, {input_name: batch.astype(np.float32)})
output_batches.append(outputs[0])
pbar.update(len(batch))
# Concatenate outputs
target_spec = np.concatenate(output_batches, axis=0)
# Convert back to spectrogram format
target_spec = _batch_to_spec(target_spec, metadata)
# Inverse STFT to get audio
target_audio = _istft(target_spec, n_fft=self.n_fft)
# Calculate the other stem (residual).
# ISTFT may return a slightly different length than the original mix;
# trim or zero-pad to match before subtracting.
n = mix.shape[1]
if target_audio.shape[1] > n:
target_audio = target_audio[:, :n]
elif target_audio.shape[1] < n:
target_audio = np.pad(target_audio, ((0, 0), (0, n - target_audio.shape[1])))
other_audio = mix - target_audio
# Create output dict based on stems configuration
sources = {}
if self.target_stem == 'vocals':
sources['vocals'] = target_audio
sources['instrumental'] = other_audio
elif self.target_stem == 'instrumental':
sources['instrumental'] = target_audio
sources['vocals'] = other_audio
elif self.target_stem == 'lead':
sources['lead'] = target_audio
sources['backing_instrumental'] = other_audio
else:
sources[self.target_stem] = target_audio
sources['other'] = other_audio
return sources
def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]:
"""Separate audio and write stems."""
output_dir = Path(output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
# Load audio
audio = _load_audio(input_path)
# Separate
sources = self._demix(audio)
# Cleanup
if self._session is not None:
del self._session
self._session = None
gc.collect()
# Save outputs
ext = self.output_format
result = {}
for name, data in sources.items():
dest = str(output_dir / f'{name}.{ext}')
_save_audio(dest, data.T, ext, self.mp3_bitrate)
result[name] = dest
return result
def separate_into_parts_chunked(
self,
input_path: str,
output_dir: str,
chunk_plan,
progress_cb=None,
) -> Dict[str, str]:
"""Chunked variant of separate_into_parts for long tracks.
Splits audio into overlapping time-domain chunks, runs _demix() on
each chunk sequentially, then reassembles via crossfade overlap-add.
Args:
input_path: Path to the source audio file.
output_dir: Directory to write separated stem files.
chunk_plan: ChunkPlan from separation.chunking.plan_chunks().
progress_cb: Optional callable(str) for per-chunk progress.
Returns:
{stem_name: absolute_path} for each written stem file.
"""
from .separation.chunking import extract_chunk, overlap_add_reconstruct
output_dir = Path(output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
audio = _load_audio(input_path)
n_chunks = len(chunk_plan.chunks)
chunk_outputs = []
for i, (start, end) in enumerate(chunk_plan.chunks):
ts_start = start / chunk_plan.sample_rate
ts_end = min(end, chunk_plan.total_samples) / chunk_plan.sample_rate
msg = f"Chunk {i + 1}/{n_chunks} ({ts_start:.0f} s – {ts_end:.0f} s)..."
if progress_cb:
progress_cb(msg)
chunk_audio = extract_chunk(audio, start, end)
sources = self._demix(chunk_audio)
chunk_outputs.append(sources)
stem_names = list(chunk_outputs[0].keys())
full_sources = overlap_add_reconstruct(chunk_outputs, chunk_plan, stem_names)
# Cleanup session
if self._session is not None:
del self._session
self._session = None
gc.collect()
# Save outputs
ext = self.output_format
result = {}
for name, data in full_sources.items():
dest = str(output_dir / f'{name}.{ext}')
_save_audio(dest, data.T, ext, self.mp3_bitrate)
result[name] = dest
return result