Files
stemengine_old/backend/services/mel_band_specialized_separator.py
T
2026-03-03 14:52:19 +09:00

670 lines
24 KiB
Python
Executable File

"""
Specialized MelBand RoFormer separators for specific use cases:
- Vocals/Instrumental separation (2-stem)
- Dereverb (dry vocals vs reverb)
- Karaoke (lead vocals vs backing+instrumental)
Uses sys.modules patching for import path fixes.
"""
import gc
import os
import subprocess
import sys
import tempfile
from pathlib import Path
from typing import Dict, Optional, List
import numpy as np
import torch
import torch.nn as nn
import yaml
from omegaconf import OmegaConf
from tqdm import tqdm
# ---------------------------------------------------------------------------
# Import path patching
# ---------------------------------------------------------------------------
if 'models' not in sys.modules:
sys.modules['models'] = type(sys)('models')
if 'models.bs_roformer' not in sys.modules:
import backend.services.bs_roformer as _bs_roformer
sys.modules['models.bs_roformer'] = _bs_roformer
if 'models.bs_roformer.attend' not in sys.modules:
from backend.services.bs_roformer import attend as _attend
sys.modules['models.bs_roformer.attend'] = _attend
from bs_roformer.mel_band_roformer import MelBandRoformer
SAMPLE_RATE = 44100
def _load_audio(path: str) -> np.ndarray:
"""Load audio as (channels, samples) float32 array."""
import soundfile as sf
try:
data, sr = sf.read(path, dtype='float32', always_2d=True)
data = data.T
except Exception:
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
tmp_path = tmp.name
try:
subprocess.run(
['ffmpeg', '-y', '-i', path, '-ar', str(SAMPLE_RATE),
'-ac', '2', '-f', 'wav', tmp_path],
check=True, capture_output=True,
)
data, sr = sf.read(tmp_path, dtype='float32', always_2d=True)
data = data.T
finally:
if os.path.exists(tmp_path):
os.remove(tmp_path)
if sr != SAMPLE_RATE:
import librosa
data = np.array([
librosa.resample(ch, orig_sr=sr, target_sr=SAMPLE_RATE)
for ch in data
], dtype=np.float32)
if data.shape[0] == 1:
data = np.vstack([data, data])
return data
def _save_audio(path: str, data: np.ndarray, output_format: str, bitrate: str = '256k') -> None:
"""Save (samples, channels) array to audio file."""
import soundfile as sf
ext = output_format.lower()
if ext in ('wav', 'flac'):
sf.write(path, data, SAMPLE_RATE, format=ext.upper(),
subtype='PCM_24' if ext == 'wav' else 'PCM_16')
else:
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
tmp_path = tmp.name
try:
sf.write(tmp_path, data, SAMPLE_RATE, format='WAV', subtype='PCM_16')
subprocess.run(
['ffmpeg', '-y', '-i', tmp_path, '-b:a', bitrate, path],
check=True, capture_output=True,
)
finally:
if os.path.exists(tmp_path):
os.remove(tmp_path)
class MelBandSeparatorBase:
"""Base class for MelBand RoFormer separators."""
def __init__(
self,
model_path: Path,
config_path: Path,
cpu_separation: bool = True,
output_format: str = 'mp3',
mp3_bitrate: str = '256k',
stems: List[str] = None,
):
self.model_path = model_path
self.config_path = config_path
self.device = 'cpu' if cpu_separation else 'cuda'
self.output_format = output_format.lower()
self.mp3_bitrate = mp3_bitrate
self.stems = stems or ['vocals', 'instrumental']
self._config = None
def _load_config(self):
if self._config is None:
with open(self.config_path, 'r') as f:
config = yaml.load(f, Loader=yaml.UnsafeLoader)
self._config = OmegaConf.create(config)
return self._config
def get_model(self) -> MelBandRoformer:
config = self._load_config()
model_cfg = OmegaConf.to_container(config.model, resolve=True)
# Convert lists to tuples where needed
if 'freqs_per_bands' in model_cfg and isinstance(model_cfg['freqs_per_bands'], list):
model_cfg['freqs_per_bands'] = tuple(model_cfg['freqs_per_bands'])
if 'multi_stft_resolutions_window_sizes' in model_cfg and isinstance(model_cfg['multi_stft_resolutions_window_sizes'], list):
model_cfg['multi_stft_resolutions_window_sizes'] = tuple(model_cfg['multi_stft_resolutions_window_sizes'])
model = MelBandRoformer(**model_cfg)
checkpoint = torch.load(str(self.model_path), map_location='cpu', weights_only=False)
if 'state_dict' in checkpoint:
state_dict = checkpoint['state_dict']
elif 'model' in checkpoint:
state_dict = checkpoint['model']
else:
state_dict = checkpoint
model.load_state_dict(state_dict)
model.to(self.device).eval()
return model
def _demix(self, mix: np.ndarray, model: MelBandRoformer,
batch_size_override: Optional[int] = None,
num_overlap_override: Optional[int] = None) -> Dict[str, np.ndarray]:
"""Separate audio into stems."""
config = self._load_config()
hop_length = config.model.stft_hop_length
segment_size = getattr(config.inference, 'dim_t', 2000)
C = hop_length * (segment_size - 1)
N = num_overlap_override if num_overlap_override is not None else getattr(config.inference, 'num_overlap', 4)
step = int(C // N)
fade_size = C // 10
batch_size = batch_size_override if batch_size_override is not None else getattr(config.inference, 'batch_size', 1)
device = self.device
mix_t = torch.tensor(mix, dtype=torch.float32)
length_init = mix_t.shape[-1]
if length_init > 2 * (C - step) and (C - step > 0):
mix_t = nn.functional.pad(mix_t, (C - step, C - step), mode='reflect')
fadein = torch.linspace(0, 1, fade_size).to(device)
fadeout = torch.linspace(1, 0, fade_size).to(device)
window_start = torch.ones(C).to(device)
window_start[-fade_size:] *= fadeout
window_finish = torch.ones(C).to(device)
window_finish[:fade_size] *= fadein
window_middle = torch.ones(C).to(device)
window_middle[:fade_size] *= fadein
window_middle[-fade_size:] *= fadeout
num_stems = len(self.stems)
req_shape = (num_stems,) + tuple(mix_t.shape)
with torch.inference_mode():
result = torch.zeros(req_shape, dtype=torch.float32, device=device)
counter = torch.zeros(req_shape, dtype=torch.float32, device=device)
batch_data, batch_locs = [], []
i = 0
total_seg = sum(1 for _ in range(0, mix_t.shape[1], step))
with tqdm(total=total_seg, desc='Separating', unit='segment', ncols=100) as pbar:
while i < mix_t.shape[1]:
part = mix_t[:, i:i + C].to(device)
l = part.shape[-1]
if l < C:
part = nn.functional.pad(
part,
(0, C - l) if l > C // 2 + 1 else (0, C - l, 0, 0),
mode='reflect' if l > C // 2 + 1 else 'constant',
)
batch_data.append(part)
batch_locs.append((i, l))
i += step
if len(batch_data) >= batch_size or i >= mix_t.shape[1]:
arr = torch.stack(batch_data, dim=0)
x = model(arr)
for j, (start, ln) in enumerate(batch_locs):
window = (window_start if start == 0
else window_finish if i >= mix_t.shape[1]
else window_middle)
result[..., start:start + ln] += x[j][..., :ln] * window[..., :ln]
counter[..., start:start + ln] += window[..., :ln]
pbar.update(len(batch_data))
batch_data, batch_locs = [], []
estimated = result / counter.clamp(min=1e-10)
if length_init > 2 * (C - step) and (C - step > 0):
estimated = estimated[..., (C - step):-(C - step)]
sources = {}
for idx, name in enumerate(self.stems):
sources[name] = estimated[idx].cpu().numpy()
return sources
def separate_into_parts(self, input_path: str, output_dir: str,
batch_size_override: Optional[int] = None,
num_overlap_override: Optional[int] = None) -> Dict[str, str]:
"""Separate audio and write stems to output_dir."""
output_dir = Path(output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
audio = _load_audio(input_path)
model = self.get_model()
sources = self._demix(audio, model,
batch_size_override=batch_size_override,
num_overlap_override=num_overlap_override)
del model
torch.cuda.empty_cache()
gc.collect()
ext = self.output_format
result = {}
for name, data in sources.items():
dest = str(output_dir / f'{name}.{ext}')
_save_audio(dest, data.T, ext, self.mp3_bitrate)
result[name] = dest
return result
class MelBandVocalsSeparator(MelBandSeparatorBase):
"""MelBand RoFormer optimized for vocals/instrumental separation."""
HF_REPO_ID = 'becruily/mel-band-roformer-vocals'
HF_REVISION = 'main'
def __init__(
self,
model_dir: Optional[str] = None,
cpu_separation: bool = True,
output_format: str = 'mp3',
mp3_bitrate: str = '256k',
):
model_dir = Path(model_dir) if model_dir else Path('/opt/mel_band_models/vocals')
model_path = model_dir / 'model.ckpt'
config_path = model_dir / 'config.yaml'
# Ensure model is downloaded
self._ensure_model(model_dir, model_path, config_path)
super().__init__(
model_path=model_path,
config_path=config_path,
cpu_separation=cpu_separation,
output_format=output_format,
mp3_bitrate=mp3_bitrate,
stems=['vocals', 'instrumental'],
)
def _ensure_model(self, model_dir: Path, model_path: Path, config_path: Path):
if model_path.exists() and config_path.exists():
return
model_dir.mkdir(parents=True, exist_ok=True)
from huggingface_hub import snapshot_download
snapshot_download(
repo_id=self.HF_REPO_ID,
revision=self.HF_REVISION,
local_dir=str(model_dir),
)
class MelBandDereverbSeparator(MelBandSeparatorBase):
"""MelBand RoFormer for dereverb - separates dry vocals from reverb."""
HF_REPO_ID = 'anvuew/dereverb_mel_band_roformer'
HF_REVISION = 'main'
def __init__(
self,
model_dir: Optional[str] = None,
cpu_separation: bool = True,
output_format: str = 'mp3',
mp3_bitrate: str = '256k',
use_less_aggressive: bool = False,
):
model_dir = Path(model_dir) if model_dir else Path('/opt/mel_band_models/dereverb')
# The dereverb repo has multiple checkpoints
if use_less_aggressive:
model_path = model_dir / 'dereverb_mel_band_roformer_less_aggressive_anvuew_sdr_18.8050.ckpt'
else:
model_path = model_dir / 'dereverb_mel_band_roformer_anvuew_sdr_19.1729.ckpt'
config_path = model_dir / 'config.yaml'
self._ensure_model(model_dir, model_path, config_path)
super().__init__(
model_path=model_path,
config_path=config_path,
cpu_separation=cpu_separation,
output_format=output_format,
mp3_bitrate=mp3_bitrate,
stems=['dry_vocals', 'reverb'],
)
def _ensure_model(self, model_dir: Path, model_path: Path, config_path: Path):
if model_path.exists() and config_path.exists():
return
model_dir.mkdir(parents=True, exist_ok=True)
from huggingface_hub import snapshot_download
snapshot_download(
repo_id=self.HF_REPO_ID,
revision=self.HF_REVISION,
local_dir=str(model_dir),
)
class MelBandKaraokeSeparator(MelBandSeparatorBase):
"""MelBand RoFormer for karaoke - separates lead vocals from backing+instrumental."""
HF_REPO_ID = 'becruily/mel-band-roformer-karaoke'
HF_REVISION = 'main'
def __init__(
self,
model_dir: Optional[str] = None,
cpu_separation: bool = True,
output_format: str = 'mp3',
mp3_bitrate: str = '256k',
):
model_dir = Path(model_dir) if model_dir else Path('/opt/mel_band_models/karaoke')
model_path = model_dir / 'model.ckpt'
config_path = model_dir / 'config.yaml'
self._ensure_model(model_dir, model_path, config_path)
super().__init__(
model_path=model_path,
config_path=config_path,
cpu_separation=cpu_separation,
output_format=output_format,
mp3_bitrate=mp3_bitrate,
stems=['lead', 'backing_instrumental'],
)
def _ensure_model(self, model_dir: Path, model_path: Path, config_path: Path):
if model_path.exists() and config_path.exists():
return
model_dir.mkdir(parents=True, exist_ok=True)
from huggingface_hub import snapshot_download
snapshot_download(
repo_id=self.HF_REPO_ID,
revision=self.HF_REVISION,
local_dir=str(model_dir),
)
class MelBandDereverbRoomSeparator(MelBandSeparatorBase):
"""MelBand RoFormer for room dereverb from MVSEP-Central-Backup."""
HF_REPO_ID = 'miercolesv/MVSEP-Central-Backup'
HF_SUBFOLDER = 'Dereverb-Room-Anvuew'
HF_REVISION = 'main'
def __init__(
self,
model_dir: Optional[str] = None,
cpu_separation: bool = True,
output_format: str = 'mp3',
mp3_bitrate: str = '256k',
):
model_dir = Path(model_dir) if model_dir else Path('/opt/mel_band_models/dereverb_room')
model_path = model_dir / 'dereverb_room_anvuew_sdr_13.7432.ckpt'
config_path = model_dir / 'dereverb_room_anvuew.yaml'
self._ensure_model(model_dir, model_path, config_path)
super().__init__(
model_path=model_path,
config_path=config_path,
cpu_separation=cpu_separation,
output_format=output_format,
mp3_bitrate=mp3_bitrate,
stems=['dry_vocals', 'room_reverb'],
)
def _ensure_model(self, model_dir: Path, model_path: Path, config_path: Path):
if model_path.exists() and config_path.exists():
return
model_dir.mkdir(parents=True, exist_ok=True)
try:
from huggingface_hub import hf_hub_download
if not model_path.exists():
hf_hub_download(
repo_id=self.HF_REPO_ID,
filename=f"{self.HF_SUBFOLDER}/dereverb_room_anvuew_sdr_13.7432.ckpt",
local_dir=model_dir,
local_dir_use_symlinks=False,
)
if not config_path.exists():
hf_hub_download(
repo_id=self.HF_REPO_ID,
filename=f"{self.HF_SUBFOLDER}/dereverb_room_anvuew.yaml",
local_dir=model_dir,
local_dir_use_symlinks=False,
)
except Exception as e:
raise RuntimeError(f"Failed to download Room Dereverb model: {e}")
class MelBandDereverbAnvuewSeparator(MelBandSeparatorBase):
"""MelBand RoFormer Anvuew dereverb from MVSEP-Central-Backup."""
HF_REPO_ID = 'miercolesv/MVSEP-Central-Backup'
HF_SUBFOLDER = 'Dereverb-MelBand-Anvuew'
HF_REVISION = 'main'
def __init__(
self,
model_dir: Optional[str] = None,
cpu_separation: bool = True,
output_format: str = 'mp3',
mp3_bitrate: str = '256k',
use_less_aggressive: bool = False,
):
model_dir = Path(model_dir) if model_dir else Path('/opt/mel_band_models/dereverb_anvuew')
if use_less_aggressive:
model_path = model_dir / 'dereverb_mel_band_roformer_less_aggressive_anvuew_sdr_18.8050.ckpt'
else:
model_path = model_dir / 'dereverb_mel_band_roformer_anvuew_sdr_19.1729.ckpt'
config_path = model_dir / 'dereverb_mel_band_roformer_anvuew.yaml'
self._ensure_model(model_dir, model_path, config_path)
super().__init__(
model_path=model_path,
config_path=config_path,
cpu_separation=cpu_separation,
output_format=output_format,
mp3_bitrate=mp3_bitrate,
stems=['dry_vocals', 'reverb'],
)
def _ensure_model(self, model_dir: Path, model_path: Path, config_path: Path):
if model_path.exists() and config_path.exists():
return
model_dir.mkdir(parents=True, exist_ok=True)
try:
from huggingface_hub import hf_hub_download
# Download the specific model variant
model_filename = model_path.name
if not model_path.exists():
hf_hub_download(
repo_id=self.HF_REPO_ID,
filename=f"{self.HF_SUBFOLDER}/{model_filename}",
local_dir=model_dir,
local_dir_use_symlinks=False,
)
# Download config
if not config_path.exists():
hf_hub_download(
repo_id=self.HF_REPO_ID,
filename=f"{self.HF_SUBFOLDER}/dereverb_mel_band_roformer_anvuew.yaml",
local_dir=model_dir,
local_dir_use_symlinks=False,
)
except Exception as e:
raise RuntimeError(f"Failed to download Anvuew Dereverb model: {e}")
class MelBandVocalsAnvuewSeparator(MelBandSeparatorBase):
"""MelBand RoFormer Anvuew vocals from MVSEP-Central-Backup."""
HF_REPO_ID = 'miercolesv/MVSEP-Central-Backup'
HF_SUBFOLDER = 'BS-RoFormer-Anvuew'
HF_REVISION = 'main'
def __init__(
self,
model_dir: Optional[str] = None,
cpu_separation: bool = True,
output_format: str = 'mp3',
mp3_bitrate: str = '256k',
):
model_dir = Path(model_dir) if model_dir else Path('/opt/mel_band_models/vocals_anvuew')
model_path = model_dir / 'bs_roformer_anvuew_sdr_12.45.ckpt'
config_path = model_dir / 'config.yaml'
self._ensure_model(model_dir, model_path, config_path)
super().__init__(
model_path=model_path,
config_path=config_path,
cpu_separation=cpu_separation,
output_format=output_format,
mp3_bitrate=mp3_bitrate,
stems=['vocals', 'instrumental'],
)
def _ensure_model(self, model_dir: Path, model_path: Path, config_path: Path):
if model_path.exists() and config_path.exists():
return
model_dir.mkdir(parents=True, exist_ok=True)
try:
from huggingface_hub import hf_hub_download
if not model_path.exists():
hf_hub_download(
repo_id=self.HF_REPO_ID,
filename=f"{self.HF_SUBFOLDER}/bs_roformer_anvuew_sdr_12.45.ckpt",
local_dir=model_dir,
local_dir_use_symlinks=False,
)
if not config_path.exists():
hf_hub_download(
repo_id=self.HF_REPO_ID,
filename=f"{self.HF_SUBFOLDER}/config.yaml",
local_dir=model_dir,
local_dir_use_symlinks=False,
)
except Exception as e:
raise RuntimeError(f"Failed to download Anvuew Vocals model: {e}")
class MelBandKaraokeGaboxSeparator(MelBandSeparatorBase):
"""MelBand RoFormer Karaoke Gabox from MVSEP-Central-Backup."""
HF_REPO_ID = 'miercolesv/MVSEP-Central-Backup'
HF_SUBFOLDER = 'MelBandRoformers/bsroformers'
HF_REVISION = 'main'
def __init__(
self,
model_dir: Optional[str] = None,
cpu_separation: bool = True,
output_format: str = 'mp3',
mp3_bitrate: str = '256k',
):
model_dir = Path(model_dir) if model_dir else Path('/opt/mel_band_models/karaoke_gabox')
model_path = model_dir / 'bs_karaoke_gabox_IS.ckpt'
config_path = model_dir / 'karaoke_bs_roformer.yaml'
self._ensure_model(model_dir, model_path, config_path)
super().__init__(
model_path=model_path,
config_path=config_path,
cpu_separation=cpu_separation,
output_format=output_format,
mp3_bitrate=mp3_bitrate,
stems=['vocals', 'karaoke'],
)
def _ensure_model(self, model_dir: Path, model_path: Path, config_path: Path):
if model_path.exists() and config_path.exists():
return
model_dir.mkdir(parents=True, exist_ok=True)
try:
from huggingface_hub import hf_hub_download
if not model_path.exists():
hf_hub_download(
repo_id=self.HF_REPO_ID,
filename=f"{self.HF_SUBFOLDER}/bs_karaoke_gabox_IS.ckpt",
local_dir=model_dir,
local_dir_use_symlinks=False,
)
if not config_path.exists():
hf_hub_download(
repo_id=self.HF_REPO_ID,
filename=f"{self.HF_SUBFOLDER}/karaoke_bs_roformer.yaml",
local_dir=model_dir,
local_dir_use_symlinks=False,
)
except Exception as e:
raise RuntimeError(f"Failed to download Karaoke Gabox model: {e}")
class MelBandDeuxBecruilySeparator(MelBandSeparatorBase):
"""MelBand RoFormer Deux Becruily - specialized 2-stem vocals model."""
HF_REPO_ID = 'miercolesv/MVSEP-Central-Backup'
HF_SUBFOLDER = 'MelBand-Roformer-Deux-Becruily'
HF_REVISION = 'main'
def __init__(
self,
model_dir: Optional[str] = None,
cpu_separation: bool = True,
output_format: str = 'mp3',
mp3_bitrate: str = '256k',
):
model_dir = Path(model_dir) if model_dir else Path('/opt/mel_band_models/deux_becruily')
model_path = model_dir / 'becruily_deux.ckpt'
config_path = model_dir / 'config_deux_becruily.yaml'
self._ensure_model(model_dir, model_path, config_path)
super().__init__(
model_path=model_path,
config_path=config_path,
cpu_separation=cpu_separation,
output_format=output_format,
mp3_bitrate=mp3_bitrate,
stems=['vocals', 'instrumental'],
)
def _ensure_model(self, model_dir: Path, model_path: Path, config_path: Path):
if model_path.exists() and config_path.exists():
return
model_dir.mkdir(parents=True, exist_ok=True)
try:
from huggingface_hub import hf_hub_download
print(f"Downloading Deux Becruily model from {self.HF_REPO_ID}")
# Download model
if not model_path.exists():
hf_hub_download(
repo_id=self.HF_REPO_ID,
filename=f"{self.HF_SUBFOLDER}/becruily_deux.ckpt",
local_dir=model_dir,
local_dir_use_symlinks=False,
)
# Download config
if not config_path.exists():
hf_hub_download(
repo_id=self.HF_REPO_ID,
filename=f"{self.HF_SUBFOLDER}/config_deux_becruily.yaml",
local_dir=model_dir,
local_dir_use_symlinks=False,
)
except Exception as e:
raise RuntimeError(f"Failed to download Deux Becruily model: {e}")