Files
stemengine_old/backend/services/bs_roformer_separator.py
T
2026-03-03 14:52:19 +09:00

323 lines
13 KiB
Python
Executable File

"""
Standalone BS-RoFormer separator — no Django, no Spleeter/TensorFlow dependency.
Audio I/O uses soundfile (WAV/FLAC) + subprocess ffmpeg (MP3 encoding).
Model architecture is unchanged from the original spleeter-web implementation.
"""
import gc
import os
import subprocess
import tempfile
from pathlib import Path
from typing import Dict, Optional
import numpy as np
import torch
import torch.nn as nn
import yaml
from omegaconf import OmegaConf
from tqdm import tqdm
from .bs_roformer import BSRoformer
# ---------------------------------------------------------------------------
# HuggingFace model coordinates
# ---------------------------------------------------------------------------
HF_REPO_ID = 'jarredou/BS-ROFO-SW-Fixed'
HF_REVISION = 'f99fc5c2e4bc201413cfc1ca5767cf62ddb1a9a4'
# ---------------------------------------------------------------------------
# Default model paths (overridable via env var BS_ROFORMER_MODEL_DIR)
# ---------------------------------------------------------------------------
_MODEL_DIR_ENV = os.environ.get('BS_ROFORMER_MODEL_DIR', '/opt/bs_roformer_models')
DEFAULT_MODEL_DIR = Path(_MODEL_DIR_ENV)
DEFAULT_MODEL_PATH = DEFAULT_MODEL_DIR / 'BS-Rofo-SW-Fixed.ckpt'
DEFAULT_CONFIG_PATH = DEFAULT_MODEL_DIR / 'BS-Rofo-SW-Fixed.yaml'
SAMPLE_RATE = 44100
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _try_download_model(
model_dir: Path = DEFAULT_MODEL_DIR,
model_path: Path = DEFAULT_MODEL_PATH,
config_path: Path = DEFAULT_CONFIG_PATH,
) -> None:
"""Download BS-RoFormer model from HuggingFace if not already present."""
if model_path.exists() and config_path.exists():
return
model_dir.mkdir(parents=True, exist_ok=True)
from huggingface_hub import snapshot_download
snapshot_download(
repo_id=HF_REPO_ID,
revision=HF_REVISION,
local_dir=str(model_dir),
)
def _load_config(config_path: Path):
"""Load YAML config, supporting !!python/tuple tags."""
with open(config_path, 'r') as f:
config = yaml.load(f, Loader=yaml.UnsafeLoader)
return OmegaConf.create(config)
def _load_audio(path: str) -> np.ndarray:
"""
Load audio file and return as float32 numpy array of shape (channels, samples).
Uses soundfile for WAV/FLAC; falls back to ffmpeg decode for MP3 and other formats.
"""
import soundfile as sf
try:
data, sr = sf.read(path, dtype='float32', always_2d=True)
# soundfile returns (samples, channels); transpose to (channels, samples)
data = data.T
except Exception:
# Fallback: decode via ffmpeg to a temporary WAV
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
tmp_path = tmp.name
try:
subprocess.run(
['ffmpeg', '-y', '-i', path, '-ar', str(SAMPLE_RATE),
'-ac', '2', '-f', 'wav', tmp_path],
check=True, capture_output=True,
)
data, sr = sf.read(tmp_path, dtype='float32', always_2d=True)
data = data.T
finally:
if os.path.exists(tmp_path):
os.remove(tmp_path)
# Resample if needed
if sr != SAMPLE_RATE:
import librosa
data = np.array([
librosa.resample(ch, orig_sr=sr, target_sr=SAMPLE_RATE)
for ch in data
], dtype=np.float32)
# Ensure stereo
if data.shape[0] == 1:
data = np.vstack([data, data])
return data # (channels, samples)
def _save_audio(path: str, data: np.ndarray, output_format: str, bitrate: str = '256k') -> None:
"""
Save (samples, channels) float32 array as audio file.
WAV and FLAC use soundfile directly; MP3 encodes via ffmpeg.
"""
import soundfile as sf
ext = output_format.lower()
if ext in ('wav', 'flac'):
sf.write(path, data, SAMPLE_RATE, format=ext.upper(),
subtype='PCM_24' if ext == 'wav' else 'PCM_16')
else:
# Encode to MP3 via ffmpeg
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
tmp_path = tmp.name
try:
sf.write(tmp_path, data, SAMPLE_RATE, format='WAV', subtype='PCM_16')
subprocess.run(
['ffmpeg', '-y', '-i', tmp_path,
'-b:a', bitrate, path],
check=True, capture_output=True,
)
finally:
if os.path.exists(tmp_path):
os.remove(tmp_path)
# ---------------------------------------------------------------------------
# Separator class
# ---------------------------------------------------------------------------
class BSRoformerSeparator:
"""
Performs source separation using the BS-RoFormer model.
Pure PyTorch — no TensorFlow / Spleeter dependency.
"""
STEM_MODE_4 = '4stem'
STEM_MODE_5_GUITAR = '5stem_guitar'
STEM_MODE_5_PIANO = '5stem_piano'
STEM_MODE_6 = '6stem'
def __init__(
self,
model_path: Optional[str] = None,
config_path: Optional[str] = None,
cpu_separation: bool = True,
output_format: str = 'mp3', # 'mp3' | 'flac' | 'wav'
mp3_bitrate: str = '256k',
stem_mode: str = '4stem',
):
self.model_path = Path(model_path) if model_path else DEFAULT_MODEL_PATH
self.config_path = Path(config_path) if config_path else DEFAULT_CONFIG_PATH
self.device = 'cpu' if cpu_separation else 'cuda'
self.output_format = output_format.lower()
self.mp3_bitrate = mp3_bitrate
self.stem_mode = stem_mode
self._config = None
# ------------------------------------------------------------------
# Model loading
# ------------------------------------------------------------------
def _ensure_model(self):
_try_download_model(DEFAULT_MODEL_DIR, self.model_path, self.config_path)
def _load_config(self):
if self._config is None:
self._config = _load_config(self.config_path)
return self._config
def get_model(self) -> BSRoformer:
self._ensure_model()
config = self._load_config()
model_cfg = OmegaConf.to_container(config.model, resolve=True)
# OmegaConf loads tuples as lists; the model requires tuples
for key in ('freqs_per_bands', 'multi_stft_resolutions_window_sizes'):
if key in model_cfg and isinstance(model_cfg[key], list):
model_cfg[key] = tuple(model_cfg[key])
model = BSRoformer(**model_cfg)
checkpoint = torch.load(str(self.model_path), map_location='cpu', weights_only=False)
model.load_state_dict(checkpoint)
model.to(self.device).eval()
return model
# ------------------------------------------------------------------
# Inference
# ------------------------------------------------------------------
def _demix(self, mix: np.ndarray, model: BSRoformer,
batch_size_override: Optional[int] = None,
num_overlap_override: Optional[int] = None) -> Dict[str, np.ndarray]:
"""Separate (channels, samples) array into a dict of stem arrays."""
config = self._load_config()
hop_length = config.model.stft_hop_length
segment_size = config.inference.dim_t
C = hop_length * (segment_size - 1)
N = num_overlap_override if num_overlap_override is not None else config.inference.num_overlap
step = int(C // N)
fade_size = C // 10
batch_size = batch_size_override if batch_size_override is not None else config.inference.batch_size
device = self.device
mix_t = torch.tensor(mix, dtype=torch.float32)
length_init = mix_t.shape[-1]
if length_init > 2 * (C - step) and (C - step > 0):
mix_t = nn.functional.pad(mix_t, (C - step, C - step), mode='reflect')
fadein = torch.linspace(0, 1, fade_size).to(device)
fadeout = torch.linspace(1, 0, fade_size).to(device)
window_start = torch.ones(C).to(device); window_start[-fade_size:] *= fadeout
window_finish = torch.ones(C).to(device); window_finish[:fade_size] *= fadein
window_middle = torch.ones(C).to(device)
window_middle[:fade_size] *= fadein; window_middle[-fade_size:] *= fadeout
S = len(config.training.instruments)
req_shape = (S,) + tuple(mix_t.shape)
with torch.inference_mode():
result = torch.zeros(req_shape, dtype=torch.float32, device=device)
counter = torch.zeros(req_shape, dtype=torch.float32, device=device)
batch_data, batch_locs = [], []
i = 0
# count total segments for tqdm
total_seg = sum(1 for _ in range(0, mix_t.shape[1], step))
with tqdm(total=total_seg, desc='Separating', unit='segment', ncols=100) as pbar:
while i < mix_t.shape[1]:
part = mix_t[:, i:i + C].to(device)
l = part.shape[-1]
if l < C:
part = nn.functional.pad(
part,
(0, C - l) if l > C // 2 + 1 else (0, C - l, 0, 0),
mode='reflect' if l > C // 2 + 1 else 'constant',
)
batch_data.append(part)
batch_locs.append((i, l))
i += step
if len(batch_data) >= batch_size or i >= mix_t.shape[1]:
arr = torch.stack(batch_data, dim=0)
x = model(arr)
for j, (start, ln) in enumerate(batch_locs):
window = (window_start if start == 0
else window_finish if i >= mix_t.shape[1]
else window_middle)
result[..., start:start + ln] += x[j][..., :ln] * window[..., :ln]
counter[..., start:start + ln] += window[..., :ln]
pbar.update(len(batch_data))
batch_data, batch_locs = [], []
estimated = result / counter.clamp(min=1e-10)
if length_init > 2 * (C - step) and (C - step > 0):
estimated = estimated[..., (C - step):-(C - step)]
sources = {}
for idx, name in enumerate(config.training.instruments):
sources[name] = estimated[idx].cpu().numpy()
return sources
def _merge_stems(self, sources_6: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]:
"""Map 6-stem model output to the requested stem_mode output."""
v, d, b, o, g, p = (
sources_6['vocals'], sources_6['drums'], sources_6['bass'],
sources_6['other'], sources_6['guitar'], sources_6['piano'],
)
if self.stem_mode == self.STEM_MODE_4:
return {'vocals': v, 'drums': d, 'bass': b, 'other': o + g + p}
if self.stem_mode == self.STEM_MODE_5_GUITAR:
return {'vocals': v, 'drums': d, 'bass': b, 'guitar': g, 'other': o + p}
if self.stem_mode == self.STEM_MODE_5_PIANO:
return {'vocals': v, 'drums': d, 'bass': b, 'piano': p, 'other': o + g}
# 6stem — return all
return sources_6
# ------------------------------------------------------------------
# Public API
# ------------------------------------------------------------------
def separate_into_parts(self, input_path: str, output_dir: str,
batch_size_override: Optional[int] = None,
num_overlap_override: Optional[int] = None) -> Dict[str, str]:
"""
Separate audio at input_path and write one file per stem into output_dir.
Returns {stem_name: output_file_path}.
"""
output_dir = Path(output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
audio = _load_audio(input_path) # (channels, samples)
model = self.get_model()
sources_6 = self._demix(audio, model,
batch_size_override=batch_size_override,
num_overlap_override=num_overlap_override)
del model
torch.cuda.empty_cache()
gc.collect()
stems = self._merge_stems(sources_6)
ext = self.output_format
result = {}
for name, data in stems.items():
dest = str(output_dir / f'{name}.{ext}')
_save_audio(dest, data.T, ext, self.mp3_bitrate) # T: (channels,samples)→(samples,channels)
result[name] = dest
return result