Files
stemengine_old/backend/services/bs_roformer_separator.py
2026-03-26 15:14:41 +09:00

466 lines
18 KiB
Python
Executable File

"""
Standalone BS-RoFormer separator — no Django, no Spleeter/TensorFlow dependency.
Audio I/O uses soundfile (WAV/FLAC) + subprocess ffmpeg (MP3 encoding).
Model architecture is unchanged from the original spleeter-web implementation.
"""
import gc
import inspect
import logging
import os
import subprocess
import tempfile
from pathlib import Path
from typing import Dict, Optional
import numpy as np
import torch
import torch.nn as nn
import yaml
from omegaconf import OmegaConf
from tqdm import tqdm
from .bs_roformer import BSRoformer
# ---------------------------------------------------------------------------
# HuggingFace model coordinates
# ---------------------------------------------------------------------------
HF_REPO_ID = 'jarredou/BS-ROFO-SW-Fixed'
HF_REVISION = 'f99fc5c2e4bc201413cfc1ca5767cf62ddb1a9a4'
# ---------------------------------------------------------------------------
# Default model paths (overridable via env var BS_ROFORMER_MODEL_DIR)
# ---------------------------------------------------------------------------
_MODEL_DIR_ENV = os.environ.get('BS_ROFORMER_MODEL_DIR', '/opt/bs_roformer_models')
DEFAULT_MODEL_DIR = Path(_MODEL_DIR_ENV)
DEFAULT_MODEL_PATH = DEFAULT_MODEL_DIR / 'BS-Rofo-SW-Fixed.ckpt'
DEFAULT_CONFIG_PATH = DEFAULT_MODEL_DIR / 'BS-Rofo-SW-Fixed.yaml'
SAMPLE_RATE = 44100
log = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _try_download_model(
model_dir: Path = DEFAULT_MODEL_DIR,
model_path: Path = DEFAULT_MODEL_PATH,
config_path: Path = DEFAULT_CONFIG_PATH,
) -> None:
"""Download BS-RoFormer model from HuggingFace if not already present."""
if model_path.exists() and config_path.exists():
return
model_dir.mkdir(parents=True, exist_ok=True)
from huggingface_hub import snapshot_download
snapshot_download(
repo_id=HF_REPO_ID,
revision=HF_REVISION,
local_dir=str(model_dir),
)
def _load_config(config_path: Path):
"""Load YAML config, supporting !!python/tuple tags."""
with open(config_path, 'r') as f:
config = yaml.load(f, Loader=yaml.UnsafeLoader)
return OmegaConf.create(config)
def _coerce_model_cfg_types(model_cfg: dict) -> dict:
normalized = dict(model_cfg)
for key in ('freqs_per_bands', 'multi_stft_resolutions_window_sizes'):
if key in normalized and isinstance(normalized[key], list):
normalized[key] = tuple(normalized[key])
return normalized
def _build_bsroformer_model(model_cfg: dict, *, config_path: Optional[Path] = None) -> BSRoformer:
kwargs = _coerce_model_cfg_types(model_cfg)
try:
return BSRoformer(**kwargs)
except TypeError as exc:
if "unexpected keyword argument" not in str(exc):
raise
signature = inspect.signature(BSRoformer.__init__)
allowed = {name for name in signature.parameters if name != 'self'}
alias_map = {
'freqs_per_bands': 'freqs_per_band',
}
filtered: dict = {}
dropped: list[str] = []
for key, value in kwargs.items():
mapped = alias_map.get(key, key)
if mapped in allowed:
filtered[mapped] = value
else:
dropped.append(key)
if dropped:
raise RuntimeError(
f"BSRoformer config/constructor mismatch for '{config_path or DEFAULT_CONFIG_PATH}'. "
f"Unsupported config keys for installed package version: {dropped}. "
"Use a matching checkpoint+config pair and/or compatible bs_roformer runtime."
)
return BSRoformer(**filtered)
def _load_audio(path: str) -> np.ndarray:
"""
Load audio file and return as float32 numpy array of shape (channels, samples).
Uses soundfile for WAV/FLAC; falls back to ffmpeg decode for MP3 and other formats.
"""
import soundfile as sf
try:
data, sr = sf.read(path, dtype='float32', always_2d=True)
# soundfile returns (samples, channels); transpose to (channels, samples)
data = data.T
except Exception:
# Fallback: decode via ffmpeg to a temporary WAV
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
tmp_path = tmp.name
try:
subprocess.run(
['ffmpeg', '-y', '-i', path, '-ar', str(SAMPLE_RATE),
'-ac', '2', '-f', 'wav', tmp_path],
check=True, capture_output=True,
)
data, sr = sf.read(tmp_path, dtype='float32', always_2d=True)
data = data.T
finally:
if os.path.exists(tmp_path):
os.remove(tmp_path)
# Resample if needed
if sr != SAMPLE_RATE:
import librosa
data = np.array([
librosa.resample(ch, orig_sr=sr, target_sr=SAMPLE_RATE)
for ch in data
], dtype=np.float32)
# Ensure stereo
if data.shape[0] == 1:
data = np.vstack([data, data])
return data # (channels, samples)
def _save_audio(path: str, data: np.ndarray, output_format: str, bitrate: str = '256k') -> None:
"""
Save (samples, channels) float32 array as audio file.
WAV and FLAC use soundfile directly; MP3 encodes via ffmpeg.
"""
import soundfile as sf
ext = output_format.lower()
if ext in ('wav', 'flac'):
sf.write(path, data, SAMPLE_RATE, format=ext.upper(),
subtype='PCM_24' if ext == 'wav' else 'PCM_16')
else:
# Encode to MP3 via ffmpeg
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
tmp_path = tmp.name
try:
sf.write(tmp_path, data, SAMPLE_RATE, format='WAV', subtype='PCM_16')
subprocess.run(
['ffmpeg', '-y', '-i', tmp_path,
'-b:a', bitrate, path],
check=True, capture_output=True,
)
finally:
if os.path.exists(tmp_path):
os.remove(tmp_path)
# ---------------------------------------------------------------------------
# Separator class
# ---------------------------------------------------------------------------
class BSRoformerSeparator:
"""
Performs source separation using the BS-RoFormer model.
Pure PyTorch — no TensorFlow / Spleeter dependency.
"""
STEM_MODE_4 = '4stem'
STEM_MODE_5_GUITAR = '5stem_guitar'
STEM_MODE_5_PIANO = '5stem_piano'
STEM_MODE_6 = '6stem'
def __init__(
self,
model_path: Optional[str] = None,
config_path: Optional[str] = None,
cpu_separation: bool = True,
output_format: str = 'mp3', # 'mp3' | 'flac' | 'wav'
mp3_bitrate: str = '256k',
stem_mode: str = '4stem',
):
self.model_path = Path(model_path) if model_path else DEFAULT_MODEL_PATH
self.config_path = Path(config_path) if config_path else DEFAULT_CONFIG_PATH
self.device = 'cpu' if cpu_separation else 'cuda'
self.output_format = output_format.lower()
self.mp3_bitrate = mp3_bitrate
self.stem_mode = stem_mode
self._config = None
# ------------------------------------------------------------------
# Model loading
# ------------------------------------------------------------------
def _ensure_model(self):
_try_download_model(DEFAULT_MODEL_DIR, self.model_path, self.config_path)
def _load_config(self):
if self._config is None:
self._config = _load_config(self.config_path)
return self._config
def get_model(self) -> BSRoformer:
self._ensure_model()
config = self._load_config()
model_cfg = OmegaConf.to_container(config.model, resolve=True)
model = _build_bsroformer_model(model_cfg, config_path=self.config_path)
checkpoint = torch.load(str(self.model_path), map_location='cpu', weights_only=False)
try:
result = model.load_state_dict(checkpoint, strict=False)
except RuntimeError as exc:
if "size mismatch" in str(exc):
raise RuntimeError(
f"BSRoformer checkpoint/config shape mismatch. "
f"checkpoint='{self.model_path.name}', config='{self.config_path.name}'. "
"This usually means the checkpoint and YAML are from different model revisions."
) from exc
raise
if result.missing_keys:
import logging
logging.getLogger(__name__).debug(
"Checkpoint missing %d key(s) (library newer than checkpoint): %s",
len(result.missing_keys), result.missing_keys[:5],
)
model.to(self.device).eval()
return model
# ------------------------------------------------------------------
# Inference
# ------------------------------------------------------------------
def _demix(self, mix: np.ndarray, model: BSRoformer,
batch_size_override: Optional[int] = None,
num_overlap_override: Optional[int] = None) -> Dict[str, np.ndarray]:
"""Separate (channels, samples) array into a dict of stem arrays."""
config = self._load_config()
hop_length = config.model.stft_hop_length
segment_size = config.inference.dim_t
C = hop_length * (segment_size - 1)
N = num_overlap_override if num_overlap_override is not None else config.inference.num_overlap
step = int(C // N)
fade_size = C // 10
batch_size = batch_size_override if batch_size_override is not None else config.inference.batch_size
device = self.device
mix_t = torch.tensor(mix, dtype=torch.float32)
length_init = mix_t.shape[-1]
if length_init > 2 * (C - step) and (C - step > 0):
mix_t = nn.functional.pad(mix_t, (C - step, C - step), mode='reflect')
fadein = torch.linspace(0, 1, fade_size).to(device)
fadeout = torch.linspace(1, 0, fade_size).to(device)
window_start = torch.ones(C).to(device); window_start[-fade_size:] *= fadeout
window_finish = torch.ones(C).to(device); window_finish[:fade_size] *= fadein
window_middle = torch.ones(C).to(device)
window_middle[:fade_size] *= fadein; window_middle[-fade_size:] *= fadeout
S = len(config.training.instruments)
req_shape = (S,) + tuple(mix_t.shape)
with torch.inference_mode(), \
torch.amp.autocast('cuda', dtype=torch.float16, enabled=(device != 'cpu')):
result = torch.zeros(req_shape, dtype=torch.float32, device=device)
counter = torch.zeros(req_shape, dtype=torch.float32, device=device)
batch_data, batch_locs = [], []
i = 0
# count total segments for tqdm
total_seg = sum(1 for _ in range(0, mix_t.shape[1], step))
with tqdm(total=total_seg, desc='Separating', unit='segment', ncols=100) as pbar:
while i < mix_t.shape[1]:
part = mix_t[:, i:i + C].to(device)
l = part.shape[-1]
if l < C:
part = nn.functional.pad(
part,
(0, C - l) if l > C // 2 + 1 else (0, C - l, 0, 0),
mode='reflect' if l > C // 2 + 1 else 'constant',
)
batch_data.append(part)
batch_locs.append((i, l))
i += step
if len(batch_data) >= batch_size or i >= mix_t.shape[1]:
arr = torch.stack(batch_data, dim=0)
x = model(arr)
for j, (start, ln) in enumerate(batch_locs):
window = (window_start if start == 0
else window_finish if i >= mix_t.shape[1]
else window_middle)
result[..., start:start + ln] += x[j][..., :ln] * window[..., :ln]
counter[..., start:start + ln] += window[..., :ln]
pbar.update(len(batch_data))
batch_data, batch_locs = [], []
estimated = result / counter.clamp(min=1e-10)
if length_init > 2 * (C - step) and (C - step > 0):
estimated = estimated[..., (C - step):-(C - step)]
sources = {}
for idx, name in enumerate(config.training.instruments):
sources[name] = estimated[idx].cpu().numpy()
return sources
def _merge_stems(self, sources_6: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]:
"""Map 6-stem model output to the requested stem_mode output."""
if self.stem_mode not in {
self.STEM_MODE_4,
self.STEM_MODE_5_GUITAR,
self.STEM_MODE_5_PIANO,
self.STEM_MODE_6,
}:
return sources_6
required = {'vocals', 'drums', 'bass', 'other', 'guitar', 'piano'}
if not required.issubset(sources_6.keys()):
return sources_6
v, d, b, o, g, p = (
sources_6['vocals'], sources_6['drums'], sources_6['bass'],
sources_6['other'], sources_6['guitar'], sources_6['piano'],
)
if self.stem_mode == self.STEM_MODE_4:
return {'vocals': v, 'drums': d, 'bass': b, 'other': o + g + p}
if self.stem_mode == self.STEM_MODE_5_GUITAR:
return {'vocals': v, 'drums': d, 'bass': b, 'guitar': g, 'other': o + p}
if self.stem_mode == self.STEM_MODE_5_PIANO:
return {'vocals': v, 'drums': d, 'bass': b, 'piano': p, 'other': o + g}
# 6stem — return all
return sources_6
# ------------------------------------------------------------------
# Public API
# ------------------------------------------------------------------
def separate_into_parts(self, input_path: str, output_dir: str,
batch_size_override: Optional[int] = None,
num_overlap_override: Optional[int] = None) -> Dict[str, str]:
"""
Separate audio at input_path and write one file per stem into output_dir.
Returns {stem_name: output_file_path}.
"""
output_dir = Path(output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
audio = _load_audio(input_path) # (channels, samples)
model = self.get_model()
sources_6 = self._demix(audio, model,
batch_size_override=batch_size_override,
num_overlap_override=num_overlap_override)
del model
torch.cuda.empty_cache()
gc.collect()
stems = self._merge_stems(sources_6)
ext = self.output_format
result = {}
for name, data in stems.items():
dest = str(output_dir / f'{name}.{ext}')
_save_audio(dest, data.T, ext, self.mp3_bitrate) # T: (channels,samples)→(samples,channels)
result[name] = dest
return result
def separate_into_parts_chunked(
self,
input_path: str,
output_dir: str,
chunk_plan,
batch_size_override: Optional[int] = None,
num_overlap_override: Optional[int] = None,
progress_cb=None,
) -> Dict[str, str]:
"""Chunked variant of separate_into_parts for long tracks.
Splits audio into overlapping time-domain chunks, runs _demix() on
each chunk sequentially, merges stem modes per chunk, then reassembles
via crossfade overlap-add. The model is loaded once; only one chunk's
activation tensors occupy VRAM at any time.
Args:
input_path: Path to the source audio file.
output_dir: Directory to write separated stem files.
chunk_plan: ChunkPlan from separation.chunking.plan_chunks().
batch_size_override: Forwarded to _demix().
num_overlap_override: Forwarded to _demix().
progress_cb: Optional callable(str) for per-chunk progress.
Returns:
{stem_name: absolute_path} for each written stem file.
"""
from .separation.chunking import extract_chunk, overlap_add_reconstruct
output_dir = Path(output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
audio = _load_audio(input_path) # (channels, samples)
model = self.get_model()
n_chunks = len(chunk_plan.chunks)
chunk_outputs = []
for i, (start, end) in enumerate(chunk_plan.chunks):
ts_start = start / chunk_plan.sample_rate
ts_end = min(end, chunk_plan.total_samples) / chunk_plan.sample_rate
msg = f"Chunk {i + 1}/{n_chunks} ({ts_start:.0f} s \u2013 {ts_end:.0f} s)..."
if progress_cb:
progress_cb(msg)
log.info('[chunked] %s', msg)
chunk_audio = extract_chunk(audio, start, end)
sources_6 = self._demix(
chunk_audio, model,
batch_size_override=batch_size_override,
num_overlap_override=num_overlap_override,
)
merged = self._merge_stems(sources_6)
chunk_outputs.append(merged)
try:
if torch.cuda.is_available():
torch.cuda.empty_cache()
except Exception:
pass
stem_names = list(chunk_outputs[0].keys())
full_sources = overlap_add_reconstruct(chunk_outputs, chunk_plan, stem_names)
del model
del audio
torch.cuda.empty_cache()
gc.collect()
ext = self.output_format
result = {}
for name, data in full_sources.items():
dest = str(output_dir / f'{name}.{ext}')
_save_audio(dest, data.T, ext, self.mp3_bitrate)
result[name] = dest
return result