Files
stemengine_old/backend/services/ensemble_separator.py
T
2026-04-14 16:32:36 +09:00

297 lines
12 KiB
Python
Executable File

"""
Ensemble separator - Combines multiple models for best quality separation
Uses model averaging/ensembling from multiple high-quality models.
"""
import os
import logging
import torch
import torchaudio
from pathlib import Path
from typing import Dict, List, Optional
import tempfile
import shutil
from ..config import ENSEMBLE_CACHE_DIR, CPU_SEPARATION
log = logging.getLogger(__name__)
# Ensemble configurations
ENSEMBLE_CONFIGS = {
"ensemble_vocal_best": {
"description": "Best vocal separation using BS-RoFormer x2 + MelBand RoFormer ft + SCNet XL IHF",
"models": [
{"type": "bs_roformer", "weight": 0.30, "stem_mode": "4stem"},
{"type": "mel_band", "weight": 0.25, "variant": "vocals"},
{"type": "scnet_xl_ihf", "weight": 0.45},
],
"stems": ["vocals", "instrumental"],
},
"ensemble_4stem": {
"description": "Best 4-stem separation using multiple models averaged",
"models": [
{"type": "demucs", "weight": 0.20, "model": "htdemucs_ft"},
{"type": "bs_roformer", "weight": 0.30, "stem_mode": "4stem"},
{"type": "mel_band", "weight": 0.25, "variant": "4stem"},
{"type": "scnet_large", "weight": 0.25},
],
"stems": ["vocals", "drums", "bass", "other"],
},
"ensemble_6stem": {
"description": "Best 6-stem separation using multiple models averaged",
"models": [
{"type": "bs_roformer", "weight": 0.35, "stem_mode": "6stem"},
{"type": "bs_roformer_sw", "weight": 0.35},
{"type": "demucs", "weight": 0.30, "model": "htdemucs_6s"},
],
"stems": ["vocals", "drums", "bass", "guitar", "piano", "other"],
},
}
class EnsembleSeparator:
"""
Ensemble separator that combines multiple models for best quality.
Uses weighted averaging of outputs from multiple separation models
to produce higher quality results than any single model alone.
"""
def __init__(
self,
ensemble_id: str = "ensemble_vocal_best",
cpu_separation: bool = CPU_SEPARATION,
output_format: str = "mp3",
mp3_bitrate: str = "256k",
):
if ensemble_id not in ENSEMBLE_CONFIGS:
raise ValueError(f"Unknown ensemble config: {ensemble_id}")
self.ensemble_id = ensemble_id
self.config = ENSEMBLE_CONFIGS[ensemble_id]
self.cpu_separation = cpu_separation
self.output_format = output_format
self.mp3_bitrate = mp3_bitrate
self.cache_dir = Path(ENSEMBLE_CACHE_DIR)
self.cache_dir.mkdir(parents=True, exist_ok=True)
def _load_audio(self, path: str) -> tuple:
"""Load audio file and return waveform and sample rate."""
waveform, sr = torchaudio.load(path)
return waveform, sr
def _save_audio(self, waveform: torch.Tensor, sr: int, path: str):
"""Save audio waveform to file."""
torchaudio.save(path, waveform, sr)
def _align_waveform_shape(self, reference: torch.Tensor, waveform: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
"""Align channel count + length so two tensors can be safely accumulated."""
if reference.dim() == 1:
reference = reference.unsqueeze(0)
if waveform.dim() == 1:
waveform = waveform.unsqueeze(0)
ref_channels = reference.shape[0]
wav_channels = waveform.shape[0]
if wav_channels != ref_channels:
if wav_channels == 1 and ref_channels > 1:
waveform = waveform.repeat(ref_channels, 1)
elif ref_channels == 1 and wav_channels > 1:
waveform = waveform.mean(dim=0, keepdim=True)
else:
target_channels = min(ref_channels, wav_channels)
reference = reference[:target_channels, :]
waveform = waveform[:target_channels, :]
min_len = min(reference.shape[-1], waveform.shape[-1])
if reference.shape[-1] != min_len:
reference = reference[..., :min_len]
if waveform.shape[-1] != min_len:
waveform = waveform[..., :min_len]
return reference, waveform
def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]:
"""
Run ensemble separation combining multiple models.
This runs multiple models and averages their outputs for best quality.
Note: This takes significantly longer than single-model separation.
Returns:
Dict mapping stem names to output file paths
"""
output_path = Path(output_dir)
output_path.mkdir(parents=True, exist_ok=True)
# Create temporary directory for intermediate results
temp_dir = Path(tempfile.mkdtemp(dir=self.cache_dir))
try:
stems = self.config["stems"]
weighted_sums: Dict[str, torch.Tensor] = {}
total_weights: Dict[str, float] = {}
stem_sample_rates: Dict[str, int] = {}
# Run each model in the ensemble
for model_config in self.config["models"]:
model_type = model_config["type"]
weight = model_config["weight"]
log.info("Running ensemble component: %s (weight: %s)", model_type, weight)
# Run individual model and get stems
model_stems = self._run_model_component(
model_config, input_path, temp_dir
)
# Add to collection for averaging
for stem_name, stem_path in model_stems.items():
if stem_name not in stems:
continue
waveform, sr = self._load_audio(stem_path)
if stem_name not in weighted_sums:
weighted_sums[stem_name] = waveform * weight
total_weights[stem_name] = float(weight)
stem_sample_rates[stem_name] = int(sr)
continue
target_sr = stem_sample_rates[stem_name]
if sr != target_sr:
waveform = torchaudio.functional.resample(waveform, sr, target_sr)
base = weighted_sums[stem_name]
base, waveform = self._align_waveform_shape(base, waveform)
weighted_sums[stem_name] = base + (waveform * weight)
total_weights[stem_name] += float(weight)
del waveform
# Compute weighted averages and save final outputs
output_paths = {}
for stem_name in stems:
if stem_name in weighted_sums and total_weights.get(stem_name, 0) > 0:
averaged = weighted_sums[stem_name] / total_weights[stem_name]
sample_rate = stem_sample_rates.get(stem_name, 44100)
# Save result
output_file = output_path / f"{stem_name}.{self.output_format}"
# Convert to desired format
if self.output_format == "mp3":
# Save as WAV first, then convert
temp_wav = temp_dir / f"{stem_name}_temp.wav"
self._save_audio(averaged, sample_rate, str(temp_wav))
# Convert to MP3
import subprocess
bitrate = self.mp3_bitrate.replace("k", "")
subprocess.run([
"ffmpeg", "-y", "-i", str(temp_wav),
"-b:a", f"{bitrate}k",
"-f", "mp3", str(output_file)
], check=True, capture_output=True)
else:
self._save_audio(averaged, sample_rate, str(output_file))
output_paths[stem_name] = str(output_file)
del averaged
try:
if torch.cuda.is_available():
torch.cuda.empty_cache()
except Exception:
pass
return output_paths
finally:
# Cleanup temp directory
shutil.rmtree(temp_dir, ignore_errors=True)
def _run_model_component(
self,
model_config: Dict,
input_path: str,
output_dir: Path
) -> Dict[str, str]:
"""
Run a single model component and return its output stems.
Returns:
Dict mapping stem names to file paths
"""
model_type = model_config["type"]
if model_type == "bs_roformer":
from .bs_roformer_separator import BSRoformerSeparator
sep = BSRoformerSeparator(
cpu_separation=self.cpu_separation,
output_format="wav", # Use WAV for lossless intermediate
mp3_bitrate="320k",
stem_mode=model_config.get("stem_mode", "4stem"),
)
return sep.separate_into_parts(input_path, str(output_dir / "bsr"))
elif model_type == "bs_roformer_sw":
from .bs_roformer_sw_separator import BSRoformerSWSeparator
sep = BSRoformerSWSeparator(
cpu_separation=self.cpu_separation,
output_format="wav",
mp3_bitrate="320k",
)
return sep.separate_into_parts(input_path, str(output_dir / "bsr_sw"))
elif model_type == "mel_band":
from .mel_band_specialized_separator import MelBandVocalsSeparator
variant = model_config.get("variant", "vocals")
if variant == "vocals":
sep = MelBandVocalsSeparator(
cpu_separation=self.cpu_separation,
output_format="wav",
mp3_bitrate="320k",
)
else:
from .mel_band_roformer_separator import MelBandRoformerSeparator
sep = MelBandRoformerSeparator(
cpu_separation=self.cpu_separation,
output_format="wav",
mp3_bitrate="320k",
)
return sep.separate_into_parts(input_path, str(output_dir / "mbr"))
elif model_type == "scnet_xl_ihf":
from .scnet_separator import SCNetSeparator
sep = SCNetSeparator(
model_id="scnet_xl_ihf",
cpu_separation=self.cpu_separation,
output_format="wav",
mp3_bitrate="320k",
)
return sep.separate_into_parts(input_path, str(output_dir / "scnet"))
elif model_type == "scnet_large":
from .scnet_separator import SCNetSeparator
sep = SCNetSeparator(
model_id="scnet_large",
cpu_separation=self.cpu_separation,
output_format="wav",
mp3_bitrate="320k",
)
return sep.separate_into_parts(input_path, str(output_dir / "scnet"))
elif model_type == "demucs":
# For Demucs, we'd need to run via CLI subprocess
# This is a placeholder
model_name = model_config.get("model", "htdemucs_ft")
# Return empty dict for now - would need actual implementation
return {}
else:
raise ValueError(f"Unknown model type in ensemble: {model_type}")