Files
stemengine_old/backend/services/ensemble_separator.py
T
2026-03-03 14:52:19 +09:00

264 lines
10 KiB
Python
Executable File

"""
Ensemble separator - Combines multiple models for best quality separation
Uses model averaging/ensembling from multiple high-quality models.
"""
import os
import torch
import torchaudio
import numpy as np
from pathlib import Path
from typing import Dict, List, Optional
import tempfile
import shutil
from ..config import ENSEMBLE_CACHE_DIR, CPU_SEPARATION
# Ensemble configurations
ENSEMBLE_CONFIGS = {
"ensemble_vocal_best": {
"description": "Best vocal separation using BS-RoFormer x2 + MelBand RoFormer ft + SCNet XL IHF",
"models": [
{"type": "bs_roformer", "weight": 0.30, "stem_mode": "4stem"},
{"type": "mel_band", "weight": 0.25, "variant": "vocals"},
{"type": "scnet_xl_ihf", "weight": 0.45},
],
"stems": ["vocals", "instrumental"],
},
"ensemble_4stem": {
"description": "Best 4-stem separation using multiple models averaged",
"models": [
{"type": "demucs", "weight": 0.20, "model": "htdemucs_ft"},
{"type": "bs_roformer", "weight": 0.30, "stem_mode": "4stem"},
{"type": "mel_band", "weight": 0.25, "variant": "4stem"},
{"type": "scnet_large", "weight": 0.25},
],
"stems": ["vocals", "drums", "bass", "other"],
},
"ensemble_6stem": {
"description": "Best 6-stem separation using multiple models averaged",
"models": [
{"type": "bs_roformer", "weight": 0.35, "stem_mode": "6stem"},
{"type": "bs_roformer_sw", "weight": 0.35},
{"type": "demucs", "weight": 0.30, "model": "htdemucs_6s"},
],
"stems": ["vocals", "drums", "bass", "guitar", "piano", "other"],
},
}
class EnsembleSeparator:
"""
Ensemble separator that combines multiple models for best quality.
Uses weighted averaging of outputs from multiple separation models
to produce higher quality results than any single model alone.
"""
def __init__(
self,
ensemble_id: str = "ensemble_vocal_best",
cpu_separation: bool = CPU_SEPARATION,
output_format: str = "mp3",
mp3_bitrate: str = "256k",
):
if ensemble_id not in ENSEMBLE_CONFIGS:
raise ValueError(f"Unknown ensemble config: {ensemble_id}")
self.ensemble_id = ensemble_id
self.config = ENSEMBLE_CONFIGS[ensemble_id]
self.cpu_separation = cpu_separation
self.output_format = output_format
self.mp3_bitrate = mp3_bitrate
self.cache_dir = Path(ENSEMBLE_CACHE_DIR)
self.cache_dir.mkdir(parents=True, exist_ok=True)
def _load_audio(self, path: str) -> tuple:
"""Load audio file and return waveform and sample rate."""
waveform, sr = torchaudio.load(path)
return waveform, sr
def _save_audio(self, waveform: torch.Tensor, sr: int, path: str):
"""Save audio waveform to file."""
torchaudio.save(path, waveform, sr)
def _weighted_average(self, audio_tensors: List[torch.Tensor], weights: List[float]) -> torch.Tensor:
"""Compute weighted average of audio tensors."""
if len(audio_tensors) != len(weights):
raise ValueError("Number of tensors must match number of weights")
# Normalize weights
total_weight = sum(weights)
normalized_weights = [w / total_weight for w in weights]
# Compute weighted sum
result = torch.zeros_like(audio_tensors[0])
for tensor, weight in zip(audio_tensors, normalized_weights):
result += tensor * weight
return result
def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]:
"""
Run ensemble separation combining multiple models.
This runs multiple models and averages their outputs for best quality.
Note: This takes significantly longer than single-model separation.
Returns:
Dict mapping stem names to output file paths
"""
output_path = Path(output_dir)
output_path.mkdir(parents=True, exist_ok=True)
# Create temporary directory for intermediate results
temp_dir = Path(tempfile.mkdtemp(dir=self.cache_dir))
try:
stems = self.config["stems"]
stem_audio_data = {stem: [] for stem in stems}
stem_weights = {stem: [] for stem in stems}
# Run each model in the ensemble
for model_config in self.config["models"]:
model_type = model_config["type"]
weight = model_config["weight"]
print(f"Running ensemble component: {model_type} (weight: {weight})")
# Run individual model and get stems
model_stems = self._run_model_component(
model_config, input_path, temp_dir
)
# Add to collection for averaging
for stem_name, stem_path in model_stems.items():
if stem_name in stem_audio_data:
waveform, sr = self._load_audio(stem_path)
stem_audio_data[stem_name].append(waveform)
stem_weights[stem_name].append(weight)
# Compute weighted averages and save final outputs
output_paths = {}
sample_rate = 44100 # Default, will be updated from actual audio
for stem_name in stems:
if stem_name in stem_audio_data and stem_audio_data[stem_name]:
# Compute weighted average
averaged = self._weighted_average(
stem_audio_data[stem_name],
stem_weights[stem_name]
)
# Save result
output_file = output_path / f"{stem_name}.{self.output_format}"
# Convert to desired format
if self.output_format == "mp3":
# Save as WAV first, then convert
temp_wav = temp_dir / f"{stem_name}_temp.wav"
self._save_audio(averaged, sample_rate, str(temp_wav))
# Convert to MP3
import subprocess
bitrate = self.mp3_bitrate.replace("k", "")
subprocess.run([
"ffmpeg", "-y", "-i", str(temp_wav),
"-b:a", f"{bitrate}k",
"-f", "mp3", str(output_file)
], check=True, capture_output=True)
else:
self._save_audio(averaged, sample_rate, str(output_file))
output_paths[stem_name] = str(output_file)
return output_paths
finally:
# Cleanup temp directory
try:
shutil.rmtree(temp_dir, ignore_errors=True)
except:
pass
def _run_model_component(
self,
model_config: Dict,
input_path: str,
output_dir: Path
) -> Dict[str, str]:
"""
Run a single model component and return its output stems.
Returns:
Dict mapping stem names to file paths
"""
model_type = model_config["type"]
if model_type == "bs_roformer":
from .bs_roformer_separator import BSRoformerSeparator
sep = BSRoformerSeparator(
cpu_separation=self.cpu_separation,
output_format="wav", # Use WAV for lossless intermediate
mp3_bitrate="320k",
stem_mode=model_config.get("stem_mode", "4stem"),
)
return sep.separate_into_parts(input_path, str(output_dir / "bsr"))
elif model_type == "bs_roformer_sw":
from .bs_roformer_sw_separator import BSRoformerSWSeparator
sep = BSRoformerSWSeparator(
cpu_separation=self.cpu_separation,
output_format="wav",
mp3_bitrate="320k",
)
return sep.separate_into_parts(input_path, str(output_dir / "bsr_sw"))
elif model_type == "mel_band":
from .mel_band_specialized_separator import MelBandVocalsSeparator
variant = model_config.get("variant", "vocals")
if variant == "vocals":
sep = MelBandVocalsSeparator(
cpu_separation=self.cpu_separation,
output_format="wav",
mp3_bitrate="320k",
)
else:
from .mel_band_roformer_separator import MelBandRoformerSeparator
sep = MelBandRoformerSeparator(
cpu_separation=self.cpu_separation,
output_format="wav",
mp3_bitrate="320k",
)
return sep.separate_into_parts(input_path, str(output_dir / "mbr"))
elif model_type == "scnet_xl_ihf":
from .scnet_separator import SCNetSeparator
sep = SCNetSeparator(
model_id="scnet_xl_ihf",
cpu_separation=self.cpu_separation,
output_format="wav",
mp3_bitrate="320k",
)
return sep.separate_into_parts(input_path, str(output_dir / "scnet"))
elif model_type == "scnet_large":
from .scnet_separator import SCNetSeparator
sep = SCNetSeparator(
model_id="scnet_large",
cpu_separation=self.cpu_separation,
output_format="wav",
mp3_bitrate="320k",
)
return sep.separate_into_parts(input_path, str(output_dir / "scnet"))
elif model_type == "demucs":
# For Demucs, we'd need to run via CLI subprocess
# This is a placeholder
model_name = model_config.get("model", "htdemucs_ft")
# Return empty dict for now - would need actual implementation
return {}
else:
raise ValueError(f"Unknown model type in ensemble: {model_type}")