""" Ensemble separator - Combines multiple models for best quality separation Uses model averaging/ensembling from multiple high-quality models. """ import os import torch import torchaudio import numpy as np from pathlib import Path from typing import Dict, List, Optional import tempfile import shutil from ..config import ENSEMBLE_CACHE_DIR, CPU_SEPARATION # Ensemble configurations ENSEMBLE_CONFIGS = { "ensemble_vocal_best": { "description": "Best vocal separation using BS-RoFormer x2 + MelBand RoFormer ft + SCNet XL IHF", "models": [ {"type": "bs_roformer", "weight": 0.30, "stem_mode": "4stem"}, {"type": "mel_band", "weight": 0.25, "variant": "vocals"}, {"type": "scnet_xl_ihf", "weight": 0.45}, ], "stems": ["vocals", "instrumental"], }, "ensemble_4stem": { "description": "Best 4-stem separation using multiple models averaged", "models": [ {"type": "demucs", "weight": 0.20, "model": "htdemucs_ft"}, {"type": "bs_roformer", "weight": 0.30, "stem_mode": "4stem"}, {"type": "mel_band", "weight": 0.25, "variant": "4stem"}, {"type": "scnet_large", "weight": 0.25}, ], "stems": ["vocals", "drums", "bass", "other"], }, "ensemble_6stem": { "description": "Best 6-stem separation using multiple models averaged", "models": [ {"type": "bs_roformer", "weight": 0.35, "stem_mode": "6stem"}, {"type": "bs_roformer_sw", "weight": 0.35}, {"type": "demucs", "weight": 0.30, "model": "htdemucs_6s"}, ], "stems": ["vocals", "drums", "bass", "guitar", "piano", "other"], }, } class EnsembleSeparator: """ Ensemble separator that combines multiple models for best quality. Uses weighted averaging of outputs from multiple separation models to produce higher quality results than any single model alone. """ def __init__( self, ensemble_id: str = "ensemble_vocal_best", cpu_separation: bool = CPU_SEPARATION, output_format: str = "mp3", mp3_bitrate: str = "256k", ): if ensemble_id not in ENSEMBLE_CONFIGS: raise ValueError(f"Unknown ensemble config: {ensemble_id}") self.ensemble_id = ensemble_id self.config = ENSEMBLE_CONFIGS[ensemble_id] self.cpu_separation = cpu_separation self.output_format = output_format self.mp3_bitrate = mp3_bitrate self.cache_dir = Path(ENSEMBLE_CACHE_DIR) self.cache_dir.mkdir(parents=True, exist_ok=True) def _load_audio(self, path: str) -> tuple: """Load audio file and return waveform and sample rate.""" waveform, sr = torchaudio.load(path) return waveform, sr def _save_audio(self, waveform: torch.Tensor, sr: int, path: str): """Save audio waveform to file.""" torchaudio.save(path, waveform, sr) def _weighted_average(self, audio_tensors: List[torch.Tensor], weights: List[float]) -> torch.Tensor: """Compute weighted average of audio tensors.""" if len(audio_tensors) != len(weights): raise ValueError("Number of tensors must match number of weights") # Normalize weights total_weight = sum(weights) normalized_weights = [w / total_weight for w in weights] # Compute weighted sum result = torch.zeros_like(audio_tensors[0]) for tensor, weight in zip(audio_tensors, normalized_weights): result += tensor * weight return result def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]: """ Run ensemble separation combining multiple models. This runs multiple models and averages their outputs for best quality. Note: This takes significantly longer than single-model separation. Returns: Dict mapping stem names to output file paths """ output_path = Path(output_dir) output_path.mkdir(parents=True, exist_ok=True) # Create temporary directory for intermediate results temp_dir = Path(tempfile.mkdtemp(dir=self.cache_dir)) try: stems = self.config["stems"] stem_audio_data = {stem: [] for stem in stems} stem_weights = {stem: [] for stem in stems} # Run each model in the ensemble for model_config in self.config["models"]: model_type = model_config["type"] weight = model_config["weight"] print(f"Running ensemble component: {model_type} (weight: {weight})") # Run individual model and get stems model_stems = self._run_model_component( model_config, input_path, temp_dir ) # Add to collection for averaging for stem_name, stem_path in model_stems.items(): if stem_name in stem_audio_data: waveform, sr = self._load_audio(stem_path) stem_audio_data[stem_name].append(waveform) stem_weights[stem_name].append(weight) # Compute weighted averages and save final outputs output_paths = {} sample_rate = 44100 # Default, will be updated from actual audio for stem_name in stems: if stem_name in stem_audio_data and stem_audio_data[stem_name]: # Compute weighted average averaged = self._weighted_average( stem_audio_data[stem_name], stem_weights[stem_name] ) # Save result output_file = output_path / f"{stem_name}.{self.output_format}" # Convert to desired format if self.output_format == "mp3": # Save as WAV first, then convert temp_wav = temp_dir / f"{stem_name}_temp.wav" self._save_audio(averaged, sample_rate, str(temp_wav)) # Convert to MP3 import subprocess bitrate = self.mp3_bitrate.replace("k", "") subprocess.run([ "ffmpeg", "-y", "-i", str(temp_wav), "-b:a", f"{bitrate}k", "-f", "mp3", str(output_file) ], check=True, capture_output=True) else: self._save_audio(averaged, sample_rate, str(output_file)) output_paths[stem_name] = str(output_file) return output_paths finally: # Cleanup temp directory try: shutil.rmtree(temp_dir, ignore_errors=True) except: pass def _run_model_component( self, model_config: Dict, input_path: str, output_dir: Path ) -> Dict[str, str]: """ Run a single model component and return its output stems. Returns: Dict mapping stem names to file paths """ model_type = model_config["type"] if model_type == "bs_roformer": from .bs_roformer_separator import BSRoformerSeparator sep = BSRoformerSeparator( cpu_separation=self.cpu_separation, output_format="wav", # Use WAV for lossless intermediate mp3_bitrate="320k", stem_mode=model_config.get("stem_mode", "4stem"), ) return sep.separate_into_parts(input_path, str(output_dir / "bsr")) elif model_type == "bs_roformer_sw": from .bs_roformer_sw_separator import BSRoformerSWSeparator sep = BSRoformerSWSeparator( cpu_separation=self.cpu_separation, output_format="wav", mp3_bitrate="320k", ) return sep.separate_into_parts(input_path, str(output_dir / "bsr_sw")) elif model_type == "mel_band": from .mel_band_specialized_separator import MelBandVocalsSeparator variant = model_config.get("variant", "vocals") if variant == "vocals": sep = MelBandVocalsSeparator( cpu_separation=self.cpu_separation, output_format="wav", mp3_bitrate="320k", ) else: from .mel_band_roformer_separator import MelBandRoformerSeparator sep = MelBandRoformerSeparator( cpu_separation=self.cpu_separation, output_format="wav", mp3_bitrate="320k", ) return sep.separate_into_parts(input_path, str(output_dir / "mbr")) elif model_type == "scnet_xl_ihf": from .scnet_separator import SCNetSeparator sep = SCNetSeparator( model_id="scnet_xl_ihf", cpu_separation=self.cpu_separation, output_format="wav", mp3_bitrate="320k", ) return sep.separate_into_parts(input_path, str(output_dir / "scnet")) elif model_type == "scnet_large": from .scnet_separator import SCNetSeparator sep = SCNetSeparator( model_id="scnet_large", cpu_separation=self.cpu_separation, output_format="wav", mp3_bitrate="320k", ) return sep.separate_into_parts(input_path, str(output_dir / "scnet")) elif model_type == "demucs": # For Demucs, we'd need to run via CLI subprocess # This is a placeholder model_name = model_config.get("model", "htdemucs_ft") # Return empty dict for now - would need actual implementation return {} else: raise ValueError(f"Unknown model type in ensemble: {model_type}")