264 lines
10 KiB
Python
Executable File
264 lines
10 KiB
Python
Executable File
"""
|
|
Ensemble separator - Combines multiple models for best quality separation
|
|
Uses model averaging/ensembling from multiple high-quality models.
|
|
"""
|
|
import os
|
|
import torch
|
|
import torchaudio
|
|
import numpy as np
|
|
from pathlib import Path
|
|
from typing import Dict, List, Optional
|
|
import tempfile
|
|
import shutil
|
|
|
|
from ..config import ENSEMBLE_CACHE_DIR, CPU_SEPARATION
|
|
|
|
|
|
# Ensemble configurations
|
|
ENSEMBLE_CONFIGS = {
|
|
"ensemble_vocal_best": {
|
|
"description": "Best vocal separation using BS-RoFormer x2 + MelBand RoFormer ft + SCNet XL IHF",
|
|
"models": [
|
|
{"type": "bs_roformer", "weight": 0.30, "stem_mode": "4stem"},
|
|
{"type": "mel_band", "weight": 0.25, "variant": "vocals"},
|
|
{"type": "scnet_xl_ihf", "weight": 0.45},
|
|
],
|
|
"stems": ["vocals", "instrumental"],
|
|
},
|
|
"ensemble_4stem": {
|
|
"description": "Best 4-stem separation using multiple models averaged",
|
|
"models": [
|
|
{"type": "demucs", "weight": 0.20, "model": "htdemucs_ft"},
|
|
{"type": "bs_roformer", "weight": 0.30, "stem_mode": "4stem"},
|
|
{"type": "mel_band", "weight": 0.25, "variant": "4stem"},
|
|
{"type": "scnet_large", "weight": 0.25},
|
|
],
|
|
"stems": ["vocals", "drums", "bass", "other"],
|
|
},
|
|
"ensemble_6stem": {
|
|
"description": "Best 6-stem separation using multiple models averaged",
|
|
"models": [
|
|
{"type": "bs_roformer", "weight": 0.35, "stem_mode": "6stem"},
|
|
{"type": "bs_roformer_sw", "weight": 0.35},
|
|
{"type": "demucs", "weight": 0.30, "model": "htdemucs_6s"},
|
|
],
|
|
"stems": ["vocals", "drums", "bass", "guitar", "piano", "other"],
|
|
},
|
|
}
|
|
|
|
|
|
class EnsembleSeparator:
|
|
"""
|
|
Ensemble separator that combines multiple models for best quality.
|
|
|
|
Uses weighted averaging of outputs from multiple separation models
|
|
to produce higher quality results than any single model alone.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
ensemble_id: str = "ensemble_vocal_best",
|
|
cpu_separation: bool = CPU_SEPARATION,
|
|
output_format: str = "mp3",
|
|
mp3_bitrate: str = "256k",
|
|
):
|
|
if ensemble_id not in ENSEMBLE_CONFIGS:
|
|
raise ValueError(f"Unknown ensemble config: {ensemble_id}")
|
|
|
|
self.ensemble_id = ensemble_id
|
|
self.config = ENSEMBLE_CONFIGS[ensemble_id]
|
|
self.cpu_separation = cpu_separation
|
|
self.output_format = output_format
|
|
self.mp3_bitrate = mp3_bitrate
|
|
self.cache_dir = Path(ENSEMBLE_CACHE_DIR)
|
|
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
def _load_audio(self, path: str) -> tuple:
|
|
"""Load audio file and return waveform and sample rate."""
|
|
waveform, sr = torchaudio.load(path)
|
|
return waveform, sr
|
|
|
|
def _save_audio(self, waveform: torch.Tensor, sr: int, path: str):
|
|
"""Save audio waveform to file."""
|
|
torchaudio.save(path, waveform, sr)
|
|
|
|
def _weighted_average(self, audio_tensors: List[torch.Tensor], weights: List[float]) -> torch.Tensor:
|
|
"""Compute weighted average of audio tensors."""
|
|
if len(audio_tensors) != len(weights):
|
|
raise ValueError("Number of tensors must match number of weights")
|
|
|
|
# Normalize weights
|
|
total_weight = sum(weights)
|
|
normalized_weights = [w / total_weight for w in weights]
|
|
|
|
# Compute weighted sum
|
|
result = torch.zeros_like(audio_tensors[0])
|
|
for tensor, weight in zip(audio_tensors, normalized_weights):
|
|
result += tensor * weight
|
|
|
|
return result
|
|
|
|
def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]:
|
|
"""
|
|
Run ensemble separation combining multiple models.
|
|
|
|
This runs multiple models and averages their outputs for best quality.
|
|
Note: This takes significantly longer than single-model separation.
|
|
|
|
Returns:
|
|
Dict mapping stem names to output file paths
|
|
"""
|
|
output_path = Path(output_dir)
|
|
output_path.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Create temporary directory for intermediate results
|
|
temp_dir = Path(tempfile.mkdtemp(dir=self.cache_dir))
|
|
|
|
try:
|
|
stems = self.config["stems"]
|
|
stem_audio_data = {stem: [] for stem in stems}
|
|
stem_weights = {stem: [] for stem in stems}
|
|
|
|
# Run each model in the ensemble
|
|
for model_config in self.config["models"]:
|
|
model_type = model_config["type"]
|
|
weight = model_config["weight"]
|
|
|
|
print(f"Running ensemble component: {model_type} (weight: {weight})")
|
|
|
|
# Run individual model and get stems
|
|
model_stems = self._run_model_component(
|
|
model_config, input_path, temp_dir
|
|
)
|
|
|
|
# Add to collection for averaging
|
|
for stem_name, stem_path in model_stems.items():
|
|
if stem_name in stem_audio_data:
|
|
waveform, sr = self._load_audio(stem_path)
|
|
stem_audio_data[stem_name].append(waveform)
|
|
stem_weights[stem_name].append(weight)
|
|
|
|
# Compute weighted averages and save final outputs
|
|
output_paths = {}
|
|
sample_rate = 44100 # Default, will be updated from actual audio
|
|
|
|
for stem_name in stems:
|
|
if stem_name in stem_audio_data and stem_audio_data[stem_name]:
|
|
# Compute weighted average
|
|
averaged = self._weighted_average(
|
|
stem_audio_data[stem_name],
|
|
stem_weights[stem_name]
|
|
)
|
|
|
|
# Save result
|
|
output_file = output_path / f"{stem_name}.{self.output_format}"
|
|
|
|
# Convert to desired format
|
|
if self.output_format == "mp3":
|
|
# Save as WAV first, then convert
|
|
temp_wav = temp_dir / f"{stem_name}_temp.wav"
|
|
self._save_audio(averaged, sample_rate, str(temp_wav))
|
|
|
|
# Convert to MP3
|
|
import subprocess
|
|
bitrate = self.mp3_bitrate.replace("k", "")
|
|
subprocess.run([
|
|
"ffmpeg", "-y", "-i", str(temp_wav),
|
|
"-b:a", f"{bitrate}k",
|
|
"-f", "mp3", str(output_file)
|
|
], check=True, capture_output=True)
|
|
else:
|
|
self._save_audio(averaged, sample_rate, str(output_file))
|
|
|
|
output_paths[stem_name] = str(output_file)
|
|
|
|
return output_paths
|
|
|
|
finally:
|
|
# Cleanup temp directory
|
|
try:
|
|
shutil.rmtree(temp_dir, ignore_errors=True)
|
|
except:
|
|
pass
|
|
|
|
def _run_model_component(
|
|
self,
|
|
model_config: Dict,
|
|
input_path: str,
|
|
output_dir: Path
|
|
) -> Dict[str, str]:
|
|
"""
|
|
Run a single model component and return its output stems.
|
|
|
|
Returns:
|
|
Dict mapping stem names to file paths
|
|
"""
|
|
model_type = model_config["type"]
|
|
|
|
if model_type == "bs_roformer":
|
|
from .bs_roformer_separator import BSRoformerSeparator
|
|
sep = BSRoformerSeparator(
|
|
cpu_separation=self.cpu_separation,
|
|
output_format="wav", # Use WAV for lossless intermediate
|
|
mp3_bitrate="320k",
|
|
stem_mode=model_config.get("stem_mode", "4stem"),
|
|
)
|
|
return sep.separate_into_parts(input_path, str(output_dir / "bsr"))
|
|
|
|
elif model_type == "bs_roformer_sw":
|
|
from .bs_roformer_sw_separator import BSRoformerSWSeparator
|
|
sep = BSRoformerSWSeparator(
|
|
cpu_separation=self.cpu_separation,
|
|
output_format="wav",
|
|
mp3_bitrate="320k",
|
|
)
|
|
return sep.separate_into_parts(input_path, str(output_dir / "bsr_sw"))
|
|
|
|
elif model_type == "mel_band":
|
|
from .mel_band_specialized_separator import MelBandVocalsSeparator
|
|
variant = model_config.get("variant", "vocals")
|
|
if variant == "vocals":
|
|
sep = MelBandVocalsSeparator(
|
|
cpu_separation=self.cpu_separation,
|
|
output_format="wav",
|
|
mp3_bitrate="320k",
|
|
)
|
|
else:
|
|
from .mel_band_roformer_separator import MelBandRoformerSeparator
|
|
sep = MelBandRoformerSeparator(
|
|
cpu_separation=self.cpu_separation,
|
|
output_format="wav",
|
|
mp3_bitrate="320k",
|
|
)
|
|
return sep.separate_into_parts(input_path, str(output_dir / "mbr"))
|
|
|
|
elif model_type == "scnet_xl_ihf":
|
|
from .scnet_separator import SCNetSeparator
|
|
sep = SCNetSeparator(
|
|
model_id="scnet_xl_ihf",
|
|
cpu_separation=self.cpu_separation,
|
|
output_format="wav",
|
|
mp3_bitrate="320k",
|
|
)
|
|
return sep.separate_into_parts(input_path, str(output_dir / "scnet"))
|
|
|
|
elif model_type == "scnet_large":
|
|
from .scnet_separator import SCNetSeparator
|
|
sep = SCNetSeparator(
|
|
model_id="scnet_large",
|
|
cpu_separation=self.cpu_separation,
|
|
output_format="wav",
|
|
mp3_bitrate="320k",
|
|
)
|
|
return sep.separate_into_parts(input_path, str(output_dir / "scnet"))
|
|
|
|
elif model_type == "demucs":
|
|
# For Demucs, we'd need to run via CLI subprocess
|
|
# This is a placeholder
|
|
model_name = model_config.get("model", "htdemucs_ft")
|
|
# Return empty dict for now - would need actual implementation
|
|
return {}
|
|
|
|
else:
|
|
raise ValueError(f"Unknown model type in ensemble: {model_type}")
|