297 lines
12 KiB
Python
Executable File
297 lines
12 KiB
Python
Executable File
"""
|
|
Ensemble separator - Combines multiple models for best quality separation
|
|
Uses model averaging/ensembling from multiple high-quality models.
|
|
"""
|
|
import os
|
|
import logging
|
|
import torch
|
|
import torchaudio
|
|
from pathlib import Path
|
|
from typing import Dict, List, Optional
|
|
import tempfile
|
|
import shutil
|
|
|
|
from ..config import ENSEMBLE_CACHE_DIR, CPU_SEPARATION
|
|
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
# Ensemble configurations
|
|
ENSEMBLE_CONFIGS = {
|
|
"ensemble_vocal_best": {
|
|
"description": "Best vocal separation using BS-RoFormer x2 + MelBand RoFormer ft + SCNet XL IHF",
|
|
"models": [
|
|
{"type": "bs_roformer", "weight": 0.30, "stem_mode": "4stem"},
|
|
{"type": "mel_band", "weight": 0.25, "variant": "vocals"},
|
|
{"type": "scnet_xl_ihf", "weight": 0.45},
|
|
],
|
|
"stems": ["vocals", "instrumental"],
|
|
},
|
|
"ensemble_4stem": {
|
|
"description": "Best 4-stem separation using multiple models averaged",
|
|
"models": [
|
|
{"type": "demucs", "weight": 0.20, "model": "htdemucs_ft"},
|
|
{"type": "bs_roformer", "weight": 0.30, "stem_mode": "4stem"},
|
|
{"type": "mel_band", "weight": 0.25, "variant": "4stem"},
|
|
{"type": "scnet_large", "weight": 0.25},
|
|
],
|
|
"stems": ["vocals", "drums", "bass", "other"],
|
|
},
|
|
"ensemble_6stem": {
|
|
"description": "Best 6-stem separation using multiple models averaged",
|
|
"models": [
|
|
{"type": "bs_roformer", "weight": 0.35, "stem_mode": "6stem"},
|
|
{"type": "bs_roformer_sw", "weight": 0.35},
|
|
{"type": "demucs", "weight": 0.30, "model": "htdemucs_6s"},
|
|
],
|
|
"stems": ["vocals", "drums", "bass", "guitar", "piano", "other"],
|
|
},
|
|
}
|
|
|
|
|
|
class EnsembleSeparator:
|
|
"""
|
|
Ensemble separator that combines multiple models for best quality.
|
|
|
|
Uses weighted averaging of outputs from multiple separation models
|
|
to produce higher quality results than any single model alone.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
ensemble_id: str = "ensemble_vocal_best",
|
|
cpu_separation: bool = CPU_SEPARATION,
|
|
output_format: str = "mp3",
|
|
mp3_bitrate: str = "256k",
|
|
):
|
|
if ensemble_id not in ENSEMBLE_CONFIGS:
|
|
raise ValueError(f"Unknown ensemble config: {ensemble_id}")
|
|
|
|
self.ensemble_id = ensemble_id
|
|
self.config = ENSEMBLE_CONFIGS[ensemble_id]
|
|
self.cpu_separation = cpu_separation
|
|
self.output_format = output_format
|
|
self.mp3_bitrate = mp3_bitrate
|
|
self.cache_dir = Path(ENSEMBLE_CACHE_DIR)
|
|
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
def _load_audio(self, path: str) -> tuple:
|
|
"""Load audio file and return waveform and sample rate."""
|
|
waveform, sr = torchaudio.load(path)
|
|
return waveform, sr
|
|
|
|
def _save_audio(self, waveform: torch.Tensor, sr: int, path: str):
|
|
"""Save audio waveform to file."""
|
|
torchaudio.save(path, waveform, sr)
|
|
|
|
def _align_waveform_shape(self, reference: torch.Tensor, waveform: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
|
"""Align channel count + length so two tensors can be safely accumulated."""
|
|
if reference.dim() == 1:
|
|
reference = reference.unsqueeze(0)
|
|
if waveform.dim() == 1:
|
|
waveform = waveform.unsqueeze(0)
|
|
|
|
ref_channels = reference.shape[0]
|
|
wav_channels = waveform.shape[0]
|
|
|
|
if wav_channels != ref_channels:
|
|
if wav_channels == 1 and ref_channels > 1:
|
|
waveform = waveform.repeat(ref_channels, 1)
|
|
elif ref_channels == 1 and wav_channels > 1:
|
|
waveform = waveform.mean(dim=0, keepdim=True)
|
|
else:
|
|
target_channels = min(ref_channels, wav_channels)
|
|
reference = reference[:target_channels, :]
|
|
waveform = waveform[:target_channels, :]
|
|
|
|
min_len = min(reference.shape[-1], waveform.shape[-1])
|
|
if reference.shape[-1] != min_len:
|
|
reference = reference[..., :min_len]
|
|
if waveform.shape[-1] != min_len:
|
|
waveform = waveform[..., :min_len]
|
|
|
|
return reference, waveform
|
|
|
|
def separate_into_parts(self, input_path: str, output_dir: str) -> Dict[str, str]:
|
|
"""
|
|
Run ensemble separation combining multiple models.
|
|
|
|
This runs multiple models and averages their outputs for best quality.
|
|
Note: This takes significantly longer than single-model separation.
|
|
|
|
Returns:
|
|
Dict mapping stem names to output file paths
|
|
"""
|
|
output_path = Path(output_dir)
|
|
output_path.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Create temporary directory for intermediate results
|
|
temp_dir = Path(tempfile.mkdtemp(dir=self.cache_dir))
|
|
|
|
try:
|
|
stems = self.config["stems"]
|
|
weighted_sums: Dict[str, torch.Tensor] = {}
|
|
total_weights: Dict[str, float] = {}
|
|
stem_sample_rates: Dict[str, int] = {}
|
|
|
|
# Run each model in the ensemble
|
|
for model_config in self.config["models"]:
|
|
model_type = model_config["type"]
|
|
weight = model_config["weight"]
|
|
|
|
log.info("Running ensemble component: %s (weight: %s)", model_type, weight)
|
|
|
|
# Run individual model and get stems
|
|
model_stems = self._run_model_component(
|
|
model_config, input_path, temp_dir
|
|
)
|
|
|
|
# Add to collection for averaging
|
|
for stem_name, stem_path in model_stems.items():
|
|
if stem_name not in stems:
|
|
continue
|
|
|
|
waveform, sr = self._load_audio(stem_path)
|
|
|
|
if stem_name not in weighted_sums:
|
|
weighted_sums[stem_name] = waveform * weight
|
|
total_weights[stem_name] = float(weight)
|
|
stem_sample_rates[stem_name] = int(sr)
|
|
continue
|
|
|
|
target_sr = stem_sample_rates[stem_name]
|
|
if sr != target_sr:
|
|
waveform = torchaudio.functional.resample(waveform, sr, target_sr)
|
|
|
|
base = weighted_sums[stem_name]
|
|
base, waveform = self._align_waveform_shape(base, waveform)
|
|
weighted_sums[stem_name] = base + (waveform * weight)
|
|
total_weights[stem_name] += float(weight)
|
|
del waveform
|
|
|
|
# Compute weighted averages and save final outputs
|
|
output_paths = {}
|
|
|
|
for stem_name in stems:
|
|
if stem_name in weighted_sums and total_weights.get(stem_name, 0) > 0:
|
|
averaged = weighted_sums[stem_name] / total_weights[stem_name]
|
|
sample_rate = stem_sample_rates.get(stem_name, 44100)
|
|
|
|
# Save result
|
|
output_file = output_path / f"{stem_name}.{self.output_format}"
|
|
|
|
# Convert to desired format
|
|
if self.output_format == "mp3":
|
|
# Save as WAV first, then convert
|
|
temp_wav = temp_dir / f"{stem_name}_temp.wav"
|
|
self._save_audio(averaged, sample_rate, str(temp_wav))
|
|
|
|
# Convert to MP3
|
|
import subprocess
|
|
bitrate = self.mp3_bitrate.replace("k", "")
|
|
subprocess.run([
|
|
"ffmpeg", "-y", "-i", str(temp_wav),
|
|
"-b:a", f"{bitrate}k",
|
|
"-f", "mp3", str(output_file)
|
|
], check=True, capture_output=True)
|
|
else:
|
|
self._save_audio(averaged, sample_rate, str(output_file))
|
|
|
|
output_paths[stem_name] = str(output_file)
|
|
|
|
del averaged
|
|
|
|
try:
|
|
if torch.cuda.is_available():
|
|
torch.cuda.empty_cache()
|
|
except Exception:
|
|
pass
|
|
|
|
return output_paths
|
|
|
|
finally:
|
|
# Cleanup temp directory
|
|
shutil.rmtree(temp_dir, ignore_errors=True)
|
|
|
|
def _run_model_component(
|
|
self,
|
|
model_config: Dict,
|
|
input_path: str,
|
|
output_dir: Path
|
|
) -> Dict[str, str]:
|
|
"""
|
|
Run a single model component and return its output stems.
|
|
|
|
Returns:
|
|
Dict mapping stem names to file paths
|
|
"""
|
|
model_type = model_config["type"]
|
|
|
|
if model_type == "bs_roformer":
|
|
from .bs_roformer_separator import BSRoformerSeparator
|
|
sep = BSRoformerSeparator(
|
|
cpu_separation=self.cpu_separation,
|
|
output_format="wav", # Use WAV for lossless intermediate
|
|
mp3_bitrate="320k",
|
|
stem_mode=model_config.get("stem_mode", "4stem"),
|
|
)
|
|
return sep.separate_into_parts(input_path, str(output_dir / "bsr"))
|
|
|
|
elif model_type == "bs_roformer_sw":
|
|
from .bs_roformer_sw_separator import BSRoformerSWSeparator
|
|
sep = BSRoformerSWSeparator(
|
|
cpu_separation=self.cpu_separation,
|
|
output_format="wav",
|
|
mp3_bitrate="320k",
|
|
)
|
|
return sep.separate_into_parts(input_path, str(output_dir / "bsr_sw"))
|
|
|
|
elif model_type == "mel_band":
|
|
from .mel_band_specialized_separator import MelBandVocalsSeparator
|
|
variant = model_config.get("variant", "vocals")
|
|
if variant == "vocals":
|
|
sep = MelBandVocalsSeparator(
|
|
cpu_separation=self.cpu_separation,
|
|
output_format="wav",
|
|
mp3_bitrate="320k",
|
|
)
|
|
else:
|
|
from .mel_band_roformer_separator import MelBandRoformerSeparator
|
|
sep = MelBandRoformerSeparator(
|
|
cpu_separation=self.cpu_separation,
|
|
output_format="wav",
|
|
mp3_bitrate="320k",
|
|
)
|
|
return sep.separate_into_parts(input_path, str(output_dir / "mbr"))
|
|
|
|
elif model_type == "scnet_xl_ihf":
|
|
from .scnet_separator import SCNetSeparator
|
|
sep = SCNetSeparator(
|
|
model_id="scnet_xl_ihf",
|
|
cpu_separation=self.cpu_separation,
|
|
output_format="wav",
|
|
mp3_bitrate="320k",
|
|
)
|
|
return sep.separate_into_parts(input_path, str(output_dir / "scnet"))
|
|
|
|
elif model_type == "scnet_large":
|
|
from .scnet_separator import SCNetSeparator
|
|
sep = SCNetSeparator(
|
|
model_id="scnet_large",
|
|
cpu_separation=self.cpu_separation,
|
|
output_format="wav",
|
|
mp3_bitrate="320k",
|
|
)
|
|
return sep.separate_into_parts(input_path, str(output_dir / "scnet"))
|
|
|
|
elif model_type == "demucs":
|
|
# For Demucs, we'd need to run via CLI subprocess
|
|
# This is a placeholder
|
|
model_name = model_config.get("model", "htdemucs_ft")
|
|
# Return empty dict for now - would need actual implementation
|
|
return {}
|
|
|
|
else:
|
|
raise ValueError(f"Unknown model type in ensemble: {model_type}")
|