Files
stemengine_old/backend/services/separator_service.py
T
2026-04-14 16:32:36 +09:00

3875 lines
141 KiB
Python
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
Stem separation service with project-based folder structure and preset mixing.
- Demucs models → CLI subprocess (python -m demucs)
- BS-RoFormer → Python API (in-process, no subprocess needed)
- Presets → Instrumental, Static, Dynamic mixing via preset_service
"""
import logging
import os
import select
import subprocess
import sys
import time
import json
import shutil
from copy import deepcopy
from pathlib import Path
from typing import Dict, List, Optional
from .. import job_manager as jm
from ..config import (
PROJECTS_DIR,
JOBS_DIR,
CPU_SEPARATION,
MELBAND_STAGE_DIAGNOSTICS,
PROJECT_SUBFOLDERS,
FALLBACK_REQUIRE_LEAD_BACK,
LONG_TRACK_DURATION_SECS,
LONG_TRACK_CHUNKING_MODE,
LONG_TRACK_CHUNK_DURATION_SECS,
LONG_TRACK_CHUNK_OVERLAP_SECS,
DEMUCS_SEGMENT_OVERRIDE,
DEMUCS_OVERLAP_OVERRIDE,
MELBAND_DIM_T_OVERRIDE,
GABOX_SMALL_MODEL_MAX_DURATION_SECS,
BS_ROFORMER_MODEL_DIR,
)
from .metadata_util import copy_metadata, read_metadata
from .project_namer import project_name, make_project_dirs
from .project_metadata import write_project_metadata
from .audio_analysis_service import analyze_audio_features
from .project_metadata import patch_project_metadata_audio_features
from .separation.model_downloads import ensure_model_available
from .separation.model_registry import (
APOLLO_MODEL_IDS,
ASPIRATION_MODEL_IDS,
BANDIT_MODEL_IDS,
BSR_ANVUEW_MODELS,
BSR_ADAPTER_MODELS,
BSR_CHORUS_MODELS,
BSR_KARAOKE_SPECIAL_MODELS,
BSR_PREFIX,
BSR_STEM_MODE,
BSR_SW_MODEL_IDS,
ENSEMBLE_MODEL_IDS,
FORMAT_LABELS,
KARAOKE_OOM_FALLBACK_CHAIN,
MBR_DEUX_BECRUILY_MODELS,
MBR_DEREVERB_MODELS,
MBR_DEREVERB_PREFIX,
MBR_KARAOKE_MODELS,
MBR_KARAOKE_PREFIX,
MBR_MODELS,
MBR_PREFIX,
MBR_VOCALS_MODELS,
MBR_VOCALS_PREFIX,
MDXNET_MODEL_IDS,
MEDLEYVOX_MODEL_IDS,
MODEL_LABELS,
SCNET_MODEL_IDS,
SEPACAP_MODEL_IDS,
SEPREFORMER_MODEL_IDS,
is_lead_back_capable,
)
from .separation.output_layout import (
next_stem_label,
normalise_stem_label,
parse_stem_label_from_filename,
stem_filename,
)
from .separation.power_presets import (
apply_global_power_policy_overrides,
build_global_power_policy,
get_power_opts,
)
from .separation.execution import flush_vram_cache, move_stems_to_project, oom_retry_call, cleanup_temp, get_audio_duration_secs
from .separation.chunking import extract_chunk, overlap_add_reconstruct, plan_chunks, should_chunk
from .separation.postprocess import apply_presets
from .stem_finalize_service import finalize_generated_stems
from .download_utils import download_file_atomic
log = logging.getLogger(__name__)
_AUDIO_EXTS = {".mp3", ".wav", ".flac", ".m4a", ".ogg"}
_BS_ADAPTER_ASSETS = {
"bs_mamba2_v1": {
"model_file": "bs_mamba2_vocals.ckpt",
"config_file": "config_bs_mamba2_vocals.yaml",
"model_url": "https://github.com/ZFTurbo/Music-Source-Separation-Training/releases/download/v1.0.19/bs_mamba2_vocals.ckpt",
"config_url": "https://github.com/ZFTurbo/Music-Source-Separation-Training/releases/download/v1.0.19/config_bs_mamba2_vocals.yaml",
},
"bs_polarformer_v1": {
"model_file": "model_bs_polarformer_float16.ckpt",
"config_file": "model_bs_polarformer_float16.yaml",
"model_url": "https://github.com/ZFTurbo/Music-Source-Separation-Training/releases/download/v1.0.20/model_bs_polarformer_float16.ckpt",
"config_url": "https://github.com/ZFTurbo/Music-Source-Separation-Training/releases/download/v1.0.20/model_bs_polarformer_float16.yaml",
},
}
def _ensure_bs_adapter_assets(model_id: str) -> tuple[Path, Path]:
spec = _BS_ADAPTER_ASSETS.get(model_id)
if not spec:
raise RuntimeError(f"No BS adapter asset spec found for model '{model_id}'")
model_dir = BS_ROFORMER_MODEL_DIR / model_id
model_dir.mkdir(parents=True, exist_ok=True)
model_path = model_dir / spec["model_file"]
config_path = model_dir / spec["config_file"]
if not model_path.exists():
log.info("Downloading %s weights from %s", model_id, spec["model_url"])
download_file_atomic(spec["model_url"], model_path, timeout=180)
if not config_path.exists():
log.info("Downloading %s config from %s", model_id, spec["config_url"])
download_file_atomic(spec["config_url"], config_path, timeout=180)
return model_path, config_path
def _diag_progress_cb(job_id: str):
def _emit(message: str) -> None:
jm.update_job(job_id, progress=message)
return _emit
def _effective_chunking_params(
mode_override: Optional[str],
duration_override: Optional[int],
overlap_override: Optional[int],
) -> tuple[str, int, int]:
mode = (mode_override or LONG_TRACK_CHUNKING_MODE or "auto").strip().lower()
if mode not in {"auto", "always", "off"}:
mode = LONG_TRACK_CHUNKING_MODE
duration = int(duration_override) if duration_override is not None else int(LONG_TRACK_CHUNK_DURATION_SECS)
overlap = int(overlap_override) if overlap_override is not None else int(LONG_TRACK_CHUNK_OVERLAP_SECS)
return mode, max(1, duration), max(0, overlap)
def _init_melband_separator_compat(sep_class, **kwargs):
"""Instantiate MelBand separator with backward-compatible kwargs."""
try:
return sep_class(**kwargs)
except TypeError as exc:
_msg = str(exc)
if "unexpected keyword argument 'dim_t_override'" in _msg:
kwargs.pop("dim_t_override", None)
kwargs.pop("segment_secs_override", None)
return sep_class(**kwargs)
if "unexpected keyword argument 'segment_secs_override'" in _msg:
kwargs.pop("segment_secs_override", None)
return sep_class(**kwargs)
raise
def _save_stem_audio(path: str, data_cs, sample_rate: int, output_format: str, bitrate: str = '256k') -> None:
"""Save stem audio array ``(channels, samples)`` to the target format."""
import tempfile
import soundfile as sf
ext = (output_format or "wav").lower()
data_sc = data_cs.T
if ext in ('wav', 'flac', 'ogg'):
fmt = 'OGG' if ext == 'ogg' else ext.upper()
sf.write(path, data_sc, sample_rate, format=fmt, subtype='PCM_16')
return
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
tmp_path = tmp.name
try:
sf.write(tmp_path, data_sc, sample_rate, format='WAV', subtype='PCM_16')
cmd = ['ffmpeg', '-y', '-i', tmp_path]
if ext == 'mp3':
cmd.extend(['-b:a', bitrate])
cmd.append(path)
subprocess.run(cmd, check=True, capture_output=True)
finally:
try:
os.remove(tmp_path)
except Exception:
pass
def _separate_into_parts_chunked_via_files(
*,
job_id: str,
input_path: str,
sep,
temp_out: Path,
chunk_duration_secs: int,
chunk_overlap_secs: int,
output_format: str,
mp3_bitrate: str,
) -> Dict[str, str]:
"""Chunk-adapter for separators that only expose file-based `separate_into_parts()`.
Uses time-domain chunking + overlap-add reconstruction to provide a safe,
family-agnostic segment policy without requiring model-native chunk APIs.
"""
import tempfile
import numpy as np
import soundfile as sf
def _read_audio_with_fallback(path: str):
try:
return sf.read(path, always_2d=True, dtype='float32')
except Exception:
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
tmp_wav = tmp.name
try:
subprocess.run(
['ffmpeg', '-y', '-i', path, '-f', 'wav', tmp_wav],
check=True,
capture_output=True,
)
return sf.read(tmp_wav, always_2d=True, dtype='float32')
finally:
try:
os.remove(tmp_wav)
except Exception:
pass
audio_sc, sample_rate = _read_audio_with_fallback(input_path)
if audio_sc.size == 0:
raise RuntimeError("Input audio is empty")
audio_cs = audio_sc.T
chunk_plan = plan_chunks(
total_samples=audio_cs.shape[-1],
sample_rate=sample_rate,
chunk_secs=float(chunk_duration_secs),
overlap_secs=float(chunk_overlap_secs),
)
total_chunks = len(chunk_plan.chunks)
jm.update_job(
job_id,
progress=(
f"Chunked mode: {total_chunks} chunk(s) × {chunk_duration_secs} s "
f"with {chunk_overlap_secs} s crossfade overlap..."
),
)
work_dir = temp_out / "chunk_adapter"
work_dir.mkdir(parents=True, exist_ok=True)
chunk_outputs: List[Dict[str, object]] = []
stem_names: Optional[List[str]] = None
channels = int(audio_cs.shape[0])
target_len = int(chunk_plan.chunk_samples)
for idx, (start, end) in enumerate(chunk_plan.chunks, start=1):
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
jm.update_job(job_id, progress=f"Processing chunk {idx}/{total_chunks}...")
chunk_cs = extract_chunk(audio_cs, start, end)
chunk_input = work_dir / f"chunk_{idx:04d}.wav"
sf.write(str(chunk_input), chunk_cs.T, sample_rate, format='WAV', subtype='PCM_16')
chunk_out_dir = work_dir / f"chunk_out_{idx:04d}"
chunk_out_dir.mkdir(parents=True, exist_ok=True)
chunk_paths = sep.separate_into_parts(str(chunk_input), str(chunk_out_dir))
if not chunk_paths:
raise RuntimeError(f"Chunk {idx} returned no stems")
current_stems = sorted(chunk_paths.keys())
if stem_names is None:
stem_names = current_stems
chunk_stems: Dict[str, object] = {}
for stem in stem_names:
stem_path = chunk_paths.get(stem)
if not stem_path:
chunk_stems[stem] = np.zeros((channels, target_len), dtype=np.float32)
continue
stem_sc, stem_sr = _read_audio_with_fallback(stem_path)
if stem_sr != sample_rate:
import librosa
stem_cs_resample = stem_sc.T
stem_cs_resample = np.array([
librosa.resample(ch, orig_sr=stem_sr, target_sr=sample_rate)
for ch in stem_cs_resample
], dtype=np.float32)
stem_sc = stem_cs_resample.T
stem_cs = stem_sc.T
if stem_cs.shape[0] == 1 and channels == 2:
stem_cs = np.vstack([stem_cs, stem_cs])
elif stem_cs.shape[0] != channels:
stem_cs = stem_cs[:channels, :]
cur_len = int(stem_cs.shape[-1])
if cur_len < target_len:
stem_cs = np.pad(stem_cs, ((0, 0), (0, target_len - cur_len)), mode='constant')
elif cur_len > target_len:
stem_cs = stem_cs[:, :target_len]
chunk_stems[stem] = stem_cs.astype(np.float32, copy=False)
chunk_outputs.append(chunk_stems)
if not stem_names:
raise RuntimeError("No stems were produced in chunked adapter path")
reconstructed = overlap_add_reconstruct(chunk_outputs, chunk_plan, stem_names)
ext = (output_format or 'wav').lower()
out_paths: Dict[str, str] = {}
for stem in stem_names:
out_path = temp_out / f"{stem}.{ext}"
_save_stem_audio(str(out_path), reconstructed[stem], sample_rate, ext, mp3_bitrate)
out_paths[stem] = str(out_path)
try:
shutil.rmtree(work_dir, ignore_errors=True)
except Exception:
pass
return out_paths
def _build_separation_diagnostics(
model: str,
duration_secs: float,
*,
chunking_mode: str,
chunk_duration_secs: int,
chunk_overlap_secs: int,
power_opts: Dict[str, Dict[str, object]],
global_power_policy: Dict[str, object],
) -> Dict[str, object]:
"""Return standardized diagnostics metadata for separation jobs."""
roformer_chunk_models = set(MBR_MODELS) | set(BSR_STEM_MODE) | set(BSR_KARAOKE_SPECIAL_MODELS) | set(BSR_ADAPTER_MODELS)
mdx_scnet_chunk_models = set(MDXNET_MODEL_IDS) | set(SCNET_MODEL_IDS)
file_chunk_adapter_models = set(APOLLO_MODEL_IDS) | set(ASPIRATION_MODEL_IDS) | set(BANDIT_MODEL_IDS)
demucs_models = {"htdemucs", "htdemucs_ft", "mdx", "mdx_extra_q"}
if model in roformer_chunk_models:
family = "roformer"
elif model in MDXNET_MODEL_IDS:
family = "mdxnet"
elif model in SCNET_MODEL_IDS:
family = "scnet"
elif model in APOLLO_MODEL_IDS:
family = "apollo"
elif model in ASPIRATION_MODEL_IDS:
family = "aspiration"
elif model in BANDIT_MODEL_IDS:
family = "bandit"
elif model in demucs_models:
family = "demucs"
elif model in MEDLEYVOX_MODEL_IDS:
family = "medleyvox"
elif model in SEPREFORMER_MODEL_IDS:
family = "sepreformer"
else:
family = "other"
chunking_supported = model in (roformer_chunk_models | mdx_scnet_chunk_models | file_chunk_adapter_models)
chunking_active = bool(
chunking_supported and should_chunk(duration_secs, chunking_mode, LONG_TRACK_DURATION_SECS)
)
demucs_auto_segment = None
if family == "demucs":
demucs_segment_pref = power_opts.get("demucs", {}).get("segment")
if demucs_segment_pref is not None:
demucs_auto_segment = int(demucs_segment_pref)
elif DEMUCS_SEGMENT_OVERRIDE is not None:
demucs_auto_segment = DEMUCS_SEGMENT_OVERRIDE
elif duration_secs < 120:
demucs_auto_segment = 12
elif duration_secs < 480:
demucs_auto_segment = 8
else:
demucs_auto_segment = 4
return {
"track_duration_secs": round(float(duration_secs), 3),
"family": family,
"chunking": {
"supported": chunking_supported,
"mode": chunking_mode,
"threshold_secs": LONG_TRACK_DURATION_SECS,
"active": chunking_active,
"chunk_duration_secs": chunk_duration_secs,
"chunk_overlap_secs": chunk_overlap_secs,
},
"demucs_tuning": {
"segment_override": power_opts.get("demucs", {}).get("segment") if power_opts.get("demucs", {}).get("segment") is not None else DEMUCS_SEGMENT_OVERRIDE,
"overlap_override": power_opts.get("demucs", {}).get("overlap") if power_opts.get("demucs", {}).get("overlap") is not None else DEMUCS_OVERLAP_OVERRIDE,
"effective_segment_secs": demucs_auto_segment,
},
"effective_power_policy": global_power_policy,
}
def _stem_filename(prefix: str, stem_name: str, suffix: str) -> str:
"""Compatibility wrapper around shared `stem_filename()` helper."""
return stem_filename(prefix, stem_name, suffix)
def _existing_project_stem_labels(project_dir: Path, project_name: str, parent_job_id: Optional[str]) -> list[str]:
labels: list[str] = []
if parent_job_id:
parent_job = jm.get_job(parent_job_id) or {}
for file_entry in parent_job.get("files", []):
if file_entry.get("type") != "stem":
continue
stem_value = (file_entry.get("stem") or "").strip()
if stem_value:
labels.append(stem_value)
continue
name = file_entry.get("name") or ""
if name:
labels.append(parse_stem_label_from_filename(Path(name).stem, project_name))
stems_root = project_dir / "Stems"
if stems_root.exists():
for path in stems_root.rglob("*"):
if not path.is_file() or path.suffix.lower() not in _AUDIO_EXTS:
continue
labels.append(parse_stem_label_from_filename(path.stem, project_name))
return labels
def _renumber_reprocessed_stems(
stem_files: Dict[str, str],
stems_dir: Path,
project_name: str,
parent_job_id: Optional[str],
job_id: str,
source_stem_label: Optional[str] = None,
) -> Dict[str, str]:
existing_labels = _existing_project_stem_labels(stems_dir.parent, project_name, parent_job_id)
assigned_labels: list[str] = []
renamed: Dict[str, str] = {}
source_label = normalise_stem_label(source_stem_label or "")
def _preferred_reprocess_label(stem_name: str) -> str:
stem_label = normalise_stem_label(stem_name)
if source_label == "backing" and stem_label in {"vocals", "vocal", "lead"}:
return "backing vocals"
return stem_label
for stem_name, path in sorted(stem_files.items()):
src = Path(path)
if not src.exists():
continue
base_label = _preferred_reprocess_label(stem_name)
# Ensure uniqueness against already existing labels and labels assigned in this run.
used_labels = [*existing_labels, *assigned_labels]
if base_label not in used_labels:
numbered_label = base_label
dest = stems_dir / _stem_filename(project_name, numbered_label, src.suffix)
else:
while True:
numbered_label = next_stem_label(base_label, used_labels)
dest = stems_dir / _stem_filename(project_name, numbered_label, src.suffix)
if not dest.exists() or dest == src:
break
assigned_labels.append(numbered_label)
used_labels = [*existing_labels, *assigned_labels]
if dest != src:
shutil.move(str(src), str(dest))
jm.update_job(job_id, progress=f"Reprocessed stem saved as '{numbered_label}'")
renamed[numbered_label] = str(dest)
assigned_labels.append(numbered_label)
return renamed
def _run_model_separation(
job_id: str,
input_path: str,
model: str,
normalised_format: str,
stems_dir: Path,
mp3_bitrate: str,
metadata: dict,
stem_prefix: str,
power_opts: dict,
chunking_mode: str,
chunk_duration_secs: int,
chunk_overlap_secs: int,
) -> Dict[str, str]:
"""Dispatch one separation request through the existing model-specific runners."""
_melband_segment_secs_override = power_opts.get('melband', {}).get('segment_secs')
if model in BSR_CHORUS_MODELS:
return _run_bsroformer_chorus(
job_id, input_path, model, normalised_format, stems_dir,
mp3_bitrate=mp3_bitrate, metadata=metadata,
stem_prefix=stem_prefix,
batch_size_override=power_opts['roformer'].get('batch_size'),
num_overlap_override=power_opts['roformer'].get('num_overlap'),
)
if model in BSR_ANVUEW_MODELS:
# Route to MelBand Vocals Anvuew since it's the same architecture
return _run_mel_band_vocals(
job_id, input_path, model, normalised_format, stems_dir,
mp3_bitrate=mp3_bitrate, metadata=metadata,
stem_prefix=stem_prefix,
batch_size_override=power_opts['roformer'].get('batch_size'),
num_overlap_override=power_opts['roformer'].get('num_overlap'),
segment_secs_override=_melband_segment_secs_override,
chunking_mode=chunking_mode,
chunk_duration_secs=chunk_duration_secs,
chunk_overlap_secs=chunk_overlap_secs,
)
if model in BSR_KARAOKE_SPECIAL_MODELS:
# BS karaoke variants are executed through the specialized karaoke path.
return _run_mel_band_karaoke(
job_id, input_path, model, normalised_format, stems_dir,
mp3_bitrate=mp3_bitrate, metadata=metadata,
stem_prefix=stem_prefix,
batch_size_override=power_opts['roformer'].get('batch_size'),
num_overlap_override=power_opts['roformer'].get('num_overlap'),
segment_secs_override=_melband_segment_secs_override,
chunking_mode=chunking_mode,
chunk_duration_secs=chunk_duration_secs,
chunk_overlap_secs=chunk_overlap_secs,
)
if model in BSR_ADAPTER_MODELS:
model_path, config_path = _ensure_bs_adapter_assets(model)
return _run_bsroformer(
job_id, input_path, model, normalised_format, stems_dir,
mp3_bitrate=mp3_bitrate, metadata=metadata,
stem_prefix=stem_prefix,
batch_size_override=power_opts['roformer'].get('batch_size'),
num_overlap_override=power_opts['roformer'].get('num_overlap'),
chunking_mode=chunking_mode,
chunk_duration_secs=chunk_duration_secs,
chunk_overlap_secs=chunk_overlap_secs,
model_path_override=str(model_path),
config_path_override=str(config_path),
)
if model.startswith(BSR_PREFIX):
return _run_bsroformer(
job_id, input_path, model, normalised_format, stems_dir,
mp3_bitrate=mp3_bitrate, metadata=metadata,
stem_prefix=stem_prefix,
batch_size_override=power_opts['roformer'].get('batch_size'),
num_overlap_override=power_opts['roformer'].get('num_overlap'),
chunking_mode=chunking_mode,
chunk_duration_secs=chunk_duration_secs,
chunk_overlap_secs=chunk_overlap_secs,
)
if model.startswith(MBR_PREFIX):
return _run_mel_band_roformer(
job_id, input_path, model, normalised_format, stems_dir,
mp3_bitrate=mp3_bitrate, metadata=metadata,
stem_prefix=stem_prefix,
batch_size_override=power_opts['roformer'].get('batch_size'),
num_overlap_override=power_opts['roformer'].get('num_overlap'),
segment_secs_override=_melband_segment_secs_override,
chunking_mode=chunking_mode,
chunk_duration_secs=chunk_duration_secs,
chunk_overlap_secs=chunk_overlap_secs,
)
if model in MBR_VOCALS_MODELS:
return _run_mel_band_vocals(
job_id, input_path, model, normalised_format, stems_dir,
mp3_bitrate=mp3_bitrate, metadata=metadata,
stem_prefix=stem_prefix,
batch_size_override=power_opts['roformer'].get('batch_size'),
num_overlap_override=power_opts['roformer'].get('num_overlap'),
segment_secs_override=_melband_segment_secs_override,
chunking_mode=chunking_mode,
chunk_duration_secs=chunk_duration_secs,
chunk_overlap_secs=chunk_overlap_secs,
)
if model in MBR_DEREVERB_MODELS:
return _run_mel_band_dereverb(
job_id, input_path, model, normalised_format, stems_dir,
mp3_bitrate=mp3_bitrate, metadata=metadata,
stem_prefix=stem_prefix,
batch_size_override=power_opts['roformer'].get('batch_size'),
num_overlap_override=power_opts['roformer'].get('num_overlap'),
dim_t_override=_melband_dim_t_override,
chunking_mode=chunking_mode,
chunk_duration_secs=chunk_duration_secs,
chunk_overlap_secs=chunk_overlap_secs,
)
if model in MBR_KARAOKE_MODELS:
return _run_mel_band_karaoke(
job_id, input_path, model, normalised_format, stems_dir,
mp3_bitrate=mp3_bitrate, metadata=metadata,
stem_prefix=stem_prefix,
batch_size_override=power_opts['roformer'].get('batch_size'),
num_overlap_override=power_opts['roformer'].get('num_overlap'),
segment_secs_override=_melband_segment_secs_override,
)
if model in MBR_DEUX_BECRUILY_MODELS:
return _run_mel_band_deux_becruily(
job_id, input_path, model, normalised_format, stems_dir,
mp3_bitrate=mp3_bitrate, metadata=metadata,
stem_prefix=stem_prefix,
batch_size_override=power_opts['roformer'].get('batch_size'),
num_overlap_override=power_opts['roformer'].get('num_overlap'),
segment_secs_override=_melband_segment_secs_override,
chunking_mode=chunking_mode,
chunk_duration_secs=chunk_duration_secs,
chunk_overlap_secs=chunk_overlap_secs,
)
if model in MDXNET_MODEL_IDS:
# Route MelBand-based MDX models appropriately
if model == 'mdxnet_dereverb':
return _run_mel_band_dereverb(
job_id, input_path, model, normalised_format, stems_dir,
mp3_bitrate=mp3_bitrate, metadata=metadata,
batch_size_override=power_opts['roformer'].get('batch_size'),
num_overlap_override=power_opts['roformer'].get('num_overlap'),
segment_secs_override=_melband_segment_secs_override,
chunking_mode=chunking_mode,
chunk_duration_secs=chunk_duration_secs,
chunk_overlap_secs=chunk_overlap_secs,
)
if model == 'mdxnet_drumsep':
return _run_mdx23c(
job_id, input_path, model, normalised_format, stems_dir,
mp3_bitrate=mp3_bitrate, metadata=metadata,
stem_prefix=stem_prefix,
chunk_size_override=power_opts.get('scnet', {}).get('chunk_size'),
num_overlap_override=power_opts.get('scnet', {}).get('num_overlap'),
chunking_mode=chunking_mode,
chunk_duration_secs=chunk_duration_secs,
chunk_overlap_secs=chunk_overlap_secs,
)
return _run_mdxnet(
job_id, input_path, model, normalised_format, stems_dir,
mp3_bitrate=mp3_bitrate, metadata=metadata,
chunking_mode=chunking_mode,
chunk_duration_secs=(power_opts.get('mdxnet', {}).get('chunk_duration_secs') or chunk_duration_secs),
chunk_overlap_secs=chunk_overlap_secs,
)
if model in SCNET_MODEL_IDS:
return _run_scnet(
job_id, input_path, model, normalised_format, stems_dir,
mp3_bitrate=mp3_bitrate, metadata=metadata,
stem_prefix=stem_prefix,
chunk_size_override=power_opts['scnet'].get('chunk_size'),
num_overlap_override=power_opts['scnet'].get('num_overlap'),
chunking_mode=chunking_mode,
chunk_duration_secs=chunk_duration_secs,
chunk_overlap_secs=chunk_overlap_secs,
)
if model in BSR_SW_MODEL_IDS:
return _run_bsroformer_sw(
job_id, input_path, model, normalised_format, stems_dir,
mp3_bitrate=mp3_bitrate, metadata=metadata,
stem_prefix=stem_prefix,
)
if model in MEDLEYVOX_MODEL_IDS:
return _run_medleyvox(
job_id, input_path, model, normalised_format, stems_dir,
mp3_bitrate=mp3_bitrate, metadata=metadata,
stem_prefix=stem_prefix,
chunk_duration_secs=(power_opts.get('medleyvox', {}).get('chunk_secs')),
)
if model in ENSEMBLE_MODEL_IDS:
return _run_ensemble(
job_id, input_path, model, normalised_format, stems_dir,
mp3_bitrate=mp3_bitrate, metadata=metadata,
stem_prefix=stem_prefix,
)
if model in APOLLO_MODEL_IDS:
return _run_apollo(
job_id, input_path, model, normalised_format, stems_dir,
mp3_bitrate=mp3_bitrate, metadata=metadata,
stem_prefix=stem_prefix,
chunking_mode=chunking_mode,
chunk_duration_secs=(power_opts.get('apollo', {}).get('chunk_duration_secs') or chunk_duration_secs),
chunk_overlap_secs=chunk_overlap_secs,
)
if model in ASPIRATION_MODEL_IDS:
return _run_aspiration(
job_id, input_path, model, normalised_format, stems_dir,
mp3_bitrate=mp3_bitrate, metadata=metadata,
stem_prefix=stem_prefix,
chunking_mode=chunking_mode,
chunk_duration_secs=(power_opts.get('aspiration', {}).get('chunk_duration_secs') or chunk_duration_secs),
chunk_overlap_secs=chunk_overlap_secs,
)
if model in BANDIT_MODEL_IDS:
return _run_bandit(
job_id, input_path, model, normalised_format, stems_dir,
mp3_bitrate=mp3_bitrate, metadata=metadata,
stem_prefix=stem_prefix,
chunking_mode=chunking_mode,
chunk_duration_secs=(power_opts.get('bandit', {}).get('chunk_duration_secs') or chunk_duration_secs),
chunk_overlap_secs=chunk_overlap_secs,
)
if model in SEPACAP_MODEL_IDS:
return _run_sepacap(
job_id, input_path, normalised_format, stems_dir,
mp3_bitrate=mp3_bitrate, metadata=metadata,
stem_prefix=stem_prefix,
chunk_samples=power_opts['sepacap']['chunk_samples'],
)
if model in SEPREFORMER_MODEL_IDS:
return _run_sepreformer(
job_id, input_path, model, normalised_format, stems_dir,
mp3_bitrate=mp3_bitrate, metadata=metadata,
stem_prefix=stem_prefix,
chunk_samples=power_opts['sepreformer']['chunk_samples'],
)
return _run_demucs(
job_id, input_path, model, normalised_format, stems_dir,
metadata=metadata,
stem_prefix=stem_prefix,
segment_override=power_opts['demucs'].get('segment'),
overlap_override=power_opts['demucs'].get('overlap'),
)
def _select_refined_output_path(target_stem: str, refined_outputs: Dict[str, str]) -> Optional[str]:
"""Choose the most likely refined output for a target instrument stem."""
if not refined_outputs:
return None
normalized_target = (target_stem or "").strip().lower()
direct = refined_outputs.get(normalized_target)
if direct:
return direct
aliases = {
"vocals": ["vocal", "voice", "lead_vocal"],
"backing": ["backing_instrumental", "backing", "karaoke", "accompaniment"],
"drums": ["drum", "percussion"],
"bass": ["bass", "low"],
"other": ["accompaniment", "instrumental", "music", "other"],
"guitar": ["guitar", "gtr"],
"piano": ["piano", "keys", "keyboard"],
}
for key, path in refined_outputs.items():
k = key.lower()
if normalized_target in k:
return path
for alias in aliases.get(normalized_target, []):
if alias in k:
return path
# Deterministic fallback.
return refined_outputs[sorted(refined_outputs.keys())[0]]
def _resolve_refinement_source_stem_path(stem_files: Dict[str, str], target_stem: str) -> Optional[str]:
"""Resolve a requested refinement stem key against known source-stem aliases."""
normalized_target = (target_stem or "").strip().lower()
if not normalized_target:
return None
if normalized_target in stem_files:
return stem_files[normalized_target]
aliases = {
"backing": ["backing_instrumental", "karaoke", "instrumental", "other"],
"vocals": ["lead", "lead_vocal", "vocal"],
}
for alias in aliases.get(normalized_target, []):
if alias in stem_files:
return stem_files[alias]
return None
def _release_idle_transcriber_models(job_id: str) -> None:
"""Unload cached Whisper models before heavy separator work starts."""
try:
from .lyric_transcriber_service import release_cached_whisper_models
released = release_cached_whisper_models()
if released:
log.info("[%s] released %d cached Whisper model(s) before separation", job_id, released)
except Exception as exc:
log.debug("[%s] failed to release cached Whisper models: %s", job_id, exc)
def _should_isolate_separator_process() -> bool:
if os.environ.get("STEMENGINE_SEPARATOR_CHILD") == "1":
return False
if CPU_SEPARATION:
return False
return os.environ.get("STEMENGINE_SEPARATION_SUBPROCESS", "true").lower() != "false"
def _serialise_separation_kwargs(kwargs: dict) -> dict:
payload = dict(kwargs)
if payload.get("output_dir_override") is not None:
payload["output_dir_override"] = str(payload["output_dir_override"])
return payload
def _run_separation_in_subprocess(**kwargs) -> None:
job_id = kwargs["job_id"]
job_dir = JOBS_DIR / job_id
job_dir.mkdir(parents=True, exist_ok=True)
payload_path = job_dir / "separator_payload.json"
payload_path.write_text(json.dumps(_serialise_separation_kwargs(kwargs)), encoding="utf-8")
env = os.environ.copy()
env["STEMENGINE_SEPARATOR_CHILD"] = "1"
try:
result = subprocess.run(
[sys.executable, "-m", "backend.services.separator_worker", str(payload_path)],
cwd=str(Path(__file__).resolve().parents[2]),
env=env,
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
text=True,
check=False,
)
except Exception as exc:
log.warning("[%s] separator subprocess launch failed, falling back to in-process execution: %s", job_id, exc)
jm.update_job(job_id, progress="Separator worker launch failed — falling back to in-process execution...")
_run_separation_impl(**kwargs)
return
if result.returncode == 0:
return
job = jm.get_job(job_id)
if job and job.get("status") in {
jm.JobStatus.DONE,
jm.JobStatus.ERROR,
jm.JobStatus.CANCELLED,
}:
return
stderr = (result.stderr or "").strip()
if len(stderr) > 800:
stderr = stderr[:800] + "..."
message = stderr or f"separator worker exited with code {result.returncode}"
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=message, progress="Error.")
def run_separation(
job_id: str,
input_path: str,
model: str,
output_format: str,
# Preset options
preset_type: str = "stems_only", # stems_only, static, instrumental, dynamic
static_option: Optional[str] = None, # for static preset: no_vocals, no_drums, etc.
dynamic_levels: Optional[Dict[str, float]] = None, # for dynamic preset
instrument_models: Optional[Dict[str, str]] = None, # optional per-instrument model mapping
instrument_refine_output_format: Optional[str] = None,
instrument_refine_mp3_bitrate: Optional[str] = None,
mp3_bitrate_override: Optional[str] = None,
# Project naming
project_name_override: Optional[str] = None,
# Power preset
sep_power_preset: str = 'standard',
# Optional request-level runtime overrides
chunking_mode_override: Optional[str] = None,
chunk_duration_secs_override: Optional[int] = None,
chunk_overlap_secs_override: Optional[int] = None,
roformer_batch_size_override: Optional[int] = None,
roformer_num_overlap_override: Optional[int] = None,
demucs_segment_override: Optional[int] = None,
demucs_overlap_override: Optional[float] = None,
scnet_chunk_size_override: Optional[int] = None,
scnet_num_overlap_override: Optional[int] = None,
# Phase 2: optional global policy overrides
global_power_tier_override: Optional[str] = None,
global_segment_secs_override: Optional[float] = None,
global_overlap_override: Optional[float] = None,
global_batch_size_override: Optional[int] = None,
# Reprocess mode: output stems directly into an existing project's Stems dir
output_dir_override: Optional[Path] = None,
parent_job_id: Optional[str] = None,
):
_release_idle_transcriber_models(job_id)
kwargs = {
"job_id": job_id,
"input_path": input_path,
"model": model,
"output_format": output_format,
"preset_type": preset_type,
"static_option": static_option,
"dynamic_levels": dynamic_levels,
"instrument_models": instrument_models,
"instrument_refine_output_format": instrument_refine_output_format,
"instrument_refine_mp3_bitrate": instrument_refine_mp3_bitrate,
"mp3_bitrate_override": mp3_bitrate_override,
"project_name_override": project_name_override,
"sep_power_preset": sep_power_preset,
"chunking_mode_override": chunking_mode_override,
"chunk_duration_secs_override": chunk_duration_secs_override,
"chunk_overlap_secs_override": chunk_overlap_secs_override,
"roformer_batch_size_override": roformer_batch_size_override,
"roformer_num_overlap_override": roformer_num_overlap_override,
"demucs_segment_override": demucs_segment_override,
"demucs_overlap_override": demucs_overlap_override,
"scnet_chunk_size_override": scnet_chunk_size_override,
"scnet_num_overlap_override": scnet_num_overlap_override,
"global_power_tier_override": global_power_tier_override,
"global_segment_secs_override": global_segment_secs_override,
"global_overlap_override": global_overlap_override,
"global_batch_size_override": global_batch_size_override,
"output_dir_override": output_dir_override,
"parent_job_id": parent_job_id,
}
if _should_isolate_separator_process():
_run_separation_in_subprocess(**kwargs)
return
_run_separation_impl(**kwargs)
def _run_separation_impl(
job_id: str,
input_path: str,
model: str,
output_format: str,
# Preset options
preset_type: str = "stems_only", # stems_only, static, instrumental, dynamic
static_option: Optional[str] = None, # for static preset: no_vocals, no_drums, etc.
dynamic_levels: Optional[Dict[str, float]] = None, # for dynamic preset
instrument_models: Optional[Dict[str, str]] = None, # optional per-instrument model mapping
instrument_refine_output_format: Optional[str] = None,
instrument_refine_mp3_bitrate: Optional[str] = None,
mp3_bitrate_override: Optional[str] = None,
# Project naming
project_name_override: Optional[str] = None,
# Power preset
sep_power_preset: str = 'standard',
# Optional request-level runtime overrides
chunking_mode_override: Optional[str] = None,
chunk_duration_secs_override: Optional[int] = None,
chunk_overlap_secs_override: Optional[int] = None,
roformer_batch_size_override: Optional[int] = None,
roformer_num_overlap_override: Optional[int] = None,
demucs_segment_override: Optional[int] = None,
demucs_overlap_override: Optional[float] = None,
scnet_chunk_size_override: Optional[int] = None,
scnet_num_overlap_override: Optional[int] = None,
# Phase 2: optional global policy overrides
global_power_tier_override: Optional[str] = None,
global_segment_secs_override: Optional[float] = None,
global_overlap_override: Optional[float] = None,
global_batch_size_override: Optional[int] = None,
# Reprocess mode: output stems directly into an existing project's Stems dir
output_dir_override: Optional[Path] = None,
parent_job_id: Optional[str] = None,
):
"""
Background task: separate audio into stems and apply presets.
Creates human-readable project folder structure:
/Artist - Title/
Stems/ (always created)
Instrumental/ (if preset includes instrumental)
Static/ (if preset is static)
Dynamic/ (if preset is dynamic)
"""
log.debug("[%s] run_separation start — model=%s format=%s preset=%s input=%s",
job_id, model, output_format, preset_type, input_path)
instrument_models_summary = {
"requested": instrument_models or {},
"applied": {},
"skipped": {},
}
# Read metadata for project naming
jm.update_job(job_id, status=jm.JobStatus.RUNNING, progress="Reading metadata...")
metadata = read_metadata(input_path)
# Ensure model weights are present — download automatically if missing
if not ensure_model_available(job_id, model):
return
# Resolve human-readable labels for tab naming
model_label = MODEL_LABELS.get(model, model)
format_label = FORMAT_LABELS.get(output_format, output_format)
tab_label = f"{model_label}_{format_label}"
# Determine project name
if project_name_override:
proj_name = project_name_override
else:
filename_stem = Path(input_path).stem
proj_name = project_name(
metadata.get('albumartist') or metadata.get('artist', ''),
metadata.get('title', ''),
filename_stem,
)
if output_dir_override is not None:
# Reprocess mode: place stems directly in the existing project's Stems dir.
# Skip project creation and source-file copy — the folder already exists.
stems_dir = output_dir_override
project_dir = output_dir_override.parent
stems_dir.mkdir(parents=True, exist_ok=True)
dirs = {'project': project_dir, 'stems': stems_dir}
source_dest = project_dir / f"{proj_name}{Path(input_path).suffix.lower()}"
else:
# Determine which folders to create based on preset
preset_flags = {
'instrumental': preset_type in ('instrumental',),
'static': preset_type == 'static',
'dynamic': preset_type == 'dynamic',
}
# Create project directories
jm.update_job(job_id, progress=f"Creating project: {proj_name}...")
dirs = make_project_dirs(proj_name, PROJECTS_DIR, preset_flags)
stems_dir = dirs['stems']
project_dir = dirs['project']
# Copy original source file to project root with full metadata
jm.update_job(job_id, progress="Copying source file to project...")
input_ext = Path(input_path).suffix.lower()
source_dest = project_dir / f"{proj_name}{input_ext}"
try:
shutil.copy2(input_path, source_dest)
copy_metadata(input_path, str(source_dest), stem_label=None, copy_lyrics=True)
except Exception as e:
jm.update_job(job_id, progress=f"Warning: Could not copy source file: {e}")
# Normalise mp3_320 → mp3 with explicit bitrate
if output_format == 'mp3_320':
mp3_bitrate = '320k'
elif output_format == 'mp3' and (mp3_bitrate_override or '').strip():
mp3_bitrate = str(mp3_bitrate_override).strip().lower()
else:
mp3_bitrate = '256k'
normalised_format = 'mp3' if output_format == 'mp3_320' else output_format
# Optional per-instrument refinement output controls.
refine_output_format_raw = (instrument_refine_output_format or '').strip().lower()
if refine_output_format_raw:
refine_normalised_format = 'mp3' if refine_output_format_raw == 'mp3_320' else refine_output_format_raw
if refine_output_format_raw == 'mp3_320':
refine_mp3_bitrate = '320k'
elif refine_normalised_format == 'mp3' and (instrument_refine_mp3_bitrate or '').strip():
refine_mp3_bitrate = str(instrument_refine_mp3_bitrate).strip().lower()
elif refine_normalised_format == 'mp3':
refine_mp3_bitrate = mp3_bitrate
else:
refine_mp3_bitrate = mp3_bitrate
else:
refine_normalised_format = normalised_format
refine_mp3_bitrate = mp3_bitrate
# Resolve power preset options (Phase 2: optional global tier override).
_effective_power_tier = (global_power_tier_override or sep_power_preset or 'standard').strip().lower()
_power = deepcopy(get_power_opts(_effective_power_tier))
# Apply global policy overrides before family-specific overrides.
_power = apply_global_power_policy_overrides(
_power,
global_segment_secs_override=global_segment_secs_override,
global_overlap_override=global_overlap_override,
global_batch_size_override=global_batch_size_override,
)
# Apply request-level overrides (Phase 4).
if roformer_batch_size_override is not None:
_power['roformer']['batch_size'] = int(roformer_batch_size_override)
if roformer_num_overlap_override is not None:
_power['roformer']['num_overlap'] = int(roformer_num_overlap_override)
if demucs_segment_override is not None:
_power['demucs']['segment'] = int(demucs_segment_override)
if demucs_overlap_override is not None:
_power['demucs']['overlap'] = float(demucs_overlap_override)
if scnet_chunk_size_override is not None:
_power['scnet']['chunk_size'] = int(scnet_chunk_size_override)
if scnet_num_overlap_override is not None:
_power['scnet']['num_overlap'] = int(scnet_num_overlap_override)
_chunk_duration_fallback = chunk_duration_secs_override
if _chunk_duration_fallback is None and global_segment_secs_override is not None:
_chunk_duration_fallback = max(1, int(round(float(global_segment_secs_override))))
_chunking_mode, _chunk_duration_secs, _chunk_overlap_secs = _effective_chunking_params(
chunking_mode_override,
_chunk_duration_fallback,
chunk_overlap_secs_override,
)
global_power_policy = build_global_power_policy(_effective_power_tier, _power)
input_duration_secs = get_audio_duration_secs(input_path)
separation_diagnostics = _build_separation_diagnostics(
model,
input_duration_secs,
chunking_mode=_chunking_mode,
chunk_duration_secs=_chunk_duration_secs,
chunk_overlap_secs=_chunk_overlap_secs,
power_opts=_power,
global_power_policy=global_power_policy,
)
request_overrides = {
k: v for k, v in {
"chunking_mode_override": chunking_mode_override,
"chunk_duration_secs_override": chunk_duration_secs_override,
"chunk_overlap_secs_override": chunk_overlap_secs_override,
"roformer_batch_size_override": roformer_batch_size_override,
"roformer_num_overlap_override": roformer_num_overlap_override,
"demucs_segment_override": demucs_segment_override,
"demucs_overlap_override": demucs_overlap_override,
"scnet_chunk_size_override": scnet_chunk_size_override,
"scnet_num_overlap_override": scnet_num_overlap_override,
"global_power_tier_override": global_power_tier_override,
"global_segment_secs_override": global_segment_secs_override,
"global_overlap_override": global_overlap_override,
"global_batch_size_override": global_batch_size_override,
}.items() if v is not None
}
# Flush PyTorch's allocator cache BEFORE applying the VRAM cap.
# On WSL2 the system reserves headroom for Windows (DWM, WDDM, etc.).
# _apply_cuda_limits() caps the process at (total_vram - STEMENGINE_GPU_VRAM_RESERVE_MB)
# / total_vram — e.g. ≈75 % on an 8 GB card, ensuring 2 GB stays free for Windows.
# The flush must come first: if stale cached allocations are still counted against
# the cap from a prior job, the cap appears tighter than it actually is.
flush_vram_cache()
try:
from .lyric_transcriber_service import _apply_cuda_limits
_apply_cuda_limits()
except Exception:
pass
# Acquire the shared VRAM lock before loading any model.
# Prevents a simultaneous lyric transcription job (running in its own
# executor thread) from loading Whisper while a separator is in VRAM.
jm.gpu_vram_lock.acquire()
# Run separation
try:
stem_files = _run_model_separation(
job_id=job_id,
input_path=input_path,
model=model,
normalised_format=normalised_format,
stems_dir=stems_dir,
mp3_bitrate=mp3_bitrate,
metadata=metadata,
stem_prefix=proj_name,
power_opts=_power,
chunking_mode=_chunking_mode,
chunk_duration_secs=_chunk_duration_secs,
chunk_overlap_secs=_chunk_overlap_secs,
)
if jm.should_cancel(job_id):
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
return
if output_dir_override is not None:
source_stem_label = parse_stem_label_from_filename(Path(input_path).stem, proj_name)
stem_files = _renumber_reprocessed_stems(
stem_files=stem_files,
stems_dir=stems_dir,
project_name=proj_name,
parent_job_id=parent_job_id,
job_id=job_id,
source_stem_label=source_stem_label,
)
# Optional per-instrument refinement pass
if instrument_models:
jm.update_job(job_id, progress="Applying per-instrument refinement...")
for stem_name, mapped_model in instrument_models.items():
if jm.should_cancel(job_id):
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
return
base_stem_path = _resolve_refinement_source_stem_path(stem_files, stem_name)
if not base_stem_path or not Path(base_stem_path).exists():
instrument_models_summary["skipped"][stem_name] = (
f"No base stem '{stem_name}' found to refine with '{mapped_model}'."
)
continue
if mapped_model == model:
instrument_models_summary["skipped"][stem_name] = (
"Mapped model matches base model; refinement skipped."
)
continue
if not ensure_model_available(job_id, mapped_model):
raise RuntimeError(f"Failed to prepare refinement model '{mapped_model}' for stem '{stem_name}'")
jm.update_job(job_id, progress=f"Refining stem '{stem_name}' with model '{mapped_model}'...")
refine_tmp = JOBS_DIR / job_id / "instrument_refine" / stem_name
cleanup_temp(refine_tmp)
refine_tmp.mkdir(parents=True, exist_ok=True)
try:
refined_outputs = _run_model_separation(
job_id=job_id,
input_path=base_stem_path,
model=mapped_model,
normalised_format=refine_normalised_format,
stems_dir=refine_tmp,
mp3_bitrate=refine_mp3_bitrate,
metadata=metadata,
stem_prefix=f"{proj_name} - {stem_name.capitalize()} Refine",
power_opts=_power,
)
selected_refined_path = _select_refined_output_path(stem_name, refined_outputs)
if not selected_refined_path:
instrument_models_summary["skipped"][stem_name] = (
f"Model '{mapped_model}' produced no usable refined output for '{stem_name}'."
)
continue
selected_refined = Path(selected_refined_path)
refined_dest = stems_dir / (
f"{proj_name} - {stem_name.capitalize()} "
f"(Refined - {MODEL_LABELS.get(mapped_model, mapped_model)}){selected_refined.suffix}"
)
shutil.move(str(selected_refined), str(refined_dest))
# Promote refined stem as canonical stem output for this instrument.
stem_files[stem_name] = str(refined_dest)
instrument_models_summary["applied"][stem_name] = {
"model": mapped_model,
"output": str(refined_dest),
}
except Exception as refine_exc:
instrument_models_summary["skipped"][stem_name] = (
f"Refinement failed for '{stem_name}' with '{mapped_model}': {refine_exc}"
)
finally:
cleanup_temp(refine_tmp)
stem_files, stem_cleanup = finalize_generated_stems(
stem_files,
stems_dir=stems_dir,
stem_prefix=proj_name,
job_id=job_id,
update_job_fn=jm.update_job,
)
preset_files = apply_presets(
preset_type=preset_type,
static_option=static_option,
dynamic_levels=dynamic_levels,
stem_files=stem_files,
dirs=dirs,
proj_name=proj_name,
input_path=input_path,
normalised_format=normalised_format,
job_id=job_id,
update_job_fn=jm.update_job,
)
# Build result file list
all_files = []
# Add source file first if it exists
if source_dest.exists():
all_files.append({
"name": source_dest.name,
"path": str(source_dest),
"type": "source"
})
for stem_name, path in sorted(stem_files.items()):
all_files.append({
"name": Path(path).name,
"path": path,
"type": "stem",
"stem": stem_name
})
all_files.extend(preset_files)
if output_dir_override is None:
# Normal mode: register a project record and write project.json
project = jm.find_or_create_project(proj_name, job_id, tab_label=tab_label)
try:
write_project_metadata(
project_dir,
project_id=project['id'],
name=proj_name,
artist=metadata.get('albumartist') or metadata.get('artist', ''),
title=metadata.get('title', ''),
model=model,
model_label=model_label,
format=normalised_format,
format_label=format_label,
source_file=str(source_dest) if source_dest.exists() else '',
stems=stem_files,
sep_job_id=job_id,
trans_jobs={},
abc_jobs={},
lrc_jobs={},
mix_job_id=None,
instrument_models=instrument_models_summary,
)
except Exception as meta_err:
jm.update_job(job_id, progress=f"Warning: could not write project.json: {meta_err}")
# Analyze BPM + musical key from the source file and cache in project.json.
# Runs after the heavier separation work so it never blocks stems delivery.
if source_dest.exists():
try:
jm.update_job(job_id, progress="Analyzing audio features (BPM / key)…")
features = analyze_audio_features(source_dest)
if features:
patch_project_metadata_audio_features(project_dir, features)
except Exception as feat_err:
log.warning("[%s] Audio feature analysis failed: %s", job_id, feat_err)
project_id_result = project['id']
else:
# Reprocess mode: merge new stems into the parent sep job's file list
# so the project page discovers them without any extra queries.
stem_entries = [f for f in all_files if f.get("type") == "stem"]
if parent_job_id:
jm.append_job_files(parent_job_id, stem_entries)
project_id_result = None
log.debug("[%s] separation complete — project=%s stems=%d files=%d",
job_id, proj_name, len(stem_files), len(all_files))
jm.update_job(
job_id,
status=jm.JobStatus.DONE,
progress="Separation complete.",
files=all_files,
result={
"reprocess": output_dir_override is not None,
"parent_job_id": parent_job_id if output_dir_override is not None else None,
"project_name": proj_name,
"project_path": str(dirs['project']),
"stems_dir": str(stems_dir),
"model_label": model_label,
"format_label": format_label,
"tab_label": tab_label,
"project_id": project_id_result,
"instrument_models": instrument_models_summary,
"stem_cleanup": stem_cleanup,
"diagnostics": separation_diagnostics,
"request_overrides": request_overrides,
}
)
except Exception as exc:
err_str = str(exc)
if jm.should_cancel(job_id) or "Cancelled by user" in err_str:
# Cancellation propagated as an exception from inside a separator.
# Treat it as a clean cancel, not an error.
log.debug("[%s] separation cancelled mid-run", job_id)
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
elif "CUDA" in err_str or "cuda" in err_str:
# CUDA context is corrupted (typically after a prior OOM).
# Auto-reset clears allocations and re-enables GPU for the next attempt.
try:
from .lyric_transcriber_service import reset_cuda
reset_cuda()
except Exception:
try:
import torch
torch.cuda.empty_cache()
except Exception:
pass
user_msg = (
f"GPU (CUDA) error: {err_str[:300]}\n\n"
"The GPU was automatically reset — please retry the separation. "
"If the error keeps occurring, restart the container."
)
log.debug("[%s] separation GPU error — %s: %s", job_id, type(exc).__name__, err_str[:200])
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=user_msg, progress="GPU error — retry to use reset GPU.")
else:
log.debug("[%s] separation failed — %s: %s", job_id, type(exc).__name__, exc)
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=err_str, progress="Error.")
finally:
# Always flush PyTorch's VRAM cache after a job ends (success, error, or
# cancel) so the next job starts with maximum headroom. gc.collect() x3
# ensures CUDA tensors are released even when model objects form reference
# cycles that a single generational GC pass misses.
try:
import gc
gc.collect()
gc.collect()
gc.collect()
except Exception:
pass
flush_vram_cache()
# Release VRAM lock so the transcription executor can proceed.
try:
jm.gpu_vram_lock.release()
except RuntimeError:
pass # already released (should not happen)
def _get_muted_stems(static_option: Optional[str]) -> List[str]:
"""Convert static option ID to list of stem names to mute."""
if not static_option:
return []
option_map = {
'no_vocals': ['vocals'],
'no_drums': ['drums'],
'no_bass': ['bass'],
'no_vocals_no_bass': ['vocals', 'bass'],
'no_drums_no_bass': ['drums', 'bass'],
}
return option_map.get(static_option, [])
def _static_name(static_option: Optional[str]) -> str:
"""Convert static option ID to human-readable name."""
name_map = {
'no_vocals': 'No Vocals',
'no_drums': 'No Drums',
'no_bass': 'No Bass',
'no_vocals_no_bass': 'No Vocals No Bass',
'no_drums_no_bass': 'No Drums No Bass',
}
return name_map.get(static_option, 'Static Mix')
# ---------------------------------------------------------------------------
# Demucs (CLI)
# ---------------------------------------------------------------------------
def _run_demucs(
job_id: str,
input_path: str,
model: str,
output_format: str,
stems_dir: Path,
metadata: Optional[Dict] = None,
stem_prefix: Optional[str] = None,
segment_override: Optional[int] = None,
overlap_override: Optional[float] = None,
) -> Dict[str, str]:
"""
Run Demucs separation. Returns dict mapping stem names to file paths.
For long tracks, automatically selects segment size based on duration
to balance quality vs. VRAM usage.
"""
from .separation.chunking import should_chunk
jm.update_job(job_id, progress="Starting Demucs separation...")
ext = output_format.lower()
fmt_flag = []
if ext == 'mp3_320':
fmt_flag = ['--mp3', '--mp3-bitrate', '320']
ext = 'mp3'
elif ext == 'mp3':
fmt_flag = ['--mp3', '--mp3-bitrate', '256']
elif ext == 'flac':
fmt_flag = ['--flac']
# Determine segment size: check env override first, then auto-scale by duration
_dur = get_audio_duration_secs(input_path)
_use_long_track_tuning = should_chunk(_dur, LONG_TRACK_CHUNKING_MODE, LONG_TRACK_DURATION_SECS)
_segment = segment_override
if _segment is None and DEMUCS_SEGMENT_OVERRIDE is not None:
_segment = DEMUCS_SEGMENT_OVERRIDE
if _segment is None and _use_long_track_tuning:
# Auto-scale segment based on duration
if _dur < 120:
_segment = 12 # Default Demucs value
elif _dur < 480:
_segment = 8 # Quality-biased for medium-length tracks
else:
_segment = 4 # VRAM-biased for long tracks
_overlap = overlap_override
if _overlap is None and DEMUCS_OVERLAP_OVERRIDE is not None:
_overlap = DEMUCS_OVERLAP_OVERRIDE
power_flags = []
if _segment is not None:
power_flags.extend(['--segment', str(_segment)])
if _overlap is not None:
power_flags.extend(['--overlap', str(_overlap)])
# Use a temp directory for demucs output
temp_out = JOBS_DIR / job_id / "demucs_out"
temp_out.mkdir(parents=True, exist_ok=True)
cmd = [
sys.executable, '-m', 'demucs',
'--name', model,
'--out', str(temp_out),
*fmt_flag,
*power_flags,
input_path,
]
try:
if _segment is not None or _use_long_track_tuning:
jm.update_job(
job_id,
progress=(
f"Long-track tuning: segment={_segment}s, overlap={_overlap or 0.25}x "
f"(duration={_dur:.0f}s)..."
),
)
process = subprocess.Popen(
cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
)
while True:
if jm.should_cancel(job_id):
process.terminate()
process.wait()
raise RuntimeError("Cancelled by user")
if process.stdout is None:
if process.poll() is not None:
break
time.sleep(0.1)
continue
ready, _, _ = select.select([process.stdout], [], [], 1.0)
if ready:
line = process.stdout.readline()
if line:
jm.update_job(job_id, progress=line.rstrip())
elif process.poll() is not None:
break
if process.returncode != 0:
raise RuntimeError(f"Demucs exited with code {process.returncode}")
# Move stems to project folder
input_stem = Path(input_path).stem
_prefix = stem_prefix or input_stem
stem_dir = temp_out / model / input_stem
stem_files = {}
if stem_dir.exists():
for f in sorted(stem_dir.iterdir()):
if f.is_file():
# Determine stem name from filename
stem_name = f.stem.lower()
# Clean up common demucs naming patterns
for prefix in [input_stem.lower(), 'htdemucs', 'mdx']:
stem_name = stem_name.replace(prefix, '').strip('_- ')
if not stem_name:
stem_name = f.stem
dest = stems_dir / _stem_filename(_prefix, stem_name, f.suffix)
shutil.move(str(f), str(dest))
copy_metadata(input_path, str(dest), stem_label=stem_name.capitalize(), copy_lyrics=True)
stem_files[stem_name] = str(dest)
# Cleanup temp directory
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
return stem_files
except Exception:
# Cleanup on error
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
raise
# ---------------------------------------------------------------------------
# BS-RoFormer Chorus (Male/Female) (Python API)
# ---------------------------------------------------------------------------
def _run_bsroformer_chorus(
job_id: str,
input_path: str,
model: str,
output_format: str,
stems_dir: Path,
mp3_bitrate: str = '256k',
metadata: Optional[Dict] = None,
stem_prefix: Optional[str] = None,
batch_size_override: Optional[int] = None,
num_overlap_override: Optional[int] = None,
) -> Dict[str, str]:
"""
Run BS-RoFormer Chorus separation (male/female/instrumental).
Returns dict mapping stem names to file paths.
"""
from .bs_roformer_chorus_separator import BSRoformerChorusSeparator
jm.update_job(
job_id, status=jm.JobStatus.RUNNING,
progress="Loading BS-RoFormer Chorus model...",
)
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
try:
import gc, torch
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
except Exception:
pass
sep = BSRoformerChorusSeparator(
cpu_separation=CPU_SEPARATION,
output_format=output_format,
mp3_bitrate=mp3_bitrate,
)
jm.update_job(job_id, progress="Separating male/female chorus... (this may take a while)")
temp_out = JOBS_DIR / job_id / "bsr_chorus_out"
temp_out.mkdir(parents=True, exist_ok=True)
_bs = batch_size_override
for _attempt in range(2):
try:
stem_paths = sep.separate_into_parts(
input_path, str(temp_out),
batch_size_override=_bs,
num_overlap_override=num_overlap_override,
)
break
except Exception as _exc:
_is_oom = "out of memory" in str(_exc).lower() or "OutOfMemory" in type(_exc).__name__
if _is_oom and _attempt == 0:
import gc, torch
jm.update_job(job_id, progress="GPU OOM — retrying with batch_size=1...")
gc.collect()
torch.cuda.empty_cache()
_bs = 1
continue
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
raise
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
_prefix = stem_prefix or Path(input_path).stem
stem_files = {}
try:
for stem_name, path in sorted(stem_paths.items()):
src = Path(path)
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
shutil.move(str(src), str(dest))
copy_metadata(input_path, str(dest), stem_label=stem_name.capitalize(), copy_lyrics=True)
stem_files[stem_name.lower()] = str(dest)
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
return stem_files
except Exception:
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
raise
# ---------------------------------------------------------------------------
# MelBand Vocals (Python API)
# ---------------------------------------------------------------------------
def _run_mel_band_vocals(
job_id: str,
input_path: str,
model: str,
output_format: str,
stems_dir: Path,
mp3_bitrate: str = '256k',
metadata: Optional[Dict] = None,
stem_prefix: Optional[str] = None,
batch_size_override: Optional[int] = None,
num_overlap_override: Optional[int] = None,
dim_t_override: Optional[int] = None,
segment_secs_override: Optional[float] = None,
chunking_mode: Optional[str] = None,
chunk_duration_secs: Optional[int] = None,
chunk_overlap_secs: Optional[int] = None,
) -> Dict[str, str]:
"""Run MelBand Vocals separation (vocals/instrumental)."""
# Route to appropriate separator based on model variant
if model == 'mel_band_vocals_gabox':
from .mel_band_specialized_separator import MelBandVocalsGaboxSeparator
sep_class = MelBandVocalsGaboxSeparator
model_name = "MelBand Vocals Gabox"
elif model == 'mel_band_vocals_anvuew':
from .mel_band_specialized_separator import MelBandVocalsAnvuewSeparator
sep_class = MelBandVocalsAnvuewSeparator
model_name = "MelBand Vocals Anvuew"
else:
from .mel_band_specialized_separator import MelBandVocalsSeparator
sep_class = MelBandVocalsSeparator
model_name = "MelBand Vocals"
jm.update_job(
job_id, status=jm.JobStatus.RUNNING,
progress=f"Loading {model_name} model...",
)
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
try:
import gc, torch
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
except Exception:
pass
_dim_t = dim_t_override if dim_t_override is not None else MELBAND_DIM_T_OVERRIDE
sep = _init_melband_separator_compat(
sep_class,
cpu_separation=CPU_SEPARATION,
output_format=output_format,
mp3_bitrate=mp3_bitrate,
diagnostics_enabled=MELBAND_STAGE_DIAGNOSTICS,
diagnostics_cb=_diag_progress_cb(job_id) if MELBAND_STAGE_DIAGNOSTICS else None,
dim_t_override=_dim_t,
segment_secs_override=segment_secs_override,
)
jm.update_job(job_id, progress="Separating vocals with MelBand...")
temp_out = JOBS_DIR / job_id / "mel_band_vocals_out"
temp_out.mkdir(parents=True, exist_ok=True)
_bs = batch_size_override
_overlap = num_overlap_override
_dur = get_audio_duration_secs(input_path)
_chunk_mode = (chunking_mode or LONG_TRACK_CHUNKING_MODE)
_chunk_secs = int(chunk_duration_secs or LONG_TRACK_CHUNK_DURATION_SECS)
_chunk_ov = int(chunk_overlap_secs or LONG_TRACK_CHUNK_OVERLAP_SECS)
_use_chunking = should_chunk(_dur, _chunk_mode, LONG_TRACK_DURATION_SECS)
try:
if _use_chunking:
_total_samples = max(1, int(round(_dur * 44100)))
_chunk_plan = plan_chunks(_total_samples, 44100, _chunk_secs, _chunk_ov)
_n = len(_chunk_plan.chunks)
jm.update_job(
job_id,
progress=(
f"Chunked mode: {_n} chunk(s) × {_chunk_secs} s "
f"with {_chunk_ov} s crossfade overlap..."
),
)
stem_paths = sep.separate_into_parts_chunked(
input_path, str(temp_out),
chunk_plan=_chunk_plan,
batch_size_override=_bs,
num_overlap_override=_overlap,
progress_cb=lambda msg: jm.update_job(job_id, progress=msg),
)
else:
for _attempt in range(3):
try:
stem_paths = sep.separate_into_parts(
input_path, str(temp_out),
batch_size_override=_bs,
num_overlap_override=_overlap,
)
break
except Exception as _exc:
_is_oom = "out of memory" in str(_exc).lower() or "OutOfMemory" in type(_exc).__name__
if not _is_oom:
raise
flush_vram_cache()
if _attempt == 0:
jm.update_job(job_id, progress="GPU OOM — retrying with batch_size=1...")
_bs = 1
elif _attempt == 1:
jm.update_job(job_id, progress="GPU OOM — retrying with batch_size=1, num_overlap=2...")
_bs = 1
_overlap = 2
else:
raise
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
_prefix = stem_prefix or Path(input_path).stem
stem_files = {}
for stem_name, path in sorted(stem_paths.items()):
src = Path(path)
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
shutil.move(str(src), str(dest))
copy_metadata(input_path, str(dest), stem_label=stem_name.capitalize(), copy_lyrics=True)
stem_files[stem_name.lower()] = str(dest)
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
return stem_files
except Exception:
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
raise
# ---------------------------------------------------------------------------
# MelBand Dereverb (Python API)
# ---------------------------------------------------------------------------
def _run_mel_band_dereverb(
job_id: str,
input_path: str,
model: str,
output_format: str,
stems_dir: Path,
mp3_bitrate: str = '256k',
metadata: Optional[Dict] = None,
stem_prefix: Optional[str] = None,
batch_size_override: Optional[int] = None,
num_overlap_override: Optional[int] = None,
dim_t_override: Optional[int] = None,
segment_secs_override: Optional[float] = None,
chunking_mode: Optional[str] = None,
chunk_duration_secs: Optional[int] = None,
chunk_overlap_secs: Optional[int] = None,
) -> Dict[str, str]:
"""Run MelBand Dereverb separation (dry vocals/reverb)."""
# Route to appropriate separator based on model variant
if model == 'mel_band_dereverb_room':
from .mel_band_specialized_separator import MelBandDereverbRoomSeparator
sep_class = MelBandDereverbRoomSeparator
model_name = "MelBand Room Dereverb"
elif model == 'mel_band_dereverb_anvuew':
from .mel_band_specialized_separator import MelBandDereverbAnvuewSeparator
sep_class = MelBandDereverbAnvuewSeparator
model_name = "MelBand Dereverb Anvuew"
else:
from .mel_band_specialized_separator import MelBandDereverbSeparator
sep_class = MelBandDereverbSeparator
model_name = "MelBand Dereverb"
jm.update_job(
job_id, status=jm.JobStatus.RUNNING,
progress=f"Loading {model_name} model...",
)
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
try:
import gc, torch
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
except Exception:
pass
_dim_t = dim_t_override if dim_t_override is not None else MELBAND_DIM_T_OVERRIDE
sep = _init_melband_separator_compat(
sep_class,
cpu_separation=CPU_SEPARATION,
output_format=output_format,
mp3_bitrate=mp3_bitrate,
diagnostics_enabled=MELBAND_STAGE_DIAGNOSTICS,
diagnostics_cb=_diag_progress_cb(job_id) if MELBAND_STAGE_DIAGNOSTICS else None,
dim_t_override=_dim_t,
segment_secs_override=segment_secs_override,
)
jm.update_job(job_id, progress="Removing reverb with MelBand...")
temp_out = JOBS_DIR / job_id / "mel_band_dereverb_out"
temp_out.mkdir(parents=True, exist_ok=True)
_bs = batch_size_override
_overlap = num_overlap_override
_dur = get_audio_duration_secs(input_path)
_chunk_mode = (chunking_mode or LONG_TRACK_CHUNKING_MODE)
_chunk_secs = int(chunk_duration_secs or LONG_TRACK_CHUNK_DURATION_SECS)
_chunk_ov = int(chunk_overlap_secs or LONG_TRACK_CHUNK_OVERLAP_SECS)
_use_chunking = should_chunk(_dur, _chunk_mode, LONG_TRACK_DURATION_SECS)
try:
if _use_chunking:
_total_samples = max(1, int(round(_dur * 44100)))
_chunk_plan = plan_chunks(_total_samples, 44100, _chunk_secs, _chunk_ov)
_n = len(_chunk_plan.chunks)
jm.update_job(
job_id,
progress=(
f"Chunked mode: {_n} chunk(s) × {_chunk_secs} s "
f"with {_chunk_ov} s crossfade overlap..."
),
)
stem_paths = sep.separate_into_parts_chunked(
input_path, str(temp_out),
chunk_plan=_chunk_plan,
batch_size_override=_bs,
num_overlap_override=_overlap,
progress_cb=lambda msg: jm.update_job(job_id, progress=msg),
)
else:
for _attempt in range(3):
try:
stem_paths = sep.separate_into_parts(
input_path, str(temp_out),
batch_size_override=_bs,
num_overlap_override=_overlap,
)
break
except Exception as _exc:
_is_oom = "out of memory" in str(_exc).lower() or "OutOfMemory" in type(_exc).__name__
if not _is_oom:
raise
flush_vram_cache()
if _attempt == 0:
jm.update_job(job_id, progress="GPU OOM — retrying with batch_size=1...")
_bs = 1
elif _attempt == 1:
jm.update_job(job_id, progress="GPU OOM — retrying with batch_size=1, num_overlap=2...")
_bs = 1
_overlap = 2
else:
raise
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
_prefix = stem_prefix or Path(input_path).stem
stem_files = {}
for stem_name, path in sorted(stem_paths.items()):
src = Path(path)
# Map internal names to display names
display_name = stem_name.replace('_', ' ').title()
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
shutil.move(str(src), str(dest))
copy_metadata(input_path, str(dest), stem_label=display_name, copy_lyrics=True)
stem_files[stem_name.lower()] = str(dest)
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
return stem_files
except Exception:
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
raise
# ---------------------------------------------------------------------------
# Karaoke OOM capability-preserving fallback
# ---------------------------------------------------------------------------
def _karaoke_oom_fallback(
job_id: str,
failed_model: str,
failed_model_name: str,
input_path: str,
output_format: str,
stems_dir: Path,
mp3_bitrate: str,
metadata: dict,
stem_prefix: Optional[str],
) -> Dict[str, str]:
"""Try lighter lead/back-capable models after exhausting karaoke OOM retries.
Iterates ``KARAOKE_OOM_FALLBACK_CHAIN`` (skipping the model that already
failed) and returns the first successful result.
Policy (controlled by ``FALLBACK_REQUIRE_LEAD_BACK``):
- ``true`` (default) — only fall back to a model that also produces
*lead* and *backing_instrumental* stems.
- ``false`` — skip the cross-model fallback entirely and raise immediately.
Raises:
RuntimeError: if all fallbacks fail or policy forbids cross-model fallback.
"""
if not FALLBACK_REQUIRE_LEAD_BACK:
raise RuntimeError(
f"'{failed_model_name}' ran out of GPU memory after all parameter "
"reductions (batch_size=1, num_overlap=2). "
"FALLBACK_REQUIRE_LEAD_BACK=false — no cross-model fallback attempted."
)
_tried: list[str] = []
for fb_model in KARAOKE_OOM_FALLBACK_CHAIN:
if fb_model == failed_model:
continue # don't retry the model that already OOM'd
fb_label = MODEL_LABELS.get(fb_model, fb_model)
jm.update_job(
job_id,
progress=(
f"'{failed_model_name}' OOM after full parameter reduction — "
f"falling back to '{fb_label}' (lead/backing-capable)..."
),
)
flush_vram_cache()
if not ensure_model_available(job_id, fb_model):
jm.update_job(
job_id,
progress=f"Fallback '{fb_label}' not available — skipping.",
)
_tried.append(fb_model)
continue
try:
# mdxnet_karaoke is the only entry in the default chain; its runner
# handles all file moves and metadata tags internally.
return _run_mdxnet(
job_id, input_path, fb_model, output_format, stems_dir,
mp3_bitrate=mp3_bitrate,
metadata=metadata,
stem_prefix=stem_prefix,
)
except Exception as fb_exc:
is_oom = (
"out of memory" in str(fb_exc).lower()
or "OutOfMemory" in type(fb_exc).__name__
)
if is_oom:
jm.update_job(
job_id,
progress=f"Fallback '{fb_label}' also OOM — no more fallbacks.",
)
flush_vram_cache()
_tried.append(fb_model)
continue
raise # non-OOM error from fallback — surface it
tried_str = ", ".join(_tried) if _tried else "none available"
raise RuntimeError(
f"'{failed_model_name}' ran out of GPU memory after all parameter reductions "
f"(batch_size=1, num_overlap=2). "
f"All lead/backing-capable fallback models also failed ({tried_str}).\n"
f"Suggested fixes:\n"
f" 1. Use the 'eco' power preset\n"
f" 2. Lower STEMENGINE_GPU_VRAM_RESERVE_MB (reduce headroom, frees VRAM)\n"
f" 3. Restart the container to clear CUDA context fragmentation\n"
f" 4. Use a shorter track (< {LONG_TRACK_DURATION_SECS} s)"
)
# ---------------------------------------------------------------------------
# MelBand Karaoke (Lead/Back) (Python API)
# ---------------------------------------------------------------------------
def _run_mel_band_karaoke(
job_id: str,
input_path: str,
model: str,
output_format: str,
stems_dir: Path,
mp3_bitrate: str = '256k',
metadata: Optional[Dict] = None,
stem_prefix: Optional[str] = None,
batch_size_override: Optional[int] = None,
num_overlap_override: Optional[int] = None,
dim_t_override: Optional[int] = None,
segment_secs_override: Optional[float] = None,
chunking_mode: Optional[str] = None,
chunk_duration_secs: Optional[int] = None,
chunk_overlap_secs: Optional[int] = None,
) -> Dict[str, str]:
"""Run MelBand Karaoke separation (lead vocals/backing+instrumental)."""
# Route to appropriate separator based on model variant
if model == 'mel_band_karaoke_gabox_small':
from .mel_band_specialized_separator import MelBandKaraokeGaboxSmallSeparator
sep_class = MelBandKaraokeGaboxSmallSeparator
model_name = "MelBand Karaoke Gabox (Small)"
elif model == 'mel_band_karaoke_gabox_v1':
from .mel_band_specialized_separator import MelBandKaraokeGaboxV1Separator
sep_class = MelBandKaraokeGaboxV1Separator
model_name = "MelBand Karaoke Gabox V1"
elif model == 'mel_band_karaoke_gabox_v2_exp':
from .mel_band_specialized_separator import MelBandKaraokeGaboxV2ExperimentalSeparator
sep_class = MelBandKaraokeGaboxV2ExperimentalSeparator
model_name = "MelBand Karaoke Gabox V2 (Experimental)"
elif model == 'mel_band_karaoke_gabox':
from .mel_band_specialized_separator import MelBandKaraokeGaboxSeparator
sep_class = MelBandKaraokeGaboxSeparator
model_name = "MelBand Karaoke Gabox"
elif model == 'bs_roformer_karaoke_anvuew':
from .mel_band_specialized_separator import BSRoformerKaraokeAnvuewSeparator
sep_class = BSRoformerKaraokeAnvuewSeparator
model_name = "BS-RoFormer Karaoke Anvuew"
elif model == 'bs_roformer_karaoke_frazer_becruily':
from .mel_band_specialized_separator import MelBandKaraokeFrazerBecruilySeparator
sep_class = MelBandKaraokeFrazerBecruilySeparator
model_name = "BS-RoFormer Karaoke Frazer+Becruily"
else:
from .mel_band_specialized_separator import MelBandKaraokeSeparator
sep_class = MelBandKaraokeSeparator
model_name = "MelBand Karaoke"
jm.update_job(
job_id, status=jm.JobStatus.RUNNING,
progress=f"Loading {model_name} model...",
)
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
# Flush allocator cache immediately before loading the model checkpoint.
# PyTorch holds reserved-but-idle blocks from previous jobs; releasing them
# here maximises contiguous VRAM available for the ~1.7 GB checkpoint.
try:
import gc, torch
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
except Exception:
pass
_dim_t = dim_t_override if dim_t_override is not None else MELBAND_DIM_T_OVERRIDE
sep = _init_melband_separator_compat(
sep_class,
cpu_separation=CPU_SEPARATION,
output_format=output_format,
mp3_bitrate=mp3_bitrate,
diagnostics_enabled=MELBAND_STAGE_DIAGNOSTICS,
diagnostics_cb=_diag_progress_cb(job_id) if MELBAND_STAGE_DIAGNOSTICS else None,
dim_t_override=_dim_t,
segment_secs_override=segment_secs_override,
)
jm.update_job(job_id, progress="Separating lead/backing vocals...")
temp_out = JOBS_DIR / job_id / "mel_band_karaoke_out"
temp_out.mkdir(parents=True, exist_ok=True)
# Duration guardrail / chunking decision.
_bs = batch_size_override
_overlap = num_overlap_override
_dur = get_audio_duration_secs(input_path)
_chunk_mode = (chunking_mode or LONG_TRACK_CHUNKING_MODE)
_chunk_secs = int(chunk_duration_secs or LONG_TRACK_CHUNK_DURATION_SECS)
_chunk_ov = int(chunk_overlap_secs or LONG_TRACK_CHUNK_OVERLAP_SECS)
_guardrail_secs = LONG_TRACK_DURATION_SECS
if model == 'mel_band_karaoke_gabox_small':
_guardrail_secs = min(_guardrail_secs, GABOX_SMALL_MODEL_MAX_DURATION_SECS)
# Chunked path: audio is processed in LONG_TRACK_CHUNK_DURATION_SECS segments
# so peak VRAM scales with chunk size, not track length. The small Gabox model
# is excluded — it is already lightweight and chunking adds unnecessary overhead.
_use_chunking = (
should_chunk(_dur, _chunk_mode, LONG_TRACK_DURATION_SECS)
and model not in ('mel_band_karaoke_gabox_small',)
)
_final_oom_exc: Optional[Exception] = None
if _use_chunking:
_total_samples = max(1, int(round(_dur * 44100)))
_chunk_plan = plan_chunks(
_total_samples, 44100,
_chunk_secs,
_chunk_ov,
)
_n = len(_chunk_plan.chunks)
jm.update_job(
job_id,
progress=(
f"Chunked mode: {_n} chunk(s) \u00d7 {_chunk_secs} s "
f"with {_chunk_ov} s crossfade overlap..."
),
)
try:
stem_paths = sep.separate_into_parts_chunked(
input_path, str(temp_out),
chunk_plan=_chunk_plan,
batch_size_override=_bs,
num_overlap_override=_overlap,
progress_cb=lambda msg: jm.update_job(job_id, progress=msg),
)
except Exception as _exc:
_is_oom = (
"out of memory" in str(_exc).lower()
or "OutOfMemory" in type(_exc).__name__
)
cleanup_temp(temp_out)
if _is_oom:
# Chunked OOM still triggers the capability fallback.
_final_oom_exc = _exc
else:
raise
else:
# Non-chunked: eco-params guardrail + 3-step OOM ladder.
# On exhaustion, _karaoke_oom_fallback() tries a lighter model.
if _dur > 0 and _dur > _guardrail_secs:
jm.update_job(
job_id,
progress=(
f"Long track ({_dur:.0f} s > {_guardrail_secs} s threshold) — "
"forcing eco params (batch_size=1, num_overlap=2) to prevent OOM..."
),
)
if _bs is None or _bs > 1:
_bs = 1
if _overlap is None or _overlap > 2:
_overlap = 2
for _attempt in range(3):
try:
stem_paths = sep.separate_into_parts(
input_path, str(temp_out),
batch_size_override=_bs,
num_overlap_override=_overlap,
)
_final_oom_exc = None
break # success
except Exception as _exc:
_is_oom = (
"out of memory" in str(_exc).lower()
or "OutOfMemory" in type(_exc).__name__
)
if not _is_oom:
cleanup_temp(temp_out)
raise
flush_vram_cache()
if _attempt == 0:
jm.update_job(job_id, progress="GPU OOM \u2014 retrying with batch_size=1...")
_bs = 1
_final_oom_exc = _exc
elif _attempt == 1:
jm.update_job(
job_id,
progress="GPU OOM \u2014 retrying with batch_size=1, num_overlap=2...",
)
_bs = 1
_overlap = 2
_final_oom_exc = _exc
else:
# All parameter reductions exhausted — will try capability fallback
_final_oom_exc = _exc
cleanup_temp(temp_out)
if _final_oom_exc is not None:
return _karaoke_oom_fallback(
job_id=job_id,
failed_model=model,
failed_model_name=model_name,
input_path=input_path,
output_format=output_format,
stems_dir=stems_dir,
mp3_bitrate=mp3_bitrate,
metadata=metadata or {},
stem_prefix=stem_prefix,
)
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
_prefix = stem_prefix or Path(input_path).stem
stem_files = {}
try:
for stem_name, path in sorted(stem_paths.items()):
src = Path(path)
display_name = stem_name.replace('_', ' ').title()
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
shutil.move(str(src), str(dest))
copy_metadata(input_path, str(dest), stem_label=display_name, copy_lyrics=True)
stem_files[stem_name.lower()] = str(dest)
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
return stem_files
except Exception:
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
raise
# ---------------------------------------------------------------------------
# MDX-Net (ONNX) (Python API)
# ---------------------------------------------------------------------------
def _run_mdxnet(
job_id: str,
input_path: str,
model: str,
output_format: str,
stems_dir: Path,
mp3_bitrate: str = '256k',
metadata: Optional[Dict] = None,
stem_prefix: Optional[str] = None,
chunking_mode: Optional[str] = None,
chunk_duration_secs: Optional[int] = None,
chunk_overlap_secs: Optional[int] = None,
) -> Dict[str, str]:
"""
Run MDX-Net separation using ONNX models.
Returns dict mapping stem names to file paths.
"""
from .mdxnet_separator import MDXNetSeparator
from .separation.chunking import plan_chunks, should_chunk
jm.update_job(
job_id, status=jm.JobStatus.RUNNING,
progress=f"Loading MDX-Net model ({model})...",
)
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
try:
import gc, torch
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
except Exception:
pass
sep = MDXNetSeparator(
model_id=model,
cpu_separation=CPU_SEPARATION,
output_format=output_format,
mp3_bitrate=mp3_bitrate,
)
jm.update_job(job_id, progress="Separating with MDX-Net...")
temp_out = JOBS_DIR / job_id / "mdxnet_out"
temp_out.mkdir(parents=True, exist_ok=True)
# Duration guardrail / chunking decision.
_dur = get_audio_duration_secs(input_path)
_chunk_mode = (chunking_mode or LONG_TRACK_CHUNKING_MODE)
_chunk_secs = int(chunk_duration_secs or LONG_TRACK_CHUNK_DURATION_SECS)
_chunk_ov = int(chunk_overlap_secs or LONG_TRACK_CHUNK_OVERLAP_SECS)
_use_chunking = should_chunk(_dur, _chunk_mode, LONG_TRACK_DURATION_SECS)
try:
if _use_chunking:
_total_samples = max(1, int(round(_dur * 44100)))
_chunk_plan = plan_chunks(
_total_samples, 44100,
_chunk_secs,
_chunk_ov,
)
_n = len(_chunk_plan.chunks)
jm.update_job(
job_id,
progress=(
f"Chunked mode: {_n} chunk(s) × {_chunk_secs} s "
f"with {_chunk_ov} s crossfade overlap..."
),
)
stem_paths = sep.separate_into_parts_chunked(
input_path, str(temp_out),
chunk_plan=_chunk_plan,
progress_cb=lambda msg: jm.update_job(job_id, progress=msg),
)
else:
stem_paths = sep.separate_into_parts(input_path, str(temp_out))
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
_prefix = stem_prefix or Path(input_path).stem
stem_files = {}
for stem_name, path in sorted(stem_paths.items()):
src = Path(path)
display_name = stem_name.replace('_', ' ').title()
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
shutil.move(str(src), str(dest))
copy_metadata(input_path, str(dest), stem_label=display_name, copy_lyrics=True)
stem_files[stem_name.lower()] = str(dest)
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
return stem_files
except Exception:
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
raise
# ---------------------------------------------------------------------------
# MelBand RoFormer (Python API)
# ---------------------------------------------------------------------------
def _run_mel_band_roformer(
job_id: str,
input_path: str,
model: str,
output_format: str,
stems_dir: Path,
mp3_bitrate: str = '256k',
metadata: Optional[Dict] = None,
stem_prefix: Optional[str] = None,
batch_size_override: Optional[int] = None,
num_overlap_override: Optional[int] = None,
dim_t_override: Optional[int] = None,
segment_secs_override: Optional[float] = None,
chunking_mode: Optional[str] = None,
chunk_duration_secs: Optional[int] = None,
chunk_overlap_secs: Optional[int] = None,
) -> Dict[str, str]:
"""
Run MelBand RoFormer separation. Returns dict mapping stem names to file paths.
"""
from .mel_band_roformer_separator import MelBandRoformerSeparator
jm.update_job(
job_id, status=jm.JobStatus.RUNNING,
progress="Loading MelBand RoFormer model...",
)
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
try:
import gc, torch
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
except Exception:
pass
_dim_t = dim_t_override if dim_t_override is not None else MELBAND_DIM_T_OVERRIDE
sep = MelBandRoformerSeparator(
cpu_separation=CPU_SEPARATION,
output_format=output_format,
mp3_bitrate=mp3_bitrate,
diagnostics_enabled=MELBAND_STAGE_DIAGNOSTICS,
diagnostics_cb=_diag_progress_cb(job_id) if MELBAND_STAGE_DIAGNOSTICS else None,
dim_t_override=_dim_t,
segment_secs_override=segment_secs_override,
)
jm.update_job(job_id, progress="Separating with MelBand RoFormer... (this may take a while)")
# Use a temp directory then move to project
temp_out = JOBS_DIR / job_id / "mel_band_out"
temp_out.mkdir(parents=True, exist_ok=True)
# Duration guardrail / chunking decision.
_bs = batch_size_override
_overlap = num_overlap_override
_dur = get_audio_duration_secs(input_path)
_chunk_mode = (chunking_mode or LONG_TRACK_CHUNKING_MODE)
_chunk_secs = int(chunk_duration_secs or LONG_TRACK_CHUNK_DURATION_SECS)
_chunk_ov = int(chunk_overlap_secs or LONG_TRACK_CHUNK_OVERLAP_SECS)
_use_chunking = should_chunk(_dur, _chunk_mode, LONG_TRACK_DURATION_SECS)
if _use_chunking:
_total_samples = max(1, int(round(_dur * 44100)))
_chunk_plan = plan_chunks(
_total_samples, 44100,
_chunk_secs,
_chunk_ov,
)
_n = len(_chunk_plan.chunks)
jm.update_job(
job_id,
progress=(
f"Chunked mode: {_n} chunk(s) \u00d7 {_chunk_secs} s "
f"with {_chunk_ov} s crossfade overlap..."
),
)
try:
stem_paths = sep.separate_into_parts_chunked(
input_path, str(temp_out),
chunk_plan=_chunk_plan,
batch_size_override=_bs,
num_overlap_override=_overlap,
progress_cb=lambda msg: jm.update_job(job_id, progress=msg),
)
except Exception:
cleanup_temp(temp_out)
raise
else:
# Non-chunked: eco-params guardrail + 3-step OOM ladder.
if _dur > 0 and _dur > LONG_TRACK_DURATION_SECS:
jm.update_job(
job_id,
progress=(
f"Long track ({_dur:.0f} s > {LONG_TRACK_DURATION_SECS} s threshold) — "
"forcing eco params (batch_size=1, num_overlap=2) to prevent OOM..."
),
)
if _bs is None or _bs > 1:
_bs = 1
if _overlap is None or _overlap > 2:
_overlap = 2
# 3-step OOM ladder: full params → batch_size=1 → batch_size=1 + num_overlap=2.
for _attempt in range(3):
try:
stem_paths = sep.separate_into_parts(
input_path, str(temp_out),
batch_size_override=_bs,
num_overlap_override=_overlap,
)
break
except Exception as _exc:
_is_oom = (
"out of memory" in str(_exc).lower()
or "OutOfMemory" in type(_exc).__name__
)
if not _is_oom:
cleanup_temp(temp_out)
raise
flush_vram_cache()
if _attempt == 0:
jm.update_job(job_id, progress="GPU OOM \u2014 retrying with batch_size=1...")
_bs = 1
elif _attempt == 1:
jm.update_job(
job_id,
progress="GPU OOM \u2014 retrying with batch_size=1, num_overlap=2...",
)
_bs = 1
_overlap = 2
else:
cleanup_temp(temp_out)
raise
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
# Move to project stems folder with proper naming
_prefix = stem_prefix or Path(input_path).stem
stem_files = {}
try:
for stem_name, path in sorted(stem_paths.items()):
src = Path(path)
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
shutil.move(str(src), str(dest))
copy_metadata(input_path, str(dest), stem_label=stem_name.capitalize(), copy_lyrics=True)
stem_files[stem_name.lower()] = str(dest)
# Cleanup temp directory
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
return stem_files
except Exception:
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
raise
# ---------------------------------------------------------------------------
# BS-RoFormer (Python API)
# ---------------------------------------------------------------------------
def _run_bsroformer(
job_id: str,
input_path: str,
model: str,
output_format: str,
stems_dir: Path,
mp3_bitrate: str = '256k',
metadata: Optional[Dict] = None,
stem_prefix: Optional[str] = None,
batch_size_override: Optional[int] = None,
num_overlap_override: Optional[int] = None,
chunking_mode: Optional[str] = None,
chunk_duration_secs: Optional[int] = None,
chunk_overlap_secs: Optional[int] = None,
model_path_override: Optional[str] = None,
config_path_override: Optional[str] = None,
) -> Dict[str, str]:
"""
Run BS-RoFormer separation. Returns dict mapping stem names to file paths.
"""
from .bs_roformer_separator import BSRoformerSeparator
stem_mode = BSR_STEM_MODE.get(model, '4stem')
jm.update_job(
job_id, status=jm.JobStatus.RUNNING,
progress=f"Loading BS-RoFormer ({stem_mode}) model...",
)
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
try:
import gc, torch
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
except Exception:
pass
sep = BSRoformerSeparator(
model_path=model_path_override,
config_path=config_path_override,
cpu_separation=CPU_SEPARATION,
output_format=output_format,
mp3_bitrate=mp3_bitrate,
stem_mode=stem_mode,
)
jm.update_job(job_id, progress="Separating with BS-RoFormer... (this may take a while)")
temp_out = JOBS_DIR / job_id / "bsr_out"
temp_out.mkdir(parents=True, exist_ok=True)
_bs = batch_size_override
_dur = get_audio_duration_secs(input_path)
_chunk_mode = (chunking_mode or LONG_TRACK_CHUNKING_MODE)
_chunk_secs = int(chunk_duration_secs or LONG_TRACK_CHUNK_DURATION_SECS)
_chunk_ov = int(chunk_overlap_secs or LONG_TRACK_CHUNK_OVERLAP_SECS)
_use_chunking = should_chunk(_dur, _chunk_mode, LONG_TRACK_DURATION_SECS)
if _use_chunking:
_total_samples = max(1, int(round(_dur * 44100)))
_chunk_plan = plan_chunks(
_total_samples, 44100,
_chunk_secs,
_chunk_ov,
)
_n = len(_chunk_plan.chunks)
jm.update_job(
job_id,
progress=(
f"Chunked mode: {_n} chunk(s) \u00d7 {_chunk_secs} s "
f"with {_chunk_ov} s crossfade overlap..."
),
)
try:
stem_paths = sep.separate_into_parts_chunked(
input_path, str(temp_out),
chunk_plan=_chunk_plan,
batch_size_override=_bs,
num_overlap_override=num_overlap_override,
progress_cb=lambda msg: jm.update_job(job_id, progress=msg),
)
except Exception:
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
raise
else:
for _attempt in range(2):
try:
stem_paths = sep.separate_into_parts(
input_path, str(temp_out),
batch_size_override=_bs,
num_overlap_override=num_overlap_override,
)
break
except Exception as _exc:
_is_oom = "out of memory" in str(_exc).lower() or "OutOfMemory" in type(_exc).__name__
if _is_oom and _attempt == 0:
import gc, torch
jm.update_job(job_id, progress="GPU OOM \u2014 retrying with batch_size=1...")
gc.collect()
torch.cuda.empty_cache()
_bs = 1
continue
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
raise
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
_prefix = stem_prefix or Path(input_path).stem
stem_files = {}
try:
for stem_name, path in sorted(stem_paths.items()):
src = Path(path)
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
shutil.move(str(src), str(dest))
copy_metadata(input_path, str(dest), stem_label=stem_name.capitalize(), copy_lyrics=True)
stem_files[stem_name.lower()] = str(dest)
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
return stem_files
except Exception:
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
raise
# ---------------------------------------------------------------------------
# MelBand RoFormer Deux Becruily (Python API)
# ---------------------------------------------------------------------------
def _run_mel_band_deux_becruily(
job_id: str,
input_path: str,
model: str,
output_format: str,
stems_dir: Path,
mp3_bitrate: str = '256k',
metadata: Optional[Dict] = None,
stem_prefix: Optional[str] = None,
batch_size_override: Optional[int] = None,
num_overlap_override: Optional[int] = None,
dim_t_override: Optional[int] = None,
segment_secs_override: Optional[float] = None,
chunking_mode: Optional[str] = None,
chunk_duration_secs: Optional[int] = None,
chunk_overlap_secs: Optional[int] = None,
) -> Dict[str, str]:
"""Run MelBand RoFormer Deux Becruily separation (2-stem vocals)."""
from .mel_band_specialized_separator import MelBandDeuxBecruilySeparator
jm.update_job(
job_id, status=jm.JobStatus.RUNNING,
progress="Loading MelBand Deux Becruily model...",
)
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
try:
import gc, torch
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
except Exception:
pass
_dim_t = dim_t_override if dim_t_override is not None else MELBAND_DIM_T_OVERRIDE
sep = MelBandDeuxBecruilySeparator(
cpu_separation=CPU_SEPARATION,
output_format=output_format,
mp3_bitrate=mp3_bitrate,
diagnostics_enabled=MELBAND_STAGE_DIAGNOSTICS,
diagnostics_cb=_diag_progress_cb(job_id) if MELBAND_STAGE_DIAGNOSTICS else None,
dim_t_override=_dim_t,
segment_secs_override=segment_secs_override,
)
jm.update_job(job_id, progress="Separating with Deux Becruily...")
temp_out = JOBS_DIR / job_id / "mel_band_deux_out"
temp_out.mkdir(parents=True, exist_ok=True)
_bs = batch_size_override
_overlap = num_overlap_override
_dur = get_audio_duration_secs(input_path)
_chunk_mode = (chunking_mode or LONG_TRACK_CHUNKING_MODE)
_chunk_secs = int(chunk_duration_secs or LONG_TRACK_CHUNK_DURATION_SECS)
_chunk_ov = int(chunk_overlap_secs or LONG_TRACK_CHUNK_OVERLAP_SECS)
_use_chunking = should_chunk(_dur, _chunk_mode, LONG_TRACK_DURATION_SECS)
try:
if _use_chunking:
_total_samples = max(1, int(round(_dur * 44100)))
_chunk_plan = plan_chunks(_total_samples, 44100, _chunk_secs, _chunk_ov)
_n = len(_chunk_plan.chunks)
jm.update_job(
job_id,
progress=(
f"Chunked mode: {_n} chunk(s) × {_chunk_secs} s "
f"with {_chunk_ov} s crossfade overlap..."
),
)
stem_paths = sep.separate_into_parts_chunked(
input_path, str(temp_out),
chunk_plan=_chunk_plan,
batch_size_override=_bs,
num_overlap_override=_overlap,
progress_cb=lambda msg: jm.update_job(job_id, progress=msg),
)
else:
for _attempt in range(3):
try:
stem_paths = sep.separate_into_parts(
input_path, str(temp_out),
batch_size_override=_bs,
num_overlap_override=_overlap,
)
break
except Exception as _exc:
_is_oom = "out of memory" in str(_exc).lower() or "OutOfMemory" in type(_exc).__name__
if not _is_oom:
raise
flush_vram_cache()
if _attempt == 0:
jm.update_job(job_id, progress="GPU OOM — retrying with batch_size=1...")
_bs = 1
elif _attempt == 1:
jm.update_job(job_id, progress="GPU OOM — retrying with batch_size=1, num_overlap=2...")
_bs = 1
_overlap = 2
else:
raise
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
_prefix = stem_prefix or Path(input_path).stem
stem_files = {}
for stem_name, path in sorted(stem_paths.items()):
src = Path(path)
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
shutil.move(str(src), str(dest))
copy_metadata(input_path, str(dest), stem_label=stem_name.capitalize(), copy_lyrics=True)
stem_files[stem_name.lower()] = str(dest)
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
return stem_files
except Exception:
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
raise
# ---------------------------------------------------------------------------
# SCNet XL (Python API)
# ---------------------------------------------------------------------------
def _run_scnet(
job_id: str,
input_path: str,
model: str,
output_format: str,
stems_dir: Path,
mp3_bitrate: str = '256k',
metadata: Optional[Dict] = None,
stem_prefix: Optional[str] = None,
chunk_size_override: Optional[int] = None,
num_overlap_override: Optional[int] = None,
chunking_mode: Optional[str] = None,
chunk_duration_secs: Optional[int] = None,
chunk_overlap_secs: Optional[int] = None,
) -> Dict[str, str]:
"""Run SCNet XL separation (state-of-the-art 2024-2025)."""
from .scnet_separator import SCNetSeparator
from .separation.chunking import plan_chunks, should_chunk
jm.update_job(
job_id, status=jm.JobStatus.RUNNING,
progress=f"Loading SCNet model ({model})...",
)
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
try:
import gc, torch
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
except Exception:
pass
sep = SCNetSeparator(
model_id=model,
cpu_separation=CPU_SEPARATION,
output_format=output_format,
mp3_bitrate=mp3_bitrate,
)
jm.update_job(job_id, progress="Separating with SCNet... (this may take a while)")
temp_out = JOBS_DIR / job_id / "scnet_out"
temp_out.mkdir(parents=True, exist_ok=True)
# Duration guardrail / chunking decision.
_dur = get_audio_duration_secs(input_path)
_chunk_mode = (chunking_mode or LONG_TRACK_CHUNKING_MODE)
_chunk_secs = int(chunk_duration_secs or LONG_TRACK_CHUNK_DURATION_SECS)
_chunk_ov = int(chunk_overlap_secs or LONG_TRACK_CHUNK_OVERLAP_SECS)
_use_chunking = should_chunk(_dur, _chunk_mode, LONG_TRACK_DURATION_SECS)
_cs = chunk_size_override
for _attempt in range(2):
try:
if _use_chunking:
_total_samples = max(1, int(round(_dur * 44100)))
_chunk_plan = plan_chunks(
_total_samples, 44100,
_chunk_secs,
_chunk_ov,
)
_n = len(_chunk_plan.chunks)
jm.update_job(
job_id,
progress=(
f"Chunked mode: {_n} chunk(s) × {_chunk_secs} s "
f"with {_chunk_ov} s crossfade overlap..."
),
)
stem_paths = sep.separate_into_parts_chunked(
input_path, str(temp_out),
chunk_plan=_chunk_plan,
chunk_size_override=_cs,
num_overlap_override=num_overlap_override,
progress_cb=lambda msg: jm.update_job(job_id, progress=msg),
)
else:
stem_paths = sep.separate_into_parts(
input_path, str(temp_out),
chunk_size_override=_cs,
num_overlap_override=num_overlap_override,
)
break
except Exception as _exc:
_is_oom = "out of memory" in str(_exc).lower() or "OutOfMemory" in type(_exc).__name__
if _is_oom and _attempt == 0:
import gc, torch
jm.update_job(job_id, progress="GPU OOM — retrying with smaller chunk size...")
gc.collect()
torch.cuda.empty_cache()
_cs = (_cs or 262144) // 2
continue
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
raise
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
_prefix = stem_prefix or Path(input_path).stem
stem_files = {}
try:
for stem_name, path in sorted(stem_paths.items()):
src = Path(path)
display_name = stem_name.replace('_', ' ').title()
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
shutil.move(str(src), str(dest))
copy_metadata(input_path, str(dest), stem_label=display_name, copy_lyrics=True)
stem_files[stem_name.lower()] = str(dest)
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
return stem_files
except Exception:
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
raise
# ---------------------------------------------------------------------------
# MDX23C TFC-TDF U-Net (Python API)
# ---------------------------------------------------------------------------
def _run_mdx23c(
job_id: str,
input_path: str,
model: str,
output_format: str,
stems_dir: Path,
mp3_bitrate: str = '256k',
metadata: Optional[Dict] = None,
stem_prefix: Optional[str] = None,
chunk_size_override: Optional[int] = None,
num_overlap_override: Optional[int] = None,
chunking_mode: Optional[str] = None,
chunk_duration_secs: Optional[int] = None,
chunk_overlap_secs: Optional[int] = None,
) -> Dict[str, str]:
"""Run MDX23C TFC-TDF U-Net separation (drum component separation)."""
from .mdx23c_separator import MDX23CSeparator
from .separation.chunking import plan_chunks, should_chunk
jm.update_job(
job_id, status=jm.JobStatus.RUNNING,
progress=f"Loading MDX23C model ({model})...",
)
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
try:
import gc, torch
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
except Exception:
pass
sep = MDX23CSeparator(
model_id=model,
cpu_separation=CPU_SEPARATION,
output_format=output_format,
mp3_bitrate=mp3_bitrate,
)
jm.update_job(job_id, progress="Separating with MDX23C... (this may take a while)")
temp_out = JOBS_DIR / job_id / "mdx23c_out"
temp_out.mkdir(parents=True, exist_ok=True)
_dur = get_audio_duration_secs(input_path)
_chunk_mode = (chunking_mode or LONG_TRACK_CHUNKING_MODE)
_chunk_secs = int(chunk_duration_secs or LONG_TRACK_CHUNK_DURATION_SECS)
_chunk_ov = int(chunk_overlap_secs or LONG_TRACK_CHUNK_OVERLAP_SECS)
_use_chunking = should_chunk(_dur, _chunk_mode, LONG_TRACK_DURATION_SECS)
_cs = chunk_size_override
for _attempt in range(2):
try:
if _use_chunking:
_total_samples = max(1, int(round(_dur * 44100)))
_chunk_plan = plan_chunks(
_total_samples, 44100,
_chunk_secs,
_chunk_ov,
)
_n = len(_chunk_plan.chunks)
jm.update_job(
job_id,
progress=(
f"Chunked mode: {_n} chunk(s) × {_chunk_secs} s "
f"with {_chunk_ov} s crossfade overlap..."
),
)
stem_paths = sep.separate_into_parts_chunked(
input_path, str(temp_out),
chunk_plan=_chunk_plan,
chunk_size_override=_cs,
num_overlap_override=num_overlap_override,
progress_cb=lambda msg: jm.update_job(job_id, progress=msg),
)
else:
stem_paths = sep.separate_into_parts(
input_path, str(temp_out),
chunk_size_override=_cs,
num_overlap_override=num_overlap_override,
)
break
except Exception as _exc:
_is_oom = "out of memory" in str(_exc).lower() or "OutOfMemory" in type(_exc).__name__
if _is_oom and _attempt == 0:
import gc, torch
jm.update_job(job_id, progress="GPU OOM — retrying with smaller chunk size...")
gc.collect()
torch.cuda.empty_cache()
_cs = (_cs or 130560) // 2
continue
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
raise
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
_prefix = stem_prefix or Path(input_path).stem
stem_files = {}
try:
for stem_name, path in sorted(stem_paths.items()):
src = Path(path)
display_name = stem_name.replace('_', ' ').title()
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
shutil.move(str(src), str(dest))
copy_metadata(input_path, str(dest), stem_label=display_name, copy_lyrics=True)
stem_files[stem_name.lower()] = str(dest)
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
return stem_files
except Exception:
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
raise
# ---------------------------------------------------------------------------
# BS-RoFormer SW (Swin Transformer variant) (Python API)
# ---------------------------------------------------------------------------
def _run_bsroformer_sw(
job_id: str,
input_path: str,
model: str,
output_format: str,
stems_dir: Path,
mp3_bitrate: str = '256k',
metadata: Optional[Dict] = None,
stem_prefix: Optional[str] = None,
) -> Dict[str, str]:
"""Run BS-RoFormer SW (Swin Transformer) separation (6-stem)."""
from .bs_roformer_sw_separator import BSRoformerSWSeparator
jm.update_job(
job_id, status=jm.JobStatus.RUNNING,
progress="Loading BS-RoFormer SW model...",
)
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
try:
import gc, torch
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
except Exception:
pass
sep = BSRoformerSWSeparator(
cpu_separation=CPU_SEPARATION,
output_format=output_format,
mp3_bitrate=mp3_bitrate,
)
jm.update_job(job_id, progress="Separating with BS-RoFormer SW... (this may take a while)")
temp_out = JOBS_DIR / job_id / "bsr_sw_out"
temp_out.mkdir(parents=True, exist_ok=True)
try:
stem_paths = sep.separate_into_parts(input_path, str(temp_out))
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
_prefix = stem_prefix or Path(input_path).stem
stem_files = {}
for stem_name, path in sorted(stem_paths.items()):
src = Path(path)
display_name = stem_name.replace('_', ' ').title()
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
shutil.move(str(src), str(dest))
copy_metadata(input_path, str(dest), stem_label=display_name, copy_lyrics=True)
stem_files[stem_name.lower()] = str(dest)
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
return stem_files
except Exception:
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
raise
# ---------------------------------------------------------------------------
# MedleyVox (HuggingFace) (Python API)
# ---------------------------------------------------------------------------
def _run_medleyvox(
job_id: str,
input_path: str,
model: str,
output_format: str,
stems_dir: Path,
mp3_bitrate: str = '256k',
metadata: Optional[Dict] = None,
stem_prefix: Optional[str] = None,
chunk_duration_secs: Optional[int] = None,
) -> Dict[str, str]:
"""Run MedleyVox separation (multi-singer vocal separation)."""
from .medleyvox_separator import MedleyVoxSeparator
jm.update_job(
job_id, status=jm.JobStatus.RUNNING,
progress=f"Loading MedleyVox model ({model})...",
)
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
try:
import gc, torch
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
except Exception:
pass
sep = MedleyVoxSeparator(
model_id=model,
cpu_separation=CPU_SEPARATION,
output_format=output_format,
mp3_bitrate=mp3_bitrate,
chunk_secs_override=chunk_duration_secs,
)
jm.update_job(job_id, progress="Separating with MedleyVox...")
temp_out = JOBS_DIR / job_id / "medleyvox_out"
temp_out.mkdir(parents=True, exist_ok=True)
try:
stem_paths = sep.separate_into_parts(input_path, str(temp_out))
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
_prefix = stem_prefix or Path(input_path).stem
stem_files = {}
for stem_name, path in sorted(stem_paths.items()):
src = Path(path)
display_name = stem_name.replace('_', ' ').title()
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
shutil.move(str(src), str(dest))
copy_metadata(input_path, str(dest), stem_label=display_name, copy_lyrics=True)
stem_files[stem_name.lower()] = str(dest)
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
return stem_files
except Exception:
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
raise
# ---------------------------------------------------------------------------
# Apollo (Vocal Enhancement/Reverb Removal) (Python API)
# ---------------------------------------------------------------------------
def _run_apollo(
job_id: str,
input_path: str,
model: str,
output_format: str,
stems_dir: Path,
mp3_bitrate: str = '256k',
metadata: Optional[Dict] = None,
stem_prefix: Optional[str] = None,
chunking_mode: Optional[str] = None,
chunk_duration_secs: Optional[int] = None,
chunk_overlap_secs: Optional[int] = None,
) -> Dict[str, str]:
"""Run Apollo vocal enhancement/reverb removal."""
from .apollo_separator import ApolloSeparator
jm.update_job(
job_id, status=jm.JobStatus.RUNNING,
progress=f"Loading Apollo model ({model})...",
)
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
try:
import gc, torch
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
except Exception:
pass
_dur = get_audio_duration_secs(input_path)
_chunk_mode = (chunking_mode or LONG_TRACK_CHUNKING_MODE)
_chunk_secs = int(chunk_duration_secs or LONG_TRACK_CHUNK_DURATION_SECS)
_chunk_ov = int(chunk_overlap_secs or LONG_TRACK_CHUNK_OVERLAP_SECS)
_use_chunking = should_chunk(_dur, _chunk_mode, LONG_TRACK_DURATION_SECS)
# Chunk adapter path reads intermediate stems back for overlap-add,
# so force lossless PCM intermediates for broad decoder compatibility.
_sep_output_format = 'wav' if _use_chunking else output_format
sep = ApolloSeparator(
model_id=model,
cpu_separation=CPU_SEPARATION,
output_format=_sep_output_format,
mp3_bitrate=mp3_bitrate,
)
jm.update_job(job_id, progress="Processing with Apollo...")
temp_out = JOBS_DIR / job_id / "apollo_out"
temp_out.mkdir(parents=True, exist_ok=True)
try:
if _use_chunking:
stem_paths = _separate_into_parts_chunked_via_files(
job_id=job_id,
input_path=input_path,
sep=sep,
temp_out=temp_out,
chunk_duration_secs=_chunk_secs,
chunk_overlap_secs=_chunk_ov,
output_format=output_format,
mp3_bitrate=mp3_bitrate,
)
else:
stem_paths = sep.separate_into_parts(input_path, str(temp_out))
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
_prefix = stem_prefix or Path(input_path).stem
stem_files = {}
for stem_name, path in sorted(stem_paths.items()):
src = Path(path)
display_name = stem_name.replace('_', ' ').title()
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
shutil.move(str(src), str(dest))
copy_metadata(input_path, str(dest), stem_label=display_name, copy_lyrics=True)
stem_files[stem_name.lower()] = str(dest)
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
return stem_files
except Exception:
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
raise
# ---------------------------------------------------------------------------
# Aspiration (Breath/Aspiration Removal) (Python API)
# ---------------------------------------------------------------------------
def _run_aspiration(
job_id: str,
input_path: str,
model: str,
output_format: str,
stems_dir: Path,
mp3_bitrate: str = '256k',
metadata: Optional[Dict] = None,
stem_prefix: Optional[str] = None,
chunking_mode: Optional[str] = None,
chunk_duration_secs: Optional[int] = None,
chunk_overlap_secs: Optional[int] = None,
) -> Dict[str, str]:
"""Run Aspiration removal (breath/aspiration separation)."""
from .aspiration_separator import AspirationSeparator
jm.update_job(
job_id, status=jm.JobStatus.RUNNING,
progress=f"Loading Aspiration model ({model})...",
)
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
try:
import gc, torch
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
except Exception:
pass
_dur = get_audio_duration_secs(input_path)
_chunk_mode = (chunking_mode or LONG_TRACK_CHUNKING_MODE)
_chunk_secs = int(chunk_duration_secs or LONG_TRACK_CHUNK_DURATION_SECS)
_chunk_ov = int(chunk_overlap_secs or LONG_TRACK_CHUNK_OVERLAP_SECS)
_use_chunking = should_chunk(_dur, _chunk_mode, LONG_TRACK_DURATION_SECS)
# Chunk adapter path reads intermediate stems back for overlap-add,
# so force lossless PCM intermediates for broad decoder compatibility.
_sep_output_format = 'wav' if _use_chunking else output_format
sep = AspirationSeparator(
model_id=model,
cpu_separation=CPU_SEPARATION,
output_format=_sep_output_format,
mp3_bitrate=mp3_bitrate,
)
jm.update_job(job_id, progress="Removing aspiration with MelBand...")
temp_out = JOBS_DIR / job_id / "aspiration_out"
temp_out.mkdir(parents=True, exist_ok=True)
try:
if _use_chunking:
stem_paths = _separate_into_parts_chunked_via_files(
job_id=job_id,
input_path=input_path,
sep=sep,
temp_out=temp_out,
chunk_duration_secs=_chunk_secs,
chunk_overlap_secs=_chunk_ov,
output_format=output_format,
mp3_bitrate=mp3_bitrate,
)
else:
stem_paths = sep.separate_into_parts(input_path, str(temp_out))
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
_prefix = stem_prefix or Path(input_path).stem
stem_files = {}
for stem_name, path in sorted(stem_paths.items()):
src = Path(path)
display_name = stem_name.replace('_', ' ').title()
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
shutil.move(str(src), str(dest))
copy_metadata(input_path, str(dest), stem_label=display_name, copy_lyrics=True)
stem_files[stem_name.lower()] = str(dest)
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
return stem_files
except Exception:
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
raise
# ---------------------------------------------------------------------------
# Bandit-v2 (Multi-Language/Multi-Stem) (Python API)
# ---------------------------------------------------------------------------
def _run_bandit(
job_id: str,
input_path: str,
model: str,
output_format: str,
stems_dir: Path,
mp3_bitrate: str = '256k',
metadata: Optional[Dict] = None,
stem_prefix: Optional[str] = None,
chunking_mode: Optional[str] = None,
chunk_duration_secs: Optional[int] = None,
chunk_overlap_secs: Optional[int] = None,
) -> Dict[str, str]:
"""Run Bandit-v2 separation (multi-language/multi-stem)."""
from .bandit_separator import BanditSeparator
jm.update_job(
job_id, status=jm.JobStatus.RUNNING,
progress=f"Loading Bandit model ({model})...",
)
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
try:
import gc, torch
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
except Exception:
pass
_dur = get_audio_duration_secs(input_path)
_chunk_mode = (chunking_mode or LONG_TRACK_CHUNKING_MODE)
_chunk_secs = int(chunk_duration_secs or LONG_TRACK_CHUNK_DURATION_SECS)
_chunk_ov = int(chunk_overlap_secs or LONG_TRACK_CHUNK_OVERLAP_SECS)
_use_chunking = should_chunk(_dur, _chunk_mode, LONG_TRACK_DURATION_SECS)
# Chunk adapter path reads intermediate stems back for overlap-add,
# so force lossless PCM intermediates for broad decoder compatibility.
_sep_output_format = 'wav' if _use_chunking else output_format
sep = BanditSeparator(
model_id=model,
cpu_separation=CPU_SEPARATION,
output_format=_sep_output_format,
mp3_bitrate=mp3_bitrate,
)
jm.update_job(job_id, progress="Separating with Bandit-v2...")
temp_out = JOBS_DIR / job_id / "bandit_out"
temp_out.mkdir(parents=True, exist_ok=True)
try:
if _use_chunking:
stem_paths = _separate_into_parts_chunked_via_files(
job_id=job_id,
input_path=input_path,
sep=sep,
temp_out=temp_out,
chunk_duration_secs=_chunk_secs,
chunk_overlap_secs=_chunk_ov,
output_format=output_format,
mp3_bitrate=mp3_bitrate,
)
else:
stem_paths = sep.separate_into_parts(input_path, str(temp_out))
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
_prefix = stem_prefix or Path(input_path).stem
stem_files = {}
for stem_name, path in sorted(stem_paths.items()):
src = Path(path)
display_name = stem_name.replace('_', ' ').title()
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
shutil.move(str(src), str(dest))
copy_metadata(input_path, str(dest), stem_label=display_name, copy_lyrics=True)
stem_files[stem_name.lower()] = str(dest)
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
return stem_files
except Exception:
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
raise
# ---------------------------------------------------------------------------
# Ensemble (Combines Multiple Models) (Python API)
# ---------------------------------------------------------------------------
def _run_ensemble(
job_id: str,
input_path: str,
model: str,
output_format: str,
stems_dir: Path,
mp3_bitrate: str = '256k',
metadata: Optional[Dict] = None,
stem_prefix: Optional[str] = None,
) -> Dict[str, str]:
"""
Run Ensemble separation (combines multiple models for best quality).
Models combined:
- ensemble_vocal_best: BS Roformer x2 + MelBand Roformer ft + SCNet XL IHF
- ensemble_4stem: Multiple 4-stem models averaged
- ensemble_6stem: Multiple 6-stem models averaged
"""
from .ensemble_separator import EnsembleSeparator
jm.update_job(
job_id, status=jm.JobStatus.RUNNING,
progress=f"Loading Ensemble models ({model})...",
)
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
try:
import gc, torch
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
except Exception:
pass
sep = EnsembleSeparator(
ensemble_id=model,
cpu_separation=CPU_SEPARATION,
output_format=output_format,
mp3_bitrate=mp3_bitrate,
)
jm.update_job(job_id, progress="Running Ensemble separation... (this will take significantly longer)")
temp_out = JOBS_DIR / job_id / "ensemble_out"
temp_out.mkdir(parents=True, exist_ok=True)
try:
stem_paths = sep.separate_into_parts(input_path, str(temp_out))
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
_prefix = stem_prefix or Path(input_path).stem
stem_files = {}
for stem_name, path in sorted(stem_paths.items()):
src = Path(path)
display_name = stem_name.replace('_', ' ').title()
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
shutil.move(str(src), str(dest))
copy_metadata(input_path, str(dest), stem_label=display_name, copy_lyrics=True)
stem_files[stem_name.lower()] = str(dest)
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
return stem_files
except Exception:
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
raise
# ---------------------------------------------------------------------------
# SepACap (A Cappella Multi-Singer Vocal Separation) (Python API)
# ---------------------------------------------------------------------------
def _run_sepacap(
job_id: str,
input_path: str,
output_format: str,
stems_dir: Path,
mp3_bitrate: str = '256k',
metadata: Optional[Dict] = None,
stem_prefix: Optional[str] = None,
chunk_samples: int = 48_000,
) -> Dict[str, str]:
"""Run SepACap a cappella vocal separation into 7 stems."""
from .sepacap_separator import SepACapSeparator
jm.update_job(
job_id, status=jm.JobStatus.RUNNING,
progress="Loading SepACap model (clones repo + downloads weights on first use)...",
)
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
try:
import gc, torch
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
except Exception:
pass
sep = SepACapSeparator(
cpu_separation=CPU_SEPARATION,
output_format=output_format,
mp3_bitrate=mp3_bitrate,
chunk_samples=chunk_samples,
)
jm.update_job(job_id, progress="Separating a cappella vocals with SepACap... (this may take a while)")
temp_out = JOBS_DIR / job_id / "sepacap_out"
temp_out.mkdir(parents=True, exist_ok=True)
try:
stem_paths = sep.separate_into_parts(input_path, str(temp_out))
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
_prefix = stem_prefix or Path(input_path).stem
stem_files = {}
for stem_name, path in sorted(stem_paths.items()):
src = Path(path)
display_name = stem_name.replace('_', ' ').title()
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
shutil.move(str(src), str(dest))
copy_metadata(input_path, str(dest), stem_label=display_name, copy_lyrics=True)
stem_files[stem_name.lower()] = str(dest)
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
return stem_files
except Exception:
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
raise
# ---------------------------------------------------------------------------
# SepReformer (2-Speaker Speech Separation) (Python API)
# ---------------------------------------------------------------------------
def _run_sepreformer(
job_id: str,
input_path: str,
model: str,
output_format: str,
stems_dir: Path,
mp3_bitrate: str = '256k',
metadata: Optional[Dict] = None,
stem_prefix: Optional[str] = None,
chunk_samples: int = 48_000,
) -> Dict[str, str]:
"""Run SepReformer 2-speaker speech separation."""
from .sepreformer_separator import SepReformerSeparator
jm.update_job(
job_id, status=jm.JobStatus.RUNNING,
progress=f"Loading SepReformer model ({model}) — clones repo on first use...",
)
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
try:
import gc, torch
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
except Exception:
pass
sep = SepReformerSeparator(
model_id=model,
cpu_separation=CPU_SEPARATION,
output_format=output_format,
mp3_bitrate=mp3_bitrate,
chunk_samples=chunk_samples,
)
jm.update_job(job_id, progress="Separating speakers with SepReformer...")
temp_out = JOBS_DIR / job_id / "sepreformer_out"
temp_out.mkdir(parents=True, exist_ok=True)
try:
stem_paths = sep.separate_into_parts(input_path, str(temp_out))
if jm.should_cancel(job_id):
raise RuntimeError("Cancelled by user")
_prefix = stem_prefix or Path(input_path).stem
stem_files = {}
for stem_name, path in sorted(stem_paths.items()):
src = Path(path)
display_name = stem_name.replace('_', ' ').title()
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
shutil.move(str(src), str(dest))
copy_metadata(input_path, str(dest), stem_label=display_name, copy_lyrics=True)
stem_files[stem_name.lower()] = str(dest)
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
return stem_files
except Exception:
try:
shutil.rmtree(temp_out, ignore_errors=True)
except Exception:
pass
raise