3875 lines
141 KiB
Python
Executable File
3875 lines
141 KiB
Python
Executable File
"""
|
||
Stem separation service with project-based folder structure and preset mixing.
|
||
- Demucs models → CLI subprocess (python -m demucs)
|
||
- BS-RoFormer → Python API (in-process, no subprocess needed)
|
||
- Presets → Instrumental, Static, Dynamic mixing via preset_service
|
||
"""
|
||
import logging
|
||
import os
|
||
import select
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
import json
|
||
import shutil
|
||
from copy import deepcopy
|
||
from pathlib import Path
|
||
from typing import Dict, List, Optional
|
||
|
||
from .. import job_manager as jm
|
||
from ..config import (
|
||
PROJECTS_DIR,
|
||
JOBS_DIR,
|
||
CPU_SEPARATION,
|
||
MELBAND_STAGE_DIAGNOSTICS,
|
||
PROJECT_SUBFOLDERS,
|
||
FALLBACK_REQUIRE_LEAD_BACK,
|
||
LONG_TRACK_DURATION_SECS,
|
||
LONG_TRACK_CHUNKING_MODE,
|
||
LONG_TRACK_CHUNK_DURATION_SECS,
|
||
LONG_TRACK_CHUNK_OVERLAP_SECS,
|
||
DEMUCS_SEGMENT_OVERRIDE,
|
||
DEMUCS_OVERLAP_OVERRIDE,
|
||
MELBAND_DIM_T_OVERRIDE,
|
||
GABOX_SMALL_MODEL_MAX_DURATION_SECS,
|
||
BS_ROFORMER_MODEL_DIR,
|
||
)
|
||
from .metadata_util import copy_metadata, read_metadata
|
||
from .project_namer import project_name, make_project_dirs
|
||
from .project_metadata import write_project_metadata
|
||
from .audio_analysis_service import analyze_audio_features
|
||
from .project_metadata import patch_project_metadata_audio_features
|
||
from .separation.model_downloads import ensure_model_available
|
||
from .separation.model_registry import (
|
||
APOLLO_MODEL_IDS,
|
||
ASPIRATION_MODEL_IDS,
|
||
BANDIT_MODEL_IDS,
|
||
BSR_ANVUEW_MODELS,
|
||
BSR_ADAPTER_MODELS,
|
||
BSR_CHORUS_MODELS,
|
||
BSR_KARAOKE_SPECIAL_MODELS,
|
||
BSR_PREFIX,
|
||
BSR_STEM_MODE,
|
||
BSR_SW_MODEL_IDS,
|
||
ENSEMBLE_MODEL_IDS,
|
||
FORMAT_LABELS,
|
||
KARAOKE_OOM_FALLBACK_CHAIN,
|
||
MBR_DEUX_BECRUILY_MODELS,
|
||
MBR_DEREVERB_MODELS,
|
||
MBR_DEREVERB_PREFIX,
|
||
MBR_KARAOKE_MODELS,
|
||
MBR_KARAOKE_PREFIX,
|
||
MBR_MODELS,
|
||
MBR_PREFIX,
|
||
MBR_VOCALS_MODELS,
|
||
MBR_VOCALS_PREFIX,
|
||
MDXNET_MODEL_IDS,
|
||
MEDLEYVOX_MODEL_IDS,
|
||
MODEL_LABELS,
|
||
SCNET_MODEL_IDS,
|
||
SEPACAP_MODEL_IDS,
|
||
SEPREFORMER_MODEL_IDS,
|
||
is_lead_back_capable,
|
||
)
|
||
from .separation.output_layout import (
|
||
next_stem_label,
|
||
normalise_stem_label,
|
||
parse_stem_label_from_filename,
|
||
stem_filename,
|
||
)
|
||
from .separation.power_presets import (
|
||
apply_global_power_policy_overrides,
|
||
build_global_power_policy,
|
||
get_power_opts,
|
||
)
|
||
from .separation.execution import flush_vram_cache, move_stems_to_project, oom_retry_call, cleanup_temp, get_audio_duration_secs
|
||
from .separation.chunking import extract_chunk, overlap_add_reconstruct, plan_chunks, should_chunk
|
||
from .separation.postprocess import apply_presets
|
||
from .stem_finalize_service import finalize_generated_stems
|
||
from .download_utils import download_file_atomic
|
||
|
||
log = logging.getLogger(__name__)
|
||
|
||
|
||
_AUDIO_EXTS = {".mp3", ".wav", ".flac", ".m4a", ".ogg"}
|
||
|
||
_BS_ADAPTER_ASSETS = {
|
||
"bs_mamba2_v1": {
|
||
"model_file": "bs_mamba2_vocals.ckpt",
|
||
"config_file": "config_bs_mamba2_vocals.yaml",
|
||
"model_url": "https://github.com/ZFTurbo/Music-Source-Separation-Training/releases/download/v1.0.19/bs_mamba2_vocals.ckpt",
|
||
"config_url": "https://github.com/ZFTurbo/Music-Source-Separation-Training/releases/download/v1.0.19/config_bs_mamba2_vocals.yaml",
|
||
},
|
||
"bs_polarformer_v1": {
|
||
"model_file": "model_bs_polarformer_float16.ckpt",
|
||
"config_file": "model_bs_polarformer_float16.yaml",
|
||
"model_url": "https://github.com/ZFTurbo/Music-Source-Separation-Training/releases/download/v1.0.20/model_bs_polarformer_float16.ckpt",
|
||
"config_url": "https://github.com/ZFTurbo/Music-Source-Separation-Training/releases/download/v1.0.20/model_bs_polarformer_float16.yaml",
|
||
},
|
||
}
|
||
|
||
|
||
def _ensure_bs_adapter_assets(model_id: str) -> tuple[Path, Path]:
|
||
spec = _BS_ADAPTER_ASSETS.get(model_id)
|
||
if not spec:
|
||
raise RuntimeError(f"No BS adapter asset spec found for model '{model_id}'")
|
||
|
||
model_dir = BS_ROFORMER_MODEL_DIR / model_id
|
||
model_dir.mkdir(parents=True, exist_ok=True)
|
||
model_path = model_dir / spec["model_file"]
|
||
config_path = model_dir / spec["config_file"]
|
||
|
||
if not model_path.exists():
|
||
log.info("Downloading %s weights from %s", model_id, spec["model_url"])
|
||
download_file_atomic(spec["model_url"], model_path, timeout=180)
|
||
if not config_path.exists():
|
||
log.info("Downloading %s config from %s", model_id, spec["config_url"])
|
||
download_file_atomic(spec["config_url"], config_path, timeout=180)
|
||
|
||
return model_path, config_path
|
||
|
||
|
||
def _diag_progress_cb(job_id: str):
|
||
def _emit(message: str) -> None:
|
||
jm.update_job(job_id, progress=message)
|
||
return _emit
|
||
|
||
|
||
def _effective_chunking_params(
|
||
mode_override: Optional[str],
|
||
duration_override: Optional[int],
|
||
overlap_override: Optional[int],
|
||
) -> tuple[str, int, int]:
|
||
mode = (mode_override or LONG_TRACK_CHUNKING_MODE or "auto").strip().lower()
|
||
if mode not in {"auto", "always", "off"}:
|
||
mode = LONG_TRACK_CHUNKING_MODE
|
||
duration = int(duration_override) if duration_override is not None else int(LONG_TRACK_CHUNK_DURATION_SECS)
|
||
overlap = int(overlap_override) if overlap_override is not None else int(LONG_TRACK_CHUNK_OVERLAP_SECS)
|
||
return mode, max(1, duration), max(0, overlap)
|
||
|
||
|
||
def _init_melband_separator_compat(sep_class, **kwargs):
|
||
"""Instantiate MelBand separator with backward-compatible kwargs."""
|
||
try:
|
||
return sep_class(**kwargs)
|
||
except TypeError as exc:
|
||
_msg = str(exc)
|
||
if "unexpected keyword argument 'dim_t_override'" in _msg:
|
||
kwargs.pop("dim_t_override", None)
|
||
kwargs.pop("segment_secs_override", None)
|
||
return sep_class(**kwargs)
|
||
if "unexpected keyword argument 'segment_secs_override'" in _msg:
|
||
kwargs.pop("segment_secs_override", None)
|
||
return sep_class(**kwargs)
|
||
raise
|
||
|
||
|
||
def _save_stem_audio(path: str, data_cs, sample_rate: int, output_format: str, bitrate: str = '256k') -> None:
|
||
"""Save stem audio array ``(channels, samples)`` to the target format."""
|
||
import tempfile
|
||
|
||
import soundfile as sf
|
||
|
||
ext = (output_format or "wav").lower()
|
||
data_sc = data_cs.T
|
||
|
||
if ext in ('wav', 'flac', 'ogg'):
|
||
fmt = 'OGG' if ext == 'ogg' else ext.upper()
|
||
sf.write(path, data_sc, sample_rate, format=fmt, subtype='PCM_16')
|
||
return
|
||
|
||
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
|
||
tmp_path = tmp.name
|
||
try:
|
||
sf.write(tmp_path, data_sc, sample_rate, format='WAV', subtype='PCM_16')
|
||
cmd = ['ffmpeg', '-y', '-i', tmp_path]
|
||
if ext == 'mp3':
|
||
cmd.extend(['-b:a', bitrate])
|
||
cmd.append(path)
|
||
subprocess.run(cmd, check=True, capture_output=True)
|
||
finally:
|
||
try:
|
||
os.remove(tmp_path)
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
def _separate_into_parts_chunked_via_files(
|
||
*,
|
||
job_id: str,
|
||
input_path: str,
|
||
sep,
|
||
temp_out: Path,
|
||
chunk_duration_secs: int,
|
||
chunk_overlap_secs: int,
|
||
output_format: str,
|
||
mp3_bitrate: str,
|
||
) -> Dict[str, str]:
|
||
"""Chunk-adapter for separators that only expose file-based `separate_into_parts()`.
|
||
|
||
Uses time-domain chunking + overlap-add reconstruction to provide a safe,
|
||
family-agnostic segment policy without requiring model-native chunk APIs.
|
||
"""
|
||
import tempfile
|
||
|
||
import numpy as np
|
||
import soundfile as sf
|
||
|
||
def _read_audio_with_fallback(path: str):
|
||
try:
|
||
return sf.read(path, always_2d=True, dtype='float32')
|
||
except Exception:
|
||
with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as tmp:
|
||
tmp_wav = tmp.name
|
||
try:
|
||
subprocess.run(
|
||
['ffmpeg', '-y', '-i', path, '-f', 'wav', tmp_wav],
|
||
check=True,
|
||
capture_output=True,
|
||
)
|
||
return sf.read(tmp_wav, always_2d=True, dtype='float32')
|
||
finally:
|
||
try:
|
||
os.remove(tmp_wav)
|
||
except Exception:
|
||
pass
|
||
|
||
audio_sc, sample_rate = _read_audio_with_fallback(input_path)
|
||
if audio_sc.size == 0:
|
||
raise RuntimeError("Input audio is empty")
|
||
audio_cs = audio_sc.T
|
||
|
||
chunk_plan = plan_chunks(
|
||
total_samples=audio_cs.shape[-1],
|
||
sample_rate=sample_rate,
|
||
chunk_secs=float(chunk_duration_secs),
|
||
overlap_secs=float(chunk_overlap_secs),
|
||
)
|
||
total_chunks = len(chunk_plan.chunks)
|
||
|
||
jm.update_job(
|
||
job_id,
|
||
progress=(
|
||
f"Chunked mode: {total_chunks} chunk(s) × {chunk_duration_secs} s "
|
||
f"with {chunk_overlap_secs} s crossfade overlap..."
|
||
),
|
||
)
|
||
|
||
work_dir = temp_out / "chunk_adapter"
|
||
work_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
chunk_outputs: List[Dict[str, object]] = []
|
||
stem_names: Optional[List[str]] = None
|
||
channels = int(audio_cs.shape[0])
|
||
target_len = int(chunk_plan.chunk_samples)
|
||
|
||
for idx, (start, end) in enumerate(chunk_plan.chunks, start=1):
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
jm.update_job(job_id, progress=f"Processing chunk {idx}/{total_chunks}...")
|
||
|
||
chunk_cs = extract_chunk(audio_cs, start, end)
|
||
chunk_input = work_dir / f"chunk_{idx:04d}.wav"
|
||
sf.write(str(chunk_input), chunk_cs.T, sample_rate, format='WAV', subtype='PCM_16')
|
||
|
||
chunk_out_dir = work_dir / f"chunk_out_{idx:04d}"
|
||
chunk_out_dir.mkdir(parents=True, exist_ok=True)
|
||
chunk_paths = sep.separate_into_parts(str(chunk_input), str(chunk_out_dir))
|
||
if not chunk_paths:
|
||
raise RuntimeError(f"Chunk {idx} returned no stems")
|
||
|
||
current_stems = sorted(chunk_paths.keys())
|
||
if stem_names is None:
|
||
stem_names = current_stems
|
||
|
||
chunk_stems: Dict[str, object] = {}
|
||
for stem in stem_names:
|
||
stem_path = chunk_paths.get(stem)
|
||
if not stem_path:
|
||
chunk_stems[stem] = np.zeros((channels, target_len), dtype=np.float32)
|
||
continue
|
||
|
||
stem_sc, stem_sr = _read_audio_with_fallback(stem_path)
|
||
if stem_sr != sample_rate:
|
||
import librosa
|
||
|
||
stem_cs_resample = stem_sc.T
|
||
stem_cs_resample = np.array([
|
||
librosa.resample(ch, orig_sr=stem_sr, target_sr=sample_rate)
|
||
for ch in stem_cs_resample
|
||
], dtype=np.float32)
|
||
stem_sc = stem_cs_resample.T
|
||
|
||
stem_cs = stem_sc.T
|
||
if stem_cs.shape[0] == 1 and channels == 2:
|
||
stem_cs = np.vstack([stem_cs, stem_cs])
|
||
elif stem_cs.shape[0] != channels:
|
||
stem_cs = stem_cs[:channels, :]
|
||
|
||
cur_len = int(stem_cs.shape[-1])
|
||
if cur_len < target_len:
|
||
stem_cs = np.pad(stem_cs, ((0, 0), (0, target_len - cur_len)), mode='constant')
|
||
elif cur_len > target_len:
|
||
stem_cs = stem_cs[:, :target_len]
|
||
|
||
chunk_stems[stem] = stem_cs.astype(np.float32, copy=False)
|
||
|
||
chunk_outputs.append(chunk_stems)
|
||
|
||
if not stem_names:
|
||
raise RuntimeError("No stems were produced in chunked adapter path")
|
||
|
||
reconstructed = overlap_add_reconstruct(chunk_outputs, chunk_plan, stem_names)
|
||
|
||
ext = (output_format or 'wav').lower()
|
||
out_paths: Dict[str, str] = {}
|
||
for stem in stem_names:
|
||
out_path = temp_out / f"{stem}.{ext}"
|
||
_save_stem_audio(str(out_path), reconstructed[stem], sample_rate, ext, mp3_bitrate)
|
||
out_paths[stem] = str(out_path)
|
||
|
||
try:
|
||
shutil.rmtree(work_dir, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
|
||
return out_paths
|
||
|
||
|
||
def _build_separation_diagnostics(
|
||
model: str,
|
||
duration_secs: float,
|
||
*,
|
||
chunking_mode: str,
|
||
chunk_duration_secs: int,
|
||
chunk_overlap_secs: int,
|
||
power_opts: Dict[str, Dict[str, object]],
|
||
global_power_policy: Dict[str, object],
|
||
) -> Dict[str, object]:
|
||
"""Return standardized diagnostics metadata for separation jobs."""
|
||
roformer_chunk_models = set(MBR_MODELS) | set(BSR_STEM_MODE) | set(BSR_KARAOKE_SPECIAL_MODELS) | set(BSR_ADAPTER_MODELS)
|
||
mdx_scnet_chunk_models = set(MDXNET_MODEL_IDS) | set(SCNET_MODEL_IDS)
|
||
file_chunk_adapter_models = set(APOLLO_MODEL_IDS) | set(ASPIRATION_MODEL_IDS) | set(BANDIT_MODEL_IDS)
|
||
demucs_models = {"htdemucs", "htdemucs_ft", "mdx", "mdx_extra_q"}
|
||
|
||
if model in roformer_chunk_models:
|
||
family = "roformer"
|
||
elif model in MDXNET_MODEL_IDS:
|
||
family = "mdxnet"
|
||
elif model in SCNET_MODEL_IDS:
|
||
family = "scnet"
|
||
elif model in APOLLO_MODEL_IDS:
|
||
family = "apollo"
|
||
elif model in ASPIRATION_MODEL_IDS:
|
||
family = "aspiration"
|
||
elif model in BANDIT_MODEL_IDS:
|
||
family = "bandit"
|
||
elif model in demucs_models:
|
||
family = "demucs"
|
||
elif model in MEDLEYVOX_MODEL_IDS:
|
||
family = "medleyvox"
|
||
elif model in SEPREFORMER_MODEL_IDS:
|
||
family = "sepreformer"
|
||
else:
|
||
family = "other"
|
||
|
||
chunking_supported = model in (roformer_chunk_models | mdx_scnet_chunk_models | file_chunk_adapter_models)
|
||
chunking_active = bool(
|
||
chunking_supported and should_chunk(duration_secs, chunking_mode, LONG_TRACK_DURATION_SECS)
|
||
)
|
||
|
||
demucs_auto_segment = None
|
||
if family == "demucs":
|
||
demucs_segment_pref = power_opts.get("demucs", {}).get("segment")
|
||
if demucs_segment_pref is not None:
|
||
demucs_auto_segment = int(demucs_segment_pref)
|
||
elif DEMUCS_SEGMENT_OVERRIDE is not None:
|
||
demucs_auto_segment = DEMUCS_SEGMENT_OVERRIDE
|
||
elif duration_secs < 120:
|
||
demucs_auto_segment = 12
|
||
elif duration_secs < 480:
|
||
demucs_auto_segment = 8
|
||
else:
|
||
demucs_auto_segment = 4
|
||
|
||
return {
|
||
"track_duration_secs": round(float(duration_secs), 3),
|
||
"family": family,
|
||
"chunking": {
|
||
"supported": chunking_supported,
|
||
"mode": chunking_mode,
|
||
"threshold_secs": LONG_TRACK_DURATION_SECS,
|
||
"active": chunking_active,
|
||
"chunk_duration_secs": chunk_duration_secs,
|
||
"chunk_overlap_secs": chunk_overlap_secs,
|
||
},
|
||
"demucs_tuning": {
|
||
"segment_override": power_opts.get("demucs", {}).get("segment") if power_opts.get("demucs", {}).get("segment") is not None else DEMUCS_SEGMENT_OVERRIDE,
|
||
"overlap_override": power_opts.get("demucs", {}).get("overlap") if power_opts.get("demucs", {}).get("overlap") is not None else DEMUCS_OVERLAP_OVERRIDE,
|
||
"effective_segment_secs": demucs_auto_segment,
|
||
},
|
||
"effective_power_policy": global_power_policy,
|
||
}
|
||
|
||
|
||
def _stem_filename(prefix: str, stem_name: str, suffix: str) -> str:
|
||
"""Compatibility wrapper around shared `stem_filename()` helper."""
|
||
return stem_filename(prefix, stem_name, suffix)
|
||
|
||
|
||
def _existing_project_stem_labels(project_dir: Path, project_name: str, parent_job_id: Optional[str]) -> list[str]:
|
||
labels: list[str] = []
|
||
|
||
if parent_job_id:
|
||
parent_job = jm.get_job(parent_job_id) or {}
|
||
for file_entry in parent_job.get("files", []):
|
||
if file_entry.get("type") != "stem":
|
||
continue
|
||
stem_value = (file_entry.get("stem") or "").strip()
|
||
if stem_value:
|
||
labels.append(stem_value)
|
||
continue
|
||
name = file_entry.get("name") or ""
|
||
if name:
|
||
labels.append(parse_stem_label_from_filename(Path(name).stem, project_name))
|
||
|
||
stems_root = project_dir / "Stems"
|
||
if stems_root.exists():
|
||
for path in stems_root.rglob("*"):
|
||
if not path.is_file() or path.suffix.lower() not in _AUDIO_EXTS:
|
||
continue
|
||
labels.append(parse_stem_label_from_filename(path.stem, project_name))
|
||
|
||
return labels
|
||
|
||
|
||
def _renumber_reprocessed_stems(
|
||
stem_files: Dict[str, str],
|
||
stems_dir: Path,
|
||
project_name: str,
|
||
parent_job_id: Optional[str],
|
||
job_id: str,
|
||
source_stem_label: Optional[str] = None,
|
||
) -> Dict[str, str]:
|
||
existing_labels = _existing_project_stem_labels(stems_dir.parent, project_name, parent_job_id)
|
||
assigned_labels: list[str] = []
|
||
renamed: Dict[str, str] = {}
|
||
|
||
source_label = normalise_stem_label(source_stem_label or "")
|
||
|
||
def _preferred_reprocess_label(stem_name: str) -> str:
|
||
stem_label = normalise_stem_label(stem_name)
|
||
if source_label == "backing" and stem_label in {"vocals", "vocal", "lead"}:
|
||
return "backing vocals"
|
||
return stem_label
|
||
|
||
for stem_name, path in sorted(stem_files.items()):
|
||
src = Path(path)
|
||
if not src.exists():
|
||
continue
|
||
|
||
base_label = _preferred_reprocess_label(stem_name)
|
||
|
||
# Ensure uniqueness against already existing labels and labels assigned in this run.
|
||
used_labels = [*existing_labels, *assigned_labels]
|
||
if base_label not in used_labels:
|
||
numbered_label = base_label
|
||
dest = stems_dir / _stem_filename(project_name, numbered_label, src.suffix)
|
||
else:
|
||
while True:
|
||
numbered_label = next_stem_label(base_label, used_labels)
|
||
dest = stems_dir / _stem_filename(project_name, numbered_label, src.suffix)
|
||
if not dest.exists() or dest == src:
|
||
break
|
||
assigned_labels.append(numbered_label)
|
||
used_labels = [*existing_labels, *assigned_labels]
|
||
|
||
if dest != src:
|
||
shutil.move(str(src), str(dest))
|
||
jm.update_job(job_id, progress=f"Reprocessed stem saved as '{numbered_label}'")
|
||
renamed[numbered_label] = str(dest)
|
||
assigned_labels.append(numbered_label)
|
||
|
||
return renamed
|
||
|
||
|
||
def _run_model_separation(
|
||
job_id: str,
|
||
input_path: str,
|
||
model: str,
|
||
normalised_format: str,
|
||
stems_dir: Path,
|
||
mp3_bitrate: str,
|
||
metadata: dict,
|
||
stem_prefix: str,
|
||
power_opts: dict,
|
||
chunking_mode: str,
|
||
chunk_duration_secs: int,
|
||
chunk_overlap_secs: int,
|
||
) -> Dict[str, str]:
|
||
"""Dispatch one separation request through the existing model-specific runners."""
|
||
_melband_segment_secs_override = power_opts.get('melband', {}).get('segment_secs')
|
||
|
||
if model in BSR_CHORUS_MODELS:
|
||
return _run_bsroformer_chorus(
|
||
job_id, input_path, model, normalised_format, stems_dir,
|
||
mp3_bitrate=mp3_bitrate, metadata=metadata,
|
||
stem_prefix=stem_prefix,
|
||
batch_size_override=power_opts['roformer'].get('batch_size'),
|
||
num_overlap_override=power_opts['roformer'].get('num_overlap'),
|
||
)
|
||
if model in BSR_ANVUEW_MODELS:
|
||
# Route to MelBand Vocals Anvuew since it's the same architecture
|
||
return _run_mel_band_vocals(
|
||
job_id, input_path, model, normalised_format, stems_dir,
|
||
mp3_bitrate=mp3_bitrate, metadata=metadata,
|
||
stem_prefix=stem_prefix,
|
||
batch_size_override=power_opts['roformer'].get('batch_size'),
|
||
num_overlap_override=power_opts['roformer'].get('num_overlap'),
|
||
segment_secs_override=_melband_segment_secs_override,
|
||
chunking_mode=chunking_mode,
|
||
chunk_duration_secs=chunk_duration_secs,
|
||
chunk_overlap_secs=chunk_overlap_secs,
|
||
)
|
||
if model in BSR_KARAOKE_SPECIAL_MODELS:
|
||
# BS karaoke variants are executed through the specialized karaoke path.
|
||
return _run_mel_band_karaoke(
|
||
job_id, input_path, model, normalised_format, stems_dir,
|
||
mp3_bitrate=mp3_bitrate, metadata=metadata,
|
||
stem_prefix=stem_prefix,
|
||
batch_size_override=power_opts['roformer'].get('batch_size'),
|
||
num_overlap_override=power_opts['roformer'].get('num_overlap'),
|
||
segment_secs_override=_melband_segment_secs_override,
|
||
chunking_mode=chunking_mode,
|
||
chunk_duration_secs=chunk_duration_secs,
|
||
chunk_overlap_secs=chunk_overlap_secs,
|
||
)
|
||
if model in BSR_ADAPTER_MODELS:
|
||
model_path, config_path = _ensure_bs_adapter_assets(model)
|
||
return _run_bsroformer(
|
||
job_id, input_path, model, normalised_format, stems_dir,
|
||
mp3_bitrate=mp3_bitrate, metadata=metadata,
|
||
stem_prefix=stem_prefix,
|
||
batch_size_override=power_opts['roformer'].get('batch_size'),
|
||
num_overlap_override=power_opts['roformer'].get('num_overlap'),
|
||
chunking_mode=chunking_mode,
|
||
chunk_duration_secs=chunk_duration_secs,
|
||
chunk_overlap_secs=chunk_overlap_secs,
|
||
model_path_override=str(model_path),
|
||
config_path_override=str(config_path),
|
||
)
|
||
if model.startswith(BSR_PREFIX):
|
||
return _run_bsroformer(
|
||
job_id, input_path, model, normalised_format, stems_dir,
|
||
mp3_bitrate=mp3_bitrate, metadata=metadata,
|
||
stem_prefix=stem_prefix,
|
||
batch_size_override=power_opts['roformer'].get('batch_size'),
|
||
num_overlap_override=power_opts['roformer'].get('num_overlap'),
|
||
chunking_mode=chunking_mode,
|
||
chunk_duration_secs=chunk_duration_secs,
|
||
chunk_overlap_secs=chunk_overlap_secs,
|
||
)
|
||
if model.startswith(MBR_PREFIX):
|
||
return _run_mel_band_roformer(
|
||
job_id, input_path, model, normalised_format, stems_dir,
|
||
mp3_bitrate=mp3_bitrate, metadata=metadata,
|
||
stem_prefix=stem_prefix,
|
||
batch_size_override=power_opts['roformer'].get('batch_size'),
|
||
num_overlap_override=power_opts['roformer'].get('num_overlap'),
|
||
segment_secs_override=_melband_segment_secs_override,
|
||
chunking_mode=chunking_mode,
|
||
chunk_duration_secs=chunk_duration_secs,
|
||
chunk_overlap_secs=chunk_overlap_secs,
|
||
)
|
||
if model in MBR_VOCALS_MODELS:
|
||
return _run_mel_band_vocals(
|
||
job_id, input_path, model, normalised_format, stems_dir,
|
||
mp3_bitrate=mp3_bitrate, metadata=metadata,
|
||
stem_prefix=stem_prefix,
|
||
batch_size_override=power_opts['roformer'].get('batch_size'),
|
||
num_overlap_override=power_opts['roformer'].get('num_overlap'),
|
||
segment_secs_override=_melband_segment_secs_override,
|
||
chunking_mode=chunking_mode,
|
||
chunk_duration_secs=chunk_duration_secs,
|
||
chunk_overlap_secs=chunk_overlap_secs,
|
||
)
|
||
if model in MBR_DEREVERB_MODELS:
|
||
return _run_mel_band_dereverb(
|
||
job_id, input_path, model, normalised_format, stems_dir,
|
||
mp3_bitrate=mp3_bitrate, metadata=metadata,
|
||
stem_prefix=stem_prefix,
|
||
batch_size_override=power_opts['roformer'].get('batch_size'),
|
||
num_overlap_override=power_opts['roformer'].get('num_overlap'),
|
||
dim_t_override=_melband_dim_t_override,
|
||
chunking_mode=chunking_mode,
|
||
chunk_duration_secs=chunk_duration_secs,
|
||
chunk_overlap_secs=chunk_overlap_secs,
|
||
)
|
||
if model in MBR_KARAOKE_MODELS:
|
||
return _run_mel_band_karaoke(
|
||
job_id, input_path, model, normalised_format, stems_dir,
|
||
mp3_bitrate=mp3_bitrate, metadata=metadata,
|
||
stem_prefix=stem_prefix,
|
||
batch_size_override=power_opts['roformer'].get('batch_size'),
|
||
num_overlap_override=power_opts['roformer'].get('num_overlap'),
|
||
segment_secs_override=_melband_segment_secs_override,
|
||
)
|
||
if model in MBR_DEUX_BECRUILY_MODELS:
|
||
return _run_mel_band_deux_becruily(
|
||
job_id, input_path, model, normalised_format, stems_dir,
|
||
mp3_bitrate=mp3_bitrate, metadata=metadata,
|
||
stem_prefix=stem_prefix,
|
||
batch_size_override=power_opts['roformer'].get('batch_size'),
|
||
num_overlap_override=power_opts['roformer'].get('num_overlap'),
|
||
segment_secs_override=_melband_segment_secs_override,
|
||
chunking_mode=chunking_mode,
|
||
chunk_duration_secs=chunk_duration_secs,
|
||
chunk_overlap_secs=chunk_overlap_secs,
|
||
)
|
||
if model in MDXNET_MODEL_IDS:
|
||
# Route MelBand-based MDX models appropriately
|
||
if model == 'mdxnet_dereverb':
|
||
return _run_mel_band_dereverb(
|
||
job_id, input_path, model, normalised_format, stems_dir,
|
||
mp3_bitrate=mp3_bitrate, metadata=metadata,
|
||
batch_size_override=power_opts['roformer'].get('batch_size'),
|
||
num_overlap_override=power_opts['roformer'].get('num_overlap'),
|
||
segment_secs_override=_melband_segment_secs_override,
|
||
chunking_mode=chunking_mode,
|
||
chunk_duration_secs=chunk_duration_secs,
|
||
chunk_overlap_secs=chunk_overlap_secs,
|
||
)
|
||
if model == 'mdxnet_drumsep':
|
||
return _run_mdx23c(
|
||
job_id, input_path, model, normalised_format, stems_dir,
|
||
mp3_bitrate=mp3_bitrate, metadata=metadata,
|
||
stem_prefix=stem_prefix,
|
||
chunk_size_override=power_opts.get('scnet', {}).get('chunk_size'),
|
||
num_overlap_override=power_opts.get('scnet', {}).get('num_overlap'),
|
||
chunking_mode=chunking_mode,
|
||
chunk_duration_secs=chunk_duration_secs,
|
||
chunk_overlap_secs=chunk_overlap_secs,
|
||
)
|
||
return _run_mdxnet(
|
||
job_id, input_path, model, normalised_format, stems_dir,
|
||
mp3_bitrate=mp3_bitrate, metadata=metadata,
|
||
chunking_mode=chunking_mode,
|
||
chunk_duration_secs=(power_opts.get('mdxnet', {}).get('chunk_duration_secs') or chunk_duration_secs),
|
||
chunk_overlap_secs=chunk_overlap_secs,
|
||
)
|
||
if model in SCNET_MODEL_IDS:
|
||
return _run_scnet(
|
||
job_id, input_path, model, normalised_format, stems_dir,
|
||
mp3_bitrate=mp3_bitrate, metadata=metadata,
|
||
stem_prefix=stem_prefix,
|
||
chunk_size_override=power_opts['scnet'].get('chunk_size'),
|
||
num_overlap_override=power_opts['scnet'].get('num_overlap'),
|
||
chunking_mode=chunking_mode,
|
||
chunk_duration_secs=chunk_duration_secs,
|
||
chunk_overlap_secs=chunk_overlap_secs,
|
||
)
|
||
if model in BSR_SW_MODEL_IDS:
|
||
return _run_bsroformer_sw(
|
||
job_id, input_path, model, normalised_format, stems_dir,
|
||
mp3_bitrate=mp3_bitrate, metadata=metadata,
|
||
stem_prefix=stem_prefix,
|
||
)
|
||
if model in MEDLEYVOX_MODEL_IDS:
|
||
return _run_medleyvox(
|
||
job_id, input_path, model, normalised_format, stems_dir,
|
||
mp3_bitrate=mp3_bitrate, metadata=metadata,
|
||
stem_prefix=stem_prefix,
|
||
chunk_duration_secs=(power_opts.get('medleyvox', {}).get('chunk_secs')),
|
||
)
|
||
if model in ENSEMBLE_MODEL_IDS:
|
||
return _run_ensemble(
|
||
job_id, input_path, model, normalised_format, stems_dir,
|
||
mp3_bitrate=mp3_bitrate, metadata=metadata,
|
||
stem_prefix=stem_prefix,
|
||
)
|
||
if model in APOLLO_MODEL_IDS:
|
||
return _run_apollo(
|
||
job_id, input_path, model, normalised_format, stems_dir,
|
||
mp3_bitrate=mp3_bitrate, metadata=metadata,
|
||
stem_prefix=stem_prefix,
|
||
chunking_mode=chunking_mode,
|
||
chunk_duration_secs=(power_opts.get('apollo', {}).get('chunk_duration_secs') or chunk_duration_secs),
|
||
chunk_overlap_secs=chunk_overlap_secs,
|
||
)
|
||
if model in ASPIRATION_MODEL_IDS:
|
||
return _run_aspiration(
|
||
job_id, input_path, model, normalised_format, stems_dir,
|
||
mp3_bitrate=mp3_bitrate, metadata=metadata,
|
||
stem_prefix=stem_prefix,
|
||
chunking_mode=chunking_mode,
|
||
chunk_duration_secs=(power_opts.get('aspiration', {}).get('chunk_duration_secs') or chunk_duration_secs),
|
||
chunk_overlap_secs=chunk_overlap_secs,
|
||
)
|
||
if model in BANDIT_MODEL_IDS:
|
||
return _run_bandit(
|
||
job_id, input_path, model, normalised_format, stems_dir,
|
||
mp3_bitrate=mp3_bitrate, metadata=metadata,
|
||
stem_prefix=stem_prefix,
|
||
chunking_mode=chunking_mode,
|
||
chunk_duration_secs=(power_opts.get('bandit', {}).get('chunk_duration_secs') or chunk_duration_secs),
|
||
chunk_overlap_secs=chunk_overlap_secs,
|
||
)
|
||
if model in SEPACAP_MODEL_IDS:
|
||
return _run_sepacap(
|
||
job_id, input_path, normalised_format, stems_dir,
|
||
mp3_bitrate=mp3_bitrate, metadata=metadata,
|
||
stem_prefix=stem_prefix,
|
||
chunk_samples=power_opts['sepacap']['chunk_samples'],
|
||
)
|
||
if model in SEPREFORMER_MODEL_IDS:
|
||
return _run_sepreformer(
|
||
job_id, input_path, model, normalised_format, stems_dir,
|
||
mp3_bitrate=mp3_bitrate, metadata=metadata,
|
||
stem_prefix=stem_prefix,
|
||
chunk_samples=power_opts['sepreformer']['chunk_samples'],
|
||
)
|
||
return _run_demucs(
|
||
job_id, input_path, model, normalised_format, stems_dir,
|
||
metadata=metadata,
|
||
stem_prefix=stem_prefix,
|
||
segment_override=power_opts['demucs'].get('segment'),
|
||
overlap_override=power_opts['demucs'].get('overlap'),
|
||
)
|
||
|
||
|
||
def _select_refined_output_path(target_stem: str, refined_outputs: Dict[str, str]) -> Optional[str]:
|
||
"""Choose the most likely refined output for a target instrument stem."""
|
||
if not refined_outputs:
|
||
return None
|
||
|
||
normalized_target = (target_stem or "").strip().lower()
|
||
direct = refined_outputs.get(normalized_target)
|
||
if direct:
|
||
return direct
|
||
|
||
aliases = {
|
||
"vocals": ["vocal", "voice", "lead_vocal"],
|
||
"backing": ["backing_instrumental", "backing", "karaoke", "accompaniment"],
|
||
"drums": ["drum", "percussion"],
|
||
"bass": ["bass", "low"],
|
||
"other": ["accompaniment", "instrumental", "music", "other"],
|
||
"guitar": ["guitar", "gtr"],
|
||
"piano": ["piano", "keys", "keyboard"],
|
||
}
|
||
|
||
for key, path in refined_outputs.items():
|
||
k = key.lower()
|
||
if normalized_target in k:
|
||
return path
|
||
for alias in aliases.get(normalized_target, []):
|
||
if alias in k:
|
||
return path
|
||
|
||
# Deterministic fallback.
|
||
return refined_outputs[sorted(refined_outputs.keys())[0]]
|
||
|
||
|
||
def _resolve_refinement_source_stem_path(stem_files: Dict[str, str], target_stem: str) -> Optional[str]:
|
||
"""Resolve a requested refinement stem key against known source-stem aliases."""
|
||
normalized_target = (target_stem or "").strip().lower()
|
||
if not normalized_target:
|
||
return None
|
||
|
||
if normalized_target in stem_files:
|
||
return stem_files[normalized_target]
|
||
|
||
aliases = {
|
||
"backing": ["backing_instrumental", "karaoke", "instrumental", "other"],
|
||
"vocals": ["lead", "lead_vocal", "vocal"],
|
||
}
|
||
|
||
for alias in aliases.get(normalized_target, []):
|
||
if alias in stem_files:
|
||
return stem_files[alias]
|
||
|
||
return None
|
||
|
||
|
||
def _release_idle_transcriber_models(job_id: str) -> None:
|
||
"""Unload cached Whisper models before heavy separator work starts."""
|
||
try:
|
||
from .lyric_transcriber_service import release_cached_whisper_models
|
||
|
||
released = release_cached_whisper_models()
|
||
if released:
|
||
log.info("[%s] released %d cached Whisper model(s) before separation", job_id, released)
|
||
except Exception as exc:
|
||
log.debug("[%s] failed to release cached Whisper models: %s", job_id, exc)
|
||
|
||
|
||
def _should_isolate_separator_process() -> bool:
|
||
if os.environ.get("STEMENGINE_SEPARATOR_CHILD") == "1":
|
||
return False
|
||
if CPU_SEPARATION:
|
||
return False
|
||
return os.environ.get("STEMENGINE_SEPARATION_SUBPROCESS", "true").lower() != "false"
|
||
|
||
|
||
def _serialise_separation_kwargs(kwargs: dict) -> dict:
|
||
payload = dict(kwargs)
|
||
if payload.get("output_dir_override") is not None:
|
||
payload["output_dir_override"] = str(payload["output_dir_override"])
|
||
return payload
|
||
|
||
|
||
def _run_separation_in_subprocess(**kwargs) -> None:
|
||
job_id = kwargs["job_id"]
|
||
job_dir = JOBS_DIR / job_id
|
||
job_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
payload_path = job_dir / "separator_payload.json"
|
||
payload_path.write_text(json.dumps(_serialise_separation_kwargs(kwargs)), encoding="utf-8")
|
||
|
||
env = os.environ.copy()
|
||
env["STEMENGINE_SEPARATOR_CHILD"] = "1"
|
||
|
||
try:
|
||
result = subprocess.run(
|
||
[sys.executable, "-m", "backend.services.separator_worker", str(payload_path)],
|
||
cwd=str(Path(__file__).resolve().parents[2]),
|
||
env=env,
|
||
stdout=subprocess.DEVNULL,
|
||
stderr=subprocess.PIPE,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
except Exception as exc:
|
||
log.warning("[%s] separator subprocess launch failed, falling back to in-process execution: %s", job_id, exc)
|
||
jm.update_job(job_id, progress="Separator worker launch failed — falling back to in-process execution...")
|
||
_run_separation_impl(**kwargs)
|
||
return
|
||
|
||
if result.returncode == 0:
|
||
return
|
||
|
||
job = jm.get_job(job_id)
|
||
if job and job.get("status") in {
|
||
jm.JobStatus.DONE,
|
||
jm.JobStatus.ERROR,
|
||
jm.JobStatus.CANCELLED,
|
||
}:
|
||
return
|
||
|
||
stderr = (result.stderr or "").strip()
|
||
if len(stderr) > 800:
|
||
stderr = stderr[:800] + "..."
|
||
message = stderr or f"separator worker exited with code {result.returncode}"
|
||
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=message, progress="Error.")
|
||
|
||
|
||
def run_separation(
|
||
job_id: str,
|
||
input_path: str,
|
||
model: str,
|
||
output_format: str,
|
||
# Preset options
|
||
preset_type: str = "stems_only", # stems_only, static, instrumental, dynamic
|
||
static_option: Optional[str] = None, # for static preset: no_vocals, no_drums, etc.
|
||
dynamic_levels: Optional[Dict[str, float]] = None, # for dynamic preset
|
||
instrument_models: Optional[Dict[str, str]] = None, # optional per-instrument model mapping
|
||
instrument_refine_output_format: Optional[str] = None,
|
||
instrument_refine_mp3_bitrate: Optional[str] = None,
|
||
mp3_bitrate_override: Optional[str] = None,
|
||
# Project naming
|
||
project_name_override: Optional[str] = None,
|
||
# Power preset
|
||
sep_power_preset: str = 'standard',
|
||
# Optional request-level runtime overrides
|
||
chunking_mode_override: Optional[str] = None,
|
||
chunk_duration_secs_override: Optional[int] = None,
|
||
chunk_overlap_secs_override: Optional[int] = None,
|
||
roformer_batch_size_override: Optional[int] = None,
|
||
roformer_num_overlap_override: Optional[int] = None,
|
||
demucs_segment_override: Optional[int] = None,
|
||
demucs_overlap_override: Optional[float] = None,
|
||
scnet_chunk_size_override: Optional[int] = None,
|
||
scnet_num_overlap_override: Optional[int] = None,
|
||
# Phase 2: optional global policy overrides
|
||
global_power_tier_override: Optional[str] = None,
|
||
global_segment_secs_override: Optional[float] = None,
|
||
global_overlap_override: Optional[float] = None,
|
||
global_batch_size_override: Optional[int] = None,
|
||
# Reprocess mode: output stems directly into an existing project's Stems dir
|
||
output_dir_override: Optional[Path] = None,
|
||
parent_job_id: Optional[str] = None,
|
||
):
|
||
_release_idle_transcriber_models(job_id)
|
||
|
||
kwargs = {
|
||
"job_id": job_id,
|
||
"input_path": input_path,
|
||
"model": model,
|
||
"output_format": output_format,
|
||
"preset_type": preset_type,
|
||
"static_option": static_option,
|
||
"dynamic_levels": dynamic_levels,
|
||
"instrument_models": instrument_models,
|
||
"instrument_refine_output_format": instrument_refine_output_format,
|
||
"instrument_refine_mp3_bitrate": instrument_refine_mp3_bitrate,
|
||
"mp3_bitrate_override": mp3_bitrate_override,
|
||
"project_name_override": project_name_override,
|
||
"sep_power_preset": sep_power_preset,
|
||
"chunking_mode_override": chunking_mode_override,
|
||
"chunk_duration_secs_override": chunk_duration_secs_override,
|
||
"chunk_overlap_secs_override": chunk_overlap_secs_override,
|
||
"roformer_batch_size_override": roformer_batch_size_override,
|
||
"roformer_num_overlap_override": roformer_num_overlap_override,
|
||
"demucs_segment_override": demucs_segment_override,
|
||
"demucs_overlap_override": demucs_overlap_override,
|
||
"scnet_chunk_size_override": scnet_chunk_size_override,
|
||
"scnet_num_overlap_override": scnet_num_overlap_override,
|
||
"global_power_tier_override": global_power_tier_override,
|
||
"global_segment_secs_override": global_segment_secs_override,
|
||
"global_overlap_override": global_overlap_override,
|
||
"global_batch_size_override": global_batch_size_override,
|
||
"output_dir_override": output_dir_override,
|
||
"parent_job_id": parent_job_id,
|
||
}
|
||
|
||
if _should_isolate_separator_process():
|
||
_run_separation_in_subprocess(**kwargs)
|
||
return
|
||
|
||
_run_separation_impl(**kwargs)
|
||
|
||
|
||
def _run_separation_impl(
|
||
job_id: str,
|
||
input_path: str,
|
||
model: str,
|
||
output_format: str,
|
||
# Preset options
|
||
preset_type: str = "stems_only", # stems_only, static, instrumental, dynamic
|
||
static_option: Optional[str] = None, # for static preset: no_vocals, no_drums, etc.
|
||
dynamic_levels: Optional[Dict[str, float]] = None, # for dynamic preset
|
||
instrument_models: Optional[Dict[str, str]] = None, # optional per-instrument model mapping
|
||
instrument_refine_output_format: Optional[str] = None,
|
||
instrument_refine_mp3_bitrate: Optional[str] = None,
|
||
mp3_bitrate_override: Optional[str] = None,
|
||
# Project naming
|
||
project_name_override: Optional[str] = None,
|
||
# Power preset
|
||
sep_power_preset: str = 'standard',
|
||
# Optional request-level runtime overrides
|
||
chunking_mode_override: Optional[str] = None,
|
||
chunk_duration_secs_override: Optional[int] = None,
|
||
chunk_overlap_secs_override: Optional[int] = None,
|
||
roformer_batch_size_override: Optional[int] = None,
|
||
roformer_num_overlap_override: Optional[int] = None,
|
||
demucs_segment_override: Optional[int] = None,
|
||
demucs_overlap_override: Optional[float] = None,
|
||
scnet_chunk_size_override: Optional[int] = None,
|
||
scnet_num_overlap_override: Optional[int] = None,
|
||
# Phase 2: optional global policy overrides
|
||
global_power_tier_override: Optional[str] = None,
|
||
global_segment_secs_override: Optional[float] = None,
|
||
global_overlap_override: Optional[float] = None,
|
||
global_batch_size_override: Optional[int] = None,
|
||
# Reprocess mode: output stems directly into an existing project's Stems dir
|
||
output_dir_override: Optional[Path] = None,
|
||
parent_job_id: Optional[str] = None,
|
||
):
|
||
"""
|
||
Background task: separate audio into stems and apply presets.
|
||
|
||
Creates human-readable project folder structure:
|
||
/Artist - Title/
|
||
Stems/ (always created)
|
||
Instrumental/ (if preset includes instrumental)
|
||
Static/ (if preset is static)
|
||
Dynamic/ (if preset is dynamic)
|
||
"""
|
||
log.debug("[%s] run_separation start — model=%s format=%s preset=%s input=%s",
|
||
job_id, model, output_format, preset_type, input_path)
|
||
|
||
instrument_models_summary = {
|
||
"requested": instrument_models or {},
|
||
"applied": {},
|
||
"skipped": {},
|
||
}
|
||
|
||
# Read metadata for project naming
|
||
jm.update_job(job_id, status=jm.JobStatus.RUNNING, progress="Reading metadata...")
|
||
metadata = read_metadata(input_path)
|
||
|
||
# Ensure model weights are present — download automatically if missing
|
||
if not ensure_model_available(job_id, model):
|
||
return
|
||
|
||
# Resolve human-readable labels for tab naming
|
||
model_label = MODEL_LABELS.get(model, model)
|
||
format_label = FORMAT_LABELS.get(output_format, output_format)
|
||
tab_label = f"{model_label}_{format_label}"
|
||
|
||
# Determine project name
|
||
if project_name_override:
|
||
proj_name = project_name_override
|
||
else:
|
||
filename_stem = Path(input_path).stem
|
||
proj_name = project_name(
|
||
metadata.get('albumartist') or metadata.get('artist', ''),
|
||
metadata.get('title', ''),
|
||
filename_stem,
|
||
)
|
||
|
||
if output_dir_override is not None:
|
||
# Reprocess mode: place stems directly in the existing project's Stems dir.
|
||
# Skip project creation and source-file copy — the folder already exists.
|
||
stems_dir = output_dir_override
|
||
project_dir = output_dir_override.parent
|
||
stems_dir.mkdir(parents=True, exist_ok=True)
|
||
dirs = {'project': project_dir, 'stems': stems_dir}
|
||
source_dest = project_dir / f"{proj_name}{Path(input_path).suffix.lower()}"
|
||
else:
|
||
# Determine which folders to create based on preset
|
||
preset_flags = {
|
||
'instrumental': preset_type in ('instrumental',),
|
||
'static': preset_type == 'static',
|
||
'dynamic': preset_type == 'dynamic',
|
||
}
|
||
|
||
# Create project directories
|
||
jm.update_job(job_id, progress=f"Creating project: {proj_name}...")
|
||
dirs = make_project_dirs(proj_name, PROJECTS_DIR, preset_flags)
|
||
|
||
stems_dir = dirs['stems']
|
||
project_dir = dirs['project']
|
||
|
||
# Copy original source file to project root with full metadata
|
||
jm.update_job(job_id, progress="Copying source file to project...")
|
||
input_ext = Path(input_path).suffix.lower()
|
||
source_dest = project_dir / f"{proj_name}{input_ext}"
|
||
try:
|
||
shutil.copy2(input_path, source_dest)
|
||
copy_metadata(input_path, str(source_dest), stem_label=None, copy_lyrics=True)
|
||
except Exception as e:
|
||
jm.update_job(job_id, progress=f"Warning: Could not copy source file: {e}")
|
||
|
||
# Normalise mp3_320 → mp3 with explicit bitrate
|
||
if output_format == 'mp3_320':
|
||
mp3_bitrate = '320k'
|
||
elif output_format == 'mp3' and (mp3_bitrate_override or '').strip():
|
||
mp3_bitrate = str(mp3_bitrate_override).strip().lower()
|
||
else:
|
||
mp3_bitrate = '256k'
|
||
normalised_format = 'mp3' if output_format == 'mp3_320' else output_format
|
||
|
||
# Optional per-instrument refinement output controls.
|
||
refine_output_format_raw = (instrument_refine_output_format or '').strip().lower()
|
||
if refine_output_format_raw:
|
||
refine_normalised_format = 'mp3' if refine_output_format_raw == 'mp3_320' else refine_output_format_raw
|
||
if refine_output_format_raw == 'mp3_320':
|
||
refine_mp3_bitrate = '320k'
|
||
elif refine_normalised_format == 'mp3' and (instrument_refine_mp3_bitrate or '').strip():
|
||
refine_mp3_bitrate = str(instrument_refine_mp3_bitrate).strip().lower()
|
||
elif refine_normalised_format == 'mp3':
|
||
refine_mp3_bitrate = mp3_bitrate
|
||
else:
|
||
refine_mp3_bitrate = mp3_bitrate
|
||
else:
|
||
refine_normalised_format = normalised_format
|
||
refine_mp3_bitrate = mp3_bitrate
|
||
|
||
# Resolve power preset options (Phase 2: optional global tier override).
|
||
_effective_power_tier = (global_power_tier_override or sep_power_preset or 'standard').strip().lower()
|
||
_power = deepcopy(get_power_opts(_effective_power_tier))
|
||
|
||
# Apply global policy overrides before family-specific overrides.
|
||
_power = apply_global_power_policy_overrides(
|
||
_power,
|
||
global_segment_secs_override=global_segment_secs_override,
|
||
global_overlap_override=global_overlap_override,
|
||
global_batch_size_override=global_batch_size_override,
|
||
)
|
||
|
||
# Apply request-level overrides (Phase 4).
|
||
if roformer_batch_size_override is not None:
|
||
_power['roformer']['batch_size'] = int(roformer_batch_size_override)
|
||
if roformer_num_overlap_override is not None:
|
||
_power['roformer']['num_overlap'] = int(roformer_num_overlap_override)
|
||
if demucs_segment_override is not None:
|
||
_power['demucs']['segment'] = int(demucs_segment_override)
|
||
if demucs_overlap_override is not None:
|
||
_power['demucs']['overlap'] = float(demucs_overlap_override)
|
||
if scnet_chunk_size_override is not None:
|
||
_power['scnet']['chunk_size'] = int(scnet_chunk_size_override)
|
||
if scnet_num_overlap_override is not None:
|
||
_power['scnet']['num_overlap'] = int(scnet_num_overlap_override)
|
||
|
||
_chunk_duration_fallback = chunk_duration_secs_override
|
||
if _chunk_duration_fallback is None and global_segment_secs_override is not None:
|
||
_chunk_duration_fallback = max(1, int(round(float(global_segment_secs_override))))
|
||
|
||
_chunking_mode, _chunk_duration_secs, _chunk_overlap_secs = _effective_chunking_params(
|
||
chunking_mode_override,
|
||
_chunk_duration_fallback,
|
||
chunk_overlap_secs_override,
|
||
)
|
||
|
||
global_power_policy = build_global_power_policy(_effective_power_tier, _power)
|
||
|
||
input_duration_secs = get_audio_duration_secs(input_path)
|
||
separation_diagnostics = _build_separation_diagnostics(
|
||
model,
|
||
input_duration_secs,
|
||
chunking_mode=_chunking_mode,
|
||
chunk_duration_secs=_chunk_duration_secs,
|
||
chunk_overlap_secs=_chunk_overlap_secs,
|
||
power_opts=_power,
|
||
global_power_policy=global_power_policy,
|
||
)
|
||
|
||
request_overrides = {
|
||
k: v for k, v in {
|
||
"chunking_mode_override": chunking_mode_override,
|
||
"chunk_duration_secs_override": chunk_duration_secs_override,
|
||
"chunk_overlap_secs_override": chunk_overlap_secs_override,
|
||
"roformer_batch_size_override": roformer_batch_size_override,
|
||
"roformer_num_overlap_override": roformer_num_overlap_override,
|
||
"demucs_segment_override": demucs_segment_override,
|
||
"demucs_overlap_override": demucs_overlap_override,
|
||
"scnet_chunk_size_override": scnet_chunk_size_override,
|
||
"scnet_num_overlap_override": scnet_num_overlap_override,
|
||
"global_power_tier_override": global_power_tier_override,
|
||
"global_segment_secs_override": global_segment_secs_override,
|
||
"global_overlap_override": global_overlap_override,
|
||
"global_batch_size_override": global_batch_size_override,
|
||
}.items() if v is not None
|
||
}
|
||
|
||
# Flush PyTorch's allocator cache BEFORE applying the VRAM cap.
|
||
# On WSL2 the system reserves headroom for Windows (DWM, WDDM, etc.).
|
||
# _apply_cuda_limits() caps the process at (total_vram - STEMENGINE_GPU_VRAM_RESERVE_MB)
|
||
# / total_vram — e.g. ≈75 % on an 8 GB card, ensuring 2 GB stays free for Windows.
|
||
# The flush must come first: if stale cached allocations are still counted against
|
||
# the cap from a prior job, the cap appears tighter than it actually is.
|
||
flush_vram_cache()
|
||
try:
|
||
from .lyric_transcriber_service import _apply_cuda_limits
|
||
_apply_cuda_limits()
|
||
except Exception:
|
||
pass
|
||
|
||
# Acquire the shared VRAM lock before loading any model.
|
||
# Prevents a simultaneous lyric transcription job (running in its own
|
||
# executor thread) from loading Whisper while a separator is in VRAM.
|
||
jm.gpu_vram_lock.acquire()
|
||
|
||
# Run separation
|
||
try:
|
||
stem_files = _run_model_separation(
|
||
job_id=job_id,
|
||
input_path=input_path,
|
||
model=model,
|
||
normalised_format=normalised_format,
|
||
stems_dir=stems_dir,
|
||
mp3_bitrate=mp3_bitrate,
|
||
metadata=metadata,
|
||
stem_prefix=proj_name,
|
||
power_opts=_power,
|
||
chunking_mode=_chunking_mode,
|
||
chunk_duration_secs=_chunk_duration_secs,
|
||
chunk_overlap_secs=_chunk_overlap_secs,
|
||
)
|
||
|
||
if jm.should_cancel(job_id):
|
||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||
return
|
||
|
||
if output_dir_override is not None:
|
||
source_stem_label = parse_stem_label_from_filename(Path(input_path).stem, proj_name)
|
||
stem_files = _renumber_reprocessed_stems(
|
||
stem_files=stem_files,
|
||
stems_dir=stems_dir,
|
||
project_name=proj_name,
|
||
parent_job_id=parent_job_id,
|
||
job_id=job_id,
|
||
source_stem_label=source_stem_label,
|
||
)
|
||
|
||
# Optional per-instrument refinement pass
|
||
if instrument_models:
|
||
jm.update_job(job_id, progress="Applying per-instrument refinement...")
|
||
for stem_name, mapped_model in instrument_models.items():
|
||
if jm.should_cancel(job_id):
|
||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||
return
|
||
|
||
base_stem_path = _resolve_refinement_source_stem_path(stem_files, stem_name)
|
||
if not base_stem_path or not Path(base_stem_path).exists():
|
||
instrument_models_summary["skipped"][stem_name] = (
|
||
f"No base stem '{stem_name}' found to refine with '{mapped_model}'."
|
||
)
|
||
continue
|
||
|
||
if mapped_model == model:
|
||
instrument_models_summary["skipped"][stem_name] = (
|
||
"Mapped model matches base model; refinement skipped."
|
||
)
|
||
continue
|
||
|
||
if not ensure_model_available(job_id, mapped_model):
|
||
raise RuntimeError(f"Failed to prepare refinement model '{mapped_model}' for stem '{stem_name}'")
|
||
|
||
jm.update_job(job_id, progress=f"Refining stem '{stem_name}' with model '{mapped_model}'...")
|
||
refine_tmp = JOBS_DIR / job_id / "instrument_refine" / stem_name
|
||
cleanup_temp(refine_tmp)
|
||
refine_tmp.mkdir(parents=True, exist_ok=True)
|
||
|
||
try:
|
||
refined_outputs = _run_model_separation(
|
||
job_id=job_id,
|
||
input_path=base_stem_path,
|
||
model=mapped_model,
|
||
normalised_format=refine_normalised_format,
|
||
stems_dir=refine_tmp,
|
||
mp3_bitrate=refine_mp3_bitrate,
|
||
metadata=metadata,
|
||
stem_prefix=f"{proj_name} - {stem_name.capitalize()} Refine",
|
||
power_opts=_power,
|
||
)
|
||
|
||
selected_refined_path = _select_refined_output_path(stem_name, refined_outputs)
|
||
if not selected_refined_path:
|
||
instrument_models_summary["skipped"][stem_name] = (
|
||
f"Model '{mapped_model}' produced no usable refined output for '{stem_name}'."
|
||
)
|
||
continue
|
||
|
||
selected_refined = Path(selected_refined_path)
|
||
refined_dest = stems_dir / (
|
||
f"{proj_name} - {stem_name.capitalize()} "
|
||
f"(Refined - {MODEL_LABELS.get(mapped_model, mapped_model)}){selected_refined.suffix}"
|
||
)
|
||
shutil.move(str(selected_refined), str(refined_dest))
|
||
|
||
# Promote refined stem as canonical stem output for this instrument.
|
||
stem_files[stem_name] = str(refined_dest)
|
||
instrument_models_summary["applied"][stem_name] = {
|
||
"model": mapped_model,
|
||
"output": str(refined_dest),
|
||
}
|
||
except Exception as refine_exc:
|
||
instrument_models_summary["skipped"][stem_name] = (
|
||
f"Refinement failed for '{stem_name}' with '{mapped_model}': {refine_exc}"
|
||
)
|
||
finally:
|
||
cleanup_temp(refine_tmp)
|
||
|
||
stem_files, stem_cleanup = finalize_generated_stems(
|
||
stem_files,
|
||
stems_dir=stems_dir,
|
||
stem_prefix=proj_name,
|
||
job_id=job_id,
|
||
update_job_fn=jm.update_job,
|
||
)
|
||
|
||
preset_files = apply_presets(
|
||
preset_type=preset_type,
|
||
static_option=static_option,
|
||
dynamic_levels=dynamic_levels,
|
||
stem_files=stem_files,
|
||
dirs=dirs,
|
||
proj_name=proj_name,
|
||
input_path=input_path,
|
||
normalised_format=normalised_format,
|
||
job_id=job_id,
|
||
update_job_fn=jm.update_job,
|
||
)
|
||
|
||
# Build result file list
|
||
all_files = []
|
||
|
||
# Add source file first if it exists
|
||
if source_dest.exists():
|
||
all_files.append({
|
||
"name": source_dest.name,
|
||
"path": str(source_dest),
|
||
"type": "source"
|
||
})
|
||
|
||
for stem_name, path in sorted(stem_files.items()):
|
||
all_files.append({
|
||
"name": Path(path).name,
|
||
"path": path,
|
||
"type": "stem",
|
||
"stem": stem_name
|
||
})
|
||
all_files.extend(preset_files)
|
||
|
||
if output_dir_override is None:
|
||
# Normal mode: register a project record and write project.json
|
||
project = jm.find_or_create_project(proj_name, job_id, tab_label=tab_label)
|
||
try:
|
||
write_project_metadata(
|
||
project_dir,
|
||
project_id=project['id'],
|
||
name=proj_name,
|
||
artist=metadata.get('albumartist') or metadata.get('artist', ''),
|
||
title=metadata.get('title', ''),
|
||
model=model,
|
||
model_label=model_label,
|
||
format=normalised_format,
|
||
format_label=format_label,
|
||
source_file=str(source_dest) if source_dest.exists() else '',
|
||
stems=stem_files,
|
||
sep_job_id=job_id,
|
||
trans_jobs={},
|
||
abc_jobs={},
|
||
lrc_jobs={},
|
||
mix_job_id=None,
|
||
instrument_models=instrument_models_summary,
|
||
)
|
||
except Exception as meta_err:
|
||
jm.update_job(job_id, progress=f"Warning: could not write project.json: {meta_err}")
|
||
|
||
# Analyze BPM + musical key from the source file and cache in project.json.
|
||
# Runs after the heavier separation work so it never blocks stems delivery.
|
||
if source_dest.exists():
|
||
try:
|
||
jm.update_job(job_id, progress="Analyzing audio features (BPM / key)…")
|
||
features = analyze_audio_features(source_dest)
|
||
if features:
|
||
patch_project_metadata_audio_features(project_dir, features)
|
||
except Exception as feat_err:
|
||
log.warning("[%s] Audio feature analysis failed: %s", job_id, feat_err)
|
||
|
||
project_id_result = project['id']
|
||
else:
|
||
# Reprocess mode: merge new stems into the parent sep job's file list
|
||
# so the project page discovers them without any extra queries.
|
||
stem_entries = [f for f in all_files if f.get("type") == "stem"]
|
||
if parent_job_id:
|
||
jm.append_job_files(parent_job_id, stem_entries)
|
||
project_id_result = None
|
||
|
||
log.debug("[%s] separation complete — project=%s stems=%d files=%d",
|
||
job_id, proj_name, len(stem_files), len(all_files))
|
||
jm.update_job(
|
||
job_id,
|
||
status=jm.JobStatus.DONE,
|
||
progress="Separation complete.",
|
||
files=all_files,
|
||
result={
|
||
"reprocess": output_dir_override is not None,
|
||
"parent_job_id": parent_job_id if output_dir_override is not None else None,
|
||
"project_name": proj_name,
|
||
"project_path": str(dirs['project']),
|
||
"stems_dir": str(stems_dir),
|
||
"model_label": model_label,
|
||
"format_label": format_label,
|
||
"tab_label": tab_label,
|
||
"project_id": project_id_result,
|
||
"instrument_models": instrument_models_summary,
|
||
"stem_cleanup": stem_cleanup,
|
||
"diagnostics": separation_diagnostics,
|
||
"request_overrides": request_overrides,
|
||
}
|
||
)
|
||
|
||
except Exception as exc:
|
||
err_str = str(exc)
|
||
if jm.should_cancel(job_id) or "Cancelled by user" in err_str:
|
||
# Cancellation propagated as an exception from inside a separator.
|
||
# Treat it as a clean cancel, not an error.
|
||
log.debug("[%s] separation cancelled mid-run", job_id)
|
||
jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.")
|
||
elif "CUDA" in err_str or "cuda" in err_str:
|
||
# CUDA context is corrupted (typically after a prior OOM).
|
||
# Auto-reset clears allocations and re-enables GPU for the next attempt.
|
||
try:
|
||
from .lyric_transcriber_service import reset_cuda
|
||
reset_cuda()
|
||
except Exception:
|
||
try:
|
||
import torch
|
||
torch.cuda.empty_cache()
|
||
except Exception:
|
||
pass
|
||
user_msg = (
|
||
f"GPU (CUDA) error: {err_str[:300]}\n\n"
|
||
"The GPU was automatically reset — please retry the separation. "
|
||
"If the error keeps occurring, restart the container."
|
||
)
|
||
log.debug("[%s] separation GPU error — %s: %s", job_id, type(exc).__name__, err_str[:200])
|
||
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=user_msg, progress="GPU error — retry to use reset GPU.")
|
||
else:
|
||
log.debug("[%s] separation failed — %s: %s", job_id, type(exc).__name__, exc)
|
||
jm.update_job(job_id, status=jm.JobStatus.ERROR, error=err_str, progress="Error.")
|
||
finally:
|
||
# Always flush PyTorch's VRAM cache after a job ends (success, error, or
|
||
# cancel) so the next job starts with maximum headroom. gc.collect() x3
|
||
# ensures CUDA tensors are released even when model objects form reference
|
||
# cycles that a single generational GC pass misses.
|
||
try:
|
||
import gc
|
||
gc.collect()
|
||
gc.collect()
|
||
gc.collect()
|
||
except Exception:
|
||
pass
|
||
flush_vram_cache()
|
||
# Release VRAM lock so the transcription executor can proceed.
|
||
try:
|
||
jm.gpu_vram_lock.release()
|
||
except RuntimeError:
|
||
pass # already released (should not happen)
|
||
|
||
|
||
def _get_muted_stems(static_option: Optional[str]) -> List[str]:
|
||
"""Convert static option ID to list of stem names to mute."""
|
||
if not static_option:
|
||
return []
|
||
|
||
option_map = {
|
||
'no_vocals': ['vocals'],
|
||
'no_drums': ['drums'],
|
||
'no_bass': ['bass'],
|
||
'no_vocals_no_bass': ['vocals', 'bass'],
|
||
'no_drums_no_bass': ['drums', 'bass'],
|
||
}
|
||
return option_map.get(static_option, [])
|
||
|
||
|
||
def _static_name(static_option: Optional[str]) -> str:
|
||
"""Convert static option ID to human-readable name."""
|
||
name_map = {
|
||
'no_vocals': 'No Vocals',
|
||
'no_drums': 'No Drums',
|
||
'no_bass': 'No Bass',
|
||
'no_vocals_no_bass': 'No Vocals No Bass',
|
||
'no_drums_no_bass': 'No Drums No Bass',
|
||
}
|
||
return name_map.get(static_option, 'Static Mix')
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Demucs (CLI)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _run_demucs(
|
||
job_id: str,
|
||
input_path: str,
|
||
model: str,
|
||
output_format: str,
|
||
stems_dir: Path,
|
||
metadata: Optional[Dict] = None,
|
||
stem_prefix: Optional[str] = None,
|
||
segment_override: Optional[int] = None,
|
||
overlap_override: Optional[float] = None,
|
||
) -> Dict[str, str]:
|
||
"""
|
||
Run Demucs separation. Returns dict mapping stem names to file paths.
|
||
|
||
For long tracks, automatically selects segment size based on duration
|
||
to balance quality vs. VRAM usage.
|
||
"""
|
||
from .separation.chunking import should_chunk
|
||
|
||
jm.update_job(job_id, progress="Starting Demucs separation...")
|
||
|
||
ext = output_format.lower()
|
||
fmt_flag = []
|
||
if ext == 'mp3_320':
|
||
fmt_flag = ['--mp3', '--mp3-bitrate', '320']
|
||
ext = 'mp3'
|
||
elif ext == 'mp3':
|
||
fmt_flag = ['--mp3', '--mp3-bitrate', '256']
|
||
elif ext == 'flac':
|
||
fmt_flag = ['--flac']
|
||
|
||
# Determine segment size: check env override first, then auto-scale by duration
|
||
_dur = get_audio_duration_secs(input_path)
|
||
_use_long_track_tuning = should_chunk(_dur, LONG_TRACK_CHUNKING_MODE, LONG_TRACK_DURATION_SECS)
|
||
|
||
_segment = segment_override
|
||
if _segment is None and DEMUCS_SEGMENT_OVERRIDE is not None:
|
||
_segment = DEMUCS_SEGMENT_OVERRIDE
|
||
if _segment is None and _use_long_track_tuning:
|
||
# Auto-scale segment based on duration
|
||
if _dur < 120:
|
||
_segment = 12 # Default Demucs value
|
||
elif _dur < 480:
|
||
_segment = 8 # Quality-biased for medium-length tracks
|
||
else:
|
||
_segment = 4 # VRAM-biased for long tracks
|
||
|
||
_overlap = overlap_override
|
||
if _overlap is None and DEMUCS_OVERLAP_OVERRIDE is not None:
|
||
_overlap = DEMUCS_OVERLAP_OVERRIDE
|
||
|
||
power_flags = []
|
||
if _segment is not None:
|
||
power_flags.extend(['--segment', str(_segment)])
|
||
if _overlap is not None:
|
||
power_flags.extend(['--overlap', str(_overlap)])
|
||
|
||
# Use a temp directory for demucs output
|
||
temp_out = JOBS_DIR / job_id / "demucs_out"
|
||
temp_out.mkdir(parents=True, exist_ok=True)
|
||
|
||
cmd = [
|
||
sys.executable, '-m', 'demucs',
|
||
'--name', model,
|
||
'--out', str(temp_out),
|
||
*fmt_flag,
|
||
*power_flags,
|
||
input_path,
|
||
]
|
||
|
||
try:
|
||
if _segment is not None or _use_long_track_tuning:
|
||
jm.update_job(
|
||
job_id,
|
||
progress=(
|
||
f"Long-track tuning: segment={_segment}s, overlap={_overlap or 0.25}x "
|
||
f"(duration={_dur:.0f}s)..."
|
||
),
|
||
)
|
||
|
||
process = subprocess.Popen(
|
||
cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
|
||
)
|
||
|
||
while True:
|
||
if jm.should_cancel(job_id):
|
||
process.terminate()
|
||
process.wait()
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
if process.stdout is None:
|
||
if process.poll() is not None:
|
||
break
|
||
time.sleep(0.1)
|
||
continue
|
||
|
||
ready, _, _ = select.select([process.stdout], [], [], 1.0)
|
||
if ready:
|
||
line = process.stdout.readline()
|
||
if line:
|
||
jm.update_job(job_id, progress=line.rstrip())
|
||
elif process.poll() is not None:
|
||
break
|
||
|
||
if process.returncode != 0:
|
||
raise RuntimeError(f"Demucs exited with code {process.returncode}")
|
||
|
||
# Move stems to project folder
|
||
input_stem = Path(input_path).stem
|
||
_prefix = stem_prefix or input_stem
|
||
stem_dir = temp_out / model / input_stem
|
||
|
||
stem_files = {}
|
||
if stem_dir.exists():
|
||
for f in sorted(stem_dir.iterdir()):
|
||
if f.is_file():
|
||
# Determine stem name from filename
|
||
stem_name = f.stem.lower()
|
||
# Clean up common demucs naming patterns
|
||
for prefix in [input_stem.lower(), 'htdemucs', 'mdx']:
|
||
stem_name = stem_name.replace(prefix, '').strip('_- ')
|
||
if not stem_name:
|
||
stem_name = f.stem
|
||
|
||
dest = stems_dir / _stem_filename(_prefix, stem_name, f.suffix)
|
||
shutil.move(str(f), str(dest))
|
||
copy_metadata(input_path, str(dest), stem_label=stem_name.capitalize(), copy_lyrics=True)
|
||
stem_files[stem_name] = str(dest)
|
||
|
||
# Cleanup temp directory
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
|
||
return stem_files
|
||
|
||
except Exception:
|
||
# Cleanup on error
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
raise
|
||
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# BS-RoFormer Chorus (Male/Female) (Python API)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _run_bsroformer_chorus(
|
||
job_id: str,
|
||
input_path: str,
|
||
model: str,
|
||
output_format: str,
|
||
stems_dir: Path,
|
||
mp3_bitrate: str = '256k',
|
||
metadata: Optional[Dict] = None,
|
||
stem_prefix: Optional[str] = None,
|
||
batch_size_override: Optional[int] = None,
|
||
num_overlap_override: Optional[int] = None,
|
||
) -> Dict[str, str]:
|
||
"""
|
||
Run BS-RoFormer Chorus separation (male/female/instrumental).
|
||
Returns dict mapping stem names to file paths.
|
||
"""
|
||
from .bs_roformer_chorus_separator import BSRoformerChorusSeparator
|
||
|
||
jm.update_job(
|
||
job_id, status=jm.JobStatus.RUNNING,
|
||
progress="Loading BS-RoFormer Chorus model...",
|
||
)
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
try:
|
||
import gc, torch
|
||
gc.collect()
|
||
if torch.cuda.is_available():
|
||
torch.cuda.empty_cache()
|
||
except Exception:
|
||
pass
|
||
|
||
sep = BSRoformerChorusSeparator(
|
||
cpu_separation=CPU_SEPARATION,
|
||
output_format=output_format,
|
||
mp3_bitrate=mp3_bitrate,
|
||
)
|
||
|
||
jm.update_job(job_id, progress="Separating male/female chorus... (this may take a while)")
|
||
|
||
temp_out = JOBS_DIR / job_id / "bsr_chorus_out"
|
||
temp_out.mkdir(parents=True, exist_ok=True)
|
||
|
||
_bs = batch_size_override
|
||
for _attempt in range(2):
|
||
try:
|
||
stem_paths = sep.separate_into_parts(
|
||
input_path, str(temp_out),
|
||
batch_size_override=_bs,
|
||
num_overlap_override=num_overlap_override,
|
||
)
|
||
break
|
||
except Exception as _exc:
|
||
_is_oom = "out of memory" in str(_exc).lower() or "OutOfMemory" in type(_exc).__name__
|
||
if _is_oom and _attempt == 0:
|
||
import gc, torch
|
||
jm.update_job(job_id, progress="GPU OOM — retrying with batch_size=1...")
|
||
gc.collect()
|
||
torch.cuda.empty_cache()
|
||
_bs = 1
|
||
continue
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
raise
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
_prefix = stem_prefix or Path(input_path).stem
|
||
stem_files = {}
|
||
|
||
try:
|
||
for stem_name, path in sorted(stem_paths.items()):
|
||
src = Path(path)
|
||
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
|
||
shutil.move(str(src), str(dest))
|
||
copy_metadata(input_path, str(dest), stem_label=stem_name.capitalize(), copy_lyrics=True)
|
||
stem_files[stem_name.lower()] = str(dest)
|
||
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
|
||
return stem_files
|
||
|
||
except Exception:
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
raise
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# MelBand Vocals (Python API)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _run_mel_band_vocals(
|
||
job_id: str,
|
||
input_path: str,
|
||
model: str,
|
||
output_format: str,
|
||
stems_dir: Path,
|
||
mp3_bitrate: str = '256k',
|
||
metadata: Optional[Dict] = None,
|
||
stem_prefix: Optional[str] = None,
|
||
batch_size_override: Optional[int] = None,
|
||
num_overlap_override: Optional[int] = None,
|
||
dim_t_override: Optional[int] = None,
|
||
segment_secs_override: Optional[float] = None,
|
||
chunking_mode: Optional[str] = None,
|
||
chunk_duration_secs: Optional[int] = None,
|
||
chunk_overlap_secs: Optional[int] = None,
|
||
) -> Dict[str, str]:
|
||
"""Run MelBand Vocals separation (vocals/instrumental)."""
|
||
# Route to appropriate separator based on model variant
|
||
if model == 'mel_band_vocals_gabox':
|
||
from .mel_band_specialized_separator import MelBandVocalsGaboxSeparator
|
||
sep_class = MelBandVocalsGaboxSeparator
|
||
model_name = "MelBand Vocals Gabox"
|
||
elif model == 'mel_band_vocals_anvuew':
|
||
from .mel_band_specialized_separator import MelBandVocalsAnvuewSeparator
|
||
sep_class = MelBandVocalsAnvuewSeparator
|
||
model_name = "MelBand Vocals Anvuew"
|
||
else:
|
||
from .mel_band_specialized_separator import MelBandVocalsSeparator
|
||
sep_class = MelBandVocalsSeparator
|
||
model_name = "MelBand Vocals"
|
||
|
||
jm.update_job(
|
||
job_id, status=jm.JobStatus.RUNNING,
|
||
progress=f"Loading {model_name} model...",
|
||
)
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
try:
|
||
import gc, torch
|
||
gc.collect()
|
||
if torch.cuda.is_available():
|
||
torch.cuda.empty_cache()
|
||
except Exception:
|
||
pass
|
||
|
||
_dim_t = dim_t_override if dim_t_override is not None else MELBAND_DIM_T_OVERRIDE
|
||
|
||
sep = _init_melband_separator_compat(
|
||
sep_class,
|
||
cpu_separation=CPU_SEPARATION,
|
||
output_format=output_format,
|
||
mp3_bitrate=mp3_bitrate,
|
||
diagnostics_enabled=MELBAND_STAGE_DIAGNOSTICS,
|
||
diagnostics_cb=_diag_progress_cb(job_id) if MELBAND_STAGE_DIAGNOSTICS else None,
|
||
dim_t_override=_dim_t,
|
||
segment_secs_override=segment_secs_override,
|
||
)
|
||
|
||
jm.update_job(job_id, progress="Separating vocals with MelBand...")
|
||
|
||
temp_out = JOBS_DIR / job_id / "mel_band_vocals_out"
|
||
temp_out.mkdir(parents=True, exist_ok=True)
|
||
|
||
_bs = batch_size_override
|
||
_overlap = num_overlap_override
|
||
_dur = get_audio_duration_secs(input_path)
|
||
_chunk_mode = (chunking_mode or LONG_TRACK_CHUNKING_MODE)
|
||
_chunk_secs = int(chunk_duration_secs or LONG_TRACK_CHUNK_DURATION_SECS)
|
||
_chunk_ov = int(chunk_overlap_secs or LONG_TRACK_CHUNK_OVERLAP_SECS)
|
||
_use_chunking = should_chunk(_dur, _chunk_mode, LONG_TRACK_DURATION_SECS)
|
||
|
||
try:
|
||
if _use_chunking:
|
||
_total_samples = max(1, int(round(_dur * 44100)))
|
||
_chunk_plan = plan_chunks(_total_samples, 44100, _chunk_secs, _chunk_ov)
|
||
_n = len(_chunk_plan.chunks)
|
||
jm.update_job(
|
||
job_id,
|
||
progress=(
|
||
f"Chunked mode: {_n} chunk(s) × {_chunk_secs} s "
|
||
f"with {_chunk_ov} s crossfade overlap..."
|
||
),
|
||
)
|
||
stem_paths = sep.separate_into_parts_chunked(
|
||
input_path, str(temp_out),
|
||
chunk_plan=_chunk_plan,
|
||
batch_size_override=_bs,
|
||
num_overlap_override=_overlap,
|
||
progress_cb=lambda msg: jm.update_job(job_id, progress=msg),
|
||
)
|
||
else:
|
||
for _attempt in range(3):
|
||
try:
|
||
stem_paths = sep.separate_into_parts(
|
||
input_path, str(temp_out),
|
||
batch_size_override=_bs,
|
||
num_overlap_override=_overlap,
|
||
)
|
||
break
|
||
except Exception as _exc:
|
||
_is_oom = "out of memory" in str(_exc).lower() or "OutOfMemory" in type(_exc).__name__
|
||
if not _is_oom:
|
||
raise
|
||
flush_vram_cache()
|
||
if _attempt == 0:
|
||
jm.update_job(job_id, progress="GPU OOM — retrying with batch_size=1...")
|
||
_bs = 1
|
||
elif _attempt == 1:
|
||
jm.update_job(job_id, progress="GPU OOM — retrying with batch_size=1, num_overlap=2...")
|
||
_bs = 1
|
||
_overlap = 2
|
||
else:
|
||
raise
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
_prefix = stem_prefix or Path(input_path).stem
|
||
stem_files = {}
|
||
|
||
for stem_name, path in sorted(stem_paths.items()):
|
||
src = Path(path)
|
||
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
|
||
shutil.move(str(src), str(dest))
|
||
copy_metadata(input_path, str(dest), stem_label=stem_name.capitalize(), copy_lyrics=True)
|
||
stem_files[stem_name.lower()] = str(dest)
|
||
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
|
||
return stem_files
|
||
|
||
except Exception:
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
raise
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# MelBand Dereverb (Python API)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _run_mel_band_dereverb(
|
||
job_id: str,
|
||
input_path: str,
|
||
model: str,
|
||
output_format: str,
|
||
stems_dir: Path,
|
||
mp3_bitrate: str = '256k',
|
||
metadata: Optional[Dict] = None,
|
||
stem_prefix: Optional[str] = None,
|
||
batch_size_override: Optional[int] = None,
|
||
num_overlap_override: Optional[int] = None,
|
||
dim_t_override: Optional[int] = None,
|
||
segment_secs_override: Optional[float] = None,
|
||
chunking_mode: Optional[str] = None,
|
||
chunk_duration_secs: Optional[int] = None,
|
||
chunk_overlap_secs: Optional[int] = None,
|
||
) -> Dict[str, str]:
|
||
"""Run MelBand Dereverb separation (dry vocals/reverb)."""
|
||
# Route to appropriate separator based on model variant
|
||
if model == 'mel_band_dereverb_room':
|
||
from .mel_band_specialized_separator import MelBandDereverbRoomSeparator
|
||
sep_class = MelBandDereverbRoomSeparator
|
||
model_name = "MelBand Room Dereverb"
|
||
elif model == 'mel_band_dereverb_anvuew':
|
||
from .mel_band_specialized_separator import MelBandDereverbAnvuewSeparator
|
||
sep_class = MelBandDereverbAnvuewSeparator
|
||
model_name = "MelBand Dereverb Anvuew"
|
||
else:
|
||
from .mel_band_specialized_separator import MelBandDereverbSeparator
|
||
sep_class = MelBandDereverbSeparator
|
||
model_name = "MelBand Dereverb"
|
||
|
||
jm.update_job(
|
||
job_id, status=jm.JobStatus.RUNNING,
|
||
progress=f"Loading {model_name} model...",
|
||
)
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
try:
|
||
import gc, torch
|
||
gc.collect()
|
||
if torch.cuda.is_available():
|
||
torch.cuda.empty_cache()
|
||
except Exception:
|
||
pass
|
||
|
||
_dim_t = dim_t_override if dim_t_override is not None else MELBAND_DIM_T_OVERRIDE
|
||
|
||
sep = _init_melband_separator_compat(
|
||
sep_class,
|
||
cpu_separation=CPU_SEPARATION,
|
||
output_format=output_format,
|
||
mp3_bitrate=mp3_bitrate,
|
||
diagnostics_enabled=MELBAND_STAGE_DIAGNOSTICS,
|
||
diagnostics_cb=_diag_progress_cb(job_id) if MELBAND_STAGE_DIAGNOSTICS else None,
|
||
dim_t_override=_dim_t,
|
||
segment_secs_override=segment_secs_override,
|
||
)
|
||
|
||
jm.update_job(job_id, progress="Removing reverb with MelBand...")
|
||
|
||
temp_out = JOBS_DIR / job_id / "mel_band_dereverb_out"
|
||
temp_out.mkdir(parents=True, exist_ok=True)
|
||
|
||
_bs = batch_size_override
|
||
_overlap = num_overlap_override
|
||
_dur = get_audio_duration_secs(input_path)
|
||
_chunk_mode = (chunking_mode or LONG_TRACK_CHUNKING_MODE)
|
||
_chunk_secs = int(chunk_duration_secs or LONG_TRACK_CHUNK_DURATION_SECS)
|
||
_chunk_ov = int(chunk_overlap_secs or LONG_TRACK_CHUNK_OVERLAP_SECS)
|
||
_use_chunking = should_chunk(_dur, _chunk_mode, LONG_TRACK_DURATION_SECS)
|
||
|
||
try:
|
||
if _use_chunking:
|
||
_total_samples = max(1, int(round(_dur * 44100)))
|
||
_chunk_plan = plan_chunks(_total_samples, 44100, _chunk_secs, _chunk_ov)
|
||
_n = len(_chunk_plan.chunks)
|
||
jm.update_job(
|
||
job_id,
|
||
progress=(
|
||
f"Chunked mode: {_n} chunk(s) × {_chunk_secs} s "
|
||
f"with {_chunk_ov} s crossfade overlap..."
|
||
),
|
||
)
|
||
stem_paths = sep.separate_into_parts_chunked(
|
||
input_path, str(temp_out),
|
||
chunk_plan=_chunk_plan,
|
||
batch_size_override=_bs,
|
||
num_overlap_override=_overlap,
|
||
progress_cb=lambda msg: jm.update_job(job_id, progress=msg),
|
||
)
|
||
else:
|
||
for _attempt in range(3):
|
||
try:
|
||
stem_paths = sep.separate_into_parts(
|
||
input_path, str(temp_out),
|
||
batch_size_override=_bs,
|
||
num_overlap_override=_overlap,
|
||
)
|
||
break
|
||
except Exception as _exc:
|
||
_is_oom = "out of memory" in str(_exc).lower() or "OutOfMemory" in type(_exc).__name__
|
||
if not _is_oom:
|
||
raise
|
||
flush_vram_cache()
|
||
if _attempt == 0:
|
||
jm.update_job(job_id, progress="GPU OOM — retrying with batch_size=1...")
|
||
_bs = 1
|
||
elif _attempt == 1:
|
||
jm.update_job(job_id, progress="GPU OOM — retrying with batch_size=1, num_overlap=2...")
|
||
_bs = 1
|
||
_overlap = 2
|
||
else:
|
||
raise
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
_prefix = stem_prefix or Path(input_path).stem
|
||
stem_files = {}
|
||
|
||
for stem_name, path in sorted(stem_paths.items()):
|
||
src = Path(path)
|
||
# Map internal names to display names
|
||
display_name = stem_name.replace('_', ' ').title()
|
||
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
|
||
shutil.move(str(src), str(dest))
|
||
copy_metadata(input_path, str(dest), stem_label=display_name, copy_lyrics=True)
|
||
stem_files[stem_name.lower()] = str(dest)
|
||
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
|
||
return stem_files
|
||
|
||
except Exception:
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
raise
|
||
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Karaoke OOM capability-preserving fallback
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _karaoke_oom_fallback(
|
||
job_id: str,
|
||
failed_model: str,
|
||
failed_model_name: str,
|
||
input_path: str,
|
||
output_format: str,
|
||
stems_dir: Path,
|
||
mp3_bitrate: str,
|
||
metadata: dict,
|
||
stem_prefix: Optional[str],
|
||
) -> Dict[str, str]:
|
||
"""Try lighter lead/back-capable models after exhausting karaoke OOM retries.
|
||
|
||
Iterates ``KARAOKE_OOM_FALLBACK_CHAIN`` (skipping the model that already
|
||
failed) and returns the first successful result.
|
||
|
||
Policy (controlled by ``FALLBACK_REQUIRE_LEAD_BACK``):
|
||
- ``true`` (default) — only fall back to a model that also produces
|
||
*lead* and *backing_instrumental* stems.
|
||
- ``false`` — skip the cross-model fallback entirely and raise immediately.
|
||
|
||
Raises:
|
||
RuntimeError: if all fallbacks fail or policy forbids cross-model fallback.
|
||
"""
|
||
if not FALLBACK_REQUIRE_LEAD_BACK:
|
||
raise RuntimeError(
|
||
f"'{failed_model_name}' ran out of GPU memory after all parameter "
|
||
"reductions (batch_size=1, num_overlap=2). "
|
||
"FALLBACK_REQUIRE_LEAD_BACK=false — no cross-model fallback attempted."
|
||
)
|
||
|
||
_tried: list[str] = []
|
||
for fb_model in KARAOKE_OOM_FALLBACK_CHAIN:
|
||
if fb_model == failed_model:
|
||
continue # don't retry the model that already OOM'd
|
||
|
||
fb_label = MODEL_LABELS.get(fb_model, fb_model)
|
||
jm.update_job(
|
||
job_id,
|
||
progress=(
|
||
f"'{failed_model_name}' OOM after full parameter reduction — "
|
||
f"falling back to '{fb_label}' (lead/backing-capable)..."
|
||
),
|
||
)
|
||
flush_vram_cache()
|
||
|
||
if not ensure_model_available(job_id, fb_model):
|
||
jm.update_job(
|
||
job_id,
|
||
progress=f"Fallback '{fb_label}' not available — skipping.",
|
||
)
|
||
_tried.append(fb_model)
|
||
continue
|
||
|
||
try:
|
||
# mdxnet_karaoke is the only entry in the default chain; its runner
|
||
# handles all file moves and metadata tags internally.
|
||
return _run_mdxnet(
|
||
job_id, input_path, fb_model, output_format, stems_dir,
|
||
mp3_bitrate=mp3_bitrate,
|
||
metadata=metadata,
|
||
stem_prefix=stem_prefix,
|
||
)
|
||
except Exception as fb_exc:
|
||
is_oom = (
|
||
"out of memory" in str(fb_exc).lower()
|
||
or "OutOfMemory" in type(fb_exc).__name__
|
||
)
|
||
if is_oom:
|
||
jm.update_job(
|
||
job_id,
|
||
progress=f"Fallback '{fb_label}' also OOM — no more fallbacks.",
|
||
)
|
||
flush_vram_cache()
|
||
_tried.append(fb_model)
|
||
continue
|
||
raise # non-OOM error from fallback — surface it
|
||
|
||
tried_str = ", ".join(_tried) if _tried else "none available"
|
||
raise RuntimeError(
|
||
f"'{failed_model_name}' ran out of GPU memory after all parameter reductions "
|
||
f"(batch_size=1, num_overlap=2). "
|
||
f"All lead/backing-capable fallback models also failed ({tried_str}).\n"
|
||
f"Suggested fixes:\n"
|
||
f" 1. Use the 'eco' power preset\n"
|
||
f" 2. Lower STEMENGINE_GPU_VRAM_RESERVE_MB (reduce headroom, frees VRAM)\n"
|
||
f" 3. Restart the container to clear CUDA context fragmentation\n"
|
||
f" 4. Use a shorter track (< {LONG_TRACK_DURATION_SECS} s)"
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# MelBand Karaoke (Lead/Back) (Python API)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _run_mel_band_karaoke(
|
||
job_id: str,
|
||
input_path: str,
|
||
model: str,
|
||
output_format: str,
|
||
stems_dir: Path,
|
||
mp3_bitrate: str = '256k',
|
||
metadata: Optional[Dict] = None,
|
||
stem_prefix: Optional[str] = None,
|
||
batch_size_override: Optional[int] = None,
|
||
num_overlap_override: Optional[int] = None,
|
||
dim_t_override: Optional[int] = None,
|
||
segment_secs_override: Optional[float] = None,
|
||
chunking_mode: Optional[str] = None,
|
||
chunk_duration_secs: Optional[int] = None,
|
||
chunk_overlap_secs: Optional[int] = None,
|
||
) -> Dict[str, str]:
|
||
"""Run MelBand Karaoke separation (lead vocals/backing+instrumental)."""
|
||
# Route to appropriate separator based on model variant
|
||
if model == 'mel_band_karaoke_gabox_small':
|
||
from .mel_band_specialized_separator import MelBandKaraokeGaboxSmallSeparator
|
||
sep_class = MelBandKaraokeGaboxSmallSeparator
|
||
model_name = "MelBand Karaoke Gabox (Small)"
|
||
elif model == 'mel_band_karaoke_gabox_v1':
|
||
from .mel_band_specialized_separator import MelBandKaraokeGaboxV1Separator
|
||
sep_class = MelBandKaraokeGaboxV1Separator
|
||
model_name = "MelBand Karaoke Gabox V1"
|
||
elif model == 'mel_band_karaoke_gabox_v2_exp':
|
||
from .mel_band_specialized_separator import MelBandKaraokeGaboxV2ExperimentalSeparator
|
||
sep_class = MelBandKaraokeGaboxV2ExperimentalSeparator
|
||
model_name = "MelBand Karaoke Gabox V2 (Experimental)"
|
||
elif model == 'mel_band_karaoke_gabox':
|
||
from .mel_band_specialized_separator import MelBandKaraokeGaboxSeparator
|
||
sep_class = MelBandKaraokeGaboxSeparator
|
||
model_name = "MelBand Karaoke Gabox"
|
||
elif model == 'bs_roformer_karaoke_anvuew':
|
||
from .mel_band_specialized_separator import BSRoformerKaraokeAnvuewSeparator
|
||
sep_class = BSRoformerKaraokeAnvuewSeparator
|
||
model_name = "BS-RoFormer Karaoke Anvuew"
|
||
elif model == 'bs_roformer_karaoke_frazer_becruily':
|
||
from .mel_band_specialized_separator import MelBandKaraokeFrazerBecruilySeparator
|
||
sep_class = MelBandKaraokeFrazerBecruilySeparator
|
||
model_name = "BS-RoFormer Karaoke Frazer+Becruily"
|
||
else:
|
||
from .mel_band_specialized_separator import MelBandKaraokeSeparator
|
||
sep_class = MelBandKaraokeSeparator
|
||
model_name = "MelBand Karaoke"
|
||
|
||
jm.update_job(
|
||
job_id, status=jm.JobStatus.RUNNING,
|
||
progress=f"Loading {model_name} model...",
|
||
)
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
# Flush allocator cache immediately before loading the model checkpoint.
|
||
# PyTorch holds reserved-but-idle blocks from previous jobs; releasing them
|
||
# here maximises contiguous VRAM available for the ~1.7 GB checkpoint.
|
||
try:
|
||
import gc, torch
|
||
gc.collect()
|
||
if torch.cuda.is_available():
|
||
torch.cuda.empty_cache()
|
||
except Exception:
|
||
pass
|
||
|
||
_dim_t = dim_t_override if dim_t_override is not None else MELBAND_DIM_T_OVERRIDE
|
||
|
||
sep = _init_melband_separator_compat(
|
||
sep_class,
|
||
cpu_separation=CPU_SEPARATION,
|
||
output_format=output_format,
|
||
mp3_bitrate=mp3_bitrate,
|
||
diagnostics_enabled=MELBAND_STAGE_DIAGNOSTICS,
|
||
diagnostics_cb=_diag_progress_cb(job_id) if MELBAND_STAGE_DIAGNOSTICS else None,
|
||
dim_t_override=_dim_t,
|
||
segment_secs_override=segment_secs_override,
|
||
)
|
||
|
||
jm.update_job(job_id, progress="Separating lead/backing vocals...")
|
||
|
||
temp_out = JOBS_DIR / job_id / "mel_band_karaoke_out"
|
||
temp_out.mkdir(parents=True, exist_ok=True)
|
||
|
||
# Duration guardrail / chunking decision.
|
||
_bs = batch_size_override
|
||
_overlap = num_overlap_override
|
||
_dur = get_audio_duration_secs(input_path)
|
||
_chunk_mode = (chunking_mode or LONG_TRACK_CHUNKING_MODE)
|
||
_chunk_secs = int(chunk_duration_secs or LONG_TRACK_CHUNK_DURATION_SECS)
|
||
_chunk_ov = int(chunk_overlap_secs or LONG_TRACK_CHUNK_OVERLAP_SECS)
|
||
_guardrail_secs = LONG_TRACK_DURATION_SECS
|
||
if model == 'mel_band_karaoke_gabox_small':
|
||
_guardrail_secs = min(_guardrail_secs, GABOX_SMALL_MODEL_MAX_DURATION_SECS)
|
||
|
||
# Chunked path: audio is processed in LONG_TRACK_CHUNK_DURATION_SECS segments
|
||
# so peak VRAM scales with chunk size, not track length. The small Gabox model
|
||
# is excluded — it is already lightweight and chunking adds unnecessary overhead.
|
||
_use_chunking = (
|
||
should_chunk(_dur, _chunk_mode, LONG_TRACK_DURATION_SECS)
|
||
and model not in ('mel_band_karaoke_gabox_small',)
|
||
)
|
||
|
||
_final_oom_exc: Optional[Exception] = None
|
||
|
||
if _use_chunking:
|
||
_total_samples = max(1, int(round(_dur * 44100)))
|
||
_chunk_plan = plan_chunks(
|
||
_total_samples, 44100,
|
||
_chunk_secs,
|
||
_chunk_ov,
|
||
)
|
||
_n = len(_chunk_plan.chunks)
|
||
jm.update_job(
|
||
job_id,
|
||
progress=(
|
||
f"Chunked mode: {_n} chunk(s) \u00d7 {_chunk_secs} s "
|
||
f"with {_chunk_ov} s crossfade overlap..."
|
||
),
|
||
)
|
||
try:
|
||
stem_paths = sep.separate_into_parts_chunked(
|
||
input_path, str(temp_out),
|
||
chunk_plan=_chunk_plan,
|
||
batch_size_override=_bs,
|
||
num_overlap_override=_overlap,
|
||
progress_cb=lambda msg: jm.update_job(job_id, progress=msg),
|
||
)
|
||
except Exception as _exc:
|
||
_is_oom = (
|
||
"out of memory" in str(_exc).lower()
|
||
or "OutOfMemory" in type(_exc).__name__
|
||
)
|
||
cleanup_temp(temp_out)
|
||
if _is_oom:
|
||
# Chunked OOM still triggers the capability fallback.
|
||
_final_oom_exc = _exc
|
||
else:
|
||
raise
|
||
else:
|
||
# Non-chunked: eco-params guardrail + 3-step OOM ladder.
|
||
# On exhaustion, _karaoke_oom_fallback() tries a lighter model.
|
||
if _dur > 0 and _dur > _guardrail_secs:
|
||
jm.update_job(
|
||
job_id,
|
||
progress=(
|
||
f"Long track ({_dur:.0f} s > {_guardrail_secs} s threshold) — "
|
||
"forcing eco params (batch_size=1, num_overlap=2) to prevent OOM..."
|
||
),
|
||
)
|
||
if _bs is None or _bs > 1:
|
||
_bs = 1
|
||
if _overlap is None or _overlap > 2:
|
||
_overlap = 2
|
||
|
||
for _attempt in range(3):
|
||
try:
|
||
stem_paths = sep.separate_into_parts(
|
||
input_path, str(temp_out),
|
||
batch_size_override=_bs,
|
||
num_overlap_override=_overlap,
|
||
)
|
||
_final_oom_exc = None
|
||
break # success
|
||
except Exception as _exc:
|
||
_is_oom = (
|
||
"out of memory" in str(_exc).lower()
|
||
or "OutOfMemory" in type(_exc).__name__
|
||
)
|
||
if not _is_oom:
|
||
cleanup_temp(temp_out)
|
||
raise
|
||
flush_vram_cache()
|
||
if _attempt == 0:
|
||
jm.update_job(job_id, progress="GPU OOM \u2014 retrying with batch_size=1...")
|
||
_bs = 1
|
||
_final_oom_exc = _exc
|
||
elif _attempt == 1:
|
||
jm.update_job(
|
||
job_id,
|
||
progress="GPU OOM \u2014 retrying with batch_size=1, num_overlap=2...",
|
||
)
|
||
_bs = 1
|
||
_overlap = 2
|
||
_final_oom_exc = _exc
|
||
else:
|
||
# All parameter reductions exhausted — will try capability fallback
|
||
_final_oom_exc = _exc
|
||
cleanup_temp(temp_out)
|
||
|
||
if _final_oom_exc is not None:
|
||
return _karaoke_oom_fallback(
|
||
job_id=job_id,
|
||
failed_model=model,
|
||
failed_model_name=model_name,
|
||
input_path=input_path,
|
||
output_format=output_format,
|
||
stems_dir=stems_dir,
|
||
mp3_bitrate=mp3_bitrate,
|
||
metadata=metadata or {},
|
||
stem_prefix=stem_prefix,
|
||
)
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
_prefix = stem_prefix or Path(input_path).stem
|
||
stem_files = {}
|
||
|
||
try:
|
||
for stem_name, path in sorted(stem_paths.items()):
|
||
src = Path(path)
|
||
display_name = stem_name.replace('_', ' ').title()
|
||
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
|
||
shutil.move(str(src), str(dest))
|
||
copy_metadata(input_path, str(dest), stem_label=display_name, copy_lyrics=True)
|
||
stem_files[stem_name.lower()] = str(dest)
|
||
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
|
||
return stem_files
|
||
|
||
except Exception:
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
raise
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# MDX-Net (ONNX) (Python API)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _run_mdxnet(
|
||
job_id: str,
|
||
input_path: str,
|
||
model: str,
|
||
output_format: str,
|
||
stems_dir: Path,
|
||
mp3_bitrate: str = '256k',
|
||
metadata: Optional[Dict] = None,
|
||
stem_prefix: Optional[str] = None,
|
||
chunking_mode: Optional[str] = None,
|
||
chunk_duration_secs: Optional[int] = None,
|
||
chunk_overlap_secs: Optional[int] = None,
|
||
) -> Dict[str, str]:
|
||
"""
|
||
Run MDX-Net separation using ONNX models.
|
||
Returns dict mapping stem names to file paths.
|
||
"""
|
||
from .mdxnet_separator import MDXNetSeparator
|
||
from .separation.chunking import plan_chunks, should_chunk
|
||
|
||
jm.update_job(
|
||
job_id, status=jm.JobStatus.RUNNING,
|
||
progress=f"Loading MDX-Net model ({model})...",
|
||
)
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
try:
|
||
import gc, torch
|
||
gc.collect()
|
||
if torch.cuda.is_available():
|
||
torch.cuda.empty_cache()
|
||
except Exception:
|
||
pass
|
||
|
||
sep = MDXNetSeparator(
|
||
model_id=model,
|
||
cpu_separation=CPU_SEPARATION,
|
||
output_format=output_format,
|
||
mp3_bitrate=mp3_bitrate,
|
||
)
|
||
|
||
jm.update_job(job_id, progress="Separating with MDX-Net...")
|
||
|
||
temp_out = JOBS_DIR / job_id / "mdxnet_out"
|
||
temp_out.mkdir(parents=True, exist_ok=True)
|
||
|
||
# Duration guardrail / chunking decision.
|
||
_dur = get_audio_duration_secs(input_path)
|
||
_chunk_mode = (chunking_mode or LONG_TRACK_CHUNKING_MODE)
|
||
_chunk_secs = int(chunk_duration_secs or LONG_TRACK_CHUNK_DURATION_SECS)
|
||
_chunk_ov = int(chunk_overlap_secs or LONG_TRACK_CHUNK_OVERLAP_SECS)
|
||
_use_chunking = should_chunk(_dur, _chunk_mode, LONG_TRACK_DURATION_SECS)
|
||
|
||
try:
|
||
if _use_chunking:
|
||
_total_samples = max(1, int(round(_dur * 44100)))
|
||
_chunk_plan = plan_chunks(
|
||
_total_samples, 44100,
|
||
_chunk_secs,
|
||
_chunk_ov,
|
||
)
|
||
_n = len(_chunk_plan.chunks)
|
||
jm.update_job(
|
||
job_id,
|
||
progress=(
|
||
f"Chunked mode: {_n} chunk(s) × {_chunk_secs} s "
|
||
f"with {_chunk_ov} s crossfade overlap..."
|
||
),
|
||
)
|
||
stem_paths = sep.separate_into_parts_chunked(
|
||
input_path, str(temp_out),
|
||
chunk_plan=_chunk_plan,
|
||
progress_cb=lambda msg: jm.update_job(job_id, progress=msg),
|
||
)
|
||
else:
|
||
stem_paths = sep.separate_into_parts(input_path, str(temp_out))
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
_prefix = stem_prefix or Path(input_path).stem
|
||
stem_files = {}
|
||
|
||
for stem_name, path in sorted(stem_paths.items()):
|
||
src = Path(path)
|
||
display_name = stem_name.replace('_', ' ').title()
|
||
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
|
||
shutil.move(str(src), str(dest))
|
||
copy_metadata(input_path, str(dest), stem_label=display_name, copy_lyrics=True)
|
||
stem_files[stem_name.lower()] = str(dest)
|
||
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
|
||
return stem_files
|
||
|
||
except Exception:
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
raise
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# MelBand RoFormer (Python API)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _run_mel_band_roformer(
|
||
job_id: str,
|
||
input_path: str,
|
||
model: str,
|
||
output_format: str,
|
||
stems_dir: Path,
|
||
mp3_bitrate: str = '256k',
|
||
metadata: Optional[Dict] = None,
|
||
stem_prefix: Optional[str] = None,
|
||
batch_size_override: Optional[int] = None,
|
||
num_overlap_override: Optional[int] = None,
|
||
dim_t_override: Optional[int] = None,
|
||
segment_secs_override: Optional[float] = None,
|
||
chunking_mode: Optional[str] = None,
|
||
chunk_duration_secs: Optional[int] = None,
|
||
chunk_overlap_secs: Optional[int] = None,
|
||
) -> Dict[str, str]:
|
||
"""
|
||
Run MelBand RoFormer separation. Returns dict mapping stem names to file paths.
|
||
"""
|
||
from .mel_band_roformer_separator import MelBandRoformerSeparator
|
||
|
||
jm.update_job(
|
||
job_id, status=jm.JobStatus.RUNNING,
|
||
progress="Loading MelBand RoFormer model...",
|
||
)
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
try:
|
||
import gc, torch
|
||
gc.collect()
|
||
if torch.cuda.is_available():
|
||
torch.cuda.empty_cache()
|
||
except Exception:
|
||
pass
|
||
|
||
_dim_t = dim_t_override if dim_t_override is not None else MELBAND_DIM_T_OVERRIDE
|
||
|
||
sep = MelBandRoformerSeparator(
|
||
cpu_separation=CPU_SEPARATION,
|
||
output_format=output_format,
|
||
mp3_bitrate=mp3_bitrate,
|
||
diagnostics_enabled=MELBAND_STAGE_DIAGNOSTICS,
|
||
diagnostics_cb=_diag_progress_cb(job_id) if MELBAND_STAGE_DIAGNOSTICS else None,
|
||
dim_t_override=_dim_t,
|
||
segment_secs_override=segment_secs_override,
|
||
)
|
||
|
||
jm.update_job(job_id, progress="Separating with MelBand RoFormer... (this may take a while)")
|
||
|
||
# Use a temp directory then move to project
|
||
temp_out = JOBS_DIR / job_id / "mel_band_out"
|
||
temp_out.mkdir(parents=True, exist_ok=True)
|
||
|
||
# Duration guardrail / chunking decision.
|
||
_bs = batch_size_override
|
||
_overlap = num_overlap_override
|
||
_dur = get_audio_duration_secs(input_path)
|
||
_chunk_mode = (chunking_mode or LONG_TRACK_CHUNKING_MODE)
|
||
_chunk_secs = int(chunk_duration_secs or LONG_TRACK_CHUNK_DURATION_SECS)
|
||
_chunk_ov = int(chunk_overlap_secs or LONG_TRACK_CHUNK_OVERLAP_SECS)
|
||
_use_chunking = should_chunk(_dur, _chunk_mode, LONG_TRACK_DURATION_SECS)
|
||
|
||
if _use_chunking:
|
||
_total_samples = max(1, int(round(_dur * 44100)))
|
||
_chunk_plan = plan_chunks(
|
||
_total_samples, 44100,
|
||
_chunk_secs,
|
||
_chunk_ov,
|
||
)
|
||
_n = len(_chunk_plan.chunks)
|
||
jm.update_job(
|
||
job_id,
|
||
progress=(
|
||
f"Chunked mode: {_n} chunk(s) \u00d7 {_chunk_secs} s "
|
||
f"with {_chunk_ov} s crossfade overlap..."
|
||
),
|
||
)
|
||
try:
|
||
stem_paths = sep.separate_into_parts_chunked(
|
||
input_path, str(temp_out),
|
||
chunk_plan=_chunk_plan,
|
||
batch_size_override=_bs,
|
||
num_overlap_override=_overlap,
|
||
progress_cb=lambda msg: jm.update_job(job_id, progress=msg),
|
||
)
|
||
except Exception:
|
||
cleanup_temp(temp_out)
|
||
raise
|
||
else:
|
||
# Non-chunked: eco-params guardrail + 3-step OOM ladder.
|
||
if _dur > 0 and _dur > LONG_TRACK_DURATION_SECS:
|
||
jm.update_job(
|
||
job_id,
|
||
progress=(
|
||
f"Long track ({_dur:.0f} s > {LONG_TRACK_DURATION_SECS} s threshold) — "
|
||
"forcing eco params (batch_size=1, num_overlap=2) to prevent OOM..."
|
||
),
|
||
)
|
||
if _bs is None or _bs > 1:
|
||
_bs = 1
|
||
if _overlap is None or _overlap > 2:
|
||
_overlap = 2
|
||
|
||
# 3-step OOM ladder: full params → batch_size=1 → batch_size=1 + num_overlap=2.
|
||
for _attempt in range(3):
|
||
try:
|
||
stem_paths = sep.separate_into_parts(
|
||
input_path, str(temp_out),
|
||
batch_size_override=_bs,
|
||
num_overlap_override=_overlap,
|
||
)
|
||
break
|
||
except Exception as _exc:
|
||
_is_oom = (
|
||
"out of memory" in str(_exc).lower()
|
||
or "OutOfMemory" in type(_exc).__name__
|
||
)
|
||
if not _is_oom:
|
||
cleanup_temp(temp_out)
|
||
raise
|
||
flush_vram_cache()
|
||
if _attempt == 0:
|
||
jm.update_job(job_id, progress="GPU OOM \u2014 retrying with batch_size=1...")
|
||
_bs = 1
|
||
elif _attempt == 1:
|
||
jm.update_job(
|
||
job_id,
|
||
progress="GPU OOM \u2014 retrying with batch_size=1, num_overlap=2...",
|
||
)
|
||
_bs = 1
|
||
_overlap = 2
|
||
else:
|
||
cleanup_temp(temp_out)
|
||
raise
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
# Move to project stems folder with proper naming
|
||
_prefix = stem_prefix or Path(input_path).stem
|
||
stem_files = {}
|
||
|
||
try:
|
||
for stem_name, path in sorted(stem_paths.items()):
|
||
src = Path(path)
|
||
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
|
||
shutil.move(str(src), str(dest))
|
||
copy_metadata(input_path, str(dest), stem_label=stem_name.capitalize(), copy_lyrics=True)
|
||
stem_files[stem_name.lower()] = str(dest)
|
||
|
||
# Cleanup temp directory
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
|
||
return stem_files
|
||
|
||
except Exception:
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
raise
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# BS-RoFormer (Python API)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _run_bsroformer(
|
||
job_id: str,
|
||
input_path: str,
|
||
model: str,
|
||
output_format: str,
|
||
stems_dir: Path,
|
||
mp3_bitrate: str = '256k',
|
||
metadata: Optional[Dict] = None,
|
||
stem_prefix: Optional[str] = None,
|
||
batch_size_override: Optional[int] = None,
|
||
num_overlap_override: Optional[int] = None,
|
||
chunking_mode: Optional[str] = None,
|
||
chunk_duration_secs: Optional[int] = None,
|
||
chunk_overlap_secs: Optional[int] = None,
|
||
model_path_override: Optional[str] = None,
|
||
config_path_override: Optional[str] = None,
|
||
) -> Dict[str, str]:
|
||
"""
|
||
Run BS-RoFormer separation. Returns dict mapping stem names to file paths.
|
||
"""
|
||
from .bs_roformer_separator import BSRoformerSeparator
|
||
|
||
stem_mode = BSR_STEM_MODE.get(model, '4stem')
|
||
jm.update_job(
|
||
job_id, status=jm.JobStatus.RUNNING,
|
||
progress=f"Loading BS-RoFormer ({stem_mode}) model...",
|
||
)
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
try:
|
||
import gc, torch
|
||
gc.collect()
|
||
if torch.cuda.is_available():
|
||
torch.cuda.empty_cache()
|
||
except Exception:
|
||
pass
|
||
|
||
sep = BSRoformerSeparator(
|
||
model_path=model_path_override,
|
||
config_path=config_path_override,
|
||
cpu_separation=CPU_SEPARATION,
|
||
output_format=output_format,
|
||
mp3_bitrate=mp3_bitrate,
|
||
stem_mode=stem_mode,
|
||
)
|
||
|
||
jm.update_job(job_id, progress="Separating with BS-RoFormer... (this may take a while)")
|
||
|
||
temp_out = JOBS_DIR / job_id / "bsr_out"
|
||
temp_out.mkdir(parents=True, exist_ok=True)
|
||
|
||
_bs = batch_size_override
|
||
_dur = get_audio_duration_secs(input_path)
|
||
_chunk_mode = (chunking_mode or LONG_TRACK_CHUNKING_MODE)
|
||
_chunk_secs = int(chunk_duration_secs or LONG_TRACK_CHUNK_DURATION_SECS)
|
||
_chunk_ov = int(chunk_overlap_secs or LONG_TRACK_CHUNK_OVERLAP_SECS)
|
||
_use_chunking = should_chunk(_dur, _chunk_mode, LONG_TRACK_DURATION_SECS)
|
||
|
||
if _use_chunking:
|
||
_total_samples = max(1, int(round(_dur * 44100)))
|
||
_chunk_plan = plan_chunks(
|
||
_total_samples, 44100,
|
||
_chunk_secs,
|
||
_chunk_ov,
|
||
)
|
||
_n = len(_chunk_plan.chunks)
|
||
jm.update_job(
|
||
job_id,
|
||
progress=(
|
||
f"Chunked mode: {_n} chunk(s) \u00d7 {_chunk_secs} s "
|
||
f"with {_chunk_ov} s crossfade overlap..."
|
||
),
|
||
)
|
||
try:
|
||
stem_paths = sep.separate_into_parts_chunked(
|
||
input_path, str(temp_out),
|
||
chunk_plan=_chunk_plan,
|
||
batch_size_override=_bs,
|
||
num_overlap_override=num_overlap_override,
|
||
progress_cb=lambda msg: jm.update_job(job_id, progress=msg),
|
||
)
|
||
except Exception:
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
raise
|
||
else:
|
||
for _attempt in range(2):
|
||
try:
|
||
stem_paths = sep.separate_into_parts(
|
||
input_path, str(temp_out),
|
||
batch_size_override=_bs,
|
||
num_overlap_override=num_overlap_override,
|
||
)
|
||
break
|
||
except Exception as _exc:
|
||
_is_oom = "out of memory" in str(_exc).lower() or "OutOfMemory" in type(_exc).__name__
|
||
if _is_oom and _attempt == 0:
|
||
import gc, torch
|
||
jm.update_job(job_id, progress="GPU OOM \u2014 retrying with batch_size=1...")
|
||
gc.collect()
|
||
torch.cuda.empty_cache()
|
||
_bs = 1
|
||
continue
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
raise
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
_prefix = stem_prefix or Path(input_path).stem
|
||
stem_files = {}
|
||
|
||
try:
|
||
for stem_name, path in sorted(stem_paths.items()):
|
||
src = Path(path)
|
||
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
|
||
shutil.move(str(src), str(dest))
|
||
copy_metadata(input_path, str(dest), stem_label=stem_name.capitalize(), copy_lyrics=True)
|
||
stem_files[stem_name.lower()] = str(dest)
|
||
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
|
||
return stem_files
|
||
|
||
except Exception:
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
raise
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# MelBand RoFormer Deux Becruily (Python API)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _run_mel_band_deux_becruily(
|
||
job_id: str,
|
||
input_path: str,
|
||
model: str,
|
||
output_format: str,
|
||
stems_dir: Path,
|
||
mp3_bitrate: str = '256k',
|
||
metadata: Optional[Dict] = None,
|
||
stem_prefix: Optional[str] = None,
|
||
batch_size_override: Optional[int] = None,
|
||
num_overlap_override: Optional[int] = None,
|
||
dim_t_override: Optional[int] = None,
|
||
segment_secs_override: Optional[float] = None,
|
||
chunking_mode: Optional[str] = None,
|
||
chunk_duration_secs: Optional[int] = None,
|
||
chunk_overlap_secs: Optional[int] = None,
|
||
) -> Dict[str, str]:
|
||
"""Run MelBand RoFormer Deux Becruily separation (2-stem vocals)."""
|
||
from .mel_band_specialized_separator import MelBandDeuxBecruilySeparator
|
||
|
||
jm.update_job(
|
||
job_id, status=jm.JobStatus.RUNNING,
|
||
progress="Loading MelBand Deux Becruily model...",
|
||
)
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
try:
|
||
import gc, torch
|
||
gc.collect()
|
||
if torch.cuda.is_available():
|
||
torch.cuda.empty_cache()
|
||
except Exception:
|
||
pass
|
||
|
||
_dim_t = dim_t_override if dim_t_override is not None else MELBAND_DIM_T_OVERRIDE
|
||
|
||
sep = MelBandDeuxBecruilySeparator(
|
||
cpu_separation=CPU_SEPARATION,
|
||
output_format=output_format,
|
||
mp3_bitrate=mp3_bitrate,
|
||
diagnostics_enabled=MELBAND_STAGE_DIAGNOSTICS,
|
||
diagnostics_cb=_diag_progress_cb(job_id) if MELBAND_STAGE_DIAGNOSTICS else None,
|
||
dim_t_override=_dim_t,
|
||
segment_secs_override=segment_secs_override,
|
||
)
|
||
|
||
jm.update_job(job_id, progress="Separating with Deux Becruily...")
|
||
|
||
temp_out = JOBS_DIR / job_id / "mel_band_deux_out"
|
||
temp_out.mkdir(parents=True, exist_ok=True)
|
||
|
||
_bs = batch_size_override
|
||
_overlap = num_overlap_override
|
||
_dur = get_audio_duration_secs(input_path)
|
||
_chunk_mode = (chunking_mode or LONG_TRACK_CHUNKING_MODE)
|
||
_chunk_secs = int(chunk_duration_secs or LONG_TRACK_CHUNK_DURATION_SECS)
|
||
_chunk_ov = int(chunk_overlap_secs or LONG_TRACK_CHUNK_OVERLAP_SECS)
|
||
_use_chunking = should_chunk(_dur, _chunk_mode, LONG_TRACK_DURATION_SECS)
|
||
|
||
try:
|
||
if _use_chunking:
|
||
_total_samples = max(1, int(round(_dur * 44100)))
|
||
_chunk_plan = plan_chunks(_total_samples, 44100, _chunk_secs, _chunk_ov)
|
||
_n = len(_chunk_plan.chunks)
|
||
jm.update_job(
|
||
job_id,
|
||
progress=(
|
||
f"Chunked mode: {_n} chunk(s) × {_chunk_secs} s "
|
||
f"with {_chunk_ov} s crossfade overlap..."
|
||
),
|
||
)
|
||
stem_paths = sep.separate_into_parts_chunked(
|
||
input_path, str(temp_out),
|
||
chunk_plan=_chunk_plan,
|
||
batch_size_override=_bs,
|
||
num_overlap_override=_overlap,
|
||
progress_cb=lambda msg: jm.update_job(job_id, progress=msg),
|
||
)
|
||
else:
|
||
for _attempt in range(3):
|
||
try:
|
||
stem_paths = sep.separate_into_parts(
|
||
input_path, str(temp_out),
|
||
batch_size_override=_bs,
|
||
num_overlap_override=_overlap,
|
||
)
|
||
break
|
||
except Exception as _exc:
|
||
_is_oom = "out of memory" in str(_exc).lower() or "OutOfMemory" in type(_exc).__name__
|
||
if not _is_oom:
|
||
raise
|
||
flush_vram_cache()
|
||
if _attempt == 0:
|
||
jm.update_job(job_id, progress="GPU OOM — retrying with batch_size=1...")
|
||
_bs = 1
|
||
elif _attempt == 1:
|
||
jm.update_job(job_id, progress="GPU OOM — retrying with batch_size=1, num_overlap=2...")
|
||
_bs = 1
|
||
_overlap = 2
|
||
else:
|
||
raise
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
_prefix = stem_prefix or Path(input_path).stem
|
||
stem_files = {}
|
||
|
||
for stem_name, path in sorted(stem_paths.items()):
|
||
src = Path(path)
|
||
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
|
||
shutil.move(str(src), str(dest))
|
||
copy_metadata(input_path, str(dest), stem_label=stem_name.capitalize(), copy_lyrics=True)
|
||
stem_files[stem_name.lower()] = str(dest)
|
||
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
|
||
return stem_files
|
||
|
||
except Exception:
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
raise
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# SCNet XL (Python API)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _run_scnet(
|
||
job_id: str,
|
||
input_path: str,
|
||
model: str,
|
||
output_format: str,
|
||
stems_dir: Path,
|
||
mp3_bitrate: str = '256k',
|
||
metadata: Optional[Dict] = None,
|
||
stem_prefix: Optional[str] = None,
|
||
chunk_size_override: Optional[int] = None,
|
||
num_overlap_override: Optional[int] = None,
|
||
chunking_mode: Optional[str] = None,
|
||
chunk_duration_secs: Optional[int] = None,
|
||
chunk_overlap_secs: Optional[int] = None,
|
||
) -> Dict[str, str]:
|
||
"""Run SCNet XL separation (state-of-the-art 2024-2025)."""
|
||
from .scnet_separator import SCNetSeparator
|
||
from .separation.chunking import plan_chunks, should_chunk
|
||
|
||
jm.update_job(
|
||
job_id, status=jm.JobStatus.RUNNING,
|
||
progress=f"Loading SCNet model ({model})...",
|
||
)
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
try:
|
||
import gc, torch
|
||
gc.collect()
|
||
if torch.cuda.is_available():
|
||
torch.cuda.empty_cache()
|
||
except Exception:
|
||
pass
|
||
|
||
sep = SCNetSeparator(
|
||
model_id=model,
|
||
cpu_separation=CPU_SEPARATION,
|
||
output_format=output_format,
|
||
mp3_bitrate=mp3_bitrate,
|
||
)
|
||
|
||
jm.update_job(job_id, progress="Separating with SCNet... (this may take a while)")
|
||
|
||
temp_out = JOBS_DIR / job_id / "scnet_out"
|
||
temp_out.mkdir(parents=True, exist_ok=True)
|
||
|
||
# Duration guardrail / chunking decision.
|
||
_dur = get_audio_duration_secs(input_path)
|
||
_chunk_mode = (chunking_mode or LONG_TRACK_CHUNKING_MODE)
|
||
_chunk_secs = int(chunk_duration_secs or LONG_TRACK_CHUNK_DURATION_SECS)
|
||
_chunk_ov = int(chunk_overlap_secs or LONG_TRACK_CHUNK_OVERLAP_SECS)
|
||
_use_chunking = should_chunk(_dur, _chunk_mode, LONG_TRACK_DURATION_SECS)
|
||
_cs = chunk_size_override
|
||
|
||
for _attempt in range(2):
|
||
try:
|
||
if _use_chunking:
|
||
_total_samples = max(1, int(round(_dur * 44100)))
|
||
_chunk_plan = plan_chunks(
|
||
_total_samples, 44100,
|
||
_chunk_secs,
|
||
_chunk_ov,
|
||
)
|
||
_n = len(_chunk_plan.chunks)
|
||
jm.update_job(
|
||
job_id,
|
||
progress=(
|
||
f"Chunked mode: {_n} chunk(s) × {_chunk_secs} s "
|
||
f"with {_chunk_ov} s crossfade overlap..."
|
||
),
|
||
)
|
||
stem_paths = sep.separate_into_parts_chunked(
|
||
input_path, str(temp_out),
|
||
chunk_plan=_chunk_plan,
|
||
chunk_size_override=_cs,
|
||
num_overlap_override=num_overlap_override,
|
||
progress_cb=lambda msg: jm.update_job(job_id, progress=msg),
|
||
)
|
||
else:
|
||
stem_paths = sep.separate_into_parts(
|
||
input_path, str(temp_out),
|
||
chunk_size_override=_cs,
|
||
num_overlap_override=num_overlap_override,
|
||
)
|
||
break
|
||
except Exception as _exc:
|
||
_is_oom = "out of memory" in str(_exc).lower() or "OutOfMemory" in type(_exc).__name__
|
||
if _is_oom and _attempt == 0:
|
||
import gc, torch
|
||
jm.update_job(job_id, progress="GPU OOM — retrying with smaller chunk size...")
|
||
gc.collect()
|
||
torch.cuda.empty_cache()
|
||
_cs = (_cs or 262144) // 2
|
||
continue
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
raise
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
_prefix = stem_prefix or Path(input_path).stem
|
||
stem_files = {}
|
||
|
||
try:
|
||
for stem_name, path in sorted(stem_paths.items()):
|
||
src = Path(path)
|
||
display_name = stem_name.replace('_', ' ').title()
|
||
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
|
||
shutil.move(str(src), str(dest))
|
||
copy_metadata(input_path, str(dest), stem_label=display_name, copy_lyrics=True)
|
||
stem_files[stem_name.lower()] = str(dest)
|
||
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
|
||
return stem_files
|
||
|
||
except Exception:
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
raise
|
||
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# MDX23C TFC-TDF U-Net (Python API)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _run_mdx23c(
|
||
job_id: str,
|
||
input_path: str,
|
||
model: str,
|
||
output_format: str,
|
||
stems_dir: Path,
|
||
mp3_bitrate: str = '256k',
|
||
metadata: Optional[Dict] = None,
|
||
stem_prefix: Optional[str] = None,
|
||
chunk_size_override: Optional[int] = None,
|
||
num_overlap_override: Optional[int] = None,
|
||
chunking_mode: Optional[str] = None,
|
||
chunk_duration_secs: Optional[int] = None,
|
||
chunk_overlap_secs: Optional[int] = None,
|
||
) -> Dict[str, str]:
|
||
"""Run MDX23C TFC-TDF U-Net separation (drum component separation)."""
|
||
from .mdx23c_separator import MDX23CSeparator
|
||
from .separation.chunking import plan_chunks, should_chunk
|
||
|
||
jm.update_job(
|
||
job_id, status=jm.JobStatus.RUNNING,
|
||
progress=f"Loading MDX23C model ({model})...",
|
||
)
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
try:
|
||
import gc, torch
|
||
gc.collect()
|
||
if torch.cuda.is_available():
|
||
torch.cuda.empty_cache()
|
||
except Exception:
|
||
pass
|
||
|
||
sep = MDX23CSeparator(
|
||
model_id=model,
|
||
cpu_separation=CPU_SEPARATION,
|
||
output_format=output_format,
|
||
mp3_bitrate=mp3_bitrate,
|
||
)
|
||
|
||
jm.update_job(job_id, progress="Separating with MDX23C... (this may take a while)")
|
||
|
||
temp_out = JOBS_DIR / job_id / "mdx23c_out"
|
||
temp_out.mkdir(parents=True, exist_ok=True)
|
||
|
||
_dur = get_audio_duration_secs(input_path)
|
||
_chunk_mode = (chunking_mode or LONG_TRACK_CHUNKING_MODE)
|
||
_chunk_secs = int(chunk_duration_secs or LONG_TRACK_CHUNK_DURATION_SECS)
|
||
_chunk_ov = int(chunk_overlap_secs or LONG_TRACK_CHUNK_OVERLAP_SECS)
|
||
_use_chunking = should_chunk(_dur, _chunk_mode, LONG_TRACK_DURATION_SECS)
|
||
_cs = chunk_size_override
|
||
|
||
for _attempt in range(2):
|
||
try:
|
||
if _use_chunking:
|
||
_total_samples = max(1, int(round(_dur * 44100)))
|
||
_chunk_plan = plan_chunks(
|
||
_total_samples, 44100,
|
||
_chunk_secs,
|
||
_chunk_ov,
|
||
)
|
||
_n = len(_chunk_plan.chunks)
|
||
jm.update_job(
|
||
job_id,
|
||
progress=(
|
||
f"Chunked mode: {_n} chunk(s) × {_chunk_secs} s "
|
||
f"with {_chunk_ov} s crossfade overlap..."
|
||
),
|
||
)
|
||
stem_paths = sep.separate_into_parts_chunked(
|
||
input_path, str(temp_out),
|
||
chunk_plan=_chunk_plan,
|
||
chunk_size_override=_cs,
|
||
num_overlap_override=num_overlap_override,
|
||
progress_cb=lambda msg: jm.update_job(job_id, progress=msg),
|
||
)
|
||
else:
|
||
stem_paths = sep.separate_into_parts(
|
||
input_path, str(temp_out),
|
||
chunk_size_override=_cs,
|
||
num_overlap_override=num_overlap_override,
|
||
)
|
||
break
|
||
except Exception as _exc:
|
||
_is_oom = "out of memory" in str(_exc).lower() or "OutOfMemory" in type(_exc).__name__
|
||
if _is_oom and _attempt == 0:
|
||
import gc, torch
|
||
jm.update_job(job_id, progress="GPU OOM — retrying with smaller chunk size...")
|
||
gc.collect()
|
||
torch.cuda.empty_cache()
|
||
_cs = (_cs or 130560) // 2
|
||
continue
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
raise
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
_prefix = stem_prefix or Path(input_path).stem
|
||
stem_files = {}
|
||
|
||
try:
|
||
for stem_name, path in sorted(stem_paths.items()):
|
||
src = Path(path)
|
||
display_name = stem_name.replace('_', ' ').title()
|
||
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
|
||
shutil.move(str(src), str(dest))
|
||
copy_metadata(input_path, str(dest), stem_label=display_name, copy_lyrics=True)
|
||
stem_files[stem_name.lower()] = str(dest)
|
||
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
|
||
return stem_files
|
||
|
||
except Exception:
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
raise
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# BS-RoFormer SW (Swin Transformer variant) (Python API)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _run_bsroformer_sw(
|
||
job_id: str,
|
||
input_path: str,
|
||
model: str,
|
||
output_format: str,
|
||
stems_dir: Path,
|
||
mp3_bitrate: str = '256k',
|
||
metadata: Optional[Dict] = None,
|
||
stem_prefix: Optional[str] = None,
|
||
) -> Dict[str, str]:
|
||
"""Run BS-RoFormer SW (Swin Transformer) separation (6-stem)."""
|
||
from .bs_roformer_sw_separator import BSRoformerSWSeparator
|
||
|
||
jm.update_job(
|
||
job_id, status=jm.JobStatus.RUNNING,
|
||
progress="Loading BS-RoFormer SW model...",
|
||
)
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
try:
|
||
import gc, torch
|
||
gc.collect()
|
||
if torch.cuda.is_available():
|
||
torch.cuda.empty_cache()
|
||
except Exception:
|
||
pass
|
||
|
||
sep = BSRoformerSWSeparator(
|
||
cpu_separation=CPU_SEPARATION,
|
||
output_format=output_format,
|
||
mp3_bitrate=mp3_bitrate,
|
||
)
|
||
|
||
jm.update_job(job_id, progress="Separating with BS-RoFormer SW... (this may take a while)")
|
||
|
||
temp_out = JOBS_DIR / job_id / "bsr_sw_out"
|
||
temp_out.mkdir(parents=True, exist_ok=True)
|
||
|
||
try:
|
||
stem_paths = sep.separate_into_parts(input_path, str(temp_out))
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
_prefix = stem_prefix or Path(input_path).stem
|
||
stem_files = {}
|
||
|
||
for stem_name, path in sorted(stem_paths.items()):
|
||
src = Path(path)
|
||
display_name = stem_name.replace('_', ' ').title()
|
||
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
|
||
shutil.move(str(src), str(dest))
|
||
copy_metadata(input_path, str(dest), stem_label=display_name, copy_lyrics=True)
|
||
stem_files[stem_name.lower()] = str(dest)
|
||
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
|
||
return stem_files
|
||
|
||
except Exception:
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
raise
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# MedleyVox (HuggingFace) (Python API)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _run_medleyvox(
|
||
job_id: str,
|
||
input_path: str,
|
||
model: str,
|
||
output_format: str,
|
||
stems_dir: Path,
|
||
mp3_bitrate: str = '256k',
|
||
metadata: Optional[Dict] = None,
|
||
stem_prefix: Optional[str] = None,
|
||
chunk_duration_secs: Optional[int] = None,
|
||
) -> Dict[str, str]:
|
||
"""Run MedleyVox separation (multi-singer vocal separation)."""
|
||
from .medleyvox_separator import MedleyVoxSeparator
|
||
|
||
jm.update_job(
|
||
job_id, status=jm.JobStatus.RUNNING,
|
||
progress=f"Loading MedleyVox model ({model})...",
|
||
)
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
try:
|
||
import gc, torch
|
||
gc.collect()
|
||
if torch.cuda.is_available():
|
||
torch.cuda.empty_cache()
|
||
except Exception:
|
||
pass
|
||
|
||
sep = MedleyVoxSeparator(
|
||
model_id=model,
|
||
cpu_separation=CPU_SEPARATION,
|
||
output_format=output_format,
|
||
mp3_bitrate=mp3_bitrate,
|
||
chunk_secs_override=chunk_duration_secs,
|
||
)
|
||
|
||
jm.update_job(job_id, progress="Separating with MedleyVox...")
|
||
|
||
temp_out = JOBS_DIR / job_id / "medleyvox_out"
|
||
temp_out.mkdir(parents=True, exist_ok=True)
|
||
|
||
try:
|
||
stem_paths = sep.separate_into_parts(input_path, str(temp_out))
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
_prefix = stem_prefix or Path(input_path).stem
|
||
stem_files = {}
|
||
|
||
for stem_name, path in sorted(stem_paths.items()):
|
||
src = Path(path)
|
||
display_name = stem_name.replace('_', ' ').title()
|
||
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
|
||
shutil.move(str(src), str(dest))
|
||
copy_metadata(input_path, str(dest), stem_label=display_name, copy_lyrics=True)
|
||
stem_files[stem_name.lower()] = str(dest)
|
||
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
|
||
return stem_files
|
||
|
||
except Exception:
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
raise
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Apollo (Vocal Enhancement/Reverb Removal) (Python API)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _run_apollo(
|
||
job_id: str,
|
||
input_path: str,
|
||
model: str,
|
||
output_format: str,
|
||
stems_dir: Path,
|
||
mp3_bitrate: str = '256k',
|
||
metadata: Optional[Dict] = None,
|
||
stem_prefix: Optional[str] = None,
|
||
chunking_mode: Optional[str] = None,
|
||
chunk_duration_secs: Optional[int] = None,
|
||
chunk_overlap_secs: Optional[int] = None,
|
||
) -> Dict[str, str]:
|
||
"""Run Apollo vocal enhancement/reverb removal."""
|
||
from .apollo_separator import ApolloSeparator
|
||
|
||
jm.update_job(
|
||
job_id, status=jm.JobStatus.RUNNING,
|
||
progress=f"Loading Apollo model ({model})...",
|
||
)
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
try:
|
||
import gc, torch
|
||
gc.collect()
|
||
if torch.cuda.is_available():
|
||
torch.cuda.empty_cache()
|
||
except Exception:
|
||
pass
|
||
|
||
_dur = get_audio_duration_secs(input_path)
|
||
_chunk_mode = (chunking_mode or LONG_TRACK_CHUNKING_MODE)
|
||
_chunk_secs = int(chunk_duration_secs or LONG_TRACK_CHUNK_DURATION_SECS)
|
||
_chunk_ov = int(chunk_overlap_secs or LONG_TRACK_CHUNK_OVERLAP_SECS)
|
||
_use_chunking = should_chunk(_dur, _chunk_mode, LONG_TRACK_DURATION_SECS)
|
||
|
||
# Chunk adapter path reads intermediate stems back for overlap-add,
|
||
# so force lossless PCM intermediates for broad decoder compatibility.
|
||
_sep_output_format = 'wav' if _use_chunking else output_format
|
||
|
||
sep = ApolloSeparator(
|
||
model_id=model,
|
||
cpu_separation=CPU_SEPARATION,
|
||
output_format=_sep_output_format,
|
||
mp3_bitrate=mp3_bitrate,
|
||
)
|
||
|
||
jm.update_job(job_id, progress="Processing with Apollo...")
|
||
|
||
temp_out = JOBS_DIR / job_id / "apollo_out"
|
||
temp_out.mkdir(parents=True, exist_ok=True)
|
||
|
||
try:
|
||
if _use_chunking:
|
||
stem_paths = _separate_into_parts_chunked_via_files(
|
||
job_id=job_id,
|
||
input_path=input_path,
|
||
sep=sep,
|
||
temp_out=temp_out,
|
||
chunk_duration_secs=_chunk_secs,
|
||
chunk_overlap_secs=_chunk_ov,
|
||
output_format=output_format,
|
||
mp3_bitrate=mp3_bitrate,
|
||
)
|
||
else:
|
||
stem_paths = sep.separate_into_parts(input_path, str(temp_out))
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
_prefix = stem_prefix or Path(input_path).stem
|
||
stem_files = {}
|
||
|
||
for stem_name, path in sorted(stem_paths.items()):
|
||
src = Path(path)
|
||
display_name = stem_name.replace('_', ' ').title()
|
||
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
|
||
shutil.move(str(src), str(dest))
|
||
copy_metadata(input_path, str(dest), stem_label=display_name, copy_lyrics=True)
|
||
stem_files[stem_name.lower()] = str(dest)
|
||
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
|
||
return stem_files
|
||
|
||
except Exception:
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
raise
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Aspiration (Breath/Aspiration Removal) (Python API)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _run_aspiration(
|
||
job_id: str,
|
||
input_path: str,
|
||
model: str,
|
||
output_format: str,
|
||
stems_dir: Path,
|
||
mp3_bitrate: str = '256k',
|
||
metadata: Optional[Dict] = None,
|
||
stem_prefix: Optional[str] = None,
|
||
chunking_mode: Optional[str] = None,
|
||
chunk_duration_secs: Optional[int] = None,
|
||
chunk_overlap_secs: Optional[int] = None,
|
||
) -> Dict[str, str]:
|
||
"""Run Aspiration removal (breath/aspiration separation)."""
|
||
from .aspiration_separator import AspirationSeparator
|
||
|
||
jm.update_job(
|
||
job_id, status=jm.JobStatus.RUNNING,
|
||
progress=f"Loading Aspiration model ({model})...",
|
||
)
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
try:
|
||
import gc, torch
|
||
gc.collect()
|
||
if torch.cuda.is_available():
|
||
torch.cuda.empty_cache()
|
||
except Exception:
|
||
pass
|
||
|
||
_dur = get_audio_duration_secs(input_path)
|
||
_chunk_mode = (chunking_mode or LONG_TRACK_CHUNKING_MODE)
|
||
_chunk_secs = int(chunk_duration_secs or LONG_TRACK_CHUNK_DURATION_SECS)
|
||
_chunk_ov = int(chunk_overlap_secs or LONG_TRACK_CHUNK_OVERLAP_SECS)
|
||
_use_chunking = should_chunk(_dur, _chunk_mode, LONG_TRACK_DURATION_SECS)
|
||
|
||
# Chunk adapter path reads intermediate stems back for overlap-add,
|
||
# so force lossless PCM intermediates for broad decoder compatibility.
|
||
_sep_output_format = 'wav' if _use_chunking else output_format
|
||
|
||
sep = AspirationSeparator(
|
||
model_id=model,
|
||
cpu_separation=CPU_SEPARATION,
|
||
output_format=_sep_output_format,
|
||
mp3_bitrate=mp3_bitrate,
|
||
)
|
||
|
||
jm.update_job(job_id, progress="Removing aspiration with MelBand...")
|
||
|
||
temp_out = JOBS_DIR / job_id / "aspiration_out"
|
||
temp_out.mkdir(parents=True, exist_ok=True)
|
||
|
||
try:
|
||
if _use_chunking:
|
||
stem_paths = _separate_into_parts_chunked_via_files(
|
||
job_id=job_id,
|
||
input_path=input_path,
|
||
sep=sep,
|
||
temp_out=temp_out,
|
||
chunk_duration_secs=_chunk_secs,
|
||
chunk_overlap_secs=_chunk_ov,
|
||
output_format=output_format,
|
||
mp3_bitrate=mp3_bitrate,
|
||
)
|
||
else:
|
||
stem_paths = sep.separate_into_parts(input_path, str(temp_out))
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
_prefix = stem_prefix or Path(input_path).stem
|
||
stem_files = {}
|
||
|
||
for stem_name, path in sorted(stem_paths.items()):
|
||
src = Path(path)
|
||
display_name = stem_name.replace('_', ' ').title()
|
||
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
|
||
shutil.move(str(src), str(dest))
|
||
copy_metadata(input_path, str(dest), stem_label=display_name, copy_lyrics=True)
|
||
stem_files[stem_name.lower()] = str(dest)
|
||
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
|
||
return stem_files
|
||
|
||
except Exception:
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
raise
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Bandit-v2 (Multi-Language/Multi-Stem) (Python API)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _run_bandit(
|
||
job_id: str,
|
||
input_path: str,
|
||
model: str,
|
||
output_format: str,
|
||
stems_dir: Path,
|
||
mp3_bitrate: str = '256k',
|
||
metadata: Optional[Dict] = None,
|
||
stem_prefix: Optional[str] = None,
|
||
chunking_mode: Optional[str] = None,
|
||
chunk_duration_secs: Optional[int] = None,
|
||
chunk_overlap_secs: Optional[int] = None,
|
||
) -> Dict[str, str]:
|
||
"""Run Bandit-v2 separation (multi-language/multi-stem)."""
|
||
from .bandit_separator import BanditSeparator
|
||
|
||
jm.update_job(
|
||
job_id, status=jm.JobStatus.RUNNING,
|
||
progress=f"Loading Bandit model ({model})...",
|
||
)
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
try:
|
||
import gc, torch
|
||
gc.collect()
|
||
if torch.cuda.is_available():
|
||
torch.cuda.empty_cache()
|
||
except Exception:
|
||
pass
|
||
|
||
_dur = get_audio_duration_secs(input_path)
|
||
_chunk_mode = (chunking_mode or LONG_TRACK_CHUNKING_MODE)
|
||
_chunk_secs = int(chunk_duration_secs or LONG_TRACK_CHUNK_DURATION_SECS)
|
||
_chunk_ov = int(chunk_overlap_secs or LONG_TRACK_CHUNK_OVERLAP_SECS)
|
||
_use_chunking = should_chunk(_dur, _chunk_mode, LONG_TRACK_DURATION_SECS)
|
||
|
||
# Chunk adapter path reads intermediate stems back for overlap-add,
|
||
# so force lossless PCM intermediates for broad decoder compatibility.
|
||
_sep_output_format = 'wav' if _use_chunking else output_format
|
||
|
||
sep = BanditSeparator(
|
||
model_id=model,
|
||
cpu_separation=CPU_SEPARATION,
|
||
output_format=_sep_output_format,
|
||
mp3_bitrate=mp3_bitrate,
|
||
)
|
||
|
||
jm.update_job(job_id, progress="Separating with Bandit-v2...")
|
||
|
||
temp_out = JOBS_DIR / job_id / "bandit_out"
|
||
temp_out.mkdir(parents=True, exist_ok=True)
|
||
|
||
try:
|
||
if _use_chunking:
|
||
stem_paths = _separate_into_parts_chunked_via_files(
|
||
job_id=job_id,
|
||
input_path=input_path,
|
||
sep=sep,
|
||
temp_out=temp_out,
|
||
chunk_duration_secs=_chunk_secs,
|
||
chunk_overlap_secs=_chunk_ov,
|
||
output_format=output_format,
|
||
mp3_bitrate=mp3_bitrate,
|
||
)
|
||
else:
|
||
stem_paths = sep.separate_into_parts(input_path, str(temp_out))
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
_prefix = stem_prefix or Path(input_path).stem
|
||
stem_files = {}
|
||
|
||
for stem_name, path in sorted(stem_paths.items()):
|
||
src = Path(path)
|
||
display_name = stem_name.replace('_', ' ').title()
|
||
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
|
||
shutil.move(str(src), str(dest))
|
||
copy_metadata(input_path, str(dest), stem_label=display_name, copy_lyrics=True)
|
||
stem_files[stem_name.lower()] = str(dest)
|
||
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
|
||
return stem_files
|
||
|
||
except Exception:
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
raise
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Ensemble (Combines Multiple Models) (Python API)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _run_ensemble(
|
||
job_id: str,
|
||
input_path: str,
|
||
model: str,
|
||
output_format: str,
|
||
stems_dir: Path,
|
||
mp3_bitrate: str = '256k',
|
||
metadata: Optional[Dict] = None,
|
||
stem_prefix: Optional[str] = None,
|
||
) -> Dict[str, str]:
|
||
"""
|
||
Run Ensemble separation (combines multiple models for best quality).
|
||
|
||
Models combined:
|
||
- ensemble_vocal_best: BS Roformer x2 + MelBand Roformer ft + SCNet XL IHF
|
||
- ensemble_4stem: Multiple 4-stem models averaged
|
||
- ensemble_6stem: Multiple 6-stem models averaged
|
||
"""
|
||
from .ensemble_separator import EnsembleSeparator
|
||
|
||
jm.update_job(
|
||
job_id, status=jm.JobStatus.RUNNING,
|
||
progress=f"Loading Ensemble models ({model})...",
|
||
)
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
try:
|
||
import gc, torch
|
||
gc.collect()
|
||
if torch.cuda.is_available():
|
||
torch.cuda.empty_cache()
|
||
except Exception:
|
||
pass
|
||
|
||
sep = EnsembleSeparator(
|
||
ensemble_id=model,
|
||
cpu_separation=CPU_SEPARATION,
|
||
output_format=output_format,
|
||
mp3_bitrate=mp3_bitrate,
|
||
)
|
||
|
||
jm.update_job(job_id, progress="Running Ensemble separation... (this will take significantly longer)")
|
||
|
||
temp_out = JOBS_DIR / job_id / "ensemble_out"
|
||
temp_out.mkdir(parents=True, exist_ok=True)
|
||
|
||
try:
|
||
stem_paths = sep.separate_into_parts(input_path, str(temp_out))
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
_prefix = stem_prefix or Path(input_path).stem
|
||
stem_files = {}
|
||
|
||
for stem_name, path in sorted(stem_paths.items()):
|
||
src = Path(path)
|
||
display_name = stem_name.replace('_', ' ').title()
|
||
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
|
||
shutil.move(str(src), str(dest))
|
||
copy_metadata(input_path, str(dest), stem_label=display_name, copy_lyrics=True)
|
||
stem_files[stem_name.lower()] = str(dest)
|
||
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
|
||
return stem_files
|
||
|
||
except Exception:
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
raise
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# SepACap (A Cappella Multi-Singer Vocal Separation) (Python API)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _run_sepacap(
|
||
job_id: str,
|
||
input_path: str,
|
||
output_format: str,
|
||
stems_dir: Path,
|
||
mp3_bitrate: str = '256k',
|
||
metadata: Optional[Dict] = None,
|
||
stem_prefix: Optional[str] = None,
|
||
chunk_samples: int = 48_000,
|
||
) -> Dict[str, str]:
|
||
"""Run SepACap a cappella vocal separation into 7 stems."""
|
||
from .sepacap_separator import SepACapSeparator
|
||
|
||
jm.update_job(
|
||
job_id, status=jm.JobStatus.RUNNING,
|
||
progress="Loading SepACap model (clones repo + downloads weights on first use)...",
|
||
)
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
try:
|
||
import gc, torch
|
||
gc.collect()
|
||
if torch.cuda.is_available():
|
||
torch.cuda.empty_cache()
|
||
except Exception:
|
||
pass
|
||
|
||
sep = SepACapSeparator(
|
||
cpu_separation=CPU_SEPARATION,
|
||
output_format=output_format,
|
||
mp3_bitrate=mp3_bitrate,
|
||
chunk_samples=chunk_samples,
|
||
)
|
||
|
||
jm.update_job(job_id, progress="Separating a cappella vocals with SepACap... (this may take a while)")
|
||
|
||
temp_out = JOBS_DIR / job_id / "sepacap_out"
|
||
temp_out.mkdir(parents=True, exist_ok=True)
|
||
|
||
try:
|
||
stem_paths = sep.separate_into_parts(input_path, str(temp_out))
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
_prefix = stem_prefix or Path(input_path).stem
|
||
stem_files = {}
|
||
|
||
for stem_name, path in sorted(stem_paths.items()):
|
||
src = Path(path)
|
||
display_name = stem_name.replace('_', ' ').title()
|
||
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
|
||
shutil.move(str(src), str(dest))
|
||
copy_metadata(input_path, str(dest), stem_label=display_name, copy_lyrics=True)
|
||
stem_files[stem_name.lower()] = str(dest)
|
||
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
|
||
return stem_files
|
||
|
||
except Exception:
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
raise
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# SepReformer (2-Speaker Speech Separation) (Python API)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _run_sepreformer(
|
||
job_id: str,
|
||
input_path: str,
|
||
model: str,
|
||
output_format: str,
|
||
stems_dir: Path,
|
||
mp3_bitrate: str = '256k',
|
||
metadata: Optional[Dict] = None,
|
||
stem_prefix: Optional[str] = None,
|
||
chunk_samples: int = 48_000,
|
||
) -> Dict[str, str]:
|
||
"""Run SepReformer 2-speaker speech separation."""
|
||
from .sepreformer_separator import SepReformerSeparator
|
||
|
||
jm.update_job(
|
||
job_id, status=jm.JobStatus.RUNNING,
|
||
progress=f"Loading SepReformer model ({model}) — clones repo on first use...",
|
||
)
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
try:
|
||
import gc, torch
|
||
gc.collect()
|
||
if torch.cuda.is_available():
|
||
torch.cuda.empty_cache()
|
||
except Exception:
|
||
pass
|
||
|
||
sep = SepReformerSeparator(
|
||
model_id=model,
|
||
cpu_separation=CPU_SEPARATION,
|
||
output_format=output_format,
|
||
mp3_bitrate=mp3_bitrate,
|
||
chunk_samples=chunk_samples,
|
||
)
|
||
|
||
jm.update_job(job_id, progress="Separating speakers with SepReformer...")
|
||
|
||
temp_out = JOBS_DIR / job_id / "sepreformer_out"
|
||
temp_out.mkdir(parents=True, exist_ok=True)
|
||
|
||
try:
|
||
stem_paths = sep.separate_into_parts(input_path, str(temp_out))
|
||
|
||
if jm.should_cancel(job_id):
|
||
raise RuntimeError("Cancelled by user")
|
||
|
||
_prefix = stem_prefix or Path(input_path).stem
|
||
stem_files = {}
|
||
|
||
for stem_name, path in sorted(stem_paths.items()):
|
||
src = Path(path)
|
||
display_name = stem_name.replace('_', ' ').title()
|
||
dest = stems_dir / _stem_filename(_prefix, stem_name, src.suffix)
|
||
shutil.move(str(src), str(dest))
|
||
copy_metadata(input_path, str(dest), stem_label=display_name, copy_lyrics=True)
|
||
stem_files[stem_name.lower()] = str(dest)
|
||
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
|
||
return stem_files
|
||
|
||
except Exception:
|
||
try:
|
||
shutil.rmtree(temp_out, ignore_errors=True)
|
||
except Exception:
|
||
pass
|
||
raise
|