272 lines
9.2 KiB
Python
272 lines
9.2 KiB
Python
"""
|
|
Shared execution helpers for all separator back-ends.
|
|
|
|
Extracts the three patterns repeated across every ``_run_*`` function in
|
|
``separator_service.py``:
|
|
|
|
1. ``flush_vram_cache()`` — clear PyTorch reserved VRAM before loading a model
|
|
2. ``move_stems_to_project()`` — move separator temp output into the project Stems
|
|
dir, copying audio metadata tags onto each file.
|
|
3. ``oom_retry_call()`` — call a separator's ``separate_into_parts()`` once,
|
|
and on CUDA OOM retry with a reduced batch/chunk param.
|
|
|
|
New model runners should use these instead of inlining the boilerplate.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
import shutil
|
|
from pathlib import Path
|
|
from typing import Any, Callable, Dict, Optional
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# VRAM helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def flush_vram_cache() -> None:
|
|
"""Release PyTorch's reserved-but-idle VRAM blocks.
|
|
|
|
Safe to call even when CUDA is not available or torch is not installed.
|
|
"""
|
|
try:
|
|
import gc
|
|
import torch
|
|
|
|
gc.collect()
|
|
if torch.cuda.is_available():
|
|
torch.cuda.empty_cache()
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Post-separation stem move
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def move_stems_to_project(
|
|
stem_paths: Dict[str, str],
|
|
stems_dir: Path,
|
|
stem_prefix: str,
|
|
input_path: str,
|
|
*,
|
|
display_fn: Optional[Callable[[str], str]] = None,
|
|
) -> Dict[str, str]:
|
|
"""Move separator output files into ``stems_dir`` with canonical names.
|
|
|
|
Args:
|
|
stem_paths: ``{stem_name: temp_file_path}`` returned by the separator.
|
|
stems_dir: Destination directory (project's ``Stems/`` folder).
|
|
stem_prefix: Human-readable prefix (project name) for the output filename.
|
|
input_path: Original source audio — used as the metadata donor file.
|
|
display_fn: Optional callable ``(stem_name) -> display_label`` used for
|
|
the metadata ``stem_label`` tag. Defaults to
|
|
``stem_name.replace("_", " ").title()``.
|
|
|
|
Returns:
|
|
``{stem_name: abs_path_str}`` for the moved files.
|
|
"""
|
|
from ..metadata_util import copy_metadata
|
|
from .output_layout import normalise_stem_label, stem_filename
|
|
|
|
if display_fn is None:
|
|
display_fn = lambda s: normalise_stem_label(s).title() # noqa: E731
|
|
|
|
stem_files: Dict[str, str] = {}
|
|
for stem_name, path in sorted(stem_paths.items()):
|
|
src = Path(path)
|
|
dest = stems_dir / stem_filename(stem_prefix, stem_name, src.suffix)
|
|
shutil.move(str(src), str(dest))
|
|
try:
|
|
copy_metadata(input_path, str(dest), stem_label=display_fn(stem_name), copy_lyrics=True)
|
|
except Exception as exc:
|
|
log.debug("copy_metadata failed for %s: %s", dest.name, exc)
|
|
stem_files[stem_name.lower()] = str(dest)
|
|
return stem_files
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# OOM-safe separator call
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def oom_retry_call(
|
|
sep_fn: Callable[..., Dict[str, str]],
|
|
*args: Any,
|
|
job_id: str,
|
|
retry_kwarg: str,
|
|
retry_value: Any,
|
|
update_job_fn: Optional[Callable[..., None]] = None,
|
|
**kwargs: Any,
|
|
) -> Dict[str, str]:
|
|
"""Call ``sep_fn(*args, **kwargs)`` with one automatic OOM retry.
|
|
|
|
On a CUDA out-of-memory error the first attempt retries with
|
|
``retry_kwarg=retry_value`` (e.g. ``batch_size_override=1``). The
|
|
``job_id`` is only used for the progress update message.
|
|
|
|
Args:
|
|
sep_fn: The callable to invoke — typically
|
|
``sep.separate_into_parts``.
|
|
*args: Positional arguments forwarded to ``sep_fn``.
|
|
job_id: Used for the OOM progress message.
|
|
retry_kwarg: Keyword argument to override on retry (e.g.
|
|
``"batch_size_override"``).
|
|
retry_value: Value to use for ``retry_kwarg`` on retry.
|
|
update_job_fn: Callable ``(job_id, progress=...) -> None`` used to emit
|
|
the OOM retry message. Pass ``jm.update_job`` here.
|
|
**kwargs: Keyword arguments forwarded to ``sep_fn``.
|
|
|
|
Returns:
|
|
Whatever ``sep_fn`` returns on success.
|
|
|
|
Raises:
|
|
The original exception if the retry also fails, or if the first failure
|
|
is not an OOM error.
|
|
"""
|
|
for attempt in range(2):
|
|
try:
|
|
return sep_fn(*args, **kwargs)
|
|
except Exception as exc:
|
|
is_oom = (
|
|
"out of memory" in str(exc).lower()
|
|
or "OutOfMemory" in type(exc).__name__
|
|
)
|
|
if is_oom and attempt == 0:
|
|
if update_job_fn is not None:
|
|
update_job_fn(
|
|
job_id,
|
|
progress=f"GPU OOM — retrying with {retry_kwarg}={retry_value}...",
|
|
)
|
|
flush_vram_cache()
|
|
kwargs[retry_kwarg] = retry_value
|
|
continue
|
|
raise
|
|
# Should never reach here, but satisfy type checkers.
|
|
raise RuntimeError("oom_retry_call exhausted all attempts")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Temp-dir cleanup helper
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def cleanup_temp(temp_dir: Path) -> None:
|
|
"""Remove a temporary separation output directory, ignoring errors."""
|
|
try:
|
|
shutil.rmtree(temp_dir, ignore_errors=True)
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Audio duration probe
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def get_audio_duration_secs(path: str) -> float:
|
|
"""Return audio duration in seconds, or 0.0 on any failure.
|
|
|
|
Tries soundfile first (zero-overhead for WAV/FLAC/OGG — reads only the
|
|
header). Falls back to ffprobe for MP3 and other container formats.
|
|
"""
|
|
try:
|
|
import soundfile as sf
|
|
return float(sf.info(path).duration)
|
|
except Exception:
|
|
pass
|
|
try:
|
|
import subprocess as _sp
|
|
result = _sp.run(
|
|
[
|
|
"ffprobe", "-v", "quiet",
|
|
"-show_entries", "format=duration",
|
|
"-of", "default=noprint_wrappers=1:nokey=1",
|
|
path,
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=15,
|
|
)
|
|
return float(result.stdout.strip())
|
|
except Exception:
|
|
return 0.0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Memory diagnostics
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _bytes_to_mib(value: int) -> str:
|
|
return f"{value / (1024 * 1024):.1f} MiB"
|
|
|
|
|
|
def get_process_rss_bytes() -> int:
|
|
"""Return current resident set size in bytes, best-effort."""
|
|
try:
|
|
with open("/proc/self/status", "r", encoding="utf-8") as handle:
|
|
for line in handle:
|
|
if line.startswith("VmRSS:"):
|
|
return int(line.split()[1]) * 1024
|
|
except Exception:
|
|
pass
|
|
try:
|
|
import resource
|
|
rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
|
|
if os.name == "posix":
|
|
return int(rss) * 1024
|
|
return int(rss)
|
|
except Exception:
|
|
return 0
|
|
|
|
|
|
def capture_cuda_memory(device: str = "cuda") -> Dict[str, int]:
|
|
"""Capture current CUDA allocation counters, returning zeros when unavailable."""
|
|
snapshot = {
|
|
"allocated": 0,
|
|
"reserved": 0,
|
|
"max_allocated": 0,
|
|
"max_reserved": 0,
|
|
}
|
|
if device == "cpu":
|
|
return snapshot
|
|
try:
|
|
import torch
|
|
if not torch.cuda.is_available():
|
|
return snapshot
|
|
snapshot["allocated"] = int(torch.cuda.memory_allocated())
|
|
snapshot["reserved"] = int(torch.cuda.memory_reserved())
|
|
snapshot["max_allocated"] = int(torch.cuda.max_memory_allocated())
|
|
snapshot["max_reserved"] = int(torch.cuda.max_memory_reserved())
|
|
except Exception:
|
|
pass
|
|
return snapshot
|
|
|
|
|
|
def describe_stage_memory(stage: str, *, device: str = "cuda") -> str:
|
|
"""Return a compact memory usage string for progress/debug reporting."""
|
|
rss = get_process_rss_bytes()
|
|
cuda = capture_cuda_memory(device=device)
|
|
return (
|
|
f"[diag] {stage} — RSS={_bytes_to_mib(rss)}, "
|
|
f"CUDA alloc={_bytes_to_mib(cuda['allocated'])}, "
|
|
f"reserved={_bytes_to_mib(cuda['reserved'])}, "
|
|
f"peak_alloc={_bytes_to_mib(cuda['max_allocated'])}, "
|
|
f"peak_reserved={_bytes_to_mib(cuda['max_reserved'])}"
|
|
)
|
|
|
|
|
|
def emit_stage_memory(
|
|
stage: str,
|
|
*,
|
|
logger: Optional[logging.Logger] = None,
|
|
device: str = "cuda",
|
|
progress_cb: Optional[Callable[[str], None]] = None,
|
|
) -> str:
|
|
"""Log and optionally surface a stage memory snapshot."""
|
|
msg = describe_stage_memory(stage, device=device)
|
|
(logger or log).debug(msg)
|
|
if progress_cb is not None:
|
|
progress_cb(msg)
|
|
return msg
|