Files
stemengine_old/backend/services/separation/execution.py
T
2026-03-26 15:14:41 +09:00

272 lines
9.2 KiB
Python

"""
Shared execution helpers for all separator back-ends.
Extracts the three patterns repeated across every ``_run_*`` function in
``separator_service.py``:
1. ``flush_vram_cache()`` — clear PyTorch reserved VRAM before loading a model
2. ``move_stems_to_project()`` — move separator temp output into the project Stems
dir, copying audio metadata tags onto each file.
3. ``oom_retry_call()`` — call a separator's ``separate_into_parts()`` once,
and on CUDA OOM retry with a reduced batch/chunk param.
New model runners should use these instead of inlining the boilerplate.
"""
from __future__ import annotations
import logging
import os
import shutil
from pathlib import Path
from typing import Any, Callable, Dict, Optional
log = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# VRAM helpers
# ---------------------------------------------------------------------------
def flush_vram_cache() -> None:
"""Release PyTorch's reserved-but-idle VRAM blocks.
Safe to call even when CUDA is not available or torch is not installed.
"""
try:
import gc
import torch
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
except Exception:
pass
# ---------------------------------------------------------------------------
# Post-separation stem move
# ---------------------------------------------------------------------------
def move_stems_to_project(
stem_paths: Dict[str, str],
stems_dir: Path,
stem_prefix: str,
input_path: str,
*,
display_fn: Optional[Callable[[str], str]] = None,
) -> Dict[str, str]:
"""Move separator output files into ``stems_dir`` with canonical names.
Args:
stem_paths: ``{stem_name: temp_file_path}`` returned by the separator.
stems_dir: Destination directory (project's ``Stems/`` folder).
stem_prefix: Human-readable prefix (project name) for the output filename.
input_path: Original source audio — used as the metadata donor file.
display_fn: Optional callable ``(stem_name) -> display_label`` used for
the metadata ``stem_label`` tag. Defaults to
``stem_name.replace("_", " ").title()``.
Returns:
``{stem_name: abs_path_str}`` for the moved files.
"""
from ..metadata_util import copy_metadata
from .output_layout import normalise_stem_label, stem_filename
if display_fn is None:
display_fn = lambda s: normalise_stem_label(s).title() # noqa: E731
stem_files: Dict[str, str] = {}
for stem_name, path in sorted(stem_paths.items()):
src = Path(path)
dest = stems_dir / stem_filename(stem_prefix, stem_name, src.suffix)
shutil.move(str(src), str(dest))
try:
copy_metadata(input_path, str(dest), stem_label=display_fn(stem_name), copy_lyrics=True)
except Exception as exc:
log.debug("copy_metadata failed for %s: %s", dest.name, exc)
stem_files[stem_name.lower()] = str(dest)
return stem_files
# ---------------------------------------------------------------------------
# OOM-safe separator call
# ---------------------------------------------------------------------------
def oom_retry_call(
sep_fn: Callable[..., Dict[str, str]],
*args: Any,
job_id: str,
retry_kwarg: str,
retry_value: Any,
update_job_fn: Optional[Callable[..., None]] = None,
**kwargs: Any,
) -> Dict[str, str]:
"""Call ``sep_fn(*args, **kwargs)`` with one automatic OOM retry.
On a CUDA out-of-memory error the first attempt retries with
``retry_kwarg=retry_value`` (e.g. ``batch_size_override=1``). The
``job_id`` is only used for the progress update message.
Args:
sep_fn: The callable to invoke — typically
``sep.separate_into_parts``.
*args: Positional arguments forwarded to ``sep_fn``.
job_id: Used for the OOM progress message.
retry_kwarg: Keyword argument to override on retry (e.g.
``"batch_size_override"``).
retry_value: Value to use for ``retry_kwarg`` on retry.
update_job_fn: Callable ``(job_id, progress=...) -> None`` used to emit
the OOM retry message. Pass ``jm.update_job`` here.
**kwargs: Keyword arguments forwarded to ``sep_fn``.
Returns:
Whatever ``sep_fn`` returns on success.
Raises:
The original exception if the retry also fails, or if the first failure
is not an OOM error.
"""
for attempt in range(2):
try:
return sep_fn(*args, **kwargs)
except Exception as exc:
is_oom = (
"out of memory" in str(exc).lower()
or "OutOfMemory" in type(exc).__name__
)
if is_oom and attempt == 0:
if update_job_fn is not None:
update_job_fn(
job_id,
progress=f"GPU OOM — retrying with {retry_kwarg}={retry_value}...",
)
flush_vram_cache()
kwargs[retry_kwarg] = retry_value
continue
raise
# Should never reach here, but satisfy type checkers.
raise RuntimeError("oom_retry_call exhausted all attempts")
# ---------------------------------------------------------------------------
# Temp-dir cleanup helper
# ---------------------------------------------------------------------------
def cleanup_temp(temp_dir: Path) -> None:
"""Remove a temporary separation output directory, ignoring errors."""
try:
shutil.rmtree(temp_dir, ignore_errors=True)
except Exception:
pass
# ---------------------------------------------------------------------------
# Audio duration probe
# ---------------------------------------------------------------------------
def get_audio_duration_secs(path: str) -> float:
"""Return audio duration in seconds, or 0.0 on any failure.
Tries soundfile first (zero-overhead for WAV/FLAC/OGG — reads only the
header). Falls back to ffprobe for MP3 and other container formats.
"""
try:
import soundfile as sf
return float(sf.info(path).duration)
except Exception:
pass
try:
import subprocess as _sp
result = _sp.run(
[
"ffprobe", "-v", "quiet",
"-show_entries", "format=duration",
"-of", "default=noprint_wrappers=1:nokey=1",
path,
],
capture_output=True,
text=True,
timeout=15,
)
return float(result.stdout.strip())
except Exception:
return 0.0
# ---------------------------------------------------------------------------
# Memory diagnostics
# ---------------------------------------------------------------------------
def _bytes_to_mib(value: int) -> str:
return f"{value / (1024 * 1024):.1f} MiB"
def get_process_rss_bytes() -> int:
"""Return current resident set size in bytes, best-effort."""
try:
with open("/proc/self/status", "r", encoding="utf-8") as handle:
for line in handle:
if line.startswith("VmRSS:"):
return int(line.split()[1]) * 1024
except Exception:
pass
try:
import resource
rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
if os.name == "posix":
return int(rss) * 1024
return int(rss)
except Exception:
return 0
def capture_cuda_memory(device: str = "cuda") -> Dict[str, int]:
"""Capture current CUDA allocation counters, returning zeros when unavailable."""
snapshot = {
"allocated": 0,
"reserved": 0,
"max_allocated": 0,
"max_reserved": 0,
}
if device == "cpu":
return snapshot
try:
import torch
if not torch.cuda.is_available():
return snapshot
snapshot["allocated"] = int(torch.cuda.memory_allocated())
snapshot["reserved"] = int(torch.cuda.memory_reserved())
snapshot["max_allocated"] = int(torch.cuda.max_memory_allocated())
snapshot["max_reserved"] = int(torch.cuda.max_memory_reserved())
except Exception:
pass
return snapshot
def describe_stage_memory(stage: str, *, device: str = "cuda") -> str:
"""Return a compact memory usage string for progress/debug reporting."""
rss = get_process_rss_bytes()
cuda = capture_cuda_memory(device=device)
return (
f"[diag] {stage} — RSS={_bytes_to_mib(rss)}, "
f"CUDA alloc={_bytes_to_mib(cuda['allocated'])}, "
f"reserved={_bytes_to_mib(cuda['reserved'])}, "
f"peak_alloc={_bytes_to_mib(cuda['max_allocated'])}, "
f"peak_reserved={_bytes_to_mib(cuda['max_reserved'])}"
)
def emit_stage_memory(
stage: str,
*,
logger: Optional[logging.Logger] = None,
device: str = "cuda",
progress_cb: Optional[Callable[[str], None]] = None,
) -> str:
"""Log and optionally surface a stage memory snapshot."""
msg = describe_stage_memory(stage, device=device)
(logger or log).debug(msg)
if progress_cb is not None:
progress_cb(msg)
return msg