Files
stemengine_old/backend/services/separation/execution.py
T

160 lines
5.7 KiB
Python

"""
Shared execution helpers for all separator back-ends.
Extracts the three patterns repeated across every ``_run_*`` function in
``separator_service.py``:
1. ``flush_vram_cache()`` — clear PyTorch reserved VRAM before loading a model
2. ``move_stems_to_project()`` — move separator temp output into the project Stems
dir, copying audio metadata tags onto each file.
3. ``oom_retry_call()`` — call a separator's ``separate_into_parts()`` once,
and on CUDA OOM retry with a reduced batch/chunk param.
New model runners should use these instead of inlining the boilerplate.
"""
from __future__ import annotations
import logging
import shutil
from pathlib import Path
from typing import Any, Callable, Dict, Optional
log = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# VRAM helpers
# ---------------------------------------------------------------------------
def flush_vram_cache() -> None:
"""Release PyTorch's reserved-but-idle VRAM blocks.
Safe to call even when CUDA is not available or torch is not installed.
"""
try:
import gc
import torch
gc.collect()
if torch.cuda.is_available():
torch.cuda.empty_cache()
except Exception:
pass
# ---------------------------------------------------------------------------
# Post-separation stem move
# ---------------------------------------------------------------------------
def move_stems_to_project(
stem_paths: Dict[str, str],
stems_dir: Path,
stem_prefix: str,
input_path: str,
*,
display_fn: Optional[Callable[[str], str]] = None,
) -> Dict[str, str]:
"""Move separator output files into ``stems_dir`` with canonical names.
Args:
stem_paths: ``{stem_name: temp_file_path}`` returned by the separator.
stems_dir: Destination directory (project's ``Stems/`` folder).
stem_prefix: Human-readable prefix (project name) for the output filename.
input_path: Original source audio — used as the metadata donor file.
display_fn: Optional callable ``(stem_name) -> display_label`` used for
the metadata ``stem_label`` tag. Defaults to
``stem_name.replace("_", " ").title()``.
Returns:
``{stem_name: abs_path_str}`` for the moved files.
"""
from ..metadata_util import copy_metadata
from .output_layout import stem_filename
if display_fn is None:
display_fn = lambda s: s.replace("_", " ").title() # noqa: E731
stem_files: Dict[str, str] = {}
for stem_name, path in sorted(stem_paths.items()):
src = Path(path)
dest = stems_dir / stem_filename(stem_prefix, stem_name, src.suffix)
shutil.move(str(src), str(dest))
try:
copy_metadata(input_path, str(dest), stem_label=display_fn(stem_name), copy_lyrics=True)
except Exception as exc:
log.debug("copy_metadata failed for %s: %s", dest.name, exc)
stem_files[stem_name.lower()] = str(dest)
return stem_files
# ---------------------------------------------------------------------------
# OOM-safe separator call
# ---------------------------------------------------------------------------
def oom_retry_call(
sep_fn: Callable[..., Dict[str, str]],
*args: Any,
job_id: str,
retry_kwarg: str,
retry_value: Any,
update_job_fn: Optional[Callable[..., None]] = None,
**kwargs: Any,
) -> Dict[str, str]:
"""Call ``sep_fn(*args, **kwargs)`` with one automatic OOM retry.
On a CUDA out-of-memory error the first attempt retries with
``retry_kwarg=retry_value`` (e.g. ``batch_size_override=1``). The
``job_id`` is only used for the progress update message.
Args:
sep_fn: The callable to invoke — typically
``sep.separate_into_parts``.
*args: Positional arguments forwarded to ``sep_fn``.
job_id: Used for the OOM progress message.
retry_kwarg: Keyword argument to override on retry (e.g.
``"batch_size_override"``).
retry_value: Value to use for ``retry_kwarg`` on retry.
update_job_fn: Callable ``(job_id, progress=...) -> None`` used to emit
the OOM retry message. Pass ``jm.update_job`` here.
**kwargs: Keyword arguments forwarded to ``sep_fn``.
Returns:
Whatever ``sep_fn`` returns on success.
Raises:
The original exception if the retry also fails, or if the first failure
is not an OOM error.
"""
for attempt in range(2):
try:
return sep_fn(*args, **kwargs)
except Exception as exc:
is_oom = (
"out of memory" in str(exc).lower()
or "OutOfMemory" in type(exc).__name__
)
if is_oom and attempt == 0:
if update_job_fn is not None:
update_job_fn(
job_id,
progress=f"GPU OOM — retrying with {retry_kwarg}={retry_value}...",
)
flush_vram_cache()
kwargs[retry_kwarg] = retry_value
continue
raise
# Should never reach here, but satisfy type checkers.
raise RuntimeError("oom_retry_call exhausted all attempts")
# ---------------------------------------------------------------------------
# Temp-dir cleanup helper
# ---------------------------------------------------------------------------
def cleanup_temp(temp_dir: Path) -> None:
"""Remove a temporary separation output directory, ignoring errors."""
try:
shutil.rmtree(temp_dir, ignore_errors=True)
except Exception:
pass