160 lines
5.7 KiB
Python
160 lines
5.7 KiB
Python
"""
|
|
Shared execution helpers for all separator back-ends.
|
|
|
|
Extracts the three patterns repeated across every ``_run_*`` function in
|
|
``separator_service.py``:
|
|
|
|
1. ``flush_vram_cache()`` — clear PyTorch reserved VRAM before loading a model
|
|
2. ``move_stems_to_project()`` — move separator temp output into the project Stems
|
|
dir, copying audio metadata tags onto each file.
|
|
3. ``oom_retry_call()`` — call a separator's ``separate_into_parts()`` once,
|
|
and on CUDA OOM retry with a reduced batch/chunk param.
|
|
|
|
New model runners should use these instead of inlining the boilerplate.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import shutil
|
|
from pathlib import Path
|
|
from typing import Any, Callable, Dict, Optional
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# VRAM helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def flush_vram_cache() -> None:
|
|
"""Release PyTorch's reserved-but-idle VRAM blocks.
|
|
|
|
Safe to call even when CUDA is not available or torch is not installed.
|
|
"""
|
|
try:
|
|
import gc
|
|
import torch
|
|
|
|
gc.collect()
|
|
if torch.cuda.is_available():
|
|
torch.cuda.empty_cache()
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Post-separation stem move
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def move_stems_to_project(
|
|
stem_paths: Dict[str, str],
|
|
stems_dir: Path,
|
|
stem_prefix: str,
|
|
input_path: str,
|
|
*,
|
|
display_fn: Optional[Callable[[str], str]] = None,
|
|
) -> Dict[str, str]:
|
|
"""Move separator output files into ``stems_dir`` with canonical names.
|
|
|
|
Args:
|
|
stem_paths: ``{stem_name: temp_file_path}`` returned by the separator.
|
|
stems_dir: Destination directory (project's ``Stems/`` folder).
|
|
stem_prefix: Human-readable prefix (project name) for the output filename.
|
|
input_path: Original source audio — used as the metadata donor file.
|
|
display_fn: Optional callable ``(stem_name) -> display_label`` used for
|
|
the metadata ``stem_label`` tag. Defaults to
|
|
``stem_name.replace("_", " ").title()``.
|
|
|
|
Returns:
|
|
``{stem_name: abs_path_str}`` for the moved files.
|
|
"""
|
|
from ..metadata_util import copy_metadata
|
|
from .output_layout import stem_filename
|
|
|
|
if display_fn is None:
|
|
display_fn = lambda s: s.replace("_", " ").title() # noqa: E731
|
|
|
|
stem_files: Dict[str, str] = {}
|
|
for stem_name, path in sorted(stem_paths.items()):
|
|
src = Path(path)
|
|
dest = stems_dir / stem_filename(stem_prefix, stem_name, src.suffix)
|
|
shutil.move(str(src), str(dest))
|
|
try:
|
|
copy_metadata(input_path, str(dest), stem_label=display_fn(stem_name), copy_lyrics=True)
|
|
except Exception as exc:
|
|
log.debug("copy_metadata failed for %s: %s", dest.name, exc)
|
|
stem_files[stem_name.lower()] = str(dest)
|
|
return stem_files
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# OOM-safe separator call
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def oom_retry_call(
|
|
sep_fn: Callable[..., Dict[str, str]],
|
|
*args: Any,
|
|
job_id: str,
|
|
retry_kwarg: str,
|
|
retry_value: Any,
|
|
update_job_fn: Optional[Callable[..., None]] = None,
|
|
**kwargs: Any,
|
|
) -> Dict[str, str]:
|
|
"""Call ``sep_fn(*args, **kwargs)`` with one automatic OOM retry.
|
|
|
|
On a CUDA out-of-memory error the first attempt retries with
|
|
``retry_kwarg=retry_value`` (e.g. ``batch_size_override=1``). The
|
|
``job_id`` is only used for the progress update message.
|
|
|
|
Args:
|
|
sep_fn: The callable to invoke — typically
|
|
``sep.separate_into_parts``.
|
|
*args: Positional arguments forwarded to ``sep_fn``.
|
|
job_id: Used for the OOM progress message.
|
|
retry_kwarg: Keyword argument to override on retry (e.g.
|
|
``"batch_size_override"``).
|
|
retry_value: Value to use for ``retry_kwarg`` on retry.
|
|
update_job_fn: Callable ``(job_id, progress=...) -> None`` used to emit
|
|
the OOM retry message. Pass ``jm.update_job`` here.
|
|
**kwargs: Keyword arguments forwarded to ``sep_fn``.
|
|
|
|
Returns:
|
|
Whatever ``sep_fn`` returns on success.
|
|
|
|
Raises:
|
|
The original exception if the retry also fails, or if the first failure
|
|
is not an OOM error.
|
|
"""
|
|
for attempt in range(2):
|
|
try:
|
|
return sep_fn(*args, **kwargs)
|
|
except Exception as exc:
|
|
is_oom = (
|
|
"out of memory" in str(exc).lower()
|
|
or "OutOfMemory" in type(exc).__name__
|
|
)
|
|
if is_oom and attempt == 0:
|
|
if update_job_fn is not None:
|
|
update_job_fn(
|
|
job_id,
|
|
progress=f"GPU OOM — retrying with {retry_kwarg}={retry_value}...",
|
|
)
|
|
flush_vram_cache()
|
|
kwargs[retry_kwarg] = retry_value
|
|
continue
|
|
raise
|
|
# Should never reach here, but satisfy type checkers.
|
|
raise RuntimeError("oom_retry_call exhausted all attempts")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Temp-dir cleanup helper
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def cleanup_temp(temp_dir: Path) -> None:
|
|
"""Remove a temporary separation output directory, ignoring errors."""
|
|
try:
|
|
shutil.rmtree(temp_dir, ignore_errors=True)
|
|
except Exception:
|
|
pass
|