Files
stemengine_old/backend/job_manager.py
T

271 lines
8.9 KiB
Python
Executable File

"""
Job manager: SQLite-backed job state + ThreadPoolExecutor for background tasks.
"""
import logging
import threading
from concurrent.futures import ThreadPoolExecutor
from enum import Enum
from typing import Callable, Optional
from .repositories import jobs as jobs_repo
from .repositories import projects as projects_repo
_log = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Cancellation flags: job_id → threading.Event (set = cancel requested)
# ---------------------------------------------------------------------------
_cancel_flags: dict[str, threading.Event] = {}
_flags_lock = threading.Lock()
# ---------------------------------------------------------------------------
# Thread pools
# Two executors:
# _general_executor — multi-worker for CPU-bound tasks (e.g. ABC conversion)
# _gpu_executor — single-worker for GPU-exclusive tasks (PyTorch + TF)
# serialises Demucs/BS-RoFormer and Omnizart so they
# never compete for VRAM; FIFO by submission order.
# _transcription_executor — single-worker queue ONLY for lyric transcription and
# LRC alignment. Kept separate from _gpu_executor so
# stem separation can run without blocking, but still
# prevents two transcription/alignment jobs from
# competing for VRAM. Jobs are submitted here instead
# of _gpu_executor for these job types.
# ---------------------------------------------------------------------------
_general_executor = ThreadPoolExecutor(max_workers=4)
_gpu_executor = ThreadPoolExecutor(max_workers=1)
_transcription_executor = ThreadPoolExecutor(max_workers=1)
# Shared VRAM coordination lock.
# Both _gpu_executor (separation) and _transcription_executor (Whisper/WhisperX)
# must hold this lock for the duration of GPU model loading + inference.
# Prevents the two independent executor threads from loading models simultaneously
# and triggering CUDA out-of-memory errors.
gpu_vram_lock = threading.Lock()
# Queue-depth counter for the transcription executor (includes the running job).
_tx_queue_lock = threading.Lock()
_tx_queue_depth = 0
# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------
class JobStatus(str, Enum):
QUEUED = "queued"
RUNNING = "running"
DONE = "done"
ERROR = "error"
CANCELLED = "cancelled"
def create_job(job_type: str) -> str:
job_id = jobs_repo.create(job_type)
with _flags_lock:
_cancel_flags[job_id] = threading.Event()
_log.info("[%s] created job_type=%s", job_id, job_type)
return job_id
def get_job(job_id: str) -> Optional[dict]:
return jobs_repo.get(job_id)
def list_jobs() -> list[dict]:
return jobs_repo.list_all()
def list_all_jobs() -> list[dict]:
"""Return every job record regardless of type."""
return list_jobs()
def delete_job(job_id: str) -> bool:
"""Permanently delete a job record. Returns True if the record existed."""
deleted = jobs_repo.delete(job_id)
with _flags_lock:
_cancel_flags.pop(job_id, None)
return deleted
def update_job(job_id: str, **kwargs):
jobs_repo.update(job_id, **kwargs)
def append_job_files(job_id: str, new_files: list):
"""Append file entries to an existing job's file list without overwriting."""
jobs_repo.append_files(job_id, new_files)
def cancel_job(job_id: str):
with _flags_lock:
flag = _cancel_flags.get(job_id)
if flag:
flag.set()
update_job(job_id, status=JobStatus.CANCELLED)
def should_cancel(job_id: str) -> bool:
with _flags_lock:
flag = _cancel_flags.get(job_id)
return flag.is_set() if flag else False
def submit(fn: Callable, *args, **kwargs):
"""Submit a CPU-bound background task (multi-worker)."""
_submit_with_logging(_general_executor, "general", fn, *args, **kwargs)
def submit_gpu(fn: Callable, *args, **kwargs):
"""Submit a GPU-exclusive background task (single-worker FIFO queue).
Use for stem separation (Demucs / BS-RoFormer) and MIDI transcription.
Tasks run strictly one-at-a-time in the order they were submitted.
"""
_submit_with_logging(_gpu_executor, "gpu", fn, *args, **kwargs)
def submit_transcription(job_id: str, fn: Callable, *args, **kwargs):
"""Submit a lyric transcription or LRC alignment job.
Uses a dedicated single-worker executor so transcription jobs are
serialised independently of stem separation, preventing VRAM contention.
Shows queue position in the job's progress field while waiting.
"""
global _tx_queue_depth
with _tx_queue_lock:
_tx_queue_depth += 1
position = _tx_queue_depth
_log.info("[%s] queued on transcription queue position=%d", job_id, position)
if position > 1:
update_job(job_id, progress=f"Queued (position {position} of {position})…")
def _wrapper():
global _tx_queue_depth
# Recalculate position at start time (earlier jobs may have finished).
with _tx_queue_lock:
pos = _tx_queue_depth
_log.info("[%s] transcription task started queue_depth=%d", job_id, pos)
if pos > 1:
update_job(job_id, progress=f"Queued — waiting for {pos - 1} job(s) to finish…")
try:
fn(*args, **kwargs)
_log.info("[%s] transcription task finished", job_id)
except Exception:
_log.exception("[%s] transcription task failed", job_id)
raise
finally:
with _tx_queue_lock:
_tx_queue_depth -= 1
_transcription_executor.submit(_wrapper)
def find_or_create_project(name: str, sep_job_id: str, tab_label: str = '') -> dict:
"""Upsert a project record by sep_job_id. Safe to call multiple times."""
return projects_repo.find_or_create(name, sep_job_id, tab_label=tab_label)
def get_project(project_id: str) -> Optional[dict]:
return projects_repo.get(project_id)
def get_project_by_sep_job(sep_job_id: str) -> Optional[dict]:
return projects_repo.get_by_sep_job(sep_job_id)
def update_project(project_id: str, **kwargs):
projects_repo.update(project_id, **kwargs)
def list_projects() -> list:
return projects_repo.list_all()
def set_project_trans_job(project_id: str, stem_name: str, job_id: str):
"""Store a per-stem transcription job_id in the project's trans_jobs dict."""
projects_repo.set_transcription_job(project_id, stem_name, job_id)
def set_project_abc_job(project_id: str, stem_name: str, job_id: str):
"""Store a per-stem ABC job_id in the project's abc_jobs dict."""
projects_repo.set_abc_job(project_id, stem_name, job_id)
def set_project_lrc_job(project_id: str, stem_name: str, job_id: str):
"""Store a per-stem LRC alignment job_id in the project's lrc_jobs dict."""
projects_repo.set_lrc_job(project_id, stem_name, job_id)
def get_executor_state() -> dict:
"""Return queue/worker visibility for operational dashboards."""
with _tx_queue_lock:
tx_depth = _tx_queue_depth
return {
"general": _executor_state(_general_executor),
"gpu": _executor_state(_gpu_executor),
"transcription": {
**_executor_state(_transcription_executor),
"queue_depth_including_running": tx_depth,
},
}
def _extract_job_id(args: tuple, kwargs: dict) -> str | None:
if args and isinstance(args[0], str):
return args[0]
candidate = kwargs.get("job_id")
if isinstance(candidate, str):
return candidate
return None
def _submit_with_logging(executor: ThreadPoolExecutor, queue_name: str, fn: Callable, *args, **kwargs):
job_id = _extract_job_id(args, kwargs)
fn_name = getattr(fn, "__name__", fn.__class__.__name__)
if job_id:
_log.info("[%s] queued on %s queue task=%s", job_id, queue_name, fn_name)
def _wrapper():
if job_id:
_log.info("[%s] started on %s queue task=%s", job_id, queue_name, fn_name)
try:
fn(*args, **kwargs)
if job_id:
_log.info("[%s] finished on %s queue task=%s", job_id, queue_name, fn_name)
except Exception:
if job_id:
_log.exception("[%s] failed on %s queue task=%s", job_id, queue_name, fn_name)
raise
executor.submit(_wrapper)
def _executor_state(executor: ThreadPoolExecutor) -> dict:
queued = None
workers = None
alive_workers = None
try:
queued = executor._work_queue.qsize()
except Exception:
pass
try:
workers = len(executor._threads)
alive_workers = sum(1 for t in executor._threads if t.is_alive())
except Exception:
pass
return {
"queued": queued,
"workers": workers,
"alive_workers": alive_workers,
}