""" Job manager: SQLite-backed job state + ThreadPoolExecutor for background tasks. """ import logging import threading import time from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timedelta, timezone from enum import Enum from typing import Callable, Optional from .config import ( STEMENGINE_JOB_RETENTION_DAYS, STEMENGINE_JOB_RETENTION_ENABLED, STEMENGINE_JOB_RETENTION_INTERVAL_SECS, STEMENGINE_JOB_RETENTION_PRUNE_BATCH, get_gpu_vram_reserve_mb, ) from .repositories import jobs as jobs_repo from .repositories import projects as projects_repo from .services.vram_estimates import VramBudgetExceededError _log = logging.getLogger(__name__) # --------------------------------------------------------------------------- # Cancellation flags: job_id → threading.Event (set = cancel requested) # --------------------------------------------------------------------------- _cancel_flags: dict[str, threading.Event] = {} _flags_lock = threading.Lock() _retention_lock = threading.Lock() _retention_last_run = 0.0 # --------------------------------------------------------------------------- # Thread pools # Two executors: # _general_executor — multi-worker for CPU-bound tasks (e.g. ABC conversion) # _gpu_executor — single-worker for GPU-exclusive tasks (PyTorch + TF) # serialises Demucs/BS-RoFormer and Omnizart so they # never compete for VRAM; FIFO by submission order. # _transcription_executor — single-worker queue ONLY for lyric transcription and # LRC alignment. Kept separate from _gpu_executor so # stem separation can run without blocking, but still # prevents two transcription/alignment jobs from # competing for VRAM. Jobs are submitted here instead # of _gpu_executor for these job types. # --------------------------------------------------------------------------- _general_executor = ThreadPoolExecutor(max_workers=4) _gpu_executor = ThreadPoolExecutor(max_workers=1) _transcription_executor = ThreadPoolExecutor(max_workers=1) # Shared VRAM coordination lock. # Both _gpu_executor (separation) and _transcription_executor (Whisper/WhisperX) # must hold this lock for the duration of GPU model loading + inference. # Prevents the two independent executor threads from loading models simultaneously # and triggering CUDA out-of-memory errors. gpu_vram_lock = threading.Lock() # Queue-depth counter for the transcription executor (includes the running job). _tx_queue_lock = threading.Lock() _tx_queue_depth = 0 # --------------------------------------------------------------------------- # Public API # --------------------------------------------------------------------------- class JobStatus(str, Enum): QUEUED = "queued" RUNNING = "running" DONE = "done" ERROR = "error" CANCELLED = "cancelled" def create_job(job_type: str) -> str: _maybe_run_retention_prune() job_id = jobs_repo.create(job_type) with _flags_lock: _cancel_flags[job_id] = threading.Event() _log.info("[%s] created job_type=%s", job_id, job_type) return job_id def get_job(job_id: str) -> Optional[dict]: return jobs_repo.get(job_id) def list_jobs() -> list[dict]: return jobs_repo.list_by_status([JobStatus.QUEUED.value, JobStatus.RUNNING.value]) def list_all_jobs() -> list[dict]: """Return every job record regardless of type.""" return jobs_repo.list_all() def list_recent_jobs(limit: int = 50) -> list[dict]: """Return a bounded recent job history, newest first.""" return jobs_repo.list_recent(limit=limit) def delete_job(job_id: str) -> bool: """Permanently delete a job record. Returns True if the record existed.""" deleted = jobs_repo.delete(job_id) cleanup_job_state(job_id) return deleted def cleanup_job_state(job_id: str) -> None: """Forget any in-memory state associated with a job ID.""" with _flags_lock: _cancel_flags.pop(job_id, None) def update_job(job_id: str, **kwargs): jobs_repo.update(job_id, **kwargs) if _is_terminal_status(kwargs.get("status")): cleanup_job_state(job_id) def append_job_files(job_id: str, new_files: list): """Append file entries to an existing job's file list without overwriting.""" jobs_repo.append_files(job_id, new_files) def cancel_job(job_id: str): with _flags_lock: flag = _cancel_flags.get(job_id) if flag: flag.set() update_job(job_id, status=JobStatus.CANCELLED) def should_cancel(job_id: str) -> bool: with _flags_lock: flag = _cancel_flags.get(job_id) if flag and flag.is_set(): return True try: job = jobs_repo.get(job_id) if job and job.get("status") == JobStatus.CANCELLED: return True except Exception: pass return False def submit(fn: Callable, *args, **kwargs): """Submit a CPU-bound background task (multi-worker).""" _submit_with_logging(_general_executor, "general", fn, *args, **kwargs) def submit_gpu(fn: Callable, *args, estimated_vram_mb: int | None = None, **kwargs): """Submit a GPU-exclusive background task (single-worker FIFO queue). Use for stem separation (Demucs / BS-RoFormer) and MIDI transcription. Tasks run strictly one-at-a-time in the order they were submitted. """ if estimated_vram_mb is not None: try: import torch if torch.cuda.is_available(): free_b, _total_b = torch.cuda.mem_get_info() free_mb = int(free_b / (1024 ** 2)) reserve_mb = int(get_gpu_vram_reserve_mb()) if int(estimated_vram_mb) > max(0, free_mb - reserve_mb): raise VramBudgetExceededError( required_mb=int(estimated_vram_mb), free_mb=free_mb, reserve_mb=reserve_mb, ) except VramBudgetExceededError: raise except Exception: # If probing fails, proceed with normal execution path. pass _submit_with_logging(_gpu_executor, "gpu", fn, *args, **kwargs) def submit_transcription(job_id: str, fn: Callable, *args, **kwargs): """Submit a lyric transcription or LRC alignment job. Uses a dedicated single-worker executor so transcription jobs are serialised independently of stem separation, preventing VRAM contention. Shows queue position in the job's progress field while waiting. """ global _tx_queue_depth with _tx_queue_lock: _tx_queue_depth += 1 position = _tx_queue_depth _log.info("[%s] queued on transcription queue position=%d", job_id, position) if position > 1: update_job(job_id, progress=f"Queued (position {position} of {position})…") def _wrapper(): global _tx_queue_depth # Recalculate position at start time (earlier jobs may have finished). with _tx_queue_lock: pos = _tx_queue_depth _log.info("[%s] transcription task started queue_depth=%d", job_id, pos) if pos > 1: update_job(job_id, progress=f"Queued — waiting for {pos - 1} job(s) to finish…") try: fn(*args, **kwargs) _log.info("[%s] transcription task finished", job_id) except Exception: _log.exception("[%s] transcription task failed", job_id) raise finally: with _tx_queue_lock: _tx_queue_depth -= 1 _transcription_executor.submit(_wrapper) def find_or_create_project(name: str, sep_job_id: str, tab_label: str = '') -> dict: """Upsert a project record by sep_job_id. Safe to call multiple times.""" return projects_repo.find_or_create(name, sep_job_id, tab_label=tab_label) def get_project(project_id: str) -> Optional[dict]: return projects_repo.get(project_id) def get_project_by_sep_job(sep_job_id: str) -> Optional[dict]: return projects_repo.get_by_sep_job(sep_job_id) def update_project(project_id: str, **kwargs): projects_repo.update(project_id, **kwargs) def list_projects() -> list: return projects_repo.list_all() def set_project_trans_job(project_id: str, stem_name: str, job_id: str): """Store a per-stem transcription job_id in the project's trans_jobs dict.""" projects_repo.set_transcription_job(project_id, stem_name, job_id) def set_project_abc_job(project_id: str, stem_name: str, job_id: str): """Store a per-stem ABC job_id in the project's abc_jobs dict.""" projects_repo.set_abc_job(project_id, stem_name, job_id) def set_project_lrc_job(project_id: str, stem_name: str, job_id: str): """Store a per-stem LRC alignment job_id in the project's lrc_jobs dict.""" projects_repo.set_lrc_job(project_id, stem_name, job_id) def get_executor_state() -> dict: """Return queue/worker visibility for operational dashboards.""" with _tx_queue_lock: tx_depth = _tx_queue_depth return { "general": _executor_state(_general_executor), "gpu": _executor_state(_gpu_executor), "transcription": { **_executor_state(_transcription_executor), "queue_depth_including_running": tx_depth, }, } def _extract_job_id(args: tuple, kwargs: dict) -> str | None: if args and isinstance(args[0], str): return args[0] candidate = kwargs.get("job_id") if isinstance(candidate, str): return candidate return None def _submit_with_logging(executor: ThreadPoolExecutor, queue_name: str, fn: Callable, *args, **kwargs): job_id = _extract_job_id(args, kwargs) fn_name = getattr(fn, "__name__", fn.__class__.__name__) if job_id: _log.info("[%s] queued on %s queue task=%s", job_id, queue_name, fn_name) def _wrapper(): if job_id: _log.info("[%s] started on %s queue task=%s", job_id, queue_name, fn_name) try: fn(*args, **kwargs) if job_id: _log.info("[%s] finished on %s queue task=%s", job_id, queue_name, fn_name) except Exception: if job_id: _log.exception("[%s] failed on %s queue task=%s", job_id, queue_name, fn_name) raise executor.submit(_wrapper) def _executor_state(executor: ThreadPoolExecutor) -> dict: queued = None workers = None alive_workers = None try: queued = executor._work_queue.qsize() except Exception: pass try: workers = len(executor._threads) alive_workers = sum(1 for t in executor._threads if t.is_alive()) except Exception: pass return { "queued": queued, "workers": workers, "alive_workers": alive_workers, } def prune_old_non_project_jobs(*, now: Optional[datetime] = None, force: bool = False) -> dict: """Prune terminal jobs not linked to any project once they age out.""" if not STEMENGINE_JOB_RETENTION_ENABLED and not force: return { "enabled": False, "deleted": 0, "considered": 0, "cutoff": None, } current = now or datetime.now(timezone.utc) cutoff = current - timedelta(days=STEMENGINE_JOB_RETENTION_DAYS) candidate_ids = jobs_repo.list_terminal_unlinked_older_than( older_than=cutoff, limit=STEMENGINE_JOB_RETENTION_PRUNE_BATCH, ) deleted = 0 for job_id in candidate_ids: if jobs_repo.delete(job_id): cleanup_job_state(job_id) deleted += 1 return { "enabled": STEMENGINE_JOB_RETENTION_ENABLED, "deleted": deleted, "considered": len(candidate_ids), "cutoff": cutoff.isoformat(), } def _maybe_run_retention_prune() -> None: if not STEMENGINE_JOB_RETENTION_ENABLED: return global _retention_last_run now_monotonic = time.monotonic() if STEMENGINE_JOB_RETENTION_INTERVAL_SECS > 0 and ( now_monotonic - _retention_last_run < STEMENGINE_JOB_RETENTION_INTERVAL_SECS ): return with _retention_lock: now_monotonic = time.monotonic() if STEMENGINE_JOB_RETENTION_INTERVAL_SECS > 0 and ( now_monotonic - _retention_last_run < STEMENGINE_JOB_RETENTION_INTERVAL_SECS ): return _retention_last_run = now_monotonic try: result = prune_old_non_project_jobs() if result.get("deleted", 0) > 0: _log.info( "Pruned %d old non-project jobs (considered=%d, cutoff=%s)", result["deleted"], result["considered"], result["cutoff"], ) except Exception: _log.exception("Job retention prune failed") def _is_terminal_status(status: object) -> bool: if status is None: return False value = getattr(status, "value", status) value = str(value) return value in { JobStatus.DONE.value, JobStatus.ERROR.value, JobStatus.CANCELLED.value, }