403 lines
13 KiB
Python
Executable File
403 lines
13 KiB
Python
Executable File
"""
|
|
Job manager: SQLite-backed job state + ThreadPoolExecutor for background tasks.
|
|
"""
|
|
import logging
|
|
import threading
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import datetime, timedelta, timezone
|
|
from enum import Enum
|
|
from typing import Callable, Optional
|
|
|
|
from .config import (
|
|
STEMENGINE_JOB_RETENTION_DAYS,
|
|
STEMENGINE_JOB_RETENTION_ENABLED,
|
|
STEMENGINE_JOB_RETENTION_INTERVAL_SECS,
|
|
STEMENGINE_JOB_RETENTION_PRUNE_BATCH,
|
|
get_gpu_vram_reserve_mb,
|
|
)
|
|
from .repositories import jobs as jobs_repo
|
|
from .repositories import projects as projects_repo
|
|
from .services.vram_estimates import VramBudgetExceededError
|
|
|
|
|
|
_log = logging.getLogger(__name__)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cancellation flags: job_id → threading.Event (set = cancel requested)
|
|
# ---------------------------------------------------------------------------
|
|
_cancel_flags: dict[str, threading.Event] = {}
|
|
_flags_lock = threading.Lock()
|
|
|
|
_retention_lock = threading.Lock()
|
|
_retention_last_run = 0.0
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Thread pools
|
|
# Two executors:
|
|
# _general_executor — multi-worker for CPU-bound tasks (e.g. ABC conversion)
|
|
# _gpu_executor — single-worker for GPU-exclusive tasks (PyTorch + TF)
|
|
# serialises Demucs/BS-RoFormer and Omnizart so they
|
|
# never compete for VRAM; FIFO by submission order.
|
|
# _transcription_executor — single-worker queue ONLY for lyric transcription and
|
|
# LRC alignment. Kept separate from _gpu_executor so
|
|
# stem separation can run without blocking, but still
|
|
# prevents two transcription/alignment jobs from
|
|
# competing for VRAM. Jobs are submitted here instead
|
|
# of _gpu_executor for these job types.
|
|
# ---------------------------------------------------------------------------
|
|
_general_executor = ThreadPoolExecutor(max_workers=4)
|
|
_gpu_executor = ThreadPoolExecutor(max_workers=1)
|
|
_transcription_executor = ThreadPoolExecutor(max_workers=1)
|
|
|
|
# Shared VRAM coordination lock.
|
|
# Both _gpu_executor (separation) and _transcription_executor (Whisper/WhisperX)
|
|
# must hold this lock for the duration of GPU model loading + inference.
|
|
# Prevents the two independent executor threads from loading models simultaneously
|
|
# and triggering CUDA out-of-memory errors.
|
|
gpu_vram_lock = threading.Lock()
|
|
|
|
# Queue-depth counter for the transcription executor (includes the running job).
|
|
_tx_queue_lock = threading.Lock()
|
|
_tx_queue_depth = 0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Public API
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class JobStatus(str, Enum):
|
|
QUEUED = "queued"
|
|
RUNNING = "running"
|
|
DONE = "done"
|
|
ERROR = "error"
|
|
CANCELLED = "cancelled"
|
|
|
|
|
|
def create_job(job_type: str) -> str:
|
|
_maybe_run_retention_prune()
|
|
job_id = jobs_repo.create(job_type)
|
|
with _flags_lock:
|
|
_cancel_flags[job_id] = threading.Event()
|
|
_log.info("[%s] created job_type=%s", job_id, job_type)
|
|
return job_id
|
|
|
|
|
|
def get_job(job_id: str) -> Optional[dict]:
|
|
return jobs_repo.get(job_id)
|
|
|
|
|
|
def list_jobs() -> list[dict]:
|
|
return jobs_repo.list_by_status([JobStatus.QUEUED.value, JobStatus.RUNNING.value])
|
|
|
|
|
|
def list_all_jobs() -> list[dict]:
|
|
"""Return every job record regardless of type."""
|
|
return jobs_repo.list_all()
|
|
|
|
|
|
def list_recent_jobs(limit: int = 50) -> list[dict]:
|
|
"""Return a bounded recent job history, newest first."""
|
|
return jobs_repo.list_recent(limit=limit)
|
|
|
|
|
|
def delete_job(job_id: str) -> bool:
|
|
"""Permanently delete a job record. Returns True if the record existed."""
|
|
deleted = jobs_repo.delete(job_id)
|
|
cleanup_job_state(job_id)
|
|
return deleted
|
|
|
|
|
|
def cleanup_job_state(job_id: str) -> None:
|
|
"""Forget any in-memory state associated with a job ID."""
|
|
with _flags_lock:
|
|
_cancel_flags.pop(job_id, None)
|
|
|
|
|
|
def update_job(job_id: str, **kwargs):
|
|
jobs_repo.update(job_id, **kwargs)
|
|
if _is_terminal_status(kwargs.get("status")):
|
|
cleanup_job_state(job_id)
|
|
|
|
|
|
def append_job_files(job_id: str, new_files: list):
|
|
"""Append file entries to an existing job's file list without overwriting."""
|
|
jobs_repo.append_files(job_id, new_files)
|
|
|
|
|
|
def cancel_job(job_id: str):
|
|
with _flags_lock:
|
|
flag = _cancel_flags.get(job_id)
|
|
if flag:
|
|
flag.set()
|
|
update_job(job_id, status=JobStatus.CANCELLED)
|
|
|
|
|
|
def should_cancel(job_id: str) -> bool:
|
|
with _flags_lock:
|
|
flag = _cancel_flags.get(job_id)
|
|
if flag and flag.is_set():
|
|
return True
|
|
|
|
try:
|
|
job = jobs_repo.get(job_id)
|
|
if job and job.get("status") == JobStatus.CANCELLED:
|
|
return True
|
|
except Exception:
|
|
pass
|
|
|
|
return False
|
|
|
|
|
|
def submit(fn: Callable, *args, **kwargs):
|
|
"""Submit a CPU-bound background task (multi-worker)."""
|
|
_submit_with_logging(_general_executor, "general", fn, *args, **kwargs)
|
|
|
|
|
|
def submit_gpu(fn: Callable, *args, estimated_vram_mb: int | None = None, **kwargs):
|
|
"""Submit a GPU-exclusive background task (single-worker FIFO queue).
|
|
|
|
Use for stem separation (Demucs / BS-RoFormer) and MIDI transcription.
|
|
Tasks run strictly one-at-a-time in the order they were submitted.
|
|
"""
|
|
if estimated_vram_mb is not None:
|
|
try:
|
|
import torch
|
|
|
|
if torch.cuda.is_available():
|
|
free_b, _total_b = torch.cuda.mem_get_info()
|
|
free_mb = int(free_b / (1024 ** 2))
|
|
reserve_mb = int(get_gpu_vram_reserve_mb())
|
|
if int(estimated_vram_mb) > max(0, free_mb - reserve_mb):
|
|
raise VramBudgetExceededError(
|
|
required_mb=int(estimated_vram_mb),
|
|
free_mb=free_mb,
|
|
reserve_mb=reserve_mb,
|
|
)
|
|
except VramBudgetExceededError:
|
|
raise
|
|
except Exception:
|
|
# If probing fails, proceed with normal execution path.
|
|
pass
|
|
|
|
_submit_with_logging(_gpu_executor, "gpu", fn, *args, **kwargs)
|
|
|
|
|
|
def submit_transcription(job_id: str, fn: Callable, *args, **kwargs):
|
|
"""Submit a lyric transcription or LRC alignment job.
|
|
|
|
Uses a dedicated single-worker executor so transcription jobs are
|
|
serialised independently of stem separation, preventing VRAM contention.
|
|
Shows queue position in the job's progress field while waiting.
|
|
"""
|
|
global _tx_queue_depth
|
|
|
|
with _tx_queue_lock:
|
|
_tx_queue_depth += 1
|
|
position = _tx_queue_depth
|
|
|
|
_log.info("[%s] queued on transcription queue position=%d", job_id, position)
|
|
|
|
if position > 1:
|
|
update_job(job_id, progress=f"Queued (position {position} of {position})…")
|
|
|
|
def _wrapper():
|
|
global _tx_queue_depth
|
|
# Recalculate position at start time (earlier jobs may have finished).
|
|
with _tx_queue_lock:
|
|
pos = _tx_queue_depth
|
|
_log.info("[%s] transcription task started queue_depth=%d", job_id, pos)
|
|
if pos > 1:
|
|
update_job(job_id, progress=f"Queued — waiting for {pos - 1} job(s) to finish…")
|
|
try:
|
|
fn(*args, **kwargs)
|
|
_log.info("[%s] transcription task finished", job_id)
|
|
except Exception:
|
|
_log.exception("[%s] transcription task failed", job_id)
|
|
raise
|
|
finally:
|
|
with _tx_queue_lock:
|
|
_tx_queue_depth -= 1
|
|
|
|
_transcription_executor.submit(_wrapper)
|
|
|
|
|
|
def find_or_create_project(name: str, sep_job_id: str, tab_label: str = '') -> dict:
|
|
"""Upsert a project record by sep_job_id. Safe to call multiple times."""
|
|
return projects_repo.find_or_create(name, sep_job_id, tab_label=tab_label)
|
|
|
|
|
|
def get_project(project_id: str) -> Optional[dict]:
|
|
return projects_repo.get(project_id)
|
|
|
|
|
|
def get_project_by_sep_job(sep_job_id: str) -> Optional[dict]:
|
|
return projects_repo.get_by_sep_job(sep_job_id)
|
|
|
|
|
|
def update_project(project_id: str, **kwargs):
|
|
projects_repo.update(project_id, **kwargs)
|
|
|
|
|
|
def list_projects() -> list:
|
|
return projects_repo.list_all()
|
|
|
|
|
|
def set_project_trans_job(project_id: str, stem_name: str, job_id: str):
|
|
"""Store a per-stem transcription job_id in the project's trans_jobs dict."""
|
|
projects_repo.set_transcription_job(project_id, stem_name, job_id)
|
|
|
|
|
|
def set_project_abc_job(project_id: str, stem_name: str, job_id: str):
|
|
"""Store a per-stem ABC job_id in the project's abc_jobs dict."""
|
|
projects_repo.set_abc_job(project_id, stem_name, job_id)
|
|
|
|
|
|
def set_project_lrc_job(project_id: str, stem_name: str, job_id: str):
|
|
"""Store a per-stem LRC alignment job_id in the project's lrc_jobs dict."""
|
|
projects_repo.set_lrc_job(project_id, stem_name, job_id)
|
|
|
|
|
|
def get_executor_state() -> dict:
|
|
"""Return queue/worker visibility for operational dashboards."""
|
|
with _tx_queue_lock:
|
|
tx_depth = _tx_queue_depth
|
|
|
|
return {
|
|
"general": _executor_state(_general_executor),
|
|
"gpu": _executor_state(_gpu_executor),
|
|
"transcription": {
|
|
**_executor_state(_transcription_executor),
|
|
"queue_depth_including_running": tx_depth,
|
|
},
|
|
}
|
|
|
|
|
|
def _extract_job_id(args: tuple, kwargs: dict) -> str | None:
|
|
if args and isinstance(args[0], str):
|
|
return args[0]
|
|
candidate = kwargs.get("job_id")
|
|
if isinstance(candidate, str):
|
|
return candidate
|
|
return None
|
|
|
|
|
|
def _submit_with_logging(executor: ThreadPoolExecutor, queue_name: str, fn: Callable, *args, **kwargs):
|
|
job_id = _extract_job_id(args, kwargs)
|
|
fn_name = getattr(fn, "__name__", fn.__class__.__name__)
|
|
|
|
if job_id:
|
|
_log.info("[%s] queued on %s queue task=%s", job_id, queue_name, fn_name)
|
|
|
|
def _wrapper():
|
|
if job_id:
|
|
_log.info("[%s] started on %s queue task=%s", job_id, queue_name, fn_name)
|
|
try:
|
|
fn(*args, **kwargs)
|
|
if job_id:
|
|
_log.info("[%s] finished on %s queue task=%s", job_id, queue_name, fn_name)
|
|
except Exception:
|
|
if job_id:
|
|
_log.exception("[%s] failed on %s queue task=%s", job_id, queue_name, fn_name)
|
|
raise
|
|
|
|
executor.submit(_wrapper)
|
|
|
|
|
|
def _executor_state(executor: ThreadPoolExecutor) -> dict:
|
|
queued = None
|
|
workers = None
|
|
alive_workers = None
|
|
|
|
try:
|
|
queued = executor._work_queue.qsize()
|
|
except Exception:
|
|
pass
|
|
try:
|
|
workers = len(executor._threads)
|
|
alive_workers = sum(1 for t in executor._threads if t.is_alive())
|
|
except Exception:
|
|
pass
|
|
|
|
return {
|
|
"queued": queued,
|
|
"workers": workers,
|
|
"alive_workers": alive_workers,
|
|
}
|
|
|
|
|
|
def prune_old_non_project_jobs(*, now: Optional[datetime] = None, force: bool = False) -> dict:
|
|
"""Prune terminal jobs not linked to any project once they age out."""
|
|
if not STEMENGINE_JOB_RETENTION_ENABLED and not force:
|
|
return {
|
|
"enabled": False,
|
|
"deleted": 0,
|
|
"considered": 0,
|
|
"cutoff": None,
|
|
}
|
|
|
|
current = now or datetime.now(timezone.utc)
|
|
cutoff = current - timedelta(days=STEMENGINE_JOB_RETENTION_DAYS)
|
|
candidate_ids = jobs_repo.list_terminal_unlinked_older_than(
|
|
older_than=cutoff,
|
|
limit=STEMENGINE_JOB_RETENTION_PRUNE_BATCH,
|
|
)
|
|
|
|
deleted = 0
|
|
for job_id in candidate_ids:
|
|
if jobs_repo.delete(job_id):
|
|
cleanup_job_state(job_id)
|
|
deleted += 1
|
|
|
|
return {
|
|
"enabled": STEMENGINE_JOB_RETENTION_ENABLED,
|
|
"deleted": deleted,
|
|
"considered": len(candidate_ids),
|
|
"cutoff": cutoff.isoformat(),
|
|
}
|
|
|
|
|
|
def _maybe_run_retention_prune() -> None:
|
|
if not STEMENGINE_JOB_RETENTION_ENABLED:
|
|
return
|
|
|
|
global _retention_last_run
|
|
now_monotonic = time.monotonic()
|
|
|
|
if STEMENGINE_JOB_RETENTION_INTERVAL_SECS > 0 and (
|
|
now_monotonic - _retention_last_run < STEMENGINE_JOB_RETENTION_INTERVAL_SECS
|
|
):
|
|
return
|
|
|
|
with _retention_lock:
|
|
now_monotonic = time.monotonic()
|
|
if STEMENGINE_JOB_RETENTION_INTERVAL_SECS > 0 and (
|
|
now_monotonic - _retention_last_run < STEMENGINE_JOB_RETENTION_INTERVAL_SECS
|
|
):
|
|
return
|
|
|
|
_retention_last_run = now_monotonic
|
|
try:
|
|
result = prune_old_non_project_jobs()
|
|
if result.get("deleted", 0) > 0:
|
|
_log.info(
|
|
"Pruned %d old non-project jobs (considered=%d, cutoff=%s)",
|
|
result["deleted"],
|
|
result["considered"],
|
|
result["cutoff"],
|
|
)
|
|
except Exception:
|
|
_log.exception("Job retention prune failed")
|
|
|
|
|
|
def _is_terminal_status(status: object) -> bool:
|
|
if status is None:
|
|
return False
|
|
value = getattr(status, "value", status)
|
|
value = str(value)
|
|
return value in {
|
|
JobStatus.DONE.value,
|
|
JobStatus.ERROR.value,
|
|
JobStatus.CANCELLED.value,
|
|
}
|