Files
2026-04-16 15:52:46 +09:00

403 lines
13 KiB
Python
Executable File

"""
Job manager: SQLite-backed job state + ThreadPoolExecutor for background tasks.
"""
import logging
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timedelta, timezone
from enum import Enum
from typing import Callable, Optional
from .config import (
STEMENGINE_JOB_RETENTION_DAYS,
STEMENGINE_JOB_RETENTION_ENABLED,
STEMENGINE_JOB_RETENTION_INTERVAL_SECS,
STEMENGINE_JOB_RETENTION_PRUNE_BATCH,
get_gpu_vram_reserve_mb,
)
from .repositories import jobs as jobs_repo
from .repositories import projects as projects_repo
from .services.vram_estimates import VramBudgetExceededError
_log = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Cancellation flags: job_id → threading.Event (set = cancel requested)
# ---------------------------------------------------------------------------
_cancel_flags: dict[str, threading.Event] = {}
_flags_lock = threading.Lock()
_retention_lock = threading.Lock()
_retention_last_run = 0.0
# ---------------------------------------------------------------------------
# Thread pools
# Two executors:
# _general_executor — multi-worker for CPU-bound tasks (e.g. ABC conversion)
# _gpu_executor — single-worker for GPU-exclusive tasks (PyTorch + TF)
# serialises Demucs/BS-RoFormer and Omnizart so they
# never compete for VRAM; FIFO by submission order.
# _transcription_executor — single-worker queue ONLY for lyric transcription and
# LRC alignment. Kept separate from _gpu_executor so
# stem separation can run without blocking, but still
# prevents two transcription/alignment jobs from
# competing for VRAM. Jobs are submitted here instead
# of _gpu_executor for these job types.
# ---------------------------------------------------------------------------
_general_executor = ThreadPoolExecutor(max_workers=4)
_gpu_executor = ThreadPoolExecutor(max_workers=1)
_transcription_executor = ThreadPoolExecutor(max_workers=1)
# Shared VRAM coordination lock.
# Both _gpu_executor (separation) and _transcription_executor (Whisper/WhisperX)
# must hold this lock for the duration of GPU model loading + inference.
# Prevents the two independent executor threads from loading models simultaneously
# and triggering CUDA out-of-memory errors.
gpu_vram_lock = threading.Lock()
# Queue-depth counter for the transcription executor (includes the running job).
_tx_queue_lock = threading.Lock()
_tx_queue_depth = 0
# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------
class JobStatus(str, Enum):
QUEUED = "queued"
RUNNING = "running"
DONE = "done"
ERROR = "error"
CANCELLED = "cancelled"
def create_job(job_type: str) -> str:
_maybe_run_retention_prune()
job_id = jobs_repo.create(job_type)
with _flags_lock:
_cancel_flags[job_id] = threading.Event()
_log.info("[%s] created job_type=%s", job_id, job_type)
return job_id
def get_job(job_id: str) -> Optional[dict]:
return jobs_repo.get(job_id)
def list_jobs() -> list[dict]:
return jobs_repo.list_by_status([JobStatus.QUEUED.value, JobStatus.RUNNING.value])
def list_all_jobs() -> list[dict]:
"""Return every job record regardless of type."""
return jobs_repo.list_all()
def list_recent_jobs(limit: int = 50) -> list[dict]:
"""Return a bounded recent job history, newest first."""
return jobs_repo.list_recent(limit=limit)
def delete_job(job_id: str) -> bool:
"""Permanently delete a job record. Returns True if the record existed."""
deleted = jobs_repo.delete(job_id)
cleanup_job_state(job_id)
return deleted
def cleanup_job_state(job_id: str) -> None:
"""Forget any in-memory state associated with a job ID."""
with _flags_lock:
_cancel_flags.pop(job_id, None)
def update_job(job_id: str, **kwargs):
jobs_repo.update(job_id, **kwargs)
if _is_terminal_status(kwargs.get("status")):
cleanup_job_state(job_id)
def append_job_files(job_id: str, new_files: list):
"""Append file entries to an existing job's file list without overwriting."""
jobs_repo.append_files(job_id, new_files)
def cancel_job(job_id: str):
with _flags_lock:
flag = _cancel_flags.get(job_id)
if flag:
flag.set()
update_job(job_id, status=JobStatus.CANCELLED)
def should_cancel(job_id: str) -> bool:
with _flags_lock:
flag = _cancel_flags.get(job_id)
if flag and flag.is_set():
return True
try:
job = jobs_repo.get(job_id)
if job and job.get("status") == JobStatus.CANCELLED:
return True
except Exception:
pass
return False
def submit(fn: Callable, *args, **kwargs):
"""Submit a CPU-bound background task (multi-worker)."""
_submit_with_logging(_general_executor, "general", fn, *args, **kwargs)
def submit_gpu(fn: Callable, *args, estimated_vram_mb: int | None = None, **kwargs):
"""Submit a GPU-exclusive background task (single-worker FIFO queue).
Use for stem separation (Demucs / BS-RoFormer) and MIDI transcription.
Tasks run strictly one-at-a-time in the order they were submitted.
"""
if estimated_vram_mb is not None:
try:
import torch
if torch.cuda.is_available():
free_b, _total_b = torch.cuda.mem_get_info()
free_mb = int(free_b / (1024 ** 2))
reserve_mb = int(get_gpu_vram_reserve_mb())
if int(estimated_vram_mb) > max(0, free_mb - reserve_mb):
raise VramBudgetExceededError(
required_mb=int(estimated_vram_mb),
free_mb=free_mb,
reserve_mb=reserve_mb,
)
except VramBudgetExceededError:
raise
except Exception:
# If probing fails, proceed with normal execution path.
pass
_submit_with_logging(_gpu_executor, "gpu", fn, *args, **kwargs)
def submit_transcription(job_id: str, fn: Callable, *args, **kwargs):
"""Submit a lyric transcription or LRC alignment job.
Uses a dedicated single-worker executor so transcription jobs are
serialised independently of stem separation, preventing VRAM contention.
Shows queue position in the job's progress field while waiting.
"""
global _tx_queue_depth
with _tx_queue_lock:
_tx_queue_depth += 1
position = _tx_queue_depth
_log.info("[%s] queued on transcription queue position=%d", job_id, position)
if position > 1:
update_job(job_id, progress=f"Queued (position {position} of {position})…")
def _wrapper():
global _tx_queue_depth
# Recalculate position at start time (earlier jobs may have finished).
with _tx_queue_lock:
pos = _tx_queue_depth
_log.info("[%s] transcription task started queue_depth=%d", job_id, pos)
if pos > 1:
update_job(job_id, progress=f"Queued — waiting for {pos - 1} job(s) to finish…")
try:
fn(*args, **kwargs)
_log.info("[%s] transcription task finished", job_id)
except Exception:
_log.exception("[%s] transcription task failed", job_id)
raise
finally:
with _tx_queue_lock:
_tx_queue_depth -= 1
_transcription_executor.submit(_wrapper)
def find_or_create_project(name: str, sep_job_id: str, tab_label: str = '') -> dict:
"""Upsert a project record by sep_job_id. Safe to call multiple times."""
return projects_repo.find_or_create(name, sep_job_id, tab_label=tab_label)
def get_project(project_id: str) -> Optional[dict]:
return projects_repo.get(project_id)
def get_project_by_sep_job(sep_job_id: str) -> Optional[dict]:
return projects_repo.get_by_sep_job(sep_job_id)
def update_project(project_id: str, **kwargs):
projects_repo.update(project_id, **kwargs)
def list_projects() -> list:
return projects_repo.list_all()
def set_project_trans_job(project_id: str, stem_name: str, job_id: str):
"""Store a per-stem transcription job_id in the project's trans_jobs dict."""
projects_repo.set_transcription_job(project_id, stem_name, job_id)
def set_project_abc_job(project_id: str, stem_name: str, job_id: str):
"""Store a per-stem ABC job_id in the project's abc_jobs dict."""
projects_repo.set_abc_job(project_id, stem_name, job_id)
def set_project_lrc_job(project_id: str, stem_name: str, job_id: str):
"""Store a per-stem LRC alignment job_id in the project's lrc_jobs dict."""
projects_repo.set_lrc_job(project_id, stem_name, job_id)
def get_executor_state() -> dict:
"""Return queue/worker visibility for operational dashboards."""
with _tx_queue_lock:
tx_depth = _tx_queue_depth
return {
"general": _executor_state(_general_executor),
"gpu": _executor_state(_gpu_executor),
"transcription": {
**_executor_state(_transcription_executor),
"queue_depth_including_running": tx_depth,
},
}
def _extract_job_id(args: tuple, kwargs: dict) -> str | None:
if args and isinstance(args[0], str):
return args[0]
candidate = kwargs.get("job_id")
if isinstance(candidate, str):
return candidate
return None
def _submit_with_logging(executor: ThreadPoolExecutor, queue_name: str, fn: Callable, *args, **kwargs):
job_id = _extract_job_id(args, kwargs)
fn_name = getattr(fn, "__name__", fn.__class__.__name__)
if job_id:
_log.info("[%s] queued on %s queue task=%s", job_id, queue_name, fn_name)
def _wrapper():
if job_id:
_log.info("[%s] started on %s queue task=%s", job_id, queue_name, fn_name)
try:
fn(*args, **kwargs)
if job_id:
_log.info("[%s] finished on %s queue task=%s", job_id, queue_name, fn_name)
except Exception:
if job_id:
_log.exception("[%s] failed on %s queue task=%s", job_id, queue_name, fn_name)
raise
executor.submit(_wrapper)
def _executor_state(executor: ThreadPoolExecutor) -> dict:
queued = None
workers = None
alive_workers = None
try:
queued = executor._work_queue.qsize()
except Exception:
pass
try:
workers = len(executor._threads)
alive_workers = sum(1 for t in executor._threads if t.is_alive())
except Exception:
pass
return {
"queued": queued,
"workers": workers,
"alive_workers": alive_workers,
}
def prune_old_non_project_jobs(*, now: Optional[datetime] = None, force: bool = False) -> dict:
"""Prune terminal jobs not linked to any project once they age out."""
if not STEMENGINE_JOB_RETENTION_ENABLED and not force:
return {
"enabled": False,
"deleted": 0,
"considered": 0,
"cutoff": None,
}
current = now or datetime.now(timezone.utc)
cutoff = current - timedelta(days=STEMENGINE_JOB_RETENTION_DAYS)
candidate_ids = jobs_repo.list_terminal_unlinked_older_than(
older_than=cutoff,
limit=STEMENGINE_JOB_RETENTION_PRUNE_BATCH,
)
deleted = 0
for job_id in candidate_ids:
if jobs_repo.delete(job_id):
cleanup_job_state(job_id)
deleted += 1
return {
"enabled": STEMENGINE_JOB_RETENTION_ENABLED,
"deleted": deleted,
"considered": len(candidate_ids),
"cutoff": cutoff.isoformat(),
}
def _maybe_run_retention_prune() -> None:
if not STEMENGINE_JOB_RETENTION_ENABLED:
return
global _retention_last_run
now_monotonic = time.monotonic()
if STEMENGINE_JOB_RETENTION_INTERVAL_SECS > 0 and (
now_monotonic - _retention_last_run < STEMENGINE_JOB_RETENTION_INTERVAL_SECS
):
return
with _retention_lock:
now_monotonic = time.monotonic()
if STEMENGINE_JOB_RETENTION_INTERVAL_SECS > 0 and (
now_monotonic - _retention_last_run < STEMENGINE_JOB_RETENTION_INTERVAL_SECS
):
return
_retention_last_run = now_monotonic
try:
result = prune_old_non_project_jobs()
if result.get("deleted", 0) > 0:
_log.info(
"Pruned %d old non-project jobs (considered=%d, cutoff=%s)",
result["deleted"],
result["considered"],
result["cutoff"],
)
except Exception:
_log.exception("Job retention prune failed")
def _is_terminal_status(status: object) -> bool:
if status is None:
return False
value = getattr(status, "value", status)
value = str(value)
return value in {
JobStatus.DONE.value,
JobStatus.ERROR.value,
JobStatus.CANCELLED.value,
}