402 lines
14 KiB
Python
Executable File
402 lines
14 KiB
Python
Executable File
"""
|
|
Job manager: SQLite-backed job state + ThreadPoolExecutor for background tasks.
|
|
"""
|
|
import re
|
|
import uuid
|
|
import threading
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import datetime
|
|
from enum import Enum
|
|
from pathlib import Path
|
|
from typing import Callable, Optional
|
|
from uuid import uuid4
|
|
|
|
from sqlalchemy import create_engine, Column, String, DateTime, JSON, Text, text, event
|
|
from sqlalchemy.orm import DeclarativeBase, Session
|
|
from sqlalchemy.orm.attributes import flag_modified
|
|
|
|
from .config import DB_PATH
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# DB setup
|
|
# ---------------------------------------------------------------------------
|
|
|
|
engine = create_engine(
|
|
f"sqlite:///{DB_PATH}",
|
|
connect_args={"check_same_thread": False, "timeout": 30},
|
|
)
|
|
|
|
|
|
@event.listens_for(engine, "connect")
|
|
def _set_sqlite_pragmas(dbapi_connection, _connection_record):
|
|
"""Enable WAL mode and set busy timeout so concurrent writes don't deadlock."""
|
|
cursor = dbapi_connection.cursor()
|
|
cursor.execute("PRAGMA journal_mode=WAL")
|
|
cursor.execute("PRAGMA synchronous=NORMAL")
|
|
cursor.execute("PRAGMA busy_timeout=30000")
|
|
cursor.close()
|
|
|
|
|
|
class Base(DeclarativeBase):
|
|
pass
|
|
|
|
|
|
class JobRecord(Base):
|
|
__tablename__ = "jobs"
|
|
|
|
id = Column(String, primary_key=True)
|
|
job_type = Column(String, nullable=False) # separate | transcribe | abc-convert | pipeline | mix
|
|
status = Column(String, default="queued") # queued | running | done | error | cancelled
|
|
progress = Column(String, default="")
|
|
error = Column(Text, default="")
|
|
files = Column(JSON, default=list) # list of {name, path} dicts
|
|
result = Column(JSON, default=dict) # arbitrary result metadata
|
|
created_at = Column(DateTime, default=datetime.utcnow)
|
|
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
|
|
|
|
|
class ProjectRecord(Base):
|
|
__tablename__ = "projects"
|
|
|
|
id = Column(String, primary_key=True, default=lambda: str(uuid4()))
|
|
name = Column(String, nullable=False)
|
|
tab_label = Column(String, default='') # display label for tab e.g. "Demucs v4_MP3 (256 kbps)"
|
|
created_at = Column(DateTime, default=datetime.utcnow)
|
|
sep_job_id = Column(String, nullable=False)
|
|
trans_jobs = Column(JSON, default=dict) # stem_name → job_id
|
|
abc_jobs = Column(JSON, default=dict) # stem_name → job_id
|
|
lrc_jobs = Column(JSON, default=dict) # stem_name → job_id
|
|
mix_job_id = Column(String, nullable=True)
|
|
|
|
|
|
Base.metadata.create_all(engine)
|
|
|
|
# Idempotent migrations: add missing columns to existing tables
|
|
for _migration_sql in (
|
|
"ALTER TABLE jobs ADD COLUMN result JSON",
|
|
"ALTER TABLE projects ADD COLUMN trans_jobs JSON",
|
|
"ALTER TABLE projects ADD COLUMN abc_jobs JSON",
|
|
"ALTER TABLE projects ADD COLUMN lrc_jobs JSON",
|
|
"ALTER TABLE projects ADD COLUMN tab_label TEXT DEFAULT ''",
|
|
):
|
|
try:
|
|
with engine.connect() as _conn:
|
|
_conn.execute(text(_migration_sql))
|
|
_conn.commit()
|
|
except Exception:
|
|
pass # Column already exists
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cancellation flags: job_id → threading.Event (set = cancel requested)
|
|
# ---------------------------------------------------------------------------
|
|
_cancel_flags: dict[str, threading.Event] = {}
|
|
_flags_lock = threading.Lock()
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Thread pools
|
|
# Two executors:
|
|
# _general_executor — multi-worker for CPU-bound tasks (e.g. ABC conversion)
|
|
# _gpu_executor — single-worker for GPU-exclusive tasks (PyTorch + TF)
|
|
# serialises Demucs/BS-RoFormer and Omnizart so they
|
|
# never compete for VRAM; FIFO by submission order.
|
|
# ---------------------------------------------------------------------------
|
|
_general_executor = ThreadPoolExecutor(max_workers=4)
|
|
_gpu_executor = ThreadPoolExecutor(max_workers=1)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Public API
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class JobStatus(str, Enum):
|
|
QUEUED = "queued"
|
|
RUNNING = "running"
|
|
DONE = "done"
|
|
ERROR = "error"
|
|
CANCELLED = "cancelled"
|
|
|
|
|
|
def create_job(job_type: str) -> str:
|
|
job_id = str(uuid.uuid4())
|
|
with Session(engine) as session:
|
|
session.add(JobRecord(id=job_id, job_type=job_type))
|
|
session.commit()
|
|
with _flags_lock:
|
|
_cancel_flags[job_id] = threading.Event()
|
|
return job_id
|
|
|
|
|
|
def get_job(job_id: str) -> Optional[dict]:
|
|
with Session(engine) as session:
|
|
record = session.get(JobRecord, job_id)
|
|
if record is None:
|
|
return None
|
|
return _record_to_dict(record)
|
|
|
|
|
|
def list_jobs() -> list[dict]:
|
|
with Session(engine) as session:
|
|
records = session.query(JobRecord).order_by(JobRecord.created_at.desc()).all()
|
|
return [_record_to_dict(r) for r in records]
|
|
|
|
|
|
def list_all_jobs() -> list[dict]:
|
|
"""Return every job record regardless of type."""
|
|
return list_jobs()
|
|
|
|
|
|
def delete_job(job_id: str) -> bool:
|
|
"""Permanently delete a job record. Returns True if the record existed."""
|
|
with Session(engine) as session:
|
|
record = session.get(JobRecord, job_id)
|
|
if record is None:
|
|
return False
|
|
session.delete(record)
|
|
session.commit()
|
|
with _flags_lock:
|
|
_cancel_flags.pop(job_id, None)
|
|
return True
|
|
|
|
|
|
def update_job(job_id: str, **kwargs):
|
|
with Session(engine) as session:
|
|
record = session.get(JobRecord, job_id)
|
|
if record is None:
|
|
return
|
|
for key, value in kwargs.items():
|
|
setattr(record, key, value)
|
|
record.updated_at = datetime.utcnow()
|
|
session.commit()
|
|
|
|
|
|
def append_job_files(job_id: str, new_files: list):
|
|
"""Append file entries to an existing job's file list without overwriting."""
|
|
with Session(engine) as session:
|
|
record = session.get(JobRecord, job_id)
|
|
if record is None:
|
|
return
|
|
existing = list(record.files or [])
|
|
existing_names = {f.get("name") for f in existing}
|
|
for f in new_files:
|
|
if f.get("name") not in existing_names:
|
|
existing.append(f)
|
|
record.files = existing
|
|
record.updated_at = datetime.utcnow()
|
|
session.commit()
|
|
|
|
|
|
def cancel_job(job_id: str):
|
|
with _flags_lock:
|
|
flag = _cancel_flags.get(job_id)
|
|
if flag:
|
|
flag.set()
|
|
update_job(job_id, status=JobStatus.CANCELLED)
|
|
|
|
|
|
def should_cancel(job_id: str) -> bool:
|
|
with _flags_lock:
|
|
flag = _cancel_flags.get(job_id)
|
|
return flag.is_set() if flag else False
|
|
|
|
|
|
def submit(fn: Callable, *args, **kwargs):
|
|
"""Submit a CPU-bound background task (multi-worker)."""
|
|
_general_executor.submit(fn, *args, **kwargs)
|
|
|
|
|
|
def submit_gpu(fn: Callable, *args, **kwargs):
|
|
"""Submit a GPU-exclusive background task (single-worker FIFO queue).
|
|
|
|
Use for any job that loads PyTorch or TensorFlow models onto the GPU:
|
|
stem separation (Demucs / BS-RoFormer), MIDI transcription (Omnizart),
|
|
lyrics alignment (wav2vec2 + Demucs vocal extraction). Tasks run
|
|
strictly one-at-a-time in the order they were submitted.
|
|
"""
|
|
_gpu_executor.submit(fn, *args, **kwargs)
|
|
|
|
|
|
def _record_to_dict(record: JobRecord) -> dict:
|
|
return {
|
|
"id": record.id,
|
|
"job_type": record.job_type,
|
|
"status": record.status,
|
|
"progress": record.progress,
|
|
"error": record.error,
|
|
"files": record.files or [],
|
|
"result": record.result or {},
|
|
"created_at": record.created_at.isoformat() if record.created_at else None,
|
|
"updated_at": record.updated_at.isoformat() if record.updated_at else None,
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Project CRUD
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def find_or_create_project(name: str, sep_job_id: str, tab_label: str = '') -> dict:
|
|
"""Upsert a project record by sep_job_id. Safe to call multiple times."""
|
|
with Session(engine) as session:
|
|
record = session.query(ProjectRecord).filter_by(sep_job_id=sep_job_id).first()
|
|
if record is None:
|
|
record = ProjectRecord(
|
|
id=str(uuid4()),
|
|
name=name,
|
|
sep_job_id=sep_job_id,
|
|
tab_label=tab_label,
|
|
)
|
|
session.add(record)
|
|
session.commit()
|
|
return _project_to_dict(record)
|
|
|
|
|
|
def get_project(project_id: str) -> Optional[dict]:
|
|
with Session(engine) as session:
|
|
record = session.get(ProjectRecord, project_id)
|
|
return _project_to_dict(record) if record else None
|
|
|
|
|
|
def get_project_by_sep_job(sep_job_id: str) -> Optional[dict]:
|
|
with Session(engine) as session:
|
|
record = session.query(ProjectRecord).filter_by(sep_job_id=sep_job_id).first()
|
|
return _project_to_dict(record) if record else None
|
|
|
|
|
|
def update_project(project_id: str, **kwargs):
|
|
with Session(engine) as session:
|
|
record = session.get(ProjectRecord, project_id)
|
|
if record is None:
|
|
return
|
|
for key, value in kwargs.items():
|
|
setattr(record, key, value)
|
|
session.commit()
|
|
|
|
|
|
def list_projects() -> list:
|
|
"""
|
|
Return all projects newest-first. Also materialises any completed separation
|
|
jobs that don't yet have a project record (idempotent migration).
|
|
"""
|
|
with Session(engine) as session:
|
|
# Find all completed separation/pipeline jobs
|
|
completed_sep_jobs = (
|
|
session.query(JobRecord)
|
|
.filter(
|
|
JobRecord.job_type.in_(["separate", "pipeline"]),
|
|
JobRecord.status == "done",
|
|
)
|
|
.all()
|
|
)
|
|
|
|
existing_sep_ids = {
|
|
r.sep_job_id for r in session.query(ProjectRecord).all()
|
|
}
|
|
|
|
for job in completed_sep_jobs:
|
|
if job.id not in existing_sep_ids:
|
|
proj_name = _extract_project_name(job)
|
|
new_proj = ProjectRecord(
|
|
id=str(uuid4()),
|
|
name=proj_name,
|
|
sep_job_id=job.id,
|
|
created_at=job.created_at,
|
|
)
|
|
session.add(new_proj)
|
|
existing_sep_ids.add(job.id)
|
|
|
|
session.commit()
|
|
|
|
records = (
|
|
session.query(ProjectRecord)
|
|
.order_by(ProjectRecord.created_at.desc())
|
|
.all()
|
|
)
|
|
return [_project_to_dict(r) for r in records]
|
|
|
|
|
|
def _extract_project_name(job: JobRecord) -> str:
|
|
"""Derive a human-readable project name from a separation job record."""
|
|
# 1. Prefer result.project_name (set for jobs after result column was added)
|
|
if job.result and isinstance(job.result, dict):
|
|
name = job.result.get("project_name") or ""
|
|
if name:
|
|
return name
|
|
pp = job.result.get("project_path") or ""
|
|
if pp:
|
|
return Path(pp).name
|
|
|
|
# 2. Derive from stem file path structure: .../ProjectName/Stems/file.mp3
|
|
for f in (job.files or []):
|
|
file_path = f.get("path", "")
|
|
if not file_path:
|
|
continue
|
|
p = Path(file_path)
|
|
parts = p.parts
|
|
# Find the "Stems" component
|
|
for i, part in enumerate(parts):
|
|
if part == "Stems" and i > 0:
|
|
return parts[i - 1]
|
|
|
|
# 3. Derive from first file name (e.g. "input_stem - Vocals.mp3" → "input_stem")
|
|
if job.files:
|
|
first_name = (job.files[0] or {}).get("name", "")
|
|
stem = Path(first_name).stem # "input_stem - Vocals"
|
|
parts = stem.rsplit(" - ", 1)
|
|
if len(parts) == 2:
|
|
return parts[0]
|
|
return stem or job.id[:8]
|
|
|
|
return job.id[:8]
|
|
|
|
|
|
def _project_to_dict(record: ProjectRecord) -> dict:
|
|
return {
|
|
"id": record.id,
|
|
"name": record.name,
|
|
"tab_label": record.tab_label or '',
|
|
"created_at": record.created_at.isoformat() if record.created_at else None,
|
|
"sep_job_id": record.sep_job_id,
|
|
"trans_jobs": record.trans_jobs or {},
|
|
"abc_jobs": record.abc_jobs or {},
|
|
"lrc_jobs": record.lrc_jobs or {},
|
|
"mix_job_id": record.mix_job_id,
|
|
}
|
|
|
|
|
|
def set_project_trans_job(project_id: str, stem_name: str, job_id: str):
|
|
"""Store a per-stem transcription job_id in the project's trans_jobs dict."""
|
|
with Session(engine) as session:
|
|
record = session.get(ProjectRecord, project_id)
|
|
if record is None:
|
|
return
|
|
jobs = dict(record.trans_jobs or {})
|
|
jobs[stem_name] = job_id
|
|
record.trans_jobs = jobs
|
|
flag_modified(record, "trans_jobs")
|
|
session.commit()
|
|
|
|
|
|
def set_project_abc_job(project_id: str, stem_name: str, job_id: str):
|
|
"""Store a per-stem ABC job_id in the project's abc_jobs dict."""
|
|
with Session(engine) as session:
|
|
record = session.get(ProjectRecord, project_id)
|
|
if record is None:
|
|
return
|
|
jobs = dict(record.abc_jobs or {})
|
|
jobs[stem_name] = job_id
|
|
record.abc_jobs = jobs
|
|
flag_modified(record, "abc_jobs")
|
|
session.commit()
|
|
|
|
|
|
def set_project_lrc_job(project_id: str, stem_name: str, job_id: str):
|
|
"""Store a per-stem LRC alignment job_id in the project's lrc_jobs dict."""
|
|
with Session(engine) as session:
|
|
record = session.get(ProjectRecord, project_id)
|
|
if record is None:
|
|
return
|
|
jobs = dict(record.lrc_jobs or {})
|
|
jobs[stem_name] = job_id
|
|
record.lrc_jobs = jobs
|
|
flag_modified(record, "lrc_jobs")
|
|
session.commit()
|