""" Job manager: SQLite-backed job state + ThreadPoolExecutor for background tasks. """ import re import uuid import threading from concurrent.futures import ThreadPoolExecutor from datetime import datetime from enum import Enum from pathlib import Path from typing import Callable, Optional from uuid import uuid4 from sqlalchemy import create_engine, Column, String, DateTime, JSON, Text, text, event from sqlalchemy.orm import DeclarativeBase, Session from sqlalchemy.orm.attributes import flag_modified from .config import DB_PATH # --------------------------------------------------------------------------- # DB setup # --------------------------------------------------------------------------- engine = create_engine( f"sqlite:///{DB_PATH}", connect_args={"check_same_thread": False, "timeout": 30}, ) @event.listens_for(engine, "connect") def _set_sqlite_pragmas(dbapi_connection, _connection_record): """Enable WAL mode and set busy timeout so concurrent writes don't deadlock.""" cursor = dbapi_connection.cursor() cursor.execute("PRAGMA journal_mode=WAL") cursor.execute("PRAGMA synchronous=NORMAL") cursor.execute("PRAGMA busy_timeout=30000") cursor.close() class Base(DeclarativeBase): pass class JobRecord(Base): __tablename__ = "jobs" id = Column(String, primary_key=True) job_type = Column(String, nullable=False) # separate | transcribe | abc-convert | pipeline | mix status = Column(String, default="queued") # queued | running | done | error | cancelled progress = Column(String, default="") error = Column(Text, default="") files = Column(JSON, default=list) # list of {name, path} dicts result = Column(JSON, default=dict) # arbitrary result metadata created_at = Column(DateTime, default=datetime.utcnow) updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) class ProjectRecord(Base): __tablename__ = "projects" id = Column(String, primary_key=True, default=lambda: str(uuid4())) name = Column(String, nullable=False) tab_label = Column(String, default='') # display label for tab e.g. "Demucs v4_MP3 (256 kbps)" created_at = Column(DateTime, default=datetime.utcnow) sep_job_id = Column(String, nullable=False) trans_jobs = Column(JSON, default=dict) # stem_name → job_id abc_jobs = Column(JSON, default=dict) # stem_name → job_id lrc_jobs = Column(JSON, default=dict) # stem_name → job_id mix_job_id = Column(String, nullable=True) Base.metadata.create_all(engine) # Idempotent migrations: add missing columns to existing tables for _migration_sql in ( "ALTER TABLE jobs ADD COLUMN result JSON", "ALTER TABLE projects ADD COLUMN trans_jobs JSON", "ALTER TABLE projects ADD COLUMN abc_jobs JSON", "ALTER TABLE projects ADD COLUMN lrc_jobs JSON", "ALTER TABLE projects ADD COLUMN tab_label TEXT DEFAULT ''", ): try: with engine.connect() as _conn: _conn.execute(text(_migration_sql)) _conn.commit() except Exception: pass # Column already exists # --------------------------------------------------------------------------- # Cancellation flags: job_id → threading.Event (set = cancel requested) # --------------------------------------------------------------------------- _cancel_flags: dict[str, threading.Event] = {} _flags_lock = threading.Lock() # --------------------------------------------------------------------------- # Thread pools # Two executors: # _general_executor — multi-worker for CPU-bound tasks (e.g. ABC conversion) # _gpu_executor — single-worker for GPU-exclusive tasks (PyTorch + TF) # serialises Demucs/BS-RoFormer and Omnizart so they # never compete for VRAM; FIFO by submission order. # --------------------------------------------------------------------------- _general_executor = ThreadPoolExecutor(max_workers=4) _gpu_executor = ThreadPoolExecutor(max_workers=1) # --------------------------------------------------------------------------- # Public API # --------------------------------------------------------------------------- class JobStatus(str, Enum): QUEUED = "queued" RUNNING = "running" DONE = "done" ERROR = "error" CANCELLED = "cancelled" def create_job(job_type: str) -> str: job_id = str(uuid.uuid4()) with Session(engine) as session: session.add(JobRecord(id=job_id, job_type=job_type)) session.commit() with _flags_lock: _cancel_flags[job_id] = threading.Event() return job_id def get_job(job_id: str) -> Optional[dict]: with Session(engine) as session: record = session.get(JobRecord, job_id) if record is None: return None return _record_to_dict(record) def list_jobs() -> list[dict]: with Session(engine) as session: records = session.query(JobRecord).order_by(JobRecord.created_at.desc()).all() return [_record_to_dict(r) for r in records] def list_all_jobs() -> list[dict]: """Return every job record regardless of type.""" return list_jobs() def delete_job(job_id: str) -> bool: """Permanently delete a job record. Returns True if the record existed.""" with Session(engine) as session: record = session.get(JobRecord, job_id) if record is None: return False session.delete(record) session.commit() with _flags_lock: _cancel_flags.pop(job_id, None) return True def update_job(job_id: str, **kwargs): with Session(engine) as session: record = session.get(JobRecord, job_id) if record is None: return for key, value in kwargs.items(): setattr(record, key, value) record.updated_at = datetime.utcnow() session.commit() def append_job_files(job_id: str, new_files: list): """Append file entries to an existing job's file list without overwriting.""" with Session(engine) as session: record = session.get(JobRecord, job_id) if record is None: return existing = list(record.files or []) existing_names = {f.get("name") for f in existing} for f in new_files: if f.get("name") not in existing_names: existing.append(f) record.files = existing record.updated_at = datetime.utcnow() session.commit() def cancel_job(job_id: str): with _flags_lock: flag = _cancel_flags.get(job_id) if flag: flag.set() update_job(job_id, status=JobStatus.CANCELLED) def should_cancel(job_id: str) -> bool: with _flags_lock: flag = _cancel_flags.get(job_id) return flag.is_set() if flag else False def submit(fn: Callable, *args, **kwargs): """Submit a CPU-bound background task (multi-worker).""" _general_executor.submit(fn, *args, **kwargs) def submit_gpu(fn: Callable, *args, **kwargs): """Submit a GPU-exclusive background task (single-worker FIFO queue). Use for any job that loads PyTorch or TensorFlow models onto the GPU: stem separation (Demucs / BS-RoFormer), MIDI transcription (Omnizart), lyrics alignment (wav2vec2 + Demucs vocal extraction). Tasks run strictly one-at-a-time in the order they were submitted. """ _gpu_executor.submit(fn, *args, **kwargs) def _record_to_dict(record: JobRecord) -> dict: return { "id": record.id, "job_type": record.job_type, "status": record.status, "progress": record.progress, "error": record.error, "files": record.files or [], "result": record.result or {}, "created_at": record.created_at.isoformat() if record.created_at else None, "updated_at": record.updated_at.isoformat() if record.updated_at else None, } # --------------------------------------------------------------------------- # Project CRUD # --------------------------------------------------------------------------- def find_or_create_project(name: str, sep_job_id: str, tab_label: str = '') -> dict: """Upsert a project record by sep_job_id. Safe to call multiple times.""" with Session(engine) as session: record = session.query(ProjectRecord).filter_by(sep_job_id=sep_job_id).first() if record is None: record = ProjectRecord( id=str(uuid4()), name=name, sep_job_id=sep_job_id, tab_label=tab_label, ) session.add(record) session.commit() return _project_to_dict(record) def get_project(project_id: str) -> Optional[dict]: with Session(engine) as session: record = session.get(ProjectRecord, project_id) return _project_to_dict(record) if record else None def get_project_by_sep_job(sep_job_id: str) -> Optional[dict]: with Session(engine) as session: record = session.query(ProjectRecord).filter_by(sep_job_id=sep_job_id).first() return _project_to_dict(record) if record else None def update_project(project_id: str, **kwargs): with Session(engine) as session: record = session.get(ProjectRecord, project_id) if record is None: return for key, value in kwargs.items(): setattr(record, key, value) session.commit() def list_projects() -> list: """ Return all projects newest-first. Also materialises any completed separation jobs that don't yet have a project record (idempotent migration). """ with Session(engine) as session: # Find all completed separation/pipeline jobs completed_sep_jobs = ( session.query(JobRecord) .filter( JobRecord.job_type.in_(["separate", "pipeline"]), JobRecord.status == "done", ) .all() ) existing_sep_ids = { r.sep_job_id for r in session.query(ProjectRecord).all() } for job in completed_sep_jobs: if job.id not in existing_sep_ids: proj_name = _extract_project_name(job) new_proj = ProjectRecord( id=str(uuid4()), name=proj_name, sep_job_id=job.id, created_at=job.created_at, ) session.add(new_proj) existing_sep_ids.add(job.id) session.commit() records = ( session.query(ProjectRecord) .order_by(ProjectRecord.created_at.desc()) .all() ) return [_project_to_dict(r) for r in records] def _extract_project_name(job: JobRecord) -> str: """Derive a human-readable project name from a separation job record.""" # 1. Prefer result.project_name (set for jobs after result column was added) if job.result and isinstance(job.result, dict): name = job.result.get("project_name") or "" if name: return name pp = job.result.get("project_path") or "" if pp: return Path(pp).name # 2. Derive from stem file path structure: .../ProjectName/Stems/file.mp3 for f in (job.files or []): file_path = f.get("path", "") if not file_path: continue p = Path(file_path) parts = p.parts # Find the "Stems" component for i, part in enumerate(parts): if part == "Stems" and i > 0: return parts[i - 1] # 3. Derive from first file name (e.g. "input_stem - Vocals.mp3" → "input_stem") if job.files: first_name = (job.files[0] or {}).get("name", "") stem = Path(first_name).stem # "input_stem - Vocals" parts = stem.rsplit(" - ", 1) if len(parts) == 2: return parts[0] return stem or job.id[:8] return job.id[:8] def _project_to_dict(record: ProjectRecord) -> dict: return { "id": record.id, "name": record.name, "tab_label": record.tab_label or '', "created_at": record.created_at.isoformat() if record.created_at else None, "sep_job_id": record.sep_job_id, "trans_jobs": record.trans_jobs or {}, "abc_jobs": record.abc_jobs or {}, "lrc_jobs": record.lrc_jobs or {}, "mix_job_id": record.mix_job_id, } def set_project_trans_job(project_id: str, stem_name: str, job_id: str): """Store a per-stem transcription job_id in the project's trans_jobs dict.""" with Session(engine) as session: record = session.get(ProjectRecord, project_id) if record is None: return jobs = dict(record.trans_jobs or {}) jobs[stem_name] = job_id record.trans_jobs = jobs flag_modified(record, "trans_jobs") session.commit() def set_project_abc_job(project_id: str, stem_name: str, job_id: str): """Store a per-stem ABC job_id in the project's abc_jobs dict.""" with Session(engine) as session: record = session.get(ProjectRecord, project_id) if record is None: return jobs = dict(record.abc_jobs or {}) jobs[stem_name] = job_id record.abc_jobs = jobs flag_modified(record, "abc_jobs") session.commit() def set_project_lrc_job(project_id: str, stem_name: str, job_id: str): """Store a per-stem LRC alignment job_id in the project's lrc_jobs dict.""" with Session(engine) as session: record = session.get(ProjectRecord, project_id) if record is None: return jobs = dict(record.lrc_jobs or {}) jobs[stem_name] = job_id record.lrc_jobs = jobs flag_modified(record, "lrc_jobs") session.commit()