Files
stemengine_old/backend/job_manager.py
T
2026-03-03 14:52:19 +09:00

402 lines
14 KiB
Python
Executable File

"""
Job manager: SQLite-backed job state + ThreadPoolExecutor for background tasks.
"""
import re
import uuid
import threading
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
from enum import Enum
from pathlib import Path
from typing import Callable, Optional
from uuid import uuid4
from sqlalchemy import create_engine, Column, String, DateTime, JSON, Text, text, event
from sqlalchemy.orm import DeclarativeBase, Session
from sqlalchemy.orm.attributes import flag_modified
from .config import DB_PATH
# ---------------------------------------------------------------------------
# DB setup
# ---------------------------------------------------------------------------
engine = create_engine(
f"sqlite:///{DB_PATH}",
connect_args={"check_same_thread": False, "timeout": 30},
)
@event.listens_for(engine, "connect")
def _set_sqlite_pragmas(dbapi_connection, _connection_record):
"""Enable WAL mode and set busy timeout so concurrent writes don't deadlock."""
cursor = dbapi_connection.cursor()
cursor.execute("PRAGMA journal_mode=WAL")
cursor.execute("PRAGMA synchronous=NORMAL")
cursor.execute("PRAGMA busy_timeout=30000")
cursor.close()
class Base(DeclarativeBase):
pass
class JobRecord(Base):
__tablename__ = "jobs"
id = Column(String, primary_key=True)
job_type = Column(String, nullable=False) # separate | transcribe | abc-convert | pipeline | mix
status = Column(String, default="queued") # queued | running | done | error | cancelled
progress = Column(String, default="")
error = Column(Text, default="")
files = Column(JSON, default=list) # list of {name, path} dicts
result = Column(JSON, default=dict) # arbitrary result metadata
created_at = Column(DateTime, default=datetime.utcnow)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
class ProjectRecord(Base):
__tablename__ = "projects"
id = Column(String, primary_key=True, default=lambda: str(uuid4()))
name = Column(String, nullable=False)
tab_label = Column(String, default='') # display label for tab e.g. "Demucs v4_MP3 (256 kbps)"
created_at = Column(DateTime, default=datetime.utcnow)
sep_job_id = Column(String, nullable=False)
trans_jobs = Column(JSON, default=dict) # stem_name → job_id
abc_jobs = Column(JSON, default=dict) # stem_name → job_id
lrc_jobs = Column(JSON, default=dict) # stem_name → job_id
mix_job_id = Column(String, nullable=True)
Base.metadata.create_all(engine)
# Idempotent migrations: add missing columns to existing tables
for _migration_sql in (
"ALTER TABLE jobs ADD COLUMN result JSON",
"ALTER TABLE projects ADD COLUMN trans_jobs JSON",
"ALTER TABLE projects ADD COLUMN abc_jobs JSON",
"ALTER TABLE projects ADD COLUMN lrc_jobs JSON",
"ALTER TABLE projects ADD COLUMN tab_label TEXT DEFAULT ''",
):
try:
with engine.connect() as _conn:
_conn.execute(text(_migration_sql))
_conn.commit()
except Exception:
pass # Column already exists
# ---------------------------------------------------------------------------
# Cancellation flags: job_id → threading.Event (set = cancel requested)
# ---------------------------------------------------------------------------
_cancel_flags: dict[str, threading.Event] = {}
_flags_lock = threading.Lock()
# ---------------------------------------------------------------------------
# Thread pools
# Two executors:
# _general_executor — multi-worker for CPU-bound tasks (e.g. ABC conversion)
# _gpu_executor — single-worker for GPU-exclusive tasks (PyTorch + TF)
# serialises Demucs/BS-RoFormer and Omnizart so they
# never compete for VRAM; FIFO by submission order.
# ---------------------------------------------------------------------------
_general_executor = ThreadPoolExecutor(max_workers=4)
_gpu_executor = ThreadPoolExecutor(max_workers=1)
# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------
class JobStatus(str, Enum):
QUEUED = "queued"
RUNNING = "running"
DONE = "done"
ERROR = "error"
CANCELLED = "cancelled"
def create_job(job_type: str) -> str:
job_id = str(uuid.uuid4())
with Session(engine) as session:
session.add(JobRecord(id=job_id, job_type=job_type))
session.commit()
with _flags_lock:
_cancel_flags[job_id] = threading.Event()
return job_id
def get_job(job_id: str) -> Optional[dict]:
with Session(engine) as session:
record = session.get(JobRecord, job_id)
if record is None:
return None
return _record_to_dict(record)
def list_jobs() -> list[dict]:
with Session(engine) as session:
records = session.query(JobRecord).order_by(JobRecord.created_at.desc()).all()
return [_record_to_dict(r) for r in records]
def list_all_jobs() -> list[dict]:
"""Return every job record regardless of type."""
return list_jobs()
def delete_job(job_id: str) -> bool:
"""Permanently delete a job record. Returns True if the record existed."""
with Session(engine) as session:
record = session.get(JobRecord, job_id)
if record is None:
return False
session.delete(record)
session.commit()
with _flags_lock:
_cancel_flags.pop(job_id, None)
return True
def update_job(job_id: str, **kwargs):
with Session(engine) as session:
record = session.get(JobRecord, job_id)
if record is None:
return
for key, value in kwargs.items():
setattr(record, key, value)
record.updated_at = datetime.utcnow()
session.commit()
def append_job_files(job_id: str, new_files: list):
"""Append file entries to an existing job's file list without overwriting."""
with Session(engine) as session:
record = session.get(JobRecord, job_id)
if record is None:
return
existing = list(record.files or [])
existing_names = {f.get("name") for f in existing}
for f in new_files:
if f.get("name") not in existing_names:
existing.append(f)
record.files = existing
record.updated_at = datetime.utcnow()
session.commit()
def cancel_job(job_id: str):
with _flags_lock:
flag = _cancel_flags.get(job_id)
if flag:
flag.set()
update_job(job_id, status=JobStatus.CANCELLED)
def should_cancel(job_id: str) -> bool:
with _flags_lock:
flag = _cancel_flags.get(job_id)
return flag.is_set() if flag else False
def submit(fn: Callable, *args, **kwargs):
"""Submit a CPU-bound background task (multi-worker)."""
_general_executor.submit(fn, *args, **kwargs)
def submit_gpu(fn: Callable, *args, **kwargs):
"""Submit a GPU-exclusive background task (single-worker FIFO queue).
Use for any job that loads PyTorch or TensorFlow models onto the GPU:
stem separation (Demucs / BS-RoFormer), MIDI transcription (Omnizart),
lyrics alignment (wav2vec2 + Demucs vocal extraction). Tasks run
strictly one-at-a-time in the order they were submitted.
"""
_gpu_executor.submit(fn, *args, **kwargs)
def _record_to_dict(record: JobRecord) -> dict:
return {
"id": record.id,
"job_type": record.job_type,
"status": record.status,
"progress": record.progress,
"error": record.error,
"files": record.files or [],
"result": record.result or {},
"created_at": record.created_at.isoformat() if record.created_at else None,
"updated_at": record.updated_at.isoformat() if record.updated_at else None,
}
# ---------------------------------------------------------------------------
# Project CRUD
# ---------------------------------------------------------------------------
def find_or_create_project(name: str, sep_job_id: str, tab_label: str = '') -> dict:
"""Upsert a project record by sep_job_id. Safe to call multiple times."""
with Session(engine) as session:
record = session.query(ProjectRecord).filter_by(sep_job_id=sep_job_id).first()
if record is None:
record = ProjectRecord(
id=str(uuid4()),
name=name,
sep_job_id=sep_job_id,
tab_label=tab_label,
)
session.add(record)
session.commit()
return _project_to_dict(record)
def get_project(project_id: str) -> Optional[dict]:
with Session(engine) as session:
record = session.get(ProjectRecord, project_id)
return _project_to_dict(record) if record else None
def get_project_by_sep_job(sep_job_id: str) -> Optional[dict]:
with Session(engine) as session:
record = session.query(ProjectRecord).filter_by(sep_job_id=sep_job_id).first()
return _project_to_dict(record) if record else None
def update_project(project_id: str, **kwargs):
with Session(engine) as session:
record = session.get(ProjectRecord, project_id)
if record is None:
return
for key, value in kwargs.items():
setattr(record, key, value)
session.commit()
def list_projects() -> list:
"""
Return all projects newest-first. Also materialises any completed separation
jobs that don't yet have a project record (idempotent migration).
"""
with Session(engine) as session:
# Find all completed separation/pipeline jobs
completed_sep_jobs = (
session.query(JobRecord)
.filter(
JobRecord.job_type.in_(["separate", "pipeline"]),
JobRecord.status == "done",
)
.all()
)
existing_sep_ids = {
r.sep_job_id for r in session.query(ProjectRecord).all()
}
for job in completed_sep_jobs:
if job.id not in existing_sep_ids:
proj_name = _extract_project_name(job)
new_proj = ProjectRecord(
id=str(uuid4()),
name=proj_name,
sep_job_id=job.id,
created_at=job.created_at,
)
session.add(new_proj)
existing_sep_ids.add(job.id)
session.commit()
records = (
session.query(ProjectRecord)
.order_by(ProjectRecord.created_at.desc())
.all()
)
return [_project_to_dict(r) for r in records]
def _extract_project_name(job: JobRecord) -> str:
"""Derive a human-readable project name from a separation job record."""
# 1. Prefer result.project_name (set for jobs after result column was added)
if job.result and isinstance(job.result, dict):
name = job.result.get("project_name") or ""
if name:
return name
pp = job.result.get("project_path") or ""
if pp:
return Path(pp).name
# 2. Derive from stem file path structure: .../ProjectName/Stems/file.mp3
for f in (job.files or []):
file_path = f.get("path", "")
if not file_path:
continue
p = Path(file_path)
parts = p.parts
# Find the "Stems" component
for i, part in enumerate(parts):
if part == "Stems" and i > 0:
return parts[i - 1]
# 3. Derive from first file name (e.g. "input_stem - Vocals.mp3" → "input_stem")
if job.files:
first_name = (job.files[0] or {}).get("name", "")
stem = Path(first_name).stem # "input_stem - Vocals"
parts = stem.rsplit(" - ", 1)
if len(parts) == 2:
return parts[0]
return stem or job.id[:8]
return job.id[:8]
def _project_to_dict(record: ProjectRecord) -> dict:
return {
"id": record.id,
"name": record.name,
"tab_label": record.tab_label or '',
"created_at": record.created_at.isoformat() if record.created_at else None,
"sep_job_id": record.sep_job_id,
"trans_jobs": record.trans_jobs or {},
"abc_jobs": record.abc_jobs or {},
"lrc_jobs": record.lrc_jobs or {},
"mix_job_id": record.mix_job_id,
}
def set_project_trans_job(project_id: str, stem_name: str, job_id: str):
"""Store a per-stem transcription job_id in the project's trans_jobs dict."""
with Session(engine) as session:
record = session.get(ProjectRecord, project_id)
if record is None:
return
jobs = dict(record.trans_jobs or {})
jobs[stem_name] = job_id
record.trans_jobs = jobs
flag_modified(record, "trans_jobs")
session.commit()
def set_project_abc_job(project_id: str, stem_name: str, job_id: str):
"""Store a per-stem ABC job_id in the project's abc_jobs dict."""
with Session(engine) as session:
record = session.get(ProjectRecord, project_id)
if record is None:
return
jobs = dict(record.abc_jobs or {})
jobs[stem_name] = job_id
record.abc_jobs = jobs
flag_modified(record, "abc_jobs")
session.commit()
def set_project_lrc_job(project_id: str, stem_name: str, job_id: str):
"""Store a per-stem LRC alignment job_id in the project's lrc_jobs dict."""
with Session(engine) as session:
record = session.get(ProjectRecord, project_id)
if record is None:
return
jobs = dict(record.lrc_jobs or {})
jobs[stem_name] = job_id
record.lrc_jobs = jobs
flag_modified(record, "lrc_jobs")
session.commit()