Files
stemengine_old/backend/routers/system.py
T

137 lines
4.5 KiB
Python
Executable File

"""System information endpoints (GPU hardware)."""
from pathlib import Path
from fastapi import APIRouter, Depends
from .. import job_manager as jm
from ..config import APP_ENV, DB_PATH, JOBS_DIR, MEDIA_DIR, PROJECTS_DIR, UPLOAD_DIR
from ..db import engine
from ..security import require_admin_token
router = APIRouter(prefix="/api/system", tags=["system"])
@router.get("/health")
def health():
"""Lightweight liveness probe."""
return {"status": "ok", "service": "stemengine", "environment": APP_ENV}
@router.get("/readiness")
def readiness():
"""Readiness probe for DB connectivity and writable core directories."""
checks = {
"db": _db_ready(),
"directories": _dirs_ready(),
}
ok = checks["db"]["ok"] and checks["directories"]["ok"]
return {"status": "ready" if ok else "not_ready", "ok": ok, "checks": checks}
@router.get("/executors")
def executors_state():
"""Queue-depth / worker visibility for background executors."""
return jm.get_executor_state()
@router.post("/gpu-reset")
def gpu_reset(_: None = Depends(require_admin_token)):
"""
Re-enable GPU acceleration for all services without restarting the container.
Resets:
- Lyric transcriber: clears _cuda_ok flag and Whisper model cache
- All services: gc.collect() + torch.cuda.empty_cache() to release
any stranded allocations left by separators or alignment models
NOTE: if the CUDA context is truly corrupted (persistent 'unknown error'),
a container restart is still required.
"""
import gc
# 1. Reset transcriber-specific CUDA state (_cuda_ok flag + Whisper cache).
transcriber_result: dict = {"ok": False, "error": None}
try:
from ..services.lyric_transcriber_service import reset_cuda
transcriber_result = reset_cuda()
except Exception as exc:
transcriber_result = {"ok": False, "error": str(exc)}
# 2. Global GPU flush — releases allocations from separators, alignment
# models, and any other PyTorch tensors still resident in VRAM.
cuda_available = False
flush_error: str | None = None
try:
import torch
cuda_available = torch.cuda.is_available()
gc.collect()
if cuda_available:
torch.cuda.empty_cache()
torch.cuda.reset_peak_memory_stats()
except Exception as exc:
flush_error = str(exc)
return {
"ok": transcriber_result.get("ok", False),
"cuda_available": cuda_available,
"cuda_ok": transcriber_result.get("cuda_ok", False),
"transcriber_reset": transcriber_result,
"flush_error": flush_error,
}
@router.get("/gpu-info")
def gpu_info():
"""Return GPU name, VRAM figures, and cuda_ok health flag."""
try:
from ..services.lyric_transcriber_service import _cuda_ok as whisper_ok
except Exception:
whisper_ok = True
try:
import torch # type: ignore
if not torch.cuda.is_available():
return {"cuda_available": False, "cuda_ok": whisper_ok, "name": None, "total_mb": 0, "free_mb": 0}
props = torch.cuda.get_device_properties(0)
free_bytes, total_bytes = torch.cuda.mem_get_info(0)
return {
"cuda_available": True,
"cuda_ok": whisper_ok,
"name": props.name,
"total_mb": total_bytes // (1024 * 1024),
"free_mb": free_bytes // (1024 * 1024),
}
except Exception as exc: # pragma: no cover
return {"cuda_available": False, "cuda_ok": False, "name": None, "total_mb": 0, "free_mb": 0, "error": str(exc)}
def _db_ready() -> dict:
try:
with engine.connect() as conn:
conn.exec_driver_sql("SELECT 1")
return {"ok": True, "path": str(DB_PATH)}
except Exception as exc:
return {"ok": False, "path": str(DB_PATH), "error": str(exc)}
def _dirs_ready() -> dict:
required: list[Path] = [MEDIA_DIR, UPLOAD_DIR, JOBS_DIR, PROJECTS_DIR]
details = {}
ok = True
for directory in required:
exists = directory.exists()
writable = exists and directory.is_dir()
if writable:
try:
probe = directory / ".write_probe"
probe.write_text("ok", encoding="utf-8")
probe.unlink(missing_ok=True)
except Exception:
writable = False
details[str(directory)] = {"exists": exists, "writable": writable}
if not (exists and writable):
ok = False
return {"ok": ok, "details": details}