137 lines
4.5 KiB
Python
Executable File
137 lines
4.5 KiB
Python
Executable File
"""System information endpoints (GPU hardware)."""
|
|
from pathlib import Path
|
|
|
|
from fastapi import APIRouter, Depends
|
|
|
|
from .. import job_manager as jm
|
|
from ..config import APP_ENV, DB_PATH, JOBS_DIR, MEDIA_DIR, PROJECTS_DIR, UPLOAD_DIR
|
|
from ..db import engine
|
|
from ..security import require_admin_token
|
|
|
|
router = APIRouter(prefix="/api/system", tags=["system"])
|
|
|
|
|
|
@router.get("/health")
|
|
def health():
|
|
"""Lightweight liveness probe."""
|
|
return {"status": "ok", "service": "stemengine", "environment": APP_ENV}
|
|
|
|
|
|
@router.get("/readiness")
|
|
def readiness():
|
|
"""Readiness probe for DB connectivity and writable core directories."""
|
|
checks = {
|
|
"db": _db_ready(),
|
|
"directories": _dirs_ready(),
|
|
}
|
|
ok = checks["db"]["ok"] and checks["directories"]["ok"]
|
|
return {"status": "ready" if ok else "not_ready", "ok": ok, "checks": checks}
|
|
|
|
|
|
@router.get("/executors")
|
|
def executors_state():
|
|
"""Queue-depth / worker visibility for background executors."""
|
|
return jm.get_executor_state()
|
|
|
|
|
|
@router.post("/gpu-reset")
|
|
def gpu_reset(_: None = Depends(require_admin_token)):
|
|
"""
|
|
Re-enable GPU acceleration for all services without restarting the container.
|
|
|
|
Resets:
|
|
- Lyric transcriber: clears _cuda_ok flag and Whisper model cache
|
|
- All services: gc.collect() + torch.cuda.empty_cache() to release
|
|
any stranded allocations left by separators or alignment models
|
|
|
|
NOTE: if the CUDA context is truly corrupted (persistent 'unknown error'),
|
|
a container restart is still required.
|
|
"""
|
|
import gc
|
|
|
|
# 1. Reset transcriber-specific CUDA state (_cuda_ok flag + Whisper cache).
|
|
transcriber_result: dict = {"ok": False, "error": None}
|
|
try:
|
|
from ..services.lyric_transcriber_service import reset_cuda
|
|
transcriber_result = reset_cuda()
|
|
except Exception as exc:
|
|
transcriber_result = {"ok": False, "error": str(exc)}
|
|
|
|
# 2. Global GPU flush — releases allocations from separators, alignment
|
|
# models, and any other PyTorch tensors still resident in VRAM.
|
|
cuda_available = False
|
|
flush_error: str | None = None
|
|
try:
|
|
import torch
|
|
cuda_available = torch.cuda.is_available()
|
|
gc.collect()
|
|
if cuda_available:
|
|
torch.cuda.empty_cache()
|
|
torch.cuda.reset_peak_memory_stats()
|
|
except Exception as exc:
|
|
flush_error = str(exc)
|
|
|
|
return {
|
|
"ok": transcriber_result.get("ok", False),
|
|
"cuda_available": cuda_available,
|
|
"cuda_ok": transcriber_result.get("cuda_ok", False),
|
|
"transcriber_reset": transcriber_result,
|
|
"flush_error": flush_error,
|
|
}
|
|
|
|
|
|
@router.get("/gpu-info")
|
|
def gpu_info():
|
|
"""Return GPU name, VRAM figures, and cuda_ok health flag."""
|
|
try:
|
|
from ..services.lyric_transcriber_service import _cuda_ok as whisper_ok
|
|
except Exception:
|
|
whisper_ok = True
|
|
|
|
try:
|
|
import torch # type: ignore
|
|
if not torch.cuda.is_available():
|
|
return {"cuda_available": False, "cuda_ok": whisper_ok, "name": None, "total_mb": 0, "free_mb": 0}
|
|
props = torch.cuda.get_device_properties(0)
|
|
free_bytes, total_bytes = torch.cuda.mem_get_info(0)
|
|
return {
|
|
"cuda_available": True,
|
|
"cuda_ok": whisper_ok,
|
|
"name": props.name,
|
|
"total_mb": total_bytes // (1024 * 1024),
|
|
"free_mb": free_bytes // (1024 * 1024),
|
|
}
|
|
except Exception as exc: # pragma: no cover
|
|
return {"cuda_available": False, "cuda_ok": False, "name": None, "total_mb": 0, "free_mb": 0, "error": str(exc)}
|
|
|
|
|
|
def _db_ready() -> dict:
|
|
try:
|
|
with engine.connect() as conn:
|
|
conn.exec_driver_sql("SELECT 1")
|
|
return {"ok": True, "path": str(DB_PATH)}
|
|
except Exception as exc:
|
|
return {"ok": False, "path": str(DB_PATH), "error": str(exc)}
|
|
|
|
|
|
def _dirs_ready() -> dict:
|
|
required: list[Path] = [MEDIA_DIR, UPLOAD_DIR, JOBS_DIR, PROJECTS_DIR]
|
|
details = {}
|
|
ok = True
|
|
|
|
for directory in required:
|
|
exists = directory.exists()
|
|
writable = exists and directory.is_dir()
|
|
if writable:
|
|
try:
|
|
probe = directory / ".write_probe"
|
|
probe.write_text("ok", encoding="utf-8")
|
|
probe.unlink(missing_ok=True)
|
|
except Exception:
|
|
writable = False
|
|
details[str(directory)] = {"exists": exists, "writable": writable}
|
|
if not (exists and writable):
|
|
ok = False
|
|
|
|
return {"ok": ok, "details": details}
|