302 lines
10 KiB
Python
302 lines
10 KiB
Python
"""
|
|
Omnizart Sidecar — minimal FastAPI HTTP wrapper around the Omnizart CLI.
|
|
|
|
Single endpoint: POST /transcribe
|
|
Request (multipart/form-data):
|
|
file — audio file (any format; converted to WAV internally)
|
|
mode — omnizart mode: music | vocal | vocal-contour
|
|
Response 200: raw MIDI bytes (Content-Type: audio/midi)
|
|
Response 4xx: JSON {"detail": "<message>"}
|
|
|
|
Environment variables:
|
|
OMNIZART_BIN — binary name or full path (default: "omnizart")
|
|
HOST — bind address (default: "0.0.0.0")
|
|
PORT — bind port (default: "9765")
|
|
|
|
Run:
|
|
uvicorn app:app --host 0.0.0.0 --port 9765
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
import shlex
|
|
import subprocess
|
|
import tempfile
|
|
from pathlib import Path
|
|
import shutil
|
|
|
|
from fastapi import FastAPI, File, Form, HTTPException, UploadFile
|
|
from fastapi.responses import Response
|
|
|
|
log = logging.getLogger("omnizart-sidecar")
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
|
|
|
OMNIZART_BIN: str = os.environ.get("OMNIZART_BIN", "omnizart")
|
|
MIN_DURATION_MS: int = 10_000
|
|
OMNIZART_OUTPUT_ROOT: Path = Path(os.environ.get("OMNIZART_OUTPUT_ROOT", "/root/.omnizart"))
|
|
OMNIZART_PROJECT_ROOT: Path = OMNIZART_OUTPUT_ROOT / "omnizart"
|
|
OMNIZART_CACHE_CHECKPOINT_ROOT: Path = OMNIZART_PROJECT_ROOT / "checkpoints"
|
|
|
|
# Keep this much VRAM free for other GPU consumers (WSL2 DWM/WDDM, etc.).
|
|
# Sidecar-specific override takes precedence; falls back to global reserve.
|
|
_OMNIZART_GPU_VRAM_RESERVE_MB: int = int(
|
|
os.environ.get(
|
|
"OMNIZART_GPU_VRAM_RESERVE_MB",
|
|
os.environ.get("STEMENGINE_GPU_VRAM_RESERVE_MB", "2048"),
|
|
)
|
|
)
|
|
|
|
|
|
def _query_total_vram_mb() -> int | None:
|
|
"""Return GPU0 total VRAM in MB using nvidia-smi, or None if unavailable."""
|
|
try:
|
|
out = subprocess.check_output(
|
|
["nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits"],
|
|
stderr=subprocess.DEVNULL,
|
|
text=True,
|
|
)
|
|
first = (out.strip().splitlines() or [""])[0].strip()
|
|
value = int(first)
|
|
return value if value > 0 else None
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _apply_tensorflow_gpu_limit() -> None:
|
|
"""Cap TensorFlow GPU usage for Omnizart so reserve headroom remains free."""
|
|
if _OMNIZART_GPU_VRAM_RESERVE_MB <= 0:
|
|
log.info("OMNIZART_GPU_VRAM_RESERVE_MB <= 0, TensorFlow GPU cap disabled")
|
|
return
|
|
|
|
try:
|
|
import tensorflow as tf # type: ignore[import]
|
|
|
|
gpus = tf.config.list_physical_devices("GPU")
|
|
if not gpus:
|
|
return
|
|
|
|
total_mb = _query_total_vram_mb()
|
|
if total_mb is None:
|
|
# Fallback when total VRAM cannot be queried.
|
|
for gpu in gpus:
|
|
tf.config.experimental.set_memory_growth(gpu, True)
|
|
log.info("Omnizart TF limiter fallback: enabled memory growth (total VRAM unknown)")
|
|
return
|
|
|
|
limit_mb = max(256, total_mb - _OMNIZART_GPU_VRAM_RESERVE_MB)
|
|
if limit_mb >= total_mb:
|
|
for gpu in gpus:
|
|
tf.config.experimental.set_memory_growth(gpu, True)
|
|
log.info(
|
|
"Omnizart TF limiter fallback: reserve too small for hard cap "
|
|
"(reserve=%dMB total=%dMB); using memory growth",
|
|
_OMNIZART_GPU_VRAM_RESERVE_MB,
|
|
total_mb,
|
|
)
|
|
return
|
|
|
|
for gpu in gpus:
|
|
tf.config.set_logical_device_configuration(
|
|
gpu,
|
|
[tf.config.LogicalDeviceConfiguration(memory_limit=limit_mb)],
|
|
)
|
|
|
|
log.info(
|
|
"Omnizart TF limiter active: total=%dMB reserve=%dMB cap=%dMB",
|
|
total_mb,
|
|
_OMNIZART_GPU_VRAM_RESERVE_MB,
|
|
limit_mb,
|
|
)
|
|
except RuntimeError as exc:
|
|
# Happens if TF runtime already initialized before configuration.
|
|
log.warning("Omnizart TF limiter not applied (runtime already initialized): %s", exc)
|
|
except Exception as exc:
|
|
log.warning("Omnizart TF limiter setup failed: %s", exc)
|
|
|
|
|
|
_apply_tensorflow_gpu_limit()
|
|
|
|
import omnizart
|
|
OMNIZART_BUILTIN_CHECKPOINT_ROOT: Path = Path(omnizart.__file__).resolve().parent / "checkpoints"
|
|
|
|
SUPPORTED_MODES = {"music", "vocal", "vocal-contour"}
|
|
CACHE_MODEL_PATHS = {
|
|
"music": OMNIZART_CACHE_CHECKPOINT_ROOT / "music" / "music_piano",
|
|
"vocal": OMNIZART_CACHE_CHECKPOINT_ROOT / "vocal" / "vocal_semi",
|
|
"vocal-contour": OMNIZART_CACHE_CHECKPOINT_ROOT / "vocal" / "vocal_contour",
|
|
}
|
|
BUILTIN_MODEL_PATHS = {
|
|
"music": OMNIZART_BUILTIN_CHECKPOINT_ROOT / "music" / "music_piano",
|
|
"vocal": OMNIZART_BUILTIN_CHECKPOINT_ROOT / "vocal" / "vocal_semi",
|
|
"vocal-contour": OMNIZART_BUILTIN_CHECKPOINT_ROOT / "vocal" / "vocal_contour",
|
|
}
|
|
|
|
app = FastAPI(title="Omnizart Sidecar", version="1.0")
|
|
|
|
|
|
@app.get("/health")
|
|
def health():
|
|
return {"status": "ok"}
|
|
|
|
|
|
@app.post("/transcribe")
|
|
async def transcribe(
|
|
file: UploadFile = File(...),
|
|
mode: str = Form("music"),
|
|
) -> Response:
|
|
if mode not in SUPPORTED_MODES:
|
|
raise HTTPException(status_code=400, detail=f"Unsupported mode: {mode!r}. Choose from: {sorted(SUPPORTED_MODES)}")
|
|
|
|
_ensure_checkpoints(mode)
|
|
|
|
audio_bytes = await file.read()
|
|
stem = Path(file.filename or "audio").stem
|
|
mid_name = f"{stem}-{mode}.mid"
|
|
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
tmp_path = Path(tmp)
|
|
src_path = tmp_path / (file.filename or "audio")
|
|
wav_path = tmp_path / f"{stem}-{mode}.wav"
|
|
|
|
# Write uploaded file
|
|
src_path.write_bytes(audio_bytes)
|
|
|
|
# Convert to WAV with minimum-length padding
|
|
_convert_to_wav(src_path, wav_path)
|
|
|
|
# Build Omnizart command
|
|
model_path = BUILTIN_MODEL_PATHS.get(mode)
|
|
model_arg = f' --model-path "{model_path}"' if model_path and model_path.exists() else ""
|
|
|
|
if mode == "vocal":
|
|
# vocal mode ignores -o; output lands in CWD as {stem}-vocal.mid
|
|
expected_output = tmp_path / mid_name
|
|
cmd = f'{OMNIZART_BIN} vocal transcribe{model_arg} "{wav_path}"'
|
|
else:
|
|
expected_output = tmp_path / mid_name
|
|
cmd = f'{OMNIZART_BIN} {mode} transcribe{model_arg} -o "{expected_output}" "{wav_path}"'
|
|
|
|
log.info("running: %s", cmd)
|
|
result = subprocess.run(
|
|
shlex.split(cmd),
|
|
cwd=str(tmp_path),
|
|
capture_output=True,
|
|
)
|
|
|
|
if result.returncode != 0:
|
|
stderr = result.stderr.decode(errors="replace")
|
|
log.error("omnizart failed (code %d): %s", result.returncode, stderr)
|
|
raise HTTPException(
|
|
status_code=500,
|
|
detail=f"Omnizart exited with code {result.returncode}. See sidecar logs.",
|
|
)
|
|
|
|
if not expected_output.exists():
|
|
raise HTTPException(
|
|
status_code=500,
|
|
detail=(
|
|
f"Omnizart produced no output at {expected_output.name}. "
|
|
"Input may be too short or the model failed silently."
|
|
),
|
|
)
|
|
|
|
midi_bytes = expected_output.read_bytes()
|
|
|
|
log.info("transcription complete: %s (%d bytes)", mid_name, len(midi_bytes))
|
|
return Response(
|
|
content=midi_bytes,
|
|
media_type="audio/midi",
|
|
headers={"Content-Disposition": f'attachment; filename="{mid_name}"'},
|
|
)
|
|
|
|
|
|
def _convert_to_wav(src: Path, dst: Path) -> None:
|
|
cmd = [
|
|
"ffmpeg", "-y",
|
|
"-i", str(src),
|
|
"-ar", "44100",
|
|
"-ac", "1",
|
|
"-af", f"apad=whole_dur={MIN_DURATION_MS}ms",
|
|
str(dst),
|
|
]
|
|
result = subprocess.run(cmd, capture_output=True)
|
|
if result.returncode != 0:
|
|
raise HTTPException(
|
|
status_code=422,
|
|
detail=(
|
|
"ffmpeg WAV conversion failed. "
|
|
+ result.stderr.decode(errors="replace")[:300]
|
|
),
|
|
)
|
|
|
|
|
|
def _ensure_checkpoints(mode: str) -> None:
|
|
"""Ensure built-in checkpoint dirs have TensorFlow weights for this mode."""
|
|
builtin_dir = BUILTIN_MODEL_PATHS.get(mode)
|
|
cache_dir = CACHE_MODEL_PATHS.get(mode)
|
|
if not builtin_dir:
|
|
return
|
|
|
|
if _has_variable_data(builtin_dir):
|
|
return
|
|
|
|
if cache_dir and _has_variable_data(cache_dir):
|
|
_copy_variable_data(cache_dir, builtin_dir)
|
|
if _has_variable_data(builtin_dir):
|
|
return
|
|
|
|
log.warning("missing Omnizart checkpoint for mode=%s, downloading checkpoints...", mode)
|
|
cmd = [
|
|
OMNIZART_BIN,
|
|
"download-checkpoints",
|
|
"--output-path",
|
|
str(OMNIZART_OUTPUT_ROOT),
|
|
]
|
|
result = subprocess.run(cmd, capture_output=True)
|
|
if result.returncode != 0:
|
|
stderr = result.stderr.decode(errors="replace")
|
|
stdout = result.stdout.decode(errors="replace")
|
|
log.error("checkpoint download failed (code %d): %s", result.returncode, stderr or stdout)
|
|
raise HTTPException(
|
|
status_code=500,
|
|
detail="Omnizart checkpoints are missing and download failed. See sidecar logs.",
|
|
)
|
|
|
|
if cache_dir:
|
|
_copy_variable_data(cache_dir, builtin_dir)
|
|
|
|
# Re-check target mode after download + sync.
|
|
if not _has_variable_data(builtin_dir):
|
|
raise HTTPException(
|
|
status_code=500,
|
|
detail=(
|
|
f"Omnizart checkpoint still missing for mode '{mode}' at {builtin_dir}. "
|
|
"See sidecar logs."
|
|
),
|
|
)
|
|
|
|
|
|
def _has_variable_data(model_dir: Path) -> bool:
|
|
return any((model_dir / "variables").glob("variables.data-*"))
|
|
|
|
|
|
def _copy_variable_data(src_model_dir: Path, dst_model_dir: Path) -> None:
|
|
src_vars = src_model_dir / "variables"
|
|
dst_vars = dst_model_dir / "variables"
|
|
dst_vars.mkdir(parents=True, exist_ok=True)
|
|
|
|
copied = False
|
|
for fp in src_vars.glob("variables.data-*"):
|
|
shutil.copy2(fp, dst_vars / fp.name)
|
|
copied = True
|
|
|
|
if copied:
|
|
log.info("checkpoint weights synced: %s -> %s", src_model_dir, dst_model_dir)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import uvicorn
|
|
uvicorn.run(app, host=os.environ.get("HOST", "0.0.0.0"), port=int(os.environ.get("PORT", "9765")))
|