diff --git a/20260314_Omnizart_MIDI_Plan.md b/20260314_Omnizart_MIDI_Plan.md new file mode 100644 index 0000000..b7a3838 --- /dev/null +++ b/20260314_Omnizart_MIDI_Plan.md @@ -0,0 +1,499 @@ +# Omnizart MIDI Transcription Integration Plan + +**Date:** 2026-03-14 +**Status:** Planning +**Scope:** Add Omnizart as a second transcription engine alongside basic-pitch. + +--- + +## Background + +StemEngine currently offers MIDI transcription via **basic-pitch** (Spotify/Google, +in-process Python API) with two modes: + +| ID | Label | Engine | +|----|-------|--------| +| `polyphonic` | Polyphonic | basic-pitch | +| `melody` | Melody / Vocal | basic-pitch | + +**Omnizart** (Music AI NTU, 2021) is a separate toolkit for music transcription. It +operates via a CLI (`omnizart {mode} transcribe ...`) and produces standard `.mid` +output. A reference integration already exists in the workspace at +`docker/x/Omnizart-UI/` (Sanic-based standalone API). The reference implementation +was the primary source for this plan. + +### Why add it? + +- basic-pitch is excellent for melody extraction but struggles with polyphonic + arrangements at instrument level (piano-roll style). +- Omnizart's `music` mode does instrument-aware multi-track transcription — it is + the best freely available option for generating separated MIDI per instrument. +- The `vocal` mode produces a melody MIDI line comparable to basic-pitch's `melody` + mode but via a different model trained specifically on vocal contours. +- The `transcribe.py` router already has a `_STEM_PREFERENCE` list ordered for + Omnizart's `music` mode (pitched instruments first, drums excluded) — the + groundwork is already there. + +--- + +## Omnizart Mode Inventory + +| Omnizart mode | CLI arg | Output | Status | +|---|---|---|---| +| `music` | `omnizart music transcribe -o ` | Multi-track MIDI | ✅ Works | +| `vocal` | `omnizart vocal transcribe ` | Single-track melody MIDI | ✅ Works (quirky `-o` behaviour, see below) | +| `vocal-contour` | `omnizart vocal-contour transcribe -o ` | Contour F0 MIDI | ✅ Works | +| `chord` | — | Chord symbols | ❌ Broken (numpy/Python conflict) | +| `drums` | — | Drum MIDI | ❌ Broken (Python 2.7 dep) | + +**Vocal mode output path quirk:** The `-o` flag is silently ignored. The output file +is always written to the current working directory as +`./{input-stem}-vocal.mid`. This must be handled explicitly (see service design). + +**Minimum file length:** Omnizart silently fails or produces empty output for inputs +shorter than ~10 seconds. The reference implementation enforces `10000 ms` as a +hard minimum via `SoundUtil.ConvertFileToWav(src, dst, 10_000)`. We must do the same. + +--- + +## New Mode IDs for StemEngine + +Three new mode IDs are proposed, prefixed with `omnizart-` to make the dispatch +branch unambiguous: + +| New ID | Label | Omnizart CLI mode | +|---|---|---| +| `omnizart-music` | Omnizart — Multi-instrument | `music` | +| `omnizart-vocal` | Omnizart — Vocal Melody | `vocal` | +| `omnizart-vocal-contour` | Omnizart — Vocal Contour | `vocal-contour` | + +Phase 1 ships `omnizart-music` only. Phases 2 and 3 add the remaining modes. + +--- + +## Files to Change + +### 1. `backend/config.py` + +**Location:** line 195 (`TRANSCRIBE_MODES` list). + +Extend the list with the new modes and add an availability flag so the frontend +can grey them out if Omnizart is not installed/reachable: + +```python +# basic-pitch transcription modes +TRANSCRIBE_MODES = [ + {"id": "polyphonic", "label": "Polyphonic", "engine": "basic-pitch"}, + {"id": "melody", "label": "Melody / Vocal", "engine": "basic-pitch"}, + # Omnizart modes — require Omnizart to be installed (see OMNIZART_ENABLED) + {"id": "omnizart-music", "label": "Omnizart — Multi-instrument", "engine": "omnizart"}, + {"id": "omnizart-vocal", "label": "Omnizart — Vocal Melody", "engine": "omnizart"}, + {"id": "omnizart-vocal-contour", "label": "Omnizart — Vocal Contour", "engine": "omnizart"}, +] +DEFAULT_TRANSCRIBE_MODE = "polyphonic" + +# Set via env var; when False the Omnizart modes are still listed but the +# router returns 503 if one is selected (allows UI to warn without hiding them). +OMNIZART_ENABLED = os.environ.get("OMNIZART_ENABLED", "false").lower() == "true" +``` + +No changes needed to `TRANSCRIBE_PRESETS` — those are basic-pitch only and +dispatched conditionally (see service below). + +--- + +### 2. `backend/services/omnizart_service.py` *(new file)* + +This service owns all Omnizart-specific logic: WAV conversion, minimum-length +padding, subprocess invocation, output path resolution, and cleanup. + +```python +""" +MIDI transcription via Omnizart CLI (subprocess). +Omnizart must be available on PATH (or OMNIZART_BIN env var). +Supports modes: music | vocal | vocal-contour +""" +import logging +import os +import shlex +import subprocess +import tempfile +from pathlib import Path + +from .. import job_manager as jm +from ..config import JOBS_DIR + +log = logging.getLogger(__name__) + +# CLI binary name / path — override with OMNIZART_BIN env var for venv/conda +OMNIZART_BIN = os.environ.get("OMNIZART_BIN", "omnizart") + +# Omnizart fails silently on inputs shorter than ~10 s; pad/reject at this limit +MIN_DURATION_MS = 10_000 + + +def _omnizart_mode(mode_id: str) -> str: + """Strip the 'omnizart-' prefix to get the raw CLI mode name.""" + return mode_id.removeprefix("omnizart-") + + +def _convert_to_wav(src: Path, dst: Path) -> None: + """ + Use ffmpeg to convert any audio file to WAV (mono, 44100 Hz). + Also enforces MIN_DURATION_MS by adding silence padding if needed. + """ + cmd = [ + "ffmpeg", "-y", + "-i", str(src), + "-ar", "44100", + "-ac", "1", # mono — Omnizart works best on mono + "-af", f"apad=whole_dur={MIN_DURATION_MS}ms", # pad to min length + str(dst), + ] + subprocess.run(cmd, check=True, capture_output=True) + + +def run_omnizart_transcription(job_id: str, input_path: str, mode_id: str) -> None: + """ + Background task: transcribe audio to MIDI using Omnizart CLI. + mode_id: 'omnizart-music' | 'omnizart-vocal' | 'omnizart-vocal-contour' + """ + mode = _omnizart_mode(mode_id) # 'music', 'vocal', or 'vocal-contour' + log.debug("[%s] omnizart start — mode=%s input=%s", job_id, mode, input_path) + + output_dir = JOBS_DIR / job_id + output_dir.mkdir(parents=True, exist_ok=True) + + jm.update_job(job_id, status=jm.JobStatus.RUNNING, progress="Converting to WAV...") + + try: + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + stem = Path(input_path).stem + wav_path = tmp_path / f"{stem}-{mode}.wav" + mid_name = f"{stem}-{mode}.mid" + + # --- WAV conversion + padding --- + _convert_to_wav(Path(input_path), wav_path) + + if jm.should_cancel(job_id): + jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.") + return + + jm.update_job(job_id, progress=f"Transcribing with Omnizart ({mode})...") + + # --- Subprocess invocation --- + # Vocal mode ignores -o; output lands in CWD as {stem}-vocal.mid + if mode == "vocal": + # Run from tmp dir so the auto-named file lands there + cmd = f'{OMNIZART_BIN} vocal transcribe "{wav_path}"' + expected_output = tmp_path / mid_name # {stem}-vocal.mid in tmp + else: + output_mid = tmp_path / mid_name + cmd = f'{OMNIZART_BIN} {mode} transcribe -o "{output_mid}" "{wav_path}"' + expected_output = output_mid + + proc = subprocess.Popen( + shlex.split(cmd), + cwd=str(tmp_path), # Important for vocal mode CWD output + shell=False, + ) + + # Poll for cancellation every 5 s (mirrors Omnizart-UI pattern) + import time + while proc.poll() is None: + if jm.should_cancel(job_id): + proc.terminate() + proc.wait() + jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.") + return + time.sleep(5) + + if proc.returncode != 0: + raise RuntimeError(f"omnizart exited with code {proc.returncode}") + + # --- Copy output to jobs dir --- + dest = output_dir / mid_name + if not expected_output.exists(): + raise FileNotFoundError( + f"Omnizart produced no output at {expected_output}. " + "Input may be too short or unsupported." + ) + dest.write_bytes(expected_output.read_bytes()) + + log.debug("[%s] omnizart complete — output=%s", job_id, dest.name) + jm.update_job( + job_id, + status=jm.JobStatus.DONE, + progress="Transcription complete.", + files=[{"name": dest.name, "path": str(dest)}], + ) + + except Exception as exc: + log.error("[%s] omnizart failed: %s", job_id, exc) + jm.update_job(job_id, status=jm.JobStatus.ERROR, error=str(exc), progress="Error.") +``` + +**Key design choices vs. Omnizart-UI reference:** + +| Concern | Omnizart-UI approach | StemEngine approach | +|---|---|---| +| WAV conversion | `SoundUtil` (custom pydub wrapper) | `ffmpeg` subprocess (already on PATH) | +| Cancellation | DB-poll in while loop | `jm.should_cancel()` in while loop | +| Vocal CWD quirk | Start process without `cwd`, track file separately | Set `cwd=tmp` so the auto-named `.mid` lands inside the temp dir | +| Job state | Custom `JobController` | Existing `jm` (job_manager) | +| Output storage | DB blob | Filesystem under `JOBS_DIR` (same as basic-pitch) | + +--- + +### 3. `backend/services/transcriber_service.py` + +Extend `run_transcription()` to branch on engine type: + +```python +""" +MIDI transcription dispatcher. +Routes to basic-pitch or Omnizart based on mode prefix. +""" +import logging +from pathlib import Path + +from .. import job_manager as jm +from ..config import JOBS_DIR, TRANSCRIBE_PRESETS, OMNIZART_ENABLED + +log = logging.getLogger(__name__) + + +def run_transcription(job_id: str, input_path: str, mode: str): + """ + Background task: transcribe audio to MIDI. + mode: 'polyphonic' | 'melody' → basic-pitch + 'omnizart-*' → Omnizart CLI + """ + if mode.startswith("omnizart-"): + if not OMNIZART_ENABLED: + jm.update_job(job_id, status=jm.JobStatus.ERROR, + error="Omnizart is not enabled on this server.", + progress="Error.") + return + from .omnizart_service import run_omnizart_transcription + run_omnizart_transcription(job_id, input_path, mode) + else: + _run_basic_pitch(job_id, input_path, mode) + + +def _run_basic_pitch(job_id: str, input_path: str, mode: str): + """Original basic-pitch path (unchanged).""" + from basic_pitch.inference import predict + from basic_pitch import ICASSP_2022_MODEL_PATH + + log.debug("[%s] basic-pitch start — mode=%s", job_id, mode) + output_dir = JOBS_DIR / job_id + output_dir.mkdir(parents=True, exist_ok=True) + + jm.update_job(job_id, status=jm.JobStatus.RUNNING, progress="Loading basic-pitch model...") + + try: + if jm.should_cancel(job_id): + jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.") + return + + params = TRANSCRIBE_PRESETS.get(mode, TRANSCRIBE_PRESETS["polyphonic"]) + jm.update_job(job_id, progress=f"Transcribing with basic-pitch ({mode})...") + + _, midi_data, _ = predict(input_path, ICASSP_2022_MODEL_PATH, **params) + + if jm.should_cancel(job_id): + jm.update_job(job_id, status=jm.JobStatus.CANCELLED, progress="Cancelled.") + return + + output_name = f"{Path(input_path).stem}-{mode}.mid" + dest = output_dir / output_name + midi_data.write(str(dest)) + + log.debug("[%s] basic-pitch complete — output=%s", job_id, dest.name) + jm.update_job( + job_id, + status=jm.JobStatus.DONE, + progress="Transcription complete.", + files=[{"name": dest.name, "path": str(dest)}], + ) + + except Exception as exc: + log.debug("[%s] basic-pitch failed: %s", job_id, exc) + jm.update_job(job_id, status=jm.JobStatus.ERROR, error=str(exc), progress="Error.") +``` + +**No other callers change.** `project_reprocess_service.py` already calls +`run_transcription(job_id, path, mode)` — it will automatically route correctly once +the mode string is set correctly when the reprocess is triggered. + +--- + +### 4. `backend/routers/transcribe.py` + +**No structural changes required.** `VALID_MODES` is built from `TRANSCRIBE_MODES` +at import time, so the new mode IDs are automatically accepted once config is updated: + +```python +VALID_MODES = {m["id"] for m in TRANSCRIBE_MODES} # already expands automatically +``` + +**Optional hardening** — add a 503 guard for Omnizart-specific modes if you want the +API to surface availability without relying on the service layer: + +```python +from ..config import OMNIZART_ENABLED + +@router.post("") +async def transcribe(file: UploadFile = File(...), mode: str = Form(DEFAULT_TRANSCRIBE_MODE)): + if mode not in VALID_MODES: + raise HTTPException(status_code=400, detail=f"Unknown mode: {mode}") + if mode.startswith("omnizart-") and not OMNIZART_ENABLED: + raise HTTPException(status_code=503, detail="Omnizart is not available on this server.") + ... +``` + +--- + +### 5. `frontend/src/pages/ProjectDetail.tsx` (and standalone transcribe UI) + +The frontend `GET /api/transcribe/modes` call already fetches the full mode list +dynamically. The main changes needed: + +1. **Group the picker by engine** — separate the mode list into two visual sections + ("Basic Pitch" and "Omnizart") using the `engine` field added to each mode + entry in config. +2. **Disable Omnizart options when unavailable** — add a `GET /api/transcribe/status` + endpoint (or embed availability in the modes response) and grey out Omnizart + entries when `OMNIZART_ENABLED=false`. +3. **Hint text** — add a tooltip/description for each Omnizart mode explaining its + strength (multi-instrument vs. melody/vocal). + +Minimal change if full grouping is deferred: just ensure the new mode IDs appear +in the ` for transcription engine + mode. */ +function TranscribeModeSelect({ + value, + onChange, + modes, +}: { + value: string + onChange: (id: string) => void + modes: ModeOption[] +}) { + const basicPitch = modes.filter((m) => m.engine !== 'omnizart') + const omnizart = modes.filter((m) => m.engine === 'omnizart') + + const renderOption = (m: ModeOption) => ( + + ) + + return ( + + ) +} + function StemMidiRow({ projectId, stemName, transJob, + modes, onAction, }: { projectId: string stemName: string transJob: Job | null + modes: ModeOption[] onAction: () => void }) { const [mode, setMode] = useState('polyphonic') @@ -1107,15 +1153,7 @@ function StemMidiRow({ {label} - +