From aecd73cb97ff107ec6ba8cdaeadeed1e5780c42e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrew=20Beveridge=20=E2=98=84=EF=B8=8F?= Date: Thu, 2 Jul 2026 00:45:54 -0400 Subject: [PATCH] =?UTF-8?q?feat(api):=20POST=20/api/parse-karaoke-titles?= =?UTF-8?q?=20=E2=80=94=20batch=20karaoke-filename=20parser=20(#864)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary New admin endpoint **`POST /api/parse-karaoke-titles`** that turns messy karaoke filenames into canonical `{artist, title, confidence}` using the Vertex Gemini stack (reuses the match-judge client pattern). This is **PR-A of the kjbox download-naming Phase 2** initiative — kjbox will call it to canonicalise downloaded-file names (esp. fixing artist/title order, e.g. KaraFun-reversed). ## Contract - Request: `{items:[{id, filename, channel?, source?}]}` - Response: `{results:[{id, artist, title, confidence}]}` — one id-aligned result per input. - Auth: `require_admin` (`X-Admin-Token`). ## Behaviour - Batched single Gemini call (JSON response schema, temperature 0, `location="global"`, Vertex — no API keys). - **Never 500s on a model error:** kill-switch (`PARSE_TITLES_ENABLED`) off, any model failure, or a batch length-mismatch → blank results (`confidence: 0.0`). Fails closed if settings can't load. kjbox degrades to its deterministic guess when this is unavailable. - New settings: `parse_titles_enabled` / `_model` (gemini-3.5-flash) / `_timeout_ms` (20s) / `_max_items` (200). ## Testing - `backend/tests/test_parse_titles.py` (11 assertions: prompt build, id-alignment, garbage-response, kill-switch, degrade-on-error, happy path) + `test_parse_titles_route.py` (admin gate + happy path). All green locally. ## Design `kjbox/docs/archive/2026-06-30-download-naming-normalization-design.md` §4; plan `2026-07-01-download-naming-p2-plan.md` (PR-A). @coderabbitai ignore --------- Co-authored-by: Claude Opus 4.8 (1M context) --- backend/api/routes/parse_titles.py | 52 ++++++++ backend/config.py | 9 ++ backend/main.py | 3 +- backend/services/parse_titles/__init__.py | 2 + backend/services/parse_titles/ai.py | 148 ++++++++++++++++++++++ backend/services/parse_titles/service.py | 50 ++++++++ backend/tests/test_parse_titles.py | 88 +++++++++++++ backend/tests/test_parse_titles_route.py | 59 +++++++++ 8 files changed, 410 insertions(+), 1 deletion(-) create mode 100644 backend/api/routes/parse_titles.py create mode 100644 backend/services/parse_titles/__init__.py create mode 100644 backend/services/parse_titles/ai.py create mode 100644 backend/services/parse_titles/service.py create mode 100644 backend/tests/test_parse_titles.py create mode 100644 backend/tests/test_parse_titles_route.py diff --git a/backend/api/routes/parse_titles.py b/backend/api/routes/parse_titles.py new file mode 100644 index 00000000..ac3b32a8 --- /dev/null +++ b/backend/api/routes/parse_titles.py @@ -0,0 +1,52 @@ +"""POST /api/parse-karaoke-titles — batch karaoke-filename → artist/title. + +Internal admin endpoint used by kjbox to canonicalise downloaded-file names. +Reuses the Vertex Gemini parser; never blocks the caller (kjbox degrades to a +deterministic guess when this is unavailable). +""" +import logging +from typing import Optional + +from fastapi import APIRouter, Depends +from pydantic import BaseModel + +from backend.api.dependencies import require_admin +from backend.services.auth_service import AuthResult +from backend.services.parse_titles import parse_titles + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["parse-titles"]) + + +class ParseItem(BaseModel): + id: str + filename: str + channel: Optional[str] = None + source: Optional[str] = None + + +class ParseRequest(BaseModel): + items: list[ParseItem] + + +class ParseResult(BaseModel): + id: str + artist: str + title: str + confidence: float + + +class ParseResponse(BaseModel): + results: list[ParseResult] + + +@router.post("/parse-karaoke-titles", response_model=ParseResponse) +async def parse_karaoke_titles( + body: ParseRequest, + auth_result: AuthResult = Depends(require_admin), +): + items = [i.model_dump() for i in body.items] + results = await parse_titles(items) + logger.info("parse-karaoke-titles: %d items", len(items)) + return {"results": results} diff --git a/backend/config.py b/backend/config.py index 35b9155c..9bd28805 100644 --- a/backend/config.py +++ b/backend/config.py @@ -117,6 +117,15 @@ class Settings(BaseSettings): # call still runs off the critical path and degrades to "none" on timeout. match_judge_timeout_ms: int = int(os.getenv("MATCH_JUDGE_TIMEOUT_MS", "12000")) + # Batch karaoke-filename parser (kjbox download-naming). Reuses the Vertex + # Gemini stack. Larger timeout than match_judge because it batches ~200 items. + parse_titles_enabled: bool = os.getenv("PARSE_TITLES_ENABLED", "true").lower() in ( + "1", "true", "yes", + ) + parse_titles_model: str = os.getenv("PARSE_TITLES_MODEL", "gemini-3.5-flash") + parse_titles_timeout_ms: int = int(os.getenv("PARSE_TITLES_TIMEOUT_MS", "20000")) + parse_titles_max_items: int = int(os.getenv("PARSE_TITLES_MAX_ITEMS", "200")) + # Cloud Tasks (for scalable worker coordination) # When enabled, workers are triggered via Cloud Tasks for guaranteed delivery # When disabled (default), workers are triggered via direct HTTP (for development) diff --git a/backend/main.py b/backend/main.py index 42bb6802..9fee3038 100644 --- a/backend/main.py +++ b/backend/main.py @@ -10,7 +10,7 @@ from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from backend.config import settings -from backend.api.routes import health, jobs, internal, file_upload, review, auth, audio_search, themes, users, admin, tenant, rate_limits, push, catalog, encoding_worker, client_errors, bulk +from backend.api.routes import health, jobs, internal, file_upload, review, auth, audio_search, themes, users, admin, tenant, rate_limits, push, catalog, encoding_worker, client_errors, bulk, parse_titles from backend.services.tracing import setup_tracing, instrument_app, get_current_trace_id from backend.services.structured_logging import setup_structured_logging from backend.services.spacy_preloader import preload_spacy_model @@ -187,6 +187,7 @@ app.include_router(admin.router, prefix="/api") # Admin dashboard and managemen app.include_router(rate_limits.router, prefix="/api") # Rate limits admin management app.include_router(push.router, prefix="/api") # Push notification subscription management app.include_router(catalog.router, prefix="/api") # Catalog proxy for song/artist autocomplete +app.include_router(parse_titles.router, prefix="/api") # kjbox karaoke-filename parser app.include_router(bulk.router, prefix="/api") # Bulk Mode: multi-job submission app.include_router(client_errors.router, prefix="/api") # Frontend crash reports app.include_router(tenant.router) # Tenant/white-label configuration (no /api prefix, router has it) diff --git a/backend/services/parse_titles/__init__.py b/backend/services/parse_titles/__init__.py new file mode 100644 index 00000000..ec2b747b --- /dev/null +++ b/backend/services/parse_titles/__init__.py @@ -0,0 +1,2 @@ +"""Batch karaoke-filename → {artist, title, confidence} parser (Vertex Gemini).""" +from backend.services.parse_titles.service import parse_titles # noqa: F401 diff --git a/backend/services/parse_titles/ai.py b/backend/services/parse_titles/ai.py new file mode 100644 index 00000000..7374b058 --- /dev/null +++ b/backend/services/parse_titles/ai.py @@ -0,0 +1,148 @@ +"""Vertex-Gemini layer for batch karaoke-filename parsing. + +Turns messy karaoke *filenames* (YouTube titles, community/producer naming, +KaraFun-reversed order) into canonical {artist, title, confidence}. Distinct +from match_judge (which judges an already-split artist/title). The model call +is injectable for tests. +""" +from __future__ import annotations + +import asyncio +import copy +import json +import logging +from typing import Awaitable, Callable, Optional + +logger = logging.getLogger(__name__) + +Generate = Callable[[str, str, str], Awaitable[dict]] + +_SYSTEM_PROMPT = ( + "You extract canonical song metadata from karaoke video/file names for a " + "karaoke library. Each item has an id and a filename (sometimes a channel or " + "source). Filenames are noisy: they carry karaoke markers ('(Karaoke Version)', " + "'KARAOKE', '[karaoke]', 'Instrumental', producer/brand tags), YouTube ids or " + "channel names, and inconsistent separators. Crucially the artist/title ORDER " + "is inconsistent — most are 'Artist - Title' but some sources (e.g. KaraFun) are " + "'Title - Artist'. Use your knowledge of real songs to put artist and title in " + "the correct fields.\n" + "Return JSON: {\"results\": [{\"id\", \"artist\", \"title\", \"confidence\"}]}. " + "Return exactly one result per input id, echoing the id verbatim.\n" + "artist/title: official formatting, karaoke noise removed, no brand codes or " + "YouTube ids. If you cannot identify one field, return it as an empty string.\n" + "confidence: 0.0-1.0 — your certainty the artist/title (and their order) are " + "correct. Be honest; low confidence for ambiguous or unknown songs. Never invent " + "a song; if unsure, return best-effort split with low confidence." +) + +RESPONSE_SCHEMA = { + "type": "object", + "properties": { + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": {"type": "string"}, + "artist": {"type": "string"}, + "title": {"type": "string"}, + "confidence": {"type": "number"}, + }, + "required": ["id"], + }, + } + }, + "required": ["results"], +} + + +def _model() -> str: + try: + from backend.config import settings + return getattr(settings, "parse_titles_model", "gemini-3.5-flash") + except Exception: # pragma: no cover - config guard + return "gemini-3.5-flash" + + +def build_prompts(items: list[dict]) -> tuple[str, str]: + lines = ["Parse these karaoke filenames:"] + for it in items: + parts = [f'id={it.get("id")!r}', f'filename={it.get("filename", "")!r}'] + if it.get("channel"): + parts.append(f'channel={it["channel"]!r}') + if it.get("source"): + parts.append(f'source={it["source"]!r}') + lines.append("- " + " ".join(parts)) + return _SYSTEM_PROMPT, "\n".join(lines) + + +def _clean(v) -> str: + return (v or "").strip() if isinstance(v, str) else "" + + +def _conf(v) -> float: + try: + f = float(v) + except (TypeError, ValueError): + return 0.0 + return max(0.0, min(1.0, f)) + + +def parse_map_from_response(data: object, items: list[dict]) -> list[dict]: + """Return one id-aligned result per input, filling any misses with blanks.""" + by_id: dict[str, dict] = {} + if isinstance(data, dict): + for r in data.get("results") or []: + if isinstance(r, dict) and r.get("id") is not None: + by_id[str(r["id"])] = r + out = [] + for it in items: + rid = str(it.get("id")) + r = by_id.get(rid, {}) + out.append({ + "id": rid, + "artist": _clean(r.get("artist")), + "title": _clean(r.get("title")), + "confidence": _conf(r.get("confidence")), + }) + return out + + +async def ai_parse( + items: list[dict], *, model: Optional[str] = None, + generate: Optional[Generate] = None, +) -> list[dict]: + gen = generate or _default_generate + system, user = build_prompts(items) + data = await gen(model or _model(), system, user) + return parse_map_from_response(data, items) + + +async def _default_generate(model: str, system: str, user: str) -> dict: + return await asyncio.to_thread(_blocking_generate, model, system, user) + + +def _blocking_generate(model: str, system: str, user: str) -> dict: + from google import genai + from google.genai import types + + from backend.config import settings + + timeout_ms = int(getattr(settings, "parse_titles_timeout_ms", 20000)) + client = genai.Client( + vertexai=True, + project=settings.google_cloud_project, + location="global", + http_options=types.HttpOptions(timeout=timeout_ms), + ) + response = client.models.generate_content( + model=model, + contents=[user], + config=types.GenerateContentConfig( + system_instruction=system, + response_mime_type="application/json", + response_schema=copy.deepcopy(RESPONSE_SCHEMA), + temperature=0, + ), + ) + return json.loads(response.text) diff --git a/backend/services/parse_titles/service.py b/backend/services/parse_titles/service.py new file mode 100644 index 00000000..759bf04d --- /dev/null +++ b/backend/services/parse_titles/service.py @@ -0,0 +1,50 @@ +"""Batch orchestration + graceful degrade for karaoke-filename parsing.""" +from __future__ import annotations + +import logging + +from backend.services.parse_titles import ai + +logger = logging.getLogger(__name__) + + +def _blanks(items: list[dict]) -> list[dict]: + return [ + {"id": str(it.get("id")), "artist": "", "title": "", "confidence": 0.0} + for it in items + ] + + +async def parse_titles( + items: list[dict], *, model=None, generate=None +) -> list[dict]: + """Parse a batch of filenames → id-aligned {id, artist, title, confidence}. + + Never raises: a disabled kill-switch or any model failure yields blank + results so the (kjbox) caller keeps its deterministic guess. + """ + if not items: + return [] + try: + from backend.config import settings + enabled = getattr(settings, "parse_titles_enabled", True) + max_items = int(getattr(settings, "parse_titles_max_items", 200)) + except Exception: # pragma: no cover + # Fail closed: if config can't load, don't reach for the external AI. + logger.warning("parse_titles settings unavailable; disabling parser") + enabled, max_items = False, 200 + if not enabled: + return _blanks(items) + + head, tail = items[:max_items], items[max_items:] + try: + results = await ai.ai_parse(head, model=model, generate=generate) + except Exception as exc: + logger.warning("parse_titles degraded (%s); returning blanks", exc) + return _blanks(items) + # ai_parse is id-aligned by construction; guard the contract defensively so a + # future refactor can't silently return a mis-aligned/short batch. + if len(results) != len(head): + logger.warning("parse_titles length mismatch; returning blanks") + return _blanks(items) + return results + _blanks(tail) diff --git a/backend/tests/test_parse_titles.py b/backend/tests/test_parse_titles.py new file mode 100644 index 00000000..90f1ea83 --- /dev/null +++ b/backend/tests/test_parse_titles.py @@ -0,0 +1,88 @@ +import asyncio + +from backend.services.parse_titles import ai +from backend.services.parse_titles import service + + +def test_settings_parse_titles_defaults(): + from backend.config import Settings + s = Settings() + assert s.parse_titles_enabled is True + assert s.parse_titles_model == "gemini-3.5-flash" + assert s.parse_titles_timeout_ms == 20000 + assert s.parse_titles_max_items == 200 + + +def test_build_prompts_lists_items_with_ids(): + system, user = ai.build_prompts( + [{"id": "a", "filename": "Santeria - Sublime _ Karaoke _ KaraFun.mp4", + "channel": "KaraFun", "source": "youtube"}] + ) + assert "artist" in system.lower() and "title" in system.lower() + assert "id='a'" in user or '"a"' in user or "id=a" in user # id must be echoed + assert "Santeria" in user + + +def test_parse_map_from_response_aligns_by_id_and_fills_misses(): + items = [{"id": "a", "filename": "x"}, {"id": "b", "filename": "y"}] + data = {"results": [{"id": "a", "artist": "Sublime", "title": "Santeria", + "confidence": 0.9}]} + out = ai.parse_map_from_response(data, items) + assert {r["id"] for r in out} == {"a", "b"} + a = next(r for r in out if r["id"] == "a") + assert (a["artist"], a["title"]) == ("Sublime", "Santeria") + b = next(r for r in out if r["id"] == "b") + assert b["artist"] == "" and b["title"] == "" and b["confidence"] == 0.0 + + +def test_ai_parse_uses_injected_generate(): + items = [{"id": "a", "filename": "Bella Kay - iloveit (Karaoke Version).mp4"}] + + async def fake_generate(model, system, user): + return {"results": [{"id": "a", "artist": "Bella Kay", + "title": "iloveit", "confidence": 0.82}]} + + out = asyncio.run(ai.ai_parse(items, generate=fake_generate)) + assert out[0]["artist"] == "Bella Kay" and out[0]["confidence"] == 0.82 + + +def test_parse_map_handles_garbage_response(): + items = [{"id": "a", "filename": "x"}] + assert ai.parse_map_from_response("not a dict", items) == [ + {"id": "a", "artist": "", "title": "", "confidence": 0.0} + ] + + +def test_parse_titles_disabled_returns_blanks(monkeypatch): + from backend.config import settings + monkeypatch.setattr(settings, "parse_titles_enabled", False) + out = asyncio.run(service.parse_titles([{"id": "a", "filename": "x"}])) + assert out == [{"id": "a", "artist": "", "title": "", "confidence": 0.0}] + + +def test_parse_titles_degrades_on_generate_error(monkeypatch): + from backend.config import settings + monkeypatch.setattr(settings, "parse_titles_enabled", True) + + async def boom(model, system, user): + raise RuntimeError("vertex down") + + out = asyncio.run(service.parse_titles( + [{"id": "a", "filename": "x"}], generate=boom)) + assert out == [{"id": "a", "artist": "", "title": "", "confidence": 0.0}] + + +def test_parse_titles_happy_path(monkeypatch): + from backend.config import settings + monkeypatch.setattr(settings, "parse_titles_enabled", True) + + async def gen(model, system, user): + return {"results": [{"id": "a", "artist": "Queen", + "title": "Bohemian Rhapsody", "confidence": 0.95}]} + + out = asyncio.run(service.parse_titles([{"id": "a", "filename": "x"}], generate=gen)) + assert out[0]["artist"] == "Queen" and out[0]["confidence"] == 0.95 + + +def test_parse_titles_empty_returns_empty(): + assert asyncio.run(service.parse_titles([])) == [] diff --git a/backend/tests/test_parse_titles_route.py b/backend/tests/test_parse_titles_route.py new file mode 100644 index 00000000..2ecb5761 --- /dev/null +++ b/backend/tests/test_parse_titles_route.py @@ -0,0 +1,59 @@ +from fastapi.testclient import TestClient + +from backend.main import app +from backend.api.dependencies import require_admin + + +def _admin_result(): + from backend.services.auth_service import AuthResult, UserType + return AuthResult(is_valid=True, user_type=UserType.ADMIN, remaining_uses=-1, + message="ok", user_email="a@nomadkaraoke.com", is_admin=True) + + +def test_parse_route_happy_path(monkeypatch): + # Self-contained: set our own admin override rather than relying on the + # conftest autouse fixture (which can be disturbed by earlier tests in a + # full-suite run and leave the endpoint returning 401). + from backend.api.routes import parse_titles as route + + async def fake_parse(items, **kw): + return [{"id": str(it["id"]), "artist": "Queen", + "title": "Bohemian Rhapsody", "confidence": 0.9} for it in items] + + monkeypatch.setattr(route, "parse_titles", fake_parse) + original = app.dependency_overrides.get(require_admin) + app.dependency_overrides[require_admin] = _admin_result + try: + client = TestClient(app) + resp = client.post("/api/parse-karaoke-titles", json={ + "items": [{"id": "1", "filename": "x.mp4", "source": "youtube"}]}) + assert resp.status_code == 200 + body = resp.json() + assert body["results"][0]["id"] == "1" + assert body["results"][0]["artist"] == "Queen" + finally: + if original is not None: + app.dependency_overrides[require_admin] = original + else: + app.dependency_overrides.pop(require_admin, None) + + +def test_parse_route_requires_admin(): + """The route is gated by require_admin: a non-admin dependency -> 403.""" + original = app.dependency_overrides.get(require_admin) + + def non_admin(): + from fastapi import HTTPException + raise HTTPException(status_code=403, detail="Admin access required") + + app.dependency_overrides[require_admin] = non_admin + try: + client = TestClient(app) + resp = client.post("/api/parse-karaoke-titles", + json={"items": [{"id": "1", "filename": "x"}]}) + assert resp.status_code == 403 + finally: + if original is not None: + app.dependency_overrides[require_admin] = original + else: + app.dependency_overrides.pop(require_admin, None)