mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
feat(api): POST /api/parse-karaoke-titles — batch karaoke-filename parser (#864)
## Summary
New admin endpoint **`POST /api/parse-karaoke-titles`** that turns messy
karaoke filenames into canonical `{artist, title, confidence}` using the
Vertex Gemini stack (reuses the match-judge client pattern). This is
**PR-A of the kjbox download-naming Phase 2** initiative — kjbox will
call it to canonicalise downloaded-file names (esp. fixing artist/title
order, e.g. KaraFun-reversed).
## Contract
- Request: `{items:[{id, filename, channel?, source?}]}`
- Response: `{results:[{id, artist, title, confidence}]}` — one
id-aligned result per input.
- Auth: `require_admin` (`X-Admin-Token`).
## Behaviour
- Batched single Gemini call (JSON response schema, temperature 0,
`location="global"`, Vertex — no API keys).
- **Never 500s on a model error:** kill-switch (`PARSE_TITLES_ENABLED`)
off, any model failure, or a batch length-mismatch → blank results
(`confidence: 0.0`). Fails closed if settings can't load. kjbox degrades
to its deterministic guess when this is unavailable.
- New settings: `parse_titles_enabled` / `_model` (gemini-3.5-flash) /
`_timeout_ms` (20s) / `_max_items` (200).
## Testing
- `backend/tests/test_parse_titles.py` (11 assertions: prompt build,
id-alignment, garbage-response, kill-switch, degrade-on-error, happy
path) + `test_parse_titles_route.py` (admin gate + happy path). All
green locally.
## Design
`kjbox/docs/archive/2026-06-30-download-naming-normalization-design.md`
§4; plan `2026-07-01-download-naming-p2-plan.md` (PR-A).
@coderabbitai ignore
---------
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d1d7c50ad2
commit
aecd73cb97
8 files changed
+410
-1
No files matched your search
@@ -0,0 +1,52 @@
|
||||
"""POST /api/parse-karaoke-titles — batch karaoke-filename → artist/title.
|
||||
|
||||
Internal admin endpoint used by kjbox to canonicalise downloaded-file names.
|
||||
Reuses the Vertex Gemini parser; never blocks the caller (kjbox degrades to a
|
||||
deterministic guess when this is unavailable).
|
||||
"""
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
|
||||
from backend.api.dependencies import require_admin
|
||||
from backend.services.auth_service import AuthResult
|
||||
from backend.services.parse_titles import parse_titles
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(tags=["parse-titles"])
|
||||
|
||||
|
||||
class ParseItem(BaseModel):
|
||||
id: str
|
||||
filename: str
|
||||
channel: Optional[str] = None
|
||||
source: Optional[str] = None
|
||||
|
||||
|
||||
class ParseRequest(BaseModel):
|
||||
items: list[ParseItem]
|
||||
|
||||
|
||||
class ParseResult(BaseModel):
|
||||
id: str
|
||||
artist: str
|
||||
title: str
|
||||
confidence: float
|
||||
|
||||
|
||||
class ParseResponse(BaseModel):
|
||||
results: list[ParseResult]
|
||||
|
||||
|
||||
@router.post("/parse-karaoke-titles", response_model=ParseResponse)
|
||||
async def parse_karaoke_titles(
|
||||
body: ParseRequest,
|
||||
auth_result: AuthResult = Depends(require_admin),
|
||||
):
|
||||
items = [i.model_dump() for i in body.items]
|
||||
results = await parse_titles(items)
|
||||
logger.info("parse-karaoke-titles: %d items", len(items))
|
||||
return {"results": results}
|
||||
@@ -117,6 +117,15 @@ class Settings(BaseSettings):
|
||||
# call still runs off the critical path and degrades to "none" on timeout.
|
||||
match_judge_timeout_ms: int = int(os.getenv("MATCH_JUDGE_TIMEOUT_MS", "12000"))
|
||||
|
||||
# Batch karaoke-filename parser (kjbox download-naming). Reuses the Vertex
|
||||
# Gemini stack. Larger timeout than match_judge because it batches ~200 items.
|
||||
parse_titles_enabled: bool = os.getenv("PARSE_TITLES_ENABLED", "true").lower() in (
|
||||
"1", "true", "yes",
|
||||
)
|
||||
parse_titles_model: str = os.getenv("PARSE_TITLES_MODEL", "gemini-3.5-flash")
|
||||
parse_titles_timeout_ms: int = int(os.getenv("PARSE_TITLES_TIMEOUT_MS", "20000"))
|
||||
parse_titles_max_items: int = int(os.getenv("PARSE_TITLES_MAX_ITEMS", "200"))
|
||||
|
||||
# Cloud Tasks (for scalable worker coordination)
|
||||
# When enabled, workers are triggered via Cloud Tasks for guaranteed delivery
|
||||
# When disabled (default), workers are triggered via direct HTTP (for development)
|
||||
|
||||
+2
-1
@@ -10,7 +10,7 @@ from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
from backend.config import settings
|
||||
from backend.api.routes import health, jobs, internal, file_upload, review, auth, audio_search, themes, users, admin, tenant, rate_limits, push, catalog, encoding_worker, client_errors, bulk
|
||||
from backend.api.routes import health, jobs, internal, file_upload, review, auth, audio_search, themes, users, admin, tenant, rate_limits, push, catalog, encoding_worker, client_errors, bulk, parse_titles
|
||||
from backend.services.tracing import setup_tracing, instrument_app, get_current_trace_id
|
||||
from backend.services.structured_logging import setup_structured_logging
|
||||
from backend.services.spacy_preloader import preload_spacy_model
|
||||
@@ -187,6 +187,7 @@ app.include_router(admin.router, prefix="/api") # Admin dashboard and managemen
|
||||
app.include_router(rate_limits.router, prefix="/api") # Rate limits admin management
|
||||
app.include_router(push.router, prefix="/api") # Push notification subscription management
|
||||
app.include_router(catalog.router, prefix="/api") # Catalog proxy for song/artist autocomplete
|
||||
app.include_router(parse_titles.router, prefix="/api") # kjbox karaoke-filename parser
|
||||
app.include_router(bulk.router, prefix="/api") # Bulk Mode: multi-job submission
|
||||
app.include_router(client_errors.router, prefix="/api") # Frontend crash reports
|
||||
app.include_router(tenant.router) # Tenant/white-label configuration (no /api prefix, router has it)
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
"""Batch karaoke-filename → {artist, title, confidence} parser (Vertex Gemini)."""
|
||||
from backend.services.parse_titles.service import parse_titles # noqa: F401
|
||||
@@ -0,0 +1,148 @@
|
||||
"""Vertex-Gemini layer for batch karaoke-filename parsing.
|
||||
|
||||
Turns messy karaoke *filenames* (YouTube titles, community/producer naming,
|
||||
KaraFun-reversed order) into canonical {artist, title, confidence}. Distinct
|
||||
from match_judge (which judges an already-split artist/title). The model call
|
||||
is injectable for tests.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import json
|
||||
import logging
|
||||
from typing import Awaitable, Callable, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
Generate = Callable[[str, str, str], Awaitable[dict]]
|
||||
|
||||
_SYSTEM_PROMPT = (
|
||||
"You extract canonical song metadata from karaoke video/file names for a "
|
||||
"karaoke library. Each item has an id and a filename (sometimes a channel or "
|
||||
"source). Filenames are noisy: they carry karaoke markers ('(Karaoke Version)', "
|
||||
"'KARAOKE', '[karaoke]', 'Instrumental', producer/brand tags), YouTube ids or "
|
||||
"channel names, and inconsistent separators. Crucially the artist/title ORDER "
|
||||
"is inconsistent — most are 'Artist - Title' but some sources (e.g. KaraFun) are "
|
||||
"'Title - Artist'. Use your knowledge of real songs to put artist and title in "
|
||||
"the correct fields.\n"
|
||||
"Return JSON: {\"results\": [{\"id\", \"artist\", \"title\", \"confidence\"}]}. "
|
||||
"Return exactly one result per input id, echoing the id verbatim.\n"
|
||||
"artist/title: official formatting, karaoke noise removed, no brand codes or "
|
||||
"YouTube ids. If you cannot identify one field, return it as an empty string.\n"
|
||||
"confidence: 0.0-1.0 — your certainty the artist/title (and their order) are "
|
||||
"correct. Be honest; low confidence for ambiguous or unknown songs. Never invent "
|
||||
"a song; if unsure, return best-effort split with low confidence."
|
||||
)
|
||||
|
||||
RESPONSE_SCHEMA = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"results": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {"type": "string"},
|
||||
"artist": {"type": "string"},
|
||||
"title": {"type": "string"},
|
||||
"confidence": {"type": "number"},
|
||||
},
|
||||
"required": ["id"],
|
||||
},
|
||||
}
|
||||
},
|
||||
"required": ["results"],
|
||||
}
|
||||
|
||||
|
||||
def _model() -> str:
|
||||
try:
|
||||
from backend.config import settings
|
||||
return getattr(settings, "parse_titles_model", "gemini-3.5-flash")
|
||||
except Exception: # pragma: no cover - config guard
|
||||
return "gemini-3.5-flash"
|
||||
|
||||
|
||||
def build_prompts(items: list[dict]) -> tuple[str, str]:
|
||||
lines = ["Parse these karaoke filenames:"]
|
||||
for it in items:
|
||||
parts = [f'id={it.get("id")!r}', f'filename={it.get("filename", "")!r}']
|
||||
if it.get("channel"):
|
||||
parts.append(f'channel={it["channel"]!r}')
|
||||
if it.get("source"):
|
||||
parts.append(f'source={it["source"]!r}')
|
||||
lines.append("- " + " ".join(parts))
|
||||
return _SYSTEM_PROMPT, "\n".join(lines)
|
||||
|
||||
|
||||
def _clean(v) -> str:
|
||||
return (v or "").strip() if isinstance(v, str) else ""
|
||||
|
||||
|
||||
def _conf(v) -> float:
|
||||
try:
|
||||
f = float(v)
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
return max(0.0, min(1.0, f))
|
||||
|
||||
|
||||
def parse_map_from_response(data: object, items: list[dict]) -> list[dict]:
|
||||
"""Return one id-aligned result per input, filling any misses with blanks."""
|
||||
by_id: dict[str, dict] = {}
|
||||
if isinstance(data, dict):
|
||||
for r in data.get("results") or []:
|
||||
if isinstance(r, dict) and r.get("id") is not None:
|
||||
by_id[str(r["id"])] = r
|
||||
out = []
|
||||
for it in items:
|
||||
rid = str(it.get("id"))
|
||||
r = by_id.get(rid, {})
|
||||
out.append({
|
||||
"id": rid,
|
||||
"artist": _clean(r.get("artist")),
|
||||
"title": _clean(r.get("title")),
|
||||
"confidence": _conf(r.get("confidence")),
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
async def ai_parse(
|
||||
items: list[dict], *, model: Optional[str] = None,
|
||||
generate: Optional[Generate] = None,
|
||||
) -> list[dict]:
|
||||
gen = generate or _default_generate
|
||||
system, user = build_prompts(items)
|
||||
data = await gen(model or _model(), system, user)
|
||||
return parse_map_from_response(data, items)
|
||||
|
||||
|
||||
async def _default_generate(model: str, system: str, user: str) -> dict:
|
||||
return await asyncio.to_thread(_blocking_generate, model, system, user)
|
||||
|
||||
|
||||
def _blocking_generate(model: str, system: str, user: str) -> dict:
|
||||
from google import genai
|
||||
from google.genai import types
|
||||
|
||||
from backend.config import settings
|
||||
|
||||
timeout_ms = int(getattr(settings, "parse_titles_timeout_ms", 20000))
|
||||
client = genai.Client(
|
||||
vertexai=True,
|
||||
project=settings.google_cloud_project,
|
||||
location="global",
|
||||
http_options=types.HttpOptions(timeout=timeout_ms),
|
||||
)
|
||||
response = client.models.generate_content(
|
||||
model=model,
|
||||
contents=[user],
|
||||
config=types.GenerateContentConfig(
|
||||
system_instruction=system,
|
||||
response_mime_type="application/json",
|
||||
response_schema=copy.deepcopy(RESPONSE_SCHEMA),
|
||||
temperature=0,
|
||||
),
|
||||
)
|
||||
return json.loads(response.text)
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Batch orchestration + graceful degrade for karaoke-filename parsing."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from backend.services.parse_titles import ai
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _blanks(items: list[dict]) -> list[dict]:
|
||||
return [
|
||||
{"id": str(it.get("id")), "artist": "", "title": "", "confidence": 0.0}
|
||||
for it in items
|
||||
]
|
||||
|
||||
|
||||
async def parse_titles(
|
||||
items: list[dict], *, model=None, generate=None
|
||||
) -> list[dict]:
|
||||
"""Parse a batch of filenames → id-aligned {id, artist, title, confidence}.
|
||||
|
||||
Never raises: a disabled kill-switch or any model failure yields blank
|
||||
results so the (kjbox) caller keeps its deterministic guess.
|
||||
"""
|
||||
if not items:
|
||||
return []
|
||||
try:
|
||||
from backend.config import settings
|
||||
enabled = getattr(settings, "parse_titles_enabled", True)
|
||||
max_items = int(getattr(settings, "parse_titles_max_items", 200))
|
||||
except Exception: # pragma: no cover
|
||||
# Fail closed: if config can't load, don't reach for the external AI.
|
||||
logger.warning("parse_titles settings unavailable; disabling parser")
|
||||
enabled, max_items = False, 200
|
||||
if not enabled:
|
||||
return _blanks(items)
|
||||
|
||||
head, tail = items[:max_items], items[max_items:]
|
||||
try:
|
||||
results = await ai.ai_parse(head, model=model, generate=generate)
|
||||
except Exception as exc:
|
||||
logger.warning("parse_titles degraded (%s); returning blanks", exc)
|
||||
return _blanks(items)
|
||||
# ai_parse is id-aligned by construction; guard the contract defensively so a
|
||||
# future refactor can't silently return a mis-aligned/short batch.
|
||||
if len(results) != len(head):
|
||||
logger.warning("parse_titles length mismatch; returning blanks")
|
||||
return _blanks(items)
|
||||
return results + _blanks(tail)
|
||||
@@ -0,0 +1,88 @@
|
||||
import asyncio
|
||||
|
||||
from backend.services.parse_titles import ai
|
||||
from backend.services.parse_titles import service
|
||||
|
||||
|
||||
def test_settings_parse_titles_defaults():
|
||||
from backend.config import Settings
|
||||
s = Settings()
|
||||
assert s.parse_titles_enabled is True
|
||||
assert s.parse_titles_model == "gemini-3.5-flash"
|
||||
assert s.parse_titles_timeout_ms == 20000
|
||||
assert s.parse_titles_max_items == 200
|
||||
|
||||
|
||||
def test_build_prompts_lists_items_with_ids():
|
||||
system, user = ai.build_prompts(
|
||||
[{"id": "a", "filename": "Santeria - Sublime _ Karaoke _ KaraFun.mp4",
|
||||
"channel": "KaraFun", "source": "youtube"}]
|
||||
)
|
||||
assert "artist" in system.lower() and "title" in system.lower()
|
||||
assert "id='a'" in user or '"a"' in user or "id=a" in user # id must be echoed
|
||||
assert "Santeria" in user
|
||||
|
||||
|
||||
def test_parse_map_from_response_aligns_by_id_and_fills_misses():
|
||||
items = [{"id": "a", "filename": "x"}, {"id": "b", "filename": "y"}]
|
||||
data = {"results": [{"id": "a", "artist": "Sublime", "title": "Santeria",
|
||||
"confidence": 0.9}]}
|
||||
out = ai.parse_map_from_response(data, items)
|
||||
assert {r["id"] for r in out} == {"a", "b"}
|
||||
a = next(r for r in out if r["id"] == "a")
|
||||
assert (a["artist"], a["title"]) == ("Sublime", "Santeria")
|
||||
b = next(r for r in out if r["id"] == "b")
|
||||
assert b["artist"] == "" and b["title"] == "" and b["confidence"] == 0.0
|
||||
|
||||
|
||||
def test_ai_parse_uses_injected_generate():
|
||||
items = [{"id": "a", "filename": "Bella Kay - iloveit (Karaoke Version).mp4"}]
|
||||
|
||||
async def fake_generate(model, system, user):
|
||||
return {"results": [{"id": "a", "artist": "Bella Kay",
|
||||
"title": "iloveit", "confidence": 0.82}]}
|
||||
|
||||
out = asyncio.run(ai.ai_parse(items, generate=fake_generate))
|
||||
assert out[0]["artist"] == "Bella Kay" and out[0]["confidence"] == 0.82
|
||||
|
||||
|
||||
def test_parse_map_handles_garbage_response():
|
||||
items = [{"id": "a", "filename": "x"}]
|
||||
assert ai.parse_map_from_response("not a dict", items) == [
|
||||
{"id": "a", "artist": "", "title": "", "confidence": 0.0}
|
||||
]
|
||||
|
||||
|
||||
def test_parse_titles_disabled_returns_blanks(monkeypatch):
|
||||
from backend.config import settings
|
||||
monkeypatch.setattr(settings, "parse_titles_enabled", False)
|
||||
out = asyncio.run(service.parse_titles([{"id": "a", "filename": "x"}]))
|
||||
assert out == [{"id": "a", "artist": "", "title": "", "confidence": 0.0}]
|
||||
|
||||
|
||||
def test_parse_titles_degrades_on_generate_error(monkeypatch):
|
||||
from backend.config import settings
|
||||
monkeypatch.setattr(settings, "parse_titles_enabled", True)
|
||||
|
||||
async def boom(model, system, user):
|
||||
raise RuntimeError("vertex down")
|
||||
|
||||
out = asyncio.run(service.parse_titles(
|
||||
[{"id": "a", "filename": "x"}], generate=boom))
|
||||
assert out == [{"id": "a", "artist": "", "title": "", "confidence": 0.0}]
|
||||
|
||||
|
||||
def test_parse_titles_happy_path(monkeypatch):
|
||||
from backend.config import settings
|
||||
monkeypatch.setattr(settings, "parse_titles_enabled", True)
|
||||
|
||||
async def gen(model, system, user):
|
||||
return {"results": [{"id": "a", "artist": "Queen",
|
||||
"title": "Bohemian Rhapsody", "confidence": 0.95}]}
|
||||
|
||||
out = asyncio.run(service.parse_titles([{"id": "a", "filename": "x"}], generate=gen))
|
||||
assert out[0]["artist"] == "Queen" and out[0]["confidence"] == 0.95
|
||||
|
||||
|
||||
def test_parse_titles_empty_returns_empty():
|
||||
assert asyncio.run(service.parse_titles([])) == []
|
||||
@@ -0,0 +1,59 @@
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from backend.main import app
|
||||
from backend.api.dependencies import require_admin
|
||||
|
||||
|
||||
def _admin_result():
|
||||
from backend.services.auth_service import AuthResult, UserType
|
||||
return AuthResult(is_valid=True, user_type=UserType.ADMIN, remaining_uses=-1,
|
||||
message="ok", user_email="a@nomadkaraoke.com", is_admin=True)
|
||||
|
||||
|
||||
def test_parse_route_happy_path(monkeypatch):
|
||||
# Self-contained: set our own admin override rather than relying on the
|
||||
# conftest autouse fixture (which can be disturbed by earlier tests in a
|
||||
# full-suite run and leave the endpoint returning 401).
|
||||
from backend.api.routes import parse_titles as route
|
||||
|
||||
async def fake_parse(items, **kw):
|
||||
return [{"id": str(it["id"]), "artist": "Queen",
|
||||
"title": "Bohemian Rhapsody", "confidence": 0.9} for it in items]
|
||||
|
||||
monkeypatch.setattr(route, "parse_titles", fake_parse)
|
||||
original = app.dependency_overrides.get(require_admin)
|
||||
app.dependency_overrides[require_admin] = _admin_result
|
||||
try:
|
||||
client = TestClient(app)
|
||||
resp = client.post("/api/parse-karaoke-titles", json={
|
||||
"items": [{"id": "1", "filename": "x.mp4", "source": "youtube"}]})
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["results"][0]["id"] == "1"
|
||||
assert body["results"][0]["artist"] == "Queen"
|
||||
finally:
|
||||
if original is not None:
|
||||
app.dependency_overrides[require_admin] = original
|
||||
else:
|
||||
app.dependency_overrides.pop(require_admin, None)
|
||||
|
||||
|
||||
def test_parse_route_requires_admin():
|
||||
"""The route is gated by require_admin: a non-admin dependency -> 403."""
|
||||
original = app.dependency_overrides.get(require_admin)
|
||||
|
||||
def non_admin():
|
||||
from fastapi import HTTPException
|
||||
raise HTTPException(status_code=403, detail="Admin access required")
|
||||
|
||||
app.dependency_overrides[require_admin] = non_admin
|
||||
try:
|
||||
client = TestClient(app)
|
||||
resp = client.post("/api/parse-karaoke-titles",
|
||||
json={"items": [{"id": "1", "filename": "x"}]})
|
||||
assert resp.status_code == 403
|
||||
finally:
|
||||
if original is not None:
|
||||
app.dependency_overrides[require_admin] = original
|
||||
else:
|
||||
app.dependency_overrides.pop(require_admin, None)
|
||||
Reference in new issue
Block a user