From cd11088be58f4412685b241fbff76784554d5a69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrew=20Beveridge=20=E2=98=84=EF=B8=8F?= Date: Fri, 6 Nov 2026 20:00:50 -0500 Subject: [PATCH] feat(match-judge): ground AI corrections in live Spotify (v0.294.0) (#1256) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary At a live kjbox night (2026-11-05, job `f01c9997`) a singer typed "Cole Phillips – Roll of Dice", a 2025 release missing from our BigQuery catalogue. Gemini "corrected" it to "Roll of the Dice" twice: 1. in the free-text resolver; 2. in the job form's full judge, after the singer had tapped "keep what I typed". Both times the catalogue check found nothing, and the guess still went out as `confident` ("No catalog match → AI verdict unchanged"). Meanwhile the audio search had found the real track on Spotify at score 1.0. This PR asks **live Spotify** before trusting Gemini, and **grounds** every AI correction in real data. Depends on karaoke-decide's new `GET /api/partner/gen/spotify-search`. Until that's deployed, the call returns `None` ("no evidence"). That is safe, but it makes AI corrections non-confident in the meantime, so **merge and deploy decide first**. ## Changes - `catalog_proxy_service.spotify_search`: calls decide's partner route with `X-Partner-Secret` (the existing `decide_partner_secret`). Cached 10 min. Returns `None` when there's no evidence. Never raises. - `match_judge/service.py`, fast and full passes: - When the catalogue can't confirm the typed song, an exact live-Spotify hit is a confident catalog verdict. No Gemini call: "Roll of dice" → "Roll of Dice". - Spotify results go to Gemini as candidates. - **Grounding rule:** a confident AI cosmetic/content verdict must name a song in the candidates, or found by one Spotify lookup of it. Otherwise `confident=False`, which both clients already show as "Did you mean…?" and never apply. - `match_judge/free_text.py` (`resolve_and_tidy`): - Live Spotify first, capped at 3 s so the worst case stays inside decide's 25 s wait. - A result that *is* the query (either order) answers it directly. - Otherwise the results go into the Gemini prompt. - An AI answer the tidy can't confirm is not confident. A timed-out tidy keeps today's behaviour (`unavailable`, not cached). - Docs: API.md (judge + resolver), LESSONS-LEARNED entry. Version 0.294.0. ## Testing - New and updated unit tests: `tests/services/match_judge/test_service.py` (Spotify confirm in fast/full, catalogue hit skips Spotify, unconfirmed downgrade with the Cole Phillips case, second-lookup confirm, any credited artist, Spotify failure, ambiguous untouched, merge), `test_match_judge_free_text.py` (exact Spotify answer without Gemini, both word orders, downgrade, AI pick from live results kept, results in prompt, Spotify timeout/failure fallback), `tests/services/test_catalog_proxy_spotify.py` (secret header, cache, no-evidence cases, network error, no secret). - `make test-backend-unit`: 7212 passed. The 8 failures in `test_emulator_integration.py` also fail on `main` without the emulator stack. - Local CodeRabbit review: no findings. @coderabbitai ignore 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_012RYLVUQ68KFYtZF7vEfhAq Co-authored-by: Claude Opus 5.5 --- backend/services/catalog_proxy_service.py | 40 +++++ backend/services/match_judge/free_text.py | 98 +++++++++- backend/services/match_judge/service.py | 98 +++++++++- .../services/match_judge/test_service.py | 167 +++++++++++++++++- .../services/test_catalog_proxy_spotify.py | 81 +++++++++ backend/tests/test_match_judge_free_text.py | 95 +++++++++- docs/API.md | 25 ++- docs/LESSONS-LEARNED.md | 17 ++ pyproject.toml | 2 +- 9 files changed, 607 insertions(+), 16 deletions(-) create mode 100644 backend/tests/services/test_catalog_proxy_spotify.py diff --git a/backend/services/catalog_proxy_service.py b/backend/services/catalog_proxy_service.py index 944f1524..1a70e0ba 100644 --- a/backend/services/catalog_proxy_service.py +++ b/backend/services/catalog_proxy_service.py @@ -103,3 +103,43 @@ async def search_tracks(query: str, artist: str | None = None, limit: int = 10) except Exception as e: logger.warning(f"Failed to search tracks from karaoke-decide: {e}") return [] + + +SPOTIFY_SEARCH_TIMEOUT = 6 # seconds; decide's own Spotify call is capped at 4 s +SPOTIFY_CACHE_TTL_SECONDS = 600 + + +async def spotify_search(query: str, limit: int = 10) -> list[dict] | None: + """Live Spotify track search via karaoke-decide (partner route, X-Partner-Secret). + + Lets the match judge confirm a song our BigQuery catalogue doesn't know (it's a July 2025 + Spotify snapshot + MusicBrainz, so 2025+ releases and small artists are missing). Returns + track dicts with ``artist_name`` / ``track_name`` (the catalogue candidate shape), or + **None when there's no evidence either way**: secret unset, decide down, or decide's + Spotify breaker open. Never raises. + """ + secret = getattr(settings, "decide_partner_secret", "") or "" + q = " ".join((query or "").split()) + if not secret or len(q) < 2: + return None + cache_key = f"spotify:{q.casefold()}:{limit}" + cached = _cache_get(cache_key) + if cached is not None: + return cached + url = f"{_get_base_url()}/api/partner/gen/spotify-search" + try: + async with httpx.AsyncClient(timeout=SPOTIFY_SEARCH_TIMEOUT) as client: + resp = await client.get( + url, params={"q": q[:300], "limit": min(limit, 10)}, + headers={"X-Partner-Secret": secret}, + ) + resp.raise_for_status() + data = resp.json() + except Exception as e: + logger.warning(f"Spotify search via karaoke-decide failed: {e}") + return None + if not isinstance(data, dict) or not data.get("available"): + return None + tracks = [t for t in data.get("tracks") or [] if isinstance(t, dict)] + _cache_set(cache_key, tracks, SPOTIFY_CACHE_TTL_SECONDS) + return tracks diff --git a/backend/services/match_judge/free_text.py b/backend/services/match_judge/free_text.py index b0c5bf08..c4fd5636 100644 --- a/backend/services/match_judge/free_text.py +++ b/backend/services/match_judge/free_text.py @@ -9,6 +9,12 @@ verdict shape (kind cosmetic/content/ambiguous/none) plus the typed split. :func:`judge_match` catalog pass, so the singer gets exactly the canonical artist/title gen's AudioSourceStep would "tidy" to (the catalog can't parse a one-line query itself — the AI only does the split + typo fix). + +Live Spotify (2026-11-06, see ``service.py``'s grounding rule): before Gemini, the +raw query goes to a live Spotify search. A result whose artist + title IS the query +(either order, after normalising) answers it without Gemini. Otherwise Gemini gets +those results as candidates, and a confident answer that neither the catalogue nor +Spotify can confirm is returned with ``confident: false``. """ from __future__ import annotations @@ -26,6 +32,7 @@ logger = logging.getLogger(__name__) Generate = Callable[[str, str, str], Awaitable[dict]] Judge = Callable[[str, str], Awaitable[MatchVerdict]] +SpotifySearch = Callable[[str], Awaitable[Optional[list[dict]]]] _VALID_KINDS = {"cosmetic", "content", "ambiguous"} @@ -45,6 +52,10 @@ GEMINI_ATTEMPT_TIMEOUT_S = 8.0 GEMINI_MIN_DEADLINE_MS = 10000 # the API's minimum request deadline GEMINI_ATTEMPTS = 2 CATALOG_TIDY_TIMEOUT_S = 4.0 +# Live Spotify lookup before Gemini. Kept short so the worst case still fits decide's +# 25 s wait: 3 + 2 x 8 (Gemini) + 4 (tidy) = 23 s. A miss just means no candidates. +SPOTIFY_TIMEOUT_S = 3.0 +SPOTIFY_CANDIDATES_FOR_AI = 8 # In-flight catalog searches. One the tidy gave up on keeps running (warming decide and # the catalog proxy's cache for the next ask); strong refs so it isn't GC'd mid-flight. @@ -75,7 +86,10 @@ _SYSTEM_PROMPT = ( " 'ambiguous' – several real songs are plausible; list them in alternatives.\n" " 'none' – not a recognisable song, or you are not sure.\n" "confident: true only when you are sure which released song is meant. Never " - "invent songs; prefer kind='none' when unsure. canonical_artist/canonical_title " + "invent songs; prefer kind='none' when unsure. When live Spotify results are " + "listed, they are real released songs: if one is the song meant, use its exact " + "artist and title. Never 'correct' a query to a different, more familiar title " + "when a listed result matches what was typed. canonical_artist/canonical_title " "must use official formatting. alternatives is a list of {artist,title} (max 4)." ) @@ -167,8 +181,18 @@ def _retryable(exc: BaseException) -> bool: return not (isinstance(code, int) and 400 <= code < 500 and code != 429) +def _user_prompt(query: str, candidates: Optional[list[dict]]) -> str: + lines = [f'Singer typed: "{query}"'] + if candidates: + lines.append("Live Spotify search results for this query (real released songs):") + lines += [f'- {c.get("artist_name", "")} — {c.get("track_name", "")}' + for c in candidates[:SPOTIFY_CANDIDATES_FOR_AI]] + return "\n".join(lines) + + async def resolve_free_text(query: str, *, generate: Optional[Generate] = None, - model: Optional[str] = None) -> dict: + model: Optional[str] = None, + candidates: Optional[list[dict]] = None) -> dict: """Split + correct a free-text song query. Never raises: failures → kind 'none'. Each Gemini attempt is bounded by GEMINI_ATTEMPT_TIMEOUT_S; a transient failure @@ -180,7 +204,7 @@ async def resolve_free_text(query: str, *, generate: Optional[Generate] = None, for attempt in range(1, GEMINI_ATTEMPTS + 1): try: data = await asyncio.wait_for( - gen(model or _model(), _SYSTEM_PROMPT, f'Singer typed: "{query}"'), + gen(model or _model(), _SYSTEM_PROMPT, _user_prompt(query, candidates)), GEMINI_ATTEMPT_TIMEOUT_S) except Exception as e: reason = f"timed out after {GEMINI_ATTEMPT_TIMEOUT_S:g}s" if isinstance(e, asyncio.TimeoutError) else str(e) @@ -192,8 +216,49 @@ async def resolve_free_text(query: str, *, generate: Optional[Generate] = None, return _none(query, "unavailable") # pragma: no cover - loop always returns +def _split_like_query(query: str, artist: str, title: str) -> tuple[str, str]: + """The query's own words as (typed_artist, typed_title), given the song it names.""" + words = query.split() + n_artist = len(artist.split()) + if normalize_for_match(" ".join(words[:n_artist])) == normalize_for_match(artist): + return " ".join(words[:n_artist]), " ".join(words[n_artist:]) + n_title = len(title.split()) + if normalize_for_match(" ".join(words[:n_title])) == normalize_for_match(title): + return " ".join(words[n_title:]), " ".join(words[:n_title]) + return artist, title + + +def _spotify_exact(query: str, live: list[dict]) -> Optional[dict]: + """A live result whose artist + title is exactly the query (either order) → its verdict.""" + typed = normalize_for_match(query) + if not typed: + return None + for c in live: + ca, ct = (c.get("artist_name") or "").strip(), (c.get("track_name") or "").strip() + if not (ca and ct): + continue + if typed in (normalize_for_match(f"{ca} {ct}"), normalize_for_match(f"{ct} {ca}")): + typed_artist, typed_title = _split_like_query(query, ca, ct) + return { + "kind": KIND_COSMETIC, "confident": True, + "typed_artist": typed_artist, "typed_title": typed_title, + "canonical_artist": ca, "canonical_title": ct, "alternatives": [], + "engine": "catalog", "reason": "live Spotify match", + } + return None + + +async def _live_spotify(spotify: SpotifySearch, query: str) -> list[dict]: + try: + return list(await asyncio.wait_for(spotify(query), SPOTIFY_TIMEOUT_S) or []) + except Exception as e: # timeout / decide down: no evidence, carry on as before + logger.warning(f"resolve_and_tidy live Spotify search failed: {e!r}") + return [] + + async def resolve_and_tidy(query: str, *, generate: Optional[Generate] = None, - judge: Optional[Judge] = None, model: Optional[str] = None) -> dict: + judge: Optional[Judge] = None, model: Optional[str] = None, + spotify: Optional[SpotifySearch] = None) -> dict: """Free-text split/correct, then gen's catalog tidy (the job flow's fast pass). The AI verdict's song — its canonical pick when confident, else the typed @@ -201,9 +266,17 @@ async def resolve_and_tidy(query: str, *, generate: Optional[Generate] = None, "fast": catalog only, no second AI call). A confident catalog match wins: its formatting becomes the canonical artist/title, kind "cosmetic" when the query already named that song (only casing/punctuation/word order differ), - else "content". No catalog match → the AI verdict unchanged. Never raises. + else "content". No catalog match → the AI verdict, but a confident answer + that live Spotify didn't list either comes back ``confident: False`` + (grounding rule). A timed-out tidy keeps the AI verdict as "unavailable" + (transient, not cached). A live Spotify result that IS the query answers + before Gemini runs. Never raises. """ - verdict = await resolve_free_text(query, generate=generate, model=model) + live = await _live_spotify(spotify or _default_spotify, query) + exact = _spotify_exact(query, live) + if exact is not None: + return exact + verdict = await resolve_free_text(query, generate=generate, model=model, candidates=live) kind = verdict.get("kind") if kind in ("cosmetic", "content") and verdict.get("confident"): artist, title = verdict["canonical_artist"], verdict["canonical_title"] @@ -231,6 +304,14 @@ async def resolve_and_tidy(query: str, *, generate: Optional[Generate] = None, return {**verdict, "reason": "unavailable"} if not (cat.confident and cat.engine == "catalog" and cat.kind in (KIND_COSMETIC, KIND_NONE) and cat.canonical_artist and cat.canonical_title): + if verdict.get("confident") and kind in ("cosmetic", "content"): + from backend.services.match_judge.service import names_song + if not names_song(live, artist, title): + # Grounding rule: neither the catalogue nor live Spotify has the song the + # AI named, so it's a guess. Offer it at most ("Did you mean…?"). + logger.info(f"resolve_and_tidy: AI answer {artist!r} / {title!r} unconfirmed; not confident") + return {**verdict, "confident": False, + "reason": f"{verdict.get('reason') or ''} [unconfirmed: not in catalog or Spotify]".strip()} return verdict ca, ct = cat.canonical_artist, cat.canonical_title typed = normalize_for_match(query) @@ -252,6 +333,11 @@ async def _default_judge(artist: str, title: str) -> MatchVerdict: return await judge_match(artist, title, stage="fast") +async def _default_spotify(query: str) -> Optional[list[dict]]: + from backend.services.catalog_proxy_service import spotify_search + return await spotify_search(query) + + async def _default_generate(model: str, system_prompt: str, user_prompt: str) -> dict: return await asyncio.to_thread(_blocking_generate, model, system_prompt, user_prompt) diff --git a/backend/services/match_judge/service.py b/backend/services/match_judge/service.py index 3cfaafff..10fa61c7 100644 --- a/backend/services/match_judge/service.py +++ b/backend/services/match_judge/service.py @@ -3,14 +3,26 @@ Composes the deterministic+catalog classifier with the AI judge. The AI judge runs ONLY when the classifier can't confidently decide, and never blocks or errors the caller — any failure degrades to "no suggestion". + +Evidence comes from two places: decide's BigQuery catalogue (a July 2025 Spotify +snapshot + MusicBrainz) and, when that can't confirm the typed song, a **live +Spotify search** (via decide's partner route). The live search is what knows 2025+ +releases and small artists. + +**Grounding rule** (2026-11-06, "Cole Phillips – Roll of Dice" became "Roll of the +Dice"): a confident AI correction must name a song the catalogue or live Spotify +actually has. Otherwise it is downgraded to ``confident=False``, which both clients +show as "Did you mean…?" and never apply. """ from __future__ import annotations import logging from typing import Awaitable, Callable, Literal, Optional -from backend.services.match_judge.classifier import classify_catalog_match -from backend.services.match_judge.verdict import KIND_NONE, MatchVerdict +from dataclasses import replace + +from backend.services.match_judge.classifier import classify_catalog_match, normalize_for_match +from backend.services.match_judge.verdict import KIND_CONTENT, KIND_COSMETIC, KIND_NONE, MatchVerdict logger = logging.getLogger(__name__) @@ -25,6 +37,8 @@ WEAK_TIER = 3 SearchTracks = Callable[[str, Optional[str], int], Awaitable[list[dict]]] AiJudge = Callable[[str, str, list[dict], Optional[int]], Awaitable[MatchVerdict]] +# Live Spotify track search: candidate dicts, or None when there's no evidence either way. +SpotifySearch = Callable[[str], Awaitable[Optional[list[dict]]]] async def _default_search_tracks(query: str, artist: Optional[str], limit: int) -> list[dict]: @@ -32,6 +46,72 @@ async def _default_search_tracks(query: str, artist: Optional[str], limit: int) return await search_tracks(query, artist, limit) +async def _default_spotify_search(query: str) -> Optional[list[dict]]: + from backend.services.catalog_proxy_service import spotify_search + return await spotify_search(query) + + +async def _safe_spotify(spotify: SpotifySearch, query: str) -> list[dict]: + try: + return list(await spotify(query) or []) + except Exception: + logger.warning("match-judge Spotify search failed; continuing without it", exc_info=True) + return [] + + +def _merge(*groups: list[dict]) -> list[dict]: + """Concatenate candidate lists, dropping repeats of the same artist/title.""" + seen: set[tuple[str, str]] = set() + out: list[dict] = [] + for group in groups: + for c in group: + key = (normalize_for_match(c.get("artist_name") or ""), normalize_for_match(c.get("track_name") or "")) + if key in seen or not all(key): + continue + seen.add(key) + out.append(c) + return out + + +def _names(candidate: dict) -> list[str]: + names = [candidate.get("artist_name") or ""] + list(candidate.get("artist_names") or []) + return [n for n in names if n] + + +def names_song(candidates: list[dict], artist: str, title: str) -> bool: + """True when a candidate is this artist/title, after normalising. Any credited artist counts.""" + key_artist, key_title = normalize_for_match(artist), normalize_for_match(title) + if not key_artist or not key_title: + return False + return any( + normalize_for_match(c.get("track_name") or "") == key_title + and any(normalize_for_match(n) == key_artist for n in _names(c)) + for c in candidates + ) + + +async def ground_verdict( + verdict: MatchVerdict, candidates: list[dict], spotify: SpotifySearch +) -> MatchVerdict: + """Apply the grounding rule to an AI verdict (see module docstring). + + A confident cosmetic/content verdict is kept only when its canonical song is + among ``candidates``, or turns up in one live Spotify search for it. + """ + if not (verdict.confident and verdict.kind in (KIND_COSMETIC, KIND_CONTENT)): + return verdict + artist, title = verdict.canonical_artist, verdict.canonical_title + if names_song(candidates, artist, title): + return verdict + if names_song(await _safe_spotify(spotify, f"{artist} {title}"), artist, title): + return verdict + logger.info( + f"match-judge: AI suggestion {artist!r} / {title!r} not found in catalog or Spotify; " + "downgraded to not confident" + ) + return replace(verdict, confident=False, reason=f"{verdict.reason} [unconfirmed: not in catalog or Spotify]".strip()) + + async def _default_ai_judge( artist: str, title: str, candidates: list[dict], audio_tier: Optional[int] ) -> MatchVerdict: @@ -55,6 +135,7 @@ async def judge_match( stage: Stage = "full", search_tracks: Optional[SearchTracks] = None, ai_judge: Optional[AiJudge] = None, + spotify_search: Optional[SpotifySearch] = None, enabled: Optional[bool] = None, ) -> MatchVerdict: """Return a :class:`MatchVerdict` for the typed artist/title. @@ -73,6 +154,7 @@ async def judge_match( """ search = search_tracks or _default_search_tracks ai = ai_judge or _default_ai_judge + spotify = spotify_search or _default_spotify_search if enabled is None: enabled = _default_enabled() @@ -86,6 +168,14 @@ async def judge_match( candidates = [] catalog_verdict = classify_catalog_match(artist, title, candidates) + if catalog_verdict is None: + # The snapshot doesn't have it: ask live Spotify. An exact (normalised) hit + # confirms the typed song, e.g. "Roll of dice" -> "Roll of Dice". + live = await _safe_spotify(spotify, f"{artist} {title}") + spotify_verdict = classify_catalog_match(artist, title, live) + if spotify_verdict is not None: + catalog_verdict = replace(spotify_verdict, reason=f"{spotify_verdict.reason} (live Spotify)") + candidates = _merge(candidates, live) # Fast pass: catalog only. Never touches the AI layer. if stage == "fast": @@ -115,6 +205,7 @@ async def judge_match( exc_info=True, ) return catalog_verdict + ai_verdict = await ground_verdict(ai_verdict, candidates, spotify) if ai_verdict.confident and ai_verdict.kind != KIND_NONE: return ai_verdict return catalog_verdict @@ -125,7 +216,7 @@ async def judge_match( ) try: - return await ai(artist, title, candidates, audio_tier) + ai_verdict = await ai(artist, title, candidates, audio_tier) except Exception as exc: from backend.services.gemini_client import note_gemini_failure @@ -139,3 +230,4 @@ async def judge_match( return MatchVerdict( KIND_NONE, True, artist, title, engine="ai", reason="ai failed" ) + return await ground_verdict(ai_verdict, candidates, spotify) diff --git a/backend/tests/services/match_judge/test_service.py b/backend/tests/services/match_judge/test_service.py index 95a55672..66c6a633 100644 --- a/backend/tests/services/match_judge/test_service.py +++ b/backend/tests/services/match_judge/test_service.py @@ -26,6 +26,22 @@ def _ai_never(*_a, **_k): raise AssertionError("AI judge must not be called") +def _spotify(*pairs, calls=None): + async def _search(query): + if calls is not None: + calls.append(query) + return [{"artist_name": a, "track_name": t} for a, t in pairs] + return _search + + +@pytest.fixture(autouse=True) +def _no_live_spotify(monkeypatch): + """Tests that don't inject a Spotify search get "no evidence" (never the network).""" + async def none(query): + return None + monkeypatch.setattr("backend.services.match_judge.service._default_spotify_search", none) + + @pytest.mark.asyncio async def test_catalog_cosmetic_match_skips_ai(): calls = {"ai": 0} @@ -68,12 +84,17 @@ async def test_no_catalog_match_calls_ai(): audio_tier=3, search_tracks=_candidates(("Queen", "Radio Ga Ga")), ai_judge=ai, + spotify_search=_spotify(("Queen", "Bohemian Rhapsody")), enabled=True, ) assert verdict.kind == KIND_CONTENT assert verdict.engine == "ai" + assert verdict.confident is True # grounded: live Spotify lists the AI's song assert seen["tier"] == 3 - assert seen["candidates"] == [{"artist_name": "Queen", "track_name": "Radio Ga Ga"}] + assert seen["candidates"] == [ + {"artist_name": "Queen", "track_name": "Radio Ga Ga"}, + {"artist_name": "Queen", "track_name": "Bohemian Rhapsody"}, + ] @pytest.mark.asyncio @@ -165,7 +186,7 @@ async def test_weak_tier_catalog_match_is_verified_and_overridden_by_ai(): verdict = await judge_match( "Queen", "Bohemian Rapsody", audio_tier=3, - search_tracks=_candidates(("Queen", "Bohemian Rapsody")), + search_tracks=_candidates(("Queen", "Bohemian Rapsody"), ("Queen", "Bohemian Rhapsody")), ai_judge=ai, enabled=True, ) @@ -174,6 +195,23 @@ async def test_weak_tier_catalog_match_is_verified_and_overridden_by_ai(): assert verdict.engine == "ai" +@pytest.mark.asyncio +async def test_weak_tier_unconfirmed_ai_override_keeps_catalog_verdict(): + async def ai(artist, title, candidates, audio_tier): + return MatchVerdict(KIND_CONTENT, True, "Queen", "Bohemian Rhapsody", engine="ai") + + verdict = await judge_match( + "Queen", "Bohemian Rapsody", + audio_tier=3, + search_tracks=_candidates(("Queen", "Bohemian Rapsody")), + ai_judge=ai, + spotify_search=_spotify(), + enabled=True, + ) + # The AI's song is in neither source, so it can't override the catalog match. + assert verdict.engine == "catalog" + + @pytest.mark.asyncio async def test_strong_tier_catalog_match_still_skips_ai(): verdict = await judge_match( @@ -270,3 +308,128 @@ async def test_catalog_search_failure_still_lets_ai_run(): enabled=True, ) assert verdict.engine == "ai" + + +# --- Live Spotify + grounding (2026-11-06: "Cole Phillips – Roll of Dice" became "Roll of the Dice") --- + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stage", ["fast", "full"]) +async def test_live_spotify_confirms_typed_song_missing_from_catalog(stage): + calls = [] + verdict = await judge_match( + "Cole Phillips", "Roll of dice", + audio_tier=2, stage=stage, + search_tracks=_candidates(), # the July 2025 snapshot doesn't have it + spotify_search=_spotify(("Cole Phillips", "Roll of Dice"), ("Bruce Springsteen", "Roll of the Dice"), + calls=calls), + ai_judge=_ai_never, + enabled=True, + ) + assert verdict.kind == KIND_COSMETIC and verdict.confident is True + assert (verdict.canonical_artist, verdict.canonical_title) == ("Cole Phillips", "Roll of Dice") + assert verdict.engine == "catalog" and "live Spotify" in verdict.reason + assert verdict.needs_ai is False + assert calls == ["Cole Phillips Roll of dice"] + + +@pytest.mark.asyncio +async def test_catalog_hit_never_calls_spotify(): + calls = [] + await judge_match( + "paramore", "big man, little dignity", + search_tracks=_candidates(("Paramore", "Big Man, Little Dignity")), + spotify_search=_spotify(calls=calls), ai_judge=_ai_never, enabled=True, + ) + assert calls == [] + + +@pytest.mark.asyncio +async def test_unconfirmed_ai_correction_is_downgraded(): + """The incident: no source has the song the AI named, so it must not be applied.""" + calls = [] + + async def ai(artist, title, candidates, audio_tier): + return MatchVerdict(KIND_CONTENT, True, "Cole Phillips", "Roll of the Dice", engine="ai", reason="typo") + + verdict = await judge_match( + "Cole Phillips", "Roll of dice", + audio_tier=2, + search_tracks=_candidates(), + spotify_search=_spotify(("Bruce Springsteen", "Roll of the Dice"), calls=calls), + ai_judge=ai, enabled=True, + ) + assert verdict.kind == KIND_CONTENT and verdict.confident is False + assert verdict.canonical_title == "Roll of the Dice" # still offered as "Did you mean…?" + assert "unconfirmed" in verdict.reason + # One search for what was typed, one for the AI's song. + assert calls == ["Cole Phillips Roll of dice", "Cole Phillips Roll of the Dice"] + + +@pytest.mark.asyncio +async def test_ai_correction_confirmed_by_second_spotify_lookup(): + async def ai(artist, title, candidates, audio_tier): + return MatchVerdict(KIND_CONTENT, True, "The Strokes", "Machu Picchu", engine="ai") + + async def spotify(query): + return [{"artist_name": "The Strokes", "track_name": "Machu Picchu"}] if "Machu" in query else [] + + verdict = await judge_match( + "the stokes", "max picu", audio_tier=3, + search_tracks=_candidates(), spotify_search=spotify, ai_judge=ai, enabled=True, + ) + assert verdict.confident is True and verdict.canonical_title == "Machu Picchu" + + +@pytest.mark.asyncio +async def test_grounding_accepts_any_credited_artist(): + async def ai(artist, title, candidates, audio_tier): + return MatchVerdict(KIND_CONTENT, True, "Sean Paul", "Breathe", engine="ai") + + async def spotify(query): + return [{"artist_name": "Blu Cantrell", "artist_names": ["Blu Cantrell", "Sean Paul"], + "track_name": "Breathe"}] + + verdict = await judge_match( + "sean pual", "breath", audio_tier=3, + search_tracks=_candidates(), spotify_search=spotify, ai_judge=ai, enabled=True, + ) + assert verdict.confident is True + + +@pytest.mark.asyncio +async def test_spotify_unavailable_or_failing_means_no_evidence(): + async def ai(artist, title, candidates, audio_tier): + return MatchVerdict(KIND_CONTENT, True, "Queen", "Bohemian Rhapsody", engine="ai") + + async def broken(query): + raise RuntimeError("decide down") + + for spotify in (broken, None): + verdict = await judge_match( + "queen", "bohemian rapsody", audio_tier=3, + search_tracks=_candidates(), spotify_search=spotify, ai_judge=ai, enabled=True, + ) + assert verdict.kind == KIND_CONTENT and verdict.confident is False + + +@pytest.mark.asyncio +async def test_ambiguous_and_none_ai_verdicts_are_not_grounded(): + calls = [] + + async def ai(artist, title, candidates, audio_tier): + return MatchVerdict(KIND_AMBIGUOUS, True, artist, title, + alternatives=[{"artist": "A", "title": "T"}], engine="ai") + + verdict = await judge_match( + "x", "y", search_tracks=_candidates(), spotify_search=_spotify(calls=calls), ai_judge=ai, enabled=True, + ) + assert verdict.kind == KIND_AMBIGUOUS and verdict.confident is True + assert calls == ["x y"] # only the typed-song lookup + + +def test_merge_drops_repeats_and_blanks(): + from backend.services.match_judge.service import _merge + a = [{"artist_name": "Queen", "track_name": "Radio Ga Ga"}, {"artist_name": "", "track_name": "x"}] + b = [{"artist_name": "queen", "track_name": "radio ga ga!"}, {"artist_name": "Queen", "track_name": "Innuendo"}] + assert [c["track_name"] for c in _merge(a, b)] == ["Radio Ga Ga", "Innuendo"] diff --git a/backend/tests/services/test_catalog_proxy_spotify.py b/backend/tests/services/test_catalog_proxy_spotify.py new file mode 100644 index 00000000..adbd99c5 --- /dev/null +++ b/backend/tests/services/test_catalog_proxy_spotify.py @@ -0,0 +1,81 @@ +"""catalog_proxy_service.spotify_search: gen -> decide's partner Spotify search.""" +import httpx +import pytest + +from backend.services import catalog_proxy_service as proxy + + +@pytest.fixture(autouse=True) +def _clean(monkeypatch): + proxy._cache.clear() + monkeypatch.setattr(proxy.settings, "decide_partner_secret", "s3cret", raising=False) + monkeypatch.setattr(proxy.settings, "karaoke_decide_api_url", "https://decide.test", raising=False) + yield + proxy._cache.clear() + + +def _mock(monkeypatch, handler): + real = httpx.AsyncClient + seen = [] + + def wrapped(request): + seen.append(request) + return handler(request) + + monkeypatch.setattr(proxy.httpx, "AsyncClient", + lambda **kw: real(transport=httpx.MockTransport(wrapped), **kw)) + return seen + + +_BODY = {"available": True, "tracks": [{"artist_name": "Cole Phillips", "track_name": "Roll of Dice"}]} + + +@pytest.mark.asyncio +async def test_sends_secret_and_returns_tracks(monkeypatch): + seen = _mock(monkeypatch, lambda r: httpx.Response(200, json=_BODY)) + tracks = await proxy.spotify_search(" Cole Phillips Roll of dice ") + assert tracks == _BODY["tracks"] + req = seen[0] + assert req.url.path == "/api/partner/gen/spotify-search" + assert req.url.params["q"] == "Cole Phillips Roll of dice" + assert req.url.params["limit"] == "10" + assert req.headers["X-Partner-Secret"] == "s3cret" + + +@pytest.mark.asyncio +async def test_results_are_cached(monkeypatch): + seen = _mock(monkeypatch, lambda r: httpx.Response(200, json=_BODY)) + await proxy.spotify_search("Cole Phillips Roll of dice") + await proxy.spotify_search("cole phillips roll of dice") + assert len(seen) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("response", [ + httpx.Response(200, json={"available": False, "tracks": []}), # decide's breaker open + httpx.Response(401, json={"detail": "partner_auth"}), + httpx.Response(503, json={"detail": "partner_disabled"}), + httpx.Response(200, json=["not", "a", "dict"]), +]) +async def test_no_evidence_is_none_and_not_cached(monkeypatch, response): + seen = _mock(monkeypatch, lambda r: response) + assert await proxy.spotify_search("some song") is None + assert await proxy.spotify_search("some song") is None + assert len(seen) == 2 + + +@pytest.mark.asyncio +async def test_network_error_is_none(monkeypatch): + def boom(request): + raise httpx.ConnectError("down", request=request) + _mock(monkeypatch, boom) + assert await proxy.spotify_search("some song") is None + + +@pytest.mark.asyncio +async def test_no_secret_or_short_query_skips_the_call(monkeypatch): + seen = _mock(monkeypatch, lambda r: httpx.Response(200, json=_BODY)) + assert await proxy.spotify_search("x") is None + monkeypatch.setattr(proxy.settings, "decide_partner_secret", "", raising=False) + assert await proxy.spotify_search("some song") is None + assert seen == [] diff --git a/backend/tests/test_match_judge_free_text.py b/backend/tests/test_match_judge_free_text.py index 9e69783e..b48e56b0 100644 --- a/backend/tests/test_match_judge_free_text.py +++ b/backend/tests/test_match_judge_free_text.py @@ -53,11 +53,18 @@ def test_ambiguous_keeps_up_to_four_alternatives(): assert v["kind"] == "ambiguous" and len(v["alternatives"]) == 4 +async def _no_spotify(q): + return None + + # ---- resolve_and_tidy: AI split → the job flow's judge_match catalog pass -def _tidy(query, ai, judge_result=None, judge_exc=None): +def _tidy(query, ai, judge_result=None, judge_exc=None, live=None): seen = [] + async def spotify(q): + return live + async def gen(model, system_prompt, user_prompt): return ai @@ -66,7 +73,7 @@ def _tidy(query, ai, judge_result=None, judge_exc=None): if judge_exc: raise judge_exc return judge_result - v = asyncio.run(resolve_and_tidy(query, generate=gen, judge=judge, model="m")) + v = asyncio.run(resolve_and_tidy(query, generate=gen, judge=judge, model="m", spotify=spotify)) return v, seen @@ -125,6 +132,87 @@ def test_no_catalog_match_keeps_ai_verdict(judge_result, judge_exc): assert v["engine"] == "ai" and v["canonical_title"] == "Push Up on Me" # A failed catalog pass is transient — the route must not cache it. assert (v["reason"] == "unavailable") is (judge_exc is not None) + # Grounding: a check that ran and found nothing makes the AI answer unconfirmed; + # a check that never ran (decide down) leaves it as it was. + assert v["confident"] is (judge_exc is not None) + + +# ---- live Spotify (2026-11-06: "Cole Phillips – Roll of Dice" became "Roll of the Dice") + +_COLE = [{"artist_name": "Cole Phillips", "track_name": "Roll of Dice"}, + {"artist_name": "Bruce Springsteen", "track_name": "Roll of the Dice"}] + + +def test_live_spotify_exact_answers_without_gemini(): + v, seen = _tidy("Roll of dice Cole Phillips", None, live=_COLE) # ai=None: Gemini must not matter + assert v["kind"] == "cosmetic" and v["confident"] is True and v["engine"] == "catalog" + assert (v["canonical_artist"], v["canonical_title"]) == ("Cole Phillips", "Roll of Dice") + assert (v["typed_artist"], v["typed_title"]) == ("Cole Phillips", "Roll of dice") + assert seen == [] + + +def test_live_spotify_exact_artist_first(): + v, _ = _tidy("cole phillips roll of dice", None, live=_COLE) + assert (v["typed_artist"], v["typed_title"]) == ("cole phillips", "roll of dice") + assert v["canonical_title"] == "Roll of Dice" + + +def test_unconfirmed_ai_correction_is_not_confident(): + """The incident: Gemini 'corrects' to a more familiar title no source has for this artist.""" + ai = {"kind": "content", "confident": True, "typed_artist": "Cole Philips", + "typed_title": "Roll of dice", "canonical_artist": "Cole Phillips", + "canonical_title": "Roll of the Dice"} + inconclusive = MatchVerdict("none", False, "Cole Phillips", "Roll of the Dice", + engine="catalog", needs_ai=True) + v, _ = _tidy("Roll of dice Cole Philips", ai, inconclusive, live=[]) + assert v["kind"] == "content" and v["confident"] is False + assert "unconfirmed" in v["reason"] + + +def test_ai_pick_from_live_results_is_kept(): + ai = {"kind": "content", "confident": True, "typed_artist": "Cole Philips", + "typed_title": "Roll of dice", "canonical_artist": "Cole Phillips", + "canonical_title": "Roll of Dice"} + inconclusive = MatchVerdict("none", False, "Cole Phillips", "Roll of Dice", + engine="catalog", needs_ai=True) + v, _ = _tidy("Roll of dice Cole Philips", ai, inconclusive, live=_COLE) + assert v["confident"] is True and v["canonical_title"] == "Roll of Dice" + + +def test_live_results_are_handed_to_gemini(): + prompts = [] + + async def gen(model, system_prompt, user_prompt): + prompts.append(user_prompt) + return {"kind": "none", "confident": False} + + async def spotify(q): + return _COLE + + asyncio.run(resolve_and_tidy("Roll of dice Cole Philips", generate=gen, model="m", spotify=spotify)) + assert "Live Spotify search results" in prompts[0] + assert "- Cole Phillips — Roll of Dice" in prompts[0] + + +def test_live_spotify_failure_falls_back_to_gemini(monkeypatch): + import backend.services.match_judge.free_text as ft + monkeypatch.setattr(ft, "SPOTIFY_TIMEOUT_S", 0.05) + prompts = [] + + async def gen(model, system_prompt, user_prompt): + prompts.append(user_prompt) + return {"kind": "none", "confident": False} + + async def slow(q): + await asyncio.sleep(1) + + async def broken(q): + raise RuntimeError("decide down") + + for spotify in (slow, broken): + v = asyncio.run(resolve_and_tidy("some query", generate=gen, model="m", spotify=spotify)) + assert v["kind"] == "none" + assert prompts == ['Singer typed: "some query"'] * 2 def test_ambiguous_and_failures_skip_the_catalog(): @@ -208,7 +296,8 @@ def test_slow_catalog_tidy_answers_with_ai_verdict_and_keeps_searching(monkeypat return MatchVerdict("cosmetic", True, "Céline Dion", "My Heart Will Go On", engine="catalog") async def main(): - v = await resolve_and_tidy("song from titanic", generate=gen, judge=judge, model="m") + v = await resolve_and_tidy("song from titanic", generate=gen, judge=judge, model="m", + spotify=_no_spotify) assert finished == [] # answered before the catalog search finished await asyncio.sleep(0.3) # ...which kept running in the background return v diff --git a/docs/API.md b/docs/API.md index 7c8215e8..bd166638 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1093,6 +1093,21 @@ the AI layer with `MATCH_JUDGE_ENABLED=false` (deterministic+catalog still run). > is designing an on-device typo-tolerant matcher that uses Gemini only as a fallback. Read > `kjbox/docs/SONG-IDENTIFICATION.md` before reusing this judge for free-text search. +**Live Spotify + grounding (v0.294.0).** When the catalogue can't confirm the typed song, the judge +(both stages) runs a live Spotify search through decide's partner route +(`GET /api/partner/gen/spotify-search`, `X-Partner-Secret`, via +`catalog_proxy_service.spotify_search`). The catalogue is a July 2025 snapshot, so new releases are +missing from it. +- An exact match after normalisation is a confident `catalog` verdict with reason + "… (live Spotify)" and **no Gemini call**. Example: "Cole Phillips" / "Roll of dice" → "Roll of Dice". +- Gemini also gets the Spotify results as candidates. +- **Grounding rule:** a confident AI `cosmetic` / `content` verdict must name a song that is in the + candidates or in one more Spotify lookup of the AI's song. Otherwise it comes back + `confident: false` with reason "… [unconfirmed: not in catalog or Spotify]". Clients show that as + "Did you mean…?" and never apply it. +- Spotify unavailable means no evidence, so AI corrections aren't confident. That is conservative on + purpose. Incident: kjbox 2026-11-05, job `f01c9997`, "Roll of Dice" became "Roll of the Dice". + `stage` (optional, `"fast"` | `"full"`, default `"full"`) supports a two-call pattern the frontend uses to keep the tidy off the critical path: @@ -3461,7 +3476,15 @@ AudioSourceStep ("Tidied to …" / "Corrected to …"). Two steps `judge_match(stage="fast")`. A confident catalog match wins: its formatting becomes `canonical_*`, `engine: "catalog"`, kind `cosmetic` when the query already named that song (only casing/punctuation/word order differ), else - `content`. No catalog match → the AI verdict unchanged. + `content`. No catalog match → the AI verdict, but a confident answer that neither the + catalogue nor live Spotify confirms comes back `confident: false` (grounding rule, above). + A tidy that timed out keeps the AI verdict with reason `unavailable` (not cached). + +**Step 0 (v0.294.0): live Spotify.** Before Gemini, the raw query goes to live Spotify (3 s cap). +A result whose artist + title *is* the query (either order, after normalising) is answered +directly: `cosmetic`, `engine: "catalog"`, reason `live Spotify match`. Otherwise the results go to +Gemini as candidates ("use the listed song's exact artist and title; never 'correct' to a more +familiar title"). kind `none` keeps the AI's `typed_artist`/`typed_title` split when it has one (kjbox pre-fills its make-it form with it). kjbox calls this only when its own diff --git a/docs/LESSONS-LEARNED.md b/docs/LESSONS-LEARNED.md index 28ee33bb..1eecbff1 100644 --- a/docs/LESSONS-LEARNED.md +++ b/docs/LESSONS-LEARNED.md @@ -6,6 +6,23 @@ Key insights for future AI agents working on this codebase. --- + +## An AI "correction" nothing can confirm is a guess: ground it in real data (Nov 2026, v0.294.0) + +At a live kjbox night a singer typed "Cole Phillips – Roll of Dice", a 2025 release missing from the +July 2025 Spotify snapshot, from MusicBrainz and from KaraokeNerds. Gemini "corrected" it to the +more familiar phrase "Roll of the Dice", and `resolve_and_tidy` shipped that as confident: +"No catalog match → AI verdict unchanged". After the singer tapped "keep what I typed", the make +wizard's full judge did it again. The job came out titled wrong (`f01c9997`) even though the audio +search had found the real track on Spotify at score 1.0. + +The LLM is weakest exactly where our data is thinnest: new releases and small artists. So silence +from the catalogue is not evidence for the AI's answer. Fix: a live Spotify search (decide partner +route, client-credentials, cached, 429 circuit breaker) confirms the typed song before Gemini. Any +AI correction must name a song the catalogue or Spotify actually has, otherwise it is +`confident: false` (offered, never applied). When a real data source is cheap and fresh, ask it +before asking a model, and check what the model says against it. + ## A lazy import on the loop can stall it for 15s: the image had no precompiled bytecode (Oct 2026, v0.282.1) The hourly `process-stale-reviews` cron stalled the API loop 10–15s, 4 times in 2 days. The culprit was `internal.py:670`: a lazy `from backend.services.conversion_emails import ...` inside an `async def`. The stall stack ended in `importlib … _write_atomic`. The backend Dockerfile never ran `compileall`, so the first import on each instance compiled the module and wrote a `.pyc` on the event loop, on a CPU-throttled instance. diff --git a/pyproject.toml b/pyproject.toml index 5388f9bc..2fef1ffb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.293.1" +version = "0.294.0" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT"