diff --git a/docs/API.md b/docs/API.md index bd166638..ac87d83b 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1105,6 +1105,7 @@ missing from it. candidates or in one more Spotify lookup of the AI's song. Otherwise it comes back `confident: false` with reason "… [unconfirmed: not in catalog or Spotify]". Clients show that as "Did you mean…?" and never apply it. +- Every live result is kept forever by decide (Firestore query cache + BigQuery `spotify_live_tracks`), and kn-data-sync adds those songs to the daily song-id export. So a song looked up once is known to the kjbox on-device matcher, decide's song search and the mobile app's offline index from the next build on (v0.294.2). - Spotify unavailable means no evidence, so AI corrections aren't confident. That is conservative on purpose. Incident: kjbox 2026-11-05, job `f01c9997`, "Roll of Dice" became "Roll of the Dice". diff --git a/infrastructure/functions/kn_data_sync/main.py b/infrastructure/functions/kn_data_sync/main.py index a4dc0036..19562f84 100644 --- a/infrastructure/functions/kn_data_sync/main.py +++ b/infrastructure/functions/kn_data_sync/main.py @@ -36,7 +36,7 @@ DATASET_ID = "karaoke_decide" # kjbox song-identification index: every song a singer might mean, exported daily # after the KN refresh and downloaded by the NomadPC's nomad-catalog-sync. Design: -# kjbox docs/SONG-IDENTIFICATION.md. Rows from three sources, merged/deduped on the +# kjbox docs/SONG-IDENTIFICATION.md. Rows from these sources, merged/deduped on the # device (scripts/build_song_id_db.py keeps the highest popularity per song): # - MusicBrainz recordings (weekly-refreshed by karaoke-decide's mb-refresh — the # source that keeps NEW songs coming). Kept when Spotify track popularity >= 30 @@ -48,6 +48,7 @@ DATASET_ID = "karaoke_decide" # (an estimate must not outrank a measured score). # - the static July 2025 Spotify snapshot (popularity >= 30) # - every KaraokeNerds song (daily; karaoke = 1) +# - every song a live Spotify search returned (SPOTIFY_LIVE_UNION below) # ListenBrainz (karaoke-decide's lb-refresh, fortnightly dumps) is mapped onto Spotify's # 0-100 scale per stats range: score = offset + slope * log10(listeners), fitted to the # median Spotify popularity of songs that have both (2026-09-29; e.g. 100 this_year @@ -119,6 +120,26 @@ UNION ALL SELECT sp.best.a, sp.best.t, sp.p, 0 FROM sp UNION ALL SELECT a, t, CAST(NULL AS INT64), 1 FROM kn """ +# Every song a live Spotify search ever returned (karaoke-decide keeps them forever in +# `spotify_live_tracks`, 2026-11-08), so a song someone looked up once, e.g. a 2025 release +# missing from the July 2025 snapshot and MusicBrainz, is in the index from then on. The API +# has no popularity, so the estimate is the D5a one: 0.8 x the artist's popularity capped at 50, +# else 30. Appended only when the table exists (decide creates it on the first live search). +SPOTIFY_LIVE_TABLE = "spotify_live_tracks" +SPOTIFY_LIVE_UNION = r""" +UNION ALL +SELECT sl.a, sl.t, COALESCE(CAST(LEAST(art.apop * 0.8, 50) AS INT64), 30), 0 +FROM ( + SELECT ANY_VALUE(artist_name) a, ANY_VALUE(track_name) t, + TRIM(REGEXP_REPLACE(REGEXP_REPLACE(la, r'[^a-z0-9 ]', ' '), r' +', ' ')) na + FROM (SELECT artist_name, track_name, LOWER(artist_name) la, LOWER(track_name) lt + FROM `{project}.{dataset}.spotify_live_tracks` + WHERE artist_name IS NOT NULL AND track_name IS NOT NULL + AND TRIM(artist_name) != '' AND TRIM(track_name) != '') + GROUP BY la, lt) sl +LEFT JOIN art ON art.name_normalized = sl.na +""" + # API endpoints SONGS_URL = "https://karaokenerds.com/Data/Songs" COMMUNITY_URL = "https://karaokenerds.com/Data/Community" @@ -256,6 +277,15 @@ def _load_community_to_bigquery(raw_bytes: bytes) -> int: return load_job.output_rows +def _table_exists(client, table: str) -> bool: + from google.api_core.exceptions import NotFound + try: + client.get_table(f"{GCP_PROJECT_ID}.{DATASET_ID}.{table}") + return True + except NotFound: + return False + + def _export_song_id_index() -> dict: """EXPORT DATA the song-identification index to GCS + write the manifest. @@ -267,6 +297,10 @@ def _export_song_id_index() -> dict: folder = f"{SONG_ID_PREFIX}/{run}" client = bigquery.Client(project=GCP_PROJECT_ID) select = SONG_ID_SQL.format(project=GCP_PROJECT_ID, dataset=DATASET_ID) + if _table_exists(client, SPOTIFY_LIVE_TABLE): + select = select.rstrip() + "\n" + SPOTIFY_LIVE_UNION.format(project=GCP_PROJECT_ID, dataset=DATASET_ID) + else: + logger.info("song-id export: %s not created yet, skipping live Spotify songs", SPOTIFY_LIVE_TABLE) export = ( f"EXPORT DATA OPTIONS(uri='gs://{GCS_BUCKET}/{folder}/songs-*.tsv.gz', format='CSV', " "field_delimiter='\t', header=false, compression='GZIP', overwrite=true) AS " + select diff --git a/infrastructure/functions/kn_data_sync/test_main.py b/infrastructure/functions/kn_data_sync/test_main.py index de21b276..8b4d2a58 100644 --- a/infrastructure/functions/kn_data_sync/test_main.py +++ b/infrastructure/functions/kn_data_sync/test_main.py @@ -137,3 +137,27 @@ def test_song_id_sql_uses_listenbrainz_and_keeps_the_four_columns(): assert f"`p.d.{table}`" in sql # the device's build_song_id_db.py reads exactly these columns, in this order assert "SELECT artist, title, popularity, karaoke FROM mb_kept" in sql + + +def test_export_adds_live_spotify_songs_when_the_table_exists(clients): + bq, _gcs = clients + bq.get_table.return_value = object() + main._export_song_id_index() + sql = bq.query.call_args[0][0] + assert "`nomadkaraoke.karaoke_decide.spotify_live_tracks`" in sql + assert sql.rstrip().endswith("LEFT JOIN art ON art.name_normalized = sl.na") + assert "{" not in sql.split(" AS ", 1)[1].replace("{project}", "") # placeholders filled + + +def test_export_skips_live_spotify_songs_before_the_table_exists(clients): + from google.api_core.exceptions import NotFound + bq, _gcs = clients + bq.get_table.side_effect = NotFound("no table") + main._export_song_id_index() + assert "spotify_live_tracks" not in bq.query.call_args[0][0] + + +def test_live_union_keeps_the_four_columns_and_estimates_popularity(): + union = main.SPOTIFY_LIVE_UNION.format(project="p", dataset="d") + assert "{" not in union and "`p.d.spotify_live_tracks`" in union + assert union.strip().startswith("UNION ALL\nSELECT sl.a, sl.t, COALESCE(CAST(LEAST(art.apop * 0.8, 50) AS INT64), 30), 0") diff --git a/pyproject.toml b/pyproject.toml index 530dc04b..094e0de3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.294.1" +version = "0.294.2" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT"