mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
feat(kn-data-sync): live Spotify songs in the song-id export (v0.294.2) (#1258)
## Summary Companion to the karaoke-decide PR that keeps every live Spotify search result forever (`karaoke_decide.spotify_live_tracks`). With this change, those songs join the daily **song-id export**, which feeds: - the kjbox on-device matcher (`song_id.db`); - decide's song-search service; - the mobile app's offline index. So a song we look up once, like "Cole Phillips – Roll of Dice" (a 2025 release missing from the July 2025 Spotify snapshot and from MusicBrainz), is known everywhere from the next build on, with no Spotify or Gemini call. ## Changes - `kn_data_sync/main.py`: `SPOTIFY_LIVE_UNION` is appended to `SONG_ID_SQL` when the table exists. Before decide's first live search creates it, the union is skipped with an INFO log. - Grouped by lowercased artist/title. - Popularity uses the D5a estimate: 0.8 × the artist's popularity (via `art`), capped at 50, else 30. The API no longer returns track popularity. - Docs: API.md. Version 0.294.2. ## Testing - `test_main.py`: union added when the table exists, skipped when it doesn't, four columns and the estimate (9 passed). - **BigQuery dry run** of the full export with the union (inline row in place of the table): valid, ~8.2 GB scanned (unchanged). - Local CodeRabbit review: 0 findings. ## Deploy The function is deployed by hand: `infrastructure/functions/kn_data_sync/deploy.sh` (`gcloud functions deploy`). CI doesn't deploy it, and it needs `/gcp-admin`. @coderabbitai ignore 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_012RYLVUQ68KFYtZF7vEfhAq Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c133b64c7d
commit
bb01e9e36a
4 files changed
+61
-2
No files matched your search
@@ -1105,6 +1105,7 @@ missing from it.
|
||||
candidates or in one more Spotify lookup of the AI's song. Otherwise it comes back
|
||||
`confident: false` with reason "… [unconfirmed: not in catalog or Spotify]". Clients show that as
|
||||
"Did you mean…?" and never apply it.
|
||||
- Every live result is kept forever by decide (Firestore query cache + BigQuery `spotify_live_tracks`), and kn-data-sync adds those songs to the daily song-id export. So a song looked up once is known to the kjbox on-device matcher, decide's song search and the mobile app's offline index from the next build on (v0.294.2).
|
||||
- Spotify unavailable means no evidence, so AI corrections aren't confident. That is conservative on
|
||||
purpose. Incident: kjbox 2026-11-05, job `f01c9997`, "Roll of Dice" became "Roll of the Dice".
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ DATASET_ID = "karaoke_decide"
|
||||
|
||||
# kjbox song-identification index: every song a singer might mean, exported daily
|
||||
# after the KN refresh and downloaded by the NomadPC's nomad-catalog-sync. Design:
|
||||
# kjbox docs/SONG-IDENTIFICATION.md. Rows from three sources, merged/deduped on the
|
||||
# kjbox docs/SONG-IDENTIFICATION.md. Rows from these sources, merged/deduped on the
|
||||
# device (scripts/build_song_id_db.py keeps the highest popularity per song):
|
||||
# - MusicBrainz recordings (weekly-refreshed by karaoke-decide's mb-refresh — the
|
||||
# source that keeps NEW songs coming). Kept when Spotify track popularity >= 30
|
||||
@@ -48,6 +48,7 @@ DATASET_ID = "karaoke_decide"
|
||||
# (an estimate must not outrank a measured score).
|
||||
# - the static July 2025 Spotify snapshot (popularity >= 30)
|
||||
# - every KaraokeNerds song (daily; karaoke = 1)
|
||||
# - every song a live Spotify search returned (SPOTIFY_LIVE_UNION below)
|
||||
# ListenBrainz (karaoke-decide's lb-refresh, fortnightly dumps) is mapped onto Spotify's
|
||||
# 0-100 scale per stats range: score = offset + slope * log10(listeners), fitted to the
|
||||
# median Spotify popularity of songs that have both (2026-09-29; e.g. 100 this_year
|
||||
@@ -119,6 +120,26 @@ UNION ALL SELECT sp.best.a, sp.best.t, sp.p, 0 FROM sp
|
||||
UNION ALL SELECT a, t, CAST(NULL AS INT64), 1 FROM kn
|
||||
"""
|
||||
|
||||
# Every song a live Spotify search ever returned (karaoke-decide keeps them forever in
|
||||
# `spotify_live_tracks`, 2026-11-08), so a song someone looked up once, e.g. a 2025 release
|
||||
# missing from the July 2025 snapshot and MusicBrainz, is in the index from then on. The API
|
||||
# has no popularity, so the estimate is the D5a one: 0.8 x the artist's popularity capped at 50,
|
||||
# else 30. Appended only when the table exists (decide creates it on the first live search).
|
||||
SPOTIFY_LIVE_TABLE = "spotify_live_tracks"
|
||||
SPOTIFY_LIVE_UNION = r"""
|
||||
UNION ALL
|
||||
SELECT sl.a, sl.t, COALESCE(CAST(LEAST(art.apop * 0.8, 50) AS INT64), 30), 0
|
||||
FROM (
|
||||
SELECT ANY_VALUE(artist_name) a, ANY_VALUE(track_name) t,
|
||||
TRIM(REGEXP_REPLACE(REGEXP_REPLACE(la, r'[^a-z0-9 ]', ' '), r' +', ' ')) na
|
||||
FROM (SELECT artist_name, track_name, LOWER(artist_name) la, LOWER(track_name) lt
|
||||
FROM `{project}.{dataset}.spotify_live_tracks`
|
||||
WHERE artist_name IS NOT NULL AND track_name IS NOT NULL
|
||||
AND TRIM(artist_name) != '' AND TRIM(track_name) != '')
|
||||
GROUP BY la, lt) sl
|
||||
LEFT JOIN art ON art.name_normalized = sl.na
|
||||
"""
|
||||
|
||||
# API endpoints
|
||||
SONGS_URL = "https://karaokenerds.com/Data/Songs"
|
||||
COMMUNITY_URL = "https://karaokenerds.com/Data/Community"
|
||||
@@ -256,6 +277,15 @@ def _load_community_to_bigquery(raw_bytes: bytes) -> int:
|
||||
return load_job.output_rows
|
||||
|
||||
|
||||
def _table_exists(client, table: str) -> bool:
|
||||
from google.api_core.exceptions import NotFound
|
||||
try:
|
||||
client.get_table(f"{GCP_PROJECT_ID}.{DATASET_ID}.{table}")
|
||||
return True
|
||||
except NotFound:
|
||||
return False
|
||||
|
||||
|
||||
def _export_song_id_index() -> dict:
|
||||
"""EXPORT DATA the song-identification index to GCS + write the manifest.
|
||||
|
||||
@@ -267,6 +297,10 @@ def _export_song_id_index() -> dict:
|
||||
folder = f"{SONG_ID_PREFIX}/{run}"
|
||||
client = bigquery.Client(project=GCP_PROJECT_ID)
|
||||
select = SONG_ID_SQL.format(project=GCP_PROJECT_ID, dataset=DATASET_ID)
|
||||
if _table_exists(client, SPOTIFY_LIVE_TABLE):
|
||||
select = select.rstrip() + "\n" + SPOTIFY_LIVE_UNION.format(project=GCP_PROJECT_ID, dataset=DATASET_ID)
|
||||
else:
|
||||
logger.info("song-id export: %s not created yet, skipping live Spotify songs", SPOTIFY_LIVE_TABLE)
|
||||
export = (
|
||||
f"EXPORT DATA OPTIONS(uri='gs://{GCS_BUCKET}/{folder}/songs-*.tsv.gz', format='CSV', "
|
||||
"field_delimiter='\t', header=false, compression='GZIP', overwrite=true) AS " + select
|
||||
|
||||
@@ -137,3 +137,27 @@ def test_song_id_sql_uses_listenbrainz_and_keeps_the_four_columns():
|
||||
assert f"`p.d.{table}`" in sql
|
||||
# the device's build_song_id_db.py reads exactly these columns, in this order
|
||||
assert "SELECT artist, title, popularity, karaoke FROM mb_kept" in sql
|
||||
|
||||
|
||||
def test_export_adds_live_spotify_songs_when_the_table_exists(clients):
|
||||
bq, _gcs = clients
|
||||
bq.get_table.return_value = object()
|
||||
main._export_song_id_index()
|
||||
sql = bq.query.call_args[0][0]
|
||||
assert "`nomadkaraoke.karaoke_decide.spotify_live_tracks`" in sql
|
||||
assert sql.rstrip().endswith("LEFT JOIN art ON art.name_normalized = sl.na")
|
||||
assert "{" not in sql.split(" AS ", 1)[1].replace("{project}", "") # placeholders filled
|
||||
|
||||
|
||||
def test_export_skips_live_spotify_songs_before_the_table_exists(clients):
|
||||
from google.api_core.exceptions import NotFound
|
||||
bq, _gcs = clients
|
||||
bq.get_table.side_effect = NotFound("no table")
|
||||
main._export_song_id_index()
|
||||
assert "spotify_live_tracks" not in bq.query.call_args[0][0]
|
||||
|
||||
|
||||
def test_live_union_keeps_the_four_columns_and_estimates_popularity():
|
||||
union = main.SPOTIFY_LIVE_UNION.format(project="p", dataset="d")
|
||||
assert "{" not in union and "`p.d.spotify_live_tracks`" in union
|
||||
assert union.strip().startswith("UNION ALL\nSELECT sl.a, sl.t, COALESCE(CAST(LEAST(art.apop * 0.8, 50) AS INT64), 30), 0")
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.294.1"
|
||||
version = "0.294.2"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
Reference in new issue
Block a user