From 37f7d578ee8cb56498b132019760e5280c19755d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrew=20Beveridge=20=E2=98=84=EF=B8=8F?= Date: Mon, 28 Sep 2026 01:48:50 -0400 Subject: [PATCH] kn-data-sync: daily export of the kjbox song-identification index (#1066) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary After the daily KN `full` refresh, `kn-data-sync` now also exports the **kjbox song-identification index**: every song a singer might mean. - **Contents:** `spotify_tracks_normalized` (popularity ≥ 30) FULL OUTER JOIN `karaokenerds_raw`, deduped on normalised artist+title. About 2.0M rows: artist, title, popularity (blank = KN-only), karaoke (0/1). - **Export:** `EXPORT DATA` to `gs://nomadkaraoke-kn-data/song-id//songs-*.tsv.gz`, then a manifest at `song-id/latest.json` naming that run's shards, so a device never mixes runs. The last 3 runs are kept. - **Cost and safety:** it's a server-side export; no rows pass through the function (512M memory is fine). The dry run scans about 160 MB (~$0.001/day). A failed export is reported in the response and doesn't fail the KN sync. - No IAM change: the SA already has project `bigquery.dataEditor` + `jobUser` and write access to the bucket. Design and reasoning: kjbox `docs/SONG-IDENTIFICATION.md` (on-device matcher first, Gemini fallback). The NomadPC side (download → build `song_id.db`) is in the kjbox PR. ## ⚠️ Deploy (manual, needs GCP write) 1. `infrastructure/functions/kn_data_sync/deploy.sh` (uploads the source zip) 2. `pulumi up`, or redeploy the function. The source object name doesn't change, so force the function to pick up the new zip. 3. Optionally trigger it once: `gcloud scheduler jobs run kn-data-sync-full-daily --location us-central1`, then check `gs://nomadkaraoke-kn-data/song-id/latest.json`. ## Tests - `infrastructure/functions/kn_data_sync/test_main.py` (5): manifest names only this run's shards, prune keeps 3 runs, empty export raises, full sync survives an export failure, community mode doesn't export. Run manually (function tests aren't in the repo testpaths). - The rendered SQL passes a BigQuery dry run. 🤖 Generated with [Claude Code](https://claude.com/claude-code) ## Summary by CodeRabbit * **New Features** * Full catalog syncs now generate a song-identification index covering popular Spotify tracks and KaraokeNerds entries. The index is exported to cloud storage, with a manifest pointing to the latest run and the three most recent runs retained. * **Bug Fixes** * If index export fails, the catalog sync still completes and reports the export failure. Community syncs do not generate an index. --------- Co-authored-by: Claude Opus 5.5 (1M context) --- docs/ARCHITECTURE.md | 2 +- infrastructure/functions/kn_data_sync/main.py | 104 +++++++++++++- .../functions/kn_data_sync/test_main.py | 129 ++++++++++++++++++ 3 files changed, 233 insertions(+), 2 deletions(-) create mode 100644 infrastructure/functions/kn_data_sync/test_main.py diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 9dbc3005..15f8ca86 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -272,7 +272,7 @@ Three Cloud Functions manage the community karaoke data pipeline, running daily | Function | Schedule | Purpose | |----------|----------|---------| | `divebar-mirror` | Daily 2:00 AM ET | Indexes 48K+ files from [diveBar Karaoke Google Drive](https://drive.google.com/drive/folders/1zxnSZcE03gzy0YVGOdnTrEIi8It_3Wu8) into BigQuery `divebar_catalog` | -| `kn-data-sync` | Daily 3:05 AM + 4:30 AM ET | Fetches KaraokeNerds catalog (281K songs) and community tracks (58K with YouTube URLs) to BigQuery + GCS | +| `kn-data-sync` | Daily 3:05 AM + 4:30 AM ET | Fetches KaraokeNerds catalog (281K songs) and community tracks (58K with YouTube URLs) to BigQuery + GCS. `full` mode then exports the **kjbox song-identification index** (≈2M songs: Spotify popularity ≥ 30 + all KN; `EXPORT DATA` → `gs://nomadkaraoke-kn-data/song-id//songs-*.tsv.gz`, manifest `song-id/latest.json`, last 3 runs kept), which the NomadPC downloads. Design: kjbox `docs/SONG-IDENTIFICATION.md` | | `divebar-lookup` | Daily 6:00 AM ET + on-demand API | Public search/lookup API for KJ Controller; rebuilds KN↔Divebar cross-reference index | **Infrastructure** (Pulumi-managed in `infrastructure/`): diff --git a/infrastructure/functions/kn_data_sync/main.py b/infrastructure/functions/kn_data_sync/main.py index 6071961d..2394dcd5 100644 --- a/infrastructure/functions/kn_data_sync/main.py +++ b/infrastructure/functions/kn_data_sync/main.py @@ -5,7 +5,8 @@ Fetches daily exports from the KaraokeNerds API and loads them into BigQuery and GCS. Replaces the legacy pipeline in projectbread-karaokay. Two modes (controlled by request body): - mode=full — Fetch full song catalog, store to GCS + refresh BigQuery + mode=full — Fetch full song catalog, store to GCS + refresh BigQuery, then + export the kjbox song-identification index (see below) mode=community — Fetch community tracks (with YouTube URLs), store to GCS + refresh BigQuery Environment variables: @@ -33,6 +34,59 @@ KARAOKENERDS_API_KEY = os.environ.get("KARAOKENERDS_API_KEY", "") GCS_BUCKET = os.environ.get("GCS_BUCKET", "nomadkaraoke-kn-data") DATASET_ID = "karaoke_decide" +# kjbox song-identification index: every song a singer might mean, exported daily +# after the KN refresh and downloaded by the NomadPC's nomad-catalog-sync. Design: +# kjbox docs/SONG-IDENTIFICATION.md. Rows from three sources, merged/deduped on the +# device (scripts/build_song_id_db.py): +# - MusicBrainz recordings (weekly-refreshed by karaoke-decide's mb-refresh — the +# source that keeps NEW songs coming): kept when Spotify track popularity >= 30 +# (via ISRC), or 3+ recordings, or the artist's Spotify popularity >= 50; songs +# with no track score get 0.8 x artist popularity, capped at 50 (an +# estimate must not outrank a measured Spotify score) +# - the static July 2025 Spotify snapshot (popularity >= 30) +# - every KaraokeNerds song (daily; karaoke = 1) +# ~6.7 GB scanned per run (~$0.04). Each run writes a dated folder + a manifest +# (song-id/latest.json) naming its shards, so the device never mixes runs. +SONG_ID_PREFIX = "song-id" +SONG_ID_KEEP_RUNS = 3 +SONG_ID_SQL = r""" +WITH +art AS ( + SELECT name_normalized, MAX(popularity) apop + FROM `{project}.{dataset}.mb_artists_normalized` + WHERE spotify_artist_id IS NOT NULL + GROUP BY name_normalized), +mb AS ( + SELECT r.artist_normalized na, r.name_normalized nt, + ARRAY_AGG(STRUCT(r.artist_credit AS a, r.title AS t) ORDER BY r.spotify_popularity DESC LIMIT 1)[OFFSET(0)] best, + COUNT(DISTINCT r.recording_mbid) recs, MAX(r.spotify_popularity) sp + FROM `{project}.{dataset}.mb_recordings_enriched` r + WHERE r.name_normalized IS NOT NULL AND r.artist_normalized IS NOT NULL + AND NOT REGEXP_CONTAINS(LOWER(IFNULL(r.disambiguation, '')), r'\b(live|demo|rehearsal|instrumental|karaoke)\b') + GROUP BY na, nt), +mb_kept AS ( + SELECT mb.best.a artist, mb.best.t title, + -- no Spotify track score (e.g. released after the July 2025 snapshot): estimate from the + -- artist's popularity, a little below it and capped at 50, so a known artist's new song still + -- ranks sensibly but never above a measured track score + COALESCE(NULLIF(mb.sp, 0), CAST(LEAST(art.apop * 0.8, 50) AS INT64), mb.sp) popularity, 0 karaoke + FROM mb LEFT JOIN art ON art.name_normalized = mb.na + WHERE mb.sp >= 30 OR mb.recs >= 3 OR art.apop >= 50), +sp AS ( + SELECT ARRAY_AGG(STRUCT(artist_name AS a, track_name AS t) ORDER BY popularity DESC LIMIT 1)[OFFSET(0)] best, + MAX(popularity) p + FROM `{project}.{dataset}.spotify_tracks_normalized` + GROUP BY normalized_artist, normalized_title), +kn AS ( + SELECT ANY_VALUE(Artist) a, ANY_VALUE(Title) t + FROM `{project}.{dataset}.karaokenerds_raw` + WHERE Artist IS NOT NULL AND Title IS NOT NULL + GROUP BY LOWER(Artist), LOWER(Title)) +SELECT artist, title, popularity, karaoke FROM mb_kept +UNION ALL SELECT sp.best.a, sp.best.t, sp.p, 0 FROM sp +UNION ALL SELECT a, t, CAST(NULL AS INT64), 1 FROM kn +""" + # API endpoints SONGS_URL = "https://karaokenerds.com/Data/Songs" COMMUNITY_URL = "https://karaokenerds.com/Data/Community" @@ -170,6 +224,43 @@ def _load_community_to_bigquery(raw_bytes: bytes) -> int: return load_job.output_rows +def _export_song_id_index() -> dict: + """EXPORT DATA the song-identification index to GCS + write the manifest. + + Server-side export (no rows pass through this function). Columns: + artist, title, popularity (blank = karaoke-only row), karaoke (0/1); TSV, + gzipped, no header, sharded by BigQuery. + """ + run = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") + folder = f"{SONG_ID_PREFIX}/{run}" + client = bigquery.Client(project=GCP_PROJECT_ID) + select = SONG_ID_SQL.format(project=GCP_PROJECT_ID, dataset=DATASET_ID) + export = ( + f"EXPORT DATA OPTIONS(uri='gs://{GCS_BUCKET}/{folder}/songs-*.tsv.gz', format='CSV', " + "field_delimiter='\t', header=false, compression='GZIP', overwrite=true) AS " + select + ) + client.query(export).result() + + gcs = storage.Client(project=GCP_PROJECT_ID) + bucket = gcs.bucket(GCS_BUCKET) + shards = sorted(b.name for b in gcs.list_blobs(GCS_BUCKET, prefix=f"{folder}/")) + if not shards: + raise RuntimeError(f"song-id export wrote no files under {folder}/") + manifest = {"run": run, "shards": [f"gs://{GCS_BUCKET}/{name}" for name in shards], + "columns": ["artist", "title", "popularity", "karaoke"]} + bucket.blob(f"{SONG_ID_PREFIX}/latest.json").upload_from_string( + json.dumps(manifest), content_type="application/json") + + # Prune old runs (keep a few so a device mid-download never loses its shards). + runs = sorted({b.name.split("/")[1] for b in gcs.list_blobs(GCS_BUCKET, prefix=f"{SONG_ID_PREFIX}/") + if b.name.count("/") >= 2}) + for old in runs[:-SONG_ID_KEEP_RUNS]: + for b in gcs.list_blobs(GCS_BUCKET, prefix=f"{SONG_ID_PREFIX}/{old}/"): + b.delete() + logger.info("song-id index exported: %s (%d shards)", folder, len(shards)) + return {"run": run, "shards": len(shards)} + + @functions_framework.http def sync_kn_data(request): """ @@ -210,6 +301,16 @@ def sync_kn_data(request): else: rows_loaded = _load_community_to_bigquery(raw_bytes) + # Step 4 (full mode): refresh the kjbox song-identification index from the + # just-loaded KN table. Its failure must not fail the KN sync itself. + song_id = None + if mode == "full": + try: + song_id = _export_song_id_index() + except Exception as e: # noqa: BLE001 + logger.exception("song-id index export failed") + song_id = {"error": str(e)} + duration = time.time() - start result = { @@ -218,6 +319,7 @@ def sync_kn_data(request): "records_fetched": record_count, "rows_loaded": rows_loaded, "gcs_path": f"gs://{GCS_BUCKET}/{gcs_path}", + "song_id_index": song_id, "duration_s": round(duration, 1), } diff --git a/infrastructure/functions/kn_data_sync/test_main.py b/infrastructure/functions/kn_data_sync/test_main.py new file mode 100644 index 00000000..6a651c9e --- /dev/null +++ b/infrastructure/functions/kn_data_sync/test_main.py @@ -0,0 +1,129 @@ +""" +Tests for kn_data_sync/main.py — the kjbox song-identification index export. + +Run: cd infrastructure/functions/kn_data_sync && python -m pytest test_main.py +(Cloud Function tests aren't in the repo-wide testpaths.) +""" +import json +import os +import sys +from unittest.mock import MagicMock + +import pytest + +_functions_framework = MagicMock() +_functions_framework.http = lambda fn: fn +for name, mod in ( + ("functions_framework", _functions_framework), + ("google.cloud.bigquery", MagicMock()), + ("google.cloud.storage", MagicMock()), +): + sys.modules.setdefault(name, mod) + +sys.path.insert(0, os.path.dirname(__file__)) + +import main # noqa: E402 + + +class _Blob: + def __init__(self, name, store): + self.name, self._store = name, store + + def delete(self): + self._store.pop(self.name, None) + + +class FakeGCS: + """In-memory bucket: list_blobs by prefix, upload, delete.""" + + def __init__(self, names=()): + self.objects = {n: b"" for n in names} + self.uploaded = {} + + def list_blobs(self, bucket, prefix=""): + return [_Blob(n, self.objects) for n in sorted(self.objects) if n.startswith(prefix)] + + def bucket(self, _name): + gcs = self + + class B: + def blob(self, name): + blob = MagicMock() + blob.upload_from_string.side_effect = lambda data, content_type=None: ( + gcs.uploaded.__setitem__(name, data), gcs.objects.__setitem__(name, data)) + return blob + return B() + + +@pytest.fixture +def clients(monkeypatch): + bq = MagicMock() + old_runs = [f"song-id/2026010{i}-000000/songs-000000000000.tsv.gz" for i in range(1, 5)] + gcs = FakeGCS(old_runs) + + def run_export(sql): + # The export writes shards into the run folder named in the URI. + folder = sql.split(f"gs://{main.GCS_BUCKET}/")[1].split("/songs-")[0] + for k in range(2): + gcs.objects[f"{folder}/songs-00000000000{k}.tsv.gz"] = b"" + job = MagicMock() + return job + bq.query.side_effect = run_export + monkeypatch.setattr(main.bigquery, "Client", lambda project=None: bq) + monkeypatch.setattr(main.storage, "Client", lambda project=None: gcs) + return bq, gcs + + +def test_export_writes_manifest_for_this_run_only(clients): + bq, gcs = clients + out = main._export_song_id_index() + sql = bq.query.call_args[0][0] + assert sql.startswith("EXPORT DATA OPTIONS(uri='gs://nomadkaraoke-kn-data/song-id/") + assert "compression='GZIP'" in sql and "karaokenerds_raw" in sql and "spotify_tracks_normalized" in sql and "mb_recordings_enriched" in sql + manifest = json.loads(gcs.uploaded["song-id/latest.json"]) + assert manifest["run"] == out["run"] and out["shards"] == 2 + assert all(f"/song-id/{out['run']}/" in s for s in manifest["shards"]) + assert manifest["columns"] == ["artist", "title", "popularity", "karaoke"] + + +def test_export_prunes_all_but_the_newest_runs(clients): + _bq, gcs = clients + main._export_song_id_index() + runs = {n.split("/")[1] for n in gcs.objects if n.count("/") >= 2} + assert len(runs) == main.SONG_ID_KEEP_RUNS + assert "20260101-000000" not in runs + + +def test_export_with_no_output_raises(clients, monkeypatch): + bq, _gcs = clients + bq.query.side_effect = None + with pytest.raises(RuntimeError, match="no files"): + main._export_song_id_index() + + +def test_full_sync_survives_export_failure(monkeypatch): + monkeypatch.setattr(main, "KARAOKENERDS_API_KEY", "k") + monkeypatch.setattr(main, "_fetch_kn_data", lambda url: (b"[]", 0)) + monkeypatch.setattr(main, "_store_to_gcs", lambda raw, prefix: "full/x.json.gz") + monkeypatch.setattr(main, "_load_songs_to_bigquery", lambda raw: 0) + monkeypatch.setattr(main, "_export_song_id_index", MagicMock(side_effect=RuntimeError("bq down"))) + req = MagicMock() + req.get_json.return_value = {"mode": "full"} + body, status, _ = main.sync_kn_data(req) + data = json.loads(body) + assert status == 200 and data["status"] == "ok" + assert data["song_id_index"] == {"error": "bq down"} + + +def test_community_sync_does_not_export(monkeypatch): + monkeypatch.setattr(main, "KARAOKENERDS_API_KEY", "k") + monkeypatch.setattr(main, "_fetch_kn_data", lambda url: (b"[]", 0)) + monkeypatch.setattr(main, "_store_to_gcs", lambda raw, prefix: "community/x.json.gz") + monkeypatch.setattr(main, "_load_community_to_bigquery", lambda raw: 0) + export = MagicMock() + monkeypatch.setattr(main, "_export_song_id_index", export) + req = MagicMock() + req.get_json.return_value = {"mode": "community"} + body, status, _ = main.sync_kn_data(req) + assert status == 200 and json.loads(body)["song_id_index"] is None + export.assert_not_called()