mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
fix: preserve gcs_path in divebar catalog sync (#649)
## Summary - **index_builder.py**: replaced `WRITE_TRUNCATE` with staging table + `MERGE` SQL to preserve `gcs_path` values during daily catalog index updates - **divebar_file_sync.py**: added `blob.exists()` check before downloading from Drive — if a file is already in GCS, just updates `gcs_path` in BigQuery without re-downloading - Added unit tests verifying the MERGE approach preserves `gcs_path` correctly **Root cause**: The daily 2 AM index sync was atomically replacing the entire `divebar_catalog` BigQuery table, wiping all `gcs_path` values. The 3 AM file sync then saw all ~48K files as unsynced and re-downloaded the entire ~779 GB catalog from Google Drive every night. ## Test plan - [x] Unit tests for staging+MERGE behavior (5 tests pass) - [ ] Verify staging table auto-creates on first run (BigQuery creates on WRITE_TRUNCATE with schema) - [ ] Monitor next nightly sync — file_sync should report "recovered from GCS" instead of downloading - [ ] Confirm gcs_path values persist across consecutive index runs @coderabbitai ignore 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
c02fdc8d9f
commit
1c846ebae7
3 files changed
+256
-48
No files matched your search
@@ -3,6 +3,9 @@ Build and upload the Divebar catalog index to BigQuery.
|
||||
|
||||
Takes the flat file listing from drive_client and the parsed metadata
|
||||
from filename_parser, and loads it into a BigQuery table.
|
||||
|
||||
Uses a staging table + MERGE to preserve gcs_path values set by the
|
||||
file sync script (divebar_file_sync.py).
|
||||
"""
|
||||
|
||||
import logging
|
||||
@@ -17,8 +20,10 @@ logger = logging.getLogger(__name__)
|
||||
# BigQuery table reference
|
||||
DATASET_ID = "karaoke_decide"
|
||||
TABLE_ID = "divebar_catalog"
|
||||
STAGING_TABLE_ID = "divebar_catalog_staging"
|
||||
|
||||
SCHEMA = [
|
||||
# Schema for the staging table (no gcs_path — that lives only in the main table)
|
||||
STAGING_SCHEMA = [
|
||||
bigquery.SchemaField("file_id", "STRING", mode="REQUIRED"),
|
||||
bigquery.SchemaField("brand", "STRING", mode="REQUIRED"),
|
||||
bigquery.SchemaField("brand_code", "STRING"),
|
||||
@@ -78,31 +83,84 @@ def load_to_bigquery(project_id: str, rows: list[dict]) -> int:
|
||||
"""
|
||||
Load rows into the BigQuery divebar_catalog table.
|
||||
|
||||
Uses WRITE_TRUNCATE to atomically replace the entire table contents.
|
||||
Returns the number of rows loaded.
|
||||
Uses a staging table + MERGE to update the catalog without losing
|
||||
gcs_path values set by the file sync script. Steps:
|
||||
1. WRITE_TRUNCATE the staging table with fresh Drive data
|
||||
2. MERGE staging into main table:
|
||||
- UPDATE existing rows (by file_id) with fresh metadata
|
||||
- INSERT new rows
|
||||
- DELETE rows no longer in Drive
|
||||
gcs_path is preserved on updates and defaults to NULL on inserts.
|
||||
|
||||
Returns the number of rows in the main table after merge.
|
||||
"""
|
||||
if not rows:
|
||||
logger.warning("No rows to load")
|
||||
return 0
|
||||
|
||||
client = bigquery.Client(project=project_id)
|
||||
table_ref = f"{project_id}.{DATASET_ID}.{TABLE_ID}"
|
||||
staging_ref = f"{project_id}.{DATASET_ID}.{STAGING_TABLE_ID}"
|
||||
main_ref = f"{project_id}.{DATASET_ID}.{TABLE_ID}"
|
||||
|
||||
# Step 1: Load fresh data into staging table (TRUNCATE is fine here)
|
||||
job_config = bigquery.LoadJobConfig(
|
||||
schema=SCHEMA,
|
||||
schema=STAGING_SCHEMA,
|
||||
write_disposition=bigquery.WriteDisposition.WRITE_TRUNCATE,
|
||||
source_format=bigquery.SourceFormat.NEWLINE_DELIMITED_JSON,
|
||||
)
|
||||
|
||||
# BigQuery load from in-memory rows
|
||||
load_job = client.load_table_from_json(
|
||||
rows,
|
||||
table_ref,
|
||||
job_config=job_config,
|
||||
)
|
||||
|
||||
# Wait for completion
|
||||
load_job = client.load_table_from_json(rows, staging_ref, job_config=job_config)
|
||||
load_job.result()
|
||||
logger.info("Loaded %d rows to staging table", load_job.output_rows)
|
||||
|
||||
logger.info("Loaded %d rows to %s", load_job.output_rows, table_ref)
|
||||
return load_job.output_rows
|
||||
# Step 2: MERGE staging into main table, preserving gcs_path
|
||||
merge_sql = f"""
|
||||
MERGE `{main_ref}` AS main
|
||||
USING `{staging_ref}` AS staging
|
||||
ON main.file_id = staging.file_id
|
||||
|
||||
WHEN MATCHED THEN UPDATE SET
|
||||
brand = staging.brand,
|
||||
brand_code = staging.brand_code,
|
||||
artist = staging.artist,
|
||||
title = staging.title,
|
||||
disc_id = staging.disc_id,
|
||||
filename = staging.filename,
|
||||
format = staging.format,
|
||||
subfolder = staging.subfolder,
|
||||
file_size = staging.file_size,
|
||||
drive_path = staging.drive_path,
|
||||
drive_md5 = staging.drive_md5,
|
||||
artist_normalized = staging.artist_normalized,
|
||||
title_normalized = staging.title_normalized,
|
||||
synced_at = staging.synced_at
|
||||
|
||||
WHEN NOT MATCHED BY TARGET THEN INSERT (
|
||||
file_id, brand, brand_code, artist, title, disc_id,
|
||||
filename, format, subfolder, file_size, drive_path,
|
||||
drive_md5, artist_normalized, title_normalized, synced_at,
|
||||
gcs_path
|
||||
) VALUES (
|
||||
staging.file_id, staging.brand, staging.brand_code,
|
||||
staging.artist, staging.title, staging.disc_id,
|
||||
staging.filename, staging.format, staging.subfolder,
|
||||
staging.file_size, staging.drive_path, staging.drive_md5,
|
||||
staging.artist_normalized, staging.title_normalized,
|
||||
staging.synced_at, NULL
|
||||
)
|
||||
|
||||
WHEN NOT MATCHED BY SOURCE THEN DELETE
|
||||
"""
|
||||
|
||||
logger.info("Merging staging into main table (preserving gcs_path)...")
|
||||
merge_job = client.query(merge_sql)
|
||||
merge_job.result()
|
||||
|
||||
# Get final row count
|
||||
count_result = list(client.query(
|
||||
f"SELECT COUNT(*) as cnt FROM `{main_ref}`"
|
||||
).result())
|
||||
final_count = count_result[0].cnt if count_result else 0
|
||||
|
||||
logger.info("Merge complete. Main table has %d rows", final_count)
|
||||
return final_count
|
||||
@@ -150,12 +150,24 @@ def update_gcs_path(bq_client, file_id, gcs_path):
|
||||
|
||||
|
||||
def sync_file(drive_service, gcs_bucket, bq_client, row):
|
||||
"""Sync a single file from Drive to GCS."""
|
||||
"""Sync a single file from Drive to GCS.
|
||||
|
||||
Checks if the file already exists in GCS before downloading from Drive.
|
||||
If it exists, just updates gcs_path in BigQuery (no re-download needed).
|
||||
"""
|
||||
file_id = row.file_id
|
||||
drive_path = row.drive_path
|
||||
gcs_path = f"files/{drive_path}"
|
||||
full_gcs_path = f"gs://{GCS_BUCKET}/{gcs_path}"
|
||||
size_mb = (row.file_size or 0) / 1024 / 1024
|
||||
|
||||
# Check if file already exists in GCS (avoids re-downloading)
|
||||
blob = gcs_bucket.blob(gcs_path)
|
||||
if blob.exists():
|
||||
update_gcs_path(bq_client, file_id, full_gcs_path)
|
||||
logger.info(" ⏭ %s already in GCS, updated gcs_path", drive_path)
|
||||
return True, 0
|
||||
|
||||
logger.info("Downloading %s (%.1f MB)...", drive_path, size_mb)
|
||||
start = time.time()
|
||||
|
||||
@@ -166,7 +178,6 @@ def sync_file(drive_service, gcs_bucket, bq_client, row):
|
||||
if ok:
|
||||
duration = time.time() - start
|
||||
speed = actual_size / 1024 / 1024 / duration if duration > 0 else 0
|
||||
full_gcs_path = f"gs://{GCS_BUCKET}/{gcs_path}"
|
||||
update_gcs_path(bq_client, file_id, full_gcs_path)
|
||||
logger.info(
|
||||
" ✓ %s (%.1f MB in %.1fs, %.1f MB/s)",
|
||||
@@ -220,54 +231,51 @@ def main():
|
||||
gcs_bucket = gcs_client.bucket(GCS_BUCKET)
|
||||
drive_service = get_drive_service()
|
||||
|
||||
synced = 0
|
||||
downloaded = 0
|
||||
recovered = 0
|
||||
failed = 0
|
||||
bytes_synced = 0
|
||||
overall_start = time.time()
|
||||
|
||||
def _tally(ok, size):
|
||||
nonlocal downloaded, recovered, failed, bytes_synced
|
||||
if ok:
|
||||
if size > 0:
|
||||
downloaded += 1
|
||||
bytes_synced += size
|
||||
else:
|
||||
recovered += 1
|
||||
else:
|
||||
failed += 1
|
||||
|
||||
def _log_progress():
|
||||
elapsed = time.time() - overall_start
|
||||
logger.info(
|
||||
"Progress: %d downloaded, %d recovered, %d failed (of %d), %.1f GB, %.0f s",
|
||||
downloaded, recovered, failed, len(files),
|
||||
bytes_synced / 1024 / 1024 / 1024, elapsed,
|
||||
)
|
||||
|
||||
if args.workers > 1:
|
||||
# Parallel downloads
|
||||
with ThreadPoolExecutor(max_workers=args.workers) as executor:
|
||||
futures = {
|
||||
executor.submit(sync_file, drive_service, gcs_bucket, bq_client, row): row
|
||||
for row in files
|
||||
}
|
||||
for future in as_completed(futures):
|
||||
ok, size = future.result()
|
||||
if ok:
|
||||
synced += 1
|
||||
bytes_synced += size
|
||||
else:
|
||||
failed += 1
|
||||
|
||||
if (synced + failed) % 50 == 0:
|
||||
elapsed = time.time() - overall_start
|
||||
logger.info(
|
||||
"Progress: %d/%d synced, %d failed, %.1f GB, %.0f s elapsed",
|
||||
synced, len(files), failed, bytes_synced / 1024 / 1024 / 1024, elapsed,
|
||||
)
|
||||
_tally(*future.result())
|
||||
if (downloaded + recovered + failed) % 50 == 0:
|
||||
_log_progress()
|
||||
else:
|
||||
# Sequential downloads
|
||||
for i, row in enumerate(files):
|
||||
ok, size = sync_file(drive_service, gcs_bucket, bq_client, row)
|
||||
if ok:
|
||||
synced += 1
|
||||
bytes_synced += size
|
||||
else:
|
||||
failed += 1
|
||||
|
||||
_tally(*sync_file(drive_service, gcs_bucket, bq_client, row))
|
||||
if (i + 1) % 50 == 0:
|
||||
elapsed = time.time() - overall_start
|
||||
logger.info(
|
||||
"Progress: %d/%d synced, %d failed, %.1f GB, %.0f s elapsed",
|
||||
synced, len(files), failed, bytes_synced / 1024 / 1024 / 1024, elapsed,
|
||||
)
|
||||
_log_progress()
|
||||
|
||||
elapsed = time.time() - overall_start
|
||||
logger.info(
|
||||
"Done: %d synced, %d failed, %.1f GB in %.0f s (%.1f MB/s avg)",
|
||||
synced, failed, bytes_synced / 1024 / 1024 / 1024, elapsed,
|
||||
bytes_synced / 1024 / 1024 / elapsed if elapsed > 0 else 0,
|
||||
"Done: %d downloaded, %d recovered from GCS, %d failed, %.1f GB in %.0f s",
|
||||
downloaded, recovered, failed, bytes_synced / 1024 / 1024 / 1024, elapsed,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
"""Tests for divebar mirror index_builder — staging+MERGE approach.
|
||||
|
||||
The Cloud Function has its own deps (google-cloud-bigquery, filename_parser)
|
||||
that aren't in the main project's poetry env. We mock them at the module level
|
||||
before importing index_builder.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pre-import mocking: google.cloud.bigquery and filename_parser are not
|
||||
# available in the CI test environment (they're Cloud Function deps).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_mock_bigquery = MagicMock()
|
||||
# SchemaField needs to be callable and return a mock
|
||||
_mock_bigquery.SchemaField = MagicMock
|
||||
|
||||
_mock_google = types.ModuleType("google")
|
||||
_mock_google.cloud = types.ModuleType("google.cloud")
|
||||
_mock_google.cloud.bigquery = _mock_bigquery
|
||||
|
||||
_mock_filename_parser = MagicMock()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _import_index_builder(monkeypatch):
|
||||
"""Import index_builder with mocked external deps."""
|
||||
monkeypatch.setitem(sys.modules, "google", _mock_google)
|
||||
monkeypatch.setitem(sys.modules, "google.cloud", _mock_google.cloud)
|
||||
monkeypatch.setitem(sys.modules, "google.cloud.bigquery", _mock_bigquery)
|
||||
monkeypatch.setitem(sys.modules, "filename_parser", _mock_filename_parser)
|
||||
|
||||
func_dir = str(
|
||||
Path(__file__).resolve().parents[2]
|
||||
/ "infrastructure"
|
||||
/ "functions"
|
||||
/ "divebar_mirror"
|
||||
)
|
||||
monkeypatch.syspath_prepend(func_dir)
|
||||
|
||||
if "index_builder" in sys.modules:
|
||||
importlib.reload(sys.modules["index_builder"])
|
||||
else:
|
||||
importlib.import_module("index_builder")
|
||||
|
||||
|
||||
def _mod():
|
||||
return sys.modules["index_builder"]
|
||||
|
||||
|
||||
class TestLoadToBigquery:
|
||||
"""Verify load_to_bigquery uses staging+MERGE instead of WRITE_TRUNCATE on main table."""
|
||||
|
||||
def _setup_client_mock(self):
|
||||
"""Set up a mock BigQuery client with staging load + merge + count."""
|
||||
client = MagicMock()
|
||||
load_job = MagicMock()
|
||||
load_job.output_rows = 1
|
||||
client.load_table_from_json.return_value = load_job
|
||||
|
||||
merge_job = MagicMock()
|
||||
count_row = MagicMock()
|
||||
count_row.cnt = 1
|
||||
count_result = MagicMock()
|
||||
count_result.result.return_value = [count_row]
|
||||
client.query.side_effect = [merge_job, count_result]
|
||||
return client
|
||||
|
||||
def test_empty_rows_returns_zero(self):
|
||||
assert _mod().load_to_bigquery("proj", []) == 0
|
||||
|
||||
def test_loads_to_staging_table_not_main(self):
|
||||
"""The load job must target the staging table, not the main table."""
|
||||
mod = _mod()
|
||||
client = self._setup_client_mock()
|
||||
|
||||
with patch.object(mod.bigquery, "Client", return_value=client):
|
||||
mod.load_to_bigquery("proj", [{"file_id": "abc"}])
|
||||
|
||||
load_target = client.load_table_from_json.call_args[0][1]
|
||||
assert mod.STAGING_TABLE_ID in load_target
|
||||
assert load_target == f"proj.{mod.DATASET_ID}.{mod.STAGING_TABLE_ID}"
|
||||
|
||||
merge_sql = client.query.call_args_list[0][0][0]
|
||||
assert "MERGE" in merge_sql
|
||||
assert f"{mod.DATASET_ID}.{mod.TABLE_ID}" in merge_sql
|
||||
assert f"{mod.DATASET_ID}.{mod.STAGING_TABLE_ID}" in merge_sql
|
||||
|
||||
def test_merge_preserves_gcs_path(self):
|
||||
"""The MERGE SQL must preserve gcs_path on UPDATE and set NULL on INSERT."""
|
||||
mod = _mod()
|
||||
client = self._setup_client_mock()
|
||||
|
||||
with patch.object(mod.bigquery, "Client", return_value=client):
|
||||
mod.load_to_bigquery("proj", [{"file_id": "x"}])
|
||||
|
||||
merge_sql = client.query.call_args_list[0][0][0]
|
||||
|
||||
# gcs_path must NOT appear in the UPDATE SET clause
|
||||
update_section = merge_sql.split("WHEN MATCHED THEN UPDATE SET")[1].split(
|
||||
"WHEN NOT MATCHED"
|
||||
)[0]
|
||||
assert "gcs_path" not in update_section
|
||||
|
||||
# gcs_path must appear in INSERT with NULL default
|
||||
insert_section = merge_sql.split("WHEN NOT MATCHED BY TARGET THEN INSERT")[
|
||||
1
|
||||
].split("WHEN NOT MATCHED BY SOURCE")[0]
|
||||
assert "gcs_path" in insert_section
|
||||
assert "NULL" in insert_section
|
||||
|
||||
def test_merge_deletes_removed_files(self):
|
||||
"""Files no longer in Drive should be deleted from the main table."""
|
||||
mod = _mod()
|
||||
client = self._setup_client_mock()
|
||||
|
||||
with patch.object(mod.bigquery, "Client", return_value=client):
|
||||
mod.load_to_bigquery("proj", [{"file_id": "x"}])
|
||||
|
||||
merge_sql = client.query.call_args_list[0][0][0]
|
||||
assert "WHEN NOT MATCHED BY SOURCE THEN DELETE" in merge_sql
|
||||
|
||||
def test_no_write_truncate_on_main_table(self):
|
||||
"""WRITE_TRUNCATE must only target the staging table, never the main table."""
|
||||
mod = _mod()
|
||||
client = self._setup_client_mock()
|
||||
|
||||
with patch.object(mod.bigquery, "Client", return_value=client):
|
||||
mod.load_to_bigquery("proj", [{"file_id": "x"}])
|
||||
|
||||
load_calls = client.load_table_from_json.call_args_list
|
||||
assert len(load_calls) == 1
|
||||
target = load_calls[0][0][1]
|
||||
assert mod.STAGING_TABLE_ID in target
|
||||
assert target != f"proj.{mod.DATASET_ID}.{mod.TABLE_ID}"
|
||||
Reference in new issue
Block a user