fix: preserve gcs_path in divebar catalog sync (#649)

## Summary

- **index_builder.py**: replaced `WRITE_TRUNCATE` with staging table +
`MERGE` SQL to preserve `gcs_path` values during daily catalog index
updates
- **divebar_file_sync.py**: added `blob.exists()` check before
downloading from Drive — if a file is already in GCS, just updates
`gcs_path` in BigQuery without re-downloading
- Added unit tests verifying the MERGE approach preserves `gcs_path`
correctly

**Root cause**: The daily 2 AM index sync was atomically replacing the
entire `divebar_catalog` BigQuery table, wiping all `gcs_path` values.
The 3 AM file sync then saw all ~48K files as unsynced and re-downloaded
the entire ~779 GB catalog from Google Drive every night.

## Test plan

- [x] Unit tests for staging+MERGE behavior (5 tests pass)
- [ ] Verify staging table auto-creates on first run (BigQuery creates
on WRITE_TRUNCATE with schema)
- [ ] Monitor next nightly sync — file_sync should report "recovered
from GCS" instead of downloading
- [ ] Confirm gcs_path values persist across consecutive index runs

@coderabbitai ignore

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Andrew Beveridge ☄️andClaude Opus 4.6 authored and GitHub committed 2026-03-31 22:31:42 +00:00
1 parent c02fdc8d9f
commit 1c846ebae7
3 files changed
+256 -48

No files matched your search

@@ -3,6 +3,9 @@ Build and upload the Divebar catalog index to BigQuery.
Takes the flat file listing from drive_client and the parsed metadata
from filename_parser, and loads it into a BigQuery table.
Uses a staging table + MERGE to preserve gcs_path values set by the
file sync script (divebar_file_sync.py).
"""
import logging
@@ -17,8 +20,10 @@ logger = logging.getLogger(__name__)
# BigQuery table reference
DATASET_ID = "karaoke_decide"
TABLE_ID = "divebar_catalog"
STAGING_TABLE_ID = "divebar_catalog_staging"
SCHEMA = [
# Schema for the staging table (no gcs_path — that lives only in the main table)
STAGING_SCHEMA = [
bigquery.SchemaField("file_id", "STRING", mode="REQUIRED"),
bigquery.SchemaField("brand", "STRING", mode="REQUIRED"),
bigquery.SchemaField("brand_code", "STRING"),
@@ -78,31 +83,84 @@ def load_to_bigquery(project_id: str, rows: list[dict]) -> int:
"""
Load rows into the BigQuery divebar_catalog table.
Uses WRITE_TRUNCATE to atomically replace the entire table contents.
Returns the number of rows loaded.
Uses a staging table + MERGE to update the catalog without losing
gcs_path values set by the file sync script. Steps:
1. WRITE_TRUNCATE the staging table with fresh Drive data
2. MERGE staging into main table:
- UPDATE existing rows (by file_id) with fresh metadata
- INSERT new rows
- DELETE rows no longer in Drive
gcs_path is preserved on updates and defaults to NULL on inserts.
Returns the number of rows in the main table after merge.
"""
if not rows:
logger.warning("No rows to load")
return 0
client = bigquery.Client(project=project_id)
table_ref = f"{project_id}.{DATASET_ID}.{TABLE_ID}"
staging_ref = f"{project_id}.{DATASET_ID}.{STAGING_TABLE_ID}"
main_ref = f"{project_id}.{DATASET_ID}.{TABLE_ID}"
# Step 1: Load fresh data into staging table (TRUNCATE is fine here)
job_config = bigquery.LoadJobConfig(
schema=SCHEMA,
schema=STAGING_SCHEMA,
write_disposition=bigquery.WriteDisposition.WRITE_TRUNCATE,
source_format=bigquery.SourceFormat.NEWLINE_DELIMITED_JSON,
)
# BigQuery load from in-memory rows
load_job = client.load_table_from_json(
rows,
table_ref,
job_config=job_config,
)
# Wait for completion
load_job = client.load_table_from_json(rows, staging_ref, job_config=job_config)
load_job.result()
logger.info("Loaded %d rows to staging table", load_job.output_rows)
logger.info("Loaded %d rows to %s", load_job.output_rows, table_ref)
return load_job.output_rows
# Step 2: MERGE staging into main table, preserving gcs_path
merge_sql = f"""
MERGE `{main_ref}` AS main
USING `{staging_ref}` AS staging
ON main.file_id = staging.file_id
WHEN MATCHED THEN UPDATE SET
brand = staging.brand,
brand_code = staging.brand_code,
artist = staging.artist,
title = staging.title,
disc_id = staging.disc_id,
filename = staging.filename,
format = staging.format,
subfolder = staging.subfolder,
file_size = staging.file_size,
drive_path = staging.drive_path,
drive_md5 = staging.drive_md5,
artist_normalized = staging.artist_normalized,
title_normalized = staging.title_normalized,
synced_at = staging.synced_at
WHEN NOT MATCHED BY TARGET THEN INSERT (
file_id, brand, brand_code, artist, title, disc_id,
filename, format, subfolder, file_size, drive_path,
drive_md5, artist_normalized, title_normalized, synced_at,
gcs_path
) VALUES (
staging.file_id, staging.brand, staging.brand_code,
staging.artist, staging.title, staging.disc_id,
staging.filename, staging.format, staging.subfolder,
staging.file_size, staging.drive_path, staging.drive_md5,
staging.artist_normalized, staging.title_normalized,
staging.synced_at, NULL
)
WHEN NOT MATCHED BY SOURCE THEN DELETE
"""
logger.info("Merging staging into main table (preserving gcs_path)...")
merge_job = client.query(merge_sql)
merge_job.result()
# Get final row count
count_result = list(client.query(
f"SELECT COUNT(*) as cnt FROM `{main_ref}`"
).result())
final_count = count_result[0].cnt if count_result else 0
logger.info("Merge complete. Main table has %d rows", final_count)
return final_count
+41 -33
View File
@@ -150,12 +150,24 @@ def update_gcs_path(bq_client, file_id, gcs_path):
def sync_file(drive_service, gcs_bucket, bq_client, row):
"""Sync a single file from Drive to GCS."""
"""Sync a single file from Drive to GCS.
Checks if the file already exists in GCS before downloading from Drive.
If it exists, just updates gcs_path in BigQuery (no re-download needed).
"""
file_id = row.file_id
drive_path = row.drive_path
gcs_path = f"files/{drive_path}"
full_gcs_path = f"gs://{GCS_BUCKET}/{gcs_path}"
size_mb = (row.file_size or 0) / 1024 / 1024
# Check if file already exists in GCS (avoids re-downloading)
blob = gcs_bucket.blob(gcs_path)
if blob.exists():
update_gcs_path(bq_client, file_id, full_gcs_path)
logger.info(" ⏭ %s already in GCS, updated gcs_path", drive_path)
return True, 0
logger.info("Downloading %s (%.1f MB)...", drive_path, size_mb)
start = time.time()
@@ -166,7 +178,6 @@ def sync_file(drive_service, gcs_bucket, bq_client, row):
if ok:
duration = time.time() - start
speed = actual_size / 1024 / 1024 / duration if duration > 0 else 0
full_gcs_path = f"gs://{GCS_BUCKET}/{gcs_path}"
update_gcs_path(bq_client, file_id, full_gcs_path)
logger.info(
" ✓ %s (%.1f MB in %.1fs, %.1f MB/s)",
@@ -220,54 +231,51 @@ def main():
gcs_bucket = gcs_client.bucket(GCS_BUCKET)
drive_service = get_drive_service()
synced = 0
downloaded = 0
recovered = 0
failed = 0
bytes_synced = 0
overall_start = time.time()
def _tally(ok, size):
nonlocal downloaded, recovered, failed, bytes_synced
if ok:
if size > 0:
downloaded += 1
bytes_synced += size
else:
recovered += 1
else:
failed += 1
def _log_progress():
elapsed = time.time() - overall_start
logger.info(
"Progress: %d downloaded, %d recovered, %d failed (of %d), %.1f GB, %.0f s",
downloaded, recovered, failed, len(files),
bytes_synced / 1024 / 1024 / 1024, elapsed,
)
if args.workers > 1:
# Parallel downloads
with ThreadPoolExecutor(max_workers=args.workers) as executor:
futures = {
executor.submit(sync_file, drive_service, gcs_bucket, bq_client, row): row
for row in files
}
for future in as_completed(futures):
ok, size = future.result()
if ok:
synced += 1
bytes_synced += size
else:
failed += 1
if (synced + failed) % 50 == 0:
elapsed = time.time() - overall_start
logger.info(
"Progress: %d/%d synced, %d failed, %.1f GB, %.0f s elapsed",
synced, len(files), failed, bytes_synced / 1024 / 1024 / 1024, elapsed,
)
_tally(*future.result())
if (downloaded + recovered + failed) % 50 == 0:
_log_progress()
else:
# Sequential downloads
for i, row in enumerate(files):
ok, size = sync_file(drive_service, gcs_bucket, bq_client, row)
if ok:
synced += 1
bytes_synced += size
else:
failed += 1
_tally(*sync_file(drive_service, gcs_bucket, bq_client, row))
if (i + 1) % 50 == 0:
elapsed = time.time() - overall_start
logger.info(
"Progress: %d/%d synced, %d failed, %.1f GB, %.0f s elapsed",
synced, len(files), failed, bytes_synced / 1024 / 1024 / 1024, elapsed,
)
_log_progress()
elapsed = time.time() - overall_start
logger.info(
"Done: %d synced, %d failed, %.1f GB in %.0f s (%.1f MB/s avg)",
synced, failed, bytes_synced / 1024 / 1024 / 1024, elapsed,
bytes_synced / 1024 / 1024 / elapsed if elapsed > 0 else 0,
"Done: %d downloaded, %d recovered from GCS, %d failed, %.1f GB in %.0f s",
downloaded, recovered, failed, bytes_synced / 1024 / 1024 / 1024, elapsed,
)
+142
View File
@@ -0,0 +1,142 @@
"""Tests for divebar mirror index_builder — staging+MERGE approach.
The Cloud Function has its own deps (google-cloud-bigquery, filename_parser)
that aren't in the main project's poetry env. We mock them at the module level
before importing index_builder.
"""
import importlib
import sys
import types
from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
# ---------------------------------------------------------------------------
# Pre-import mocking: google.cloud.bigquery and filename_parser are not
# available in the CI test environment (they're Cloud Function deps).
# ---------------------------------------------------------------------------
_mock_bigquery = MagicMock()
# SchemaField needs to be callable and return a mock
_mock_bigquery.SchemaField = MagicMock
_mock_google = types.ModuleType("google")
_mock_google.cloud = types.ModuleType("google.cloud")
_mock_google.cloud.bigquery = _mock_bigquery
_mock_filename_parser = MagicMock()
@pytest.fixture(autouse=True)
def _import_index_builder(monkeypatch):
"""Import index_builder with mocked external deps."""
monkeypatch.setitem(sys.modules, "google", _mock_google)
monkeypatch.setitem(sys.modules, "google.cloud", _mock_google.cloud)
monkeypatch.setitem(sys.modules, "google.cloud.bigquery", _mock_bigquery)
monkeypatch.setitem(sys.modules, "filename_parser", _mock_filename_parser)
func_dir = str(
Path(__file__).resolve().parents[2]
/ "infrastructure"
/ "functions"
/ "divebar_mirror"
)
monkeypatch.syspath_prepend(func_dir)
if "index_builder" in sys.modules:
importlib.reload(sys.modules["index_builder"])
else:
importlib.import_module("index_builder")
def _mod():
return sys.modules["index_builder"]
class TestLoadToBigquery:
"""Verify load_to_bigquery uses staging+MERGE instead of WRITE_TRUNCATE on main table."""
def _setup_client_mock(self):
"""Set up a mock BigQuery client with staging load + merge + count."""
client = MagicMock()
load_job = MagicMock()
load_job.output_rows = 1
client.load_table_from_json.return_value = load_job
merge_job = MagicMock()
count_row = MagicMock()
count_row.cnt = 1
count_result = MagicMock()
count_result.result.return_value = [count_row]
client.query.side_effect = [merge_job, count_result]
return client
def test_empty_rows_returns_zero(self):
assert _mod().load_to_bigquery("proj", []) == 0
def test_loads_to_staging_table_not_main(self):
"""The load job must target the staging table, not the main table."""
mod = _mod()
client = self._setup_client_mock()
with patch.object(mod.bigquery, "Client", return_value=client):
mod.load_to_bigquery("proj", [{"file_id": "abc"}])
load_target = client.load_table_from_json.call_args[0][1]
assert mod.STAGING_TABLE_ID in load_target
assert load_target == f"proj.{mod.DATASET_ID}.{mod.STAGING_TABLE_ID}"
merge_sql = client.query.call_args_list[0][0][0]
assert "MERGE" in merge_sql
assert f"{mod.DATASET_ID}.{mod.TABLE_ID}" in merge_sql
assert f"{mod.DATASET_ID}.{mod.STAGING_TABLE_ID}" in merge_sql
def test_merge_preserves_gcs_path(self):
"""The MERGE SQL must preserve gcs_path on UPDATE and set NULL on INSERT."""
mod = _mod()
client = self._setup_client_mock()
with patch.object(mod.bigquery, "Client", return_value=client):
mod.load_to_bigquery("proj", [{"file_id": "x"}])
merge_sql = client.query.call_args_list[0][0][0]
# gcs_path must NOT appear in the UPDATE SET clause
update_section = merge_sql.split("WHEN MATCHED THEN UPDATE SET")[1].split(
"WHEN NOT MATCHED"
)[0]
assert "gcs_path" not in update_section
# gcs_path must appear in INSERT with NULL default
insert_section = merge_sql.split("WHEN NOT MATCHED BY TARGET THEN INSERT")[
1
].split("WHEN NOT MATCHED BY SOURCE")[0]
assert "gcs_path" in insert_section
assert "NULL" in insert_section
def test_merge_deletes_removed_files(self):
"""Files no longer in Drive should be deleted from the main table."""
mod = _mod()
client = self._setup_client_mock()
with patch.object(mod.bigquery, "Client", return_value=client):
mod.load_to_bigquery("proj", [{"file_id": "x"}])
merge_sql = client.query.call_args_list[0][0][0]
assert "WHEN NOT MATCHED BY SOURCE THEN DELETE" in merge_sql
def test_no_write_truncate_on_main_table(self):
"""WRITE_TRUNCATE must only target the staging table, never the main table."""
mod = _mod()
client = self._setup_client_mock()
with patch.object(mod.bigquery, "Client", return_value=client):
mod.load_to_bigquery("proj", [{"file_id": "x"}])
load_calls = client.load_table_from_json.call_args_list
assert len(load_calls) == 1
target = load_calls[0][0][1]
assert mod.STAGING_TABLE_ID in target
assert target != f"proj.{mod.DATASET_ID}.{mod.TABLE_ID}"