diff --git a/infrastructure/functions/divebar_mirror/index_builder.py b/infrastructure/functions/divebar_mirror/index_builder.py index 56103505..dfcc50b1 100644 --- a/infrastructure/functions/divebar_mirror/index_builder.py +++ b/infrastructure/functions/divebar_mirror/index_builder.py @@ -3,6 +3,9 @@ Build and upload the Divebar catalog index to BigQuery. Takes the flat file listing from drive_client and the parsed metadata from filename_parser, and loads it into a BigQuery table. + +Uses a staging table + MERGE to preserve gcs_path values set by the +file sync script (divebar_file_sync.py). """ import logging @@ -17,8 +20,10 @@ logger = logging.getLogger(__name__) # BigQuery table reference DATASET_ID = "karaoke_decide" TABLE_ID = "divebar_catalog" +STAGING_TABLE_ID = "divebar_catalog_staging" -SCHEMA = [ +# Schema for the staging table (no gcs_path — that lives only in the main table) +STAGING_SCHEMA = [ bigquery.SchemaField("file_id", "STRING", mode="REQUIRED"), bigquery.SchemaField("brand", "STRING", mode="REQUIRED"), bigquery.SchemaField("brand_code", "STRING"), @@ -78,31 +83,84 @@ def load_to_bigquery(project_id: str, rows: list[dict]) -> int: """ Load rows into the BigQuery divebar_catalog table. - Uses WRITE_TRUNCATE to atomically replace the entire table contents. - Returns the number of rows loaded. + Uses a staging table + MERGE to update the catalog without losing + gcs_path values set by the file sync script. Steps: + 1. WRITE_TRUNCATE the staging table with fresh Drive data + 2. MERGE staging into main table: + - UPDATE existing rows (by file_id) with fresh metadata + - INSERT new rows + - DELETE rows no longer in Drive + gcs_path is preserved on updates and defaults to NULL on inserts. + + Returns the number of rows in the main table after merge. """ if not rows: logger.warning("No rows to load") return 0 client = bigquery.Client(project=project_id) - table_ref = f"{project_id}.{DATASET_ID}.{TABLE_ID}" + staging_ref = f"{project_id}.{DATASET_ID}.{STAGING_TABLE_ID}" + main_ref = f"{project_id}.{DATASET_ID}.{TABLE_ID}" + # Step 1: Load fresh data into staging table (TRUNCATE is fine here) job_config = bigquery.LoadJobConfig( - schema=SCHEMA, + schema=STAGING_SCHEMA, write_disposition=bigquery.WriteDisposition.WRITE_TRUNCATE, source_format=bigquery.SourceFormat.NEWLINE_DELIMITED_JSON, ) - # BigQuery load from in-memory rows - load_job = client.load_table_from_json( - rows, - table_ref, - job_config=job_config, - ) - - # Wait for completion + load_job = client.load_table_from_json(rows, staging_ref, job_config=job_config) load_job.result() + logger.info("Loaded %d rows to staging table", load_job.output_rows) - logger.info("Loaded %d rows to %s", load_job.output_rows, table_ref) - return load_job.output_rows + # Step 2: MERGE staging into main table, preserving gcs_path + merge_sql = f""" + MERGE `{main_ref}` AS main + USING `{staging_ref}` AS staging + ON main.file_id = staging.file_id + + WHEN MATCHED THEN UPDATE SET + brand = staging.brand, + brand_code = staging.brand_code, + artist = staging.artist, + title = staging.title, + disc_id = staging.disc_id, + filename = staging.filename, + format = staging.format, + subfolder = staging.subfolder, + file_size = staging.file_size, + drive_path = staging.drive_path, + drive_md5 = staging.drive_md5, + artist_normalized = staging.artist_normalized, + title_normalized = staging.title_normalized, + synced_at = staging.synced_at + + WHEN NOT MATCHED BY TARGET THEN INSERT ( + file_id, brand, brand_code, artist, title, disc_id, + filename, format, subfolder, file_size, drive_path, + drive_md5, artist_normalized, title_normalized, synced_at, + gcs_path + ) VALUES ( + staging.file_id, staging.brand, staging.brand_code, + staging.artist, staging.title, staging.disc_id, + staging.filename, staging.format, staging.subfolder, + staging.file_size, staging.drive_path, staging.drive_md5, + staging.artist_normalized, staging.title_normalized, + staging.synced_at, NULL + ) + + WHEN NOT MATCHED BY SOURCE THEN DELETE + """ + + logger.info("Merging staging into main table (preserving gcs_path)...") + merge_job = client.query(merge_sql) + merge_job.result() + + # Get final row count + count_result = list(client.query( + f"SELECT COUNT(*) as cnt FROM `{main_ref}`" + ).result()) + final_count = count_result[0].cnt if count_result else 0 + + logger.info("Merge complete. Main table has %d rows", final_count) + return final_count diff --git a/infrastructure/scripts/divebar_file_sync.py b/infrastructure/scripts/divebar_file_sync.py index ee781111..b4d0d6e3 100644 --- a/infrastructure/scripts/divebar_file_sync.py +++ b/infrastructure/scripts/divebar_file_sync.py @@ -150,12 +150,24 @@ def update_gcs_path(bq_client, file_id, gcs_path): def sync_file(drive_service, gcs_bucket, bq_client, row): - """Sync a single file from Drive to GCS.""" + """Sync a single file from Drive to GCS. + + Checks if the file already exists in GCS before downloading from Drive. + If it exists, just updates gcs_path in BigQuery (no re-download needed). + """ file_id = row.file_id drive_path = row.drive_path gcs_path = f"files/{drive_path}" + full_gcs_path = f"gs://{GCS_BUCKET}/{gcs_path}" size_mb = (row.file_size or 0) / 1024 / 1024 + # Check if file already exists in GCS (avoids re-downloading) + blob = gcs_bucket.blob(gcs_path) + if blob.exists(): + update_gcs_path(bq_client, file_id, full_gcs_path) + logger.info(" ⏭ %s already in GCS, updated gcs_path", drive_path) + return True, 0 + logger.info("Downloading %s (%.1f MB)...", drive_path, size_mb) start = time.time() @@ -166,7 +178,6 @@ def sync_file(drive_service, gcs_bucket, bq_client, row): if ok: duration = time.time() - start speed = actual_size / 1024 / 1024 / duration if duration > 0 else 0 - full_gcs_path = f"gs://{GCS_BUCKET}/{gcs_path}" update_gcs_path(bq_client, file_id, full_gcs_path) logger.info( " ✓ %s (%.1f MB in %.1fs, %.1f MB/s)", @@ -220,54 +231,51 @@ def main(): gcs_bucket = gcs_client.bucket(GCS_BUCKET) drive_service = get_drive_service() - synced = 0 + downloaded = 0 + recovered = 0 failed = 0 bytes_synced = 0 overall_start = time.time() + def _tally(ok, size): + nonlocal downloaded, recovered, failed, bytes_synced + if ok: + if size > 0: + downloaded += 1 + bytes_synced += size + else: + recovered += 1 + else: + failed += 1 + + def _log_progress(): + elapsed = time.time() - overall_start + logger.info( + "Progress: %d downloaded, %d recovered, %d failed (of %d), %.1f GB, %.0f s", + downloaded, recovered, failed, len(files), + bytes_synced / 1024 / 1024 / 1024, elapsed, + ) + if args.workers > 1: - # Parallel downloads with ThreadPoolExecutor(max_workers=args.workers) as executor: futures = { executor.submit(sync_file, drive_service, gcs_bucket, bq_client, row): row for row in files } for future in as_completed(futures): - ok, size = future.result() - if ok: - synced += 1 - bytes_synced += size - else: - failed += 1 - - if (synced + failed) % 50 == 0: - elapsed = time.time() - overall_start - logger.info( - "Progress: %d/%d synced, %d failed, %.1f GB, %.0f s elapsed", - synced, len(files), failed, bytes_synced / 1024 / 1024 / 1024, elapsed, - ) + _tally(*future.result()) + if (downloaded + recovered + failed) % 50 == 0: + _log_progress() else: - # Sequential downloads for i, row in enumerate(files): - ok, size = sync_file(drive_service, gcs_bucket, bq_client, row) - if ok: - synced += 1 - bytes_synced += size - else: - failed += 1 - + _tally(*sync_file(drive_service, gcs_bucket, bq_client, row)) if (i + 1) % 50 == 0: - elapsed = time.time() - overall_start - logger.info( - "Progress: %d/%d synced, %d failed, %.1f GB, %.0f s elapsed", - synced, len(files), failed, bytes_synced / 1024 / 1024 / 1024, elapsed, - ) + _log_progress() elapsed = time.time() - overall_start logger.info( - "Done: %d synced, %d failed, %.1f GB in %.0f s (%.1f MB/s avg)", - synced, failed, bytes_synced / 1024 / 1024 / 1024, elapsed, - bytes_synced / 1024 / 1024 / elapsed if elapsed > 0 else 0, + "Done: %d downloaded, %d recovered from GCS, %d failed, %.1f GB in %.0f s", + downloaded, recovered, failed, bytes_synced / 1024 / 1024 / 1024, elapsed, ) diff --git a/tests/unit/test_divebar_index_builder.py b/tests/unit/test_divebar_index_builder.py new file mode 100644 index 00000000..026dcc63 --- /dev/null +++ b/tests/unit/test_divebar_index_builder.py @@ -0,0 +1,142 @@ +"""Tests for divebar mirror index_builder — staging+MERGE approach. + +The Cloud Function has its own deps (google-cloud-bigquery, filename_parser) +that aren't in the main project's poetry env. We mock them at the module level +before importing index_builder. +""" + +import importlib +import sys +import types +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +# --------------------------------------------------------------------------- +# Pre-import mocking: google.cloud.bigquery and filename_parser are not +# available in the CI test environment (they're Cloud Function deps). +# --------------------------------------------------------------------------- + +_mock_bigquery = MagicMock() +# SchemaField needs to be callable and return a mock +_mock_bigquery.SchemaField = MagicMock + +_mock_google = types.ModuleType("google") +_mock_google.cloud = types.ModuleType("google.cloud") +_mock_google.cloud.bigquery = _mock_bigquery + +_mock_filename_parser = MagicMock() + + +@pytest.fixture(autouse=True) +def _import_index_builder(monkeypatch): + """Import index_builder with mocked external deps.""" + monkeypatch.setitem(sys.modules, "google", _mock_google) + monkeypatch.setitem(sys.modules, "google.cloud", _mock_google.cloud) + monkeypatch.setitem(sys.modules, "google.cloud.bigquery", _mock_bigquery) + monkeypatch.setitem(sys.modules, "filename_parser", _mock_filename_parser) + + func_dir = str( + Path(__file__).resolve().parents[2] + / "infrastructure" + / "functions" + / "divebar_mirror" + ) + monkeypatch.syspath_prepend(func_dir) + + if "index_builder" in sys.modules: + importlib.reload(sys.modules["index_builder"]) + else: + importlib.import_module("index_builder") + + +def _mod(): + return sys.modules["index_builder"] + + +class TestLoadToBigquery: + """Verify load_to_bigquery uses staging+MERGE instead of WRITE_TRUNCATE on main table.""" + + def _setup_client_mock(self): + """Set up a mock BigQuery client with staging load + merge + count.""" + client = MagicMock() + load_job = MagicMock() + load_job.output_rows = 1 + client.load_table_from_json.return_value = load_job + + merge_job = MagicMock() + count_row = MagicMock() + count_row.cnt = 1 + count_result = MagicMock() + count_result.result.return_value = [count_row] + client.query.side_effect = [merge_job, count_result] + return client + + def test_empty_rows_returns_zero(self): + assert _mod().load_to_bigquery("proj", []) == 0 + + def test_loads_to_staging_table_not_main(self): + """The load job must target the staging table, not the main table.""" + mod = _mod() + client = self._setup_client_mock() + + with patch.object(mod.bigquery, "Client", return_value=client): + mod.load_to_bigquery("proj", [{"file_id": "abc"}]) + + load_target = client.load_table_from_json.call_args[0][1] + assert mod.STAGING_TABLE_ID in load_target + assert load_target == f"proj.{mod.DATASET_ID}.{mod.STAGING_TABLE_ID}" + + merge_sql = client.query.call_args_list[0][0][0] + assert "MERGE" in merge_sql + assert f"{mod.DATASET_ID}.{mod.TABLE_ID}" in merge_sql + assert f"{mod.DATASET_ID}.{mod.STAGING_TABLE_ID}" in merge_sql + + def test_merge_preserves_gcs_path(self): + """The MERGE SQL must preserve gcs_path on UPDATE and set NULL on INSERT.""" + mod = _mod() + client = self._setup_client_mock() + + with patch.object(mod.bigquery, "Client", return_value=client): + mod.load_to_bigquery("proj", [{"file_id": "x"}]) + + merge_sql = client.query.call_args_list[0][0][0] + + # gcs_path must NOT appear in the UPDATE SET clause + update_section = merge_sql.split("WHEN MATCHED THEN UPDATE SET")[1].split( + "WHEN NOT MATCHED" + )[0] + assert "gcs_path" not in update_section + + # gcs_path must appear in INSERT with NULL default + insert_section = merge_sql.split("WHEN NOT MATCHED BY TARGET THEN INSERT")[ + 1 + ].split("WHEN NOT MATCHED BY SOURCE")[0] + assert "gcs_path" in insert_section + assert "NULL" in insert_section + + def test_merge_deletes_removed_files(self): + """Files no longer in Drive should be deleted from the main table.""" + mod = _mod() + client = self._setup_client_mock() + + with patch.object(mod.bigquery, "Client", return_value=client): + mod.load_to_bigquery("proj", [{"file_id": "x"}]) + + merge_sql = client.query.call_args_list[0][0][0] + assert "WHEN NOT MATCHED BY SOURCE THEN DELETE" in merge_sql + + def test_no_write_truncate_on_main_table(self): + """WRITE_TRUNCATE must only target the staging table, never the main table.""" + mod = _mod() + client = self._setup_client_mock() + + with patch.object(mod.bigquery, "Client", return_value=client): + mod.load_to_bigquery("proj", [{"file_id": "x"}]) + + load_calls = client.load_table_from_json.call_args_list + assert len(load_calls) == 1 + target = load_calls[0][0][1] + assert mod.STAGING_TABLE_ID in target + assert target != f"proj.{mod.DATASET_ID}.{mod.TABLE_ID}"