mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
## Summary - **index_builder.py**: replaced `WRITE_TRUNCATE` with staging table + `MERGE` SQL to preserve `gcs_path` values during daily catalog index updates - **divebar_file_sync.py**: added `blob.exists()` check before downloading from Drive — if a file is already in GCS, just updates `gcs_path` in BigQuery without re-downloading - Added unit tests verifying the MERGE approach preserves `gcs_path` correctly **Root cause**: The daily 2 AM index sync was atomically replacing the entire `divebar_catalog` BigQuery table, wiping all `gcs_path` values. The 3 AM file sync then saw all ~48K files as unsynced and re-downloaded the entire ~779 GB catalog from Google Drive every night. ## Test plan - [x] Unit tests for staging+MERGE behavior (5 tests pass) - [ ] Verify staging table auto-creates on first run (BigQuery creates on WRITE_TRUNCATE with schema) - [ ] Monitor next nightly sync — file_sync should report "recovered from GCS" instead of downloading - [ ] Confirm gcs_path values persist across consecutive index runs @coderabbitai ignore 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
284 lines
9.1 KiB
Python
284 lines
9.1 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Divebar File Sync — Download files from Google Drive to GCS.
|
|
|
|
Reads the BigQuery divebar_catalog index and downloads all files that don't
|
|
yet exist in GCS. Progress is tracked by updating a `gcs_path` column in
|
|
BigQuery after each successful upload.
|
|
|
|
Usage:
|
|
# Sync all files (initial bulk or incremental)
|
|
python divebar_file_sync.py
|
|
|
|
# Sync with concurrency (faster but uses more Drive API quota)
|
|
python divebar_file_sync.py --workers 4
|
|
|
|
# Dry run (show what would be downloaded)
|
|
python divebar_file_sync.py --dry-run
|
|
|
|
# Sync only specific brand
|
|
python divebar_file_sync.py --brand "WTF Karaoke Videos"
|
|
|
|
# Limit number of files (useful for testing)
|
|
python divebar_file_sync.py --limit 100
|
|
|
|
Requirements:
|
|
pip install google-api-python-client google-cloud-storage google-cloud-bigquery
|
|
|
|
Environment:
|
|
GOOGLE_CLOUD_PROJECT (default: nomadkaraoke)
|
|
GCS_BUCKET (default: nomadkaraoke-divebar-files)
|
|
"""
|
|
|
|
import argparse
|
|
import io
|
|
import logging
|
|
import os
|
|
import sys
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
|
|
from google.auth import default
|
|
from google.cloud import bigquery, storage
|
|
from googleapiclient.discovery import build
|
|
from googleapiclient.http import MediaIoBaseDownload
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s %(levelname)s %(message)s",
|
|
datefmt="%H:%M:%S",
|
|
)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
PROJECT_ID = os.environ.get("GOOGLE_CLOUD_PROJECT", "nomadkaraoke")
|
|
GCS_BUCKET = os.environ.get("GCS_BUCKET", "nomadkaraoke-divebar-files")
|
|
DATASET = "karaoke_decide"
|
|
TABLE = "divebar_catalog"
|
|
|
|
|
|
def get_drive_service():
|
|
"""Create Google Drive API service."""
|
|
credentials, _ = default(scopes=["https://www.googleapis.com/auth/drive.readonly"])
|
|
return build("drive", "v3", credentials=credentials)
|
|
|
|
|
|
def get_unsynced_files(bq_client, brand=None, limit=None):
|
|
"""Get files from BigQuery that haven't been synced to GCS yet."""
|
|
where = "WHERE gcs_path IS NULL"
|
|
if brand:
|
|
where += f" AND brand = @brand"
|
|
|
|
query = f"""
|
|
SELECT file_id, brand, filename, drive_path, file_size, drive_md5
|
|
FROM `{PROJECT_ID}.{DATASET}.{TABLE}`
|
|
{where}
|
|
ORDER BY file_size ASC
|
|
"""
|
|
if limit:
|
|
query += f" LIMIT {limit}"
|
|
|
|
params = []
|
|
if brand:
|
|
params.append(bigquery.ScalarQueryParameter("brand", "STRING", brand))
|
|
|
|
job_config = bigquery.QueryJobConfig(query_parameters=params) if params else None
|
|
rows = list(bq_client.query(query, job_config=job_config).result())
|
|
return rows
|
|
|
|
|
|
def get_sync_stats(bq_client):
|
|
"""Get current sync progress."""
|
|
query = f"""
|
|
SELECT
|
|
COUNT(*) as total,
|
|
COUNTIF(gcs_path IS NOT NULL) as synced,
|
|
COUNTIF(gcs_path IS NULL) as pending,
|
|
ROUND(SUM(CASE WHEN gcs_path IS NOT NULL THEN file_size ELSE 0 END) / 1024/1024/1024, 1) as synced_gb,
|
|
ROUND(SUM(CASE WHEN gcs_path IS NULL THEN file_size ELSE 0 END) / 1024/1024/1024, 1) as pending_gb
|
|
FROM `{PROJECT_ID}.{DATASET}.{TABLE}`
|
|
"""
|
|
row = list(bq_client.query(query).result())[0]
|
|
return {
|
|
"total": row.total,
|
|
"synced": row.synced,
|
|
"pending": row.pending,
|
|
"synced_gb": row.synced_gb,
|
|
"pending_gb": row.pending_gb,
|
|
}
|
|
|
|
|
|
def download_and_upload(drive_service, gcs_bucket, file_id, gcs_path, expected_size=None):
|
|
"""Download a file from Drive and upload to GCS. Returns True on success."""
|
|
try:
|
|
# Download from Drive
|
|
request = drive_service.files().get_media(fileId=file_id)
|
|
buffer = io.BytesIO()
|
|
downloader = MediaIoBaseDownload(buffer, request)
|
|
|
|
done = False
|
|
while not done:
|
|
_, done = downloader.next_chunk()
|
|
|
|
buffer.seek(0)
|
|
actual_size = buffer.getbuffer().nbytes
|
|
|
|
# Upload to GCS
|
|
blob = gcs_bucket.blob(gcs_path)
|
|
blob.upload_from_file(buffer, timeout=300)
|
|
|
|
return True, actual_size
|
|
|
|
except Exception as e:
|
|
logger.error("Failed %s: %s", gcs_path, e)
|
|
return False, 0
|
|
|
|
|
|
def update_gcs_path(bq_client, file_id, gcs_path):
|
|
"""Update the gcs_path in BigQuery to mark a file as synced."""
|
|
query = f"""
|
|
UPDATE `{PROJECT_ID}.{DATASET}.{TABLE}`
|
|
SET gcs_path = @gcs_path
|
|
WHERE file_id = @file_id
|
|
"""
|
|
job_config = bigquery.QueryJobConfig(
|
|
query_parameters=[
|
|
bigquery.ScalarQueryParameter("gcs_path", "STRING", gcs_path),
|
|
bigquery.ScalarQueryParameter("file_id", "STRING", file_id),
|
|
]
|
|
)
|
|
bq_client.query(query, job_config=job_config).result()
|
|
|
|
|
|
def sync_file(drive_service, gcs_bucket, bq_client, row):
|
|
"""Sync a single file from Drive to GCS.
|
|
|
|
Checks if the file already exists in GCS before downloading from Drive.
|
|
If it exists, just updates gcs_path in BigQuery (no re-download needed).
|
|
"""
|
|
file_id = row.file_id
|
|
drive_path = row.drive_path
|
|
gcs_path = f"files/{drive_path}"
|
|
full_gcs_path = f"gs://{GCS_BUCKET}/{gcs_path}"
|
|
size_mb = (row.file_size or 0) / 1024 / 1024
|
|
|
|
# Check if file already exists in GCS (avoids re-downloading)
|
|
blob = gcs_bucket.blob(gcs_path)
|
|
if blob.exists():
|
|
update_gcs_path(bq_client, file_id, full_gcs_path)
|
|
logger.info(" ⏭ %s already in GCS, updated gcs_path", drive_path)
|
|
return True, 0
|
|
|
|
logger.info("Downloading %s (%.1f MB)...", drive_path, size_mb)
|
|
start = time.time()
|
|
|
|
ok, actual_size = download_and_upload(
|
|
drive_service, gcs_bucket, file_id, gcs_path, row.file_size
|
|
)
|
|
|
|
if ok:
|
|
duration = time.time() - start
|
|
speed = actual_size / 1024 / 1024 / duration if duration > 0 else 0
|
|
update_gcs_path(bq_client, file_id, full_gcs_path)
|
|
logger.info(
|
|
" ✓ %s (%.1f MB in %.1fs, %.1f MB/s)",
|
|
drive_path, actual_size / 1024 / 1024, duration, speed,
|
|
)
|
|
return True, actual_size
|
|
else:
|
|
return False, 0
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Sync Divebar files from Google Drive to GCS")
|
|
parser.add_argument("--workers", type=int, default=1, help="Concurrent download workers (default: 1)")
|
|
parser.add_argument("--dry-run", action="store_true", help="Show what would be downloaded")
|
|
parser.add_argument("--brand", help="Only sync files from this brand")
|
|
parser.add_argument("--limit", type=int, help="Maximum files to sync")
|
|
args = parser.parse_args()
|
|
|
|
bq_client = bigquery.Client(project=PROJECT_ID)
|
|
gcs_client = storage.Client(project=PROJECT_ID)
|
|
|
|
# Show current progress
|
|
stats = get_sync_stats(bq_client)
|
|
logger.info(
|
|
"Sync status: %d/%d files synced (%.1f/%.1f GB), %d pending (%.1f GB)",
|
|
stats["synced"], stats["total"], stats["synced_gb"],
|
|
stats["synced_gb"] + stats["pending_gb"],
|
|
stats["pending"], stats["pending_gb"],
|
|
)
|
|
|
|
# Get unsynced files
|
|
files = get_unsynced_files(bq_client, brand=args.brand, limit=args.limit)
|
|
if not files:
|
|
logger.info("All files are synced!")
|
|
return
|
|
|
|
total_size = sum(r.file_size or 0 for r in files)
|
|
logger.info(
|
|
"Will sync %d files (%.1f GB)%s",
|
|
len(files), total_size / 1024 / 1024 / 1024,
|
|
" [DRY RUN]" if args.dry_run else "",
|
|
)
|
|
|
|
if args.dry_run:
|
|
for row in files[:20]:
|
|
logger.info(" Would download: %s (%.1f MB)", row.drive_path, (row.file_size or 0) / 1024 / 1024)
|
|
if len(files) > 20:
|
|
logger.info(" ... and %d more", len(files) - 20)
|
|
return
|
|
|
|
gcs_bucket = gcs_client.bucket(GCS_BUCKET)
|
|
drive_service = get_drive_service()
|
|
|
|
downloaded = 0
|
|
recovered = 0
|
|
failed = 0
|
|
bytes_synced = 0
|
|
overall_start = time.time()
|
|
|
|
def _tally(ok, size):
|
|
nonlocal downloaded, recovered, failed, bytes_synced
|
|
if ok:
|
|
if size > 0:
|
|
downloaded += 1
|
|
bytes_synced += size
|
|
else:
|
|
recovered += 1
|
|
else:
|
|
failed += 1
|
|
|
|
def _log_progress():
|
|
elapsed = time.time() - overall_start
|
|
logger.info(
|
|
"Progress: %d downloaded, %d recovered, %d failed (of %d), %.1f GB, %.0f s",
|
|
downloaded, recovered, failed, len(files),
|
|
bytes_synced / 1024 / 1024 / 1024, elapsed,
|
|
)
|
|
|
|
if args.workers > 1:
|
|
with ThreadPoolExecutor(max_workers=args.workers) as executor:
|
|
futures = {
|
|
executor.submit(sync_file, drive_service, gcs_bucket, bq_client, row): row
|
|
for row in files
|
|
}
|
|
for future in as_completed(futures):
|
|
_tally(*future.result())
|
|
if (downloaded + recovered + failed) % 50 == 0:
|
|
_log_progress()
|
|
else:
|
|
for i, row in enumerate(files):
|
|
_tally(*sync_file(drive_service, gcs_bucket, bq_client, row))
|
|
if (i + 1) % 50 == 0:
|
|
_log_progress()
|
|
|
|
elapsed = time.time() - overall_start
|
|
logger.info(
|
|
"Done: %d downloaded, %d recovered from GCS, %d failed, %.1f GB in %.0f s",
|
|
downloaded, recovered, failed, bytes_synced / 1024 / 1024 / 1024, elapsed,
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|