Files
karaoke-gen/scripts/backfill_gdrive_uploads.py
Andrew Beveridge ☄️andClaude Opus 4.6 0f5ddb0092 Fix silent Google Drive upload failures with retry + alerting (#417)
## Summary
- Added tenacity retry with connection reset to Google Drive API calls
(`get_or_create_folder`, `upload_file`, `delete_file`) to handle stale
HTTP connections on idle Cloud Run containers
- Added `distribution_warnings` field to `OrchestratorResult` and
`_handle_native_distribution()` to track non-fatal upload failures
(YouTube, Dropbox, GDrive)
- Added Pushbullet notification when distribution uploads fail, so
operator is alerted immediately
- Added backfill script (`scripts/backfill_gdrive_uploads.py`) for
affected jobs - already run successfully for 12 affected jobs
- Updated docs: LESSONS-LEARNED.md and TROUBLESHOOTING.md with GDrive
stale connection pattern

## Context
Every GDrive upload silently failed for ~1 month (64 errors, 12+
affected jobs). The root cause was the singleton `GoogleDriveService`
caching its HTTP connection, which went stale when Cloud Run containers
sat idle between jobs. The orchestrator caught all exceptions and
continued, so jobs completed with `gdrive_files: {}` and no alert was
sent.

## Test plan
- [x] 5 new retry behavior tests in `test_gdrive_service.py`
(BrokenPipeError, SSLError, give up after 3, service reset, no retry on
non-transient)
- [x] 3 new distribution warnings tests in
`test_video_worker_orchestrator.py`
- [x] All 76 affected tests pass
- [x] Full backend test suite passes
- [x] Backfill script already run successfully for all 12 affected jobs

@coderabbitai ignore

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-27 19:47:30 +00:00

304 lines
9.6 KiB
Python

#!/usr/bin/env python3
"""
Backfill Google Drive uploads for jobs that completed with empty gdrive_files.
This addresses the issue where GDrive uploads silently failed due to stale
HTTP connections (BrokenPipeError / SSLError) on idle Cloud Run containers.
Usage:
# Dry run - show what would be uploaded
python scripts/backfill_gdrive_uploads.py --dry-run --job-ids ca408e46,99baa198
# Upload specific jobs
python scripts/backfill_gdrive_uploads.py --job-ids ca408e46,99baa198
# Find and upload all jobs with empty gdrive_files
python scripts/backfill_gdrive_uploads.py --all-missing
# Override gdrive folder ID (instead of using DEFAULT_GDRIVE_FOLDER_ID env var)
python scripts/backfill_gdrive_uploads.py --job-ids ca408e46 --gdrive-folder-id 1abc...
Requires:
- google-cloud-firestore
- google-cloud-storage
- google-api-python-client, google-auth
- GOOGLE_APPLICATION_CREDENTIALS or gcloud auth application-default login
"""
import argparse
import logging
import os
import sys
import tempfile
# Add project root to path so we can import backend modules
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from google.cloud import firestore, storage
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(message)s",
)
logger = logging.getLogger(__name__)
GCS_BUCKET = "karaoke-gen-storage-nomadkaraoke"
PROJECT = "nomadkaraoke"
# Default GDrive public share folder (from infrastructure/config.py)
DEFAULT_GDRIVE_FOLDER_ID = "1laRKAyxo0v817SstfM5XkpbWiNKNAMSX"
# Known affected jobs from the incident
KNOWN_AFFECTED_JOBS = [
"ca408e46", "99baa198", "1e15d1dd", "ebba0996",
"cb744c24", "1389f765", "fde3ae7d", "21f4b34f",
"443c34a1", "56192d3a", "fdb4dc63", "4eeaa729",
]
def get_job(db: firestore.Client, job_id: str) -> dict | None:
"""Fetch job document from Firestore."""
doc = db.collection("jobs").document(job_id).get()
if doc.exists:
return doc.to_dict()
return None
def find_jobs_with_missing_gdrive(db: firestore.Client) -> list[dict]:
"""Find completed jobs that have empty gdrive_files in state_data.
Looks for jobs that have a brand_code (indicating they went through
distribution) but have empty or missing gdrive_files.
"""
jobs = []
query = db.collection("jobs").where("status", "==", "complete")
for doc in query.stream():
data = doc.to_dict()
state_data = data.get("state_data", {})
gdrive_files = state_data.get("gdrive_files", {})
brand_code = state_data.get("brand_code")
# Job should have had GDrive uploads if it has a brand_code
# but gdrive_files is empty/missing
if brand_code and not gdrive_files:
jobs.append({"job_id": doc.id, **data})
return jobs
def _classify_blob(filename: str) -> str | None:
"""Classify a GCS blob filename to an output_files key, or None to skip.
Only returns keys for the 3 file types that get uploaded to GDrive:
lossy 4K MP4, lossy 720p MP4, and CDG ZIP.
"""
lower = filename.lower()
if lower.endswith(".zip") and "cdg" in lower:
return "final_karaoke_cdg_zip"
if "720p" in lower and lower.endswith(".mp4"):
return "final_karaoke_lossy_720p_mp4"
if "lossy" in lower and lower.endswith(".mp4") and "720p" not in lower:
return "final_karaoke_lossy_mp4"
return None
def download_finals(gcs_client: storage.Client, job_id: str, temp_dir: str) -> dict:
"""Download only the final output files needed for GDrive upload.
Only downloads lossy 4K MP4, lossy 720p MP4, and CDG ZIP — skips
lossless files, MKVs, karaoke video, etc. to save bandwidth.
Returns dict mapping output_files keys to local paths.
"""
bucket = gcs_client.bucket(GCS_BUCKET)
prefix = f"jobs/{job_id}/finals/"
blobs = list(bucket.list_blobs(prefix=prefix))
output_files = {}
for blob in blobs:
filename = os.path.basename(blob.name)
if not filename:
continue
key = _classify_blob(filename)
if key is None:
continue
if key in output_files:
continue # already have this type
local_path = os.path.join(temp_dir, filename)
logger.info(f" Downloading {filename} ({blob.size / 1024 / 1024:.1f} MB)")
blob.download_to_filename(local_path)
output_files[key] = local_path
return output_files
def list_finals(gcs_client: storage.Client, job_id: str) -> list[str]:
"""List uploadable final output files in GCS without downloading."""
bucket = gcs_client.bucket(GCS_BUCKET)
prefix = f"jobs/{job_id}/finals/"
blobs = list(bucket.list_blobs(prefix=prefix))
return [
f"{os.path.basename(b.name)} ({b.size / 1024 / 1024:.1f} MB)"
for b in blobs
if os.path.basename(b.name) and _classify_blob(os.path.basename(b.name))
]
def backfill_job(
db: firestore.Client,
gcs_client: storage.Client,
job_id: str,
gdrive_folder_id: str,
dry_run: bool = False,
) -> bool:
"""Backfill GDrive upload for a single job.
Returns True if successful (or dry run).
"""
logger.info(f"Processing job {job_id}")
# Fetch job data
job_data = get_job(db, job_id)
if not job_data:
logger.error(f" Job {job_id} not found in Firestore")
return False
state_data = job_data.get("state_data", {})
artist = job_data.get("artist", "Unknown")
title = job_data.get("title", "Unknown")
brand_code = state_data.get("brand_code")
status = job_data.get("status")
logger.info(f" {artist} - {title} (brand: {brand_code}, status: {status})")
# Check existing gdrive_files
existing = state_data.get("gdrive_files", {})
if existing:
logger.info(f" Already has gdrive_files: {existing}")
logger.info(f" Skipping (already uploaded)")
return True
if not brand_code:
logger.warning(f" No brand code, skipping")
return False
if dry_run:
# List what files are available in GCS
files = list_finals(gcs_client, job_id)
if files:
logger.info(f" [DRY RUN] Would upload {len(files)} files to GDrive folder {gdrive_folder_id}:")
for f in files:
logger.info(f" - {f}")
else:
logger.warning(f" [DRY RUN] No final files found in GCS")
return True
# Download files and upload
with tempfile.TemporaryDirectory(prefix=f"backfill-{job_id}-") as temp_dir:
# Download finals from GCS
logger.info(f" Downloading finals from GCS...")
output_files = download_finals(gcs_client, job_id, temp_dir)
if not output_files:
logger.warning(f" No final files found in GCS")
return False
logger.info(f" Found {len(output_files)} files to upload")
# Upload to GDrive
from backend.services.gdrive_service import GoogleDriveService
gdrive = GoogleDriveService()
if not gdrive.is_configured:
logger.error(f" Google Drive not configured")
return False
base_name = f"{artist} - {title}"
uploaded = gdrive.upload_to_public_share(
root_folder_id=gdrive_folder_id,
brand_code=brand_code,
base_name=base_name,
output_files=output_files,
)
logger.info(f" Uploaded {len(uploaded)} files: {uploaded}")
# Update Firestore
db.collection("jobs").document(job_id).update({
"state_data.gdrive_files": uploaded,
})
logger.info(f" Updated Firestore state_data.gdrive_files")
return True
def main():
parser = argparse.ArgumentParser(
description="Backfill Google Drive uploads for jobs with empty gdrive_files"
)
parser.add_argument(
"--job-ids",
help="Comma-separated list of job IDs to process",
)
parser.add_argument(
"--all-missing",
action="store_true",
help="Find and process all jobs with missing gdrive_files",
)
parser.add_argument(
"--dry-run",
action="store_true",
help="Show what would be done without making changes",
)
parser.add_argument(
"--gdrive-folder-id",
default=os.getenv("DEFAULT_GDRIVE_FOLDER_ID", DEFAULT_GDRIVE_FOLDER_ID),
help=f"Google Drive folder ID for public share (default: {DEFAULT_GDRIVE_FOLDER_ID})",
)
args = parser.parse_args()
if not args.job_ids and not args.all_missing:
parser.error("Specify --job-ids or --all-missing")
db = firestore.Client(project=PROJECT)
gcs_client = storage.Client(project=PROJECT)
if args.job_ids:
job_ids = [j.strip() for j in args.job_ids.split(",")]
elif args.all_missing:
logger.info("Searching for jobs with missing GDrive uploads...")
jobs = find_jobs_with_missing_gdrive(db)
job_ids = [j["job_id"] for j in jobs]
logger.info(f"Found {len(job_ids)} jobs with missing uploads")
if args.dry_run:
logger.info("[DRY RUN MODE]")
logger.info(f"GDrive folder: {args.gdrive_folder_id}")
success_count = 0
fail_count = 0
for job_id in job_ids:
try:
ok = backfill_job(
db, gcs_client, job_id,
gdrive_folder_id=args.gdrive_folder_id,
dry_run=args.dry_run,
)
if ok:
success_count += 1
else:
fail_count += 1
except Exception as e:
logger.error(f" Failed: {e}", exc_info=True)
fail_count += 1
logger.info(f"\nDone: {success_count} succeeded, {fail_count} failed out of {len(job_ids)} jobs")
if __name__ == "__main__":
main()