Fix silent Google Drive upload failures with retry + alerting (#417)

## Summary
- Added tenacity retry with connection reset to Google Drive API calls
(`get_or_create_folder`, `upload_file`, `delete_file`) to handle stale
HTTP connections on idle Cloud Run containers
- Added `distribution_warnings` field to `OrchestratorResult` and
`_handle_native_distribution()` to track non-fatal upload failures
(YouTube, Dropbox, GDrive)
- Added Pushbullet notification when distribution uploads fail, so
operator is alerted immediately
- Added backfill script (`scripts/backfill_gdrive_uploads.py`) for
affected jobs - already run successfully for 12 affected jobs
- Updated docs: LESSONS-LEARNED.md and TROUBLESHOOTING.md with GDrive
stale connection pattern

## Context
Every GDrive upload silently failed for ~1 month (64 errors, 12+
affected jobs). The root cause was the singleton `GoogleDriveService`
caching its HTTP connection, which went stale when Cloud Run containers
sat idle between jobs. The orchestrator caught all exceptions and
continued, so jobs completed with `gdrive_files: {}` and no alert was
sent.

## Test plan
- [x] 5 new retry behavior tests in `test_gdrive_service.py`
(BrokenPipeError, SSLError, give up after 3, service reset, no retry on
non-transient)
- [x] 3 new distribution warnings tests in
`test_video_worker_orchestrator.py`
- [x] All 76 affected tests pass
- [x] Full backend test suite passes
- [x] Backfill script already run successfully for all 12 affected jobs

@coderabbitai ignore

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Andrew Beveridge ☄️andClaude Opus 4.6 authored and GitHub committed 2026-02-27 19:47:30 +00:00
1 parent 23a8b0fb98
commit 0f5ddb0092
9 files changed
+775 -84

No files matched your search

+110 -66
View File
@@ -12,13 +12,19 @@ shared with YouTube credentials if scopes include drive.file.
import json
import logging
import os
import ssl
from typing import Any, Dict, Optional
from tenacity import Retrying, stop_after_attempt, wait_exponential, retry_if_exception_type
from backend.config import get_settings
from karaoke_gen.utils import sanitize_filename
logger = logging.getLogger(__name__)
# Transient errors that indicate a stale HTTP connection (Cloud Run idle containers)
TRANSIENT_ERRORS = (BrokenPipeError, ConnectionResetError, ConnectionError, ssl.SSLError)
class GoogleDriveService:
"""Google Drive operations using native API."""
@@ -33,6 +39,11 @@ class GoogleDriveService:
self._credentials_data: Optional[Dict[str, Any]] = None
self._loaded = False
def _reset_service(self):
"""Reset the cached Drive service to force a fresh connection on next use."""
logger.info("Resetting Google Drive service connection")
self._service = None
def _load_credentials(self) -> Optional[Dict[str, Any]]:
"""Load OAuth credentials from Secret Manager."""
if self._loaded:
@@ -127,6 +138,9 @@ class GoogleDriveService:
"""
Get existing folder or create new one, return folder ID.
Retries on transient connection errors (BrokenPipeError, SSLError)
that occur when Cloud Run containers sit idle between jobs.
Args:
parent_id: Parent folder ID
folder_name: Name of folder to find or create
@@ -134,33 +148,41 @@ class GoogleDriveService:
Returns:
Folder ID
"""
logger.info(f"Looking for folder '{folder_name}' in parent {parent_id}")
for attempt in Retrying(
stop=stop_after_attempt(3),
wait=wait_exponential(multiplier=1, min=2, max=8),
retry=retry_if_exception_type(TRANSIENT_ERRORS),
before_sleep=lambda retry_state: self._reset_service(),
reraise=True,
):
with attempt:
logger.info(f"Looking for folder '{folder_name}' in parent {parent_id}")
# Search for existing folder
# Escape single quotes in folder name for Google Drive API query syntax
escaped_folder_name = folder_name.replace("'", "\\'")
query = (
f"name='{escaped_folder_name}' and '{parent_id}' in parents "
f"and mimeType='application/vnd.google-apps.folder' and trashed=false"
)
results = self.service.files().list(q=query, fields="files(id, name)").execute()
# Search for existing folder
# Escape single quotes in folder name for Google Drive API query syntax
escaped_folder_name = folder_name.replace("'", "\\'")
query = (
f"name='{escaped_folder_name}' and '{parent_id}' in parents "
f"and mimeType='application/vnd.google-apps.folder' and trashed=false"
)
results = self.service.files().list(q=query, fields="files(id, name)").execute()
if results.get("files"):
folder_id = results["files"][0]["id"]
logger.info(f"Found existing folder '{folder_name}': {folder_id}")
return folder_id
if results.get("files"):
folder_id = results["files"][0]["id"]
logger.info(f"Found existing folder '{folder_name}': {folder_id}")
return folder_id
# Create folder
logger.info(f"Creating new folder '{folder_name}'")
metadata = {
"name": folder_name,
"mimeType": "application/vnd.google-apps.folder",
"parents": [parent_id],
}
folder = self.service.files().create(body=metadata, fields="id").execute()
folder_id = folder["id"]
logger.info(f"Created folder '{folder_name}': {folder_id}")
return folder_id
# Create folder
logger.info(f"Creating new folder '{folder_name}'")
metadata = {
"name": folder_name,
"mimeType": "application/vnd.google-apps.folder",
"parents": [parent_id],
}
folder = self.service.files().create(body=metadata, fields="id").execute()
folder_id = folder["id"]
logger.info(f"Created folder '{folder_name}': {folder_id}")
return folder_id
def upload_file(
self,
@@ -172,6 +194,9 @@ class GoogleDriveService:
"""
Upload a file to a specific Drive folder.
Retries on transient connection errors (BrokenPipeError, SSLError)
that occur when Cloud Run containers sit idle between jobs.
Args:
local_path: Local file path
parent_id: Parent folder ID in Google Drive
@@ -183,49 +208,57 @@ class GoogleDriveService:
"""
from googleapiclient.http import MediaFileUpload
file_size = os.path.getsize(local_path)
logger.info(
f"Uploading {local_path} ({file_size / 1024 / 1024:.1f} MB) "
f"as '{filename}' to folder {parent_id}"
)
for attempt in Retrying(
stop=stop_after_attempt(3),
wait=wait_exponential(multiplier=1, min=2, max=8),
retry=retry_if_exception_type(TRANSIENT_ERRORS),
before_sleep=lambda retry_state: self._reset_service(),
reraise=True,
):
with attempt:
file_size = os.path.getsize(local_path)
logger.info(
f"Uploading {local_path} ({file_size / 1024 / 1024:.1f} MB) "
f"as '{filename}' to folder {parent_id}"
)
# Determine MIME type
ext = os.path.splitext(local_path)[1].lower()
mime_types = {
".mp4": "video/mp4",
".mkv": "video/x-matroska",
".zip": "application/zip",
".flac": "audio/flac",
".mp3": "audio/mpeg",
".wav": "audio/wav",
".png": "image/png",
".jpg": "image/jpeg",
".jpeg": "image/jpeg",
}
mime_type = mime_types.get(ext, "application/octet-stream")
# Determine MIME type
ext = os.path.splitext(local_path)[1].lower()
mime_types = {
".mp4": "video/mp4",
".mkv": "video/x-matroska",
".zip": "application/zip",
".flac": "audio/flac",
".mp3": "audio/mpeg",
".wav": "audio/wav",
".png": "image/png",
".jpg": "image/jpeg",
".jpeg": "image/jpeg",
}
mime_type = mime_types.get(ext, "application/octet-stream")
# Check for existing file with same name
if replace_existing:
# Escape single quotes in filename for Google Drive API query syntax
escaped_filename = filename.replace("'", "\\'")
query = (
f"name='{escaped_filename}' and '{parent_id}' in parents and trashed=false"
)
results = self.service.files().list(q=query, fields="files(id)").execute()
for existing_file in results.get("files", []):
logger.info(f"Deleting existing file: {existing_file['id']}")
self.service.files().delete(fileId=existing_file["id"]).execute()
# Check for existing file with same name
if replace_existing:
# Escape single quotes in filename for Google Drive API query syntax
escaped_filename = filename.replace("'", "\\'")
query = (
f"name='{escaped_filename}' and '{parent_id}' in parents and trashed=false"
)
results = self.service.files().list(q=query, fields="files(id)").execute()
for existing_file in results.get("files", []):
logger.info(f"Deleting existing file: {existing_file['id']}")
self.service.files().delete(fileId=existing_file["id"]).execute()
# Upload file
metadata = {"name": filename, "parents": [parent_id]}
media = MediaFileUpload(local_path, mimetype=mime_type, resumable=True)
# Upload file
metadata = {"name": filename, "parents": [parent_id]}
media = MediaFileUpload(local_path, mimetype=mime_type, resumable=True)
file_result = (
self.service.files().create(body=metadata, media_body=media, fields="id").execute()
)
file_id = file_result["id"]
logger.info(f"Successfully uploaded '{filename}': {file_id}")
return file_id
file_result = (
self.service.files().create(body=metadata, media_body=media, fields="id").execute()
)
file_id = file_result["id"]
logger.info(f"Successfully uploaded '{filename}': {file_id}")
return file_id
def upload_to_public_share(
self,
@@ -308,6 +341,9 @@ class GoogleDriveService:
"""
Delete a file from Google Drive.
Retries on transient connection errors (BrokenPipeError, SSLError)
that occur when Cloud Run containers sit idle between jobs.
Args:
file_id: Google Drive file ID to delete
@@ -317,9 +353,17 @@ class GoogleDriveService:
logger.info(f"Deleting Google Drive file: {file_id}")
try:
self.service.files().delete(fileId=file_id).execute()
logger.info(f"Successfully deleted file: {file_id}")
return True
for attempt in Retrying(
stop=stop_after_attempt(3),
wait=wait_exponential(multiplier=1, min=2, max=8),
retry=retry_if_exception_type(TRANSIENT_ERRORS),
before_sleep=lambda retry_state: self._reset_service(),
reraise=True,
):
with attempt:
self.service.files().delete(fileId=file_id).execute()
logger.info(f"Successfully deleted file: {file_id}")
return True
except Exception as e:
# Check if it's a 404 (already deleted)
if hasattr(e, 'resp') and e.resp.status == 404:
+181
View File
@@ -9,6 +9,7 @@ These tests mock the Google API client and Secret Manager to verify:
"""
import json
import os
import ssl
import pytest
from unittest.mock import Mock, MagicMock, patch
@@ -501,6 +502,186 @@ class TestUploadToPublicShare:
assert len(result) == 1
class TestRetryBehavior:
"""Test retry behavior for transient connection errors."""
@patch("backend.services.gdrive_service.get_settings")
def test_retries_on_broken_pipe(self, mock_get_settings):
"""Test that BrokenPipeError triggers retry and succeeds on second attempt."""
from backend.services.gdrive_service import GoogleDriveService
mock_settings = Mock()
mock_settings.get_secret.return_value = json.dumps({
"refresh_token": "token", "client_id": "id", "client_secret": "secret",
})
mock_get_settings.return_value = mock_settings
service = GoogleDriveService()
mock_files_api = Mock()
mock_list_result = Mock()
# First call raises BrokenPipeError, second succeeds
mock_list_result.execute.side_effect = [
BrokenPipeError("[Errno 32] Broken pipe"),
{"files": [{"id": "folder-123", "name": "MP4"}]},
]
mock_files_api.list.return_value = mock_list_result
mock_drive = Mock()
mock_drive.files.return_value = mock_files_api
service._service = mock_drive
service._loaded = True
service._credentials_data = {"refresh_token": "token", "client_id": "id", "client_secret": "secret"}
# Patch _reset_service to clear but re-set the mock (avoid real OAuth)
def mock_reset():
service._service = mock_drive
service._reset_service = mock_reset
folder_id = service.get_or_create_folder("parent-123", "MP4")
assert folder_id == "folder-123"
assert mock_list_result.execute.call_count == 2
@patch("backend.services.gdrive_service.get_settings")
def test_gives_up_after_3_attempts(self, mock_get_settings):
"""Test that 3 consecutive failures raises the exception."""
from backend.services.gdrive_service import GoogleDriveService
mock_settings = Mock()
mock_settings.get_secret.return_value = json.dumps({
"refresh_token": "token", "client_id": "id", "client_secret": "secret",
})
mock_get_settings.return_value = mock_settings
service = GoogleDriveService()
mock_files_api = Mock()
mock_list_result = Mock()
mock_list_result.execute.side_effect = BrokenPipeError("[Errno 32] Broken pipe")
mock_files_api.list.return_value = mock_list_result
mock_drive = Mock()
mock_drive.files.return_value = mock_files_api
service._service = mock_drive
service._loaded = True
service._credentials_data = {"refresh_token": "token", "client_id": "id", "client_secret": "secret"}
# Patch _reset_service to re-set the mock (avoid real OAuth)
service._reset_service = lambda: setattr(service, '_service', mock_drive)
with pytest.raises(BrokenPipeError):
service.get_or_create_folder("parent-123", "MP4")
assert mock_list_result.execute.call_count == 3
@patch("backend.services.gdrive_service.get_settings")
def test_service_reset_on_retry(self, mock_get_settings):
"""Test that _service is reset to None between retry attempts."""
from backend.services.gdrive_service import GoogleDriveService
mock_settings = Mock()
mock_settings.get_secret.return_value = json.dumps({
"refresh_token": "token", "client_id": "id", "client_secret": "secret",
})
mock_get_settings.return_value = mock_settings
service = GoogleDriveService()
mock_files_api = Mock()
mock_list_result = Mock()
mock_list_result.execute.side_effect = [
BrokenPipeError("[Errno 32] Broken pipe"),
{"files": [{"id": "folder-123", "name": "MP4"}]},
]
mock_files_api.list.return_value = mock_list_result
mock_drive = Mock()
mock_drive.files.return_value = mock_files_api
service._service = mock_drive
service._loaded = True
service._credentials_data = {"refresh_token": "token", "client_id": "id", "client_secret": "secret"}
# Patch _reset_service to track calls but still do the reset
original_reset = service._reset_service
reset_calls = []
def tracked_reset():
reset_calls.append(True)
original_reset()
# Re-set mock service so the retry works
service._service = mock_drive
service._reset_service = tracked_reset
folder_id = service.get_or_create_folder("parent-123", "MP4")
assert folder_id == "folder-123"
assert len(reset_calls) == 1 # Reset called once between attempt 1 and 2
@patch("backend.services.gdrive_service.get_settings")
def test_retries_on_ssl_error(self, mock_get_settings):
"""Test that SSLError triggers retry."""
from backend.services.gdrive_service import GoogleDriveService
mock_settings = Mock()
mock_settings.get_secret.return_value = json.dumps({
"refresh_token": "token", "client_id": "id", "client_secret": "secret",
})
mock_get_settings.return_value = mock_settings
service = GoogleDriveService()
mock_files_api = Mock()
mock_list_result = Mock()
mock_list_result.execute.side_effect = [
ssl.SSLError("SSL: UNEXPECTED_EOF_WHILE_READING"),
{"files": [{"id": "folder-123", "name": "MP4"}]},
]
mock_files_api.list.return_value = mock_list_result
mock_drive = Mock()
mock_drive.files.return_value = mock_files_api
service._service = mock_drive
service._loaded = True
service._credentials_data = {"refresh_token": "token", "client_id": "id", "client_secret": "secret"}
# Patch _reset_service to re-set the mock (avoid real OAuth)
service._reset_service = lambda: setattr(service, '_service', mock_drive)
folder_id = service.get_or_create_folder("parent-123", "MP4")
assert folder_id == "folder-123"
assert mock_list_result.execute.call_count == 2
@patch("backend.services.gdrive_service.get_settings")
def test_no_retry_on_non_transient_error(self, mock_get_settings):
"""Test that non-transient errors are not retried."""
from backend.services.gdrive_service import GoogleDriveService
mock_settings = Mock()
mock_settings.get_secret.return_value = json.dumps({
"refresh_token": "token", "client_id": "id", "client_secret": "secret",
})
mock_get_settings.return_value = mock_settings
service = GoogleDriveService()
mock_files_api = Mock()
mock_list_result = Mock()
mock_list_result.execute.side_effect = ValueError("bad value")
mock_files_api.list.return_value = mock_list_result
mock_drive = Mock()
mock_drive.files.return_value = mock_files_api
service._service = mock_drive
service._loaded = True
service._credentials_data = {"refresh_token": "token", "client_id": "id", "client_secret": "secret"}
with pytest.raises(ValueError):
service.get_or_create_folder("parent-123", "MP4")
assert mock_list_result.execute.call_count == 1
class TestGetGdriveService:
"""Test get_gdrive_service singleton."""
@@ -1512,3 +1512,64 @@ class TestOrchestratorCountdownPadding:
assert captured_input is not None, "encode() should have been called"
assert captured_input.options.get("countdown_padding_seconds") == 3.0, \
"Orchestrator must pass countdown_padding_seconds in EncodingInput.options"
class TestDistributionWarnings:
"""Test distribution_warnings tracking in OrchestratorResult."""
def test_result_has_distribution_warnings_field(self):
"""Test that OrchestratorResult has distribution_warnings as empty list by default."""
result = OrchestratorResult(success=True)
assert result.distribution_warnings == []
@pytest.mark.asyncio
async def test_gdrive_failure_populates_warnings(self):
"""Test that Google Drive upload failure populates distribution_warnings."""
config = OrchestratorConfig(
job_id="test-job",
artist="Test Artist",
title="Test Title",
title_video_path="/path/title.mov",
karaoke_video_path="/path/karaoke.mov",
instrumental_audio_path="/path/audio.flac",
gdrive_folder_id="folder-123",
)
orchestrator = VideoWorkerOrchestrator(config)
orchestrator.result.brand_code = "NOMAD-0001"
with patch("backend.services.gdrive_service.get_gdrive_service") as mock_get:
mock_gdrive = MagicMock()
mock_gdrive.is_configured = True
mock_gdrive.upload_to_public_share.side_effect = Exception("Connection refused")
mock_get.return_value = mock_gdrive
await orchestrator._upload_to_gdrive()
assert len(orchestrator.result.distribution_warnings) == 1
assert "Google Drive upload failed" in orchestrator.result.distribution_warnings[0]
@pytest.mark.asyncio
async def test_dropbox_failure_populates_warnings(self):
"""Test that Dropbox upload failure populates distribution_warnings."""
config = OrchestratorConfig(
job_id="test-job",
artist="Test Artist",
title="Test Title",
title_video_path="/path/title.mov",
karaoke_video_path="/path/karaoke.mov",
instrumental_audio_path="/path/audio.flac",
dropbox_path="/Public Share",
)
orchestrator = VideoWorkerOrchestrator(config)
orchestrator.result.brand_code = "NOMAD-0001"
with patch("backend.services.dropbox_service.get_dropbox_service") as mock_get:
mock_dropbox = MagicMock()
mock_dropbox.is_configured = True
mock_dropbox.upload_folder.side_effect = Exception("Auth expired")
mock_get.return_value = mock_dropbox
await orchestrator._upload_to_dropbox()
assert len(orchestrator.result.distribution_warnings) == 1
assert "Dropbox upload failed" in orchestrator.result.distribution_warnings[0]
+77 -17
View File
@@ -339,15 +339,26 @@ async def generate_video_orchestrated(job_id: str) -> bool:
# Store result metadata in job BEFORE transitioning to COMPLETE
# This ensures youtube_url is available when completion email is sent
logger.info(f"[job:{job_id}] Video generation complete")
job_manager.update_job(job_id, {
'state_data': {
**job.state_data,
'brand_code': result.brand_code,
'youtube_url': result.youtube_url,
'dropbox_link': result.dropbox_link,
'gdrive_files': result.gdrive_files,
}
})
state_update = {
**job.state_data,
'brand_code': result.brand_code,
'youtube_url': result.youtube_url,
'dropbox_link': result.dropbox_link,
'gdrive_files': result.gdrive_files,
}
if result.distribution_warnings:
state_update['distribution_warnings'] = result.distribution_warnings
job_manager.update_job(job_id, {'state_data': state_update})
# Send Pushbullet alert if any distribution uploads failed
if result.distribution_warnings:
_send_distribution_warning_notification(
job_id=job_id,
artist=job.artist,
title=job.title,
warnings=result.distribution_warnings,
settings=settings,
)
# Mark job as complete (triggers completion email with youtube_url now available)
job_manager.transition_to_state(
@@ -841,6 +852,7 @@ async def _handle_native_distribution(
job_log.warning(f"Dropbox SDK not installed: {e}")
except Exception as e:
job_log.error(f"Native Dropbox upload failed: {e}", exc_info=True)
result.setdefault('distribution_warnings', []).append(f"Dropbox upload failed: {e}")
# Don't fail the job - distribution is optional
# Upload to Google Drive using native API
@@ -884,6 +896,7 @@ async def _handle_native_distribution(
job_log.warning(f"Google API packages not installed: {e}")
except Exception as e:
job_log.error(f"Native Google Drive upload failed: {e}", exc_info=True)
result.setdefault('distribution_warnings', []).append(f"Google Drive upload failed: {e}")
# Don't fail the job - distribution is optional
elif existing_gdrive_files:
job_log.info(f"Skipping Google Drive upload - orchestrator already uploaded {len(existing_gdrive_files)} files")
@@ -891,17 +904,64 @@ async def _handle_native_distribution(
# Update job state_data with brand code and links
if brand_code or result.get('dropbox_link') or result.get('gdrive_files'):
try:
job_manager.update_job(job_id, {
'state_data': {
**job.state_data,
'brand_code': brand_code,
'dropbox_link': result.get('dropbox_link'),
'gdrive_files': result.get('gdrive_files'),
}
})
state_update = {
**job.state_data,
'brand_code': brand_code,
'dropbox_link': result.get('dropbox_link'),
'gdrive_files': result.get('gdrive_files'),
}
if result.get('distribution_warnings'):
state_update['distribution_warnings'] = result['distribution_warnings']
job_manager.update_job(job_id, {'state_data': state_update})
except Exception as e:
job_log.warning(f"Failed to update job state_data: {e}")
# Send Pushbullet alert if any distribution uploads failed
if result.get('distribution_warnings'):
_send_distribution_warning_notification(
job_id=job_id,
artist=job.artist,
title=job.title,
warnings=result['distribution_warnings'],
settings=settings,
)
def _send_distribution_warning_notification(
job_id: str,
artist: str,
title: str,
warnings: list[str],
settings=None,
) -> None:
"""Send Pushbullet notification when distribution uploads fail."""
try:
if settings is None:
settings = get_settings()
api_key = settings.get_secret("pushbullet-api-key")
if not api_key:
logger.warning("Pushbullet API key not found, skipping distribution warning notification")
return
import requests as req
push_title = f"Distribution Warning: {artist} - {title}"
body = f"Job {job_id} completed but {len(warnings)} upload(s) failed:\n\n"
body += "\n".join(f" - {w}" for w in warnings)
resp = req.post(
"https://api.pushbullet.com/v2/pushes",
headers={"Access-Token": api_key, "Content-Type": "application/json"},
json={"type": "note", "title": push_title, "body": body},
timeout=10,
)
if resp.ok:
logger.info(f"[job:{job_id}] Distribution warning notification sent via Pushbullet")
else:
logger.warning(f"[job:{job_id}] Pushbullet notification failed: {resp.status_code} {resp.text}")
except Exception as e:
logger.warning(f"[job:{job_id}] Failed to send Pushbullet notification: {e}")
def _validate_prerequisites(job) -> bool:
"""Validate that all prerequisites are met for video generation."""
@@ -109,6 +109,9 @@ class OrchestratorResult:
dropbox_link: Optional[str] = None
gdrive_files: Optional[Dict[str, str]] = field(default_factory=dict)
# Distribution warnings (non-fatal upload failures)
distribution_warnings: List[str] = field(default_factory=list)
# Timing
encoding_time_seconds: Optional[float] = None
total_time_seconds: Optional[float] = None
@@ -603,6 +606,7 @@ class VideoWorkerOrchestrator:
except Exception as e:
self.job_log.error(f"YouTube upload failed: {e}")
self.result.distribution_warnings.append(f"YouTube upload failed: {e}")
# Don't fail the pipeline - YouTube is optional
async def _upload_to_dropbox(self):
@@ -639,6 +643,7 @@ class VideoWorkerOrchestrator:
except Exception as e:
self.job_log.error(f"Dropbox upload failed: {e}")
self.result.distribution_warnings.append(f"Dropbox upload failed: {e}")
# Don't fail the pipeline - Dropbox is optional
async def _upload_to_gdrive(self):
@@ -675,6 +680,7 @@ class VideoWorkerOrchestrator:
except Exception as e:
self.job_log.error(f"Google Drive upload failed: {e}")
self.result.distribution_warnings.append(f"Google Drive upload failed: {e}")
# Don't fail the pipeline - GDrive is optional
async def _run_notifications(self):
+12
View File
@@ -222,6 +222,18 @@ When deferring processing from one worker to another, you MUST update the job st
### Google Drive Query Escaping
Escape special chars in Google Drive API queries: `'` → `\'`, `\` → `\\\\`.
### Google Drive Stale Connection on Cloud Run (Feb 2026)
**What happened**: Every GDrive upload silently failed for ~1 month (64 errors, 12+ jobs). Errors: `[Errno 32] Broken pipe` or `[SSL: UNEXPECTED_EOF_WHILE_READING]`. Jobs completed with `gdrive_files: {}` and no alert was sent.
**Root cause**: The singleton `GoogleDriveService` caches its HTTP connection. When Cloud Run containers sit idle between jobs, the underlying TCP connection goes stale. The next API call hits a dead socket. The orchestrator catches all exceptions in distribution and continues silently.
**Fix** (v0.119.7):
1. Added tenacity retry with `_reset_service()` (sets `self._service = None` to force fresh connection) on transient errors (`BrokenPipeError`, `ConnectionResetError`, `ssl.SSLError`)
2. Added `distribution_warnings` field to `OrchestratorResult` to track non-fatal upload failures
3. Added Pushbullet notification when distribution uploads fail, so operator is alerted immediately
**Pattern**: Any singleton service that caches HTTP connections should have retry logic with connection reset for stale connection errors. Cloud Run containers can idle for minutes between requests, killing keep-alive connections. Also, never silently swallow distribution failures - always track and alert on them.
### FFmpeg Path Escaping
For subprocess without shell, FFmpeg filter paths need: apostrophes escaped as `'\\''`, special chars as `\\[char]`.
+24
View File
@@ -68,3 +68,27 @@ gcloud compute ssh encoding-worker --zone=us-central1-c --project=nomadkaraoke \
gcloud compute ssh encoding-worker --zone=us-central1-c --project=nomadkaraoke \
--command="sudo systemctl restart encoding-worker"
```
---
## Google Drive uploads missing (gdrive_files empty)
**Symptoms:** Jobs complete successfully but `state_data.gdrive_files` is empty `{}`. Cloud Run logs show `BrokenPipeError` or `SSL: UNEXPECTED_EOF_WHILE_READING`.
**Cause:** Stale HTTP connections in the singleton `GoogleDriveService`. Since v0.119.7, this is handled automatically with retry + connection reset.
**Backfill affected jobs:**
```bash
# Dry run - list affected jobs
GCS_BUCKET_NAME=karaoke-gen-storage-nomadkaraoke \
GOOGLE_CLOUD_PROJECT=nomadkaraoke \
python scripts/backfill_gdrive_uploads.py --all-missing --dry-run
# Run for real
GCS_BUCKET_NAME=karaoke-gen-storage-nomadkaraoke \
GOOGLE_CLOUD_PROJECT=nomadkaraoke \
python scripts/backfill_gdrive_uploads.py --all-missing
# Specific jobs
python scripts/backfill_gdrive_uploads.py --job-ids JOB1,JOB2
```
+1 -1
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "karaoke-gen"
version = "0.119.7"
version = "0.119.8"
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
license = "MIT"
+303
View File
@@ -0,0 +1,303 @@
#!/usr/bin/env python3
"""
Backfill Google Drive uploads for jobs that completed with empty gdrive_files.
This addresses the issue where GDrive uploads silently failed due to stale
HTTP connections (BrokenPipeError / SSLError) on idle Cloud Run containers.
Usage:
# Dry run - show what would be uploaded
python scripts/backfill_gdrive_uploads.py --dry-run --job-ids ca408e46,99baa198
# Upload specific jobs
python scripts/backfill_gdrive_uploads.py --job-ids ca408e46,99baa198
# Find and upload all jobs with empty gdrive_files
python scripts/backfill_gdrive_uploads.py --all-missing
# Override gdrive folder ID (instead of using DEFAULT_GDRIVE_FOLDER_ID env var)
python scripts/backfill_gdrive_uploads.py --job-ids ca408e46 --gdrive-folder-id 1abc...
Requires:
- google-cloud-firestore
- google-cloud-storage
- google-api-python-client, google-auth
- GOOGLE_APPLICATION_CREDENTIALS or gcloud auth application-default login
"""
import argparse
import logging
import os
import sys
import tempfile
# Add project root to path so we can import backend modules
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from google.cloud import firestore, storage
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(message)s",
)
logger = logging.getLogger(__name__)
GCS_BUCKET = "karaoke-gen-storage-nomadkaraoke"
PROJECT = "nomadkaraoke"
# Default GDrive public share folder (from infrastructure/config.py)
DEFAULT_GDRIVE_FOLDER_ID = "1laRKAyxo0v817SstfM5XkpbWiNKNAMSX"
# Known affected jobs from the incident
KNOWN_AFFECTED_JOBS = [
"ca408e46", "99baa198", "1e15d1dd", "ebba0996",
"cb744c24", "1389f765", "fde3ae7d", "21f4b34f",
"443c34a1", "56192d3a", "fdb4dc63", "4eeaa729",
]
def get_job(db: firestore.Client, job_id: str) -> dict | None:
"""Fetch job document from Firestore."""
doc = db.collection("jobs").document(job_id).get()
if doc.exists:
return doc.to_dict()
return None
def find_jobs_with_missing_gdrive(db: firestore.Client) -> list[dict]:
"""Find completed jobs that have empty gdrive_files in state_data.
Looks for jobs that have a brand_code (indicating they went through
distribution) but have empty or missing gdrive_files.
"""
jobs = []
query = db.collection("jobs").where("status", "==", "complete")
for doc in query.stream():
data = doc.to_dict()
state_data = data.get("state_data", {})
gdrive_files = state_data.get("gdrive_files", {})
brand_code = state_data.get("brand_code")
# Job should have had GDrive uploads if it has a brand_code
# but gdrive_files is empty/missing
if brand_code and not gdrive_files:
jobs.append({"job_id": doc.id, **data})
return jobs
def _classify_blob(filename: str) -> str | None:
"""Classify a GCS blob filename to an output_files key, or None to skip.
Only returns keys for the 3 file types that get uploaded to GDrive:
lossy 4K MP4, lossy 720p MP4, and CDG ZIP.
"""
lower = filename.lower()
if lower.endswith(".zip") and "cdg" in lower:
return "final_karaoke_cdg_zip"
if "720p" in lower and lower.endswith(".mp4"):
return "final_karaoke_lossy_720p_mp4"
if "lossy" in lower and lower.endswith(".mp4") and "720p" not in lower:
return "final_karaoke_lossy_mp4"
return None
def download_finals(gcs_client: storage.Client, job_id: str, temp_dir: str) -> dict:
"""Download only the final output files needed for GDrive upload.
Only downloads lossy 4K MP4, lossy 720p MP4, and CDG ZIP — skips
lossless files, MKVs, karaoke video, etc. to save bandwidth.
Returns dict mapping output_files keys to local paths.
"""
bucket = gcs_client.bucket(GCS_BUCKET)
prefix = f"jobs/{job_id}/finals/"
blobs = list(bucket.list_blobs(prefix=prefix))
output_files = {}
for blob in blobs:
filename = os.path.basename(blob.name)
if not filename:
continue
key = _classify_blob(filename)
if key is None:
continue
if key in output_files:
continue # already have this type
local_path = os.path.join(temp_dir, filename)
logger.info(f" Downloading {filename} ({blob.size / 1024 / 1024:.1f} MB)")
blob.download_to_filename(local_path)
output_files[key] = local_path
return output_files
def list_finals(gcs_client: storage.Client, job_id: str) -> list[str]:
"""List uploadable final output files in GCS without downloading."""
bucket = gcs_client.bucket(GCS_BUCKET)
prefix = f"jobs/{job_id}/finals/"
blobs = list(bucket.list_blobs(prefix=prefix))
return [
f"{os.path.basename(b.name)} ({b.size / 1024 / 1024:.1f} MB)"
for b in blobs
if os.path.basename(b.name) and _classify_blob(os.path.basename(b.name))
]
def backfill_job(
db: firestore.Client,
gcs_client: storage.Client,
job_id: str,
gdrive_folder_id: str,
dry_run: bool = False,
) -> bool:
"""Backfill GDrive upload for a single job.
Returns True if successful (or dry run).
"""
logger.info(f"Processing job {job_id}")
# Fetch job data
job_data = get_job(db, job_id)
if not job_data:
logger.error(f" Job {job_id} not found in Firestore")
return False
state_data = job_data.get("state_data", {})
artist = job_data.get("artist", "Unknown")
title = job_data.get("title", "Unknown")
brand_code = state_data.get("brand_code")
status = job_data.get("status")
logger.info(f" {artist} - {title} (brand: {brand_code}, status: {status})")
# Check existing gdrive_files
existing = state_data.get("gdrive_files", {})
if existing:
logger.info(f" Already has gdrive_files: {existing}")
logger.info(f" Skipping (already uploaded)")
return True
if not brand_code:
logger.warning(f" No brand code, skipping")
return False
if dry_run:
# List what files are available in GCS
files = list_finals(gcs_client, job_id)
if files:
logger.info(f" [DRY RUN] Would upload {len(files)} files to GDrive folder {gdrive_folder_id}:")
for f in files:
logger.info(f" - {f}")
else:
logger.warning(f" [DRY RUN] No final files found in GCS")
return True
# Download files and upload
with tempfile.TemporaryDirectory(prefix=f"backfill-{job_id}-") as temp_dir:
# Download finals from GCS
logger.info(f" Downloading finals from GCS...")
output_files = download_finals(gcs_client, job_id, temp_dir)
if not output_files:
logger.warning(f" No final files found in GCS")
return False
logger.info(f" Found {len(output_files)} files to upload")
# Upload to GDrive
from backend.services.gdrive_service import GoogleDriveService
gdrive = GoogleDriveService()
if not gdrive.is_configured:
logger.error(f" Google Drive not configured")
return False
base_name = f"{artist} - {title}"
uploaded = gdrive.upload_to_public_share(
root_folder_id=gdrive_folder_id,
brand_code=brand_code,
base_name=base_name,
output_files=output_files,
)
logger.info(f" Uploaded {len(uploaded)} files: {uploaded}")
# Update Firestore
db.collection("jobs").document(job_id).update({
"state_data.gdrive_files": uploaded,
})
logger.info(f" Updated Firestore state_data.gdrive_files")
return True
def main():
parser = argparse.ArgumentParser(
description="Backfill Google Drive uploads for jobs with empty gdrive_files"
)
parser.add_argument(
"--job-ids",
help="Comma-separated list of job IDs to process",
)
parser.add_argument(
"--all-missing",
action="store_true",
help="Find and process all jobs with missing gdrive_files",
)
parser.add_argument(
"--dry-run",
action="store_true",
help="Show what would be done without making changes",
)
parser.add_argument(
"--gdrive-folder-id",
default=os.getenv("DEFAULT_GDRIVE_FOLDER_ID", DEFAULT_GDRIVE_FOLDER_ID),
help=f"Google Drive folder ID for public share (default: {DEFAULT_GDRIVE_FOLDER_ID})",
)
args = parser.parse_args()
if not args.job_ids and not args.all_missing:
parser.error("Specify --job-ids or --all-missing")
db = firestore.Client(project=PROJECT)
gcs_client = storage.Client(project=PROJECT)
if args.job_ids:
job_ids = [j.strip() for j in args.job_ids.split(",")]
elif args.all_missing:
logger.info("Searching for jobs with missing GDrive uploads...")
jobs = find_jobs_with_missing_gdrive(db)
job_ids = [j["job_id"] for j in jobs]
logger.info(f"Found {len(job_ids)} jobs with missing uploads")
if args.dry_run:
logger.info("[DRY RUN MODE]")
logger.info(f"GDrive folder: {args.gdrive_folder_id}")
success_count = 0
fail_count = 0
for job_id in job_ids:
try:
ok = backfill_job(
db, gcs_client, job_id,
gdrive_folder_id=args.gdrive_folder_id,
dry_run=args.dry_run,
)
if ok:
success_count += 1
else:
fail_count += 1
except Exception as e:
logger.error(f" Failed: {e}", exc_info=True)
fail_count += 1
logger.info(f"\nDone: {success_count} succeeded, {fail_count} failed out of {len(job_ids)} jobs")
if __name__ == "__main__":
main()