diff --git a/backend/services/gdrive_service.py b/backend/services/gdrive_service.py index 91fafa60..69352daf 100644 --- a/backend/services/gdrive_service.py +++ b/backend/services/gdrive_service.py @@ -12,13 +12,19 @@ shared with YouTube credentials if scopes include drive.file. import json import logging import os +import ssl from typing import Any, Dict, Optional +from tenacity import Retrying, stop_after_attempt, wait_exponential, retry_if_exception_type + from backend.config import get_settings from karaoke_gen.utils import sanitize_filename logger = logging.getLogger(__name__) +# Transient errors that indicate a stale HTTP connection (Cloud Run idle containers) +TRANSIENT_ERRORS = (BrokenPipeError, ConnectionResetError, ConnectionError, ssl.SSLError) + class GoogleDriveService: """Google Drive operations using native API.""" @@ -33,6 +39,11 @@ class GoogleDriveService: self._credentials_data: Optional[Dict[str, Any]] = None self._loaded = False + def _reset_service(self): + """Reset the cached Drive service to force a fresh connection on next use.""" + logger.info("Resetting Google Drive service connection") + self._service = None + def _load_credentials(self) -> Optional[Dict[str, Any]]: """Load OAuth credentials from Secret Manager.""" if self._loaded: @@ -127,6 +138,9 @@ class GoogleDriveService: """ Get existing folder or create new one, return folder ID. + Retries on transient connection errors (BrokenPipeError, SSLError) + that occur when Cloud Run containers sit idle between jobs. + Args: parent_id: Parent folder ID folder_name: Name of folder to find or create @@ -134,33 +148,41 @@ class GoogleDriveService: Returns: Folder ID """ - logger.info(f"Looking for folder '{folder_name}' in parent {parent_id}") + for attempt in Retrying( + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=1, min=2, max=8), + retry=retry_if_exception_type(TRANSIENT_ERRORS), + before_sleep=lambda retry_state: self._reset_service(), + reraise=True, + ): + with attempt: + logger.info(f"Looking for folder '{folder_name}' in parent {parent_id}") - # Search for existing folder - # Escape single quotes in folder name for Google Drive API query syntax - escaped_folder_name = folder_name.replace("'", "\\'") - query = ( - f"name='{escaped_folder_name}' and '{parent_id}' in parents " - f"and mimeType='application/vnd.google-apps.folder' and trashed=false" - ) - results = self.service.files().list(q=query, fields="files(id, name)").execute() + # Search for existing folder + # Escape single quotes in folder name for Google Drive API query syntax + escaped_folder_name = folder_name.replace("'", "\\'") + query = ( + f"name='{escaped_folder_name}' and '{parent_id}' in parents " + f"and mimeType='application/vnd.google-apps.folder' and trashed=false" + ) + results = self.service.files().list(q=query, fields="files(id, name)").execute() - if results.get("files"): - folder_id = results["files"][0]["id"] - logger.info(f"Found existing folder '{folder_name}': {folder_id}") - return folder_id + if results.get("files"): + folder_id = results["files"][0]["id"] + logger.info(f"Found existing folder '{folder_name}': {folder_id}") + return folder_id - # Create folder - logger.info(f"Creating new folder '{folder_name}'") - metadata = { - "name": folder_name, - "mimeType": "application/vnd.google-apps.folder", - "parents": [parent_id], - } - folder = self.service.files().create(body=metadata, fields="id").execute() - folder_id = folder["id"] - logger.info(f"Created folder '{folder_name}': {folder_id}") - return folder_id + # Create folder + logger.info(f"Creating new folder '{folder_name}'") + metadata = { + "name": folder_name, + "mimeType": "application/vnd.google-apps.folder", + "parents": [parent_id], + } + folder = self.service.files().create(body=metadata, fields="id").execute() + folder_id = folder["id"] + logger.info(f"Created folder '{folder_name}': {folder_id}") + return folder_id def upload_file( self, @@ -172,6 +194,9 @@ class GoogleDriveService: """ Upload a file to a specific Drive folder. + Retries on transient connection errors (BrokenPipeError, SSLError) + that occur when Cloud Run containers sit idle between jobs. + Args: local_path: Local file path parent_id: Parent folder ID in Google Drive @@ -183,49 +208,57 @@ class GoogleDriveService: """ from googleapiclient.http import MediaFileUpload - file_size = os.path.getsize(local_path) - logger.info( - f"Uploading {local_path} ({file_size / 1024 / 1024:.1f} MB) " - f"as '{filename}' to folder {parent_id}" - ) + for attempt in Retrying( + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=1, min=2, max=8), + retry=retry_if_exception_type(TRANSIENT_ERRORS), + before_sleep=lambda retry_state: self._reset_service(), + reraise=True, + ): + with attempt: + file_size = os.path.getsize(local_path) + logger.info( + f"Uploading {local_path} ({file_size / 1024 / 1024:.1f} MB) " + f"as '{filename}' to folder {parent_id}" + ) - # Determine MIME type - ext = os.path.splitext(local_path)[1].lower() - mime_types = { - ".mp4": "video/mp4", - ".mkv": "video/x-matroska", - ".zip": "application/zip", - ".flac": "audio/flac", - ".mp3": "audio/mpeg", - ".wav": "audio/wav", - ".png": "image/png", - ".jpg": "image/jpeg", - ".jpeg": "image/jpeg", - } - mime_type = mime_types.get(ext, "application/octet-stream") + # Determine MIME type + ext = os.path.splitext(local_path)[1].lower() + mime_types = { + ".mp4": "video/mp4", + ".mkv": "video/x-matroska", + ".zip": "application/zip", + ".flac": "audio/flac", + ".mp3": "audio/mpeg", + ".wav": "audio/wav", + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + } + mime_type = mime_types.get(ext, "application/octet-stream") - # Check for existing file with same name - if replace_existing: - # Escape single quotes in filename for Google Drive API query syntax - escaped_filename = filename.replace("'", "\\'") - query = ( - f"name='{escaped_filename}' and '{parent_id}' in parents and trashed=false" - ) - results = self.service.files().list(q=query, fields="files(id)").execute() - for existing_file in results.get("files", []): - logger.info(f"Deleting existing file: {existing_file['id']}") - self.service.files().delete(fileId=existing_file["id"]).execute() + # Check for existing file with same name + if replace_existing: + # Escape single quotes in filename for Google Drive API query syntax + escaped_filename = filename.replace("'", "\\'") + query = ( + f"name='{escaped_filename}' and '{parent_id}' in parents and trashed=false" + ) + results = self.service.files().list(q=query, fields="files(id)").execute() + for existing_file in results.get("files", []): + logger.info(f"Deleting existing file: {existing_file['id']}") + self.service.files().delete(fileId=existing_file["id"]).execute() - # Upload file - metadata = {"name": filename, "parents": [parent_id]} - media = MediaFileUpload(local_path, mimetype=mime_type, resumable=True) + # Upload file + metadata = {"name": filename, "parents": [parent_id]} + media = MediaFileUpload(local_path, mimetype=mime_type, resumable=True) - file_result = ( - self.service.files().create(body=metadata, media_body=media, fields="id").execute() - ) - file_id = file_result["id"] - logger.info(f"Successfully uploaded '{filename}': {file_id}") - return file_id + file_result = ( + self.service.files().create(body=metadata, media_body=media, fields="id").execute() + ) + file_id = file_result["id"] + logger.info(f"Successfully uploaded '{filename}': {file_id}") + return file_id def upload_to_public_share( self, @@ -308,6 +341,9 @@ class GoogleDriveService: """ Delete a file from Google Drive. + Retries on transient connection errors (BrokenPipeError, SSLError) + that occur when Cloud Run containers sit idle between jobs. + Args: file_id: Google Drive file ID to delete @@ -317,9 +353,17 @@ class GoogleDriveService: logger.info(f"Deleting Google Drive file: {file_id}") try: - self.service.files().delete(fileId=file_id).execute() - logger.info(f"Successfully deleted file: {file_id}") - return True + for attempt in Retrying( + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=1, min=2, max=8), + retry=retry_if_exception_type(TRANSIENT_ERRORS), + before_sleep=lambda retry_state: self._reset_service(), + reraise=True, + ): + with attempt: + self.service.files().delete(fileId=file_id).execute() + logger.info(f"Successfully deleted file: {file_id}") + return True except Exception as e: # Check if it's a 404 (already deleted) if hasattr(e, 'resp') and e.resp.status == 404: diff --git a/backend/tests/test_gdrive_service.py b/backend/tests/test_gdrive_service.py index efb4f935..f76439fd 100644 --- a/backend/tests/test_gdrive_service.py +++ b/backend/tests/test_gdrive_service.py @@ -9,6 +9,7 @@ These tests mock the Google API client and Secret Manager to verify: """ import json import os +import ssl import pytest from unittest.mock import Mock, MagicMock, patch @@ -501,6 +502,186 @@ class TestUploadToPublicShare: assert len(result) == 1 +class TestRetryBehavior: + """Test retry behavior for transient connection errors.""" + + @patch("backend.services.gdrive_service.get_settings") + def test_retries_on_broken_pipe(self, mock_get_settings): + """Test that BrokenPipeError triggers retry and succeeds on second attempt.""" + from backend.services.gdrive_service import GoogleDriveService + + mock_settings = Mock() + mock_settings.get_secret.return_value = json.dumps({ + "refresh_token": "token", "client_id": "id", "client_secret": "secret", + }) + mock_get_settings.return_value = mock_settings + + service = GoogleDriveService() + + mock_files_api = Mock() + mock_list_result = Mock() + # First call raises BrokenPipeError, second succeeds + mock_list_result.execute.side_effect = [ + BrokenPipeError("[Errno 32] Broken pipe"), + {"files": [{"id": "folder-123", "name": "MP4"}]}, + ] + mock_files_api.list.return_value = mock_list_result + + mock_drive = Mock() + mock_drive.files.return_value = mock_files_api + service._service = mock_drive + service._loaded = True + service._credentials_data = {"refresh_token": "token", "client_id": "id", "client_secret": "secret"} + + # Patch _reset_service to clear but re-set the mock (avoid real OAuth) + def mock_reset(): + service._service = mock_drive + + service._reset_service = mock_reset + + folder_id = service.get_or_create_folder("parent-123", "MP4") + assert folder_id == "folder-123" + assert mock_list_result.execute.call_count == 2 + + @patch("backend.services.gdrive_service.get_settings") + def test_gives_up_after_3_attempts(self, mock_get_settings): + """Test that 3 consecutive failures raises the exception.""" + from backend.services.gdrive_service import GoogleDriveService + + mock_settings = Mock() + mock_settings.get_secret.return_value = json.dumps({ + "refresh_token": "token", "client_id": "id", "client_secret": "secret", + }) + mock_get_settings.return_value = mock_settings + + service = GoogleDriveService() + + mock_files_api = Mock() + mock_list_result = Mock() + mock_list_result.execute.side_effect = BrokenPipeError("[Errno 32] Broken pipe") + mock_files_api.list.return_value = mock_list_result + + mock_drive = Mock() + mock_drive.files.return_value = mock_files_api + service._service = mock_drive + service._loaded = True + service._credentials_data = {"refresh_token": "token", "client_id": "id", "client_secret": "secret"} + + # Patch _reset_service to re-set the mock (avoid real OAuth) + service._reset_service = lambda: setattr(service, '_service', mock_drive) + + with pytest.raises(BrokenPipeError): + service.get_or_create_folder("parent-123", "MP4") + + assert mock_list_result.execute.call_count == 3 + + @patch("backend.services.gdrive_service.get_settings") + def test_service_reset_on_retry(self, mock_get_settings): + """Test that _service is reset to None between retry attempts.""" + from backend.services.gdrive_service import GoogleDriveService + + mock_settings = Mock() + mock_settings.get_secret.return_value = json.dumps({ + "refresh_token": "token", "client_id": "id", "client_secret": "secret", + }) + mock_get_settings.return_value = mock_settings + + service = GoogleDriveService() + + mock_files_api = Mock() + mock_list_result = Mock() + mock_list_result.execute.side_effect = [ + BrokenPipeError("[Errno 32] Broken pipe"), + {"files": [{"id": "folder-123", "name": "MP4"}]}, + ] + mock_files_api.list.return_value = mock_list_result + + mock_drive = Mock() + mock_drive.files.return_value = mock_files_api + service._service = mock_drive + service._loaded = True + service._credentials_data = {"refresh_token": "token", "client_id": "id", "client_secret": "secret"} + + # Patch _reset_service to track calls but still do the reset + original_reset = service._reset_service + reset_calls = [] + + def tracked_reset(): + reset_calls.append(True) + original_reset() + # Re-set mock service so the retry works + service._service = mock_drive + + service._reset_service = tracked_reset + + folder_id = service.get_or_create_folder("parent-123", "MP4") + assert folder_id == "folder-123" + assert len(reset_calls) == 1 # Reset called once between attempt 1 and 2 + + @patch("backend.services.gdrive_service.get_settings") + def test_retries_on_ssl_error(self, mock_get_settings): + """Test that SSLError triggers retry.""" + from backend.services.gdrive_service import GoogleDriveService + + mock_settings = Mock() + mock_settings.get_secret.return_value = json.dumps({ + "refresh_token": "token", "client_id": "id", "client_secret": "secret", + }) + mock_get_settings.return_value = mock_settings + + service = GoogleDriveService() + + mock_files_api = Mock() + mock_list_result = Mock() + mock_list_result.execute.side_effect = [ + ssl.SSLError("SSL: UNEXPECTED_EOF_WHILE_READING"), + {"files": [{"id": "folder-123", "name": "MP4"}]}, + ] + mock_files_api.list.return_value = mock_list_result + + mock_drive = Mock() + mock_drive.files.return_value = mock_files_api + service._service = mock_drive + service._loaded = True + service._credentials_data = {"refresh_token": "token", "client_id": "id", "client_secret": "secret"} + + # Patch _reset_service to re-set the mock (avoid real OAuth) + service._reset_service = lambda: setattr(service, '_service', mock_drive) + + folder_id = service.get_or_create_folder("parent-123", "MP4") + assert folder_id == "folder-123" + assert mock_list_result.execute.call_count == 2 + + @patch("backend.services.gdrive_service.get_settings") + def test_no_retry_on_non_transient_error(self, mock_get_settings): + """Test that non-transient errors are not retried.""" + from backend.services.gdrive_service import GoogleDriveService + + mock_settings = Mock() + mock_settings.get_secret.return_value = json.dumps({ + "refresh_token": "token", "client_id": "id", "client_secret": "secret", + }) + mock_get_settings.return_value = mock_settings + + service = GoogleDriveService() + + mock_files_api = Mock() + mock_list_result = Mock() + mock_list_result.execute.side_effect = ValueError("bad value") + mock_files_api.list.return_value = mock_list_result + + mock_drive = Mock() + mock_drive.files.return_value = mock_files_api + service._service = mock_drive + service._loaded = True + service._credentials_data = {"refresh_token": "token", "client_id": "id", "client_secret": "secret"} + + with pytest.raises(ValueError): + service.get_or_create_folder("parent-123", "MP4") + + assert mock_list_result.execute.call_count == 1 + + class TestGetGdriveService: """Test get_gdrive_service singleton.""" diff --git a/backend/tests/test_video_worker_orchestrator.py b/backend/tests/test_video_worker_orchestrator.py index 7acd6365..8e260b67 100644 --- a/backend/tests/test_video_worker_orchestrator.py +++ b/backend/tests/test_video_worker_orchestrator.py @@ -1512,3 +1512,64 @@ class TestOrchestratorCountdownPadding: assert captured_input is not None, "encode() should have been called" assert captured_input.options.get("countdown_padding_seconds") == 3.0, \ "Orchestrator must pass countdown_padding_seconds in EncodingInput.options" + + +class TestDistributionWarnings: + """Test distribution_warnings tracking in OrchestratorResult.""" + + def test_result_has_distribution_warnings_field(self): + """Test that OrchestratorResult has distribution_warnings as empty list by default.""" + result = OrchestratorResult(success=True) + assert result.distribution_warnings == [] + + @pytest.mark.asyncio + async def test_gdrive_failure_populates_warnings(self): + """Test that Google Drive upload failure populates distribution_warnings.""" + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + gdrive_folder_id="folder-123", + ) + orchestrator = VideoWorkerOrchestrator(config) + orchestrator.result.brand_code = "NOMAD-0001" + + with patch("backend.services.gdrive_service.get_gdrive_service") as mock_get: + mock_gdrive = MagicMock() + mock_gdrive.is_configured = True + mock_gdrive.upload_to_public_share.side_effect = Exception("Connection refused") + mock_get.return_value = mock_gdrive + + await orchestrator._upload_to_gdrive() + + assert len(orchestrator.result.distribution_warnings) == 1 + assert "Google Drive upload failed" in orchestrator.result.distribution_warnings[0] + + @pytest.mark.asyncio + async def test_dropbox_failure_populates_warnings(self): + """Test that Dropbox upload failure populates distribution_warnings.""" + config = OrchestratorConfig( + job_id="test-job", + artist="Test Artist", + title="Test Title", + title_video_path="/path/title.mov", + karaoke_video_path="/path/karaoke.mov", + instrumental_audio_path="/path/audio.flac", + dropbox_path="/Public Share", + ) + orchestrator = VideoWorkerOrchestrator(config) + orchestrator.result.brand_code = "NOMAD-0001" + + with patch("backend.services.dropbox_service.get_dropbox_service") as mock_get: + mock_dropbox = MagicMock() + mock_dropbox.is_configured = True + mock_dropbox.upload_folder.side_effect = Exception("Auth expired") + mock_get.return_value = mock_dropbox + + await orchestrator._upload_to_dropbox() + + assert len(orchestrator.result.distribution_warnings) == 1 + assert "Dropbox upload failed" in orchestrator.result.distribution_warnings[0] diff --git a/backend/workers/video_worker.py b/backend/workers/video_worker.py index 2e6f0476..d6de016f 100644 --- a/backend/workers/video_worker.py +++ b/backend/workers/video_worker.py @@ -339,15 +339,26 @@ async def generate_video_orchestrated(job_id: str) -> bool: # Store result metadata in job BEFORE transitioning to COMPLETE # This ensures youtube_url is available when completion email is sent logger.info(f"[job:{job_id}] Video generation complete") - job_manager.update_job(job_id, { - 'state_data': { - **job.state_data, - 'brand_code': result.brand_code, - 'youtube_url': result.youtube_url, - 'dropbox_link': result.dropbox_link, - 'gdrive_files': result.gdrive_files, - } - }) + state_update = { + **job.state_data, + 'brand_code': result.brand_code, + 'youtube_url': result.youtube_url, + 'dropbox_link': result.dropbox_link, + 'gdrive_files': result.gdrive_files, + } + if result.distribution_warnings: + state_update['distribution_warnings'] = result.distribution_warnings + job_manager.update_job(job_id, {'state_data': state_update}) + + # Send Pushbullet alert if any distribution uploads failed + if result.distribution_warnings: + _send_distribution_warning_notification( + job_id=job_id, + artist=job.artist, + title=job.title, + warnings=result.distribution_warnings, + settings=settings, + ) # Mark job as complete (triggers completion email with youtube_url now available) job_manager.transition_to_state( @@ -841,6 +852,7 @@ async def _handle_native_distribution( job_log.warning(f"Dropbox SDK not installed: {e}") except Exception as e: job_log.error(f"Native Dropbox upload failed: {e}", exc_info=True) + result.setdefault('distribution_warnings', []).append(f"Dropbox upload failed: {e}") # Don't fail the job - distribution is optional # Upload to Google Drive using native API @@ -884,6 +896,7 @@ async def _handle_native_distribution( job_log.warning(f"Google API packages not installed: {e}") except Exception as e: job_log.error(f"Native Google Drive upload failed: {e}", exc_info=True) + result.setdefault('distribution_warnings', []).append(f"Google Drive upload failed: {e}") # Don't fail the job - distribution is optional elif existing_gdrive_files: job_log.info(f"Skipping Google Drive upload - orchestrator already uploaded {len(existing_gdrive_files)} files") @@ -891,17 +904,64 @@ async def _handle_native_distribution( # Update job state_data with brand code and links if brand_code or result.get('dropbox_link') or result.get('gdrive_files'): try: - job_manager.update_job(job_id, { - 'state_data': { - **job.state_data, - 'brand_code': brand_code, - 'dropbox_link': result.get('dropbox_link'), - 'gdrive_files': result.get('gdrive_files'), - } - }) + state_update = { + **job.state_data, + 'brand_code': brand_code, + 'dropbox_link': result.get('dropbox_link'), + 'gdrive_files': result.get('gdrive_files'), + } + if result.get('distribution_warnings'): + state_update['distribution_warnings'] = result['distribution_warnings'] + job_manager.update_job(job_id, {'state_data': state_update}) except Exception as e: job_log.warning(f"Failed to update job state_data: {e}") + # Send Pushbullet alert if any distribution uploads failed + if result.get('distribution_warnings'): + _send_distribution_warning_notification( + job_id=job_id, + artist=job.artist, + title=job.title, + warnings=result['distribution_warnings'], + settings=settings, + ) + + +def _send_distribution_warning_notification( + job_id: str, + artist: str, + title: str, + warnings: list[str], + settings=None, +) -> None: + """Send Pushbullet notification when distribution uploads fail.""" + try: + if settings is None: + settings = get_settings() + api_key = settings.get_secret("pushbullet-api-key") + if not api_key: + logger.warning("Pushbullet API key not found, skipping distribution warning notification") + return + + import requests as req + + push_title = f"Distribution Warning: {artist} - {title}" + body = f"Job {job_id} completed but {len(warnings)} upload(s) failed:\n\n" + body += "\n".join(f" - {w}" for w in warnings) + + resp = req.post( + "https://api.pushbullet.com/v2/pushes", + headers={"Access-Token": api_key, "Content-Type": "application/json"}, + json={"type": "note", "title": push_title, "body": body}, + timeout=10, + ) + if resp.ok: + logger.info(f"[job:{job_id}] Distribution warning notification sent via Pushbullet") + else: + logger.warning(f"[job:{job_id}] Pushbullet notification failed: {resp.status_code} {resp.text}") + except Exception as e: + logger.warning(f"[job:{job_id}] Failed to send Pushbullet notification: {e}") + def _validate_prerequisites(job) -> bool: """Validate that all prerequisites are met for video generation.""" diff --git a/backend/workers/video_worker_orchestrator.py b/backend/workers/video_worker_orchestrator.py index 0c63c5c4..4ec2e7a3 100644 --- a/backend/workers/video_worker_orchestrator.py +++ b/backend/workers/video_worker_orchestrator.py @@ -109,6 +109,9 @@ class OrchestratorResult: dropbox_link: Optional[str] = None gdrive_files: Optional[Dict[str, str]] = field(default_factory=dict) + # Distribution warnings (non-fatal upload failures) + distribution_warnings: List[str] = field(default_factory=list) + # Timing encoding_time_seconds: Optional[float] = None total_time_seconds: Optional[float] = None @@ -603,6 +606,7 @@ class VideoWorkerOrchestrator: except Exception as e: self.job_log.error(f"YouTube upload failed: {e}") + self.result.distribution_warnings.append(f"YouTube upload failed: {e}") # Don't fail the pipeline - YouTube is optional async def _upload_to_dropbox(self): @@ -639,6 +643,7 @@ class VideoWorkerOrchestrator: except Exception as e: self.job_log.error(f"Dropbox upload failed: {e}") + self.result.distribution_warnings.append(f"Dropbox upload failed: {e}") # Don't fail the pipeline - Dropbox is optional async def _upload_to_gdrive(self): @@ -675,6 +680,7 @@ class VideoWorkerOrchestrator: except Exception as e: self.job_log.error(f"Google Drive upload failed: {e}") + self.result.distribution_warnings.append(f"Google Drive upload failed: {e}") # Don't fail the pipeline - GDrive is optional async def _run_notifications(self): diff --git a/docs/LESSONS-LEARNED.md b/docs/LESSONS-LEARNED.md index 4a24f58a..9fb36fa2 100644 --- a/docs/LESSONS-LEARNED.md +++ b/docs/LESSONS-LEARNED.md @@ -222,6 +222,18 @@ When deferring processing from one worker to another, you MUST update the job st ### Google Drive Query Escaping Escape special chars in Google Drive API queries: `'` → `\'`, `\` → `\\\\`. +### Google Drive Stale Connection on Cloud Run (Feb 2026) +**What happened**: Every GDrive upload silently failed for ~1 month (64 errors, 12+ jobs). Errors: `[Errno 32] Broken pipe` or `[SSL: UNEXPECTED_EOF_WHILE_READING]`. Jobs completed with `gdrive_files: {}` and no alert was sent. + +**Root cause**: The singleton `GoogleDriveService` caches its HTTP connection. When Cloud Run containers sit idle between jobs, the underlying TCP connection goes stale. The next API call hits a dead socket. The orchestrator catches all exceptions in distribution and continues silently. + +**Fix** (v0.119.7): +1. Added tenacity retry with `_reset_service()` (sets `self._service = None` to force fresh connection) on transient errors (`BrokenPipeError`, `ConnectionResetError`, `ssl.SSLError`) +2. Added `distribution_warnings` field to `OrchestratorResult` to track non-fatal upload failures +3. Added Pushbullet notification when distribution uploads fail, so operator is alerted immediately + +**Pattern**: Any singleton service that caches HTTP connections should have retry logic with connection reset for stale connection errors. Cloud Run containers can idle for minutes between requests, killing keep-alive connections. Also, never silently swallow distribution failures - always track and alert on them. + ### FFmpeg Path Escaping For subprocess without shell, FFmpeg filter paths need: apostrophes escaped as `'\\''`, special chars as `\\[char]`. diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index fc71ff1c..26eb86df 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -68,3 +68,27 @@ gcloud compute ssh encoding-worker --zone=us-central1-c --project=nomadkaraoke \ gcloud compute ssh encoding-worker --zone=us-central1-c --project=nomadkaraoke \ --command="sudo systemctl restart encoding-worker" ``` + +--- + +## Google Drive uploads missing (gdrive_files empty) + +**Symptoms:** Jobs complete successfully but `state_data.gdrive_files` is empty `{}`. Cloud Run logs show `BrokenPipeError` or `SSL: UNEXPECTED_EOF_WHILE_READING`. + +**Cause:** Stale HTTP connections in the singleton `GoogleDriveService`. Since v0.119.7, this is handled automatically with retry + connection reset. + +**Backfill affected jobs:** +```bash +# Dry run - list affected jobs +GCS_BUCKET_NAME=karaoke-gen-storage-nomadkaraoke \ +GOOGLE_CLOUD_PROJECT=nomadkaraoke \ +python scripts/backfill_gdrive_uploads.py --all-missing --dry-run + +# Run for real +GCS_BUCKET_NAME=karaoke-gen-storage-nomadkaraoke \ +GOOGLE_CLOUD_PROJECT=nomadkaraoke \ +python scripts/backfill_gdrive_uploads.py --all-missing + +# Specific jobs +python scripts/backfill_gdrive_uploads.py --job-ids JOB1,JOB2 +``` diff --git a/pyproject.toml b/pyproject.toml index 81a01b6a..e6b20845 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.119.7" +version = "0.119.8" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT" diff --git a/scripts/backfill_gdrive_uploads.py b/scripts/backfill_gdrive_uploads.py new file mode 100644 index 00000000..c082b0ae --- /dev/null +++ b/scripts/backfill_gdrive_uploads.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 +""" +Backfill Google Drive uploads for jobs that completed with empty gdrive_files. + +This addresses the issue where GDrive uploads silently failed due to stale +HTTP connections (BrokenPipeError / SSLError) on idle Cloud Run containers. + +Usage: + # Dry run - show what would be uploaded + python scripts/backfill_gdrive_uploads.py --dry-run --job-ids ca408e46,99baa198 + + # Upload specific jobs + python scripts/backfill_gdrive_uploads.py --job-ids ca408e46,99baa198 + + # Find and upload all jobs with empty gdrive_files + python scripts/backfill_gdrive_uploads.py --all-missing + + # Override gdrive folder ID (instead of using DEFAULT_GDRIVE_FOLDER_ID env var) + python scripts/backfill_gdrive_uploads.py --job-ids ca408e46 --gdrive-folder-id 1abc... + +Requires: + - google-cloud-firestore + - google-cloud-storage + - google-api-python-client, google-auth + - GOOGLE_APPLICATION_CREDENTIALS or gcloud auth application-default login +""" +import argparse +import logging +import os +import sys +import tempfile + +# Add project root to path so we can import backend modules +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from google.cloud import firestore, storage + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", +) +logger = logging.getLogger(__name__) + +GCS_BUCKET = "karaoke-gen-storage-nomadkaraoke" +PROJECT = "nomadkaraoke" + +# Default GDrive public share folder (from infrastructure/config.py) +DEFAULT_GDRIVE_FOLDER_ID = "1laRKAyxo0v817SstfM5XkpbWiNKNAMSX" + +# Known affected jobs from the incident +KNOWN_AFFECTED_JOBS = [ + "ca408e46", "99baa198", "1e15d1dd", "ebba0996", + "cb744c24", "1389f765", "fde3ae7d", "21f4b34f", + "443c34a1", "56192d3a", "fdb4dc63", "4eeaa729", +] + + +def get_job(db: firestore.Client, job_id: str) -> dict | None: + """Fetch job document from Firestore.""" + doc = db.collection("jobs").document(job_id).get() + if doc.exists: + return doc.to_dict() + return None + + +def find_jobs_with_missing_gdrive(db: firestore.Client) -> list[dict]: + """Find completed jobs that have empty gdrive_files in state_data. + + Looks for jobs that have a brand_code (indicating they went through + distribution) but have empty or missing gdrive_files. + """ + jobs = [] + query = db.collection("jobs").where("status", "==", "complete") + + for doc in query.stream(): + data = doc.to_dict() + state_data = data.get("state_data", {}) + gdrive_files = state_data.get("gdrive_files", {}) + brand_code = state_data.get("brand_code") + + # Job should have had GDrive uploads if it has a brand_code + # but gdrive_files is empty/missing + if brand_code and not gdrive_files: + jobs.append({"job_id": doc.id, **data}) + + return jobs + + +def _classify_blob(filename: str) -> str | None: + """Classify a GCS blob filename to an output_files key, or None to skip. + + Only returns keys for the 3 file types that get uploaded to GDrive: + lossy 4K MP4, lossy 720p MP4, and CDG ZIP. + """ + lower = filename.lower() + if lower.endswith(".zip") and "cdg" in lower: + return "final_karaoke_cdg_zip" + if "720p" in lower and lower.endswith(".mp4"): + return "final_karaoke_lossy_720p_mp4" + if "lossy" in lower and lower.endswith(".mp4") and "720p" not in lower: + return "final_karaoke_lossy_mp4" + return None + + +def download_finals(gcs_client: storage.Client, job_id: str, temp_dir: str) -> dict: + """Download only the final output files needed for GDrive upload. + + Only downloads lossy 4K MP4, lossy 720p MP4, and CDG ZIP — skips + lossless files, MKVs, karaoke video, etc. to save bandwidth. + + Returns dict mapping output_files keys to local paths. + """ + bucket = gcs_client.bucket(GCS_BUCKET) + prefix = f"jobs/{job_id}/finals/" + blobs = list(bucket.list_blobs(prefix=prefix)) + + output_files = {} + for blob in blobs: + filename = os.path.basename(blob.name) + if not filename: + continue + + key = _classify_blob(filename) + if key is None: + continue + if key in output_files: + continue # already have this type + + local_path = os.path.join(temp_dir, filename) + logger.info(f" Downloading {filename} ({blob.size / 1024 / 1024:.1f} MB)") + blob.download_to_filename(local_path) + output_files[key] = local_path + + return output_files + + +def list_finals(gcs_client: storage.Client, job_id: str) -> list[str]: + """List uploadable final output files in GCS without downloading.""" + bucket = gcs_client.bucket(GCS_BUCKET) + prefix = f"jobs/{job_id}/finals/" + blobs = list(bucket.list_blobs(prefix=prefix)) + return [ + f"{os.path.basename(b.name)} ({b.size / 1024 / 1024:.1f} MB)" + for b in blobs + if os.path.basename(b.name) and _classify_blob(os.path.basename(b.name)) + ] + + +def backfill_job( + db: firestore.Client, + gcs_client: storage.Client, + job_id: str, + gdrive_folder_id: str, + dry_run: bool = False, +) -> bool: + """Backfill GDrive upload for a single job. + + Returns True if successful (or dry run). + """ + logger.info(f"Processing job {job_id}") + + # Fetch job data + job_data = get_job(db, job_id) + if not job_data: + logger.error(f" Job {job_id} not found in Firestore") + return False + + state_data = job_data.get("state_data", {}) + artist = job_data.get("artist", "Unknown") + title = job_data.get("title", "Unknown") + brand_code = state_data.get("brand_code") + status = job_data.get("status") + + logger.info(f" {artist} - {title} (brand: {brand_code}, status: {status})") + + # Check existing gdrive_files + existing = state_data.get("gdrive_files", {}) + if existing: + logger.info(f" Already has gdrive_files: {existing}") + logger.info(f" Skipping (already uploaded)") + return True + + if not brand_code: + logger.warning(f" No brand code, skipping") + return False + + if dry_run: + # List what files are available in GCS + files = list_finals(gcs_client, job_id) + if files: + logger.info(f" [DRY RUN] Would upload {len(files)} files to GDrive folder {gdrive_folder_id}:") + for f in files: + logger.info(f" - {f}") + else: + logger.warning(f" [DRY RUN] No final files found in GCS") + return True + + # Download files and upload + with tempfile.TemporaryDirectory(prefix=f"backfill-{job_id}-") as temp_dir: + # Download finals from GCS + logger.info(f" Downloading finals from GCS...") + output_files = download_finals(gcs_client, job_id, temp_dir) + + if not output_files: + logger.warning(f" No final files found in GCS") + return False + + logger.info(f" Found {len(output_files)} files to upload") + + # Upload to GDrive + from backend.services.gdrive_service import GoogleDriveService + + gdrive = GoogleDriveService() + if not gdrive.is_configured: + logger.error(f" Google Drive not configured") + return False + + base_name = f"{artist} - {title}" + uploaded = gdrive.upload_to_public_share( + root_folder_id=gdrive_folder_id, + brand_code=brand_code, + base_name=base_name, + output_files=output_files, + ) + + logger.info(f" Uploaded {len(uploaded)} files: {uploaded}") + + # Update Firestore + db.collection("jobs").document(job_id).update({ + "state_data.gdrive_files": uploaded, + }) + logger.info(f" Updated Firestore state_data.gdrive_files") + + return True + + +def main(): + parser = argparse.ArgumentParser( + description="Backfill Google Drive uploads for jobs with empty gdrive_files" + ) + parser.add_argument( + "--job-ids", + help="Comma-separated list of job IDs to process", + ) + parser.add_argument( + "--all-missing", + action="store_true", + help="Find and process all jobs with missing gdrive_files", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Show what would be done without making changes", + ) + parser.add_argument( + "--gdrive-folder-id", + default=os.getenv("DEFAULT_GDRIVE_FOLDER_ID", DEFAULT_GDRIVE_FOLDER_ID), + help=f"Google Drive folder ID for public share (default: {DEFAULT_GDRIVE_FOLDER_ID})", + ) + args = parser.parse_args() + + if not args.job_ids and not args.all_missing: + parser.error("Specify --job-ids or --all-missing") + + db = firestore.Client(project=PROJECT) + gcs_client = storage.Client(project=PROJECT) + + if args.job_ids: + job_ids = [j.strip() for j in args.job_ids.split(",")] + elif args.all_missing: + logger.info("Searching for jobs with missing GDrive uploads...") + jobs = find_jobs_with_missing_gdrive(db) + job_ids = [j["job_id"] for j in jobs] + logger.info(f"Found {len(job_ids)} jobs with missing uploads") + + if args.dry_run: + logger.info("[DRY RUN MODE]") + + logger.info(f"GDrive folder: {args.gdrive_folder_id}") + + success_count = 0 + fail_count = 0 + + for job_id in job_ids: + try: + ok = backfill_job( + db, gcs_client, job_id, + gdrive_folder_id=args.gdrive_folder_id, + dry_run=args.dry_run, + ) + if ok: + success_count += 1 + else: + fail_count += 1 + except Exception as e: + logger.error(f" Failed: {e}", exc_info=True) + fail_count += 1 + + logger.info(f"\nDone: {success_count} succeeded, {fail_count} failed out of {len(job_ids)} jobs") + + +if __name__ == "__main__": + main()