mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
Fix silent Google Drive upload failures with retry + alerting (#417)
## Summary
- Added tenacity retry with connection reset to Google Drive API calls
(`get_or_create_folder`, `upload_file`, `delete_file`) to handle stale
HTTP connections on idle Cloud Run containers
- Added `distribution_warnings` field to `OrchestratorResult` and
`_handle_native_distribution()` to track non-fatal upload failures
(YouTube, Dropbox, GDrive)
- Added Pushbullet notification when distribution uploads fail, so
operator is alerted immediately
- Added backfill script (`scripts/backfill_gdrive_uploads.py`) for
affected jobs - already run successfully for 12 affected jobs
- Updated docs: LESSONS-LEARNED.md and TROUBLESHOOTING.md with GDrive
stale connection pattern
## Context
Every GDrive upload silently failed for ~1 month (64 errors, 12+
affected jobs). The root cause was the singleton `GoogleDriveService`
caching its HTTP connection, which went stale when Cloud Run containers
sat idle between jobs. The orchestrator caught all exceptions and
continued, so jobs completed with `gdrive_files: {}` and no alert was
sent.
## Test plan
- [x] 5 new retry behavior tests in `test_gdrive_service.py`
(BrokenPipeError, SSLError, give up after 3, service reset, no retry on
non-transient)
- [x] 3 new distribution warnings tests in
`test_video_worker_orchestrator.py`
- [x] All 76 affected tests pass
- [x] Full backend test suite passes
- [x] Backfill script already run successfully for all 12 affected jobs
@coderabbitai ignore
🤖 Generated with [Claude Code](https://claude.com/claude-code)
---------
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
23a8b0fb98
commit
0f5ddb0092
9 files changed
+775
-84
No files matched your search
@@ -12,13 +12,19 @@ shared with YouTube credentials if scopes include drive.file.
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import ssl
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from tenacity import Retrying, stop_after_attempt, wait_exponential, retry_if_exception_type
|
||||
|
||||
from backend.config import get_settings
|
||||
from karaoke_gen.utils import sanitize_filename
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Transient errors that indicate a stale HTTP connection (Cloud Run idle containers)
|
||||
TRANSIENT_ERRORS = (BrokenPipeError, ConnectionResetError, ConnectionError, ssl.SSLError)
|
||||
|
||||
|
||||
class GoogleDriveService:
|
||||
"""Google Drive operations using native API."""
|
||||
@@ -33,6 +39,11 @@ class GoogleDriveService:
|
||||
self._credentials_data: Optional[Dict[str, Any]] = None
|
||||
self._loaded = False
|
||||
|
||||
def _reset_service(self):
|
||||
"""Reset the cached Drive service to force a fresh connection on next use."""
|
||||
logger.info("Resetting Google Drive service connection")
|
||||
self._service = None
|
||||
|
||||
def _load_credentials(self) -> Optional[Dict[str, Any]]:
|
||||
"""Load OAuth credentials from Secret Manager."""
|
||||
if self._loaded:
|
||||
@@ -127,6 +138,9 @@ class GoogleDriveService:
|
||||
"""
|
||||
Get existing folder or create new one, return folder ID.
|
||||
|
||||
Retries on transient connection errors (BrokenPipeError, SSLError)
|
||||
that occur when Cloud Run containers sit idle between jobs.
|
||||
|
||||
Args:
|
||||
parent_id: Parent folder ID
|
||||
folder_name: Name of folder to find or create
|
||||
@@ -134,33 +148,41 @@ class GoogleDriveService:
|
||||
Returns:
|
||||
Folder ID
|
||||
"""
|
||||
logger.info(f"Looking for folder '{folder_name}' in parent {parent_id}")
|
||||
for attempt in Retrying(
|
||||
stop=stop_after_attempt(3),
|
||||
wait=wait_exponential(multiplier=1, min=2, max=8),
|
||||
retry=retry_if_exception_type(TRANSIENT_ERRORS),
|
||||
before_sleep=lambda retry_state: self._reset_service(),
|
||||
reraise=True,
|
||||
):
|
||||
with attempt:
|
||||
logger.info(f"Looking for folder '{folder_name}' in parent {parent_id}")
|
||||
|
||||
# Search for existing folder
|
||||
# Escape single quotes in folder name for Google Drive API query syntax
|
||||
escaped_folder_name = folder_name.replace("'", "\\'")
|
||||
query = (
|
||||
f"name='{escaped_folder_name}' and '{parent_id}' in parents "
|
||||
f"and mimeType='application/vnd.google-apps.folder' and trashed=false"
|
||||
)
|
||||
results = self.service.files().list(q=query, fields="files(id, name)").execute()
|
||||
# Search for existing folder
|
||||
# Escape single quotes in folder name for Google Drive API query syntax
|
||||
escaped_folder_name = folder_name.replace("'", "\\'")
|
||||
query = (
|
||||
f"name='{escaped_folder_name}' and '{parent_id}' in parents "
|
||||
f"and mimeType='application/vnd.google-apps.folder' and trashed=false"
|
||||
)
|
||||
results = self.service.files().list(q=query, fields="files(id, name)").execute()
|
||||
|
||||
if results.get("files"):
|
||||
folder_id = results["files"][0]["id"]
|
||||
logger.info(f"Found existing folder '{folder_name}': {folder_id}")
|
||||
return folder_id
|
||||
if results.get("files"):
|
||||
folder_id = results["files"][0]["id"]
|
||||
logger.info(f"Found existing folder '{folder_name}': {folder_id}")
|
||||
return folder_id
|
||||
|
||||
# Create folder
|
||||
logger.info(f"Creating new folder '{folder_name}'")
|
||||
metadata = {
|
||||
"name": folder_name,
|
||||
"mimeType": "application/vnd.google-apps.folder",
|
||||
"parents": [parent_id],
|
||||
}
|
||||
folder = self.service.files().create(body=metadata, fields="id").execute()
|
||||
folder_id = folder["id"]
|
||||
logger.info(f"Created folder '{folder_name}': {folder_id}")
|
||||
return folder_id
|
||||
# Create folder
|
||||
logger.info(f"Creating new folder '{folder_name}'")
|
||||
metadata = {
|
||||
"name": folder_name,
|
||||
"mimeType": "application/vnd.google-apps.folder",
|
||||
"parents": [parent_id],
|
||||
}
|
||||
folder = self.service.files().create(body=metadata, fields="id").execute()
|
||||
folder_id = folder["id"]
|
||||
logger.info(f"Created folder '{folder_name}': {folder_id}")
|
||||
return folder_id
|
||||
|
||||
def upload_file(
|
||||
self,
|
||||
@@ -172,6 +194,9 @@ class GoogleDriveService:
|
||||
"""
|
||||
Upload a file to a specific Drive folder.
|
||||
|
||||
Retries on transient connection errors (BrokenPipeError, SSLError)
|
||||
that occur when Cloud Run containers sit idle between jobs.
|
||||
|
||||
Args:
|
||||
local_path: Local file path
|
||||
parent_id: Parent folder ID in Google Drive
|
||||
@@ -183,49 +208,57 @@ class GoogleDriveService:
|
||||
"""
|
||||
from googleapiclient.http import MediaFileUpload
|
||||
|
||||
file_size = os.path.getsize(local_path)
|
||||
logger.info(
|
||||
f"Uploading {local_path} ({file_size / 1024 / 1024:.1f} MB) "
|
||||
f"as '{filename}' to folder {parent_id}"
|
||||
)
|
||||
for attempt in Retrying(
|
||||
stop=stop_after_attempt(3),
|
||||
wait=wait_exponential(multiplier=1, min=2, max=8),
|
||||
retry=retry_if_exception_type(TRANSIENT_ERRORS),
|
||||
before_sleep=lambda retry_state: self._reset_service(),
|
||||
reraise=True,
|
||||
):
|
||||
with attempt:
|
||||
file_size = os.path.getsize(local_path)
|
||||
logger.info(
|
||||
f"Uploading {local_path} ({file_size / 1024 / 1024:.1f} MB) "
|
||||
f"as '{filename}' to folder {parent_id}"
|
||||
)
|
||||
|
||||
# Determine MIME type
|
||||
ext = os.path.splitext(local_path)[1].lower()
|
||||
mime_types = {
|
||||
".mp4": "video/mp4",
|
||||
".mkv": "video/x-matroska",
|
||||
".zip": "application/zip",
|
||||
".flac": "audio/flac",
|
||||
".mp3": "audio/mpeg",
|
||||
".wav": "audio/wav",
|
||||
".png": "image/png",
|
||||
".jpg": "image/jpeg",
|
||||
".jpeg": "image/jpeg",
|
||||
}
|
||||
mime_type = mime_types.get(ext, "application/octet-stream")
|
||||
# Determine MIME type
|
||||
ext = os.path.splitext(local_path)[1].lower()
|
||||
mime_types = {
|
||||
".mp4": "video/mp4",
|
||||
".mkv": "video/x-matroska",
|
||||
".zip": "application/zip",
|
||||
".flac": "audio/flac",
|
||||
".mp3": "audio/mpeg",
|
||||
".wav": "audio/wav",
|
||||
".png": "image/png",
|
||||
".jpg": "image/jpeg",
|
||||
".jpeg": "image/jpeg",
|
||||
}
|
||||
mime_type = mime_types.get(ext, "application/octet-stream")
|
||||
|
||||
# Check for existing file with same name
|
||||
if replace_existing:
|
||||
# Escape single quotes in filename for Google Drive API query syntax
|
||||
escaped_filename = filename.replace("'", "\\'")
|
||||
query = (
|
||||
f"name='{escaped_filename}' and '{parent_id}' in parents and trashed=false"
|
||||
)
|
||||
results = self.service.files().list(q=query, fields="files(id)").execute()
|
||||
for existing_file in results.get("files", []):
|
||||
logger.info(f"Deleting existing file: {existing_file['id']}")
|
||||
self.service.files().delete(fileId=existing_file["id"]).execute()
|
||||
# Check for existing file with same name
|
||||
if replace_existing:
|
||||
# Escape single quotes in filename for Google Drive API query syntax
|
||||
escaped_filename = filename.replace("'", "\\'")
|
||||
query = (
|
||||
f"name='{escaped_filename}' and '{parent_id}' in parents and trashed=false"
|
||||
)
|
||||
results = self.service.files().list(q=query, fields="files(id)").execute()
|
||||
for existing_file in results.get("files", []):
|
||||
logger.info(f"Deleting existing file: {existing_file['id']}")
|
||||
self.service.files().delete(fileId=existing_file["id"]).execute()
|
||||
|
||||
# Upload file
|
||||
metadata = {"name": filename, "parents": [parent_id]}
|
||||
media = MediaFileUpload(local_path, mimetype=mime_type, resumable=True)
|
||||
# Upload file
|
||||
metadata = {"name": filename, "parents": [parent_id]}
|
||||
media = MediaFileUpload(local_path, mimetype=mime_type, resumable=True)
|
||||
|
||||
file_result = (
|
||||
self.service.files().create(body=metadata, media_body=media, fields="id").execute()
|
||||
)
|
||||
file_id = file_result["id"]
|
||||
logger.info(f"Successfully uploaded '{filename}': {file_id}")
|
||||
return file_id
|
||||
file_result = (
|
||||
self.service.files().create(body=metadata, media_body=media, fields="id").execute()
|
||||
)
|
||||
file_id = file_result["id"]
|
||||
logger.info(f"Successfully uploaded '{filename}': {file_id}")
|
||||
return file_id
|
||||
|
||||
def upload_to_public_share(
|
||||
self,
|
||||
@@ -308,6 +341,9 @@ class GoogleDriveService:
|
||||
"""
|
||||
Delete a file from Google Drive.
|
||||
|
||||
Retries on transient connection errors (BrokenPipeError, SSLError)
|
||||
that occur when Cloud Run containers sit idle between jobs.
|
||||
|
||||
Args:
|
||||
file_id: Google Drive file ID to delete
|
||||
|
||||
@@ -317,9 +353,17 @@ class GoogleDriveService:
|
||||
logger.info(f"Deleting Google Drive file: {file_id}")
|
||||
|
||||
try:
|
||||
self.service.files().delete(fileId=file_id).execute()
|
||||
logger.info(f"Successfully deleted file: {file_id}")
|
||||
return True
|
||||
for attempt in Retrying(
|
||||
stop=stop_after_attempt(3),
|
||||
wait=wait_exponential(multiplier=1, min=2, max=8),
|
||||
retry=retry_if_exception_type(TRANSIENT_ERRORS),
|
||||
before_sleep=lambda retry_state: self._reset_service(),
|
||||
reraise=True,
|
||||
):
|
||||
with attempt:
|
||||
self.service.files().delete(fileId=file_id).execute()
|
||||
logger.info(f"Successfully deleted file: {file_id}")
|
||||
return True
|
||||
except Exception as e:
|
||||
# Check if it's a 404 (already deleted)
|
||||
if hasattr(e, 'resp') and e.resp.status == 404:
|
||||
|
||||
@@ -9,6 +9,7 @@ These tests mock the Google API client and Secret Manager to verify:
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import ssl
|
||||
import pytest
|
||||
from unittest.mock import Mock, MagicMock, patch
|
||||
|
||||
@@ -501,6 +502,186 @@ class TestUploadToPublicShare:
|
||||
assert len(result) == 1
|
||||
|
||||
|
||||
class TestRetryBehavior:
|
||||
"""Test retry behavior for transient connection errors."""
|
||||
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_retries_on_broken_pipe(self, mock_get_settings):
|
||||
"""Test that BrokenPipeError triggers retry and succeeds on second attempt."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.return_value = json.dumps({
|
||||
"refresh_token": "token", "client_id": "id", "client_secret": "secret",
|
||||
})
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
service = GoogleDriveService()
|
||||
|
||||
mock_files_api = Mock()
|
||||
mock_list_result = Mock()
|
||||
# First call raises BrokenPipeError, second succeeds
|
||||
mock_list_result.execute.side_effect = [
|
||||
BrokenPipeError("[Errno 32] Broken pipe"),
|
||||
{"files": [{"id": "folder-123", "name": "MP4"}]},
|
||||
]
|
||||
mock_files_api.list.return_value = mock_list_result
|
||||
|
||||
mock_drive = Mock()
|
||||
mock_drive.files.return_value = mock_files_api
|
||||
service._service = mock_drive
|
||||
service._loaded = True
|
||||
service._credentials_data = {"refresh_token": "token", "client_id": "id", "client_secret": "secret"}
|
||||
|
||||
# Patch _reset_service to clear but re-set the mock (avoid real OAuth)
|
||||
def mock_reset():
|
||||
service._service = mock_drive
|
||||
|
||||
service._reset_service = mock_reset
|
||||
|
||||
folder_id = service.get_or_create_folder("parent-123", "MP4")
|
||||
assert folder_id == "folder-123"
|
||||
assert mock_list_result.execute.call_count == 2
|
||||
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_gives_up_after_3_attempts(self, mock_get_settings):
|
||||
"""Test that 3 consecutive failures raises the exception."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.return_value = json.dumps({
|
||||
"refresh_token": "token", "client_id": "id", "client_secret": "secret",
|
||||
})
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
service = GoogleDriveService()
|
||||
|
||||
mock_files_api = Mock()
|
||||
mock_list_result = Mock()
|
||||
mock_list_result.execute.side_effect = BrokenPipeError("[Errno 32] Broken pipe")
|
||||
mock_files_api.list.return_value = mock_list_result
|
||||
|
||||
mock_drive = Mock()
|
||||
mock_drive.files.return_value = mock_files_api
|
||||
service._service = mock_drive
|
||||
service._loaded = True
|
||||
service._credentials_data = {"refresh_token": "token", "client_id": "id", "client_secret": "secret"}
|
||||
|
||||
# Patch _reset_service to re-set the mock (avoid real OAuth)
|
||||
service._reset_service = lambda: setattr(service, '_service', mock_drive)
|
||||
|
||||
with pytest.raises(BrokenPipeError):
|
||||
service.get_or_create_folder("parent-123", "MP4")
|
||||
|
||||
assert mock_list_result.execute.call_count == 3
|
||||
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_service_reset_on_retry(self, mock_get_settings):
|
||||
"""Test that _service is reset to None between retry attempts."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.return_value = json.dumps({
|
||||
"refresh_token": "token", "client_id": "id", "client_secret": "secret",
|
||||
})
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
service = GoogleDriveService()
|
||||
|
||||
mock_files_api = Mock()
|
||||
mock_list_result = Mock()
|
||||
mock_list_result.execute.side_effect = [
|
||||
BrokenPipeError("[Errno 32] Broken pipe"),
|
||||
{"files": [{"id": "folder-123", "name": "MP4"}]},
|
||||
]
|
||||
mock_files_api.list.return_value = mock_list_result
|
||||
|
||||
mock_drive = Mock()
|
||||
mock_drive.files.return_value = mock_files_api
|
||||
service._service = mock_drive
|
||||
service._loaded = True
|
||||
service._credentials_data = {"refresh_token": "token", "client_id": "id", "client_secret": "secret"}
|
||||
|
||||
# Patch _reset_service to track calls but still do the reset
|
||||
original_reset = service._reset_service
|
||||
reset_calls = []
|
||||
|
||||
def tracked_reset():
|
||||
reset_calls.append(True)
|
||||
original_reset()
|
||||
# Re-set mock service so the retry works
|
||||
service._service = mock_drive
|
||||
|
||||
service._reset_service = tracked_reset
|
||||
|
||||
folder_id = service.get_or_create_folder("parent-123", "MP4")
|
||||
assert folder_id == "folder-123"
|
||||
assert len(reset_calls) == 1 # Reset called once between attempt 1 and 2
|
||||
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_retries_on_ssl_error(self, mock_get_settings):
|
||||
"""Test that SSLError triggers retry."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.return_value = json.dumps({
|
||||
"refresh_token": "token", "client_id": "id", "client_secret": "secret",
|
||||
})
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
service = GoogleDriveService()
|
||||
|
||||
mock_files_api = Mock()
|
||||
mock_list_result = Mock()
|
||||
mock_list_result.execute.side_effect = [
|
||||
ssl.SSLError("SSL: UNEXPECTED_EOF_WHILE_READING"),
|
||||
{"files": [{"id": "folder-123", "name": "MP4"}]},
|
||||
]
|
||||
mock_files_api.list.return_value = mock_list_result
|
||||
|
||||
mock_drive = Mock()
|
||||
mock_drive.files.return_value = mock_files_api
|
||||
service._service = mock_drive
|
||||
service._loaded = True
|
||||
service._credentials_data = {"refresh_token": "token", "client_id": "id", "client_secret": "secret"}
|
||||
|
||||
# Patch _reset_service to re-set the mock (avoid real OAuth)
|
||||
service._reset_service = lambda: setattr(service, '_service', mock_drive)
|
||||
|
||||
folder_id = service.get_or_create_folder("parent-123", "MP4")
|
||||
assert folder_id == "folder-123"
|
||||
assert mock_list_result.execute.call_count == 2
|
||||
|
||||
@patch("backend.services.gdrive_service.get_settings")
|
||||
def test_no_retry_on_non_transient_error(self, mock_get_settings):
|
||||
"""Test that non-transient errors are not retried."""
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
mock_settings = Mock()
|
||||
mock_settings.get_secret.return_value = json.dumps({
|
||||
"refresh_token": "token", "client_id": "id", "client_secret": "secret",
|
||||
})
|
||||
mock_get_settings.return_value = mock_settings
|
||||
|
||||
service = GoogleDriveService()
|
||||
|
||||
mock_files_api = Mock()
|
||||
mock_list_result = Mock()
|
||||
mock_list_result.execute.side_effect = ValueError("bad value")
|
||||
mock_files_api.list.return_value = mock_list_result
|
||||
|
||||
mock_drive = Mock()
|
||||
mock_drive.files.return_value = mock_files_api
|
||||
service._service = mock_drive
|
||||
service._loaded = True
|
||||
service._credentials_data = {"refresh_token": "token", "client_id": "id", "client_secret": "secret"}
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
service.get_or_create_folder("parent-123", "MP4")
|
||||
|
||||
assert mock_list_result.execute.call_count == 1
|
||||
|
||||
|
||||
class TestGetGdriveService:
|
||||
"""Test get_gdrive_service singleton."""
|
||||
|
||||
|
||||
@@ -1512,3 +1512,64 @@ class TestOrchestratorCountdownPadding:
|
||||
assert captured_input is not None, "encode() should have been called"
|
||||
assert captured_input.options.get("countdown_padding_seconds") == 3.0, \
|
||||
"Orchestrator must pass countdown_padding_seconds in EncodingInput.options"
|
||||
|
||||
|
||||
class TestDistributionWarnings:
|
||||
"""Test distribution_warnings tracking in OrchestratorResult."""
|
||||
|
||||
def test_result_has_distribution_warnings_field(self):
|
||||
"""Test that OrchestratorResult has distribution_warnings as empty list by default."""
|
||||
result = OrchestratorResult(success=True)
|
||||
assert result.distribution_warnings == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_gdrive_failure_populates_warnings(self):
|
||||
"""Test that Google Drive upload failure populates distribution_warnings."""
|
||||
config = OrchestratorConfig(
|
||||
job_id="test-job",
|
||||
artist="Test Artist",
|
||||
title="Test Title",
|
||||
title_video_path="/path/title.mov",
|
||||
karaoke_video_path="/path/karaoke.mov",
|
||||
instrumental_audio_path="/path/audio.flac",
|
||||
gdrive_folder_id="folder-123",
|
||||
)
|
||||
orchestrator = VideoWorkerOrchestrator(config)
|
||||
orchestrator.result.brand_code = "NOMAD-0001"
|
||||
|
||||
with patch("backend.services.gdrive_service.get_gdrive_service") as mock_get:
|
||||
mock_gdrive = MagicMock()
|
||||
mock_gdrive.is_configured = True
|
||||
mock_gdrive.upload_to_public_share.side_effect = Exception("Connection refused")
|
||||
mock_get.return_value = mock_gdrive
|
||||
|
||||
await orchestrator._upload_to_gdrive()
|
||||
|
||||
assert len(orchestrator.result.distribution_warnings) == 1
|
||||
assert "Google Drive upload failed" in orchestrator.result.distribution_warnings[0]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dropbox_failure_populates_warnings(self):
|
||||
"""Test that Dropbox upload failure populates distribution_warnings."""
|
||||
config = OrchestratorConfig(
|
||||
job_id="test-job",
|
||||
artist="Test Artist",
|
||||
title="Test Title",
|
||||
title_video_path="/path/title.mov",
|
||||
karaoke_video_path="/path/karaoke.mov",
|
||||
instrumental_audio_path="/path/audio.flac",
|
||||
dropbox_path="/Public Share",
|
||||
)
|
||||
orchestrator = VideoWorkerOrchestrator(config)
|
||||
orchestrator.result.brand_code = "NOMAD-0001"
|
||||
|
||||
with patch("backend.services.dropbox_service.get_dropbox_service") as mock_get:
|
||||
mock_dropbox = MagicMock()
|
||||
mock_dropbox.is_configured = True
|
||||
mock_dropbox.upload_folder.side_effect = Exception("Auth expired")
|
||||
mock_get.return_value = mock_dropbox
|
||||
|
||||
await orchestrator._upload_to_dropbox()
|
||||
|
||||
assert len(orchestrator.result.distribution_warnings) == 1
|
||||
assert "Dropbox upload failed" in orchestrator.result.distribution_warnings[0]
|
||||
@@ -339,15 +339,26 @@ async def generate_video_orchestrated(job_id: str) -> bool:
|
||||
# Store result metadata in job BEFORE transitioning to COMPLETE
|
||||
# This ensures youtube_url is available when completion email is sent
|
||||
logger.info(f"[job:{job_id}] Video generation complete")
|
||||
job_manager.update_job(job_id, {
|
||||
'state_data': {
|
||||
**job.state_data,
|
||||
'brand_code': result.brand_code,
|
||||
'youtube_url': result.youtube_url,
|
||||
'dropbox_link': result.dropbox_link,
|
||||
'gdrive_files': result.gdrive_files,
|
||||
}
|
||||
})
|
||||
state_update = {
|
||||
**job.state_data,
|
||||
'brand_code': result.brand_code,
|
||||
'youtube_url': result.youtube_url,
|
||||
'dropbox_link': result.dropbox_link,
|
||||
'gdrive_files': result.gdrive_files,
|
||||
}
|
||||
if result.distribution_warnings:
|
||||
state_update['distribution_warnings'] = result.distribution_warnings
|
||||
job_manager.update_job(job_id, {'state_data': state_update})
|
||||
|
||||
# Send Pushbullet alert if any distribution uploads failed
|
||||
if result.distribution_warnings:
|
||||
_send_distribution_warning_notification(
|
||||
job_id=job_id,
|
||||
artist=job.artist,
|
||||
title=job.title,
|
||||
warnings=result.distribution_warnings,
|
||||
settings=settings,
|
||||
)
|
||||
|
||||
# Mark job as complete (triggers completion email with youtube_url now available)
|
||||
job_manager.transition_to_state(
|
||||
@@ -841,6 +852,7 @@ async def _handle_native_distribution(
|
||||
job_log.warning(f"Dropbox SDK not installed: {e}")
|
||||
except Exception as e:
|
||||
job_log.error(f"Native Dropbox upload failed: {e}", exc_info=True)
|
||||
result.setdefault('distribution_warnings', []).append(f"Dropbox upload failed: {e}")
|
||||
# Don't fail the job - distribution is optional
|
||||
|
||||
# Upload to Google Drive using native API
|
||||
@@ -884,6 +896,7 @@ async def _handle_native_distribution(
|
||||
job_log.warning(f"Google API packages not installed: {e}")
|
||||
except Exception as e:
|
||||
job_log.error(f"Native Google Drive upload failed: {e}", exc_info=True)
|
||||
result.setdefault('distribution_warnings', []).append(f"Google Drive upload failed: {e}")
|
||||
# Don't fail the job - distribution is optional
|
||||
elif existing_gdrive_files:
|
||||
job_log.info(f"Skipping Google Drive upload - orchestrator already uploaded {len(existing_gdrive_files)} files")
|
||||
@@ -891,17 +904,64 @@ async def _handle_native_distribution(
|
||||
# Update job state_data with brand code and links
|
||||
if brand_code or result.get('dropbox_link') or result.get('gdrive_files'):
|
||||
try:
|
||||
job_manager.update_job(job_id, {
|
||||
'state_data': {
|
||||
**job.state_data,
|
||||
'brand_code': brand_code,
|
||||
'dropbox_link': result.get('dropbox_link'),
|
||||
'gdrive_files': result.get('gdrive_files'),
|
||||
}
|
||||
})
|
||||
state_update = {
|
||||
**job.state_data,
|
||||
'brand_code': brand_code,
|
||||
'dropbox_link': result.get('dropbox_link'),
|
||||
'gdrive_files': result.get('gdrive_files'),
|
||||
}
|
||||
if result.get('distribution_warnings'):
|
||||
state_update['distribution_warnings'] = result['distribution_warnings']
|
||||
job_manager.update_job(job_id, {'state_data': state_update})
|
||||
except Exception as e:
|
||||
job_log.warning(f"Failed to update job state_data: {e}")
|
||||
|
||||
# Send Pushbullet alert if any distribution uploads failed
|
||||
if result.get('distribution_warnings'):
|
||||
_send_distribution_warning_notification(
|
||||
job_id=job_id,
|
||||
artist=job.artist,
|
||||
title=job.title,
|
||||
warnings=result['distribution_warnings'],
|
||||
settings=settings,
|
||||
)
|
||||
|
||||
|
||||
def _send_distribution_warning_notification(
|
||||
job_id: str,
|
||||
artist: str,
|
||||
title: str,
|
||||
warnings: list[str],
|
||||
settings=None,
|
||||
) -> None:
|
||||
"""Send Pushbullet notification when distribution uploads fail."""
|
||||
try:
|
||||
if settings is None:
|
||||
settings = get_settings()
|
||||
api_key = settings.get_secret("pushbullet-api-key")
|
||||
if not api_key:
|
||||
logger.warning("Pushbullet API key not found, skipping distribution warning notification")
|
||||
return
|
||||
|
||||
import requests as req
|
||||
|
||||
push_title = f"Distribution Warning: {artist} - {title}"
|
||||
body = f"Job {job_id} completed but {len(warnings)} upload(s) failed:\n\n"
|
||||
body += "\n".join(f" - {w}" for w in warnings)
|
||||
|
||||
resp = req.post(
|
||||
"https://api.pushbullet.com/v2/pushes",
|
||||
headers={"Access-Token": api_key, "Content-Type": "application/json"},
|
||||
json={"type": "note", "title": push_title, "body": body},
|
||||
timeout=10,
|
||||
)
|
||||
if resp.ok:
|
||||
logger.info(f"[job:{job_id}] Distribution warning notification sent via Pushbullet")
|
||||
else:
|
||||
logger.warning(f"[job:{job_id}] Pushbullet notification failed: {resp.status_code} {resp.text}")
|
||||
except Exception as e:
|
||||
logger.warning(f"[job:{job_id}] Failed to send Pushbullet notification: {e}")
|
||||
|
||||
|
||||
def _validate_prerequisites(job) -> bool:
|
||||
"""Validate that all prerequisites are met for video generation."""
|
||||
|
||||
@@ -109,6 +109,9 @@ class OrchestratorResult:
|
||||
dropbox_link: Optional[str] = None
|
||||
gdrive_files: Optional[Dict[str, str]] = field(default_factory=dict)
|
||||
|
||||
# Distribution warnings (non-fatal upload failures)
|
||||
distribution_warnings: List[str] = field(default_factory=list)
|
||||
|
||||
# Timing
|
||||
encoding_time_seconds: Optional[float] = None
|
||||
total_time_seconds: Optional[float] = None
|
||||
@@ -603,6 +606,7 @@ class VideoWorkerOrchestrator:
|
||||
|
||||
except Exception as e:
|
||||
self.job_log.error(f"YouTube upload failed: {e}")
|
||||
self.result.distribution_warnings.append(f"YouTube upload failed: {e}")
|
||||
# Don't fail the pipeline - YouTube is optional
|
||||
|
||||
async def _upload_to_dropbox(self):
|
||||
@@ -639,6 +643,7 @@ class VideoWorkerOrchestrator:
|
||||
|
||||
except Exception as e:
|
||||
self.job_log.error(f"Dropbox upload failed: {e}")
|
||||
self.result.distribution_warnings.append(f"Dropbox upload failed: {e}")
|
||||
# Don't fail the pipeline - Dropbox is optional
|
||||
|
||||
async def _upload_to_gdrive(self):
|
||||
@@ -675,6 +680,7 @@ class VideoWorkerOrchestrator:
|
||||
|
||||
except Exception as e:
|
||||
self.job_log.error(f"Google Drive upload failed: {e}")
|
||||
self.result.distribution_warnings.append(f"Google Drive upload failed: {e}")
|
||||
# Don't fail the pipeline - GDrive is optional
|
||||
|
||||
async def _run_notifications(self):
|
||||
|
||||
@@ -222,6 +222,18 @@ When deferring processing from one worker to another, you MUST update the job st
|
||||
### Google Drive Query Escaping
|
||||
Escape special chars in Google Drive API queries: `'` → `\'`, `\` → `\\\\`.
|
||||
|
||||
### Google Drive Stale Connection on Cloud Run (Feb 2026)
|
||||
**What happened**: Every GDrive upload silently failed for ~1 month (64 errors, 12+ jobs). Errors: `[Errno 32] Broken pipe` or `[SSL: UNEXPECTED_EOF_WHILE_READING]`. Jobs completed with `gdrive_files: {}` and no alert was sent.
|
||||
|
||||
**Root cause**: The singleton `GoogleDriveService` caches its HTTP connection. When Cloud Run containers sit idle between jobs, the underlying TCP connection goes stale. The next API call hits a dead socket. The orchestrator catches all exceptions in distribution and continues silently.
|
||||
|
||||
**Fix** (v0.119.7):
|
||||
1. Added tenacity retry with `_reset_service()` (sets `self._service = None` to force fresh connection) on transient errors (`BrokenPipeError`, `ConnectionResetError`, `ssl.SSLError`)
|
||||
2. Added `distribution_warnings` field to `OrchestratorResult` to track non-fatal upload failures
|
||||
3. Added Pushbullet notification when distribution uploads fail, so operator is alerted immediately
|
||||
|
||||
**Pattern**: Any singleton service that caches HTTP connections should have retry logic with connection reset for stale connection errors. Cloud Run containers can idle for minutes between requests, killing keep-alive connections. Also, never silently swallow distribution failures - always track and alert on them.
|
||||
|
||||
### FFmpeg Path Escaping
|
||||
For subprocess without shell, FFmpeg filter paths need: apostrophes escaped as `'\\''`, special chars as `\\[char]`.
|
||||
|
||||
|
||||
@@ -68,3 +68,27 @@ gcloud compute ssh encoding-worker --zone=us-central1-c --project=nomadkaraoke \
|
||||
gcloud compute ssh encoding-worker --zone=us-central1-c --project=nomadkaraoke \
|
||||
--command="sudo systemctl restart encoding-worker"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Google Drive uploads missing (gdrive_files empty)
|
||||
|
||||
**Symptoms:** Jobs complete successfully but `state_data.gdrive_files` is empty `{}`. Cloud Run logs show `BrokenPipeError` or `SSL: UNEXPECTED_EOF_WHILE_READING`.
|
||||
|
||||
**Cause:** Stale HTTP connections in the singleton `GoogleDriveService`. Since v0.119.7, this is handled automatically with retry + connection reset.
|
||||
|
||||
**Backfill affected jobs:**
|
||||
```bash
|
||||
# Dry run - list affected jobs
|
||||
GCS_BUCKET_NAME=karaoke-gen-storage-nomadkaraoke \
|
||||
GOOGLE_CLOUD_PROJECT=nomadkaraoke \
|
||||
python scripts/backfill_gdrive_uploads.py --all-missing --dry-run
|
||||
|
||||
# Run for real
|
||||
GCS_BUCKET_NAME=karaoke-gen-storage-nomadkaraoke \
|
||||
GOOGLE_CLOUD_PROJECT=nomadkaraoke \
|
||||
python scripts/backfill_gdrive_uploads.py --all-missing
|
||||
|
||||
# Specific jobs
|
||||
python scripts/backfill_gdrive_uploads.py --job-ids JOB1,JOB2
|
||||
```
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.119.7"
|
||||
version = "0.119.8"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
@@ -0,0 +1,303 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Backfill Google Drive uploads for jobs that completed with empty gdrive_files.
|
||||
|
||||
This addresses the issue where GDrive uploads silently failed due to stale
|
||||
HTTP connections (BrokenPipeError / SSLError) on idle Cloud Run containers.
|
||||
|
||||
Usage:
|
||||
# Dry run - show what would be uploaded
|
||||
python scripts/backfill_gdrive_uploads.py --dry-run --job-ids ca408e46,99baa198
|
||||
|
||||
# Upload specific jobs
|
||||
python scripts/backfill_gdrive_uploads.py --job-ids ca408e46,99baa198
|
||||
|
||||
# Find and upload all jobs with empty gdrive_files
|
||||
python scripts/backfill_gdrive_uploads.py --all-missing
|
||||
|
||||
# Override gdrive folder ID (instead of using DEFAULT_GDRIVE_FOLDER_ID env var)
|
||||
python scripts/backfill_gdrive_uploads.py --job-ids ca408e46 --gdrive-folder-id 1abc...
|
||||
|
||||
Requires:
|
||||
- google-cloud-firestore
|
||||
- google-cloud-storage
|
||||
- google-api-python-client, google-auth
|
||||
- GOOGLE_APPLICATION_CREDENTIALS or gcloud auth application-default login
|
||||
"""
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
# Add project root to path so we can import backend modules
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from google.cloud import firestore, storage
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(message)s",
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
GCS_BUCKET = "karaoke-gen-storage-nomadkaraoke"
|
||||
PROJECT = "nomadkaraoke"
|
||||
|
||||
# Default GDrive public share folder (from infrastructure/config.py)
|
||||
DEFAULT_GDRIVE_FOLDER_ID = "1laRKAyxo0v817SstfM5XkpbWiNKNAMSX"
|
||||
|
||||
# Known affected jobs from the incident
|
||||
KNOWN_AFFECTED_JOBS = [
|
||||
"ca408e46", "99baa198", "1e15d1dd", "ebba0996",
|
||||
"cb744c24", "1389f765", "fde3ae7d", "21f4b34f",
|
||||
"443c34a1", "56192d3a", "fdb4dc63", "4eeaa729",
|
||||
]
|
||||
|
||||
|
||||
def get_job(db: firestore.Client, job_id: str) -> dict | None:
|
||||
"""Fetch job document from Firestore."""
|
||||
doc = db.collection("jobs").document(job_id).get()
|
||||
if doc.exists:
|
||||
return doc.to_dict()
|
||||
return None
|
||||
|
||||
|
||||
def find_jobs_with_missing_gdrive(db: firestore.Client) -> list[dict]:
|
||||
"""Find completed jobs that have empty gdrive_files in state_data.
|
||||
|
||||
Looks for jobs that have a brand_code (indicating they went through
|
||||
distribution) but have empty or missing gdrive_files.
|
||||
"""
|
||||
jobs = []
|
||||
query = db.collection("jobs").where("status", "==", "complete")
|
||||
|
||||
for doc in query.stream():
|
||||
data = doc.to_dict()
|
||||
state_data = data.get("state_data", {})
|
||||
gdrive_files = state_data.get("gdrive_files", {})
|
||||
brand_code = state_data.get("brand_code")
|
||||
|
||||
# Job should have had GDrive uploads if it has a brand_code
|
||||
# but gdrive_files is empty/missing
|
||||
if brand_code and not gdrive_files:
|
||||
jobs.append({"job_id": doc.id, **data})
|
||||
|
||||
return jobs
|
||||
|
||||
|
||||
def _classify_blob(filename: str) -> str | None:
|
||||
"""Classify a GCS blob filename to an output_files key, or None to skip.
|
||||
|
||||
Only returns keys for the 3 file types that get uploaded to GDrive:
|
||||
lossy 4K MP4, lossy 720p MP4, and CDG ZIP.
|
||||
"""
|
||||
lower = filename.lower()
|
||||
if lower.endswith(".zip") and "cdg" in lower:
|
||||
return "final_karaoke_cdg_zip"
|
||||
if "720p" in lower and lower.endswith(".mp4"):
|
||||
return "final_karaoke_lossy_720p_mp4"
|
||||
if "lossy" in lower and lower.endswith(".mp4") and "720p" not in lower:
|
||||
return "final_karaoke_lossy_mp4"
|
||||
return None
|
||||
|
||||
|
||||
def download_finals(gcs_client: storage.Client, job_id: str, temp_dir: str) -> dict:
|
||||
"""Download only the final output files needed for GDrive upload.
|
||||
|
||||
Only downloads lossy 4K MP4, lossy 720p MP4, and CDG ZIP — skips
|
||||
lossless files, MKVs, karaoke video, etc. to save bandwidth.
|
||||
|
||||
Returns dict mapping output_files keys to local paths.
|
||||
"""
|
||||
bucket = gcs_client.bucket(GCS_BUCKET)
|
||||
prefix = f"jobs/{job_id}/finals/"
|
||||
blobs = list(bucket.list_blobs(prefix=prefix))
|
||||
|
||||
output_files = {}
|
||||
for blob in blobs:
|
||||
filename = os.path.basename(blob.name)
|
||||
if not filename:
|
||||
continue
|
||||
|
||||
key = _classify_blob(filename)
|
||||
if key is None:
|
||||
continue
|
||||
if key in output_files:
|
||||
continue # already have this type
|
||||
|
||||
local_path = os.path.join(temp_dir, filename)
|
||||
logger.info(f" Downloading {filename} ({blob.size / 1024 / 1024:.1f} MB)")
|
||||
blob.download_to_filename(local_path)
|
||||
output_files[key] = local_path
|
||||
|
||||
return output_files
|
||||
|
||||
|
||||
def list_finals(gcs_client: storage.Client, job_id: str) -> list[str]:
|
||||
"""List uploadable final output files in GCS without downloading."""
|
||||
bucket = gcs_client.bucket(GCS_BUCKET)
|
||||
prefix = f"jobs/{job_id}/finals/"
|
||||
blobs = list(bucket.list_blobs(prefix=prefix))
|
||||
return [
|
||||
f"{os.path.basename(b.name)} ({b.size / 1024 / 1024:.1f} MB)"
|
||||
for b in blobs
|
||||
if os.path.basename(b.name) and _classify_blob(os.path.basename(b.name))
|
||||
]
|
||||
|
||||
|
||||
def backfill_job(
|
||||
db: firestore.Client,
|
||||
gcs_client: storage.Client,
|
||||
job_id: str,
|
||||
gdrive_folder_id: str,
|
||||
dry_run: bool = False,
|
||||
) -> bool:
|
||||
"""Backfill GDrive upload for a single job.
|
||||
|
||||
Returns True if successful (or dry run).
|
||||
"""
|
||||
logger.info(f"Processing job {job_id}")
|
||||
|
||||
# Fetch job data
|
||||
job_data = get_job(db, job_id)
|
||||
if not job_data:
|
||||
logger.error(f" Job {job_id} not found in Firestore")
|
||||
return False
|
||||
|
||||
state_data = job_data.get("state_data", {})
|
||||
artist = job_data.get("artist", "Unknown")
|
||||
title = job_data.get("title", "Unknown")
|
||||
brand_code = state_data.get("brand_code")
|
||||
status = job_data.get("status")
|
||||
|
||||
logger.info(f" {artist} - {title} (brand: {brand_code}, status: {status})")
|
||||
|
||||
# Check existing gdrive_files
|
||||
existing = state_data.get("gdrive_files", {})
|
||||
if existing:
|
||||
logger.info(f" Already has gdrive_files: {existing}")
|
||||
logger.info(f" Skipping (already uploaded)")
|
||||
return True
|
||||
|
||||
if not brand_code:
|
||||
logger.warning(f" No brand code, skipping")
|
||||
return False
|
||||
|
||||
if dry_run:
|
||||
# List what files are available in GCS
|
||||
files = list_finals(gcs_client, job_id)
|
||||
if files:
|
||||
logger.info(f" [DRY RUN] Would upload {len(files)} files to GDrive folder {gdrive_folder_id}:")
|
||||
for f in files:
|
||||
logger.info(f" - {f}")
|
||||
else:
|
||||
logger.warning(f" [DRY RUN] No final files found in GCS")
|
||||
return True
|
||||
|
||||
# Download files and upload
|
||||
with tempfile.TemporaryDirectory(prefix=f"backfill-{job_id}-") as temp_dir:
|
||||
# Download finals from GCS
|
||||
logger.info(f" Downloading finals from GCS...")
|
||||
output_files = download_finals(gcs_client, job_id, temp_dir)
|
||||
|
||||
if not output_files:
|
||||
logger.warning(f" No final files found in GCS")
|
||||
return False
|
||||
|
||||
logger.info(f" Found {len(output_files)} files to upload")
|
||||
|
||||
# Upload to GDrive
|
||||
from backend.services.gdrive_service import GoogleDriveService
|
||||
|
||||
gdrive = GoogleDriveService()
|
||||
if not gdrive.is_configured:
|
||||
logger.error(f" Google Drive not configured")
|
||||
return False
|
||||
|
||||
base_name = f"{artist} - {title}"
|
||||
uploaded = gdrive.upload_to_public_share(
|
||||
root_folder_id=gdrive_folder_id,
|
||||
brand_code=brand_code,
|
||||
base_name=base_name,
|
||||
output_files=output_files,
|
||||
)
|
||||
|
||||
logger.info(f" Uploaded {len(uploaded)} files: {uploaded}")
|
||||
|
||||
# Update Firestore
|
||||
db.collection("jobs").document(job_id).update({
|
||||
"state_data.gdrive_files": uploaded,
|
||||
})
|
||||
logger.info(f" Updated Firestore state_data.gdrive_files")
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Backfill Google Drive uploads for jobs with empty gdrive_files"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--job-ids",
|
||||
help="Comma-separated list of job IDs to process",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--all-missing",
|
||||
action="store_true",
|
||||
help="Find and process all jobs with missing gdrive_files",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Show what would be done without making changes",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--gdrive-folder-id",
|
||||
default=os.getenv("DEFAULT_GDRIVE_FOLDER_ID", DEFAULT_GDRIVE_FOLDER_ID),
|
||||
help=f"Google Drive folder ID for public share (default: {DEFAULT_GDRIVE_FOLDER_ID})",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.job_ids and not args.all_missing:
|
||||
parser.error("Specify --job-ids or --all-missing")
|
||||
|
||||
db = firestore.Client(project=PROJECT)
|
||||
gcs_client = storage.Client(project=PROJECT)
|
||||
|
||||
if args.job_ids:
|
||||
job_ids = [j.strip() for j in args.job_ids.split(",")]
|
||||
elif args.all_missing:
|
||||
logger.info("Searching for jobs with missing GDrive uploads...")
|
||||
jobs = find_jobs_with_missing_gdrive(db)
|
||||
job_ids = [j["job_id"] for j in jobs]
|
||||
logger.info(f"Found {len(job_ids)} jobs with missing uploads")
|
||||
|
||||
if args.dry_run:
|
||||
logger.info("[DRY RUN MODE]")
|
||||
|
||||
logger.info(f"GDrive folder: {args.gdrive_folder_id}")
|
||||
|
||||
success_count = 0
|
||||
fail_count = 0
|
||||
|
||||
for job_id in job_ids:
|
||||
try:
|
||||
ok = backfill_job(
|
||||
db, gcs_client, job_id,
|
||||
gdrive_folder_id=args.gdrive_folder_id,
|
||||
dry_run=args.dry_run,
|
||||
)
|
||||
if ok:
|
||||
success_count += 1
|
||||
else:
|
||||
fail_count += 1
|
||||
except Exception as e:
|
||||
logger.error(f" Failed: {e}", exc_info=True)
|
||||
fail_count += 1
|
||||
|
||||
logger.info(f"\nDone: {success_count} succeeded, {fail_count} failed out of {len(job_ids)} jobs")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user