Files
karaoke-gen/backend/workers/youtube_queue_processor.py
T
Andrew Beveridge ☄️andClaude Opus 4.6 6ded014acf fix: use nested file_urls structure in queue processor (#457)
## Summary

- Fix queue processor to read video files from `finals.lossless_4k_mkv`
(nested dict) instead of `final_video_mkv` (flat key)
- Fix thumbnail path to use `screens.title_jpg` instead of flat
`title_jpg`
- Required for queuing the 5 jobs that missed YouTube uploads

## Changes
- `youtube_queue_processor.py` — Updated `_download_video_from_gcs` and
`_download_thumbnail_from_gcs` to use nested `file_urls` structure
- `test_youtube_queue_processor.py` — Updated test fixtures to match
nested structure

## Testing
- [x] 13 queue processor tests pass

@coderabbitai ignore

---
🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-03 21:43:28 +00:00

320 lines
11 KiB
Python

"""
YouTube upload queue processor.
Processes deferred YouTube uploads when quota is available.
Called by Cloud Scheduler via an internal endpoint (hourly).
"""
import logging
import os
import shutil
import tempfile
from typing import Dict, Any, Optional
from backend.config import get_settings
from backend.services.job_manager import JobManager
from backend.services.storage_service import StorageService
from backend.services.youtube_quota_service import get_youtube_quota_service
from backend.services.youtube_upload_queue_service import get_youtube_upload_queue_service
logger = logging.getLogger(__name__)
async def process_youtube_upload_queue() -> Dict[str, Any]:
"""
Process queued YouTube uploads.
Checks quota availability, then processes queued uploads one at a time.
Stops if quota is exhausted during processing.
Returns:
Summary dict with counts of processed, failed, and remaining items
"""
settings = get_settings()
quota_service = get_youtube_quota_service()
queue_service = get_youtube_upload_queue_service()
# Check if any quota is available
allowed, remaining, message = quota_service.check_quota_available()
if not allowed:
logger.info(f"YouTube queue processor: no quota available, skipping. {message}")
return {
"status": "skipped",
"reason": "no_quota",
"message": message,
"processed": 0,
"failed": 0,
"remaining": len(queue_service.get_queued_uploads()),
}
# Get queued uploads
queued = queue_service.get_queued_uploads(limit=20)
if not queued:
logger.info("YouTube queue processor: no uploads queued")
return {
"status": "empty",
"message": "No uploads queued",
"processed": 0,
"failed": 0,
"remaining": 0,
}
logger.info(f"YouTube queue processor: processing {len(queued)} queued uploads")
processed = 0
failed = 0
for entry in queued:
job_id = entry["job_id"]
# Re-check quota before each upload
allowed, remaining, message = quota_service.check_quota_available()
if not allowed:
logger.info(f"YouTube queue processor: quota exhausted after {processed} uploads")
break
# Claim the entry
if not queue_service.mark_processing(job_id):
logger.info(f"YouTube queue processor: could not claim job {job_id}, skipping")
continue
try:
youtube_url = await _process_single_upload(job_id, entry, quota_service, settings)
if youtube_url:
queue_service.mark_completed(job_id, youtube_url)
# Update job state_data with the YouTube URL
_update_job_youtube_url(job_id, youtube_url)
# Send follow-up email
await _send_youtube_upload_notification(job_id, entry, youtube_url)
processed += 1
else:
queue_service.mark_failed(job_id, "Upload returned no URL")
failed += 1
except Exception as e:
error_str = str(e)
logger.exception(f"YouTube queue processor: failed to process job {job_id}: {e}")
# If quota exceeded, stop processing entirely
if "quotaExceeded" in error_str:
queue_service.mark_failed(job_id, f"Quota exceeded: {error_str}")
logger.warning("YouTube queue processor: quota exceeded, stopping")
failed += 1
break
queue_service.mark_failed(job_id, error_str)
failed += 1
remaining_count = len(queue_service.get_queued_uploads())
logger.info(
f"YouTube queue processor: done. processed={processed} failed={failed} remaining={remaining_count}"
)
return {
"status": "processed",
"processed": processed,
"failed": failed,
"remaining": remaining_count,
}
async def _process_single_upload(
job_id: str,
entry: Dict[str, Any],
quota_service,
settings,
) -> Optional[str]:
"""
Process a single queued YouTube upload.
Downloads the video from GCS, uploads to YouTube, records quota.
Returns:
YouTube URL if successful, None otherwise
"""
job_manager = JobManager()
storage = StorageService()
job = job_manager.get_job(job_id)
if not job:
logger.error(f"YouTube queue processor: job {job_id} not found")
return None
# Create temp directory for the download
temp_dir = tempfile.mkdtemp(prefix=f"yt-queue-{job_id[:8]}-")
try:
# Find the video file in GCS (prefer MKV, then lossless MP4, then lossy)
video_path = _download_video_from_gcs(job_id, job, storage, temp_dir)
if not video_path:
logger.error(f"YouTube queue processor: no video file found for job {job_id}")
return None
# Download thumbnail if available
thumbnail_path = _download_thumbnail_from_gcs(job_id, job, storage, temp_dir)
# Build YouTube service with fresh credentials
youtube_service = _create_youtube_service(settings)
if not youtube_service:
logger.error("YouTube queue processor: failed to create YouTube service")
return None
# Build metadata
artist = entry.get("artist", job.artist or "Unknown")
title = entry.get("title", job.title or "Unknown")
youtube_title = f"{artist} - {title} (Karaoke)"
if len(youtube_title) > 95:
youtube_title = youtube_title[:92] + " ..."
description = settings.default_youtube_description or ""
brand_code = entry.get("brand_code") or job.state_data.get("brand_code")
if brand_code:
description = f"{description}\n\nBrand Code: {brand_code}".strip()
# Upload
video_id, video_url = youtube_service.upload_video(
video_path=video_path,
title=youtube_title,
description=description,
thumbnail_path=thumbnail_path,
tags=["karaoke", artist, title],
replace_existing=True,
)
if video_url:
# Record quota consumption
user_email = entry.get("user_email", "unknown")
quota_service.record_operation(job_id, user_email, "search.list")
quota_service.record_operation(job_id, user_email, "videos.insert")
if thumbnail_path:
quota_service.record_operation(job_id, user_email, "thumbnails.set")
logger.info(f"YouTube queue processor: uploaded job {job_id} -> {video_url}")
return video_url
return None
finally:
# Cleanup temp directory
if os.path.exists(temp_dir):
shutil.rmtree(temp_dir, ignore_errors=True)
def _download_video_from_gcs(
job_id: str, job, storage: StorageService, temp_dir: str
) -> Optional[str]:
"""Download the best available video file from GCS."""
file_urls = job.file_urls or {}
finals = file_urls.get("finals", {}) if isinstance(file_urls.get("finals"), dict) else {}
# Priority order: MKV (FLAC audio) > lossless MP4 > lossy 4K MP4 > lossy 720p MP4
candidates = [
(finals.get("lossless_4k_mkv"), "lossless_4k_mkv", ".mkv"),
(finals.get("lossless_4k_mp4"), "lossless_4k_mp4", ".mp4"),
(finals.get("lossy_4k_mp4"), "lossy_4k_mp4", ".mp4"),
(finals.get("lossy_720p_mp4"), "lossy_720p_mp4", ".mp4"),
]
for gcs_path, key, ext in candidates:
if gcs_path:
local_path = os.path.join(temp_dir, f"video{ext}")
try:
storage.download_file(gcs_path, local_path)
if os.path.isfile(local_path) and os.path.getsize(local_path) > 0:
logger.info(f"Downloaded {key} from GCS for job {job_id}")
return local_path
except Exception as e:
logger.warning(f"Failed to download {key} for job {job_id}: {e}")
return None
def _download_thumbnail_from_gcs(
job_id: str, job, storage: StorageService, temp_dir: str
) -> Optional[str]:
"""Download the thumbnail from GCS if available."""
file_urls = job.file_urls or {}
screens = file_urls.get("screens", {}) if isinstance(file_urls.get("screens"), dict) else {}
thumbnail_url = screens.get("title_jpg")
if not thumbnail_url:
return None
local_path = os.path.join(temp_dir, "thumbnail.jpg")
try:
storage.download_file(thumbnail_url, local_path)
if os.path.isfile(local_path) and os.path.getsize(local_path) > 0:
return local_path
except Exception as e:
logger.warning(f"Failed to download thumbnail for job {job_id}: {e}")
return None
def _create_youtube_service(settings):
"""Create a YouTube upload service with fresh credentials from Secret Manager."""
try:
from backend.services.youtube_upload_service import YouTubeUploadService
import json
youtube_creds_json = settings.get_secret("youtube-oauth-credentials")
if not youtube_creds_json:
logger.error("YouTube OAuth credentials not found in Secret Manager")
return None
credentials = json.loads(youtube_creds_json)
return YouTubeUploadService(
credentials=credentials,
non_interactive=True,
server_side_mode=True,
logger=logger,
)
except Exception as e:
logger.exception(f"Failed to create YouTube service: {e}")
return None
def _update_job_youtube_url(job_id: str, youtube_url: str) -> None:
"""Update job state_data with the YouTube URL after deferred upload."""
try:
job_manager = JobManager()
job = job_manager.get_job(job_id)
if job:
state_data = job.state_data or {}
state_data["youtube_url"] = youtube_url
state_data["youtube_upload_queued"] = False # No longer queued
job_manager.update_job(job_id, {"state_data": state_data})
logger.info(f"Updated job {job_id} state_data with YouTube URL")
except Exception as e:
logger.error(f"Failed to update job {job_id} with YouTube URL: {e}")
async def _send_youtube_upload_notification(
job_id: str, entry: Dict[str, Any], youtube_url: str
) -> None:
"""Send follow-up email notifying user their YouTube upload is complete."""
try:
from backend.services.job_notification_service import get_job_notification_service
notification_service = get_job_notification_service()
await notification_service.send_youtube_upload_complete_email(
job_id=job_id,
user_email=entry.get("user_email", ""),
artist=entry.get("artist", ""),
title=entry.get("title", ""),
youtube_url=youtube_url,
brand_code=entry.get("brand_code"),
)
except Exception as e:
logger.error(f"Failed to send YouTube upload notification for job {job_id}: {e}")
# Don't fail the upload over a notification error - mark it in the queue
try:
from backend.services.youtube_upload_queue_service import get_youtube_upload_queue_service
queue_service = get_youtube_upload_queue_service()
doc_ref = queue_service.db.collection("youtube_upload_queue").document(job_id)
doc_ref.update({"notification_sent": False, "notification_error": str(e)})
except Exception:
pass