mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
## Summary - Replace hard-coded 10 uploads/day limit with YouTube API quota unit tracking (10,000 units/day, ~33 uploads) - Queue uploads when quota is exhausted instead of silently skipping them - Process queued uploads hourly via Cloud Scheduler, notify users via email when complete - Add admin dashboard for quota monitoring and queue management ## Changes ### Backend - New Services - `youtube_quota_service.py` — Tracks YouTube API quota units per day in Firestore (`youtube_quota` collection, date-keyed by Pacific Time) - `youtube_upload_queue_service.py` — Manages deferred upload queue in Firestore (`youtube_upload_queue` collection) with atomic claiming, retry logic, and failure handling - `youtube_queue_processor.py` — Worker that processes queued uploads when quota available (downloads from GCS, uploads to YouTube, sends notification) ### Backend - Modified - `video_worker_orchestrator.py` — Queue on quota exhaustion instead of skip; record quota units after each API call - `rate_limit_service.py` — Deprecated YouTube methods delegate to new quota service with legacy fallback - `job_notification_service.py` + `template_service.py` + `email_service.py` — Follow-up email when deferred upload completes - `rate_limits.py` — 3 new admin endpoints: GET queue, POST retry, POST process - `internal.py` — Internal endpoint for Cloud Scheduler to trigger queue processing - `config.py` — New settings: `youtube_quota_daily_limit`, `youtube_quota_upload_cost`, `youtube_quota_safety_margin` ### Frontend - `rate-limits/page.tsx` — YouTube Queue tab with quota progress bar, queued uploads table, retry/process buttons - `api.ts` — Types and API methods for queue management ### Infrastructure (Pulumi) - Cloud Tasks queue `youtube-upload-queue` - Cloud Scheduler job: hourly, Pacific Time, calls internal process endpoint ### Tests (90 passing) - `test_youtube_quota_service.py` — 16 tests - `test_youtube_upload_queue_service.py` — 15 tests - `test_youtube_queue_processor.py` — 14 tests (async) - `test_rate_limits_api.py` — 4 new endpoint tests - `test_rate_limit_service.py` — 5 updated/new delegation tests ### Docs - Updated ARCHITECTURE.md, API.md, README.md, LESSONS-LEARNED.md ## Motivation 4 jobs on 2026-03-03 had YouTube uploads silently skipped because the hard-coded rate limit of 10 uploads/day was reached. Users expected uploads but got none with no notification. The actual YouTube API quota (10,000 units/day) supports ~33 uploads — the rate limit was overly conservative. ## Testing - [x] 90 targeted backend tests pass (4.03s) - [x] Frontend Jest tests pass (503 tests) - [x] CodeRabbit CLI review completed, feedback addressed @coderabbitai ignore --- 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
171 lines
6.2 KiB
Python
171 lines
6.2 KiB
Python
"""
|
|
Cloud Tasks resources.
|
|
|
|
Manages task queues for asynchronous worker processing.
|
|
"""
|
|
|
|
import pulumi
|
|
import pulumi_gcp as gcp
|
|
from pulumi_gcp import cloudtasks
|
|
|
|
from config import REGION, QueueConfigs
|
|
|
|
|
|
def create_queues() -> dict[str, cloudtasks.Queue]:
|
|
"""
|
|
Create all Cloud Tasks queues.
|
|
|
|
These queues provide guaranteed delivery and horizontal scaling for worker tasks.
|
|
Workers are triggered via Cloud Tasks instead of BackgroundTasks, ensuring:
|
|
- Tasks survive container restarts
|
|
- Each task gets dedicated resources
|
|
- Automatic retries on failure
|
|
- Rate limiting to protect external APIs
|
|
|
|
Returns:
|
|
dict: Dictionary mapping queue names to Queue resources.
|
|
"""
|
|
queues = {}
|
|
|
|
# Audio worker queue - calls Modal API for audio separation
|
|
queues["audio-worker-queue"] = cloudtasks.Queue(
|
|
"audio-worker-queue",
|
|
name="audio-worker-queue",
|
|
location=REGION,
|
|
rate_limits=cloudtasks.QueueRateLimitsArgs(
|
|
max_dispatches_per_second=QueueConfigs.Audio.MAX_DISPATCHES_PER_SECOND,
|
|
max_concurrent_dispatches=QueueConfigs.Audio.MAX_CONCURRENT_DISPATCHES,
|
|
),
|
|
retry_config=cloudtasks.QueueRetryConfigArgs(
|
|
max_attempts=3,
|
|
min_backoff="10s",
|
|
max_backoff="300s",
|
|
max_retry_duration=QueueConfigs.Audio.MAX_RETRY_DURATION,
|
|
),
|
|
)
|
|
|
|
# Lyrics worker queue - calls AudioShake API for transcription
|
|
queues["lyrics-worker-queue"] = cloudtasks.Queue(
|
|
"lyrics-worker-queue",
|
|
name="lyrics-worker-queue",
|
|
location=REGION,
|
|
rate_limits=cloudtasks.QueueRateLimitsArgs(
|
|
max_dispatches_per_second=QueueConfigs.Lyrics.MAX_DISPATCHES_PER_SECOND,
|
|
max_concurrent_dispatches=QueueConfigs.Lyrics.MAX_CONCURRENT_DISPATCHES,
|
|
),
|
|
retry_config=cloudtasks.QueueRetryConfigArgs(
|
|
max_attempts=3,
|
|
min_backoff="10s",
|
|
max_backoff="300s",
|
|
max_retry_duration=QueueConfigs.Lyrics.MAX_RETRY_DURATION,
|
|
),
|
|
)
|
|
|
|
# Screens worker queue - generates title/end screens (fast, CPU-light)
|
|
queues["screens-worker-queue"] = cloudtasks.Queue(
|
|
"screens-worker-queue",
|
|
name="screens-worker-queue",
|
|
location=REGION,
|
|
rate_limits=cloudtasks.QueueRateLimitsArgs(
|
|
max_dispatches_per_second=QueueConfigs.Screens.MAX_DISPATCHES_PER_SECOND,
|
|
max_concurrent_dispatches=QueueConfigs.Screens.MAX_CONCURRENT_DISPATCHES,
|
|
),
|
|
retry_config=cloudtasks.QueueRetryConfigArgs(
|
|
max_attempts=3,
|
|
min_backoff="5s",
|
|
max_backoff="60s",
|
|
max_retry_duration=QueueConfigs.Screens.MAX_RETRY_DURATION,
|
|
),
|
|
)
|
|
|
|
# Render worker queue - LyricsTranscriber + FFmpeg (CPU-intensive)
|
|
queues["render-worker-queue"] = cloudtasks.Queue(
|
|
"render-worker-queue",
|
|
name="render-worker-queue",
|
|
location=REGION,
|
|
rate_limits=cloudtasks.QueueRateLimitsArgs(
|
|
max_dispatches_per_second=QueueConfigs.Render.MAX_DISPATCHES_PER_SECOND,
|
|
max_concurrent_dispatches=QueueConfigs.Render.MAX_CONCURRENT_DISPATCHES,
|
|
),
|
|
retry_config=cloudtasks.QueueRetryConfigArgs(
|
|
max_attempts=2,
|
|
min_backoff="30s",
|
|
max_backoff="300s",
|
|
max_retry_duration=QueueConfigs.Render.MAX_RETRY_DURATION,
|
|
),
|
|
)
|
|
|
|
# Video worker queue - final encoding (very CPU-intensive, longest running)
|
|
queues["video-worker-queue"] = cloudtasks.Queue(
|
|
"video-worker-queue",
|
|
name="video-worker-queue",
|
|
location=REGION,
|
|
rate_limits=cloudtasks.QueueRateLimitsArgs(
|
|
max_dispatches_per_second=QueueConfigs.Video.MAX_DISPATCHES_PER_SECOND,
|
|
max_concurrent_dispatches=QueueConfigs.Video.MAX_CONCURRENT_DISPATCHES,
|
|
),
|
|
retry_config=cloudtasks.QueueRetryConfigArgs(
|
|
max_attempts=2,
|
|
min_backoff="60s",
|
|
max_backoff="600s",
|
|
max_retry_duration=QueueConfigs.Video.MAX_RETRY_DURATION,
|
|
),
|
|
)
|
|
|
|
# Idle reminder queue - delayed tasks for sending reminder emails
|
|
# Tasks are scheduled with a 5-minute delay when jobs enter blocking states
|
|
queues["idle-reminder-queue"] = cloudtasks.Queue(
|
|
"idle-reminder-queue",
|
|
name="idle-reminder-queue",
|
|
location=REGION,
|
|
rate_limits=cloudtasks.QueueRateLimitsArgs(
|
|
max_dispatches_per_second=10, # Email sending is fast
|
|
max_concurrent_dispatches=50,
|
|
),
|
|
retry_config=cloudtasks.QueueRetryConfigArgs(
|
|
max_attempts=3,
|
|
min_backoff="10s",
|
|
max_backoff="60s",
|
|
max_retry_duration="600s", # 10 min total
|
|
),
|
|
)
|
|
|
|
# YouTube upload queue - deferred uploads when quota is exhausted
|
|
# Processor runs hourly via Cloud Scheduler, processing one upload at a time
|
|
queues["youtube-upload-queue"] = cloudtasks.Queue(
|
|
"youtube-upload-queue",
|
|
name="youtube-upload-queue",
|
|
location=REGION,
|
|
rate_limits=cloudtasks.QueueRateLimitsArgs(
|
|
max_dispatches_per_second=1, # One upload at a time
|
|
max_concurrent_dispatches=1,
|
|
),
|
|
retry_config=cloudtasks.QueueRetryConfigArgs(
|
|
max_attempts=3,
|
|
min_backoff="60s",
|
|
max_backoff="600s",
|
|
max_retry_duration="3600s", # 1 hour total
|
|
),
|
|
)
|
|
|
|
# GDrive validation queue - delayed tasks for post-job validation
|
|
# Tasks are scheduled with a 5-minute delay after job completion to allow
|
|
# E2E test cleanup to finish before the validator checks for sequence gaps
|
|
queues["gdrive-validation-queue"] = cloudtasks.Queue(
|
|
"gdrive-validation-queue",
|
|
name="gdrive-validation-queue",
|
|
location=REGION,
|
|
rate_limits=cloudtasks.QueueRateLimitsArgs(
|
|
max_dispatches_per_second=1, # Low rate - validation is infrequent
|
|
max_concurrent_dispatches=1, # Only one validation at a time
|
|
),
|
|
retry_config=cloudtasks.QueueRetryConfigArgs(
|
|
max_attempts=2,
|
|
min_backoff="30s",
|
|
max_backoff="120s",
|
|
max_retry_duration="300s", # 5 min total
|
|
),
|
|
)
|
|
|
|
return queues
|