mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
## Summary Follow-up to #1077. `video-encoding-job` runs both the post-review render (`render_video_worker`) and the final video (`video_worker`). In us-central1 each run waited **2–5 min** for Cloud Run Jobs to start it, so every customer job paid that twice. This moves it to us-east4 alongside the download, lyrics and bulk-search jobs. ## Cost (measured, not estimated) - Over the last 30 days the job read **156 GB** from Google services, mostly the us-central1 bucket, across 494 runs (Cloud Monitoring `container/network/received_bytes_count`, kind=google). That's about 0.32 GB per run. - Pricing from the Billing Catalog: GCS inter-region within North America is **free for the first 100 GB/month, then $0.02/GB**. Cloud Run writes to Google APIs are **$0**. - Internet egress (224 GB/month, Dropbox etc.) and Cloud Run CPU cost the same in either region. - Existing cross-region reads are about 24 GB/month (GPU job and lyrics), which the billing export shows at about $0. - **Net: about $1.60/month**, or about $3/month worst case. Time saved: roughly 4–8 min per customer job. ## Changes - **backend**: `trigger_video_worker` (via `_trigger_cloud_run_job`) and `trigger_render_video_worker` now use `cpu_jobs_region`. `GCP_REGION` stays us-central1; encoding VM and zone logic are unchanged. - **infra (Pulumi, already applied)**: new us-east4 `video-encoding-job`, with the legacy us-central1 copy kept for rollback and its deletion protection turned off. A preview after applying showed 290 unchanged. - **ci**: the video job image update now targets us-east4, using the us-east4 CPU image copy. - **Also updated**: rollback.sh, tests (video and render routing), TROUBLESHOOTING/ARCHITECTURE/LESSONS-LEARNED. ## Testing - `make test` passes: backend tests plus 1443 frontend tests. - Prod smoke test of the new us-east4 job: it imports `video_worker` and `render_video_worker` and calls the encoding VM's `/health` on its public IP (port 8080 is open to 0.0.0.0/0 with API-key auth). **Started in 25 s, exit 0.** 🤖 Generated with [Claude Code](https://claude.com/claude-code) <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit - **Updates** - Video encoding and render jobs now run in the `us-east4` region, alongside other CPU jobs. - Video encoding continues to access storage and encoding resources in `us-central1`; cross-region transfer may incur costs. <!-- end of auto-generated comment: release notes by coderabbit.ai --> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
714 lines
36 KiB
Python
714 lines
36 KiB
Python
"""
|
|
Cloud Run resources.
|
|
|
|
Manages Cloud Run domain mapping and Cloud Run Jobs.
|
|
"""
|
|
|
|
import pulumi
|
|
import pulumi_gcp as gcp
|
|
from pulumi_gcp import cloudrun, cloudrunv2
|
|
|
|
from config import PROJECT_ID, REGION
|
|
|
|
AUDIO_WORKER_GPU_REGION = "us-east4" # L4 GPU quota available here
|
|
|
|
# Region for the latency-critical CPU jobs (audio-download, lyrics-transcription,
|
|
# bulk-search, video-encoding). Cloud Run Jobs in us-central1 queue 2-5 min before the container
|
|
# starts, regardless of image size or CPU (measured 2026-09-28 with Google's tiny
|
|
# sample job image: us-central1 4-5 min, us-east4 8-19s). Must match the
|
|
# backend's CPU_JOBS_REGION default (backend/config.py). GCP_REGION inside these
|
|
# jobs stays REGION: the bucket, queues and other resources are still there.
|
|
CPU_JOBS_REGION = "us-east4"
|
|
|
|
|
|
def cpu_job_image(region: str) -> str:
|
|
"""CPU backend image for a job in ``region``, pulled from a same-region registry.
|
|
|
|
CI pushes the CPU app image to both us-central1 (karaoke-repo) and us-east4
|
|
(the existing karaoke-backend-gpu repo, as ``karaoke-backend-cpu``) so jobs
|
|
never pull the multi-GB image across regions (Artifact Registry egress).
|
|
"""
|
|
if region == REGION:
|
|
return f"{REGION}-docker.pkg.dev/{PROJECT_ID}/karaoke-repo/karaoke-backend:latest"
|
|
if region == AUDIO_WORKER_GPU_REGION:
|
|
return f"{region}-docker.pkg.dev/{PROJECT_ID}/karaoke-backend-gpu/karaoke-backend-cpu:latest"
|
|
raise ValueError(f"No CPU backend image registry configured for region {region}")
|
|
|
|
|
|
def _job_resource_name(base: str, region: str) -> str:
|
|
"""Pulumi logical name: unchanged for REGION, suffixed for other regions so the
|
|
us-east4 job can coexist with the legacy us-central1 one during cutover."""
|
|
return base if region == REGION else f"{base}-{region}"
|
|
|
|
|
|
def create_domain_mapping() -> cloudrun.DomainMapping:
|
|
"""
|
|
Create the Cloud Run domain mapping for api.nomadkaraoke.com.
|
|
|
|
Returns:
|
|
cloudrun.DomainMapping: The domain mapping resource.
|
|
"""
|
|
domain_mapping = cloudrun.DomainMapping(
|
|
"karaoke-backend-domain",
|
|
location=REGION,
|
|
name="api.nomadkaraoke.com",
|
|
metadata=cloudrun.DomainMappingMetadataArgs(
|
|
namespace=PROJECT_ID,
|
|
),
|
|
spec=cloudrun.DomainMappingSpecArgs(
|
|
route_name="karaoke-backend",
|
|
),
|
|
)
|
|
|
|
return domain_mapping
|
|
|
|
|
|
def create_lyrics_transcription_job(
|
|
bucket: gcp.storage.Bucket,
|
|
service_account: gcp.serviceaccount.Account,
|
|
region: str = REGION,
|
|
) -> cloudrunv2.Job:
|
|
"""
|
|
Create the Cloud Run Job for lyrics transcription.
|
|
|
|
Cloud Run Jobs run to completion without HTTP request lifecycle concerns,
|
|
avoiding the instance termination issue where Cloud Run would shut down
|
|
instances mid-processing when using BackgroundTasks (e.g., job c94cc9d6).
|
|
|
|
Typical duration: 5-15 minutes (AudioShake + agentic AI correction)
|
|
|
|
Args:
|
|
bucket: The GCS bucket for job artifacts.
|
|
service_account: The service account to run the job.
|
|
|
|
Returns:
|
|
cloudrunv2.Job: The Cloud Run Job resource.
|
|
"""
|
|
lyrics_transcription_job = cloudrunv2.Job(
|
|
_job_resource_name("lyrics-transcription-job", region),
|
|
name="lyrics-transcription-job",
|
|
location=region,
|
|
deletion_protection=False, # Allow the legacy-region copy to be removed
|
|
template=cloudrunv2.JobTemplateArgs(
|
|
template=cloudrunv2.JobTemplateTemplateArgs(
|
|
containers=[
|
|
cloudrunv2.JobTemplateTemplateContainerArgs(
|
|
image=cpu_job_image(region),
|
|
args=["python", "-m", "backend.workers.lyrics_worker"],
|
|
resources=cloudrunv2.JobTemplateTemplateContainerResourcesArgs(
|
|
limits={
|
|
"cpu": "2",
|
|
"memory": "4Gi",
|
|
},
|
|
),
|
|
envs=[
|
|
# Admin token for Cloud Tasks auth header
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="ADMIN_TOKENS",
|
|
value_source=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceArgs(
|
|
secret_key_ref=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceSecretKeyRefArgs(
|
|
secret=f"projects/{PROJECT_ID}/secrets/admin-tokens",
|
|
version="latest",
|
|
),
|
|
),
|
|
),
|
|
# AudioShake API token (required for transcription)
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="AUDIOSHAKE_API_TOKEN",
|
|
value_source=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceArgs(
|
|
secret_key_ref=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceSecretKeyRefArgs(
|
|
secret=f"projects/{PROJECT_ID}/secrets/audioshake-api-key",
|
|
version="latest",
|
|
),
|
|
),
|
|
),
|
|
# Cloud Run service URL for Cloud Tasks targeting
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="CLOUD_RUN_SERVICE_URL",
|
|
value="https://api.nomadkaraoke.com",
|
|
),
|
|
# Enable Cloud Tasks mode (vs direct HTTP)
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="ENABLE_CLOUD_TASKS",
|
|
value="true",
|
|
),
|
|
# Basic configuration
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="ENVIRONMENT",
|
|
value="production",
|
|
),
|
|
# Cap the duration-scaled transcription timeout below the
|
|
# task `timeout` above so the worker raises a clean
|
|
# RuntimeError before Cloud Run hard-kills the task.
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="TRANSCRIPTION_TIMEOUT_CAP_SECONDS",
|
|
value="2700",
|
|
),
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="GCP_REGION",
|
|
value=REGION,
|
|
),
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="GCS_BUCKET_NAME",
|
|
value=bucket.name,
|
|
),
|
|
# Genius API token (for lyrics fetching)
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="GENIUS_API_TOKEN",
|
|
value_source=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceArgs(
|
|
secret_key_ref=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceSecretKeyRefArgs(
|
|
secret=f"projects/{PROJECT_ID}/secrets/genius-api-key",
|
|
version="latest",
|
|
),
|
|
),
|
|
),
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="GOOGLE_CLOUD_PROJECT",
|
|
value=PROJECT_ID,
|
|
),
|
|
# RapidAPI key (for Musixmatch lyrics)
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="RAPIDAPI_KEY",
|
|
value_source=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceArgs(
|
|
secret_key_ref=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceSecretKeyRefArgs(
|
|
secret=f"projects/{PROJECT_ID}/secrets/rapidapi-key",
|
|
version="latest",
|
|
),
|
|
),
|
|
),
|
|
# Spotify cookie (for Spotify lyrics)
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="SPOTIFY_COOKIE_SP_DC",
|
|
value_source=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceArgs(
|
|
secret_key_ref=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceSecretKeyRefArgs(
|
|
secret=f"projects/{PROJECT_ID}/secrets/spotify-cookie",
|
|
version="latest",
|
|
),
|
|
),
|
|
),
|
|
],
|
|
)
|
|
],
|
|
service_account=service_account.email,
|
|
timeout="3000s", # 50 minutes max per task (long mashups can take 30+ min to transcribe)
|
|
max_retries=2,
|
|
),
|
|
),
|
|
)
|
|
|
|
return lyrics_transcription_job
|
|
|
|
|
|
def create_audio_separation_job(
|
|
bucket: gcp.storage.Bucket,
|
|
service_account: gcp.serviceaccount.Account,
|
|
) -> cloudrunv2.Job:
|
|
"""
|
|
Create the Cloud Run Job for audio separation with L4 GPU.
|
|
|
|
Runs the Separator class directly with GPU acceleration, eliminating
|
|
the HTTP call to the separate audio-separator service.
|
|
|
|
Deployed in us-east4 where L4 GPU quota is available.
|
|
|
|
Typical duration: 5-10 minutes (local GPU separation)
|
|
|
|
Args:
|
|
bucket: The GCS bucket for job artifacts.
|
|
service_account: The service account to run the job.
|
|
|
|
Returns:
|
|
cloudrunv2.Job: The Cloud Run Job resource.
|
|
"""
|
|
audio_separation_job = cloudrunv2.Job(
|
|
"audio-separation-job",
|
|
name="audio-separation-job",
|
|
location=AUDIO_WORKER_GPU_REGION,
|
|
deletion_protection=False, # Allow Pulumi to replace (region change requires delete+create)
|
|
template=cloudrunv2.JobTemplateArgs(
|
|
template=cloudrunv2.JobTemplateTemplateArgs(
|
|
containers=[
|
|
cloudrunv2.JobTemplateTemplateContainerArgs(
|
|
image=f"{AUDIO_WORKER_GPU_REGION}-docker.pkg.dev/{PROJECT_ID}/karaoke-backend-gpu/karaoke-backend:latest",
|
|
args=["python", "-m", "backend.workers.audio_worker"],
|
|
resources=cloudrunv2.JobTemplateTemplateContainerResourcesArgs(
|
|
limits={
|
|
"cpu": "4",
|
|
"memory": "16Gi",
|
|
"nvidia.com/gpu": "1",
|
|
},
|
|
),
|
|
envs=[
|
|
# Admin token for Cloud Tasks auth header
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="ADMIN_TOKENS",
|
|
value_source=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceArgs(
|
|
secret_key_ref=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceSecretKeyRefArgs(
|
|
secret=f"projects/{PROJECT_ID}/secrets/admin-tokens",
|
|
version="latest",
|
|
),
|
|
),
|
|
),
|
|
# Model directory (baked into GPU image, triggers local GPU mode)
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="MODEL_DIR",
|
|
value="/models",
|
|
),
|
|
# Cloud Run service URL for Cloud Tasks targeting
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="CLOUD_RUN_SERVICE_URL",
|
|
value="https://api.nomadkaraoke.com",
|
|
),
|
|
# Enable Cloud Tasks mode (vs direct HTTP)
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="ENABLE_CLOUD_TASKS",
|
|
value="true",
|
|
),
|
|
# Basic configuration
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="ENVIRONMENT",
|
|
value="production",
|
|
),
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="GCP_REGION",
|
|
value=AUDIO_WORKER_GPU_REGION,
|
|
),
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="GCS_BUCKET_NAME",
|
|
value=bucket.name,
|
|
),
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="GOOGLE_CLOUD_PROJECT",
|
|
value=PROJECT_ID,
|
|
),
|
|
],
|
|
)
|
|
],
|
|
node_selector=cloudrunv2.JobTemplateTemplateNodeSelectorArgs(
|
|
accelerator="nvidia-l4",
|
|
),
|
|
gpu_zonal_redundancy_disabled=True, # Required for GPU Jobs (capacity limitation)
|
|
service_account=service_account.email,
|
|
timeout="1800s", # 30 minutes (local GPU is faster than remote API)
|
|
max_retries=2,
|
|
),
|
|
),
|
|
)
|
|
|
|
return audio_separation_job
|
|
|
|
|
|
def create_audio_download_job(
|
|
bucket: gcp.storage.Bucket,
|
|
service_account: gcp.serviceaccount.Account,
|
|
region: str = REGION,
|
|
) -> cloudrunv2.Job:
|
|
"""
|
|
Create the Cloud Run Job for audio downloading.
|
|
|
|
Downloads audio from Spotify/YouTube/RED/OPS via the flacfetch service
|
|
(public HTTPS URL from the flacfetch-api-url secret), then triggers audio
|
|
separation and lyrics workers.
|
|
|
|
Typical duration: 30s-5 minutes (depending on source)
|
|
|
|
Args:
|
|
bucket: The GCS bucket for job artifacts.
|
|
service_account: The service account to run the job.
|
|
|
|
Returns:
|
|
cloudrunv2.Job: The Cloud Run Job resource.
|
|
"""
|
|
|
|
audio_download_job = cloudrunv2.Job(
|
|
_job_resource_name("audio-download-job", region),
|
|
name="audio-download-job",
|
|
location=region,
|
|
deletion_protection=False, # Allow the legacy-region copy to be removed
|
|
template=cloudrunv2.JobTemplateArgs(
|
|
template=cloudrunv2.JobTemplateTemplateArgs(
|
|
containers=[
|
|
cloudrunv2.JobTemplateTemplateContainerArgs(
|
|
image=cpu_job_image(region),
|
|
args=["python", "-m", "backend.workers.audio_download_worker"],
|
|
resources=cloudrunv2.JobTemplateTemplateContainerResourcesArgs(
|
|
limits={
|
|
"cpu": "1",
|
|
"memory": "1Gi",
|
|
},
|
|
),
|
|
envs=[
|
|
# Admin token for worker auth
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="ADMIN_TOKENS",
|
|
value_source=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceArgs(
|
|
secret_key_ref=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceSecretKeyRefArgs(
|
|
secret=f"projects/{PROJECT_ID}/secrets/admin-tokens",
|
|
version="latest",
|
|
),
|
|
),
|
|
),
|
|
# Cloud Run service URL for triggering workers
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="CLOUD_RUN_SERVICE_URL",
|
|
value="https://api.nomadkaraoke.com",
|
|
),
|
|
# Enable Cloud Tasks mode for worker triggers
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="ENABLE_CLOUD_TASKS",
|
|
value="true",
|
|
),
|
|
# Basic configuration
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="ENVIRONMENT",
|
|
value="production",
|
|
),
|
|
# Flacfetch API key
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="FLACFETCH_API_KEY",
|
|
value_source=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceArgs(
|
|
secret_key_ref=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceSecretKeyRefArgs(
|
|
secret=f"projects/{PROJECT_ID}/secrets/flacfetch-api-key",
|
|
version="latest",
|
|
),
|
|
),
|
|
),
|
|
# Flacfetch API URL (public HTTPS endpoint; no VPC egress)
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="FLACFETCH_API_URL",
|
|
value_source=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceArgs(
|
|
secret_key_ref=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceSecretKeyRefArgs(
|
|
secret=f"projects/{PROJECT_ID}/secrets/flacfetch-api-url",
|
|
version="latest",
|
|
),
|
|
),
|
|
),
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="GCP_REGION",
|
|
value=REGION,
|
|
),
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="GCS_BUCKET_NAME",
|
|
value=bucket.name,
|
|
),
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="GOOGLE_CLOUD_PROJECT",
|
|
value=PROJECT_ID,
|
|
),
|
|
# Email provider (Postmark). The worker's duration
|
|
# reconcile can send an over-limit cancellation email
|
|
# to the customer; without this the EmailService "fail
|
|
# loudly" guard raised in production and crashed the
|
|
# whole download (job a453d1d5).
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="POSTMARK_SERVER_TOKEN",
|
|
value_source=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceArgs(
|
|
secret_key_ref=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceSecretKeyRefArgs(
|
|
secret=f"projects/{PROJECT_ID}/secrets/postmark-server-token",
|
|
version="latest",
|
|
),
|
|
),
|
|
),
|
|
],
|
|
)
|
|
],
|
|
service_account=service_account.email,
|
|
# 20 min max. Slow torrent (RED/OPS) downloads can exceed 10 min;
|
|
# a SIGKILL at the old 600s bypassed graceful failure handling and
|
|
# left in-flight auto-retries racing manual retries (job 1cd29294).
|
|
# The worker's DOWNLOAD_WAIT_TIMEOUT_SECONDS (1080s) is kept
|
|
# strictly below this so it fails cleanly before Cloud Run kills it.
|
|
timeout="1200s",
|
|
max_retries=2,
|
|
),
|
|
),
|
|
)
|
|
|
|
return audio_download_job
|
|
|
|
|
|
def create_bulk_search_job(
|
|
bucket: gcp.storage.Bucket,
|
|
service_account: gcp.serviceaccount.Account,
|
|
region: str = REGION,
|
|
) -> cloudrunv2.Job:
|
|
"""
|
|
Create the Cloud Run Job for Bulk Mode batch audio search.
|
|
|
|
Processes every job in a batch (state_data.batch_id): runs each audio search
|
|
via flacfetch, auto-selects confident lossless matches (triggering the
|
|
audio-download-job), and parks the rest in AWAITING_AUDIO_SELECTION.
|
|
|
|
Longer timeout than the download
|
|
job because a batch can contain up to 100 songs. Idempotent across retries
|
|
(only re-processes jobs still flagged bulk_pending_search), so retries=1.
|
|
"""
|
|
|
|
bulk_search_job = cloudrunv2.Job(
|
|
_job_resource_name("bulk-search-job", region),
|
|
name="bulk-search-job",
|
|
location=region,
|
|
deletion_protection=False, # Allow the legacy-region copy to be removed
|
|
template=cloudrunv2.JobTemplateArgs(
|
|
template=cloudrunv2.JobTemplateTemplateArgs(
|
|
containers=[
|
|
cloudrunv2.JobTemplateTemplateContainerArgs(
|
|
image=cpu_job_image(region),
|
|
args=["python", "-m", "backend.workers.bulk_search_worker"],
|
|
resources=cloudrunv2.JobTemplateTemplateContainerResourcesArgs(
|
|
limits={
|
|
"cpu": "1",
|
|
"memory": "1Gi",
|
|
},
|
|
),
|
|
envs=[
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="ADMIN_TOKENS",
|
|
value_source=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceArgs(
|
|
secret_key_ref=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceSecretKeyRefArgs(
|
|
secret=f"projects/{PROJECT_ID}/secrets/admin-tokens",
|
|
version="latest",
|
|
),
|
|
),
|
|
),
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="CLOUD_RUN_SERVICE_URL",
|
|
value="https://api.nomadkaraoke.com",
|
|
),
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="ENABLE_CLOUD_TASKS",
|
|
value="true",
|
|
),
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="ENVIRONMENT",
|
|
value="production",
|
|
),
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="FLACFETCH_API_KEY",
|
|
value_source=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceArgs(
|
|
secret_key_ref=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceSecretKeyRefArgs(
|
|
secret=f"projects/{PROJECT_ID}/secrets/flacfetch-api-key",
|
|
version="latest",
|
|
),
|
|
),
|
|
),
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="FLACFETCH_API_URL",
|
|
value_source=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceArgs(
|
|
secret_key_ref=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceSecretKeyRefArgs(
|
|
secret=f"projects/{PROJECT_ID}/secrets/flacfetch-api-url",
|
|
version="latest",
|
|
),
|
|
),
|
|
),
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="GCP_REGION",
|
|
value=REGION,
|
|
),
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="GCS_BUCKET_NAME",
|
|
value=bucket.name,
|
|
),
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="GOOGLE_CLOUD_PROJECT",
|
|
value=PROJECT_ID,
|
|
),
|
|
],
|
|
)
|
|
],
|
|
service_account=service_account.email,
|
|
timeout="3600s", # up to 100 songs per batch
|
|
max_retries=1,
|
|
),
|
|
),
|
|
)
|
|
|
|
return bulk_search_job
|
|
|
|
|
|
def create_video_encoding_job(
|
|
bucket: gcp.storage.Bucket,
|
|
service_account: gcp.serviceaccount.Account,
|
|
region: str = REGION,
|
|
) -> cloudrunv2.Job:
|
|
"""
|
|
Create the Cloud Run Job for video encoding.
|
|
|
|
Cloud Run Jobs run to completion (up to 1 hour), immune to Cloud Run Service
|
|
deployment rollouts. This prevents the BackgroundTask termination issue where
|
|
a deployment kills the encoding orchestrator mid-flight (see incident 2026-03-08).
|
|
|
|
The video worker orchestrates: CDG/TXT packaging, GCE encoding submission/polling,
|
|
file download from GCS, YouTube/Dropbox/GDrive upload, and Discord notification.
|
|
It needs the same env vars and secrets as the Cloud Run Service for these features.
|
|
|
|
Args:
|
|
bucket: The GCS bucket for job artifacts.
|
|
service_account: The service account to run the job.
|
|
|
|
Returns:
|
|
cloudrunv2.Job: The Cloud Run Job resource.
|
|
"""
|
|
video_encoding_job = cloudrunv2.Job(
|
|
_job_resource_name("video-encoding-job", region),
|
|
name="video-encoding-job",
|
|
location=region,
|
|
deletion_protection=False, # Allow the legacy-region copy to be removed
|
|
template=cloudrunv2.JobTemplateArgs(
|
|
template=cloudrunv2.JobTemplateTemplateArgs(
|
|
containers=[
|
|
cloudrunv2.JobTemplateTemplateContainerArgs(
|
|
image=cpu_job_image(region),
|
|
args=["python", "-m", "backend.workers.video_worker"],
|
|
resources=cloudrunv2.JobTemplateTemplateContainerResourcesArgs(
|
|
limits={
|
|
"cpu": "4",
|
|
"memory": "8Gi",
|
|
},
|
|
),
|
|
envs=[
|
|
# Admin token for worker-to-service auth
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="ADMIN_TOKENS",
|
|
value_source=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceArgs(
|
|
secret_key_ref=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceSecretKeyRefArgs(
|
|
secret=f"projects/{PROJECT_ID}/secrets/admin-tokens",
|
|
version="latest",
|
|
),
|
|
),
|
|
),
|
|
# Cloud Run service URL for triggering downstream workers
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="CLOUD_RUN_SERVICE_URL",
|
|
value="https://api.nomadkaraoke.com",
|
|
),
|
|
# Distribution defaults
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="DEFAULT_BRAND_PREFIX",
|
|
value="NOMAD",
|
|
),
|
|
# Discord webhook for release notifications
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="DEFAULT_DISCORD_WEBHOOK_URL",
|
|
value_source=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceArgs(
|
|
secret_key_ref=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceSecretKeyRefArgs(
|
|
secret=f"projects/{PROJECT_ID}/secrets/discord-releases-webhook",
|
|
version="latest",
|
|
),
|
|
),
|
|
),
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="DEFAULT_DROPBOX_PATH",
|
|
value="/MediaUnsynced/Karaoke/Tracks-Organized",
|
|
),
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="DEFAULT_ENABLE_YOUTUBE_UPLOAD",
|
|
value="true",
|
|
),
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="DEFAULT_GDRIVE_FOLDER_ID",
|
|
value="1laRKAyxo0v817SstfM5XkpbWiNKNAMSX",
|
|
),
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="DEFAULT_PRIVATE_BRAND_PREFIX",
|
|
value="NOMADNP",
|
|
),
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="DEFAULT_PRIVATE_DROPBOX_PATH",
|
|
value="/MediaUnsynced/Karaoke/Tracks-NonPublished",
|
|
),
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="ENABLE_CLOUD_TASKS",
|
|
value="true",
|
|
),
|
|
# GCE encoding worker
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="ENCODING_WORKER_API_KEY",
|
|
value_source=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceArgs(
|
|
secret_key_ref=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceSecretKeyRefArgs(
|
|
secret=f"projects/{PROJECT_ID}/secrets/encoding-worker-api-key",
|
|
version="latest",
|
|
),
|
|
),
|
|
),
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="ENCODING_WORKER_URL",
|
|
value_source=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceArgs(
|
|
secret_key_ref=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceSecretKeyRefArgs(
|
|
secret=f"projects/{PROJECT_ID}/secrets/encoding-worker-url",
|
|
version="latest",
|
|
),
|
|
),
|
|
),
|
|
# Multi-zone capacity-fallback VMs (JSON list).
|
|
# Without this, the encoding worker manager only
|
|
# tries the primary zone — when -c is exhausted
|
|
# the job fails. Cloud Run Service has the same
|
|
# secret mounted; the Cloud Run Job needed it too.
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="ENCODING_WORKER_FALLBACK_VMS",
|
|
value_source=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceArgs(
|
|
secret_key_ref=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceSecretKeyRefArgs(
|
|
secret=f"projects/{PROJECT_ID}/secrets/encoding-worker-fallback-vms",
|
|
version="latest",
|
|
),
|
|
),
|
|
),
|
|
# Core configuration
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="ENVIRONMENT",
|
|
value="production",
|
|
),
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="GCP_REGION",
|
|
value=REGION,
|
|
),
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="GCS_BUCKET_NAME",
|
|
value=bucket.name,
|
|
),
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="GOOGLE_CLOUD_PROJECT",
|
|
value=PROJECT_ID,
|
|
),
|
|
# Email provider for completion emails (Postmark)
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="POSTMARK_SERVER_TOKEN",
|
|
value_source=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceArgs(
|
|
secret_key_ref=cloudrunv2.JobTemplateTemplateContainerEnvValueSourceSecretKeyRefArgs(
|
|
secret=f"projects/{PROJECT_ID}/secrets/postmark-server-token",
|
|
version="latest",
|
|
),
|
|
),
|
|
),
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="USE_GCE_ENCODING",
|
|
value="true",
|
|
),
|
|
# The render worker also runs in this job (args
|
|
# override → backend.workers.render_video_worker)
|
|
# and triggers the video worker in-process when the
|
|
# render completes. Without these flags that
|
|
# downstream trigger would fall back to the
|
|
# deploy-vulnerable Cloud Tasks → BackgroundTask
|
|
# path this job exists to avoid.
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="USE_CLOUD_RUN_JOBS_FOR_VIDEO",
|
|
value="true",
|
|
),
|
|
cloudrunv2.JobTemplateTemplateContainerEnvArgs(
|
|
name="USE_CLOUD_RUN_JOBS_FOR_RENDER",
|
|
value="true",
|
|
),
|
|
],
|
|
)
|
|
],
|
|
service_account=service_account.email,
|
|
timeout="3600s", # 1 hour max per task
|
|
max_retries=2,
|
|
),
|
|
),
|
|
)
|
|
|
|
return video_encoding_job
|