diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d7270dbc..1ea19db4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2152,16 +2152,15 @@ jobs: # daily E2E run and the codified `scripts/rollback.sh` (R2) for recovery # if a bad image reaches them. run: | - IMAGE="${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.APP_IMAGE_NAME }}:${{ github.sha }}" GPU_IMAGE="${{ env.GPU_REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.GPU_REPOSITORY }}/${{ env.GPU_APP_IMAGE_NAME }}:${{ github.sha }}" CPU_JOBS_IMAGE="${{ env.CPU_JOBS_REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.CPU_JOBS_REPOSITORY }}/${{ env.CPU_JOBS_IMAGE_NAME }}:${{ github.sha }}" echo "📦 Updating Cloud Run Jobs..." - # Video encoding job (CPU, us-central1) + # Video encoding job (CPU, us-east4; also runs render_video_worker) gcloud run jobs update video-encoding-job \ - --image "$IMAGE" \ - --region us-central1 \ + --image "$CPU_JOBS_IMAGE" \ + --region us-east4 \ --project ${{ env.PROJECT_ID }} \ --quiet diff --git a/backend/config.py b/backend/config.py index 4260cf70..0fdaf966 100644 --- a/backend/config.py +++ b/backend/config.py @@ -206,12 +206,14 @@ class Settings(BaseSettings): # gcp_region 404s from those workers ("Queue does not exist"). cloud_tasks_region: str = os.getenv("CLOUD_TASKS_REGION", "us-central1") # Region of the latency-critical CPU Cloud Run Jobs (audio-download-job, - # lyrics-transcription-job, bulk-search-job). Cloud Run Jobs in us-central1 - # queue 2-5 min before the container starts (measured 2026-09-28 with a tiny - # sample image: us-central1 4-5 min vs us-east4 8-19s), independent of image - # size or CPU. Rollback: set CPU_JOBS_REGION=us-central1 on the service AND - # the jobs (the download worker triggers the lyrics job). Deliberately - # separate from gcp_region, which these workers use for other resources. + # lyrics-transcription-job, bulk-search-job, video-encoding-job). Cloud Run + # Jobs in us-central1 queue 2-5 min before the container starts (measured + # 2026-09-28 with a tiny sample image: us-central1 4-5 min vs us-east4 8-19s), + # independent of image size or CPU. video-encoding-job reads ~0.3 GB/run from + # the us-central1 bucket cross-region (~$1.60/mo, 2026-09-29 model). Rollback: + # set CPU_JOBS_REGION=us-central1 on the service AND the jobs (the download + # worker triggers the lyrics job). Deliberately separate from gcp_region, + # which these workers use for other resources (queues, encoding VM zones). cpu_jobs_region: str = os.getenv("CPU_JOBS_REGION", "us-east4") # Cloud Run Jobs (for long-running video encoding) diff --git a/backend/services/worker_service.py b/backend/services/worker_service.py index 3a1cda23..415a7372 100644 --- a/backend/services/worker_service.py +++ b/backend/services/worker_service.py @@ -590,6 +590,7 @@ class WorkerService: job_id=job_id, cloud_run_job_name="video-encoding-job", worker_module="video_worker", + location=self.settings.cpu_jobs_region, ) async def _run_job_with_retry(self, client, request, log_prefix: str): @@ -739,6 +740,7 @@ class WorkerService: job_id=job_id, cloud_run_job_name="video-encoding-job", worker_module="render_video_worker", + location=self.settings.cpu_jobs_region, ) return await self.trigger_worker("render-video", job_id) diff --git a/backend/tests/test_worker_service_cpu_jobs_region.py b/backend/tests/test_worker_service_cpu_jobs_region.py index 3035d18c..4acf56ab 100644 --- a/backend/tests/test_worker_service_cpu_jobs_region.py +++ b/backend/tests/test_worker_service_cpu_jobs_region.py @@ -2,10 +2,9 @@ Cloud Run Jobs in us-central1 queue 2-5 min before the container starts, even for a tiny sample image (measured 2026-09-28: us-central1 4-5 min, us-east4 8-19s). -audio-download-job, lyrics-transcription-job and bulk-search-job therefore run in -us-east4, while the jobs and workers keep GCP_REGION=us-central1 for everything -else. video-encoding-job stays in gcp_region: it moves multi-GB outputs through -the us-central1 bucket, so a cross-region copy would add GCS egress. +audio-download-job, lyrics-transcription-job, bulk-search-job and +video-encoding-job (video + post-review render workers) therefore run in us-east4, +while the jobs and workers keep GCP_REGION=us-central1 for everything else. """ from __future__ import annotations @@ -68,9 +67,23 @@ async def test_cpu_jobs_region_env_override_for_rollback(monkeypatch) -> None: @pytest.mark.asyncio -async def test_video_encoding_job_stays_in_gcp_region(monkeypatch) -> None: +async def test_video_encoding_job_triggered_in_us_east4(monkeypatch) -> None: name = await _triggered_job_name(_settings(monkeypatch), lambda s: s._trigger_cloud_run_job("job1")) - assert name == "projects/test-project/locations/us-central1/jobs/video-encoding-job" + assert name == "projects/test-project/locations/us-east4/jobs/video-encoding-job" + + +@pytest.mark.asyncio +async def test_render_video_job_triggered_in_us_east4(monkeypatch) -> None: + monkeypatch.setenv("USE_CLOUD_RUN_JOBS_FOR_RENDER", "true") + settings = _settings(monkeypatch) + + async def trigger(service): + with patch.object(service, "_bump_worker_generation"), \ + patch.object(service, "_start_encoding_worker_warmup"): + return await service.trigger_render_video_worker("job1") + + name = await _triggered_job_name(settings, trigger) + assert name == "projects/test-project/locations/us-east4/jobs/video-encoding-job" def test_gcp_region_unaffected_by_cpu_jobs_region(monkeypatch) -> None: diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index ae27ff88..3a0e2e5a 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -539,7 +539,7 @@ Audio and lyrics workers run as **Cloud Run Jobs** - standalone batch containers │ lyrics-transcription-job - 5-15 min (AudioShake + correction, us-east4)│ │ bulk-search-job - batch audio search (us-east4) │ │ audio-separation-job - 10-20 min (L4 GPU, us-east4, direct) │ -│ video-encoding-job - up to 60 min (us-central1) │ +│ video-encoding-job - up to 60 min (us-east4) │ │ │ │ Triggered via: google.cloud.run_v2.JobsClient.run_job() │ │ Job ID passed as: --job-id argument │ @@ -547,9 +547,9 @@ Audio and lyrics workers run as **Cloud Run Jobs** - standalone batch containers └─────────────────────────────────────────────────────────────────┘ ``` -**Job regions**: the CPU jobs a customer waits on (download, lyrics, bulk search) run in `us-east4` -(`CPU_JOBS_REGION`), because Cloud Run Jobs in `us-central1` wait 2-5 min before the container starts. -`video-encoding-job` stays in `us-central1` next to the bucket. See TROUBLESHOOTING.md, "Cloud Run Jobs +**Job regions**: the CPU jobs a customer waits on (download, lyrics, bulk search, video/render) run in +`us-east4` (`CPU_JOBS_REGION`), because Cloud Run Jobs in `us-central1` wait 2-5 min before the +container starts. The bucket and GCE encoding VMs stay in `us-central1`. See TROUBLESHOOTING.md, "Cloud Run Jobs take 2-5 minutes to start". **Why Cloud Run Jobs**: When using FastAPI BackgroundTasks, Cloud Run would terminate instances when HTTP requests completed, even if background work was still running. Cloud Run Jobs solve this by running workers as standalone processes that complete naturally. This applies to audio downloads (jobs 51b8231d, 89e497b1 were killed mid-download), lyrics transcription (job c94cc9d6 killed mid-processing), and audio separation. diff --git a/docs/LESSONS-LEARNED.md b/docs/LESSONS-LEARNED.md index 397f31ca..7d2e96f7 100644 --- a/docs/LESSONS-LEARNED.md +++ b/docs/LESSONS-LEARNED.md @@ -21,10 +21,12 @@ their limits. `Started` condition's `lastTransitionTime`, and the first app log line. Only the last gap is ours. - **An A/B with a trivial image across regions** isolates platform behaviour in minutes and costs pennies. Do it before optimizing image size or imports. -- **When moving a job's region, keep the data-heavy jobs next to the bucket.** Download, lyrics and - bulk search move small files, so they moved to us-east4. `video-encoding-job` pulls multi-GB - outputs, so it stayed. Also push the image to a same-region registry so the multi-GB image never - crosses regions (Artifact Registry egress). +- **Measure cross-region transfer before assuming it's expensive.** Cloud Monitoring's + `run.googleapis.com/container/network/received_bytes_count` (kind=google) gives bytes per job, and + the Billing Catalog API gives exact SKU prices. `video-encoding-job` turned out to read only + ~156 GB/month, about $1.60/month after GCS's 100 GB free inter-region tier, so it moved too + (v0.249.1). Writes to GCS from Cloud Run are free. Also push the image to a same-region registry so + the multi-GB image never crosses regions (Artifact Registry egress). - **Keep region-for-jobs separate from `GCP_REGION`.** The workers still use `GCP_REGION` for other resources. Same lesson as `cloud_tasks_region` (job b8bda9c2). diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 635635e1..f2c496b2 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -747,7 +747,7 @@ client.run_job(request=run_v2.RunJobRequest( | `audio-download-job` | `us-east4` | `audio_download_worker` | No | | `lyrics-transcription-job` | `us-east4` | `lyrics_worker` | No | | `bulk-search-job` | `us-east4` | `bulk_search_worker` | No | -| `video-encoding-job` | `us-central1` | `video_worker` / `render_video_worker` | No | +| `video-encoding-job` | `us-east4` | `video_worker` / `render_video_worker` | No | The CPU jobs above run in `us-east4` (`CPU_JOBS_REGION`) even though everything else lives in `us-central1`. See "Cloud Run Jobs take 2-5 minutes to start" below. @@ -767,14 +767,17 @@ and `us-east1` took 8-25s. decide's 203 MB image on 4 CPU was just as slow in `u slimming our image or adding CPU doesn't help. Quotas were nowhere near their limits. **Fix in place:** the latency-critical CPU jobs (`audio-download-job`, `lyrics-transcription-job`, -`bulk-search-job`) run in `us-east4`. The backend setting is `CPU_JOBS_REGION` (default `us-east4`), +`bulk-search-job`, `video-encoding-job`) run in `us-east4`. The backend setting is `CPU_JOBS_REGION` (default `us-east4`), and the Pulumi constant `CPU_JOBS_REGION` must match it (a unit test checks this). CI pushes a copy of the CPU image to `us-east4-docker.pkg.dev/nomadkaraoke/karaoke-backend-gpu/karaoke-backend-cpu` so the jobs pull from their own region. Inside the jobs, `GCP_REGION` stays `us-central1`. A new image digest takes a one-time ~2.5 min "Container image import" on its first execution in a region, so after each deploy CI starts one no-op `audio-download-job` execution (`python -c pass`) to do that import. -`video-encoding-job` stays in `us-central1` on purpose: it moves multi-GB outputs through the -us-central1 bucket, so running it in us-east4 would add inter-region GCS egress. +`video-encoding-job` (video + post-review render) moved in v0.249.1. It reads about 0.3 GB per run +from the us-central1 bucket, which is roughly 156 GB/month cross-region: about $1.60/month after +GCS's 100 GB/month free inter-region tier, or $3/month worst case. Its writes to GCS and its +internet uploads cost the same from either region. The GCE encoding VMs stay in `us-central1`; they +have public IPs (port 8080, API key), so the job reaches them from us-east4. **Measure:** ```bash @@ -790,6 +793,7 @@ done gcloud run services update karaoke-backend --region us-central1 --update-env-vars CPU_JOBS_REGION=us-central1 gcloud run jobs update audio-download-job --region us-central1 --update-env-vars CPU_JOBS_REGION=us-central1 # legacy job triggers lyrics gcloud run jobs update bulk-search-job --region us-central1 --update-env-vars CPU_JOBS_REGION=us-central1 # legacy job triggers downloads +gcloud run jobs update video-encoding-job --region us-central1 --update-env-vars CPU_JOBS_REGION=us-central1 # legacy job triggers render/video ``` The next CI deploy resets the service env. Make it permanent by adding `CPU_JOBS_REGION` to `--set-env-vars` in `ci.yml`. diff --git a/infrastructure/__main__.py b/infrastructure/__main__.py index ba47135b..916cdcd0 100644 --- a/infrastructure/__main__.py +++ b/infrastructure/__main__.py @@ -186,7 +186,6 @@ edge_resources = edge_security.configure_edge_security() # Cloud Run Jobs for batch processing # These use Cloud Run Jobs instead of Cloud Tasks to avoid instance termination # during long-running processing (the BackgroundTasks issue) -video_encoding_job = cloud_run.create_video_encoding_job(bucket, backend_service_account) audio_separation_job = cloud_run.create_audio_separation_job(bucket, backend_service_account) # Latency-critical CPU jobs run in CPU_JOBS_REGION (us-east4): us-central1 Cloud # Run Jobs queue 2-5 min before starting. The legacy us-central1 copies are kept @@ -201,6 +200,10 @@ audio_download_job = cloud_run.create_audio_download_job( bulk_search_job = cloud_run.create_bulk_search_job( bucket, backend_service_account, region=cloud_run.CPU_JOBS_REGION ) +video_encoding_job = cloud_run.create_video_encoding_job( + bucket, backend_service_account, region=cloud_run.CPU_JOBS_REGION +) +legacy_video_encoding_job = cloud_run.create_video_encoding_job(bucket, backend_service_account) legacy_lyrics_transcription_job = cloud_run.create_lyrics_transcription_job(bucket, backend_service_account) legacy_audio_download_job = cloud_run.create_audio_download_job(bucket, backend_service_account) legacy_bulk_search_job = cloud_run.create_bulk_search_job(bucket, backend_service_account) diff --git a/infrastructure/modules/cloud_run.py b/infrastructure/modules/cloud_run.py index 4a76d785..599f1fd0 100644 --- a/infrastructure/modules/cloud_run.py +++ b/infrastructure/modules/cloud_run.py @@ -13,7 +13,7 @@ from config import PROJECT_ID, REGION AUDIO_WORKER_GPU_REGION = "us-east4" # L4 GPU quota available here # Region for the latency-critical CPU jobs (audio-download, lyrics-transcription, -# bulk-search). Cloud Run Jobs in us-central1 queue 2-5 min before the container +# bulk-search, video-encoding). Cloud Run Jobs in us-central1 queue 2-5 min before the container # starts, regardless of image size or CPU (measured 2026-09-28 with Google's tiny # sample job image: us-central1 4-5 min, us-east4 8-19s). Must match the # backend's CPU_JOBS_REGION default (backend/config.py). GCP_REGION inside these @@ -529,6 +529,7 @@ def create_bulk_search_job( def create_video_encoding_job( bucket: gcp.storage.Bucket, service_account: gcp.serviceaccount.Account, + region: str = REGION, ) -> cloudrunv2.Job: """ Create the Cloud Run Job for video encoding. @@ -549,14 +550,15 @@ def create_video_encoding_job( cloudrunv2.Job: The Cloud Run Job resource. """ video_encoding_job = cloudrunv2.Job( - "video-encoding-job", + _job_resource_name("video-encoding-job", region), name="video-encoding-job", - location=REGION, + location=region, + deletion_protection=False, # Allow the legacy-region copy to be removed template=cloudrunv2.JobTemplateArgs( template=cloudrunv2.JobTemplateTemplateArgs( containers=[ cloudrunv2.JobTemplateTemplateContainerArgs( - image=f"{REGION}-docker.pkg.dev/{PROJECT_ID}/karaoke-repo/karaoke-backend:latest", + image=cpu_job_image(region), args=["python", "-m", "backend.workers.video_worker"], resources=cloudrunv2.JobTemplateTemplateContainerResourcesArgs( limits={ diff --git a/pyproject.toml b/pyproject.toml index bd5dccc1..59b4c61d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.249.0" +version = "0.249.1" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT" diff --git a/scripts/rollback.sh b/scripts/rollback.sh index f44abb5a..9b1e13da 100755 --- a/scripts/rollback.sh +++ b/scripts/rollback.sh @@ -8,7 +8,7 @@ # This script is the single, documented, parameterized rollback path for: # - the `karaoke-backend` Cloud Run SERVICE, and # - the three Cloud Run JOBS that share the backend image: -# * video-encoding-job (CPU, us-central1) +# * video-encoding-job (CPU, us-east4 — CPU image copy) # * lyrics-transcription-job (CPU, us-east4 — CPU image copy) # * audio-separation-job (GPU, us-east4 — uses the GPU image) # @@ -70,7 +70,7 @@ CPU_EAST_IMAGE_REPO="us-east4-docker.pkg.dev/${PROJECT_ID}/karaoke-backend-gpu/k # audio-download-job / bulk-search-job are NOT re-pinned: they track :latest and # CI never re-pins them, so a pin here would stick past the next deploy. JOBS=( - "video-encoding-job:us-central1:${CPU_IMAGE_REPO}" + "video-encoding-job:us-east4:${CPU_EAST_IMAGE_REPO}" "lyrics-transcription-job:us-east4:${CPU_EAST_IMAGE_REPO}" "audio-separation-job:us-east4:${GPU_IMAGE_REPO}" )