diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 897250fe..d7270dbc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1106,6 +1106,13 @@ jobs: GPU_REPOSITORY: karaoke-backend-gpu GPU_BASE_IMAGE_NAME: karaoke-backend-gpu-base GPU_APP_IMAGE_NAME: karaoke-backend + # CPU app image copy for the latency-critical CPU jobs (audio-download, + # lyrics-transcription, bulk-search) in us-east4, where Cloud Run Jobs start + # in ~15s instead of 2-5 min. Lives in the existing us-east4 repo so the jobs + # pull same-region; only changed app layers cross regions on each push. + CPU_JOBS_REGISTRY: us-east4-docker.pkg.dev + CPU_JOBS_REPOSITORY: karaoke-backend-gpu + CPU_JOBS_IMAGE_NAME: karaoke-backend-cpu steps: - name: Check disk space (aggressive for Docker builds) run: | @@ -1859,12 +1866,32 @@ jobs: ${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.APP_IMAGE_NAME }}:latest ${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.APP_IMAGE_NAME }}:${{ github.sha }} ${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.APP_IMAGE_NAME }}:v${{ steps.version_check.outputs.version }} + ${{ env.CPU_JOBS_REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.CPU_JOBS_REPOSITORY }}/${{ env.CPU_JOBS_IMAGE_NAME }}:latest + ${{ env.CPU_JOBS_REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.CPU_JOBS_REPOSITORY }}/${{ env.CPU_JOBS_IMAGE_NAME }}:${{ github.sha }} + ${{ env.CPU_JOBS_REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.CPU_JOBS_REPOSITORY }}/${{ env.CPU_JOBS_IMAGE_NAME }}:v${{ steps.version_check.outputs.version }} # Registry cache for app layers # ignore-error=true allows build to continue if cache doesn't exist yet cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.APP_IMAGE_NAME }}:cache,ignore-error=true cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.APP_IMAGE_NAME }}:cache,mode=max,ignore-error=true provenance: false + - name: Pre-import CPU image for us-east4 jobs + # A new image digest costs a one-time ~2.5 min "Container image import" on + # its FIRST execution in a region (measured 2026-09-29); later executions + # of any job using that digest start in ~8s. audio-download-job runs + # :latest (just pushed), so a no-op execution now performs the import + # while the service deploy + smoke test run, instead of on the first + # customer download. Async on purpose: the import overlaps the rest of + # the deploy. Non-fatal: worst case the first download pays the import. + run: | + gcloud run jobs execute audio-download-job \ + --region us-east4 \ + --project ${{ env.PROJECT_ID }} \ + --args="python,-c,pass" \ + --async \ + --quiet \ + || echo "::warning::us-east4 CPU image pre-import execution failed to start (non-fatal)" + # ============================================ # GPU Image Build (for audio worker with L4 GPU) # Same pattern as base image: hash check → rebuild if needed. @@ -2127,6 +2154,7 @@ jobs: run: | IMAGE="${{ env.REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.REPOSITORY }}/${{ env.APP_IMAGE_NAME }}:${{ github.sha }}" GPU_IMAGE="${{ env.GPU_REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.GPU_REPOSITORY }}/${{ env.GPU_APP_IMAGE_NAME }}:${{ github.sha }}" + CPU_JOBS_IMAGE="${{ env.CPU_JOBS_REGISTRY }}/${{ env.PROJECT_ID }}/${{ env.CPU_JOBS_REPOSITORY }}/${{ env.CPU_JOBS_IMAGE_NAME }}:${{ github.sha }}" echo "📦 Updating Cloud Run Jobs..." @@ -2137,7 +2165,10 @@ jobs: --project ${{ env.PROJECT_ID }} \ --quiet - # Lyrics transcription job (CPU, us-central1) + # Lyrics transcription job (CPU, us-east4 — see CPU_JOBS_REGION in + # backend/config.py). The us-east4 job is created by Pulumi, so + # `pulumi up` must run BEFORE merging a change that introduces it. audio-download-job and bulk-search-job track the + # :latest tag pushed above, so they need no update here. # NOTE: image-only update. Adding --update-secrets here crashed gcloud # ("Invalid secret path ... in annotation") because this job's existing # secrets are stored via Cloud Run secret *aliases* that --update-secrets @@ -2147,8 +2178,8 @@ jobs: # doc); until then the proactive flag stays unset and the worker's # _run_proactive_auto_correct() returns early (no-op). gcloud run jobs update lyrics-transcription-job \ - --image "$IMAGE" \ - --region us-central1 \ + --image "$CPU_JOBS_IMAGE" \ + --region us-east4 \ --project ${{ env.PROJECT_ID }} \ --quiet diff --git a/backend/config.py b/backend/config.py index 1aadbdf1..4260cf70 100644 --- a/backend/config.py +++ b/backend/config.py @@ -205,6 +205,14 @@ class Settings(BaseSettings): # must still enqueue to the real queue location — building queue paths from # gcp_region 404s from those workers ("Queue does not exist"). cloud_tasks_region: str = os.getenv("CLOUD_TASKS_REGION", "us-central1") + # Region of the latency-critical CPU Cloud Run Jobs (audio-download-job, + # lyrics-transcription-job, bulk-search-job). Cloud Run Jobs in us-central1 + # queue 2-5 min before the container starts (measured 2026-09-28 with a tiny + # sample image: us-central1 4-5 min vs us-east4 8-19s), independent of image + # size or CPU. Rollback: set CPU_JOBS_REGION=us-central1 on the service AND + # the jobs (the download worker triggers the lyrics job). Deliberately + # separate from gcp_region, which these workers use for other resources. + cpu_jobs_region: str = os.getenv("CPU_JOBS_REGION", "us-east4") # Cloud Run Jobs (for long-running video encoding) # When enabled AND enable_cloud_tasks is true, video worker uses Cloud Run Jobs diff --git a/backend/services/worker_service.py b/backend/services/worker_service.py index 14e6af20..3a1cda23 100644 --- a/backend/services/worker_service.py +++ b/backend/services/worker_service.py @@ -397,6 +397,7 @@ class WorkerService: job_id=job_id, cloud_run_job_name="audio-download-job", worker_module="audio_download_worker", + location=self.settings.cpu_jobs_region, ) async def trigger_bulk_search_worker(self, batch_id: str) -> bool: @@ -419,7 +420,7 @@ class WorkerService: logger.error("GOOGLE_CLOUD_PROJECT not set, cannot trigger bulk search job") return False - location = self.settings.gcp_region + location = self.settings.cpu_jobs_region job_name = f"projects/{project}/locations/{location}/jobs/bulk-search-job" client = run_v2.JobsClient() request = run_v2.RunJobRequest( @@ -472,6 +473,7 @@ class WorkerService: job_id=job_id, cloud_run_job_name="lyrics-transcription-job", worker_module="lyrics_worker", + location=self.settings.cpu_jobs_region, ) async def trigger_screens_worker(self, job_id: str) -> bool: diff --git a/backend/tests/test_worker_service_cpu_jobs_region.py b/backend/tests/test_worker_service_cpu_jobs_region.py new file mode 100644 index 00000000..3035d18c --- /dev/null +++ b/backend/tests/test_worker_service_cpu_jobs_region.py @@ -0,0 +1,87 @@ +"""Latency-critical CPU Cloud Run Jobs are triggered in cpu_jobs_region (us-east4). + +Cloud Run Jobs in us-central1 queue 2-5 min before the container starts, even for +a tiny sample image (measured 2026-09-28: us-central1 4-5 min, us-east4 8-19s). +audio-download-job, lyrics-transcription-job and bulk-search-job therefore run in +us-east4, while the jobs and workers keep GCP_REGION=us-central1 for everything +else. video-encoding-job stays in gcp_region: it moves multi-GB outputs through +the us-central1 bucket, so a cross-region copy would add GCS egress. +""" +from __future__ import annotations + +import re +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def _settings(monkeypatch, cpu_jobs_region: str | None = None): + monkeypatch.setenv("ENABLE_CLOUD_TASKS", "true") + monkeypatch.setenv("GCP_REGION", "us-central1") + monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "test-project") + if cpu_jobs_region is None: + monkeypatch.delenv("CPU_JOBS_REGION", raising=False) + else: + monkeypatch.setenv("CPU_JOBS_REGION", cpu_jobs_region) + from backend.config import Settings + return Settings() + + +async def _triggered_job_name(settings, trigger) -> str: + """Run ``trigger(service)`` against a mocked run_v2 and return the job path.""" + from backend.services.worker_service import WorkerService + + mock_run_v2 = MagicMock() + mock_run_v2.JobsClient.return_value.run_job.return_value = MagicMock(metadata="m") + + import google.cloud + with patch("backend.services.worker_service.get_settings", return_value=settings), \ + patch.dict("sys.modules", {"google.cloud.run_v2": mock_run_v2}), \ + patch.object(google.cloud, "run_v2", mock_run_v2, create=True): + assert await trigger(WorkerService()) is True + + return mock_run_v2.RunJobRequest.call_args.kwargs["name"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "trigger, job", + [ + (lambda s: s.trigger_audio_download_worker("job1"), "audio-download-job"), + (lambda s: s.trigger_lyrics_worker("job1"), "lyrics-transcription-job"), + (lambda s: s.trigger_bulk_search_worker("batch1"), "bulk-search-job"), + ], +) +async def test_cpu_jobs_triggered_in_us_east4(monkeypatch, trigger, job) -> None: + name = await _triggered_job_name(_settings(monkeypatch), trigger) + assert name == f"projects/test-project/locations/us-east4/jobs/{job}" + + +@pytest.mark.asyncio +async def test_cpu_jobs_region_env_override_for_rollback(monkeypatch) -> None: + settings = _settings(monkeypatch, cpu_jobs_region="us-central1") + name = await _triggered_job_name(settings, lambda s: s.trigger_audio_download_worker("job1")) + assert name == "projects/test-project/locations/us-central1/jobs/audio-download-job" + + +@pytest.mark.asyncio +async def test_video_encoding_job_stays_in_gcp_region(monkeypatch) -> None: + name = await _triggered_job_name(_settings(monkeypatch), lambda s: s._trigger_cloud_run_job("job1")) + assert name == "projects/test-project/locations/us-central1/jobs/video-encoding-job" + + +def test_gcp_region_unaffected_by_cpu_jobs_region(monkeypatch) -> None: + settings = _settings(monkeypatch) + assert settings.cpu_jobs_region == "us-east4" + assert settings.gcp_region == "us-central1" + + +def test_backend_default_matches_pulumi_cpu_jobs_region(monkeypatch) -> None: + """The backend triggers jobs where Pulumi creates them; the two must agree.""" + src = (REPO_ROOT / "infrastructure" / "modules" / "cloud_run.py").read_text() + m = re.search(r'^CPU_JOBS_REGION = "([a-z0-9-]+)"', src, re.MULTILINE) + assert m, "could not locate CPU_JOBS_REGION in infrastructure/modules/cloud_run.py" + assert _settings(monkeypatch).cpu_jobs_region == m.group(1) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 26833727..178c9a4a 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -535,16 +535,23 @@ Audio and lyrics workers run as **Cloud Run Jobs** - standalone batch containers ┌─────────────────────────────────────────────────────────────────┐ │ Cloud Run Jobs (via WorkerService) │ │ │ -│ audio-download-job - 30s-5 min (flacfetch/YouTube) │ -│ lyrics-transcription-job - 5-15 min (AudioShake + correction)│ +│ audio-download-job - 30s-5 min (flacfetch/YouTube, us-east4) │ +│ lyrics-transcription-job - 5-15 min (AudioShake + correction, us-east4)│ +│ bulk-search-job - batch audio search (us-east4) │ │ audio-separation-job - 10-20 min (L4 GPU, us-east4, direct) │ -│ video-encoding-job - up to 60 min (optional) │ +│ video-encoding-job - up to 60 min (us-central1) │ │ │ │ Triggered via: google.cloud.run_v2.JobsClient.run_job() │ │ Job ID passed as: --job-id argument │ +│ (bulk-search-job takes --batch-id instead) │ └─────────────────────────────────────────────────────────────────┘ ``` +**Job regions**: the CPU jobs a customer waits on (download, lyrics, bulk search) run in `us-east4` +(`CPU_JOBS_REGION`), because Cloud Run Jobs in `us-central1` wait 2-5 min before the container starts. +`video-encoding-job` stays in `us-central1` next to the bucket. See TROUBLESHOOTING.md, "Cloud Run Jobs +take 2-5 minutes to start". + **Why Cloud Run Jobs**: When using FastAPI BackgroundTasks, Cloud Run would terminate instances when HTTP requests completed, even if background work was still running. Cloud Run Jobs solve this by running workers as standalone processes that complete naturally. This applies to audio downloads (jobs 51b8231d, 89e497b1 were killed mid-download), lyrics transcription (job c94cc9d6 killed mid-processing), and audio separation. **CLI Entry Points**: Each worker has a `main()` function for Cloud Run Job execution: diff --git a/docs/LESSONS-LEARNED.md b/docs/LESSONS-LEARNED.md index 381c9e4f..397f31ca 100644 --- a/docs/LESSONS-LEARNED.md +++ b/docs/LESSONS-LEARNED.md @@ -6,6 +6,28 @@ Key insights for future AI agents working on this codebase. --- +## Cloud Run Job start latency was the region, not our image (Sep 2026, v0.247.1) + +Every us-central1 CPU Cloud Run Job (download, lyrics, encoding, error monitor, even decide's 203 MB +image on 4 CPU) waited 2-5 min between "execution created" and the container starting. The obvious +suspects were image size, lazy imports and CPU, and all three were wrong. The execution's `Started` +condition fired about 10s before the first Python log, so Python wasn't the problem. The decisive +test was a throwaway job running Google's tiny sample image in three regions: +**us-central1 4-5 min, us-east4 8-19s, us-east1 8-25s.** All four us-central1 probes started in the +same second, so executions are queued and released in batches on Google's side. Quotas were far from +their limits. + +- **Separate your own startup time from the platform's first.** Compare `status.startTime`, the + `Started` condition's `lastTransitionTime`, and the first app log line. Only the last gap is ours. +- **An A/B with a trivial image across regions** isolates platform behaviour in minutes and costs + pennies. Do it before optimizing image size or imports. +- **When moving a job's region, keep the data-heavy jobs next to the bucket.** Download, lyrics and + bulk search move small files, so they moved to us-east4. `video-encoding-job` pulls multi-GB + outputs, so it stayed. Also push the image to a same-region registry so the multi-GB image never + crosses regions (Artifact Registry egress). +- **Keep region-for-jobs separate from `GCP_REGION`.** The workers still use `GCP_REGION` for other + resources. Same lesson as `cloud_tasks_region` (job b8bda9c2). + ## "Eager" prep keyed to the wrong pipeline stage silently became lazy (Sep 2026, v0.238.0) **Symptom:** clicking the backing-vocals waveform in the preview modal played nothing for 7-15 s. diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index ae0b57d9..635635e1 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -72,7 +72,10 @@ state *now* rather than wait for a forward fix to build and deploy. revision — instant, no rebuild), and - the three Cloud Run **jobs** that share the backend image (`video-encoding-job`, `lyrics-transcription-job`, `audio-separation-job`) by - re-pinning them to a previous image tag. Cloud Run Jobs can't traffic-split, so + re-pinning them to a previous image tag. `lyrics-transcription-job` runs in us-east4 from the + `karaoke-backend-cpu` image copy. That copy only has tags from releases after the region move, so + `rollback.sh` skips (with a warning) any job whose target tag is missing and re-pins the rest. `audio-download-job` and `bulk-search-job` + track `:latest` and aren't re-pinned. Cloud Run Jobs can't traffic-split, so they are re-pinned; the old image takes effect on each job's **next** invocation (in-flight runs are unaffected). @@ -741,31 +744,55 @@ client.run_job(request=run_v2.RunJobRequest( | Job Name | Region | Module | GPU | |----------|--------|--------|-----| | `audio-separation-job` | `us-east4` | `audio_worker` | L4 | -| `lyrics-transcription-job` | `us-central1` | `lyrics_worker` | No | -| `video-encoding-job` | `us-central1` | `video_worker` | No | +| `audio-download-job` | `us-east4` | `audio_download_worker` | No | +| `lyrics-transcription-job` | `us-east4` | `lyrics_worker` | No | +| `bulk-search-job` | `us-east4` | `bulk_search_worker` | No | +| `video-encoding-job` | `us-central1` | `video_worker` / `render_video_worker` | No | + +The CPU jobs above run in `us-east4` (`CPU_JOBS_REGION`) even though everything else lives in +`us-central1`. See "Cloud Run Jobs take 2-5 minutes to start" below. --- -## CI does not update `lyrics-transcription-job` image +## Cloud Run Jobs take 2-5 minutes to start -**Symptoms:** Deployed code fixes don't take effect for lyrics processing. The backend service shows the new version, but lyrics jobs still crash with the old bug. +**Symptoms:** A job execution sits between "created" and its first Python log line for 2-5 minutes. +`gcloud run jobs executions describe ` shows `Started deployed execution in 3m...`. Concurrent +executions often start in the same second, as if a queue drained. -**Cause:** The CI workflow (`.github/workflows/ci.yml`) updates `audio-separation-job` and `video-encoding-job` after deploy, but does NOT update `lyrics-transcription-job`. The job uses the `:latest` tag, but Cloud Run Jobs resolve the tag to a digest at update time — pushing a new `:latest` image doesn't automatically update running jobs. +**Cause:** Google-side scheduling of Cloud Run **Jobs** in `us-central1`. It doesn't depend on our +image, CPU or code. Measured 2026-09-28 with Google's tiny sample job image +(`us-docker.pkg.dev/cloudrun/container/job`) on 1 CPU: `us-central1` took 4-5 min, while `us-east4` +and `us-east1` took 8-25s. decide's 203 MB image on 4 CPU was just as slow in `us-central1`, so +slimming our image or adding CPU doesn't help. Quotas were nowhere near their limits. -**Manual fix after deploy:** +**Fix in place:** the latency-critical CPU jobs (`audio-download-job`, `lyrics-transcription-job`, +`bulk-search-job`) run in `us-east4`. The backend setting is `CPU_JOBS_REGION` (default `us-east4`), +and the Pulumi constant `CPU_JOBS_REGION` must match it (a unit test checks this). CI pushes a copy of +the CPU image to `us-east4-docker.pkg.dev/nomadkaraoke/karaoke-backend-gpu/karaoke-backend-cpu` so the +jobs pull from their own region. Inside the jobs, `GCP_REGION` stays `us-central1`. A new image digest takes a +one-time ~2.5 min "Container image import" on its first execution in a region, so after each +deploy CI starts one no-op `audio-download-job` execution (`python -c pass`) to do that import. +`video-encoding-job` stays in `us-central1` on purpose: it moves multi-GB outputs through the +us-central1 bucket, so running it in us-east4 would add inter-region GCS egress. + +**Measure:** ```bash -# Pin to the specific version tag (preferred) -gcloud run jobs update lyrics-transcription-job \ - --image us-central1-docker.pkg.dev/nomadkaraoke/karaoke-repo/karaoke-backend:vX.Y.Z \ - --region us-central1 --project nomadkaraoke - -# Or force re-resolve :latest -gcloud run jobs update lyrics-transcription-job \ - --image us-central1-docker.pkg.dev/nomadkaraoke/karaoke-repo/karaoke-backend:latest \ - --region us-central1 --project nomadkaraoke +for e in $(gcloud run jobs executions list --job audio-download-job --region us-east4 \ + --project nomadkaraoke --limit 8 --format='value(name)'); do + gcloud run jobs executions describe $e --region us-east4 --project nomadkaraoke --format=json \ + | jq -r '[.metadata.creationTimestamp, (.status.conditions[]|select(.type=="Started")|.message)]|@tsv' +done ``` -**Permanent fix:** Add `lyrics-transcription-job` update to CI deploy step in `.github/workflows/ci.yml` alongside the existing `video-encoding-job` and `audio-separation-job` updates (~line 1568). +**Rollback to us-central1** (the legacy jobs still exist until they're removed in a follow-up): +```bash +gcloud run services update karaoke-backend --region us-central1 --update-env-vars CPU_JOBS_REGION=us-central1 +gcloud run jobs update audio-download-job --region us-central1 --update-env-vars CPU_JOBS_REGION=us-central1 # legacy job triggers lyrics +gcloud run jobs update bulk-search-job --region us-central1 --update-env-vars CPU_JOBS_REGION=us-central1 # legacy job triggers downloads +``` +The next CI deploy resets the service env. Make it permanent by adding `CPU_JOBS_REGION` to +`--set-env-vars` in `ci.yml`. ## Frontend error alert ("New Error Pattern Detected", service `frontend`) diff --git a/infrastructure/__main__.py b/infrastructure/__main__.py index d7583056..ba47135b 100644 --- a/infrastructure/__main__.py +++ b/infrastructure/__main__.py @@ -187,10 +187,23 @@ edge_resources = edge_security.configure_edge_security() # These use Cloud Run Jobs instead of Cloud Tasks to avoid instance termination # during long-running processing (the BackgroundTasks issue) video_encoding_job = cloud_run.create_video_encoding_job(bucket, backend_service_account) -lyrics_transcription_job = cloud_run.create_lyrics_transcription_job(bucket, backend_service_account) audio_separation_job = cloud_run.create_audio_separation_job(bucket, backend_service_account) -audio_download_job = cloud_run.create_audio_download_job(bucket, backend_service_account) -bulk_search_job = cloud_run.create_bulk_search_job(bucket, backend_service_account) +# Latency-critical CPU jobs run in CPU_JOBS_REGION (us-east4): us-central1 Cloud +# Run Jobs queue 2-5 min before starting. The legacy us-central1 copies are kept +# until the backend has cut over (rollback path: CPU_JOBS_REGION=us-central1); +# remove them in a follow-up once the us-east4 jobs are verified in prod. +lyrics_transcription_job = cloud_run.create_lyrics_transcription_job( + bucket, backend_service_account, region=cloud_run.CPU_JOBS_REGION +) +audio_download_job = cloud_run.create_audio_download_job( + bucket, backend_service_account, region=cloud_run.CPU_JOBS_REGION +) +bulk_search_job = cloud_run.create_bulk_search_job( + bucket, backend_service_account, region=cloud_run.CPU_JOBS_REGION +) +legacy_lyrics_transcription_job = cloud_run.create_lyrics_transcription_job(bucket, backend_service_account) +legacy_audio_download_job = cloud_run.create_audio_download_job(bucket, backend_service_account) +legacy_bulk_search_job = cloud_run.create_bulk_search_job(bucket, backend_service_account) # ==================== Error Monitor ==================== diff --git a/infrastructure/modules/cloud_run.py b/infrastructure/modules/cloud_run.py index ce3163c0..4a76d785 100644 --- a/infrastructure/modules/cloud_run.py +++ b/infrastructure/modules/cloud_run.py @@ -12,6 +12,34 @@ from config import PROJECT_ID, REGION AUDIO_WORKER_GPU_REGION = "us-east4" # L4 GPU quota available here +# Region for the latency-critical CPU jobs (audio-download, lyrics-transcription, +# bulk-search). Cloud Run Jobs in us-central1 queue 2-5 min before the container +# starts, regardless of image size or CPU (measured 2026-09-28 with Google's tiny +# sample job image: us-central1 4-5 min, us-east4 8-19s). Must match the +# backend's CPU_JOBS_REGION default (backend/config.py). GCP_REGION inside these +# jobs stays REGION: the bucket, queues and other resources are still there. +CPU_JOBS_REGION = "us-east4" + + +def cpu_job_image(region: str) -> str: + """CPU backend image for a job in ``region``, pulled from a same-region registry. + + CI pushes the CPU app image to both us-central1 (karaoke-repo) and us-east4 + (the existing karaoke-backend-gpu repo, as ``karaoke-backend-cpu``) so jobs + never pull the multi-GB image across regions (Artifact Registry egress). + """ + if region == REGION: + return f"{REGION}-docker.pkg.dev/{PROJECT_ID}/karaoke-repo/karaoke-backend:latest" + if region == AUDIO_WORKER_GPU_REGION: + return f"{region}-docker.pkg.dev/{PROJECT_ID}/karaoke-backend-gpu/karaoke-backend-cpu:latest" + raise ValueError(f"No CPU backend image registry configured for region {region}") + + +def _job_resource_name(base: str, region: str) -> str: + """Pulumi logical name: unchanged for REGION, suffixed for other regions so the + us-east4 job can coexist with the legacy us-central1 one during cutover.""" + return base if region == REGION else f"{base}-{region}" + def create_domain_mapping() -> cloudrun.DomainMapping: """ @@ -38,6 +66,7 @@ def create_domain_mapping() -> cloudrun.DomainMapping: def create_lyrics_transcription_job( bucket: gcp.storage.Bucket, service_account: gcp.serviceaccount.Account, + region: str = REGION, ) -> cloudrunv2.Job: """ Create the Cloud Run Job for lyrics transcription. @@ -56,14 +85,15 @@ def create_lyrics_transcription_job( cloudrunv2.Job: The Cloud Run Job resource. """ lyrics_transcription_job = cloudrunv2.Job( - "lyrics-transcription-job", + _job_resource_name("lyrics-transcription-job", region), name="lyrics-transcription-job", - location=REGION, + location=region, + deletion_protection=False, # Allow the legacy-region copy to be removed template=cloudrunv2.JobTemplateArgs( template=cloudrunv2.JobTemplateTemplateArgs( containers=[ cloudrunv2.JobTemplateTemplateContainerArgs( - image=f"{REGION}-docker.pkg.dev/{PROJECT_ID}/karaoke-repo/karaoke-backend:latest", + image=cpu_job_image(region), args=["python", "-m", "backend.workers.lyrics_worker"], resources=cloudrunv2.JobTemplateTemplateContainerResourcesArgs( limits={ @@ -271,6 +301,7 @@ def create_audio_separation_job( def create_audio_download_job( bucket: gcp.storage.Bucket, service_account: gcp.serviceaccount.Account, + region: str = REGION, ) -> cloudrunv2.Job: """ Create the Cloud Run Job for audio downloading. @@ -290,14 +321,15 @@ def create_audio_download_job( """ audio_download_job = cloudrunv2.Job( - "audio-download-job", + _job_resource_name("audio-download-job", region), name="audio-download-job", - location=REGION, + location=region, + deletion_protection=False, # Allow the legacy-region copy to be removed template=cloudrunv2.JobTemplateArgs( template=cloudrunv2.JobTemplateTemplateArgs( containers=[ cloudrunv2.JobTemplateTemplateContainerArgs( - image=f"{REGION}-docker.pkg.dev/{PROJECT_ID}/karaoke-repo/karaoke-backend:latest", + image=cpu_job_image(region), args=["python", "-m", "backend.workers.audio_download_worker"], resources=cloudrunv2.JobTemplateTemplateContainerResourcesArgs( limits={ @@ -398,6 +430,7 @@ def create_audio_download_job( def create_bulk_search_job( bucket: gcp.storage.Bucket, service_account: gcp.serviceaccount.Account, + region: str = REGION, ) -> cloudrunv2.Job: """ Create the Cloud Run Job for Bulk Mode batch audio search. @@ -412,14 +445,15 @@ def create_bulk_search_job( """ bulk_search_job = cloudrunv2.Job( - "bulk-search-job", + _job_resource_name("bulk-search-job", region), name="bulk-search-job", - location=REGION, + location=region, + deletion_protection=False, # Allow the legacy-region copy to be removed template=cloudrunv2.JobTemplateArgs( template=cloudrunv2.JobTemplateTemplateArgs( containers=[ cloudrunv2.JobTemplateTemplateContainerArgs( - image=f"{REGION}-docker.pkg.dev/{PROJECT_ID}/karaoke-repo/karaoke-backend:latest", + image=cpu_job_image(region), args=["python", "-m", "backend.workers.bulk_search_worker"], resources=cloudrunv2.JobTemplateTemplateContainerResourcesArgs( limits={ diff --git a/pyproject.toml b/pyproject.toml index 49553e77..fadfa602 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.247.0" +version = "0.247.1" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT" diff --git a/scripts/rollback.sh b/scripts/rollback.sh index 414bdcd5..f44abb5a 100755 --- a/scripts/rollback.sh +++ b/scripts/rollback.sh @@ -9,7 +9,7 @@ # - the `karaoke-backend` Cloud Run SERVICE, and # - the three Cloud Run JOBS that share the backend image: # * video-encoding-job (CPU, us-central1) -# * lyrics-transcription-job (CPU, us-central1) +# * lyrics-transcription-job (CPU, us-east4 — CPU image copy) # * audio-separation-job (GPU, us-east4 — uses the GPU image) # # TWO DIFFERENT MECHANISMS (intentional — see docs/TROUBLESHOOTING.md "Fast rollback"): @@ -63,11 +63,15 @@ SERVICE_REGION="us-central1" CPU_IMAGE_REPO="us-central1-docker.pkg.dev/${PROJECT_ID}/karaoke-repo/karaoke-backend" # GPU (audio separation) image repository, co-located with the L4 GPU in us-east4. GPU_IMAGE_REPO="us-east4-docker.pkg.dev/${PROJECT_ID}/karaoke-backend-gpu/karaoke-backend" +# CPU image copy for the CPU jobs that run in us-east4 (CPU_JOBS_REGION). +CPU_EAST_IMAGE_REPO="us-east4-docker.pkg.dev/${PROJECT_ID}/karaoke-backend-gpu/karaoke-backend-cpu" # job_name:region:repo — audio-separation-job uses the GPU repo/region. +# audio-download-job / bulk-search-job are NOT re-pinned: they track :latest and +# CI never re-pins them, so a pin here would stick past the next deploy. JOBS=( "video-encoding-job:us-central1:${CPU_IMAGE_REPO}" - "lyrics-transcription-job:us-central1:${CPU_IMAGE_REPO}" + "lyrics-transcription-job:us-east4:${CPU_EAST_IMAGE_REPO}" "audio-separation-job:us-east4:${GPU_IMAGE_REPO}" ) @@ -222,6 +226,19 @@ if [[ "$ROLL_JOBS" == "true" ]]; then for entry in "${JOBS[@]}"; do IFS=':' read -r job_name job_region job_repo <<< "$entry" echo "--- ${job_name} (${job_region}) ---" + # The us-east4 CPU image copy only has tags from v0.247.1 onward. Skip a job + # ONLY when its target tag genuinely doesn't exist, so the remaining jobs + # still get re-pinned; any other lookup error (auth, network) aborts loudly + # rather than letting the rollback look complete with a job on the bad image. + if ! describe_err=$(gcloud artifacts docker images describe "${job_repo}:${IMAGE_VERSION}" \ + --project="${PROJECT_ID}" 2>&1 >/dev/null); then + if grep -qiE "not ?found|NOT_FOUND|manifest unknown" <<< "$describe_err"; then + echo "⚠️ ${job_repo}:${IMAGE_VERSION} not found — SKIPPING ${job_name} (left on its current image)" >&2 + continue + fi + echo "❌ Could not inspect ${job_repo}:${IMAGE_VERSION}: ${describe_err}" >&2 + exit 1 + fi run_cmd gcloud run jobs update "${job_name}" \ --image="${job_repo}:${IMAGE_VERSION}" \ --region="${job_region}" \