From 452a8a3e282c0b5e53bb7a31024ad6023c602893 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrew=20Beveridge=20=E2=98=84=EF=B8=8F?= Date: Sat, 31 Oct 2026 16:07:15 -0400 Subject: [PATCH] fix(encoding): stems-ready resumes reach the AWS video worker (v0.289.1) (#1241) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Found by the prod E2E on 2026-10-31. - With `aws_video_worker` on, the render worker dispatched the video worker to AWS, and it parked waiting for stems. - The **stems-ready resume** came from `audio-separation-job`, which has no `ENCODING_WORKER_FALLBACK_VMS` env. It never saw the AWS worker, so it silently started the Cloud Run job. - The job completed fine, because the Cloud Run job is the fallback, but its distribution ran from GCP. ## Changes - `EncodingWorkerManager.fallback_entries()`: in production, when the env var is missing, read the `encoding-worker-fallback-vms` secret (cached per process by `get_secret`). This affects the audio, lyrics and download jobs, which also trigger and warm the encoder. Outside production it never fetches. - `_trigger_video_worker_on_aws`: logs when the AWS dispatch is skipped because AWS isn't the top-ranked worker, so it can't fail silently again. ## Testing - 3 new tests: secret fallback in prod, no fetch outside prod, env preferred over the secret. All 171 related tests pass. - Local CodeRabbit: no findings. - After deploy: another prod E2E to confirm the resume reaches AWS. @coderabbitai ignore 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_012RYLVSR5qgwsxTQtvGimKF Co-authored-by: Claude Opus 5.5 --- backend/services/encoding_worker_manager.py | 12 +++++- backend/services/worker_service.py | 4 ++ .../tests/test_encoding_worker_manager_aws.py | 42 +++++++++++++++++++ pyproject.toml | 2 +- 4 files changed, 57 insertions(+), 3 deletions(-) diff --git a/backend/services/encoding_worker_manager.py b/backend/services/encoding_worker_manager.py index 1fea61a0..419da1d8 100644 --- a/backend/services/encoding_worker_manager.py +++ b/backend/services/encoding_worker_manager.py @@ -239,9 +239,17 @@ class EncodingWorkerManager: raw = self._fallback_vms_raw if raw is None: try: - from backend.config import get_settings + from backend.config import get_settings, is_production - raw = get_settings().encoding_worker_fallback_vms + settings = get_settings() + raw = settings.encoding_worker_fallback_vms + if not raw and is_production(): + # Only the API service and video-encoding-job get the env var; + # the audio/lyrics/download jobs also trigger the video worker + # and warm the encoder, so read the secret directly (cached + # per process by get_secret). Without this they never see the + # AWS worker and silently keep the old path. + raw = settings.get_secret("encoding-worker-fallback-vms") except Exception as e: # noqa: BLE001 logger.warning("Could not read ENCODING_WORKER_FALLBACK_VMS: %s", e) raw = None diff --git a/backend/services/worker_service.py b/backend/services/worker_service.py index eec5bd76..d5107764 100644 --- a/backend/services/worker_service.py +++ b/backend/services/worker_service.py @@ -669,6 +669,10 @@ class WorkerService: return False candidates = await asyncio.to_thread(manager.ranked_candidates, config) if not candidates or candidates[0].provider != "aws": + logger.info( + f"[job:{job_id}] AWS video worker enabled but not the preferred encoder " + f"(top: {candidates[0].vm_name if candidates else 'none'}); using Cloud Run" + ) return False aws = candidates[0] result = await asyncio.to_thread(manager.ensure_any_running, candidates) diff --git a/backend/tests/test_encoding_worker_manager_aws.py b/backend/tests/test_encoding_worker_manager_aws.py index 9e842eda..5f5c8473 100644 --- a/backend/tests/test_encoding_worker_manager_aws.py +++ b/backend/tests/test_encoding_worker_manager_aws.py @@ -144,3 +144,45 @@ def test_aws_control_plane_fault_fails_over_to_gce(): result = m.ensure_any_running(m.ranked_candidates()) assert result["vm_name"] == "encoding-worker-a" + + +def _settings(env_value, secret_value): + s = MagicMock() + s.encoding_worker_fallback_vms = env_value + s.get_secret = MagicMock(return_value=secret_value) + return s + + +def test_fallback_entries_read_secret_in_prod_when_env_missing(monkeypatch): + """Jobs without the env var (audio/lyrics/download) still see the AWS worker: + they trigger the video worker after stems, so they must rank identically.""" + from unittest.mock import patch + db, _ = _db() + m = EncodingWorkerManager(db=db, compute_client=MagicMock(), aws_compute=MagicMock()) + settings = _settings(None, json.dumps([AWS_ENTRY])) + with patch("backend.config.get_settings", return_value=settings), \ + patch("backend.config.is_production", return_value=True): + assert [e["vm"] for e in m.fallback_entries()] == ["aws-encoding-worker-1"] + settings.get_secret.assert_called_once_with("encoding-worker-fallback-vms") + + +def test_fallback_entries_never_fetch_secret_outside_prod(): + from unittest.mock import patch + db, _ = _db() + m = EncodingWorkerManager(db=db, compute_client=MagicMock(), aws_compute=MagicMock()) + settings = _settings(None, json.dumps([AWS_ENTRY])) + with patch("backend.config.get_settings", return_value=settings), \ + patch("backend.config.is_production", return_value=False): + assert m.fallback_entries() == [] + settings.get_secret.assert_not_called() + + +def test_fallback_entries_prefer_env_over_secret(): + from unittest.mock import patch + db, _ = _db() + m = EncodingWorkerManager(db=db, compute_client=MagicMock(), aws_compute=MagicMock()) + settings = _settings(json.dumps([C2D_ENTRY]), json.dumps([AWS_ENTRY])) + with patch("backend.config.get_settings", return_value=settings), \ + patch("backend.config.is_production", return_value=True): + assert [e["vm"] for e in m.fallback_entries()] == ["encoding-worker-fallback-c2df"] + settings.get_secret.assert_not_called() diff --git a/pyproject.toml b/pyproject.toml index 643dc786..b871b2f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.289.0" +version = "0.289.1" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT"