fix(encoding): stems-ready resumes reach the AWS video worker (v0.289.1) (#1241)

## Summary
Found by the prod E2E on 2026-10-31.
- With `aws_video_worker` on, the render worker dispatched the video
worker to AWS, and it parked waiting for stems.
- The **stems-ready resume** came from `audio-separation-job`, which has
no `ENCODING_WORKER_FALLBACK_VMS` env. It never saw the AWS worker, so
it silently started the Cloud Run job.
- The job completed fine, because the Cloud Run job is the fallback, but
its distribution ran from GCP.

## Changes
- `EncodingWorkerManager.fallback_entries()`: in production, when the
env var is missing, read the `encoding-worker-fallback-vms` secret
(cached per process by `get_secret`). This affects the audio, lyrics and
download jobs, which also trigger and warm the encoder. Outside
production it never fetches.
- `_trigger_video_worker_on_aws`: logs when the AWS dispatch is skipped
because AWS isn't the top-ranked worker, so it can't fail silently
again.

## Testing
- 3 new tests: secret fallback in prod, no fetch outside prod, env
preferred over the secret. All 171 related tests pass.
- Local CodeRabbit: no findings.
- After deploy: another prod E2E to confirm the resume reaches AWS.

@coderabbitai ignore

🤖 Generated with [Claude Code](https://claude.com/claude-code)

https://claude.ai/code/session_012RYLVSR5qgwsxTQtvGimKF

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Andrew Beveridge ☄️andClaude Opus 5.5 authored and GitHub committed 2026-10-31 16:07:15 -04:00
1 parent 78264fa23a
commit 452a8a3e28
4 files changed
+57 -3

No files matched your search

+10 -2
View File
@@ -239,9 +239,17 @@ class EncodingWorkerManager:
raw = self._fallback_vms_raw
if raw is None:
try:
from backend.config import get_settings
from backend.config import get_settings, is_production
raw = get_settings().encoding_worker_fallback_vms
settings = get_settings()
raw = settings.encoding_worker_fallback_vms
if not raw and is_production():
# Only the API service and video-encoding-job get the env var;
# the audio/lyrics/download jobs also trigger the video worker
# and warm the encoder, so read the secret directly (cached
# per process by get_secret). Without this they never see the
# AWS worker and silently keep the old path.
raw = settings.get_secret("encoding-worker-fallback-vms")
except Exception as e: # noqa: BLE001
logger.warning("Could not read ENCODING_WORKER_FALLBACK_VMS: %s", e)
raw = None
+4
View File
@@ -669,6 +669,10 @@ class WorkerService:
return False
candidates = await asyncio.to_thread(manager.ranked_candidates, config)
if not candidates or candidates[0].provider != "aws":
logger.info(
f"[job:{job_id}] AWS video worker enabled but not the preferred encoder "
f"(top: {candidates[0].vm_name if candidates else 'none'}); using Cloud Run"
)
return False
aws = candidates[0]
result = await asyncio.to_thread(manager.ensure_any_running, candidates)
@@ -144,3 +144,45 @@ def test_aws_control_plane_fault_fails_over_to_gce():
result = m.ensure_any_running(m.ranked_candidates())
assert result["vm_name"] == "encoding-worker-a"
def _settings(env_value, secret_value):
s = MagicMock()
s.encoding_worker_fallback_vms = env_value
s.get_secret = MagicMock(return_value=secret_value)
return s
def test_fallback_entries_read_secret_in_prod_when_env_missing(monkeypatch):
"""Jobs without the env var (audio/lyrics/download) still see the AWS worker:
they trigger the video worker after stems, so they must rank identically."""
from unittest.mock import patch
db, _ = _db()
m = EncodingWorkerManager(db=db, compute_client=MagicMock(), aws_compute=MagicMock())
settings = _settings(None, json.dumps([AWS_ENTRY]))
with patch("backend.config.get_settings", return_value=settings), \
patch("backend.config.is_production", return_value=True):
assert [e["vm"] for e in m.fallback_entries()] == ["aws-encoding-worker-1"]
settings.get_secret.assert_called_once_with("encoding-worker-fallback-vms")
def test_fallback_entries_never_fetch_secret_outside_prod():
from unittest.mock import patch
db, _ = _db()
m = EncodingWorkerManager(db=db, compute_client=MagicMock(), aws_compute=MagicMock())
settings = _settings(None, json.dumps([AWS_ENTRY]))
with patch("backend.config.get_settings", return_value=settings), \
patch("backend.config.is_production", return_value=False):
assert m.fallback_entries() == []
settings.get_secret.assert_not_called()
def test_fallback_entries_prefer_env_over_secret():
from unittest.mock import patch
db, _ = _db()
m = EncodingWorkerManager(db=db, compute_client=MagicMock(), aws_compute=MagicMock())
settings = _settings(json.dumps([C2D_ENTRY]), json.dumps([AWS_ENTRY]))
with patch("backend.config.get_settings", return_value=settings), \
patch("backend.config.is_production", return_value=True):
assert [e["vm"] for e in m.fallback_entries()] == ["encoding-worker-fallback-c2df"]
settings.get_secret.assert_not_called()
+1 -1
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "karaoke-gen"
version = "0.289.0"
version = "0.289.1"
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
license = "MIT"