mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
fix(encoding): stems-ready resumes reach the AWS video worker (v0.289.1) (#1241)
## Summary Found by the prod E2E on 2026-10-31. - With `aws_video_worker` on, the render worker dispatched the video worker to AWS, and it parked waiting for stems. - The **stems-ready resume** came from `audio-separation-job`, which has no `ENCODING_WORKER_FALLBACK_VMS` env. It never saw the AWS worker, so it silently started the Cloud Run job. - The job completed fine, because the Cloud Run job is the fallback, but its distribution ran from GCP. ## Changes - `EncodingWorkerManager.fallback_entries()`: in production, when the env var is missing, read the `encoding-worker-fallback-vms` secret (cached per process by `get_secret`). This affects the audio, lyrics and download jobs, which also trigger and warm the encoder. Outside production it never fetches. - `_trigger_video_worker_on_aws`: logs when the AWS dispatch is skipped because AWS isn't the top-ranked worker, so it can't fail silently again. ## Testing - 3 new tests: secret fallback in prod, no fetch outside prod, env preferred over the secret. All 171 related tests pass. - Local CodeRabbit: no findings. - After deploy: another prod E2E to confirm the resume reaches AWS. @coderabbitai ignore 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_012RYLVSR5qgwsxTQtvGimKF Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
78264fa23a
commit
452a8a3e28
4 files changed
+57
-3
No files matched your search
@@ -239,9 +239,17 @@ class EncodingWorkerManager:
|
||||
raw = self._fallback_vms_raw
|
||||
if raw is None:
|
||||
try:
|
||||
from backend.config import get_settings
|
||||
from backend.config import get_settings, is_production
|
||||
|
||||
raw = get_settings().encoding_worker_fallback_vms
|
||||
settings = get_settings()
|
||||
raw = settings.encoding_worker_fallback_vms
|
||||
if not raw and is_production():
|
||||
# Only the API service and video-encoding-job get the env var;
|
||||
# the audio/lyrics/download jobs also trigger the video worker
|
||||
# and warm the encoder, so read the secret directly (cached
|
||||
# per process by get_secret). Without this they never see the
|
||||
# AWS worker and silently keep the old path.
|
||||
raw = settings.get_secret("encoding-worker-fallback-vms")
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("Could not read ENCODING_WORKER_FALLBACK_VMS: %s", e)
|
||||
raw = None
|
||||
|
||||
@@ -669,6 +669,10 @@ class WorkerService:
|
||||
return False
|
||||
candidates = await asyncio.to_thread(manager.ranked_candidates, config)
|
||||
if not candidates or candidates[0].provider != "aws":
|
||||
logger.info(
|
||||
f"[job:{job_id}] AWS video worker enabled but not the preferred encoder "
|
||||
f"(top: {candidates[0].vm_name if candidates else 'none'}); using Cloud Run"
|
||||
)
|
||||
return False
|
||||
aws = candidates[0]
|
||||
result = await asyncio.to_thread(manager.ensure_any_running, candidates)
|
||||
|
||||
@@ -144,3 +144,45 @@ def test_aws_control_plane_fault_fails_over_to_gce():
|
||||
result = m.ensure_any_running(m.ranked_candidates())
|
||||
|
||||
assert result["vm_name"] == "encoding-worker-a"
|
||||
|
||||
|
||||
def _settings(env_value, secret_value):
|
||||
s = MagicMock()
|
||||
s.encoding_worker_fallback_vms = env_value
|
||||
s.get_secret = MagicMock(return_value=secret_value)
|
||||
return s
|
||||
|
||||
|
||||
def test_fallback_entries_read_secret_in_prod_when_env_missing(monkeypatch):
|
||||
"""Jobs without the env var (audio/lyrics/download) still see the AWS worker:
|
||||
they trigger the video worker after stems, so they must rank identically."""
|
||||
from unittest.mock import patch
|
||||
db, _ = _db()
|
||||
m = EncodingWorkerManager(db=db, compute_client=MagicMock(), aws_compute=MagicMock())
|
||||
settings = _settings(None, json.dumps([AWS_ENTRY]))
|
||||
with patch("backend.config.get_settings", return_value=settings), \
|
||||
patch("backend.config.is_production", return_value=True):
|
||||
assert [e["vm"] for e in m.fallback_entries()] == ["aws-encoding-worker-1"]
|
||||
settings.get_secret.assert_called_once_with("encoding-worker-fallback-vms")
|
||||
|
||||
|
||||
def test_fallback_entries_never_fetch_secret_outside_prod():
|
||||
from unittest.mock import patch
|
||||
db, _ = _db()
|
||||
m = EncodingWorkerManager(db=db, compute_client=MagicMock(), aws_compute=MagicMock())
|
||||
settings = _settings(None, json.dumps([AWS_ENTRY]))
|
||||
with patch("backend.config.get_settings", return_value=settings), \
|
||||
patch("backend.config.is_production", return_value=False):
|
||||
assert m.fallback_entries() == []
|
||||
settings.get_secret.assert_not_called()
|
||||
|
||||
|
||||
def test_fallback_entries_prefer_env_over_secret():
|
||||
from unittest.mock import patch
|
||||
db, _ = _db()
|
||||
m = EncodingWorkerManager(db=db, compute_client=MagicMock(), aws_compute=MagicMock())
|
||||
settings = _settings(json.dumps([C2D_ENTRY]), json.dumps([AWS_ENTRY]))
|
||||
with patch("backend.config.get_settings", return_value=settings), \
|
||||
patch("backend.config.is_production", return_value=True):
|
||||
assert [e["vm"] for e in m.fallback_entries()] == ["encoding-worker-fallback-c2df"]
|
||||
settings.get_secret.assert_not_called()
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.289.0"
|
||||
version = "0.289.1"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
Reference in new issue
Block a user