From 77bec3327cc0b86cb16c34d65b9fd868ec37705e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrew=20Beveridge=20=E2=98=84=EF=B8=8F?= Date: Mon, 12 Oct 2026 17:29:43 -0400 Subject: [PATCH] fix: encoding-worker boot loop (poisoned gcloud cache) + stale file_urls after change-to-public (v0.268.3) (#1138) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Two bugs, both hit by job `82f48738` (a private → public visibility change) on 2026-10-12. ### 1. Encoding worker VMs boot into an unrecoverable restart loop If gcloud's first call (the API key fetch in `startup.sh`) happens before DHCP, gcloud caches "not on GCE" in `/root/.config/gcloud/gce`. Every later gcloud call in that boot then has no credentials, so the API key comes back empty, the v0.266.4 self-heal refuses to serve, and systemd re-runs `startup.sh` against the same poisoned cache until the VM idle-stops. worker-b was up 20:30–20:42 and never answered /health, so the render failed with `Cannot connect to host 34.10.189.118:8080`. The same thing happened on 10-09 (`f4310aea`): it was fixed by hand and the durable fix was noted but never shipped. - `startup.sh`: `rm -f /root/.config/gcloud/gce`, wait up to 60s for the metadata server, retry the secret fetch 6× (clearing the cache each time). CI uploads this script on every deploy and bootstrap downloads it on every service start, so existing VM disks pick it up. - Packer `startup-fallback.sh`: clears the cache too (new images). ### 2. Retry after a failed change-to-public 404s forever `change_to_public` deletes screens, `videos/with_vocals.mkv` and the finals but left their `file_urls`. `/retry` then saw `videos.with_vocals` + `instrumental_selection`, skipped the render and 404'd on the deleted MKV. It now drops those refs, as `ThemeRerenderService` already does. ## Testing - New `backend/tests/test_encoding_worker_startup_script.py` and `test_change_visibility.py::test_drops_file_urls_for_deleted_artifacts` - `pytest backend/tests/test_encoding_worker_startup_script.py backend/tests/test_encoding_worker_self_heal.py backend/tests/test_encoding_worker_code_paths.py backend/tests/test_change_visibility.py`: all pass 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_01JUFVxMZME6ydzkmLvyuXRo ## Summary by CodeRabbit * **Bug Fixes** * When a private job is made public, references to deleted screen and video files are cleared before processing resumes, preventing stale files from appearing. * Improved encoding worker startup reliability by retrying API-key retrieval when the metadata service or secret store is temporarily unavailable. @coderabbitai ignore --------- Co-authored-by: Claude Opus 5.5 --- backend/services/visibility_change_service.py | 6 ++ backend/tests/test_change_visibility.py | 29 +++++++++ .../test_encoding_worker_startup_script.py | 62 +++++++++++++++++++ docs/TROUBLESHOOTING.md | 12 ++++ infrastructure/encoding-worker/startup.sh | 33 +++++++++- infrastructure/packer/scripts/provision.sh | 5 +- pyproject.toml | 2 +- 7 files changed, 144 insertions(+), 5 deletions(-) create mode 100644 backend/tests/test_encoding_worker_startup_script.py diff --git a/backend/services/visibility_change_service.py b/backend/services/visibility_change_service.py index 744f0e79..188fd626 100644 --- a/backend/services/visibility_change_service.py +++ b/backend/services/visibility_change_service.py @@ -231,6 +231,12 @@ class VisibilityChangeService: "state_data.render_progress": DELETE_FIELD, "state_data.video_progress": DELETE_FIELD, "state_data.encoding_progress": DELETE_FIELD, + # Their blobs were deleted above. A stale videos.with_vocals ref made + # a retry after a failed render skip the render and 404 on the + # missing MKV (job 82f48738); the workers re-add all three. + "file_urls.screens": DELETE_FIELD, + "file_urls.videos.with_vocals": DELETE_FIELD, + "file_urls.finals": DELETE_FIELD, "outputs_deleted_at": None, "outputs_deleted_by": None, "updated_at": datetime.now(timezone.utc), diff --git a/backend/tests/test_change_visibility.py b/backend/tests/test_change_visibility.py index 2064aa86..5b70f292 100644 --- a/backend/tests/test_change_visibility.py +++ b/backend/tests/test_change_visibility.py @@ -263,6 +263,35 @@ class TestChangeToPublic: assert style_update["is_private"] is False assert style_update["status"] == "lyrics_complete" + @pytest.mark.asyncio + async def test_drops_file_urls_for_deleted_artifacts(self): + """Screens, lyrics video and finals blobs are deleted, so their refs must go + too: a stale videos.with_vocals made retry skip the render and 404 (82f48738).""" + # The service's own sentinel: other tests may swap google.cloud.firestore_v1 for a mock. + from backend.services.visibility_change_service import DELETE_FIELD + + mock_job_manager = MagicMock() + mock_job_ref = MagicMock() + mock_job_manager.firestore.db.collection.return_value.document.return_value = mock_job_ref + + service = VisibilityChangeService(job_manager=mock_job_manager) + job = _make_job(is_private=True, state_data={}) + + with patch("backend.services.visibility_change_service.VisibilityChangeService._delete_distributed_outputs", new_callable=AsyncMock), \ + patch("backend.services.visibility_change_service.StorageService") as mock_storage_cls, \ + patch("backend.api.routes.file_upload._prepare_theme_for_job", return_value=("p.json", {}, None)), \ + patch("backend.services.worker_service.get_worker_service") as mock_worker_svc: + + mock_storage_cls.return_value = MagicMock() + mock_worker_svc.return_value.trigger_screens_worker = AsyncMock(return_value=True) + + await service.change_to_public("test-123", job, "user@example.com") + + payload = next(c.args[0] for c in mock_job_ref.update.call_args_list if "theme_id" in c.args[0]) + assert payload["file_urls.screens"] is DELETE_FIELD + assert payload["file_urls.videos.with_vocals"] is DELETE_FIELD + assert payload["file_urls.finals"] is DELETE_FIELD + @pytest.mark.asyncio async def test_rollback_on_worker_failure(self): mock_job_manager = MagicMock() diff --git a/backend/tests/test_encoding_worker_startup_script.py b/backend/tests/test_encoding_worker_startup_script.py new file mode 100644 index 00000000..2c94653a --- /dev/null +++ b/backend/tests/test_encoding_worker_startup_script.py @@ -0,0 +1,62 @@ +"""Guards for the encoding-worker boot scripts' gcloud on-GCE cache handling. + +A gcloud call made before DHCP finishes writes "False" to /root/.config/gcloud/gce, +so every later gcloud call in that boot has no credentials. The worker then had +no API key, self-heal refused to serve, and systemd re-ran startup.sh against the +same poisoned cache until the VM idle-stopped (jobs f4310aea 2026-10-09, +82f48738 2026-10-12). The scripts must drop that cache before fetching the key. +""" +import re +import subprocess +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +STARTUP = REPO / "infrastructure" / "encoding-worker" / "startup.sh" +PROVISION = REPO / "infrastructure" / "packer" / "scripts" / "provision.sh" + +CACHE_RM = "rm -f /root/.config/gcloud/gce" +SECRET_FETCH = "gcloud secrets versions access latest --secret=encoding-worker-api-key" + + +def test_startup_script_is_valid_bash(): + subprocess.run(["bash", "-n", str(STARTUP)], check=True) + + +def test_startup_clears_gce_cache_before_first_secret_fetch(): + text = STARTUP.read_text() + assert CACHE_RM in text + assert text.index(CACHE_RM) < text.index(SECRET_FETCH) + + +def test_startup_waits_for_metadata_server_before_fetch(): + text = STARTUP.read_text() + metadata_probe = text.index("computeMetadata/v1/instance/id") + assert metadata_probe < text.index(SECRET_FETCH) + + +def test_startup_retries_secret_fetch(): + text = STARTUP.read_text() + loops = re.findall(r"for attempt in \$\(seq 1 \d+\); do\n(.*?)\ndone", text, re.S) + assert any(SECRET_FETCH in body and CACHE_RM in body for body in loops) + + +def test_packer_fallback_clears_gce_cache_before_secret_fetch(): + text = PROVISION.read_text() + fallback = text[text.index("startup-fallback.sh << 'FALLBACK'"):] + assert CACHE_RM in fallback + assert fallback.index(CACHE_RM) < fallback.index(SECRET_FETCH) + + +def test_startup_metadata_probe_has_total_timeout(): + text = STARTUP.read_text() + lines = text.splitlines() + url_idx = next(i for i, l in enumerate(lines) if "computeMetadata/v1/instance/id" in l) + # The curl flags sit on the line before the URL continuation. + assert "--max-time" in lines[url_idx - 1] + lines[url_idx] + + +def test_startup_exits_nonzero_without_api_key(): + text = STARTUP.read_text() + block = text[text.index('if [ -z "$ENCODING_API_KEY" ]; then'):] + block = block[: block.index("fi")] + assert "exit 1" in block diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 77707d7e..d77e2593 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -515,6 +515,18 @@ gcloud compute ssh encoding-worker-b --zone=us-central1-c --project=nomadkaraoke --- +## Render fails "Cannot connect to host :8080" and the VM never answers `/health` + +**Symptoms:** a render that cold-started the primary VM retries 8 times (~90s) and fails with `ClientConnectorError ... Connect call failed`. The backend keeps logging `encoding-worker-b /health unconfirmed; keeping alive (fail-safe)` until the VM idle-stops. + +**Cause (2026-10-09 `f4310aea`, 2026-10-12 `82f48738`):** `startup.sh`'s first gcloud call ran before DHCP, so gcloud cached "not on GCE" (`/root/.config/gcloud/gce` = `False`). Every later gcloud call in that boot had no credentials: no API key, the self-heal refuses to serve, systemd re-runs `startup.sh`, and it hits the same cache forever. + +**Diagnose:** the serial log shows repeated `WARNING: No API key found in Secret Manager`, and on the VM `sudo cat /root/.config/gcloud/gce` prints `False`. + +**Fix:** since v0.268.3 `startup.sh` deletes that cache, waits for the metadata server and retries the secret fetch, so the next restart recovers by itself. By hand: `sudo rm -f /root/.config/gcloud/gce && sudo systemctl restart encoding-worker`. + +--- + ## Job stuck at `rendering_video` (orphaned render) **Symptoms:** Job frozen at `rendering_video` (step 7/10) for a long time with **no error** and **no Retry button** (it's a processing state, not `failed`). diff --git a/infrastructure/encoding-worker/startup.sh b/infrastructure/encoding-worker/startup.sh index 15a54592..42afb50f 100644 --- a/infrastructure/encoding-worker/startup.sh +++ b/infrastructure/encoding-worker/startup.sh @@ -33,12 +33,39 @@ rm -rf "${TMP_DIR}" mkdir -p "${TMP_DIR}" # 1. Fetch API key from Secret Manager +# +# Boot race (2026-10-09, 2026-10-12): if gcloud's first call runs before DHCP +# brings up the metadata server, gcloud writes "False" to its on-GCE cache +# (/root/.config/gcloud/gce) and every later gcloud call in this boot runs +# with no credentials -> empty API key -> the worker's self-heal refuses to +# serve -> systemd re-runs this script, which hits the same poisoned cache, +# forever (until the VM idle-stops). So: drop the cache, wait for the metadata +# server, and retry the fetch instead of giving up after one attempt. echo "[1/5] Fetching API key from Secret Manager..." -ENCODING_API_KEY=$(gcloud secrets versions access latest --secret=encoding-worker-api-key 2>/dev/null || echo "") +rm -f /root/.config/gcloud/gce "${HOME:-/root}/.config/gcloud/gce" +for attempt in $(seq 1 30); do + if curl -sf --connect-timeout 2 --max-time 3 -H "Metadata-Flavor: Google" \ + "http://metadata.google.internal/computeMetadata/v1/instance/id" >/dev/null 2>&1; then + break + fi + echo "Metadata server not reachable yet (attempt ${attempt}/30), waiting 2s..." + sleep 2 +done + +ENCODING_API_KEY="" +for attempt in $(seq 1 6); do + ENCODING_API_KEY=$(gcloud secrets versions access latest --secret=encoding-worker-api-key 2>/dev/null || echo "") + [ -n "$ENCODING_API_KEY" ] && break + echo "API key fetch failed (attempt ${attempt}/6), retrying in 5s..." + rm -f /root/.config/gcloud/gce "${HOME:-/root}/.config/gcloud/gce" + sleep 5 +done if [ -z "$ENCODING_API_KEY" ]; then - echo "WARNING: No API key found in Secret Manager" - echo "Service will start but may reject requests" + # The worker refuses to serve without a key (self_heal boot check), so fail + # here and let systemd re-run the bootstrap rather than start broken. + echo "FATAL: No API key found in Secret Manager; exiting so systemd retries" + exit 1 fi # Read this VM's GCE instance name so the encoding worker can tag its diff --git a/infrastructure/packer/scripts/provision.sh b/infrastructure/packer/scripts/provision.sh index f333cc61..4c86b2a6 100644 --- a/infrastructure/packer/scripts/provision.sh +++ b/infrastructure/packer/scripts/provision.sh @@ -282,7 +282,10 @@ WORKER_DIR="/opt/encoding-worker" BUCKET="gs://karaoke-gen-storage-nomadkaraoke" # Fetch API key. Without it the worker would accept unauthenticated requests, -# so fail and let systemd retry the whole bootstrap instead. +# so fail and let systemd retry the whole bootstrap instead. Drop gcloud's +# on-GCE cache first: a pre-DHCP gcloud call poisons it with "False" (see +# infrastructure/encoding-worker/startup.sh). +rm -f /root/.config/gcloud/gce ENCODING_API_KEY=$(gcloud secrets versions access latest --secret=encoding-worker-api-key 2>/dev/null || echo "") if [ -z "$ENCODING_API_KEY" ]; then echo "FATAL: could not fetch encoding-worker-api-key; exiting so systemd retries" diff --git a/pyproject.toml b/pyproject.toml index a6860f05..cd05e299 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.268.2" +version = "0.268.3" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT"