mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
fix: encoding-worker boot loop (poisoned gcloud cache) + stale file_urls after change-to-public (v0.268.3) (#1138)
## Summary Two bugs, both hit by job `82f48738` (a private → public visibility change) on 2026-10-12. ### 1. Encoding worker VMs boot into an unrecoverable restart loop If gcloud's first call (the API key fetch in `startup.sh`) happens before DHCP, gcloud caches "not on GCE" in `/root/.config/gcloud/gce`. Every later gcloud call in that boot then has no credentials, so the API key comes back empty, the v0.266.4 self-heal refuses to serve, and systemd re-runs `startup.sh` against the same poisoned cache until the VM idle-stops. worker-b was up 20:30–20:42 and never answered /health, so the render failed with `Cannot connect to host 34.10.189.118:8080`. The same thing happened on 10-09 (`f4310aea`): it was fixed by hand and the durable fix was noted but never shipped. - `startup.sh`: `rm -f /root/.config/gcloud/gce`, wait up to 60s for the metadata server, retry the secret fetch 6× (clearing the cache each time). CI uploads this script on every deploy and bootstrap downloads it on every service start, so existing VM disks pick it up. - Packer `startup-fallback.sh`: clears the cache too (new images). ### 2. Retry after a failed change-to-public 404s forever `change_to_public` deletes screens, `videos/with_vocals.mkv` and the finals but left their `file_urls`. `/retry` then saw `videos.with_vocals` + `instrumental_selection`, skipped the render and 404'd on the deleted MKV. It now drops those refs, as `ThemeRerenderService` already does. ## Testing - New `backend/tests/test_encoding_worker_startup_script.py` and `test_change_visibility.py::test_drops_file_urls_for_deleted_artifacts` - `pytest backend/tests/test_encoding_worker_startup_script.py backend/tests/test_encoding_worker_self_heal.py backend/tests/test_encoding_worker_code_paths.py backend/tests/test_change_visibility.py`: all pass 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_01JUFVxMZME6ydzkmLvyuXRo <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **Bug Fixes** * When a private job is made public, references to deleted screen and video files are cleared before processing resumes, preventing stale files from appearing. * Improved encoding worker startup reliability by retrying API-key retrieval when the metadata service or secret store is temporarily unavailable. <!-- end of auto-generated comment: release notes by coderabbit.ai --> @coderabbitai ignore --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
983f6ecdcd
commit
77bec3327c
7 files changed
+144
-5
No files matched your search
@@ -231,6 +231,12 @@ class VisibilityChangeService:
|
||||
"state_data.render_progress": DELETE_FIELD,
|
||||
"state_data.video_progress": DELETE_FIELD,
|
||||
"state_data.encoding_progress": DELETE_FIELD,
|
||||
# Their blobs were deleted above. A stale videos.with_vocals ref made
|
||||
# a retry after a failed render skip the render and 404 on the
|
||||
# missing MKV (job 82f48738); the workers re-add all three.
|
||||
"file_urls.screens": DELETE_FIELD,
|
||||
"file_urls.videos.with_vocals": DELETE_FIELD,
|
||||
"file_urls.finals": DELETE_FIELD,
|
||||
"outputs_deleted_at": None,
|
||||
"outputs_deleted_by": None,
|
||||
"updated_at": datetime.now(timezone.utc),
|
||||
|
||||
@@ -263,6 +263,35 @@ class TestChangeToPublic:
|
||||
assert style_update["is_private"] is False
|
||||
assert style_update["status"] == "lyrics_complete"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_drops_file_urls_for_deleted_artifacts(self):
|
||||
"""Screens, lyrics video and finals blobs are deleted, so their refs must go
|
||||
too: a stale videos.with_vocals made retry skip the render and 404 (82f48738)."""
|
||||
# The service's own sentinel: other tests may swap google.cloud.firestore_v1 for a mock.
|
||||
from backend.services.visibility_change_service import DELETE_FIELD
|
||||
|
||||
mock_job_manager = MagicMock()
|
||||
mock_job_ref = MagicMock()
|
||||
mock_job_manager.firestore.db.collection.return_value.document.return_value = mock_job_ref
|
||||
|
||||
service = VisibilityChangeService(job_manager=mock_job_manager)
|
||||
job = _make_job(is_private=True, state_data={})
|
||||
|
||||
with patch("backend.services.visibility_change_service.VisibilityChangeService._delete_distributed_outputs", new_callable=AsyncMock), \
|
||||
patch("backend.services.visibility_change_service.StorageService") as mock_storage_cls, \
|
||||
patch("backend.api.routes.file_upload._prepare_theme_for_job", return_value=("p.json", {}, None)), \
|
||||
patch("backend.services.worker_service.get_worker_service") as mock_worker_svc:
|
||||
|
||||
mock_storage_cls.return_value = MagicMock()
|
||||
mock_worker_svc.return_value.trigger_screens_worker = AsyncMock(return_value=True)
|
||||
|
||||
await service.change_to_public("test-123", job, "user@example.com")
|
||||
|
||||
payload = next(c.args[0] for c in mock_job_ref.update.call_args_list if "theme_id" in c.args[0])
|
||||
assert payload["file_urls.screens"] is DELETE_FIELD
|
||||
assert payload["file_urls.videos.with_vocals"] is DELETE_FIELD
|
||||
assert payload["file_urls.finals"] is DELETE_FIELD
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rollback_on_worker_failure(self):
|
||||
mock_job_manager = MagicMock()
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Guards for the encoding-worker boot scripts' gcloud on-GCE cache handling.
|
||||
|
||||
A gcloud call made before DHCP finishes writes "False" to /root/.config/gcloud/gce,
|
||||
so every later gcloud call in that boot has no credentials. The worker then had
|
||||
no API key, self-heal refused to serve, and systemd re-ran startup.sh against the
|
||||
same poisoned cache until the VM idle-stopped (jobs f4310aea 2026-10-09,
|
||||
82f48738 2026-10-12). The scripts must drop that cache before fetching the key.
|
||||
"""
|
||||
import re
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parents[2]
|
||||
STARTUP = REPO / "infrastructure" / "encoding-worker" / "startup.sh"
|
||||
PROVISION = REPO / "infrastructure" / "packer" / "scripts" / "provision.sh"
|
||||
|
||||
CACHE_RM = "rm -f /root/.config/gcloud/gce"
|
||||
SECRET_FETCH = "gcloud secrets versions access latest --secret=encoding-worker-api-key"
|
||||
|
||||
|
||||
def test_startup_script_is_valid_bash():
|
||||
subprocess.run(["bash", "-n", str(STARTUP)], check=True)
|
||||
|
||||
|
||||
def test_startup_clears_gce_cache_before_first_secret_fetch():
|
||||
text = STARTUP.read_text()
|
||||
assert CACHE_RM in text
|
||||
assert text.index(CACHE_RM) < text.index(SECRET_FETCH)
|
||||
|
||||
|
||||
def test_startup_waits_for_metadata_server_before_fetch():
|
||||
text = STARTUP.read_text()
|
||||
metadata_probe = text.index("computeMetadata/v1/instance/id")
|
||||
assert metadata_probe < text.index(SECRET_FETCH)
|
||||
|
||||
|
||||
def test_startup_retries_secret_fetch():
|
||||
text = STARTUP.read_text()
|
||||
loops = re.findall(r"for attempt in \$\(seq 1 \d+\); do\n(.*?)\ndone", text, re.S)
|
||||
assert any(SECRET_FETCH in body and CACHE_RM in body for body in loops)
|
||||
|
||||
|
||||
def test_packer_fallback_clears_gce_cache_before_secret_fetch():
|
||||
text = PROVISION.read_text()
|
||||
fallback = text[text.index("startup-fallback.sh << 'FALLBACK'"):]
|
||||
assert CACHE_RM in fallback
|
||||
assert fallback.index(CACHE_RM) < fallback.index(SECRET_FETCH)
|
||||
|
||||
|
||||
def test_startup_metadata_probe_has_total_timeout():
|
||||
text = STARTUP.read_text()
|
||||
lines = text.splitlines()
|
||||
url_idx = next(i for i, l in enumerate(lines) if "computeMetadata/v1/instance/id" in l)
|
||||
# The curl flags sit on the line before the URL continuation.
|
||||
assert "--max-time" in lines[url_idx - 1] + lines[url_idx]
|
||||
|
||||
|
||||
def test_startup_exits_nonzero_without_api_key():
|
||||
text = STARTUP.read_text()
|
||||
block = text[text.index('if [ -z "$ENCODING_API_KEY" ]; then'):]
|
||||
block = block[: block.index("fi")]
|
||||
assert "exit 1" in block
|
||||
@@ -515,6 +515,18 @@ gcloud compute ssh encoding-worker-b --zone=us-central1-c --project=nomadkaraoke
|
||||
|
||||
---
|
||||
|
||||
## Render fails "Cannot connect to host <worker-ip>:8080" and the VM never answers `/health`
|
||||
|
||||
**Symptoms:** a render that cold-started the primary VM retries 8 times (~90s) and fails with `ClientConnectorError ... Connect call failed`. The backend keeps logging `encoding-worker-b /health unconfirmed; keeping alive (fail-safe)` until the VM idle-stops.
|
||||
|
||||
**Cause (2026-10-09 `f4310aea`, 2026-10-12 `82f48738`):** `startup.sh`'s first gcloud call ran before DHCP, so gcloud cached "not on GCE" (`/root/.config/gcloud/gce` = `False`). Every later gcloud call in that boot had no credentials: no API key, the self-heal refuses to serve, systemd re-runs `startup.sh`, and it hits the same cache forever.
|
||||
|
||||
**Diagnose:** the serial log shows repeated `WARNING: No API key found in Secret Manager`, and on the VM `sudo cat /root/.config/gcloud/gce` prints `False`.
|
||||
|
||||
**Fix:** since v0.268.3 `startup.sh` deletes that cache, waits for the metadata server and retries the secret fetch, so the next restart recovers by itself. By hand: `sudo rm -f /root/.config/gcloud/gce && sudo systemctl restart encoding-worker`.
|
||||
|
||||
---
|
||||
|
||||
## Job stuck at `rendering_video` (orphaned render)
|
||||
|
||||
**Symptoms:** Job frozen at `rendering_video` (step 7/10) for a long time with **no error** and **no Retry button** (it's a processing state, not `failed`).
|
||||
|
||||
@@ -33,12 +33,39 @@ rm -rf "${TMP_DIR}"
|
||||
mkdir -p "${TMP_DIR}"
|
||||
|
||||
# 1. Fetch API key from Secret Manager
|
||||
#
|
||||
# Boot race (2026-10-09, 2026-10-12): if gcloud's first call runs before DHCP
|
||||
# brings up the metadata server, gcloud writes "False" to its on-GCE cache
|
||||
# (/root/.config/gcloud/gce) and every later gcloud call in this boot runs
|
||||
# with no credentials -> empty API key -> the worker's self-heal refuses to
|
||||
# serve -> systemd re-runs this script, which hits the same poisoned cache,
|
||||
# forever (until the VM idle-stops). So: drop the cache, wait for the metadata
|
||||
# server, and retry the fetch instead of giving up after one attempt.
|
||||
echo "[1/5] Fetching API key from Secret Manager..."
|
||||
ENCODING_API_KEY=$(gcloud secrets versions access latest --secret=encoding-worker-api-key 2>/dev/null || echo "")
|
||||
rm -f /root/.config/gcloud/gce "${HOME:-/root}/.config/gcloud/gce"
|
||||
for attempt in $(seq 1 30); do
|
||||
if curl -sf --connect-timeout 2 --max-time 3 -H "Metadata-Flavor: Google" \
|
||||
"http://metadata.google.internal/computeMetadata/v1/instance/id" >/dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
echo "Metadata server not reachable yet (attempt ${attempt}/30), waiting 2s..."
|
||||
sleep 2
|
||||
done
|
||||
|
||||
ENCODING_API_KEY=""
|
||||
for attempt in $(seq 1 6); do
|
||||
ENCODING_API_KEY=$(gcloud secrets versions access latest --secret=encoding-worker-api-key 2>/dev/null || echo "")
|
||||
[ -n "$ENCODING_API_KEY" ] && break
|
||||
echo "API key fetch failed (attempt ${attempt}/6), retrying in 5s..."
|
||||
rm -f /root/.config/gcloud/gce "${HOME:-/root}/.config/gcloud/gce"
|
||||
sleep 5
|
||||
done
|
||||
|
||||
if [ -z "$ENCODING_API_KEY" ]; then
|
||||
echo "WARNING: No API key found in Secret Manager"
|
||||
echo "Service will start but may reject requests"
|
||||
# The worker refuses to serve without a key (self_heal boot check), so fail
|
||||
# here and let systemd re-run the bootstrap rather than start broken.
|
||||
echo "FATAL: No API key found in Secret Manager; exiting so systemd retries"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Read this VM's GCE instance name so the encoding worker can tag its
|
||||
|
||||
@@ -282,7 +282,10 @@ WORKER_DIR="/opt/encoding-worker"
|
||||
BUCKET="gs://karaoke-gen-storage-nomadkaraoke"
|
||||
|
||||
# Fetch API key. Without it the worker would accept unauthenticated requests,
|
||||
# so fail and let systemd retry the whole bootstrap instead.
|
||||
# so fail and let systemd retry the whole bootstrap instead. Drop gcloud's
|
||||
# on-GCE cache first: a pre-DHCP gcloud call poisons it with "False" (see
|
||||
# infrastructure/encoding-worker/startup.sh).
|
||||
rm -f /root/.config/gcloud/gce
|
||||
ENCODING_API_KEY=$(gcloud secrets versions access latest --secret=encoding-worker-api-key 2>/dev/null || echo "")
|
||||
if [ -z "$ENCODING_API_KEY" ]; then
|
||||
echo "FATAL: could not fetch encoding-worker-api-key; exiting so systemd retries"
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.268.2"
|
||||
version = "0.268.3"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
Reference in new issue
Block a user