Files
karaoke-gen/backend/tests/test_encoding_worker_self_heal.py
Andrew Beveridge ☄️andClaude Opus 5.5 0ea2a23766 fix(encoding): worker self-heals after a bad boot; default metadata mTLS off (v0.266.4) (#1131)
## Summary

On 2026-10-06 the primary encoding VM (`encoding-worker-b`) booted
before its network was up. `bootstrap.sh` couldn't download `startup.sh`
from GCS (`Network is unreachable`) and fell back to the image's
`startup-fallback.sh`. That script writes an env file without
`ENCODING_API_KEY` or `GCE_METADATA_MTLS_MODE=none`, so google-auth used
mTLS to the metadata server and every render and preview failed with
`CERTIFICATE_VERIFY_FAILED`. The backend deliberately keeps retrying the
primary on infra errors, so a Randy Vild render (job `ebfe4344`) looped
in `render_pending_capacity` for an hour until the service was restarted
by hand.

## Changes

- **Worker defaults `GCE_METADATA_MTLS_MODE=none`** in
`backend/services/gce_encoding/main.py` before any google import, so
every boot path gets it. Ships with the wheel; no image rebuild needed.
- **Worker self-heal** (`backend/services/gce_encoding/self_heal.py`),
active only on worker VMs (systemd `INVOCATION_ID` **and**
`/opt/encoding-worker/bootstrap.sh`):
- Boot: refuses to start if the API key is missing or it can't get GCP
credentials. uvicorn exits, and systemd re-runs the bootstrap.
- Runtime: after a job fails with an infra/auth error, it waits 30s (so
the backend sees the failure), re-probes credentials, and exits once
idle if they're still failing.
- **Packer bootstrap** (`provision.sh`) retries the `startup.sh`
download for about 2 minutes. The fallback writes
`GCE_METADATA_MTLS_MODE=none` and exits if it can't fetch the API key,
instead of starting unauthenticated. Takes effect with the next image
build.
- **Alerting:** an infra failure while the primary is serving now logs
at ERROR (was WARNING).
- **UI:** `state_data.render_pending_capacity` is cleared once a retried
render reports progress, so the job card stops saying "(waiting for
encoding server availability)" during a healthy render.
- Docs: TROUBLESHOOTING runbook, LESSONS-LEARNED, README.

## Testing

- New: `test_encoding_worker_self_heal.py` (probe/boot/restart logic,
`main.py` wiring, mTLS default) and
`test_render_capacity_marker_clear.py`.
- `make test`: 5799 backend unit, 150 emulator, 1646 frontend, all
passing.
- Local CodeRabbit review: no findings.

@coderabbitai ignore

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-06 11:55:51 -04:00

247 lines
8.6 KiB
Python

"""Encoding worker self-heal (backend/services/gce_encoding/self_heal.py).
Incident 2026-10-06: the primary booted before its network was up, fell back to a
startup script that omitted ENCODING_API_KEY and GCE_METADATA_MTLS_MODE=none, and
then failed every job with CERTIFICATE_VERIFY_FAILED until restarted by hand.
"""
import threading
import pytest
from backend.services.gce_encoding import self_heal
INFRA_ERROR = (
"Failed to retrieve https://metadata.google.internal/computeMetadata/v1/instance/"
"service-accounts/default/?recursive=true from the Google Compute Engine metadata "
"service. Compute Engine Metadata server unavailable."
)
@pytest.fixture
def on_worker_vm(monkeypatch):
monkeypatch.setattr(self_heal, "running_on_worker_vm", lambda: True)
@pytest.fixture(autouse=True)
def reset_restart_flag():
self_heal._restart_pending = False
yield
self_heal._restart_pending = False
class TestRunningOnWorkerVm:
def test_requires_systemd_and_bootstrap(self, monkeypatch, tmp_path):
bootstrap = tmp_path / "bootstrap.sh"
monkeypatch.setattr(self_heal, "BOOTSTRAP_PATH", str(bootstrap))
monkeypatch.setenv("INVOCATION_ID", "abc")
assert self_heal.running_on_worker_vm() is False # no bootstrap.sh (e.g. CI runner)
bootstrap.touch()
assert self_heal.running_on_worker_vm() is True
monkeypatch.delenv("INVOCATION_ID")
assert self_heal.running_on_worker_vm() is False
class TestProbeCredentials:
def test_healthy(self):
assert self_heal.probe_credentials(refresh=lambda: None) is None
def test_retries_then_succeeds(self):
calls = {"n": 0}
def refresh():
calls["n"] += 1
if calls["n"] < 3:
raise RuntimeError("blip")
assert self_heal.probe_credentials(attempts=3, retry_seconds=0, refresh=refresh) is None
assert calls["n"] == 3
def test_returns_last_error_when_always_failing(self):
def refresh():
raise RuntimeError(INFRA_ERROR)
assert self_heal.probe_credentials(attempts=2, retry_seconds=0, refresh=refresh) == INFRA_ERROR
class TestBootHealthProblem:
def test_off_vm_never_reports(self, monkeypatch):
monkeypatch.setattr(self_heal, "running_on_worker_vm", lambda: False)
monkeypatch.delenv("ENCODING_API_KEY", raising=False)
assert self_heal.boot_health_problem(probe=lambda: "broken") is None
def test_healthy_boot(self, on_worker_vm, monkeypatch):
monkeypatch.setenv("ENCODING_API_KEY", "k")
assert self_heal.boot_health_problem(probe=lambda: None) is None
def test_missing_api_key(self, on_worker_vm, monkeypatch):
monkeypatch.delenv("ENCODING_API_KEY", raising=False)
problem = self_heal.boot_health_problem(probe=lambda: None)
assert "ENCODING_API_KEY" in problem
def test_credentials_failing(self, on_worker_vm, monkeypatch):
monkeypatch.setenv("ENCODING_API_KEY", "k")
problem = self_heal.boot_health_problem(probe=lambda: INFRA_ERROR)
assert "cannot obtain GCP credentials" in problem
def _wait_for(event, timeout=5):
assert event.wait(timeout), "self-heal thread did not finish"
class TestScheduleRestart:
def test_ignores_non_infra_errors(self, on_worker_vm):
exited = []
assert not self_heal.schedule_restart_if_unhealthy(
"ffmpeg: invalid codec", lambda: False, delay_seconds=0,
probe=lambda: INFRA_ERROR, exit_process=exited.append,
)
assert exited == []
def test_off_vm_does_nothing(self, monkeypatch):
monkeypatch.setattr(self_heal, "running_on_worker_vm", lambda: False)
assert not self_heal.schedule_restart_if_unhealthy(
INFRA_ERROR, lambda: False, delay_seconds=0,
probe=lambda: INFRA_ERROR, exit_process=lambda code: None,
)
def test_exits_when_still_broken_and_idle(self, on_worker_vm):
done = threading.Event()
exited = []
def exit_process(code):
exited.append(code)
done.set()
assert self_heal.schedule_restart_if_unhealthy(
INFRA_ERROR, lambda: False, delay_seconds=0,
probe=lambda: INFRA_ERROR, exit_process=exit_process,
)
_wait_for(done)
assert exited == [1]
def test_no_exit_when_credentials_recover(self, on_worker_vm):
done = threading.Event()
exited = []
def probe():
done.set()
return None
assert self_heal.schedule_restart_if_unhealthy(
INFRA_ERROR, lambda: False, delay_seconds=0,
probe=probe, exit_process=exited.append,
)
_wait_for(done)
# Let the thread finish and release the pending flag.
for _ in range(100):
if not self_heal._restart_pending:
break
threading.Event().wait(0.01)
assert exited == []
assert self_heal._restart_pending is False
def test_waits_for_in_flight_jobs_before_exiting(self, on_worker_vm):
done = threading.Event()
exited = []
active = iter([True, True, False])
def exit_process(code):
exited.append(code)
done.set()
self_heal.schedule_restart_if_unhealthy(
INFRA_ERROR, lambda: next(active), delay_seconds=0,
probe=lambda: INFRA_ERROR, exit_process=exit_process,
)
_wait_for(done)
assert exited == [1]
def test_only_one_check_at_a_time(self, on_worker_vm):
release = threading.Event()
def probe():
release.wait(5)
return None
assert self_heal.schedule_restart_if_unhealthy(
INFRA_ERROR, lambda: False, delay_seconds=0, probe=probe, exit_process=lambda c: None,
)
assert not self_heal.schedule_restart_if_unhealthy(
INFRA_ERROR, lambda: False, delay_seconds=0, probe=probe, exit_process=lambda c: None,
)
release.set()
# ---------------------------------------------------------------------------
# Wiring in gce_encoding/main.py
# ---------------------------------------------------------------------------
import asyncio
import os
import sys
from unittest.mock import MagicMock, patch
@pytest.fixture
def worker_module():
with patch.dict(os.environ, {}, clear=False):
os.environ.pop("GCE_METADATA_MTLS_MODE", None)
with patch("google.cloud.storage.Client", return_value=MagicMock()):
sys.modules.pop("backend.services.gce_encoding.main", None)
import backend.services.gce_encoding.main as m
yield m
sys.modules.pop("backend.services.gce_encoding.main", None)
def test_import_defaults_metadata_mtls_off(worker_module):
assert os.environ["GCE_METADATA_MTLS_MODE"] == "none"
def test_import_keeps_explicit_mtls_mode():
with patch.dict(os.environ, {"GCE_METADATA_MTLS_MODE": "strict"}, clear=False):
with patch("google.cloud.storage.Client", return_value=MagicMock()):
sys.modules.pop("backend.services.gce_encoding.main", None)
import backend.services.gce_encoding.main # noqa: F401
sys.modules.pop("backend.services.gce_encoding.main", None)
assert os.environ["GCE_METADATA_MTLS_MODE"] == "strict"
def _run_lifespan(m):
async def go():
async with m.lifespan(m.app):
pass
asyncio.run(go())
def test_lifespan_refuses_to_start_when_boot_unhealthy(worker_module, monkeypatch):
monkeypatch.setattr(worker_module.self_heal, "boot_health_problem", lambda: "no API key")
with pytest.raises(RuntimeError, match="booted unhealthy: no API key"):
_run_lifespan(worker_module)
def test_lifespan_starts_when_healthy(worker_module, monkeypatch):
monkeypatch.setattr(worker_module.self_heal, "boot_health_problem", lambda: None)
monkeypatch.setattr(worker_module.persister, "mark_orphans_failed_on_startup", lambda jobs: 0)
_run_lifespan(worker_module)
def test_failed_job_schedules_self_heal(worker_module, monkeypatch):
seen = []
monkeypatch.setattr(
worker_module.self_heal, "schedule_restart_if_unhealthy",
lambda error_text, has_active: seen.append((error_text, has_active())),
)
worker_module.jobs.clear()
worker_module._self_heal_after_failure(RuntimeError(INFRA_ERROR))
assert seen == [(INFRA_ERROR, False)]
def test_self_heal_scheduling_errors_never_propagate(worker_module, monkeypatch):
def boom(*a, **k):
raise RuntimeError("thread start failed")
monkeypatch.setattr(worker_module.self_heal, "schedule_restart_if_unhealthy", boom)
worker_module._self_heal_after_failure(RuntimeError(INFRA_ERROR)) # must not raise