From 424b908d71ab13b3f03b6a658351be9c66cd25fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrew=20Beveridge=20=E2=98=84=EF=B8=8F?= Date: Sun, 17 May 2026 22:42:01 -0400 Subject: [PATCH] fix(runner-dispatcher): omit TERMINATE for e2 families (unblocks ephemeral cutover) (#776) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Fixes three cutover bugs that left the ephemeral runner dispatcher silently broken since 2026-05-17T04:56Z: 1. **`on_host_maintenance="TERMINATE"` set unconditionally** — e2 machine types reject TERMINATE unless preemptible, so 100% of general/build VM creates failed. Fix: only set TERMINATE on GPU (n1) families where it's required because attached GPUs can't live-migrate. 3 regression tests added in `test_ephemeral.py`. 2. **GPU `disk_size_gb=150` but image is 200GB** — GCE rejects boot disks smaller than the source image. Fix: raise GPU disk to 200GB. 3. **Runner user has no passwordless sudo** — workflow steps like `sudo apt-get install -y google-cloud-cli-firestore-emulator` (backend-emulator-tests) and `sudo apt-get install -y ffmpeg` (package-*, GPU integration tests) failed with "a terminal is required to read the password". The legacy GHA runner VMs had NOPASSWD sudo. Fix: add `/etc/sudoers.d/runner` in the image provision script. Fixes 1 + 2 are dispatcher code (already applied via `pulumi up --target ...:runner-manager-source ...:runner-manager-function` from this branch). Fix 3 requires a new image build (triggered: build-runner-images.yml on this branch). ## Why the trivial file touches `backend/main.py` and `karaoke_gen/__init__.py` carry one-line comments to flip the `backend` and `package` `dorny/paths-filter` outputs so this PR exercises **all five PR-triggered self-hosted jobs** end-to-end on ephemeral runners. Verification PR (paired): nomadkaraoke/python-audio-separator#286. ## Test plan - [x] `pytest test_ephemeral.py` — 26 passed locally (3 new scheduling tests) - [x] Pulumi applied locally (both iterations, function update verified) - [x] First ephemeral general VM create succeeded (4 RUNNING; previously 100% failure) - [ ] Image rebuild completes for general+build+gpu with new sudo provision - [ ] PR CI passes with all 5 self-hosted jobs running on the new image - [ ] python-audio-separator#286 GPU integration tests pass on the new GPU image - [ ] After merge: `deploy-backend` succeeds on ephemeral build runner; prod /api/health/detailed reports new version - [ ] Phase 4 decommission of 7 legacy VMs+disks (separate PR, ~$220/mo saving) ## Context - `karaoke-gen/docs/EPHEMERAL-GHA-RUNNERS.md` - `docs/archive/2026-05-16-ephemeral-gha-runners-plan.md` (workspace) - Memory: `project_cost_sprint_may2026.md` @coderabbitai ignore 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- backend/main.py | 3 ++ .../functions/runner_manager/ephemeral.py | 17 ++++++---- .../runner_manager/test_ephemeral.py | 34 +++++++++++++++++++ .../scripts/runner-image-provision.sh | 10 ++++++ karaoke_gen/__init__.py | 1 + pyproject.toml | 2 +- 6 files changed, 59 insertions(+), 8 deletions(-) diff --git a/backend/main.py b/backend/main.py index ddc09ba3..ef08b381 100644 --- a/backend/main.py +++ b/backend/main.py @@ -1,5 +1,8 @@ """ FastAPI application entry point for karaoke generation backend. + +(Trivial comment touch to trigger backend CI on ephemeral runners after + the 2026-05-17 dispatcher e2 fix; safe to remove on next backend edit.) """ import logging from contextlib import asynccontextmanager diff --git a/infrastructure/functions/runner_manager/ephemeral.py b/infrastructure/functions/runner_manager/ephemeral.py index 11581c40..5813ad97 100644 --- a/infrastructure/functions/runner_manager/ephemeral.py +++ b/infrastructure/functions/runner_manager/ephemeral.py @@ -77,7 +77,9 @@ FAMILIES: dict[str, FamilySpec] = { "gpu": FamilySpec( name="gpu", machine_type="n1-standard-4", - disk_size_gb=150, + # GPU image bakes ~14GB of audio-separator models and is built on a + # 200GB boot disk, so the disk we create here must be >= 200GB. + disk_size_gb=200, image_family="gha-runner-gpu", extra_runner_labels=("x64", "gcp", "gpu"), has_gpu=True, @@ -283,12 +285,13 @@ def _build_instance( ] ) - scheduling = compute_v1.Scheduling( - automatic_restart=False, - # GPU VMs cannot live-migrate; everyone uses TERMINATE for simplicity. - on_host_maintenance="TERMINATE", - preemptible=False, - ) + # GPU VMs cannot live-migrate, so they must use TERMINATE. e2 machine types + # reject TERMINATE unless preemptible, so we leave on_host_maintenance unset + # (GCE defaults to MIGRATE) for the non-GPU e2 families. + scheduling_kwargs = {"automatic_restart": False, "preemptible": False} + if family.has_gpu: + scheduling_kwargs["on_host_maintenance"] = "TERMINATE" + scheduling = compute_v1.Scheduling(**scheduling_kwargs) instance = compute_v1.Instance( name=name, diff --git a/infrastructure/functions/runner_manager/test_ephemeral.py b/infrastructure/functions/runner_manager/test_ephemeral.py index 8bb1df24..262145c5 100644 --- a/infrastructure/functions/runner_manager/test_ephemeral.py +++ b/infrastructure/functions/runner_manager/test_ephemeral.py @@ -205,6 +205,40 @@ class TestCreateEphemeralRunner: assert compute_client.insert.call_count == 1 +class TestSchedulingPerFamily: + """e2 instances reject on_host_maintenance=TERMINATE unless preemptible. + + Regression test for the 2026-05-17 cutover bug: dispatcher set TERMINATE + unconditionally, causing every general/build VM create to fail with + `BadRequest('e2 instances do not support onHostMaintenance=TERMINATE + unless they are preemptible.')`. + """ + + def _build_for(self, family_name): + ep = _fresh_module() + ep._build_instance( + name=f"gha-{family_name}-test", + family=ep.FAMILIES[family_name], + zone="us-central1-a", + jit_config="JIT", + image_self_link="projects/p/global/images/family/x", + use_external_ip=False, + ) + return _compute_stub.Scheduling.call_args.kwargs + + def test_e2_general_omits_terminate(self): + kwargs = self._build_for("general") + assert "on_host_maintenance" not in kwargs + + def test_e2_build_omits_terminate(self): + kwargs = self._build_for("build") + assert "on_host_maintenance" not in kwargs + + def test_gpu_keeps_terminate(self): + kwargs = self._build_for("gpu") + assert kwargs.get("on_host_maintenance") == "TERMINATE" + + def _make_instance(name, age_minutes, zone="us-central1-a"): inst = MagicMock() inst.name = name diff --git a/infrastructure/scripts/runner-image-provision.sh b/infrastructure/scripts/runner-image-provision.sh index 5376216f..4353da33 100755 --- a/infrastructure/scripts/runner-image-provision.sh +++ b/infrastructure/scripts/runner-image-provision.sh @@ -283,6 +283,16 @@ if [[ "$VARIANT" != "gpu" ]]; then usermod -aG docker runner fi +# Many CI workflows install OS packages with `sudo apt-get …`. The legacy +# GHA runner VMs had passwordless sudo for the runner user; without it, +# steps like `sudo apt-get install -y google-cloud-cli-firestore-emulator` +# in backend-emulator-tests fail with "a terminal is required to read the +# password". Match the legacy behaviour. +mkdir -p /etc/sudoers.d +echo "runner ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/runner +chmod 0440 /etc/sudoers.d/runner +visudo -cf /etc/sudoers.d/runner + # ==================== GitHub Actions runner binary ==================== ck "phase: github actions runner binary" RUNNER_VERSION="2.332.0" diff --git a/karaoke_gen/__init__.py b/karaoke_gen/__init__.py index 0fcea562..b4ef3a1e 100644 --- a/karaoke_gen/__init__.py +++ b/karaoke_gen/__init__.py @@ -1,5 +1,6 @@ import warnings +# Trigger CI on ephemeral runners after the dispatcher e2 fix. # Suppress specific SyntaxWarnings from third-party packages warnings.filterwarnings("ignore", category=SyntaxWarning, module="pydub.*") warnings.filterwarnings("ignore", category=SyntaxWarning, module="syrics.*") diff --git a/pyproject.toml b/pyproject.toml index c09eb692..0284266b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.174.10" +version = "0.174.11" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT"