diff --git a/backend/main.py b/backend/main.py index ddc09ba3..ef08b381 100644 --- a/backend/main.py +++ b/backend/main.py @@ -1,5 +1,8 @@ """ FastAPI application entry point for karaoke generation backend. + +(Trivial comment touch to trigger backend CI on ephemeral runners after + the 2026-05-17 dispatcher e2 fix; safe to remove on next backend edit.) """ import logging from contextlib import asynccontextmanager diff --git a/infrastructure/functions/runner_manager/ephemeral.py b/infrastructure/functions/runner_manager/ephemeral.py index 11581c40..5813ad97 100644 --- a/infrastructure/functions/runner_manager/ephemeral.py +++ b/infrastructure/functions/runner_manager/ephemeral.py @@ -77,7 +77,9 @@ FAMILIES: dict[str, FamilySpec] = { "gpu": FamilySpec( name="gpu", machine_type="n1-standard-4", - disk_size_gb=150, + # GPU image bakes ~14GB of audio-separator models and is built on a + # 200GB boot disk, so the disk we create here must be >= 200GB. + disk_size_gb=200, image_family="gha-runner-gpu", extra_runner_labels=("x64", "gcp", "gpu"), has_gpu=True, @@ -283,12 +285,13 @@ def _build_instance( ] ) - scheduling = compute_v1.Scheduling( - automatic_restart=False, - # GPU VMs cannot live-migrate; everyone uses TERMINATE for simplicity. - on_host_maintenance="TERMINATE", - preemptible=False, - ) + # GPU VMs cannot live-migrate, so they must use TERMINATE. e2 machine types + # reject TERMINATE unless preemptible, so we leave on_host_maintenance unset + # (GCE defaults to MIGRATE) for the non-GPU e2 families. + scheduling_kwargs = {"automatic_restart": False, "preemptible": False} + if family.has_gpu: + scheduling_kwargs["on_host_maintenance"] = "TERMINATE" + scheduling = compute_v1.Scheduling(**scheduling_kwargs) instance = compute_v1.Instance( name=name, diff --git a/infrastructure/functions/runner_manager/test_ephemeral.py b/infrastructure/functions/runner_manager/test_ephemeral.py index 8bb1df24..262145c5 100644 --- a/infrastructure/functions/runner_manager/test_ephemeral.py +++ b/infrastructure/functions/runner_manager/test_ephemeral.py @@ -205,6 +205,40 @@ class TestCreateEphemeralRunner: assert compute_client.insert.call_count == 1 +class TestSchedulingPerFamily: + """e2 instances reject on_host_maintenance=TERMINATE unless preemptible. + + Regression test for the 2026-05-17 cutover bug: dispatcher set TERMINATE + unconditionally, causing every general/build VM create to fail with + `BadRequest('e2 instances do not support onHostMaintenance=TERMINATE + unless they are preemptible.')`. + """ + + def _build_for(self, family_name): + ep = _fresh_module() + ep._build_instance( + name=f"gha-{family_name}-test", + family=ep.FAMILIES[family_name], + zone="us-central1-a", + jit_config="JIT", + image_self_link="projects/p/global/images/family/x", + use_external_ip=False, + ) + return _compute_stub.Scheduling.call_args.kwargs + + def test_e2_general_omits_terminate(self): + kwargs = self._build_for("general") + assert "on_host_maintenance" not in kwargs + + def test_e2_build_omits_terminate(self): + kwargs = self._build_for("build") + assert "on_host_maintenance" not in kwargs + + def test_gpu_keeps_terminate(self): + kwargs = self._build_for("gpu") + assert kwargs.get("on_host_maintenance") == "TERMINATE" + + def _make_instance(name, age_minutes, zone="us-central1-a"): inst = MagicMock() inst.name = name diff --git a/infrastructure/scripts/runner-image-provision.sh b/infrastructure/scripts/runner-image-provision.sh index 5376216f..4353da33 100755 --- a/infrastructure/scripts/runner-image-provision.sh +++ b/infrastructure/scripts/runner-image-provision.sh @@ -283,6 +283,16 @@ if [[ "$VARIANT" != "gpu" ]]; then usermod -aG docker runner fi +# Many CI workflows install OS packages with `sudo apt-get …`. The legacy +# GHA runner VMs had passwordless sudo for the runner user; without it, +# steps like `sudo apt-get install -y google-cloud-cli-firestore-emulator` +# in backend-emulator-tests fail with "a terminal is required to read the +# password". Match the legacy behaviour. +mkdir -p /etc/sudoers.d +echo "runner ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/runner +chmod 0440 /etc/sudoers.d/runner +visudo -cf /etc/sudoers.d/runner + # ==================== GitHub Actions runner binary ==================== ck "phase: github actions runner binary" RUNNER_VERSION="2.332.0" diff --git a/karaoke_gen/__init__.py b/karaoke_gen/__init__.py index 0fcea562..b4ef3a1e 100644 --- a/karaoke_gen/__init__.py +++ b/karaoke_gen/__init__.py @@ -1,5 +1,6 @@ import warnings +# Trigger CI on ephemeral runners after the dispatcher e2 fix. # Suppress specific SyntaxWarnings from third-party packages warnings.filterwarnings("ignore", category=SyntaxWarning, module="pydub.*") warnings.filterwarnings("ignore", category=SyntaxWarning, module="syrics.*") diff --git a/pyproject.toml b/pyproject.toml index c09eb692..0284266b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.174.10" +version = "0.174.11" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT"