mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
fix(runner-dispatcher): omit TERMINATE for e2 families (unblocks ephemeral cutover) (#776)
## Summary Fixes three cutover bugs that left the ephemeral runner dispatcher silently broken since 2026-05-17T04:56Z: 1. **`on_host_maintenance="TERMINATE"` set unconditionally** — e2 machine types reject TERMINATE unless preemptible, so 100% of general/build VM creates failed. Fix: only set TERMINATE on GPU (n1) families where it's required because attached GPUs can't live-migrate. 3 regression tests added in `test_ephemeral.py`. 2. **GPU `disk_size_gb=150` but image is 200GB** — GCE rejects boot disks smaller than the source image. Fix: raise GPU disk to 200GB. 3. **Runner user has no passwordless sudo** — workflow steps like `sudo apt-get install -y google-cloud-cli-firestore-emulator` (backend-emulator-tests) and `sudo apt-get install -y ffmpeg` (package-*, GPU integration tests) failed with "a terminal is required to read the password". The legacy GHA runner VMs had NOPASSWD sudo. Fix: add `/etc/sudoers.d/runner` in the image provision script. Fixes 1 + 2 are dispatcher code (already applied via `pulumi up --target ...:runner-manager-source ...:runner-manager-function` from this branch). Fix 3 requires a new image build (triggered: build-runner-images.yml on this branch). ## Why the trivial file touches `backend/main.py` and `karaoke_gen/__init__.py` carry one-line comments to flip the `backend` and `package` `dorny/paths-filter` outputs so this PR exercises **all five PR-triggered self-hosted jobs** end-to-end on ephemeral runners. Verification PR (paired): nomadkaraoke/python-audio-separator#286. ## Test plan - [x] `pytest test_ephemeral.py` — 26 passed locally (3 new scheduling tests) - [x] Pulumi applied locally (both iterations, function update verified) - [x] First ephemeral general VM create succeeded (4 RUNNING; previously 100% failure) - [ ] Image rebuild completes for general+build+gpu with new sudo provision - [ ] PR CI passes with all 5 self-hosted jobs running on the new image - [ ] python-audio-separator#286 GPU integration tests pass on the new GPU image - [ ] After merge: `deploy-backend` succeeds on ephemeral build runner; prod /api/health/detailed reports new version - [ ] Phase 4 decommission of 7 legacy VMs+disks (separate PR, ~$220/mo saving) ## Context - `karaoke-gen/docs/EPHEMERAL-GHA-RUNNERS.md` - `docs/archive/2026-05-16-ephemeral-gha-runners-plan.md` (workspace) - Memory: `project_cost_sprint_may2026.md` @coderabbitai ignore 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
241ad41096
commit
424b908d71
6 files changed
+59
-8
No files matched your search
@@ -1,5 +1,8 @@
|
||||
"""
|
||||
FastAPI application entry point for karaoke generation backend.
|
||||
|
||||
(Trivial comment touch to trigger backend CI on ephemeral runners after
|
||||
the 2026-05-17 dispatcher e2 fix; safe to remove on next backend edit.)
|
||||
"""
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
@@ -77,7 +77,9 @@ FAMILIES: dict[str, FamilySpec] = {
|
||||
"gpu": FamilySpec(
|
||||
name="gpu",
|
||||
machine_type="n1-standard-4",
|
||||
disk_size_gb=150,
|
||||
# GPU image bakes ~14GB of audio-separator models and is built on a
|
||||
# 200GB boot disk, so the disk we create here must be >= 200GB.
|
||||
disk_size_gb=200,
|
||||
image_family="gha-runner-gpu",
|
||||
extra_runner_labels=("x64", "gcp", "gpu"),
|
||||
has_gpu=True,
|
||||
@@ -283,12 +285,13 @@ def _build_instance(
|
||||
]
|
||||
)
|
||||
|
||||
scheduling = compute_v1.Scheduling(
|
||||
automatic_restart=False,
|
||||
# GPU VMs cannot live-migrate; everyone uses TERMINATE for simplicity.
|
||||
on_host_maintenance="TERMINATE",
|
||||
preemptible=False,
|
||||
)
|
||||
# GPU VMs cannot live-migrate, so they must use TERMINATE. e2 machine types
|
||||
# reject TERMINATE unless preemptible, so we leave on_host_maintenance unset
|
||||
# (GCE defaults to MIGRATE) for the non-GPU e2 families.
|
||||
scheduling_kwargs = {"automatic_restart": False, "preemptible": False}
|
||||
if family.has_gpu:
|
||||
scheduling_kwargs["on_host_maintenance"] = "TERMINATE"
|
||||
scheduling = compute_v1.Scheduling(**scheduling_kwargs)
|
||||
|
||||
instance = compute_v1.Instance(
|
||||
name=name,
|
||||
|
||||
@@ -205,6 +205,40 @@ class TestCreateEphemeralRunner:
|
||||
assert compute_client.insert.call_count == 1
|
||||
|
||||
|
||||
class TestSchedulingPerFamily:
|
||||
"""e2 instances reject on_host_maintenance=TERMINATE unless preemptible.
|
||||
|
||||
Regression test for the 2026-05-17 cutover bug: dispatcher set TERMINATE
|
||||
unconditionally, causing every general/build VM create to fail with
|
||||
`BadRequest('e2 instances do not support onHostMaintenance=TERMINATE
|
||||
unless they are preemptible.')`.
|
||||
"""
|
||||
|
||||
def _build_for(self, family_name):
|
||||
ep = _fresh_module()
|
||||
ep._build_instance(
|
||||
name=f"gha-{family_name}-test",
|
||||
family=ep.FAMILIES[family_name],
|
||||
zone="us-central1-a",
|
||||
jit_config="JIT",
|
||||
image_self_link="projects/p/global/images/family/x",
|
||||
use_external_ip=False,
|
||||
)
|
||||
return _compute_stub.Scheduling.call_args.kwargs
|
||||
|
||||
def test_e2_general_omits_terminate(self):
|
||||
kwargs = self._build_for("general")
|
||||
assert "on_host_maintenance" not in kwargs
|
||||
|
||||
def test_e2_build_omits_terminate(self):
|
||||
kwargs = self._build_for("build")
|
||||
assert "on_host_maintenance" not in kwargs
|
||||
|
||||
def test_gpu_keeps_terminate(self):
|
||||
kwargs = self._build_for("gpu")
|
||||
assert kwargs.get("on_host_maintenance") == "TERMINATE"
|
||||
|
||||
|
||||
def _make_instance(name, age_minutes, zone="us-central1-a"):
|
||||
inst = MagicMock()
|
||||
inst.name = name
|
||||
|
||||
@@ -283,6 +283,16 @@ if [[ "$VARIANT" != "gpu" ]]; then
|
||||
usermod -aG docker runner
|
||||
fi
|
||||
|
||||
# Many CI workflows install OS packages with `sudo apt-get …`. The legacy
|
||||
# GHA runner VMs had passwordless sudo for the runner user; without it,
|
||||
# steps like `sudo apt-get install -y google-cloud-cli-firestore-emulator`
|
||||
# in backend-emulator-tests fail with "a terminal is required to read the
|
||||
# password". Match the legacy behaviour.
|
||||
mkdir -p /etc/sudoers.d
|
||||
echo "runner ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/runner
|
||||
chmod 0440 /etc/sudoers.d/runner
|
||||
visudo -cf /etc/sudoers.d/runner
|
||||
|
||||
# ==================== GitHub Actions runner binary ====================
|
||||
ck "phase: github actions runner binary"
|
||||
RUNNER_VERSION="2.332.0"
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import warnings
|
||||
|
||||
# Trigger CI on ephemeral runners after the dispatcher e2 fix.
|
||||
# Suppress specific SyntaxWarnings from third-party packages
|
||||
warnings.filterwarnings("ignore", category=SyntaxWarning, module="pydub.*")
|
||||
warnings.filterwarnings("ignore", category=SyntaxWarning, module="syrics.*")
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.174.10"
|
||||
version = "0.174.11"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
Reference in new issue
Block a user