From a1cbe1c0d4edf3347a51a3d48fe79a61d08efcfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrew=20Beveridge=20=E2=98=84=EF=B8=8F?= Date: Thu, 5 Nov 2026 18:50:54 -0500 Subject: [PATCH] feat(encoding): retire the GCE c4d pair; run on AWS + c2df only (0.293.0) (#1255) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why Since 2026-10-31 the AWS EC2 worker (`aws-encoding-worker-1`) has handled every encode and distribution job. 27 jobs went to it, with no Cloud Run fallbacks, no `encoding_stuck`, and nothing encoding-related in #alerts. The c4d Spot pair (`encoding-worker-a`/`-b`) now only costs ~$8/mo of Hyperdisk. `encoding-worker-fallback-c2df` stays as the GCP fallback. ## What Deleting the pair needs code changes first, because several paths assumed it exists: - **`encoding_worker_manager`** - The primary is optional: an empty `primary_vm` means none. - `ranked_candidates` skips a missing primary and always sets `is_primary`, so with no primary a successful start records the AWS/c2df override. Previously the legacy index-0 rule would have cleared it. - `ensure_primary_running` and `ensure_preferred_running` start a ranked candidate. If there are no candidates they raise `NO_CANDIDATES`. - `demote_active_worker` looks up the override's `machine_type`, so the AWS box can now be cooled down. It then clears the override so the retry goes elsewhere. - **`encoding_service`** - When `active_url` is empty, it targets the top-ranked candidate. - It also invalidates its cached URL after a single-candidate fallback. - **CI deploy (`ci.yml` + `deploy_promote.py`)** - While AWS serves, a c2df green is validate-only: it writes `validated_version`, keeps the AWS override, and stops the green. Before this, it would have replaced the AWS override with c2df. - The worker-change gate now diffs against `validated_version`, because `primary_version` stops moving. - An in-place restart of a serving override first waits for it to drain (15-min cap). If it is still busy, the restart is skipped. - **Idle function:** a missing VM is logged as a WARNING returning `NOT_FOUND`, not an ERROR. - **Error monitor:** watches c2df instead of a/b. - **Pulumi:** drops the a/b VMs and static IPs. The prod preview shows exactly 4 deletes. A separate, unrelated `youtube-comment-autoreply-scheduler` drift shows up too; it will not be applied. - **Docs:** CLAUDE.md, ARCHITECTURE, DEVELOPMENT and TROUBLESHOOTING. ## Rollout (after merge + deploy) 1. Back up `config/encoding-worker`, then set `primary_vm`, `primary_ip`, `secondary_vm` and `secondary_ip` to `""`. 2. Apply `pulumi destroy --target` on the 4 a/b URNs only. 3. Redeploy the idle function with `deploy.sh`. 4. Verify: one prod encode on AWS, warmup, and the logs. **Rollback:** in Firestore, delete `speed_rank_overrides` and set `aws_video_worker=false`. Restoring the pair would need a revert plus `pulumi up`, which gives new static IPs. ## Testing - New `backend/tests/test_encoding_worker_no_primary.py` (13 tests), plus new deploy_promote, idle-function and error-monitor tests. - Full backend suite: 7188 passed. The 8 `test_emulator_integration` failures also fail on main, because of a local emulator from another session. - CodeRabbit CLI: no findings. An independent review found 2 issues, both fixed: the demote retry path, and the in-place restart drain. 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_012RYLVTkatGnrYFhCzq4PTe @coderabbitai ignore Co-authored-by: Claude Opus 5.5 --- .github/workflows/ci.yml | 50 ++++- CLAUDE.md | 23 +- backend/services/encoding_service.py | 14 +- backend/services/encoding_worker_manager.py | 89 ++++++-- backend/services/error_monitor/config.py | 8 +- backend/tests/test_deploy_promote.py | 37 +++- .../tests/test_encoding_worker_no_primary.py | 196 ++++++++++++++++++ docs/ARCHITECTURE.md | 12 +- docs/DEVELOPMENT.md | 2 +- docs/TROUBLESHOOTING.md | 14 +- infrastructure/__main__.py | 16 +- infrastructure/compute/__init__.py | 4 - infrastructure/compute/encoding_worker_vm.py | 111 +--------- infrastructure/config.py | 17 +- .../encoding-worker/deploy_promote.py | 21 ++ .../functions/encoding_worker_idle/main.py | 6 + .../test_idle_shutdown.py | 23 ++ infrastructure/test_encoding_worker_config.py | 30 +-- pyproject.toml | 2 +- .../services/error_monitor/test_config.py | 10 +- 20 files changed, 470 insertions(+), 215 deletions(-) create mode 100644 backend/tests/test_encoding_worker_no_primary.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0923bdaa..591d31f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1439,7 +1439,12 @@ jobs: print("::error::Encoding worker config not found in Firestore") raise SystemExit(1) data = doc.to_dict() - if (data.get("active_override_provider") or "").lower() == "aws": + aws_override = (data.get("active_override_provider") or "").lower() == "aws" + # Version the serving worker was last validated at: the CI gate diffs + # worker code against it. primary_version stops moving once the c4d + # pair is retired, so prefer validated_version (written on promote). + data["validated_version"] = data.get("validated_version") or data.get("primary_version") + if aws_override: # The AWS encoding worker (EC2) is serving by preference. This # GCE blue-green flow can't gcloud-start/stop it and must not # clear its override; it picks up the new wheel on its next job. @@ -1449,7 +1454,8 @@ jobs: for key in ("active_override_vm", "active_override_ip", "active_override_zone"): data[key] = None with open(os.environ["GITHUB_OUTPUT"], "a") as f: - for key in ["primary_vm", "primary_ip", "primary_version", + f.write(f"aws_override={str(aws_override).lower()}\n") + for key in ["primary_vm", "primary_ip", "primary_version", "validated_version", "secondary_vm", "secondary_ip", "active_override_vm", "active_override_ip", "active_override_zone"]: # `or ''`: a key stored as null must be empty, not the string @@ -1475,7 +1481,7 @@ jobs: id: worker_gate env: FORCE: ${{ github.event_name == 'workflow_dispatch' && inputs.force_encoding_worker_deploy == true }} - SERVING_VERSION: ${{ steps.config.outputs.primary_version }} + SERVING_VERSION: ${{ steps.config.outputs.validated_version }} OVERRIDE_VM: ${{ steps.config.outputs.active_override_vm }} run: | mapfile -t PATHS < <(grep -v '^\s*#' infrastructure/encoding-worker/worker_code_paths.txt | grep -v '^\s*$') @@ -1621,7 +1627,34 @@ jobs: # serving override in place (accepts a brief blip; auto-resubmit # covers any in-flight render). If there's no override either, fail. ov = os.environ.get("OVERRIDE_VM") + ov_ip = os.environ.get("OVERRIDE_IP") + + def _active_jobs(ip): + import urllib.request + if not ip: + return None + try: + with urllib.request.urlopen(f"http://{ip}:8080/health", timeout=5) as r: + return json.load(r).get("active_jobs") + except Exception: + return None + + # Since the c4d pair was retired (2026-11) this is the normal path when + # AWS is in cooldown and c2df serves: never stop it mid-render. Wait + # (bounded) for a confirmed idle worker; if it stays busy, skip — it + # idle-stops after 5 min and cold-boots the new wheel next time. + drained = True if ov: + drain_deadline = time.monotonic() + 900 + n = _active_jobs(ov_ip) + while n != 0 and time.monotonic() < drain_deadline: + print(f" serving override {ov} active_jobs={n}; waiting to restart") + time.sleep(15) + n = _active_jobs(ov_ip) + drained = n == 0 or vm_status(ov, os.environ.get("OVERRIDE_ZONE") or "us-central1-f") != "RUNNING" + if not drained: + print(f"::warning::Serving override {ov} still busy (active_jobs={n}); skipping in-place restart") + if ov and drained: zone = os.environ.get("OVERRIDE_ZONE") or "us-central1-f" print(f"::warning::No separate green could start; restarting serving override {ov} in place") subprocess.run(["gcloud", "compute", "instances", "stop", ov, @@ -1646,7 +1679,7 @@ jobs: if final_status != "RUNNING": inplace_failed = True print(f"::error::In-place restart of {ov} did not reach RUNNING (status={final_status or 'unknown'})") - else: + elif not ov: print("::error::No green target could be started and no active override exists") def _out(**kv): @@ -1841,6 +1874,7 @@ jobs: OVERRIDE_VM: ${{ steps.config.outputs.active_override_vm }} OVERRIDE_IP: ${{ steps.config.outputs.active_override_ip }} OVERRIDE_ZONE: ${{ steps.config.outputs.active_override_zone }} + AWS_OVERRIDE: ${{ steps.config.outputs.aws_override }} run: | python3 << 'PYEOF' import json, os, subprocess, sys, time, urllib.request @@ -1859,6 +1893,9 @@ jobs: "active_override_vm": os.environ.get("OVERRIDE_VM") or None, "active_override_ip": os.environ.get("OVERRIDE_IP") or None, "active_override_zone": os.environ.get("OVERRIDE_ZONE") or None, + # The config step hid an AWS override from the GCE flow; tell the + # promote plan so a fallback green doesn't displace it. + "active_override_provider": "aws" if os.environ.get("AWS_OVERRIDE") == "true" else None, } now = datetime.now(timezone.utc).isoformat() plan = deploy_promote.promote_plan(green, config, now=now, version=os.environ["NEW_VERSION"]) @@ -1866,7 +1903,10 @@ jobs: db = firestore.Client(project="nomadkaraoke") db.collection("config").document("encoding-worker").update(plan["firestore_updates"]) print("Applied config update:", json.dumps(plan["firestore_updates"], default=str)) - print(f"Green {green['vm']} ({green['kind']}) is now serving") + if any(t.get("vm") == green["vm"] for t in plan["drain_and_stop"]): + print(f"Green {green['vm']} ({green['kind']}) validated; AWS worker stays serving, stopping the green") + else: + print(f"Green {green['vm']} ({green['kind']}) is now serving") def active_jobs(ip): # /health is unauthenticated; None == "unknown / maybe busy". diff --git a/CLAUDE.md b/CLAUDE.md index 585bfe16..1e241c62 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -151,21 +151,22 @@ EOF ``` ### GCE Encoding Worker -The worker is a **Spot** c4d-highcpu-16 blue-green primary pair (`encoding-worker-a`/`-b`, -us-central1-c) plus ONE stopped on-demand fallback (`encoding-worker-fallback-c2df`, -c2d-highcpu-16, us-central1-f) — 3 VMs since 2026-09-26. Idle VMs stop after 5 min. -CI blue-green redeploys the worker only when worker code changed -(`infrastructure/encoding-worker/worker_code_paths.txt`); force it via Actions → CI → -Run workflow → "Force a blue-green redeploy of the GCE encoding worker". The currently-serving -VM is `active_override_vm` in `config/encoding-worker` if set, else `primary_vm`. -Use that VM name + its zone below (there is no VM literally named `encoding-worker`): +Since 2026-11 the only GCE encoding VM is the stopped on-demand fallback +`encoding-worker-fallback-c2df` (c2d-highcpu-16, us-central1-f); the c4d Spot pair +(`encoding-worker-a`/`-b`) was retired and deleted, and `primary_vm`/`secondary_vm` in +`config/encoding-worker` are empty. The serving worker is `active_override_vm` (normally the +AWS box below). Idle VMs stop after 5 min. CI's "blue-green" deploy now only runs when worker +code changed (`infrastructure/encoding-worker/worker_code_paths.txt`, diffed against +`validated_version`): it boots c2df, runs a test encode, records `validated_version` and stops +it again, leaving the AWS override serving. Force it via Actions → CI → Run workflow → +"Force a blue-green redeploy of the GCE encoding worker". ```bash -# Check health (includes wheel_version). Example uses the primary pair's VM. -gcloud compute ssh encoding-worker-a --zone=us-central1-c --project=nomadkaraoke \ +# Check health (includes wheel_version) on the GCE fallback. +gcloud compute ssh encoding-worker-fallback-c2df --zone=us-central1-f --project=nomadkaraoke \ --command="curl -s http://localhost:8080/health" # Restart the service (pick up new worker-side code after a deploy) -gcloud compute ssh encoding-worker-a --zone=us-central1-c --project=nomadkaraoke \ +gcloud compute ssh encoding-worker-fallback-c2df --zone=us-central1-f --project=nomadkaraoke \ --command="sudo systemctl restart encoding-worker" ``` diff --git a/backend/services/encoding_service.py b/backend/services/encoding_service.py index ea7cfa6e..4419603c 100644 --- a/backend/services/encoding_service.py +++ b/backend/services/encoding_service.py @@ -239,7 +239,15 @@ class EncodingService: if self._worker_manager: config = self._worker_manager.get_config() - self._cached_url = config.active_url + url = config.active_url + if not url: + # No primary pair and no override recorded yet: target the + # top-ranked candidate. If it's stopped, the connection failure + # triggers the warmup fallback, which starts it and records it + # as the override. + candidates = self._worker_manager.ranked_candidates(config) + url = candidates[0].url if candidates else "" + self._cached_url = url self._url_cached_at = now return self._cached_url @@ -456,6 +464,10 @@ class EncodingService: self._invalidate_cached_url() else: result = await asyncio.to_thread(self._worker_manager.ensure_primary_running) + # With no GCE primary pair, ensure_primary_running starts a ranked + # candidate and records it as the override. + if result.get("fell_back"): + self._invalidate_cached_url() except EncodingWorkerStartError: # Every candidate exhausted (or single-zone primary failed) — both # capacity errors and other start failures (e.g. 503 diff --git a/backend/services/encoding_worker_manager.py b/backend/services/encoding_worker_manager.py index 419da1d8..55486946 100644 --- a/backend/services/encoding_worker_manager.py +++ b/backend/services/encoding_worker_manager.py @@ -119,8 +119,15 @@ class EncodingWorkerConfig: # also runs on it instead of the Cloud Run job (worker_service). Default off. aws_video_worker: bool = False + @property + def has_primary(self) -> bool: + """False once the GCE blue-green pair is retired (empty primary_vm).""" + return bool(self.primary_vm) + @property def primary_url(self) -> str: + if not self.primary_ip: + return "" return f"http://{self.primary_ip}:{ENCODING_WORKER_PORT}" @property @@ -134,7 +141,8 @@ class EncodingWorkerConfig: Returns the capacity-fallback override if set, otherwise the primary URL. The override mechanism lets us route around a zone that's temporarily out of c4d (Spot) capacity without rewriting the - blue-green primary/secondary tracking. + blue-green primary/secondary tracking. Empty when neither is set (no + primary pair and no override yet); callers resolve a candidate then. """ if self.active_override_ip: return f"http://{self.active_override_ip}:{ENCODING_WORKER_PORT}" @@ -159,8 +167,11 @@ class EncodingWorkerCandidate: # True iff this candidate is the blue-green PRIMARY VM. The override routing # (set on fallback, clear on primary) keys off this flag, NOT list position — # the preference logic may rank a fallback ahead of a stocked-out primary, so - # "first candidate" is no longer synonymous with "primary". - is_primary: bool = False + # "first candidate" is no longer synonymous with "primary". None ⇒ unknown + # (hand-built lists): the legacy "index 0 is the primary" rule applies. + # ranked_candidates always sets True/False, so with no primary pair every + # candidate is a fallback and a successful start always sets the override. + is_primary: Optional[bool] = None # "aws" for the EC2 worker (routed to AwsComputeAdapter); None ⇒ GCE. provider: Optional[str] = None @@ -282,14 +293,18 @@ class EncodingWorkerManager: config = config or self.get_config() primary_mt = getattr(config, "primary_machine_type", None) or PRIMARY_MACHINE_TYPE - pool = [{ - "vm": config.primary_vm, - "zone": self._zone, - "ip": config.primary_ip, - "machine_type": primary_mt, - "kind": "primary", - "is_primary": True, - }] + pool = [] + # No primary once the GCE c4d pair is retired (2026-11): the pool is then + # just the fallbacks (AWS worker + GCE c2df). + if config.primary_vm: + pool.append({ + "vm": config.primary_vm, + "zone": self._zone, + "ip": config.primary_ip, + "machine_type": primary_mt, + "kind": "primary", + "is_primary": True, + }) pool.extend(self.fallback_entries()) ranked = ordered_candidates( pool, @@ -302,7 +317,7 @@ class EncodingWorkerManager: zone=c["zone"], ip=c["ip"], machine_type=c.get("machine_type"), - is_primary=c.get("is_primary", False), + is_primary=bool(c.get("is_primary", False)), provider=c.get("provider"), ) for c in ranked @@ -320,8 +335,13 @@ class EncodingWorkerManager: """ config = self.get_config() candidates = self.ranked_candidates(config) - if not candidates or candidates[0].is_primary: + if config.has_primary and (not candidates or candidates[0].is_primary): return self.ensure_primary_running() + if not candidates: + raise EncodingWorkerStartError( + "No encoding worker candidates configured (no primary, no fallbacks)", + vm_name="", zone="", code="NO_CANDIDATES", + ) return self.ensure_any_running(candidates) def _doc_ref(self): @@ -341,13 +361,15 @@ class EncodingWorkerManager: f"{CONFIG_COLLECTION}/{CONFIG_DOCUMENT}" ) data = snapshot.to_dict() + # The primary/secondary pair is optional: empty/absent once the GCE c4d + # pair is retired, leaving only the fallback pool (AWS + c2df). return EncodingWorkerConfig( - primary_vm=data["primary_vm"], - primary_ip=data["primary_ip"], - primary_version=data["primary_version"], + primary_vm=data.get("primary_vm") or "", + primary_ip=data.get("primary_ip") or "", + primary_version=data.get("primary_version") or "", primary_deployed_at=data.get("primary_deployed_at"), - secondary_vm=data["secondary_vm"], - secondary_ip=data["secondary_ip"], + secondary_vm=data.get("secondary_vm") or "", + secondary_ip=data.get("secondary_ip") or "", secondary_version=data.get("secondary_version"), secondary_deployed_at=data.get("secondary_deployed_at"), last_swap_at=data.get("last_swap_at"), @@ -495,6 +517,16 @@ class EncodingWorkerManager: EncodingWorkerStartError: any other start failure. """ config = self.get_config() + if not config.has_primary: + # No GCE primary pair: "the primary" is whichever ranked candidate + # starts (the AWS worker when preferred). Same contract as before. + candidates = self.ranked_candidates(config) + if not candidates: + raise EncodingWorkerStartError( + "No encoding worker candidates configured (no primary, no fallbacks)", + vm_name="", zone="", code="NO_CANDIDATES", + ) + return self.ensure_any_running(candidates) vm_name = config.primary_vm status = self.get_vm_status(vm_name) @@ -594,7 +626,9 @@ class EncodingWorkerManager: last_non_transient_error: Optional[EncodingWorkerStartError] = None # When the caller marks a primary explicitly, route override off that flag; # otherwise preserve the legacy "index 0 is the primary" behaviour. - has_explicit_primary = any(getattr(c, "is_primary", False) for c in candidates) + has_explicit_primary = any( + getattr(c, "is_primary", None) is not None for c in candidates + ) for index, candidate in enumerate(candidates): try: try: @@ -645,7 +679,7 @@ class EncodingWorkerManager: # explicit flag when present (ordering can rank a fallback ahead of # a stocked-out primary, so it may not be index 0); otherwise the # legacy position-based rule. - is_primary = candidate.is_primary if has_explicit_primary else (index == 0) + is_primary = bool(candidate.is_primary) if has_explicit_primary else (index == 0) fell_back = not is_primary if fell_back: self._set_active_override(candidate) @@ -798,6 +832,12 @@ class EncodingWorkerManager: vm = config.active_override_vm zone = config.active_override_zone + if not vm and not config.has_primary: + logger.warning( + "demote_active_worker: no override and no primary pair; nothing to demote (%s)", + reason, + ) + return if not vm: # Primary pair was serving — see docstring; leave selection untouched. # Logged at ERROR so it alerts: on 2026-10-06 a primary that booted @@ -810,7 +850,12 @@ class EncodingWorkerManager: ) return - key = cooldown_key({"vm_name": vm, "zone": zone}) + # Fallback entries carry machine_type (the AWS worker's name has no family + # token, so without it the AWS box could never be cooled down). + machine_type = next( + (e.get("machine_type") for e in self.fallback_entries() if e.get("vm") == vm), None + ) + key = cooldown_key({"vm_name": vm, "zone": zone, "machine_type": machine_type}) if key: try: self._doc_ref().set( @@ -824,6 +869,8 @@ class EncodingWorkerManager: logger.warning( "demote_active_worker: could not record cooldown for %s: %s", key, e ) + # With no primary pair, an empty override makes encoding_service resolve + # the top-ranked candidate, which the cooldown above now demotes. self._clear_active_override() # ------------------------------------------------------------------ diff --git a/backend/services/error_monitor/config.py b/backend/services/error_monitor/config.py index eb54566f..0dc59614 100644 --- a/backend/services/error_monitor/config.py +++ b/backend/services/error_monitor/config.py @@ -77,8 +77,9 @@ MONITORED_CLOUD_RUN_JOBS: list[str] = [ MONITORED_CLOUD_FUNCTIONS: list[str] = [] MONITORED_GCE_INSTANCES: list[str] = [ - "encoding-worker-a", - "encoding-worker-b", + # GCE encoding fallback; the c4d pair (encoding-worker-a/b) was retired + # 2026-11 and the AWS worker (aws-encoding-worker-1) logs on the box. + "encoding-worker-fallback-c2df", "flacfetch-vm", "divebar-sync-vm", ] @@ -96,7 +97,8 @@ karaoke-backend (main API) → audio-separation-job (Cloud Run Job, runs on Cloud Run GPU) → audio-download-job (Cloud Run Job, uses flacfetch-vm) → gdrive-validator (Cloud Function, validates output files) - → encoding-worker-a/b (GCE VMs, long-running encoding workers) + → aws-encoding-worker-1 (AWS EC2, preferred encoding + distribution worker) + → encoding-worker-fallback-c2df (GCE VM, encoding fallback) karaoke-decide (catalog / discovery API) → kn_data_sync (Cloud Function, syncs KaraokeNerds catalog) diff --git a/backend/tests/test_deploy_promote.py b/backend/tests/test_deploy_promote.py index 47145593..aa28e2df 100644 --- a/backend/tests/test_deploy_promote.py +++ b/backend/tests/test_deploy_promote.py @@ -247,12 +247,45 @@ def test_secondary_promotion_leaves_aws_override_serving(): assert [d["vm"] for d in plan["drain_and_stop"]] == ["encoding-worker-a"] -def test_fallback_promotion_does_not_drain_aws_override(): +def test_fallback_green_is_validate_only_while_aws_serves(): + """With the AWS worker serving, a GCE fallback green must not displace it: + record the validated version, keep the override, stop the green.""" config = {"active_override_vm": "aws-encoding-worker-1", "active_override_zone": "us-east-2b", "active_override_provider": "aws"} green = {"vm": "encoding-worker-fallback-c2df", "ip": "2.2.2.2", "zone": "us-central1-f", "kind": "fallback"} plan = deploy_promote.promote_plan(green, config, now="2026-10-29T00:00:00Z", version="2.0") - assert plan["drain_and_stop"] == [] + updates = plan["firestore_updates"] + assert not any(k.startswith("active_override") for k in updates) + assert updates["validated_version"] == "2.0" + assert updates["deploy_in_progress"] is False + # Never the AWS override; only the green we booted for validation. + assert plan["drain_and_stop"] == [ + {"vm": "encoding-worker-fallback-c2df", "ip": "2.2.2.2", "zone": "us-central1-f"} + ] + + +def test_aws_override_hidden_by_ci_is_still_recognised(): + """CI blanks the AWS override's vm/ip/zone but passes the provider; that alone + must keep the AWS worker serving (no c4d pair, 2026-11).""" + config = {"active_override_vm": None, "active_override_provider": "aws"} + green = {"vm": "encoding-worker-fallback-c2df", "ip": "2.2.2.2", "zone": "us-central1-f", "kind": "fallback"} + plan = deploy_promote.promote_plan(green, config, now="t", version="2.1") + assert "active_override_vm" not in plan["firestore_updates"] + + +def test_no_pair_green_selection_is_fallbacks_only(): + """Once the c4d pair is retired (empty secondary), the only GCE green is c2df.""" + config = {"secondary_vm": None, "active_override_vm": None} + fallbacks = [{"vm": "encoding-worker-fallback-c2df", "zone": "us-central1-f", "ip": "2.2.2.2"}, _AWS] + names = [c["vm"] for c in deploy_promote.select_green_candidates(config, fallbacks)] + assert names == ["encoding-worker-fallback-c2df"] + + +@pytest.mark.parametrize("kind", ["secondary", "fallback"]) +def test_every_promote_records_validated_version(kind): + green = {"vm": "g", "ip": "9.9.9.9", "zone": "us-central1-f", "kind": kind} + plan = deploy_promote.promote_plan(green, {"primary_vm": "p"}, now="t", version="3.0") + assert plan["firestore_updates"]["validated_version"] == "3.0" def test_fallback_promotion_clears_provider_tag(): diff --git a/backend/tests/test_encoding_worker_no_primary.py b/backend/tests/test_encoding_worker_no_primary.py new file mode 100644 index 00000000..8a1a4ace --- /dev/null +++ b/backend/tests/test_encoding_worker_no_primary.py @@ -0,0 +1,196 @@ +"""EncodingWorkerManager once the GCE c4d blue-green pair is retired (2026-11). + +config/encoding-worker then has empty (or absent) primary_vm/secondary_vm, and the +pool is just the fallbacks: the AWS worker (preferred) + the GCE c2df VM. Every +path must route to a fallback and record it as the active override, never to a +blank primary. +""" +import json +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from backend.services.encoding_errors import EncodingWorkerStartError +from backend.services.encoding_worker_manager import ( + EncodingWorkerCandidate, + EncodingWorkerManager, +) + +AWS_ENTRY = { + "vm": "aws-encoding-worker-1", "zone": "us-east-2b", "ip": "3.3.3.3", + "machine_type": "c7a-4xlarge", "provider": "aws", "instance_id": "i-0abc", "region": "us-east-2", +} +C2D_ENTRY = {"vm": "encoding-worker-fallback-c2df", "zone": "us-central1-f", "ip": "2.2.2.2", + "machine_type": "c2d-highcpu-16"} + +NO_PAIR = {"primary_vm": "", "primary_ip": "", "primary_version": "0.291.0", + "secondary_vm": "", "secondary_ip": ""} + + +def _db(**config): + data = {"last_activity_at": None, "capacity_state": {}, "speed_rank_overrides": {"c7a": 5}, + **config} + db = MagicMock() + snap = MagicMock(exists=True) + snap.to_dict.return_value = data + doc = MagicMock() + doc.get.return_value = snap + db.collection.return_value.document.return_value = doc + return db, doc + + +def _manager(db, entries=(C2D_ENTRY, AWS_ENTRY)): + gce = MagicMock(name="gce") + aws = MagicMock(name="aws") + aws.handles.side_effect = lambda vm: vm == AWS_ENTRY["vm"] + m = EncodingWorkerManager(db=db, compute_client=gce, + fallback_vms_raw=json.dumps(list(entries)), aws_compute=aws) + return m, gce, aws + + +def test_get_config_tolerates_absent_pair_keys(): + db, _ = _db() # no primary/secondary keys at all + m, _, _ = _manager(db) + config = m.get_config() + assert config.primary_vm == "" and config.secondary_vm == "" + assert not config.has_primary + assert config.primary_url == "" + assert config.active_url == "" + + +def test_active_url_follows_override_without_primary(): + db, _ = _db(**NO_PAIR, active_override_vm="aws-encoding-worker-1", active_override_ip="3.3.3.3") + m, _, _ = _manager(db) + assert m.get_config().active_url == "http://3.3.3.3:8080" + + +def test_ranked_candidates_skip_blank_primary(): + db, _ = _db(**NO_PAIR) + m, _, _ = _manager(db) + ranked = m.ranked_candidates() + assert [c.vm_name for c in ranked] == ["aws-encoding-worker-1", "encoding-worker-fallback-c2df"] + assert all(c.is_primary is False for c in ranked) + + +def test_aws_landing_sets_override_not_clears_it(): + """With no primary, index 0 (AWS) is a fallback: starting it must SET the + override (the legacy 'index 0 is the primary' rule would clear it, leaving + active_url empty).""" + db, doc = _db(**NO_PAIR) + m, _, aws = _manager(db) + aws.get.return_value = MagicMock(status="RUNNING") + m.update_activity = MagicMock() + result = m.ensure_any_running(m.ranked_candidates()) + assert result["vm_name"] == "aws-encoding-worker-1" + assert result["fell_back"] is True + update = doc.update.call_args[0][0] + assert update["active_override_vm"] == "aws-encoding-worker-1" + assert update["active_override_provider"] == "aws" + + +def test_hand_built_candidates_keep_legacy_index0_rule(): + """Candidates built without is_primary (None) keep the old position rule.""" + db, doc = _db(**NO_PAIR) + m, gce, _ = _manager(db) + gce.get.return_value = MagicMock(status="RUNNING") + m.update_activity = MagicMock() + m._clear_active_override = MagicMock() + m._set_active_override = MagicMock() + result = m.ensure_any_running([EncodingWorkerCandidate("vm-x", "us-central1-f", "9.9.9.9")]) + assert result["fell_back"] is False + m._clear_active_override.assert_called_once() + m._set_active_override.assert_not_called() + + +def test_ensure_primary_running_without_pair_starts_ranked_candidate(): + db, _ = _db(**NO_PAIR) + m, gce, _ = _manager(db) + m.ensure_any_running = MagicMock(return_value={"started": True, "vm_name": "aws-encoding-worker-1", + "fell_back": True}) + result = m.ensure_primary_running() + assert result["vm_name"] == "aws-encoding-worker-1" + (candidates,), _ = m.ensure_any_running.call_args + assert candidates[0].vm_name == "aws-encoding-worker-1" + gce.get.assert_not_called() # never looks up a blank primary on GCE + + +def test_ensure_preferred_running_without_pair_never_takes_primary_path(): + # No rank override → c2df ranks first; still no primary path. + db, _ = _db(**NO_PAIR, speed_rank_overrides={}) + m, _, _ = _manager(db) + m.ensure_any_running = MagicMock(return_value={"started": False}) + m.ensure_preferred_running() + m.ensure_any_running.assert_called_once() + + +@pytest.mark.parametrize("method", ["ensure_preferred_running", "ensure_primary_running"]) +def test_no_candidates_at_all_raises_start_error(method): + db, _ = _db(**NO_PAIR) + m, _, _ = _manager(db, entries=()) + with pytest.raises(EncodingWorkerStartError) as ei: + getattr(m, method)() + assert ei.value.code == "NO_CANDIDATES" + + +def test_demote_aws_without_primary_retries_on_another_worker(): + """A broken AWS worker is cooled down (via its machine_type) and the override + cleared, so the retry's URL resolves to the GCE fallback instead.""" + from backend.services.encoding_service import EncodingService + + db, doc = _db(**NO_PAIR, active_override_vm="aws-encoding-worker-1", + active_override_zone="us-east-2b", active_override_ip="3.3.3.3") + m, _, _ = _manager(db) + m.demote_active_worker("metadata unreachable") + state = doc.set.call_args[0][0]["capacity_state"] + assert list(state) == ["c7a-4xlarge@us-east-2b"] + assert doc.update.call_args[0][0]["active_override_vm"] is None + + # Next read: override gone, AWS in cooldown → URL is c2df. + db2, _ = _db(**NO_PAIR, capacity_state=state) + m2, _, _ = _manager(db2) + svc = EncodingService.__new__(EncodingService) + svc._cached_url = None + svc._url_cached_at = 0.0 + svc._worker_manager = m2 + assert svc._get_worker_url() == "http://2.2.2.2:8080" + + +def test_demote_with_nothing_serving_is_quiet(caplog): + db, _ = _db(**NO_PAIR) + m, _, _ = _manager(db) + m.demote_active_worker("x") + assert not [r for r in caplog.records if r.levelname == "ERROR"] + + +def test_encoding_service_resolves_top_candidate_when_no_url(): + from backend.services.encoding_service import EncodingService + + db, _ = _db(**NO_PAIR) + m, _, _ = _manager(db) + svc = EncodingService.__new__(EncodingService) + svc._cached_url = None + svc._url_cached_at = 0.0 + svc._worker_manager = m + assert svc._get_worker_url() == "http://3.3.3.3:8080" + + +@pytest.mark.asyncio +async def test_single_candidate_warmup_invalidates_cache_on_fallback(): + from backend.services.encoding_service import EncodingService + + svc = EncodingService.__new__(EncodingService) + svc._cached_url = "http://stale:8080" + svc._url_cached_at = 1e12 + manager = MagicMock() + manager.get_config.return_value = MagicMock() + manager.ranked_candidates.return_value = [ + EncodingWorkerCandidate("aws-encoding-worker-1", "us-east-2b", "3.3.3.3", is_primary=False) + ] + manager.ensure_primary_running.return_value = { + "started": False, "vm_name": "aws-encoding-worker-1", "zone": "us-east-2b", + "primary_url": "http://3.3.3.3:8080", "fell_back": True, + } + manager.wait_for_worker_ready = AsyncMock() + svc._worker_manager = manager + await svc._warmup_encoding_worker_fallback("job1") + assert svc._cached_url is None diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 68d49169..89d20550 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -38,11 +38,11 @@ │ (jobs) │ │ (files) │ │ (API keys) │ │ Worker (*) │ └──────────┘ └──────────┘ └──────────────┘ └──────────────┘ -(*) GCE Encoding Worker: a Spot c4d-highcpu-16 blue-green pair (AMD EPYC - Turin, the preferred pick) plus ONE on-demand c2d-highcpu-16 fallback in a - different zone for Spot/c4d stockouts (3 VMs since 2026-09-26 — see - docs/archive/2026-09-26-encoding-cost-cuts.md). Candidates are selected - fastest-first with a capacity cooldown. +(*) Encoding Worker: since 2026-11 the AWS EC2 worker (aws-encoding-worker-1, + c7a.4xlarge, preferred via speed_rank_overrides) plus ONE on-demand GCE + c2d-highcpu-16 fallback (encoding-worker-fallback-c2df). The c4d Spot + blue-green pair was retired. Candidates are selected fastest-first with a + capacity cooldown. Used for both final video encoding and preview video generation. Uses immutable deployment pattern - see infrastructure/encoding-worker/README.md and the "Encoding Worker" section below. @@ -469,6 +469,8 @@ karaoke-gen shares a GCP project (`nomadkaraoke`) with karaoke-decide, but uses ## Encoding Worker: blue-green primary pair + multi-family fallback pool +> **2026-11 update:** the c4d Spot pair (`encoding-worker-a`/`-b`) is **retired and deleted**. The preferred encoder (and distribution worker) is the AWS EC2 box `aws-encoding-worker-1`; the only GCE VM is the on-demand fallback `encoding-worker-fallback-c2df` (us-central1-f). `config/encoding-worker` keeps `primary_vm`/`secondary_vm` empty, so every selection path routes to a fallback and records it as `active_override_*`. CI blue-green deploys now just validate the new wheel on c2df and stop it again (the AWS override keeps serving); the worker-change gate diffs against `validated_version`. The pair-specific text below is historical. + The GCE encoding worker is a **blue-green primary pair** (`encoding-worker-a`/`-b`, both **Spot** c4d-highcpu-16 in us-central1-c) plus **one on-demand fallback** (`encoding-worker-fallback-c2df`, c2d-highcpu-16, us-central1-f). **3 VMs total** diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 0cbf1790..dd712f8f 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -235,7 +235,7 @@ TESTMAIL_NAMESPACE=... # testmail.app namespace ## Encoding Worker Operations -The encoding worker is a Spot blue-green primary pair (`encoding-worker-a`/`-b`, c4d-highcpu-16) plus one stopped on-demand fallback (`encoding-worker-fallback-c2df`, c2d-highcpu-16, us-central1-f) — 3 VMs total since 2026-09-26. CI only blue-green redeploys the worker when a path in `infrastructure/encoding-worker/worker_code_paths.txt` changed since the serving version (or via Actions → CI → Run workflow with "Force a blue-green redeploy of the GCE encoding worker"). Candidate ordering (runtime + deploy) is the shared `backend/services/encoding_worker_preference.py`. See [ARCHITECTURE.md](ARCHITECTURE.md#encoding-worker-blue-green-primary-pair--multi-family-fallback-pool) for the full design. +The encoding worker is the AWS EC2 box `aws-encoding-worker-1` (preferred, also runs distribution) plus one stopped on-demand GCE fallback (`encoding-worker-fallback-c2df`, c2d-highcpu-16, us-central1-f). The c4d Spot pair (`encoding-worker-a`/`-b`) was retired and deleted in 2026-11; examples below that name it should use c2df (`--zone=us-central1-f`). CI only blue-green redeploys the worker when a path in `infrastructure/encoding-worker/worker_code_paths.txt` changed since the serving version (or via Actions → CI → Run workflow with "Force a blue-green redeploy of the GCE encoding worker"). Candidate ordering (runtime + deploy) is the shared `backend/services/encoding_worker_preference.py`. See [ARCHITECTURE.md](ARCHITECTURE.md#encoding-worker-blue-green-primary-pair--multi-family-fallback-pool) for the full design. ### Seed Firestore config (initial setup or reset) diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index c9a6fbe8..ce1c3ea0 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -359,17 +359,17 @@ The script is idempotent — re-running it skips regeneration if the CDG ZIP alr **Cause:** Worker picks up a new deploy but doesn't restart automatically. Target the **currently-serving** VM — `active_override_vm` in `config/encoding-worker` -if set, else `primary_vm` (with its zone). There is no VM literally named -`encoding-worker`; the examples below use the primary `encoding-worker-a` — swap in -the active VM/zone if a fallback is serving. +if set, else `primary_vm` (with its zone). Since 2026-11 `primary_vm` is empty (the c4d +pair was deleted): the serving VM is the AWS box (SSM shell, see +`infrastructure/aws/README.md`) or the GCE fallback `encoding-worker-fallback-c2df`. ```bash -# Check current wheel version (example: the primary a in us-central1-c) -gcloud compute ssh encoding-worker-a --zone=us-central1-c --project=nomadkaraoke \ +# Check current wheel version (example: the GCE fallback in us-central1-f) +gcloud compute ssh encoding-worker-fallback-c2df --zone=us-central1-f --project=nomadkaraoke \ --command="curl -s http://localhost:8080/health | python3 -m json.tool" # Restart to pick up latest wheel -gcloud compute ssh encoding-worker-a --zone=us-central1-c --project=nomadkaraoke \ +gcloud compute ssh encoding-worker-fallback-c2df --zone=us-central1-f --project=nomadkaraoke \ --command="sudo systemctl restart encoding-worker" ``` @@ -626,7 +626,7 @@ gcloud compute ssh encoding-worker-b --zone=us-central1-c --project=nomadkaraoke **Background — what this state means:** GCE returned `ZONE_RESOURCE_POOL_EXHAUSTED` or transient `503 SERVICE_UNAVAILABLE` from `compute.instances.start` on every encoding-worker VM. The render worker parks the job in `RENDER_PENDING_CAPACITY` instead of failing it; Cloud Scheduler retries it every 10 min via `/api/internal/retry-pending-render-jobs`. Hard timeout is 24 hours (then transitions to `failed` with a clear permanent-failure message). -Since 2026-09-26 the fleet is 3 VMs: **Spot** `c4d-highcpu-16` primaries (`encoding-worker-a`/`-b`) in `us-central1-c`, plus ONE stopped on-demand fallback `encoding-worker-fallback-c2df` (`c2d-highcpu-16`, `us-central1-f`). Candidates are tried **fastest-first with a 15-min capacity cooldown** — see `backend/services/encoding_worker_preference.py`. If both the Spot pair and the c2d fallback return stockout, that's a genuinely severe regional event — wait, or (re-)add a fallback in another family/zone to `EncodingWorkerConfig.FALLBACKS` **and** the `encoding-worker-fallback-vms` secret (the 2026-08-15 pool had 8 fallbacks across 6 families). NOTE: creating a *new* fallback VM needs a one-time boot allocation, so a deep enough crunch can even block provisioning. +Since 2026-11 the fleet is the AWS worker plus ONE GCE VM (the c4d Spot pair `encoding-worker-a`/`-b` was deleted). Before that (2026-09-26) it was **Spot** `c4d-highcpu-16` primaries in `us-central1-c`, plus ONE stopped on-demand fallback `encoding-worker-fallback-c2df` (`c2d-highcpu-16`, `us-central1-f`). Candidates are tried **fastest-first with a 15-min capacity cooldown** — see `backend/services/encoding_worker_preference.py`. If both the Spot pair and the c2d fallback return stockout, that's a genuinely severe regional event — wait, or (re-)add a fallback in another family/zone to `EncodingWorkerConfig.FALLBACKS` **and** the `encoding-worker-fallback-vms` secret (the 2026-08-15 pool had 8 fallbacks across 6 families). NOTE: creating a *new* fallback VM needs a one-time boot allocation, so a deep enough crunch can even block provisioning. **Spot preemption mid-encode** is expected occasionally: the backend sees the pinned worker VM is TERMINATED/STOPPING, logs `Encoding worker VM stopped mid-job (... ) — likely Spot preemption`, and resubmits the encode as `_retry_` (up to `ENCODING_RESUBMIT_MAX`, default 2). Repeated preemptions → the job fails with the lost-job error; check `gcloud compute operations list --filter="operationType=compute.instances.preempted"`. diff --git a/infrastructure/__main__.py b/infrastructure/__main__.py index d043ffa4..073c8b43 100644 --- a/infrastructure/__main__.py +++ b/infrastructure/__main__.py @@ -756,11 +756,9 @@ if AUDIO_SEPARATOR_SERVICE_ENABLED: # ==================== Compute VMs ==================== -# Encoding Worker VMs (blue-green pair for zero-downtime deployments) -encoding_worker_ips = encoding_worker_vm.create_encoding_worker_ips() -encoding_worker_instances = encoding_worker_vm.create_encoding_worker_vms( - encoding_worker_ips, encoding_worker_sa -) +# The c4d Spot blue-green pair (encoding-worker-a/-b + static IPs) was retired +# 2026-11: the AWS EC2 worker (infrastructure/aws/) is the preferred encoder and +# runs distribution; the c2df VM below is the GCP fallback. # Capacity-fallback Encoding Worker VM — one ON-DEMAND VM in a different zone AND # machine family from the c4d Spot pair, so a Spot/c4d stockout can't take out @@ -890,12 +888,6 @@ pulumi.export("recover_stuck_downloads_scheduler_name", recover_stuck_downloads_ pulumi.export("disposable_domains_sync_scheduler_name", disposable_domains_sync_scheduler.name) pulumi.export("gdrive_validator_service_account", gdrive_validator_sa.email) -# Encoding workers (blue-green pair) -pulumi.export("encoding_worker_a_ip", encoding_worker_ips[0].address) -pulumi.export("encoding_worker_b_ip", encoding_worker_ips[1].address) -pulumi.export("encoding_worker_a_url", encoding_worker_ips[0].address.apply(lambda ip: f"http://{ip}:8080")) -pulumi.export("encoding_worker_b_url", encoding_worker_ips[1].address.apply(lambda ip: f"http://{ip}:8080")) - # Capacity-fallback VM exports (one per EncodingWorkerConfig.FALLBACKS entry, # keyed by suffix, e.g. encoding_worker_fallback_c2df_ip). for _fb, _fb_ip in zip(EncodingWorkerConfig.FALLBACKS, encoding_worker_fallback_ips): @@ -905,8 +897,6 @@ for _fb, _fb_ip in zip(EncodingWorkerConfig.FALLBACKS, encoding_worker_fallback_ _fb_ip.address.apply(lambda ip: f"http://{ip}:8080"), ) pulumi.export("encoding_worker_service_account", encoding_worker_sa.email) -pulumi.export("encoding_worker_a_name", encoding_worker_instances[0].name) -pulumi.export("encoding_worker_b_name", encoding_worker_instances[1].name) # GitHub runners pulumi.export("github_runner_service_account", github_runner_sa.email) diff --git a/infrastructure/compute/__init__.py b/infrastructure/compute/__init__.py index 7ca154d4..dd1f5d3f 100644 --- a/infrastructure/compute/__init__.py +++ b/infrastructure/compute/__init__.py @@ -7,8 +7,6 @@ Contains VM definitions for: """ from .encoding_worker_vm import ( - create_encoding_worker_vms, - create_encoding_worker_ips, create_encoding_worker_fallback_vms, create_encoding_worker_fallback_ips, create_encoding_worker_firewall, @@ -16,8 +14,6 @@ from .encoding_worker_vm import ( from .github_runners import create_github_runners, create_build_runner, create_cloud_nat __all__ = [ - "create_encoding_worker_vms", - "create_encoding_worker_ips", "create_encoding_worker_fallback_vms", "create_encoding_worker_fallback_ips", "create_encoding_worker_firewall", diff --git a/infrastructure/compute/encoding_worker_vm.py b/infrastructure/compute/encoding_worker_vm.py index f41c5ae6..e20e6be0 100644 --- a/infrastructure/compute/encoding_worker_vm.py +++ b/infrastructure/compute/encoding_worker_vm.py @@ -1,117 +1,20 @@ """ -Encoding Worker VM resources — blue-green deployment pair. +Encoding Worker VM resources — the GCE capacity-fallback fleet. -Manages two identical VMs (a/b) for zero-downtime deployments. -Only one is active (primary) at a time; the other is stopped (secondary). -Both auto-shutdown when idle to minimize cost, and both run on SPOT capacity -(preempted → STOPPED, restarted on demand; lost encodes are resubmitted). One -on-demand fallback VM in a different family/zone absorbs Spot stockouts. +The c4d Spot blue-green pair (encoding-worker-a/-b) was retired 2026-11; the AWS +EC2 worker (infrastructure/aws/) is now the preferred encoder. What remains here +is the on-demand fallback (c2df), provisioned stopped and started on demand. -See docs/superpowers/specs/2026-03-24-blue-green-encoding-worker-design.md +History: docs/superpowers/specs/2026-03-24-blue-green-encoding-worker-design.md """ import pulumi from pulumi_gcp import compute, serviceaccount -from config import REGION, ENCODING_WORKER_ZONE, PROJECT_ID, MachineTypes, DiskSizes, EncodingWorkerConfig +from config import REGION, PROJECT_ID, MachineTypes, DiskSizes, EncodingWorkerConfig from .startup_scripts import read_script -def create_encoding_worker_ips() -> list[compute.Address]: - """Create static IPs for both encoding worker VMs.""" - ips = [] - for name in EncodingWorkerConfig.IP_NAMES: - ip = compute.Address( - name, - name=name, - region=REGION, - address_type="EXTERNAL", - description=f"Static external IP for {name}", - ) - ips.append(ip) - return ips - - -def create_encoding_worker_vms( - ips: list[compute.Address], - service_account: serviceaccount.Account, -) -> list[compute.Instance]: - """Create the blue-green encoding worker VM pair.""" - startup_script = read_script("encoding_worker.sh") - custom_image = f"projects/{PROJECT_ID}/global/images/family/encoding-worker" - - vms = [] - for vm_name, ip in zip(EncodingWorkerConfig.VM_NAMES, ips): - vm = compute.Instance( - vm_name, - name=vm_name, - machine_type=MachineTypes.ENCODING_WORKER, - zone=ENCODING_WORKER_ZONE, - boot_disk=compute.InstanceBootDiskArgs( - initialize_params=compute.InstanceBootDiskInitializeParamsArgs( - image=custom_image, - size=DiskSizes.ENCODING_WORKER, - type="hyperdisk-balanced", - # NOTE: Hyperdisk provisioned IOPS/throughput are deliberately - # NOT set here. Specifying them forces a full VM *replacement* - # (the provider treats boot-disk init params as immutable), and - # recreating a primary depends on us-central1-c capacity — the - # exact shortage the fallback fleet exists to absorb. So the - # disks are pinned to the Hyperdisk Balanced free baseline - # (3000 IOPS / 140 MB/s) live via `gcloud compute disks update` - # instead — non-disruptive, and pulumi does not manage this - # attribute so there is no drift. They were over-provisioned at - # 3600 IOPS / 290 MB/s (~$8.70/disk/mo wasted). Re-apply the - # gcloud tune after any worker-image rebuild that recreates the - # disks. See docs/archive/2026-06-14-gcp-cost-reduction-plan.md. - ), - ), - network_interfaces=[compute.InstanceNetworkInterfaceArgs( - network="default", - access_configs=[compute.InstanceNetworkInterfaceAccessConfigArgs( - nat_ip=ip.address, - )], - )], - service_account=compute.InstanceServiceAccountArgs( - email=service_account.email, - scopes=["cloud-platform"], - ), - metadata_startup_script=startup_script, - tags=["encoding-worker"], - allow_stopping_for_update=True, - advanced_machine_features=compute.InstanceAdvancedMachineFeaturesArgs( - threads_per_core=2, - ), - scheduling=_primary_pair_scheduling(), - ) - vms.append(vm) - return vms - - -def _primary_pair_scheduling() -> compute.InstanceSchedulingArgs: - """Scheduling for the a/b pair: Spot with STOP-on-preemption. - - STOP (not DELETE) keeps the VM, its disk and static IP, so a preempted - worker is just a TERMINATED instance that the backend's ensure-running path - restarts like any idle-stopped worker. automatic_restart must be False and - on_host_maintenance TERMINATE for Spot. - """ - if not EncodingWorkerConfig.PRIMARY_PAIR_SPOT: - return compute.InstanceSchedulingArgs( - provisioning_model="STANDARD", - preemptible=False, - automatic_restart=True, - on_host_maintenance="MIGRATE", - ) - return compute.InstanceSchedulingArgs( - provisioning_model="SPOT", - preemptible=True, - automatic_restart=False, - on_host_maintenance="TERMINATE", - instance_termination_action="STOP", - ) - - def create_encoding_worker_fallback_ips() -> list[compute.Address]: """Create static IPs for the capacity-fallback VMs. @@ -128,7 +31,7 @@ def create_encoding_worker_fallback_ips() -> list[compute.Address]: region=REGION, address_type="EXTERNAL", # Keep this string byte-identical to the originally-deployed value so - # `pulumi up` does not diff (and needlessly churn) the existing a/b IPs. + # `pulumi up` does not diff (and needlessly churn) the existing IPs. description=f"Static external IP for {ip_name} (zone-fallback)", ) ips.append(ip) diff --git a/infrastructure/config.py b/infrastructure/config.py index 60d6eff7..e65aee18 100644 --- a/infrastructure/config.py +++ b/infrastructure/config.py @@ -33,8 +33,8 @@ def get_project_number() -> str: REGION = "us-central1" ZONE = f"{REGION}-a" -# Encoding worker zone - uses us-central1-c due to c4d availability -# (us-central1-a and us-central1-b often lack capacity for high-end C4D instances) +# Default zone the encoding idle-shutdown function targets (GCP_ZONE). The c4d +# pair that lived here was retired 2026-11; fallbacks carry their own zones. ENCODING_WORKER_ZONE = f"{REGION}-c" # Pulumi config accessor @@ -52,7 +52,8 @@ class MachineTypes: # only kept ~66-78% of 32 vCPUs busy; ffmpeg auto-threads adapt to 16). The # worker serializes heavy jobs (ENCODING_HEAVY_CONCURRENCY=1), and highcpu-16 # still has ~30-32 GB RAM — enough for one 4K encode (~18 GB peak). - ENCODING_WORKER = "c4d-highcpu-16" # 16 vCPU / 30 GB, AMD EPYC Turin — Spot blue-green pair + # (The c4d-highcpu-16 Spot blue-green pair was retired 2026-11 in favour of + # the AWS EC2 worker, infrastructure/aws/.) # The ONE on-demand capacity fallback. Deliberately a DIFFERENT machine family # and zone from the c4d Spot pair: c4d suffered a region-wide # ZONE_RESOURCE_POOL_EXHAUSTED stockout (2026-08-12), and Spot capacity can @@ -307,9 +308,7 @@ GITHUB_REPO_NAME = "karaoke-gen" class EncodingWorkerConfig: - """Configuration for blue-green encoding worker VMs.""" - VM_NAMES = ["encoding-worker-a", "encoding-worker-b"] - IP_NAMES = ["encoding-worker-ip-a", "encoding-worker-ip-b"] + """Configuration for the GCE encoding worker fallback fleet + idle shutdown.""" # Stop idle workers fast: VMs bill per second while RUNNING, and with only # ~5-20 jobs/day most of the old 15-min tail was paid idle time. The function # still NEVER stops a VM with active jobs (fail-safe on unconfirmed /health). @@ -319,12 +318,6 @@ class EncodingWorkerConfig: FUNCTION_MEMORY = "512M" # Increased from 256M — OOM with gRPC/Firestore/Compute client libs FUNCTION_TIMEOUT = 120 # 2 minutes - # The a/b blue-green pair runs on SPOT capacity (~60-70% cheaper). A preempted - # Spot VM is STOPPED (not deleted) — the backend restarts it on the next - # request, and an encode lost to preemption is resubmitted automatically - # (encoding_service.wait_for_completion → EncodingJobLostError). - PRIMARY_PAIR_SPOT = True - # Capacity-resilience fallback. Provisioned STOPPED and started on demand only # when the Spot pair can't start (ZONE_RESOURCE_POOL_EXHAUSTED / Spot # stockout). ON-DEMAND (never preempted) and a different family + zone from the diff --git a/infrastructure/encoding-worker/deploy_promote.py b/infrastructure/encoding-worker/deploy_promote.py index 129b0ec6..7a9c74b9 100644 --- a/infrastructure/encoding-worker/deploy_promote.py +++ b/infrastructure/encoding-worker/deploy_promote.py @@ -130,6 +130,13 @@ def promote_plan( serving worker). Leave primary/secondary untouched so c4d resumes as primary when capacity returns. Drain+stop the previous override (if any, and different from the green). + * green.kind == "fallback" while the AWS worker is the serving override: + the green was only a validation target. Keep the AWS override serving, + record the validated version, and stop the green (it served nothing). + + Every successful promote records ``validated_version``: the version the CI + worker-change gate diffs against. ``primary_version`` alone stops moving + once the c4d pair is retired (2026-11). """ drain: List[Dict[str, Optional[str]]] = [] @@ -149,6 +156,7 @@ def promote_plan( "secondary_ip": config.get("primary_ip"), "secondary_version": config.get("primary_version"), "secondary_deployed_at": now, + "validated_version": version, "last_swap_at": now, "deploy_in_progress": False, "deploy_in_progress_since": None, @@ -179,6 +187,18 @@ def promote_plan( }) return _finalize(updates) + if _is_aws_override(config): + # The AWS worker serves by preference and hot-updates its wheel per job; + # a GCE fallback green must not displace it. It was validated only. + return { + "firestore_updates": { + "validated_version": version, + "deploy_in_progress": False, + "deploy_in_progress_since": None, + }, + "drain_and_stop": [{"vm": green["vm"], "ip": green.get("ip"), "zone": green.get("zone")}], + } + # kind == "fallback": the green becomes the serving override. updates = { "active_override_vm": green["vm"], @@ -189,6 +209,7 @@ def promote_plan( # The green is a GCE VM: never leave a stale "aws" tag on it (CI hides # an AWS override from this flow, so it may be the override replaced). "active_override_provider": None, + "validated_version": version, "deploy_in_progress": False, "deploy_in_progress_since": None, } diff --git a/infrastructure/functions/encoding_worker_idle/main.py b/infrastructure/functions/encoding_worker_idle/main.py index db398377..0735270d 100644 --- a/infrastructure/functions/encoding_worker_idle/main.py +++ b/infrastructure/functions/encoding_worker_idle/main.py @@ -35,6 +35,7 @@ from datetime import datetime, timezone, timedelta import functions_framework import requests +from google.api_core import exceptions as gcp_exceptions from google.cloud import compute_v1, firestore logger = logging.getLogger(__name__) @@ -68,6 +69,11 @@ def get_vm_status(vm_name, zone=ZONE): try: instance = client.get(project=PROJECT_ID, zone=zone, instance=vm_name) return instance.status + except gcp_exceptions.NotFound: + # A VM still named in config/secret but deleted (e.g. the retired c4d + # pair): expected, not page-worthy. + logger.warning(f"VM {vm_name} not found in {zone}; skipping") + return "NOT_FOUND" except Exception as e: logger.error(f"Failed to get status for {vm_name} in {zone}: {e}") return "UNKNOWN" diff --git a/infrastructure/functions/encoding_worker_idle/test_idle_shutdown.py b/infrastructure/functions/encoding_worker_idle/test_idle_shutdown.py index 7d7e9c77..3d8d3d82 100644 --- a/infrastructure/functions/encoding_worker_idle/test_idle_shutdown.py +++ b/infrastructure/functions/encoding_worker_idle/test_idle_shutdown.py @@ -592,3 +592,26 @@ class TestCheckActiveJobs: patch("main.time.sleep"): assert main.check_active_jobs("1.2.3.4", attempts=3) == 1 assert get.call_count == 2 + + +def test_missing_vm_logs_warning_not_error(mock_compute, caplog): + """A VM deleted but still named somewhere (the retired c4d pair) is expected: + WARNING + NOT_FOUND, never ERROR (the error monitor alerts on ERROR).""" + from google.api_core import exceptions as gcp_exceptions + + mock_compute.get.side_effect = gcp_exceptions.NotFound("instance gone") + with caplog.at_level("WARNING"): + assert main.get_vm_status("encoding-worker-a") == "NOT_FOUND" + assert not [r for r in caplog.records if r.levelname == "ERROR"] + + +def test_blank_pair_names_are_skipped(mock_compute, mock_firestore): + """After the c4d pair is retired config holds empty primary/secondary names.""" + snap = MagicMock(exists=True) + snap.to_dict.return_value = _make_config(primary_vm="", secondary_vm="") + mock_firestore.collection.return_value.document.return_value.get.return_value = snap + with patch.object(main, "_parse_fallback_vms", return_value=[]): + body, status = main.idle_shutdown(MagicMock()) + assert status == 200 + assert json.loads(body)["results"] == {} + mock_compute.get.assert_not_called() diff --git a/infrastructure/test_encoding_worker_config.py b/infrastructure/test_encoding_worker_config.py index bfa7611d..36c993ea 100644 --- a/infrastructure/test_encoding_worker_config.py +++ b/infrastructure/test_encoding_worker_config.py @@ -10,30 +10,18 @@ Run locally with: `pytest infrastructure/test_encoding_worker_config.py` resource graph via `pulumi preview`). """ -from config import ENCODING_WORKER_ZONE, EncodingWorkerConfig, MachineTypes +from config import EncodingWorkerConfig, MachineTypes -def test_fallback_is_a_different_machine_family_and_zone_from_primary(): - """The on-demand fallback must not share the Spot pair's family or zone, - otherwise a single c4d / Spot / zone stockout takes out every lane at once - (incident 2026-08-12).""" - assert EncodingWorkerConfig.FALLBACKS, "need at least one capacity fallback" - primary_family = _family(MachineTypes.ENCODING_WORKER) - primary_zone_suffix = ENCODING_WORKER_ZONE.rsplit("-", 1)[1] - for fb in EncodingWorkerConfig.FALLBACKS: - assert _family(fb["machine_type"]) != primary_family, fb - assert fb["zone_suffix"] != primary_zone_suffix, fb - - -def test_fleet_is_cost_cut_shape(): - """2026-09-26 cost cut: Spot a/b pair + exactly one on-demand fallback, all - 16 vCPU. Growing this again should be a deliberate decision (update the - fallback secret + docs), not an accident.""" - assert len(EncodingWorkerConfig.VM_NAMES) == 2 +def test_fleet_is_single_on_demand_fallback(): + """Since 2026-11 the GCE fleet is ONE on-demand fallback (c2df): the c4d Spot + a/b pair was retired for the AWS EC2 worker. Growing this again should be a + deliberate decision (update the fallback secret + docs), not an accident.""" + assert not hasattr(EncodingWorkerConfig, "VM_NAMES") + assert not hasattr(MachineTypes, "ENCODING_WORKER") assert len(EncodingWorkerConfig.FALLBACKS) == 1 - assert EncodingWorkerConfig.PRIMARY_PAIR_SPOT is True - for mt in [MachineTypes.ENCODING_WORKER] + [fb["machine_type"] for fb in EncodingWorkerConfig.FALLBACKS]: - assert mt.endswith("-16"), mt + for fb in EncodingWorkerConfig.FALLBACKS: + assert fb["machine_type"].endswith("-16"), fb def test_idle_shutdown_is_fast_but_outlives_review_heartbeat(): diff --git a/pyproject.toml b/pyproject.toml index f44e8675..2c6c11c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.292.0" +version = "0.293.0" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT" diff --git a/tests/unit/services/error_monitor/test_config.py b/tests/unit/services/error_monitor/test_config.py index 21c9fd09..c2e7b09c 100644 --- a/tests/unit/services/error_monitor/test_config.py +++ b/tests/unit/services/error_monitor/test_config.py @@ -74,22 +74,24 @@ class TestMonitoredServiceLists: assert config.MONITORED_CLOUD_FUNCTIONS == [] - def test_gce_instances_count_at_least_4(self): + def test_gce_instances_count_at_least_3(self): from backend.services.error_monitor import config - assert len(config.MONITORED_GCE_INSTANCES) >= 4 + assert len(config.MONITORED_GCE_INSTANCES) >= 3 def test_gce_instances_contains_expected(self): from backend.services.error_monitor import config expected = [ - "encoding-worker-a", - "encoding-worker-b", + "encoding-worker-fallback-c2df", "flacfetch-vm", "divebar-sync-vm", ] for instance in expected: assert instance in config.MONITORED_GCE_INSTANCES, f"Expected '{instance}' in MONITORED_GCE_INSTANCES" + # The c4d pair was retired 2026-11; monitoring deleted VMs is noise. + assert "encoding-worker-a" not in config.MONITORED_GCE_INSTANCES + assert "encoding-worker-b" not in config.MONITORED_GCE_INSTANCES class TestDefaultValues: