mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
feat(encoding): retire the GCE c4d pair; run on AWS + c2df only (0.293.0) (#1255)
## Why Since 2026-10-31 the AWS EC2 worker (`aws-encoding-worker-1`) has handled every encode and distribution job. 27 jobs went to it, with no Cloud Run fallbacks, no `encoding_stuck`, and nothing encoding-related in #alerts. The c4d Spot pair (`encoding-worker-a`/`-b`) now only costs ~$8/mo of Hyperdisk. `encoding-worker-fallback-c2df` stays as the GCP fallback. ## What Deleting the pair needs code changes first, because several paths assumed it exists: - **`encoding_worker_manager`** - The primary is optional: an empty `primary_vm` means none. - `ranked_candidates` skips a missing primary and always sets `is_primary`, so with no primary a successful start records the AWS/c2df override. Previously the legacy index-0 rule would have cleared it. - `ensure_primary_running` and `ensure_preferred_running` start a ranked candidate. If there are no candidates they raise `NO_CANDIDATES`. - `demote_active_worker` looks up the override's `machine_type`, so the AWS box can now be cooled down. It then clears the override so the retry goes elsewhere. - **`encoding_service`** - When `active_url` is empty, it targets the top-ranked candidate. - It also invalidates its cached URL after a single-candidate fallback. - **CI deploy (`ci.yml` + `deploy_promote.py`)** - While AWS serves, a c2df green is validate-only: it writes `validated_version`, keeps the AWS override, and stops the green. Before this, it would have replaced the AWS override with c2df. - The worker-change gate now diffs against `validated_version`, because `primary_version` stops moving. - An in-place restart of a serving override first waits for it to drain (15-min cap). If it is still busy, the restart is skipped. - **Idle function:** a missing VM is logged as a WARNING returning `NOT_FOUND`, not an ERROR. - **Error monitor:** watches c2df instead of a/b. - **Pulumi:** drops the a/b VMs and static IPs. The prod preview shows exactly 4 deletes. A separate, unrelated `youtube-comment-autoreply-scheduler` drift shows up too; it will not be applied. - **Docs:** CLAUDE.md, ARCHITECTURE, DEVELOPMENT and TROUBLESHOOTING. ## Rollout (after merge + deploy) 1. Back up `config/encoding-worker`, then set `primary_vm`, `primary_ip`, `secondary_vm` and `secondary_ip` to `""`. 2. Apply `pulumi destroy --target` on the 4 a/b URNs only. 3. Redeploy the idle function with `deploy.sh`. 4. Verify: one prod encode on AWS, warmup, and the logs. **Rollback:** in Firestore, delete `speed_rank_overrides` and set `aws_video_worker=false`. Restoring the pair would need a revert plus `pulumi up`, which gives new static IPs. ## Testing - New `backend/tests/test_encoding_worker_no_primary.py` (13 tests), plus new deploy_promote, idle-function and error-monitor tests. - Full backend suite: 7188 passed. The 8 `test_emulator_integration` failures also fail on main, because of a local emulator from another session. - CodeRabbit CLI: no findings. An independent review found 2 issues, both fixed: the demote retry path, and the in-place restart drain. 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_012RYLVTkatGnrYFhCzq4PTe @coderabbitai ignore Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
d63c9781e1
commit
a1cbe1c0d4
20 files changed
+470
-215
No files matched your search
@@ -1439,7 +1439,12 @@ jobs:
|
||||
print("::error::Encoding worker config not found in Firestore")
|
||||
raise SystemExit(1)
|
||||
data = doc.to_dict()
|
||||
if (data.get("active_override_provider") or "").lower() == "aws":
|
||||
aws_override = (data.get("active_override_provider") or "").lower() == "aws"
|
||||
# Version the serving worker was last validated at: the CI gate diffs
|
||||
# worker code against it. primary_version stops moving once the c4d
|
||||
# pair is retired, so prefer validated_version (written on promote).
|
||||
data["validated_version"] = data.get("validated_version") or data.get("primary_version")
|
||||
if aws_override:
|
||||
# The AWS encoding worker (EC2) is serving by preference. This
|
||||
# GCE blue-green flow can't gcloud-start/stop it and must not
|
||||
# clear its override; it picks up the new wheel on its next job.
|
||||
@@ -1449,7 +1454,8 @@ jobs:
|
||||
for key in ("active_override_vm", "active_override_ip", "active_override_zone"):
|
||||
data[key] = None
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as f:
|
||||
for key in ["primary_vm", "primary_ip", "primary_version",
|
||||
f.write(f"aws_override={str(aws_override).lower()}\n")
|
||||
for key in ["primary_vm", "primary_ip", "primary_version", "validated_version",
|
||||
"secondary_vm", "secondary_ip",
|
||||
"active_override_vm", "active_override_ip", "active_override_zone"]:
|
||||
# `or ''`: a key stored as null must be empty, not the string
|
||||
@@ -1475,7 +1481,7 @@ jobs:
|
||||
id: worker_gate
|
||||
env:
|
||||
FORCE: ${{ github.event_name == 'workflow_dispatch' && inputs.force_encoding_worker_deploy == true }}
|
||||
SERVING_VERSION: ${{ steps.config.outputs.primary_version }}
|
||||
SERVING_VERSION: ${{ steps.config.outputs.validated_version }}
|
||||
OVERRIDE_VM: ${{ steps.config.outputs.active_override_vm }}
|
||||
run: |
|
||||
mapfile -t PATHS < <(grep -v '^\s*#' infrastructure/encoding-worker/worker_code_paths.txt | grep -v '^\s*$')
|
||||
@@ -1621,7 +1627,34 @@ jobs:
|
||||
# serving override in place (accepts a brief blip; auto-resubmit
|
||||
# covers any in-flight render). If there's no override either, fail.
|
||||
ov = os.environ.get("OVERRIDE_VM")
|
||||
ov_ip = os.environ.get("OVERRIDE_IP")
|
||||
|
||||
def _active_jobs(ip):
|
||||
import urllib.request
|
||||
if not ip:
|
||||
return None
|
||||
try:
|
||||
with urllib.request.urlopen(f"http://{ip}:8080/health", timeout=5) as r:
|
||||
return json.load(r).get("active_jobs")
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
# Since the c4d pair was retired (2026-11) this is the normal path when
|
||||
# AWS is in cooldown and c2df serves: never stop it mid-render. Wait
|
||||
# (bounded) for a confirmed idle worker; if it stays busy, skip — it
|
||||
# idle-stops after 5 min and cold-boots the new wheel next time.
|
||||
drained = True
|
||||
if ov:
|
||||
drain_deadline = time.monotonic() + 900
|
||||
n = _active_jobs(ov_ip)
|
||||
while n != 0 and time.monotonic() < drain_deadline:
|
||||
print(f" serving override {ov} active_jobs={n}; waiting to restart")
|
||||
time.sleep(15)
|
||||
n = _active_jobs(ov_ip)
|
||||
drained = n == 0 or vm_status(ov, os.environ.get("OVERRIDE_ZONE") or "us-central1-f") != "RUNNING"
|
||||
if not drained:
|
||||
print(f"::warning::Serving override {ov} still busy (active_jobs={n}); skipping in-place restart")
|
||||
if ov and drained:
|
||||
zone = os.environ.get("OVERRIDE_ZONE") or "us-central1-f"
|
||||
print(f"::warning::No separate green could start; restarting serving override {ov} in place")
|
||||
subprocess.run(["gcloud", "compute", "instances", "stop", ov,
|
||||
@@ -1646,7 +1679,7 @@ jobs:
|
||||
if final_status != "RUNNING":
|
||||
inplace_failed = True
|
||||
print(f"::error::In-place restart of {ov} did not reach RUNNING (status={final_status or 'unknown'})")
|
||||
else:
|
||||
elif not ov:
|
||||
print("::error::No green target could be started and no active override exists")
|
||||
|
||||
def _out(**kv):
|
||||
@@ -1841,6 +1874,7 @@ jobs:
|
||||
OVERRIDE_VM: ${{ steps.config.outputs.active_override_vm }}
|
||||
OVERRIDE_IP: ${{ steps.config.outputs.active_override_ip }}
|
||||
OVERRIDE_ZONE: ${{ steps.config.outputs.active_override_zone }}
|
||||
AWS_OVERRIDE: ${{ steps.config.outputs.aws_override }}
|
||||
run: |
|
||||
python3 << 'PYEOF'
|
||||
import json, os, subprocess, sys, time, urllib.request
|
||||
@@ -1859,6 +1893,9 @@ jobs:
|
||||
"active_override_vm": os.environ.get("OVERRIDE_VM") or None,
|
||||
"active_override_ip": os.environ.get("OVERRIDE_IP") or None,
|
||||
"active_override_zone": os.environ.get("OVERRIDE_ZONE") or None,
|
||||
# The config step hid an AWS override from the GCE flow; tell the
|
||||
# promote plan so a fallback green doesn't displace it.
|
||||
"active_override_provider": "aws" if os.environ.get("AWS_OVERRIDE") == "true" else None,
|
||||
}
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
plan = deploy_promote.promote_plan(green, config, now=now, version=os.environ["NEW_VERSION"])
|
||||
@@ -1866,7 +1903,10 @@ jobs:
|
||||
db = firestore.Client(project="nomadkaraoke")
|
||||
db.collection("config").document("encoding-worker").update(plan["firestore_updates"])
|
||||
print("Applied config update:", json.dumps(plan["firestore_updates"], default=str))
|
||||
print(f"Green {green['vm']} ({green['kind']}) is now serving")
|
||||
if any(t.get("vm") == green["vm"] for t in plan["drain_and_stop"]):
|
||||
print(f"Green {green['vm']} ({green['kind']}) validated; AWS worker stays serving, stopping the green")
|
||||
else:
|
||||
print(f"Green {green['vm']} ({green['kind']}) is now serving")
|
||||
|
||||
def active_jobs(ip):
|
||||
# /health is unauthenticated; None == "unknown / maybe busy".
|
||||
|
||||
@@ -151,21 +151,22 @@ EOF
|
||||
```
|
||||
|
||||
### GCE Encoding Worker
|
||||
The worker is a **Spot** c4d-highcpu-16 blue-green primary pair (`encoding-worker-a`/`-b`,
|
||||
us-central1-c) plus ONE stopped on-demand fallback (`encoding-worker-fallback-c2df`,
|
||||
c2d-highcpu-16, us-central1-f) — 3 VMs since 2026-09-26. Idle VMs stop after 5 min.
|
||||
CI blue-green redeploys the worker only when worker code changed
|
||||
(`infrastructure/encoding-worker/worker_code_paths.txt`); force it via Actions → CI →
|
||||
Run workflow → "Force a blue-green redeploy of the GCE encoding worker". The currently-serving
|
||||
VM is `active_override_vm` in `config/encoding-worker` if set, else `primary_vm`.
|
||||
Use that VM name + its zone below (there is no VM literally named `encoding-worker`):
|
||||
Since 2026-11 the only GCE encoding VM is the stopped on-demand fallback
|
||||
`encoding-worker-fallback-c2df` (c2d-highcpu-16, us-central1-f); the c4d Spot pair
|
||||
(`encoding-worker-a`/`-b`) was retired and deleted, and `primary_vm`/`secondary_vm` in
|
||||
`config/encoding-worker` are empty. The serving worker is `active_override_vm` (normally the
|
||||
AWS box below). Idle VMs stop after 5 min. CI's "blue-green" deploy now only runs when worker
|
||||
code changed (`infrastructure/encoding-worker/worker_code_paths.txt`, diffed against
|
||||
`validated_version`): it boots c2df, runs a test encode, records `validated_version` and stops
|
||||
it again, leaving the AWS override serving. Force it via Actions → CI → Run workflow →
|
||||
"Force a blue-green redeploy of the GCE encoding worker".
|
||||
```bash
|
||||
# Check health (includes wheel_version). Example uses the primary pair's VM.
|
||||
gcloud compute ssh encoding-worker-a --zone=us-central1-c --project=nomadkaraoke \
|
||||
# Check health (includes wheel_version) on the GCE fallback.
|
||||
gcloud compute ssh encoding-worker-fallback-c2df --zone=us-central1-f --project=nomadkaraoke \
|
||||
--command="curl -s http://localhost:8080/health"
|
||||
|
||||
# Restart the service (pick up new worker-side code after a deploy)
|
||||
gcloud compute ssh encoding-worker-a --zone=us-central1-c --project=nomadkaraoke \
|
||||
gcloud compute ssh encoding-worker-fallback-c2df --zone=us-central1-f --project=nomadkaraoke \
|
||||
--command="sudo systemctl restart encoding-worker"
|
||||
```
|
||||
|
||||
|
||||
@@ -239,7 +239,15 @@ class EncodingService:
|
||||
|
||||
if self._worker_manager:
|
||||
config = self._worker_manager.get_config()
|
||||
self._cached_url = config.active_url
|
||||
url = config.active_url
|
||||
if not url:
|
||||
# No primary pair and no override recorded yet: target the
|
||||
# top-ranked candidate. If it's stopped, the connection failure
|
||||
# triggers the warmup fallback, which starts it and records it
|
||||
# as the override.
|
||||
candidates = self._worker_manager.ranked_candidates(config)
|
||||
url = candidates[0].url if candidates else ""
|
||||
self._cached_url = url
|
||||
self._url_cached_at = now
|
||||
return self._cached_url
|
||||
|
||||
@@ -456,6 +464,10 @@ class EncodingService:
|
||||
self._invalidate_cached_url()
|
||||
else:
|
||||
result = await asyncio.to_thread(self._worker_manager.ensure_primary_running)
|
||||
# With no GCE primary pair, ensure_primary_running starts a ranked
|
||||
# candidate and records it as the override.
|
||||
if result.get("fell_back"):
|
||||
self._invalidate_cached_url()
|
||||
except EncodingWorkerStartError:
|
||||
# Every candidate exhausted (or single-zone primary failed) — both
|
||||
# capacity errors and other start failures (e.g. 503
|
||||
|
||||
@@ -119,8 +119,15 @@ class EncodingWorkerConfig:
|
||||
# also runs on it instead of the Cloud Run job (worker_service). Default off.
|
||||
aws_video_worker: bool = False
|
||||
|
||||
@property
|
||||
def has_primary(self) -> bool:
|
||||
"""False once the GCE blue-green pair is retired (empty primary_vm)."""
|
||||
return bool(self.primary_vm)
|
||||
|
||||
@property
|
||||
def primary_url(self) -> str:
|
||||
if not self.primary_ip:
|
||||
return ""
|
||||
return f"http://{self.primary_ip}:{ENCODING_WORKER_PORT}"
|
||||
|
||||
@property
|
||||
@@ -134,7 +141,8 @@ class EncodingWorkerConfig:
|
||||
Returns the capacity-fallback override if set, otherwise the primary
|
||||
URL. The override mechanism lets us route around a zone that's
|
||||
temporarily out of c4d (Spot) capacity without rewriting the
|
||||
blue-green primary/secondary tracking.
|
||||
blue-green primary/secondary tracking. Empty when neither is set (no
|
||||
primary pair and no override yet); callers resolve a candidate then.
|
||||
"""
|
||||
if self.active_override_ip:
|
||||
return f"http://{self.active_override_ip}:{ENCODING_WORKER_PORT}"
|
||||
@@ -159,8 +167,11 @@ class EncodingWorkerCandidate:
|
||||
# True iff this candidate is the blue-green PRIMARY VM. The override routing
|
||||
# (set on fallback, clear on primary) keys off this flag, NOT list position —
|
||||
# the preference logic may rank a fallback ahead of a stocked-out primary, so
|
||||
# "first candidate" is no longer synonymous with "primary".
|
||||
is_primary: bool = False
|
||||
# "first candidate" is no longer synonymous with "primary". None ⇒ unknown
|
||||
# (hand-built lists): the legacy "index 0 is the primary" rule applies.
|
||||
# ranked_candidates always sets True/False, so with no primary pair every
|
||||
# candidate is a fallback and a successful start always sets the override.
|
||||
is_primary: Optional[bool] = None
|
||||
# "aws" for the EC2 worker (routed to AwsComputeAdapter); None ⇒ GCE.
|
||||
provider: Optional[str] = None
|
||||
|
||||
@@ -282,14 +293,18 @@ class EncodingWorkerManager:
|
||||
|
||||
config = config or self.get_config()
|
||||
primary_mt = getattr(config, "primary_machine_type", None) or PRIMARY_MACHINE_TYPE
|
||||
pool = [{
|
||||
"vm": config.primary_vm,
|
||||
"zone": self._zone,
|
||||
"ip": config.primary_ip,
|
||||
"machine_type": primary_mt,
|
||||
"kind": "primary",
|
||||
"is_primary": True,
|
||||
}]
|
||||
pool = []
|
||||
# No primary once the GCE c4d pair is retired (2026-11): the pool is then
|
||||
# just the fallbacks (AWS worker + GCE c2df).
|
||||
if config.primary_vm:
|
||||
pool.append({
|
||||
"vm": config.primary_vm,
|
||||
"zone": self._zone,
|
||||
"ip": config.primary_ip,
|
||||
"machine_type": primary_mt,
|
||||
"kind": "primary",
|
||||
"is_primary": True,
|
||||
})
|
||||
pool.extend(self.fallback_entries())
|
||||
ranked = ordered_candidates(
|
||||
pool,
|
||||
@@ -302,7 +317,7 @@ class EncodingWorkerManager:
|
||||
zone=c["zone"],
|
||||
ip=c["ip"],
|
||||
machine_type=c.get("machine_type"),
|
||||
is_primary=c.get("is_primary", False),
|
||||
is_primary=bool(c.get("is_primary", False)),
|
||||
provider=c.get("provider"),
|
||||
)
|
||||
for c in ranked
|
||||
@@ -320,8 +335,13 @@ class EncodingWorkerManager:
|
||||
"""
|
||||
config = self.get_config()
|
||||
candidates = self.ranked_candidates(config)
|
||||
if not candidates or candidates[0].is_primary:
|
||||
if config.has_primary and (not candidates or candidates[0].is_primary):
|
||||
return self.ensure_primary_running()
|
||||
if not candidates:
|
||||
raise EncodingWorkerStartError(
|
||||
"No encoding worker candidates configured (no primary, no fallbacks)",
|
||||
vm_name="", zone="", code="NO_CANDIDATES",
|
||||
)
|
||||
return self.ensure_any_running(candidates)
|
||||
|
||||
def _doc_ref(self):
|
||||
@@ -341,13 +361,15 @@ class EncodingWorkerManager:
|
||||
f"{CONFIG_COLLECTION}/{CONFIG_DOCUMENT}"
|
||||
)
|
||||
data = snapshot.to_dict()
|
||||
# The primary/secondary pair is optional: empty/absent once the GCE c4d
|
||||
# pair is retired, leaving only the fallback pool (AWS + c2df).
|
||||
return EncodingWorkerConfig(
|
||||
primary_vm=data["primary_vm"],
|
||||
primary_ip=data["primary_ip"],
|
||||
primary_version=data["primary_version"],
|
||||
primary_vm=data.get("primary_vm") or "",
|
||||
primary_ip=data.get("primary_ip") or "",
|
||||
primary_version=data.get("primary_version") or "",
|
||||
primary_deployed_at=data.get("primary_deployed_at"),
|
||||
secondary_vm=data["secondary_vm"],
|
||||
secondary_ip=data["secondary_ip"],
|
||||
secondary_vm=data.get("secondary_vm") or "",
|
||||
secondary_ip=data.get("secondary_ip") or "",
|
||||
secondary_version=data.get("secondary_version"),
|
||||
secondary_deployed_at=data.get("secondary_deployed_at"),
|
||||
last_swap_at=data.get("last_swap_at"),
|
||||
@@ -495,6 +517,16 @@ class EncodingWorkerManager:
|
||||
EncodingWorkerStartError: any other start failure.
|
||||
"""
|
||||
config = self.get_config()
|
||||
if not config.has_primary:
|
||||
# No GCE primary pair: "the primary" is whichever ranked candidate
|
||||
# starts (the AWS worker when preferred). Same contract as before.
|
||||
candidates = self.ranked_candidates(config)
|
||||
if not candidates:
|
||||
raise EncodingWorkerStartError(
|
||||
"No encoding worker candidates configured (no primary, no fallbacks)",
|
||||
vm_name="", zone="", code="NO_CANDIDATES",
|
||||
)
|
||||
return self.ensure_any_running(candidates)
|
||||
vm_name = config.primary_vm
|
||||
|
||||
status = self.get_vm_status(vm_name)
|
||||
@@ -594,7 +626,9 @@ class EncodingWorkerManager:
|
||||
last_non_transient_error: Optional[EncodingWorkerStartError] = None
|
||||
# When the caller marks a primary explicitly, route override off that flag;
|
||||
# otherwise preserve the legacy "index 0 is the primary" behaviour.
|
||||
has_explicit_primary = any(getattr(c, "is_primary", False) for c in candidates)
|
||||
has_explicit_primary = any(
|
||||
getattr(c, "is_primary", None) is not None for c in candidates
|
||||
)
|
||||
for index, candidate in enumerate(candidates):
|
||||
try:
|
||||
try:
|
||||
@@ -645,7 +679,7 @@ class EncodingWorkerManager:
|
||||
# explicit flag when present (ordering can rank a fallback ahead of
|
||||
# a stocked-out primary, so it may not be index 0); otherwise the
|
||||
# legacy position-based rule.
|
||||
is_primary = candidate.is_primary if has_explicit_primary else (index == 0)
|
||||
is_primary = bool(candidate.is_primary) if has_explicit_primary else (index == 0)
|
||||
fell_back = not is_primary
|
||||
if fell_back:
|
||||
self._set_active_override(candidate)
|
||||
@@ -798,6 +832,12 @@ class EncodingWorkerManager:
|
||||
|
||||
vm = config.active_override_vm
|
||||
zone = config.active_override_zone
|
||||
if not vm and not config.has_primary:
|
||||
logger.warning(
|
||||
"demote_active_worker: no override and no primary pair; nothing to demote (%s)",
|
||||
reason,
|
||||
)
|
||||
return
|
||||
if not vm:
|
||||
# Primary pair was serving — see docstring; leave selection untouched.
|
||||
# Logged at ERROR so it alerts: on 2026-10-06 a primary that booted
|
||||
@@ -810,7 +850,12 @@ class EncodingWorkerManager:
|
||||
)
|
||||
return
|
||||
|
||||
key = cooldown_key({"vm_name": vm, "zone": zone})
|
||||
# Fallback entries carry machine_type (the AWS worker's name has no family
|
||||
# token, so without it the AWS box could never be cooled down).
|
||||
machine_type = next(
|
||||
(e.get("machine_type") for e in self.fallback_entries() if e.get("vm") == vm), None
|
||||
)
|
||||
key = cooldown_key({"vm_name": vm, "zone": zone, "machine_type": machine_type})
|
||||
if key:
|
||||
try:
|
||||
self._doc_ref().set(
|
||||
@@ -824,6 +869,8 @@ class EncodingWorkerManager:
|
||||
logger.warning(
|
||||
"demote_active_worker: could not record cooldown for %s: %s", key, e
|
||||
)
|
||||
# With no primary pair, an empty override makes encoding_service resolve
|
||||
# the top-ranked candidate, which the cooldown above now demotes.
|
||||
self._clear_active_override()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@@ -77,8 +77,9 @@ MONITORED_CLOUD_RUN_JOBS: list[str] = [
|
||||
MONITORED_CLOUD_FUNCTIONS: list[str] = []
|
||||
|
||||
MONITORED_GCE_INSTANCES: list[str] = [
|
||||
"encoding-worker-a",
|
||||
"encoding-worker-b",
|
||||
# GCE encoding fallback; the c4d pair (encoding-worker-a/b) was retired
|
||||
# 2026-11 and the AWS worker (aws-encoding-worker-1) logs on the box.
|
||||
"encoding-worker-fallback-c2df",
|
||||
"flacfetch-vm",
|
||||
"divebar-sync-vm",
|
||||
]
|
||||
@@ -96,7 +97,8 @@ karaoke-backend (main API)
|
||||
→ audio-separation-job (Cloud Run Job, runs on Cloud Run GPU)
|
||||
→ audio-download-job (Cloud Run Job, uses flacfetch-vm)
|
||||
→ gdrive-validator (Cloud Function, validates output files)
|
||||
→ encoding-worker-a/b (GCE VMs, long-running encoding workers)
|
||||
→ aws-encoding-worker-1 (AWS EC2, preferred encoding + distribution worker)
|
||||
→ encoding-worker-fallback-c2df (GCE VM, encoding fallback)
|
||||
|
||||
karaoke-decide (catalog / discovery API)
|
||||
→ kn_data_sync (Cloud Function, syncs KaraokeNerds catalog)
|
||||
|
||||
@@ -247,12 +247,45 @@ def test_secondary_promotion_leaves_aws_override_serving():
|
||||
assert [d["vm"] for d in plan["drain_and_stop"]] == ["encoding-worker-a"]
|
||||
|
||||
|
||||
def test_fallback_promotion_does_not_drain_aws_override():
|
||||
def test_fallback_green_is_validate_only_while_aws_serves():
|
||||
"""With the AWS worker serving, a GCE fallback green must not displace it:
|
||||
record the validated version, keep the override, stop the green."""
|
||||
config = {"active_override_vm": "aws-encoding-worker-1", "active_override_zone": "us-east-2b",
|
||||
"active_override_provider": "aws"}
|
||||
green = {"vm": "encoding-worker-fallback-c2df", "ip": "2.2.2.2", "zone": "us-central1-f", "kind": "fallback"}
|
||||
plan = deploy_promote.promote_plan(green, config, now="2026-10-29T00:00:00Z", version="2.0")
|
||||
assert plan["drain_and_stop"] == []
|
||||
updates = plan["firestore_updates"]
|
||||
assert not any(k.startswith("active_override") for k in updates)
|
||||
assert updates["validated_version"] == "2.0"
|
||||
assert updates["deploy_in_progress"] is False
|
||||
# Never the AWS override; only the green we booted for validation.
|
||||
assert plan["drain_and_stop"] == [
|
||||
{"vm": "encoding-worker-fallback-c2df", "ip": "2.2.2.2", "zone": "us-central1-f"}
|
||||
]
|
||||
|
||||
|
||||
def test_aws_override_hidden_by_ci_is_still_recognised():
|
||||
"""CI blanks the AWS override's vm/ip/zone but passes the provider; that alone
|
||||
must keep the AWS worker serving (no c4d pair, 2026-11)."""
|
||||
config = {"active_override_vm": None, "active_override_provider": "aws"}
|
||||
green = {"vm": "encoding-worker-fallback-c2df", "ip": "2.2.2.2", "zone": "us-central1-f", "kind": "fallback"}
|
||||
plan = deploy_promote.promote_plan(green, config, now="t", version="2.1")
|
||||
assert "active_override_vm" not in plan["firestore_updates"]
|
||||
|
||||
|
||||
def test_no_pair_green_selection_is_fallbacks_only():
|
||||
"""Once the c4d pair is retired (empty secondary), the only GCE green is c2df."""
|
||||
config = {"secondary_vm": None, "active_override_vm": None}
|
||||
fallbacks = [{"vm": "encoding-worker-fallback-c2df", "zone": "us-central1-f", "ip": "2.2.2.2"}, _AWS]
|
||||
names = [c["vm"] for c in deploy_promote.select_green_candidates(config, fallbacks)]
|
||||
assert names == ["encoding-worker-fallback-c2df"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["secondary", "fallback"])
|
||||
def test_every_promote_records_validated_version(kind):
|
||||
green = {"vm": "g", "ip": "9.9.9.9", "zone": "us-central1-f", "kind": kind}
|
||||
plan = deploy_promote.promote_plan(green, {"primary_vm": "p"}, now="t", version="3.0")
|
||||
assert plan["firestore_updates"]["validated_version"] == "3.0"
|
||||
|
||||
|
||||
def test_fallback_promotion_clears_provider_tag():
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
"""EncodingWorkerManager once the GCE c4d blue-green pair is retired (2026-11).
|
||||
|
||||
config/encoding-worker then has empty (or absent) primary_vm/secondary_vm, and the
|
||||
pool is just the fallbacks: the AWS worker (preferred) + the GCE c2df VM. Every
|
||||
path must route to a fallback and record it as the active override, never to a
|
||||
blank primary.
|
||||
"""
|
||||
import json
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from backend.services.encoding_errors import EncodingWorkerStartError
|
||||
from backend.services.encoding_worker_manager import (
|
||||
EncodingWorkerCandidate,
|
||||
EncodingWorkerManager,
|
||||
)
|
||||
|
||||
AWS_ENTRY = {
|
||||
"vm": "aws-encoding-worker-1", "zone": "us-east-2b", "ip": "3.3.3.3",
|
||||
"machine_type": "c7a-4xlarge", "provider": "aws", "instance_id": "i-0abc", "region": "us-east-2",
|
||||
}
|
||||
C2D_ENTRY = {"vm": "encoding-worker-fallback-c2df", "zone": "us-central1-f", "ip": "2.2.2.2",
|
||||
"machine_type": "c2d-highcpu-16"}
|
||||
|
||||
NO_PAIR = {"primary_vm": "", "primary_ip": "", "primary_version": "0.291.0",
|
||||
"secondary_vm": "", "secondary_ip": ""}
|
||||
|
||||
|
||||
def _db(**config):
|
||||
data = {"last_activity_at": None, "capacity_state": {}, "speed_rank_overrides": {"c7a": 5},
|
||||
**config}
|
||||
db = MagicMock()
|
||||
snap = MagicMock(exists=True)
|
||||
snap.to_dict.return_value = data
|
||||
doc = MagicMock()
|
||||
doc.get.return_value = snap
|
||||
db.collection.return_value.document.return_value = doc
|
||||
return db, doc
|
||||
|
||||
|
||||
def _manager(db, entries=(C2D_ENTRY, AWS_ENTRY)):
|
||||
gce = MagicMock(name="gce")
|
||||
aws = MagicMock(name="aws")
|
||||
aws.handles.side_effect = lambda vm: vm == AWS_ENTRY["vm"]
|
||||
m = EncodingWorkerManager(db=db, compute_client=gce,
|
||||
fallback_vms_raw=json.dumps(list(entries)), aws_compute=aws)
|
||||
return m, gce, aws
|
||||
|
||||
|
||||
def test_get_config_tolerates_absent_pair_keys():
|
||||
db, _ = _db() # no primary/secondary keys at all
|
||||
m, _, _ = _manager(db)
|
||||
config = m.get_config()
|
||||
assert config.primary_vm == "" and config.secondary_vm == ""
|
||||
assert not config.has_primary
|
||||
assert config.primary_url == ""
|
||||
assert config.active_url == ""
|
||||
|
||||
|
||||
def test_active_url_follows_override_without_primary():
|
||||
db, _ = _db(**NO_PAIR, active_override_vm="aws-encoding-worker-1", active_override_ip="3.3.3.3")
|
||||
m, _, _ = _manager(db)
|
||||
assert m.get_config().active_url == "http://3.3.3.3:8080"
|
||||
|
||||
|
||||
def test_ranked_candidates_skip_blank_primary():
|
||||
db, _ = _db(**NO_PAIR)
|
||||
m, _, _ = _manager(db)
|
||||
ranked = m.ranked_candidates()
|
||||
assert [c.vm_name for c in ranked] == ["aws-encoding-worker-1", "encoding-worker-fallback-c2df"]
|
||||
assert all(c.is_primary is False for c in ranked)
|
||||
|
||||
|
||||
def test_aws_landing_sets_override_not_clears_it():
|
||||
"""With no primary, index 0 (AWS) is a fallback: starting it must SET the
|
||||
override (the legacy 'index 0 is the primary' rule would clear it, leaving
|
||||
active_url empty)."""
|
||||
db, doc = _db(**NO_PAIR)
|
||||
m, _, aws = _manager(db)
|
||||
aws.get.return_value = MagicMock(status="RUNNING")
|
||||
m.update_activity = MagicMock()
|
||||
result = m.ensure_any_running(m.ranked_candidates())
|
||||
assert result["vm_name"] == "aws-encoding-worker-1"
|
||||
assert result["fell_back"] is True
|
||||
update = doc.update.call_args[0][0]
|
||||
assert update["active_override_vm"] == "aws-encoding-worker-1"
|
||||
assert update["active_override_provider"] == "aws"
|
||||
|
||||
|
||||
def test_hand_built_candidates_keep_legacy_index0_rule():
|
||||
"""Candidates built without is_primary (None) keep the old position rule."""
|
||||
db, doc = _db(**NO_PAIR)
|
||||
m, gce, _ = _manager(db)
|
||||
gce.get.return_value = MagicMock(status="RUNNING")
|
||||
m.update_activity = MagicMock()
|
||||
m._clear_active_override = MagicMock()
|
||||
m._set_active_override = MagicMock()
|
||||
result = m.ensure_any_running([EncodingWorkerCandidate("vm-x", "us-central1-f", "9.9.9.9")])
|
||||
assert result["fell_back"] is False
|
||||
m._clear_active_override.assert_called_once()
|
||||
m._set_active_override.assert_not_called()
|
||||
|
||||
|
||||
def test_ensure_primary_running_without_pair_starts_ranked_candidate():
|
||||
db, _ = _db(**NO_PAIR)
|
||||
m, gce, _ = _manager(db)
|
||||
m.ensure_any_running = MagicMock(return_value={"started": True, "vm_name": "aws-encoding-worker-1",
|
||||
"fell_back": True})
|
||||
result = m.ensure_primary_running()
|
||||
assert result["vm_name"] == "aws-encoding-worker-1"
|
||||
(candidates,), _ = m.ensure_any_running.call_args
|
||||
assert candidates[0].vm_name == "aws-encoding-worker-1"
|
||||
gce.get.assert_not_called() # never looks up a blank primary on GCE
|
||||
|
||||
|
||||
def test_ensure_preferred_running_without_pair_never_takes_primary_path():
|
||||
# No rank override → c2df ranks first; still no primary path.
|
||||
db, _ = _db(**NO_PAIR, speed_rank_overrides={})
|
||||
m, _, _ = _manager(db)
|
||||
m.ensure_any_running = MagicMock(return_value={"started": False})
|
||||
m.ensure_preferred_running()
|
||||
m.ensure_any_running.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method", ["ensure_preferred_running", "ensure_primary_running"])
|
||||
def test_no_candidates_at_all_raises_start_error(method):
|
||||
db, _ = _db(**NO_PAIR)
|
||||
m, _, _ = _manager(db, entries=())
|
||||
with pytest.raises(EncodingWorkerStartError) as ei:
|
||||
getattr(m, method)()
|
||||
assert ei.value.code == "NO_CANDIDATES"
|
||||
|
||||
|
||||
def test_demote_aws_without_primary_retries_on_another_worker():
|
||||
"""A broken AWS worker is cooled down (via its machine_type) and the override
|
||||
cleared, so the retry's URL resolves to the GCE fallback instead."""
|
||||
from backend.services.encoding_service import EncodingService
|
||||
|
||||
db, doc = _db(**NO_PAIR, active_override_vm="aws-encoding-worker-1",
|
||||
active_override_zone="us-east-2b", active_override_ip="3.3.3.3")
|
||||
m, _, _ = _manager(db)
|
||||
m.demote_active_worker("metadata unreachable")
|
||||
state = doc.set.call_args[0][0]["capacity_state"]
|
||||
assert list(state) == ["c7a-4xlarge@us-east-2b"]
|
||||
assert doc.update.call_args[0][0]["active_override_vm"] is None
|
||||
|
||||
# Next read: override gone, AWS in cooldown → URL is c2df.
|
||||
db2, _ = _db(**NO_PAIR, capacity_state=state)
|
||||
m2, _, _ = _manager(db2)
|
||||
svc = EncodingService.__new__(EncodingService)
|
||||
svc._cached_url = None
|
||||
svc._url_cached_at = 0.0
|
||||
svc._worker_manager = m2
|
||||
assert svc._get_worker_url() == "http://2.2.2.2:8080"
|
||||
|
||||
|
||||
def test_demote_with_nothing_serving_is_quiet(caplog):
|
||||
db, _ = _db(**NO_PAIR)
|
||||
m, _, _ = _manager(db)
|
||||
m.demote_active_worker("x")
|
||||
assert not [r for r in caplog.records if r.levelname == "ERROR"]
|
||||
|
||||
|
||||
def test_encoding_service_resolves_top_candidate_when_no_url():
|
||||
from backend.services.encoding_service import EncodingService
|
||||
|
||||
db, _ = _db(**NO_PAIR)
|
||||
m, _, _ = _manager(db)
|
||||
svc = EncodingService.__new__(EncodingService)
|
||||
svc._cached_url = None
|
||||
svc._url_cached_at = 0.0
|
||||
svc._worker_manager = m
|
||||
assert svc._get_worker_url() == "http://3.3.3.3:8080"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_candidate_warmup_invalidates_cache_on_fallback():
|
||||
from backend.services.encoding_service import EncodingService
|
||||
|
||||
svc = EncodingService.__new__(EncodingService)
|
||||
svc._cached_url = "http://stale:8080"
|
||||
svc._url_cached_at = 1e12
|
||||
manager = MagicMock()
|
||||
manager.get_config.return_value = MagicMock()
|
||||
manager.ranked_candidates.return_value = [
|
||||
EncodingWorkerCandidate("aws-encoding-worker-1", "us-east-2b", "3.3.3.3", is_primary=False)
|
||||
]
|
||||
manager.ensure_primary_running.return_value = {
|
||||
"started": False, "vm_name": "aws-encoding-worker-1", "zone": "us-east-2b",
|
||||
"primary_url": "http://3.3.3.3:8080", "fell_back": True,
|
||||
}
|
||||
manager.wait_for_worker_ready = AsyncMock()
|
||||
svc._worker_manager = manager
|
||||
await svc._warmup_encoding_worker_fallback("job1")
|
||||
assert svc._cached_url is None
|
||||
@@ -38,11 +38,11 @@
|
||||
│ (jobs) │ │ (files) │ │ (API keys) │ │ Worker (*) │
|
||||
└──────────┘ └──────────┘ └──────────────┘ └──────────────┘
|
||||
|
||||
(*) GCE Encoding Worker: a Spot c4d-highcpu-16 blue-green pair (AMD EPYC
|
||||
Turin, the preferred pick) plus ONE on-demand c2d-highcpu-16 fallback in a
|
||||
different zone for Spot/c4d stockouts (3 VMs since 2026-09-26 — see
|
||||
docs/archive/2026-09-26-encoding-cost-cuts.md). Candidates are selected
|
||||
fastest-first with a capacity cooldown.
|
||||
(*) Encoding Worker: since 2026-11 the AWS EC2 worker (aws-encoding-worker-1,
|
||||
c7a.4xlarge, preferred via speed_rank_overrides) plus ONE on-demand GCE
|
||||
c2d-highcpu-16 fallback (encoding-worker-fallback-c2df). The c4d Spot
|
||||
blue-green pair was retired. Candidates are selected fastest-first with a
|
||||
capacity cooldown.
|
||||
Used for both final video encoding and preview video generation.
|
||||
Uses immutable deployment pattern - see infrastructure/encoding-worker/README.md
|
||||
and the "Encoding Worker" section below.
|
||||
@@ -469,6 +469,8 @@ karaoke-gen shares a GCP project (`nomadkaraoke`) with karaoke-decide, but uses
|
||||
|
||||
## Encoding Worker: blue-green primary pair + multi-family fallback pool
|
||||
|
||||
> **2026-11 update:** the c4d Spot pair (`encoding-worker-a`/`-b`) is **retired and deleted**. The preferred encoder (and distribution worker) is the AWS EC2 box `aws-encoding-worker-1`; the only GCE VM is the on-demand fallback `encoding-worker-fallback-c2df` (us-central1-f). `config/encoding-worker` keeps `primary_vm`/`secondary_vm` empty, so every selection path routes to a fallback and records it as `active_override_*`. CI blue-green deploys now just validate the new wheel on c2df and stop it again (the AWS override keeps serving); the worker-change gate diffs against `validated_version`. The pair-specific text below is historical.
|
||||
|
||||
The GCE encoding worker is a **blue-green primary pair** (`encoding-worker-a`/`-b`,
|
||||
both **Spot** c4d-highcpu-16 in us-central1-c) plus **one on-demand fallback**
|
||||
(`encoding-worker-fallback-c2df`, c2d-highcpu-16, us-central1-f). **3 VMs total**
|
||||
|
||||
+1
-1
@@ -235,7 +235,7 @@ TESTMAIL_NAMESPACE=... # testmail.app namespace
|
||||
|
||||
## Encoding Worker Operations
|
||||
|
||||
The encoding worker is a Spot blue-green primary pair (`encoding-worker-a`/`-b`, c4d-highcpu-16) plus one stopped on-demand fallback (`encoding-worker-fallback-c2df`, c2d-highcpu-16, us-central1-f) — 3 VMs total since 2026-09-26. CI only blue-green redeploys the worker when a path in `infrastructure/encoding-worker/worker_code_paths.txt` changed since the serving version (or via Actions → CI → Run workflow with "Force a blue-green redeploy of the GCE encoding worker"). Candidate ordering (runtime + deploy) is the shared `backend/services/encoding_worker_preference.py`. See [ARCHITECTURE.md](ARCHITECTURE.md#encoding-worker-blue-green-primary-pair--multi-family-fallback-pool) for the full design.
|
||||
The encoding worker is the AWS EC2 box `aws-encoding-worker-1` (preferred, also runs distribution) plus one stopped on-demand GCE fallback (`encoding-worker-fallback-c2df`, c2d-highcpu-16, us-central1-f). The c4d Spot pair (`encoding-worker-a`/`-b`) was retired and deleted in 2026-11; examples below that name it should use c2df (`--zone=us-central1-f`). CI only blue-green redeploys the worker when a path in `infrastructure/encoding-worker/worker_code_paths.txt` changed since the serving version (or via Actions → CI → Run workflow with "Force a blue-green redeploy of the GCE encoding worker"). Candidate ordering (runtime + deploy) is the shared `backend/services/encoding_worker_preference.py`. See [ARCHITECTURE.md](ARCHITECTURE.md#encoding-worker-blue-green-primary-pair--multi-family-fallback-pool) for the full design.
|
||||
|
||||
### Seed Firestore config (initial setup or reset)
|
||||
|
||||
|
||||
@@ -359,17 +359,17 @@ The script is idempotent — re-running it skips regeneration if the CDG ZIP alr
|
||||
**Cause:** Worker picks up a new deploy but doesn't restart automatically.
|
||||
|
||||
Target the **currently-serving** VM — `active_override_vm` in `config/encoding-worker`
|
||||
if set, else `primary_vm` (with its zone). There is no VM literally named
|
||||
`encoding-worker`; the examples below use the primary `encoding-worker-a` — swap in
|
||||
the active VM/zone if a fallback is serving.
|
||||
if set, else `primary_vm` (with its zone). Since 2026-11 `primary_vm` is empty (the c4d
|
||||
pair was deleted): the serving VM is the AWS box (SSM shell, see
|
||||
`infrastructure/aws/README.md`) or the GCE fallback `encoding-worker-fallback-c2df`.
|
||||
|
||||
```bash
|
||||
# Check current wheel version (example: the primary a in us-central1-c)
|
||||
gcloud compute ssh encoding-worker-a --zone=us-central1-c --project=nomadkaraoke \
|
||||
# Check current wheel version (example: the GCE fallback in us-central1-f)
|
||||
gcloud compute ssh encoding-worker-fallback-c2df --zone=us-central1-f --project=nomadkaraoke \
|
||||
--command="curl -s http://localhost:8080/health | python3 -m json.tool"
|
||||
|
||||
# Restart to pick up latest wheel
|
||||
gcloud compute ssh encoding-worker-a --zone=us-central1-c --project=nomadkaraoke \
|
||||
gcloud compute ssh encoding-worker-fallback-c2df --zone=us-central1-f --project=nomadkaraoke \
|
||||
--command="sudo systemctl restart encoding-worker"
|
||||
```
|
||||
|
||||
@@ -626,7 +626,7 @@ gcloud compute ssh encoding-worker-b --zone=us-central1-c --project=nomadkaraoke
|
||||
|
||||
**Background — what this state means:** GCE returned `ZONE_RESOURCE_POOL_EXHAUSTED` or transient `503 SERVICE_UNAVAILABLE` from `compute.instances.start` on every encoding-worker VM. The render worker parks the job in `RENDER_PENDING_CAPACITY` instead of failing it; Cloud Scheduler retries it every 10 min via `/api/internal/retry-pending-render-jobs`. Hard timeout is 24 hours (then transitions to `failed` with a clear permanent-failure message).
|
||||
|
||||
Since 2026-09-26 the fleet is 3 VMs: **Spot** `c4d-highcpu-16` primaries (`encoding-worker-a`/`-b`) in `us-central1-c`, plus ONE stopped on-demand fallback `encoding-worker-fallback-c2df` (`c2d-highcpu-16`, `us-central1-f`). Candidates are tried **fastest-first with a 15-min capacity cooldown** — see `backend/services/encoding_worker_preference.py`. If both the Spot pair and the c2d fallback return stockout, that's a genuinely severe regional event — wait, or (re-)add a fallback in another family/zone to `EncodingWorkerConfig.FALLBACKS` **and** the `encoding-worker-fallback-vms` secret (the 2026-08-15 pool had 8 fallbacks across 6 families). NOTE: creating a *new* fallback VM needs a one-time boot allocation, so a deep enough crunch can even block provisioning.
|
||||
Since 2026-11 the fleet is the AWS worker plus ONE GCE VM (the c4d Spot pair `encoding-worker-a`/`-b` was deleted). Before that (2026-09-26) it was **Spot** `c4d-highcpu-16` primaries in `us-central1-c`, plus ONE stopped on-demand fallback `encoding-worker-fallback-c2df` (`c2d-highcpu-16`, `us-central1-f`). Candidates are tried **fastest-first with a 15-min capacity cooldown** — see `backend/services/encoding_worker_preference.py`. If both the Spot pair and the c2d fallback return stockout, that's a genuinely severe regional event — wait, or (re-)add a fallback in another family/zone to `EncodingWorkerConfig.FALLBACKS` **and** the `encoding-worker-fallback-vms` secret (the 2026-08-15 pool had 8 fallbacks across 6 families). NOTE: creating a *new* fallback VM needs a one-time boot allocation, so a deep enough crunch can even block provisioning.
|
||||
|
||||
**Spot preemption mid-encode** is expected occasionally: the backend sees the pinned worker VM is TERMINATED/STOPPING, logs `Encoding worker VM stopped mid-job (... ) — likely Spot preemption`, and resubmits the encode as `<job>_retry_<hex>` (up to `ENCODING_RESUBMIT_MAX`, default 2). Repeated preemptions → the job fails with the lost-job error; check `gcloud compute operations list --filter="operationType=compute.instances.preempted"`.
|
||||
|
||||
|
||||
@@ -756,11 +756,9 @@ if AUDIO_SEPARATOR_SERVICE_ENABLED:
|
||||
|
||||
# ==================== Compute VMs ====================
|
||||
|
||||
# Encoding Worker VMs (blue-green pair for zero-downtime deployments)
|
||||
encoding_worker_ips = encoding_worker_vm.create_encoding_worker_ips()
|
||||
encoding_worker_instances = encoding_worker_vm.create_encoding_worker_vms(
|
||||
encoding_worker_ips, encoding_worker_sa
|
||||
)
|
||||
# The c4d Spot blue-green pair (encoding-worker-a/-b + static IPs) was retired
|
||||
# 2026-11: the AWS EC2 worker (infrastructure/aws/) is the preferred encoder and
|
||||
# runs distribution; the c2df VM below is the GCP fallback.
|
||||
|
||||
# Capacity-fallback Encoding Worker VM — one ON-DEMAND VM in a different zone AND
|
||||
# machine family from the c4d Spot pair, so a Spot/c4d stockout can't take out
|
||||
@@ -890,12 +888,6 @@ pulumi.export("recover_stuck_downloads_scheduler_name", recover_stuck_downloads_
|
||||
pulumi.export("disposable_domains_sync_scheduler_name", disposable_domains_sync_scheduler.name)
|
||||
pulumi.export("gdrive_validator_service_account", gdrive_validator_sa.email)
|
||||
|
||||
# Encoding workers (blue-green pair)
|
||||
pulumi.export("encoding_worker_a_ip", encoding_worker_ips[0].address)
|
||||
pulumi.export("encoding_worker_b_ip", encoding_worker_ips[1].address)
|
||||
pulumi.export("encoding_worker_a_url", encoding_worker_ips[0].address.apply(lambda ip: f"http://{ip}:8080"))
|
||||
pulumi.export("encoding_worker_b_url", encoding_worker_ips[1].address.apply(lambda ip: f"http://{ip}:8080"))
|
||||
|
||||
# Capacity-fallback VM exports (one per EncodingWorkerConfig.FALLBACKS entry,
|
||||
# keyed by suffix, e.g. encoding_worker_fallback_c2df_ip).
|
||||
for _fb, _fb_ip in zip(EncodingWorkerConfig.FALLBACKS, encoding_worker_fallback_ips):
|
||||
@@ -905,8 +897,6 @@ for _fb, _fb_ip in zip(EncodingWorkerConfig.FALLBACKS, encoding_worker_fallback_
|
||||
_fb_ip.address.apply(lambda ip: f"http://{ip}:8080"),
|
||||
)
|
||||
pulumi.export("encoding_worker_service_account", encoding_worker_sa.email)
|
||||
pulumi.export("encoding_worker_a_name", encoding_worker_instances[0].name)
|
||||
pulumi.export("encoding_worker_b_name", encoding_worker_instances[1].name)
|
||||
|
||||
# GitHub runners
|
||||
pulumi.export("github_runner_service_account", github_runner_sa.email)
|
||||
|
||||
@@ -7,8 +7,6 @@ Contains VM definitions for:
|
||||
"""
|
||||
|
||||
from .encoding_worker_vm import (
|
||||
create_encoding_worker_vms,
|
||||
create_encoding_worker_ips,
|
||||
create_encoding_worker_fallback_vms,
|
||||
create_encoding_worker_fallback_ips,
|
||||
create_encoding_worker_firewall,
|
||||
@@ -16,8 +14,6 @@ from .encoding_worker_vm import (
|
||||
from .github_runners import create_github_runners, create_build_runner, create_cloud_nat
|
||||
|
||||
__all__ = [
|
||||
"create_encoding_worker_vms",
|
||||
"create_encoding_worker_ips",
|
||||
"create_encoding_worker_fallback_vms",
|
||||
"create_encoding_worker_fallback_ips",
|
||||
"create_encoding_worker_firewall",
|
||||
|
||||
@@ -1,117 +1,20 @@
|
||||
"""
|
||||
Encoding Worker VM resources — blue-green deployment pair.
|
||||
Encoding Worker VM resources — the GCE capacity-fallback fleet.
|
||||
|
||||
Manages two identical VMs (a/b) for zero-downtime deployments.
|
||||
Only one is active (primary) at a time; the other is stopped (secondary).
|
||||
Both auto-shutdown when idle to minimize cost, and both run on SPOT capacity
|
||||
(preempted → STOPPED, restarted on demand; lost encodes are resubmitted). One
|
||||
on-demand fallback VM in a different family/zone absorbs Spot stockouts.
|
||||
The c4d Spot blue-green pair (encoding-worker-a/-b) was retired 2026-11; the AWS
|
||||
EC2 worker (infrastructure/aws/) is now the preferred encoder. What remains here
|
||||
is the on-demand fallback (c2df), provisioned stopped and started on demand.
|
||||
|
||||
See docs/superpowers/specs/2026-03-24-blue-green-encoding-worker-design.md
|
||||
History: docs/superpowers/specs/2026-03-24-blue-green-encoding-worker-design.md
|
||||
"""
|
||||
|
||||
import pulumi
|
||||
from pulumi_gcp import compute, serviceaccount
|
||||
|
||||
from config import REGION, ENCODING_WORKER_ZONE, PROJECT_ID, MachineTypes, DiskSizes, EncodingWorkerConfig
|
||||
from config import REGION, PROJECT_ID, MachineTypes, DiskSizes, EncodingWorkerConfig
|
||||
from .startup_scripts import read_script
|
||||
|
||||
|
||||
def create_encoding_worker_ips() -> list[compute.Address]:
|
||||
"""Create static IPs for both encoding worker VMs."""
|
||||
ips = []
|
||||
for name in EncodingWorkerConfig.IP_NAMES:
|
||||
ip = compute.Address(
|
||||
name,
|
||||
name=name,
|
||||
region=REGION,
|
||||
address_type="EXTERNAL",
|
||||
description=f"Static external IP for {name}",
|
||||
)
|
||||
ips.append(ip)
|
||||
return ips
|
||||
|
||||
|
||||
def create_encoding_worker_vms(
|
||||
ips: list[compute.Address],
|
||||
service_account: serviceaccount.Account,
|
||||
) -> list[compute.Instance]:
|
||||
"""Create the blue-green encoding worker VM pair."""
|
||||
startup_script = read_script("encoding_worker.sh")
|
||||
custom_image = f"projects/{PROJECT_ID}/global/images/family/encoding-worker"
|
||||
|
||||
vms = []
|
||||
for vm_name, ip in zip(EncodingWorkerConfig.VM_NAMES, ips):
|
||||
vm = compute.Instance(
|
||||
vm_name,
|
||||
name=vm_name,
|
||||
machine_type=MachineTypes.ENCODING_WORKER,
|
||||
zone=ENCODING_WORKER_ZONE,
|
||||
boot_disk=compute.InstanceBootDiskArgs(
|
||||
initialize_params=compute.InstanceBootDiskInitializeParamsArgs(
|
||||
image=custom_image,
|
||||
size=DiskSizes.ENCODING_WORKER,
|
||||
type="hyperdisk-balanced",
|
||||
# NOTE: Hyperdisk provisioned IOPS/throughput are deliberately
|
||||
# NOT set here. Specifying them forces a full VM *replacement*
|
||||
# (the provider treats boot-disk init params as immutable), and
|
||||
# recreating a primary depends on us-central1-c capacity — the
|
||||
# exact shortage the fallback fleet exists to absorb. So the
|
||||
# disks are pinned to the Hyperdisk Balanced free baseline
|
||||
# (3000 IOPS / 140 MB/s) live via `gcloud compute disks update`
|
||||
# instead — non-disruptive, and pulumi does not manage this
|
||||
# attribute so there is no drift. They were over-provisioned at
|
||||
# 3600 IOPS / 290 MB/s (~$8.70/disk/mo wasted). Re-apply the
|
||||
# gcloud tune after any worker-image rebuild that recreates the
|
||||
# disks. See docs/archive/2026-06-14-gcp-cost-reduction-plan.md.
|
||||
),
|
||||
),
|
||||
network_interfaces=[compute.InstanceNetworkInterfaceArgs(
|
||||
network="default",
|
||||
access_configs=[compute.InstanceNetworkInterfaceAccessConfigArgs(
|
||||
nat_ip=ip.address,
|
||||
)],
|
||||
)],
|
||||
service_account=compute.InstanceServiceAccountArgs(
|
||||
email=service_account.email,
|
||||
scopes=["cloud-platform"],
|
||||
),
|
||||
metadata_startup_script=startup_script,
|
||||
tags=["encoding-worker"],
|
||||
allow_stopping_for_update=True,
|
||||
advanced_machine_features=compute.InstanceAdvancedMachineFeaturesArgs(
|
||||
threads_per_core=2,
|
||||
),
|
||||
scheduling=_primary_pair_scheduling(),
|
||||
)
|
||||
vms.append(vm)
|
||||
return vms
|
||||
|
||||
|
||||
def _primary_pair_scheduling() -> compute.InstanceSchedulingArgs:
|
||||
"""Scheduling for the a/b pair: Spot with STOP-on-preemption.
|
||||
|
||||
STOP (not DELETE) keeps the VM, its disk and static IP, so a preempted
|
||||
worker is just a TERMINATED instance that the backend's ensure-running path
|
||||
restarts like any idle-stopped worker. automatic_restart must be False and
|
||||
on_host_maintenance TERMINATE for Spot.
|
||||
"""
|
||||
if not EncodingWorkerConfig.PRIMARY_PAIR_SPOT:
|
||||
return compute.InstanceSchedulingArgs(
|
||||
provisioning_model="STANDARD",
|
||||
preemptible=False,
|
||||
automatic_restart=True,
|
||||
on_host_maintenance="MIGRATE",
|
||||
)
|
||||
return compute.InstanceSchedulingArgs(
|
||||
provisioning_model="SPOT",
|
||||
preemptible=True,
|
||||
automatic_restart=False,
|
||||
on_host_maintenance="TERMINATE",
|
||||
instance_termination_action="STOP",
|
||||
)
|
||||
|
||||
|
||||
def create_encoding_worker_fallback_ips() -> list[compute.Address]:
|
||||
"""Create static IPs for the capacity-fallback VMs.
|
||||
|
||||
@@ -128,7 +31,7 @@ def create_encoding_worker_fallback_ips() -> list[compute.Address]:
|
||||
region=REGION,
|
||||
address_type="EXTERNAL",
|
||||
# Keep this string byte-identical to the originally-deployed value so
|
||||
# `pulumi up` does not diff (and needlessly churn) the existing a/b IPs.
|
||||
# `pulumi up` does not diff (and needlessly churn) the existing IPs.
|
||||
description=f"Static external IP for {ip_name} (zone-fallback)",
|
||||
)
|
||||
ips.append(ip)
|
||||
|
||||
@@ -33,8 +33,8 @@ def get_project_number() -> str:
|
||||
REGION = "us-central1"
|
||||
ZONE = f"{REGION}-a"
|
||||
|
||||
# Encoding worker zone - uses us-central1-c due to c4d availability
|
||||
# (us-central1-a and us-central1-b often lack capacity for high-end C4D instances)
|
||||
# Default zone the encoding idle-shutdown function targets (GCP_ZONE). The c4d
|
||||
# pair that lived here was retired 2026-11; fallbacks carry their own zones.
|
||||
ENCODING_WORKER_ZONE = f"{REGION}-c"
|
||||
|
||||
# Pulumi config accessor
|
||||
@@ -52,7 +52,8 @@ class MachineTypes:
|
||||
# only kept ~66-78% of 32 vCPUs busy; ffmpeg auto-threads adapt to 16). The
|
||||
# worker serializes heavy jobs (ENCODING_HEAVY_CONCURRENCY=1), and highcpu-16
|
||||
# still has ~30-32 GB RAM — enough for one 4K encode (~18 GB peak).
|
||||
ENCODING_WORKER = "c4d-highcpu-16" # 16 vCPU / 30 GB, AMD EPYC Turin — Spot blue-green pair
|
||||
# (The c4d-highcpu-16 Spot blue-green pair was retired 2026-11 in favour of
|
||||
# the AWS EC2 worker, infrastructure/aws/.)
|
||||
# The ONE on-demand capacity fallback. Deliberately a DIFFERENT machine family
|
||||
# and zone from the c4d Spot pair: c4d suffered a region-wide
|
||||
# ZONE_RESOURCE_POOL_EXHAUSTED stockout (2026-08-12), and Spot capacity can
|
||||
@@ -307,9 +308,7 @@ GITHUB_REPO_NAME = "karaoke-gen"
|
||||
|
||||
|
||||
class EncodingWorkerConfig:
|
||||
"""Configuration for blue-green encoding worker VMs."""
|
||||
VM_NAMES = ["encoding-worker-a", "encoding-worker-b"]
|
||||
IP_NAMES = ["encoding-worker-ip-a", "encoding-worker-ip-b"]
|
||||
"""Configuration for the GCE encoding worker fallback fleet + idle shutdown."""
|
||||
# Stop idle workers fast: VMs bill per second while RUNNING, and with only
|
||||
# ~5-20 jobs/day most of the old 15-min tail was paid idle time. The function
|
||||
# still NEVER stops a VM with active jobs (fail-safe on unconfirmed /health).
|
||||
@@ -319,12 +318,6 @@ class EncodingWorkerConfig:
|
||||
FUNCTION_MEMORY = "512M" # Increased from 256M — OOM with gRPC/Firestore/Compute client libs
|
||||
FUNCTION_TIMEOUT = 120 # 2 minutes
|
||||
|
||||
# The a/b blue-green pair runs on SPOT capacity (~60-70% cheaper). A preempted
|
||||
# Spot VM is STOPPED (not deleted) — the backend restarts it on the next
|
||||
# request, and an encode lost to preemption is resubmitted automatically
|
||||
# (encoding_service.wait_for_completion → EncodingJobLostError).
|
||||
PRIMARY_PAIR_SPOT = True
|
||||
|
||||
# Capacity-resilience fallback. Provisioned STOPPED and started on demand only
|
||||
# when the Spot pair can't start (ZONE_RESOURCE_POOL_EXHAUSTED / Spot
|
||||
# stockout). ON-DEMAND (never preempted) and a different family + zone from the
|
||||
|
||||
@@ -130,6 +130,13 @@ def promote_plan(
|
||||
serving worker). Leave primary/secondary untouched so c4d resumes as
|
||||
primary when capacity returns. Drain+stop the previous override (if any,
|
||||
and different from the green).
|
||||
* green.kind == "fallback" while the AWS worker is the serving override:
|
||||
the green was only a validation target. Keep the AWS override serving,
|
||||
record the validated version, and stop the green (it served nothing).
|
||||
|
||||
Every successful promote records ``validated_version``: the version the CI
|
||||
worker-change gate diffs against. ``primary_version`` alone stops moving
|
||||
once the c4d pair is retired (2026-11).
|
||||
"""
|
||||
drain: List[Dict[str, Optional[str]]] = []
|
||||
|
||||
@@ -149,6 +156,7 @@ def promote_plan(
|
||||
"secondary_ip": config.get("primary_ip"),
|
||||
"secondary_version": config.get("primary_version"),
|
||||
"secondary_deployed_at": now,
|
||||
"validated_version": version,
|
||||
"last_swap_at": now,
|
||||
"deploy_in_progress": False,
|
||||
"deploy_in_progress_since": None,
|
||||
@@ -179,6 +187,18 @@ def promote_plan(
|
||||
})
|
||||
return _finalize(updates)
|
||||
|
||||
if _is_aws_override(config):
|
||||
# The AWS worker serves by preference and hot-updates its wheel per job;
|
||||
# a GCE fallback green must not displace it. It was validated only.
|
||||
return {
|
||||
"firestore_updates": {
|
||||
"validated_version": version,
|
||||
"deploy_in_progress": False,
|
||||
"deploy_in_progress_since": None,
|
||||
},
|
||||
"drain_and_stop": [{"vm": green["vm"], "ip": green.get("ip"), "zone": green.get("zone")}],
|
||||
}
|
||||
|
||||
# kind == "fallback": the green becomes the serving override.
|
||||
updates = {
|
||||
"active_override_vm": green["vm"],
|
||||
@@ -189,6 +209,7 @@ def promote_plan(
|
||||
# The green is a GCE VM: never leave a stale "aws" tag on it (CI hides
|
||||
# an AWS override from this flow, so it may be the override replaced).
|
||||
"active_override_provider": None,
|
||||
"validated_version": version,
|
||||
"deploy_in_progress": False,
|
||||
"deploy_in_progress_since": None,
|
||||
}
|
||||
|
||||
@@ -35,6 +35,7 @@ from datetime import datetime, timezone, timedelta
|
||||
|
||||
import functions_framework
|
||||
import requests
|
||||
from google.api_core import exceptions as gcp_exceptions
|
||||
from google.cloud import compute_v1, firestore
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -68,6 +69,11 @@ def get_vm_status(vm_name, zone=ZONE):
|
||||
try:
|
||||
instance = client.get(project=PROJECT_ID, zone=zone, instance=vm_name)
|
||||
return instance.status
|
||||
except gcp_exceptions.NotFound:
|
||||
# A VM still named in config/secret but deleted (e.g. the retired c4d
|
||||
# pair): expected, not page-worthy.
|
||||
logger.warning(f"VM {vm_name} not found in {zone}; skipping")
|
||||
return "NOT_FOUND"
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get status for {vm_name} in {zone}: {e}")
|
||||
return "UNKNOWN"
|
||||
|
||||
@@ -592,3 +592,26 @@ class TestCheckActiveJobs:
|
||||
patch("main.time.sleep"):
|
||||
assert main.check_active_jobs("1.2.3.4", attempts=3) == 1
|
||||
assert get.call_count == 2
|
||||
|
||||
|
||||
def test_missing_vm_logs_warning_not_error(mock_compute, caplog):
|
||||
"""A VM deleted but still named somewhere (the retired c4d pair) is expected:
|
||||
WARNING + NOT_FOUND, never ERROR (the error monitor alerts on ERROR)."""
|
||||
from google.api_core import exceptions as gcp_exceptions
|
||||
|
||||
mock_compute.get.side_effect = gcp_exceptions.NotFound("instance gone")
|
||||
with caplog.at_level("WARNING"):
|
||||
assert main.get_vm_status("encoding-worker-a") == "NOT_FOUND"
|
||||
assert not [r for r in caplog.records if r.levelname == "ERROR"]
|
||||
|
||||
|
||||
def test_blank_pair_names_are_skipped(mock_compute, mock_firestore):
|
||||
"""After the c4d pair is retired config holds empty primary/secondary names."""
|
||||
snap = MagicMock(exists=True)
|
||||
snap.to_dict.return_value = _make_config(primary_vm="", secondary_vm="")
|
||||
mock_firestore.collection.return_value.document.return_value.get.return_value = snap
|
||||
with patch.object(main, "_parse_fallback_vms", return_value=[]):
|
||||
body, status = main.idle_shutdown(MagicMock())
|
||||
assert status == 200
|
||||
assert json.loads(body)["results"] == {}
|
||||
mock_compute.get.assert_not_called()
|
||||
@@ -10,30 +10,18 @@ Run locally with: `pytest infrastructure/test_encoding_worker_config.py`
|
||||
resource graph via `pulumi preview`).
|
||||
"""
|
||||
|
||||
from config import ENCODING_WORKER_ZONE, EncodingWorkerConfig, MachineTypes
|
||||
from config import EncodingWorkerConfig, MachineTypes
|
||||
|
||||
|
||||
def test_fallback_is_a_different_machine_family_and_zone_from_primary():
|
||||
"""The on-demand fallback must not share the Spot pair's family or zone,
|
||||
otherwise a single c4d / Spot / zone stockout takes out every lane at once
|
||||
(incident 2026-08-12)."""
|
||||
assert EncodingWorkerConfig.FALLBACKS, "need at least one capacity fallback"
|
||||
primary_family = _family(MachineTypes.ENCODING_WORKER)
|
||||
primary_zone_suffix = ENCODING_WORKER_ZONE.rsplit("-", 1)[1]
|
||||
for fb in EncodingWorkerConfig.FALLBACKS:
|
||||
assert _family(fb["machine_type"]) != primary_family, fb
|
||||
assert fb["zone_suffix"] != primary_zone_suffix, fb
|
||||
|
||||
|
||||
def test_fleet_is_cost_cut_shape():
|
||||
"""2026-09-26 cost cut: Spot a/b pair + exactly one on-demand fallback, all
|
||||
16 vCPU. Growing this again should be a deliberate decision (update the
|
||||
fallback secret + docs), not an accident."""
|
||||
assert len(EncodingWorkerConfig.VM_NAMES) == 2
|
||||
def test_fleet_is_single_on_demand_fallback():
|
||||
"""Since 2026-11 the GCE fleet is ONE on-demand fallback (c2df): the c4d Spot
|
||||
a/b pair was retired for the AWS EC2 worker. Growing this again should be a
|
||||
deliberate decision (update the fallback secret + docs), not an accident."""
|
||||
assert not hasattr(EncodingWorkerConfig, "VM_NAMES")
|
||||
assert not hasattr(MachineTypes, "ENCODING_WORKER")
|
||||
assert len(EncodingWorkerConfig.FALLBACKS) == 1
|
||||
assert EncodingWorkerConfig.PRIMARY_PAIR_SPOT is True
|
||||
for mt in [MachineTypes.ENCODING_WORKER] + [fb["machine_type"] for fb in EncodingWorkerConfig.FALLBACKS]:
|
||||
assert mt.endswith("-16"), mt
|
||||
for fb in EncodingWorkerConfig.FALLBACKS:
|
||||
assert fb["machine_type"].endswith("-16"), fb
|
||||
|
||||
|
||||
def test_idle_shutdown_is_fast_but_outlives_review_heartbeat():
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.292.0"
|
||||
version = "0.293.0"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
@@ -74,22 +74,24 @@ class TestMonitoredServiceLists:
|
||||
|
||||
assert config.MONITORED_CLOUD_FUNCTIONS == []
|
||||
|
||||
def test_gce_instances_count_at_least_4(self):
|
||||
def test_gce_instances_count_at_least_3(self):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
assert len(config.MONITORED_GCE_INSTANCES) >= 4
|
||||
assert len(config.MONITORED_GCE_INSTANCES) >= 3
|
||||
|
||||
def test_gce_instances_contains_expected(self):
|
||||
from backend.services.error_monitor import config
|
||||
|
||||
expected = [
|
||||
"encoding-worker-a",
|
||||
"encoding-worker-b",
|
||||
"encoding-worker-fallback-c2df",
|
||||
"flacfetch-vm",
|
||||
"divebar-sync-vm",
|
||||
]
|
||||
for instance in expected:
|
||||
assert instance in config.MONITORED_GCE_INSTANCES, f"Expected '{instance}' in MONITORED_GCE_INSTANCES"
|
||||
# The c4d pair was retired 2026-11; monitoring deleted VMs is noise.
|
||||
assert "encoding-worker-a" not in config.MONITORED_GCE_INSTANCES
|
||||
assert "encoding-worker-b" not in config.MONITORED_GCE_INSTANCES
|
||||
|
||||
|
||||
class TestDefaultValues:
|
||||
|
||||
Reference in new issue
Block a user