diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aa006738..b70c4c24 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,15 @@ on: branches: [main] types: [opened, synchronize, reopened, labeled] workflow_dispatch: + inputs: + force_encoding_worker_deploy: + # Manual override for the encoding-worker deploy gate (see "Decide + # encoding-worker deploy" in deploy-backend). Dispatch on main re-runs + # package-build + deploy-backend; with this ticked the GCE worker is + # blue-green redeployed even if no worker code changed. + description: "Force a blue-green redeploy of the GCE encoding worker" + type: boolean + default: false permissions: contents: write @@ -824,7 +833,7 @@ jobs: package-build: name: "Package - Build" runs-on: ubuntu-latest - if: github.ref == 'refs/heads/main' && github.event_name == 'push' + if: github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') steps: - name: Free disk space (only if low) # GitHub-hosted runners (free for this public repo) currently have ~86 GB @@ -1079,7 +1088,7 @@ jobs: name: "Deploy - Backend (Cloud Run)" needs: [package-build] runs-on: [self-hosted, linux, gcp, docker-build] - if: github.ref == 'refs/heads/main' && github.event_name == 'push' + if: github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') permissions: contents: read id-token: write @@ -1289,7 +1298,9 @@ jobs: for key in ["primary_vm", "primary_ip", "primary_version", "secondary_vm", "secondary_ip", "active_override_vm", "active_override_ip", "active_override_zone"]: - f.write(f"{key}={data.get(key, '')}\n") + # `or ''`: a key stored as null must be empty, not the string + # "None" (which later steps would treat as a real VM name). + f.write(f"{key}={data.get(key) or ''}\n") f.write(f"deploy_in_progress={str(data.get('deploy_in_progress', False)).lower()}\n") # capacity_state is a map — emit as single-line JSON so the green # selection step can demote recently-stocked-out machine types. @@ -1298,8 +1309,47 @@ jobs: f"active_override={data.get('active_override_vm')}") PYEOF + # Gate the blue-green worker deploy on worker-affecting changes. It boots a + # fresh green VM + runs a real test encode, which used to happen on EVERY + # push to main (~5x/day) even for frontend/backend-only changes. The wheel, + # startup.sh and version.txt above are still uploaded every time, so any + # worker that cold-boots (they idle-stop after 5 min) runs the latest code. + # Diff base = the version the serving worker was last validated at (not + # github.event.before), so a cancelled/skipped deploy can never lose a + # worker change. Any doubt → deploy (fail open). + - name: Decide encoding-worker deploy + id: worker_gate + env: + FORCE: ${{ github.event_name == 'workflow_dispatch' && inputs.force_encoding_worker_deploy == true }} + SERVING_VERSION: ${{ steps.config.outputs.primary_version }} + OVERRIDE_VM: ${{ steps.config.outputs.active_override_vm }} + run: | + mapfile -t PATHS < <(grep -v '^\s*#' infrastructure/encoding-worker/worker_code_paths.txt | grep -v '^\s*$') + DEPLOY=true + if [ "$FORCE" = "true" ]; then + REASON="manual override (workflow_dispatch force_encoding_worker_deploy)" + elif [ -n "$OVERRIDE_VM" ] && [ "$OVERRIDE_VM" != "None" ]; then + REASON="capacity override $OVERRIDE_VM is serving — refresh it" + elif [ -z "$SERVING_VERSION" ] || [ "$SERVING_VERSION" = "None" ]; then + REASON="serving worker version unknown" + elif ! git fetch --no-tags --depth=1 origin "refs/tags/v${SERVING_VERSION}:refs/tags/v${SERVING_VERSION}"; then + REASON="tag v${SERVING_VERSION} not fetchable" + elif ! CHANGED=$(git diff --name-only "v${SERVING_VERSION}" HEAD -- "${PATHS[@]}"); then + REASON="git diff against v${SERVING_VERSION} failed" + elif [ -z "$CHANGED" ]; then + DEPLOY=false + REASON="no worker-affecting changes since serving v${SERVING_VERSION}" + else + REASON="worker-affecting changes since v${SERVING_VERSION}" + echo "Changed worker paths:"; echo "$CHANGED" | head -50 + fi + echo "deploy=${DEPLOY}" >> "$GITHUB_OUTPUT" + echo "Encoding-worker blue-green deploy: ${DEPLOY} (${REASON})" + echo "### Encoding worker deploy: ${DEPLOY}" >> "$GITHUB_STEP_SUMMARY" + echo "${REASON}" >> "$GITHUB_STEP_SUMMARY" + - name: Wait for concurrent deploy - if: steps.config.outputs.deploy_in_progress == 'true' + if: steps.worker_gate.outputs.deploy == 'true' && steps.config.outputs.deploy_in_progress == 'true' run: | echo "Another deploy is in progress, waiting..." python3 << 'PYEOF' @@ -1318,6 +1368,7 @@ jobs: PYEOF - name: Set deploy in progress + if: steps.worker_gate.outputs.deploy == 'true' run: | python3 << 'PYEOF' from datetime import datetime, timezone @@ -1338,6 +1389,7 @@ jobs: # infrastructure/encoding-worker/deploy_promote.py. - name: Select & start green target id: green + if: steps.worker_gate.outputs.deploy == 'true' # Never hard-fail the whole backend deploy just because no encoding # worker could be validated — emit green_ok=false and let the gated # health/encode/promote steps skip while the Cloud Run deploy proceeds. @@ -1578,7 +1630,7 @@ jobs: # always() so the deploy_in_progress flag is ALWAYS cleared, even when the # green step found no worker (its downstream steps skip, but the flag was # already set). Skips only on a fully successful promote (passed == true). - if: always() && steps.encode-test.outputs.passed != 'true' + if: always() && steps.worker_gate.outputs.deploy == 'true' && steps.encode-test.outputs.passed != 'true' run: | GREEN_VM="${{ steps.green.outputs.green_vm }}" GREEN_ZONE="${{ steps.green.outputs.green_zone }}" @@ -1719,7 +1771,7 @@ jobs: # step guarantees the lock is released so the next deploy doesn't stall 10 min # in "Wait for concurrent deploy". Idempotent with the other clears. - name: Clear encoding-worker deploy lock - if: always() + if: always() && steps.worker_gate.outputs.deploy == 'true' run: | python3 -c " from google.cloud import firestore diff --git a/CLAUDE.md b/CLAUDE.md index 5772cd80..c67ab52f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -149,9 +149,12 @@ EOF ``` ### GCE Encoding Worker -The worker is a c4d blue-green primary pair (`encoding-worker-a`/`-b`, us-central1-c) -plus 8 stopped fallback VMs across 6 machine families for stockout resilience -(`encoding-worker-fallback-{a,b,n2c,n2f,c4a,n4db,c2df,n2da}`). The currently-serving +The worker is a **Spot** c4d-highcpu-16 blue-green primary pair (`encoding-worker-a`/`-b`, +us-central1-c) plus ONE stopped on-demand fallback (`encoding-worker-fallback-c2df`, +c2d-highcpu-16, us-central1-f) — 3 VMs since 2026-09-26. Idle VMs stop after 5 min. +CI blue-green redeploys the worker only when worker code changed +(`infrastructure/encoding-worker/worker_code_paths.txt`); force it via Actions → CI → +Run workflow → "Force a blue-green redeploy of the GCE encoding worker". The currently-serving VM is `active_override_vm` in `config/encoding-worker` if set, else `primary_vm`. Use that VM name + its zone below (there is no VM literally named `encoding-worker`): ```bash diff --git a/backend/api/routes/health.py b/backend/api/routes/health.py index 21909896..1329d36f 100644 --- a/backend/api/routes/health.py +++ b/backend/api/routes/health.py @@ -450,7 +450,7 @@ async def system_status( # alternate family) the override points at the live VM; # otherwise it's the blue-green primary. The primary pair is # always c4d — fallbacks carry their family in the VM name - # (…-fallback-n2c, …-fallback-c4a), which infer_machine_type + # (…-fallback-c2df), which infer_machine_type # decodes when no explicit machine_type is stored on the config. on_fallback = bool(config.active_override_vm) active_vm = config.active_override_vm or config.primary_vm diff --git a/backend/config.py b/backend/config.py index b54b0f95..1aadbdf1 100644 --- a/backend/config.py +++ b/backend/config.py @@ -326,6 +326,19 @@ class Settings(BaseSettings): default_private_dropbox_path: Optional[str] = os.getenv("DEFAULT_PRIVATE_DROPBOX_PATH") default_private_brand_prefix: Optional[str] = os.getenv("DEFAULT_PRIVATE_BRAND_PREFIX", "NOMADNP") + # Output files NOT uploaded to Nomad's own Dropbox folders (DEFAULT_DROPBOX_PATH / + # DEFAULT_PRIVATE_DROPBOX_PATH), as "|"-separated filename suffixes. They stay in + # GCS (job-page downloads, YouTube upload, AWS backup) — this only saves the + # multi-GB internet egress + upload time. Tenant Dropbox folders are exempt + # (B2B tenants expect the full set). Default skips only the lossless 4K MP4, + # which no customer-facing text promises in Dropbox. The lossless MKV IS promised + # there (completion email + the Fiverr bot), so add + # " (Final Karaoke Lossless 4k).mkv" only after updating those. + # See docs/archive/2026-09-26-encoding-cost-cuts.md. + dropbox_skip_output_suffixes: str = os.getenv( + "DROPBOX_SKIP_OUTPUT_SUFFIXES", " (Final Karaoke Lossless 4k).mp4" + ) + # YouTube Data API v3 Quota Configuration # Daily quota limit (YouTube grants 10,000 units/day by default) youtube_quota_daily_limit: int = int(os.getenv("YOUTUBE_QUOTA_DAILY_LIMIT", "10000")) diff --git a/backend/services/dropbox_service.py b/backend/services/dropbox_service.py index 83e60ec2..20b2bc5c 100644 --- a/backend/services/dropbox_service.py +++ b/backend/services/dropbox_service.py @@ -12,7 +12,7 @@ import json import logging import os import re -from typing import Optional +from typing import Optional, Sequence, Tuple from google.cloud import secretmanager @@ -249,13 +249,20 @@ class DropboxService: ) cursor.offset = file_obj.tell() - def upload_folder(self, local_dir: str, remote_path: str) -> None: + def upload_folder( + self, + local_dir: str, + remote_path: str, + exclude_suffixes: Sequence[str] = (), + ) -> None: """ Recursively upload all files and subdirectories to Dropbox folder. Args: local_dir: Local directory to upload remote_path: Dropbox destination folder path + exclude_suffixes: Filename suffixes to skip (see + dropbox_skip_suffixes_for). Skipped files stay on local disk / GCS. """ # Ensure remote path starts with / if not remote_path.startswith("/"): @@ -264,6 +271,7 @@ class DropboxService: logger.info(f"Uploading folder {local_dir} to {remote_path}") uploaded_count = 0 + skipped_count = 0 for root, _dirs, files in os.walk(local_dir): # Calculate the relative path from local_dir to current root rel_root = os.path.relpath(root, local_dir) @@ -273,12 +281,19 @@ class DropboxService: current_remote = f"{remote_path}/{rel_root}" for filename in files: + if exclude_suffixes and filename.endswith(tuple(exclude_suffixes)): + logger.info(f"Skipping {filename} (excluded from Dropbox upload)") + skipped_count += 1 + continue local_file = os.path.join(root, filename) remote_file = f"{current_remote}/{filename}" self.upload_file(local_file, remote_file) uploaded_count += 1 - logger.info(f"Uploaded {uploaded_count} files to {remote_path}") + logger.info( + f"Uploaded {uploaded_count} files to {remote_path}" + + (f" (skipped {skipped_count} excluded)" if skipped_count else "") + ) def create_shared_link(self, path: str) -> str: """ @@ -351,3 +366,27 @@ class DropboxService: def get_dropbox_service() -> DropboxService: """Factory function to get a DropboxService instance.""" return DropboxService() + + +def dropbox_skip_suffixes_for(dropbox_path: Optional[str]) -> Tuple[str, ...]: + """Filename suffixes to leave out of a Dropbox upload to ``dropbox_path``. + + Only Nomad's own folders (DEFAULT_DROPBOX_PATH / DEFAULT_PRIVATE_DROPBOX_PATH) + skip anything — tenant folders get the full output set. The suffix list comes + from settings.dropbox_skip_output_suffixes ("|"-separated). Skipped files are + still in GCS, so job-page downloads and YouTube uploads are unaffected. + """ + from backend.config import get_settings + + settings = get_settings() + if not dropbox_path: + return () + own_paths = { + p.rstrip("/") + for p in (settings.default_dropbox_path, settings.default_private_dropbox_path) + if p + } + if dropbox_path.rstrip("/") not in own_paths: + return () + raw = settings.dropbox_skip_output_suffixes or "" + return tuple(s for s in raw.split("|") if s.strip()) diff --git a/backend/services/encoding_service.py b/backend/services/encoding_service.py index 6f025ae4..3041e1df 100644 --- a/backend/services/encoding_service.py +++ b/backend/services/encoding_service.py @@ -116,6 +116,26 @@ def _parse_resubmit_max() -> int: return 2 +# GCE statuses meaning the worker VM is not running (idle-stop, deploy stop, or — +# since the a/b pair moved to Spot on 2026-09-26 — a Spot preemption, which STOPs +# the VM). A job submitted to a VM in one of these states can never complete: +# its in-memory job registry and ffmpeg process are gone. +# PROVISIONING/STAGING/REPAIRING are included too: the job was accepted by a +# RUNNING VM, so seeing the VM in a (re)start state means it went down and came +# back (e.g. another request restarted a preempted worker) — the job is gone. +_STOPPED_VM_STATUSES = frozenset({ + "STOPPING", "STOPPED", "TERMINATED", "SUSPENDING", "SUSPENDED", + "PROVISIONING", "STAGING", "REPAIRING", +}) + +# While polling a job, refresh config/encoding-worker.last_activity_at at most +# this often. The idle-shutdown function keeps the routed VM alive only while +# last_activity_at is < IDLE_TIMEOUT_MINUTES (5) old, and its /health +# active_jobs counts only RUNNING jobs — so without this, a VM whose encode ran +# longer than the idle window could be stopped in the seconds between the job +# finishing and our next poll collecting the result. +ACTIVITY_TOUCH_INTERVAL_SECONDS = 60.0 + QUEUE_TIMEOUT_SECONDS = _parse_queue_timeout() ENCODING_RESUBMIT_MAX = _parse_resubmit_max() @@ -272,6 +292,104 @@ class EncodingService: async with sem: yield + def _fallback_entries(self) -> list: + """Parse ENCODING_WORKER_FALLBACK_VMS into candidate dicts. + + Schema: '[{"vm":"encoding-worker-fallback-c2df","zone":"us-central1-f", + "ip":"34.x.x.x","machine_type":"c2d-highcpu-16"}, ...]'. machine_type is + optional (inferred from the VM name for legacy entries). Malformed input + degrades to fewer/no fallbacks — this runs during connection recovery and + must never raise. + """ + import json as _json + raw = self.settings.encoding_worker_fallback_vms + if not raw: + return [] + try: + parsed = _json.loads(raw) + except (ValueError, TypeError) as e: + logger.warning(f"Invalid ENCODING_WORKER_FALLBACK_VMS JSON: {e}") + return [] + # A non-list root (null, dict, scalar) parses fine but would blow up + # `for item in parsed` and escape during connection recovery — degrade + # to no fallbacks instead. + if not isinstance(parsed, list): + logger.warning("ENCODING_WORKER_FALLBACK_VMS must be a JSON list; ignoring") + return [] + entries = [] + for item in parsed: + try: + entries.append({ + "vm": item["vm"], + "zone": item["zone"], + "ip": item["ip"], + "machine_type": item.get("machine_type"), + "kind": "fallback", + }) + except (KeyError, TypeError) as e: + logger.warning(f"Skipping malformed fallback candidate {item}: {e}") + continue + return entries + + def _vm_for_worker_url(self, worker_url: Optional[str]) -> Optional[tuple]: + """Map a worker base URL (http://:8080) back to its (vm_name, zone). + + Looks at the blue-green primary/secondary, the active override, and the + configured fallbacks. Returns None when unknown (no manager, unparseable + URL, or an IP no longer in the fleet). BLOCKING (Firestore read). + """ + if not self._worker_manager or not worker_url: + return None + from urllib.parse import urlparse + + host = urlparse(worker_url).hostname + if not host: + return None + config = self._worker_manager.get_config() + default_zone = self._worker_manager._zone + known = [ + (config.primary_ip, config.primary_vm, default_zone), + (config.secondary_ip, config.secondary_vm, default_zone), + (config.active_override_ip, config.active_override_vm, + config.active_override_zone or default_zone), + ] + known.extend((fb["ip"], fb["vm"], fb["zone"]) for fb in self._fallback_entries()) + for ip, vm, zone in known: + if ip and vm and ip == host: + return vm, zone + return None + + async def _touch_worker_activity(self, job_id: str) -> None: + """Best-effort refresh of last_activity_at (keeps the VM from idle-stop).""" + try: + await asyncio.to_thread(self._worker_manager.update_activity) + except Exception as e: # noqa: BLE001 — keep-alive only, never fail the poll + logger.debug(f"[job:{job_id}] Could not refresh worker activity: {_format_exception(e)}") + + async def _stopped_worker_status(self, worker_url: Optional[str]) -> Optional[str]: + """If the VM behind ``worker_url`` is stopped/preempted, return its GCE status. + + Returns None when the VM is running, unknown, or the lookup fails — the + caller then falls back to the ordinary poll-failure tolerance. Blocking + Firestore/Compute calls run off the event loop. + """ + if not self._worker_manager: + return None + try: + # Unpinned (legacy) polls target the current active URL. + url = worker_url or await asyncio.to_thread(self._get_worker_url) + found = await asyncio.to_thread(self._vm_for_worker_url, url) + if not found: + return None + vm_name, zone = found + status = await asyncio.to_thread(self._worker_manager.get_vm_status, vm_name, zone) + except Exception as e: # noqa: BLE001 — best-effort diagnosis only + logger.debug(f"Could not check worker VM status for {worker_url}: {_format_exception(e)}") + return None + if status in _STOPPED_VM_STATUSES: + return f"{vm_name} is {status}" + return None + def _build_worker_candidates(self) -> list: """Build the ranked candidate list for ensure_any_running. @@ -310,37 +428,9 @@ class EncodingService: "is_primary": True, }] - # Optional capacity-fallback VMs in alternate zones/families, configured via - # env var. Schema: '[{"vm":"encoding-worker-fallback-c4a","zone":"us-central1-a", - # "ip":"34.x.x.x","machine_type":"c4-highcpu-32"}, ...]'. machine_type is - # optional (inferred from the VM name for legacy entries). Empty by default; - # populated after `pulumi up` provisions the fallback VMs. - import json as _json - raw = self.settings.encoding_worker_fallback_vms - if raw: - try: - parsed = _json.loads(raw) - except (ValueError, TypeError) as e: - logger.warning(f"Invalid ENCODING_WORKER_FALLBACK_VMS JSON: {e}") - parsed = [] - # A non-list root (null, dict, scalar) parses fine but would blow up - # `for item in parsed` and escape this method during connection - # recovery — degrade to no fallbacks instead. - if not isinstance(parsed, list): - logger.warning("ENCODING_WORKER_FALLBACK_VMS must be a JSON list; ignoring") - parsed = [] - for item in parsed: - try: - pool.append({ - "vm": item["vm"], - "zone": item["zone"], - "ip": item["ip"], - "machine_type": item.get("machine_type"), - "kind": "fallback", - }) - except (KeyError, TypeError) as e: - logger.warning(f"Skipping malformed fallback candidate {item}: {e}") - continue + # Optional capacity-fallback VMs in alternate zones/families (see + # _fallback_entries). Empty when unconfigured. + pool.extend(self._fallback_entries()) # Rank fastest-first, demoting types that recently stocked out. Keeps the # primary (c4d) at the top whenever it has capacity. @@ -778,6 +868,7 @@ class EncodingService: # counts from here; time spent "pending" in the worker's serialized # heavy queue counts against `queue_timeout` instead. run_started_at: Optional[float] = None + last_activity_touch: Optional[float] = None while True: now = asyncio.get_event_loop().time() @@ -809,6 +900,25 @@ class EncodingService: f"Encoding job {job_id} was lost by the worker (restarted mid-run)", job_id=job_id, ) from e + # The worker VM itself went away (Spot preemption, idle/deploy + # stop). Nothing on that VM can finish this job, so waiting out the + # remaining poll tolerance (several minutes of retries) only delays + # recovery — and "lost contact" is NOT resubmitted. Surface it as a + # lost job so run_with_lost_job_resubmit re-submits it; the + # resubmit's connection failure then restarts a worker (the Spot + # VM, or the on-demand fallback if Spot capacity is gone). + stopped = await self._stopped_worker_status(worker_url) + if stopped: + logger.warning( + f"[job:{job_id}] Encoding worker VM stopped mid-job ({stopped}) — " + f"likely Spot preemption; treating job as lost for resubmission" + ) + self._invalidate_cached_url() + raise EncodingJobLostError( + f"Encoding job {job_id} was lost: worker VM {stopped} " + f"(Spot preemption or stop)", + job_id=job_id, + ) from e consecutive_failures += 1 if consecutive_failures >= MAX_CONSECUTIVE_POLL_FAILURES: logger.error( @@ -834,6 +944,12 @@ class EncodingService: status = {} job_seen = True + if self._worker_manager and ( + last_activity_touch is None + or now - last_activity_touch >= ACTIVITY_TOUCH_INTERVAL_SECONDS + ): + last_activity_touch = now + await self._touch_worker_activity(job_id) job_status = status.get("status", "unknown") progress = status.get("progress", 0) if run_started_at is None and (job_status == "running" or progress): diff --git a/backend/services/encoding_worker_manager.py b/backend/services/encoding_worker_manager.py index b330e252..e4a51be8 100644 --- a/backend/services/encoding_worker_manager.py +++ b/backend/services/encoding_worker_manager.py @@ -34,6 +34,7 @@ from typing import Any, Optional import aiohttp +from google.api_core import exceptions as gcp_exceptions from google.cloud import firestore from backend.services.encoding_errors import ( @@ -126,7 +127,7 @@ class EncodingWorkerConfig: Returns the capacity-fallback override if set, otherwise the primary URL. The override mechanism lets us route around a zone that's - temporarily out of c4d-highcpu-32 capacity without rewriting the + temporarily out of c4d (Spot) capacity without rewriting the blue-green primary/secondary tracking. """ if self.active_override_ip: @@ -141,7 +142,7 @@ class EncodingWorkerCandidate: Carries everything needed to talk to the worker: VM name + zone (so the GCE compute client targets the right zone) and external IP (so successful starts can be persisted as the active URL override). ``machine_type`` (e.g. - "c4d-highcpu-32") lets the manager record/clear per-(type,zone) capacity + "c4d-highcpu-16") lets the manager record/clear per-(type,zone) capacity state so the shared preference logic can demote a stocked-out type. """ @@ -444,7 +445,20 @@ class EncodingWorkerManager: has_explicit_primary = any(getattr(c, "is_primary", False) for c in candidates) for index, candidate in enumerate(candidates): try: - status = self.get_vm_status(candidate.vm_name, zone=candidate.zone) + try: + status = self.get_vm_status(candidate.vm_name, zone=candidate.zone) + except gcp_exceptions.NotFound as nf: + # A candidate that no longer exists (e.g. a fallback VM removed + # from the fleet while an instance still holds the old + # ENCODING_WORKER_FALLBACK_VMS list) must be SKIPPED, not + # abort the whole failover — otherwise one stale entry blocks + # every later (real) candidate. + raise EncodingWorkerStartError( + f"VM {candidate.vm_name} not found in {candidate.zone}: {nf}", + vm_name=candidate.vm_name, + zone=candidate.zone, + code="NOT_FOUND", + ) from nf started = False if status not in ("RUNNING", "STAGING"): logger.info( diff --git a/backend/services/encoding_worker_preference.py b/backend/services/encoding_worker_preference.py index e6ac3690..848a237e 100644 --- a/backend/services/encoding_worker_preference.py +++ b/backend/services/encoding_worker_preference.py @@ -33,21 +33,24 @@ from __future__ import annotations from datetime import datetime, timezone from typing import Any, Dict, List, Optional -# Lower rank = faster = preferred. Seeded from CPU architecture; refine with the -# measured medians from infrastructure/encoding-worker/benchmark_types.py. -# c4d AMD EPYC Turin (Zen5) — current primary, fastest +# Lower rank = faster = preferred. Keyed by machine FAMILY (the prefix before the +# first "-"), so the ranking is independent of the vCPU size — the fleet moved +# from *-highcpu-32 to *-highcpu-16 on 2026-09-26 and could move again. Seeded +# from CPU architecture; refine with the measured medians from +# infrastructure/encoding-worker/benchmark_types.py. +# c4d AMD EPYC Turin (Zen5) — Spot blue-green primary pair, fastest # c4 Intel Emerald Rapids — newest Intel perf tier # n4d AMD (Titanium) — flexible/deep AMD -# c2d AMD Milan (Zen3) — mature, deep pool -# n2d AMD Rome/Milan — deep pool (32 GB floor) -# n2 Intel Cascade/Ice Lake — current fallback (32 GB floor) +# c2d AMD Milan (Zen3) — mature, deep pool (current on-demand fallback) +# n2d AMD Rome/Milan — deep pool +# n2 Intel Cascade/Ice Lake — deep pool SPEED_RANK: Dict[str, int] = { - "c4d-highcpu-32": 10, - "c4-highcpu-32": 20, - "n4d-highcpu-32": 30, - "c2d-highcpu-32": 50, - "n2d-highcpu-32": 60, - "n2-highcpu-32": 70, + "c4d": 10, + "c4": 20, + "n4d": 30, + "c2d": 50, + "n2d": 60, + "n2": 70, } # Rank for a candidate whose machine type is unknown and cannot be inferred: @@ -58,20 +61,26 @@ UNKNOWN_RANK = 1000 COOLDOWN_SECONDS = 900 # 15 minutes # Machine type of the blue-green primary/secondary pair (encoding-worker-a/-b). -# They are always c4d; used when a candidate carries no explicit machine_type. -PRIMARY_MACHINE_TYPE = "c4d-highcpu-32" +# They are always c4d (Spot); used when a candidate carries no explicit +# machine_type. Keep in sync with infrastructure/config.py MachineTypes.ENCODING_WORKER. +PRIMARY_MACHINE_TYPE = "c4d-highcpu-16" -# vm-name substring → machine type, for backward-compat inference when a fallback -# entry predates the machine_type field. Checked in order — longer / more-specific -# family tokens FIRST so "n2d" is not shadowed by "n2", nor "c4d" by "c4". -_NAME_TOKEN_TO_TYPE = ( - ("c4d", "c4d-highcpu-32"), - ("n4d", "n4d-highcpu-32"), - ("n2d", "n2d-highcpu-32"), - ("c2d", "c2d-highcpu-32"), - ("c4", "c4-highcpu-32"), - ("n2", "n2-highcpu-32"), -) +# Current vCPU size of the fleet — used only to turn a family inferred from a VM +# name into a full machine type (for display + cooldown keys). +_FLEET_SIZE_SUFFIX = "-highcpu-16" + +# vm-name substring → machine family, for backward-compat inference when a +# fallback entry predates the machine_type field. Checked in order — longer / +# more-specific family tokens FIRST so "n2d" is not shadowed by "n2", nor "c4d" +# by "c4". +_NAME_TOKEN_TO_FAMILY = ("c4d", "n4d", "n2d", "c2d", "c4", "n2") + + +def machine_family(machine_type: Optional[str]) -> Optional[str]: + """``"c4d-highcpu-16"`` → ``"c4d"``; None/empty → None.""" + if not machine_type: + return None + return machine_type.split("-", 1)[0].lower() def _candidate_name(candidate: Dict[str, Any]) -> str: @@ -83,21 +92,22 @@ def infer_machine_type(candidate: Dict[str, Any]) -> Optional[str]: """Best-effort machine type: explicit ``machine_type`` field, else vm-name token. Fallback secret entries created before the ``machine_type`` field carry only - vm/zone/ip; the VM-name family token (…-fallback-n2c, …-fallback-c4a) is the - only hint. Returns None when nothing matches (caller treats as UNKNOWN_RANK). + vm/zone/ip; the VM-name family token (…-fallback-c2df) is the only hint and + is expanded to the current fleet size. Returns None when nothing matches + (caller treats as UNKNOWN_RANK). """ explicit = candidate.get("machine_type") if explicit: return explicit name = _candidate_name(candidate).lower() - for token, mtype in _NAME_TOKEN_TO_TYPE: - if token in name: - return mtype + for family in _NAME_TOKEN_TO_FAMILY: + if family in name: + return f"{family}{_FLEET_SIZE_SUFFIX}" return None def _speed_rank(candidate: Dict[str, Any]) -> int: - return SPEED_RANK.get(infer_machine_type(candidate), UNKNOWN_RANK) + return SPEED_RANK.get(machine_family(infer_machine_type(candidate)), UNKNOWN_RANK) def cooldown_key(candidate: Dict[str, Any]) -> Optional[str]: diff --git a/backend/tests/test_deploy_promote.py b/backend/tests/test_deploy_promote.py index 60a66f3a..f33fb1d0 100644 --- a/backend/tests/test_deploy_promote.py +++ b/backend/tests/test_deploy_promote.py @@ -176,15 +176,15 @@ class TestSelectGreenRankingSharedWithRuntime: # 6-type pool with explicit machine_type (mirrors the post-rollout secret). POOL = [ {"vm": "encoding-worker-fallback-c4a", "zone": "us-central1-a", "ip": "10.0.0.1", - "machine_type": "c4-highcpu-32"}, + "machine_type": "c4-highcpu-16"}, {"vm": "encoding-worker-fallback-n4db", "zone": "us-central1-b", "ip": "10.0.0.2", - "machine_type": "n4d-highcpu-32"}, + "machine_type": "n4d-highcpu-16"}, {"vm": "encoding-worker-fallback-c2df", "zone": "us-central1-f", "ip": "10.0.0.3", - "machine_type": "c2d-highcpu-32"}, + "machine_type": "c2d-highcpu-16"}, {"vm": "encoding-worker-fallback-n2da", "zone": "us-central1-a", "ip": "10.0.0.4", - "machine_type": "n2d-highcpu-32"}, + "machine_type": "n2d-highcpu-16"}, {"vm": "encoding-worker-fallback-n2c", "zone": "us-central1-c", "ip": "10.0.0.5", - "machine_type": "n2-highcpu-32"}, + "machine_type": "n2-highcpu-16"}, ] def test_secondary_c4d_leads_then_fastest_fallbacks(self): @@ -205,7 +205,7 @@ class TestSelectGreenRankingSharedWithRuntime: from datetime import datetime, timezone cfg = _config( active_override_vm=None, - capacity_state={"c4d-highcpu-32@us-central1-c": datetime.now(timezone.utc).isoformat()}, + capacity_state={"c4d-highcpu-16@us-central1-c": datetime.now(timezone.utc).isoformat()}, ) cands = select_green_candidates(cfg, self.POOL) vms = [c["vm"] for c in cands] @@ -218,4 +218,4 @@ class TestSelectGreenRankingSharedWithRuntime: def test_machine_type_carried_through(self): cands = select_green_candidates(_config(active_override_vm=None), self.POOL) c4 = next(c for c in cands if c["vm"] == "encoding-worker-fallback-c4a") - assert c4["machine_type"] == "c4-highcpu-32" + assert c4["machine_type"] == "c4-highcpu-16" diff --git a/backend/tests/test_dropbox_service.py b/backend/tests/test_dropbox_service.py index e70bca71..926e25dd 100644 --- a/backend/tests/test_dropbox_service.py +++ b/backend/tests/test_dropbox_service.py @@ -531,3 +531,69 @@ class TestFileExists: service.file_exists("Karaoke/x.flac") mock_client.files_get_metadata.assert_called_once_with("/Karaoke/x.flac") + + +class TestDropboxLosslessExclusion: + """2026-09-26 cost cut: Nomad's own Dropbox folders skip the lossless 4K MP4 + (it stays in GCS); tenant folders still get everything.""" + + @patch("backend.services.dropbox_service.secretmanager.SecretManagerServiceClient") + def test_upload_folder_skips_excluded_suffixes(self, mock_sm_client_class, tmp_path): + from backend.services.dropbox_service import DropboxService + + mock_response = Mock() + mock_response.payload.data = json.dumps({"access_token": "token"}).encode() + mock_sm_client_class.return_value.access_secret_version.return_value = mock_response + + (tmp_path / "A - B (Final Karaoke Lossless 4k).mp4").write_text("x") + (tmp_path / "A - B (Final Karaoke Lossless 4k).mkv").write_text("x") + (tmp_path / "A - B (Final Karaoke Lossy 4k).mp4").write_text("x") + + service = DropboxService() + with patch.object(service, "upload_file") as mock_upload: + service.upload_folder( + str(tmp_path), "/Uploads/folder", + exclude_suffixes=(" (Final Karaoke Lossless 4k).mp4",), + ) + + uploaded = sorted(c.args[1].rsplit("/", 1)[1] for c in mock_upload.call_args_list) + assert uploaded == [ + "A - B (Final Karaoke Lossless 4k).mkv", + "A - B (Final Karaoke Lossy 4k).mp4", + ] + + def _settings(self, suffixes=" (Final Karaoke Lossless 4k).mp4"): + s = Mock() + s.default_dropbox_path = "/MediaUnsynced/Karaoke/Tracks-Organized" + s.default_private_dropbox_path = "/MediaUnsynced/Karaoke/Tracks-NonPublished" + s.dropbox_skip_output_suffixes = suffixes + return s + + def test_own_folders_skip_lossless_mp4_only_by_default(self): + from backend.config import Settings + from backend.services.dropbox_service import dropbox_skip_suffixes_for + + default = Settings().dropbox_skip_output_suffixes + with patch("backend.config.get_settings", return_value=self._settings(default)): + for path in ("/MediaUnsynced/Karaoke/Tracks-Organized", + "/MediaUnsynced/Karaoke/Tracks-NonPublished/"): + skip = dropbox_skip_suffixes_for(path) + assert skip == (" (Final Karaoke Lossless 4k).mp4",) + # The MKV is promised in the Dropbox folder (completion email, Fiverr bot). + assert not any(s.endswith(".mkv") for s in skip) + + def test_tenant_folder_gets_everything(self): + from backend.services.dropbox_service import dropbox_skip_suffixes_for + + with patch("backend.config.get_settings", return_value=self._settings()): + assert dropbox_skip_suffixes_for("/MediaUnsynced/Karaoke/Tracks-VocalStar") == () + assert dropbox_skip_suffixes_for(None) == () + + def test_multiple_suffixes_and_empty_config(self): + from backend.services.dropbox_service import dropbox_skip_suffixes_for + + both = " (Final Karaoke Lossless 4k).mp4| (Final Karaoke Lossless 4k).mkv" + with patch("backend.config.get_settings", return_value=self._settings(both)): + assert len(dropbox_skip_suffixes_for("/MediaUnsynced/Karaoke/Tracks-Organized")) == 2 + with patch("backend.config.get_settings", return_value=self._settings("")): + assert dropbox_skip_suffixes_for("/MediaUnsynced/Karaoke/Tracks-Organized") == () diff --git a/backend/tests/test_encoding_service.py b/backend/tests/test_encoding_service.py index 1436359c..9d82f567 100644 --- a/backend/tests/test_encoding_service.py +++ b/backend/tests/test_encoding_service.py @@ -869,6 +869,194 @@ class TestWaitForCompletionLostJob: assert result["status"] == "complete" +class TestWaitForCompletionWorkerStopped: + """The a/b workers run on Spot (2026-09-26). A preempted (STOPPED) worker VM + must surface as a lost job immediately so run_with_lost_job_resubmit re-submits + it, instead of burning the poll tolerance and failing with "lost contact".""" + + def _with_manager(self, encoding_service, vm_status, fallbacks_json=None): + cfg = MagicMock() + cfg.primary_vm, cfg.primary_ip = "encoding-worker-a", "10.0.0.1" + cfg.secondary_vm, cfg.secondary_ip = "encoding-worker-b", "10.0.0.2" + cfg.active_override_vm = cfg.active_override_ip = cfg.active_override_zone = None + mgr = MagicMock() + mgr._zone = "us-central1-c" + mgr.get_config.return_value = cfg + mgr.get_vm_status.return_value = vm_status + encoding_service._worker_manager = mgr + encoding_service.settings.encoding_worker_fallback_vms = fallbacks_json + return mgr + + @staticmethod + async def _conn_refused(job_id, worker_url=None): + raise aiohttp.ClientConnectorError( + connection_key=MagicMock(), os_error=OSError("Connection refused") + ) + + @pytest.mark.asyncio + async def test_preempted_worker_raises_lost_error_on_first_failure(self, encoding_service): + mgr = self._with_manager(encoding_service, "TERMINATED") + calls = 0 + + async def status(job_id, worker_url=None): + nonlocal calls + calls += 1 + if calls == 1: + return {"status": "running", "progress": 40} + return await self._conn_refused(job_id, worker_url) + + with patch.object(encoding_service, "get_job_status", side_effect=status), \ + patch("asyncio.sleep", new_callable=AsyncMock), \ + patch("asyncio.get_event_loop") as mock_loop: + mock_loop.return_value.time.return_value = 0 + with pytest.raises(EncodingJobLostError, match="encoding-worker-a is TERMINATED"): + await encoding_service.wait_for_completion( + "j1", worker_url="http://10.0.0.1:8080" + ) + + assert calls == 2 # bailed on the first failed poll + mgr.get_vm_status.assert_called_with("encoding-worker-a", "us-central1-c") + + @pytest.mark.asyncio + async def test_stopped_fallback_worker_resolved_via_fallback_list(self, encoding_service): + import json + mgr = self._with_manager( + encoding_service, "STOPPING", + fallbacks_json=json.dumps([{"vm": "encoding-worker-fallback-c2df", + "zone": "us-central1-f", "ip": "10.0.0.9"}]), + ) + with patch.object(encoding_service, "get_job_status", side_effect=self._conn_refused), \ + patch("asyncio.sleep", new_callable=AsyncMock), \ + patch("asyncio.get_event_loop") as mock_loop: + mock_loop.return_value.time.return_value = 0 + with pytest.raises(EncodingJobLostError): + await encoding_service.wait_for_completion( + "j1", worker_url="http://10.0.0.9:8080" + ) + mgr.get_vm_status.assert_called_with("encoding-worker-fallback-c2df", "us-central1-f") + + @pytest.mark.asyncio + async def test_running_worker_keeps_normal_poll_tolerance(self, encoding_service): + """A RUNNING VM that's just slow/unreachable is NOT a lost job.""" + self._with_manager(encoding_service, "RUNNING") + with patch.object(encoding_service, "get_job_status", side_effect=self._conn_refused), \ + patch("asyncio.sleep", new_callable=AsyncMock), \ + patch("asyncio.get_event_loop") as mock_loop: + mock_loop.return_value.time.return_value = 0 + with pytest.raises(RuntimeError, match="consecutive poll failures"): + await encoding_service.wait_for_completion( + "j1", worker_url="http://10.0.0.1:8080" + ) + + @pytest.mark.asyncio + async def test_unknown_worker_ip_keeps_normal_poll_tolerance(self, encoding_service): + mgr = self._with_manager(encoding_service, "TERMINATED") + with patch.object(encoding_service, "get_job_status", side_effect=self._conn_refused), \ + patch("asyncio.sleep", new_callable=AsyncMock), \ + patch("asyncio.get_event_loop") as mock_loop: + mock_loop.return_value.time.return_value = 0 + with pytest.raises(RuntimeError, match="consecutive poll failures"): + await encoding_service.wait_for_completion( + "j1", worker_url="http://10.9.9.9:8080" + ) + mgr.get_vm_status.assert_not_called() + + @pytest.mark.asyncio + async def test_status_lookup_failure_is_non_fatal(self, encoding_service): + mgr = self._with_manager(encoding_service, "TERMINATED") + mgr.get_vm_status.side_effect = Exception("compute API down") + with patch.object(encoding_service, "get_job_status", side_effect=self._conn_refused), \ + patch("asyncio.sleep", new_callable=AsyncMock), \ + patch("asyncio.get_event_loop") as mock_loop: + mock_loop.return_value.time.return_value = 0 + with pytest.raises(RuntimeError, match="consecutive poll failures"): + await encoding_service.wait_for_completion( + "j1", worker_url="http://10.0.0.1:8080" + ) + + @pytest.mark.asyncio + async def test_restarted_worker_mid_boot_counts_as_lost(self, encoding_service): + """Another request restarted the preempted VM before our poll — STAGING + means it went down, so the job (accepted while RUNNING) is gone.""" + self._with_manager(encoding_service, "STAGING") + with patch.object(encoding_service, "get_job_status", side_effect=self._conn_refused), \ + patch("asyncio.sleep", new_callable=AsyncMock), \ + patch("asyncio.get_event_loop") as mock_loop: + mock_loop.return_value.time.return_value = 0 + with pytest.raises(EncodingJobLostError): + await encoding_service.wait_for_completion( + "j1", worker_url="http://10.0.0.1:8080" + ) + + @pytest.mark.asyncio + async def test_polling_refreshes_worker_activity_at_most_once_a_minute(self, encoding_service): + """Keeps the routed VM inside the 5-min idle window for the whole encode, + so it isn't stopped between the job finishing and us collecting it.""" + mgr = self._with_manager(encoding_service, "RUNNING") + # First call is wait_for_completion's start_time, then one per poll. + clock = iter([0, 0, 10, 20, 70, 80, 150]) + statuses = iter([ + {"status": "running", "progress": 10}, + {"status": "running", "progress": 20}, + {"status": "running", "progress": 30}, + {"status": "running", "progress": 40}, + {"status": "running", "progress": 50}, + {"status": "complete", "output_files": ["a.mp4"]}, + ]) + + async def status(job_id, worker_url=None): + return next(statuses) + + with patch.object(encoding_service, "get_job_status", side_effect=status), \ + patch("asyncio.sleep", new_callable=AsyncMock), \ + patch("asyncio.get_event_loop") as mock_loop: + mock_loop.return_value.time.side_effect = lambda: next(clock) + result = await encoding_service.wait_for_completion( + "j1", worker_url="http://10.0.0.1:8080" + ) + + assert result["status"] == "complete" + # Touched at t=0, t=70, t=150 (>= 60s apart), not on every poll. + assert mgr.update_activity.call_count == 3 + + @pytest.mark.asyncio + async def test_activity_refresh_failure_does_not_fail_the_poll(self, encoding_service): + mgr = self._with_manager(encoding_service, "RUNNING") + mgr.update_activity.side_effect = Exception("firestore down") + + async def status(job_id, worker_url=None): + return {"status": "complete", "output_files": ["a.mp4"]} + + with patch.object(encoding_service, "get_job_status", side_effect=status), \ + patch("asyncio.sleep", new_callable=AsyncMock), \ + patch("asyncio.get_event_loop") as mock_loop: + mock_loop.return_value.time.return_value = 0 + result = await encoding_service.wait_for_completion("j1") + assert result["status"] == "complete" + + @pytest.mark.asyncio + async def test_preemption_is_resubmitted_by_run_with_lost_job_resubmit(self, encoding_service): + """End-to-end: a preempted first attempt is resubmitted under a fresh id.""" + self._with_manager(encoding_service, "TERMINATED") + attempts = [] + + async def operation(job_id): + attempts.append(job_id) + if len(attempts) == 1: + with patch.object(encoding_service, "get_job_status", + side_effect=self._conn_refused), \ + patch("asyncio.sleep", new_callable=AsyncMock): + return await encoding_service.wait_for_completion( + job_id, worker_url="http://10.0.0.1:8080" + ) + return {"status": "complete"} + + result = await run_with_lost_job_resubmit(operation, "job123") + assert result == {"status": "complete"} + assert attempts[0] == "job123" + assert attempts[1].startswith("job123_retry_") + + class TestPreviewQueueTimeout: """A queued preview must not wait the long default queue_timeout — an interactive user is waiting, so total wait is capped at the short timeout.""" @@ -1178,15 +1366,15 @@ class TestBuildWorkerCandidates: cands = svc._build_worker_candidates() assert cands[0].vm_name == "encoding-worker-a" assert cands[0].is_primary is True - assert cands[0].machine_type == "c4d-highcpu-32" + assert cands[0].machine_type == "c4d-highcpu-16" def test_fallbacks_ranked_fastest_first(self, encoding_service): import json fallbacks = json.dumps([ {"vm": "encoding-worker-fallback-n2f", "zone": "us-central1-f", "ip": "10.0.0.5", - "machine_type": "n2-highcpu-32"}, + "machine_type": "n2-highcpu-16"}, {"vm": "encoding-worker-fallback-c4a", "zone": "us-central1-a", "ip": "10.0.0.6", - "machine_type": "c4-highcpu-32"}, + "machine_type": "c4-highcpu-16"}, ]) svc = self._service_with_fallbacks(encoding_service, fallbacks) order = [c.vm_name for c in svc._build_worker_candidates()] @@ -1199,10 +1387,10 @@ class TestBuildWorkerCandidates: from datetime import datetime, timezone fallbacks = json.dumps([ {"vm": "encoding-worker-fallback-c4a", "zone": "us-central1-a", "ip": "10.0.0.6", - "machine_type": "c4-highcpu-32"}, + "machine_type": "c4-highcpu-16"}, ]) # c4d@us-central1-c stocked out "now" → demoted below c4. - cap = {"c4d-highcpu-32@us-central1-c": datetime.now(timezone.utc).isoformat()} + cap = {"c4d-highcpu-16@us-central1-c": datetime.now(timezone.utc).isoformat()} svc = self._service_with_fallbacks(encoding_service, fallbacks, capacity_state=cap) cands = svc._build_worker_candidates() order = [c.vm_name for c in cands] diff --git a/backend/tests/test_encoding_worker_code_paths.py b/backend/tests/test_encoding_worker_code_paths.py new file mode 100644 index 00000000..603f33a8 --- /dev/null +++ b/backend/tests/test_encoding_worker_code_paths.py @@ -0,0 +1,105 @@ +"""Guard for the CI encoding-worker deploy gate. + +CI only blue-green redeploys the GCE encoding worker when a path listed in +infrastructure/encoding-worker/worker_code_paths.txt changed since the serving +worker's version. If the worker starts importing a backend module that is NOT +listed, changes to it would ship to the worker only on its next cold boot and +without the pre-promote test encode. This test derives the worker's import +closure (every ``backend.*`` module reachable from the worker entrypoint and the +karaoke_gen package) and asserts each one is covered by the path list. +""" +import ast +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +PATHS_FILE = REPO / "infrastructure" / "encoding-worker" / "worker_code_paths.txt" + + +def _pathspecs(): + specs = [] + for line in PATHS_FILE.read_text().splitlines(): + line = line.strip() + if line and not line.startswith("#"): + specs.append(line) + return specs + + +def _covered(rel_path: str, specs) -> bool: + return any(rel_path == s or (s.endswith("/") and rel_path.startswith(s)) for s in specs) + + +def _backend_imports(py_file: Path): + """All ``backend.*`` module names imported anywhere in the file (incl. lazily).""" + tree = ast.parse(py_file.read_text(), filename=str(py_file)) + mods = set() + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom) and node.module and node.level == 0: + if node.module == "backend" or node.module.startswith("backend."): + mods.add(node.module) + for alias in node.names: # `from backend.services import x` → module x + mods.add(f"{node.module}.{alias.name}") + elif isinstance(node, ast.Import): + for alias in node.names: + if alias.name.startswith("backend."): + mods.add(alias.name) + return mods + + +def _module_file(mod: str): + base = REPO / Path(*mod.split(".")) + for candidate in (base.with_suffix(".py"), base / "__init__.py"): + if candidate.is_file(): + return candidate + return None # a name imported from a module, not a module itself + + +def _worker_closure(): + roots = list((REPO / "karaoke_gen").rglob("*.py")) + roots += list((REPO / "backend" / "services" / "gce_encoding").rglob("*.py")) + seen_files = set() + queue = list(roots) + backend_files = set() + while queue: + f = queue.pop() + if f in seen_files: + continue + seen_files.add(f) + for mod in _backend_imports(f): + mf = _module_file(mod) + if not mf: + continue + # Importing backend.a.b also executes backend/__init__.py and + # backend/a/__init__.py — they're part of the worker's code too. + parts = mod.split(".") + pkg_inits = [_module_file(".".join(parts[:i])) for i in range(1, len(parts))] + for dep in [mf, *pkg_inits]: + if dep and dep not in seen_files: + backend_files.add(dep) + queue.append(dep) + return backend_files + + +def test_worker_code_paths_file_is_well_formed(): + specs = _pathspecs() + assert "karaoke_gen/" in specs + assert "backend/services/gce_encoding/" in specs + # pyproject.toml changes on EVERY PR (version bump) — listing it would make + # the gate fire on every push, defeating it. Dependency changes are caught + # via poetry.lock. + assert "pyproject.toml" not in specs + assert "poetry.lock" in specs + for s in specs: + assert (REPO / s.rstrip("/")).exists(), f"stale path in {PATHS_FILE.name}: {s}" + + +def test_every_backend_module_the_worker_imports_is_gated(): + specs = _pathspecs() + missing = sorted( + str(f.relative_to(REPO)) for f in _worker_closure() + if not _covered(str(f.relative_to(REPO)), specs) + ) + assert not missing, ( + "The encoding worker imports backend modules not listed in " + f"{PATHS_FILE.relative_to(REPO)} — add them so CI redeploys the worker " + f"when they change: {missing}" + ) diff --git a/backend/tests/test_encoding_worker_manager.py b/backend/tests/test_encoding_worker_manager.py index 877c09c6..1153b909 100644 --- a/backend/tests/test_encoding_worker_manager.py +++ b/backend/tests/test_encoding_worker_manager.py @@ -494,6 +494,56 @@ class TestEnsureAnyRunning: # Only the first candidate's start was attempted. mock_compute.start.assert_called_once() + def test_skips_deleted_candidate_and_starts_next(self, manager, mock_db, mock_compute): + """A candidate VM that no longer exists (removed from the fleet while an + instance still holds a stale fallback list) is skipped, not fatal.""" + from google.api_core import exceptions as gcp_exceptions + + stopped = MagicMock() + stopped.status = "TERMINATED" + + def _get(project, zone, instance): + if instance == "encoding-worker-fallback-gone": + raise gcp_exceptions.NotFound("instance not found") + return stopped + + mock_compute.get.side_effect = _get + # Primary: Spot capacity exhausted. Deleted fallback: NotFound. Live fallback: ok. + mock_compute.start.side_effect = [_capacity_op(), _ok_op()] + + candidates = [ + EncodingWorkerCandidate(vm_name="encoding-worker-blue", zone="us-central1-c", + ip="10.0.0.1", is_primary=True), + EncodingWorkerCandidate(vm_name="encoding-worker-fallback-gone", zone="us-central1-a", + ip="10.0.0.2"), + EncodingWorkerCandidate(vm_name="encoding-worker-fallback-c2df", zone="us-central1-f", + ip="10.0.0.3"), + ] + + with patch("backend.services.encoding_worker_manager.datetime") as mock_dt: + mock_dt.now.return_value = datetime(2026, 9, 26, 9, 0, 0, tzinfo=UTC) + mock_dt.side_effect = lambda *a, **kw: datetime(*a, **kw) + result = manager.ensure_any_running(candidates) + + assert result["vm_name"] == "encoding-worker-fallback-c2df" + assert result["fell_back"] is True + # The deleted VM was never asked to start. + started = [c.kwargs["instance"] for c in mock_compute.start.call_args_list] + assert started == ["encoding-worker-blue", "encoding-worker-fallback-c2df"] + + def test_all_candidates_missing_raises_start_error(self, manager, mock_compute): + """If every candidate is gone, surface a typed start error (not a bare NotFound).""" + from google.api_core import exceptions as gcp_exceptions + + mock_compute.get.side_effect = gcp_exceptions.NotFound("gone") + candidates = [ + EncodingWorkerCandidate(vm_name="vm-x", zone="us-central1-a", ip="10.0.0.9"), + ] + with pytest.raises(EncodingWorkerStartError) as exc_info: + manager.ensure_any_running(candidates) + assert exc_info.value.code == "NOT_FOUND" + mock_compute.start.assert_not_called() + def test_falls_through_to_next_zone_on_capacity_error(self, manager, mock_db, mock_compute): """Capacity error on candidate 1 triggers a try of candidate 2.""" mock_instance = MagicMock() @@ -953,9 +1003,9 @@ class TestCapacityStateFeedback: candidates = [ EncodingWorkerCandidate(vm_name="encoding-worker-a", zone="us-central1-c", - ip="10.0.0.1", machine_type="c4d-highcpu-32", is_primary=True), + ip="10.0.0.1", machine_type="c4d-highcpu-16", is_primary=True), EncodingWorkerCandidate(vm_name="encoding-worker-fallback-n2f", zone="us-central1-f", - ip="10.0.0.2", machine_type="n2-highcpu-32"), + ip="10.0.0.2", machine_type="n2-highcpu-16"), ] with patch("backend.services.encoding_worker_manager.datetime") as mock_dt: @@ -970,7 +1020,7 @@ class TestCapacityStateFeedback: recorded = {} for c in set_calls: recorded.update(c.args[0]["capacity_state"]) - assert recorded.get("c4d-highcpu-32@us-central1-c") == "2026-05-05T09:00:00+00:00" + assert recorded.get("c4d-highcpu-16@us-central1-c") == "2026-05-05T09:00:00+00:00" def test_clears_stockout_for_type_on_clean_start(self, manager, mock_db, mock_compute): mock_instance = MagicMock() @@ -980,7 +1030,7 @@ class TestCapacityStateFeedback: candidates = [ EncodingWorkerCandidate(vm_name="encoding-worker-a", zone="us-central1-c", - ip="10.0.0.1", machine_type="c4d-highcpu-32", is_primary=True), + ip="10.0.0.1", machine_type="c4d-highcpu-16", is_primary=True), ] with patch("backend.services.encoding_worker_manager.datetime") as mock_dt: @@ -994,7 +1044,7 @@ class TestCapacityStateFeedback: payload = c.args[0] if c.args else {} if "capacity_state" in payload: cleared.update(payload["capacity_state"]) - assert cleared.get("c4d-highcpu-32@us-central1-c") is None + assert cleared.get("c4d-highcpu-16@us-central1-c") is None def test_stockout_write_failure_does_not_break_start(self, manager, mock_db, mock_compute): """A Firestore error while recording a stockout must not abort failover.""" @@ -1008,9 +1058,9 @@ class TestCapacityStateFeedback: candidates = [ EncodingWorkerCandidate(vm_name="encoding-worker-a", zone="us-central1-c", - ip="10.0.0.1", machine_type="c4d-highcpu-32", is_primary=True), + ip="10.0.0.1", machine_type="c4d-highcpu-16", is_primary=True), EncodingWorkerCandidate(vm_name="encoding-worker-fallback-n2f", zone="us-central1-f", - ip="10.0.0.2", machine_type="n2-highcpu-32"), + ip="10.0.0.2", machine_type="n2-highcpu-16"), ] with patch("backend.services.encoding_worker_manager.datetime") as mock_dt: @@ -1035,9 +1085,9 @@ class TestCapacityStateFeedback: # Fallback is first (primary demoted), primary flagged but ranked second. candidates = [ EncodingWorkerCandidate(vm_name="encoding-worker-fallback-c2df", zone="us-central1-f", - ip="10.0.0.9", machine_type="c2d-highcpu-32", is_primary=False), + ip="10.0.0.9", machine_type="c2d-highcpu-16", is_primary=False), EncodingWorkerCandidate(vm_name="encoding-worker-a", zone="us-central1-c", - ip="10.0.0.1", machine_type="c4d-highcpu-32", is_primary=True), + ip="10.0.0.1", machine_type="c4d-highcpu-16", is_primary=True), ] with patch("backend.services.encoding_worker_manager.datetime") as mock_dt: @@ -1101,7 +1151,7 @@ class TestDemoteActiveWorker: assert doc_ref.set.called set_args, set_kwargs = doc_ref.set.call_args assert "capacity_state" in set_args[0] - assert "c4-highcpu-32@us-central1-c" in set_args[0]["capacity_state"] + assert "c4-highcpu-16@us-central1-c" in set_args[0]["capacity_state"] assert set_kwargs.get("merge") is True # Cleared the active_override so the retry starts from the primary. diff --git a/backend/tests/test_encoding_worker_preference.py b/backend/tests/test_encoding_worker_preference.py index 7f0ed015..aaed0757 100644 --- a/backend/tests/test_encoding_worker_preference.py +++ b/backend/tests/test_encoding_worker_preference.py @@ -31,14 +31,14 @@ def _c(vm, zone, machine_type=None, **extra): # --------------------------------------------------------------------------- def test_explicit_machine_type_wins(): - assert infer_machine_type(_c("whatever", "z", "c2d-highcpu-32")) == "c2d-highcpu-32" + assert infer_machine_type(_c("whatever", "z", "c2d-highcpu-16")) == "c2d-highcpu-16" def test_infer_from_vm_name_specific_before_generic(): # "n2d"/"c4d" must not be shadowed by "n2"/"c4". - assert infer_machine_type(_c("encoding-worker-fallback-n2da", "z")) == "n2d-highcpu-32" - assert infer_machine_type(_c("encoding-worker-fallback-n2f", "z")) == "n2-highcpu-32" - assert infer_machine_type(_c("encoding-worker-fallback-c4x", "z")) == "c4-highcpu-32" + assert infer_machine_type(_c("encoding-worker-fallback-n2da", "z")) == "n2d-highcpu-16" + assert infer_machine_type(_c("encoding-worker-fallback-n2f", "z")) == "n2-highcpu-16" + assert infer_machine_type(_c("encoding-worker-fallback-c4x", "z")) == "c4-highcpu-16" def test_infer_unknown_returns_none(): @@ -47,7 +47,7 @@ def test_infer_unknown_returns_none(): def test_runtime_dict_shape_uses_vm_name_key(): - assert infer_machine_type({"vm_name": "x-n2d-y", "zone": "z"}) == "n2d-highcpu-32" + assert infer_machine_type({"vm_name": "x-n2d-y", "zone": "z"}) == "n2d-highcpu-16" # --------------------------------------------------------------------------- @@ -56,10 +56,10 @@ def test_runtime_dict_shape_uses_vm_name_key(): def test_orders_fastest_first(): pool = [ - _c("n2", "us-central1-c", "n2-highcpu-32"), - _c("c4d", "us-central1-c", "c4d-highcpu-32"), - _c("c2d", "us-central1-f", "c2d-highcpu-32"), - _c("c4", "us-central1-a", "c4-highcpu-32"), + _c("n2", "us-central1-c", "n2-highcpu-16"), + _c("c4d", "us-central1-c", "c4d-highcpu-16"), + _c("c2d", "us-central1-f", "c2d-highcpu-16"), + _c("c4", "us-central1-a", "c4-highcpu-16"), ] got = [c["vm"] for c in ordered_candidates(pool, now=NOW)] assert got == ["c4d", "c4", "c2d", "n2"] @@ -68,18 +68,18 @@ def test_orders_fastest_first(): def test_unknown_type_sorts_last_but_kept(): pool = [ _c("mystery", "z"), # no machine_type, unknown name - _c("c4d", "z", "c4d-highcpu-32"), + _c("c4d", "z", "c4d-highcpu-16"), ] got = [c["vm"] for c in ordered_candidates(pool, now=NOW)] assert got == ["c4d", "mystery"] - assert SPEED_RANK["c4d-highcpu-32"] < UNKNOWN_RANK + assert SPEED_RANK["c4d"] < UNKNOWN_RANK def test_stable_within_equal_rank_preserves_zone_spread(): # Two n2 in different zones keep input order (caller's secondary preference). pool = [ - _c("n2f", "us-central1-f", "n2-highcpu-32"), - _c("n2c", "us-central1-c", "n2-highcpu-32"), + _c("n2f", "us-central1-f", "n2-highcpu-16"), + _c("n2c", "us-central1-c", "n2-highcpu-16"), ] got = [c["vm"] for c in ordered_candidates(pool, now=NOW)] assert got == ["n2f", "n2c"] @@ -91,24 +91,24 @@ def test_stable_within_equal_rank_preserves_zone_spread(): def test_recent_stockout_demotes_type_to_back(): pool = [ - _c("c4d", "us-central1-c", "c4d-highcpu-32"), - _c("c4", "us-central1-a", "c4-highcpu-32"), - _c("n2", "us-central1-c", "n2-highcpu-32"), + _c("c4d", "us-central1-c", "c4d-highcpu-16"), + _c("c4", "us-central1-a", "c4-highcpu-16"), + _c("n2", "us-central1-c", "n2-highcpu-16"), ] # c4d stocked out 1 minute ago → demoted behind the still-healthy types. - capacity_state = {"c4d-highcpu-32@us-central1-c": (NOW - timedelta(minutes=1)).isoformat()} + capacity_state = {"c4d-highcpu-16@us-central1-c": (NOW - timedelta(minutes=1)).isoformat()} got = [c["vm"] for c in ordered_candidates(pool, capacity_state, now=NOW)] assert got == ["c4", "n2", "c4d"] def test_cooldown_expires_and_snaps_back_to_fastest(): pool = [ - _c("c4d", "us-central1-c", "c4d-highcpu-32"), - _c("c4", "us-central1-a", "c4-highcpu-32"), + _c("c4d", "us-central1-c", "c4d-highcpu-16"), + _c("c4", "us-central1-a", "c4-highcpu-16"), ] # Stockout older than COOLDOWN → c4d is hot again and returns to the top. old = (NOW - timedelta(seconds=COOLDOWN_SECONDS + 60)).isoformat() - capacity_state = {"c4d-highcpu-32@us-central1-c": old} + capacity_state = {"c4d-highcpu-16@us-central1-c": old} got = [c["vm"] for c in ordered_candidates(pool, capacity_state, now=NOW)] assert got == ["c4d", "c4"] @@ -116,15 +116,15 @@ def test_cooldown_expires_and_snaps_back_to_fastest(): def test_two_types_stocked_out_still_serves_from_deep_pool(): # Both newest-gen types (c4d, c4) exhausted; deep pools keep serving in order. pool = [ - _c("c4d", "us-central1-c", "c4d-highcpu-32"), - _c("c4", "us-central1-a", "c4-highcpu-32"), - _c("c2d", "us-central1-f", "c2d-highcpu-32"), - _c("n2d", "us-central1-a", "n2d-highcpu-32"), + _c("c4d", "us-central1-c", "c4d-highcpu-16"), + _c("c4", "us-central1-a", "c4-highcpu-16"), + _c("c2d", "us-central1-f", "c2d-highcpu-16"), + _c("n2d", "us-central1-a", "n2d-highcpu-16"), ] recent = (NOW - timedelta(minutes=2)).isoformat() capacity_state = { - "c4d-highcpu-32@us-central1-c": recent, - "c4-highcpu-32@us-central1-a": recent, + "c4d-highcpu-16@us-central1-c": recent, + "c4-highcpu-16@us-central1-a": recent, } got = [c["vm"] for c in ordered_candidates(pool, capacity_state, now=NOW)] # Hot deep pools first (fastest-first among them), then the two cold types. @@ -133,31 +133,31 @@ def test_two_types_stocked_out_still_serves_from_deep_pool(): def test_type_only_capacity_key_demotes_all_zones(): pool = [ - _c("c4d-c", "us-central1-c", "c4d-highcpu-32"), - _c("n2", "us-central1-c", "n2-highcpu-32"), + _c("c4d-c", "us-central1-c", "c4d-highcpu-16"), + _c("n2", "us-central1-c", "n2-highcpu-16"), ] - capacity_state = {"c4d-highcpu-32": (NOW - timedelta(minutes=1)).isoformat()} + capacity_state = {"c4d-highcpu-16": (NOW - timedelta(minutes=1)).isoformat()} got = [c["vm"] for c in ordered_candidates(pool, capacity_state, now=NOW)] assert got == ["n2", "c4d-c"] def test_empty_capacity_state_is_pure_speed_order(): - pool = [_c("n2", "z", "n2-highcpu-32"), _c("c4d", "z", "c4d-highcpu-32")] + pool = [_c("n2", "z", "n2-highcpu-16"), _c("c4d", "z", "c4d-highcpu-16")] assert [c["vm"] for c in ordered_candidates(pool, {}, now=NOW)] == ["c4d", "n2"] assert [c["vm"] for c in ordered_candidates(pool, None, now=NOW)] == ["c4d", "n2"] def test_naive_timestamp_tolerated(): - pool = [_c("c4d", "us-central1-c", "c4d-highcpu-32"), _c("n2", "z", "n2-highcpu-32")] + pool = [_c("c4d", "us-central1-c", "c4d-highcpu-16"), _c("n2", "z", "n2-highcpu-16")] naive = (NOW - timedelta(minutes=1)).replace(tzinfo=None).isoformat() - capacity_state = {"c4d-highcpu-32@us-central1-c": naive} + capacity_state = {"c4d-highcpu-16@us-central1-c": naive} got = [c["vm"] for c in ordered_candidates(pool, capacity_state, now=NOW)] assert got == ["n2", "c4d"] def test_malformed_timestamp_ignored(): - pool = [_c("c4d", "us-central1-c", "c4d-highcpu-32"), _c("n2", "z", "n2-highcpu-32")] - capacity_state = {"c4d-highcpu-32@us-central1-c": "not-a-date"} + pool = [_c("c4d", "us-central1-c", "c4d-highcpu-16"), _c("n2", "z", "n2-highcpu-16")] + capacity_state = {"c4d-highcpu-16@us-central1-c": "not-a-date"} got = [c["vm"] for c in ordered_candidates(pool, capacity_state, now=NOW)] assert got == ["c4d", "n2"] # unparseable → not in cooldown @@ -167,6 +167,6 @@ def test_empty_pool(): def test_cooldown_key_shape(): - assert cooldown_key(_c("x", "us-central1-c", "c4d-highcpu-32")) == "c4d-highcpu-32@us-central1-c" - assert cooldown_key(_c("x", None, "n2-highcpu-32")) == "n2-highcpu-32" + assert cooldown_key(_c("x", "us-central1-c", "c4d-highcpu-16")) == "c4d-highcpu-16@us-central1-c" + assert cooldown_key(_c("x", None, "n2-highcpu-16")) == "n2-highcpu-16" assert cooldown_key(_c("no-token", "z")) is None diff --git a/backend/tests/test_system_status_endpoint.py b/backend/tests/test_system_status_endpoint.py index 8abc735c..6801b8b4 100644 --- a/backend/tests/test_system_status_endpoint.py +++ b/backend/tests/test_system_status_endpoint.py @@ -167,7 +167,7 @@ class TestSystemStatusAdmin: assert admin["on_fallback"] is False assert admin["active_vm"] == "encoding-worker-b" assert admin["active_zone"] == "us-central1-c" - assert admin["active_machine_type"] == "c4d-highcpu-32" + assert admin["active_machine_type"] == "c4d-highcpu-16" @pytest.mark.asyncio async def test_admin_sees_capacity_fallback_worker( @@ -203,5 +203,5 @@ class TestSystemStatusAdmin: assert admin["on_fallback"] is True assert admin["active_vm"] == "encoding-worker-fallback-n2c" assert admin["active_zone"] == "us-central1-a" - # "n2c" family token → n2-highcpu-32 (inferred from the VM name). - assert admin["active_machine_type"] == "n2-highcpu-32" + # "n2c" family token → n2-highcpu-16 (inferred from the VM name). + assert admin["active_machine_type"] == "n2-highcpu-16" diff --git a/backend/workers/video_worker.py b/backend/workers/video_worker.py index 17044969..ce8f298c 100644 --- a/backend/workers/video_worker.py +++ b/backend/workers/video_worker.py @@ -1180,7 +1180,12 @@ async def _handle_native_distribution( remote_folder = f"{dropbox_path}/{folder_name}" job_log.info(f"Uploading to Dropbox folder: {remote_folder}") - dropbox.upload_folder(temp_dir, remote_folder) + from backend.services.dropbox_service import dropbox_skip_suffixes_for + + dropbox.upload_folder( + temp_dir, remote_folder, + exclude_suffixes=dropbox_skip_suffixes_for(dropbox_path), + ) # Create sharing link try: diff --git a/backend/workers/video_worker_orchestrator.py b/backend/workers/video_worker_orchestrator.py index 45e3a183..152cea00 100644 --- a/backend/workers/video_worker_orchestrator.py +++ b/backend/workers/video_worker_orchestrator.py @@ -931,8 +931,15 @@ class VideoWorkerOrchestrator: folder_name = f"{self.result.brand_code or 'TRACK-0000'} - {base_name}" remote_folder = f"{self.config.dropbox_path}/{folder_name}" - # Upload entire output directory - dropbox.upload_folder(self.config.output_dir, remote_folder) + # Upload the output directory, minus files Nomad's own Dropbox archive + # doesn't need (lossless 4K MP4 by default — it stays in GCS). Tenant + # folders get everything. See dropbox_skip_suffixes_for. + from backend.services.dropbox_service import dropbox_skip_suffixes_for + + skip = dropbox_skip_suffixes_for(self.config.dropbox_path) + if skip: + self.job_log.info(f"Excluding from Dropbox upload (kept in GCS): {list(skip)}") + dropbox.upload_folder(self.config.output_dir, remote_folder, exclude_suffixes=skip) # Create sharing link try: diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 04774314..3512063a 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -38,10 +38,11 @@ │ (jobs) │ │ (files) │ │ (API keys) │ │ Worker (*) │ └──────────┘ └──────────┘ └──────────────┘ └──────────────┘ -(*) GCE Encoding Worker: a ranked pool of 6 x86_64 32-vCPU highcpu machine - families (c4d/c4/n4d/c2d/n2d/n2). c4d-highcpu-32 (AMD EPYC Turin) is the - fastest and stays the primary/preferred pick; the other families are - stockout-resilience fallbacks selected fastest-first with a capacity cooldown. +(*) GCE Encoding Worker: a Spot c4d-highcpu-16 blue-green pair (AMD EPYC + Turin, the preferred pick) plus ONE on-demand c2d-highcpu-16 fallback in a + different zone for Spot/c4d stockouts (3 VMs since 2026-09-26 — see + docs/archive/2026-09-26-encoding-cost-cuts.md). Candidates are selected + fastest-first with a capacity cooldown. Used for both final video encoding and preview video generation. Uses immutable deployment pattern - see infrastructure/encoding-worker/README.md and the "Encoding Worker" section below. @@ -305,16 +306,25 @@ karaoke-gen shares a GCP project (`nomadkaraoke`) with karaoke-decide, but uses ## Encoding Worker: blue-green primary pair + multi-family fallback pool The GCE encoding worker is a **blue-green primary pair** (`encoding-worker-a`/`-b`, -both c4d-highcpu-32 in us-central1-c) plus a **ranked pool of stockout-resilience -fallback VMs** spanning 6 x86_64 machine families (see "Multi-instance-type pool" -below). **10 VMs total**, all TERMINATED by default and started on demand. +both **Spot** c4d-highcpu-16 in us-central1-c) plus **one on-demand fallback** +(`encoding-worker-fallback-c2df`, c2d-highcpu-16, us-central1-f). **3 VMs total** +(cut from 10 on 2026-09-26 — docs/archive/2026-09-26-encoding-cost-cuts.md), all +TERMINATED by default, started on demand, and idle-stopped after 5 min. + +**Spot preemption:** the pair uses `instance_termination_action=STOP`, so a +preempted worker is just a TERMINATED VM that the next request restarts. An +encode in flight when the VM is preempted is detected by +`EncodingService.wait_for_completion` (the pinned worker VM is no longer RUNNING) +and raised as `EncodingJobLostError`, which `run_with_lost_job_resubmit` resubmits +— the resubmit's connection failure then starts the Spot VM again, or the +on-demand fallback if Spot capacity is gone. ### Architecture ```text Frontend → Backend (Cloud Run) → Firestore config → active worker (primary, or a ↑ capacity-fallback "override") -Cloud Scheduler (5 min) → Cloud Function → checks idle + stops VMs +Cloud Scheduler (2 min) → Cloud Function → checks idle (5 min) + stops VMs ↑ CI (GitHub Actions) → selects a fresh "green" (ranked pool) → health + encode test → promote (swap primary OR set active_override) → stop old @@ -323,10 +333,13 @@ CI (GitHub Actions) → selects a fresh "green" (ranked pool) → health + encod ### Multi-instance-type pool (stockout resilience, v0.195.0) A single machine family can hit a region-wide `ZONE_RESOURCE_POOL_EXHAUSTED` -stockout across every zone at once. To survive that, the fallback fleet spans -**6 families** (c4d, c4, n4d, c2d, n2d, n2 — all `-highcpu-32`, ≥32 GB) across -zones a/b/c/f: `encoding-worker-fallback-a`/`-b` (c4d), `-n2c`/`-n2f` (n2), -`-c4a` (c4), `-n4db` (n4d), `-c2df` (c2d), `-n2da` (n2d). +stockout across every zone at once. To survive that, the fallback lives in a +different family AND zone from the c4d pair: `encoding-worker-fallback-c2df` +(c2d-highcpu-16, on-demand, us-central1-f). The pool was 8 fallbacks across 6 +families (v0.195.0) until the 2026-09-26 cost cut; the selection machinery below +still handles any number of fallbacks — re-add entries to +`EncodingWorkerConfig.FALLBACKS` + the `encoding-worker-fallback-vms` secret if +stockouts become a problem again. **Candidate ordering is one shared pure module**, `backend/services/encoding_worker_preference.py::ordered_candidates(pool, capacity_state)`, diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index d992691d..474b416f 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -221,7 +221,7 @@ TESTMAIL_NAMESPACE=... # testmail.app namespace ## Encoding Worker Operations -The encoding worker is a blue-green primary pair (`encoding-worker-a`/`-b`, c4d) plus a ranked pool of 8 stopped fallback VMs spanning 6 machine families (c4d/c4/n4d/c2d/n2d/n2) for stockout resilience — 10 VMs total. Candidate ordering (runtime + deploy) is the shared `backend/services/encoding_worker_preference.py`. See [ARCHITECTURE.md](ARCHITECTURE.md#encoding-worker-blue-green-primary-pair--multi-family-fallback-pool) for the full design. +The encoding worker is a Spot blue-green primary pair (`encoding-worker-a`/`-b`, c4d-highcpu-16) plus one stopped on-demand fallback (`encoding-worker-fallback-c2df`, c2d-highcpu-16, us-central1-f) — 3 VMs total since 2026-09-26. CI only blue-green redeploys the worker when a path in `infrastructure/encoding-worker/worker_code_paths.txt` changed since the serving version (or via Actions → CI → Run workflow with "Force a blue-green redeploy of the GCE encoding worker"). Candidate ordering (runtime + deploy) is the shared `backend/services/encoding_worker_preference.py`. See [ARCHITECTURE.md](ARCHITECTURE.md#encoding-worker-blue-green-primary-pair--multi-family-fallback-pool) for the full design. ### Seed Firestore config (initial setup or reset) diff --git a/docs/GCP-COST-OPTIMIZATION.md b/docs/GCP-COST-OPTIMIZATION.md index 2fa71748..ea12dfb8 100644 --- a/docs/GCP-COST-OPTIMIZATION.md +++ b/docs/GCP-COST-OPTIMIZATION.md @@ -80,11 +80,11 @@ The `concurrency=1` setting may have been added deliberately for performance rea ### 🔍 GCE Encoding Worker **Current State (updated v0.195.0 — the opportunities below are largely SHIPPED):** -- No longer a single 24/7 VM. It is a pool of **on-demand** VMs: a c4d-highcpu-32 - blue-green primary pair + 8 stopped fallbacks across 6 machine families - (c4d/c4/n4d/c2d/n2d/n2) for stockout resilience. -- **Idle auto-shutdown is live** (JIT start on the lyrics-review page; a Cloud - Function stops idle VMs after 15 min). When idle, cost is just boot disks +- No longer a single 24/7 VM. Since 2026-09-26 it is 3 VMs: a **Spot** + c4d-highcpu-16 blue-green pair + 1 stopped on-demand c2d-highcpu-16 fallback + (was 10 × 32-vCPU on-demand VMs — see docs/archive/2026-09-26-encoding-cost-cuts.md). +- **Idle auto-shutdown is live** (JIT start + heartbeat on the lyrics-review page; + a Cloud Function checks every 2 min and stops idle VMs after 5 min). When idle, cost is just boot disks (50 GB since 2026-09-26, ~$4-5/VM/mo; was 100 GB / ~$10 — see docs/archive/2026-09-26-encoding-worker-disk-cost.md); compute is billed only while encoding — so the historical "~$700/mo running 24/7" no longer applies. diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 545c7397..9a227b0b 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -438,7 +438,9 @@ gcloud compute instances describe encoding-worker-a \ **Background — what this state means:** GCE returned `ZONE_RESOURCE_POOL_EXHAUSTED` or transient `503 SERVICE_UNAVAILABLE` from `compute.instances.start` on every encoding-worker VM. The render worker parks the job in `RENDER_PENDING_CAPACITY` instead of failing it; Cloud Scheduler retries it every 10 min via `/api/internal/retry-pending-render-jobs`. Hard timeout is 24 hours (then transitions to `failed` with a clear permanent-failure message). -The fallback fleet is diversified across **6 machine families** (broadened to the full pool in v0.195.0): `c4d-highcpu-32` primaries (`encoding-worker-a`/`-b`) in `us-central1-c`, plus 8 stopped fallbacks — `c4d` (`-fallback-a`/`-b`, zones a/b), `n2` (`-n2c`/`-n2f`, zones c/f), `c4` (`-c4a`, zone a), `n4d` (`-n4db`, zone b), `c2d` (`-c2df`, zone f), `n2d` (`-n2da`, zone a). Candidates are tried **fastest-first with a 15-min capacity cooldown** (a family that stocks out is demoted, then re-probed) — see `backend/services/encoding_worker_preference.py`. A single-family region-wide stockout (which took out c4d a/b/c at once on 2026-08-12) now still finds capacity in the 5 other families. If you see EVERY family return stockout, that's a genuinely severe regional event — wait, or add a cross-region fallback. NOTE: creating a *new* fallback VM (or re-creating a deleted one) still needs a one-time boot allocation, so a deep enough crunch can even block provisioning — a `pulumi up` may show the missing VM as "to create / errored" until capacity returns. +Since 2026-09-26 the fleet is 3 VMs: **Spot** `c4d-highcpu-16` primaries (`encoding-worker-a`/`-b`) in `us-central1-c`, plus ONE stopped on-demand fallback `encoding-worker-fallback-c2df` (`c2d-highcpu-16`, `us-central1-f`). Candidates are tried **fastest-first with a 15-min capacity cooldown** — see `backend/services/encoding_worker_preference.py`. If both the Spot pair and the c2d fallback return stockout, that's a genuinely severe regional event — wait, or (re-)add a fallback in another family/zone to `EncodingWorkerConfig.FALLBACKS` **and** the `encoding-worker-fallback-vms` secret (the 2026-08-15 pool had 8 fallbacks across 6 families). NOTE: creating a *new* fallback VM needs a one-time boot allocation, so a deep enough crunch can even block provisioning. + +**Spot preemption mid-encode** is expected occasionally: the backend sees the pinned worker VM is TERMINATED/STOPPING, logs `Encoding worker VM stopped mid-job (... ) — likely Spot preemption`, and resubmits the encode as `_retry_` (up to `ENCODING_RESUBMIT_MAX`, default 2). Repeated preemptions → the job fails with the lost-job error; check `gcloud compute operations list --filter="operationType=compute.instances.preempted"`. **Related symptom — preview 524 / "NetworkError":** the same capacity exhaustion also makes `POST /api/review/{id}/preview-video` fall back to slow local encoding (~130–160 s), which exceeds Cloudflare's ~100 s edge timeout → the browser shows a **524** or Firefox *"NetworkError when attempting to fetch resource"* even though the backend returns 200. If a user reports the review preview failing, check for a concurrent stockout; the already-rendered preview mp4 (if any) is fetchable via `GET /api/review/{id}/preview-video/{hash}` (302 → signed GCS URL) with a Bearer admin token. Completing the review does **not** require the preview. @@ -466,18 +468,13 @@ gcloud logging read 'protoPayload.methodName:"instances.start" AND "ZONE_RESOURC --project=nomadkaraoke --freshness=30m --limit=5 \ --format='value(timestamp, protoPayload.resourceName)' -# Try starting a fallback in each of the 6 families; if ALL refuse, it's severe — wait. +# Try starting each worker; if ALL refuse, it's severe — wait. # Synchronous (no --async) so the start op surfaces stockout inline, and we capture # gcloud's own exit status rather than piping (a pipe would return sed's status). for VM_ZONE in \ - "encoding-worker-fallback-a:us-central1-a" \ - "encoding-worker-fallback-b:us-central1-b" \ - "encoding-worker-fallback-n2c:us-central1-c" \ - "encoding-worker-fallback-n2f:us-central1-f" \ - "encoding-worker-fallback-c4a:us-central1-a" \ - "encoding-worker-fallback-n4db:us-central1-b" \ - "encoding-worker-fallback-c2df:us-central1-f" \ - "encoding-worker-fallback-n2da:us-central1-a"; do + "encoding-worker-a:us-central1-c" \ + "encoding-worker-b:us-central1-c" \ + "encoding-worker-fallback-c2df:us-central1-f"; do VM="${VM_ZONE%%:*}"; ZONE="${VM_ZONE##*:}" if OUT=$(gcloud compute instances start "$VM" --zone="$ZONE" --project=nomadkaraoke 2>&1); then echo "$VM: OK (started — remember to stop it)" diff --git a/docs/archive/2026-09-26-encoding-cost-cuts.md b/docs/archive/2026-09-26-encoding-cost-cuts.md new file mode 100644 index 00000000..e3f93c97 --- /dev/null +++ b/docs/archive/2026-09-26-encoding-cost-cuts.md @@ -0,0 +1,86 @@ +# Encoding-worker cost cuts (2026-09-26) + +Part of the "whole GCP project under $300/mo" push (workspace plan: +`docs/archive/2026-09-26-gcp-under-300-plan.md`). The encoding VMs were costing +~$300/mo for ~5-20 jobs/day. Andrew approved all of these, including the speed +trade-offs. + +## What changed + +| | Before | After | +|---|---|---| +| Primary pair `encoding-worker-a`/`-b` | c4d-highcpu-32, on-demand | **c4d-highcpu-16, Spot** (STOP on preemption) | +| Fallbacks | 8 stopped VMs across 6 families (+8 static IPs, 8 boot disks) | **1**: `encoding-worker-fallback-c2df`, c2d-highcpu-16, **on-demand**, us-central1-f | +| Fleet | 10 VMs | 3 VMs | +| Idle shutdown | 15 min, checked every 5 min | **5 min, checked every 2 min** | +| CI worker blue-green deploy | every push to main (~5x/day, boots a green + test encode) | only when `infrastructure/encoding-worker/worker_code_paths.txt` paths changed since the serving version; manual override via workflow_dispatch input `force_encoding_worker_deploy` | +| Nomad Dropbox upload | whole output folder | minus the lossless 4K **MP4** (`DROPBOX_SKIP_OUTPUT_SUFFIXES`); tenant folders unchanged | + +Dropped from scope by Andrew mid-task: moving review previews off the VMs +(`USE_GCE_PREVIEW_ENCODING` stays `true`) and removing the review-page warmup. The +warmup is deliberate (VM warm for preview + final render), so instead the review +page now heartbeats every 2 min while the tab is visible and was used in the last +15 min (`frontend/lib/lyrics-review/hooks/useEncodingWorkerKeepAlive.ts`), keeping +the VM inside the new 5-min idle window during a review. Previously the heartbeat +only fired on lyric edits. + +## How it was applied (infra) + +- **Spot conversion without recreating the VMs.** Pulumi (pulumi-gcp 9.22.0) + treats a `scheduling` change as *replace*; recreating a c4d VM risks a stockout + at create time (and the VM would be gone). Instead each VM was converted in + place while STOPPED: + `gcloud compute instances set-scheduling --zone us-central1-c --preemptible --provisioning-model=SPOT --instance-termination-action=STOP --no-restart-on-failure --maintenance-policy=TERMINATE`, + then `pulumi refresh --target ` (state now matches the code) and a targeted + `pulumi up` for the in-place machine-type change (`allow_stopping_for_update`). + Note `--preemptible` is required alongside `--provisioning-model=SPOT`. +- Fallback removal: targeted `pulumi up` deleted 7 VMs (+ their auto-delete boot + disks) and 7 static IPs; c2df resized in place to c2d-highcpu-16. +- `encoding-worker-fallback-vms` secret → version 6 (only c2df). The value is + managed manually (Pulumi only creates the secret). +- Idle function env + scheduler updated by Pulumi; `main.py` default also 5. + +## Behaviour notes + +- **Spot preemption mid-encode**: `EncodingService.wait_for_completion` now checks + the pinned worker VM's status on a failed poll; if it's STOPPING/TERMINATED/etc. + it raises `EncodingJobLostError`, so `run_with_lost_job_resubmit` resubmits the + encode (fresh `_retry_` id; outputs overwrite the same GCS paths). Before, a + vanished VM surfaced after ~7 min as a non-resubmitted "lost contact" failure. +- **Idle-stop vs. an encode in flight**: the idle function's `/health active_jobs` + counts only RUNNING jobs and `last_activity_at` used to be written only when a + VM was started, so with a 5-min window a long encode's VM could be stopped in + the seconds between the job finishing and the backend's next poll. The poll + loop now refreshes `last_activity_at` at most once a minute while waiting. + (A VM seen in STAGING/PROVISIONING after accepting a job also counts as lost.) +- **Stale fallback entries** (a removed VM still listed in an instance's cached + `ENCODING_WORKER_FALLBACK_VMS`) are now skipped (`NOT_FOUND` start error) + instead of aborting the whole failover. +- `SPEED_RANK` is keyed by machine *family* now, so the ranking survives vCPU + resizes. +- 16 vCPU: final encodes/renders will be slower (x264 kept ~66-78% of 32 vCPUs + busy). RAM is ~30-32 GB — one 4K encode peaks ~18 GB and heavy jobs are + serialized (`ENCODING_HEAVY_CONCURRENCY=1`), so no OOM risk. + +## Dropbox lossless (item 7) — why only the MP4 + +No automated consumer reads lossless files from Dropbox (YouTube reads the local / +GCS MKV; GDrive, the NOMAD master mirror, kjbox and the AWS backup use GCS or +lossy files). But the lossless **MKV** is promised *in the Dropbox folder* by the +customer completion email (`backend/services/template_service.py`, +`backend/translations/*.json` `fileList`) and by the live Fiverr auto-reply bot +(`fiverrbot/fiverrbot/triage/prompts.py` "What's delivered"), and B2B tenant +VocalStar expects lossless. So the default skips only the lossless 4K MP4 (not +promised anywhere in Dropbox) and only for Nomad's own folders. To also skip the +MKV: update those two texts, then set +`DROPBOX_SKIP_OUTPUT_SUFFIXES=" (Final Karaoke Lossless 4k).mp4| (Final Karaoke Lossless 4k).mkv"` +on the `video-encoding-job` (Pulumi `infrastructure/modules/cloud_run.py`) or +change the default in `backend/config.py`. + +## Open risks + +- Spot capacity for c4d in us-central1-c can disappear; then all work lands on + the single on-demand c2d fallback (slower, and a single lane). +- Two consecutive preemptions of the same encode exhaust `ENCODING_RESUBMIT_MAX=2`. +- Skipped CI worker deploys mean `config/encoding-worker.primary_version` can lag + the backend version; VMs still boot the latest wheel from GCS. diff --git a/frontend/components/lyrics-review/LyricsAnalyzer.tsx b/frontend/components/lyrics-review/LyricsAnalyzer.tsx index e9bbd9c8..6af09fa7 100644 --- a/frontend/components/lyrics-review/LyricsAnalyzer.tsx +++ b/frontend/components/lyrics-review/LyricsAnalyzer.tsx @@ -27,7 +27,8 @@ import { } from '@/lib/lyrics-review/types' import type { EditLog, EditLogEntry, EditFeedbackReason } from '@/lib/lyrics-review/types' import type { InstrumentalSelectionType, LyricsReviewApiClient } from '@/lib/api' -import { lyricsReviewApi, warmupEncodingWorker, heartbeatEncodingWorker, getAccessToken } from '@/lib/api' +import { lyricsReviewApi, getAccessToken } from '@/lib/api' +import { useEncodingWorkerKeepAlive } from '@/lib/lyrics-review/hooks/useEncodingWorkerKeepAlive' import { cn } from '@/lib/utils' import ReferenceView from './ReferenceView' import TranscriptionView from './TranscriptionView' @@ -492,25 +493,12 @@ export default function LyricsAnalyzer({ setSessionRestoreOpen(true) }, [sessionClient]) - // Warm up encoding worker VM when lyrics review page loads. - // Only in cloud mode (jobId present) — the endpoint is review-auth scoped to - // the job, and there's no cloud encoding VM to warm in local mode. - useEffect(() => { - if (!isReadOnly && jobId) { - warmupEncodingWorker(jobId) - } - }, []) // eslint-disable-line react-hooks/exhaustive-deps - - // Debounced heartbeat — keeps VM alive during active editing - const lastHeartbeat = useRef(0) - const sendHeartbeat = useCallback(() => { - if (!jobId) return - const now = Date.now() - if (now - lastHeartbeat.current > 60_000) { - lastHeartbeat.current = now - heartbeatEncodingWorker(jobId) - } - }, [jobId]) + // Keep the encoding worker VM warm while reviewing: warmup on load, heartbeat + // on edits AND periodically while the tab is visible and in use, so the 5-min + // backend idle shutdown doesn't stop it under an active reviewer. Only in cloud + // mode (jobId present) — the endpoints are review-auth scoped to the job, and + // there's no cloud encoding VM to warm in local mode. + const { sendHeartbeat } = useEncodingWorkerKeepAlive(jobId, !isReadOnly) // Heartbeat the encoding worker as the reviewer works. Crash recovery / // restore now flows entirely through review sessions (server in cloud mode, diff --git a/frontend/lib/lyrics-review/hooks/__tests__/useEncodingWorkerKeepAlive.test.tsx b/frontend/lib/lyrics-review/hooks/__tests__/useEncodingWorkerKeepAlive.test.tsx new file mode 100644 index 00000000..90dda7f7 --- /dev/null +++ b/frontend/lib/lyrics-review/hooks/__tests__/useEncodingWorkerKeepAlive.test.tsx @@ -0,0 +1,114 @@ +import { renderHook, act } from '@testing-library/react' +import { + useEncodingWorkerKeepAlive, + REVIEW_HEARTBEAT_INTERVAL_MS, + REVIEW_ACTIVE_WINDOW_MS, + HEARTBEAT_MIN_GAP_MS, +} from '../useEncodingWorkerKeepAlive' +import { warmupEncodingWorker, heartbeatEncodingWorker } from '@/lib/api' + +jest.mock('@/lib/api', () => ({ + warmupEncodingWorker: jest.fn(), + heartbeatEncodingWorker: jest.fn(), +})) + +const warmup = warmupEncodingWorker as jest.Mock +const heartbeat = heartbeatEncodingWorker as jest.Mock + +function setVisibility(state: 'visible' | 'hidden') { + Object.defineProperty(document, 'visibilityState', { configurable: true, get: () => state }) + document.dispatchEvent(new Event('visibilitychange')) +} + +describe('useEncodingWorkerKeepAlive', () => { + beforeEach(() => { + jest.useFakeTimers() + warmup.mockClear() + heartbeat.mockClear() + Object.defineProperty(document, 'visibilityState', { configurable: true, get: () => 'visible' }) + }) + + afterEach(() => { + jest.useRealTimers() + }) + + it('heartbeat interval stays well under the 5-minute backend idle timeout', () => { + expect(REVIEW_HEARTBEAT_INTERVAL_MS * 2).toBeLessThanOrEqual(5 * 60_000) + }) + + it('warms up once on mount', () => { + renderHook(() => useEncodingWorkerKeepAlive('job-1', true)) + expect(warmup).toHaveBeenCalledTimes(1) + expect(warmup).toHaveBeenCalledWith('job-1') + }) + + it('does nothing when disabled (read-only) or without a job id', () => { + renderHook(() => useEncodingWorkerKeepAlive('job-1', false)) + renderHook(() => useEncodingWorkerKeepAlive(undefined, true)) + act(() => { + jest.advanceTimersByTime(REVIEW_HEARTBEAT_INTERVAL_MS * 3) + }) + expect(warmup).not.toHaveBeenCalled() + expect(heartbeat).not.toHaveBeenCalled() + }) + + it('heartbeats periodically while visible even without edits', () => { + renderHook(() => useEncodingWorkerKeepAlive('job-1', true)) + act(() => { + jest.advanceTimersByTime(REVIEW_HEARTBEAT_INTERVAL_MS * 3) + }) + expect(heartbeat).toHaveBeenCalledTimes(3) + }) + + it('skips periodic heartbeats while the tab is hidden', () => { + renderHook(() => useEncodingWorkerKeepAlive('job-1', true)) + act(() => { + setVisibility('hidden') + jest.advanceTimersByTime(REVIEW_HEARTBEAT_INTERVAL_MS * 3) + }) + expect(heartbeat).not.toHaveBeenCalled() + }) + + it('re-warms the worker when the tab becomes visible again', () => { + renderHook(() => useEncodingWorkerKeepAlive('job-1', true)) + expect(warmup).toHaveBeenCalledTimes(1) + act(() => { + setVisibility('hidden') + setVisibility('visible') + }) + expect(warmup).toHaveBeenCalledTimes(2) + }) + + it('stops keeping the VM alive after the reviewer goes inactive', () => { + renderHook(() => useEncodingWorkerKeepAlive('job-1', true)) + act(() => { + jest.advanceTimersByTime(REVIEW_ACTIVE_WINDOW_MS + REVIEW_HEARTBEAT_INTERVAL_MS * 3) + }) + const callsWhileIdle = heartbeat.mock.calls.length + act(() => { + jest.advanceTimersByTime(REVIEW_HEARTBEAT_INTERVAL_MS * 3) + }) + expect(heartbeat.mock.calls.length).toBe(callsWhileIdle) + + // Any interaction resumes the keep-alive. + act(() => { + window.dispatchEvent(new Event('pointerdown')) + jest.advanceTimersByTime(REVIEW_HEARTBEAT_INTERVAL_MS) + }) + expect(heartbeat.mock.calls.length).toBe(callsWhileIdle + 1) + }) + + it('debounces edit-triggered heartbeats', () => { + const { result } = renderHook(() => useEncodingWorkerKeepAlive('job-1', true)) + act(() => { + result.current.sendHeartbeat() + result.current.sendHeartbeat() + }) + expect(heartbeat).toHaveBeenCalledTimes(1) + act(() => { + jest.advanceTimersByTime(HEARTBEAT_MIN_GAP_MS + 1) + result.current.sendHeartbeat() + }) + expect(heartbeat).toHaveBeenCalledTimes(2) + }) +}) diff --git a/frontend/lib/lyrics-review/hooks/useEncodingWorkerKeepAlive.ts b/frontend/lib/lyrics-review/hooks/useEncodingWorkerKeepAlive.ts new file mode 100644 index 00000000..e33926d1 --- /dev/null +++ b/frontend/lib/lyrics-review/hooks/useEncodingWorkerKeepAlive.ts @@ -0,0 +1,85 @@ +import { useCallback, useEffect, useRef } from 'react' +import { warmupEncodingWorker, heartbeatEncodingWorker } from '@/lib/api' + +/** + * Keeps the encoding worker VM warm while a reviewer is on the lyrics-review page, + * so the preview video and the final render don't pay a VM cold start. + * + * The backend idle-shutdown stops the serving VM once `last_activity_at` is older + * than IDLE_TIMEOUT_MINUTES (5 min since 2026-09-26, checked every 2 min — see + * infrastructure/config.py EncodingWorkerConfig). Previously the only heartbeat + * fired on lyric *edits*, so a reviewer reading or listening without editing would + * let the VM stop. Now: + * - warmup once on mount (starts the VM if stopped), + * - heartbeat on edits (debounced to one per HEARTBEAT_MIN_GAP_MS), + * - heartbeat every REVIEW_HEARTBEAT_INTERVAL_MS while the tab is visible AND + * the reviewer interacted within REVIEW_ACTIVE_WINDOW_MS (an abandoned tab + * stops keeping the VM alive), + * - warmup again when the tab becomes visible after being hidden (the VM may + * have idled out meanwhile). + */ + +// Must stay comfortably below the backend idle timeout (5 min) — asserted by +// infrastructure/test_encoding_worker_config.py::test_idle_shutdown_is_fast_but_outlives_review_heartbeat. +export const REVIEW_HEARTBEAT_INTERVAL_MS = 2 * 60_000 +export const HEARTBEAT_MIN_GAP_MS = 60_000 +// Keep-alive stops this long after the last interaction (≈ the old 15-min idle window). +export const REVIEW_ACTIVE_WINDOW_MS = 15 * 60_000 + +const INTERACTION_EVENTS = ['pointerdown', 'keydown', 'wheel', 'touchstart'] as const + +export function useEncodingWorkerKeepAlive(jobId: string | null | undefined, enabled: boolean) { + const lastHeartbeat = useRef(0) + const lastInteraction = useRef(Date.now()) + const active = enabled && !!jobId + + const sendHeartbeat = useCallback(() => { + if (!active || !jobId) return + lastInteraction.current = Date.now() + const now = Date.now() + if (now - lastHeartbeat.current > HEARTBEAT_MIN_GAP_MS) { + lastHeartbeat.current = now + heartbeatEncodingWorker(jobId) + } + }, [active, jobId]) + + // Warm up once when the review page loads (cloud mode only). + useEffect(() => { + if (active && jobId) warmupEncodingWorker(jobId) + }, []) // eslint-disable-line react-hooks/exhaustive-deps + + useEffect(() => { + if (!active || !jobId) return + + const markInteraction = () => { + lastInteraction.current = Date.now() + } + INTERACTION_EVENTS.forEach((evt) => window.addEventListener(evt, markInteraction, { passive: true })) + + const tick = () => { + if (document.visibilityState !== 'visible') return + if (Date.now() - lastInteraction.current > REVIEW_ACTIVE_WINDOW_MS) return + lastHeartbeat.current = Date.now() + heartbeatEncodingWorker(jobId) + } + const intervalId = setInterval(tick, REVIEW_HEARTBEAT_INTERVAL_MS) + + const onVisibilityChange = () => { + if (document.visibilityState !== 'visible') return + lastInteraction.current = Date.now() + lastHeartbeat.current = Date.now() + // Starts the VM if it idled out while the tab was hidden; otherwise just + // refreshes last_activity_at. + warmupEncodingWorker(jobId) + } + document.addEventListener('visibilitychange', onVisibilityChange) + + return () => { + INTERACTION_EVENTS.forEach((evt) => window.removeEventListener(evt, markInteraction)) + clearInterval(intervalId) + document.removeEventListener('visibilitychange', onVisibilityChange) + } + }, [active, jobId]) + + return { sendHeartbeat } +} diff --git a/infrastructure/__main__.py b/infrastructure/__main__.py index 8e73ca68..d7583056 100644 --- a/infrastructure/__main__.py +++ b/infrastructure/__main__.py @@ -35,6 +35,7 @@ from config import ( NUM_GITHUB_RUNNERS, MachineTypes, DiskSizes, + EncodingWorkerConfig, ) from modules import database, storage as storage_module, artifact_registry, secrets from modules import cloud_tasks, cloud_run, monitoring, runner_manager @@ -663,11 +664,10 @@ encoding_worker_instances = encoding_worker_vm.create_encoding_worker_vms( encoding_worker_ips, encoding_worker_sa ) -# Capacity-fallback Encoding Worker VMs — diversified across alternate zones -# AND an alternate machine family (c4d in us-central1-a/-b + n2 in -c/-f) so a -# region-wide c4d-highcpu-32 ZONE_RESOURCE_POOL_EXHAUSTED can't take out every -# lane (incident 2026-08-12). Provisioned stopped; only started by the -# application when the primary rejects a start. Cost when idle: ~$10/mo each. +# Capacity-fallback Encoding Worker VM — one ON-DEMAND VM in a different zone AND +# machine family from the c4d Spot pair, so a Spot/c4d stockout can't take out +# every lane (incident 2026-08-12). Provisioned stopped; only started by the +# application when the primary rejects a start. Cost when idle: the boot disk. # See EncodingWorkerConfig.FALLBACKS for the fleet definition. encoding_worker_fallback_ips = encoding_worker_vm.create_encoding_worker_fallback_ips() encoding_worker_fallback_instances = encoding_worker_vm.create_encoding_worker_fallback_vms( @@ -679,7 +679,7 @@ encoding_worker_firewall = encoding_worker_vm.create_encoding_worker_firewall() # Grant backend SA permission to start encoding worker VMs backend_compute_perms = worker_sas.grant_backend_compute_permissions(backend_service_account) -# Encoding Worker Idle Shutdown (auto-stop after 15 min idle) +# Encoding Worker Idle Shutdown (auto-stop after 5 min idle, checked every 2 min) encoding_worker_idle_resources = encoding_worker_manager.create_idle_shutdown_resources() # GitHub Runners (CI/CD self-hosted runners) @@ -789,17 +789,14 @@ pulumi.export("encoding_worker_b_ip", encoding_worker_ips[1].address) pulumi.export("encoding_worker_a_url", encoding_worker_ips[0].address.apply(lambda ip: f"http://{ip}:8080")) pulumi.export("encoding_worker_b_url", encoding_worker_ips[1].address.apply(lambda ip: f"http://{ip}:8080")) -# Capacity-fallback VM exports (us-central1-a, -b) -pulumi.export("encoding_worker_fallback_a_ip", encoding_worker_fallback_ips[0].address) -pulumi.export("encoding_worker_fallback_b_ip", encoding_worker_fallback_ips[1].address) -pulumi.export( - "encoding_worker_fallback_a_url", - encoding_worker_fallback_ips[0].address.apply(lambda ip: f"http://{ip}:8080"), -) -pulumi.export( - "encoding_worker_fallback_b_url", - encoding_worker_fallback_ips[1].address.apply(lambda ip: f"http://{ip}:8080"), -) +# Capacity-fallback VM exports (one per EncodingWorkerConfig.FALLBACKS entry, +# keyed by suffix, e.g. encoding_worker_fallback_c2df_ip). +for _fb, _fb_ip in zip(EncodingWorkerConfig.FALLBACKS, encoding_worker_fallback_ips): + pulumi.export(f"encoding_worker_fallback_{_fb['suffix']}_ip", _fb_ip.address) + pulumi.export( + f"encoding_worker_fallback_{_fb['suffix']}_url", + _fb_ip.address.apply(lambda ip: f"http://{ip}:8080"), + ) pulumi.export("encoding_worker_service_account", encoding_worker_sa.email) pulumi.export("encoding_worker_a_name", encoding_worker_instances[0].name) pulumi.export("encoding_worker_b_name", encoding_worker_instances[1].name) diff --git a/infrastructure/compute/encoding_worker_vm.py b/infrastructure/compute/encoding_worker_vm.py index 41e9926d..f41c5ae6 100644 --- a/infrastructure/compute/encoding_worker_vm.py +++ b/infrastructure/compute/encoding_worker_vm.py @@ -3,7 +3,9 @@ Encoding Worker VM resources — blue-green deployment pair. Manages two identical VMs (a/b) for zero-downtime deployments. Only one is active (primary) at a time; the other is stopped (secondary). -Both auto-shutdown when idle to minimize cost. +Both auto-shutdown when idle to minimize cost, and both run on SPOT capacity +(preempted → STOPPED, restarted on demand; lost encodes are resubmitted). One +on-demand fallback VM in a different family/zone absorbs Spot stockouts. See docs/superpowers/specs/2026-03-24-blue-green-encoding-worker-design.md """ @@ -80,18 +82,42 @@ def create_encoding_worker_vms( advanced_machine_features=compute.InstanceAdvancedMachineFeaturesArgs( threads_per_core=2, ), + scheduling=_primary_pair_scheduling(), ) vms.append(vm) return vms +def _primary_pair_scheduling() -> compute.InstanceSchedulingArgs: + """Scheduling for the a/b pair: Spot with STOP-on-preemption. + + STOP (not DELETE) keeps the VM, its disk and static IP, so a preempted + worker is just a TERMINATED instance that the backend's ensure-running path + restarts like any idle-stopped worker. automatic_restart must be False and + on_host_maintenance TERMINATE for Spot. + """ + if not EncodingWorkerConfig.PRIMARY_PAIR_SPOT: + return compute.InstanceSchedulingArgs( + provisioning_model="STANDARD", + preemptible=False, + automatic_restart=True, + on_host_maintenance="MIGRATE", + ) + return compute.InstanceSchedulingArgs( + provisioning_model="SPOT", + preemptible=True, + automatic_restart=False, + on_host_maintenance="TERMINATE", + instance_termination_action="STOP", + ) + + def create_encoding_worker_fallback_ips() -> list[compute.Address]: """Create static IPs for the capacity-fallback VMs. - These IPs back the fallback fleet that absorbs a primary-family stockout — - both alternate zones AND an alternate machine family (n2-highcpu-32) so a - region-wide c4d-highcpu-32 ZONE_RESOURCE_POOL_EXHAUSTED can't take out every - lane. One IP per entry in EncodingWorkerConfig.FALLBACKS (order-aligned). + These IPs back the on-demand fallback that absorbs a Spot/c4d stockout of + the primary pair (alternate zone AND machine family). One IP per entry in + EncodingWorkerConfig.FALLBACKS (order-aligned). """ ips = [] for fb in EncodingWorkerConfig.FALLBACKS: @@ -115,14 +141,13 @@ def create_encoding_worker_fallback_vms( ) -> list[compute.Instance]: """Create capacity-fallback encoding worker VMs. - Provisioned stopped — only started by the application when the primary zone - rejects starts with ZONE_RESOURCE_POOL_EXHAUSTED. The fleet diversifies - across both alternate zones AND an alternate machine family - (n2-highcpu-32) so a region-wide c4d-highcpu-32 stockout cannot exhaust - every lane at once (incident 2026-08-12). Each entry's machine_type and - disk_type come from EncodingWorkerConfig.FALLBACKS — n2 uses pd-balanced - because it does not support hyperdisk-balanced. Cost when stopped is just - the 50 GB boot disk (~$4-5/mo each). + Provisioned stopped — only started by the application when the Spot primary + pair can't start (ZONE_RESOURCE_POOL_EXHAUSTED / Spot stockout). On-demand + (default STANDARD scheduling — never preempted), in a different machine + family and zone from the c4d pair (incident 2026-08-12). Each entry's + machine_type and disk_type come from EncodingWorkerConfig.FALLBACKS — c2d/n2 + use pd-balanced because they don't support hyperdisk-balanced. Cost when + stopped is just the 50 GB boot disk. """ startup_script = read_script("encoding_worker.sh") custom_image = f"projects/{PROJECT_ID}/global/images/family/encoding-worker" diff --git a/infrastructure/config.py b/infrastructure/config.py index 77c2b9dd..85012dd8 100644 --- a/infrastructure/config.py +++ b/infrastructure/config.py @@ -33,7 +33,7 @@ def get_project_number() -> str: REGION = "us-central1" ZONE = f"{REGION}-a" -# Encoding worker zone - uses us-central1-c due to c4d-highcpu-32 availability +# Encoding worker zone - uses us-central1-c due to c4d availability # (us-central1-a and us-central1-b often lack capacity for high-end C4D instances) ENCODING_WORKER_ZONE = f"{REGION}-c" @@ -47,23 +47,19 @@ class MachineTypes: GITHUB_RUNNER = "e2-standard-4" # 4 vCPU, 16GB RAM GITHUB_BUILD_RUNNER = "e2-standard-8" # 8 vCPU, 32GB RAM - dedicated Docker build runner GITHUB_GPU_RUNNER = "n1-standard-4" # 4 vCPU, 15GB RAM - GPU runners need N1 series - ENCODING_WORKER = "c4d-highcpu-32" # 32 vCPU, AMD EPYC 9B45 Turin - 4.92x faster than c4-standard-8 - # Capacity-fallback machine family, deliberately DIFFERENT from ENCODING_WORKER. - # c4d-highcpu-32 suffered a region-wide ZONE_RESOURCE_POOL_EXHAUSTED stockout across - # us-central1-a/-b/-c simultaneously (2026-08-12) which took out every same-family lane. - # n2-highcpu-32 (Intel Cascade/Ice Lake) draws from a much deeper, independent pool, so a - # c4d shortage cannot exhaust it. n2 does NOT support hyperdisk-balanced → uses pd-balanced. - ENCODING_WORKER_ALT = "n2-highcpu-32" # 32 vCPU, Intel Cascade/Ice Lake - deep-capacity fallback pool - # Broadened ranked pool (2026-08-15) — ≥5 x86_64, 32-vCPU, ≥32 GB highcpu types - # across 4 distinct silicon lineages, so a stockout of the newest-gen cohort - # (c4d/c4, the crunch) cannot exhaust every lane. Encode-speed ranking lives in - # backend/services/encoding_worker_preference.py::SPEED_RANK. Disk-type rule: - # next-gen Titanium families (c4/c4d/n4/n4d) support hyperdisk-balanced ONLY; - # older families (c2d/n2/n2d) use pd-balanced. Getting this wrong = pulumi error. - ENCODING_WORKER_C4 = "c4-highcpu-32" # 32 vCPU / 64 GB, Intel Emerald Rapids (hyperdisk-only) - ENCODING_WORKER_N4D = "n4d-highcpu-32" # 32 vCPU / 64 GB, AMD Titanium (hyperdisk-only) - ENCODING_WORKER_C2D = "c2d-highcpu-32" # 32 vCPU / 64 GB, AMD Milan (Zen3), deep pool (pd-balanced) - ENCODING_WORKER_N2D = "n2d-highcpu-32" # 32 vCPU / 32 GB, AMD Rome/Milan, deep pool (pd-balanced) + # Encoding workers (2026-09-26 cost cut, docs/archive/2026-09-26-encoding-cost-cuts.md): + # 16 vCPU instead of 32 — an approved speed-for-cost trade (finalization x264 + # only kept ~66-78% of 32 vCPUs busy; ffmpeg auto-threads adapt to 16). The + # worker serializes heavy jobs (ENCODING_HEAVY_CONCURRENCY=1), and highcpu-16 + # still has ~30-32 GB RAM — enough for one 4K encode (~18 GB peak). + ENCODING_WORKER = "c4d-highcpu-16" # 16 vCPU / 30 GB, AMD EPYC Turin — Spot blue-green pair + # The ONE on-demand capacity fallback. Deliberately a DIFFERENT machine family + # and zone from the c4d Spot pair: c4d suffered a region-wide + # ZONE_RESOURCE_POOL_EXHAUSTED stockout (2026-08-12), and Spot capacity can + # vanish at any time. c2d (AMD Milan) draws from a deep, mature pool and is + # on-demand, so it is not preemptible. c2d does NOT support hyperdisk → + # pd-balanced boot disk. + ENCODING_WORKER_FALLBACK = "c2d-highcpu-16" # 16 vCPU / 32 GB, AMD Milan (Zen3), on-demand FLACFETCH = "e2-small" # 0.5 vCPU, 2GB RAM @@ -308,56 +304,38 @@ class EncodingWorkerConfig: """Configuration for blue-green encoding worker VMs.""" VM_NAMES = ["encoding-worker-a", "encoding-worker-b"] IP_NAMES = ["encoding-worker-ip-a", "encoding-worker-ip-b"] - IDLE_CHECK_SCHEDULE = "*/5 * * * *" # Every 5 minutes - IDLE_TIMEOUT_MINUTES = 15 + # Stop idle workers fast: VMs bill per second while RUNNING, and with only + # ~5-20 jobs/day most of the old 15-min tail was paid idle time. The function + # still NEVER stops a VM with active jobs (fail-safe on unconfirmed /health). + IDLE_CHECK_SCHEDULE = "*/2 * * * *" # Every 2 minutes + IDLE_TIMEOUT_MINUTES = 5 FUNCTION_NAME = "encoding-worker-idle-shutdown" FUNCTION_MEMORY = "512M" # Increased from 256M — OOM with gRPC/Firestore/Compute client libs FUNCTION_TIMEOUT = 120 # 2 minutes - # Capacity-resilience fallback fleet. Each VM is provisioned STOPPED in an - # alternate zone / machine family and is started on demand only when the - # primary zone rejects a start with ZONE_RESOURCE_POOL_EXHAUSTED. Cost when - # stopped is just the 50 GB boot disk (~$4-5/mo each). + # The a/b blue-green pair runs on SPOT capacity (~60-70% cheaper). A preempted + # Spot VM is STOPPED (not deleted) — the backend restarts it on the next + # request, and an encode lost to preemption is resubmitted automatically + # (encoding_service.wait_for_completion → EncodingJobLostError). + PRIMARY_PAIR_SPOT = True + + # Capacity-resilience fallback. Provisioned STOPPED and started on demand only + # when the Spot pair can't start (ZONE_RESOURCE_POOL_EXHAUSTED / Spot + # stockout). ON-DEMAND (never preempted) and a different family + zone from the + # c4d pair so one shortage can't take out both lanes. Cost when stopped is just + # the 50 GB boot disk. # - # Machine-family diversity is DELIBERATE. The primary pair and the two c4d - # fallbacks are all c4d-highcpu-32, so a region-wide c4d stockout (observed - # 2026-08-12 across us-central1-a/-b/-c at once) exhausts every lane - # simultaneously and forces slow local encoding (→ 524 on preview, parked - # renders). The n2-highcpu-32 fallbacks draw from an independent, much deeper - # pool so a c4d shortage cannot take them out. n2 does not support - # hyperdisk-balanced, hence pd-balanced boot disks on those entries. + # Was 8 fallbacks across 6 families (2026-08-15) — cut to 1 on 2026-09-26 + # (fleet 10 → 3 VMs) to save the idle disk/IP spend; c2df was kept because it + # is a deep-pool, pd-balanced (cheaper disk) family in a separate zone. # # Each entry: name/IP suffix, zone suffix ({REGION}-{zone_suffix}), - # machine_type, boot disk_type. ORDER MATTERS — IPs and VMs are zipped by - # position. (Candidate PRIORITY is no longer positional: it is decided at - # runtime/deploy by the shared speed-rank + cooldown in - # backend/services/encoding_worker_preference.py.) Keep the existing a/b/n2c/n2f - # entries first and byte-identical so Pulumi does not recreate those VMs; the - # broadened-pool entries are APPENDED so the change is purely additive - # (verify `pulumi preview` shows only new IPs+VMs, zero replace). - # - # Zone spread: one machine type per zone where possible so a single - # (type × zone) stockout can't correlate across the pool. c4d primary lives in - # zone c (the primary pair), so the c4 fallback goes to a, n4d to b, c2d to f, - # n2d to a — 4 lineages across 4 zones. + # machine_type, boot disk_type. IPs and VMs are zipped by position. The + # runtime/deploy candidate PRIORITY is decided by the shared speed-rank + + # cooldown in backend/services/encoding_worker_preference.py. FALLBACKS = [ - {"suffix": "a", "zone_suffix": "a", - "machine_type": MachineTypes.ENCODING_WORKER, "disk_type": "hyperdisk-balanced"}, - {"suffix": "b", "zone_suffix": "b", - "machine_type": MachineTypes.ENCODING_WORKER, "disk_type": "hyperdisk-balanced"}, - {"suffix": "n2c", "zone_suffix": "c", - "machine_type": MachineTypes.ENCODING_WORKER_ALT, "disk_type": "pd-balanced"}, - {"suffix": "n2f", "zone_suffix": "f", - "machine_type": MachineTypes.ENCODING_WORKER_ALT, "disk_type": "pd-balanced"}, - # --- Broadened pool (2026-08-15), appended (additive-only) --- - {"suffix": "c4a", "zone_suffix": "a", - "machine_type": MachineTypes.ENCODING_WORKER_C4, "disk_type": "hyperdisk-balanced"}, - {"suffix": "n4db", "zone_suffix": "b", - "machine_type": MachineTypes.ENCODING_WORKER_N4D, "disk_type": "hyperdisk-balanced"}, {"suffix": "c2df", "zone_suffix": "f", - "machine_type": MachineTypes.ENCODING_WORKER_C2D, "disk_type": "pd-balanced"}, - {"suffix": "n2da", "zone_suffix": "a", - "machine_type": MachineTypes.ENCODING_WORKER_N2D, "disk_type": "pd-balanced"}, + "machine_type": MachineTypes.ENCODING_WORKER_FALLBACK, "disk_type": "pd-balanced"}, ] # Derived name lists (kept for readability / any external reference). diff --git a/infrastructure/encoding-worker/benchmark_types.py b/infrastructure/encoding-worker/benchmark_types.py index 9440545d..942f55d2 100644 --- a/infrastructure/encoding-worker/benchmark_types.py +++ b/infrastructure/encoding-worker/benchmark_types.py @@ -84,7 +84,8 @@ def _inventory(only_types: Optional[set]) -> List[Dict[str, Any]]: if data.get("primary_vm"): workers.append({ "vm": data["primary_vm"], "zone": DEFAULT_C4D_ZONE, - "ip": data.get("primary_ip"), "machine_type": "c4d-highcpu-32", + # Keep in sync with encoding_worker_preference.PRIMARY_MACHINE_TYPE. + "ip": data.get("primary_ip"), "machine_type": "c4d-highcpu-16", }) except Exception as e: # noqa: BLE001 print(f"WARN: could not read primary from Firestore ({e}); benchmarking fallbacks only") @@ -238,11 +239,19 @@ def benchmark(args) -> Dict[str, float]: def _suggest_ranks(medians: Dict[str, float]) -> Dict[str, int]: - """Map medians → SPEED_RANK ints (fastest = smallest), spaced by 10.""" + """Map medians → SPEED_RANK ints (fastest = smallest), spaced by 10. + + Keyed by machine FAMILY ("c4d-highcpu-16" → "c4d"), matching + encoding_worker_preference.SPEED_RANK (family-keyed since 2026-09-26). + """ ordered = sorted(medians.items(), key=lambda kv: kv[1]) return {mt: (idx + 1) * 10 for idx, (mt, _) in enumerate(ordered)} +def _family(machine_type: str) -> str: + return machine_type.split("-", 1)[0].lower() + + def main() -> int: p = argparse.ArgumentParser(description=__doc__) p.add_argument("--input-gcs", dest="input_gcs", required=True, @@ -282,7 +291,10 @@ def main() -> int: for mt, secs in sorted(medians.items(), key=lambda kv: kv[1]): print(f" {mt:>16}: {secs:>7.1f}s → suggested SPEED_RANK {ranks[mt]}") - payload = {"medians_seconds": medians, "suggested_speed_rank": ranks} + family_ranks: Dict[str, int] = {} + for mt, rank in sorted(ranks.items(), key=lambda kv: kv[1]): + family_ranks.setdefault(_family(mt), rank) # fastest size wins per family + payload = {"medians_seconds": medians, "suggested_speed_rank": family_ranks} if args.json_out: with open(args.json_out, "w") as f: json.dump(payload, f, indent=2) diff --git a/infrastructure/encoding-worker/deploy_promote.py b/infrastructure/encoding-worker/deploy_promote.py index 2193c281..359b8181 100644 --- a/infrastructure/encoding-worker/deploy_promote.py +++ b/infrastructure/encoding-worker/deploy_promote.py @@ -6,13 +6,14 @@ delegates the *decisions* to the pure functions here so the branching — the pa that's easy to get wrong and impossible to run locally in CI — has real unit tests (`backend/tests/test_deploy_promote.py`). -Background: us-central1 is in a persistent c4d Spot stockout, so the c4d -primary/secondary are usually down and all traffic runs on an n2 fallback -recorded as `active_override_vm`. A naive "start the c4d secondary and swap" -deploy can't start the secondary and never refreshes the serving fallback, so -worker-side changes never reach production. These helpers make the deploy pick a -*fresh* green target (c4d secondary if it starts, else a different n2 fallback) -and then promote that validated green to serving with zero downtime. +Background: the c4d primary/secondary are Spot VMs (since 2026-09-26) and c4d +capacity in us-central1 can run out, in which case traffic runs on the on-demand +fallback recorded as `active_override_vm`. A naive "start the c4d secondary and +swap" deploy can't start the secondary and never refreshes the serving fallback, +so worker-side changes never reach production. These helpers make the deploy pick +a *fresh* green target (c4d secondary if it starts, else a fallback) and then +promote that validated green to serving with zero downtime. CI only runs this when +worker code changed (infrastructure/encoding-worker/worker_code_paths.txt). """ from __future__ import annotations diff --git a/infrastructure/encoding-worker/worker_code_paths.txt b/infrastructure/encoding-worker/worker_code_paths.txt new file mode 100644 index 00000000..af478cd0 --- /dev/null +++ b/infrastructure/encoding-worker/worker_code_paths.txt @@ -0,0 +1,34 @@ +# Paths whose changes require a blue-green redeploy of the GCE encoding worker. +# +# CI (.github/workflows/ci.yml, "Decide encoding-worker deploy") diffs these +# paths between the version the serving worker was last validated at +# (config/encoding-worker.primary_version → git tag v) and HEAD. No +# diff → the fresh-green boot + test encode + swap is skipped (it used to run on +# every push to main). The wheel/startup.sh/version.txt are STILL uploaded to +# GCS on every deploy, so any worker that cold-boots picks up the latest wheel. +# +# The worker process runs backend/services/gce_encoding/main.py and imports +# karaoke_gen (render/encode) plus a few backend modules. +# backend/tests/test_encoding_worker_code_paths.py derives that import closure +# and fails if a module the worker imports is not covered here — so adding a new +# worker import can't silently bypass the gate. +# +# One git pathspec per line; blank lines and # comments are ignored. +karaoke_gen/ +backend/__init__.py +backend/services/__init__.py +backend/models/__init__.py +backend/services/gce_encoding/ +backend/services/local_encoding_service.py +backend/services/local_preview_encoding_service.py +backend/services/theme_service.py +backend/services/storage_service.py +backend/services/langfuse_preloader.py +backend/services/nltk_preloader.py +backend/services/spacy_preloader.py +backend/models/theme.py +backend/utils/ +backend/config.py +infrastructure/encoding-worker/ +infrastructure/packer/ +poetry.lock diff --git a/infrastructure/functions/encoding_worker_idle/main.py b/infrastructure/functions/encoding_worker_idle/main.py index 4866e4a7..c87a51cb 100644 --- a/infrastructure/functions/encoding_worker_idle/main.py +++ b/infrastructure/functions/encoding_worker_idle/main.py @@ -1,7 +1,7 @@ """ Encoding Worker Idle Shutdown Cloud Function. -Triggered by Cloud Scheduler every 5 minutes. Checks all encoding worker +Triggered by Cloud Scheduler every 2 minutes. Checks all encoding worker VMs (primary, secondary, AND multi-zone fallbacks) and stops any that are idle. Without checking the fallbacks, a fallback VM started during a capacity event could be left running indefinitely after the event clears @@ -11,7 +11,7 @@ iterated primary/secondary in the default zone. Idle criteria (all must be true to stop a VM): - No active encoding jobs (from /health endpoint active_jobs) -- No recent user activity (from Firestore last_activity_at > 15 min) +- No recent user activity (from Firestore last_activity_at > IDLE_TIMEOUT_MINUTES) - Only applies to the *currently routed* VM: `active_override_vm` if set, else primary. Secondary and non-active fallbacks always stop on idle so they don't leak after a capacity event. @@ -20,7 +20,8 @@ Idle criteria (all must be true to stop a VM): Environment variables: - GCP_PROJECT: GCP project ID (default: nomadkaraoke) - GCP_ZONE: Default zone for primary/secondary (default: us-central1-c) -- IDLE_TIMEOUT_MINUTES: Minutes of inactivity before shutdown (default: 15) +- IDLE_TIMEOUT_MINUTES: Minutes of inactivity before shutdown (default: 5; + Pulumi sets it from EncodingWorkerConfig.IDLE_TIMEOUT_MINUTES) - ENCODING_WORKER_FALLBACK_VMS: JSON list of {vm, zone, ip} for fallback VMs in alternate zones. Same secret the backend uses for multi-zone failover. Empty/missing → no fallbacks checked. @@ -40,7 +41,7 @@ logger = logging.getLogger(__name__) PROJECT_ID = os.environ.get("GCP_PROJECT", "nomadkaraoke") ZONE = os.environ.get("GCP_ZONE", "us-central1-c") -IDLE_TIMEOUT_MINUTES = int(os.environ.get("IDLE_TIMEOUT_MINUTES", "15")) +IDLE_TIMEOUT_MINUTES = int(os.environ.get("IDLE_TIMEOUT_MINUTES", "5")) DEPLOY_STALE_TIMEOUT_MINUTES = 20 ENCODING_WORKER_PORT = 8080 @@ -218,7 +219,7 @@ def idle_shutdown(request): # Fail SAFE: never stop a VM unless /health gives us a *confirmed* # active_jobs == 0. If we can't reach the VM or confirm its job count, - # keep it alive this cycle (the function reruns every 5 min, so a + # keep it alive this cycle (the function reruns every 2 min, so a # genuinely idle VM is reclaimed once /health responds). This prevents # stopping a VM mid-render — the cause of "lost contact with worker" # (incident 2026-06-15). diff --git a/infrastructure/functions/encoding_worker_idle/test_idle_shutdown.py b/infrastructure/functions/encoding_worker_idle/test_idle_shutdown.py index 2bb86c9d..82780084 100644 --- a/infrastructure/functions/encoding_worker_idle/test_idle_shutdown.py +++ b/infrastructure/functions/encoding_worker_idle/test_idle_shutdown.py @@ -107,7 +107,7 @@ class TestIdleShutdown: def test_skips_deploy_in_progress(self, mock_compute, mock_firestore): """Should skip idle check when a deploy is in progress.""" - recent = (datetime.now(timezone.utc) - timedelta(minutes=5)).isoformat() + recent = (datetime.now(timezone.utc) - timedelta(minutes=2)).isoformat() config = _make_config( deploy_in_progress=True, deploy_in_progress_since=recent, @@ -147,7 +147,7 @@ class TestIdleShutdown: def test_recent_activity_keeps_primary_alive(self, mock_compute, mock_firestore): """Recent activity should keep the primary VM alive but stop the secondary.""" - recent = (datetime.now(timezone.utc) - timedelta(minutes=5)).isoformat() + recent = (datetime.now(timezone.utc) - timedelta(minutes=2)).isoformat() config = _make_config(last_activity_at=recent) _setup_firestore(mock_firestore, config) @@ -163,6 +163,33 @@ class TestIdleShutdown: # Only secondary should be stopped assert mock_compute.stop.call_count == 1 + def test_default_idle_timeout_is_five_minutes(self): + """2026-09-26 cost cut: idle VMs stop after 5 min (was 15).""" + assert main.IDLE_TIMEOUT_MINUTES == 5 + + def test_review_heartbeat_within_timeout_keeps_primary_alive(self, mock_compute, mock_firestore): + """The lyrics-review page heartbeats every ~2 min; a heartbeat 4 min ago is + still inside the 5-min window, so the warm primary must NOT be stopped + while someone is reviewing (it's kept warm for the preview + final render).""" + heartbeat = (datetime.now(timezone.utc) - timedelta(minutes=4)).isoformat() + _setup_firestore(mock_firestore, _make_config(last_activity_at=heartbeat)) + mock_compute.get.return_value = _make_instance(status="RUNNING", ip="1.2.3.4") + + with patch("main.check_active_jobs", return_value=0): + response, _ = main.idle_shutdown(MagicMock()) + + assert json.loads(response)["results"]["encoding-worker-a"] == "active_session" + + def test_activity_older_than_timeout_stops_idle_primary(self, mock_compute, mock_firestore): + stale = (datetime.now(timezone.utc) - timedelta(minutes=6)).isoformat() + _setup_firestore(mock_firestore, _make_config(last_activity_at=stale)) + mock_compute.get.return_value = _make_instance(status="RUNNING", ip="1.2.3.4") + + with patch("main.check_active_jobs", return_value=0): + response, _ = main.idle_shutdown(MagicMock()) + + assert json.loads(response)["results"]["encoding-worker-a"] == "stopped" + def test_skips_terminated_vms(self, mock_compute, mock_firestore): """VMs that are already TERMINATED should be skipped.""" config = _make_config() @@ -299,7 +326,7 @@ class TestFallbackVms: def test_idle_fallback_gets_stopped(self, mock_compute, mock_firestore, fallback_env): """A fallback VM that's RUNNING but idle (no jobs, not active) must be stopped. This is the bug the fix addresses — previously fallbacks were never checked.""" - recent = (datetime.now(timezone.utc) - timedelta(minutes=5)).isoformat() + recent = (datetime.now(timezone.utc) - timedelta(minutes=2)).isoformat() config = _make_config(last_activity_at=recent) _setup_firestore(mock_firestore, config) @@ -345,7 +372,7 @@ class TestFallbackVms: {"vm": "encoding-worker-fallback-n2f", "zone": "us-central1-f", "ip": "4.4.4.4"}, ]), ) - recent = (datetime.now(timezone.utc) - timedelta(minutes=5)).isoformat() + recent = (datetime.now(timezone.utc) - timedelta(minutes=2)).isoformat() _setup_firestore(mock_firestore, _make_config(last_activity_at=recent)) # Only the n2-f fallback is RUNNING+idle → it must be stopped in -f. @@ -376,7 +403,7 @@ class TestFallbackVms: """When a capacity-fallback override is set, that fallback is the currently-routed VM and gets the keep-alive treatment (matching what primary normally gets). Primary should still stop on idle.""" - recent = (datetime.now(timezone.utc) - timedelta(minutes=5)).isoformat() + recent = (datetime.now(timezone.utc) - timedelta(minutes=2)).isoformat() config = { **_make_config(last_activity_at=recent), "active_override_vm": "encoding-worker-fallback-a", diff --git a/infrastructure/test_encoding_worker_config.py b/infrastructure/test_encoding_worker_config.py index 54c0dda4..bfa7611d 100644 --- a/infrastructure/test_encoding_worker_config.py +++ b/infrastructure/test_encoding_worker_config.py @@ -10,19 +10,40 @@ Run locally with: `pytest infrastructure/test_encoding_worker_config.py` resource graph via `pulumi preview`). """ -from config import EncodingWorkerConfig, MachineTypes +from config import ENCODING_WORKER_ZONE, EncodingWorkerConfig, MachineTypes -def test_fallback_fleet_has_machine_family_diversity(): - """At least one fallback must use a machine family OTHER than the primary, - otherwise a single-family stockout takes out every lane at once.""" - families = {fb["machine_type"] for fb in EncodingWorkerConfig.FALLBACKS} - assert MachineTypes.ENCODING_WORKER in families - assert any(mt != MachineTypes.ENCODING_WORKER for mt in families), ( - "fallback fleet is single-machine-family — a c4d stockout would exhaust " - "every lane (see incident 2026-08-12)" - ) - assert MachineTypes.ENCODING_WORKER_ALT in families +def test_fallback_is_a_different_machine_family_and_zone_from_primary(): + """The on-demand fallback must not share the Spot pair's family or zone, + otherwise a single c4d / Spot / zone stockout takes out every lane at once + (incident 2026-08-12).""" + assert EncodingWorkerConfig.FALLBACKS, "need at least one capacity fallback" + primary_family = _family(MachineTypes.ENCODING_WORKER) + primary_zone_suffix = ENCODING_WORKER_ZONE.rsplit("-", 1)[1] + for fb in EncodingWorkerConfig.FALLBACKS: + assert _family(fb["machine_type"]) != primary_family, fb + assert fb["zone_suffix"] != primary_zone_suffix, fb + + +def test_fleet_is_cost_cut_shape(): + """2026-09-26 cost cut: Spot a/b pair + exactly one on-demand fallback, all + 16 vCPU. Growing this again should be a deliberate decision (update the + fallback secret + docs), not an accident.""" + assert len(EncodingWorkerConfig.VM_NAMES) == 2 + assert len(EncodingWorkerConfig.FALLBACKS) == 1 + assert EncodingWorkerConfig.PRIMARY_PAIR_SPOT is True + for mt in [MachineTypes.ENCODING_WORKER] + [fb["machine_type"] for fb in EncodingWorkerConfig.FALLBACKS]: + assert mt.endswith("-16"), mt + + +def test_idle_shutdown_is_fast_but_outlives_review_heartbeat(): + """Idle VMs stop after 5 min, checked every 2 min. The lyrics-review page + heartbeats every 2 min (frontend REVIEW_HEARTBEAT_INTERVAL_MS) so an active + review keeps the warm VM alive — the timeout must leave margin over it.""" + assert EncodingWorkerConfig.IDLE_TIMEOUT_MINUTES == 5 + assert EncodingWorkerConfig.IDLE_CHECK_SCHEDULE == "*/2 * * * *" + review_heartbeat_minutes = 2 + assert EncodingWorkerConfig.IDLE_TIMEOUT_MINUTES >= 2 * review_heartbeat_minutes def test_n2_fallbacks_use_pd_balanced_disk(): @@ -65,18 +86,6 @@ def test_every_fallback_disk_type_matches_its_family_capability(): assert fb["disk_type"] == _FAMILY_DISK_TYPE[fam], fb -def test_pool_has_at_least_five_types_across_multiple_lineages(): - """The broadened pool must be ≥5 distinct machine types spanning ≥3 lineages - (AMD Turin / Intel Emerald / AMD Milan / Intel Cascade / AMD Rome) so a - newest-gen stockout can't exhaust every lane.""" - types = {fb["machine_type"] for fb in EncodingWorkerConfig.FALLBACKS} - # primary (c4d) + the fallback types. - types.add(MachineTypes.ENCODING_WORKER) - assert len(types) >= 5, f"pool has only {len(types)} types: {types}" - families = {_family(mt) for mt in types} - assert len(families) >= 3, f"pool spans only {families}" - - def test_zone_spread_avoids_same_type_same_zone(): """No machine type should sit twice in the same zone (correlated stockout).""" seen = set() @@ -102,18 +111,6 @@ def test_fallback_names_and_ips_are_unique_and_aligned(): assert len(set(EncodingWorkerConfig.FALLBACK_IP_NAMES)) == len(fb) -def test_original_c4d_fallbacks_unchanged(): - """The first two entries must remain the original c4d a/b fallbacks - (byte-identical) so `pulumi up` does NOT recreate the existing VMs — - recreating a c4d VM would need the very capacity we're trying to avoid - depending on.""" - a, b = EncodingWorkerConfig.FALLBACKS[0], EncodingWorkerConfig.FALLBACKS[1] - assert a == {"suffix": "a", "zone_suffix": "a", - "machine_type": MachineTypes.ENCODING_WORKER, "disk_type": "hyperdisk-balanced"} - assert b == {"suffix": "b", "zone_suffix": "b", - "machine_type": MachineTypes.ENCODING_WORKER, "disk_type": "hyperdisk-balanced"} - - def test_worker_boot_disk_fits_packer_image_size(): """GCE cannot create a boot disk smaller than its source image's disk size. The Packer image's `disk_size` must stay <= DiskSizes.ENCODING_WORKER, or diff --git a/pyproject.toml b/pyproject.toml index 38962937..29d065ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.244.1" +version = "0.245.0" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT"