diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9167756b..1bd65e8b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,7 +87,7 @@ jobs: # ============================================ runner-manager-tests: - name: "Infra - runner_manager function tests" + name: "Infra - Cloud Function tests" runs-on: ubuntu-latest needs: changes if: github.ref != 'refs/heads/main' && (needs.changes.outputs.infra == 'true' || needs.changes.outputs.deps == 'true') @@ -100,6 +100,9 @@ jobs: # google.cloud namespace package is genuinely installed. - run: pip install pytest -r infrastructure/functions/runner_manager/requirements.txt - run: pytest infrastructure/functions/runner_manager -q -p no:cacheprovider -o addopts="" + - run: pip install -r infrastructure/functions/encoding_worker_idle/requirements.txt + # Separate invocation: both functions have a top-level `main` module. + - run: pytest infrastructure/functions/encoding_worker_idle -q -p no:cacheprovider -o addopts="" frontend-lint: name: "Frontend - Lint (Rules of Hooks)" @@ -1399,6 +1402,11 @@ jobs: echo "Uploading startup.sh..." gsutil cp infrastructure/encoding-worker/startup.sh \ "${BUCKET}/encoding-worker/startup.sh" + # AWS encoding worker (EC2): its startup + idle self-stop, fetched on boot + gsutil cp infrastructure/encoding-worker/aws/startup-aws.sh \ + "${BUCKET}/encoding-worker/startup-aws.sh" + gsutil cp infrastructure/encoding-worker/aws/idle-watchdog.py \ + "${BUCKET}/encoding-worker/idle-watchdog.py" # 4. Write version manifest for verification echo "Writing version manifest..." @@ -1425,6 +1433,15 @@ jobs: print("::error::Encoding worker config not found in Firestore") raise SystemExit(1) data = doc.to_dict() + if (data.get("active_override_provider") or "").lower() == "aws": + # The AWS encoding worker (EC2) is serving by preference. This + # GCE blue-green flow can't gcloud-start/stop it and must not + # clear its override; it picks up the new wheel on its next job. + # Treat it as "no GCE override" so every step below ignores it. + print(f"AWS worker {data.get('active_override_vm')} is the serving override; " + "leaving it untouched by the GCE blue-green deploy") + for key in ("active_override_vm", "active_override_ip", "active_override_zone"): + data[key] = None with open(os.environ["GITHUB_OUTPUT"], "a") as f: for key in ["primary_vm", "primary_ip", "primary_version", "secondary_vm", "secondary_ip", diff --git a/CLAUDE.md b/CLAUDE.md index dceeb56b..8392f78e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -171,6 +171,12 @@ gcloud compute ssh encoding-worker-a --zone=us-central1-c --project=nomadkaraoke For job stuck at `encoding` status, see [docs/TROUBLESHOOTING.md](docs/TROUBLESHOOTING.md). +**AWS encoding worker** (`aws-encoding-worker-1`, EC2 c7a.8xlarge Spot, us-east-2) is a pool +candidate too (`provider: aws` in `encoding-worker-fallback-vms`): last resort by default, +preferred when Firestore `config/encoding-worker.speed_rank_overrides = {"c7a": 5}`. Not +blue-green deployed, idle-stops itself, shell via SSM, logs are on the box (journald). See +`infrastructure/aws/README.md`. + ### What Doesn't Work - `gcloud auth print-identity-token` - Wrong token type for internal endpoints - Cloud Tasks without proper service account - Gets 401 from Cloud Run diff --git a/backend/api/routes/encoding_worker.py b/backend/api/routes/encoding_worker.py index 13efc9a1..b0c0a4e6 100644 --- a/backend/api/routes/encoding_worker.py +++ b/backend/api/routes/encoding_worker.py @@ -51,7 +51,7 @@ async def warmup_encoding_worker( _auth: Tuple[str, str] = Depends(require_review_auth), manager=Depends(get_worker_manager), ): - """Start the primary encoding worker VM if it's stopped. + """Start the preferred encoding worker if it's stopped (normally the GCE primary). Called on lyrics-review page load so the VM is warm by the time the reviewer previews a render. ``job_id`` scopes auth to the job under review. @@ -63,7 +63,7 @@ async def warmup_encoding_worker( # WHOLE instance for ~11s on every review-page open with a cold VM, # stalling that same page's audio/lyrics requests behind it # (2026-09-19, follow-up to the concurrent-load investigation). - result = await asyncio.to_thread(manager.ensure_primary_running) + result = await asyncio.to_thread(manager.ensure_preferred_running) if result["started"]: logger.info(f"Started encoding worker VM: {result['vm_name']}") return result diff --git a/backend/services/aws_encoding_compute.py b/backend/services/aws_encoding_compute.py new file mode 100644 index 00000000..62adcdbd --- /dev/null +++ b/backend/services/aws_encoding_compute.py @@ -0,0 +1,167 @@ +"""EC2 adapter for the AWS encoding worker. + +The AWS worker (an EC2 Spot instance running the same ``gce_encoding`` HTTP +service) is just another candidate in the encoding-worker pool. Rather than +teach every caller about EC2, this module mimics the slice of +``google.cloud.compute_v1.InstancesClient`` that ``EncodingWorkerManager`` uses +(``get`` / ``start`` / ``stop``), so the manager routes AWS VMs here and the +GCE code paths stay untouched: + +- EC2 states map to GCE statuses (``running`` → ``RUNNING``, ``stopped`` → + ``TERMINATED``, …) so the existing RUNNING/STAGING/stopped checks just work. +- ``start`` returns an operation object shaped like a compute ExtendedOperation + (``result()``, ``error_code``, ``error_message``); EC2 capacity errors surface + as codes in ``CAPACITY_ERROR_CODES`` so stockout cooldowns apply unchanged. +- A missing instance raises ``google.api_core.exceptions.NotFound``, which the + manager already treats as "skip this candidate". + +Credentials: an IAM user limited to Start/Stop/Describe on instances tagged +``Project=nomad-encoding``, stored as JSON ``{"access_key_id", "secret_access_key"}`` +in Secret Manager ``aws-encoding-controller-credentials`` (env +``AWS_ENCODING_CONTROLLER_CREDENTIALS`` overrides, for tests/local runs). +""" + +from __future__ import annotations + +import json +import logging +import os +import threading +from dataclasses import dataclass +from typing import Any, Dict, Optional + +from google.api_core import exceptions as gcp_exceptions + +logger = logging.getLogger(__name__) + +CREDENTIALS_SECRET = "aws-encoding-controller-credentials" +CREDENTIALS_ENV = "AWS_ENCODING_CONTROLLER_CREDENTIALS" + +# EC2 instance state → GCE instance status. +_EC2_TO_GCE_STATUS = { + "pending": "STAGING", + "running": "RUNNING", + "stopping": "STOPPING", + "stopped": "TERMINATED", + "shutting-down": "STOPPING", + "terminated": "TERMINATED", +} + + +@dataclass(frozen=True) +class AwsVm: + """Where an AWS worker lives. Built from a fallback-pool entry.""" + + instance_id: str + region: str + + +@dataclass +class _Instance: + status: str + + +class AwsStartOperation: + """Duck-types a compute ExtendedOperation for ``_wait_for_compute_operation``.""" + + def __init__(self, run): + self._run = run + self.error_code = "" + self.error_message = "" + + def result(self, timeout: Optional[float] = None) -> None: + from botocore.exceptions import ClientError + + try: + self._run() + except ClientError as e: + err = e.response.get("Error", {}) + self.error_code = err.get("Code", "") or "ClientError" + self.error_message = err.get("Message", "") or str(e) + + +def _load_credentials(project_id: str) -> Dict[str, str]: + raw = os.environ.get(CREDENTIALS_ENV) + if not raw: + from google.cloud import secretmanager + + client = secretmanager.SecretManagerServiceClient() + name = f"projects/{project_id}/secrets/{CREDENTIALS_SECRET}/versions/latest" + raw = client.access_secret_version(name=name).payload.data.decode("utf-8") + creds = json.loads(raw) + return {"access_key_id": creds["access_key_id"], "secret_access_key": creds["secret_access_key"]} + + +class AwsComputeAdapter: + """``get``/``start``/``stop`` for registered AWS worker VMs, by VM name.""" + + def __init__(self, vms: Dict[str, AwsVm], project_id: str = "nomadkaraoke", client_factory=None): + self._vms = dict(vms) + self._project_id = project_id + self._client_factory = client_factory + self._clients: Dict[str, Any] = {} + self._lock = threading.Lock() + + def handles(self, vm_name: str) -> bool: + return vm_name in self._vms + + def _client(self, region: str): + with self._lock: + if region not in self._clients: + if self._client_factory is not None: + self._clients[region] = self._client_factory(region) + else: + import boto3 + + creds = _load_credentials(self._project_id) + self._clients[region] = boto3.client( + "ec2", + region_name=region, + aws_access_key_id=creds["access_key_id"], + aws_secret_access_key=creds["secret_access_key"], + ) + return self._clients[region] + + def _vm(self, instance: str) -> AwsVm: + vm = self._vms.get(instance) + if vm is None: + raise gcp_exceptions.NotFound(f"AWS worker {instance} is not registered") + return vm + + def get(self, project: str = "", zone: str = "", instance: str = "") -> _Instance: + from botocore.exceptions import ClientError + + vm = self._vm(instance) + try: + resp = self._client(vm.region).describe_instances(InstanceIds=[vm.instance_id]) + except ClientError as e: + if e.response.get("Error", {}).get("Code", "").startswith("InvalidInstanceID"): + raise gcp_exceptions.NotFound(f"AWS worker {instance} ({vm.instance_id}) not found") from e + raise + reservations = resp.get("Reservations") or [] + if not reservations or not reservations[0].get("Instances"): + raise gcp_exceptions.NotFound(f"AWS worker {instance} ({vm.instance_id}) not found") + state = reservations[0]["Instances"][0]["State"]["Name"] + return _Instance(status=_EC2_TO_GCE_STATUS.get(state, state.upper())) + + def start(self, project: str = "", zone: str = "", instance: str = "") -> AwsStartOperation: + vm = self._vm(instance) + logger.info("Starting AWS worker %s (%s in %s)", instance, vm.instance_id, vm.region) + return AwsStartOperation(lambda: self._client(vm.region).start_instances(InstanceIds=[vm.instance_id])) + + def stop(self, project: str = "", zone: str = "", instance: str = "") -> None: + vm = self._vm(instance) + logger.info("Stopping AWS worker %s (%s in %s)", instance, vm.instance_id, vm.region) + self._client(vm.region).stop_instances(InstanceIds=[vm.instance_id]) + + +def aws_vms_from_pool(entries) -> Dict[str, AwsVm]: + """``{vm_name: AwsVm}`` for the ``provider == "aws"`` entries of a fallback pool.""" + vms: Dict[str, AwsVm] = {} + for e in entries or []: + if (e.get("provider") or "").lower() != "aws": + continue + name = e.get("vm") or e.get("vm_name") + if name and e.get("instance_id") and e.get("region"): + vms[name] = AwsVm(instance_id=e["instance_id"], region=e["region"]) + return vms diff --git a/backend/services/encoding_errors.py b/backend/services/encoding_errors.py index 59690c77..06e4fca6 100644 --- a/backend/services/encoding_errors.py +++ b/backend/services/encoding_errors.py @@ -12,6 +12,12 @@ CAPACITY_ERROR_CODES: FrozenSet[str] = frozenset({ "ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS", "STOCKOUT", "QUOTA_EXCEEDED", + # AWS EC2 (the AWS encoding worker; see aws_encoding_compute.py) + "InsufficientInstanceCapacity", + "InsufficientHostCapacity", + "InsufficientCapacity", + "VcpuLimitExceeded", + "MaxSpotInstanceCountExceeded", }) diff --git a/backend/services/encoding_service.py b/backend/services/encoding_service.py index bfef792e..ea7cfa6e 100644 --- a/backend/services/encoding_service.py +++ b/backend/services/encoding_service.py @@ -294,43 +294,14 @@ class EncodingService: yield def _fallback_entries(self) -> list: - """Parse ENCODING_WORKER_FALLBACK_VMS into candidate dicts. + """Parse ENCODING_WORKER_FALLBACK_VMS into candidate dicts (never raises). - Schema: '[{"vm":"encoding-worker-fallback-c2df","zone":"us-central1-f", - "ip":"34.x.x.x","machine_type":"c2d-highcpu-16"}, ...]'. machine_type is - optional (inferred from the VM name for legacy entries). Malformed input - degrades to fewer/no fallbacks — this runs during connection recovery and - must never raise. + See ``encoding_worker_manager.parse_fallback_vms`` for the schema (GCE + entries, plus AWS ones carrying provider/instance_id/region). """ - import json as _json - raw = self.settings.encoding_worker_fallback_vms - if not raw: - return [] - try: - parsed = _json.loads(raw) - except (ValueError, TypeError) as e: - logger.warning(f"Invalid ENCODING_WORKER_FALLBACK_VMS JSON: {e}") - return [] - # A non-list root (null, dict, scalar) parses fine but would blow up - # `for item in parsed` and escape during connection recovery — degrade - # to no fallbacks instead. - if not isinstance(parsed, list): - logger.warning("ENCODING_WORKER_FALLBACK_VMS must be a JSON list; ignoring") - return [] - entries = [] - for item in parsed: - try: - entries.append({ - "vm": item["vm"], - "zone": item["zone"], - "ip": item["ip"], - "machine_type": item.get("machine_type"), - "kind": "fallback", - }) - except (KeyError, TypeError) as e: - logger.warning(f"Skipping malformed fallback candidate {item}: {e}") - continue - return entries + from backend.services.encoding_worker_manager import parse_fallback_vms + + return parse_fallback_vms(self.settings.encoding_worker_fallback_vms) def _vm_for_worker_url(self, worker_url: Optional[str]) -> Optional[tuple]: """Map a worker base URL (http://:8080) back to its (vm_name, zone). @@ -394,13 +365,12 @@ class EncodingService: def _build_worker_candidates(self) -> list: """Build the ranked candidate list for ensure_any_running. - Reads optional fallback VMs from settings (env var - ENCODING_WORKER_FALLBACK_VMS, JSON list of {vm, zone, ip, machine_type?}). - The primary + all fallbacks are ranked by the shared preference logic - (fastest-first, demoting recently-stocked-out types) so runtime selection - stays in lock-step with the deploy green selection. Returns an empty list - when no worker manager / no configured fallbacks — caller falls back to - the single-VM ensure_primary_running path. + Delegates to ``EncodingWorkerManager.ranked_candidates`` (primary + + ENCODING_WORKER_FALLBACK_VMS, fastest-first, demoting recently + stocked-out types, honouring ``speed_rank_overrides``) so runtime + selection, the warmups and the deploy green selection rank identically. + Returns an empty list when there is no worker manager or its config + can't be read — caller falls back to ensure_primary_running. """ if not self._worker_manager: return [] @@ -409,43 +379,7 @@ class EncodingService: except Exception as e: # noqa: BLE001 logger.warning(f"Could not read worker config for candidates: {_format_exception(e)}") return [] - - from backend.services.encoding_worker_manager import EncodingWorkerCandidate - from backend.services.encoding_worker_preference import ( - PRIMARY_MACHINE_TYPE, - ordered_candidates, - ) - - # Build a dict pool (primary + fallbacks), each tagged with machine_type so - # the shared preference logic can rank them. The primary/secondary pair is - # always c4d; the config may override via primary_machine_type. - primary_mt = getattr(config, "primary_machine_type", None) or PRIMARY_MACHINE_TYPE - pool = [{ - "vm": config.primary_vm, - "zone": self._worker_manager._zone, - "ip": config.primary_ip, - "machine_type": primary_mt, - "kind": "primary", - "is_primary": True, - }] - - # Optional capacity-fallback VMs in alternate zones/families (see - # _fallback_entries). Empty when unconfigured. - pool.extend(self._fallback_entries()) - - # Rank fastest-first, demoting types that recently stocked out. Keeps the - # primary (c4d) at the top whenever it has capacity. - ranked = ordered_candidates(pool, capacity_state=config.capacity_state) - return [ - EncodingWorkerCandidate( - vm_name=c["vm"], - zone=c["zone"], - ip=c["ip"], - machine_type=c.get("machine_type"), - is_primary=c.get("is_primary", False), - ) - for c in ranked - ] + return self._worker_manager.ranked_candidates(config) def _load_credentials(self): """Load encoding worker URL and API key from config/secrets.""" diff --git a/backend/services/encoding_worker_manager.py b/backend/services/encoding_worker_manager.py index 199d7373..970ee7bd 100644 --- a/backend/services/encoding_worker_manager.py +++ b/backend/services/encoding_worker_manager.py @@ -112,6 +112,9 @@ class EncodingWorkerConfig: # shared preference logic to demote recently-exhausted types. See # backend/services/encoding_worker_preference.py. capacity_state: dict = field(default_factory=dict) + # {machine_family: rank} replacing SPEED_RANK at runtime, e.g. {"c7a": 5} + # makes the AWS worker preferred. Absent/empty ⇒ the built-in ranking. + speed_rank_overrides: dict = field(default_factory=dict) @property def primary_url(self) -> str: @@ -155,6 +158,8 @@ class EncodingWorkerCandidate: # the preference logic may rank a fallback ahead of a stocked-out primary, so # "first candidate" is no longer synonymous with "primary". is_primary: bool = False + # "aws" for the EC2 worker (routed to AwsComputeAdapter); None ⇒ GCE. + provider: Optional[str] = None @property def url(self) -> str: @@ -165,14 +170,148 @@ class EncodingWorkerCandidate: return {"vm": self.vm_name, "zone": self.zone, "machine_type": self.machine_type} +def parse_fallback_vms(raw: Optional[str]) -> list: + """Parse ENCODING_WORKER_FALLBACK_VMS into candidate dicts. Never raises. + + Schema: '[{"vm","zone","ip","machine_type"?}, ...]'. AWS workers add + ``"provider": "aws", "instance_id", "region"`` (passed through so the + manager can route them to the EC2 adapter). machine_type is optional + (inferred from the VM name for legacy entries). Malformed input degrades to + fewer/no fallbacks — this runs during connection recovery. + """ + import json as _json + + if not raw: + return [] + try: + parsed = _json.loads(raw) + except (ValueError, TypeError) as e: + logger.warning("Invalid ENCODING_WORKER_FALLBACK_VMS JSON: %s", e) + return [] + if not isinstance(parsed, list): + logger.warning("ENCODING_WORKER_FALLBACK_VMS must be a JSON list; ignoring") + return [] + entries = [] + for item in parsed: + try: + entry = { + "vm": item["vm"], + "zone": item["zone"], + "ip": item["ip"], + "machine_type": item.get("machine_type"), + "kind": "fallback", + } + for key in ("provider", "instance_id", "region"): + if item.get(key): + entry[key] = item[key] + entries.append(entry) + except (KeyError, TypeError) as e: + logger.warning("Skipping malformed fallback candidate %s: %s", item, e) + continue + return entries + + class EncodingWorkerManager: """Manages blue-green encoding worker VMs via Firestore config and Compute API.""" - def __init__(self, db, compute_client=None, project_id="nomadkaraoke", zone="us-central1-c"): + def __init__( + self, + db, + compute_client=None, + project_id="nomadkaraoke", + zone="us-central1-c", + fallback_vms_raw: Optional[str] = None, + aws_compute=None, + ): self._db = db self._compute = compute_client self._project_id = project_id self._zone = zone + # Raw ENCODING_WORKER_FALLBACK_VMS JSON; None → read from settings lazily. + self._fallback_vms_raw = fallback_vms_raw + self._aws_compute = aws_compute # injectable for tests; built lazily otherwise + + def fallback_entries(self) -> list: + """The configured capacity-fallback candidates (GCE and AWS), never raising.""" + raw = self._fallback_vms_raw + if raw is None: + try: + from backend.config import get_settings + + raw = get_settings().encoding_worker_fallback_vms + except Exception as e: # noqa: BLE001 + logger.warning("Could not read ENCODING_WORKER_FALLBACK_VMS: %s", e) + raw = None + return parse_fallback_vms(raw) + + def _compute_for(self, vm_name: str): + """The compute client for a VM: the EC2 adapter for AWS workers, else GCE.""" + if self._aws_compute is None: + from backend.services.aws_encoding_compute import AwsComputeAdapter, aws_vms_from_pool + + self._aws_compute = AwsComputeAdapter( + aws_vms_from_pool(self.fallback_entries()), project_id=self._project_id + ) + if self._aws_compute.handles(vm_name): + return self._aws_compute + return self._compute + + def ranked_candidates(self, config: Optional["EncodingWorkerConfig"] = None) -> list: + """Primary + fallbacks ranked by the shared preference logic. + + Fastest-first (SPEED_RANK, overridable per family via + ``config/encoding-worker.speed_rank_overrides``), demoting types that + recently stocked out. Shared by the encode path and the warmups so they + always start the worker that requests will actually be routed to. + """ + from backend.services.encoding_worker_preference import ( + PRIMARY_MACHINE_TYPE, + ordered_candidates, + ) + + config = config or self.get_config() + primary_mt = getattr(config, "primary_machine_type", None) or PRIMARY_MACHINE_TYPE + pool = [{ + "vm": config.primary_vm, + "zone": self._zone, + "ip": config.primary_ip, + "machine_type": primary_mt, + "kind": "primary", + "is_primary": True, + }] + pool.extend(self.fallback_entries()) + ranked = ordered_candidates( + pool, + capacity_state=config.capacity_state, + rank_overrides=getattr(config, "speed_rank_overrides", None), + ) + return [ + EncodingWorkerCandidate( + vm_name=c["vm"], + zone=c["zone"], + ip=c["ip"], + machine_type=c.get("machine_type"), + is_primary=c.get("is_primary", False), + provider=c.get("provider"), + ) + for c in ranked + ] + + def ensure_preferred_running(self) -> dict: + """Warm the worker requests will be routed to (ranked, with failover). + + When the GCE primary ranks first (the normal case) this is exactly + ``ensure_primary_running`` — unchanged behaviour. When something else + ranks first (a ``speed_rank_overrides`` preferring the AWS worker, or the + primary's type in stockout cooldown) it runs the same ranked + ``ensure_any_running`` the encode path uses, so the worker that requests + will be routed to is the one warmed, and the active override points at it. + """ + config = self.get_config() + candidates = self.ranked_candidates(config) + if not candidates or candidates[0].is_primary: + return self.ensure_primary_running() + return self.ensure_any_running(candidates) def _doc_ref(self): """Return a reference to the config/encoding-worker document.""" @@ -209,6 +348,7 @@ class EncodingWorkerManager: active_override_zone=data.get("active_override_zone"), active_override_set_at=data.get("active_override_set_at"), capacity_state=data.get("capacity_state") or {}, + speed_rank_overrides=data.get("speed_rank_overrides") or {}, ) def update_activity(self) -> None: @@ -271,7 +411,7 @@ class EncodingWorkerManager: Pass `zone` for fallback VMs that live in alternate zones. """ - instance = self._compute.get( + instance = self._compute_for(vm_name).get( project=self._project_id, zone=zone or self._zone, instance=vm_name, @@ -303,7 +443,7 @@ class EncodingWorkerManager: logger.info("VM %s is already %s, skipping start", vm_name, status) return logger.info("Starting VM %s in zone %s (current status: %s)", vm_name, target_zone, status) - operation = self._compute.start( + operation = self._compute_for(vm_name).start( project=self._project_id, zone=target_zone, instance=vm_name, @@ -318,7 +458,7 @@ class EncodingWorkerManager: logger.info("VM %s is already %s, skipping stop", vm_name, status) return logger.info("Stopping VM %s in zone %s (current status: %s)", vm_name, target_zone, status) - self._compute.stop( + self._compute_for(vm_name).stop( project=self._project_id, zone=target_zone, instance=vm_name, @@ -459,6 +599,22 @@ class EncodingWorkerManager: zone=candidate.zone, code="NOT_FOUND", ) from nf + except EncodingWorkerStartError: + raise + except Exception as err: # noqa: BLE001 + if getattr(candidate, "provider", None) != "aws": + raise # GCE: unchanged behaviour + # An AWS control-plane fault (bad/missing controller creds, + # throttling, botocore error) must fail over to the GCE + # candidates, not abort selection — when the AWS worker is + # preferred it ranks first, so escaping here would block all + # encoding. + raise EncodingWorkerStartError( + f"VM {candidate.vm_name} status check failed in {candidate.zone}: {err}", + vm_name=candidate.vm_name, + zone=candidate.zone, + code="STATUS_CHECK_FAILED", + ) from err started = False if status not in ("RUNNING", "STAGING"): logger.info( @@ -575,6 +731,10 @@ class EncodingWorkerManager: "active_override_ip": candidate.ip, "active_override_zone": candidate.zone, "active_override_set_at": now, + # "aws" for the EC2 worker; the CI blue-green deploy leaves an AWS + # override serving (it can't gcloud-stop it, and it hot-updates its + # wheel per job). None for GCE. + "active_override_provider": getattr(candidate, "provider", None), }) logger.info( "Set active_override to %s in %s (capacity fallback)", @@ -597,6 +757,7 @@ class EncodingWorkerManager: "active_override_ip": None, "active_override_zone": None, "active_override_set_at": None, + "active_override_provider": None, }) logger.info("Cleared active_override (primary is healthy again)") diff --git a/backend/services/encoding_worker_preference.py b/backend/services/encoding_worker_preference.py index 848a237e..291eabd6 100644 --- a/backend/services/encoding_worker_preference.py +++ b/backend/services/encoding_worker_preference.py @@ -51,6 +51,11 @@ SPEED_RANK: Dict[str, int] = { "c2d": 50, "n2d": 60, "n2": 70, + # AWS EC2 c7a (AMD Genoa, us-east-2) — the AWS encoding worker + # (provider "aws"; machine_type "c7a-8xlarge", "." → "-" for Firestore-safe + # cooldown keys). Last resort by default; make it preferred at runtime with + # config/encoding-worker.speed_rank_overrides = {"c7a": 5}. + "c7a": 90, } # Rank for a candidate whose machine type is unknown and cannot be inferred: @@ -106,8 +111,14 @@ def infer_machine_type(candidate: Dict[str, Any]) -> Optional[str]: return None -def _speed_rank(candidate: Dict[str, Any]) -> int: - return SPEED_RANK.get(machine_family(infer_machine_type(candidate)), UNKNOWN_RANK) +def _speed_rank(candidate: Dict[str, Any], rank_overrides: Optional[Dict[str, Any]] = None) -> int: + family = machine_family(infer_machine_type(candidate)) + if rank_overrides and family in rank_overrides: + try: + return int(rank_overrides[family]) + except (TypeError, ValueError): + pass # a malformed override must never break worker selection + return SPEED_RANK.get(family, UNKNOWN_RANK) def cooldown_key(candidate: Dict[str, Any]) -> Optional[str]: @@ -164,6 +175,7 @@ def ordered_candidates( capacity_state: Optional[Dict[str, Any]] = None, now: Optional[datetime] = None, cooldown_seconds: float = COOLDOWN_SECONDS, + rank_overrides: Optional[Dict[str, Any]] = None, ) -> List[Dict[str, Any]]: """Rank ``pool`` fastest-first, demoting recently-stocked-out types to the back. @@ -175,6 +187,9 @@ def ordered_candidates( Absent/empty ⇒ pure fastest-first order (today's behaviour). now: current UTC time (injectable for tests); defaults to ``datetime.now(UTC)``. cooldown_seconds: demotion window; defaults to ``COOLDOWN_SECONDS``. + rank_overrides: ``{family: rank}`` from Firestore + ``config/encoding-worker.speed_rank_overrides``; replaces SPEED_RANK + for those families (e.g. ``{"c7a": 5}`` prefers the AWS worker). The speed sort is STABLE, so equal-rank candidates keep their input order — callers express a secondary preference (e.g. zone spread) purely via input @@ -182,7 +197,7 @@ def ordered_candidates( """ if now is None: now = datetime.now(timezone.utc) - ranked = sorted(pool, key=_speed_rank) # stable: fastest first + ranked = sorted(pool, key=lambda c: _speed_rank(c, rank_overrides)) # stable: fastest first hot = [c for c in ranked if not _recently_stocked_out(c, capacity_state, now, cooldown_seconds)] cold = [c for c in ranked if _recently_stocked_out(c, capacity_state, now, cooldown_seconds)] return hot + cold diff --git a/backend/services/gce_encoding/main.py b/backend/services/gce_encoding/main.py index b6cec11d..202a91e3 100644 --- a/backend/services/gce_encoding/main.py +++ b/backend/services/gce_encoding/main.py @@ -707,6 +707,18 @@ def verify_wheel_imports(): return True +def _gcs_cli() -> list: + """The GCS CLI for wheel listing/copying on this worker. + + GCE workers use ``gsutil`` (unchanged). The AWS worker authenticates to GCP + via Workload Identity Federation, which ``gsutil`` doesn't support, so it + uses ``gcloud storage`` (same ``ls``/``cp`` syntax). + """ + if os.environ.get("ENCODING_WORKER_PROVIDER", "").lower() == "aws": + return ["gcloud", "storage"] + return ["gsutil"] + + def ensure_latest_wheel(): '''Ensure the latest karaoke-gen wheel is installed & verified (thread-safe). @@ -748,7 +760,7 @@ def _ensure_latest_wheel_locked(): # job and blow the timeout on fresh workers — the historical cause of # "no wheel found" / partial installs. Listing + single copy is ~2s. ls_result = subprocess.run( - ["gsutil", "ls", "gs://karaoke-gen-storage-nomadkaraoke/wheels/karaoke_gen-*.whl"], + [*_gcs_cli(), "ls", "gs://karaoke-gen-storage-nomadkaraoke/wheels/karaoke_gen-*.whl"], capture_output=True, text=True, timeout=60 ) # Filter out karaoke_gen-current.whl (not a valid PEP 427 wheel name) @@ -787,7 +799,7 @@ def _ensure_latest_wheel_locked(): ) cp_result = subprocess.run( - ["gsutil", "cp", latest_uri, wheel_path], + [*_gcs_cli(), "cp", latest_uri, wheel_path], capture_output=True, text=True, timeout=120 ) if cp_result.returncode != 0: diff --git a/backend/services/worker_service.py b/backend/services/worker_service.py index 465b4668..069757fd 100644 --- a/backend/services/worker_service.py +++ b/backend/services/worker_service.py @@ -624,7 +624,7 @@ class WorkerService: compute_client=compute_client, project_id=self.settings.google_cloud_project, ) - result = manager.ensure_primary_running() + result = manager.ensure_preferred_running() if result["started"]: logger.info(f"[job:{job_id}] Encoding worker warmup: started VM {result['vm_name']}") else: diff --git a/backend/tests/test_aws_encoding_compute.py b/backend/tests/test_aws_encoding_compute.py new file mode 100644 index 00000000..73b4f8d0 --- /dev/null +++ b/backend/tests/test_aws_encoding_compute.py @@ -0,0 +1,115 @@ +"""Unit tests for the EC2 adapter behind the AWS encoding worker. + +The adapter must look exactly like the compute_v1.InstancesClient slice the +manager uses, so the manager's GCE-shaped status checks, capacity cooldowns +and NotFound skips apply to the AWS worker unchanged. +""" +import json + +import boto3 +import pytest +from botocore.stub import Stubber +from google.api_core import exceptions as gcp_exceptions + +from backend.services.aws_encoding_compute import ( + AwsComputeAdapter, + AwsVm, + _load_credentials, + aws_vms_from_pool, +) +from backend.services.encoding_errors import CAPACITY_ERROR_CODES + +VM = "aws-encoding-worker-1" +IID = "i-0123456789abcdef0" + + +@pytest.fixture +def ec2(): + client = boto3.client("ec2", region_name="us-east-2", aws_access_key_id="x", aws_secret_access_key="y") + with Stubber(client) as stub: + yield client, stub + + +def _adapter(client): + return AwsComputeAdapter({VM: AwsVm(IID, "us-east-2")}, client_factory=lambda region: client) + + +def _describe(state): + return {"Reservations": [{"Instances": [{"InstanceId": IID, "State": {"Name": state, "Code": 0}}]}]} + + +@pytest.mark.parametrize("ec2_state,gce_status", [ + ("running", "RUNNING"), + ("pending", "STAGING"), + ("stopping", "STOPPING"), + ("stopped", "TERMINATED"), + ("shutting-down", "STOPPING"), +]) +def test_get_maps_ec2_state_to_gce_status(ec2, ec2_state, gce_status): + client, stub = ec2 + stub.add_response("describe_instances", _describe(ec2_state), {"InstanceIds": [IID]}) + assert _adapter(client).get(project="p", zone="us-east-2b", instance=VM).status == gce_status + + +def test_get_missing_instance_raises_not_found(ec2): + client, stub = ec2 + stub.add_client_error("describe_instances", service_error_code="InvalidInstanceID.NotFound") + with pytest.raises(gcp_exceptions.NotFound): + _adapter(client).get(instance=VM) + + +def test_unregistered_vm_raises_not_found(ec2): + client, _ = ec2 + with pytest.raises(gcp_exceptions.NotFound): + _adapter(client).get(instance="encoding-worker-a") + + +def test_start_success_leaves_no_error_code(ec2): + client, stub = ec2 + stub.add_response("start_instances", {"StartingInstances": []}, {"InstanceIds": [IID]}) + op = _adapter(client).start(instance=VM) + op.result(timeout=30) + assert op.error_code == "" + + +def test_start_capacity_error_is_a_capacity_code(ec2): + client, stub = ec2 + stub.add_client_error( + "start_instances", service_error_code="InsufficientInstanceCapacity", + service_message="no c7a.8xlarge capacity in us-east-2b", + ) + op = _adapter(client).start(instance=VM) + op.result() + assert op.error_code == "InsufficientInstanceCapacity" + assert op.error_code in CAPACITY_ERROR_CODES + assert "capacity" in op.error_message + + +def test_stop_calls_stop_instances(ec2): + client, stub = ec2 + stub.add_response("stop_instances", {"StoppingInstances": []}, {"InstanceIds": [IID]}) + _adapter(client).stop(instance=VM) + stub.assert_no_pending_responses() + + +def test_handles_only_registered_vms(): + adapter = AwsComputeAdapter({VM: AwsVm(IID, "us-east-2")}, client_factory=lambda r: None) + assert adapter.handles(VM) + assert not adapter.handles("encoding-worker-a") + + +def test_aws_vms_from_pool_keeps_only_complete_aws_entries(): + pool = [ + {"vm": "encoding-worker-fallback-c2df", "zone": "us-central1-f", "ip": "1.2.3.4"}, + {"vm": VM, "zone": "us-east-2b", "ip": "5.6.7.8", "provider": "aws", "instance_id": IID, "region": "us-east-2"}, + {"vm": "aws-incomplete", "zone": "us-east-2b", "ip": "9.9.9.9", "provider": "aws"}, + ] + assert aws_vms_from_pool(pool) == {VM: AwsVm(IID, "us-east-2")} + + +def test_credentials_env_override(monkeypatch): + monkeypatch.setenv( + "AWS_ENCODING_CONTROLLER_CREDENTIALS", + json.dumps({"access_key_id": "AKIA", "secret_access_key": "s3cr3t", "extra": 1}), + ) + assert _load_credentials("nomadkaraoke") == {"access_key_id": "AKIA", "secret_access_key": "s3cr3t"} diff --git a/backend/tests/test_aws_idle_watchdog.py b/backend/tests/test_aws_idle_watchdog.py new file mode 100644 index 00000000..5939c3e5 --- /dev/null +++ b/backend/tests/test_aws_idle_watchdog.py @@ -0,0 +1,54 @@ +"""Tests for the AWS encoding worker's idle self-stop decision.""" +import importlib.util +from datetime import datetime, timedelta, timezone +from pathlib import Path + +_spec = importlib.util.spec_from_file_location("idle_watchdog", Path(__file__).resolve().parents[2] / "infrastructure" / "encoding-worker" / "aws" / "idle-watchdog.py") +wd = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(wd) + +NOW = datetime(2026, 10, 29, 12, 0, tzinfo=timezone.utc) +VM = "aws-encoding-worker-1" + + +def _decide(active_jobs=0, config=None, unknown_since=None): + return wd.decide(now=NOW, active_jobs=active_jobs, config=config, vm_name=VM, unknown_since=unknown_since)[0] + + +def _routed(minutes_ago): + return {"active_override_vm": VM, "last_activity_at": (NOW - timedelta(minutes=minutes_ago)).isoformat()} + + +def test_busy_worker_keeps_running_even_if_not_routed(): + assert _decide(active_jobs=2, config={"active_override_vm": "encoding-worker-a"}) == "keep" + + +def test_idle_and_not_routed_stops(): + assert _decide(config={"active_override_vm": None}) == "stop" + + +def test_routed_with_recent_activity_keeps_running(): + assert _decide(config=_routed(2)) == "keep" + + +def test_routed_but_idle_past_timeout_stops(): + assert _decide(config=_routed(6)) == "stop" + + +def test_routed_without_activity_timestamp_stops(): + assert _decide(config={"active_override_vm": VM}) == "stop" + + +def test_naive_timestamp_treated_as_utc(): + assert _decide(config={"active_override_vm": VM, "last_activity_at": "2026-10-29T11:58:00"}) == "keep" + + +def test_unreadable_health_starts_grace_then_stops(): + assert _decide(active_jobs=None, config=_routed(1)) == "unknown" + assert _decide(active_jobs=None, config=_routed(1), unknown_since=NOW - timedelta(minutes=5)) == "unknown" + assert _decide(active_jobs=None, config=_routed(1), unknown_since=NOW - timedelta(minutes=16)) == "stop" + + +def test_unreadable_firestore_starts_grace_then_stops(): + assert _decide(config=None) == "unknown" + assert _decide(config=None, unknown_since=NOW - timedelta(minutes=20)) == "stop" diff --git a/backend/tests/test_deploy_promote.py b/backend/tests/test_deploy_promote.py index f33fb1d0..47145593 100644 --- a/backend/tests/test_deploy_promote.py +++ b/backend/tests/test_deploy_promote.py @@ -219,3 +219,43 @@ class TestSelectGreenRankingSharedWithRuntime: cands = select_green_candidates(_config(active_override_vm=None), self.POOL) c4 = next(c for c in cands if c["vm"] == "encoding-worker-fallback-c4a") assert c4["machine_type"] == "c4-highcpu-16" + + +# --- AWS encoding worker: never touched by the GCE blue-green deploy ---------- + +_AWS = {"vm": "aws-encoding-worker-1", "zone": "us-east-2b", "ip": "3.3.3.3", + "machine_type": "c7a-8xlarge", "provider": "aws", "instance_id": "i-0abc", "region": "us-east-2"} + + +def test_aws_worker_is_never_a_green_candidate(): + config = {"secondary_vm": "encoding-worker-b", "secondary_ip": "1.1.1.2"} + fallbacks = [{"vm": "encoding-worker-fallback-c2df", "zone": "us-central1-f", "ip": "2.2.2.2"}, _AWS] + names = [c["vm"] for c in deploy_promote.select_green_candidates(config, fallbacks)] + assert "aws-encoding-worker-1" not in names + assert names == ["encoding-worker-b", "encoding-worker-fallback-c2df"] + + +def test_secondary_promotion_leaves_aws_override_serving(): + config = { + "primary_vm": "encoding-worker-a", "primary_ip": "1.1.1.1", "primary_version": "1.0", + "active_override_vm": "aws-encoding-worker-1", "active_override_ip": "3.3.3.3", + "active_override_zone": "us-east-2b", "active_override_provider": "aws", + } + green = {"vm": "encoding-worker-b", "ip": "1.1.1.2", "zone": "us-central1-c", "kind": "secondary"} + plan = deploy_promote.promote_plan(green, config, now="2026-10-29T00:00:00Z", version="2.0") + assert "active_override_vm" not in plan["firestore_updates"] + assert [d["vm"] for d in plan["drain_and_stop"]] == ["encoding-worker-a"] + + +def test_fallback_promotion_does_not_drain_aws_override(): + config = {"active_override_vm": "aws-encoding-worker-1", "active_override_zone": "us-east-2b", + "active_override_provider": "aws"} + green = {"vm": "encoding-worker-fallback-c2df", "ip": "2.2.2.2", "zone": "us-central1-f", "kind": "fallback"} + plan = deploy_promote.promote_plan(green, config, now="2026-10-29T00:00:00Z", version="2.0") + assert plan["drain_and_stop"] == [] + + +def test_fallback_promotion_clears_provider_tag(): + green = {"vm": "encoding-worker-fallback-c2df", "ip": "2.2.2.2", "zone": "us-central1-f", "kind": "fallback"} + plan = deploy_promote.promote_plan(green, {}, now="2026-10-29T00:00:00Z", version="2.0") + assert plan["firestore_updates"]["active_override_provider"] is None diff --git a/backend/tests/test_encoding_service.py b/backend/tests/test_encoding_service.py index 9d82f567..154b6d48 100644 --- a/backend/tests/test_encoding_service.py +++ b/backend/tests/test_encoding_service.py @@ -1351,12 +1351,19 @@ class TestBuildWorkerCandidates: cfg.primary_ip = "10.0.0.1" cfg.primary_machine_type = None # → defaults to c4d cfg.capacity_state = capacity_state or {} + cfg.speed_rank_overrides = {} return cfg def _service_with_fallbacks(self, encoding_service, fallbacks_json, capacity_state=None): - mgr = MagicMock() - mgr._zone = "us-central1-c" - mgr.get_config.return_value = self._config(capacity_state) + # Ranking lives in EncodingWorkerManager.ranked_candidates (shared with the + # warmups), so use a real manager with a mocked config read. + from backend.services.encoding_worker_manager import EncodingWorkerManager + + mgr = EncodingWorkerManager( + db=MagicMock(), compute_client=MagicMock(), zone="us-central1-c", + fallback_vms_raw=fallbacks_json, + ) + mgr.get_config = MagicMock(return_value=self._config(capacity_state)) encoding_service._worker_manager = mgr encoding_service.settings.encoding_worker_fallback_vms = fallbacks_json return encoding_service diff --git a/backend/tests/test_encoding_worker_manager_aws.py b/backend/tests/test_encoding_worker_manager_aws.py new file mode 100644 index 00000000..9e842eda --- /dev/null +++ b/backend/tests/test_encoding_worker_manager_aws.py @@ -0,0 +1,146 @@ +"""EncodingWorkerManager with the AWS encoding worker in the pool. + +Covers routing AWS VMs to the EC2 adapter (GCE untouched), the shared fallback +parser passing provider fields through, ranking with Firestore +speed_rank_overrides, the ranked warmup, and the override's provider tag. +""" +import json +from unittest.mock import MagicMock + +import pytest + +from backend.services.aws_encoding_compute import AwsStartOperation +from backend.services.encoding_errors import EncodingWorkerCapacityError +from backend.services.encoding_worker_manager import EncodingWorkerManager, parse_fallback_vms + +AWS_ENTRY = { + "vm": "aws-encoding-worker-1", "zone": "us-east-2b", "ip": "3.3.3.3", + "machine_type": "c7a-8xlarge", "provider": "aws", "instance_id": "i-0abc", "region": "us-east-2", +} +C2D_ENTRY = {"vm": "encoding-worker-fallback-c2df", "zone": "us-central1-f", "ip": "2.2.2.2", + "machine_type": "c2d-highcpu-16"} + + +def _db(**config): + data = { + "primary_vm": "encoding-worker-a", "primary_ip": "1.1.1.1", "primary_version": "1.0", + "secondary_vm": "encoding-worker-b", "secondary_ip": "1.1.1.2", + "last_activity_at": None, "capacity_state": {}, **config, + } + db = MagicMock() + snap = MagicMock(exists=True) + snap.to_dict.return_value = data + doc = MagicMock() + doc.get.return_value = snap + db.collection.return_value.document.return_value = doc + return db, doc + + +def _manager(db, aws=None, entries=(C2D_ENTRY, AWS_ENTRY)): + gce = MagicMock(name="gce") + aws = aws or MagicMock(name="aws") + aws.handles.side_effect = lambda vm: vm == AWS_ENTRY["vm"] + m = EncodingWorkerManager(db=db, compute_client=gce, fallback_vms_raw=json.dumps(list(entries)), aws_compute=aws) + return m, gce, aws + + +def test_parse_fallback_vms_passes_aws_fields_through(): + parsed = parse_fallback_vms(json.dumps([C2D_ENTRY, AWS_ENTRY])) + assert parsed[0] == {**C2D_ENTRY, "kind": "fallback"} + assert parsed[1]["provider"] == "aws" + assert parsed[1]["instance_id"] == "i-0abc" and parsed[1]["region"] == "us-east-2" + + +@pytest.mark.parametrize("raw", [None, "", "not json", "{}", "null"]) +def test_parse_fallback_vms_never_raises(raw): + assert parse_fallback_vms(raw) == [] + + +def test_aws_vm_ops_go_to_adapter_and_gce_untouched(): + db, _ = _db() + m, gce, aws = _manager(db) + aws.get.return_value = MagicMock(status="RUNNING") + gce.get.return_value = MagicMock(status="TERMINATED") + + assert m.get_vm_status("aws-encoding-worker-1", zone="us-east-2b") == "RUNNING" + assert m.get_vm_status("encoding-worker-a") == "TERMINATED" + aws.get.assert_called_once() + gce.get.assert_called_once() + + +def test_aws_capacity_error_raises_typed_capacity_error(): + db, _ = _db() + m, _, aws = _manager(db) + aws.get.return_value = MagicMock(status="TERMINATED") + + def _no_capacity(): + from botocore.exceptions import ClientError + raise ClientError({"Error": {"Code": "InsufficientInstanceCapacity", "Message": "none"}}, "StartInstances") + + aws.start.return_value = AwsStartOperation(_no_capacity) + with pytest.raises(EncodingWorkerCapacityError): + m.start_vm("aws-encoding-worker-1", zone="us-east-2b") + + +def test_ranked_candidates_default_keeps_gce_primary_first_and_aws_last(): + db, _ = _db() + m, _, _ = _manager(db) + ranked = m.ranked_candidates() + assert [c.vm_name for c in ranked] == ["encoding-worker-a", "encoding-worker-fallback-c2df", "aws-encoding-worker-1"] + assert ranked[0].is_primary and ranked[-1].provider == "aws" + + +def test_ranked_candidates_honour_speed_rank_overrides(): + db, _ = _db(speed_rank_overrides={"c7a": 5}) + m, _, _ = _manager(db) + assert m.ranked_candidates()[0].vm_name == "aws-encoding-worker-1" + + +def test_ensure_preferred_running_is_primary_path_when_primary_ranks_first(): + db, _ = _db() + m, _, _ = _manager(db) + m.ensure_primary_running = MagicMock(return_value={"started": False}) + m.ensure_any_running = MagicMock() + m.ensure_preferred_running() + m.ensure_primary_running.assert_called_once() + m.ensure_any_running.assert_not_called() + + +def test_ensure_preferred_running_starts_aws_when_preferred(): + db, _ = _db(speed_rank_overrides={"c7a": 5}) + m, _, _ = _manager(db) + m.ensure_primary_running = MagicMock() + m.ensure_any_running = MagicMock(return_value={"started": True, "vm_name": "aws-encoding-worker-1"}) + m.ensure_preferred_running() + m.ensure_primary_running.assert_not_called() + (candidates,), _ = m.ensure_any_running.call_args + assert candidates[0].vm_name == "aws-encoding-worker-1" + + +def test_active_override_records_provider(): + db, doc = _db() + m, _, _ = _manager(db) + aws_candidate = next(c for c in m.ranked_candidates() if c.provider == "aws") + m._set_active_override(aws_candidate) + update = doc.update.call_args[0][0] + assert update["active_override_vm"] == "aws-encoding-worker-1" + assert update["active_override_provider"] == "aws" + + db2, doc2 = _db(active_override_vm="aws-encoding-worker-1", active_override_provider="aws") + m2, _, _ = _manager(db2) + m2._clear_active_override() + assert doc2.update.call_args[0][0]["active_override_provider"] is None + + +def test_aws_control_plane_fault_fails_over_to_gce(): + db, _ = _db(speed_rank_overrides={"c7a": 5}) + m, gce, aws = _manager(db, entries=(AWS_ENTRY,)) + aws.get.side_effect = RuntimeError("could not load aws-encoding-controller-credentials") + gce.get.return_value = MagicMock(status="RUNNING") + m.update_activity = MagicMock() + m._clear_active_override = MagicMock() + m._clear_stockout = MagicMock() + + result = m.ensure_any_running(m.ranked_candidates()) + + assert result["vm_name"] == "encoding-worker-a" diff --git a/backend/tests/test_encoding_worker_preference.py b/backend/tests/test_encoding_worker_preference.py index aaed0757..f049048d 100644 --- a/backend/tests/test_encoding_worker_preference.py +++ b/backend/tests/test_encoding_worker_preference.py @@ -170,3 +170,56 @@ def test_cooldown_key_shape(): assert cooldown_key(_c("x", "us-central1-c", "c4d-highcpu-16")) == "c4d-highcpu-16@us-central1-c" assert cooldown_key(_c("x", None, "n2-highcpu-16")) == "n2-highcpu-16" assert cooldown_key(_c("no-token", "z")) is None + + +# --- AWS encoding worker (c7a) + runtime rank overrides ----------------------- + +AWS = dict(provider="aws", instance_id="i-0abc", region="us-east-2") + + +def test_aws_worker_is_last_resort_by_default(): + pool = [ + _c("aws-encoding-worker-1", "us-east-2b", "c7a-8xlarge", **AWS), + _c("encoding-worker-fallback-c2df", "us-central1-f", "c2d-highcpu-16"), + _c("encoding-worker-a", "us-central1-c", "c4d-highcpu-16"), + ] + ranked = ordered_candidates(pool, now=NOW) + assert [c["vm"] for c in ranked] == [ + "encoding-worker-a", "encoding-worker-fallback-c2df", "aws-encoding-worker-1", + ] + + +def test_rank_override_prefers_aws_worker(): + pool = [ + _c("encoding-worker-a", "us-central1-c", "c4d-highcpu-16"), + _c("encoding-worker-fallback-c2df", "us-central1-f", "c2d-highcpu-16"), + _c("aws-encoding-worker-1", "us-east-2b", "c7a-8xlarge", **AWS), + ] + ranked = ordered_candidates(pool, now=NOW, rank_overrides={"c7a": 5}) + assert ranked[0]["vm"] == "aws-encoding-worker-1" + # The rest keep the built-in order behind it. + assert [c["vm"] for c in ranked[1:]] == ["encoding-worker-a", "encoding-worker-fallback-c2df"] + + +def test_stocked_out_aws_worker_still_demoted_despite_override(): + aws = _c("aws-encoding-worker-1", "us-east-2b", "c7a-8xlarge", **AWS) + pool = [aws, _c("encoding-worker-a", "us-central1-c", "c4d-highcpu-16")] + state = {cooldown_key(aws): (NOW - timedelta(seconds=60)).isoformat()} + ranked = ordered_candidates(pool, capacity_state=state, now=NOW, rank_overrides={"c7a": 5}) + assert ranked[-1]["vm"] == "aws-encoding-worker-1" + + +def test_malformed_rank_override_falls_back_to_builtin_rank(): + pool = [ + _c("aws-encoding-worker-1", "us-east-2b", "c7a-8xlarge", **AWS), + _c("encoding-worker-a", "us-central1-c", "c4d-highcpu-16"), + ] + ranked = ordered_candidates(pool, now=NOW, rank_overrides={"c7a": "first please"}) + assert ranked[0]["vm"] == "encoding-worker-a" + + +def test_aws_cooldown_key_is_firestore_safe(): + key = cooldown_key(_c("aws-encoding-worker-1", "us-east-2b", "c7a-8xlarge", **AWS)) + assert key == "c7a-8xlarge@us-east-2b" + assert "." not in key + assert SPEED_RANK["c7a"] > SPEED_RANK["c2d"] diff --git a/backend/tests/test_encoding_worker_routes.py b/backend/tests/test_encoding_worker_routes.py index 85922ee7..c681c5f9 100644 --- a/backend/tests/test_encoding_worker_routes.py +++ b/backend/tests/test_encoding_worker_routes.py @@ -23,12 +23,12 @@ class TestWarmupHandler: def test_success_passthrough(self): manager = MagicMock() - manager.ensure_primary_running.return_value = {"started": True, "vm_name": "encoding-worker-a"} + manager.ensure_preferred_running.return_value = {"started": True, "vm_name": "encoding-worker-a"} assert self._run(manager)["started"] is True def test_start_error_is_soft(self): manager = MagicMock() - manager.ensure_primary_running.side_effect = EncodingWorkerStartError("zone exhausted") + manager.ensure_preferred_running.side_effect = EncodingWorkerStartError("zone exhausted") result = self._run(manager) assert result["started"] is False assert "zone exhausted" in result["error"] @@ -41,7 +41,7 @@ class TestWarmupHandler: time.sleep(0.5) return {"started": False} - manager.ensure_primary_running.side_effect = slow_warmup + manager.ensure_preferred_running.side_effect = slow_warmup async def scenario(): ticks = 0 diff --git a/backend/tests/test_gce_encoding_worker.py b/backend/tests/test_gce_encoding_worker.py index 2b95e3c2..fbe111fe 100644 --- a/backend/tests/test_gce_encoding_worker.py +++ b/backend/tests/test_gce_encoding_worker.py @@ -733,3 +733,17 @@ class TestVerifyWheelImports: monkeypatch.setattr(main.subprocess, "run", _timeout) assert main.verify_wheel_imports() is False + + +class TestGcsCli: + """The AWS worker (WIF auth) can't use gsutil; GCE workers keep it.""" + + def test_gce_worker_uses_gsutil(self, monkeypatch): + from backend.services.gce_encoding import main as worker + monkeypatch.delenv("ENCODING_WORKER_PROVIDER", raising=False) + assert worker._gcs_cli() == ["gsutil"] + + def test_aws_worker_uses_gcloud_storage(self, monkeypatch): + from backend.services.gce_encoding import main as worker + monkeypatch.setenv("ENCODING_WORKER_PROVIDER", "aws") + assert worker._gcs_cli() == ["gcloud", "storage"] diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 5a169fb8..4f1ef471 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -476,6 +476,23 @@ and raised as `EncodingJobLostError`, which `run_with_lost_job_resubmit` resubmi — the resubmit's connection failure then starts the Spot VM again, or the on-demand fallback if Spot capacity is gone. +### AWS encoding worker (EC2, provider "aws") + +**`aws-encoding-worker-1`** is a c7a.8xlarge persistent Spot instance in us-east-2. It runs the same `gce_encoding` service and is one more entry in the `encoding-worker-fallback-vms` pool (`"provider": "aws"`, plus `instance_id` and `region`). It is paid for by AWS Activate credits. IaC lives in `infrastructure/aws/` (its own Pulumi stack; see its README). + +- **Control:** + - The manager routes AWS VMs to `backend/services/aws_encoding_compute.py`, an EC2 adapter that mimics the compute client. EC2 states map to GCE statuses, and EC2 capacity errors join `CAPACITY_ERROR_CODES`. + - So failover, overrides, lost-job detection and stockout cooldowns apply unchanged. + - The override records `active_override_provider: "aws"`. The CI blue-green deploy and the idle Cloud Function both ignore the AWS worker. +- **Ranking:** + - It is a last resort by default (`SPEED_RANK["c7a"] = 90`). + - Firestore `config/encoding-worker.speed_rank_overrides = {"c7a": 5}` makes it preferred with no deploy. + - Both warmups (`ensure_preferred_running`) follow the same ranking, so they warm the worker requests are routed to. +- **Auth and lifecycle:** + - GCP access is Workload Identity Federation as `encoding-worker@`: no keys. + - It stops itself when idle (`infrastructure/encoding-worker/aws/idle-watchdog.py`). + - It gets new code at boot and per job, with no blue-green deploy. + ### Architecture ```text diff --git a/docs/archive/2026-10-29-aws-encoding-worker-plan.md b/docs/archive/2026-10-29-aws-encoding-worker-plan.md new file mode 100644 index 00000000..ba680cde --- /dev/null +++ b/docs/archive/2026-10-29-aws-encoding-worker-plan.md @@ -0,0 +1,91 @@ +# AWS encoding worker: implementation plan (2026-10-29) + +**Goal:** run gen's encoding worker on EC2 Spot, paid for with the AWS Activate credits (acct 790139457751, expiring 2028-10-31). Then run distribution (Dropbox/YouTube/Drive uploads) from AWS too, so the large output egress moves off the GCP bill. **Constraint:** job→published must not get slower, and the move runs in parallel with GCP first. + +Evidence: workspace `docs/archive/2026-10-29-aws-encoding-worker-migration-plan.md`. A c7a.4xlarge matched or beat the GCE c4d-highcpu-16 on 3 replayed prod jobs. + +## Phase 1: AWS box as an encoding-worker candidate (this PR set) + +The box runs the **unchanged worker HTTP service** (`backend.services.gce_encoding.main`). It joins the existing candidate pool, so failover, overrides, lost-job detection and cooldowns all apply unchanged. + +### Design + +**Instance** +- `aws-encoding-worker-1`: **c7a.8xlarge** (32 vCPU / 64 GB), us-east-2. +- A **persistent Spot** request with `stop` on interruption, matching GCE's STOP-on-preempt, plus an Elastic IP. +- Port 8080 open with API-key auth, the same as the GCE firewall today. +- The root EBS volume persists across stop/start, like the GCE boot disks. It is provisioned once by user-data (a port of packer `provision.sh`). +- **Why 8xlarge:** ~100 h/mo × ~$0.46/h ≈ $46/mo of *credits* (we accrue ~$205/mo of credits). It encodes faster than today's 16 vCPU and fits two heavy jobs in RAM. + +**GCP auth** +- Workload Identity Federation: an `aws-encoding` pool plus an AWS provider for acct 790139457751. +- The `nomad-encoding-worker` role is bound to the existing **encoding-worker SA** (`roles/iam.workloadIdentityUser`). No SA keys. +- The box uses an `external_account` credential config (not secret) with IMDSv2, for both the client libraries and `gcloud`. + +**Backend control of EC2** +- An IAM user `nomad-encoding-controller` limited to `ec2:Start/Stop/DescribeInstances` on instances tagged `Project=nomad-encoding`. +- Its keys go in Secret Manager `aws-encoding-controller-credentials`, the same pattern as backup-to-aws; the backend SA can read it. + +**Registry** +- The box is an entry in the existing `encoding-worker-fallback-vms` secret: + `{"vm":"aws-encoding-worker-1","zone":"us-east-2b","ip":,"machine_type":"c7a-8xlarge","provider":"aws","instance_id":"i-…","region":"us-east-2"}` +- The `.` in the AWS type becomes `-`, so Firestore cooldown keys stay valid. + +**Manager** +- `AwsComputeAdapter` (`backend/services/aws_encoding_compute.py`) mimics the `compute_v1.InstancesClient` surface the manager uses (`get/start/stop`). It maps EC2 states to GCE ones (`running→RUNNING`, `pending→STAGING`, `stopping→STOPPING`, `stopped→TERMINATED`), EC2 capacity errors to `CAPACITY_ERROR_CODES`, and `InvalidInstanceID.NotFound` to `NotFound`. +- `EncodingWorkerManager._compute_for(vm)` routes AWS VMs to the adapter. GCE code paths are untouched. + +**Ranking** +- `SPEED_RANK["c7a"] = 90`, so by default AWS is a **last-resort** fallback (stage A). +- A new Firestore field `config/encoding-worker.speed_rank_overrides` (e.g. `{"c7a": 5}`) makes it **preferred** (stage C) with no deploy. Rollback deletes the field. + +**Warmups** +- `worker_service._warmup_encoding_worker` and `POST /internal/encoding-worker/warmup` currently start only the GCE primary. +- They will use the same ranked path as the encode itself (`manager.ensure_preferred_running()`). Otherwise, with AWS preferred, they would warm a VM that isn't routed to. + +**Exclusions from GCE-only machinery** +- CI blue-green green selection: `deploy_promote.select_green_candidates` and the jq in ci.yml. +- The idle Cloud Function's `_parse_fallback_vms`. +- The box gets new code through the existing per-job `ensure_latest_wheel` hot update, so it needs no blue-green deploy. + +**Idle stop** +- A systemd timer **on the box**, independent of the uvicorn process, runs every 2 min. +- It stops the instance (`shutdown -h now`, instance-initiated shutdown = stop) when either: + - `/health` reports `active_jobs == 0`, and `last_activity_at` is older than 5 min or the box isn't the routed worker; or + - `/health` has failed for 15 min. +- This mirrors the idle function's rules. + +**Worker boot on AWS** +- `startup-aws.sh` has no GCE metadata server. It fetches the API key and wheel with `gcloud` over WIF and sets `ENCODING_WORKER_VM_NAME`. +- `ensure_latest_wheel` uses `gsutil`, which doesn't support WIF credential files. On AWS (`ENCODING_WORKER_PROVIDER=aws`) it uses `gcloud storage`; the GCE path is unchanged. + +### Rollout +- **A:** deploy and register the box at rank 90. It's only exercised on c4d+c2d stockouts. +- **B:** set `speed_rank_overrides {"c7a": 5}` for a supervised window. Run the daily E2E plus admin test jobs, and compare render/encode times and outputs (including the portrait parity check) against GCE. +- **C:** leave the override on; AWS is preferred. After a week clean, delete the c4d pair (Pulumi), leaving c2df as the GCP fallback. + +### Testing +- **Unit:** + - adapter (botocore `Stubber`): state map, capacity/NotFound mapping, start/stop idempotency + - manager routing (`_compute_for`), `ensure_preferred_running` + - preference `rank_overrides`, `c7a` family + - fallback parsing passthrough of `provider`/`instance_id`/`region` + - deploy_promote and idle-fn exclusion of aws entries + - the idle watchdog decision function + - `ensure_latest_wheel` command selection + - Extend the existing test files listed in the GCE-coupling map. +- **Integration (prod, supervised):** stage B runs real jobs through the AWS worker. The prod daily E2E (payment + happy path) is the end-to-end check. + +## Phase 2: distribution from AWS (next PR set, after Phase 1 is verified) + +The video worker (`backend.workers.video_worker`) runs **on the box** with local encoding (`LocalEncodingBackend`): +- Inputs come from GCS once (~300 MB). +- Finals are encoded locally, uploaded to Dropbox/YouTube/Drive from AWS, and pushed to GCS `finals/` (AWS egress, on credits). +- This removes the ~$18/mo of Cloud Run internet egress and the video-encoding-job CPU (~$11/mo). + +Needs: +- a `/run-worker` endpoint and env/secrets on the box (the video-encoding-job env list); +- a trigger branch in `WorkerService.trigger_video_worker`, with Cloud Run as the fallback; +- the box's WIF identity mapped to the backend SA's permissions; +- **Spot-interruption recovery for in-flight runs.** Today nothing recovers a video worker that dies after `GENERATING_VIDEO`; build a supervised re-dispatch first; +- the deferred YouTube queue processor runs on the API, so it still egresses from GCP (low volume). diff --git a/infrastructure/__main__.py b/infrastructure/__main__.py index a7fb341b..5b563c8d 100644 --- a/infrastructure/__main__.py +++ b/infrastructure/__main__.py @@ -163,6 +163,12 @@ encoding_worker_iam_bindings = worker_sas.grant_encoding_worker_permissions( encoding_worker_sa, bucket ) +# AWS encoding worker (EC2) → GCP as encoding-worker@ via Workload Identity +# Federation (no keys). See modules/aws_encoding_wif.py. +from modules import aws_encoding_wif # noqa: E402 + +aws_encoding_wif_resources = aws_encoding_wif.create_aws_encoding_wif(encoding_worker_sa) + gdrive_validator_sa = worker_sas.create_gdrive_validator_service_account() gdrive_validator_iam_bindings = worker_sas.grant_gdrive_validator_permissions( gdrive_validator_sa diff --git a/infrastructure/aws/Pulumi.prod.yaml b/infrastructure/aws/Pulumi.prod.yaml new file mode 100644 index 00000000..bdd65715 --- /dev/null +++ b/infrastructure/aws/Pulumi.prod.yaml @@ -0,0 +1,2 @@ +config: + aws:region: us-east-2 diff --git a/infrastructure/aws/Pulumi.yaml b/infrastructure/aws/Pulumi.yaml new file mode 100644 index 00000000..52f09365 --- /dev/null +++ b/infrastructure/aws/Pulumi.yaml @@ -0,0 +1,6 @@ +name: nomad-aws +runtime: + name: python + options: + virtualenv: venv +description: Nomad Karaoke AWS account (790139457751) — the EC2 encoding worker. See README.md. diff --git a/infrastructure/aws/README.md b/infrastructure/aws/README.md new file mode 100644 index 00000000..f7edbbfd --- /dev/null +++ b/infrastructure/aws/README.md @@ -0,0 +1,43 @@ +# nomad-aws: Nomad Karaoke's AWS account (790139457751) + +This stack holds the **AWS encoding worker**, an EC2 Spot instance running the same `gce_encoding` HTTP service as the GCE encoding VMs. It is paid for by AWS Activate credits ($5k, expiring 2028-10-31). The plan and benchmark are in `docs/archive/2026-10-29-aws-encoding-worker-plan.md`. + +## What's here + +| Resource | Purpose | +|---|---| +| `aws-encoding-worker-1` | c7a.8xlarge **persistent Spot**, us-east-2b. Stops on interruption and on OS shutdown; the EBS root survives. The Elastic IP serves `:8080`. | +| `nomad-encoding-worker` role | The box's identity. GCP federates it to `encoding-worker@nomadkaraoke` (WIF pool `aws-encoding`, in the GCP stack's `modules/aws_encoding_wif.py`). It also has SSM for a shell. | +| `nomad-encoding-controller` user | Backend rights: start/stop instances tagged `Project=nomad-encoding`, plus describe. | + +`provision.sh` (in `infrastructure/encoding-worker/aws/`) runs once as user-data. Every boot after that runs `bootstrap.sh` → `startup-aws.sh`, which CI uploads to GCS. The new wheel installs at boot and per job, so there is **no blue-green deploy** for this box. `idle-watchdog.py` stops the box when it's idle. + +## Apply + +`pulumi up` runs locally and never in CI, with Nomad's AWS keys from the workspace `.envrc`: + +```bash +cd infrastructure/aws && python3 -m venv venv && ./venv/bin/pip install -r requirements.txt +AWS_ACCESS_KEY_ID=$NOMAD_AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$NOMAD_AWS_SECRET_ACCESS_KEY \ + pulumi up --stack nomadkaraoke/prod +``` + +`ami` and `user_data` are `ignore_changes`, so a newer Ubuntu or an edited `provision.sh` never replaces the worker's disk. To re-provision on purpose, use `pulumi up --replace `. + +## One-time wiring after the first `up` + +1. **Controller key → GCP secret** (needs `/gcp-admin`): + ```bash + aws iam create-access-key --user-name nomad-encoding-controller \ + | jq -c '{access_key_id:.AccessKey.AccessKeyId, secret_access_key:.AccessKey.SecretAccessKey}' \ + | scripts/gcp-admin gcloud secrets versions add aws-encoding-controller-credentials --data-file=- --project=nomadkaraoke + ``` +2. **Register the worker in the pool:** append `pulumi stack output fallback_vms_entry` to the JSON list in secret `encoding-worker-fallback-vms`. The backend picks up `:latest` on new instances, and the next deploy picks it up everywhere. +3. **Rank:** by default the AWS worker is a **last resort** (`SPEED_RANK["c7a"] = 90`). To prefer it, set Firestore `config/encoding-worker.speed_rank_overrides = {"c7a": 5}`. Delete the field to roll back; no deploy is needed either way. + +## Operating it + +- **Shell:** `aws ssm start-session --target --region us-east-2`. +- **Logs:** `journalctl -u encoding-worker`, `/var/log/encoding-worker-startup.log`, `journalctl -u encoding-worker-idle`. They are **not** in Cloud Logging. +- **Health:** `curl http://:8080/health`. +- **Spot interruption:** EC2 *stops* the box. An in-flight encode/render is detected as lost (the box is no longer RUNNING) and resubmitted. If c7a capacity is short, the start fails with `InsufficientInstanceCapacity`; that is recorded in `capacity_state`, demoted for 15 min, and GCE takes over. diff --git a/infrastructure/aws/__main__.py b/infrastructure/aws/__main__.py new file mode 100644 index 00000000..8ba28086 --- /dev/null +++ b/infrastructure/aws/__main__.py @@ -0,0 +1,137 @@ +"""Nomad Karaoke AWS account: the EC2 encoding worker. + +Paid for by the AWS Activate credits ($5k, acct 790139457751, exp 2028-10-31). +The worker runs the same ``gce_encoding`` HTTP service as the GCE encoding VMs +and is one more candidate in the backend's encoding-worker pool (provider +"aws"). Plan: karaoke-gen docs/archive/2026-10-29-aws-encoding-worker-plan.md. + +- Persistent Spot instance, ``stop`` on interruption and on OS shutdown (the idle + watchdog), so the EBS root (venv, caches) survives like a GCE boot disk. +- Elastic IP + port 8080 (API-key auth, same as the GCE firewall). +- GCP access via Workload Identity Federation as encoding-worker@ (the + ``nomad-encoding-worker`` role is the federated identity; see + infrastructure/modules/aws_encoding_wif.py in the GCP stack). +- ``nomad-encoding-controller`` IAM user: the backend's start/stop/describe + rights, scoped by tag. Its access key is created by hand and stored in GCP + Secret Manager ``aws-encoding-controller-credentials`` (see README). +""" + +import json +from pathlib import Path + +import pulumi +import pulumi_aws as aws + +PROJECT_TAG = {"Project": "nomad-encoding"} +VM_NAME = "aws-encoding-worker-1" +INSTANCE_TYPE = "c7a.8xlarge" # 32 vCPU / 64 GB — see the plan doc for sizing +AZ = "us-east-2b" +WIF_PROVIDER = ( + "projects/718638054799/locations/global/workloadIdentityPools/" + "aws-encoding/providers/aws-790139457751" +) + +# --- Worker identity (federated into GCP; also SSM for operator shell) ---------- +role = aws.iam.Role( + "worker-role", + name="nomad-encoding-worker", + assume_role_policy=json.dumps({ + "Version": "2012-10-17", + "Statement": [{"Effect": "Allow", "Principal": {"Service": "ec2.amazonaws.com"}, "Action": "sts:AssumeRole"}], + }), + tags=PROJECT_TAG, +) +aws.iam.RolePolicyAttachment( + "worker-ssm", + role=role.name, + policy_arn="arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore", +) +profile = aws.iam.InstanceProfile("worker-profile", name="nomad-encoding-worker", role=role.name) + +# --- Backend controller (start/stop/describe the tagged worker) ----------------- +controller = aws.iam.User("controller", name="nomad-encoding-controller", tags=PROJECT_TAG) +aws.iam.UserPolicy( + "controller-policy", + user=controller.name, + policy=json.dumps({ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["ec2:StartInstances", "ec2:StopInstances"], + "Resource": "arn:aws:ec2:*:790139457751:instance/*", + "Condition": {"StringEquals": {"aws:ResourceTag/Project": "nomad-encoding"}}, + }, + {"Effect": "Allow", "Action": ["ec2:DescribeInstances"], "Resource": "*"}, + ], + }), +) + +# --- Network --------------------------------------------------------------------- +vpc = aws.ec2.get_vpc(default=True) +subnet = aws.ec2.get_subnet(filters=[ + aws.ec2.GetSubnetFilterArgs(name="vpc-id", values=[vpc.id]), + aws.ec2.GetSubnetFilterArgs(name="availability-zone", values=[AZ]), + aws.ec2.GetSubnetFilterArgs(name="default-for-az", values=["true"]), +]) +sg = aws.ec2.SecurityGroup( + "worker-sg", + name="nomad-encoding-worker", + description="Encoding worker HTTP (API-key auth), same as the GCE encoding-worker firewall", + vpc_id=vpc.id, + ingress=[aws.ec2.SecurityGroupIngressArgs(protocol="tcp", from_port=8080, to_port=8080, cidr_blocks=["0.0.0.0/0"])], + egress=[aws.ec2.SecurityGroupEgressArgs(protocol="-1", from_port=0, to_port=0, cidr_blocks=["0.0.0.0/0"])], + tags=PROJECT_TAG, +) + +# --- Instance -------------------------------------------------------------------- +ami = aws.ssm.get_parameter(name="/aws/service/canonical/ubuntu/server/24.04/stable/current/amd64/hvm/ebs-gp3/ami-id") +provision = Path(__file__).parent.parent.joinpath("encoding-worker", "aws", "provision.sh").read_text() +user_data = provision.replace( + "set -euxo pipefail\n", + f"set -euxo pipefail\nexport WIF_PROVIDER='{WIF_PROVIDER}' VM_NAME='{VM_NAME}'\n", + 1, +) + +instance = aws.ec2.Instance( + "worker", + ami=ami.value, + instance_type=INSTANCE_TYPE, + subnet_id=subnet.id, + vpc_security_group_ids=[sg.id], + iam_instance_profile=profile.name, + user_data=user_data, + instance_initiated_shutdown_behavior="stop", + instance_market_options=aws.ec2.InstanceInstanceMarketOptionsArgs( + market_type="spot", + spot_options=aws.ec2.InstanceInstanceMarketOptionsSpotOptionsArgs( + spot_instance_type="persistent", + instance_interruption_behavior="stop", + ), + ), + metadata_options=aws.ec2.InstanceMetadataOptionsArgs(http_tokens="required", http_put_response_hop_limit=2), + root_block_device=aws.ec2.InstanceRootBlockDeviceArgs( + volume_size=100, volume_type="gp3", iops=6000, throughput=400, delete_on_termination=True, + tags={**PROJECT_TAG, "Name": VM_NAME}, + ), + tags={**PROJECT_TAG, "Name": VM_NAME}, + # The root volume is the worker's state (like a GCE boot disk): a newer + # Ubuntu AMI or an edited provision.sh must never silently replace it. + opts=pulumi.ResourceOptions(ignore_changes=["ami", "user_data"]), +) +eip = aws.ec2.Eip("worker-eip", domain="vpc", tags={**PROJECT_TAG, "Name": VM_NAME}) +aws.ec2.EipAssociation("worker-eip-assoc", instance_id=instance.id, allocation_id=eip.id) + +pulumi.export("vm_name", VM_NAME) +pulumi.export("instance_id", instance.id) +pulumi.export("public_ip", eip.public_ip) +pulumi.export("availability_zone", instance.availability_zone) +pulumi.export( + "fallback_vms_entry", + pulumi.Output.all(instance.id, eip.public_ip, instance.availability_zone).apply( + lambda a: json.dumps({ + "vm": VM_NAME, "zone": a[2], "ip": a[1], "machine_type": INSTANCE_TYPE.replace(".", "-"), + "provider": "aws", "instance_id": a[0], "region": "us-east-2", + }) + ), +) diff --git a/infrastructure/aws/requirements.txt b/infrastructure/aws/requirements.txt new file mode 100644 index 00000000..9a1b8d0f --- /dev/null +++ b/infrastructure/aws/requirements.txt @@ -0,0 +1,2 @@ +pulumi>=3.0.0,<4.0.0 +pulumi-aws>=7.0.0,<8.0.0 diff --git a/infrastructure/encoding-worker/aws/idle-watchdog.py b/infrastructure/encoding-worker/aws/idle-watchdog.py new file mode 100644 index 00000000..8f58ffd3 --- /dev/null +++ b/infrastructure/encoding-worker/aws/idle-watchdog.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Idle self-stop for the AWS encoding worker (systemd timer, every 2 min). + +The GCE workers are stopped by the encoding-worker-idle Cloud Function; the AWS +worker isn't a GCE VM, so it stops itself with the same rules. This runs as its +own systemd unit (not inside uvicorn), so a wedged worker process still gets +stopped: + +- busy (``/health`` active_jobs > 0) → keep running +- idle and not the routed worker → stop +- idle, routed, last_activity_at older than 5 min → stop +- health or Firestore unreadable for 15 min → stop (never run forever) + +"Stop" = ``shutdown -h now``; the instance's shutdown behaviour is ``stop``, so +the EBS root (venv, caches) survives and the backend restarts it on demand. +CI uploads this to gs://karaoke-gen-storage-nomadkaraoke/encoding-worker/idle-watchdog.py. +""" + +from __future__ import annotations + +import json +import subprocess +import urllib.request +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Optional + +IDLE_TIMEOUT = timedelta(minutes=5) +UNKNOWN_GRACE = timedelta(minutes=15) +STATE_FILE = Path("/var/lib/encoding-worker-idle/unknown-since") +VM_NAME_FILE = Path("/opt/encoding-worker/vm-name") + + +def decide( + *, + now: datetime, + active_jobs: Optional[int], + config: Optional[dict], + vm_name: str, + unknown_since: Optional[datetime], +) -> tuple[str, str]: + """Return ("stop" | "keep" | "unknown", reason). Pure, for tests. + + ``active_jobs``/``config`` are None when /health or Firestore couldn't be + read. "unknown" asks the caller to start (or keep) the grace clock. + """ + if active_jobs is not None and active_jobs > 0: + return "keep", f"{active_jobs} active job(s)" + if active_jobs is None or config is None: + what = "health" if active_jobs is None else "Firestore config" + if unknown_since and now - unknown_since >= UNKNOWN_GRACE: + return "stop", f"{what} unreadable since {unknown_since.isoformat()}" + return "unknown", f"{what} unreadable" + if config.get("active_override_vm") != vm_name: + return "stop", f"idle and not the routed worker (routed: {config.get('active_override_vm')})" + last = config.get("last_activity_at") + try: + last_dt = datetime.fromisoformat(last) if last else None + except (TypeError, ValueError): + last_dt = None + if last_dt is not None and last_dt.tzinfo is None: + last_dt = last_dt.replace(tzinfo=timezone.utc) + if last_dt is None or now - last_dt >= IDLE_TIMEOUT: + return "stop", f"idle since {last or 'never'}" + return "keep", f"routed, last activity {last}" + + +def _active_jobs() -> Optional[int]: + try: + with urllib.request.urlopen("http://127.0.0.1:8080/health", timeout=10) as r: + return int(json.load(r).get("active_jobs", 0)) + except Exception: # noqa: BLE001 + return None + + +def _config() -> Optional[dict]: + try: + from google.cloud import firestore + + snap = firestore.Client(project="nomadkaraoke").collection("config").document("encoding-worker").get() + return snap.to_dict() if snap.exists else None + except Exception: # noqa: BLE001 + return None + + +def main() -> None: + now = datetime.now(timezone.utc) + vm_name = VM_NAME_FILE.read_text().strip() if VM_NAME_FILE.exists() else "" + try: + unknown_since = datetime.fromisoformat(STATE_FILE.read_text().strip()) + except (OSError, ValueError): + unknown_since = None + + action, reason = decide( + now=now, active_jobs=_active_jobs(), config=_config(), vm_name=vm_name, unknown_since=unknown_since + ) + print(f"idle-watchdog: {action}: {reason}") + if action == "unknown": + if unknown_since is None: + STATE_FILE.parent.mkdir(parents=True, exist_ok=True) + STATE_FILE.write_text(now.isoformat()) + return + STATE_FILE.unlink(missing_ok=True) + if action == "stop": + subprocess.run(["shutdown", "-h", "now"], check=False) + + +if __name__ == "__main__": + main() diff --git a/infrastructure/encoding-worker/aws/provision.sh b/infrastructure/encoding-worker/aws/provision.sh new file mode 100755 index 00000000..2b61c610 --- /dev/null +++ b/infrastructure/encoding-worker/aws/provision.sh @@ -0,0 +1,117 @@ +#!/bin/bash +# One-time provisioning of the AWS encoding worker (EC2 user-data, first boot). +# +# AWS counterpart of infrastructure/packer/scripts/provision.sh: same static +# ffmpeg, fonts, Python 3.13 venv with the full karaoke-gen dependency tree +# (CPU-only torch), and the same systemd unit layout under /opt/encoding-worker +# (self_heal.running_on_worker_vm keys off /opt/encoding-worker/bootstrap.sh). +# The root EBS volume persists across stop/start, so this runs once. +# +# GCP auth: Workload Identity Federation (no keys). The credential config is a +# non-secret JSON pointing at the AWS IMDSv2 role creds; gcloud and the client +# libraries both use it to impersonate encoding-worker@nomadkaraoke. +set -euxo pipefail +exec > >(tee -a /var/log/encoding-worker-provision.log) 2>&1 + +WORKER_DIR=/opt/encoding-worker +BUCKET=gs://karaoke-gen-storage-nomadkaraoke +WIF_PROVIDER="${WIF_PROVIDER:?}" # projects//locations/global/workloadIdentityPools//providers/ +WORKER_SA="${WORKER_SA:-encoding-worker@nomadkaraoke.iam.gserviceaccount.com}" +VM_NAME="${VM_NAME:-aws-encoding-worker-1}" + +export DEBIAN_FRONTEND=noninteractive +apt-get update -q +apt-get install -y -q curl xz-utils unzip git build-essential libffi-dev libssl-dev \ + fonts-noto fonts-noto-cjk fontconfig apt-transport-https ca-certificates gnupg + +# Google Cloud CLI (gcloud storage / secrets over WIF) +curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg +echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" \ + > /etc/apt/sources.list.d/google-cloud-sdk.list +apt-get update -q && apt-get install -y -q google-cloud-cli + +# Static FFmpeg 7.x (same source as the GCE image) +curl -fsSL https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz -o /tmp/ff.tar.xz +tar -xf /tmp/ff.tar.xz -C /tmp && cp /tmp/ffmpeg-*-amd64-static/ff{mpeg,probe} /usr/local/bin/ && rm -rf /tmp/ff* + +mkdir -p "${WORKER_DIR}" +gcloud iam workload-identity-pools create-cred-config "${WIF_PROVIDER}" \ + --service-account="${WORKER_SA}" --aws --enable-imdsv2 \ + --output-file="${WORKER_DIR}/gcp-wif.json" +gcloud auth login --cred-file="${WORKER_DIR}/gcp-wif.json" --quiet +gcloud config set project nomadkaraoke --quiet +export GOOGLE_APPLICATION_CREDENTIALS="${WORKER_DIR}/gcp-wif.json" + +# Python 3.13 venv (uv-managed interpreter; --seed so pip exists for startup/ensure_latest_wheel) +curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR=/usr/local/bin sh +UV_PYTHON_INSTALL_DIR=/opt/python uv venv --seed --python 3.13 "${WORKER_DIR}/venv" +VERSION=$(gcloud storage cat "${BUCKET}/encoding-worker/version.txt" | tr -d '[:space:]') +WHEEL="/tmp/karaoke_gen-${VERSION}-py3-none-any.whl" +gcloud storage cp "${BUCKET}/wheels/karaoke_gen-current.whl" "${WHEEL}" +"${WORKER_DIR}/venv/bin/pip" install --quiet fastapi uvicorn google-cloud-storage aiofiles aiohttp packaging +"${WORKER_DIR}/venv/bin/pip" install --quiet \ + --index-url https://download.pytorch.org/whl/cpu --extra-index-url https://pypi.org/simple "${WHEEL}" +rm -f "${WHEEL}" + +# Bootstrap (ExecStartPre): fetch the latest startup-aws.sh from GCS, else the baked copy. +cat > "${WORKER_DIR}/bootstrap.sh" <<'BOOT' +#!/bin/bash +set -e +if gcloud storage cp gs://karaoke-gen-storage-nomadkaraoke/encoding-worker/startup-aws.sh /opt/encoding-worker/startup-aws.sh.new; then + mv /opt/encoding-worker/startup-aws.sh.new /opt/encoding-worker/startup-aws.sh +fi +exec bash /opt/encoding-worker/startup-aws.sh +BOOT +chmod +x "${WORKER_DIR}/bootstrap.sh" +gcloud storage cp "${BUCKET}/encoding-worker/startup-aws.sh" "${WORKER_DIR}/startup-aws.sh" +gcloud storage cp "${BUCKET}/encoding-worker/idle-watchdog.py" "${WORKER_DIR}/idle-watchdog.py" +echo "${VM_NAME}" > "${WORKER_DIR}/vm-name" + +cat > /etc/systemd/system/encoding-worker.service < /etc/systemd/system/encoding-worker-idle.service < /etc/systemd/system/encoding-worker-idle.timer < >(tee -a /var/log/encoding-worker-startup.log) 2>&1 + +echo "=== AWS encoding worker startup: $(date) ===" +rm -rf "${TMP_DIR}" && mkdir -p "${TMP_DIR}" + +echo "[1/5] Fetching API key from Secret Manager (WIF)..." +ENCODING_API_KEY="" +for attempt in $(seq 1 10); do + ENCODING_API_KEY=$(gcloud secrets versions access latest --secret=encoding-worker-api-key --project=nomadkaraoke 2>/dev/null || echo "") + [ -n "$ENCODING_API_KEY" ] && break + echo "API key fetch failed (attempt ${attempt}/10), retrying in 5s..." + sleep 5 +done +if [ -z "$ENCODING_API_KEY" ]; then + echo "FATAL: No API key from Secret Manager; exiting so systemd retries" + exit 1 +fi +VM_NAME=$(cat "${WORKER_DIR}/vm-name" 2>/dev/null || hostname) + +cat > "${WORKER_DIR}/env" </dev/null | tr -d '[:space:]' || echo "") +echo "Expected version: ${EXPECTED_VERSION:-unknown}" + +echo "[3/5] Downloading current wheel..." +WHEEL_PATH="${TMP_DIR}/karaoke_gen-${EXPECTED_VERSION:-0.0.0}-py3-none-any.whl" +gcloud storage cp "${BUCKET}/wheels/karaoke_gen-current.whl" "${WHEEL_PATH}" + +echo "[4/5] Installing wheel (code only; deps are baked by provision.sh)..." +"${VENV}/bin/pip" install --upgrade --quiet --force-reinstall --no-deps "${WHEEL_PATH}" + +echo "[5/5] Verifying installed version..." +INSTALLED_VERSION=$("${VENV}/bin/pip" show karaoke-gen 2>/dev/null | grep "^Version:" | cut -d' ' -f2 | tr -d '[:space:]') +if [ -n "$EXPECTED_VERSION" ] && [ "${INSTALLED_VERSION}" != "${EXPECTED_VERSION}" ]; then + echo "FATAL: version mismatch (expected ${EXPECTED_VERSION}, installed ${INSTALLED_VERSION})" + exit 1 +fi +rm -rf "${TMP_DIR}" +echo "=== Startup complete: karaoke-gen v${INSTALLED_VERSION} on ${VM_NAME} ===" diff --git a/infrastructure/encoding-worker/deploy_promote.py b/infrastructure/encoding-worker/deploy_promote.py index 359b8181..129b0ec6 100644 --- a/infrastructure/encoding-worker/deploy_promote.py +++ b/infrastructure/encoding-worker/deploy_promote.py @@ -85,9 +85,13 @@ def select_green_candidates( # Only complete entries are usable — a fallback missing vm/zone/ip would # produce an unstartable candidate (e.g. `gcloud ... --zone=None`) and # silently drop capacity from the selection path. + # AWS workers (provider "aws") are excluded: this workflow starts/stops + # greens with `gcloud compute`, and the AWS worker hot-updates its wheel per + # job instead of being blue-green deployed. usable = [ f for f in (fallback_vms or []) if f.get("vm") and f.get("zone") and f.get("ip") and f.get("vm") != override_vm + and (f.get("provider") or "").lower() != "aws" ] for f in usable: pool.append({ @@ -101,6 +105,10 @@ def select_green_candidates( return ordered_candidates(pool, capacity_state=config.get("capacity_state")) +def _is_aws_override(config: Dict[str, Any]) -> bool: + return (config.get("active_override_provider") or "").lower() == "aws" + + def promote_plan( green: Dict[str, Any], config: Dict[str, Any], @@ -152,14 +160,17 @@ def promote_plan( "ip": config.get("primary_ip"), "zone": config.get("primary_zone") or DEFAULT_C4D_ZONE, }) - # A stale override must be cleared so active_url follows the fresh primary. - if config.get("active_override_vm"): + # A stale override must be cleared so active_url follows the fresh primary — + # except an AWS override, which is serving by preference (rank override), + # can't be gcloud-stopped, and picks up the new wheel on its next job. + if config.get("active_override_vm") and not _is_aws_override(config): updates.update({ "active_override_vm": None, "active_override_ip": None, "active_override_zone": None, "active_override_set_at": None, "active_override_version": None, + "active_override_provider": None, }) drain.append({ "vm": config["active_override_vm"], @@ -175,11 +186,14 @@ def promote_plan( "active_override_zone": green["zone"], "active_override_set_at": now, "active_override_version": version, + # The green is a GCE VM: never leave a stale "aws" tag on it (CI hides + # an AWS override from this flow, so it may be the override replaced). + "active_override_provider": None, "deploy_in_progress": False, "deploy_in_progress_since": None, } old_override = config.get("active_override_vm") - if old_override and old_override != green["vm"]: + if old_override and old_override != green["vm"] and not _is_aws_override(config): drain.append({ "vm": old_override, "ip": config.get("active_override_ip"), diff --git a/infrastructure/functions/encoding_worker_idle/main.py b/infrastructure/functions/encoding_worker_idle/main.py index c87a51cb..db398377 100644 --- a/infrastructure/functions/encoding_worker_idle/main.py +++ b/infrastructure/functions/encoding_worker_idle/main.py @@ -144,6 +144,10 @@ def _parse_fallback_vms(): fallbacks = [] for item in parsed: try: + if (item.get("provider") or "").lower() == "aws": + # The AWS worker is not a GCE VM; it idle-stops itself + # (infrastructure/encoding-worker/aws/idle-watchdog.py). + continue fallbacks.append((item["vm"], item["zone"])) except (KeyError, TypeError) as exc: logger.warning(f"Skipping malformed fallback entry {item}: {exc}") diff --git a/infrastructure/functions/encoding_worker_idle/test_idle_shutdown.py b/infrastructure/functions/encoding_worker_idle/test_idle_shutdown.py index 82780084..7d7e9c77 100644 --- a/infrastructure/functions/encoding_worker_idle/test_idle_shutdown.py +++ b/infrastructure/functions/encoding_worker_idle/test_idle_shutdown.py @@ -505,6 +505,18 @@ class TestParseFallbackVms: monkeypatch.setenv("ENCODING_WORKER_FALLBACK_VMS", "not json") assert main._parse_fallback_vms() == [] + def test_skips_aws_worker(self, monkeypatch): + # The AWS encoding worker isn't a GCE VM; it idle-stops itself. + monkeypatch.setenv( + "ENCODING_WORKER_FALLBACK_VMS", + json.dumps([ + {"vm": "fb-a", "zone": "us-central1-a", "ip": "1.1.1.1"}, + {"vm": "aws-encoding-worker-1", "zone": "us-east-2b", "ip": "3.3.3.3", + "provider": "aws", "instance_id": "i-0abc", "region": "us-east-2"}, + ]), + ) + assert main._parse_fallback_vms() == [("fb-a", "us-central1-a")] + def test_returns_tuples_of_vm_zone(self, monkeypatch): monkeypatch.setenv( "ENCODING_WORKER_FALLBACK_VMS", diff --git a/infrastructure/modules/aws_encoding_wif.py b/infrastructure/modules/aws_encoding_wif.py new file mode 100644 index 00000000..2e56ab8a --- /dev/null +++ b/infrastructure/modules/aws_encoding_wif.py @@ -0,0 +1,59 @@ +"""Workload Identity Federation for the AWS encoding worker. + +The AWS encoding worker (EC2 in Nomad's AWS account) authenticates to GCP as +the existing ``encoding-worker`` service account with no keys: the instance's +IAM role ``nomad-encoding-worker`` is exchanged for a short-lived GCP token via +this pool/provider. The box's credential config is generated from the provider +name in infrastructure/encoding-worker/aws/provision.sh. +""" + +import pulumi +import pulumi_gcp as gcp + +from config import PROJECT_ID + +AWS_ACCOUNT_ID = "790139457751" +AWS_WORKER_ROLE = "nomad-encoding-worker" +POOL_ID = "aws-encoding" +PROVIDER_ID = f"aws-{AWS_ACCOUNT_ID}" + + +def create_aws_encoding_wif(encoding_worker_sa: gcp.serviceaccount.Account) -> dict: + pool = gcp.iam.WorkloadIdentityPool( + "aws-encoding-wif-pool", + workload_identity_pool_id=POOL_ID, + display_name="AWS encoding worker", + description="EC2 encoding worker in Nomad's AWS account (karaoke-gen)", + ) + provider = gcp.iam.WorkloadIdentityPoolProvider( + "aws-encoding-wif-provider", + workload_identity_pool_id=pool.workload_identity_pool_id, + workload_identity_pool_provider_id=PROVIDER_ID, + display_name="Nomad AWS account", + aws=gcp.iam.WorkloadIdentityPoolProviderAwsArgs(account_id=AWS_ACCOUNT_ID), + attribute_mapping={ + "google.subject": "assertion.arn", + "attribute.aws_role": ( + "assertion.arn.contains('assumed-role') ? " + "assertion.arn.extract('{account_arn}assumed-role/') + 'assumed-role/' + " + "assertion.arn.extract('assumed-role/{role_name}/') : assertion.arn" + ), + }, + # Only the worker's role in Nomad's account may federate. + attribute_condition=( + f"attribute.aws_role == 'arn:aws:sts::{AWS_ACCOUNT_ID}:assumed-role/{AWS_WORKER_ROLE}'" + ), + ) + binding = gcp.serviceaccount.IAMMember( + "aws-encoding-worker-wif-user", + service_account_id=encoding_worker_sa.name, + role="roles/iam.workloadIdentityUser", + member=pool.name.apply( + lambda name: ( + f"principalSet://iam.googleapis.com/{name}/attribute.aws_role/" + f"arn:aws:sts::{AWS_ACCOUNT_ID}:assumed-role/{AWS_WORKER_ROLE}" + ) + ), + ) + pulumi.export("aws_encoding_wif_provider", provider.name) + return {"pool": pool, "provider": provider, "binding": binding} diff --git a/infrastructure/modules/secrets.py b/infrastructure/modules/secrets.py index 13ecc264..303abb49 100644 --- a/infrastructure/modules/secrets.py +++ b/infrastructure/modules/secrets.py @@ -50,6 +50,10 @@ def create_secrets() -> dict[str, secretmanager.Secret]: # comma-delimited and the JSON value contains commas; the value itself is # non-secret (just public IPs of fallback VMs). "encoding-worker-fallback-vms", + # AWS encoding worker: IAM user keys ({"access_key_id","secret_access_key"}) + # the backend uses to start/stop/describe the EC2 worker + # (backend/services/aws_encoding_compute.py). Version added by hand. + "aws-encoding-controller-credentials", # Notifications "pushbullet-api-key", diff --git a/poetry.lock b/poetry.lock index baea6023..36350562 100644 --- a/poetry.lock +++ b/poetry.lock @@ -660,6 +660,46 @@ files = [ [package.dependencies] numpy = {version = ">=1.19.0,<3.0.0", markers = "python_version >= \"3.9\""} +[[package]] +name = "boto3" +version = "1.43.122" +description = "The AWS SDK for Python (Boto3)" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "boto3-1.43.122-py3-none-any.whl", hash = "sha256:cca51b3650080594a1df344b8a2e76007eab6d5fa50504a0b622dfdc14cdeaa1"}, + {file = "boto3-1.43.122.tar.gz", hash = "sha256:c923650d09e4c445564ca14d5ce89466c3083884b6f6c189f0f445fc17f02eb5"}, +] + +[package.dependencies] +botocore = ">=1.43.122,<1.44.0" +jmespath = ">=0.7.1,<2.0.0" +s3transfer = ">=0.19.0,<0.20.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "botocore" +version = "1.43.122" +description = "Low-level, data-driven core of boto 3." +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "botocore-1.43.122-py3-none-any.whl", hash = "sha256:668e608edf8847fc4c3c6b38688e183c3bcf19b8a1f7f58d33c03a6f3039bd49"}, + {file = "botocore-1.43.122.tar.gz", hash = "sha256:2fc53b68b35314b1738cad46e6170d0556496863913451428fc3afccb8c7fa95"}, +] + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = ">=1.25.4,<2.2.0 || >2.2.0,<3" + +[package.extras] +crt = ["awscrt (==0.38.0)"] + [[package]] name = "build" version = "1.3.0" @@ -3579,6 +3619,18 @@ files = [ click = ">=8.1.8" rapidfuzz = ">=3.9.7" +[[package]] +name = "jmespath" +version = "1.1.0" +description = "JSON Matching Expressions" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64"}, + {file = "jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d"}, +] + [[package]] name = "joblib" version = "1.5.3" @@ -7979,6 +8031,24 @@ oletools = ">=0.56" dev = ["coverage (>=7.2.2,<7.3.0)", "lxml (>=6.0,<7.0)", "mypy (>=1.1,<2.0)", "pdoc3 (>=0.10.0,<0.11.0)"] msg-parse = ["extract_msg (>=0.27,<1.0)"] +[[package]] +name = "s3transfer" +version = "0.19.3" +description = "An Amazon S3 Transfer Manager" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "s3transfer-0.19.3-py3-none-any.whl", hash = "sha256:24eb7535804306355e31c952a90022bf5b2f9d760f9a11062c7493a217c08553"}, + {file = "s3transfer-0.19.3.tar.gz", hash = "sha256:8326acc3d056d7f7b8a37110c7d2aea41f67e83f6ae79cd3b5fbcca34c726406"}, +] + +[package.dependencies] +botocore = ">=1.37.4,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.37.4,<2.0a.0)"] + [[package]] name = "safetensors" version = "0.7.0" @@ -10738,4 +10808,4 @@ local-whisper = ["whisper-timestamped"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.14" -content-hash = "ea37b6f69b799819b84da84381a2e2971bbcac8d409c375d94a26fa8f230696b" +content-hash = "59291df081a5b57850eefee645e27751b7fd78491e517c5eb180a82c43e2c219" diff --git a/pyproject.toml b/pyproject.toml index 54f403fb..b7cff807 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.287.2" +version = "0.288.0" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT" @@ -98,6 +98,8 @@ httpx = ">=0.25.0" tenacity = "^8.2.0" # Retry logic for external service calls google-cloud-firestore = "2.26.0" google-cloud-compute = "^1.0.0" +# AWS encoding worker control (EC2 start/stop/describe) — backend/services/aws_encoding_compute.py +boto3 = "^1.34" google-cloud-storage = "3.10.1" google-cloud-secret-manager = "2.26.0" google-cloud-monitoring = "2.29.1" # Cloud Monitoring for YouTube API quota metrics