From 78264fa23afce6e0f96e4bb00e45107563784170 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrew=20Beveridge=20=E2=98=84=EF=B8=8F?= Date: Sat, 31 Oct 2026 15:02:42 -0400 Subject: [PATCH] feat(encoding): run the video worker (distribution) on the AWS box (v0.289.0) (#1240) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Phase 2 of moving encoding to AWS; Phase 1 was #1236 and #1239. Design and rationale are in `docs/archive/2026-10-29-aws-encoding-worker-plan.md` ยง Phase 2. When Firestore `config/encoding-worker.aws_video_worker` is true **and** the AWS worker ranks first, `trigger_video_worker` runs the video worker **on the AWS box** instead of starting the Cloud Run job `video-encoding-job`. - **Uploads move to AWS:** the Dropbox/YouTube/Drive uploads (~187 GB/mo, ~$18/mo of Cloud Run internet egress) leave from AWS, on credits. - **No Cloud Run CPU** for the job (~$11/mo). - **Off by default.** Any failure falls back to the Cloud Run job, so this path can't drop a job. ## Changes - **`POST /run-worker` (worker):** - Spawns `python -m backend.workers.video_worker --job-id โ€ฆ` with the box's video-worker env. - Status is reported via `/status/{id}`. - A run counts in `/health` `active_jobs` from the moment it's accepted, so the idle watchdog won't stop the box mid-run. - Returns 503 unless the env file exists. - `job_id` is pattern-validated. - Any startup failure marks the run failed. - **`backend/services/local_encoded_outputs.py`:** a same-box handoff. - When the co-located video worker asks (`stash_local_outputs`, set only where `LOCAL_ENCODED_OUTPUTS_DIR` exists), the encoder keeps local copies of the finals, keyed by blob path. - The video worker takes those instead of downloading them back out of GCS. Without this, the move would *add* ~400 MB/job of GCS egress. - Copies are made off the event loop and purged after 6 h. - No change on GCE or Cloud Run. - **`video-worker-env.py` (box, every boot):** - Mirrors the **live** `video-encoding-job` env, with secrets resolved, so nothing is hand-copied and nothing drifts. - Runs as the job's identity, `karaoke-backend@`, via a second WIF binding (Pulumi `aws-encoding-worker-wif-backend-user`). - On failure it writes no env file, so `/run-worker` answers 503 and the Cloud Run job is used. - **`WorkerService._trigger_video_worker_on_aws`:** checks the flag and that the AWS worker ranks first, ensures it's running, waits for health, then POSTs `/run-worker`. Returns False on any doubt. - **Also:** `startup-aws.sh` creates the backend credential config and runs the env helper; CI uploads `video-worker-env.py`; `worker_code_paths.txt` now gates `local_encoded_outputs.py`. ## Rollout (needs `/gcp-admin`) 1. **Before merge:** targeted `pulumi up` of the backend WIF binding. It's additive. 2. **After stage B of Phase 1** (AWS worker preferred, timings verified): stop/start the box once so the new `startup-aws.sh` writes the env, then set `aws_video_worker: true`. Set it to `false` to roll back. ## Testing - **27 new tests:** - handoff stash/take/purge, path-escape guard, and the per-request stash flag; - `/run-worker`: disabled, bad module, dedupe, active immediately, spawn args/env, failure log tail, wheel-not-ready, safe start, unsafe ids; - env builder; - the trigger's AWS branch: off, not first, happy path, didn't start, refusals, errors. - **Full backend + unit suites:** no new failures (the 13 local failures are identical on main). - **Local CodeRabbit:** 5 findings, all fixed. The code-paths gate test also caught a missing worker path, now fixed. @coderabbitai ignore ๐Ÿค– Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_012RYLVSR5qgwsxTQtvGimKF --------- Co-authored-by: Claude Opus 5.5 --- .github/workflows/ci.yml | 2 + backend/services/encoding_interface.py | 6 + backend/services/encoding_worker_manager.py | 4 + backend/services/gce_encoding/main.py | 135 +++++++- backend/services/local_encoded_outputs.py | 91 ++++++ backend/services/worker_service.py | 61 ++++ backend/tests/test_aws_video_worker.py | 304 ++++++++++++++++++ backend/workers/video_worker_orchestrator.py | 10 +- docs/ARCHITECTURE.md | 1 + .../2026-10-29-aws-encoding-worker-plan.md | 47 ++- infrastructure/__main__.py | 4 +- infrastructure/aws/README.md | 2 + .../encoding-worker/aws/startup-aws.sh | 19 +- .../encoding-worker/aws/video-worker-env.py | 85 +++++ .../encoding-worker/worker_code_paths.txt | 1 + infrastructure/modules/aws_encoding_wif.py | 21 +- pyproject.toml | 2 +- 17 files changed, 776 insertions(+), 19 deletions(-) create mode 100644 backend/services/local_encoded_outputs.py create mode 100644 backend/tests/test_aws_video_worker.py create mode 100644 infrastructure/encoding-worker/aws/video-worker-env.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1bd65e8b..9edb99a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1407,6 +1407,8 @@ jobs: "${BUCKET}/encoding-worker/startup-aws.sh" gsutil cp infrastructure/encoding-worker/aws/idle-watchdog.py \ "${BUCKET}/encoding-worker/idle-watchdog.py" + gsutil cp infrastructure/encoding-worker/aws/video-worker-env.py \ + "${BUCKET}/encoding-worker/video-worker-env.py" # 4. Write version manifest for verification echo "Writing version manifest..." diff --git a/backend/services/encoding_interface.py b/backend/services/encoding_interface.py index 965f5d3e..afb7cec0 100644 --- a/backend/services/encoding_interface.py +++ b/backend/services/encoding_interface.py @@ -467,6 +467,12 @@ class GCEEncodingBackend(EncodingBackend): # Screens the theme leaves out: the encoder must skip them rather than # rebuild them from whatever matching PNG it finds in the job folder. + # On the AWS box the video worker sits next to the encoder: ask it + # to keep local copies so we take them instead of downloading back + # from GCS (local_encoded_outputs). Unset everywhere else. + if os.environ.get("LOCAL_ENCODED_OUTPUTS_DIR"): + encoding_config["stash_local_outputs"] = True + for flag in ("include_title_screen", "include_end_screen"): if input_config.options.get(flag) is False: encoding_config[flag] = False diff --git a/backend/services/encoding_worker_manager.py b/backend/services/encoding_worker_manager.py index 970ee7bd..1fea61a0 100644 --- a/backend/services/encoding_worker_manager.py +++ b/backend/services/encoding_worker_manager.py @@ -115,6 +115,9 @@ class EncodingWorkerConfig: # {machine_family: rank} replacing SPEED_RANK at runtime, e.g. {"c7a": 5} # makes the AWS worker preferred. Absent/empty โ‡’ the built-in ranking. speed_rank_overrides: dict = field(default_factory=dict) + # True โ‡’ when the AWS worker ranks first, the video worker (distribution) + # also runs on it instead of the Cloud Run job (worker_service). Default off. + aws_video_worker: bool = False @property def primary_url(self) -> str: @@ -349,6 +352,7 @@ class EncodingWorkerManager: active_override_set_at=data.get("active_override_set_at"), capacity_state=data.get("capacity_state") or {}, speed_rank_overrides=data.get("speed_rank_overrides") or {}, + aws_video_worker=bool(data.get("aws_video_worker", False)), ) def update_activity(self) -> None: diff --git a/backend/services/gce_encoding/main.py b/backend/services/gce_encoding/main.py index 202a91e3..c87e7466 100644 --- a/backend/services/gce_encoding/main.py +++ b/backend/services/gce_encoding/main.py @@ -27,7 +27,7 @@ os.environ.setdefault("GCE_METADATA_MTLS_MODE", "none") from fastapi import FastAPI, BackgroundTasks, HTTPException, Header, Depends from google.cloud import storage from packaging.version import Version -from pydantic import BaseModel +from pydantic import BaseModel, Field from . import self_heal from .persistence import JobStatePersister @@ -120,6 +120,12 @@ async def lifespan(app: FastAPI): logger.critical("Encoding worker booted unhealthy, exiting for systemd restart: %s", problem) raise RuntimeError(f"Encoding worker booted unhealthy: {problem}") + # AWS box only (no-op elsewhere): drop local finals no video worker took. + from backend.services import local_encoded_outputs + removed = await asyncio.to_thread(local_encoded_outputs.purge_stale) + if removed: + logger.info("Purged %d stale locally-stashed encoded outputs", removed) + # Startup: read back any in-progress jobs from Firestore. They were # interrupted by the restart that brought us here โ€” their work_dir is # gone, so mark them failed with the restart code so polls return a @@ -1341,6 +1347,21 @@ async def process_job(job_id: str, request: EncodeRequest): jobs[job_id]["output_files"] = blob_paths logger.info(f"Output files (blob paths): {blob_paths}") + # On the AWS box the video worker runs here too: keep local copies + # (keyed by blob path) so it needn't download them back out of GCS. + # Only when that co-located video worker asked (stash_local_outputs), + # and only where LOCAL_ENCODED_OUTPUTS_DIR is set. Off the event loop: + # it copies ~400 MB. + if (request.encoding_config or {}).get("stash_local_outputs"): + from backend.services import local_encoded_outputs + + def _stash_all(): + local_encoded_outputs.purge_stale() + for local_path, blob_path in zip(local_output_files, blob_paths): + local_encoded_outputs.stash(Path(local_path), blob_path) + + await asyncio.to_thread(_stash_all) + jobs[job_id]["status"] = "complete" jobs[job_id]["progress"] = 100 _persist(job_id) @@ -1494,6 +1515,118 @@ async def submit_render_video_job(request: RenderVideoRequest, background_tasks: return {"status": "accepted", "job_id": job_id} +# --------------------------------------------------------------------------- +# /run-worker โ€” run a backend worker (the video worker) on this box +# --------------------------------------------------------------------------- +# AWS box only: there the video worker (packaging + encode + Dropbox/YouTube/Drive +# distribution) runs next to the encoder, so the big uploads leave from AWS +# (credits) instead of GCP, and finals are handed over locally +# (local_encoded_outputs). Enabled only where the environment file written by +# startup-aws.sh exists; the backend falls back to the Cloud Run job otherwise. +RUN_WORKER_ENV_FILE = os.environ.get("RUN_WORKER_ENV_FILE", "/opt/encoding-worker/video-worker-env.json") +RUN_WORKER_LOG_DIR = Path(os.environ.get("RUN_WORKER_LOG_DIR", "/var/log/encoding-worker-runs")) +_RUN_WORKER_MODULES = frozenset({"video_worker"}) + + +class RunWorkerRequest(BaseModel): + # Used in a log file name and as a CLI argument: gen job ids only. + job_id: str = Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$") + module: str = "video_worker" + + +def run_worker_id(module: str, job_id: str) -> str: + return f"run-{module}-{job_id}" + + +def _run_worker_env() -> Optional[dict]: + """Env for the worker subprocess: ours + the video-worker env file, or None if absent.""" + try: + with open(RUN_WORKER_ENV_FILE) as f: + extra = json.load(f) + except (OSError, ValueError): + return None + if not isinstance(extra, dict): + return None + env = dict(os.environ) + env.pop("ENCODING_API_KEY", None) # the worker's own key isn't the subprocess's business + env.update({str(k): str(v) for k, v in extra.items()}) + return env + + +def _watch_worker_run(run_id: str, proc: subprocess.Popen, log_path: Path) -> None: + rc = proc.wait() + state = jobs.get(run_id) + if state is None: + return + if rc == 0: + state["status"] = "complete" + state["progress"] = 100 + else: + tail = "" + try: + tail = log_path.read_text(errors="replace")[-1500:] + except OSError: + pass + state["status"] = "failed" + state["error"] = f"{state.get('module')} exited {rc}: {tail}" + state["returncode"] = rc + _persist(run_id) + logger.info("Worker run %s finished rc=%s (log %s)", run_id, rc, log_path) + + +def _start_worker_run(run_id: str, request: RunWorkerRequest, env: dict) -> None: + # The subprocess imports the installed wheel: make sure it's the latest. + if not ensure_latest_wheel(): + jobs[run_id].update(status="failed", error="karaoke-gen wheel not ready on this worker") + _persist(run_id) + return + RUN_WORKER_LOG_DIR.mkdir(parents=True, exist_ok=True) + log_path = RUN_WORKER_LOG_DIR / f"{request.job_id}-{uuid.uuid4().hex[:8]}.log" + with open(log_path, "ab") as log: + proc = subprocess.Popen( + [sys.executable, "-m", f"backend.workers.{request.module}", "--job-id", request.job_id], + env=env, stdout=log, stderr=subprocess.STDOUT, cwd="/tmp", + ) + jobs[run_id].update(status="running", pid=proc.pid, log=str(log_path)) + _persist(run_id) + logger.info("Started %s for job %s (pid %s, log %s)", request.module, request.job_id, proc.pid, log_path) + threading.Thread(target=_watch_worker_run, args=(run_id, proc, log_path), daemon=True).start() + + +def _start_worker_run_safe(run_id: str, request: RunWorkerRequest, env: dict) -> None: + """Never leave a run stuck "running": that would hold the box up (idle + watchdog) and lose the job (the backend got "accepted", so no fallback).""" + try: + _start_worker_run(run_id, request, env) + except Exception as exc: # noqa: BLE001 + logger.exception("Failed to start worker run %s", run_id) + jobs[run_id].update(status="failed", error=f"failed to start worker: {exc}") + _persist(run_id) + + +@app.post("/run-worker") +async def submit_worker_run(request: RunWorkerRequest, _auth: bool = Depends(verify_api_key)): + if request.module not in _RUN_WORKER_MODULES: + raise HTTPException(status_code=400, detail=f"module {request.module!r} not allowed") + env = _run_worker_env() + if env is None: + raise HTTPException(status_code=503, detail="worker runs are not enabled on this worker") + run_id = run_worker_id(request.module, request.job_id) + existing = jobs.get(run_id) + if existing and existing.get("status") == "running": + return {"status": "in_progress", "job_id": run_id} + # "running" from the start: /health counts only running jobs, and the + # idle watchdog must not stop the box while the wheel check precedes spawn. + jobs[run_id] = { + "job_id": run_id, "status": "running", "progress": 0, "error": None, + "output_files": None, "kind": "worker_run", "module": request.module, + } + _persist(run_id) + loop = asyncio.get_event_loop() + loop.run_in_executor(light_executor, _start_worker_run_safe, run_id, request, env) + return {"status": "accepted", "job_id": run_id} + + @app.post("/encode") async def submit_encode_job(request: EncodeRequest, background_tasks: BackgroundTasks, _auth: bool = Depends(verify_api_key)): # Submit an encoding job diff --git a/backend/services/local_encoded_outputs.py b/backend/services/local_encoded_outputs.py new file mode 100644 index 00000000..945ff2ed --- /dev/null +++ b/backend/services/local_encoded_outputs.py @@ -0,0 +1,91 @@ +"""Same-box handoff of encoded finals from the encoding worker to the video worker. + +On the AWS encoding worker, the video worker (distribution) runs on the same box +as the encoding service. Without this, the video worker would download every +final the encoder just uploaded back out of GCS โ€” ~400 MB of out-of-pocket GCS +egress per job, more than moving distribution to AWS saves. So the encoder +"stashes" a local copy of each output keyed by its GCS blob path, and the video +worker "takes" it instead of downloading. + +Active only when ``LOCAL_ENCODED_OUTPUTS_DIR`` is set (the AWS box). Everywhere +else ``stash`` is a no-op and ``take`` returns False, i.e. today's behaviour. +Best effort: any failure falls back to GCS. +""" + +from __future__ import annotations + +import logging +import os +import shutil +import time +from pathlib import Path +from typing import Optional + +logger = logging.getLogger(__name__) + +ENV_VAR = "LOCAL_ENCODED_OUTPUTS_DIR" +MAX_AGE_SECONDS = 6 * 3600 # a take happens minutes after the stash + + +def _root() -> Optional[Path]: + raw = os.environ.get(ENV_VAR, "").strip() + return Path(raw) if raw else None + + +def _path_for(root: Path, blob_path: str) -> Path: + rel = blob_path.lstrip("/") + target = (root / rel).resolve() + if root.resolve() not in target.parents: + raise ValueError(f"blob path escapes the stash dir: {blob_path!r}") + return target + + +def stash(local_file: Path, blob_path: str) -> bool: + """Keep a copy of ``local_file`` for a co-located video worker. Never raises.""" + root = _root() + if root is None: + return False + try: + dest = _path_for(root, blob_path) + dest.parent.mkdir(parents=True, exist_ok=True) + tmp = dest.with_name(dest.name + ".partial") + shutil.copyfile(local_file, tmp) + os.replace(tmp, dest) + return True + except Exception as e: # noqa: BLE001 + logger.warning("Could not stash %s for local handoff: %s", blob_path, e) + return False + + +def take(blob_path: str, dest: str) -> bool: + """Move the stashed copy of ``blob_path`` to ``dest``. False โ‡’ download from GCS.""" + root = _root() + if root is None: + return False + try: + src = _path_for(root, blob_path) + if not src.is_file(): + return False + Path(dest).parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(src), dest) + return True + except Exception as e: # noqa: BLE001 + logger.warning("Local handoff of %s failed, falling back to GCS: %s", blob_path, e) + return False + + +def purge_stale(max_age_seconds: float = MAX_AGE_SECONDS) -> int: + """Delete stashed files older than ``max_age_seconds`` (never taken). Returns count.""" + root = _root() + if root is None or not root.is_dir(): + return 0 + cutoff = time.time() - max_age_seconds + removed = 0 + for p in root.rglob("*"): + try: + if p.is_file() and p.stat().st_mtime < cutoff: + p.unlink() + removed += 1 + except OSError: + continue + return removed diff --git a/backend/services/worker_service.py b/backend/services/worker_service.py index 069757fd..eec5bd76 100644 --- a/backend/services/worker_service.py +++ b/backend/services/worker_service.py @@ -645,8 +645,69 @@ class WorkerService: await asyncio.to_thread(self._bump_worker_generation, job_id) # Firestore write self._start_encoding_worker_warmup(job_id) if self._use_cloud_tasks and self.settings.use_cloud_run_jobs_for_video: + if await self._trigger_video_worker_on_aws(job_id): + return True return await self._trigger_cloud_run_job(job_id) return await self.trigger_worker("video", job_id) + + async def _trigger_video_worker_on_aws(self, job_id: str) -> bool: + """Run the video worker on the AWS encoding box instead of Cloud Run, if enabled. + + Only when Firestore ``config/encoding-worker.aws_video_worker`` is true AND + the AWS worker ranks first (it's the preferred encoder, via + ``speed_rank_overrides``). There the distribution uploads (Dropbox/YouTube/ + Drive) leave from AWS, and finals are handed over locally instead of + downloaded back out of GCS. Returns False on any doubt or failure so the + caller falls back to the Cloud Run job โ€” this path must never lose a job. + """ + try: + from backend.services.encoding_service import get_encoding_service + + manager = self._build_encoding_worker_manager() + config = await asyncio.to_thread(manager.get_config) + if not getattr(config, "aws_video_worker", False): + return False + candidates = await asyncio.to_thread(manager.ranked_candidates, config) + if not candidates or candidates[0].provider != "aws": + return False + aws = candidates[0] + result = await asyncio.to_thread(manager.ensure_any_running, candidates) + if result.get("vm_name") != aws.vm_name: + logger.warning( + f"[job:{job_id}] AWS video worker: {aws.vm_name} unavailable " + f"(started {result.get('vm_name')}); using Cloud Run" + ) + return False + await manager.wait_for_worker_ready(aws.vm_name, f"{aws.url}/health", zone=aws.zone) + + service = get_encoding_service() + service._load_credentials() + async with httpx.AsyncClient(timeout=30.0) as client: + resp = await client.post( + f"{aws.url}/run-worker", + json={"job_id": job_id, "module": "video_worker"}, + headers={"X-API-Key": service._api_key}, + ) + if resp.status_code == 200 and resp.json().get("status") in ("accepted", "in_progress"): + logger.info(f"[job:{job_id}] Video worker started on AWS ({aws.vm_name})") + return True + logger.warning( + f"[job:{job_id}] AWS video worker refused ({resp.status_code}: {resp.text[:300]}); using Cloud Run" + ) + return False + except Exception as e: # noqa: BLE001 โ€” fall back to Cloud Run, never drop the job + logger.warning(f"[job:{job_id}] AWS video worker dispatch failed ({e}); using Cloud Run") + return False + + def _build_encoding_worker_manager(self): + from backend.services.encoding_worker_manager import EncodingWorkerManager + from google.cloud import compute_v1, firestore + + return EncodingWorkerManager( + db=firestore.Client(project=self.settings.google_cloud_project), + compute_client=compute_v1.InstancesClient(), + project_id=self.settings.google_cloud_project, + ) async def _trigger_cloud_run_job(self, job_id: str) -> bool: """ diff --git a/backend/tests/test_aws_video_worker.py b/backend/tests/test_aws_video_worker.py new file mode 100644 index 00000000..4390d4b0 --- /dev/null +++ b/backend/tests/test_aws_video_worker.py @@ -0,0 +1,304 @@ +"""Phase 2: the video worker (distribution) on the AWS encoding box. + +Covers the same-box finals handoff, the worker's /run-worker endpoint, the +box's env builder (mirrors the live Cloud Run job), and the backend trigger's +AWS branch โ€” which must fall back to the Cloud Run job on any doubt. +""" +import asyncio +import importlib.util +import json +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from fastapi import HTTPException + +from backend.services import local_encoded_outputs as leo + +# --- local_encoded_outputs ------------------------------------------------------ + +BLOB = "jobs/abc123/finals/Artist - Title (Final Karaoke Lossless 4k).mp4" + + +def test_handoff_is_noop_without_env(tmp_path, monkeypatch): + monkeypatch.delenv(leo.ENV_VAR, raising=False) + f = tmp_path / "x.mp4" + f.write_bytes(b"v") + assert leo.stash(f, BLOB) is False + assert leo.take(BLOB, str(tmp_path / "out.mp4")) is False + + +def test_stash_then_take_moves_file(tmp_path, monkeypatch): + monkeypatch.setenv(leo.ENV_VAR, str(tmp_path / "stash")) + src = tmp_path / "encoded.mp4" + src.write_bytes(b"video-bytes") + assert leo.stash(src, BLOB) is True + dest = tmp_path / "work" / "final.mp4" + assert leo.take(BLOB, str(dest)) is True + assert dest.read_bytes() == b"video-bytes" + assert leo.take(BLOB, str(dest)) is False # moved, so a second take falls back to GCS + + +def test_take_missing_falls_back(tmp_path, monkeypatch): + monkeypatch.setenv(leo.ENV_VAR, str(tmp_path)) + assert leo.take(BLOB, str(tmp_path / "o.mp4")) is False + + +def test_blob_path_cannot_escape_stash_dir(tmp_path, monkeypatch): + monkeypatch.setenv(leo.ENV_VAR, str(tmp_path / "stash")) + f = tmp_path / "x" + f.write_bytes(b"1") + assert leo.stash(f, "../../etc/evil") is False + + +def test_purge_stale_removes_only_old_files(tmp_path, monkeypatch): + import os + import time + monkeypatch.setenv(leo.ENV_VAR, str(tmp_path)) + old, new = tmp_path / "jobs/a/old.mp4", tmp_path / "jobs/b/new.mp4" + for p in (old, new): + p.parent.mkdir(parents=True) + p.write_bytes(b"1") + os.utime(old, (time.time() - 2 * 86400,) * 2) + assert leo.purge_stale() == 1 + assert not old.exists() and new.exists() + + +# --- worker /run-worker ----------------------------------------------------------- + +@pytest.fixture +def worker(monkeypatch, tmp_path): + from backend.services.gce_encoding import main + env_file = tmp_path / "video-worker-env.json" + monkeypatch.setattr(main, "RUN_WORKER_ENV_FILE", str(env_file)) + monkeypatch.setattr(main, "RUN_WORKER_LOG_DIR", tmp_path / "runs") + monkeypatch.setattr(main, "_persist", lambda job_id: None) + main.jobs.clear() + yield main, env_file + main.jobs.clear() + + +def _req(main, job_id="abc123", module="video_worker"): + return main.RunWorkerRequest(job_id=job_id, module=module) + + +def test_run_worker_disabled_without_env_file(worker): + main, _ = worker + with pytest.raises(HTTPException) as exc: + asyncio.run(main.submit_worker_run(_req(main))) + assert exc.value.status_code == 503 + + +def test_run_worker_rejects_unknown_module(worker): + main, env_file = worker + env_file.write_text("{}") + with pytest.raises(HTTPException) as exc: + asyncio.run(main.submit_worker_run(_req(main, module="audio_worker"))) + assert exc.value.status_code == 400 + + +def test_run_worker_dedupes_running_job(worker): + main, env_file = worker + env_file.write_text("{}") + main.jobs[main.run_worker_id("video_worker", "abc123")] = {"status": "running"} + out = asyncio.run(main.submit_worker_run(_req(main))) + assert out["status"] == "in_progress" + + +def test_run_worker_counts_as_active_immediately(worker): + main, env_file = worker + env_file.write_text(json.dumps({"ENVIRONMENT": "production"})) + with patch.object(main, "_start_worker_run"): + out = asyncio.run(main.submit_worker_run(_req(main))) + assert out["status"] == "accepted" + assert main.jobs[out["job_id"]]["status"] == "running" # /health active_jobs counts it + + +def test_start_worker_run_spawns_module_with_box_env(worker): + main, env_file = worker + env_file.write_text(json.dumps({"ENVIRONMENT": "production", "LOCAL_ENCODED_OUTPUTS_DIR": "/x"})) + run_id = main.run_worker_id("video_worker", "abc123") + main.jobs[run_id] = {"status": "running", "module": "video_worker"} + proc = MagicMock(pid=4242) + proc.wait.return_value = 0 + with patch.object(main, "ensure_latest_wheel", return_value=True), \ + patch.object(main.subprocess, "Popen", return_value=proc) as popen: + main._start_worker_run(run_id, _req(main), main._run_worker_env()) + args, kwargs = popen.call_args + assert args[0][1:] == ["-m", "backend.workers.video_worker", "--job-id", "abc123"] + assert kwargs["env"]["LOCAL_ENCODED_OUTPUTS_DIR"] == "/x" + assert "ENCODING_API_KEY" not in kwargs["env"] + main._watch_worker_run(run_id, proc, Path(main.jobs[run_id]["log"])) + assert main.jobs[run_id]["status"] == "complete" + + +def test_failed_run_reports_log_tail(worker, tmp_path): + main, _ = worker + run_id = main.run_worker_id("video_worker", "abc123") + main.jobs[run_id] = {"status": "running", "module": "video_worker"} + log = tmp_path / "run.log" + log.write_text("Traceback ... Dropbox upload failed") + proc = MagicMock() + proc.wait.return_value = 1 + main._watch_worker_run(run_id, proc, log) + assert main.jobs[run_id]["status"] == "failed" + assert "Dropbox upload failed" in main.jobs[run_id]["error"] + + +def test_wheel_not_ready_fails_run_without_spawning(worker): + main, env_file = worker + env_file.write_text("{}") + run_id = main.run_worker_id("video_worker", "abc123") + main.jobs[run_id] = {"status": "running"} + with patch.object(main, "ensure_latest_wheel", return_value=False), \ + patch.object(main.subprocess, "Popen") as popen: + main._start_worker_run(run_id, _req(main), {}) + popen.assert_not_called() + assert main.jobs[run_id]["status"] == "failed" + + +# --- video-worker-env.py ------------------------------------------------------------ + +_ENV_PATH = Path(__file__).resolve().parents[2] / "infrastructure" / "encoding-worker" / "aws" / "video-worker-env.py" +_spec = importlib.util.spec_from_file_location("video_worker_env", _ENV_PATH) +vwe = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(vwe) + + +def test_build_env_mirrors_job_and_resolves_secrets(): + job = {"template": {"template": {"containers": [{"env": [ + {"name": "ENVIRONMENT", "value": "production"}, + {"name": "POSTMARK_SERVER_TOKEN", "valueSource": {"secretKeyRef": { + "secret": "projects/nomadkaraoke/secrets/postmark-server-token", "version": "latest"}}}, + ]}]}}} + env = vwe.build_env(job, lambda secret, version: f"<{secret}@{version}>") + assert env["ENVIRONMENT"] == "production" + assert env["POSTMARK_SERVER_TOKEN"] == "" + assert env["GOOGLE_APPLICATION_CREDENTIALS"].endswith("gcp-wif-backend.json") + assert env["LOCAL_ENCODED_OUTPUTS_DIR"] and env["CLOUD_RUN_JOB"] == "video-encoding-job" + + +# --- backend trigger ------------------------------------------------------------------ + +def _candidate(provider, vm="aws-encoding-worker-1"): + return SimpleNamespace(provider=provider, vm_name=vm, url="http://3.3.3.3:8080", zone="us-east-2b") + + +@pytest.fixture +def service(): + from backend.services.worker_service import WorkerService + svc = WorkerService.__new__(WorkerService) + svc.settings = MagicMock(google_cloud_project="nomadkaraoke") + return svc + + +def _manager(aws_video_worker=True, top="aws", started="aws-encoding-worker-1"): + m = MagicMock() + m.get_config.return_value = SimpleNamespace(aws_video_worker=aws_video_worker) + m.ranked_candidates.return_value = [_candidate(top, "aws-encoding-worker-1" if top == "aws" else "encoding-worker-a")] + m.ensure_any_running.return_value = {"vm_name": started} + m.wait_for_worker_ready = AsyncMock() + return m + + +def _run(service, manager, response=None, post_exc=None): + client = MagicMock() + client.__aenter__ = AsyncMock(return_value=client) + client.__aexit__ = AsyncMock(return_value=False) + client.post = AsyncMock(side_effect=post_exc, return_value=response) + enc = MagicMock(_api_key="k") + with patch.object(service, "_build_encoding_worker_manager", return_value=manager), \ + patch("backend.services.encoding_service.get_encoding_service", return_value=enc), \ + patch("backend.services.worker_service.httpx.AsyncClient", return_value=client): + return asyncio.run(service._trigger_video_worker_on_aws("abc123")), client + + +def test_aws_dispatch_off_by_default(service): + ok, client = _run(service, _manager(aws_video_worker=False)) + assert ok is False + client.post.assert_not_called() + + +def test_aws_dispatch_requires_aws_to_rank_first(service): + ok, client = _run(service, _manager(top=None)) + assert ok is False + client.post.assert_not_called() + + +def test_aws_dispatch_posts_run_worker(service): + resp = MagicMock(status_code=200) + resp.json.return_value = {"status": "accepted"} + ok, client = _run(service, _manager(), response=resp) + assert ok is True + url = client.post.call_args.args[0] + assert url == "http://3.3.3.3:8080/run-worker" + assert client.post.call_args.kwargs["json"] == {"job_id": "abc123", "module": "video_worker"} + + +def test_aws_dispatch_falls_back_when_aws_did_not_start(service): + ok, client = _run(service, _manager(started="encoding-worker-a")) + assert ok is False + client.post.assert_not_called() + + +@pytest.mark.parametrize("status,body", [(503, {"detail": "not enabled"}), (200, {"status": "weird"})]) +def test_aws_dispatch_falls_back_on_refusal(service, status, body): + resp = MagicMock(status_code=status, text=json.dumps(body)) + resp.json.return_value = body + ok, _ = _run(service, _manager(), response=resp) + assert ok is False + + +def test_aws_dispatch_falls_back_on_error(service): + ok, _ = _run(service, _manager(), post_exc=ConnectionError("boom")) + assert ok is False + + +def test_start_failure_never_leaves_run_stuck(worker): + main, env_file = worker + run_id = main.run_worker_id("video_worker", "abc123") + main.jobs[run_id] = {"status": "running"} + with patch.object(main, "_start_worker_run", side_effect=OSError("disk full")): + main._start_worker_run_safe(run_id, _req(main), {}) + assert main.jobs[run_id]["status"] == "failed" + assert "disk full" in main.jobs[run_id]["error"] + + +@pytest.mark.parametrize("bad", ["../../tmp/x", "-h", "", "a b", "x" * 80]) +def test_run_worker_rejects_unsafe_job_ids(bad): + from pydantic import ValidationError + from backend.services.gce_encoding import main + with pytest.raises(ValidationError): + main.RunWorkerRequest(job_id=bad) + + +def test_gce_backend_requests_stash_only_on_the_box(monkeypatch): + """The encode payload asks for local copies only where the video worker + shares the box with the encoder (LOCAL_ENCODED_OUTPUTS_DIR set).""" + from backend.services.encoding_interface import EncodingInput, GCEEncodingBackend + + async def capture(env_value): + if env_value: + monkeypatch.setenv("LOCAL_ENCODED_OUTPUTS_DIR", env_value) + else: + monkeypatch.delenv("LOCAL_ENCODED_OUTPUTS_DIR", raising=False) + svc = MagicMock() + svc.encode_videos = AsyncMock(return_value={"status": "complete", "output_files": []}) + backend = GCEEncodingBackend() + backend._service = svc + with patch.object(GCEEncodingBackend, "_get_service", return_value=svc): + await backend.encode(EncodingInput( + title_video_path="t", karaoke_video_path="k", instrumental_audio_path="i", + artist="A", title="T", output_dir="/tmp", + options={"job_id": "abc123_g1", "input_gcs_path": "gs://b/jobs/abc123/", + "output_gcs_path": "gs://b/jobs/abc123/finals/"}, + )) + return svc.encode_videos.call_args + + on = asyncio.run(capture("/var/lib/encoding-worker/outputs")) + off = asyncio.run(capture(None)) + on_cfg = on.kwargs.get("encoding_config") or on.args[-1] + off_cfg = off.kwargs.get("encoding_config") or off.args[-1] + assert on_cfg.get("stash_local_outputs") is True + assert "stash_local_outputs" not in off_cfg diff --git a/backend/workers/video_worker_orchestrator.py b/backend/workers/video_worker_orchestrator.py index dcd28ae6..57654138 100644 --- a/backend/workers/video_worker_orchestrator.py +++ b/backend/workers/video_worker_orchestrator.py @@ -691,8 +691,14 @@ class VideoWorkerOrchestrator: local_path = os.path.join(self.config.output_dir, filename) try: - self.job_log.info(f"Downloading {filename} from GCS") - self.storage.download_file(gcs_path, local_path) + # On the AWS encoding box the encoder left a local copy: take it + # instead of downloading back out of GCS (see local_encoded_outputs). + from backend.services import local_encoded_outputs + if local_encoded_outputs.take(gcs_path, local_path): + self.job_log.info(f"Took {filename} from the co-located encoder (no GCS download)") + else: + self.job_log.info(f"Downloading {filename} from GCS") + self.storage.download_file(gcs_path, local_path) # Update the result to point to local file setattr(self.result, result_attr, local_path) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index eb8b423f..b0349d75 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -488,6 +488,7 @@ on-demand fallback if Spot capacity is gone. - It is a last resort by default (`SPEED_RANK["c7a"] = 90`). - Firestore `config/encoding-worker.speed_rank_overrides = {"c7a": 5}` makes it preferred with no deploy. - Both warmups (`ensure_preferred_running`) follow the same ranking, so they warm the worker requests are routed to. +- **Distribution on AWS (Phase 2, off by default):** with `config/encoding-worker.aws_video_worker = true` and the AWS worker ranked first, `trigger_video_worker` runs the video worker on the box (`POST /run-worker`, env mirrored from the live `video-encoding-job`, identity `karaoke-backend@` via WIF). Uploads leave from AWS, and finals are handed over locally (`local_encoded_outputs`) instead of downloaded back from GCS. Any failure falls back to the Cloud Run job. - **Auth and lifecycle:** - GCP access is Workload Identity Federation as `encoding-worker@`: no keys. - It stops itself when idle (`infrastructure/encoding-worker/aws/idle-watchdog.py`). diff --git a/docs/archive/2026-10-29-aws-encoding-worker-plan.md b/docs/archive/2026-10-29-aws-encoding-worker-plan.md index 59dbdd3b..41ef84b4 100644 --- a/docs/archive/2026-10-29-aws-encoding-worker-plan.md +++ b/docs/archive/2026-10-29-aws-encoding-worker-plan.md @@ -83,19 +83,44 @@ The box runs the **unchanged worker HTTP service** (`backend.services.gce_encodi - Extend the existing test files listed in the GCE-coupling map. - **Integration (prod, supervised):** stage B runs real jobs through the AWS worker. The prod daily E2E (payment + happy path) is the end-to-end check. -## Phase 2: distribution from AWS (next PR set, after Phase 1 is verified) +## Phase 2: distribution from AWS (implemented 2026-10-31, off by default) -The video worker (`backend.workers.video_worker`) runs **on the box** with local encoding (`LocalEncodingBackend`): -- Inputs come from GCS once (~300 MB). -- Finals are encoded locally, uploaded to Dropbox/YouTube/Drive from AWS, and pushed to GCS `finals/` (AWS egress, on credits). -- This removes the ~$18/mo of Cloud Run internet egress and the video-encoding-job CPU (~$11/mo). +The video worker (packaging, encode, Dropbox/YouTube/Drive uploads, notifications) can run **on the AWS box** instead of in the Cloud Run job `video-encoding-job`. The large uploads then leave from AWS (credits) instead of GCP (the ~$18/mo of Cloud Run internet egress), and the job's Cloud Run CPU (~$11/mo) goes away. -Needs: -- a `/run-worker` endpoint and env/secrets on the box (the video-encoding-job env list); -- a trigger branch in `WorkerService.trigger_video_worker`, with Cloud Run as the fallback; -- the box's WIF identity mapped to the backend SA's permissions; -- **Spot-interruption recovery for in-flight runs.** Today nothing recovers a video worker that dies after `GENERATING_VIDEO`; build a supervised re-dispatch first; -- the deferred YouTube queue processor runs on the API, so it still egresses from GCP (low volume). +**Encoding path unchanged (output parity)** +- The video worker on the box still encodes through the worker HTTP API, so the deliverables (portrait included) are identical. +- `LocalEncodingBackend` was rejected: it lacks the worker's portrait render. + +**Same-box handoff** (`backend/services/local_encoded_outputs.py`) +- Without it, the video worker would download every final back out of GCS: ~400 MB/job of out-of-pocket egress, which costs more than the move saves. +- The encoder stashes each output under `LOCAL_ENCODED_OUTPUTS_DIR`, keyed by its GCS blob path, and the video worker takes it from there. +- The video worker asks for it per encode (`stash_local_outputs`, set only where the env var exists), so GCE and Cloud Run are unaffected. Copies not taken within 6 h are purged on every stash and at worker boot. + +**`POST /run-worker`** (worker) +- Spawns `python -m backend.workers.video_worker --job-id โ€ฆ` with the box's video-worker env. +- Status is reported through the usual `/status/{id}`. A run counts in `/health` `active_jobs`, so the idle watchdog won't stop the box mid-run. +- It answers 503 unless `/opt/encoding-worker/video-worker-env.json` exists. + +**Env and identity** +- At boot, `video-worker-env.py` reads the **live** `video-encoding-job` definition and resolves its secret refs, so nothing is hand-copied and nothing drifts. +- It adds `LOCAL_ENCODED_OUTPUTS_DIR` and `CLOUD_RUN_JOB`. +- The run uses the job's own identity, `karaoke-backend@`, via a second WIF binding (`aws-encoding-worker-wif-backend-user`). + +**Trigger** (`WorkerService._trigger_video_worker_on_aws`) +- It applies only when Firestore `config/encoding-worker.aws_video_worker` is true **and** the AWS worker ranks first. +- It then ensures the box is running, waits for `/health`, and POSTs `/run-worker`. +- **Any** doubt or failure returns False, and the caller starts the Cloud Run job as today. + +**Not covered** +- Deferred YouTube uploads (the hourly queue processor on the API) still leave from GCP; volume is low. +- Video-worker stdout stays on the box at `/var/log/encoding-worker-runs/`. The job's Firestore worker logs are unaffected, but the error monitor doesn't see the box's stderr. +- With the box on-demand (see the amendment above), Spot-interruption recovery is moot. A systemd restart of the worker during a run would still kill it; the existing `encoding_stuck` flag at 50 min surfaces that. + +**Enable (after stage B, needs `/gcp-admin` for the IAM binding)** +1. Apply the GCP WIF backend binding. +2. Reboot the box once (`stop`/`start`) so `startup-aws.sh` writes the backend credentials and the env. +3. Set `aws_video_worker: true`. +4. Roll back by setting it to false. ## Stage A status (2026-10-29/31) diff --git a/infrastructure/__main__.py b/infrastructure/__main__.py index 1a544e72..d043ffa4 100644 --- a/infrastructure/__main__.py +++ b/infrastructure/__main__.py @@ -168,7 +168,9 @@ encoding_worker_iam_bindings = worker_sas.grant_encoding_worker_permissions( # Federation (no keys). See modules/aws_encoding_wif.py. from modules import aws_encoding_wif # noqa: E402 -aws_encoding_wif_resources = aws_encoding_wif.create_aws_encoding_wif(encoding_worker_sa) +aws_encoding_wif_resources = aws_encoding_wif.create_aws_encoding_wif( + encoding_worker_sa, backend_service_account +) gdrive_validator_sa = worker_sas.create_gdrive_validator_service_account() gdrive_validator_iam_bindings = worker_sas.grant_gdrive_validator_permissions( diff --git a/infrastructure/aws/README.md b/infrastructure/aws/README.md index 047667a0..cd6f06c0 100644 --- a/infrastructure/aws/README.md +++ b/infrastructure/aws/README.md @@ -35,6 +35,8 @@ AWS_ACCESS_KEY_ID=$NOMAD_AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$NOMAD_AWS_SECR 2. **Register the worker in the pool:** append `pulumi stack output fallback_vms_entry` to the JSON list in secret `encoding-worker-fallback-vms`. The backend picks up `:latest` on new instances, and the next deploy picks it up everywhere. 3. **Rank:** by default the AWS worker is a **last resort** (`SPEED_RANK["c7a"] = 90`). To prefer it, set Firestore `config/encoding-worker.speed_rank_overrides = {"c7a": 5}`. Delete the field to roll back; no deploy is needed either way. +4. **Distribution on AWS (Phase 2):** once the AWS worker is preferred (step 3), set Firestore `config/encoding-worker.aws_video_worker = true`. The video worker (Dropbox/YouTube/Drive uploads) then runs on the box via `/run-worker`, with the Cloud Run job as an automatic fallback. Set it to `false` to roll back. Run logs are in `/var/log/encoding-worker-runs/`. + ## Operating it - **Shell:** `aws ssm start-session --target --region us-east-2`. diff --git a/infrastructure/encoding-worker/aws/startup-aws.sh b/infrastructure/encoding-worker/aws/startup-aws.sh index 2d68487f..32a4aba1 100755 --- a/infrastructure/encoding-worker/aws/startup-aws.sh +++ b/infrastructure/encoding-worker/aws/startup-aws.sh @@ -38,11 +38,28 @@ ENCODING_API_KEY=${ENCODING_API_KEY} ENCODING_WORKER_VM_NAME=${VM_NAME} ENCODING_WORKER_PROVIDER=aws GOOGLE_APPLICATION_CREDENTIALS=${WORKER_DIR}/gcp-wif.json +LOCAL_ENCODED_OUTPUTS_DIR=/var/lib/encoding-worker/outputs ENV chmod 600 "${WORKER_DIR}/env" +mkdir -p /var/lib/encoding-worker/outputs -# Keep the idle watchdog current with the deployed code (best effort). +# Keep the idle watchdog + video-worker env helper current (best effort). gcloud storage cp "${BUCKET}/encoding-worker/idle-watchdog.py" "${WORKER_DIR}/idle-watchdog.py" || true +gcloud storage cp "${BUCKET}/encoding-worker/video-worker-env.py" "${WORKER_DIR}/video-worker-env.py" || true + +# Video worker on this box (/run-worker) runs as karaoke-backend@, the Cloud Run +# job's identity, via the same WIF provider. Its env mirrors the live job. +if [ ! -s "${WORKER_DIR}/gcp-wif-backend.json" ]; then + WIF_PROVIDER=$(python3 -c "import json;print(json.load(open('${WORKER_DIR}/gcp-wif.json'))['audience'].split('//iam.googleapis.com/')[1])" 2>/dev/null) || WIF_PROVIDER="" + if [ -n "${WIF_PROVIDER}" ]; then + gcloud iam workload-identity-pools create-cred-config "${WIF_PROVIDER}" \ + --service-account=karaoke-backend@nomadkaraoke.iam.gserviceaccount.com --aws --enable-imdsv2 \ + --output-file="${WORKER_DIR}/gcp-wif-backend.json" || true + fi +fi +if [ -f "${WORKER_DIR}/video-worker-env.py" ]; then + "${VENV}/bin/python" "${WORKER_DIR}/video-worker-env.py" || true +fi echo "[2/5] Downloading version manifest..." EXPECTED_VERSION=$(gcloud storage cat "${BUCKET}/encoding-worker/version.txt" 2>/dev/null | tr -d '[:space:]' || echo "") diff --git a/infrastructure/encoding-worker/aws/video-worker-env.py b/infrastructure/encoding-worker/aws/video-worker-env.py new file mode 100644 index 00000000..db4b8e5e --- /dev/null +++ b/infrastructure/encoding-worker/aws/video-worker-env.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Write the env for running the video worker on the AWS encoding box. + +Mirrors the LIVE Cloud Run job ``video-encoding-job`` (plain env + secret refs +resolved from Secret Manager), so the box runs the video worker with exactly +the configuration Cloud Run does โ€” no hand-copied list to drift. Authenticates +as karaoke-backend@ (the job's service account) via Workload Identity +Federation, and the env it writes points the video worker at the same identity. + +On any failure it removes the env file: /run-worker then answers 503 and the +backend keeps using the Cloud Run job. Run by startup-aws.sh on every boot. +""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path + +PROJECT = "nomadkaraoke" +JOB = "video-encoding-job" +REGION = os.environ.get("CPU_JOBS_REGION", "us-east4") +OUT = Path(os.environ.get("RUN_WORKER_ENV_FILE", "/opt/encoding-worker/video-worker-env.json")) +BACKEND_CREDS = "/opt/encoding-worker/gcp-wif-backend.json" + +# Box-specific settings on top of the Cloud Run job's env. +BOX_ENV = { + "GOOGLE_APPLICATION_CREDENTIALS": BACKEND_CREDS, + # Take finals from the co-located encoder instead of GCS (local_encoded_outputs). + "LOCAL_ENCODED_OUTPUTS_DIR": "/var/lib/encoding-worker/outputs", + # Run-to-completion process like a Cloud Run job (synchronous log writes etc.). + "CLOUD_RUN_JOB": JOB, +} + + +def build_env(job: dict, secret_value) -> dict: + """Plain env + resolved secrets from a run_v2 Job dict, plus BOX_ENV.""" + containers = job["template"]["template"]["containers"] + env: dict = {} + for item in containers[0].get("env", []): + name = item["name"] + if "value" in item: + env[name] = item["value"] + elif "valueSource" in item: + ref = item["valueSource"]["secretKeyRef"] + secret = ref["secret"].rsplit("/", 1)[-1] + env[name] = secret_value(secret, ref.get("version") or "latest") + env.update(BOX_ENV) + return env + + +def main() -> int: + os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = BACKEND_CREDS + try: + import google.auth + import google.auth.transport.requests + from google.cloud import secretmanager + + creds, _ = google.auth.default(scopes=["https://www.googleapis.com/auth/cloud-platform"]) + session = google.auth.transport.requests.AuthorizedSession(creds) + resp = session.get(f"https://run.googleapis.com/v2/projects/{PROJECT}/locations/{REGION}/jobs/{JOB}", timeout=30) + resp.raise_for_status() + sm = secretmanager.SecretManagerServiceClient(credentials=creds) + + def secret_value(secret: str, version: str) -> str: + name = f"projects/{PROJECT}/secrets/{secret}/versions/{version}" + return sm.access_secret_version(name=name).payload.data.decode("utf-8") + + env = build_env(resp.json(), secret_value) + OUT.parent.mkdir(parents=True, exist_ok=True) + tmp = OUT.with_suffix(".tmp") + tmp.write_text(json.dumps(env)) + os.chmod(tmp, 0o600) + os.replace(tmp, OUT) + print(f"video-worker env: {len(env)} vars from {JOB} ({REGION}) -> {OUT}") + return 0 + except Exception as e: # noqa: BLE001 + OUT.unlink(missing_ok=True) + print(f"video-worker env NOT written (worker runs disabled, Cloud Run fallback): {e}", file=sys.stderr) + return 0 # never block the encoder's startup + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/infrastructure/encoding-worker/worker_code_paths.txt b/infrastructure/encoding-worker/worker_code_paths.txt index b09f1558..e2351f9e 100644 --- a/infrastructure/encoding-worker/worker_code_paths.txt +++ b/infrastructure/encoding-worker/worker_code_paths.txt @@ -33,3 +33,4 @@ backend/config.py infrastructure/encoding-worker/ infrastructure/packer/ poetry.lock +backend/services/local_encoded_outputs.py diff --git a/infrastructure/modules/aws_encoding_wif.py b/infrastructure/modules/aws_encoding_wif.py index 2e56ab8a..792c0fe2 100644 --- a/infrastructure/modules/aws_encoding_wif.py +++ b/infrastructure/modules/aws_encoding_wif.py @@ -18,7 +18,10 @@ POOL_ID = "aws-encoding" PROVIDER_ID = f"aws-{AWS_ACCOUNT_ID}" -def create_aws_encoding_wif(encoding_worker_sa: gcp.serviceaccount.Account) -> dict: +def create_aws_encoding_wif( + encoding_worker_sa: gcp.serviceaccount.Account, + backend_sa: gcp.serviceaccount.Account, +) -> dict: pool = gcp.iam.WorkloadIdentityPool( "aws-encoding-wif-pool", workload_identity_pool_id=POOL_ID, @@ -55,5 +58,19 @@ def create_aws_encoding_wif(encoding_worker_sa: gcp.serviceaccount.Account) -> d ) ), ) + # The video worker can also run on the box (/run-worker): it uses the Cloud + # Run video-encoding-job's identity, karaoke-backend@, for exact permission + # parity (see infrastructure/encoding-worker/aws/video-worker-env.py). + backend_binding = gcp.serviceaccount.IAMMember( + "aws-encoding-worker-wif-backend-user", + service_account_id=backend_sa.name, + role="roles/iam.workloadIdentityUser", + member=pool.name.apply( + lambda name: ( + f"principalSet://iam.googleapis.com/{name}/attribute.aws_role/" + f"arn:aws:sts::{AWS_ACCOUNT_ID}:assumed-role/{AWS_WORKER_ROLE}" + ) + ), + ) pulumi.export("aws_encoding_wif_provider", provider.name) - return {"pool": pool, "provider": provider, "binding": binding} + return {"pool": pool, "provider": provider, "binding": binding, "backend_binding": backend_binding} diff --git a/pyproject.toml b/pyproject.toml index bb1a2d2e..643dc786 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.288.1" +version = "0.289.0" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT"