mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
feat(encoding): run the video worker (distribution) on the AWS box (v0.289.0) (#1240)
## Summary Phase 2 of moving encoding to AWS; Phase 1 was #1236 and #1239. Design and rationale are in `docs/archive/2026-10-29-aws-encoding-worker-plan.md` § Phase 2. When Firestore `config/encoding-worker.aws_video_worker` is true **and** the AWS worker ranks first, `trigger_video_worker` runs the video worker **on the AWS box** instead of starting the Cloud Run job `video-encoding-job`. - **Uploads move to AWS:** the Dropbox/YouTube/Drive uploads (~187 GB/mo, ~$18/mo of Cloud Run internet egress) leave from AWS, on credits. - **No Cloud Run CPU** for the job (~$11/mo). - **Off by default.** Any failure falls back to the Cloud Run job, so this path can't drop a job. ## Changes - **`POST /run-worker` (worker):** - Spawns `python -m backend.workers.video_worker --job-id …` with the box's video-worker env. - Status is reported via `/status/{id}`. - A run counts in `/health` `active_jobs` from the moment it's accepted, so the idle watchdog won't stop the box mid-run. - Returns 503 unless the env file exists. - `job_id` is pattern-validated. - Any startup failure marks the run failed. - **`backend/services/local_encoded_outputs.py`:** a same-box handoff. - When the co-located video worker asks (`stash_local_outputs`, set only where `LOCAL_ENCODED_OUTPUTS_DIR` exists), the encoder keeps local copies of the finals, keyed by blob path. - The video worker takes those instead of downloading them back out of GCS. Without this, the move would *add* ~400 MB/job of GCS egress. - Copies are made off the event loop and purged after 6 h. - No change on GCE or Cloud Run. - **`video-worker-env.py` (box, every boot):** - Mirrors the **live** `video-encoding-job` env, with secrets resolved, so nothing is hand-copied and nothing drifts. - Runs as the job's identity, `karaoke-backend@`, via a second WIF binding (Pulumi `aws-encoding-worker-wif-backend-user`). - On failure it writes no env file, so `/run-worker` answers 503 and the Cloud Run job is used. - **`WorkerService._trigger_video_worker_on_aws`:** checks the flag and that the AWS worker ranks first, ensures it's running, waits for health, then POSTs `/run-worker`. Returns False on any doubt. - **Also:** `startup-aws.sh` creates the backend credential config and runs the env helper; CI uploads `video-worker-env.py`; `worker_code_paths.txt` now gates `local_encoded_outputs.py`. ## Rollout (needs `/gcp-admin`) 1. **Before merge:** targeted `pulumi up` of the backend WIF binding. It's additive. 2. **After stage B of Phase 1** (AWS worker preferred, timings verified): stop/start the box once so the new `startup-aws.sh` writes the env, then set `aws_video_worker: true`. Set it to `false` to roll back. ## Testing - **27 new tests:** - handoff stash/take/purge, path-escape guard, and the per-request stash flag; - `/run-worker`: disabled, bad module, dedupe, active immediately, spawn args/env, failure log tail, wheel-not-ready, safe start, unsafe ids; - env builder; - the trigger's AWS branch: off, not first, happy path, didn't start, refusals, errors. - **Full backend + unit suites:** no new failures (the 13 local failures are identical on main). - **Local CodeRabbit:** 5 findings, all fixed. The code-paths gate test also caught a missing worker path, now fixed. @coderabbitai ignore 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_012RYLVSR5qgwsxTQtvGimKF --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
20b957958c
commit
78264fa23a
17 files changed
+776
-19
No files matched your search
@@ -1407,6 +1407,8 @@ jobs:
|
||||
"${BUCKET}/encoding-worker/startup-aws.sh"
|
||||
gsutil cp infrastructure/encoding-worker/aws/idle-watchdog.py \
|
||||
"${BUCKET}/encoding-worker/idle-watchdog.py"
|
||||
gsutil cp infrastructure/encoding-worker/aws/video-worker-env.py \
|
||||
"${BUCKET}/encoding-worker/video-worker-env.py"
|
||||
|
||||
# 4. Write version manifest for verification
|
||||
echo "Writing version manifest..."
|
||||
|
||||
@@ -467,6 +467,12 @@ class GCEEncodingBackend(EncodingBackend):
|
||||
|
||||
# Screens the theme leaves out: the encoder must skip them rather than
|
||||
# rebuild them from whatever matching PNG it finds in the job folder.
|
||||
# On the AWS box the video worker sits next to the encoder: ask it
|
||||
# to keep local copies so we take them instead of downloading back
|
||||
# from GCS (local_encoded_outputs). Unset everywhere else.
|
||||
if os.environ.get("LOCAL_ENCODED_OUTPUTS_DIR"):
|
||||
encoding_config["stash_local_outputs"] = True
|
||||
|
||||
for flag in ("include_title_screen", "include_end_screen"):
|
||||
if input_config.options.get(flag) is False:
|
||||
encoding_config[flag] = False
|
||||
|
||||
@@ -115,6 +115,9 @@ class EncodingWorkerConfig:
|
||||
# {machine_family: rank} replacing SPEED_RANK at runtime, e.g. {"c7a": 5}
|
||||
# makes the AWS worker preferred. Absent/empty ⇒ the built-in ranking.
|
||||
speed_rank_overrides: dict = field(default_factory=dict)
|
||||
# True ⇒ when the AWS worker ranks first, the video worker (distribution)
|
||||
# also runs on it instead of the Cloud Run job (worker_service). Default off.
|
||||
aws_video_worker: bool = False
|
||||
|
||||
@property
|
||||
def primary_url(self) -> str:
|
||||
@@ -349,6 +352,7 @@ class EncodingWorkerManager:
|
||||
active_override_set_at=data.get("active_override_set_at"),
|
||||
capacity_state=data.get("capacity_state") or {},
|
||||
speed_rank_overrides=data.get("speed_rank_overrides") or {},
|
||||
aws_video_worker=bool(data.get("aws_video_worker", False)),
|
||||
)
|
||||
|
||||
def update_activity(self) -> None:
|
||||
|
||||
@@ -27,7 +27,7 @@ os.environ.setdefault("GCE_METADATA_MTLS_MODE", "none")
|
||||
from fastapi import FastAPI, BackgroundTasks, HTTPException, Header, Depends
|
||||
from google.cloud import storage
|
||||
from packaging.version import Version
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from . import self_heal
|
||||
from .persistence import JobStatePersister
|
||||
@@ -120,6 +120,12 @@ async def lifespan(app: FastAPI):
|
||||
logger.critical("Encoding worker booted unhealthy, exiting for systemd restart: %s", problem)
|
||||
raise RuntimeError(f"Encoding worker booted unhealthy: {problem}")
|
||||
|
||||
# AWS box only (no-op elsewhere): drop local finals no video worker took.
|
||||
from backend.services import local_encoded_outputs
|
||||
removed = await asyncio.to_thread(local_encoded_outputs.purge_stale)
|
||||
if removed:
|
||||
logger.info("Purged %d stale locally-stashed encoded outputs", removed)
|
||||
|
||||
# Startup: read back any in-progress jobs from Firestore. They were
|
||||
# interrupted by the restart that brought us here — their work_dir is
|
||||
# gone, so mark them failed with the restart code so polls return a
|
||||
@@ -1341,6 +1347,21 @@ async def process_job(job_id: str, request: EncodeRequest):
|
||||
jobs[job_id]["output_files"] = blob_paths
|
||||
logger.info(f"Output files (blob paths): {blob_paths}")
|
||||
|
||||
# On the AWS box the video worker runs here too: keep local copies
|
||||
# (keyed by blob path) so it needn't download them back out of GCS.
|
||||
# Only when that co-located video worker asked (stash_local_outputs),
|
||||
# and only where LOCAL_ENCODED_OUTPUTS_DIR is set. Off the event loop:
|
||||
# it copies ~400 MB.
|
||||
if (request.encoding_config or {}).get("stash_local_outputs"):
|
||||
from backend.services import local_encoded_outputs
|
||||
|
||||
def _stash_all():
|
||||
local_encoded_outputs.purge_stale()
|
||||
for local_path, blob_path in zip(local_output_files, blob_paths):
|
||||
local_encoded_outputs.stash(Path(local_path), blob_path)
|
||||
|
||||
await asyncio.to_thread(_stash_all)
|
||||
|
||||
jobs[job_id]["status"] = "complete"
|
||||
jobs[job_id]["progress"] = 100
|
||||
_persist(job_id)
|
||||
@@ -1494,6 +1515,118 @@ async def submit_render_video_job(request: RenderVideoRequest, background_tasks:
|
||||
return {"status": "accepted", "job_id": job_id}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# /run-worker — run a backend worker (the video worker) on this box
|
||||
# ---------------------------------------------------------------------------
|
||||
# AWS box only: there the video worker (packaging + encode + Dropbox/YouTube/Drive
|
||||
# distribution) runs next to the encoder, so the big uploads leave from AWS
|
||||
# (credits) instead of GCP, and finals are handed over locally
|
||||
# (local_encoded_outputs). Enabled only where the environment file written by
|
||||
# startup-aws.sh exists; the backend falls back to the Cloud Run job otherwise.
|
||||
RUN_WORKER_ENV_FILE = os.environ.get("RUN_WORKER_ENV_FILE", "/opt/encoding-worker/video-worker-env.json")
|
||||
RUN_WORKER_LOG_DIR = Path(os.environ.get("RUN_WORKER_LOG_DIR", "/var/log/encoding-worker-runs"))
|
||||
_RUN_WORKER_MODULES = frozenset({"video_worker"})
|
||||
|
||||
|
||||
class RunWorkerRequest(BaseModel):
|
||||
# Used in a log file name and as a CLI argument: gen job ids only.
|
||||
job_id: str = Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$")
|
||||
module: str = "video_worker"
|
||||
|
||||
|
||||
def run_worker_id(module: str, job_id: str) -> str:
|
||||
return f"run-{module}-{job_id}"
|
||||
|
||||
|
||||
def _run_worker_env() -> Optional[dict]:
|
||||
"""Env for the worker subprocess: ours + the video-worker env file, or None if absent."""
|
||||
try:
|
||||
with open(RUN_WORKER_ENV_FILE) as f:
|
||||
extra = json.load(f)
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
if not isinstance(extra, dict):
|
||||
return None
|
||||
env = dict(os.environ)
|
||||
env.pop("ENCODING_API_KEY", None) # the worker's own key isn't the subprocess's business
|
||||
env.update({str(k): str(v) for k, v in extra.items()})
|
||||
return env
|
||||
|
||||
|
||||
def _watch_worker_run(run_id: str, proc: subprocess.Popen, log_path: Path) -> None:
|
||||
rc = proc.wait()
|
||||
state = jobs.get(run_id)
|
||||
if state is None:
|
||||
return
|
||||
if rc == 0:
|
||||
state["status"] = "complete"
|
||||
state["progress"] = 100
|
||||
else:
|
||||
tail = ""
|
||||
try:
|
||||
tail = log_path.read_text(errors="replace")[-1500:]
|
||||
except OSError:
|
||||
pass
|
||||
state["status"] = "failed"
|
||||
state["error"] = f"{state.get('module')} exited {rc}: {tail}"
|
||||
state["returncode"] = rc
|
||||
_persist(run_id)
|
||||
logger.info("Worker run %s finished rc=%s (log %s)", run_id, rc, log_path)
|
||||
|
||||
|
||||
def _start_worker_run(run_id: str, request: RunWorkerRequest, env: dict) -> None:
|
||||
# The subprocess imports the installed wheel: make sure it's the latest.
|
||||
if not ensure_latest_wheel():
|
||||
jobs[run_id].update(status="failed", error="karaoke-gen wheel not ready on this worker")
|
||||
_persist(run_id)
|
||||
return
|
||||
RUN_WORKER_LOG_DIR.mkdir(parents=True, exist_ok=True)
|
||||
log_path = RUN_WORKER_LOG_DIR / f"{request.job_id}-{uuid.uuid4().hex[:8]}.log"
|
||||
with open(log_path, "ab") as log:
|
||||
proc = subprocess.Popen(
|
||||
[sys.executable, "-m", f"backend.workers.{request.module}", "--job-id", request.job_id],
|
||||
env=env, stdout=log, stderr=subprocess.STDOUT, cwd="/tmp",
|
||||
)
|
||||
jobs[run_id].update(status="running", pid=proc.pid, log=str(log_path))
|
||||
_persist(run_id)
|
||||
logger.info("Started %s for job %s (pid %s, log %s)", request.module, request.job_id, proc.pid, log_path)
|
||||
threading.Thread(target=_watch_worker_run, args=(run_id, proc, log_path), daemon=True).start()
|
||||
|
||||
|
||||
def _start_worker_run_safe(run_id: str, request: RunWorkerRequest, env: dict) -> None:
|
||||
"""Never leave a run stuck "running": that would hold the box up (idle
|
||||
watchdog) and lose the job (the backend got "accepted", so no fallback)."""
|
||||
try:
|
||||
_start_worker_run(run_id, request, env)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.exception("Failed to start worker run %s", run_id)
|
||||
jobs[run_id].update(status="failed", error=f"failed to start worker: {exc}")
|
||||
_persist(run_id)
|
||||
|
||||
|
||||
@app.post("/run-worker")
|
||||
async def submit_worker_run(request: RunWorkerRequest, _auth: bool = Depends(verify_api_key)):
|
||||
if request.module not in _RUN_WORKER_MODULES:
|
||||
raise HTTPException(status_code=400, detail=f"module {request.module!r} not allowed")
|
||||
env = _run_worker_env()
|
||||
if env is None:
|
||||
raise HTTPException(status_code=503, detail="worker runs are not enabled on this worker")
|
||||
run_id = run_worker_id(request.module, request.job_id)
|
||||
existing = jobs.get(run_id)
|
||||
if existing and existing.get("status") == "running":
|
||||
return {"status": "in_progress", "job_id": run_id}
|
||||
# "running" from the start: /health counts only running jobs, and the
|
||||
# idle watchdog must not stop the box while the wheel check precedes spawn.
|
||||
jobs[run_id] = {
|
||||
"job_id": run_id, "status": "running", "progress": 0, "error": None,
|
||||
"output_files": None, "kind": "worker_run", "module": request.module,
|
||||
}
|
||||
_persist(run_id)
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.run_in_executor(light_executor, _start_worker_run_safe, run_id, request, env)
|
||||
return {"status": "accepted", "job_id": run_id}
|
||||
|
||||
|
||||
@app.post("/encode")
|
||||
async def submit_encode_job(request: EncodeRequest, background_tasks: BackgroundTasks, _auth: bool = Depends(verify_api_key)):
|
||||
# Submit an encoding job
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Same-box handoff of encoded finals from the encoding worker to the video worker.
|
||||
|
||||
On the AWS encoding worker, the video worker (distribution) runs on the same box
|
||||
as the encoding service. Without this, the video worker would download every
|
||||
final the encoder just uploaded back out of GCS — ~400 MB of out-of-pocket GCS
|
||||
egress per job, more than moving distribution to AWS saves. So the encoder
|
||||
"stashes" a local copy of each output keyed by its GCS blob path, and the video
|
||||
worker "takes" it instead of downloading.
|
||||
|
||||
Active only when ``LOCAL_ENCODED_OUTPUTS_DIR`` is set (the AWS box). Everywhere
|
||||
else ``stash`` is a no-op and ``take`` returns False, i.e. today's behaviour.
|
||||
Best effort: any failure falls back to GCS.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ENV_VAR = "LOCAL_ENCODED_OUTPUTS_DIR"
|
||||
MAX_AGE_SECONDS = 6 * 3600 # a take happens minutes after the stash
|
||||
|
||||
|
||||
def _root() -> Optional[Path]:
|
||||
raw = os.environ.get(ENV_VAR, "").strip()
|
||||
return Path(raw) if raw else None
|
||||
|
||||
|
||||
def _path_for(root: Path, blob_path: str) -> Path:
|
||||
rel = blob_path.lstrip("/")
|
||||
target = (root / rel).resolve()
|
||||
if root.resolve() not in target.parents:
|
||||
raise ValueError(f"blob path escapes the stash dir: {blob_path!r}")
|
||||
return target
|
||||
|
||||
|
||||
def stash(local_file: Path, blob_path: str) -> bool:
|
||||
"""Keep a copy of ``local_file`` for a co-located video worker. Never raises."""
|
||||
root = _root()
|
||||
if root is None:
|
||||
return False
|
||||
try:
|
||||
dest = _path_for(root, blob_path)
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = dest.with_name(dest.name + ".partial")
|
||||
shutil.copyfile(local_file, tmp)
|
||||
os.replace(tmp, dest)
|
||||
return True
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("Could not stash %s for local handoff: %s", blob_path, e)
|
||||
return False
|
||||
|
||||
|
||||
def take(blob_path: str, dest: str) -> bool:
|
||||
"""Move the stashed copy of ``blob_path`` to ``dest``. False ⇒ download from GCS."""
|
||||
root = _root()
|
||||
if root is None:
|
||||
return False
|
||||
try:
|
||||
src = _path_for(root, blob_path)
|
||||
if not src.is_file():
|
||||
return False
|
||||
Path(dest).parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.move(str(src), dest)
|
||||
return True
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("Local handoff of %s failed, falling back to GCS: %s", blob_path, e)
|
||||
return False
|
||||
|
||||
|
||||
def purge_stale(max_age_seconds: float = MAX_AGE_SECONDS) -> int:
|
||||
"""Delete stashed files older than ``max_age_seconds`` (never taken). Returns count."""
|
||||
root = _root()
|
||||
if root is None or not root.is_dir():
|
||||
return 0
|
||||
cutoff = time.time() - max_age_seconds
|
||||
removed = 0
|
||||
for p in root.rglob("*"):
|
||||
try:
|
||||
if p.is_file() and p.stat().st_mtime < cutoff:
|
||||
p.unlink()
|
||||
removed += 1
|
||||
except OSError:
|
||||
continue
|
||||
return removed
|
||||
@@ -645,8 +645,69 @@ class WorkerService:
|
||||
await asyncio.to_thread(self._bump_worker_generation, job_id) # Firestore write
|
||||
self._start_encoding_worker_warmup(job_id)
|
||||
if self._use_cloud_tasks and self.settings.use_cloud_run_jobs_for_video:
|
||||
if await self._trigger_video_worker_on_aws(job_id):
|
||||
return True
|
||||
return await self._trigger_cloud_run_job(job_id)
|
||||
return await self.trigger_worker("video", job_id)
|
||||
|
||||
async def _trigger_video_worker_on_aws(self, job_id: str) -> bool:
|
||||
"""Run the video worker on the AWS encoding box instead of Cloud Run, if enabled.
|
||||
|
||||
Only when Firestore ``config/encoding-worker.aws_video_worker`` is true AND
|
||||
the AWS worker ranks first (it's the preferred encoder, via
|
||||
``speed_rank_overrides``). There the distribution uploads (Dropbox/YouTube/
|
||||
Drive) leave from AWS, and finals are handed over locally instead of
|
||||
downloaded back out of GCS. Returns False on any doubt or failure so the
|
||||
caller falls back to the Cloud Run job — this path must never lose a job.
|
||||
"""
|
||||
try:
|
||||
from backend.services.encoding_service import get_encoding_service
|
||||
|
||||
manager = self._build_encoding_worker_manager()
|
||||
config = await asyncio.to_thread(manager.get_config)
|
||||
if not getattr(config, "aws_video_worker", False):
|
||||
return False
|
||||
candidates = await asyncio.to_thread(manager.ranked_candidates, config)
|
||||
if not candidates or candidates[0].provider != "aws":
|
||||
return False
|
||||
aws = candidates[0]
|
||||
result = await asyncio.to_thread(manager.ensure_any_running, candidates)
|
||||
if result.get("vm_name") != aws.vm_name:
|
||||
logger.warning(
|
||||
f"[job:{job_id}] AWS video worker: {aws.vm_name} unavailable "
|
||||
f"(started {result.get('vm_name')}); using Cloud Run"
|
||||
)
|
||||
return False
|
||||
await manager.wait_for_worker_ready(aws.vm_name, f"{aws.url}/health", zone=aws.zone)
|
||||
|
||||
service = get_encoding_service()
|
||||
service._load_credentials()
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
resp = await client.post(
|
||||
f"{aws.url}/run-worker",
|
||||
json={"job_id": job_id, "module": "video_worker"},
|
||||
headers={"X-API-Key": service._api_key},
|
||||
)
|
||||
if resp.status_code == 200 and resp.json().get("status") in ("accepted", "in_progress"):
|
||||
logger.info(f"[job:{job_id}] Video worker started on AWS ({aws.vm_name})")
|
||||
return True
|
||||
logger.warning(
|
||||
f"[job:{job_id}] AWS video worker refused ({resp.status_code}: {resp.text[:300]}); using Cloud Run"
|
||||
)
|
||||
return False
|
||||
except Exception as e: # noqa: BLE001 — fall back to Cloud Run, never drop the job
|
||||
logger.warning(f"[job:{job_id}] AWS video worker dispatch failed ({e}); using Cloud Run")
|
||||
return False
|
||||
|
||||
def _build_encoding_worker_manager(self):
|
||||
from backend.services.encoding_worker_manager import EncodingWorkerManager
|
||||
from google.cloud import compute_v1, firestore
|
||||
|
||||
return EncodingWorkerManager(
|
||||
db=firestore.Client(project=self.settings.google_cloud_project),
|
||||
compute_client=compute_v1.InstancesClient(),
|
||||
project_id=self.settings.google_cloud_project,
|
||||
)
|
||||
|
||||
async def _trigger_cloud_run_job(self, job_id: str) -> bool:
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,304 @@
|
||||
"""Phase 2: the video worker (distribution) on the AWS encoding box.
|
||||
|
||||
Covers the same-box finals handoff, the worker's /run-worker endpoint, the
|
||||
box's env builder (mirrors the live Cloud Run job), and the backend trigger's
|
||||
AWS branch — which must fall back to the Cloud Run job on any doubt.
|
||||
"""
|
||||
import asyncio
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
from backend.services import local_encoded_outputs as leo
|
||||
|
||||
# --- local_encoded_outputs ------------------------------------------------------
|
||||
|
||||
BLOB = "jobs/abc123/finals/Artist - Title (Final Karaoke Lossless 4k).mp4"
|
||||
|
||||
|
||||
def test_handoff_is_noop_without_env(tmp_path, monkeypatch):
|
||||
monkeypatch.delenv(leo.ENV_VAR, raising=False)
|
||||
f = tmp_path / "x.mp4"
|
||||
f.write_bytes(b"v")
|
||||
assert leo.stash(f, BLOB) is False
|
||||
assert leo.take(BLOB, str(tmp_path / "out.mp4")) is False
|
||||
|
||||
|
||||
def test_stash_then_take_moves_file(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv(leo.ENV_VAR, str(tmp_path / "stash"))
|
||||
src = tmp_path / "encoded.mp4"
|
||||
src.write_bytes(b"video-bytes")
|
||||
assert leo.stash(src, BLOB) is True
|
||||
dest = tmp_path / "work" / "final.mp4"
|
||||
assert leo.take(BLOB, str(dest)) is True
|
||||
assert dest.read_bytes() == b"video-bytes"
|
||||
assert leo.take(BLOB, str(dest)) is False # moved, so a second take falls back to GCS
|
||||
|
||||
|
||||
def test_take_missing_falls_back(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv(leo.ENV_VAR, str(tmp_path))
|
||||
assert leo.take(BLOB, str(tmp_path / "o.mp4")) is False
|
||||
|
||||
|
||||
def test_blob_path_cannot_escape_stash_dir(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv(leo.ENV_VAR, str(tmp_path / "stash"))
|
||||
f = tmp_path / "x"
|
||||
f.write_bytes(b"1")
|
||||
assert leo.stash(f, "../../etc/evil") is False
|
||||
|
||||
|
||||
def test_purge_stale_removes_only_old_files(tmp_path, monkeypatch):
|
||||
import os
|
||||
import time
|
||||
monkeypatch.setenv(leo.ENV_VAR, str(tmp_path))
|
||||
old, new = tmp_path / "jobs/a/old.mp4", tmp_path / "jobs/b/new.mp4"
|
||||
for p in (old, new):
|
||||
p.parent.mkdir(parents=True)
|
||||
p.write_bytes(b"1")
|
||||
os.utime(old, (time.time() - 2 * 86400,) * 2)
|
||||
assert leo.purge_stale() == 1
|
||||
assert not old.exists() and new.exists()
|
||||
|
||||
|
||||
# --- worker /run-worker -----------------------------------------------------------
|
||||
|
||||
@pytest.fixture
|
||||
def worker(monkeypatch, tmp_path):
|
||||
from backend.services.gce_encoding import main
|
||||
env_file = tmp_path / "video-worker-env.json"
|
||||
monkeypatch.setattr(main, "RUN_WORKER_ENV_FILE", str(env_file))
|
||||
monkeypatch.setattr(main, "RUN_WORKER_LOG_DIR", tmp_path / "runs")
|
||||
monkeypatch.setattr(main, "_persist", lambda job_id: None)
|
||||
main.jobs.clear()
|
||||
yield main, env_file
|
||||
main.jobs.clear()
|
||||
|
||||
|
||||
def _req(main, job_id="abc123", module="video_worker"):
|
||||
return main.RunWorkerRequest(job_id=job_id, module=module)
|
||||
|
||||
|
||||
def test_run_worker_disabled_without_env_file(worker):
|
||||
main, _ = worker
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
asyncio.run(main.submit_worker_run(_req(main)))
|
||||
assert exc.value.status_code == 503
|
||||
|
||||
|
||||
def test_run_worker_rejects_unknown_module(worker):
|
||||
main, env_file = worker
|
||||
env_file.write_text("{}")
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
asyncio.run(main.submit_worker_run(_req(main, module="audio_worker")))
|
||||
assert exc.value.status_code == 400
|
||||
|
||||
|
||||
def test_run_worker_dedupes_running_job(worker):
|
||||
main, env_file = worker
|
||||
env_file.write_text("{}")
|
||||
main.jobs[main.run_worker_id("video_worker", "abc123")] = {"status": "running"}
|
||||
out = asyncio.run(main.submit_worker_run(_req(main)))
|
||||
assert out["status"] == "in_progress"
|
||||
|
||||
|
||||
def test_run_worker_counts_as_active_immediately(worker):
|
||||
main, env_file = worker
|
||||
env_file.write_text(json.dumps({"ENVIRONMENT": "production"}))
|
||||
with patch.object(main, "_start_worker_run"):
|
||||
out = asyncio.run(main.submit_worker_run(_req(main)))
|
||||
assert out["status"] == "accepted"
|
||||
assert main.jobs[out["job_id"]]["status"] == "running" # /health active_jobs counts it
|
||||
|
||||
|
||||
def test_start_worker_run_spawns_module_with_box_env(worker):
|
||||
main, env_file = worker
|
||||
env_file.write_text(json.dumps({"ENVIRONMENT": "production", "LOCAL_ENCODED_OUTPUTS_DIR": "/x"}))
|
||||
run_id = main.run_worker_id("video_worker", "abc123")
|
||||
main.jobs[run_id] = {"status": "running", "module": "video_worker"}
|
||||
proc = MagicMock(pid=4242)
|
||||
proc.wait.return_value = 0
|
||||
with patch.object(main, "ensure_latest_wheel", return_value=True), \
|
||||
patch.object(main.subprocess, "Popen", return_value=proc) as popen:
|
||||
main._start_worker_run(run_id, _req(main), main._run_worker_env())
|
||||
args, kwargs = popen.call_args
|
||||
assert args[0][1:] == ["-m", "backend.workers.video_worker", "--job-id", "abc123"]
|
||||
assert kwargs["env"]["LOCAL_ENCODED_OUTPUTS_DIR"] == "/x"
|
||||
assert "ENCODING_API_KEY" not in kwargs["env"]
|
||||
main._watch_worker_run(run_id, proc, Path(main.jobs[run_id]["log"]))
|
||||
assert main.jobs[run_id]["status"] == "complete"
|
||||
|
||||
|
||||
def test_failed_run_reports_log_tail(worker, tmp_path):
|
||||
main, _ = worker
|
||||
run_id = main.run_worker_id("video_worker", "abc123")
|
||||
main.jobs[run_id] = {"status": "running", "module": "video_worker"}
|
||||
log = tmp_path / "run.log"
|
||||
log.write_text("Traceback ... Dropbox upload failed")
|
||||
proc = MagicMock()
|
||||
proc.wait.return_value = 1
|
||||
main._watch_worker_run(run_id, proc, log)
|
||||
assert main.jobs[run_id]["status"] == "failed"
|
||||
assert "Dropbox upload failed" in main.jobs[run_id]["error"]
|
||||
|
||||
|
||||
def test_wheel_not_ready_fails_run_without_spawning(worker):
|
||||
main, env_file = worker
|
||||
env_file.write_text("{}")
|
||||
run_id = main.run_worker_id("video_worker", "abc123")
|
||||
main.jobs[run_id] = {"status": "running"}
|
||||
with patch.object(main, "ensure_latest_wheel", return_value=False), \
|
||||
patch.object(main.subprocess, "Popen") as popen:
|
||||
main._start_worker_run(run_id, _req(main), {})
|
||||
popen.assert_not_called()
|
||||
assert main.jobs[run_id]["status"] == "failed"
|
||||
|
||||
|
||||
# --- video-worker-env.py ------------------------------------------------------------
|
||||
|
||||
_ENV_PATH = Path(__file__).resolve().parents[2] / "infrastructure" / "encoding-worker" / "aws" / "video-worker-env.py"
|
||||
_spec = importlib.util.spec_from_file_location("video_worker_env", _ENV_PATH)
|
||||
vwe = importlib.util.module_from_spec(_spec)
|
||||
_spec.loader.exec_module(vwe)
|
||||
|
||||
|
||||
def test_build_env_mirrors_job_and_resolves_secrets():
|
||||
job = {"template": {"template": {"containers": [{"env": [
|
||||
{"name": "ENVIRONMENT", "value": "production"},
|
||||
{"name": "POSTMARK_SERVER_TOKEN", "valueSource": {"secretKeyRef": {
|
||||
"secret": "projects/nomadkaraoke/secrets/postmark-server-token", "version": "latest"}}},
|
||||
]}]}}}
|
||||
env = vwe.build_env(job, lambda secret, version: f"<{secret}@{version}>")
|
||||
assert env["ENVIRONMENT"] == "production"
|
||||
assert env["POSTMARK_SERVER_TOKEN"] == "<postmark-server-token@latest>"
|
||||
assert env["GOOGLE_APPLICATION_CREDENTIALS"].endswith("gcp-wif-backend.json")
|
||||
assert env["LOCAL_ENCODED_OUTPUTS_DIR"] and env["CLOUD_RUN_JOB"] == "video-encoding-job"
|
||||
|
||||
|
||||
# --- backend trigger ------------------------------------------------------------------
|
||||
|
||||
def _candidate(provider, vm="aws-encoding-worker-1"):
|
||||
return SimpleNamespace(provider=provider, vm_name=vm, url="http://3.3.3.3:8080", zone="us-east-2b")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def service():
|
||||
from backend.services.worker_service import WorkerService
|
||||
svc = WorkerService.__new__(WorkerService)
|
||||
svc.settings = MagicMock(google_cloud_project="nomadkaraoke")
|
||||
return svc
|
||||
|
||||
|
||||
def _manager(aws_video_worker=True, top="aws", started="aws-encoding-worker-1"):
|
||||
m = MagicMock()
|
||||
m.get_config.return_value = SimpleNamespace(aws_video_worker=aws_video_worker)
|
||||
m.ranked_candidates.return_value = [_candidate(top, "aws-encoding-worker-1" if top == "aws" else "encoding-worker-a")]
|
||||
m.ensure_any_running.return_value = {"vm_name": started}
|
||||
m.wait_for_worker_ready = AsyncMock()
|
||||
return m
|
||||
|
||||
|
||||
def _run(service, manager, response=None, post_exc=None):
|
||||
client = MagicMock()
|
||||
client.__aenter__ = AsyncMock(return_value=client)
|
||||
client.__aexit__ = AsyncMock(return_value=False)
|
||||
client.post = AsyncMock(side_effect=post_exc, return_value=response)
|
||||
enc = MagicMock(_api_key="k")
|
||||
with patch.object(service, "_build_encoding_worker_manager", return_value=manager), \
|
||||
patch("backend.services.encoding_service.get_encoding_service", return_value=enc), \
|
||||
patch("backend.services.worker_service.httpx.AsyncClient", return_value=client):
|
||||
return asyncio.run(service._trigger_video_worker_on_aws("abc123")), client
|
||||
|
||||
|
||||
def test_aws_dispatch_off_by_default(service):
|
||||
ok, client = _run(service, _manager(aws_video_worker=False))
|
||||
assert ok is False
|
||||
client.post.assert_not_called()
|
||||
|
||||
|
||||
def test_aws_dispatch_requires_aws_to_rank_first(service):
|
||||
ok, client = _run(service, _manager(top=None))
|
||||
assert ok is False
|
||||
client.post.assert_not_called()
|
||||
|
||||
|
||||
def test_aws_dispatch_posts_run_worker(service):
|
||||
resp = MagicMock(status_code=200)
|
||||
resp.json.return_value = {"status": "accepted"}
|
||||
ok, client = _run(service, _manager(), response=resp)
|
||||
assert ok is True
|
||||
url = client.post.call_args.args[0]
|
||||
assert url == "http://3.3.3.3:8080/run-worker"
|
||||
assert client.post.call_args.kwargs["json"] == {"job_id": "abc123", "module": "video_worker"}
|
||||
|
||||
|
||||
def test_aws_dispatch_falls_back_when_aws_did_not_start(service):
|
||||
ok, client = _run(service, _manager(started="encoding-worker-a"))
|
||||
assert ok is False
|
||||
client.post.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status,body", [(503, {"detail": "not enabled"}), (200, {"status": "weird"})])
|
||||
def test_aws_dispatch_falls_back_on_refusal(service, status, body):
|
||||
resp = MagicMock(status_code=status, text=json.dumps(body))
|
||||
resp.json.return_value = body
|
||||
ok, _ = _run(service, _manager(), response=resp)
|
||||
assert ok is False
|
||||
|
||||
|
||||
def test_aws_dispatch_falls_back_on_error(service):
|
||||
ok, _ = _run(service, _manager(), post_exc=ConnectionError("boom"))
|
||||
assert ok is False
|
||||
|
||||
|
||||
def test_start_failure_never_leaves_run_stuck(worker):
|
||||
main, env_file = worker
|
||||
run_id = main.run_worker_id("video_worker", "abc123")
|
||||
main.jobs[run_id] = {"status": "running"}
|
||||
with patch.object(main, "_start_worker_run", side_effect=OSError("disk full")):
|
||||
main._start_worker_run_safe(run_id, _req(main), {})
|
||||
assert main.jobs[run_id]["status"] == "failed"
|
||||
assert "disk full" in main.jobs[run_id]["error"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bad", ["../../tmp/x", "-h", "", "a b", "x" * 80])
|
||||
def test_run_worker_rejects_unsafe_job_ids(bad):
|
||||
from pydantic import ValidationError
|
||||
from backend.services.gce_encoding import main
|
||||
with pytest.raises(ValidationError):
|
||||
main.RunWorkerRequest(job_id=bad)
|
||||
|
||||
|
||||
def test_gce_backend_requests_stash_only_on_the_box(monkeypatch):
|
||||
"""The encode payload asks for local copies only where the video worker
|
||||
shares the box with the encoder (LOCAL_ENCODED_OUTPUTS_DIR set)."""
|
||||
from backend.services.encoding_interface import EncodingInput, GCEEncodingBackend
|
||||
|
||||
async def capture(env_value):
|
||||
if env_value:
|
||||
monkeypatch.setenv("LOCAL_ENCODED_OUTPUTS_DIR", env_value)
|
||||
else:
|
||||
monkeypatch.delenv("LOCAL_ENCODED_OUTPUTS_DIR", raising=False)
|
||||
svc = MagicMock()
|
||||
svc.encode_videos = AsyncMock(return_value={"status": "complete", "output_files": []})
|
||||
backend = GCEEncodingBackend()
|
||||
backend._service = svc
|
||||
with patch.object(GCEEncodingBackend, "_get_service", return_value=svc):
|
||||
await backend.encode(EncodingInput(
|
||||
title_video_path="t", karaoke_video_path="k", instrumental_audio_path="i",
|
||||
artist="A", title="T", output_dir="/tmp",
|
||||
options={"job_id": "abc123_g1", "input_gcs_path": "gs://b/jobs/abc123/",
|
||||
"output_gcs_path": "gs://b/jobs/abc123/finals/"},
|
||||
))
|
||||
return svc.encode_videos.call_args
|
||||
|
||||
on = asyncio.run(capture("/var/lib/encoding-worker/outputs"))
|
||||
off = asyncio.run(capture(None))
|
||||
on_cfg = on.kwargs.get("encoding_config") or on.args[-1]
|
||||
off_cfg = off.kwargs.get("encoding_config") or off.args[-1]
|
||||
assert on_cfg.get("stash_local_outputs") is True
|
||||
assert "stash_local_outputs" not in off_cfg
|
||||
@@ -691,8 +691,14 @@ class VideoWorkerOrchestrator:
|
||||
local_path = os.path.join(self.config.output_dir, filename)
|
||||
|
||||
try:
|
||||
self.job_log.info(f"Downloading {filename} from GCS")
|
||||
self.storage.download_file(gcs_path, local_path)
|
||||
# On the AWS encoding box the encoder left a local copy: take it
|
||||
# instead of downloading back out of GCS (see local_encoded_outputs).
|
||||
from backend.services import local_encoded_outputs
|
||||
if local_encoded_outputs.take(gcs_path, local_path):
|
||||
self.job_log.info(f"Took {filename} from the co-located encoder (no GCS download)")
|
||||
else:
|
||||
self.job_log.info(f"Downloading {filename} from GCS")
|
||||
self.storage.download_file(gcs_path, local_path)
|
||||
|
||||
# Update the result to point to local file
|
||||
setattr(self.result, result_attr, local_path)
|
||||
|
||||
@@ -488,6 +488,7 @@ on-demand fallback if Spot capacity is gone.
|
||||
- It is a last resort by default (`SPEED_RANK["c7a"] = 90`).
|
||||
- Firestore `config/encoding-worker.speed_rank_overrides = {"c7a": 5}` makes it preferred with no deploy.
|
||||
- Both warmups (`ensure_preferred_running`) follow the same ranking, so they warm the worker requests are routed to.
|
||||
- **Distribution on AWS (Phase 2, off by default):** with `config/encoding-worker.aws_video_worker = true` and the AWS worker ranked first, `trigger_video_worker` runs the video worker on the box (`POST /run-worker`, env mirrored from the live `video-encoding-job`, identity `karaoke-backend@` via WIF). Uploads leave from AWS, and finals are handed over locally (`local_encoded_outputs`) instead of downloaded back from GCS. Any failure falls back to the Cloud Run job.
|
||||
- **Auth and lifecycle:**
|
||||
- GCP access is Workload Identity Federation as `encoding-worker@`: no keys.
|
||||
- It stops itself when idle (`infrastructure/encoding-worker/aws/idle-watchdog.py`).
|
||||
|
||||
@@ -83,19 +83,44 @@ The box runs the **unchanged worker HTTP service** (`backend.services.gce_encodi
|
||||
- Extend the existing test files listed in the GCE-coupling map.
|
||||
- **Integration (prod, supervised):** stage B runs real jobs through the AWS worker. The prod daily E2E (payment + happy path) is the end-to-end check.
|
||||
|
||||
## Phase 2: distribution from AWS (next PR set, after Phase 1 is verified)
|
||||
## Phase 2: distribution from AWS (implemented 2026-10-31, off by default)
|
||||
|
||||
The video worker (`backend.workers.video_worker`) runs **on the box** with local encoding (`LocalEncodingBackend`):
|
||||
- Inputs come from GCS once (~300 MB).
|
||||
- Finals are encoded locally, uploaded to Dropbox/YouTube/Drive from AWS, and pushed to GCS `finals/` (AWS egress, on credits).
|
||||
- This removes the ~$18/mo of Cloud Run internet egress and the video-encoding-job CPU (~$11/mo).
|
||||
The video worker (packaging, encode, Dropbox/YouTube/Drive uploads, notifications) can run **on the AWS box** instead of in the Cloud Run job `video-encoding-job`. The large uploads then leave from AWS (credits) instead of GCP (the ~$18/mo of Cloud Run internet egress), and the job's Cloud Run CPU (~$11/mo) goes away.
|
||||
|
||||
Needs:
|
||||
- a `/run-worker` endpoint and env/secrets on the box (the video-encoding-job env list);
|
||||
- a trigger branch in `WorkerService.trigger_video_worker`, with Cloud Run as the fallback;
|
||||
- the box's WIF identity mapped to the backend SA's permissions;
|
||||
- **Spot-interruption recovery for in-flight runs.** Today nothing recovers a video worker that dies after `GENERATING_VIDEO`; build a supervised re-dispatch first;
|
||||
- the deferred YouTube queue processor runs on the API, so it still egresses from GCP (low volume).
|
||||
**Encoding path unchanged (output parity)**
|
||||
- The video worker on the box still encodes through the worker HTTP API, so the deliverables (portrait included) are identical.
|
||||
- `LocalEncodingBackend` was rejected: it lacks the worker's portrait render.
|
||||
|
||||
**Same-box handoff** (`backend/services/local_encoded_outputs.py`)
|
||||
- Without it, the video worker would download every final back out of GCS: ~400 MB/job of out-of-pocket egress, which costs more than the move saves.
|
||||
- The encoder stashes each output under `LOCAL_ENCODED_OUTPUTS_DIR`, keyed by its GCS blob path, and the video worker takes it from there.
|
||||
- The video worker asks for it per encode (`stash_local_outputs`, set only where the env var exists), so GCE and Cloud Run are unaffected. Copies not taken within 6 h are purged on every stash and at worker boot.
|
||||
|
||||
**`POST /run-worker`** (worker)
|
||||
- Spawns `python -m backend.workers.video_worker --job-id …` with the box's video-worker env.
|
||||
- Status is reported through the usual `/status/{id}`. A run counts in `/health` `active_jobs`, so the idle watchdog won't stop the box mid-run.
|
||||
- It answers 503 unless `/opt/encoding-worker/video-worker-env.json` exists.
|
||||
|
||||
**Env and identity**
|
||||
- At boot, `video-worker-env.py` reads the **live** `video-encoding-job` definition and resolves its secret refs, so nothing is hand-copied and nothing drifts.
|
||||
- It adds `LOCAL_ENCODED_OUTPUTS_DIR` and `CLOUD_RUN_JOB`.
|
||||
- The run uses the job's own identity, `karaoke-backend@`, via a second WIF binding (`aws-encoding-worker-wif-backend-user`).
|
||||
|
||||
**Trigger** (`WorkerService._trigger_video_worker_on_aws`)
|
||||
- It applies only when Firestore `config/encoding-worker.aws_video_worker` is true **and** the AWS worker ranks first.
|
||||
- It then ensures the box is running, waits for `/health`, and POSTs `/run-worker`.
|
||||
- **Any** doubt or failure returns False, and the caller starts the Cloud Run job as today.
|
||||
|
||||
**Not covered**
|
||||
- Deferred YouTube uploads (the hourly queue processor on the API) still leave from GCP; volume is low.
|
||||
- Video-worker stdout stays on the box at `/var/log/encoding-worker-runs/`. The job's Firestore worker logs are unaffected, but the error monitor doesn't see the box's stderr.
|
||||
- With the box on-demand (see the amendment above), Spot-interruption recovery is moot. A systemd restart of the worker during a run would still kill it; the existing `encoding_stuck` flag at 50 min surfaces that.
|
||||
|
||||
**Enable (after stage B, needs `/gcp-admin` for the IAM binding)**
|
||||
1. Apply the GCP WIF backend binding.
|
||||
2. Reboot the box once (`stop`/`start`) so `startup-aws.sh` writes the backend credentials and the env.
|
||||
3. Set `aws_video_worker: true`.
|
||||
4. Roll back by setting it to false.
|
||||
|
||||
## Stage A status (2026-10-29/31)
|
||||
|
||||
|
||||
@@ -168,7 +168,9 @@ encoding_worker_iam_bindings = worker_sas.grant_encoding_worker_permissions(
|
||||
# Federation (no keys). See modules/aws_encoding_wif.py.
|
||||
from modules import aws_encoding_wif # noqa: E402
|
||||
|
||||
aws_encoding_wif_resources = aws_encoding_wif.create_aws_encoding_wif(encoding_worker_sa)
|
||||
aws_encoding_wif_resources = aws_encoding_wif.create_aws_encoding_wif(
|
||||
encoding_worker_sa, backend_service_account
|
||||
)
|
||||
|
||||
gdrive_validator_sa = worker_sas.create_gdrive_validator_service_account()
|
||||
gdrive_validator_iam_bindings = worker_sas.grant_gdrive_validator_permissions(
|
||||
|
||||
@@ -35,6 +35,8 @@ AWS_ACCESS_KEY_ID=$NOMAD_AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$NOMAD_AWS_SECR
|
||||
2. **Register the worker in the pool:** append `pulumi stack output fallback_vms_entry` to the JSON list in secret `encoding-worker-fallback-vms`. The backend picks up `:latest` on new instances, and the next deploy picks it up everywhere.
|
||||
3. **Rank:** by default the AWS worker is a **last resort** (`SPEED_RANK["c7a"] = 90`). To prefer it, set Firestore `config/encoding-worker.speed_rank_overrides = {"c7a": 5}`. Delete the field to roll back; no deploy is needed either way.
|
||||
|
||||
4. **Distribution on AWS (Phase 2):** once the AWS worker is preferred (step 3), set Firestore `config/encoding-worker.aws_video_worker = true`. The video worker (Dropbox/YouTube/Drive uploads) then runs on the box via `/run-worker`, with the Cloud Run job as an automatic fallback. Set it to `false` to roll back. Run logs are in `/var/log/encoding-worker-runs/`.
|
||||
|
||||
## Operating it
|
||||
|
||||
- **Shell:** `aws ssm start-session --target <instance_id> --region us-east-2`.
|
||||
|
||||
@@ -38,11 +38,28 @@ ENCODING_API_KEY=${ENCODING_API_KEY}
|
||||
ENCODING_WORKER_VM_NAME=${VM_NAME}
|
||||
ENCODING_WORKER_PROVIDER=aws
|
||||
GOOGLE_APPLICATION_CREDENTIALS=${WORKER_DIR}/gcp-wif.json
|
||||
LOCAL_ENCODED_OUTPUTS_DIR=/var/lib/encoding-worker/outputs
|
||||
ENV
|
||||
chmod 600 "${WORKER_DIR}/env"
|
||||
mkdir -p /var/lib/encoding-worker/outputs
|
||||
|
||||
# Keep the idle watchdog current with the deployed code (best effort).
|
||||
# Keep the idle watchdog + video-worker env helper current (best effort).
|
||||
gcloud storage cp "${BUCKET}/encoding-worker/idle-watchdog.py" "${WORKER_DIR}/idle-watchdog.py" || true
|
||||
gcloud storage cp "${BUCKET}/encoding-worker/video-worker-env.py" "${WORKER_DIR}/video-worker-env.py" || true
|
||||
|
||||
# Video worker on this box (/run-worker) runs as karaoke-backend@, the Cloud Run
|
||||
# job's identity, via the same WIF provider. Its env mirrors the live job.
|
||||
if [ ! -s "${WORKER_DIR}/gcp-wif-backend.json" ]; then
|
||||
WIF_PROVIDER=$(python3 -c "import json;print(json.load(open('${WORKER_DIR}/gcp-wif.json'))['audience'].split('//iam.googleapis.com/')[1])" 2>/dev/null) || WIF_PROVIDER=""
|
||||
if [ -n "${WIF_PROVIDER}" ]; then
|
||||
gcloud iam workload-identity-pools create-cred-config "${WIF_PROVIDER}" \
|
||||
--service-account=karaoke-backend@nomadkaraoke.iam.gserviceaccount.com --aws --enable-imdsv2 \
|
||||
--output-file="${WORKER_DIR}/gcp-wif-backend.json" || true
|
||||
fi
|
||||
fi
|
||||
if [ -f "${WORKER_DIR}/video-worker-env.py" ]; then
|
||||
"${VENV}/bin/python" "${WORKER_DIR}/video-worker-env.py" || true
|
||||
fi
|
||||
|
||||
echo "[2/5] Downloading version manifest..."
|
||||
EXPECTED_VERSION=$(gcloud storage cat "${BUCKET}/encoding-worker/version.txt" 2>/dev/null | tr -d '[:space:]' || echo "")
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Write the env for running the video worker on the AWS encoding box.
|
||||
|
||||
Mirrors the LIVE Cloud Run job ``video-encoding-job`` (plain env + secret refs
|
||||
resolved from Secret Manager), so the box runs the video worker with exactly
|
||||
the configuration Cloud Run does — no hand-copied list to drift. Authenticates
|
||||
as karaoke-backend@ (the job's service account) via Workload Identity
|
||||
Federation, and the env it writes points the video worker at the same identity.
|
||||
|
||||
On any failure it removes the env file: /run-worker then answers 503 and the
|
||||
backend keeps using the Cloud Run job. Run by startup-aws.sh on every boot.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT = "nomadkaraoke"
|
||||
JOB = "video-encoding-job"
|
||||
REGION = os.environ.get("CPU_JOBS_REGION", "us-east4")
|
||||
OUT = Path(os.environ.get("RUN_WORKER_ENV_FILE", "/opt/encoding-worker/video-worker-env.json"))
|
||||
BACKEND_CREDS = "/opt/encoding-worker/gcp-wif-backend.json"
|
||||
|
||||
# Box-specific settings on top of the Cloud Run job's env.
|
||||
BOX_ENV = {
|
||||
"GOOGLE_APPLICATION_CREDENTIALS": BACKEND_CREDS,
|
||||
# Take finals from the co-located encoder instead of GCS (local_encoded_outputs).
|
||||
"LOCAL_ENCODED_OUTPUTS_DIR": "/var/lib/encoding-worker/outputs",
|
||||
# Run-to-completion process like a Cloud Run job (synchronous log writes etc.).
|
||||
"CLOUD_RUN_JOB": JOB,
|
||||
}
|
||||
|
||||
|
||||
def build_env(job: dict, secret_value) -> dict:
|
||||
"""Plain env + resolved secrets from a run_v2 Job dict, plus BOX_ENV."""
|
||||
containers = job["template"]["template"]["containers"]
|
||||
env: dict = {}
|
||||
for item in containers[0].get("env", []):
|
||||
name = item["name"]
|
||||
if "value" in item:
|
||||
env[name] = item["value"]
|
||||
elif "valueSource" in item:
|
||||
ref = item["valueSource"]["secretKeyRef"]
|
||||
secret = ref["secret"].rsplit("/", 1)[-1]
|
||||
env[name] = secret_value(secret, ref.get("version") or "latest")
|
||||
env.update(BOX_ENV)
|
||||
return env
|
||||
|
||||
|
||||
def main() -> int:
|
||||
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = BACKEND_CREDS
|
||||
try:
|
||||
import google.auth
|
||||
import google.auth.transport.requests
|
||||
from google.cloud import secretmanager
|
||||
|
||||
creds, _ = google.auth.default(scopes=["https://www.googleapis.com/auth/cloud-platform"])
|
||||
session = google.auth.transport.requests.AuthorizedSession(creds)
|
||||
resp = session.get(f"https://run.googleapis.com/v2/projects/{PROJECT}/locations/{REGION}/jobs/{JOB}", timeout=30)
|
||||
resp.raise_for_status()
|
||||
sm = secretmanager.SecretManagerServiceClient(credentials=creds)
|
||||
|
||||
def secret_value(secret: str, version: str) -> str:
|
||||
name = f"projects/{PROJECT}/secrets/{secret}/versions/{version}"
|
||||
return sm.access_secret_version(name=name).payload.data.decode("utf-8")
|
||||
|
||||
env = build_env(resp.json(), secret_value)
|
||||
OUT.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = OUT.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(env))
|
||||
os.chmod(tmp, 0o600)
|
||||
os.replace(tmp, OUT)
|
||||
print(f"video-worker env: {len(env)} vars from {JOB} ({REGION}) -> {OUT}")
|
||||
return 0
|
||||
except Exception as e: # noqa: BLE001
|
||||
OUT.unlink(missing_ok=True)
|
||||
print(f"video-worker env NOT written (worker runs disabled, Cloud Run fallback): {e}", file=sys.stderr)
|
||||
return 0 # never block the encoder's startup
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -33,3 +33,4 @@ backend/config.py
|
||||
infrastructure/encoding-worker/
|
||||
infrastructure/packer/
|
||||
poetry.lock
|
||||
backend/services/local_encoded_outputs.py
|
||||
@@ -18,7 +18,10 @@ POOL_ID = "aws-encoding"
|
||||
PROVIDER_ID = f"aws-{AWS_ACCOUNT_ID}"
|
||||
|
||||
|
||||
def create_aws_encoding_wif(encoding_worker_sa: gcp.serviceaccount.Account) -> dict:
|
||||
def create_aws_encoding_wif(
|
||||
encoding_worker_sa: gcp.serviceaccount.Account,
|
||||
backend_sa: gcp.serviceaccount.Account,
|
||||
) -> dict:
|
||||
pool = gcp.iam.WorkloadIdentityPool(
|
||||
"aws-encoding-wif-pool",
|
||||
workload_identity_pool_id=POOL_ID,
|
||||
@@ -55,5 +58,19 @@ def create_aws_encoding_wif(encoding_worker_sa: gcp.serviceaccount.Account) -> d
|
||||
)
|
||||
),
|
||||
)
|
||||
# The video worker can also run on the box (/run-worker): it uses the Cloud
|
||||
# Run video-encoding-job's identity, karaoke-backend@, for exact permission
|
||||
# parity (see infrastructure/encoding-worker/aws/video-worker-env.py).
|
||||
backend_binding = gcp.serviceaccount.IAMMember(
|
||||
"aws-encoding-worker-wif-backend-user",
|
||||
service_account_id=backend_sa.name,
|
||||
role="roles/iam.workloadIdentityUser",
|
||||
member=pool.name.apply(
|
||||
lambda name: (
|
||||
f"principalSet://iam.googleapis.com/{name}/attribute.aws_role/"
|
||||
f"arn:aws:sts::{AWS_ACCOUNT_ID}:assumed-role/{AWS_WORKER_ROLE}"
|
||||
)
|
||||
),
|
||||
)
|
||||
pulumi.export("aws_encoding_wif_provider", provider.name)
|
||||
return {"pool": pool, "provider": provider, "binding": binding}
|
||||
return {"pool": pool, "provider": provider, "binding": binding, "backend_binding": backend_binding}
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.288.1"
|
||||
version = "0.289.0"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
Reference in new issue
Block a user