diff --git a/backend/services/encoding_interface.py b/backend/services/encoding_interface.py index 965f5d3e..afb7cec0 100644 --- a/backend/services/encoding_interface.py +++ b/backend/services/encoding_interface.py @@ -467,6 +467,12 @@ class GCEEncodingBackend(EncodingBackend): # Screens the theme leaves out: the encoder must skip them rather than # rebuild them from whatever matching PNG it finds in the job folder. + # On the AWS box the video worker sits next to the encoder: ask it + # to keep local copies so we take them instead of downloading back + # from GCS (local_encoded_outputs). Unset everywhere else. + if os.environ.get("LOCAL_ENCODED_OUTPUTS_DIR"): + encoding_config["stash_local_outputs"] = True + for flag in ("include_title_screen", "include_end_screen"): if input_config.options.get(flag) is False: encoding_config[flag] = False diff --git a/backend/services/gce_encoding/main.py b/backend/services/gce_encoding/main.py index f24c063b..c87e7466 100644 --- a/backend/services/gce_encoding/main.py +++ b/backend/services/gce_encoding/main.py @@ -27,7 +27,7 @@ os.environ.setdefault("GCE_METADATA_MTLS_MODE", "none") from fastapi import FastAPI, BackgroundTasks, HTTPException, Header, Depends from google.cloud import storage from packaging.version import Version -from pydantic import BaseModel +from pydantic import BaseModel, Field from . import self_heal from .persistence import JobStatePersister @@ -1349,10 +1349,18 @@ async def process_job(job_id: str, request: EncodeRequest): # On the AWS box the video worker runs here too: keep local copies # (keyed by blob path) so it needn't download them back out of GCS. - # No-op unless LOCAL_ENCODED_OUTPUTS_DIR is set. - from backend.services import local_encoded_outputs - for local_path, blob_path in zip(local_output_files, blob_paths): - local_encoded_outputs.stash(Path(local_path), blob_path) + # Only when that co-located video worker asked (stash_local_outputs), + # and only where LOCAL_ENCODED_OUTPUTS_DIR is set. Off the event loop: + # it copies ~400 MB. + if (request.encoding_config or {}).get("stash_local_outputs"): + from backend.services import local_encoded_outputs + + def _stash_all(): + local_encoded_outputs.purge_stale() + for local_path, blob_path in zip(local_output_files, blob_paths): + local_encoded_outputs.stash(Path(local_path), blob_path) + + await asyncio.to_thread(_stash_all) jobs[job_id]["status"] = "complete" jobs[job_id]["progress"] = 100 @@ -1521,7 +1529,8 @@ _RUN_WORKER_MODULES = frozenset({"video_worker"}) class RunWorkerRequest(BaseModel): - job_id: str + # Used in a log file name and as a CLI argument: gen job ids only. + job_id: str = Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$") module: str = "video_worker" @@ -1584,6 +1593,17 @@ def _start_worker_run(run_id: str, request: RunWorkerRequest, env: dict) -> None threading.Thread(target=_watch_worker_run, args=(run_id, proc, log_path), daemon=True).start() +def _start_worker_run_safe(run_id: str, request: RunWorkerRequest, env: dict) -> None: + """Never leave a run stuck "running": that would hold the box up (idle + watchdog) and lose the job (the backend got "accepted", so no fallback).""" + try: + _start_worker_run(run_id, request, env) + except Exception as exc: # noqa: BLE001 + logger.exception("Failed to start worker run %s", run_id) + jobs[run_id].update(status="failed", error=f"failed to start worker: {exc}") + _persist(run_id) + + @app.post("/run-worker") async def submit_worker_run(request: RunWorkerRequest, _auth: bool = Depends(verify_api_key)): if request.module not in _RUN_WORKER_MODULES: @@ -1603,7 +1623,7 @@ async def submit_worker_run(request: RunWorkerRequest, _auth: bool = Depends(ver } _persist(run_id) loop = asyncio.get_event_loop() - loop.run_in_executor(light_executor, _start_worker_run, run_id, request, env) + loop.run_in_executor(light_executor, _start_worker_run_safe, run_id, request, env) return {"status": "accepted", "job_id": run_id} diff --git a/backend/services/local_encoded_outputs.py b/backend/services/local_encoded_outputs.py index 11b582f7..945ff2ed 100644 --- a/backend/services/local_encoded_outputs.py +++ b/backend/services/local_encoded_outputs.py @@ -24,7 +24,7 @@ from typing import Optional logger = logging.getLogger(__name__) ENV_VAR = "LOCAL_ENCODED_OUTPUTS_DIR" -MAX_AGE_SECONDS = 24 * 3600 +MAX_AGE_SECONDS = 6 * 3600 # a take happens minutes after the stash def _root() -> Optional[Path]: diff --git a/backend/tests/test_aws_video_worker.py b/backend/tests/test_aws_video_worker.py index 4c4b25f5..4390d4b0 100644 --- a/backend/tests/test_aws_video_worker.py +++ b/backend/tests/test_aws_video_worker.py @@ -253,3 +253,52 @@ def test_aws_dispatch_falls_back_on_refusal(service, status, body): def test_aws_dispatch_falls_back_on_error(service): ok, _ = _run(service, _manager(), post_exc=ConnectionError("boom")) assert ok is False + + +def test_start_failure_never_leaves_run_stuck(worker): + main, env_file = worker + run_id = main.run_worker_id("video_worker", "abc123") + main.jobs[run_id] = {"status": "running"} + with patch.object(main, "_start_worker_run", side_effect=OSError("disk full")): + main._start_worker_run_safe(run_id, _req(main), {}) + assert main.jobs[run_id]["status"] == "failed" + assert "disk full" in main.jobs[run_id]["error"] + + +@pytest.mark.parametrize("bad", ["../../tmp/x", "-h", "", "a b", "x" * 80]) +def test_run_worker_rejects_unsafe_job_ids(bad): + from pydantic import ValidationError + from backend.services.gce_encoding import main + with pytest.raises(ValidationError): + main.RunWorkerRequest(job_id=bad) + + +def test_gce_backend_requests_stash_only_on_the_box(monkeypatch): + """The encode payload asks for local copies only where the video worker + shares the box with the encoder (LOCAL_ENCODED_OUTPUTS_DIR set).""" + from backend.services.encoding_interface import EncodingInput, GCEEncodingBackend + + async def capture(env_value): + if env_value: + monkeypatch.setenv("LOCAL_ENCODED_OUTPUTS_DIR", env_value) + else: + monkeypatch.delenv("LOCAL_ENCODED_OUTPUTS_DIR", raising=False) + svc = MagicMock() + svc.encode_videos = AsyncMock(return_value={"status": "complete", "output_files": []}) + backend = GCEEncodingBackend() + backend._service = svc + with patch.object(GCEEncodingBackend, "_get_service", return_value=svc): + await backend.encode(EncodingInput( + title_video_path="t", karaoke_video_path="k", instrumental_audio_path="i", + artist="A", title="T", output_dir="/tmp", + options={"job_id": "abc123_g1", "input_gcs_path": "gs://b/jobs/abc123/", + "output_gcs_path": "gs://b/jobs/abc123/finals/"}, + )) + return svc.encode_videos.call_args + + on = asyncio.run(capture("/var/lib/encoding-worker/outputs")) + off = asyncio.run(capture(None)) + on_cfg = on.kwargs.get("encoding_config") or on.args[-1] + off_cfg = off.kwargs.get("encoding_config") or off.args[-1] + assert on_cfg.get("stash_local_outputs") is True + assert "stash_local_outputs" not in off_cfg diff --git a/docs/archive/2026-10-29-aws-encoding-worker-plan.md b/docs/archive/2026-10-29-aws-encoding-worker-plan.md index 497d4f40..41ef84b4 100644 --- a/docs/archive/2026-10-29-aws-encoding-worker-plan.md +++ b/docs/archive/2026-10-29-aws-encoding-worker-plan.md @@ -94,7 +94,7 @@ The video worker (packaging, encode, Dropbox/YouTube/Drive uploads, notification **Same-box handoff** (`backend/services/local_encoded_outputs.py`) - Without it, the video worker would download every final back out of GCS: ~400 MB/job of out-of-pocket egress, which costs more than the move saves. - The encoder stashes each output under `LOCAL_ENCODED_OUTPUTS_DIR`, keyed by its GCS blob path, and the video worker takes it from there. -- It is a no-op wherever the env var is unset (GCE, Cloud Run). Stale stashes are purged at worker boot. +- The video worker asks for it per encode (`stash_local_outputs`, set only where the env var exists), so GCE and Cloud Run are unaffected. Copies not taken within 6 h are purged on every stash and at worker boot. **`POST /run-worker`** (worker) - Spawns `python -m backend.workers.video_worker --job-id …` with the box's video-worker env. diff --git a/infrastructure/encoding-worker/aws/startup-aws.sh b/infrastructure/encoding-worker/aws/startup-aws.sh index ea1b5722..32a4aba1 100755 --- a/infrastructure/encoding-worker/aws/startup-aws.sh +++ b/infrastructure/encoding-worker/aws/startup-aws.sh @@ -50,10 +50,12 @@ gcloud storage cp "${BUCKET}/encoding-worker/video-worker-env.py" "${WORKER_DIR} # Video worker on this box (/run-worker) runs as karaoke-backend@, the Cloud Run # job's identity, via the same WIF provider. Its env mirrors the live job. if [ ! -s "${WORKER_DIR}/gcp-wif-backend.json" ]; then - WIF_PROVIDER=$(python3 -c "import json;print(json.load(open('${WORKER_DIR}/gcp-wif.json'))['audience'].split('//iam.googleapis.com/')[1])") - gcloud iam workload-identity-pools create-cred-config "${WIF_PROVIDER}" \ - --service-account=karaoke-backend@nomadkaraoke.iam.gserviceaccount.com --aws --enable-imdsv2 \ - --output-file="${WORKER_DIR}/gcp-wif-backend.json" || true + WIF_PROVIDER=$(python3 -c "import json;print(json.load(open('${WORKER_DIR}/gcp-wif.json'))['audience'].split('//iam.googleapis.com/')[1])" 2>/dev/null) || WIF_PROVIDER="" + if [ -n "${WIF_PROVIDER}" ]; then + gcloud iam workload-identity-pools create-cred-config "${WIF_PROVIDER}" \ + --service-account=karaoke-backend@nomadkaraoke.iam.gserviceaccount.com --aws --enable-imdsv2 \ + --output-file="${WORKER_DIR}/gcp-wif-backend.json" || true + fi fi if [ -f "${WORKER_DIR}/video-worker-env.py" ]; then "${VENV}/bin/python" "${WORKER_DIR}/video-worker-env.py" || true diff --git a/infrastructure/encoding-worker/worker_code_paths.txt b/infrastructure/encoding-worker/worker_code_paths.txt index b09f1558..e2351f9e 100644 --- a/infrastructure/encoding-worker/worker_code_paths.txt +++ b/infrastructure/encoding-worker/worker_code_paths.txt @@ -33,3 +33,4 @@ backend/config.py infrastructure/encoding-worker/ infrastructure/packer/ poetry.lock +backend/services/local_encoded_outputs.py