fix: review findings — safe run start, job_id validation, stash only on request + off-loop, guard WIF derivation; gate local_encoded_outputs for worker deploys

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012RYLVSR5qgwsxTQtvGimKF
This commit is contained in:
Andrew BeveridgeandClaude Opus 5.5 committed 2026-10-31 13:31:44 -04:00
1 parent ddc9351895
commit 7b2886cc26
7 files changed
+91 -13

No files matched your search

+6
View File
@@ -467,6 +467,12 @@ class GCEEncodingBackend(EncodingBackend):
# Screens the theme leaves out: the encoder must skip them rather than
# rebuild them from whatever matching PNG it finds in the job folder.
# On the AWS box the video worker sits next to the encoder: ask it
# to keep local copies so we take them instead of downloading back
# from GCS (local_encoded_outputs). Unset everywhere else.
if os.environ.get("LOCAL_ENCODED_OUTPUTS_DIR"):
encoding_config["stash_local_outputs"] = True
for flag in ("include_title_screen", "include_end_screen"):
if input_config.options.get(flag) is False:
encoding_config[flag] = False
+27 -7
View File
@@ -27,7 +27,7 @@ os.environ.setdefault("GCE_METADATA_MTLS_MODE", "none")
from fastapi import FastAPI, BackgroundTasks, HTTPException, Header, Depends
from google.cloud import storage
from packaging.version import Version
from pydantic import BaseModel
from pydantic import BaseModel, Field
from . import self_heal
from .persistence import JobStatePersister
@@ -1349,10 +1349,18 @@ async def process_job(job_id: str, request: EncodeRequest):
# On the AWS box the video worker runs here too: keep local copies
# (keyed by blob path) so it needn't download them back out of GCS.
# No-op unless LOCAL_ENCODED_OUTPUTS_DIR is set.
from backend.services import local_encoded_outputs
for local_path, blob_path in zip(local_output_files, blob_paths):
local_encoded_outputs.stash(Path(local_path), blob_path)
# Only when that co-located video worker asked (stash_local_outputs),
# and only where LOCAL_ENCODED_OUTPUTS_DIR is set. Off the event loop:
# it copies ~400 MB.
if (request.encoding_config or {}).get("stash_local_outputs"):
from backend.services import local_encoded_outputs
def _stash_all():
local_encoded_outputs.purge_stale()
for local_path, blob_path in zip(local_output_files, blob_paths):
local_encoded_outputs.stash(Path(local_path), blob_path)
await asyncio.to_thread(_stash_all)
jobs[job_id]["status"] = "complete"
jobs[job_id]["progress"] = 100
@@ -1521,7 +1529,8 @@ _RUN_WORKER_MODULES = frozenset({"video_worker"})
class RunWorkerRequest(BaseModel):
job_id: str
# Used in a log file name and as a CLI argument: gen job ids only.
job_id: str = Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$")
module: str = "video_worker"
@@ -1584,6 +1593,17 @@ def _start_worker_run(run_id: str, request: RunWorkerRequest, env: dict) -> None
threading.Thread(target=_watch_worker_run, args=(run_id, proc, log_path), daemon=True).start()
def _start_worker_run_safe(run_id: str, request: RunWorkerRequest, env: dict) -> None:
"""Never leave a run stuck "running": that would hold the box up (idle
watchdog) and lose the job (the backend got "accepted", so no fallback)."""
try:
_start_worker_run(run_id, request, env)
except Exception as exc: # noqa: BLE001
logger.exception("Failed to start worker run %s", run_id)
jobs[run_id].update(status="failed", error=f"failed to start worker: {exc}")
_persist(run_id)
@app.post("/run-worker")
async def submit_worker_run(request: RunWorkerRequest, _auth: bool = Depends(verify_api_key)):
if request.module not in _RUN_WORKER_MODULES:
@@ -1603,7 +1623,7 @@ async def submit_worker_run(request: RunWorkerRequest, _auth: bool = Depends(ver
}
_persist(run_id)
loop = asyncio.get_event_loop()
loop.run_in_executor(light_executor, _start_worker_run, run_id, request, env)
loop.run_in_executor(light_executor, _start_worker_run_safe, run_id, request, env)
return {"status": "accepted", "job_id": run_id}
+1 -1
View File
@@ -24,7 +24,7 @@ from typing import Optional
logger = logging.getLogger(__name__)
ENV_VAR = "LOCAL_ENCODED_OUTPUTS_DIR"
MAX_AGE_SECONDS = 24 * 3600
MAX_AGE_SECONDS = 6 * 3600 # a take happens minutes after the stash
def _root() -> Optional[Path]:
+49
View File
@@ -253,3 +253,52 @@ def test_aws_dispatch_falls_back_on_refusal(service, status, body):
def test_aws_dispatch_falls_back_on_error(service):
ok, _ = _run(service, _manager(), post_exc=ConnectionError("boom"))
assert ok is False
def test_start_failure_never_leaves_run_stuck(worker):
main, env_file = worker
run_id = main.run_worker_id("video_worker", "abc123")
main.jobs[run_id] = {"status": "running"}
with patch.object(main, "_start_worker_run", side_effect=OSError("disk full")):
main._start_worker_run_safe(run_id, _req(main), {})
assert main.jobs[run_id]["status"] == "failed"
assert "disk full" in main.jobs[run_id]["error"]
@pytest.mark.parametrize("bad", ["../../tmp/x", "-h", "", "a b", "x" * 80])
def test_run_worker_rejects_unsafe_job_ids(bad):
from pydantic import ValidationError
from backend.services.gce_encoding import main
with pytest.raises(ValidationError):
main.RunWorkerRequest(job_id=bad)
def test_gce_backend_requests_stash_only_on_the_box(monkeypatch):
"""The encode payload asks for local copies only where the video worker
shares the box with the encoder (LOCAL_ENCODED_OUTPUTS_DIR set)."""
from backend.services.encoding_interface import EncodingInput, GCEEncodingBackend
async def capture(env_value):
if env_value:
monkeypatch.setenv("LOCAL_ENCODED_OUTPUTS_DIR", env_value)
else:
monkeypatch.delenv("LOCAL_ENCODED_OUTPUTS_DIR", raising=False)
svc = MagicMock()
svc.encode_videos = AsyncMock(return_value={"status": "complete", "output_files": []})
backend = GCEEncodingBackend()
backend._service = svc
with patch.object(GCEEncodingBackend, "_get_service", return_value=svc):
await backend.encode(EncodingInput(
title_video_path="t", karaoke_video_path="k", instrumental_audio_path="i",
artist="A", title="T", output_dir="/tmp",
options={"job_id": "abc123_g1", "input_gcs_path": "gs://b/jobs/abc123/",
"output_gcs_path": "gs://b/jobs/abc123/finals/"},
))
return svc.encode_videos.call_args
on = asyncio.run(capture("/var/lib/encoding-worker/outputs"))
off = asyncio.run(capture(None))
on_cfg = on.kwargs.get("encoding_config") or on.args[-1]
off_cfg = off.kwargs.get("encoding_config") or off.args[-1]
assert on_cfg.get("stash_local_outputs") is True
assert "stash_local_outputs" not in off_cfg
@@ -94,7 +94,7 @@ The video worker (packaging, encode, Dropbox/YouTube/Drive uploads, notification
**Same-box handoff** (`backend/services/local_encoded_outputs.py`)
- Without it, the video worker would download every final back out of GCS: ~400 MB/job of out-of-pocket egress, which costs more than the move saves.
- The encoder stashes each output under `LOCAL_ENCODED_OUTPUTS_DIR`, keyed by its GCS blob path, and the video worker takes it from there.
- It is a no-op wherever the env var is unset (GCE, Cloud Run). Stale stashes are purged at worker boot.
- The video worker asks for it per encode (`stash_local_outputs`, set only where the env var exists), so GCE and Cloud Run are unaffected. Copies not taken within 6 h are purged on every stash and at worker boot.
**`POST /run-worker`** (worker)
- Spawns `python -m backend.workers.video_worker --job-id …` with the box's video-worker env.
@@ -50,10 +50,12 @@ gcloud storage cp "${BUCKET}/encoding-worker/video-worker-env.py" "${WORKER_DIR}
# Video worker on this box (/run-worker) runs as karaoke-backend@, the Cloud Run
# job's identity, via the same WIF provider. Its env mirrors the live job.
if [ ! -s "${WORKER_DIR}/gcp-wif-backend.json" ]; then
WIF_PROVIDER=$(python3 -c "import json;print(json.load(open('${WORKER_DIR}/gcp-wif.json'))['audience'].split('//iam.googleapis.com/')[1])")
gcloud iam workload-identity-pools create-cred-config "${WIF_PROVIDER}" \
--service-account=karaoke-backend@nomadkaraoke.iam.gserviceaccount.com --aws --enable-imdsv2 \
--output-file="${WORKER_DIR}/gcp-wif-backend.json" || true
WIF_PROVIDER=$(python3 -c "import json;print(json.load(open('${WORKER_DIR}/gcp-wif.json'))['audience'].split('//iam.googleapis.com/')[1])" 2>/dev/null) || WIF_PROVIDER=""
if [ -n "${WIF_PROVIDER}" ]; then
gcloud iam workload-identity-pools create-cred-config "${WIF_PROVIDER}" \
--service-account=karaoke-backend@nomadkaraoke.iam.gserviceaccount.com --aws --enable-imdsv2 \
--output-file="${WORKER_DIR}/gcp-wif-backend.json" || true
fi
fi
if [ -f "${WORKER_DIR}/video-worker-env.py" ]; then
"${VENV}/bin/python" "${WORKER_DIR}/video-worker-env.py" || true
@@ -33,3 +33,4 @@ backend/config.py
infrastructure/encoding-worker/
infrastructure/packer/
poetry.lock
backend/services/local_encoded_outputs.py