mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
fix: review findings — safe run start, job_id validation, stash only on request + off-loop, guard WIF derivation; gate local_encoded_outputs for worker deploys
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012RYLVSR5qgwsxTQtvGimKF
This commit is contained in:
1 parent
ddc9351895
commit
7b2886cc26
7 files changed
+91
-13
No files matched your search
@@ -467,6 +467,12 @@ class GCEEncodingBackend(EncodingBackend):
|
||||
|
||||
# Screens the theme leaves out: the encoder must skip them rather than
|
||||
# rebuild them from whatever matching PNG it finds in the job folder.
|
||||
# On the AWS box the video worker sits next to the encoder: ask it
|
||||
# to keep local copies so we take them instead of downloading back
|
||||
# from GCS (local_encoded_outputs). Unset everywhere else.
|
||||
if os.environ.get("LOCAL_ENCODED_OUTPUTS_DIR"):
|
||||
encoding_config["stash_local_outputs"] = True
|
||||
|
||||
for flag in ("include_title_screen", "include_end_screen"):
|
||||
if input_config.options.get(flag) is False:
|
||||
encoding_config[flag] = False
|
||||
|
||||
@@ -27,7 +27,7 @@ os.environ.setdefault("GCE_METADATA_MTLS_MODE", "none")
|
||||
from fastapi import FastAPI, BackgroundTasks, HTTPException, Header, Depends
|
||||
from google.cloud import storage
|
||||
from packaging.version import Version
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from . import self_heal
|
||||
from .persistence import JobStatePersister
|
||||
@@ -1349,10 +1349,18 @@ async def process_job(job_id: str, request: EncodeRequest):
|
||||
|
||||
# On the AWS box the video worker runs here too: keep local copies
|
||||
# (keyed by blob path) so it needn't download them back out of GCS.
|
||||
# No-op unless LOCAL_ENCODED_OUTPUTS_DIR is set.
|
||||
from backend.services import local_encoded_outputs
|
||||
for local_path, blob_path in zip(local_output_files, blob_paths):
|
||||
local_encoded_outputs.stash(Path(local_path), blob_path)
|
||||
# Only when that co-located video worker asked (stash_local_outputs),
|
||||
# and only where LOCAL_ENCODED_OUTPUTS_DIR is set. Off the event loop:
|
||||
# it copies ~400 MB.
|
||||
if (request.encoding_config or {}).get("stash_local_outputs"):
|
||||
from backend.services import local_encoded_outputs
|
||||
|
||||
def _stash_all():
|
||||
local_encoded_outputs.purge_stale()
|
||||
for local_path, blob_path in zip(local_output_files, blob_paths):
|
||||
local_encoded_outputs.stash(Path(local_path), blob_path)
|
||||
|
||||
await asyncio.to_thread(_stash_all)
|
||||
|
||||
jobs[job_id]["status"] = "complete"
|
||||
jobs[job_id]["progress"] = 100
|
||||
@@ -1521,7 +1529,8 @@ _RUN_WORKER_MODULES = frozenset({"video_worker"})
|
||||
|
||||
|
||||
class RunWorkerRequest(BaseModel):
|
||||
job_id: str
|
||||
# Used in a log file name and as a CLI argument: gen job ids only.
|
||||
job_id: str = Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$")
|
||||
module: str = "video_worker"
|
||||
|
||||
|
||||
@@ -1584,6 +1593,17 @@ def _start_worker_run(run_id: str, request: RunWorkerRequest, env: dict) -> None
|
||||
threading.Thread(target=_watch_worker_run, args=(run_id, proc, log_path), daemon=True).start()
|
||||
|
||||
|
||||
def _start_worker_run_safe(run_id: str, request: RunWorkerRequest, env: dict) -> None:
|
||||
"""Never leave a run stuck "running": that would hold the box up (idle
|
||||
watchdog) and lose the job (the backend got "accepted", so no fallback)."""
|
||||
try:
|
||||
_start_worker_run(run_id, request, env)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.exception("Failed to start worker run %s", run_id)
|
||||
jobs[run_id].update(status="failed", error=f"failed to start worker: {exc}")
|
||||
_persist(run_id)
|
||||
|
||||
|
||||
@app.post("/run-worker")
|
||||
async def submit_worker_run(request: RunWorkerRequest, _auth: bool = Depends(verify_api_key)):
|
||||
if request.module not in _RUN_WORKER_MODULES:
|
||||
@@ -1603,7 +1623,7 @@ async def submit_worker_run(request: RunWorkerRequest, _auth: bool = Depends(ver
|
||||
}
|
||||
_persist(run_id)
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.run_in_executor(light_executor, _start_worker_run, run_id, request, env)
|
||||
loop.run_in_executor(light_executor, _start_worker_run_safe, run_id, request, env)
|
||||
return {"status": "accepted", "job_id": run_id}
|
||||
|
||||
|
||||
|
||||
@@ -24,7 +24,7 @@ from typing import Optional
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ENV_VAR = "LOCAL_ENCODED_OUTPUTS_DIR"
|
||||
MAX_AGE_SECONDS = 24 * 3600
|
||||
MAX_AGE_SECONDS = 6 * 3600 # a take happens minutes after the stash
|
||||
|
||||
|
||||
def _root() -> Optional[Path]:
|
||||
|
||||
@@ -253,3 +253,52 @@ def test_aws_dispatch_falls_back_on_refusal(service, status, body):
|
||||
def test_aws_dispatch_falls_back_on_error(service):
|
||||
ok, _ = _run(service, _manager(), post_exc=ConnectionError("boom"))
|
||||
assert ok is False
|
||||
|
||||
|
||||
def test_start_failure_never_leaves_run_stuck(worker):
|
||||
main, env_file = worker
|
||||
run_id = main.run_worker_id("video_worker", "abc123")
|
||||
main.jobs[run_id] = {"status": "running"}
|
||||
with patch.object(main, "_start_worker_run", side_effect=OSError("disk full")):
|
||||
main._start_worker_run_safe(run_id, _req(main), {})
|
||||
assert main.jobs[run_id]["status"] == "failed"
|
||||
assert "disk full" in main.jobs[run_id]["error"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bad", ["../../tmp/x", "-h", "", "a b", "x" * 80])
|
||||
def test_run_worker_rejects_unsafe_job_ids(bad):
|
||||
from pydantic import ValidationError
|
||||
from backend.services.gce_encoding import main
|
||||
with pytest.raises(ValidationError):
|
||||
main.RunWorkerRequest(job_id=bad)
|
||||
|
||||
|
||||
def test_gce_backend_requests_stash_only_on_the_box(monkeypatch):
|
||||
"""The encode payload asks for local copies only where the video worker
|
||||
shares the box with the encoder (LOCAL_ENCODED_OUTPUTS_DIR set)."""
|
||||
from backend.services.encoding_interface import EncodingInput, GCEEncodingBackend
|
||||
|
||||
async def capture(env_value):
|
||||
if env_value:
|
||||
monkeypatch.setenv("LOCAL_ENCODED_OUTPUTS_DIR", env_value)
|
||||
else:
|
||||
monkeypatch.delenv("LOCAL_ENCODED_OUTPUTS_DIR", raising=False)
|
||||
svc = MagicMock()
|
||||
svc.encode_videos = AsyncMock(return_value={"status": "complete", "output_files": []})
|
||||
backend = GCEEncodingBackend()
|
||||
backend._service = svc
|
||||
with patch.object(GCEEncodingBackend, "_get_service", return_value=svc):
|
||||
await backend.encode(EncodingInput(
|
||||
title_video_path="t", karaoke_video_path="k", instrumental_audio_path="i",
|
||||
artist="A", title="T", output_dir="/tmp",
|
||||
options={"job_id": "abc123_g1", "input_gcs_path": "gs://b/jobs/abc123/",
|
||||
"output_gcs_path": "gs://b/jobs/abc123/finals/"},
|
||||
))
|
||||
return svc.encode_videos.call_args
|
||||
|
||||
on = asyncio.run(capture("/var/lib/encoding-worker/outputs"))
|
||||
off = asyncio.run(capture(None))
|
||||
on_cfg = on.kwargs.get("encoding_config") or on.args[-1]
|
||||
off_cfg = off.kwargs.get("encoding_config") or off.args[-1]
|
||||
assert on_cfg.get("stash_local_outputs") is True
|
||||
assert "stash_local_outputs" not in off_cfg
|
||||
@@ -94,7 +94,7 @@ The video worker (packaging, encode, Dropbox/YouTube/Drive uploads, notification
|
||||
**Same-box handoff** (`backend/services/local_encoded_outputs.py`)
|
||||
- Without it, the video worker would download every final back out of GCS: ~400 MB/job of out-of-pocket egress, which costs more than the move saves.
|
||||
- The encoder stashes each output under `LOCAL_ENCODED_OUTPUTS_DIR`, keyed by its GCS blob path, and the video worker takes it from there.
|
||||
- It is a no-op wherever the env var is unset (GCE, Cloud Run). Stale stashes are purged at worker boot.
|
||||
- The video worker asks for it per encode (`stash_local_outputs`, set only where the env var exists), so GCE and Cloud Run are unaffected. Copies not taken within 6 h are purged on every stash and at worker boot.
|
||||
|
||||
**`POST /run-worker`** (worker)
|
||||
- Spawns `python -m backend.workers.video_worker --job-id …` with the box's video-worker env.
|
||||
|
||||
@@ -50,10 +50,12 @@ gcloud storage cp "${BUCKET}/encoding-worker/video-worker-env.py" "${WORKER_DIR}
|
||||
# Video worker on this box (/run-worker) runs as karaoke-backend@, the Cloud Run
|
||||
# job's identity, via the same WIF provider. Its env mirrors the live job.
|
||||
if [ ! -s "${WORKER_DIR}/gcp-wif-backend.json" ]; then
|
||||
WIF_PROVIDER=$(python3 -c "import json;print(json.load(open('${WORKER_DIR}/gcp-wif.json'))['audience'].split('//iam.googleapis.com/')[1])")
|
||||
gcloud iam workload-identity-pools create-cred-config "${WIF_PROVIDER}" \
|
||||
--service-account=karaoke-backend@nomadkaraoke.iam.gserviceaccount.com --aws --enable-imdsv2 \
|
||||
--output-file="${WORKER_DIR}/gcp-wif-backend.json" || true
|
||||
WIF_PROVIDER=$(python3 -c "import json;print(json.load(open('${WORKER_DIR}/gcp-wif.json'))['audience'].split('//iam.googleapis.com/')[1])" 2>/dev/null) || WIF_PROVIDER=""
|
||||
if [ -n "${WIF_PROVIDER}" ]; then
|
||||
gcloud iam workload-identity-pools create-cred-config "${WIF_PROVIDER}" \
|
||||
--service-account=karaoke-backend@nomadkaraoke.iam.gserviceaccount.com --aws --enable-imdsv2 \
|
||||
--output-file="${WORKER_DIR}/gcp-wif-backend.json" || true
|
||||
fi
|
||||
fi
|
||||
if [ -f "${WORKER_DIR}/video-worker-env.py" ]; then
|
||||
"${VENV}/bin/python" "${WORKER_DIR}/video-worker-env.py" || true
|
||||
|
||||
@@ -33,3 +33,4 @@ backend/config.py
|
||||
infrastructure/encoding-worker/
|
||||
infrastructure/packer/
|
||||
poetry.lock
|
||||
backend/services/local_encoded_outputs.py
|
||||
Reference in new issue
Block a user