mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
feat(encoding-worker): persist job state to Firestore (hardening 2/3) (#757)
## Summary Addresses **hardening item 2 of 3** from \`docs/archive/2026-05-06-capacity-resilience-session.md\`. (Replaces #755, which was auto-closed when its base branch was merged + deleted.) Encoding worker held job state in a process-local dict. When systemd restarted the service (deploy, crash, OOM) the dict was empty, so polls from the Cloud Run side returned 404 and either failed the user's job or triggered wasteful re-renders. Rebased onto main after #754 merged. Item 3 (#756) is stacked on this PR. ## Changes - **\`backend/services/gce_encoding/persistence.py\`** (new): \`JobStatePersister\` class. Lazy Firestore client (no creds at import time). Per-VM tagging via \`vm_name\`. 7-day TTL via \`expires_at\`. All write paths swallow Firestore errors so encoding correctness is never blocked by persistence. - **\`backend/services/gce_encoding/main.py\`**: \`_persist(job_id)\` helper called after each status transition. FastAPI lifespan startup runs \`mark_orphans_failed_on_startup()\`, flipping any in-progress jobs from before the restart to \`failed\` with \`restart_failure_code: encoding_worker_restart\`. Subsequent polls see clear failure instead of 404. - **\`infrastructure/encoding-worker/startup.sh\`**: reads the GCE instance name from the metadata server and exports \`ENCODING_WORKER_VM_NAME\` so persistence tags rows with the actual VM identifier. - **\`infrastructure/modules/iam/worker_sas.py\`**: \`roles/datastore.user\` for the encoding-worker SA. - **\`backend/services/gce_encoding/requirements.txt\`**: \`google-cloud-firestore>=2.14.0\`. - **\`pyproject.toml\`**: 0.174.4 → 0.174.5 ## ⚠ Deploy note **Run \`pulumi up\` locally before merging.** Without the IAM binding the persister logs warnings and disables — encoding still works exactly as before, but the persistence benefit only kicks in once IAM applies. ## Test plan - [x] 10 new tests in \`test_encoding_worker_persistence.py\` (passed locally) - [x] Existing encoding-worker tests still pass - [ ] Prod verification: 1. \`pulumi up\` (apply IAM) 2. Deploy via CI (uploads new wheel + startup.sh, blue-green VM swap) 3. Submit a render via the orchestrator 4. SSH to the encoding-worker VM and \`sudo systemctl restart encoding-worker\` while the job is running 5. Observe: poll from orchestrator returns \`status: failed\` with \`error\` containing \`encoding_worker_restart\` (not 404) 6. Confirm Firestore \`encoding_worker_jobs\` collection has the row tagged with the correct \`vm_name\` @coderabbitai ignore
This commit is contained in:
1 parent
9b903d95d3
commit
db42594740
8 files changed
+617
-7
No files matched your search
@@ -11,6 +11,7 @@ import sys
|
||||
import tempfile
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
@@ -19,10 +20,48 @@ from google.cloud import storage
|
||||
from packaging.version import Version
|
||||
from pydantic import BaseModel
|
||||
|
||||
from .persistence import JobStatePersister
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
app = FastAPI(title="Encoding Worker", version="1.0.0")
|
||||
|
||||
# Job tracking — process-local dict, mirrored to Firestore via `persister`
|
||||
# so polls after a worker restart return the *last known* status instead of
|
||||
# 404. Hot path stays in-memory; persistence is best-effort.
|
||||
jobs: dict[str, dict] = {}
|
||||
executor = ThreadPoolExecutor(max_workers=4) # 4 parallel encoding jobs
|
||||
|
||||
# Persister is initialized lazily — see persistence.py — so import-time
|
||||
# stays free of GCP creds. Tests can monkey-patch this attribute.
|
||||
persister = JobStatePersister()
|
||||
|
||||
|
||||
def _persist(job_id: str) -> None:
|
||||
"""Mirror a job's current state to Firestore (best-effort, never raises)."""
|
||||
state = jobs.get(job_id)
|
||||
if state is not None:
|
||||
persister.save(state)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
# Startup: read back any in-progress jobs from Firestore. They were
|
||||
# interrupted by the restart that brought us here — their work_dir is
|
||||
# gone, so mark them failed with the restart code so polls return a
|
||||
# clear failure instead of 404.
|
||||
try:
|
||||
marked = persister.mark_orphans_failed_on_startup(jobs)
|
||||
if marked:
|
||||
logger.warning(
|
||||
"Marked %d orphaned encoding job(s) failed on startup", marked,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error("Encoding worker startup orphan-recovery failed: %r", exc)
|
||||
yield
|
||||
|
||||
|
||||
app = FastAPI(title="Encoding Worker", version="1.0.0", lifespan=lifespan)
|
||||
|
||||
# API key authentication
|
||||
API_KEY = os.environ.get("ENCODING_API_KEY", "")
|
||||
@@ -37,10 +76,6 @@ async def verify_api_key(x_api_key: str = Header(None)):
|
||||
raise HTTPException(status_code=401, detail="Invalid API key")
|
||||
return True
|
||||
|
||||
# Job tracking
|
||||
jobs: dict[str, dict] = {}
|
||||
executor = ThreadPoolExecutor(max_workers=4) # 4 parallel encoding jobs
|
||||
|
||||
# GCS client
|
||||
storage_client = storage.Client()
|
||||
|
||||
@@ -213,6 +248,7 @@ def run_preview_encoding(job_id: str, work_dir: Path, request: "EncodePreviewReq
|
||||
# Run FFmpeg encoding for preview video (480x270, fast settings)
|
||||
jobs[job_id]["status"] = "running"
|
||||
jobs[job_id]["progress"] = 10
|
||||
_persist(job_id)
|
||||
|
||||
try:
|
||||
# Download input files
|
||||
@@ -309,6 +345,7 @@ def run_preview_encoding(job_id: str, work_dir: Path, request: "EncodePreviewReq
|
||||
logger.error(f"Preview encoding failed: {e}")
|
||||
jobs[job_id]["status"] = "failed"
|
||||
jobs[job_id]["error"] = str(e)
|
||||
_persist(job_id)
|
||||
raise
|
||||
|
||||
|
||||
@@ -324,6 +361,7 @@ def run_render_video(job_id: str, work_dir: Path, request: "RenderVideoRequest")
|
||||
"""
|
||||
jobs[job_id]["status"] = "running"
|
||||
jobs[job_id]["progress"] = 5
|
||||
_persist(job_id)
|
||||
|
||||
try:
|
||||
# Import from installed karaoke-gen wheel
|
||||
@@ -490,12 +528,14 @@ def run_render_video(job_id: str, work_dir: Path, request: "RenderVideoRequest")
|
||||
logger.error(error_msg)
|
||||
jobs[job_id]["status"] = "failed"
|
||||
jobs[job_id]["error"] = error_msg
|
||||
_persist(job_id)
|
||||
raise RuntimeError(error_msg) from e
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"[job:{job_id}] Render video failed: {e}")
|
||||
jobs[job_id]["status"] = "failed"
|
||||
jobs[job_id]["error"] = str(e)
|
||||
_persist(job_id)
|
||||
raise
|
||||
|
||||
|
||||
@@ -617,6 +657,7 @@ def run_encoding(job_id: str, work_dir: Path, config: dict):
|
||||
'''
|
||||
jobs[job_id]["status"] = "running"
|
||||
jobs[job_id]["progress"] = 10
|
||||
_persist(job_id)
|
||||
|
||||
try:
|
||||
# Import LocalEncodingService from installed wheel (required, no fallback)
|
||||
@@ -803,12 +844,14 @@ def run_encoding(job_id: str, work_dir: Path, config: dict):
|
||||
logger.error(error_msg)
|
||||
jobs[job_id]["status"] = "failed"
|
||||
jobs[job_id]["error"] = error_msg
|
||||
_persist(job_id)
|
||||
raise RuntimeError(error_msg) from e
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Encoding failed: {e}")
|
||||
jobs[job_id]["status"] = "failed"
|
||||
jobs[job_id]["error"] = str(e)
|
||||
_persist(job_id)
|
||||
raise
|
||||
|
||||
|
||||
@@ -871,12 +914,14 @@ async def process_job(job_id: str, request: EncodeRequest):
|
||||
|
||||
jobs[job_id]["status"] = "complete"
|
||||
jobs[job_id]["progress"] = 100
|
||||
_persist(job_id)
|
||||
logger.info(f"Job {job_id} complete")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Job {job_id} failed: {e}")
|
||||
jobs[job_id]["status"] = "failed"
|
||||
jobs[job_id]["error"] = str(e)
|
||||
_persist(job_id)
|
||||
|
||||
|
||||
async def process_preview_job(job_id: str, request: EncodePreviewRequest):
|
||||
@@ -899,12 +944,14 @@ async def process_preview_job(job_id: str, request: EncodePreviewRequest):
|
||||
|
||||
jobs[job_id]["status"] = "complete"
|
||||
jobs[job_id]["progress"] = 100
|
||||
_persist(job_id)
|
||||
logger.info(f"Preview job {job_id} complete")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Preview job {job_id} failed: {e}")
|
||||
jobs[job_id]["status"] = "failed"
|
||||
jobs[job_id]["error"] = str(e)
|
||||
_persist(job_id)
|
||||
|
||||
|
||||
async def process_render_video_job(job_id: str, request: RenderVideoRequest):
|
||||
@@ -929,12 +976,14 @@ async def process_render_video_job(job_id: str, request: RenderVideoRequest):
|
||||
|
||||
jobs[job_id]["status"] = "complete"
|
||||
jobs[job_id]["progress"] = 100
|
||||
_persist(job_id)
|
||||
logger.info(f"Render video job {job_id} complete")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Render video job {job_id} failed: {e}")
|
||||
jobs[job_id]["status"] = "failed"
|
||||
jobs[job_id]["error"] = str(e)
|
||||
_persist(job_id)
|
||||
|
||||
|
||||
@app.post("/encode-preview")
|
||||
@@ -962,6 +1011,7 @@ async def submit_preview_encode_job(request: EncodePreviewRequest, background_ta
|
||||
"error": None,
|
||||
"output_files": None,
|
||||
}
|
||||
_persist(job_id)
|
||||
|
||||
background_tasks.add_task(process_preview_job, job_id, request)
|
||||
|
||||
@@ -993,6 +1043,7 @@ async def submit_render_video_job(request: RenderVideoRequest, background_tasks:
|
||||
"output_files": None,
|
||||
"metadata": None,
|
||||
}
|
||||
_persist(job_id)
|
||||
|
||||
background_tasks.add_task(process_render_video_job, job_id, request)
|
||||
|
||||
@@ -1024,6 +1075,7 @@ async def submit_encode_job(request: EncodeRequest, background_tasks: Background
|
||||
"error": None,
|
||||
"output_files": None,
|
||||
}
|
||||
_persist(job_id)
|
||||
|
||||
background_tasks.add_task(process_job, job_id, request)
|
||||
|
||||
|
||||
@@ -0,0 +1,266 @@
|
||||
"""Firestore persistence for the encoding worker's in-memory job state.
|
||||
|
||||
Why this exists
|
||||
---------------
|
||||
The encoding worker holds job state in a process-local `dict`. When the
|
||||
systemd service restarts (deploy, crash, OOM, manual restart) the dict is
|
||||
empty, so polls from the Cloud Run side return 404 and the orchestrator
|
||||
either fails the user's job or triggers wasteful re-renders.
|
||||
|
||||
This module mirrors per-job state to Firestore so that:
|
||||
|
||||
1. Polls after a restart return the *last known* status (typically `failed`
|
||||
with `code: encoding_worker_restart`), not 404 — orchestrator can react
|
||||
intelligently instead of guessing.
|
||||
2. On startup, the worker reads back any jobs it had in flight and marks
|
||||
them failed with that restart code, so the in-memory dict and Firestore
|
||||
stay in sync after a crash.
|
||||
|
||||
Scope
|
||||
-----
|
||||
Each row is keyed by `job_id` and tagged with `vm_name`. Loads are filtered
|
||||
to the current VM — VMs don't see each other's jobs. (We never have two
|
||||
VMs simultaneously holding the same `job_id` — Cloud Run only submits to
|
||||
one VM at a time, and on restart that same VM is the only one that owned
|
||||
the doc.)
|
||||
|
||||
A Firestore field-level TTL on `expires_at` cleans completed/failed rows
|
||||
automatically after 7 days; we set it on every write.
|
||||
|
||||
Cost / latency
|
||||
--------------
|
||||
Writes are best-effort. A Firestore failure logs a warning but does NOT
|
||||
fail the encoding job — persistence is observability/recovery, not
|
||||
correctness. The hot path stays in-memory.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
import threading
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
_RESTART_FAIL_CODE = "encoding_worker_restart"
|
||||
_TERMINAL_STATUSES = frozenset({"complete", "failed"})
|
||||
_DEFAULT_TTL_DAYS = 7
|
||||
_COLLECTION = "encoding_worker_jobs"
|
||||
|
||||
|
||||
def _now_utc() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def get_vm_name() -> str:
|
||||
"""Return the VM's identifier.
|
||||
|
||||
Prefer the GCE instance name (set by infrastructure/encoding-worker/
|
||||
startup.sh as `ENCODING_WORKER_VM_NAME`), falling back to the OS
|
||||
hostname so this still works in tests/local dev.
|
||||
"""
|
||||
return (
|
||||
os.environ.get("ENCODING_WORKER_VM_NAME")
|
||||
or socket.gethostname()
|
||||
or "unknown"
|
||||
)
|
||||
|
||||
|
||||
class JobStatePersister:
|
||||
"""Mirror in-memory job state to Firestore.
|
||||
|
||||
Lazily initializes the Firestore client so import-time has zero cost
|
||||
and tests/local-dev that don't have GCP creds don't blow up.
|
||||
|
||||
All operations are thread-safe — the encoding worker uses a
|
||||
ThreadPoolExecutor and may call save() from multiple worker threads.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
vm_name: Optional[str] = None,
|
||||
collection: str = _COLLECTION,
|
||||
ttl_days: int = _DEFAULT_TTL_DAYS,
|
||||
client: Any = None, # for tests
|
||||
) -> None:
|
||||
self.vm_name = vm_name or get_vm_name()
|
||||
self.collection = collection
|
||||
self.ttl_days = ttl_days
|
||||
self._client = client
|
||||
self._client_lock = threading.Lock()
|
||||
self._enabled: Optional[bool] = None # None = not yet probed
|
||||
|
||||
def _get_client(self):
|
||||
"""Return a Firestore client or None if creds are missing."""
|
||||
if self._client is not None:
|
||||
return self._client
|
||||
with self._client_lock:
|
||||
if self._client is not None:
|
||||
return self._client
|
||||
try:
|
||||
from google.cloud import firestore # type: ignore
|
||||
self._client = firestore.Client()
|
||||
self._enabled = True
|
||||
logger.info(
|
||||
"JobStatePersister initialized (vm=%s, collection=%s)",
|
||||
self.vm_name, self.collection,
|
||||
)
|
||||
except Exception as exc:
|
||||
# Don't fail the worker if creds aren't available — just
|
||||
# disable persistence and log loudly so it's discoverable.
|
||||
self._enabled = False
|
||||
logger.warning(
|
||||
"JobStatePersister disabled (Firestore unavailable): %s",
|
||||
exc,
|
||||
)
|
||||
return self._client
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
if self._enabled is None:
|
||||
self._get_client()
|
||||
return bool(self._enabled)
|
||||
|
||||
def save(self, job_state: dict) -> None:
|
||||
"""Persist a single job's state.
|
||||
|
||||
Adds `vm_name`, `updated_at`, and `expires_at` (TTL) fields. Never
|
||||
raises — Firestore failures are logged and swallowed because
|
||||
encoding is the source of truth and persistence is best-effort.
|
||||
|
||||
On the first PermissionDenied (typically: IAM not yet applied for
|
||||
the encoding worker SA), self-disables to avoid logging a warning
|
||||
on every save call. Caller can re-enable by re-instantiating after
|
||||
IAM lands.
|
||||
"""
|
||||
client = self._get_client()
|
||||
if client is None:
|
||||
return
|
||||
job_id = job_state.get("job_id")
|
||||
if not job_id:
|
||||
logger.warning("JobStatePersister.save called without job_id; skipping")
|
||||
return
|
||||
try:
|
||||
doc = {
|
||||
**job_state,
|
||||
"vm_name": self.vm_name,
|
||||
"updated_at": _now_utc(),
|
||||
"expires_at": _now_utc() + timedelta(days=self.ttl_days),
|
||||
}
|
||||
client.collection(self.collection).document(job_id).set(doc)
|
||||
except Exception as exc:
|
||||
# If this is a permanent auth failure (IAM not applied yet),
|
||||
# log once and self-disable so we don't spam ERROR per save.
|
||||
# Other transient errors are logged at WARNING per occurrence.
|
||||
if self._looks_like_auth_error(exc):
|
||||
if self._enabled is not False: # only log on transition
|
||||
logger.warning(
|
||||
"JobStatePersister disabling: Firestore auth failed "
|
||||
"(likely IAM not yet applied to encoding-worker-sa: "
|
||||
"needs roles/datastore.user). Subsequent saves will "
|
||||
"be no-ops. Error: %r",
|
||||
exc,
|
||||
)
|
||||
self._enabled = False
|
||||
self._client = None
|
||||
return
|
||||
logger.warning(
|
||||
"[job:%s] Failed to persist job state to Firestore: %r",
|
||||
job_id, exc,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _looks_like_auth_error(exc: BaseException) -> bool:
|
||||
"""Heuristic — google.api_core.exceptions.PermissionDenied or 403/401."""
|
||||
type_name = type(exc).__name__
|
||||
if type_name in ("PermissionDenied", "Unauthenticated", "Forbidden"):
|
||||
return True
|
||||
msg = str(exc).lower()
|
||||
return any(s in msg for s in ("permission_denied", "permissiondenied",
|
||||
"403", "401", "unauthenticated",
|
||||
"missing or insufficient permissions"))
|
||||
|
||||
def load_active_jobs(self) -> dict[str, dict]:
|
||||
"""Return any non-terminal jobs persisted for this VM.
|
||||
|
||||
Used at startup to recover state from before the restart. Terminal
|
||||
jobs (complete/failed) are also persisted but not returned here —
|
||||
their docs serve only post-restart status polls.
|
||||
|
||||
Self-disables on auth errors (same pattern as save).
|
||||
"""
|
||||
client = self._get_client()
|
||||
if client is None:
|
||||
return {}
|
||||
try:
|
||||
query = (
|
||||
client.collection(self.collection)
|
||||
.where("vm_name", "==", self.vm_name)
|
||||
)
|
||||
recovered: dict[str, dict] = {}
|
||||
for doc_snapshot in query.stream():
|
||||
data = doc_snapshot.to_dict() or {}
|
||||
status = data.get("status")
|
||||
if status in _TERMINAL_STATUSES:
|
||||
continue
|
||||
# Strip the persistence-only fields before handing back
|
||||
for k in ("vm_name", "updated_at", "expires_at"):
|
||||
data.pop(k, None)
|
||||
recovered[doc_snapshot.id] = data
|
||||
return recovered
|
||||
except Exception as exc:
|
||||
if self._looks_like_auth_error(exc):
|
||||
if self._enabled is not False:
|
||||
logger.warning(
|
||||
"JobStatePersister disabling: Firestore auth failed "
|
||||
"during load_active_jobs (likely IAM not applied). "
|
||||
"Error: %r", exc,
|
||||
)
|
||||
self._enabled = False
|
||||
self._client = None
|
||||
return {}
|
||||
logger.warning(
|
||||
"JobStatePersister.load_active_jobs failed: %r", exc,
|
||||
)
|
||||
return {}
|
||||
|
||||
def mark_orphans_failed_on_startup(self, jobs: dict[str, dict]) -> int:
|
||||
"""Mark any in-progress jobs as failed with the restart code.
|
||||
|
||||
Called at worker startup. Mutates `jobs` in place AND persists each
|
||||
change so polls from the orchestrator see consistent state.
|
||||
|
||||
The work_dir for an interrupted job is gone (tmpfs / temp dir wiped
|
||||
by systemd restart), so we cannot resume — the only safe thing is
|
||||
to surface a clear failure so the caller can decide to retry.
|
||||
|
||||
Returns the number of jobs marked failed.
|
||||
"""
|
||||
active = self.load_active_jobs()
|
||||
if not active:
|
||||
return 0
|
||||
marked = 0
|
||||
for job_id, job_state in active.items():
|
||||
job_state["status"] = "failed"
|
||||
job_state["error"] = (
|
||||
f"Encoding worker restarted mid-job (code: {_RESTART_FAIL_CODE}). "
|
||||
"Worker process restart wiped in-memory state and the temp "
|
||||
"work directory; resubmit to retry."
|
||||
)
|
||||
job_state["restart_failure_code"] = _RESTART_FAIL_CODE
|
||||
jobs[job_id] = job_state
|
||||
self.save(job_state)
|
||||
marked += 1
|
||||
logger.warning(
|
||||
"[job:%s] Marked failed at startup due to encoding worker restart "
|
||||
"(was status=%s, progress=%s)",
|
||||
job_id,
|
||||
job_state.get("_pre_restart_status", "unknown"),
|
||||
job_state.get("progress"),
|
||||
)
|
||||
return marked
|
||||
@@ -8,6 +8,10 @@ uvicorn>=0.27.0
|
||||
# GCS client
|
||||
google-cloud-storage>=2.14.0
|
||||
|
||||
# Firestore client - persists job state across systemd restarts so polls
|
||||
# survive a worker restart instead of returning 404. See persistence.py.
|
||||
google-cloud-firestore>=2.14.0
|
||||
|
||||
# Async utilities (optional, for future improvements)
|
||||
aiofiles>=23.2.0
|
||||
aiohttp>=3.9.0
|
||||
|
||||
@@ -0,0 +1,267 @@
|
||||
"""Tests for encoding-worker job-state persistence to Firestore.
|
||||
|
||||
These tests cover the contract documented in
|
||||
backend/services/gce_encoding/persistence.py:
|
||||
- save() mirrors job state with vm_name + ttl fields
|
||||
- save() never raises on Firestore errors (best-effort)
|
||||
- load_active_jobs() filters by vm_name and skips terminal-status rows
|
||||
- mark_orphans_failed_on_startup() flips in-progress rows to failed and
|
||||
persists the change (so a poll after restart returns clear failure
|
||||
instead of 404)
|
||||
- When Firestore creds are unavailable, persister silently disables
|
||||
rather than crashing the worker
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from backend.services.gce_encoding.persistence import (
|
||||
JobStatePersister,
|
||||
get_vm_name,
|
||||
_RESTART_FAIL_CODE,
|
||||
)
|
||||
|
||||
|
||||
def _fake_firestore_client():
|
||||
"""Return a mock Firestore client that records writes and supports stream()."""
|
||||
client = MagicMock()
|
||||
# Make .collection().document().set() work
|
||||
docs: dict[str, dict] = {}
|
||||
|
||||
def collection(name):
|
||||
coll = MagicMock()
|
||||
|
||||
def document(doc_id):
|
||||
doc = MagicMock()
|
||||
|
||||
def set(data):
|
||||
docs[doc_id] = data
|
||||
|
||||
doc.set.side_effect = set
|
||||
return doc
|
||||
|
||||
coll.document.side_effect = document
|
||||
|
||||
def where(*args, **kwargs):
|
||||
# Capture the filter so stream() can apply it
|
||||
filter_field = args[0] if args else kwargs.get("field_path")
|
||||
filter_value = args[2] if len(args) > 2 else kwargs.get("value")
|
||||
|
||||
query = MagicMock()
|
||||
|
||||
def stream():
|
||||
for doc_id, data in docs.items():
|
||||
if filter_field == "vm_name" and data.get("vm_name") != filter_value:
|
||||
continue
|
||||
snapshot = MagicMock()
|
||||
snapshot.id = doc_id
|
||||
snapshot.to_dict.return_value = dict(data)
|
||||
yield snapshot
|
||||
|
||||
query.stream.side_effect = stream
|
||||
return query
|
||||
|
||||
coll.where.side_effect = where
|
||||
coll._docs = docs # so tests can inspect
|
||||
return coll
|
||||
|
||||
client.collection.side_effect = collection
|
||||
client._docs = docs
|
||||
return client
|
||||
|
||||
|
||||
def test_get_vm_name_uses_env_var(monkeypatch):
|
||||
"""ENCODING_WORKER_VM_NAME wins over hostname so we tag rows by GCE
|
||||
instance name, not the (similar-but-not-guaranteed-equal) hostname."""
|
||||
monkeypatch.setenv("ENCODING_WORKER_VM_NAME", "encoding-worker-fallback-a")
|
||||
assert get_vm_name() == "encoding-worker-fallback-a"
|
||||
|
||||
|
||||
def test_get_vm_name_falls_back_to_hostname(monkeypatch):
|
||||
monkeypatch.delenv("ENCODING_WORKER_VM_NAME", raising=False)
|
||||
name = get_vm_name()
|
||||
# Whatever the test machine's hostname is, just confirm we got *something*
|
||||
assert name and isinstance(name, str)
|
||||
|
||||
|
||||
def test_save_writes_doc_with_vm_name_and_ttl():
|
||||
client = _fake_firestore_client()
|
||||
persister = JobStatePersister(vm_name="encoding-worker-a", client=client)
|
||||
|
||||
persister.save({
|
||||
"job_id": "abc123",
|
||||
"status": "running",
|
||||
"progress": 42,
|
||||
})
|
||||
|
||||
assert "abc123" in client._docs
|
||||
doc = client._docs["abc123"]
|
||||
assert doc["job_id"] == "abc123"
|
||||
assert doc["status"] == "running"
|
||||
assert doc["progress"] == 42
|
||||
assert doc["vm_name"] == "encoding-worker-a"
|
||||
# TTL field set ~7 days in the future (default)
|
||||
assert isinstance(doc["expires_at"], datetime)
|
||||
assert doc["expires_at"] > datetime.now(timezone.utc) + timedelta(days=6)
|
||||
|
||||
|
||||
def test_save_swallows_firestore_errors():
|
||||
"""A Firestore failure must NOT propagate — encoding correctness comes
|
||||
first; persistence is best-effort observability/recovery."""
|
||||
client = MagicMock()
|
||||
client.collection.side_effect = RuntimeError("Firestore down")
|
||||
persister = JobStatePersister(vm_name="x", client=client)
|
||||
|
||||
# Should not raise
|
||||
persister.save({"job_id": "abc", "status": "running"})
|
||||
|
||||
|
||||
def test_save_skips_when_job_id_missing():
|
||||
client = _fake_firestore_client()
|
||||
persister = JobStatePersister(vm_name="x", client=client)
|
||||
persister.save({"status": "pending"}) # no job_id
|
||||
assert client._docs == {}
|
||||
|
||||
|
||||
def test_load_active_jobs_filters_by_vm_and_skips_terminal_statuses():
|
||||
client = _fake_firestore_client()
|
||||
persister = JobStatePersister(vm_name="encoding-worker-a", client=client)
|
||||
|
||||
# Pre-seed Firestore with mixed-VM, mixed-status rows
|
||||
persister.save({"job_id": "running-on-a", "status": "running", "progress": 50})
|
||||
persister.save({"job_id": "complete-on-a", "status": "complete", "progress": 100})
|
||||
persister.save({"job_id": "failed-on-a", "status": "failed"})
|
||||
|
||||
other_vm = JobStatePersister(vm_name="encoding-worker-b", client=client)
|
||||
other_vm.save({"job_id": "running-on-b", "status": "running"})
|
||||
|
||||
active = persister.load_active_jobs()
|
||||
# Only the in-progress job for VM A
|
||||
assert set(active.keys()) == {"running-on-a"}
|
||||
# Persistence-only fields stripped
|
||||
assert "vm_name" not in active["running-on-a"]
|
||||
assert "expires_at" not in active["running-on-a"]
|
||||
assert active["running-on-a"]["status"] == "running"
|
||||
|
||||
|
||||
def test_load_active_jobs_returns_empty_on_firestore_error():
|
||||
"""An unreadable Firestore must NOT prevent the worker from starting up
|
||||
— we just lose the recovery capability for this restart."""
|
||||
client = MagicMock()
|
||||
client.collection.side_effect = RuntimeError("Firestore unreachable")
|
||||
persister = JobStatePersister(vm_name="x", client=client)
|
||||
assert persister.load_active_jobs() == {}
|
||||
|
||||
|
||||
def test_mark_orphans_failed_on_startup_flips_in_progress_jobs():
|
||||
"""On worker restart, any job that was running/pending must be marked
|
||||
failed in the in-memory jobs dict AND persisted back to Firestore so
|
||||
a subsequent poll sees clear failure (not 404, not still-running)."""
|
||||
client = _fake_firestore_client()
|
||||
persister = JobStatePersister(vm_name="encoding-worker-a", client=client)
|
||||
|
||||
# Simulate state from before the restart
|
||||
persister.save({"job_id": "interrupted", "status": "running", "progress": 75})
|
||||
persister.save({"job_id": "pending-not-started", "status": "pending", "progress": 0})
|
||||
persister.save({"job_id": "old-completed", "status": "complete", "progress": 100})
|
||||
|
||||
jobs: dict[str, dict] = {}
|
||||
marked = persister.mark_orphans_failed_on_startup(jobs)
|
||||
|
||||
assert marked == 2 # interrupted + pending-not-started
|
||||
assert set(jobs.keys()) == {"interrupted", "pending-not-started"}
|
||||
|
||||
for job_id in ("interrupted", "pending-not-started"):
|
||||
assert jobs[job_id]["status"] == "failed"
|
||||
assert jobs[job_id]["restart_failure_code"] == _RESTART_FAIL_CODE
|
||||
assert _RESTART_FAIL_CODE in jobs[job_id]["error"]
|
||||
# Persisted back to Firestore so polls see the failed state
|
||||
assert client._docs[job_id]["status"] == "failed"
|
||||
|
||||
|
||||
def test_mark_orphans_failed_on_startup_returns_zero_when_clean():
|
||||
"""Clean restart (no in-progress rows) must be a fast no-op."""
|
||||
client = _fake_firestore_client()
|
||||
persister = JobStatePersister(vm_name="encoding-worker-a", client=client)
|
||||
# Only completed rows
|
||||
persister.save({"job_id": "old-1", "status": "complete"})
|
||||
persister.save({"job_id": "old-2", "status": "failed"})
|
||||
|
||||
jobs: dict[str, dict] = {}
|
||||
assert persister.mark_orphans_failed_on_startup(jobs) == 0
|
||||
assert jobs == {}
|
||||
|
||||
|
||||
def test_persister_self_disables_on_permission_denied():
|
||||
"""When IAM hasn't been applied yet, the first save() call hits
|
||||
PermissionDenied. The persister must log ONCE and disable, not log a
|
||||
warning per save (which would spam during normal job processing
|
||||
while waiting for the operator to run `pulumi up`).
|
||||
"""
|
||||
class FakePermissionDenied(Exception):
|
||||
"""Stand-in matched by name in _looks_like_auth_error — avoids
|
||||
importing google.api_core just for the type."""
|
||||
|
||||
FakePermissionDenied.__name__ = "PermissionDenied"
|
||||
|
||||
save_count = 0
|
||||
client = MagicMock()
|
||||
|
||||
def collection_factory(name):
|
||||
coll = MagicMock()
|
||||
|
||||
def document(doc_id):
|
||||
nonlocal save_count
|
||||
doc = MagicMock()
|
||||
|
||||
def set(data):
|
||||
nonlocal save_count
|
||||
save_count += 1
|
||||
raise FakePermissionDenied("missing roles/datastore.user")
|
||||
|
||||
doc.set.side_effect = set
|
||||
return doc
|
||||
|
||||
coll.document.side_effect = document
|
||||
return coll
|
||||
|
||||
client.collection.side_effect = collection_factory
|
||||
persister = JobStatePersister(vm_name="x", client=client)
|
||||
|
||||
# First save triggers PermissionDenied → persister disables itself
|
||||
persister.save({"job_id": "a", "status": "running"})
|
||||
assert save_count == 1
|
||||
assert persister.enabled is False
|
||||
|
||||
# Subsequent saves are no-ops — must NOT hit Firestore again
|
||||
persister.save({"job_id": "b", "status": "running"})
|
||||
persister.save({"job_id": "c", "status": "running"})
|
||||
assert save_count == 1
|
||||
|
||||
|
||||
def test_persister_disabled_when_firestore_client_construction_fails(monkeypatch):
|
||||
"""If Firestore client construction raises (e.g. local dev without creds),
|
||||
the persister must silently disable instead of breaking the worker.
|
||||
All ops become no-ops.
|
||||
|
||||
Patch via the actual `google.cloud.firestore` module attribute that
|
||||
persistence._get_client() resolves at runtime — patching by string
|
||||
path is unreliable across CI environments where the import order can
|
||||
differ.
|
||||
"""
|
||||
import google.cloud.firestore as fs_module
|
||||
monkeypatch.setattr(
|
||||
fs_module, "Client",
|
||||
MagicMock(side_effect=Exception("no creds")),
|
||||
)
|
||||
|
||||
persister = JobStatePersister(vm_name="x")
|
||||
# First touch triggers init — should disable, not raise
|
||||
assert persister.enabled is False
|
||||
# All ops safe to call (no-ops)
|
||||
persister.save({"job_id": "abc", "status": "running"})
|
||||
assert persister.load_active_jobs() == {}
|
||||
@@ -80,7 +80,7 @@ Render fails → typed error classification
|
||||
The 3 items below were identified during the May 5–6 session. Status updated as each ships.
|
||||
|
||||
1. **Render worker doesn't handle SIGTERM gracefully.** ✅ **Shipped in v0.174.4** — render worker registers with `worker_registry`; FastAPI shutdown waits up to 480s, then calls `park_active_render_jobs_for_shutdown()` which transitions any still-active render jobs to `RENDER_PENDING_CAPACITY` (last_code: `WORKER_SHUTDOWN`) before exit. Auto-retry scheduler picks them up within 5 min. Reduced wait from 600s → 480s leaves 120s for cleanup before SIGKILL.
|
||||
2. **Encoding worker loses in-memory job state on systemctl restart.** Observed once on `693c2254` — submit job to fallback-a, worker restarts (deploy or crash), poll for status returns `not found`. Workaround: retry the job (usually succeeds). Fix: persist job state to GCS or Firestore.
|
||||
2. **Encoding worker loses in-memory job state on systemctl restart.** ✅ **Shipped in v0.174.6** — `JobStatePersister` mirrors `jobs[]` to Firestore collection `encoding_worker_jobs`, tagged with `vm_name` from `ENCODING_WORKER_VM_NAME` env var (set by startup.sh from GCE metadata). FastAPI lifespan startup loads any in-progress jobs and marks them `failed` with `restart_failure_code: encoding_worker_restart` (work_dir is gone, can't resume). Polls after restart now return clear failure instead of 404. Datastore TTL set to 7 days for cleanup. Best-effort writes — Firestore failures log a warning but don't fail encoding. Self-disables on PermissionDenied so log noise stays bounded if IAM lags the deploy. Requires `roles/datastore.user` on `encoding-worker-sa` (added in IAM module).
|
||||
3. **Concurrent renders against a single fallback VM cause `Connection reset by peer`.** ✅ **Shipped in v0.174.5** — `EncodingService._submission_slot()` adds a per-URL `asyncio.Semaphore` (default cap 3, configurable via `ENCODING_SUBMISSION_CONCURRENCY` env var). Wraps `render_video_on_gce` and `encode_videos` for the full submit + wait cycle so the worker isn't asked to encode more than 3 jobs in parallel from one Cloud Run instance. Different URLs (primary vs fallback) have separate semaphores so multi-zone fallback isn't serialized. Cap leaves 1 slot of the worker's 4-thread pool free for status polls / previews. **Caveat:** this is per-Cloud-Run-instance, not global — with N instances each contributing 3, a single worker VM can still see 3N concurrent. Acceptable for current scale; revisit if multi-instance fallback storms recur.
|
||||
|
||||
## Operational reference
|
||||
|
||||
@@ -41,6 +41,16 @@ if [ -z "$ENCODING_API_KEY" ]; then
|
||||
echo "Service will start but may reject requests"
|
||||
fi
|
||||
|
||||
# Read this VM's GCE instance name so the encoding worker can tag its
|
||||
# Firestore-persisted job state with vm_name. This lets each VM only see
|
||||
# its own jobs at startup (different fallback VMs have separate state).
|
||||
# Falls back to the OS hostname (which on GCE is the instance name anyway)
|
||||
# if the metadata server isn't reachable for any reason.
|
||||
VM_NAME=$(curl -s --connect-timeout 2 -H "Metadata-Flavor: Google" \
|
||||
"http://metadata.google.internal/computeMetadata/v1/instance/name" 2>/dev/null \
|
||||
|| hostname)
|
||||
echo "VM name: ${VM_NAME}"
|
||||
|
||||
# Write environment file for systemd.
|
||||
#
|
||||
# GCE_METADATA_MTLS_MODE=none forces HTTP metadata server access on port 80.
|
||||
@@ -54,6 +64,7 @@ fi
|
||||
cat > "${WORKER_DIR}/env" <<EOF
|
||||
ENCODING_API_KEY=${ENCODING_API_KEY}
|
||||
GCE_METADATA_MTLS_MODE=none
|
||||
ENCODING_WORKER_VM_NAME=${VM_NAME}
|
||||
EOF
|
||||
chmod 600 "${WORKER_DIR}/env"
|
||||
|
||||
|
||||
@@ -65,6 +65,16 @@ def grant_encoding_worker_permissions(
|
||||
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
|
||||
)
|
||||
|
||||
# Datastore User - read/write `encoding_worker_jobs` collection so per-VM
|
||||
# job state survives a systemd restart (so polls return last-known status
|
||||
# instead of 404). See backend/services/gce_encoding/persistence.py.
|
||||
bindings["firestore_access"] = gcp.projects.IAMMember(
|
||||
"encoding-worker-firestore-access",
|
||||
project=PROJECT_ID,
|
||||
role="roles/datastore.user",
|
||||
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
|
||||
)
|
||||
|
||||
return bindings
|
||||
|
||||
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.174.5"
|
||||
version = "0.174.6"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
Reference in new issue
Block a user