feat(encoding-worker): persist job state to Firestore (hardening 2/3) (#757)

## Summary

Addresses **hardening item 2 of 3** from
\`docs/archive/2026-05-06-capacity-resilience-session.md\`.

(Replaces #755, which was auto-closed when its base branch was merged +
deleted.)

Encoding worker held job state in a process-local dict. When systemd
restarted the service (deploy, crash, OOM) the dict was empty, so polls
from the Cloud Run side returned 404 and either failed the user's job or
triggered wasteful re-renders.

Rebased onto main after #754 merged. Item 3 (#756) is stacked on this
PR.

## Changes

- **\`backend/services/gce_encoding/persistence.py\`** (new):
\`JobStatePersister\` class. Lazy Firestore client (no creds at import
time). Per-VM tagging via \`vm_name\`. 7-day TTL via \`expires_at\`. All
write paths swallow Firestore errors so encoding correctness is never
blocked by persistence.
- **\`backend/services/gce_encoding/main.py\`**: \`_persist(job_id)\`
helper called after each status transition. FastAPI lifespan startup
runs \`mark_orphans_failed_on_startup()\`, flipping any in-progress jobs
from before the restart to \`failed\` with \`restart_failure_code:
encoding_worker_restart\`. Subsequent polls see clear failure instead of
404.
- **\`infrastructure/encoding-worker/startup.sh\`**: reads the GCE
instance name from the metadata server and exports
\`ENCODING_WORKER_VM_NAME\` so persistence tags rows with the actual VM
identifier.
- **\`infrastructure/modules/iam/worker_sas.py\`**:
\`roles/datastore.user\` for the encoding-worker SA.
- **\`backend/services/gce_encoding/requirements.txt\`**:
\`google-cloud-firestore>=2.14.0\`.
- **\`pyproject.toml\`**: 0.174.4 → 0.174.5

## ⚠ Deploy note

**Run \`pulumi up\` locally before merging.** Without the IAM binding
the persister logs warnings and disables — encoding still works exactly
as before, but the persistence benefit only kicks in once IAM applies.

## Test plan

- [x] 10 new tests in \`test_encoding_worker_persistence.py\` (passed
locally)
- [x] Existing encoding-worker tests still pass
- [ ] Prod verification:
  1. \`pulumi up\` (apply IAM)
  2. Deploy via CI (uploads new wheel + startup.sh, blue-green VM swap)
  3. Submit a render via the orchestrator
4. SSH to the encoding-worker VM and \`sudo systemctl restart
encoding-worker\` while the job is running
5. Observe: poll from orchestrator returns \`status: failed\` with
\`error\` containing \`encoding_worker_restart\` (not 404)
6. Confirm Firestore \`encoding_worker_jobs\` collection has the row
tagged with the correct \`vm_name\`

@coderabbitai ignore
This commit is contained in:
Andrew Beveridge ☄️ authored and GitHub committed 2026-05-06 22:46:00 -04:00
1 parent 9b903d95d3
commit db42594740
8 files changed
+617 -7

No files matched your search

+57 -5
View File
@@ -11,6 +11,7 @@ import sys
import tempfile
import uuid
from concurrent.futures import ThreadPoolExecutor
from contextlib import asynccontextmanager
from pathlib import Path
from typing import Optional
@@ -19,10 +20,48 @@ from google.cloud import storage
from packaging.version import Version
from pydantic import BaseModel
from .persistence import JobStatePersister
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
app = FastAPI(title="Encoding Worker", version="1.0.0")
# Job tracking — process-local dict, mirrored to Firestore via `persister`
# so polls after a worker restart return the *last known* status instead of
# 404. Hot path stays in-memory; persistence is best-effort.
jobs: dict[str, dict] = {}
executor = ThreadPoolExecutor(max_workers=4) # 4 parallel encoding jobs
# Persister is initialized lazily — see persistence.py — so import-time
# stays free of GCP creds. Tests can monkey-patch this attribute.
persister = JobStatePersister()
def _persist(job_id: str) -> None:
"""Mirror a job's current state to Firestore (best-effort, never raises)."""
state = jobs.get(job_id)
if state is not None:
persister.save(state)
@asynccontextmanager
async def lifespan(app: FastAPI):
# Startup: read back any in-progress jobs from Firestore. They were
# interrupted by the restart that brought us here — their work_dir is
# gone, so mark them failed with the restart code so polls return a
# clear failure instead of 404.
try:
marked = persister.mark_orphans_failed_on_startup(jobs)
if marked:
logger.warning(
"Marked %d orphaned encoding job(s) failed on startup", marked,
)
except Exception as exc:
logger.error("Encoding worker startup orphan-recovery failed: %r", exc)
yield
app = FastAPI(title="Encoding Worker", version="1.0.0", lifespan=lifespan)
# API key authentication
API_KEY = os.environ.get("ENCODING_API_KEY", "")
@@ -37,10 +76,6 @@ async def verify_api_key(x_api_key: str = Header(None)):
raise HTTPException(status_code=401, detail="Invalid API key")
return True
# Job tracking
jobs: dict[str, dict] = {}
executor = ThreadPoolExecutor(max_workers=4) # 4 parallel encoding jobs
# GCS client
storage_client = storage.Client()
@@ -213,6 +248,7 @@ def run_preview_encoding(job_id: str, work_dir: Path, request: "EncodePreviewReq
# Run FFmpeg encoding for preview video (480x270, fast settings)
jobs[job_id]["status"] = "running"
jobs[job_id]["progress"] = 10
_persist(job_id)
try:
# Download input files
@@ -309,6 +345,7 @@ def run_preview_encoding(job_id: str, work_dir: Path, request: "EncodePreviewReq
logger.error(f"Preview encoding failed: {e}")
jobs[job_id]["status"] = "failed"
jobs[job_id]["error"] = str(e)
_persist(job_id)
raise
@@ -324,6 +361,7 @@ def run_render_video(job_id: str, work_dir: Path, request: "RenderVideoRequest")
"""
jobs[job_id]["status"] = "running"
jobs[job_id]["progress"] = 5
_persist(job_id)
try:
# Import from installed karaoke-gen wheel
@@ -490,12 +528,14 @@ def run_render_video(job_id: str, work_dir: Path, request: "RenderVideoRequest")
logger.error(error_msg)
jobs[job_id]["status"] = "failed"
jobs[job_id]["error"] = error_msg
_persist(job_id)
raise RuntimeError(error_msg) from e
except Exception as e:
logger.error(f"[job:{job_id}] Render video failed: {e}")
jobs[job_id]["status"] = "failed"
jobs[job_id]["error"] = str(e)
_persist(job_id)
raise
@@ -617,6 +657,7 @@ def run_encoding(job_id: str, work_dir: Path, config: dict):
'''
jobs[job_id]["status"] = "running"
jobs[job_id]["progress"] = 10
_persist(job_id)
try:
# Import LocalEncodingService from installed wheel (required, no fallback)
@@ -803,12 +844,14 @@ def run_encoding(job_id: str, work_dir: Path, config: dict):
logger.error(error_msg)
jobs[job_id]["status"] = "failed"
jobs[job_id]["error"] = error_msg
_persist(job_id)
raise RuntimeError(error_msg) from e
except Exception as e:
logger.error(f"Encoding failed: {e}")
jobs[job_id]["status"] = "failed"
jobs[job_id]["error"] = str(e)
_persist(job_id)
raise
@@ -871,12 +914,14 @@ async def process_job(job_id: str, request: EncodeRequest):
jobs[job_id]["status"] = "complete"
jobs[job_id]["progress"] = 100
_persist(job_id)
logger.info(f"Job {job_id} complete")
except Exception as e:
logger.error(f"Job {job_id} failed: {e}")
jobs[job_id]["status"] = "failed"
jobs[job_id]["error"] = str(e)
_persist(job_id)
async def process_preview_job(job_id: str, request: EncodePreviewRequest):
@@ -899,12 +944,14 @@ async def process_preview_job(job_id: str, request: EncodePreviewRequest):
jobs[job_id]["status"] = "complete"
jobs[job_id]["progress"] = 100
_persist(job_id)
logger.info(f"Preview job {job_id} complete")
except Exception as e:
logger.error(f"Preview job {job_id} failed: {e}")
jobs[job_id]["status"] = "failed"
jobs[job_id]["error"] = str(e)
_persist(job_id)
async def process_render_video_job(job_id: str, request: RenderVideoRequest):
@@ -929,12 +976,14 @@ async def process_render_video_job(job_id: str, request: RenderVideoRequest):
jobs[job_id]["status"] = "complete"
jobs[job_id]["progress"] = 100
_persist(job_id)
logger.info(f"Render video job {job_id} complete")
except Exception as e:
logger.error(f"Render video job {job_id} failed: {e}")
jobs[job_id]["status"] = "failed"
jobs[job_id]["error"] = str(e)
_persist(job_id)
@app.post("/encode-preview")
@@ -962,6 +1011,7 @@ async def submit_preview_encode_job(request: EncodePreviewRequest, background_ta
"error": None,
"output_files": None,
}
_persist(job_id)
background_tasks.add_task(process_preview_job, job_id, request)
@@ -993,6 +1043,7 @@ async def submit_render_video_job(request: RenderVideoRequest, background_tasks:
"output_files": None,
"metadata": None,
}
_persist(job_id)
background_tasks.add_task(process_render_video_job, job_id, request)
@@ -1024,6 +1075,7 @@ async def submit_encode_job(request: EncodeRequest, background_tasks: Background
"error": None,
"output_files": None,
}
_persist(job_id)
background_tasks.add_task(process_job, job_id, request)
@@ -0,0 +1,266 @@
"""Firestore persistence for the encoding worker's in-memory job state.
Why this exists
---------------
The encoding worker holds job state in a process-local `dict`. When the
systemd service restarts (deploy, crash, OOM, manual restart) the dict is
empty, so polls from the Cloud Run side return 404 and the orchestrator
either fails the user's job or triggers wasteful re-renders.
This module mirrors per-job state to Firestore so that:
1. Polls after a restart return the *last known* status (typically `failed`
with `code: encoding_worker_restart`), not 404 — orchestrator can react
intelligently instead of guessing.
2. On startup, the worker reads back any jobs it had in flight and marks
them failed with that restart code, so the in-memory dict and Firestore
stay in sync after a crash.
Scope
-----
Each row is keyed by `job_id` and tagged with `vm_name`. Loads are filtered
to the current VM — VMs don't see each other's jobs. (We never have two
VMs simultaneously holding the same `job_id` — Cloud Run only submits to
one VM at a time, and on restart that same VM is the only one that owned
the doc.)
A Firestore field-level TTL on `expires_at` cleans completed/failed rows
automatically after 7 days; we set it on every write.
Cost / latency
--------------
Writes are best-effort. A Firestore failure logs a warning but does NOT
fail the encoding job — persistence is observability/recovery, not
correctness. The hot path stays in-memory.
"""
from __future__ import annotations
import logging
import os
import socket
import threading
from datetime import datetime, timedelta, timezone
from typing import Any, Optional
logger = logging.getLogger(__name__)
_RESTART_FAIL_CODE = "encoding_worker_restart"
_TERMINAL_STATUSES = frozenset({"complete", "failed"})
_DEFAULT_TTL_DAYS = 7
_COLLECTION = "encoding_worker_jobs"
def _now_utc() -> datetime:
return datetime.now(timezone.utc)
def get_vm_name() -> str:
"""Return the VM's identifier.
Prefer the GCE instance name (set by infrastructure/encoding-worker/
startup.sh as `ENCODING_WORKER_VM_NAME`), falling back to the OS
hostname so this still works in tests/local dev.
"""
return (
os.environ.get("ENCODING_WORKER_VM_NAME")
or socket.gethostname()
or "unknown"
)
class JobStatePersister:
"""Mirror in-memory job state to Firestore.
Lazily initializes the Firestore client so import-time has zero cost
and tests/local-dev that don't have GCP creds don't blow up.
All operations are thread-safe — the encoding worker uses a
ThreadPoolExecutor and may call save() from multiple worker threads.
"""
def __init__(
self,
*,
vm_name: Optional[str] = None,
collection: str = _COLLECTION,
ttl_days: int = _DEFAULT_TTL_DAYS,
client: Any = None, # for tests
) -> None:
self.vm_name = vm_name or get_vm_name()
self.collection = collection
self.ttl_days = ttl_days
self._client = client
self._client_lock = threading.Lock()
self._enabled: Optional[bool] = None # None = not yet probed
def _get_client(self):
"""Return a Firestore client or None if creds are missing."""
if self._client is not None:
return self._client
with self._client_lock:
if self._client is not None:
return self._client
try:
from google.cloud import firestore # type: ignore
self._client = firestore.Client()
self._enabled = True
logger.info(
"JobStatePersister initialized (vm=%s, collection=%s)",
self.vm_name, self.collection,
)
except Exception as exc:
# Don't fail the worker if creds aren't available — just
# disable persistence and log loudly so it's discoverable.
self._enabled = False
logger.warning(
"JobStatePersister disabled (Firestore unavailable): %s",
exc,
)
return self._client
@property
def enabled(self) -> bool:
if self._enabled is None:
self._get_client()
return bool(self._enabled)
def save(self, job_state: dict) -> None:
"""Persist a single job's state.
Adds `vm_name`, `updated_at`, and `expires_at` (TTL) fields. Never
raises — Firestore failures are logged and swallowed because
encoding is the source of truth and persistence is best-effort.
On the first PermissionDenied (typically: IAM not yet applied for
the encoding worker SA), self-disables to avoid logging a warning
on every save call. Caller can re-enable by re-instantiating after
IAM lands.
"""
client = self._get_client()
if client is None:
return
job_id = job_state.get("job_id")
if not job_id:
logger.warning("JobStatePersister.save called without job_id; skipping")
return
try:
doc = {
**job_state,
"vm_name": self.vm_name,
"updated_at": _now_utc(),
"expires_at": _now_utc() + timedelta(days=self.ttl_days),
}
client.collection(self.collection).document(job_id).set(doc)
except Exception as exc:
# If this is a permanent auth failure (IAM not applied yet),
# log once and self-disable so we don't spam ERROR per save.
# Other transient errors are logged at WARNING per occurrence.
if self._looks_like_auth_error(exc):
if self._enabled is not False: # only log on transition
logger.warning(
"JobStatePersister disabling: Firestore auth failed "
"(likely IAM not yet applied to encoding-worker-sa: "
"needs roles/datastore.user). Subsequent saves will "
"be no-ops. Error: %r",
exc,
)
self._enabled = False
self._client = None
return
logger.warning(
"[job:%s] Failed to persist job state to Firestore: %r",
job_id, exc,
)
@staticmethod
def _looks_like_auth_error(exc: BaseException) -> bool:
"""Heuristic — google.api_core.exceptions.PermissionDenied or 403/401."""
type_name = type(exc).__name__
if type_name in ("PermissionDenied", "Unauthenticated", "Forbidden"):
return True
msg = str(exc).lower()
return any(s in msg for s in ("permission_denied", "permissiondenied",
"403", "401", "unauthenticated",
"missing or insufficient permissions"))
def load_active_jobs(self) -> dict[str, dict]:
"""Return any non-terminal jobs persisted for this VM.
Used at startup to recover state from before the restart. Terminal
jobs (complete/failed) are also persisted but not returned here —
their docs serve only post-restart status polls.
Self-disables on auth errors (same pattern as save).
"""
client = self._get_client()
if client is None:
return {}
try:
query = (
client.collection(self.collection)
.where("vm_name", "==", self.vm_name)
)
recovered: dict[str, dict] = {}
for doc_snapshot in query.stream():
data = doc_snapshot.to_dict() or {}
status = data.get("status")
if status in _TERMINAL_STATUSES:
continue
# Strip the persistence-only fields before handing back
for k in ("vm_name", "updated_at", "expires_at"):
data.pop(k, None)
recovered[doc_snapshot.id] = data
return recovered
except Exception as exc:
if self._looks_like_auth_error(exc):
if self._enabled is not False:
logger.warning(
"JobStatePersister disabling: Firestore auth failed "
"during load_active_jobs (likely IAM not applied). "
"Error: %r", exc,
)
self._enabled = False
self._client = None
return {}
logger.warning(
"JobStatePersister.load_active_jobs failed: %r", exc,
)
return {}
def mark_orphans_failed_on_startup(self, jobs: dict[str, dict]) -> int:
"""Mark any in-progress jobs as failed with the restart code.
Called at worker startup. Mutates `jobs` in place AND persists each
change so polls from the orchestrator see consistent state.
The work_dir for an interrupted job is gone (tmpfs / temp dir wiped
by systemd restart), so we cannot resume — the only safe thing is
to surface a clear failure so the caller can decide to retry.
Returns the number of jobs marked failed.
"""
active = self.load_active_jobs()
if not active:
return 0
marked = 0
for job_id, job_state in active.items():
job_state["status"] = "failed"
job_state["error"] = (
f"Encoding worker restarted mid-job (code: {_RESTART_FAIL_CODE}). "
"Worker process restart wiped in-memory state and the temp "
"work directory; resubmit to retry."
)
job_state["restart_failure_code"] = _RESTART_FAIL_CODE
jobs[job_id] = job_state
self.save(job_state)
marked += 1
logger.warning(
"[job:%s] Marked failed at startup due to encoding worker restart "
"(was status=%s, progress=%s)",
job_id,
job_state.get("_pre_restart_status", "unknown"),
job_state.get("progress"),
)
return marked
@@ -8,6 +8,10 @@ uvicorn>=0.27.0
# GCS client
google-cloud-storage>=2.14.0
# Firestore client - persists job state across systemd restarts so polls
# survive a worker restart instead of returning 404. See persistence.py.
google-cloud-firestore>=2.14.0
# Async utilities (optional, for future improvements)
aiofiles>=23.2.0
aiohttp>=3.9.0
@@ -0,0 +1,267 @@
"""Tests for encoding-worker job-state persistence to Firestore.
These tests cover the contract documented in
backend/services/gce_encoding/persistence.py:
- save() mirrors job state with vm_name + ttl fields
- save() never raises on Firestore errors (best-effort)
- load_active_jobs() filters by vm_name and skips terminal-status rows
- mark_orphans_failed_on_startup() flips in-progress rows to failed and
persists the change (so a poll after restart returns clear failure
instead of 404)
- When Firestore creds are unavailable, persister silently disables
rather than crashing the worker
"""
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from unittest.mock import MagicMock, patch
import pytest
from backend.services.gce_encoding.persistence import (
JobStatePersister,
get_vm_name,
_RESTART_FAIL_CODE,
)
def _fake_firestore_client():
"""Return a mock Firestore client that records writes and supports stream()."""
client = MagicMock()
# Make .collection().document().set() work
docs: dict[str, dict] = {}
def collection(name):
coll = MagicMock()
def document(doc_id):
doc = MagicMock()
def set(data):
docs[doc_id] = data
doc.set.side_effect = set
return doc
coll.document.side_effect = document
def where(*args, **kwargs):
# Capture the filter so stream() can apply it
filter_field = args[0] if args else kwargs.get("field_path")
filter_value = args[2] if len(args) > 2 else kwargs.get("value")
query = MagicMock()
def stream():
for doc_id, data in docs.items():
if filter_field == "vm_name" and data.get("vm_name") != filter_value:
continue
snapshot = MagicMock()
snapshot.id = doc_id
snapshot.to_dict.return_value = dict(data)
yield snapshot
query.stream.side_effect = stream
return query
coll.where.side_effect = where
coll._docs = docs # so tests can inspect
return coll
client.collection.side_effect = collection
client._docs = docs
return client
def test_get_vm_name_uses_env_var(monkeypatch):
"""ENCODING_WORKER_VM_NAME wins over hostname so we tag rows by GCE
instance name, not the (similar-but-not-guaranteed-equal) hostname."""
monkeypatch.setenv("ENCODING_WORKER_VM_NAME", "encoding-worker-fallback-a")
assert get_vm_name() == "encoding-worker-fallback-a"
def test_get_vm_name_falls_back_to_hostname(monkeypatch):
monkeypatch.delenv("ENCODING_WORKER_VM_NAME", raising=False)
name = get_vm_name()
# Whatever the test machine's hostname is, just confirm we got *something*
assert name and isinstance(name, str)
def test_save_writes_doc_with_vm_name_and_ttl():
client = _fake_firestore_client()
persister = JobStatePersister(vm_name="encoding-worker-a", client=client)
persister.save({
"job_id": "abc123",
"status": "running",
"progress": 42,
})
assert "abc123" in client._docs
doc = client._docs["abc123"]
assert doc["job_id"] == "abc123"
assert doc["status"] == "running"
assert doc["progress"] == 42
assert doc["vm_name"] == "encoding-worker-a"
# TTL field set ~7 days in the future (default)
assert isinstance(doc["expires_at"], datetime)
assert doc["expires_at"] > datetime.now(timezone.utc) + timedelta(days=6)
def test_save_swallows_firestore_errors():
"""A Firestore failure must NOT propagate — encoding correctness comes
first; persistence is best-effort observability/recovery."""
client = MagicMock()
client.collection.side_effect = RuntimeError("Firestore down")
persister = JobStatePersister(vm_name="x", client=client)
# Should not raise
persister.save({"job_id": "abc", "status": "running"})
def test_save_skips_when_job_id_missing():
client = _fake_firestore_client()
persister = JobStatePersister(vm_name="x", client=client)
persister.save({"status": "pending"}) # no job_id
assert client._docs == {}
def test_load_active_jobs_filters_by_vm_and_skips_terminal_statuses():
client = _fake_firestore_client()
persister = JobStatePersister(vm_name="encoding-worker-a", client=client)
# Pre-seed Firestore with mixed-VM, mixed-status rows
persister.save({"job_id": "running-on-a", "status": "running", "progress": 50})
persister.save({"job_id": "complete-on-a", "status": "complete", "progress": 100})
persister.save({"job_id": "failed-on-a", "status": "failed"})
other_vm = JobStatePersister(vm_name="encoding-worker-b", client=client)
other_vm.save({"job_id": "running-on-b", "status": "running"})
active = persister.load_active_jobs()
# Only the in-progress job for VM A
assert set(active.keys()) == {"running-on-a"}
# Persistence-only fields stripped
assert "vm_name" not in active["running-on-a"]
assert "expires_at" not in active["running-on-a"]
assert active["running-on-a"]["status"] == "running"
def test_load_active_jobs_returns_empty_on_firestore_error():
"""An unreadable Firestore must NOT prevent the worker from starting up
— we just lose the recovery capability for this restart."""
client = MagicMock()
client.collection.side_effect = RuntimeError("Firestore unreachable")
persister = JobStatePersister(vm_name="x", client=client)
assert persister.load_active_jobs() == {}
def test_mark_orphans_failed_on_startup_flips_in_progress_jobs():
"""On worker restart, any job that was running/pending must be marked
failed in the in-memory jobs dict AND persisted back to Firestore so
a subsequent poll sees clear failure (not 404, not still-running)."""
client = _fake_firestore_client()
persister = JobStatePersister(vm_name="encoding-worker-a", client=client)
# Simulate state from before the restart
persister.save({"job_id": "interrupted", "status": "running", "progress": 75})
persister.save({"job_id": "pending-not-started", "status": "pending", "progress": 0})
persister.save({"job_id": "old-completed", "status": "complete", "progress": 100})
jobs: dict[str, dict] = {}
marked = persister.mark_orphans_failed_on_startup(jobs)
assert marked == 2 # interrupted + pending-not-started
assert set(jobs.keys()) == {"interrupted", "pending-not-started"}
for job_id in ("interrupted", "pending-not-started"):
assert jobs[job_id]["status"] == "failed"
assert jobs[job_id]["restart_failure_code"] == _RESTART_FAIL_CODE
assert _RESTART_FAIL_CODE in jobs[job_id]["error"]
# Persisted back to Firestore so polls see the failed state
assert client._docs[job_id]["status"] == "failed"
def test_mark_orphans_failed_on_startup_returns_zero_when_clean():
"""Clean restart (no in-progress rows) must be a fast no-op."""
client = _fake_firestore_client()
persister = JobStatePersister(vm_name="encoding-worker-a", client=client)
# Only completed rows
persister.save({"job_id": "old-1", "status": "complete"})
persister.save({"job_id": "old-2", "status": "failed"})
jobs: dict[str, dict] = {}
assert persister.mark_orphans_failed_on_startup(jobs) == 0
assert jobs == {}
def test_persister_self_disables_on_permission_denied():
"""When IAM hasn't been applied yet, the first save() call hits
PermissionDenied. The persister must log ONCE and disable, not log a
warning per save (which would spam during normal job processing
while waiting for the operator to run `pulumi up`).
"""
class FakePermissionDenied(Exception):
"""Stand-in matched by name in _looks_like_auth_error — avoids
importing google.api_core just for the type."""
FakePermissionDenied.__name__ = "PermissionDenied"
save_count = 0
client = MagicMock()
def collection_factory(name):
coll = MagicMock()
def document(doc_id):
nonlocal save_count
doc = MagicMock()
def set(data):
nonlocal save_count
save_count += 1
raise FakePermissionDenied("missing roles/datastore.user")
doc.set.side_effect = set
return doc
coll.document.side_effect = document
return coll
client.collection.side_effect = collection_factory
persister = JobStatePersister(vm_name="x", client=client)
# First save triggers PermissionDenied → persister disables itself
persister.save({"job_id": "a", "status": "running"})
assert save_count == 1
assert persister.enabled is False
# Subsequent saves are no-ops — must NOT hit Firestore again
persister.save({"job_id": "b", "status": "running"})
persister.save({"job_id": "c", "status": "running"})
assert save_count == 1
def test_persister_disabled_when_firestore_client_construction_fails(monkeypatch):
"""If Firestore client construction raises (e.g. local dev without creds),
the persister must silently disable instead of breaking the worker.
All ops become no-ops.
Patch via the actual `google.cloud.firestore` module attribute that
persistence._get_client() resolves at runtime — patching by string
path is unreliable across CI environments where the import order can
differ.
"""
import google.cloud.firestore as fs_module
monkeypatch.setattr(
fs_module, "Client",
MagicMock(side_effect=Exception("no creds")),
)
persister = JobStatePersister(vm_name="x")
# First touch triggers init — should disable, not raise
assert persister.enabled is False
# All ops safe to call (no-ops)
persister.save({"job_id": "abc", "status": "running"})
assert persister.load_active_jobs() == {}
@@ -80,7 +80,7 @@ Render fails → typed error classification
The 3 items below were identified during the May 5–6 session. Status updated as each ships.
1. **Render worker doesn't handle SIGTERM gracefully.** ✅ **Shipped in v0.174.4** — render worker registers with `worker_registry`; FastAPI shutdown waits up to 480s, then calls `park_active_render_jobs_for_shutdown()` which transitions any still-active render jobs to `RENDER_PENDING_CAPACITY` (last_code: `WORKER_SHUTDOWN`) before exit. Auto-retry scheduler picks them up within 5 min. Reduced wait from 600s → 480s leaves 120s for cleanup before SIGKILL.
2. **Encoding worker loses in-memory job state on systemctl restart.** Observed once on `693c2254` — submit job to fallback-a, worker restarts (deploy or crash), poll for status returns `not found`. Workaround: retry the job (usually succeeds). Fix: persist job state to GCS or Firestore.
2. **Encoding worker loses in-memory job state on systemctl restart.** ✅ **Shipped in v0.174.6** — `JobStatePersister` mirrors `jobs[]` to Firestore collection `encoding_worker_jobs`, tagged with `vm_name` from `ENCODING_WORKER_VM_NAME` env var (set by startup.sh from GCE metadata). FastAPI lifespan startup loads any in-progress jobs and marks them `failed` with `restart_failure_code: encoding_worker_restart` (work_dir is gone, can't resume). Polls after restart now return clear failure instead of 404. Datastore TTL set to 7 days for cleanup. Best-effort writes — Firestore failures log a warning but don't fail encoding. Self-disables on PermissionDenied so log noise stays bounded if IAM lags the deploy. Requires `roles/datastore.user` on `encoding-worker-sa` (added in IAM module).
3. **Concurrent renders against a single fallback VM cause `Connection reset by peer`.** ✅ **Shipped in v0.174.5** — `EncodingService._submission_slot()` adds a per-URL `asyncio.Semaphore` (default cap 3, configurable via `ENCODING_SUBMISSION_CONCURRENCY` env var). Wraps `render_video_on_gce` and `encode_videos` for the full submit + wait cycle so the worker isn't asked to encode more than 3 jobs in parallel from one Cloud Run instance. Different URLs (primary vs fallback) have separate semaphores so multi-zone fallback isn't serialized. Cap leaves 1 slot of the worker's 4-thread pool free for status polls / previews. **Caveat:** this is per-Cloud-Run-instance, not global — with N instances each contributing 3, a single worker VM can still see 3N concurrent. Acceptable for current scale; revisit if multi-instance fallback storms recur.
## Operational reference
+11
View File
@@ -41,6 +41,16 @@ if [ -z "$ENCODING_API_KEY" ]; then
echo "Service will start but may reject requests"
fi
# Read this VM's GCE instance name so the encoding worker can tag its
# Firestore-persisted job state with vm_name. This lets each VM only see
# its own jobs at startup (different fallback VMs have separate state).
# Falls back to the OS hostname (which on GCE is the instance name anyway)
# if the metadata server isn't reachable for any reason.
VM_NAME=$(curl -s --connect-timeout 2 -H "Metadata-Flavor: Google" \
"http://metadata.google.internal/computeMetadata/v1/instance/name" 2>/dev/null \
|| hostname)
echo "VM name: ${VM_NAME}"
# Write environment file for systemd.
#
# GCE_METADATA_MTLS_MODE=none forces HTTP metadata server access on port 80.
@@ -54,6 +64,7 @@ fi
cat > "${WORKER_DIR}/env" <<EOF
ENCODING_API_KEY=${ENCODING_API_KEY}
GCE_METADATA_MTLS_MODE=none
ENCODING_WORKER_VM_NAME=${VM_NAME}
EOF
chmod 600 "${WORKER_DIR}/env"
+10
View File
@@ -65,6 +65,16 @@ def grant_encoding_worker_permissions(
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
)
# Datastore User - read/write `encoding_worker_jobs` collection so per-VM
# job state survives a systemd restart (so polls return last-known status
# instead of 404). See backend/services/gce_encoding/persistence.py.
bindings["firestore_access"] = gcp.projects.IAMMember(
"encoding-worker-firestore-access",
project=PROJECT_ID,
role="roles/datastore.user",
member=service_account.email.apply(lambda email: f"serviceAccount:{email}"),
)
return bindings
+1 -1
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "karaoke-gen"
version = "0.174.5"
version = "0.174.6"
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
license = "MIT"