Files
karaoke-gen/backend/tests/test_encoding_worker_persistence.py
Andrew Beveridge ☄️ db42594740 feat(encoding-worker): persist job state to Firestore (hardening 2/3) (#757)
## Summary

Addresses **hardening item 2 of 3** from
\`docs/archive/2026-05-06-capacity-resilience-session.md\`.

(Replaces #755, which was auto-closed when its base branch was merged +
deleted.)

Encoding worker held job state in a process-local dict. When systemd
restarted the service (deploy, crash, OOM) the dict was empty, so polls
from the Cloud Run side returned 404 and either failed the user's job or
triggered wasteful re-renders.

Rebased onto main after #754 merged. Item 3 (#756) is stacked on this
PR.

## Changes

- **\`backend/services/gce_encoding/persistence.py\`** (new):
\`JobStatePersister\` class. Lazy Firestore client (no creds at import
time). Per-VM tagging via \`vm_name\`. 7-day TTL via \`expires_at\`. All
write paths swallow Firestore errors so encoding correctness is never
blocked by persistence.
- **\`backend/services/gce_encoding/main.py\`**: \`_persist(job_id)\`
helper called after each status transition. FastAPI lifespan startup
runs \`mark_orphans_failed_on_startup()\`, flipping any in-progress jobs
from before the restart to \`failed\` with \`restart_failure_code:
encoding_worker_restart\`. Subsequent polls see clear failure instead of
404.
- **\`infrastructure/encoding-worker/startup.sh\`**: reads the GCE
instance name from the metadata server and exports
\`ENCODING_WORKER_VM_NAME\` so persistence tags rows with the actual VM
identifier.
- **\`infrastructure/modules/iam/worker_sas.py\`**:
\`roles/datastore.user\` for the encoding-worker SA.
- **\`backend/services/gce_encoding/requirements.txt\`**:
\`google-cloud-firestore>=2.14.0\`.
- **\`pyproject.toml\`**: 0.174.4 → 0.174.5

## ⚠ Deploy note

**Run \`pulumi up\` locally before merging.** Without the IAM binding
the persister logs warnings and disables — encoding still works exactly
as before, but the persistence benefit only kicks in once IAM applies.

## Test plan

- [x] 10 new tests in \`test_encoding_worker_persistence.py\` (passed
locally)
- [x] Existing encoding-worker tests still pass
- [ ] Prod verification:
  1. \`pulumi up\` (apply IAM)
  2. Deploy via CI (uploads new wheel + startup.sh, blue-green VM swap)
  3. Submit a render via the orchestrator
4. SSH to the encoding-worker VM and \`sudo systemctl restart
encoding-worker\` while the job is running
5. Observe: poll from orchestrator returns \`status: failed\` with
\`error\` containing \`encoding_worker_restart\` (not 404)
6. Confirm Firestore \`encoding_worker_jobs\` collection has the row
tagged with the correct \`vm_name\`

@coderabbitai ignore
2026-05-06 22:46:00 -04:00

268 lines
9.8 KiB
Python

"""Tests for encoding-worker job-state persistence to Firestore.
These tests cover the contract documented in
backend/services/gce_encoding/persistence.py:
- save() mirrors job state with vm_name + ttl fields
- save() never raises on Firestore errors (best-effort)
- load_active_jobs() filters by vm_name and skips terminal-status rows
- mark_orphans_failed_on_startup() flips in-progress rows to failed and
persists the change (so a poll after restart returns clear failure
instead of 404)
- When Firestore creds are unavailable, persister silently disables
rather than crashing the worker
"""
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from unittest.mock import MagicMock, patch
import pytest
from backend.services.gce_encoding.persistence import (
JobStatePersister,
get_vm_name,
_RESTART_FAIL_CODE,
)
def _fake_firestore_client():
"""Return a mock Firestore client that records writes and supports stream()."""
client = MagicMock()
# Make .collection().document().set() work
docs: dict[str, dict] = {}
def collection(name):
coll = MagicMock()
def document(doc_id):
doc = MagicMock()
def set(data):
docs[doc_id] = data
doc.set.side_effect = set
return doc
coll.document.side_effect = document
def where(*args, **kwargs):
# Capture the filter so stream() can apply it
filter_field = args[0] if args else kwargs.get("field_path")
filter_value = args[2] if len(args) > 2 else kwargs.get("value")
query = MagicMock()
def stream():
for doc_id, data in docs.items():
if filter_field == "vm_name" and data.get("vm_name") != filter_value:
continue
snapshot = MagicMock()
snapshot.id = doc_id
snapshot.to_dict.return_value = dict(data)
yield snapshot
query.stream.side_effect = stream
return query
coll.where.side_effect = where
coll._docs = docs # so tests can inspect
return coll
client.collection.side_effect = collection
client._docs = docs
return client
def test_get_vm_name_uses_env_var(monkeypatch):
"""ENCODING_WORKER_VM_NAME wins over hostname so we tag rows by GCE
instance name, not the (similar-but-not-guaranteed-equal) hostname."""
monkeypatch.setenv("ENCODING_WORKER_VM_NAME", "encoding-worker-fallback-a")
assert get_vm_name() == "encoding-worker-fallback-a"
def test_get_vm_name_falls_back_to_hostname(monkeypatch):
monkeypatch.delenv("ENCODING_WORKER_VM_NAME", raising=False)
name = get_vm_name()
# Whatever the test machine's hostname is, just confirm we got *something*
assert name and isinstance(name, str)
def test_save_writes_doc_with_vm_name_and_ttl():
client = _fake_firestore_client()
persister = JobStatePersister(vm_name="encoding-worker-a", client=client)
persister.save({
"job_id": "abc123",
"status": "running",
"progress": 42,
})
assert "abc123" in client._docs
doc = client._docs["abc123"]
assert doc["job_id"] == "abc123"
assert doc["status"] == "running"
assert doc["progress"] == 42
assert doc["vm_name"] == "encoding-worker-a"
# TTL field set ~7 days in the future (default)
assert isinstance(doc["expires_at"], datetime)
assert doc["expires_at"] > datetime.now(timezone.utc) + timedelta(days=6)
def test_save_swallows_firestore_errors():
"""A Firestore failure must NOT propagate — encoding correctness comes
first; persistence is best-effort observability/recovery."""
client = MagicMock()
client.collection.side_effect = RuntimeError("Firestore down")
persister = JobStatePersister(vm_name="x", client=client)
# Should not raise
persister.save({"job_id": "abc", "status": "running"})
def test_save_skips_when_job_id_missing():
client = _fake_firestore_client()
persister = JobStatePersister(vm_name="x", client=client)
persister.save({"status": "pending"}) # no job_id
assert client._docs == {}
def test_load_active_jobs_filters_by_vm_and_skips_terminal_statuses():
client = _fake_firestore_client()
persister = JobStatePersister(vm_name="encoding-worker-a", client=client)
# Pre-seed Firestore with mixed-VM, mixed-status rows
persister.save({"job_id": "running-on-a", "status": "running", "progress": 50})
persister.save({"job_id": "complete-on-a", "status": "complete", "progress": 100})
persister.save({"job_id": "failed-on-a", "status": "failed"})
other_vm = JobStatePersister(vm_name="encoding-worker-b", client=client)
other_vm.save({"job_id": "running-on-b", "status": "running"})
active = persister.load_active_jobs()
# Only the in-progress job for VM A
assert set(active.keys()) == {"running-on-a"}
# Persistence-only fields stripped
assert "vm_name" not in active["running-on-a"]
assert "expires_at" not in active["running-on-a"]
assert active["running-on-a"]["status"] == "running"
def test_load_active_jobs_returns_empty_on_firestore_error():
"""An unreadable Firestore must NOT prevent the worker from starting up
— we just lose the recovery capability for this restart."""
client = MagicMock()
client.collection.side_effect = RuntimeError("Firestore unreachable")
persister = JobStatePersister(vm_name="x", client=client)
assert persister.load_active_jobs() == {}
def test_mark_orphans_failed_on_startup_flips_in_progress_jobs():
"""On worker restart, any job that was running/pending must be marked
failed in the in-memory jobs dict AND persisted back to Firestore so
a subsequent poll sees clear failure (not 404, not still-running)."""
client = _fake_firestore_client()
persister = JobStatePersister(vm_name="encoding-worker-a", client=client)
# Simulate state from before the restart
persister.save({"job_id": "interrupted", "status": "running", "progress": 75})
persister.save({"job_id": "pending-not-started", "status": "pending", "progress": 0})
persister.save({"job_id": "old-completed", "status": "complete", "progress": 100})
jobs: dict[str, dict] = {}
marked = persister.mark_orphans_failed_on_startup(jobs)
assert marked == 2 # interrupted + pending-not-started
assert set(jobs.keys()) == {"interrupted", "pending-not-started"}
for job_id in ("interrupted", "pending-not-started"):
assert jobs[job_id]["status"] == "failed"
assert jobs[job_id]["restart_failure_code"] == _RESTART_FAIL_CODE
assert _RESTART_FAIL_CODE in jobs[job_id]["error"]
# Persisted back to Firestore so polls see the failed state
assert client._docs[job_id]["status"] == "failed"
def test_mark_orphans_failed_on_startup_returns_zero_when_clean():
"""Clean restart (no in-progress rows) must be a fast no-op."""
client = _fake_firestore_client()
persister = JobStatePersister(vm_name="encoding-worker-a", client=client)
# Only completed rows
persister.save({"job_id": "old-1", "status": "complete"})
persister.save({"job_id": "old-2", "status": "failed"})
jobs: dict[str, dict] = {}
assert persister.mark_orphans_failed_on_startup(jobs) == 0
assert jobs == {}
def test_persister_self_disables_on_permission_denied():
"""When IAM hasn't been applied yet, the first save() call hits
PermissionDenied. The persister must log ONCE and disable, not log a
warning per save (which would spam during normal job processing
while waiting for the operator to run `pulumi up`).
"""
class FakePermissionDenied(Exception):
"""Stand-in matched by name in _looks_like_auth_error — avoids
importing google.api_core just for the type."""
FakePermissionDenied.__name__ = "PermissionDenied"
save_count = 0
client = MagicMock()
def collection_factory(name):
coll = MagicMock()
def document(doc_id):
nonlocal save_count
doc = MagicMock()
def set(data):
nonlocal save_count
save_count += 1
raise FakePermissionDenied("missing roles/datastore.user")
doc.set.side_effect = set
return doc
coll.document.side_effect = document
return coll
client.collection.side_effect = collection_factory
persister = JobStatePersister(vm_name="x", client=client)
# First save triggers PermissionDenied → persister disables itself
persister.save({"job_id": "a", "status": "running"})
assert save_count == 1
assert persister.enabled is False
# Subsequent saves are no-ops — must NOT hit Firestore again
persister.save({"job_id": "b", "status": "running"})
persister.save({"job_id": "c", "status": "running"})
assert save_count == 1
def test_persister_disabled_when_firestore_client_construction_fails(monkeypatch):
"""If Firestore client construction raises (e.g. local dev without creds),
the persister must silently disable instead of breaking the worker.
All ops become no-ops.
Patch via the actual `google.cloud.firestore` module attribute that
persistence._get_client() resolves at runtime — patching by string
path is unreliable across CI environments where the import order can
differ.
"""
import google.cloud.firestore as fs_module
monkeypatch.setattr(
fs_module, "Client",
MagicMock(side_effect=Exception("no creds")),
)
persister = JobStatePersister(vm_name="x")
# First touch triggers init — should disable, not raise
assert persister.enabled is False
# All ops safe to call (no-ops)
persister.save({"job_id": "abc", "status": "running"})
assert persister.load_active_jobs() == {}