Files
karaoke-gen/backend/tests/test_encoding_service.py
T
Andrew Beveridge ☄️andClaude Opus 4.8 c7959ea2d5 feat(encoding): mature multi-instance-type pool with availability-aware selection (#911)
## Summary
- Replaces "c4d primary + single n2 fallback family" with a **ranked
pool of 6 x86_64 / 32-vCPU / ≥32 GB highcpu types** across 4 silicon
lineages (c4d, c4, n4d, c2d, n2d, n2), selected **fastest-first** but
**availability-aware** — a type that stocks out is demoted for a 15-min
cooldown then re-probed. c4d stays the top pick whenever it can start; a
render no longer stalls even if 2+ types are fully stocked out in
us-central1.
- One **pure preference module**
(`encoding_worker_preference.ordered_candidates`) is now the single
source of truth for candidate ranking, driving **both** runtime
selection and deploy green selection so they can't drift.
- Keeps the per-VM start/stop model (MIG deferred with documented
triggers; no reservation). Full plan + rollout in
`docs/archive/2026-08-15-encoding-instance-type-diversity-plan.md`.

## Changes
- **New** `backend/services/encoding_worker_preference.py` — speed-rank
+ Firestore `capacity_state` cooldown; stdlib-only so `deploy_promote`
imports it (with a shadow-proof by-path fallback).
- **Runtime** (`encoding_service`, `encoding_worker_manager`): builder
ranks primary + fallbacks; manager records/clears per-(type,zone)
stockout; override routing now keys off an explicit `is_primary` flag
instead of list position.
- **Deploy** (`deploy_promote`, `ci.yml`): green selection uses the
shared ranker + threads `capacity_state`.
- **IaC** (`config.py`, tests): `FALLBACKS` broadened to 6 types
(additive-only) with family-correct disk types (Titanium c4/c4d/n4/n4d =
hyperdisk-balanced; c2d/n2/n2d = pd-balanced); secret schema gains
optional `machine_type`.
- **Packer** (`provision.sh`, `startup.sh`): bake full dep tree with
**CPU-only torch** so fresh type VMs boot self-sufficient; build-time
import verify.
- **New** `benchmark_types.py` to measure per-type encode wall-time and
refine the seeded `SPEED_RANK`.

## Testing
- [x] Full backend unit suite: **3871 passed, 36 skipped** (emulator
tests need external services)
- [x] New/updated tests: preference module (16), capacity_state +
`is_primary` manager tests, builder-ordering + non-list-JSON service
tests, deploy ranking tests, infra disk-capability + pool-breadth
invariants
- [ ] Operator rollout (needs write creds): `pulumi up` (verify +8
create, 0 replace) → rebuild image (dep-bake) → grow fallback secret
with `machine_type` → run `benchmark_types.py`

## Review
- [x] Local CodeRabbit review completed (4 major findings fixed; cycle 2
clean)
- [x] Review feedback addressed

@coderabbitai ignore

---
🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
2026-08-15 20:52:56 -04:00

1154 lines
52 KiB
Python

"""
Tests for the GCE encoding service (encoding_service.py).
Tests resilience features: idempotent submission handling, cached/in_progress
responses, 409 fallback behavior, deployment retry tolerance, and poll failure handling.
"""
import asyncio
import aiohttp
import pytest
from unittest.mock import AsyncMock, patch, MagicMock
from backend.services.encoding_service import (
EncodingService,
MAX_RETRIES,
INITIAL_BACKOFF_SECONDS,
MAX_BACKOFF_SECONDS,
MAX_CONSECUTIVE_POLL_FAILURES,
run_with_lost_job_resubmit,
)
from backend.services.encoding_errors import EncodingJobLostError, EncodingJobNotFoundError
@pytest.fixture
def encoding_service():
"""Create an EncodingService with mocked credentials."""
service = EncodingService()
service._url = "http://fake-worker:8080"
service._api_key = "test-key"
service._initialized = True
return service
class TestSubmitEncodingJob:
"""Tests for submit_encoding_job() resilience."""
@pytest.mark.asyncio
async def test_submit_returns_accepted(self, encoding_service):
"""Normal submission returns accepted response."""
mock_resp = {"status": 200, "json": {"status": "accepted", "job_id": "j1"}, "text": None}
with patch.object(encoding_service, "_request_with_retry", new_callable=AsyncMock, return_value=mock_resp):
result = await encoding_service.submit_encoding_job("j1", "gs://in", "gs://out", {})
assert result["status"] == "accepted"
@pytest.mark.asyncio
async def test_submit_returns_cached(self, encoding_service):
"""When worker returns cached (job already complete), pass through."""
mock_resp = {"status": 200, "json": {"status": "cached", "job_id": "j1", "output_files": ["a.mp4"]}, "text": None}
with patch.object(encoding_service, "_request_with_retry", new_callable=AsyncMock, return_value=mock_resp):
result = await encoding_service.submit_encoding_job("j1", "gs://in", "gs://out", {})
assert result["status"] == "cached"
assert result["output_files"] == ["a.mp4"]
@pytest.mark.asyncio
async def test_submit_returns_in_progress(self, encoding_service):
"""When worker returns in_progress, pass through."""
mock_resp = {"status": 200, "json": {"status": "in_progress", "job_id": "j1"}, "text": None}
with patch.object(encoding_service, "_request_with_retry", new_callable=AsyncMock, return_value=mock_resp):
result = await encoding_service.submit_encoding_job("j1", "gs://in", "gs://out", {})
assert result["status"] == "in_progress"
@pytest.mark.asyncio
async def test_409_fallback_returns_cached_if_complete(self, encoding_service):
"""On 409, fall back to get_job_status(); return cached if complete."""
mock_resp = {"status": 409, "json": None, "text": "Job j1 already exists"}
mock_status = {"status": "complete", "output_files": ["a.mp4", "b.mp4"]}
with patch.object(encoding_service, "_request_with_retry", new_callable=AsyncMock, return_value=mock_resp), \
patch.object(encoding_service, "get_job_status", new_callable=AsyncMock, return_value=mock_status):
result = await encoding_service.submit_encoding_job("j1", "gs://in", "gs://out", {})
assert result["status"] == "cached"
assert result["output_files"] == ["a.mp4", "b.mp4"]
@pytest.mark.asyncio
async def test_409_fallback_returns_in_progress_if_running(self, encoding_service):
"""On 409, fall back to get_job_status(); return in_progress if running."""
mock_resp = {"status": 409, "json": None, "text": "Job j1 already exists"}
mock_status = {"status": "running", "progress": 42}
with patch.object(encoding_service, "_request_with_retry", new_callable=AsyncMock, return_value=mock_resp), \
patch.object(encoding_service, "get_job_status", new_callable=AsyncMock, return_value=mock_status):
result = await encoding_service.submit_encoding_job("j1", "gs://in", "gs://out", {})
assert result["status"] == "in_progress"
@pytest.mark.asyncio
async def test_409_fallback_returns_in_progress_if_pending(self, encoding_service):
"""On 409, fall back to get_job_status(); return in_progress if pending."""
mock_resp = {"status": 409, "json": None, "text": "Job j1 already exists"}
mock_status = {"status": "pending", "progress": 0}
with patch.object(encoding_service, "_request_with_retry", new_callable=AsyncMock, return_value=mock_resp), \
patch.object(encoding_service, "get_job_status", new_callable=AsyncMock, return_value=mock_status):
result = await encoding_service.submit_encoding_job("j1", "gs://in", "gs://out", {})
assert result["status"] == "in_progress"
@pytest.mark.asyncio
async def test_409_fallback_raises_if_failed(self, encoding_service):
"""On 409, fall back to get_job_status(); raise if status is failed."""
mock_resp = {"status": 409, "json": None, "text": "Job j1 already exists"}
mock_status = {"status": "failed", "error": "ffmpeg crash"}
with patch.object(encoding_service, "_request_with_retry", new_callable=AsyncMock, return_value=mock_resp), \
patch.object(encoding_service, "get_job_status", new_callable=AsyncMock, return_value=mock_status):
with pytest.raises(RuntimeError, match="already exists with status: failed"):
await encoding_service.submit_encoding_job("j1", "gs://in", "gs://out", {})
@pytest.mark.asyncio
async def test_409_fallback_raises_if_status_check_404(self, encoding_service):
"""On 409, if status check returns 404 (worker restarted), raise conflict error."""
mock_resp = {"status": 409, "json": None, "text": "Job j1 already exists"}
with patch.object(encoding_service, "_request_with_retry", new_callable=AsyncMock, return_value=mock_resp), \
patch.object(encoding_service, "get_job_status", new_callable=AsyncMock, side_effect=RuntimeError("Encoding job j1 not found")):
with pytest.raises(RuntimeError, match="conflict: 409 but job not found"):
await encoding_service.submit_encoding_job("j1", "gs://in", "gs://out", {})
class TestEncodeVideos:
"""Tests for encode_videos() handling of submit result status."""
@pytest.mark.asyncio
async def test_cached_submit_returns_immediately(self, encoding_service):
"""When submit returns cached, return immediately without polling."""
cached_result = {"status": "cached", "job_id": "j1", "output_files": ["a.mp4"]}
with patch.object(encoding_service, "submit_encoding_job", new_callable=AsyncMock, return_value=cached_result) as mock_submit, \
patch.object(encoding_service, "wait_for_completion", new_callable=AsyncMock) as mock_wait:
result = await encoding_service.encode_videos("j1", "gs://in", "gs://out")
assert result["status"] == "complete"
assert result["output_files"] == ["a.mp4"]
mock_submit.assert_called_once()
mock_wait.assert_not_called()
@pytest.mark.asyncio
async def test_in_progress_submit_falls_through_to_wait(self, encoding_service):
"""When submit returns in_progress, fall through to wait_for_completion."""
in_progress_result = {"status": "in_progress", "job_id": "j1"}
completed_result = {"status": "complete", "output_files": ["a.mp4"]}
with patch.object(encoding_service, "submit_encoding_job", new_callable=AsyncMock, return_value=in_progress_result), \
patch.object(encoding_service, "wait_for_completion", new_callable=AsyncMock, return_value=completed_result) as mock_wait:
result = await encoding_service.encode_videos("j1", "gs://in", "gs://out")
assert result["status"] == "complete"
mock_wait.assert_called_once()
@pytest.mark.asyncio
async def test_accepted_submit_falls_through_to_wait(self, encoding_service):
"""Normal accepted submission falls through to wait_for_completion."""
accepted_result = {"status": "accepted", "job_id": "j1"}
completed_result = {"status": "complete", "output_files": ["a.mp4"]}
with patch.object(encoding_service, "submit_encoding_job", new_callable=AsyncMock, return_value=accepted_result), \
patch.object(encoding_service, "wait_for_completion", new_callable=AsyncMock, return_value=completed_result) as mock_wait:
result = await encoding_service.encode_videos("j1", "gs://in", "gs://out")
assert result["status"] == "complete"
mock_wait.assert_called_once()
class TestRetryConfiguration:
"""Tests for deployment-resilient retry configuration."""
def test_retry_config_provides_sufficient_window(self):
"""Retry config must provide at least 60s of retry window for worker restarts.
A worker restart (download wheel, install, start uvicorn) takes 30-90s.
The retry window must exceed this to avoid failing jobs during deployments.
"""
# Calculate total retry window: sum of all backoff delays
total_wait = 0.0
backoff = INITIAL_BACKOFF_SECONDS
for _ in range(MAX_RETRIES):
total_wait += backoff
backoff = min(backoff * 2, MAX_BACKOFF_SECONDS)
assert total_wait >= 60.0, (
f"Retry window ({total_wait:.0f}s) is less than 60s minimum for worker restarts. "
f"Config: MAX_RETRIES={MAX_RETRIES}, INITIAL_BACKOFF={INITIAL_BACKOFF_SECONDS}s, "
f"MAX_BACKOFF={MAX_BACKOFF_SECONDS}s"
)
def test_retry_config_values(self):
"""Verify retry config matches documented values."""
assert MAX_RETRIES == 7
assert INITIAL_BACKOFF_SECONDS == 5.0
assert MAX_BACKOFF_SECONDS == 15.0
def test_poll_failure_tolerance_config(self):
"""Verify poll failure tolerance is set."""
assert MAX_CONSECUTIVE_POLL_FAILURES >= 3, (
"Must tolerate at least 3 poll failures for worker restart tolerance"
)
class TestRequestWithRetry:
"""Tests for _request_with_retry() retry behavior.
These tests patch at the aiohttp.ClientSession level to simulate connection
failures during worker restarts.
"""
@pytest.mark.asyncio
async def test_retries_on_connection_error(self, encoding_service):
"""Retries on aiohttp.ClientConnectorError and succeeds when worker comes back."""
call_count = 0
success_resp = {"status": 200, "json": {"status": "ok"}, "text": None}
original_request = encoding_service._request_with_retry
async def mock_request_with_retry(method, url, headers, json_payload=None, timeout=30.0, job_id="unknown", path=None):
nonlocal call_count
call_count += 1
if call_count == 1:
return success_resp
return success_resp
# Instead of fighting aiohttp mocking, test at the submit level
# which uses _request_with_retry internally. We test the retry logic
# by verifying the config values are correct for surviving restarts.
# The actual retry loop is exercised by integration tests.
with patch.object(encoding_service, "_request_with_retry", side_effect=mock_request_with_retry):
result = await encoding_service.submit_encoding_job("j1", "gs://in", "gs://out", {})
assert result["status"] == "ok"
@pytest.mark.asyncio
async def test_submit_propagates_connection_error(self, encoding_service):
"""Connection errors from _request_with_retry propagate to caller."""
async def fail_with_connection_error(*args, **kwargs):
raise aiohttp.ClientConnectorError(
connection_key=MagicMock(), os_error=OSError("Connection refused")
)
with patch.object(encoding_service, "_request_with_retry", side_effect=fail_with_connection_error):
with pytest.raises(aiohttp.ClientConnectorError):
await encoding_service.submit_encoding_job("j1", "gs://in", "gs://out", {})
@pytest.mark.asyncio
async def test_request_aborts_on_capacity_error_from_warmup(self, encoding_service):
"""When the warmup raises a capacity error, abort retries immediately.
Hammering the HTTP endpoint another 7 times costs ~7 min of wall clock
and produces no useful information — the VM literally cannot start.
The capacity error must surface to the worker so the job can be parked.
"""
from backend.services.encoding_errors import EncodingWorkerCapacityError
# Make ClientSession() raise synchronously when entered, so the very
# first attempt fails with TimeoutError without needing to mock the
# full aiohttp request lifecycle.
class TimingOutSession:
async def __aenter__(self):
raise asyncio.TimeoutError()
async def __aexit__(self, *a):
return False
async def warmup_raises_capacity(job_id):
raise EncodingWorkerCapacityError(
"ZONE_RESOURCE_POOL_EXHAUSTED",
vm_name="encoding-worker-b",
zone="us-central1-c",
code="ZONE_RESOURCE_POOL_EXHAUSTED",
)
with patch("aiohttp.ClientSession", return_value=TimingOutSession()), \
patch.object(
encoding_service,
"_warmup_encoding_worker_fallback",
side_effect=warmup_raises_capacity,
) as warmup_mock, \
patch("asyncio.sleep", new_callable=AsyncMock):
with pytest.raises(EncodingWorkerCapacityError):
await encoding_service._request_with_retry(
method="POST",
url="http://1.2.3.4:8080/encode",
headers={},
json_payload={},
timeout=1.0,
job_id="cap-test",
)
# Warmup is invoked exactly once (after the first failure) and its
# capacity error short-circuits the retry loop.
assert warmup_mock.await_count == 1
@pytest.mark.asyncio
async def test_url_reresolves_after_warmup_so_fallback_takes_effect(self, encoding_service):
"""After warmup updates active_override, retry attempts must hit the new URL.
Regression guard for the bug seen on job fae3eadc 2026-05-06: warmup
successfully routed to fallback-a in us-central1-a (set the active_
override, invalidated the URL cache), but the retry loop kept hitting
the original primary URL it captured before the loop started, all 8
attempts timed out, and the render worker failed the job hard with
TimeoutError() instead of letting the (working) fallback handle it.
"""
attempts_seen_urls = []
# Sequence of URLs returned by _get_worker_url():
# first call (used to build initial url): primary (dead)
# subsequent calls (after warmup invalidated cache): fallback (live)
url_sequence = iter([
"http://primary.dead:8080",
"http://fallback.live:8080",
"http://fallback.live:8080",
"http://fallback.live:8080",
])
# Track which URLs the loop actually hits
class FailingThenOkSession:
def __init__(self, url):
self.url = url
async def __aenter__(self):
attempts_seen_urls.append(self.url)
if "primary.dead" in self.url:
raise asyncio.TimeoutError()
# On fallback URL, return a fake successful response
resp = AsyncMock()
resp.status = 200
resp.json = AsyncMock(return_value={"ok": True})
resp.text = AsyncMock(return_value="ok")
return resp
async def __aexit__(self, *a):
return False
class FakeSession:
async def __aenter__(self):
return self
async def __aexit__(self, *a):
return False
def post(self, url, **kwargs):
return FailingThenOkSession(url)
async def warmup_noop(job_id):
return None # represents successful warmup that updated override
# Re-resolve only fires when worker_manager is wired (production mode).
encoding_service._worker_manager = MagicMock()
with patch.object(encoding_service, "_get_worker_url", side_effect=lambda: next(url_sequence)), \
patch("aiohttp.ClientSession", return_value=FakeSession()), \
patch.object(
encoding_service,
"_warmup_encoding_worker_fallback",
side_effect=warmup_noop,
), \
patch("asyncio.sleep", new_callable=AsyncMock):
initial_url = encoding_service._get_worker_url() + "/render-video"
result = await encoding_service._request_with_retry(
method="POST",
url=initial_url,
headers={},
json_payload={"job_id": "test"},
timeout=1.0,
job_id="test",
path="/render-video",
)
# First attempt hits primary (fails). Second attempt re-resolves URL
# post-warmup and hits fallback (succeeds).
assert result["status"] == 200
assert len(attempts_seen_urls) == 2
assert "primary.dead" in attempts_seen_urls[0]
assert "fallback.live" in attempts_seen_urls[1]
class TestWarmupFallback:
"""Tests for _warmup_encoding_worker_fallback() in the retry loop."""
@pytest.mark.asyncio
async def test_warmup_fallback_called_on_first_connection_failure(self, encoding_service):
"""Warmup fallback is called when the first connection attempt fails."""
mock_manager = MagicMock()
mock_manager.ensure_primary_running.return_value = {
"started": False, "vm_name": "encoding-worker-b", "primary_url": "http://1.2.3.4:8080"
}
encoding_service._worker_manager = mock_manager
await encoding_service._warmup_encoding_worker_fallback("test-job")
mock_manager.ensure_primary_running.assert_called_once()
@pytest.mark.asyncio
async def test_warmup_fallback_noop_without_worker_manager(self, encoding_service):
"""Warmup fallback is a no-op when worker_manager is not set (dev mode)."""
encoding_service._worker_manager = None
# Should not raise
await encoding_service._warmup_encoding_worker_fallback("test-job")
@pytest.mark.asyncio
async def test_warmup_fallback_swallows_unexpected_exceptions(self, encoding_service):
"""Generic warmup failures (e.g. transient Compute API blips) are logged but non-fatal.
Capacity errors are an exception — they propagate so the caller can park
the job for auto-retry. See `test_warmup_fallback_propagates_capacity_error`.
"""
mock_manager = MagicMock()
mock_manager.ensure_primary_running.side_effect = Exception("Compute API down")
encoding_service._worker_manager = mock_manager
# Should not raise
await encoding_service._warmup_encoding_worker_fallback("test-job")
@pytest.mark.asyncio
async def test_warmup_fallback_propagates_capacity_error(self, encoding_service):
"""Capacity errors must propagate so the job can be parked for auto-retry.
Without this, a ZONE_RESOURCE_POOL_EXHAUSTED hides behind the cold-start
timeout and surfaces 7+ minutes later as a useless `TimeoutError`.
"""
from backend.services.encoding_errors import EncodingWorkerCapacityError
mock_manager = MagicMock()
mock_manager.ensure_primary_running.side_effect = EncodingWorkerCapacityError(
"VM encoding-worker-b could not be started in us-central1-c: "
"ZONE_RESOURCE_POOL_EXHAUSTED — out of capacity",
vm_name="encoding-worker-b",
zone="us-central1-c",
code="ZONE_RESOURCE_POOL_EXHAUSTED",
)
encoding_service._worker_manager = mock_manager
with pytest.raises(EncodingWorkerCapacityError):
await encoding_service._warmup_encoding_worker_fallback("test-job")
@pytest.mark.asyncio
async def test_warmup_skips_readiness_wait_when_vm_already_running(self, encoding_service):
"""When started=False (deploy restart), DO NOT await readiness — fall back to fast retry."""
mock_manager = MagicMock()
mock_manager.ensure_primary_running.return_value = {
"started": False, "vm_name": "encoding-worker-b", "primary_url": "http://1.2.3.4:8080"
}
mock_manager.wait_for_worker_ready = AsyncMock()
encoding_service._worker_manager = mock_manager
await encoding_service._warmup_encoding_worker_fallback("test-job")
mock_manager.wait_for_worker_ready.assert_not_called()
@pytest.mark.asyncio
async def test_warmup_awaits_readiness_when_cold_started(self, encoding_service):
"""When started=True (VM was TERMINATED), AWAIT wait_for_worker_ready."""
mock_manager = MagicMock()
mock_manager.ensure_primary_running.return_value = {
"started": True, "vm_name": "encoding-worker-a", "primary_url": "http://1.2.3.4:8080"
}
mock_manager.wait_for_worker_ready = AsyncMock()
encoding_service._worker_manager = mock_manager
await encoding_service._warmup_encoding_worker_fallback("test-job")
mock_manager.wait_for_worker_ready.assert_awaited_once()
call = mock_manager.wait_for_worker_ready.call_args
assert call.args[0] == "encoding-worker-a"
assert call.args[1] == "http://1.2.3.4:8080/health"
# zone is passed (None for single-zone path; capacity-fallback paths
# pass the candidate's zone — see test_falls_through_on_generic_start_error_too)
assert "zone" in call.kwargs
@pytest.mark.asyncio
async def test_warmup_swallows_readiness_timeout(self, encoding_service):
"""If wait_for_worker_ready times out, log and return — main retry loop will surface it."""
mock_manager = MagicMock()
mock_manager.ensure_primary_running.return_value = {
"started": True, "vm_name": "encoding-worker-a", "primary_url": "http://1.2.3.4:8080"
}
mock_manager.wait_for_worker_ready = AsyncMock(side_effect=TimeoutError("VM stuck in STAGING"))
encoding_service._worker_manager = mock_manager
# Should not raise
await encoding_service._warmup_encoding_worker_fallback("test-job")
mock_manager.wait_for_worker_ready.assert_awaited_once()
@pytest.mark.asyncio
async def test_warmup_swallows_readiness_unexpected_error(self, encoding_service):
"""If wait_for_worker_ready raises a non-TimeoutError, log and return — never propagate."""
mock_manager = MagicMock()
mock_manager.ensure_primary_running.return_value = {
"started": True, "vm_name": "encoding-worker-a", "primary_url": "http://1.2.3.4:8080"
}
mock_manager.wait_for_worker_ready = AsyncMock(side_effect=RuntimeError("compute API error"))
encoding_service._worker_manager = mock_manager
# Should not raise
await encoding_service._warmup_encoding_worker_fallback("test-job")
mock_manager.wait_for_worker_ready.assert_awaited_once()
class TestColdStartIntegration:
"""End-to-end: first request fails, warmup awaits readiness, retry succeeds."""
@pytest.mark.asyncio
async def test_cold_start_recovery_no_retry_exhaustion(self, encoding_service):
"""
Simulates the 2026-04-24 incident path with the fix in place:
1. First HTTP attempt raises ClientConnectorError (VM was TERMINATED).
2. Warmup fallback runs, ensure_primary_running returns started=True.
3. wait_for_worker_ready resolves quickly (mocked).
4. Second HTTP attempt succeeds.
With the fix, no 8-retry exhaustion happens.
"""
# Mock worker manager
mock_manager = MagicMock()
mock_manager.ensure_primary_running.return_value = {
"started": True, "vm_name": "encoding-worker-a", "primary_url": "http://1.2.3.4:8080"
}
mock_manager.wait_for_worker_ready = AsyncMock()
encoding_service._worker_manager = mock_manager
# First call raises, second call succeeds
call_count = {"n": 0}
class _Resp:
def __init__(self, status, payload):
self.status = status
self._payload = payload
async def __aenter__(self): return self
async def __aexit__(self, *a): return None
async def json(self): return self._payload
async def text(self): return ""
class _Session:
async def __aenter__(self): return self
async def __aexit__(self, *a): return None
def post(self, *a, **kw):
call_count["n"] += 1
if call_count["n"] == 1:
raise aiohttp.ClientConnectorError(MagicMock(), OSError())
return _Resp(200, {"status": "accepted", "job_id": "j1"})
with patch("backend.services.encoding_service.aiohttp.ClientSession", return_value=_Session()), \
patch("backend.services.encoding_service.asyncio.sleep", new_callable=AsyncMock):
result = await encoding_service._request_with_retry(
"POST",
"http://1.2.3.4:8080/encode",
headers={},
json_payload={},
timeout=5.0,
job_id="j1",
)
assert result["status"] == 200
assert call_count["n"] == 2 # one fail, one success — no retry exhaustion
mock_manager.ensure_primary_running.assert_called_once()
mock_manager.wait_for_worker_ready.assert_awaited_once()
@pytest.mark.asyncio
async def test_warmup_only_runs_on_first_attempt(self, encoding_service):
"""
Regression guard: the `if attempt == 0` guard in _request_with_retry must
keep the warmup fallback from re-running on every retry. If someone
refactored that guard out, every retry would re-trigger the readiness
wait — wasteful and possibly buggy. This test pins the contract.
"""
mock_manager = MagicMock()
mock_manager.ensure_primary_running.return_value = {
"started": False, "vm_name": "encoding-worker-a", "primary_url": "http://1.2.3.4:8080"
}
mock_manager.wait_for_worker_ready = AsyncMock()
encoding_service._worker_manager = mock_manager
# All 3 calls fail — exhausts retries to confirm the guard holds across many attempts
call_count = {"n": 0}
class _Session:
async def __aenter__(self): return self
async def __aexit__(self, *a): return None
def post(self, *a, **kw):
call_count["n"] += 1
raise aiohttp.ClientConnectorError(MagicMock(), OSError())
with patch("backend.services.encoding_service.aiohttp.ClientSession", return_value=_Session()), \
patch("backend.services.encoding_service.asyncio.sleep", new_callable=AsyncMock), \
pytest.raises(aiohttp.ClientConnectorError):
await encoding_service._request_with_retry(
"POST",
"http://1.2.3.4:8080/encode",
headers={},
json_payload={},
timeout=5.0,
job_id="j1",
)
# All MAX_RETRIES + 1 attempts ran, but warmup fired exactly once.
assert call_count["n"] == MAX_RETRIES + 1
mock_manager.ensure_primary_running.assert_called_once()
class TestWaitForCompletionPollTolerance:
"""Tests for wait_for_completion() transient failure tolerance."""
@pytest.mark.asyncio
async def test_tolerates_transient_poll_failures(self, encoding_service):
"""Tolerates up to MAX_CONSECUTIVE_POLL_FAILURES-1 consecutive failures."""
call_count = 0
async def mock_get_status(job_id, worker_url=None):
nonlocal call_count
call_count += 1
if call_count <= 2:
raise aiohttp.ClientConnectorError(
connection_key=MagicMock(), os_error=OSError("Connection refused")
)
# Succeed on 3rd poll
return {"status": "complete", "output_files": ["a.mp4"]}
with patch.object(encoding_service, "get_job_status", side_effect=mock_get_status), \
patch("asyncio.sleep", new_callable=AsyncMock), \
patch("asyncio.get_event_loop") as mock_loop:
mock_loop.return_value.time.return_value = 0
result = await encoding_service.wait_for_completion("j1")
assert result["status"] == "complete"
assert call_count == 3
@pytest.mark.asyncio
async def test_fails_after_max_consecutive_poll_failures(self, encoding_service):
"""Fails after MAX_CONSECUTIVE_POLL_FAILURES consecutive failures."""
async def always_fail(job_id, worker_url=None):
raise aiohttp.ClientConnectorError(
connection_key=MagicMock(), os_error=OSError("Connection refused")
)
with patch.object(encoding_service, "get_job_status", side_effect=always_fail), \
patch("asyncio.sleep", new_callable=AsyncMock), \
patch("asyncio.get_event_loop") as mock_loop:
mock_loop.return_value.time.return_value = 0
with pytest.raises(RuntimeError, match="consecutive poll failures"):
await encoding_service.wait_for_completion("j1")
@pytest.mark.asyncio
async def test_resets_failure_counter_on_success(self, encoding_service):
"""A successful poll resets the consecutive failure counter."""
call_count = 0
async def intermittent_failures(job_id, worker_url=None):
nonlocal call_count
call_count += 1
# Fail for 2, succeed (running), fail for 2 more, succeed (complete)
if call_count in (1, 2):
raise aiohttp.ClientConnectorError(
connection_key=MagicMock(), os_error=OSError("Connection refused")
)
if call_count == 3:
return {"status": "running", "progress": 50}
if call_count in (4, 5):
raise aiohttp.ClientConnectorError(
connection_key=MagicMock(), os_error=OSError("Connection refused")
)
return {"status": "complete", "output_files": ["a.mp4"]}
with patch.object(encoding_service, "get_job_status", side_effect=intermittent_failures), \
patch("asyncio.sleep", new_callable=AsyncMock), \
patch("asyncio.get_event_loop") as mock_loop:
mock_loop.return_value.time.return_value = 0
result = await encoding_service.wait_for_completion("j1")
assert result["status"] == "complete"
assert call_count == 6 # 2 fail + 1 success + 2 fail + 1 success
class TestWaitForCompletionLostJob:
"""A worker restart (OOM/deploy) that wipes an in-flight job must surface a
distinct, recoverable signal instead of the generic poll-failure timeout."""
@pytest.mark.asyncio
async def test_404_after_job_seen_raises_lost_error(self, encoding_service):
"""Once we've seen the job run, a later 404 = the worker lost it → resubmit."""
call_count = 0
async def mock_get_status(job_id, worker_url=None):
nonlocal call_count
call_count += 1
if call_count == 1:
return {"status": "running", "progress": 30}
raise EncodingJobNotFoundError(f"Encoding job {job_id} not found")
with patch.object(encoding_service, "get_job_status", side_effect=mock_get_status), \
patch("asyncio.sleep", new_callable=AsyncMock), \
patch("asyncio.get_event_loop") as mock_loop:
mock_loop.return_value.time.return_value = 0
with pytest.raises(EncodingJobLostError):
await encoding_service.wait_for_completion("j1")
# Should bail on the FIRST 404, not burn the full poll tolerance.
assert call_count == 2
@pytest.mark.asyncio
async def test_404_before_job_seen_is_tolerated(self, encoding_service):
"""A 404 before the job is ever seen is a submit/poll race, not a lost job —
keep the transient tolerance (avoids false resubmits)."""
async def always_404(job_id, worker_url=None):
raise EncodingJobNotFoundError(f"Encoding job {job_id} not found")
with patch.object(encoding_service, "get_job_status", side_effect=always_404), \
patch("asyncio.sleep", new_callable=AsyncMock), \
patch("asyncio.get_event_loop") as mock_loop:
mock_loop.return_value.time.return_value = 0
# Falls through to the generic tolerance, NOT EncodingJobLostError.
with pytest.raises(RuntimeError, match="consecutive poll failures"):
await encoding_service.wait_for_completion("j1")
@pytest.mark.asyncio
async def test_restart_failure_code_raises_lost_error(self, encoding_service):
"""A terminal failure carrying the restart marker is recoverable → lost error."""
async def mock_get_status(job_id, worker_url=None):
return {
"status": "failed",
"error": "Encoding worker restarted mid-job",
"restart_failure_code": "encoding_worker_restart",
}
with patch.object(encoding_service, "get_job_status", side_effect=mock_get_status), \
patch("asyncio.sleep", new_callable=AsyncMock), \
patch("asyncio.get_event_loop") as mock_loop:
mock_loop.return_value.time.return_value = 0
with pytest.raises(EncodingJobLostError):
await encoding_service.wait_for_completion("j1")
@pytest.mark.asyncio
async def test_plain_failure_still_raises_runtimeerror(self, encoding_service):
"""A normal (non-restart) failure stays a hard RuntimeError, not a resubmit."""
async def mock_get_status(job_id, worker_url=None):
return {"status": "failed", "error": "ffmpeg exploded"}
with patch.object(encoding_service, "get_job_status", side_effect=mock_get_status), \
patch("asyncio.sleep", new_callable=AsyncMock), \
patch("asyncio.get_event_loop") as mock_loop:
mock_loop.return_value.time.return_value = 0
with pytest.raises(RuntimeError, match="ffmpeg exploded"):
await encoding_service.wait_for_completion("j1")
# And it must not be the recoverable subtype.
try:
await encoding_service.wait_for_completion("j1")
except EncodingJobLostError: # pragma: no cover
pytest.fail("plain failure should not be EncodingJobLostError")
except RuntimeError:
pass
@pytest.mark.asyncio
async def test_pending_does_not_count_toward_run_timeout(self, encoding_service):
"""Time spent queued (pending) is bounded by queue_timeout, not the per-run
timeout — a burst can wait in the serialized heavy queue and still succeed."""
statuses = [
{"status": "pending", "progress": 0, "queue_position": 3},
{"status": "pending", "progress": 0, "queue_position": 2},
{"status": "pending", "progress": 0, "queue_position": 1},
{"status": "running", "progress": 10},
{"status": "complete", "output_files": ["a.mp4"]},
]
idx = 0
async def mock_get_status(job_id, worker_url=None):
nonlocal idx
s = statuses[idx]
idx += 1
return s
# start_time read, then one read per loop iteration. Job is pending until
# t=300 (>> timeout=50) then runs and completes at t=310 (10s of run time).
seq = [0, 0, 100, 200, 300, 310]
def fake_time():
return seq.pop(0) if len(seq) > 1 else seq[0]
with patch.object(encoding_service, "get_job_status", side_effect=mock_get_status), \
patch("asyncio.sleep", new_callable=AsyncMock), \
patch("asyncio.get_event_loop") as mock_loop:
mock_loop.return_value.time.side_effect = fake_time
result = await encoding_service.wait_for_completion(
"j1", timeout=50, queue_timeout=1000
)
assert result["status"] == "complete"
class TestPreviewQueueTimeout:
"""A queued preview must not wait the long default queue_timeout — an
interactive user is waiting, so total wait is capped at the short timeout."""
@pytest.mark.asyncio
async def test_preview_caps_queue_timeout_to_timeout(self, encoding_service):
submit = AsyncMock(return_value={"status": "accepted", "job_id": "p1"})
wait = AsyncMock(return_value={"status": "complete"})
with patch.object(encoding_service, "submit_preview_encoding_job", submit), \
patch.object(encoding_service, "wait_for_completion", wait):
await encoding_service.encode_preview_video(
job_id="p1",
ass_gcs_path="gs://b/x.ass",
audio_gcs_path="gs://b/a.flac",
output_gcs_path="gs://b/out.mp4",
timeout=90.0,
)
_, kwargs = wait.call_args
assert kwargs["timeout"] == 90.0
assert kwargs["queue_timeout"] == 90.0
class TestRunWithLostJobResubmit:
"""The bounded resubmit wrapper used by the render + encode workers."""
@pytest.mark.asyncio
async def test_resubmits_with_fresh_id_then_succeeds(self):
seen_ids = []
async def op(job_id):
seen_ids.append(job_id)
if len(seen_ids) == 1:
raise EncodingJobLostError("lost", job_id=job_id)
return {"status": "complete"}
result = await run_with_lost_job_resubmit(op, "base123", max_resubmits=2)
assert result["status"] == "complete"
assert seen_ids[0] == "base123"
assert seen_ids[1].startswith("base123_retry_")
assert len(seen_ids) == 2
@pytest.mark.asyncio
async def test_gives_up_after_max_resubmits(self):
attempts = []
async def op(job_id):
attempts.append(job_id)
raise EncodingJobLostError("lost", job_id=job_id)
with pytest.raises(EncodingJobLostError):
await run_with_lost_job_resubmit(op, "base", max_resubmits=2)
# 1 initial + 2 resubmits = 3 attempts.
assert len(attempts) == 3
assert attempts[0] == "base"
assert all(a.startswith("base_retry_") for a in attempts[1:])
@pytest.mark.asyncio
async def test_other_errors_are_not_retried(self):
attempts = []
async def op(job_id):
attempts.append(job_id)
raise RuntimeError("some other failure")
with pytest.raises(RuntimeError, match="some other failure"):
await run_with_lost_job_resubmit(op, "base", max_resubmits=2)
assert len(attempts) == 1 # not retried
class TestDynamicURLResolution:
"""Tests that EncodingService reads URL from Firestore, not static config."""
def test_url_from_worker_manager(self):
"""Should read primary_url from worker manager."""
mock_manager = MagicMock()
mock_manager.get_config.return_value = MagicMock(
primary_url="http://34.1.2.3:8080",
active_url="http://34.1.2.3:8080",
)
service = EncodingService()
service._initialized = True
service._api_key = "test-key"
service.set_worker_manager(mock_manager)
url = service._get_worker_url()
assert url == "http://34.1.2.3:8080"
mock_manager.get_config.assert_called_once()
def test_url_caches_within_ttl(self):
"""Should cache URL and not re-read within TTL."""
mock_manager = MagicMock()
mock_manager.get_config.return_value = MagicMock(
primary_url="http://34.1.2.3:8080",
active_url="http://34.1.2.3:8080",
)
service = EncodingService()
service._initialized = True
service.set_worker_manager(mock_manager)
service._get_worker_url()
service._get_worker_url()
assert mock_manager.get_config.call_count == 1
def test_url_refreshes_after_ttl(self):
"""Should re-read URL after TTL expires."""
mock_manager = MagicMock()
mock_manager.get_config.return_value = MagicMock(
primary_url="http://34.1.2.3:8080",
active_url="http://34.1.2.3:8080",
)
service = EncodingService()
service._initialized = True
service.set_worker_manager(mock_manager)
service._URL_CACHE_TTL = 0 # Expire immediately
service._get_worker_url()
service._get_worker_url()
assert mock_manager.get_config.call_count == 2
def test_fallback_to_static_url_without_manager(self):
"""Should fall back to static URL when no worker manager set."""
service = EncodingService()
service._url = "http://static:8080"
service._api_key = "test-key"
service._initialized = True
url = service._get_worker_url()
assert url == "http://static:8080"
class TestFormatException:
"""Tests for the _format_exception helper used in retry logs."""
def test_renders_message_when_present(self):
from backend.services.encoding_service import _format_exception
e = RuntimeError("something broke")
assert _format_exception(e) == "RuntimeError: something broke"
def test_renders_type_only_when_message_empty(self):
"""aiohttp.ClientConnectorError often has empty str(e) — show type."""
from backend.services.encoding_service import _format_exception
class _SilentError(Exception):
def __str__(self):
return ""
assert _format_exception(_SilentError()) == "_SilentError"
def test_handles_real_aiohttp_connector_error(self):
from backend.services.encoding_service import _format_exception
e = aiohttp.ClientConnectorError(MagicMock(), OSError())
# Type name should always appear
assert "ClientConnectorError" in _format_exception(e)
class TestWorkerUrlPinning:
"""Status polls must stay pinned to the worker that ACCEPTED the job.
Regression for incident 2026-06-16 (job d3af33ae): a blue-green deploy
swapped the primary pointer mid-render, and because status polls re-resolved
`active_url` on every call, they migrated from the worker actually encoding
the job (old primary) to the freshly-swapped new primary — which 404'd
"Encoding job ... not found" and failed the render after 5 polls, even though
the original worker finished the encode fine.
"""
@pytest.mark.asyncio
async def test_get_job_status_targets_pinned_url(self, encoding_service):
"""A pinned poll hits the given worker_url, not active_url, and disables
failover (a pinned poll must never re-resolve or spin up a fallback VM —
a fresh VM cannot have the in-flight job)."""
# active_url would resolve to worker-b, but the job lives on worker-a.
captured = {}
async def fake_request(**kwargs):
captured.update(kwargs)
return {"status": 200, "json": {"status": "running", "progress": 40}, "text": None}
with patch.object(encoding_service, "_get_worker_url", return_value="http://worker-b:8080"), \
patch.object(encoding_service, "_request_with_retry", side_effect=fake_request):
result = await encoding_service.get_job_status("d3af33ae", worker_url="http://worker-a:8080")
assert result["status"] == "running"
assert captured["url"] == "http://worker-a:8080/status/d3af33ae"
assert captured["allow_failover"] is False
@pytest.mark.asyncio
async def test_get_job_status_without_pin_uses_active_url(self, encoding_service):
"""Regression: unpinned polls still resolve the current active_url."""
captured = {}
async def fake_request(**kwargs):
captured.update(kwargs)
return {"status": 200, "json": {"status": "running"}, "text": None}
with patch.object(encoding_service, "_get_worker_url", return_value="http://worker-b:8080"), \
patch.object(encoding_service, "_request_with_retry", side_effect=fake_request):
await encoding_service.get_job_status("d3af33ae")
assert captured["url"] == "http://worker-b:8080/status/d3af33ae"
# Unpinned keeps the existing failover/re-resolution behaviour.
assert captured.get("allow_failover", True) is True
@pytest.mark.asyncio
async def test_wait_for_completion_threads_pinned_url_into_polls(self, encoding_service):
"""wait_for_completion forwards its worker_url to every status poll."""
seen_urls = []
async def mock_get_status(job_id, worker_url=None):
seen_urls.append(worker_url)
return {"status": "complete", "output_files": ["a.mp4"]}
with patch.object(encoding_service, "get_job_status", side_effect=mock_get_status), \
patch("asyncio.sleep", new_callable=AsyncMock), \
patch("asyncio.get_event_loop") as mock_loop:
mock_loop.return_value.time.return_value = 0
result = await encoding_service.wait_for_completion(
"d3af33ae", worker_url="http://worker-a:8080"
)
assert result["status"] == "complete"
assert seen_urls == ["http://worker-a:8080"]
@pytest.mark.asyncio
async def test_request_with_retry_no_failover_when_disabled(self, encoding_service):
"""With allow_failover=False, a connection error must NOT trigger the
fallback-VM warmup nor URL re-resolution — it just retries the same URL
and surfaces the failure to the caller's own poll-tolerance loop."""
encoding_service.set_worker_manager(MagicMock())
warmup = AsyncMock()
with patch.object(encoding_service, "_warmup_encoding_worker_fallback", warmup), \
patch("asyncio.sleep", new_callable=AsyncMock), \
patch("aiohttp.ClientSession") as mock_session:
mock_session.side_effect = aiohttp.ClientConnectorError(
MagicMock(), OSError("Connection refused")
)
with pytest.raises(aiohttp.ClientConnectorError):
await encoding_service._request_with_retry(
method="GET",
url="http://worker-a:8080/status/j1",
headers={},
job_id="j1",
path=None,
allow_failover=False,
)
warmup.assert_not_called()
@pytest.mark.asyncio
async def test_render_video_keeps_polling_submission_worker_after_swap(self, encoding_service):
"""End-to-end: when active_url has swapped to a worker that 404s, the
render keeps polling the worker the job was submitted to and completes.
This FAILS before the fix (the poll follows active_url to worker-b and
gets "not found"), and PASSES after (the poll is pinned to worker-a)."""
encoding_service.set_worker_manager(MagicMock())
async def fake_submit(job_id, render_config):
return {"status": "accepted", "job_id": job_id}
async def fake_get_status(job_id, worker_url=None):
# worker-a owns the job; the post-swap active worker (worker-b, or an
# unpinned None) has never seen it.
if worker_url == "http://worker-a:8080":
return {"status": "complete", "output_files": ["4k.mp4"]}
raise RuntimeError(f"Encoding job {job_id} not found")
with patch.object(encoding_service, "_get_worker_url", return_value="http://worker-a:8080"), \
patch.object(encoding_service, "submit_render_video_job", side_effect=fake_submit), \
patch.object(encoding_service, "get_job_status", side_effect=fake_get_status), \
patch("asyncio.sleep", new_callable=AsyncMock), \
patch("asyncio.get_event_loop") as mock_loop:
mock_loop.return_value.time.return_value = 0
result = await encoding_service.render_video_on_gce("d3af33ae", {"foo": "bar"})
assert result["status"] == "complete"
assert result["output_files"] == ["4k.mp4"]
class TestBuildWorkerCandidates:
"""_build_worker_candidates ranks primary + fallbacks via the shared preference."""
def _config(self, capacity_state=None):
cfg = MagicMock()
cfg.primary_vm = "encoding-worker-a"
cfg.primary_ip = "10.0.0.1"
cfg.primary_machine_type = None # → defaults to c4d
cfg.capacity_state = capacity_state or {}
return cfg
def _service_with_fallbacks(self, encoding_service, fallbacks_json, capacity_state=None):
mgr = MagicMock()
mgr._zone = "us-central1-c"
mgr.get_config.return_value = self._config(capacity_state)
encoding_service._worker_manager = mgr
encoding_service.settings.encoding_worker_fallback_vms = fallbacks_json
return encoding_service
def test_primary_first_and_flagged(self, encoding_service):
svc = self._service_with_fallbacks(encoding_service, None)
cands = svc._build_worker_candidates()
assert cands[0].vm_name == "encoding-worker-a"
assert cands[0].is_primary is True
assert cands[0].machine_type == "c4d-highcpu-32"
def test_fallbacks_ranked_fastest_first(self, encoding_service):
import json
fallbacks = json.dumps([
{"vm": "encoding-worker-fallback-n2f", "zone": "us-central1-f", "ip": "10.0.0.5",
"machine_type": "n2-highcpu-32"},
{"vm": "encoding-worker-fallback-c4a", "zone": "us-central1-a", "ip": "10.0.0.6",
"machine_type": "c4-highcpu-32"},
])
svc = self._service_with_fallbacks(encoding_service, fallbacks)
order = [c.vm_name for c in svc._build_worker_candidates()]
# c4d primary, then c4 (faster), then n2.
assert order == ["encoding-worker-a", "encoding-worker-fallback-c4a",
"encoding-worker-fallback-n2f"]
def test_stocked_out_primary_demoted(self, encoding_service):
import json
from datetime import datetime, timezone
fallbacks = json.dumps([
{"vm": "encoding-worker-fallback-c4a", "zone": "us-central1-a", "ip": "10.0.0.6",
"machine_type": "c4-highcpu-32"},
])
# c4d@us-central1-c stocked out "now" → demoted below c4.
cap = {"c4d-highcpu-32@us-central1-c": datetime.now(timezone.utc).isoformat()}
svc = self._service_with_fallbacks(encoding_service, fallbacks, capacity_state=cap)
cands = svc._build_worker_candidates()
order = [c.vm_name for c in cands]
assert order == ["encoding-worker-fallback-c4a", "encoding-worker-a"]
# is_primary still correctly attached to the demoted c4d.
primary = [c for c in cands if c.is_primary][0]
assert primary.vm_name == "encoding-worker-a"
def test_legacy_fallback_without_machine_type_inferred(self, encoding_service):
import json
fallbacks = json.dumps([
{"vm": "encoding-worker-fallback-n2c", "zone": "us-central1-c", "ip": "10.0.0.7"},
])
svc = self._service_with_fallbacks(encoding_service, fallbacks)
cands = svc._build_worker_candidates()
n2 = [c for c in cands if "n2c" in c.vm_name][0]
assert n2.machine_type is None # not fabricated on the object…
# …but inference placed it after the c4d primary.
assert [c.vm_name for c in cands] == ["encoding-worker-a", "encoding-worker-fallback-n2c"]
def test_no_worker_manager_returns_empty(self, encoding_service):
encoding_service._worker_manager = None
assert encoding_service._build_worker_candidates() == []
def test_non_list_fallback_json_degrades_to_primary_only(self, encoding_service):
# A JSON scalar (e.g. "null") parses fine but must not blow up iteration.
svc = self._service_with_fallbacks(encoding_service, "null")
cands = svc._build_worker_candidates()
assert [c.vm_name for c in cands] == ["encoding-worker-a"]
def test_dict_fallback_json_degrades_to_primary_only(self, encoding_service):
svc = self._service_with_fallbacks(encoding_service, '{"vm": "x"}')
cands = svc._build_worker_candidates()
assert [c.vm_name for c in cands] == ["encoding-worker-a"]