mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
## Summary - Replaces "c4d primary + single n2 fallback family" with a **ranked pool of 6 x86_64 / 32-vCPU / ≥32 GB highcpu types** across 4 silicon lineages (c4d, c4, n4d, c2d, n2d, n2), selected **fastest-first** but **availability-aware** — a type that stocks out is demoted for a 15-min cooldown then re-probed. c4d stays the top pick whenever it can start; a render no longer stalls even if 2+ types are fully stocked out in us-central1. - One **pure preference module** (`encoding_worker_preference.ordered_candidates`) is now the single source of truth for candidate ranking, driving **both** runtime selection and deploy green selection so they can't drift. - Keeps the per-VM start/stop model (MIG deferred with documented triggers; no reservation). Full plan + rollout in `docs/archive/2026-08-15-encoding-instance-type-diversity-plan.md`. ## Changes - **New** `backend/services/encoding_worker_preference.py` — speed-rank + Firestore `capacity_state` cooldown; stdlib-only so `deploy_promote` imports it (with a shadow-proof by-path fallback). - **Runtime** (`encoding_service`, `encoding_worker_manager`): builder ranks primary + fallbacks; manager records/clears per-(type,zone) stockout; override routing now keys off an explicit `is_primary` flag instead of list position. - **Deploy** (`deploy_promote`, `ci.yml`): green selection uses the shared ranker + threads `capacity_state`. - **IaC** (`config.py`, tests): `FALLBACKS` broadened to 6 types (additive-only) with family-correct disk types (Titanium c4/c4d/n4/n4d = hyperdisk-balanced; c2d/n2/n2d = pd-balanced); secret schema gains optional `machine_type`. - **Packer** (`provision.sh`, `startup.sh`): bake full dep tree with **CPU-only torch** so fresh type VMs boot self-sufficient; build-time import verify. - **New** `benchmark_types.py` to measure per-type encode wall-time and refine the seeded `SPEED_RANK`. ## Testing - [x] Full backend unit suite: **3871 passed, 36 skipped** (emulator tests need external services) - [x] New/updated tests: preference module (16), capacity_state + `is_primary` manager tests, builder-ordering + non-list-JSON service tests, deploy ranking tests, infra disk-capability + pool-breadth invariants - [ ] Operator rollout (needs write creds): `pulumi up` (verify +8 create, 0 replace) → rebuild image (dep-bake) → grow fallback secret with `machine_type` → run `benchmark_types.py` ## Review - [x] Local CodeRabbit review completed (4 major findings fixed; cycle 2 clean) - [x] Review feedback addressed @coderabbitai ignore --- 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
1154 lines
52 KiB
Python
1154 lines
52 KiB
Python
"""
|
|
Tests for the GCE encoding service (encoding_service.py).
|
|
|
|
Tests resilience features: idempotent submission handling, cached/in_progress
|
|
responses, 409 fallback behavior, deployment retry tolerance, and poll failure handling.
|
|
"""
|
|
import asyncio
|
|
|
|
import aiohttp
|
|
import pytest
|
|
from unittest.mock import AsyncMock, patch, MagicMock
|
|
|
|
from backend.services.encoding_service import (
|
|
EncodingService,
|
|
MAX_RETRIES,
|
|
INITIAL_BACKOFF_SECONDS,
|
|
MAX_BACKOFF_SECONDS,
|
|
MAX_CONSECUTIVE_POLL_FAILURES,
|
|
run_with_lost_job_resubmit,
|
|
)
|
|
from backend.services.encoding_errors import EncodingJobLostError, EncodingJobNotFoundError
|
|
|
|
|
|
@pytest.fixture
|
|
def encoding_service():
|
|
"""Create an EncodingService with mocked credentials."""
|
|
service = EncodingService()
|
|
service._url = "http://fake-worker:8080"
|
|
service._api_key = "test-key"
|
|
service._initialized = True
|
|
return service
|
|
|
|
|
|
class TestSubmitEncodingJob:
|
|
"""Tests for submit_encoding_job() resilience."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_submit_returns_accepted(self, encoding_service):
|
|
"""Normal submission returns accepted response."""
|
|
mock_resp = {"status": 200, "json": {"status": "accepted", "job_id": "j1"}, "text": None}
|
|
with patch.object(encoding_service, "_request_with_retry", new_callable=AsyncMock, return_value=mock_resp):
|
|
result = await encoding_service.submit_encoding_job("j1", "gs://in", "gs://out", {})
|
|
assert result["status"] == "accepted"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_submit_returns_cached(self, encoding_service):
|
|
"""When worker returns cached (job already complete), pass through."""
|
|
mock_resp = {"status": 200, "json": {"status": "cached", "job_id": "j1", "output_files": ["a.mp4"]}, "text": None}
|
|
with patch.object(encoding_service, "_request_with_retry", new_callable=AsyncMock, return_value=mock_resp):
|
|
result = await encoding_service.submit_encoding_job("j1", "gs://in", "gs://out", {})
|
|
assert result["status"] == "cached"
|
|
assert result["output_files"] == ["a.mp4"]
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_submit_returns_in_progress(self, encoding_service):
|
|
"""When worker returns in_progress, pass through."""
|
|
mock_resp = {"status": 200, "json": {"status": "in_progress", "job_id": "j1"}, "text": None}
|
|
with patch.object(encoding_service, "_request_with_retry", new_callable=AsyncMock, return_value=mock_resp):
|
|
result = await encoding_service.submit_encoding_job("j1", "gs://in", "gs://out", {})
|
|
assert result["status"] == "in_progress"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_409_fallback_returns_cached_if_complete(self, encoding_service):
|
|
"""On 409, fall back to get_job_status(); return cached if complete."""
|
|
mock_resp = {"status": 409, "json": None, "text": "Job j1 already exists"}
|
|
mock_status = {"status": "complete", "output_files": ["a.mp4", "b.mp4"]}
|
|
|
|
with patch.object(encoding_service, "_request_with_retry", new_callable=AsyncMock, return_value=mock_resp), \
|
|
patch.object(encoding_service, "get_job_status", new_callable=AsyncMock, return_value=mock_status):
|
|
result = await encoding_service.submit_encoding_job("j1", "gs://in", "gs://out", {})
|
|
|
|
assert result["status"] == "cached"
|
|
assert result["output_files"] == ["a.mp4", "b.mp4"]
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_409_fallback_returns_in_progress_if_running(self, encoding_service):
|
|
"""On 409, fall back to get_job_status(); return in_progress if running."""
|
|
mock_resp = {"status": 409, "json": None, "text": "Job j1 already exists"}
|
|
mock_status = {"status": "running", "progress": 42}
|
|
|
|
with patch.object(encoding_service, "_request_with_retry", new_callable=AsyncMock, return_value=mock_resp), \
|
|
patch.object(encoding_service, "get_job_status", new_callable=AsyncMock, return_value=mock_status):
|
|
result = await encoding_service.submit_encoding_job("j1", "gs://in", "gs://out", {})
|
|
|
|
assert result["status"] == "in_progress"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_409_fallback_returns_in_progress_if_pending(self, encoding_service):
|
|
"""On 409, fall back to get_job_status(); return in_progress if pending."""
|
|
mock_resp = {"status": 409, "json": None, "text": "Job j1 already exists"}
|
|
mock_status = {"status": "pending", "progress": 0}
|
|
|
|
with patch.object(encoding_service, "_request_with_retry", new_callable=AsyncMock, return_value=mock_resp), \
|
|
patch.object(encoding_service, "get_job_status", new_callable=AsyncMock, return_value=mock_status):
|
|
result = await encoding_service.submit_encoding_job("j1", "gs://in", "gs://out", {})
|
|
|
|
assert result["status"] == "in_progress"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_409_fallback_raises_if_failed(self, encoding_service):
|
|
"""On 409, fall back to get_job_status(); raise if status is failed."""
|
|
mock_resp = {"status": 409, "json": None, "text": "Job j1 already exists"}
|
|
mock_status = {"status": "failed", "error": "ffmpeg crash"}
|
|
|
|
with patch.object(encoding_service, "_request_with_retry", new_callable=AsyncMock, return_value=mock_resp), \
|
|
patch.object(encoding_service, "get_job_status", new_callable=AsyncMock, return_value=mock_status):
|
|
with pytest.raises(RuntimeError, match="already exists with status: failed"):
|
|
await encoding_service.submit_encoding_job("j1", "gs://in", "gs://out", {})
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_409_fallback_raises_if_status_check_404(self, encoding_service):
|
|
"""On 409, if status check returns 404 (worker restarted), raise conflict error."""
|
|
mock_resp = {"status": 409, "json": None, "text": "Job j1 already exists"}
|
|
|
|
with patch.object(encoding_service, "_request_with_retry", new_callable=AsyncMock, return_value=mock_resp), \
|
|
patch.object(encoding_service, "get_job_status", new_callable=AsyncMock, side_effect=RuntimeError("Encoding job j1 not found")):
|
|
with pytest.raises(RuntimeError, match="conflict: 409 but job not found"):
|
|
await encoding_service.submit_encoding_job("j1", "gs://in", "gs://out", {})
|
|
|
|
|
|
class TestEncodeVideos:
|
|
"""Tests for encode_videos() handling of submit result status."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_cached_submit_returns_immediately(self, encoding_service):
|
|
"""When submit returns cached, return immediately without polling."""
|
|
cached_result = {"status": "cached", "job_id": "j1", "output_files": ["a.mp4"]}
|
|
|
|
with patch.object(encoding_service, "submit_encoding_job", new_callable=AsyncMock, return_value=cached_result) as mock_submit, \
|
|
patch.object(encoding_service, "wait_for_completion", new_callable=AsyncMock) as mock_wait:
|
|
result = await encoding_service.encode_videos("j1", "gs://in", "gs://out")
|
|
|
|
assert result["status"] == "complete"
|
|
assert result["output_files"] == ["a.mp4"]
|
|
mock_submit.assert_called_once()
|
|
mock_wait.assert_not_called()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_in_progress_submit_falls_through_to_wait(self, encoding_service):
|
|
"""When submit returns in_progress, fall through to wait_for_completion."""
|
|
in_progress_result = {"status": "in_progress", "job_id": "j1"}
|
|
completed_result = {"status": "complete", "output_files": ["a.mp4"]}
|
|
|
|
with patch.object(encoding_service, "submit_encoding_job", new_callable=AsyncMock, return_value=in_progress_result), \
|
|
patch.object(encoding_service, "wait_for_completion", new_callable=AsyncMock, return_value=completed_result) as mock_wait:
|
|
result = await encoding_service.encode_videos("j1", "gs://in", "gs://out")
|
|
|
|
assert result["status"] == "complete"
|
|
mock_wait.assert_called_once()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_accepted_submit_falls_through_to_wait(self, encoding_service):
|
|
"""Normal accepted submission falls through to wait_for_completion."""
|
|
accepted_result = {"status": "accepted", "job_id": "j1"}
|
|
completed_result = {"status": "complete", "output_files": ["a.mp4"]}
|
|
|
|
with patch.object(encoding_service, "submit_encoding_job", new_callable=AsyncMock, return_value=accepted_result), \
|
|
patch.object(encoding_service, "wait_for_completion", new_callable=AsyncMock, return_value=completed_result) as mock_wait:
|
|
result = await encoding_service.encode_videos("j1", "gs://in", "gs://out")
|
|
|
|
assert result["status"] == "complete"
|
|
mock_wait.assert_called_once()
|
|
|
|
|
|
class TestRetryConfiguration:
|
|
"""Tests for deployment-resilient retry configuration."""
|
|
|
|
def test_retry_config_provides_sufficient_window(self):
|
|
"""Retry config must provide at least 60s of retry window for worker restarts.
|
|
|
|
A worker restart (download wheel, install, start uvicorn) takes 30-90s.
|
|
The retry window must exceed this to avoid failing jobs during deployments.
|
|
"""
|
|
# Calculate total retry window: sum of all backoff delays
|
|
total_wait = 0.0
|
|
backoff = INITIAL_BACKOFF_SECONDS
|
|
for _ in range(MAX_RETRIES):
|
|
total_wait += backoff
|
|
backoff = min(backoff * 2, MAX_BACKOFF_SECONDS)
|
|
|
|
assert total_wait >= 60.0, (
|
|
f"Retry window ({total_wait:.0f}s) is less than 60s minimum for worker restarts. "
|
|
f"Config: MAX_RETRIES={MAX_RETRIES}, INITIAL_BACKOFF={INITIAL_BACKOFF_SECONDS}s, "
|
|
f"MAX_BACKOFF={MAX_BACKOFF_SECONDS}s"
|
|
)
|
|
|
|
def test_retry_config_values(self):
|
|
"""Verify retry config matches documented values."""
|
|
assert MAX_RETRIES == 7
|
|
assert INITIAL_BACKOFF_SECONDS == 5.0
|
|
assert MAX_BACKOFF_SECONDS == 15.0
|
|
|
|
def test_poll_failure_tolerance_config(self):
|
|
"""Verify poll failure tolerance is set."""
|
|
assert MAX_CONSECUTIVE_POLL_FAILURES >= 3, (
|
|
"Must tolerate at least 3 poll failures for worker restart tolerance"
|
|
)
|
|
|
|
|
|
class TestRequestWithRetry:
|
|
"""Tests for _request_with_retry() retry behavior.
|
|
|
|
These tests patch at the aiohttp.ClientSession level to simulate connection
|
|
failures during worker restarts.
|
|
"""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_retries_on_connection_error(self, encoding_service):
|
|
"""Retries on aiohttp.ClientConnectorError and succeeds when worker comes back."""
|
|
call_count = 0
|
|
success_resp = {"status": 200, "json": {"status": "ok"}, "text": None}
|
|
|
|
original_request = encoding_service._request_with_retry
|
|
|
|
async def mock_request_with_retry(method, url, headers, json_payload=None, timeout=30.0, job_id="unknown", path=None):
|
|
nonlocal call_count
|
|
call_count += 1
|
|
if call_count == 1:
|
|
return success_resp
|
|
return success_resp
|
|
|
|
# Instead of fighting aiohttp mocking, test at the submit level
|
|
# which uses _request_with_retry internally. We test the retry logic
|
|
# by verifying the config values are correct for surviving restarts.
|
|
# The actual retry loop is exercised by integration tests.
|
|
with patch.object(encoding_service, "_request_with_retry", side_effect=mock_request_with_retry):
|
|
result = await encoding_service.submit_encoding_job("j1", "gs://in", "gs://out", {})
|
|
|
|
assert result["status"] == "ok"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_submit_propagates_connection_error(self, encoding_service):
|
|
"""Connection errors from _request_with_retry propagate to caller."""
|
|
async def fail_with_connection_error(*args, **kwargs):
|
|
raise aiohttp.ClientConnectorError(
|
|
connection_key=MagicMock(), os_error=OSError("Connection refused")
|
|
)
|
|
|
|
with patch.object(encoding_service, "_request_with_retry", side_effect=fail_with_connection_error):
|
|
with pytest.raises(aiohttp.ClientConnectorError):
|
|
await encoding_service.submit_encoding_job("j1", "gs://in", "gs://out", {})
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_request_aborts_on_capacity_error_from_warmup(self, encoding_service):
|
|
"""When the warmup raises a capacity error, abort retries immediately.
|
|
|
|
Hammering the HTTP endpoint another 7 times costs ~7 min of wall clock
|
|
and produces no useful information — the VM literally cannot start.
|
|
The capacity error must surface to the worker so the job can be parked.
|
|
"""
|
|
from backend.services.encoding_errors import EncodingWorkerCapacityError
|
|
|
|
# Make ClientSession() raise synchronously when entered, so the very
|
|
# first attempt fails with TimeoutError without needing to mock the
|
|
# full aiohttp request lifecycle.
|
|
class TimingOutSession:
|
|
async def __aenter__(self):
|
|
raise asyncio.TimeoutError()
|
|
async def __aexit__(self, *a):
|
|
return False
|
|
|
|
async def warmup_raises_capacity(job_id):
|
|
raise EncodingWorkerCapacityError(
|
|
"ZONE_RESOURCE_POOL_EXHAUSTED",
|
|
vm_name="encoding-worker-b",
|
|
zone="us-central1-c",
|
|
code="ZONE_RESOURCE_POOL_EXHAUSTED",
|
|
)
|
|
|
|
with patch("aiohttp.ClientSession", return_value=TimingOutSession()), \
|
|
patch.object(
|
|
encoding_service,
|
|
"_warmup_encoding_worker_fallback",
|
|
side_effect=warmup_raises_capacity,
|
|
) as warmup_mock, \
|
|
patch("asyncio.sleep", new_callable=AsyncMock):
|
|
|
|
with pytest.raises(EncodingWorkerCapacityError):
|
|
await encoding_service._request_with_retry(
|
|
method="POST",
|
|
url="http://1.2.3.4:8080/encode",
|
|
headers={},
|
|
json_payload={},
|
|
timeout=1.0,
|
|
job_id="cap-test",
|
|
)
|
|
|
|
# Warmup is invoked exactly once (after the first failure) and its
|
|
# capacity error short-circuits the retry loop.
|
|
assert warmup_mock.await_count == 1
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_url_reresolves_after_warmup_so_fallback_takes_effect(self, encoding_service):
|
|
"""After warmup updates active_override, retry attempts must hit the new URL.
|
|
|
|
Regression guard for the bug seen on job fae3eadc 2026-05-06: warmup
|
|
successfully routed to fallback-a in us-central1-a (set the active_
|
|
override, invalidated the URL cache), but the retry loop kept hitting
|
|
the original primary URL it captured before the loop started, all 8
|
|
attempts timed out, and the render worker failed the job hard with
|
|
TimeoutError() instead of letting the (working) fallback handle it.
|
|
"""
|
|
attempts_seen_urls = []
|
|
|
|
# Sequence of URLs returned by _get_worker_url():
|
|
# first call (used to build initial url): primary (dead)
|
|
# subsequent calls (after warmup invalidated cache): fallback (live)
|
|
url_sequence = iter([
|
|
"http://primary.dead:8080",
|
|
"http://fallback.live:8080",
|
|
"http://fallback.live:8080",
|
|
"http://fallback.live:8080",
|
|
])
|
|
|
|
# Track which URLs the loop actually hits
|
|
class FailingThenOkSession:
|
|
def __init__(self, url):
|
|
self.url = url
|
|
async def __aenter__(self):
|
|
attempts_seen_urls.append(self.url)
|
|
if "primary.dead" in self.url:
|
|
raise asyncio.TimeoutError()
|
|
# On fallback URL, return a fake successful response
|
|
resp = AsyncMock()
|
|
resp.status = 200
|
|
resp.json = AsyncMock(return_value={"ok": True})
|
|
resp.text = AsyncMock(return_value="ok")
|
|
return resp
|
|
async def __aexit__(self, *a):
|
|
return False
|
|
|
|
class FakeSession:
|
|
async def __aenter__(self):
|
|
return self
|
|
async def __aexit__(self, *a):
|
|
return False
|
|
def post(self, url, **kwargs):
|
|
return FailingThenOkSession(url)
|
|
|
|
async def warmup_noop(job_id):
|
|
return None # represents successful warmup that updated override
|
|
|
|
# Re-resolve only fires when worker_manager is wired (production mode).
|
|
encoding_service._worker_manager = MagicMock()
|
|
|
|
with patch.object(encoding_service, "_get_worker_url", side_effect=lambda: next(url_sequence)), \
|
|
patch("aiohttp.ClientSession", return_value=FakeSession()), \
|
|
patch.object(
|
|
encoding_service,
|
|
"_warmup_encoding_worker_fallback",
|
|
side_effect=warmup_noop,
|
|
), \
|
|
patch("asyncio.sleep", new_callable=AsyncMock):
|
|
|
|
initial_url = encoding_service._get_worker_url() + "/render-video"
|
|
result = await encoding_service._request_with_retry(
|
|
method="POST",
|
|
url=initial_url,
|
|
headers={},
|
|
json_payload={"job_id": "test"},
|
|
timeout=1.0,
|
|
job_id="test",
|
|
path="/render-video",
|
|
)
|
|
|
|
# First attempt hits primary (fails). Second attempt re-resolves URL
|
|
# post-warmup and hits fallback (succeeds).
|
|
assert result["status"] == 200
|
|
assert len(attempts_seen_urls) == 2
|
|
assert "primary.dead" in attempts_seen_urls[0]
|
|
assert "fallback.live" in attempts_seen_urls[1]
|
|
|
|
|
|
class TestWarmupFallback:
|
|
"""Tests for _warmup_encoding_worker_fallback() in the retry loop."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_warmup_fallback_called_on_first_connection_failure(self, encoding_service):
|
|
"""Warmup fallback is called when the first connection attempt fails."""
|
|
mock_manager = MagicMock()
|
|
mock_manager.ensure_primary_running.return_value = {
|
|
"started": False, "vm_name": "encoding-worker-b", "primary_url": "http://1.2.3.4:8080"
|
|
}
|
|
encoding_service._worker_manager = mock_manager
|
|
|
|
await encoding_service._warmup_encoding_worker_fallback("test-job")
|
|
|
|
mock_manager.ensure_primary_running.assert_called_once()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_warmup_fallback_noop_without_worker_manager(self, encoding_service):
|
|
"""Warmup fallback is a no-op when worker_manager is not set (dev mode)."""
|
|
encoding_service._worker_manager = None
|
|
# Should not raise
|
|
await encoding_service._warmup_encoding_worker_fallback("test-job")
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_warmup_fallback_swallows_unexpected_exceptions(self, encoding_service):
|
|
"""Generic warmup failures (e.g. transient Compute API blips) are logged but non-fatal.
|
|
|
|
Capacity errors are an exception — they propagate so the caller can park
|
|
the job for auto-retry. See `test_warmup_fallback_propagates_capacity_error`.
|
|
"""
|
|
mock_manager = MagicMock()
|
|
mock_manager.ensure_primary_running.side_effect = Exception("Compute API down")
|
|
encoding_service._worker_manager = mock_manager
|
|
|
|
# Should not raise
|
|
await encoding_service._warmup_encoding_worker_fallback("test-job")
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_warmup_fallback_propagates_capacity_error(self, encoding_service):
|
|
"""Capacity errors must propagate so the job can be parked for auto-retry.
|
|
|
|
Without this, a ZONE_RESOURCE_POOL_EXHAUSTED hides behind the cold-start
|
|
timeout and surfaces 7+ minutes later as a useless `TimeoutError`.
|
|
"""
|
|
from backend.services.encoding_errors import EncodingWorkerCapacityError
|
|
|
|
mock_manager = MagicMock()
|
|
mock_manager.ensure_primary_running.side_effect = EncodingWorkerCapacityError(
|
|
"VM encoding-worker-b could not be started in us-central1-c: "
|
|
"ZONE_RESOURCE_POOL_EXHAUSTED — out of capacity",
|
|
vm_name="encoding-worker-b",
|
|
zone="us-central1-c",
|
|
code="ZONE_RESOURCE_POOL_EXHAUSTED",
|
|
)
|
|
encoding_service._worker_manager = mock_manager
|
|
|
|
with pytest.raises(EncodingWorkerCapacityError):
|
|
await encoding_service._warmup_encoding_worker_fallback("test-job")
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_warmup_skips_readiness_wait_when_vm_already_running(self, encoding_service):
|
|
"""When started=False (deploy restart), DO NOT await readiness — fall back to fast retry."""
|
|
mock_manager = MagicMock()
|
|
mock_manager.ensure_primary_running.return_value = {
|
|
"started": False, "vm_name": "encoding-worker-b", "primary_url": "http://1.2.3.4:8080"
|
|
}
|
|
mock_manager.wait_for_worker_ready = AsyncMock()
|
|
encoding_service._worker_manager = mock_manager
|
|
|
|
await encoding_service._warmup_encoding_worker_fallback("test-job")
|
|
|
|
mock_manager.wait_for_worker_ready.assert_not_called()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_warmup_awaits_readiness_when_cold_started(self, encoding_service):
|
|
"""When started=True (VM was TERMINATED), AWAIT wait_for_worker_ready."""
|
|
mock_manager = MagicMock()
|
|
mock_manager.ensure_primary_running.return_value = {
|
|
"started": True, "vm_name": "encoding-worker-a", "primary_url": "http://1.2.3.4:8080"
|
|
}
|
|
mock_manager.wait_for_worker_ready = AsyncMock()
|
|
encoding_service._worker_manager = mock_manager
|
|
|
|
await encoding_service._warmup_encoding_worker_fallback("test-job")
|
|
|
|
mock_manager.wait_for_worker_ready.assert_awaited_once()
|
|
call = mock_manager.wait_for_worker_ready.call_args
|
|
assert call.args[0] == "encoding-worker-a"
|
|
assert call.args[1] == "http://1.2.3.4:8080/health"
|
|
# zone is passed (None for single-zone path; capacity-fallback paths
|
|
# pass the candidate's zone — see test_falls_through_on_generic_start_error_too)
|
|
assert "zone" in call.kwargs
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_warmup_swallows_readiness_timeout(self, encoding_service):
|
|
"""If wait_for_worker_ready times out, log and return — main retry loop will surface it."""
|
|
mock_manager = MagicMock()
|
|
mock_manager.ensure_primary_running.return_value = {
|
|
"started": True, "vm_name": "encoding-worker-a", "primary_url": "http://1.2.3.4:8080"
|
|
}
|
|
mock_manager.wait_for_worker_ready = AsyncMock(side_effect=TimeoutError("VM stuck in STAGING"))
|
|
encoding_service._worker_manager = mock_manager
|
|
|
|
# Should not raise
|
|
await encoding_service._warmup_encoding_worker_fallback("test-job")
|
|
|
|
mock_manager.wait_for_worker_ready.assert_awaited_once()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_warmup_swallows_readiness_unexpected_error(self, encoding_service):
|
|
"""If wait_for_worker_ready raises a non-TimeoutError, log and return — never propagate."""
|
|
mock_manager = MagicMock()
|
|
mock_manager.ensure_primary_running.return_value = {
|
|
"started": True, "vm_name": "encoding-worker-a", "primary_url": "http://1.2.3.4:8080"
|
|
}
|
|
mock_manager.wait_for_worker_ready = AsyncMock(side_effect=RuntimeError("compute API error"))
|
|
encoding_service._worker_manager = mock_manager
|
|
|
|
# Should not raise
|
|
await encoding_service._warmup_encoding_worker_fallback("test-job")
|
|
|
|
mock_manager.wait_for_worker_ready.assert_awaited_once()
|
|
|
|
|
|
class TestColdStartIntegration:
|
|
"""End-to-end: first request fails, warmup awaits readiness, retry succeeds."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_cold_start_recovery_no_retry_exhaustion(self, encoding_service):
|
|
"""
|
|
Simulates the 2026-04-24 incident path with the fix in place:
|
|
1. First HTTP attempt raises ClientConnectorError (VM was TERMINATED).
|
|
2. Warmup fallback runs, ensure_primary_running returns started=True.
|
|
3. wait_for_worker_ready resolves quickly (mocked).
|
|
4. Second HTTP attempt succeeds.
|
|
With the fix, no 8-retry exhaustion happens.
|
|
"""
|
|
# Mock worker manager
|
|
mock_manager = MagicMock()
|
|
mock_manager.ensure_primary_running.return_value = {
|
|
"started": True, "vm_name": "encoding-worker-a", "primary_url": "http://1.2.3.4:8080"
|
|
}
|
|
mock_manager.wait_for_worker_ready = AsyncMock()
|
|
encoding_service._worker_manager = mock_manager
|
|
|
|
# First call raises, second call succeeds
|
|
call_count = {"n": 0}
|
|
|
|
class _Resp:
|
|
def __init__(self, status, payload):
|
|
self.status = status
|
|
self._payload = payload
|
|
async def __aenter__(self): return self
|
|
async def __aexit__(self, *a): return None
|
|
async def json(self): return self._payload
|
|
async def text(self): return ""
|
|
|
|
class _Session:
|
|
async def __aenter__(self): return self
|
|
async def __aexit__(self, *a): return None
|
|
def post(self, *a, **kw):
|
|
call_count["n"] += 1
|
|
if call_count["n"] == 1:
|
|
raise aiohttp.ClientConnectorError(MagicMock(), OSError())
|
|
return _Resp(200, {"status": "accepted", "job_id": "j1"})
|
|
|
|
with patch("backend.services.encoding_service.aiohttp.ClientSession", return_value=_Session()), \
|
|
patch("backend.services.encoding_service.asyncio.sleep", new_callable=AsyncMock):
|
|
result = await encoding_service._request_with_retry(
|
|
"POST",
|
|
"http://1.2.3.4:8080/encode",
|
|
headers={},
|
|
json_payload={},
|
|
timeout=5.0,
|
|
job_id="j1",
|
|
)
|
|
|
|
assert result["status"] == 200
|
|
assert call_count["n"] == 2 # one fail, one success — no retry exhaustion
|
|
mock_manager.ensure_primary_running.assert_called_once()
|
|
mock_manager.wait_for_worker_ready.assert_awaited_once()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_warmup_only_runs_on_first_attempt(self, encoding_service):
|
|
"""
|
|
Regression guard: the `if attempt == 0` guard in _request_with_retry must
|
|
keep the warmup fallback from re-running on every retry. If someone
|
|
refactored that guard out, every retry would re-trigger the readiness
|
|
wait — wasteful and possibly buggy. This test pins the contract.
|
|
"""
|
|
mock_manager = MagicMock()
|
|
mock_manager.ensure_primary_running.return_value = {
|
|
"started": False, "vm_name": "encoding-worker-a", "primary_url": "http://1.2.3.4:8080"
|
|
}
|
|
mock_manager.wait_for_worker_ready = AsyncMock()
|
|
encoding_service._worker_manager = mock_manager
|
|
|
|
# All 3 calls fail — exhausts retries to confirm the guard holds across many attempts
|
|
call_count = {"n": 0}
|
|
|
|
class _Session:
|
|
async def __aenter__(self): return self
|
|
async def __aexit__(self, *a): return None
|
|
def post(self, *a, **kw):
|
|
call_count["n"] += 1
|
|
raise aiohttp.ClientConnectorError(MagicMock(), OSError())
|
|
|
|
with patch("backend.services.encoding_service.aiohttp.ClientSession", return_value=_Session()), \
|
|
patch("backend.services.encoding_service.asyncio.sleep", new_callable=AsyncMock), \
|
|
pytest.raises(aiohttp.ClientConnectorError):
|
|
await encoding_service._request_with_retry(
|
|
"POST",
|
|
"http://1.2.3.4:8080/encode",
|
|
headers={},
|
|
json_payload={},
|
|
timeout=5.0,
|
|
job_id="j1",
|
|
)
|
|
|
|
# All MAX_RETRIES + 1 attempts ran, but warmup fired exactly once.
|
|
assert call_count["n"] == MAX_RETRIES + 1
|
|
mock_manager.ensure_primary_running.assert_called_once()
|
|
|
|
|
|
class TestWaitForCompletionPollTolerance:
|
|
"""Tests for wait_for_completion() transient failure tolerance."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_tolerates_transient_poll_failures(self, encoding_service):
|
|
"""Tolerates up to MAX_CONSECUTIVE_POLL_FAILURES-1 consecutive failures."""
|
|
call_count = 0
|
|
|
|
async def mock_get_status(job_id, worker_url=None):
|
|
nonlocal call_count
|
|
call_count += 1
|
|
if call_count <= 2:
|
|
raise aiohttp.ClientConnectorError(
|
|
connection_key=MagicMock(), os_error=OSError("Connection refused")
|
|
)
|
|
# Succeed on 3rd poll
|
|
return {"status": "complete", "output_files": ["a.mp4"]}
|
|
|
|
with patch.object(encoding_service, "get_job_status", side_effect=mock_get_status), \
|
|
patch("asyncio.sleep", new_callable=AsyncMock), \
|
|
patch("asyncio.get_event_loop") as mock_loop:
|
|
mock_loop.return_value.time.return_value = 0
|
|
result = await encoding_service.wait_for_completion("j1")
|
|
|
|
assert result["status"] == "complete"
|
|
assert call_count == 3
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fails_after_max_consecutive_poll_failures(self, encoding_service):
|
|
"""Fails after MAX_CONSECUTIVE_POLL_FAILURES consecutive failures."""
|
|
async def always_fail(job_id, worker_url=None):
|
|
raise aiohttp.ClientConnectorError(
|
|
connection_key=MagicMock(), os_error=OSError("Connection refused")
|
|
)
|
|
|
|
with patch.object(encoding_service, "get_job_status", side_effect=always_fail), \
|
|
patch("asyncio.sleep", new_callable=AsyncMock), \
|
|
patch("asyncio.get_event_loop") as mock_loop:
|
|
mock_loop.return_value.time.return_value = 0
|
|
with pytest.raises(RuntimeError, match="consecutive poll failures"):
|
|
await encoding_service.wait_for_completion("j1")
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_resets_failure_counter_on_success(self, encoding_service):
|
|
"""A successful poll resets the consecutive failure counter."""
|
|
call_count = 0
|
|
|
|
async def intermittent_failures(job_id, worker_url=None):
|
|
nonlocal call_count
|
|
call_count += 1
|
|
# Fail for 2, succeed (running), fail for 2 more, succeed (complete)
|
|
if call_count in (1, 2):
|
|
raise aiohttp.ClientConnectorError(
|
|
connection_key=MagicMock(), os_error=OSError("Connection refused")
|
|
)
|
|
if call_count == 3:
|
|
return {"status": "running", "progress": 50}
|
|
if call_count in (4, 5):
|
|
raise aiohttp.ClientConnectorError(
|
|
connection_key=MagicMock(), os_error=OSError("Connection refused")
|
|
)
|
|
return {"status": "complete", "output_files": ["a.mp4"]}
|
|
|
|
with patch.object(encoding_service, "get_job_status", side_effect=intermittent_failures), \
|
|
patch("asyncio.sleep", new_callable=AsyncMock), \
|
|
patch("asyncio.get_event_loop") as mock_loop:
|
|
mock_loop.return_value.time.return_value = 0
|
|
result = await encoding_service.wait_for_completion("j1")
|
|
|
|
assert result["status"] == "complete"
|
|
assert call_count == 6 # 2 fail + 1 success + 2 fail + 1 success
|
|
|
|
|
|
class TestWaitForCompletionLostJob:
|
|
"""A worker restart (OOM/deploy) that wipes an in-flight job must surface a
|
|
distinct, recoverable signal instead of the generic poll-failure timeout."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_404_after_job_seen_raises_lost_error(self, encoding_service):
|
|
"""Once we've seen the job run, a later 404 = the worker lost it → resubmit."""
|
|
call_count = 0
|
|
|
|
async def mock_get_status(job_id, worker_url=None):
|
|
nonlocal call_count
|
|
call_count += 1
|
|
if call_count == 1:
|
|
return {"status": "running", "progress": 30}
|
|
raise EncodingJobNotFoundError(f"Encoding job {job_id} not found")
|
|
|
|
with patch.object(encoding_service, "get_job_status", side_effect=mock_get_status), \
|
|
patch("asyncio.sleep", new_callable=AsyncMock), \
|
|
patch("asyncio.get_event_loop") as mock_loop:
|
|
mock_loop.return_value.time.return_value = 0
|
|
with pytest.raises(EncodingJobLostError):
|
|
await encoding_service.wait_for_completion("j1")
|
|
|
|
# Should bail on the FIRST 404, not burn the full poll tolerance.
|
|
assert call_count == 2
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_404_before_job_seen_is_tolerated(self, encoding_service):
|
|
"""A 404 before the job is ever seen is a submit/poll race, not a lost job —
|
|
keep the transient tolerance (avoids false resubmits)."""
|
|
async def always_404(job_id, worker_url=None):
|
|
raise EncodingJobNotFoundError(f"Encoding job {job_id} not found")
|
|
|
|
with patch.object(encoding_service, "get_job_status", side_effect=always_404), \
|
|
patch("asyncio.sleep", new_callable=AsyncMock), \
|
|
patch("asyncio.get_event_loop") as mock_loop:
|
|
mock_loop.return_value.time.return_value = 0
|
|
# Falls through to the generic tolerance, NOT EncodingJobLostError.
|
|
with pytest.raises(RuntimeError, match="consecutive poll failures"):
|
|
await encoding_service.wait_for_completion("j1")
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_restart_failure_code_raises_lost_error(self, encoding_service):
|
|
"""A terminal failure carrying the restart marker is recoverable → lost error."""
|
|
async def mock_get_status(job_id, worker_url=None):
|
|
return {
|
|
"status": "failed",
|
|
"error": "Encoding worker restarted mid-job",
|
|
"restart_failure_code": "encoding_worker_restart",
|
|
}
|
|
|
|
with patch.object(encoding_service, "get_job_status", side_effect=mock_get_status), \
|
|
patch("asyncio.sleep", new_callable=AsyncMock), \
|
|
patch("asyncio.get_event_loop") as mock_loop:
|
|
mock_loop.return_value.time.return_value = 0
|
|
with pytest.raises(EncodingJobLostError):
|
|
await encoding_service.wait_for_completion("j1")
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_plain_failure_still_raises_runtimeerror(self, encoding_service):
|
|
"""A normal (non-restart) failure stays a hard RuntimeError, not a resubmit."""
|
|
async def mock_get_status(job_id, worker_url=None):
|
|
return {"status": "failed", "error": "ffmpeg exploded"}
|
|
|
|
with patch.object(encoding_service, "get_job_status", side_effect=mock_get_status), \
|
|
patch("asyncio.sleep", new_callable=AsyncMock), \
|
|
patch("asyncio.get_event_loop") as mock_loop:
|
|
mock_loop.return_value.time.return_value = 0
|
|
with pytest.raises(RuntimeError, match="ffmpeg exploded"):
|
|
await encoding_service.wait_for_completion("j1")
|
|
# And it must not be the recoverable subtype.
|
|
try:
|
|
await encoding_service.wait_for_completion("j1")
|
|
except EncodingJobLostError: # pragma: no cover
|
|
pytest.fail("plain failure should not be EncodingJobLostError")
|
|
except RuntimeError:
|
|
pass
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_pending_does_not_count_toward_run_timeout(self, encoding_service):
|
|
"""Time spent queued (pending) is bounded by queue_timeout, not the per-run
|
|
timeout — a burst can wait in the serialized heavy queue and still succeed."""
|
|
statuses = [
|
|
{"status": "pending", "progress": 0, "queue_position": 3},
|
|
{"status": "pending", "progress": 0, "queue_position": 2},
|
|
{"status": "pending", "progress": 0, "queue_position": 1},
|
|
{"status": "running", "progress": 10},
|
|
{"status": "complete", "output_files": ["a.mp4"]},
|
|
]
|
|
idx = 0
|
|
|
|
async def mock_get_status(job_id, worker_url=None):
|
|
nonlocal idx
|
|
s = statuses[idx]
|
|
idx += 1
|
|
return s
|
|
|
|
# start_time read, then one read per loop iteration. Job is pending until
|
|
# t=300 (>> timeout=50) then runs and completes at t=310 (10s of run time).
|
|
seq = [0, 0, 100, 200, 300, 310]
|
|
|
|
def fake_time():
|
|
return seq.pop(0) if len(seq) > 1 else seq[0]
|
|
|
|
with patch.object(encoding_service, "get_job_status", side_effect=mock_get_status), \
|
|
patch("asyncio.sleep", new_callable=AsyncMock), \
|
|
patch("asyncio.get_event_loop") as mock_loop:
|
|
mock_loop.return_value.time.side_effect = fake_time
|
|
result = await encoding_service.wait_for_completion(
|
|
"j1", timeout=50, queue_timeout=1000
|
|
)
|
|
|
|
assert result["status"] == "complete"
|
|
|
|
|
|
class TestPreviewQueueTimeout:
|
|
"""A queued preview must not wait the long default queue_timeout — an
|
|
interactive user is waiting, so total wait is capped at the short timeout."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_preview_caps_queue_timeout_to_timeout(self, encoding_service):
|
|
submit = AsyncMock(return_value={"status": "accepted", "job_id": "p1"})
|
|
wait = AsyncMock(return_value={"status": "complete"})
|
|
with patch.object(encoding_service, "submit_preview_encoding_job", submit), \
|
|
patch.object(encoding_service, "wait_for_completion", wait):
|
|
await encoding_service.encode_preview_video(
|
|
job_id="p1",
|
|
ass_gcs_path="gs://b/x.ass",
|
|
audio_gcs_path="gs://b/a.flac",
|
|
output_gcs_path="gs://b/out.mp4",
|
|
timeout=90.0,
|
|
)
|
|
_, kwargs = wait.call_args
|
|
assert kwargs["timeout"] == 90.0
|
|
assert kwargs["queue_timeout"] == 90.0
|
|
|
|
|
|
class TestRunWithLostJobResubmit:
|
|
"""The bounded resubmit wrapper used by the render + encode workers."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_resubmits_with_fresh_id_then_succeeds(self):
|
|
seen_ids = []
|
|
|
|
async def op(job_id):
|
|
seen_ids.append(job_id)
|
|
if len(seen_ids) == 1:
|
|
raise EncodingJobLostError("lost", job_id=job_id)
|
|
return {"status": "complete"}
|
|
|
|
result = await run_with_lost_job_resubmit(op, "base123", max_resubmits=2)
|
|
|
|
assert result["status"] == "complete"
|
|
assert seen_ids[0] == "base123"
|
|
assert seen_ids[1].startswith("base123_retry_")
|
|
assert len(seen_ids) == 2
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_gives_up_after_max_resubmits(self):
|
|
attempts = []
|
|
|
|
async def op(job_id):
|
|
attempts.append(job_id)
|
|
raise EncodingJobLostError("lost", job_id=job_id)
|
|
|
|
with pytest.raises(EncodingJobLostError):
|
|
await run_with_lost_job_resubmit(op, "base", max_resubmits=2)
|
|
|
|
# 1 initial + 2 resubmits = 3 attempts.
|
|
assert len(attempts) == 3
|
|
assert attempts[0] == "base"
|
|
assert all(a.startswith("base_retry_") for a in attempts[1:])
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_other_errors_are_not_retried(self):
|
|
attempts = []
|
|
|
|
async def op(job_id):
|
|
attempts.append(job_id)
|
|
raise RuntimeError("some other failure")
|
|
|
|
with pytest.raises(RuntimeError, match="some other failure"):
|
|
await run_with_lost_job_resubmit(op, "base", max_resubmits=2)
|
|
|
|
assert len(attempts) == 1 # not retried
|
|
|
|
|
|
class TestDynamicURLResolution:
|
|
"""Tests that EncodingService reads URL from Firestore, not static config."""
|
|
|
|
def test_url_from_worker_manager(self):
|
|
"""Should read primary_url from worker manager."""
|
|
mock_manager = MagicMock()
|
|
mock_manager.get_config.return_value = MagicMock(
|
|
primary_url="http://34.1.2.3:8080",
|
|
active_url="http://34.1.2.3:8080",
|
|
)
|
|
service = EncodingService()
|
|
service._initialized = True
|
|
service._api_key = "test-key"
|
|
service.set_worker_manager(mock_manager)
|
|
|
|
url = service._get_worker_url()
|
|
assert url == "http://34.1.2.3:8080"
|
|
mock_manager.get_config.assert_called_once()
|
|
|
|
def test_url_caches_within_ttl(self):
|
|
"""Should cache URL and not re-read within TTL."""
|
|
mock_manager = MagicMock()
|
|
mock_manager.get_config.return_value = MagicMock(
|
|
primary_url="http://34.1.2.3:8080",
|
|
active_url="http://34.1.2.3:8080",
|
|
)
|
|
service = EncodingService()
|
|
service._initialized = True
|
|
service.set_worker_manager(mock_manager)
|
|
|
|
service._get_worker_url()
|
|
service._get_worker_url()
|
|
|
|
assert mock_manager.get_config.call_count == 1
|
|
|
|
def test_url_refreshes_after_ttl(self):
|
|
"""Should re-read URL after TTL expires."""
|
|
mock_manager = MagicMock()
|
|
mock_manager.get_config.return_value = MagicMock(
|
|
primary_url="http://34.1.2.3:8080",
|
|
active_url="http://34.1.2.3:8080",
|
|
)
|
|
service = EncodingService()
|
|
service._initialized = True
|
|
service.set_worker_manager(mock_manager)
|
|
service._URL_CACHE_TTL = 0 # Expire immediately
|
|
|
|
service._get_worker_url()
|
|
service._get_worker_url()
|
|
|
|
assert mock_manager.get_config.call_count == 2
|
|
|
|
def test_fallback_to_static_url_without_manager(self):
|
|
"""Should fall back to static URL when no worker manager set."""
|
|
service = EncodingService()
|
|
service._url = "http://static:8080"
|
|
service._api_key = "test-key"
|
|
service._initialized = True
|
|
|
|
url = service._get_worker_url()
|
|
assert url == "http://static:8080"
|
|
|
|
|
|
class TestFormatException:
|
|
"""Tests for the _format_exception helper used in retry logs."""
|
|
|
|
def test_renders_message_when_present(self):
|
|
from backend.services.encoding_service import _format_exception
|
|
e = RuntimeError("something broke")
|
|
assert _format_exception(e) == "RuntimeError: something broke"
|
|
|
|
def test_renders_type_only_when_message_empty(self):
|
|
"""aiohttp.ClientConnectorError often has empty str(e) — show type."""
|
|
from backend.services.encoding_service import _format_exception
|
|
|
|
class _SilentError(Exception):
|
|
def __str__(self):
|
|
return ""
|
|
|
|
assert _format_exception(_SilentError()) == "_SilentError"
|
|
|
|
def test_handles_real_aiohttp_connector_error(self):
|
|
from backend.services.encoding_service import _format_exception
|
|
e = aiohttp.ClientConnectorError(MagicMock(), OSError())
|
|
# Type name should always appear
|
|
assert "ClientConnectorError" in _format_exception(e)
|
|
|
|
|
|
class TestWorkerUrlPinning:
|
|
"""Status polls must stay pinned to the worker that ACCEPTED the job.
|
|
|
|
Regression for incident 2026-06-16 (job d3af33ae): a blue-green deploy
|
|
swapped the primary pointer mid-render, and because status polls re-resolved
|
|
`active_url` on every call, they migrated from the worker actually encoding
|
|
the job (old primary) to the freshly-swapped new primary — which 404'd
|
|
"Encoding job ... not found" and failed the render after 5 polls, even though
|
|
the original worker finished the encode fine.
|
|
"""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_job_status_targets_pinned_url(self, encoding_service):
|
|
"""A pinned poll hits the given worker_url, not active_url, and disables
|
|
failover (a pinned poll must never re-resolve or spin up a fallback VM —
|
|
a fresh VM cannot have the in-flight job)."""
|
|
# active_url would resolve to worker-b, but the job lives on worker-a.
|
|
captured = {}
|
|
|
|
async def fake_request(**kwargs):
|
|
captured.update(kwargs)
|
|
return {"status": 200, "json": {"status": "running", "progress": 40}, "text": None}
|
|
|
|
with patch.object(encoding_service, "_get_worker_url", return_value="http://worker-b:8080"), \
|
|
patch.object(encoding_service, "_request_with_retry", side_effect=fake_request):
|
|
result = await encoding_service.get_job_status("d3af33ae", worker_url="http://worker-a:8080")
|
|
|
|
assert result["status"] == "running"
|
|
assert captured["url"] == "http://worker-a:8080/status/d3af33ae"
|
|
assert captured["allow_failover"] is False
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_job_status_without_pin_uses_active_url(self, encoding_service):
|
|
"""Regression: unpinned polls still resolve the current active_url."""
|
|
captured = {}
|
|
|
|
async def fake_request(**kwargs):
|
|
captured.update(kwargs)
|
|
return {"status": 200, "json": {"status": "running"}, "text": None}
|
|
|
|
with patch.object(encoding_service, "_get_worker_url", return_value="http://worker-b:8080"), \
|
|
patch.object(encoding_service, "_request_with_retry", side_effect=fake_request):
|
|
await encoding_service.get_job_status("d3af33ae")
|
|
|
|
assert captured["url"] == "http://worker-b:8080/status/d3af33ae"
|
|
# Unpinned keeps the existing failover/re-resolution behaviour.
|
|
assert captured.get("allow_failover", True) is True
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_wait_for_completion_threads_pinned_url_into_polls(self, encoding_service):
|
|
"""wait_for_completion forwards its worker_url to every status poll."""
|
|
seen_urls = []
|
|
|
|
async def mock_get_status(job_id, worker_url=None):
|
|
seen_urls.append(worker_url)
|
|
return {"status": "complete", "output_files": ["a.mp4"]}
|
|
|
|
with patch.object(encoding_service, "get_job_status", side_effect=mock_get_status), \
|
|
patch("asyncio.sleep", new_callable=AsyncMock), \
|
|
patch("asyncio.get_event_loop") as mock_loop:
|
|
mock_loop.return_value.time.return_value = 0
|
|
result = await encoding_service.wait_for_completion(
|
|
"d3af33ae", worker_url="http://worker-a:8080"
|
|
)
|
|
|
|
assert result["status"] == "complete"
|
|
assert seen_urls == ["http://worker-a:8080"]
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_request_with_retry_no_failover_when_disabled(self, encoding_service):
|
|
"""With allow_failover=False, a connection error must NOT trigger the
|
|
fallback-VM warmup nor URL re-resolution — it just retries the same URL
|
|
and surfaces the failure to the caller's own poll-tolerance loop."""
|
|
encoding_service.set_worker_manager(MagicMock())
|
|
warmup = AsyncMock()
|
|
|
|
with patch.object(encoding_service, "_warmup_encoding_worker_fallback", warmup), \
|
|
patch("asyncio.sleep", new_callable=AsyncMock), \
|
|
patch("aiohttp.ClientSession") as mock_session:
|
|
mock_session.side_effect = aiohttp.ClientConnectorError(
|
|
MagicMock(), OSError("Connection refused")
|
|
)
|
|
with pytest.raises(aiohttp.ClientConnectorError):
|
|
await encoding_service._request_with_retry(
|
|
method="GET",
|
|
url="http://worker-a:8080/status/j1",
|
|
headers={},
|
|
job_id="j1",
|
|
path=None,
|
|
allow_failover=False,
|
|
)
|
|
|
|
warmup.assert_not_called()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_render_video_keeps_polling_submission_worker_after_swap(self, encoding_service):
|
|
"""End-to-end: when active_url has swapped to a worker that 404s, the
|
|
render keeps polling the worker the job was submitted to and completes.
|
|
|
|
This FAILS before the fix (the poll follows active_url to worker-b and
|
|
gets "not found"), and PASSES after (the poll is pinned to worker-a)."""
|
|
encoding_service.set_worker_manager(MagicMock())
|
|
|
|
async def fake_submit(job_id, render_config):
|
|
return {"status": "accepted", "job_id": job_id}
|
|
|
|
async def fake_get_status(job_id, worker_url=None):
|
|
# worker-a owns the job; the post-swap active worker (worker-b, or an
|
|
# unpinned None) has never seen it.
|
|
if worker_url == "http://worker-a:8080":
|
|
return {"status": "complete", "output_files": ["4k.mp4"]}
|
|
raise RuntimeError(f"Encoding job {job_id} not found")
|
|
|
|
with patch.object(encoding_service, "_get_worker_url", return_value="http://worker-a:8080"), \
|
|
patch.object(encoding_service, "submit_render_video_job", side_effect=fake_submit), \
|
|
patch.object(encoding_service, "get_job_status", side_effect=fake_get_status), \
|
|
patch("asyncio.sleep", new_callable=AsyncMock), \
|
|
patch("asyncio.get_event_loop") as mock_loop:
|
|
mock_loop.return_value.time.return_value = 0
|
|
result = await encoding_service.render_video_on_gce("d3af33ae", {"foo": "bar"})
|
|
|
|
assert result["status"] == "complete"
|
|
assert result["output_files"] == ["4k.mp4"]
|
|
|
|
|
|
class TestBuildWorkerCandidates:
|
|
"""_build_worker_candidates ranks primary + fallbacks via the shared preference."""
|
|
|
|
def _config(self, capacity_state=None):
|
|
cfg = MagicMock()
|
|
cfg.primary_vm = "encoding-worker-a"
|
|
cfg.primary_ip = "10.0.0.1"
|
|
cfg.primary_machine_type = None # → defaults to c4d
|
|
cfg.capacity_state = capacity_state or {}
|
|
return cfg
|
|
|
|
def _service_with_fallbacks(self, encoding_service, fallbacks_json, capacity_state=None):
|
|
mgr = MagicMock()
|
|
mgr._zone = "us-central1-c"
|
|
mgr.get_config.return_value = self._config(capacity_state)
|
|
encoding_service._worker_manager = mgr
|
|
encoding_service.settings.encoding_worker_fallback_vms = fallbacks_json
|
|
return encoding_service
|
|
|
|
def test_primary_first_and_flagged(self, encoding_service):
|
|
svc = self._service_with_fallbacks(encoding_service, None)
|
|
cands = svc._build_worker_candidates()
|
|
assert cands[0].vm_name == "encoding-worker-a"
|
|
assert cands[0].is_primary is True
|
|
assert cands[0].machine_type == "c4d-highcpu-32"
|
|
|
|
def test_fallbacks_ranked_fastest_first(self, encoding_service):
|
|
import json
|
|
fallbacks = json.dumps([
|
|
{"vm": "encoding-worker-fallback-n2f", "zone": "us-central1-f", "ip": "10.0.0.5",
|
|
"machine_type": "n2-highcpu-32"},
|
|
{"vm": "encoding-worker-fallback-c4a", "zone": "us-central1-a", "ip": "10.0.0.6",
|
|
"machine_type": "c4-highcpu-32"},
|
|
])
|
|
svc = self._service_with_fallbacks(encoding_service, fallbacks)
|
|
order = [c.vm_name for c in svc._build_worker_candidates()]
|
|
# c4d primary, then c4 (faster), then n2.
|
|
assert order == ["encoding-worker-a", "encoding-worker-fallback-c4a",
|
|
"encoding-worker-fallback-n2f"]
|
|
|
|
def test_stocked_out_primary_demoted(self, encoding_service):
|
|
import json
|
|
from datetime import datetime, timezone
|
|
fallbacks = json.dumps([
|
|
{"vm": "encoding-worker-fallback-c4a", "zone": "us-central1-a", "ip": "10.0.0.6",
|
|
"machine_type": "c4-highcpu-32"},
|
|
])
|
|
# c4d@us-central1-c stocked out "now" → demoted below c4.
|
|
cap = {"c4d-highcpu-32@us-central1-c": datetime.now(timezone.utc).isoformat()}
|
|
svc = self._service_with_fallbacks(encoding_service, fallbacks, capacity_state=cap)
|
|
cands = svc._build_worker_candidates()
|
|
order = [c.vm_name for c in cands]
|
|
assert order == ["encoding-worker-fallback-c4a", "encoding-worker-a"]
|
|
# is_primary still correctly attached to the demoted c4d.
|
|
primary = [c for c in cands if c.is_primary][0]
|
|
assert primary.vm_name == "encoding-worker-a"
|
|
|
|
def test_legacy_fallback_without_machine_type_inferred(self, encoding_service):
|
|
import json
|
|
fallbacks = json.dumps([
|
|
{"vm": "encoding-worker-fallback-n2c", "zone": "us-central1-c", "ip": "10.0.0.7"},
|
|
])
|
|
svc = self._service_with_fallbacks(encoding_service, fallbacks)
|
|
cands = svc._build_worker_candidates()
|
|
n2 = [c for c in cands if "n2c" in c.vm_name][0]
|
|
assert n2.machine_type is None # not fabricated on the object…
|
|
# …but inference placed it after the c4d primary.
|
|
assert [c.vm_name for c in cands] == ["encoding-worker-a", "encoding-worker-fallback-n2c"]
|
|
|
|
def test_no_worker_manager_returns_empty(self, encoding_service):
|
|
encoding_service._worker_manager = None
|
|
assert encoding_service._build_worker_candidates() == []
|
|
|
|
def test_non_list_fallback_json_degrades_to_primary_only(self, encoding_service):
|
|
# A JSON scalar (e.g. "null") parses fine but must not blow up iteration.
|
|
svc = self._service_with_fallbacks(encoding_service, "null")
|
|
cands = svc._build_worker_candidates()
|
|
assert [c.vm_name for c in cands] == ["encoding-worker-a"]
|
|
|
|
def test_dict_fallback_json_degrades_to_primary_only(self, encoding_service):
|
|
svc = self._service_with_fallbacks(encoding_service, '{"vm": "x"}')
|
|
cands = svc._build_worker_candidates()
|
|
assert [c.vm_name for c in cands] == ["encoding-worker-a"]
|