Files
karaoke-gen/backend/tests/test_encoding_contract.py
Andrew Beveridge ☄️andClaude Opus 4.8 ed7c6006be Incident hardening: encode→publish seam test + typed contract + guard discipline (T1/P3/P1/P2) (#986)
Part 2 of 3 of the **Incident hardening (post-NOMAD-1632)** initiative
(plan: `docs/archive/2026-09-12-incident-hardening-plan.md`). Both
incident bugs lived in the **encode→publish seam**, yet no test composed
the *real* `GCEEncodingBackend.encode()` with the *real* orchestrator —
so green mocked unit tests gave false confidence in both directions.

## T1 — composed encode→publish seam integration test
New `TestEncodePublishSeam` in
`backend/tests/test_video_worker_orchestrator.py` runs the **real**
backend through the **real** orchestrator `_run_encoding`, patching only
the network boundary (`encode_videos`) and storage. Asserts all four
shapes:
- **empty** `output_files` → completeness guard does **NOT** trip →
orchestrator downloads 0 → raises "stale" → **re-encodes** and recovers
(the Failure-B path the original fix broke);
- **partial** (720p missing) → guard trips → job fails loud (the
Failure-A defense);
- **complete** → success, all result attrs populated (uses the
NOMAD-1632 "Portrait of Jennie" title so the 720p classifies correctly);
- **malformed** (stray/wrong-extension names) → classified out, treated
as empty (no mis-slotting, no crash).

## P3 — kill mock drift: fixtures + typed contract
- Golden fixtures under `backend/tests/fixtures/encoding/`
(`success`/`cached`/`empty`/`partial`/`malformed`) + README.
- Typed `GceEncodeResponse` (TypedDict) in `encoding_interface.py`,
shared by producer (`EncodingService.encode_videos`) and consumer
(`GCEEncodingBackend.encode`) — type-only import, no runtime cost/cycle.
- `backend/tests/test_encoding_contract.py` replays every fixture
through `classify_encoded_output` / `encode()` and validates the schema.
The "empty" shape that hid Failure B is now in the corpus.

## P1/P2 — process docs
`docs/TESTING.md`: **"New Guard / New Enforcement Checklist"**
(recoverable ≠ wrong, fail-open on uncertainty, test
empty/partial/malformed/complete, shadow-first, composed integration
test) and a **"Critical-Path Change Tier"** note + blameless post-mortem
template.

## Tests
`138 passed` locally — new seam tests + contract tests + existing
orchestrator/interface regression. No behavior change to app code (typed
annotation only).

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
2026-09-12 21:48:55 +00:00

152 lines
6.2 KiB
Python

"""Contract test: replay golden GCE worker responses through the real encode path.
This is the "kill mock drift" backstop (hardening plan P3). Every encoding mock
in the suite is hand-authored, and the real "cached / empty" response shape was
simply never in the corpus — the textbook cause of NOMAD-1632's Failure B. Here
we capture the *real* worker response shapes as golden JSON fixtures
(``backend/tests/fixtures/encoding/``) and replay them through the real
``classify_encoded_output`` and ``GCEEncodingBackend.encode()`` so the classifier
and the completeness guard are exercised against the actual contract.
Kept lightweight and deterministic: no network, no GCS, no filesystem writes —
the only boundary patched is the worker call (``encode_videos``).
"""
import json
from pathlib import Path
import pytest
from unittest.mock import AsyncMock, MagicMock, patch
from backend.services.encoding_interface import (
GceEncodeResponse,
GCEEncodingBackend,
EncodingInput,
classify_encoded_output,
)
FIXTURES_DIR = Path(__file__).parent / "fixtures" / "encoding"
# The four formats GCEEncodingBackend.encode() always requests (and the guard
# enforces). Keep in sync with encoding_interface.encode().
REQUESTED_FORMATS = {"mp4_4k_lossless", "mp4_4k_lossy", "mkv_4k", "mp4_720p"}
def _load(name: str) -> GceEncodeResponse:
with open(FIXTURES_DIR / f"{name}.json") as fh:
return json.load(fh)
async def _run_encode(response: GceEncodeResponse):
"""Drive the REAL encode() with a mocked worker returning ``response``."""
backend = GCEEncodingBackend(logger=MagicMock())
mock_service = MagicMock()
mock_service.encode_videos = AsyncMock(return_value=response)
input_config = EncodingInput(
title_video_path="/tmp/title.mov",
karaoke_video_path="/tmp/karaoke.mkv",
instrumental_audio_path="/tmp/audio.flac",
artist="Nat King Cole",
title="Portrait of Jennie",
options={
"job_id": "NOMAD-1632",
"input_gcs_path": "gs://bucket/jobs/NOMAD-1632/",
"output_gcs_path": "gs://bucket/jobs/NOMAD-1632/finals/",
},
)
with patch(
"backend.services.encoding_service.get_encoding_service",
return_value=mock_service,
):
return await backend.encode(input_config)
class TestEncodingResponseContract:
"""Replay golden fixtures through the real classifier + encode()."""
ALL_FIXTURES = ["success", "cached", "empty", "partial", "malformed"]
@pytest.mark.parametrize("name", ALL_FIXTURES)
def test_fixture_conforms_to_schema(self, name):
"""Every fixture matches the GceEncodeResponse contract keys/types."""
data = _load(name)
assert set(data).issubset(set(GceEncodeResponse.__annotations__)), (
f"{name}.json has keys outside GceEncodeResponse"
)
assert isinstance(data["status"], str)
assert isinstance(data["output_files"], list)
assert all(isinstance(p, str) for p in data["output_files"])
def test_classify_success_fixture(self):
"""NOMAD-1632 regression: 'Portrait of Jennie' 720p must NOT misclassify
as the portrait video — classification keys off the trailing tag."""
files = _load("success")["output_files"]
classified = {classify_encoded_output(p) for p in files}
assert classified == REQUESTED_FORMATS
# The 720p whose TITLE contains 'Portrait' maps to mp4_720p, not portrait_mp4.
seven20 = next(p for p in files if "720p" in p)
assert classify_encoded_output(seven20) == "mp4_720p"
def test_classify_cached_fixture_covers_all_branches(self):
"""The cached full-set fixture exercises every classifier branch."""
files = _load("cached")["output_files"]
classified = {classify_encoded_output(p) for p in files}
assert classified == REQUESTED_FORMATS | {
"portrait_mp4",
"with_vocals_mp4",
"title_mov",
"end_mov",
"cdg_zip",
}
def test_classify_malformed_fixture_classified_out(self):
"""Stray / wrong-extension names classify to None (no false slotting)."""
files = _load("malformed")["output_files"]
assert all(classify_encoded_output(p) is None for p in files)
# Specifically: a 720p tag with the wrong extension must be rejected.
wrong_ext = next(p for p in files if p.endswith(".webm"))
assert classify_encoded_output(wrong_ext) is None
@pytest.mark.asyncio
async def test_encode_success_fixture(self):
out = await _run_encode(_load("success"))
assert out.success is True
assert out.lossy_720p_mp4_path is not None
assert set(out.output_files) == REQUESTED_FORMATS
@pytest.mark.asyncio
async def test_encode_cached_fixture(self):
out = await _run_encode(_load("cached"))
assert out.success is True
assert out.portrait_mp4_path is not None
assert out.with_vocals_mp4_path is not None
assert REQUESTED_FORMATS.issubset(set(out.output_files))
@pytest.mark.asyncio
async def test_encode_empty_fixture_is_recoverable(self):
"""Empty output_files must NOT trip the guard — it's recoverable
(stale cache) and the orchestrator re-encodes. Failure B defense."""
out = await _run_encode(_load("empty"))
assert out.success is True
assert out.output_files == {}
assert out.error_message is None
@pytest.mark.asyncio
async def test_encode_partial_fixture_trips_guard(self):
"""Partial (720p missing) must fail loud. Failure A defense."""
out = await _run_encode(_load("partial"))
assert out.success is False
assert "mp4_720p" in out.error_message
assert "incomplete" in out.error_message.lower()
@pytest.mark.asyncio
async def test_encode_malformed_fixture_treated_as_empty(self):
"""Nothing classifies into a slot → treated as empty (no guard trip,
no crash), which the orchestrator then recovers from as a stale cache."""
out = await _run_encode(_load("malformed"))
assert out.success is True
assert out.output_files == {}
assert out.error_message is None