Compare commits

...
Author SHA1 Message Date
Andrew BeveridgeandClaude Opus 4.5 b89423c8e7 fix: move mock-based test to unit tests (CI fix)
The test_reset_endpoint_returns_200_with_none_return_value test was in
the emulator directory but doesn't actually use the emulator - it's a
pure mock test. When run with emulators, other parts of the code tried
to access real Firestore.

Moved the mock-based test to test_admin_job_reset.py (unit tests) and
kept only the true emulator integration test in the emulator directory.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-02-03 16:31:14 -08:00
Andrew BeveridgeandClaude Opus 4.5 91c39dc478 test: fix mock return values to match real API contract
The admin reset endpoint (PR #371) always returned 500 because tests
mocked update_job() to return True, but the real method returns None.
Code checking `if not success:` was always True with None.

Changes:
- Fix 22 mocks from `return_value = True` to `return_value = None`
- Add emulator integration tests proving update_job() returns None
- Add test using real production job 984da08b as fixture
- Document lesson in LESSONS-LEARNED.md
- Add mock guidance to TESTING.md

This ensures our tests would have caught the bug before it shipped.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-02-03 16:22:49 -08:00
6 changed files with 321 additions and 20 deletions

No files matched your search

@@ -0,0 +1,122 @@
"""
Integration tests for admin job reset endpoint.
These tests use the real Firestore emulator and real JobManager,
catching bugs that unit tests with mocks might miss (e.g., incorrect
assumptions about return values).
Run with: scripts/run-emulator-tests.sh
"""
import pytest
from datetime import datetime, UTC
from backend.tests.emulator.conftest import emulators_running
# Skip all tests if emulators aren't running
pytestmark = pytest.mark.skipif(
not emulators_running(),
reason="GCP emulators not running. Start with: scripts/start-emulators.sh"
)
# Only import if emulators are running
if emulators_running():
from backend.models.job import Job, JobStatus
from backend.services.job_manager import JobManager
from backend.services.firestore_service import FirestoreService
class TestUpdateJobReturnValue:
"""
Critical tests documenting that update_job() returns None, not boolean.
This test class exists because a production bug (PR #371) was caused by
assuming update_job() returns a boolean success indicator. The real API
returns None on success and raises on failure. Mocks that return True
hide this behavior.
"""
@pytest.fixture
def job_manager(self):
"""Create a real JobManager using emulator."""
return JobManager()
@pytest.fixture
def firestore_service(self):
"""Create a real FirestoreService using emulator."""
return FirestoreService()
@pytest.fixture
def test_job(self, firestore_service):
"""Create a test job for verification."""
job_id = f"update-return-test-{datetime.now(UTC).timestamp()}"
job = Job(
job_id=job_id,
status=JobStatus.COMPLETE,
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC),
artist="Test Artist",
title="Test Song",
)
firestore_service.create_job(job)
yield job
# Cleanup
try:
firestore_service.delete_job(job_id)
except Exception:
pass
def test_update_job_returns_none_not_boolean(self, job_manager, test_job):
"""
Verify update_job() returns None, not a boolean.
This test exists because a production bug was introduced when code
assumed update_job() returns a boolean. The method actually returns
None on success and raises an exception on failure.
Bug pattern that this test prevents:
```python
success = job_manager.update_job(job_id, updates)
if not success: # BUG: None is falsy, so this is always True!
raise HTTPException(status_code=500, ...)
```
Correct pattern:
```python
# update_job() raises on failure, returns None on success
job_manager.update_job(job_id, updates)
```
"""
# update_job() should return None, not True/False
result = job_manager.update_job(
test_job.job_id,
{"status": JobStatus.PENDING.value}
)
# The actual return value is None, not True
assert result is None, (
"update_job() should return None, not a boolean. "
"Code checking 'if not update_job(...)' will always be True!"
)
def test_update_job_on_nonexistent_job(self, job_manager):
"""
Document update_job() behavior for non-existent jobs.
In production Firestore, update() on non-existent doc raises NotFound.
The key insight for preventing the admin reset bug is understanding
that update_job() returns None (not boolean) on success.
"""
try:
result = job_manager.update_job(
"nonexistent-job-id-12345",
{"status": JobStatus.PENDING.value}
)
# If no exception raised (emulator quirk), verify return is still None
assert result is None, "update_job should return None even on non-existent doc"
except Exception:
# Exception is the expected behavior in production Firestore
pass
@@ -0,0 +1,69 @@
"""
Integration tests for admin job reset using REAL job fixture from production.
This test validates that update_job() returns None, not a boolean
(the root cause of PR #371 bug).
Uses actual data from production job 984da08b which needed admin reset.
Run with: scripts/run-emulator-tests.sh
"""
import pytest
from datetime import datetime, UTC
from backend.tests.emulator.conftest import emulators_running
pytestmark = pytest.mark.skipif(
not emulators_running(),
reason="GCP emulators not running. Start with: scripts/start-emulators.sh"
)
if emulators_running():
from backend.models.job import Job, JobStatus
from backend.services.job_manager import JobManager
class TestAdminResetBugFix:
"""
Tests that prove the PR #371 bug fix works using real emulator.
The bug: admin reset endpoint returned 500 because:
success = job_manager.update_job(job_id, updates)
if not success: # BUG: update_job returns None, not True!
raise HTTPException(500)
"""
def test_update_job_returns_none_not_boolean(self):
"""
Prove that update_job() returns None, not a boolean.
This is the root cause of the bug. Any code that does
'if not update_job(...)' will always be True.
"""
jm = JobManager()
job_id = f"test-{datetime.now(UTC).timestamp()}"
job = Job(
job_id=job_id,
status=JobStatus.COMPLETE,
created_at=datetime.now(UTC),
updated_at=datetime.now(UTC),
artist="Mark Mallman",
title="Minneapolis",
)
try:
jm.firestore.create_job(job)
# THE KEY TEST: update_job returns None
result = jm.update_job(job_id, {"status": "pending"})
assert result is None, (
"update_job() must return None. "
"Code checking 'if not success:' would always fail!"
)
finally:
try:
jm.firestore.delete_job(job_id)
except Exception:
pass
+73 -19
View File
@@ -84,7 +84,7 @@ class TestResetJobToPending:
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = True
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm_class.return_value = mock_jm
response = client.post(
@@ -104,7 +104,7 @@ class TestResetJobToPending:
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = True
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm_class.return_value = mock_jm
response = client.post(
@@ -128,7 +128,7 @@ class TestResetJobToAwaitingAudioSelection:
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = True
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm_class.return_value = mock_jm
response = client.post(
@@ -149,7 +149,7 @@ class TestResetJobToAwaitingReview:
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = True
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm_class.return_value = mock_jm
response = client.post(
@@ -247,7 +247,7 @@ class TestResetJobLogging:
patch('backend.api.routes.admin.logger') as mock_logger:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = True
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm_class.return_value = mock_jm
response = client.post(
@@ -270,7 +270,7 @@ class TestResetJobTimeline:
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = True
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm_class.return_value = mock_jm
response = client.post(
@@ -319,7 +319,7 @@ class TestResetJobClearsStateData:
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = True
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm_class.return_value = mock_jm
response = client.post(
@@ -359,7 +359,7 @@ class TestResetJobClearsStateData:
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = True
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm_class.return_value = mock_jm
response = client.post(
@@ -382,7 +382,7 @@ class TestResetJobClearsStateData:
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = True
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm_class.return_value = mock_jm
response = client.post(
@@ -425,7 +425,7 @@ class TestResetJobToInstrumentalSelected:
patch('backend.services.worker_service.get_worker_service') as mock_ws:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = True
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm_class.return_value = mock_jm
# Mock worker service with async method
@@ -450,7 +450,7 @@ class TestResetJobToInstrumentalSelected:
patch('backend.services.worker_service.get_worker_service') as mock_ws:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = True
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm_class.return_value = mock_jm
# Mock worker service
@@ -478,7 +478,7 @@ class TestResetJobToInstrumentalSelected:
patch('backend.services.worker_service.get_worker_service') as mock_ws:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = True
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm_class.return_value = mock_jm
# Mock worker service failure
@@ -515,7 +515,7 @@ class TestResetJobToInstrumentalSelected:
patch('backend.services.worker_service.get_worker_service') as mock_ws:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = True
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm_class.return_value = mock_jm
mock_worker_service = Mock()
@@ -550,7 +550,7 @@ class TestResetJobClearsErrorState:
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = True
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm_class.return_value = mock_jm
response = client.post(
@@ -654,7 +654,7 @@ class TestResetIncludesEncodingProgress:
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = True
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm_class.return_value = mock_jm
response = client.post(
@@ -679,7 +679,7 @@ class TestResetWorkerTriggerResponseFields:
patch('backend.services.worker_service.get_worker_service') as mock_ws:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = True
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm_class.return_value = mock_jm
mock_worker_service = Mock()
@@ -704,7 +704,7 @@ class TestResetWorkerTriggerResponseFields:
patch('backend.services.worker_service.get_worker_service') as mock_ws:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = True
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm_class.return_value = mock_jm
mock_worker_service = Mock()
@@ -730,7 +730,7 @@ class TestResetWorkerTriggerResponseFields:
patch('backend.services.worker_service.get_worker_service') as mock_ws:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = True
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm_class.return_value = mock_jm
mock_worker_service = Mock()
@@ -754,7 +754,7 @@ class TestResetWorkerTriggerResponseFields:
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
mock_jm = Mock()
mock_jm.get_job.return_value = mock_job
mock_jm.update_job.return_value = True
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
mock_jm_class.return_value = mock_jm
response = client.post(
@@ -962,3 +962,57 @@ class TestTriggerWorkerEndpoint:
app.dependency_overrides[require_admin] = original_override
else:
app.dependency_overrides[require_admin] = get_mock_admin
class TestResetEndpointWithRealJobData:
"""
Tests using real production job data (job 984da08b) as fixture.
This validates the admin reset endpoint works correctly with realistic
state_data from a production job that needed manual admin reset.
"""
# Real state_data from production job 984da08b
REAL_JOB_STATE_DATA = {
"backing_vocals_analysis": {"has_audible_content": True},
"video_progress": {"stage": "running"}, # Stale progress
"lyrics_complete": True,
"audio_complete": True
}
def test_reset_endpoint_returns_200_with_none_return_value(self, client):
"""
Test that reset endpoint returns 200 (not 500) when update_job
returns None, matching the real API behavior.
Before fix: endpoint checked 'if not success:' which was always True
After fix: endpoint doesn't check return value
Uses real production job 984da08b data as fixture.
"""
# Mock job with real production data
mock_job = Mock(spec=Job)
mock_job.job_id = "984da08b"
mock_job.status = JobStatus.COMPLETE
mock_job.artist = "Mark Mallman"
mock_job.title = "Minneapolis"
mock_job.theme_id = "nomad"
mock_job.user_email = "madeforyou@nomadkaraoke.com"
mock_job.file_urls = {"stems": {}, "lyrics": {}}
mock_job.state_data = self.REAL_JOB_STATE_DATA.copy()
mock_job.timeline = []
with patch('backend.api.routes.admin.JobManager') as mock_jm:
mock_jm.return_value.get_job.return_value = mock_job
# KEY: return None (matching real API contract - not boolean!)
mock_jm.return_value.update_job.return_value = None
response = client.post(
"/api/admin/jobs/984da08b/reset",
json={"target_state": "pending"},
)
# This is the bug fix - before it was 500, now 200
assert response.status_code == 200
assert response.json()["status"] == "success"
assert response.json()["new_status"] == "pending"
+32
View File
@@ -312,6 +312,38 @@ const debouncedUpdate = useCallback((val: string) => {
**Testing**: Verify (1) typing multi-digit values doesn't jump cursor, (2) debounce fires after delay, (3) blur syncs immediately, (4) external updates respected when not focused. See `frontend/components/lyrics-review/__tests__/TimeInput.test.tsx` for comprehensive test suite.
### Mocks Must Match Real Return Values (Feb 2026)
**Problem**: PR #371 introduced a bug where admin reset always returned 500. The code checked `if not success:` after calling `update_job()`, but `update_job()` returns `None`, not a boolean. Since `not None` is `True`, the error was always raised.
**Why tests didn't catch it**: All 22 test mocks had:
```python
mock_jm.update_job.return_value = True # WRONG - real method returns None
```
The mock returned `True`, so `if not True:` was `False` and tests passed. But in production, `if not None:` is `True`, causing 500 errors.
**Root cause**: Mocks were written based on an assumed API contract (boolean return) rather than the actual contract (void return, raises on error).
**Fix**:
1. Updated all mocks to `return_value = None` with comment explaining why
2. Added emulator integration test (`test_admin_reset_integration.py`) that uses real `JobManager`
3. Added explicit test `test_update_job_returns_none_not_boolean` to document the API contract
**Pattern for preventing this**:
1. **Mocks should match real signatures**: Check the actual method before setting `return_value`. Void methods should mock `return_value = None`.
2. **Add emulator tests for critical paths**: Integration tests with real services catch contract mismatches mocks hide.
3. **Document API contracts in tests**: Add explicit tests for non-obvious behaviors (`update_job` returns None, raises on failure).
**Code pattern**:
```python
# ❌ WRONG - assumes boolean return
mock_jm.update_job.return_value = True
# ✅ CORRECT - matches real API
mock_jm.update_job.return_value = None # Real API: returns None, raises on error
```
### Emulator Tests Catch Real Bugs
Firestore emulator tests catch issues (like missing indexes) that unit tests with mocks miss.
+24
View File
@@ -158,6 +158,30 @@ def test_create_job_applies_default_theme(mock_firestore, mock_gcs):
assert job.theme_id == "nomad" # Default theme applied
```
**Critical: Mocks Must Match Real API Contracts**
When mocking methods, ensure the mock's `return_value` matches what the real method returns. A common bug pattern:
```python
# ❌ BAD: Assumes method returns boolean
mock_service.update_job.return_value = True
# Code under test:
success = job_manager.update_job(job_id, updates)
if not success: # BUG: if real method returns None, this is always True!
raise HTTPException(500)
```
The fix:
```python
# ✅ GOOD: Check real method signature first
# def update_job(self, job_id: str, updates: Dict) -> None:
mock_service.update_job.return_value = None # Match real API
```
**Before writing a mock**, check the actual method's return type. Void methods should mock `return_value = None`. Methods that raise on error (rather than returning False) should have mocks that raise exceptions for error cases. See [LESSONS-LEARNED.md](LESSONS-LEARNED.md#mocks-must-match-real-return-values-feb-2026) for a production bug caused by this.
### Integration Tests (`tests/integration/`)
**Purpose:** Test CLI workflows and component interactions.
+1 -1
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "karaoke-gen"
version = "0.114.16"
version = "0.114.17"
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
license = "MIT"