diff --git a/backend/tests/emulator/test_admin_reset_integration.py b/backend/tests/emulator/test_admin_reset_integration.py new file mode 100644 index 00000000..4e5c9011 --- /dev/null +++ b/backend/tests/emulator/test_admin_reset_integration.py @@ -0,0 +1,122 @@ +""" +Integration tests for admin job reset endpoint. + +These tests use the real Firestore emulator and real JobManager, +catching bugs that unit tests with mocks might miss (e.g., incorrect +assumptions about return values). + +Run with: scripts/run-emulator-tests.sh +""" +import pytest +from datetime import datetime, UTC + +from backend.tests.emulator.conftest import emulators_running + +# Skip all tests if emulators aren't running +pytestmark = pytest.mark.skipif( + not emulators_running(), + reason="GCP emulators not running. Start with: scripts/start-emulators.sh" +) + +# Only import if emulators are running +if emulators_running(): + from backend.models.job import Job, JobStatus + from backend.services.job_manager import JobManager + from backend.services.firestore_service import FirestoreService + + +class TestUpdateJobReturnValue: + """ + Critical tests documenting that update_job() returns None, not boolean. + + This test class exists because a production bug (PR #371) was caused by + assuming update_job() returns a boolean success indicator. The real API + returns None on success and raises on failure. Mocks that return True + hide this behavior. + """ + + @pytest.fixture + def job_manager(self): + """Create a real JobManager using emulator.""" + return JobManager() + + @pytest.fixture + def firestore_service(self): + """Create a real FirestoreService using emulator.""" + return FirestoreService() + + @pytest.fixture + def test_job(self, firestore_service): + """Create a test job for verification.""" + job_id = f"update-return-test-{datetime.now(UTC).timestamp()}" + + job = Job( + job_id=job_id, + status=JobStatus.COMPLETE, + created_at=datetime.now(UTC), + updated_at=datetime.now(UTC), + artist="Test Artist", + title="Test Song", + ) + + firestore_service.create_job(job) + + yield job + + # Cleanup + try: + firestore_service.delete_job(job_id) + except Exception: + pass + + def test_update_job_returns_none_not_boolean(self, job_manager, test_job): + """ + Verify update_job() returns None, not a boolean. + + This test exists because a production bug was introduced when code + assumed update_job() returns a boolean. The method actually returns + None on success and raises an exception on failure. + + Bug pattern that this test prevents: + ```python + success = job_manager.update_job(job_id, updates) + if not success: # BUG: None is falsy, so this is always True! + raise HTTPException(status_code=500, ...) + ``` + + Correct pattern: + ```python + # update_job() raises on failure, returns None on success + job_manager.update_job(job_id, updates) + ``` + """ + # update_job() should return None, not True/False + result = job_manager.update_job( + test_job.job_id, + {"status": JobStatus.PENDING.value} + ) + + # The actual return value is None, not True + assert result is None, ( + "update_job() should return None, not a boolean. " + "Code checking 'if not update_job(...)' will always be True!" + ) + + def test_update_job_on_nonexistent_job(self, job_manager): + """ + Document update_job() behavior for non-existent jobs. + + In production Firestore, update() on non-existent doc raises NotFound. + The key insight for preventing the admin reset bug is understanding + that update_job() returns None (not boolean) on success. + """ + try: + result = job_manager.update_job( + "nonexistent-job-id-12345", + {"status": JobStatus.PENDING.value} + ) + # If no exception raised (emulator quirk), verify return is still None + assert result is None, "update_job should return None even on non-existent doc" + except Exception: + # Exception is the expected behavior in production Firestore + pass diff --git a/backend/tests/emulator/test_admin_reset_real_job_fixture.py b/backend/tests/emulator/test_admin_reset_real_job_fixture.py new file mode 100644 index 00000000..cac26ff6 --- /dev/null +++ b/backend/tests/emulator/test_admin_reset_real_job_fixture.py @@ -0,0 +1,69 @@ +""" +Integration tests for admin job reset using REAL job fixture from production. + +This test validates that update_job() returns None, not a boolean +(the root cause of PR #371 bug). + +Uses actual data from production job 984da08b which needed admin reset. + +Run with: scripts/run-emulator-tests.sh +""" +import pytest +from datetime import datetime, UTC + +from backend.tests.emulator.conftest import emulators_running + +pytestmark = pytest.mark.skipif( + not emulators_running(), + reason="GCP emulators not running. Start with: scripts/start-emulators.sh" +) + +if emulators_running(): + from backend.models.job import Job, JobStatus + from backend.services.job_manager import JobManager + + +class TestAdminResetBugFix: + """ + Tests that prove the PR #371 bug fix works using real emulator. + + The bug: admin reset endpoint returned 500 because: + success = job_manager.update_job(job_id, updates) + if not success: # BUG: update_job returns None, not True! + raise HTTPException(500) + """ + + def test_update_job_returns_none_not_boolean(self): + """ + Prove that update_job() returns None, not a boolean. + + This is the root cause of the bug. Any code that does + 'if not update_job(...)' will always be True. + """ + jm = JobManager() + job_id = f"test-{datetime.now(UTC).timestamp()}" + + job = Job( + job_id=job_id, + status=JobStatus.COMPLETE, + created_at=datetime.now(UTC), + updated_at=datetime.now(UTC), + artist="Mark Mallman", + title="Minneapolis", + ) + + try: + jm.firestore.create_job(job) + + # THE KEY TEST: update_job returns None + result = jm.update_job(job_id, {"status": "pending"}) + + assert result is None, ( + "update_job() must return None. " + "Code checking 'if not success:' would always fail!" + ) + finally: + try: + jm.firestore.delete_job(job_id) + except Exception: + pass diff --git a/backend/tests/test_admin_job_reset.py b/backend/tests/test_admin_job_reset.py index b7427116..1c3cef20 100644 --- a/backend/tests/test_admin_job_reset.py +++ b/backend/tests/test_admin_job_reset.py @@ -84,7 +84,7 @@ class TestResetJobToPending: with patch('backend.api.routes.admin.JobManager') as mock_jm_class: mock_jm = Mock() mock_jm.get_job.return_value = mock_job - mock_jm.update_job.return_value = True + mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error mock_jm_class.return_value = mock_jm response = client.post( @@ -104,7 +104,7 @@ class TestResetJobToPending: with patch('backend.api.routes.admin.JobManager') as mock_jm_class: mock_jm = Mock() mock_jm.get_job.return_value = mock_job - mock_jm.update_job.return_value = True + mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error mock_jm_class.return_value = mock_jm response = client.post( @@ -128,7 +128,7 @@ class TestResetJobToAwaitingAudioSelection: with patch('backend.api.routes.admin.JobManager') as mock_jm_class: mock_jm = Mock() mock_jm.get_job.return_value = mock_job - mock_jm.update_job.return_value = True + mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error mock_jm_class.return_value = mock_jm response = client.post( @@ -149,7 +149,7 @@ class TestResetJobToAwaitingReview: with patch('backend.api.routes.admin.JobManager') as mock_jm_class: mock_jm = Mock() mock_jm.get_job.return_value = mock_job - mock_jm.update_job.return_value = True + mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error mock_jm_class.return_value = mock_jm response = client.post( @@ -247,7 +247,7 @@ class TestResetJobLogging: patch('backend.api.routes.admin.logger') as mock_logger: mock_jm = Mock() mock_jm.get_job.return_value = mock_job - mock_jm.update_job.return_value = True + mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error mock_jm_class.return_value = mock_jm response = client.post( @@ -270,7 +270,7 @@ class TestResetJobTimeline: with patch('backend.api.routes.admin.JobManager') as mock_jm_class: mock_jm = Mock() mock_jm.get_job.return_value = mock_job - mock_jm.update_job.return_value = True + mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error mock_jm_class.return_value = mock_jm response = client.post( @@ -319,7 +319,7 @@ class TestResetJobClearsStateData: with patch('backend.api.routes.admin.JobManager') as mock_jm_class: mock_jm = Mock() mock_jm.get_job.return_value = mock_job - mock_jm.update_job.return_value = True + mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error mock_jm_class.return_value = mock_jm response = client.post( @@ -359,7 +359,7 @@ class TestResetJobClearsStateData: with patch('backend.api.routes.admin.JobManager') as mock_jm_class: mock_jm = Mock() mock_jm.get_job.return_value = mock_job - mock_jm.update_job.return_value = True + mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error mock_jm_class.return_value = mock_jm response = client.post( @@ -382,7 +382,7 @@ class TestResetJobClearsStateData: with patch('backend.api.routes.admin.JobManager') as mock_jm_class: mock_jm = Mock() mock_jm.get_job.return_value = mock_job - mock_jm.update_job.return_value = True + mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error mock_jm_class.return_value = mock_jm response = client.post( @@ -425,7 +425,7 @@ class TestResetJobToInstrumentalSelected: patch('backend.services.worker_service.get_worker_service') as mock_ws: mock_jm = Mock() mock_jm.get_job.return_value = mock_job - mock_jm.update_job.return_value = True + mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error mock_jm_class.return_value = mock_jm # Mock worker service with async method @@ -450,7 +450,7 @@ class TestResetJobToInstrumentalSelected: patch('backend.services.worker_service.get_worker_service') as mock_ws: mock_jm = Mock() mock_jm.get_job.return_value = mock_job - mock_jm.update_job.return_value = True + mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error mock_jm_class.return_value = mock_jm # Mock worker service @@ -478,7 +478,7 @@ class TestResetJobToInstrumentalSelected: patch('backend.services.worker_service.get_worker_service') as mock_ws: mock_jm = Mock() mock_jm.get_job.return_value = mock_job - mock_jm.update_job.return_value = True + mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error mock_jm_class.return_value = mock_jm # Mock worker service failure @@ -515,7 +515,7 @@ class TestResetJobToInstrumentalSelected: patch('backend.services.worker_service.get_worker_service') as mock_ws: mock_jm = Mock() mock_jm.get_job.return_value = mock_job - mock_jm.update_job.return_value = True + mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error mock_jm_class.return_value = mock_jm mock_worker_service = Mock() @@ -550,7 +550,7 @@ class TestResetJobClearsErrorState: with patch('backend.api.routes.admin.JobManager') as mock_jm_class: mock_jm = Mock() mock_jm.get_job.return_value = mock_job - mock_jm.update_job.return_value = True + mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error mock_jm_class.return_value = mock_jm response = client.post( @@ -654,7 +654,7 @@ class TestResetIncludesEncodingProgress: with patch('backend.api.routes.admin.JobManager') as mock_jm_class: mock_jm = Mock() mock_jm.get_job.return_value = mock_job - mock_jm.update_job.return_value = True + mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error mock_jm_class.return_value = mock_jm response = client.post( @@ -679,7 +679,7 @@ class TestResetWorkerTriggerResponseFields: patch('backend.services.worker_service.get_worker_service') as mock_ws: mock_jm = Mock() mock_jm.get_job.return_value = mock_job - mock_jm.update_job.return_value = True + mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error mock_jm_class.return_value = mock_jm mock_worker_service = Mock() @@ -704,7 +704,7 @@ class TestResetWorkerTriggerResponseFields: patch('backend.services.worker_service.get_worker_service') as mock_ws: mock_jm = Mock() mock_jm.get_job.return_value = mock_job - mock_jm.update_job.return_value = True + mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error mock_jm_class.return_value = mock_jm mock_worker_service = Mock() @@ -730,7 +730,7 @@ class TestResetWorkerTriggerResponseFields: patch('backend.services.worker_service.get_worker_service') as mock_ws: mock_jm = Mock() mock_jm.get_job.return_value = mock_job - mock_jm.update_job.return_value = True + mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error mock_jm_class.return_value = mock_jm mock_worker_service = Mock() @@ -754,7 +754,7 @@ class TestResetWorkerTriggerResponseFields: with patch('backend.api.routes.admin.JobManager') as mock_jm_class: mock_jm = Mock() mock_jm.get_job.return_value = mock_job - mock_jm.update_job.return_value = True + mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error mock_jm_class.return_value = mock_jm response = client.post( @@ -962,3 +962,57 @@ class TestTriggerWorkerEndpoint: app.dependency_overrides[require_admin] = original_override else: app.dependency_overrides[require_admin] = get_mock_admin + + +class TestResetEndpointWithRealJobData: + """ + Tests using real production job data (job 984da08b) as fixture. + + This validates the admin reset endpoint works correctly with realistic + state_data from a production job that needed manual admin reset. + """ + + # Real state_data from production job 984da08b + REAL_JOB_STATE_DATA = { + "backing_vocals_analysis": {"has_audible_content": True}, + "video_progress": {"stage": "running"}, # Stale progress + "lyrics_complete": True, + "audio_complete": True + } + + def test_reset_endpoint_returns_200_with_none_return_value(self, client): + """ + Test that reset endpoint returns 200 (not 500) when update_job + returns None, matching the real API behavior. + + Before fix: endpoint checked 'if not success:' which was always True + After fix: endpoint doesn't check return value + + Uses real production job 984da08b data as fixture. + """ + # Mock job with real production data + mock_job = Mock(spec=Job) + mock_job.job_id = "984da08b" + mock_job.status = JobStatus.COMPLETE + mock_job.artist = "Mark Mallman" + mock_job.title = "Minneapolis" + mock_job.theme_id = "nomad" + mock_job.user_email = "madeforyou@nomadkaraoke.com" + mock_job.file_urls = {"stems": {}, "lyrics": {}} + mock_job.state_data = self.REAL_JOB_STATE_DATA.copy() + mock_job.timeline = [] + + with patch('backend.api.routes.admin.JobManager') as mock_jm: + mock_jm.return_value.get_job.return_value = mock_job + # KEY: return None (matching real API contract - not boolean!) + mock_jm.return_value.update_job.return_value = None + + response = client.post( + "/api/admin/jobs/984da08b/reset", + json={"target_state": "pending"}, + ) + + # This is the bug fix - before it was 500, now 200 + assert response.status_code == 200 + assert response.json()["status"] == "success" + assert response.json()["new_status"] == "pending" diff --git a/docs/LESSONS-LEARNED.md b/docs/LESSONS-LEARNED.md index 1811c807..ff69463e 100644 --- a/docs/LESSONS-LEARNED.md +++ b/docs/LESSONS-LEARNED.md @@ -312,6 +312,38 @@ const debouncedUpdate = useCallback((val: string) => { **Testing**: Verify (1) typing multi-digit values doesn't jump cursor, (2) debounce fires after delay, (3) blur syncs immediately, (4) external updates respected when not focused. See `frontend/components/lyrics-review/__tests__/TimeInput.test.tsx` for comprehensive test suite. +### Mocks Must Match Real Return Values (Feb 2026) + +**Problem**: PR #371 introduced a bug where admin reset always returned 500. The code checked `if not success:` after calling `update_job()`, but `update_job()` returns `None`, not a boolean. Since `not None` is `True`, the error was always raised. + +**Why tests didn't catch it**: All 22 test mocks had: +```python +mock_jm.update_job.return_value = True # WRONG - real method returns None +``` + +The mock returned `True`, so `if not True:` was `False` and tests passed. But in production, `if not None:` is `True`, causing 500 errors. + +**Root cause**: Mocks were written based on an assumed API contract (boolean return) rather than the actual contract (void return, raises on error). + +**Fix**: +1. Updated all mocks to `return_value = None` with comment explaining why +2. Added emulator integration test (`test_admin_reset_integration.py`) that uses real `JobManager` +3. Added explicit test `test_update_job_returns_none_not_boolean` to document the API contract + +**Pattern for preventing this**: +1. **Mocks should match real signatures**: Check the actual method before setting `return_value`. Void methods should mock `return_value = None`. +2. **Add emulator tests for critical paths**: Integration tests with real services catch contract mismatches mocks hide. +3. **Document API contracts in tests**: Add explicit tests for non-obvious behaviors (`update_job` returns None, raises on failure). + +**Code pattern**: +```python +# ❌ WRONG - assumes boolean return +mock_jm.update_job.return_value = True + +# ✅ CORRECT - matches real API +mock_jm.update_job.return_value = None # Real API: returns None, raises on error +``` + ### Emulator Tests Catch Real Bugs Firestore emulator tests catch issues (like missing indexes) that unit tests with mocks miss. diff --git a/docs/TESTING.md b/docs/TESTING.md index 80f632df..4f585f56 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -158,6 +158,30 @@ def test_create_job_applies_default_theme(mock_firestore, mock_gcs): assert job.theme_id == "nomad" # Default theme applied ``` +**Critical: Mocks Must Match Real API Contracts** + +When mocking methods, ensure the mock's `return_value` matches what the real method returns. A common bug pattern: + +```python +# ❌ BAD: Assumes method returns boolean +mock_service.update_job.return_value = True + +# Code under test: +success = job_manager.update_job(job_id, updates) +if not success: # BUG: if real method returns None, this is always True! + raise HTTPException(500) +``` + +The fix: + +```python +# ✅ GOOD: Check real method signature first +# def update_job(self, job_id: str, updates: Dict) -> None: +mock_service.update_job.return_value = None # Match real API +``` + +**Before writing a mock**, check the actual method's return type. Void methods should mock `return_value = None`. Methods that raise on error (rather than returning False) should have mocks that raise exceptions for error cases. See [LESSONS-LEARNED.md](LESSONS-LEARNED.md#mocks-must-match-real-return-values-feb-2026) for a production bug caused by this. + ### Integration Tests (`tests/integration/`) **Purpose:** Test CLI workflows and component interactions. diff --git a/pyproject.toml b/pyproject.toml index 9891ff43..73b1ae76 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.114.16" +version = "0.114.17" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT"