mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b89423c8e7 | ||
|
|
91c39dc478 |
No files matched your search
@@ -0,0 +1,122 @@
|
||||
"""
|
||||
Integration tests for admin job reset endpoint.
|
||||
|
||||
These tests use the real Firestore emulator and real JobManager,
|
||||
catching bugs that unit tests with mocks might miss (e.g., incorrect
|
||||
assumptions about return values).
|
||||
|
||||
Run with: scripts/run-emulator-tests.sh
|
||||
"""
|
||||
import pytest
|
||||
from datetime import datetime, UTC
|
||||
|
||||
from backend.tests.emulator.conftest import emulators_running
|
||||
|
||||
# Skip all tests if emulators aren't running
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not emulators_running(),
|
||||
reason="GCP emulators not running. Start with: scripts/start-emulators.sh"
|
||||
)
|
||||
|
||||
# Only import if emulators are running
|
||||
if emulators_running():
|
||||
from backend.models.job import Job, JobStatus
|
||||
from backend.services.job_manager import JobManager
|
||||
from backend.services.firestore_service import FirestoreService
|
||||
|
||||
|
||||
class TestUpdateJobReturnValue:
|
||||
"""
|
||||
Critical tests documenting that update_job() returns None, not boolean.
|
||||
|
||||
This test class exists because a production bug (PR #371) was caused by
|
||||
assuming update_job() returns a boolean success indicator. The real API
|
||||
returns None on success and raises on failure. Mocks that return True
|
||||
hide this behavior.
|
||||
"""
|
||||
|
||||
@pytest.fixture
|
||||
def job_manager(self):
|
||||
"""Create a real JobManager using emulator."""
|
||||
return JobManager()
|
||||
|
||||
@pytest.fixture
|
||||
def firestore_service(self):
|
||||
"""Create a real FirestoreService using emulator."""
|
||||
return FirestoreService()
|
||||
|
||||
@pytest.fixture
|
||||
def test_job(self, firestore_service):
|
||||
"""Create a test job for verification."""
|
||||
job_id = f"update-return-test-{datetime.now(UTC).timestamp()}"
|
||||
|
||||
job = Job(
|
||||
job_id=job_id,
|
||||
status=JobStatus.COMPLETE,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
artist="Test Artist",
|
||||
title="Test Song",
|
||||
)
|
||||
|
||||
firestore_service.create_job(job)
|
||||
|
||||
yield job
|
||||
|
||||
# Cleanup
|
||||
try:
|
||||
firestore_service.delete_job(job_id)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def test_update_job_returns_none_not_boolean(self, job_manager, test_job):
|
||||
"""
|
||||
Verify update_job() returns None, not a boolean.
|
||||
|
||||
This test exists because a production bug was introduced when code
|
||||
assumed update_job() returns a boolean. The method actually returns
|
||||
None on success and raises an exception on failure.
|
||||
|
||||
Bug pattern that this test prevents:
|
||||
```python
|
||||
success = job_manager.update_job(job_id, updates)
|
||||
if not success: # BUG: None is falsy, so this is always True!
|
||||
raise HTTPException(status_code=500, ...)
|
||||
```
|
||||
|
||||
Correct pattern:
|
||||
```python
|
||||
# update_job() raises on failure, returns None on success
|
||||
job_manager.update_job(job_id, updates)
|
||||
```
|
||||
"""
|
||||
# update_job() should return None, not True/False
|
||||
result = job_manager.update_job(
|
||||
test_job.job_id,
|
||||
{"status": JobStatus.PENDING.value}
|
||||
)
|
||||
|
||||
# The actual return value is None, not True
|
||||
assert result is None, (
|
||||
"update_job() should return None, not a boolean. "
|
||||
"Code checking 'if not update_job(...)' will always be True!"
|
||||
)
|
||||
|
||||
def test_update_job_on_nonexistent_job(self, job_manager):
|
||||
"""
|
||||
Document update_job() behavior for non-existent jobs.
|
||||
|
||||
In production Firestore, update() on non-existent doc raises NotFound.
|
||||
The key insight for preventing the admin reset bug is understanding
|
||||
that update_job() returns None (not boolean) on success.
|
||||
"""
|
||||
try:
|
||||
result = job_manager.update_job(
|
||||
"nonexistent-job-id-12345",
|
||||
{"status": JobStatus.PENDING.value}
|
||||
)
|
||||
# If no exception raised (emulator quirk), verify return is still None
|
||||
assert result is None, "update_job should return None even on non-existent doc"
|
||||
except Exception:
|
||||
# Exception is the expected behavior in production Firestore
|
||||
pass
|
||||
@@ -0,0 +1,69 @@
|
||||
"""
|
||||
Integration tests for admin job reset using REAL job fixture from production.
|
||||
|
||||
This test validates that update_job() returns None, not a boolean
|
||||
(the root cause of PR #371 bug).
|
||||
|
||||
Uses actual data from production job 984da08b which needed admin reset.
|
||||
|
||||
Run with: scripts/run-emulator-tests.sh
|
||||
"""
|
||||
import pytest
|
||||
from datetime import datetime, UTC
|
||||
|
||||
from backend.tests.emulator.conftest import emulators_running
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not emulators_running(),
|
||||
reason="GCP emulators not running. Start with: scripts/start-emulators.sh"
|
||||
)
|
||||
|
||||
if emulators_running():
|
||||
from backend.models.job import Job, JobStatus
|
||||
from backend.services.job_manager import JobManager
|
||||
|
||||
|
||||
class TestAdminResetBugFix:
|
||||
"""
|
||||
Tests that prove the PR #371 bug fix works using real emulator.
|
||||
|
||||
The bug: admin reset endpoint returned 500 because:
|
||||
success = job_manager.update_job(job_id, updates)
|
||||
if not success: # BUG: update_job returns None, not True!
|
||||
raise HTTPException(500)
|
||||
"""
|
||||
|
||||
def test_update_job_returns_none_not_boolean(self):
|
||||
"""
|
||||
Prove that update_job() returns None, not a boolean.
|
||||
|
||||
This is the root cause of the bug. Any code that does
|
||||
'if not update_job(...)' will always be True.
|
||||
"""
|
||||
jm = JobManager()
|
||||
job_id = f"test-{datetime.now(UTC).timestamp()}"
|
||||
|
||||
job = Job(
|
||||
job_id=job_id,
|
||||
status=JobStatus.COMPLETE,
|
||||
created_at=datetime.now(UTC),
|
||||
updated_at=datetime.now(UTC),
|
||||
artist="Mark Mallman",
|
||||
title="Minneapolis",
|
||||
)
|
||||
|
||||
try:
|
||||
jm.firestore.create_job(job)
|
||||
|
||||
# THE KEY TEST: update_job returns None
|
||||
result = jm.update_job(job_id, {"status": "pending"})
|
||||
|
||||
assert result is None, (
|
||||
"update_job() must return None. "
|
||||
"Code checking 'if not success:' would always fail!"
|
||||
)
|
||||
finally:
|
||||
try:
|
||||
jm.firestore.delete_job(job_id)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -84,7 +84,7 @@ class TestResetJobToPending:
|
||||
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.post(
|
||||
@@ -104,7 +104,7 @@ class TestResetJobToPending:
|
||||
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.post(
|
||||
@@ -128,7 +128,7 @@ class TestResetJobToAwaitingAudioSelection:
|
||||
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.post(
|
||||
@@ -149,7 +149,7 @@ class TestResetJobToAwaitingReview:
|
||||
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.post(
|
||||
@@ -247,7 +247,7 @@ class TestResetJobLogging:
|
||||
patch('backend.api.routes.admin.logger') as mock_logger:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.post(
|
||||
@@ -270,7 +270,7 @@ class TestResetJobTimeline:
|
||||
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.post(
|
||||
@@ -319,7 +319,7 @@ class TestResetJobClearsStateData:
|
||||
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.post(
|
||||
@@ -359,7 +359,7 @@ class TestResetJobClearsStateData:
|
||||
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.post(
|
||||
@@ -382,7 +382,7 @@ class TestResetJobClearsStateData:
|
||||
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.post(
|
||||
@@ -425,7 +425,7 @@ class TestResetJobToInstrumentalSelected:
|
||||
patch('backend.services.worker_service.get_worker_service') as mock_ws:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
# Mock worker service with async method
|
||||
@@ -450,7 +450,7 @@ class TestResetJobToInstrumentalSelected:
|
||||
patch('backend.services.worker_service.get_worker_service') as mock_ws:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
# Mock worker service
|
||||
@@ -478,7 +478,7 @@ class TestResetJobToInstrumentalSelected:
|
||||
patch('backend.services.worker_service.get_worker_service') as mock_ws:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
# Mock worker service failure
|
||||
@@ -515,7 +515,7 @@ class TestResetJobToInstrumentalSelected:
|
||||
patch('backend.services.worker_service.get_worker_service') as mock_ws:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
mock_worker_service = Mock()
|
||||
@@ -550,7 +550,7 @@ class TestResetJobClearsErrorState:
|
||||
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.post(
|
||||
@@ -654,7 +654,7 @@ class TestResetIncludesEncodingProgress:
|
||||
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.post(
|
||||
@@ -679,7 +679,7 @@ class TestResetWorkerTriggerResponseFields:
|
||||
patch('backend.services.worker_service.get_worker_service') as mock_ws:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
mock_worker_service = Mock()
|
||||
@@ -704,7 +704,7 @@ class TestResetWorkerTriggerResponseFields:
|
||||
patch('backend.services.worker_service.get_worker_service') as mock_ws:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
mock_worker_service = Mock()
|
||||
@@ -730,7 +730,7 @@ class TestResetWorkerTriggerResponseFields:
|
||||
patch('backend.services.worker_service.get_worker_service') as mock_ws:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
mock_worker_service = Mock()
|
||||
@@ -754,7 +754,7 @@ class TestResetWorkerTriggerResponseFields:
|
||||
with patch('backend.api.routes.admin.JobManager') as mock_jm_class:
|
||||
mock_jm = Mock()
|
||||
mock_jm.get_job.return_value = mock_job
|
||||
mock_jm.update_job.return_value = True
|
||||
mock_jm.update_job.return_value = None # Matches real API: returns None, raises on error
|
||||
mock_jm_class.return_value = mock_jm
|
||||
|
||||
response = client.post(
|
||||
@@ -962,3 +962,57 @@ class TestTriggerWorkerEndpoint:
|
||||
app.dependency_overrides[require_admin] = original_override
|
||||
else:
|
||||
app.dependency_overrides[require_admin] = get_mock_admin
|
||||
|
||||
|
||||
class TestResetEndpointWithRealJobData:
|
||||
"""
|
||||
Tests using real production job data (job 984da08b) as fixture.
|
||||
|
||||
This validates the admin reset endpoint works correctly with realistic
|
||||
state_data from a production job that needed manual admin reset.
|
||||
"""
|
||||
|
||||
# Real state_data from production job 984da08b
|
||||
REAL_JOB_STATE_DATA = {
|
||||
"backing_vocals_analysis": {"has_audible_content": True},
|
||||
"video_progress": {"stage": "running"}, # Stale progress
|
||||
"lyrics_complete": True,
|
||||
"audio_complete": True
|
||||
}
|
||||
|
||||
def test_reset_endpoint_returns_200_with_none_return_value(self, client):
|
||||
"""
|
||||
Test that reset endpoint returns 200 (not 500) when update_job
|
||||
returns None, matching the real API behavior.
|
||||
|
||||
Before fix: endpoint checked 'if not success:' which was always True
|
||||
After fix: endpoint doesn't check return value
|
||||
|
||||
Uses real production job 984da08b data as fixture.
|
||||
"""
|
||||
# Mock job with real production data
|
||||
mock_job = Mock(spec=Job)
|
||||
mock_job.job_id = "984da08b"
|
||||
mock_job.status = JobStatus.COMPLETE
|
||||
mock_job.artist = "Mark Mallman"
|
||||
mock_job.title = "Minneapolis"
|
||||
mock_job.theme_id = "nomad"
|
||||
mock_job.user_email = "madeforyou@nomadkaraoke.com"
|
||||
mock_job.file_urls = {"stems": {}, "lyrics": {}}
|
||||
mock_job.state_data = self.REAL_JOB_STATE_DATA.copy()
|
||||
mock_job.timeline = []
|
||||
|
||||
with patch('backend.api.routes.admin.JobManager') as mock_jm:
|
||||
mock_jm.return_value.get_job.return_value = mock_job
|
||||
# KEY: return None (matching real API contract - not boolean!)
|
||||
mock_jm.return_value.update_job.return_value = None
|
||||
|
||||
response = client.post(
|
||||
"/api/admin/jobs/984da08b/reset",
|
||||
json={"target_state": "pending"},
|
||||
)
|
||||
|
||||
# This is the bug fix - before it was 500, now 200
|
||||
assert response.status_code == 200
|
||||
assert response.json()["status"] == "success"
|
||||
assert response.json()["new_status"] == "pending"
|
||||
@@ -312,6 +312,38 @@ const debouncedUpdate = useCallback((val: string) => {
|
||||
|
||||
**Testing**: Verify (1) typing multi-digit values doesn't jump cursor, (2) debounce fires after delay, (3) blur syncs immediately, (4) external updates respected when not focused. See `frontend/components/lyrics-review/__tests__/TimeInput.test.tsx` for comprehensive test suite.
|
||||
|
||||
### Mocks Must Match Real Return Values (Feb 2026)
|
||||
|
||||
**Problem**: PR #371 introduced a bug where admin reset always returned 500. The code checked `if not success:` after calling `update_job()`, but `update_job()` returns `None`, not a boolean. Since `not None` is `True`, the error was always raised.
|
||||
|
||||
**Why tests didn't catch it**: All 22 test mocks had:
|
||||
```python
|
||||
mock_jm.update_job.return_value = True # WRONG - real method returns None
|
||||
```
|
||||
|
||||
The mock returned `True`, so `if not True:` was `False` and tests passed. But in production, `if not None:` is `True`, causing 500 errors.
|
||||
|
||||
**Root cause**: Mocks were written based on an assumed API contract (boolean return) rather than the actual contract (void return, raises on error).
|
||||
|
||||
**Fix**:
|
||||
1. Updated all mocks to `return_value = None` with comment explaining why
|
||||
2. Added emulator integration test (`test_admin_reset_integration.py`) that uses real `JobManager`
|
||||
3. Added explicit test `test_update_job_returns_none_not_boolean` to document the API contract
|
||||
|
||||
**Pattern for preventing this**:
|
||||
1. **Mocks should match real signatures**: Check the actual method before setting `return_value`. Void methods should mock `return_value = None`.
|
||||
2. **Add emulator tests for critical paths**: Integration tests with real services catch contract mismatches mocks hide.
|
||||
3. **Document API contracts in tests**: Add explicit tests for non-obvious behaviors (`update_job` returns None, raises on failure).
|
||||
|
||||
**Code pattern**:
|
||||
```python
|
||||
# ❌ WRONG - assumes boolean return
|
||||
mock_jm.update_job.return_value = True
|
||||
|
||||
# ✅ CORRECT - matches real API
|
||||
mock_jm.update_job.return_value = None # Real API: returns None, raises on error
|
||||
```
|
||||
|
||||
### Emulator Tests Catch Real Bugs
|
||||
Firestore emulator tests catch issues (like missing indexes) that unit tests with mocks miss.
|
||||
|
||||
|
||||
@@ -158,6 +158,30 @@ def test_create_job_applies_default_theme(mock_firestore, mock_gcs):
|
||||
assert job.theme_id == "nomad" # Default theme applied
|
||||
```
|
||||
|
||||
**Critical: Mocks Must Match Real API Contracts**
|
||||
|
||||
When mocking methods, ensure the mock's `return_value` matches what the real method returns. A common bug pattern:
|
||||
|
||||
```python
|
||||
# ❌ BAD: Assumes method returns boolean
|
||||
mock_service.update_job.return_value = True
|
||||
|
||||
# Code under test:
|
||||
success = job_manager.update_job(job_id, updates)
|
||||
if not success: # BUG: if real method returns None, this is always True!
|
||||
raise HTTPException(500)
|
||||
```
|
||||
|
||||
The fix:
|
||||
|
||||
```python
|
||||
# ✅ GOOD: Check real method signature first
|
||||
# def update_job(self, job_id: str, updates: Dict) -> None:
|
||||
mock_service.update_job.return_value = None # Match real API
|
||||
```
|
||||
|
||||
**Before writing a mock**, check the actual method's return type. Void methods should mock `return_value = None`. Methods that raise on error (rather than returning False) should have mocks that raise exceptions for error cases. See [LESSONS-LEARNED.md](LESSONS-LEARNED.md#mocks-must-match-real-return-values-feb-2026) for a production bug caused by this.
|
||||
|
||||
### Integration Tests (`tests/integration/`)
|
||||
|
||||
**Purpose:** Test CLI workflows and component interactions.
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.114.16"
|
||||
version = "0.114.17"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
Reference in new issue
Block a user